From ef1eb0315e5830e2c3ea76d4f1ec6c50842da7bb Mon Sep 17 00:00:00 2001 From: Sizov Nikita Date: Fri, 8 Mar 2024 12:45:58 +0300 Subject: [PATCH 0001/1674] [AArch64] Add neon bici test for haddu and shadd (#84073) Add neon bici test for haddu and shadd, prerequisite for #76644 --- .../AArch64/aarch64-known-bits-hadd.ll | 129 ++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100644 llvm/test/CodeGen/AArch64/aarch64-known-bits-hadd.ll diff --git a/llvm/test/CodeGen/AArch64/aarch64-known-bits-hadd.ll b/llvm/test/CodeGen/AArch64/aarch64-known-bits-hadd.ll new file mode 100644 index 000000000000..017f38277489 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/aarch64-known-bits-hadd.ll @@ -0,0 +1,129 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +; RUN: llc -mtriple=aarch64 < %s | FileCheck %s + +declare <8 x i16> @llvm.aarch64.neon.uhadd.v8i16(<8 x i16>, <8 x i16>) +declare <8 x i16> @llvm.aarch64.neon.urhadd.v8i16(<8 x i16>, <8 x i16>) +declare <8 x i16> @llvm.aarch64.neon.shadd.v8i16(<8 x i16>, <8 x i16>) +declare <8 x i16> @llvm.aarch64.neon.srhadd.v8i16(<8 x i16>, <8 x i16>) + +define <8 x i16> @haddu_zext(<8 x i8> %a0, <8 x i8> %a1) { +; CHECK-LABEL: haddu_zext: +; CHECK: // %bb.0: +; CHECK-NEXT: ushll v0.8h, v0.8b, #0 +; CHECK-NEXT: ushll v1.8h, v1.8b, #0 +; CHECK-NEXT: uhadd v0.8h, v0.8h, v1.8h +; CHECK-NEXT: bic v0.8h, #254, lsl #8 +; CHECK-NEXT: ret + %x0 = zext <8 x i8> %a0 to <8 x i16> + %x1 = zext <8 x i8> %a1 to <8 x i16> + %hadd = call <8 x i16> @llvm.aarch64.neon.uhadd.v8i16(<8 x i16> %x0, <8 x i16> %x1) + %res = and <8 x i16> %hadd, + ret <8 x i16> %res +} + +define <8 x i16> @rhaddu_zext(<8 x i8> %a0, <8 x i8> %a1) { +; CHECK-LABEL: rhaddu_zext: +; CHECK: // %bb.0: +; CHECK-NEXT: ushll v0.8h, v0.8b, #0 +; CHECK-NEXT: ushll v1.8h, v1.8b, #0 +; CHECK-NEXT: urhadd v0.8h, v0.8h, v1.8h +; CHECK-NEXT: bic v0.8h, #254, lsl #8 +; CHECK-NEXT: ret + %x0 = zext <8 x i8> %a0 to <8 x i16> + %x1 = zext <8 x i8> %a1 to <8 x i16> + %hadd = call <8 x i16> @llvm.aarch64.neon.urhadd.v8i16(<8 x i16> %x0, <8 x i16> %x1) + %res = and <8 x i16> %hadd, + ret <8 x i16> %res +} + +define <8 x i16> @hadds_zext(<8 x i8> %a0, <8 x i8> %a1) { +; CHECK-LABEL: hadds_zext: +; CHECK: // %bb.0: +; CHECK-NEXT: ushll v0.8h, v0.8b, #0 +; CHECK-NEXT: ushll v1.8h, v1.8b, #0 +; CHECK-NEXT: shadd v0.8h, v0.8h, v1.8h +; CHECK-NEXT: bic v0.8h, #254, lsl #8 +; CHECK-NEXT: ret + %x0 = zext <8 x i8> %a0 to <8 x i16> + %x1 = zext <8 x i8> %a1 to <8 x i16> + %hadd = call <8 x i16> @llvm.aarch64.neon.shadd.v8i16(<8 x i16> %x0, <8 x i16> %x1) + %res = and <8 x i16> %hadd, + ret <8 x i16> %res +} + +define <8 x i16> @shaddu_zext(<8 x i8> %a0, <8 x i8> %a1) { +; CHECK-LABEL: shaddu_zext: +; CHECK: // %bb.0: +; CHECK-NEXT: ushll v0.8h, v0.8b, #0 +; CHECK-NEXT: ushll v1.8h, v1.8b, #0 +; CHECK-NEXT: srhadd v0.8h, v0.8h, v1.8h +; CHECK-NEXT: bic v0.8h, #254, lsl #8 +; CHECK-NEXT: ret + %x0 = zext <8 x i8> %a0 to <8 x i16> + %x1 = zext <8 x i8> %a1 to <8 x i16> + %hadd = call <8 x i16> @llvm.aarch64.neon.srhadd.v8i16(<8 x i16> %x0, <8 x i16> %x1) + %res = and <8 x i16> %hadd, + ret <8 x i16> %res +} + +; ; negative tests + +define <8 x i16> @haddu_sext(<8 x i8> %a0, <8 x i8> %a1) { +; CHECK-LABEL: haddu_sext: +; CHECK: // %bb.0: +; CHECK-NEXT: sshll v0.8h, v0.8b, #0 +; CHECK-NEXT: sshll v1.8h, v1.8b, #0 +; CHECK-NEXT: uhadd v0.8h, v0.8h, v1.8h +; CHECK-NEXT: bic v0.8h, #254, lsl #8 +; CHECK-NEXT: ret + %x0 = sext <8 x i8> %a0 to <8 x i16> + %x1 = sext <8 x i8> %a1 to <8 x i16> + %hadd = call <8 x i16> @llvm.aarch64.neon.uhadd.v8i16(<8 x i16> %x0, <8 x i16> %x1) + %res = and <8 x i16> %hadd, + ret <8 x i16> %res +} + +define <8 x i16> @urhadd_sext(<8 x i8> %a0, <8 x i8> %a1) { +; CHECK-LABEL: urhadd_sext: +; CHECK: // %bb.0: +; CHECK-NEXT: sshll v0.8h, v0.8b, #0 +; CHECK-NEXT: sshll v1.8h, v1.8b, #0 +; CHECK-NEXT: urhadd v0.8h, v0.8h, v1.8h +; CHECK-NEXT: bic v0.8h, #254, lsl #8 +; CHECK-NEXT: ret + %x0 = sext <8 x i8> %a0 to <8 x i16> + %x1 = sext <8 x i8> %a1 to <8 x i16> + %hadd = call <8 x i16> @llvm.aarch64.neon.urhadd.v8i16(<8 x i16> %x0, <8 x i16> %x1) + %res = and <8 x i16> %hadd, + ret <8 x i16> %res +} + +define <8 x i16> @hadds_sext(<8 x i8> %a0, <8 x i8> %a1) { +; CHECK-LABEL: hadds_sext: +; CHECK: // %bb.0: +; CHECK-NEXT: sshll v0.8h, v0.8b, #0 +; CHECK-NEXT: sshll v1.8h, v1.8b, #0 +; CHECK-NEXT: shadd v0.8h, v0.8h, v1.8h +; CHECK-NEXT: bic v0.8h, #254, lsl #8 +; CHECK-NEXT: ret + %x0 = sext <8 x i8> %a0 to <8 x i16> + %x1 = sext <8 x i8> %a1 to <8 x i16> + %hadd = call <8 x i16> @llvm.aarch64.neon.shadd.v8i16(<8 x i16> %x0, <8 x i16> %x1) + %res = and <8 x i16> %hadd, + ret <8 x i16> %res +} + +define <8 x i16> @shaddu_sext(<8 x i8> %a0, <8 x i8> %a1) { +; CHECK-LABEL: shaddu_sext: +; CHECK: // %bb.0: +; CHECK-NEXT: sshll v0.8h, v0.8b, #0 +; CHECK-NEXT: sshll v1.8h, v1.8b, #0 +; CHECK-NEXT: srhadd v0.8h, v0.8h, v1.8h +; CHECK-NEXT: bic v0.8h, #254, lsl #8 +; CHECK-NEXT: ret + %x0 = sext <8 x i8> %a0 to <8 x i16> + %x1 = sext <8 x i8> %a1 to <8 x i16> + %hadd = call <8 x i16> @llvm.aarch64.neon.srhadd.v8i16(<8 x i16> %x0, <8 x i16> %x1) + %res = and <8 x i16> %hadd, + ret <8 x i16> %res +} -- GitLab From d36d805373c27103edf13164260ef679efd30dc1 Mon Sep 17 00:00:00 2001 From: David Green Date: Fri, 8 Mar 2024 09:52:49 +0000 Subject: [PATCH 0002/1674] [AArch64] Ensure Neoverse-N2 scheduling model includes all SVE pseudos. Similar to #84187, this enables the existing test we have for checking the scheduling info of the pseudos matches the real instructions, and adjusts the scheduling info in the NeoverseN2 model to make sure all cases were handled. --- .../Target/AArch64/AArch64SchedNeoverseN2.td | 210 +++++++++--------- .../AArch64/AArch64SVESchedPseudoTest.cpp | 4 + 2 files changed, 114 insertions(+), 100 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64SchedNeoverseN2.td b/llvm/lib/Target/AArch64/AArch64SchedNeoverseN2.td index c18045e7c8f9..a4ac344510de 100644 --- a/llvm/lib/Target/AArch64/AArch64SchedNeoverseN2.td +++ b/llvm/lib/Target/AArch64/AArch64SchedNeoverseN2.td @@ -1577,7 +1577,8 @@ def : InstRW<[N2Write_2cyc_1M], (instregex "^(ZIP|UZP)[12]_PPP_[BHSDQ]$")>; // ----------------------------------------------------------------------------- // Arithmetic, absolute diff -def : InstRW<[N2Write_2cyc_1V], (instregex "^[SU]ABD_ZPmZ_[BHSD]$")>; +def : InstRW<[N2Write_2cyc_1V], (instregex "^[SU]ABD_ZPmZ_[BHSD]", + "^[SU]ABD_ZPZZ_[BHSD]")>; // Arithmetic, absolute diff accum def : InstRW<[N2Write_4cyc_1V1], (instregex "^[SU]ABA_ZZZ_[BHSD]$")>; @@ -1590,24 +1591,25 @@ def : InstRW<[N2Write_2cyc_1V], (instregex "^[SU]ABDL[TB]_ZZZ_[HSD]$")>; // Arithmetic, basic def : InstRW<[N2Write_2cyc_1V], - (instregex "^(ABS|ADD|CNOT|NEG|SUB|SUBR)_ZPmZ_[BHSD]$", - "^(ADD|SUB)_ZZZ_[BHSD]$", - "^(ADD|SUB|SUBR)_ZI_[BHSD]$", - "^ADR_[SU]XTW_ZZZ_D_[0123]$", - "^ADR_LSL_ZZZ_[SD]_[0123]$", - "^[SU](ADD|SUB)[LW][BT]_ZZZ_[HSD]$", - "^SADDLBT_ZZZ_[HSD]$", - "^[SU]H(ADD|SUB|SUBR)_ZPmZ_[BHSD]$", - "^SSUBL(BT|TB)_ZZZ_[HSD]$")>; + (instregex "^(ABS|ADD|CNOT|NEG|SUB|SUBR)_ZPmZ_[BHSD]", + "^(ADD|SUB)_ZZZ_[BHSD]", + "^(ADD|SUB|SUBR)_ZPZZ_[BHSD]", + "^(ADD|SUB|SUBR)_ZI_[BHSD]", + "^ADR_[SU]XTW_ZZZ_D_[0123]", + "^ADR_LSL_ZZZ_[SD]_[0123]", + "^[SU](ADD|SUB)[LW][BT]_ZZZ_[HSD]", + "^SADDLBT_ZZZ_[HSD]", + "^[SU]H(ADD|SUB|SUBR)_ZPmZ_[BHSD]", + "^SSUBL(BT|TB)_ZZZ_[HSD]")>; // Arithmetic, complex def : InstRW<[N2Write_2cyc_1V], - (instregex "^R?(ADD|SUB)HN[BT]_ZZZ_[BHS]$", - "^SQ(ABS|ADD|NEG|SUB|SUBR)_ZPmZ_[BHSD]$", - "^[SU]Q(ADD|SUB)_ZZZ_[BHSD]$", - "^[SU]Q(ADD|SUB)_ZI_[BHSD]$", - "^(SRH|SUQ|UQ|USQ|URH)ADD_ZPmZ_[BHSD]$", - "^(UQSUB|UQSUBR)_ZPmZ_[BHSD]$")>; + (instregex "^R?(ADD|SUB)HN[BT]_ZZZ_[BHS]", + "^SQ(ABS|ADD|NEG|SUB|SUBR)_ZPmZ_[BHSD]", + "^[SU]Q(ADD|SUB)_ZZZ_[BHSD]", + "^[SU]Q(ADD|SUB)_ZI_[BHSD]", + "^(SRH|SUQ|UQ|USQ|URH)ADD_ZPmZ_[BHSD]", + "^(UQSUB|UQSUBR)_ZPmZ_[BHSD]")>; // Arithmetic, large integer def : InstRW<[N2Write_2cyc_1V], (instregex "^(AD|SB)CL[BT]_ZZZ_[SD]$")>; @@ -1620,12 +1622,13 @@ def : InstRW<[N2Write_4cyc_1V1], (instregex "^[SU]ADALP_ZPmZ_[HSD]$")>; // Arithmetic, shift def : InstRW<[N2Write_2cyc_1V1], - (instregex "^(ASR|LSL|LSR)_WIDE_ZPmZ_[BHS]$", - "^(ASR|LSL|LSR)_WIDE_ZZZ_[BHS]$", - "^(ASR|LSL|LSR)_ZPmI_[BHSD]$", - "^(ASR|LSL|LSR)_ZPmZ_[BHSD]$", - "^(ASR|LSL|LSR)_ZZI_[BHSD]$", - "^(ASRR|LSLR|LSRR)_ZPmZ_[BHSD]$")>; + (instregex "^(ASR|LSL|LSR)_WIDE_ZPmZ_[BHS]", + "^(ASR|LSL|LSR)_WIDE_ZZZ_[BHS]", + "^(ASR|LSL|LSR)_ZPmI_[BHSD]", + "^(ASR|LSL|LSR)_ZPmZ_[BHSD]", + "^(ASR|LSL|LSR)_ZZI_[BHSD]", + "^(ASR|LSL|LSR)_ZPZ[IZ]_[BHSD]", + "^(ASRR|LSLR|LSRR)_ZPmZ_[BHSD]")>; // Arithmetic, shift and accumulate def : InstRW<[N2Write_4cyc_1V1], @@ -1638,29 +1641,29 @@ def : InstRW<[N2Write_2cyc_1V1], // Arithmetic, shift complex def : InstRW<[N2Write_4cyc_1V1], - (instregex "^(SQ)?RSHRU?N[BT]_ZZI_[BHS]$", - "^(SQRSHL|SQRSHLR|SQSHL|SQSHLR|UQRSHL|UQRSHLR|UQSHL|UQSHLR)_ZPmZ_[BHSD]$", - "^(SQSHL|SQSHLU|UQSHL)_ZPmI_[BHSD]$", - "^SQSHRU?N[BT]_ZZI_[BHS]$", - "^UQR?SHRN[BT]_ZZI_[BHS]$")>; + (instregex "^(SQ)?RSHRU?N[BT]_ZZI_[BHS]", + "^(SQRSHL|SQRSHLR|SQSHL|SQSHLR|UQRSHL|UQRSHLR|UQSHL|UQSHLR)_ZPmZ_[BHSD]", + "^[SU]QR?SHL_ZPZZ_[BHSD]", + "^(SQSHL|SQSHLU|UQSHL)_(ZPmI|ZPZI)_[BHSD]", + "^SQSHRU?N[BT]_ZZI_[BHS]", + "^UQR?SHRN[BT]_ZZI_[BHS]")>; // Arithmetic, shift right for divide -def : InstRW<[N2Write_4cyc_1V1], (instregex "^ASRD_ZPmI_[BHSD]$")>; +def : InstRW<[N2Write_4cyc_1V1], (instregex "^ASRD_(ZPmI|ZPZI)_[BHSD]")>; // Arithmetic, shift rounding -def : InstRW<[N2Write_4cyc_1V1], - (instregex "^(SRSHL|SRSHLR|URSHL|URSHLR)_ZPmZ_[BHSD]$", - "^[SU]RSHR_ZPmI_[BHSD]$")>; +def : InstRW<[N2Write_4cyc_1V1], (instregex "^[SU]RSHLR?_ZPmZ_[BHSD]", + "^[SU]RSHL_ZPZZ_[BHSD]", + "^[SU]RSHR_(ZPmI|ZPZI)_[BHSD]")>; // Bit manipulation -def : InstRW<[N2Write_6cyc_2V1], - (instregex "^(BDEP|BEXT|BGRP)_ZZZ_[BHSD]$")>; +def : InstRW<[N2Write_6cyc_2V1], (instregex "^(BDEP|BEXT|BGRP)_ZZZ_[BHSD]")>; // Bitwise select def : InstRW<[N2Write_2cyc_1V], (instregex "^(BSL|BSL1N|BSL2N|NBSL)_ZZZZ$")>; // Count/reverse bits -def : InstRW<[N2Write_2cyc_1V], (instregex "^(CLS|CLZ|CNT|RBIT)_ZPmZ_[BHSD]$")>; +def : InstRW<[N2Write_2cyc_1V], (instregex "^(CLS|CLZ|CNT|RBIT)_ZPmZ_[BHSD]")>; // Broadcast logical bitmask immediate to vector def : InstRW<[N2Write_2cyc_1V], (instrs DUPM_ZI)>; @@ -1695,19 +1698,14 @@ def : InstRW<[N2Write_3cyc_1V1], (instregex "^CLAST[AB]_[VZ]PZ_[BHSD]$", "^SPLICE_ZPZZ?_[BHSD]$")>; // Convert to floating point, 64b to float or convert to double -def : InstRW<[N2Write_3cyc_1V0], (instregex "^[SU]CVTF_ZPmZ_Dto[SD]$")>; - -// Convert to floating point, 64b to half -def : InstRW<[N2Write_3cyc_1V0], (instregex "^[SU]CVTF_ZPmZ_DtoH$")>; +def : InstRW<[N2Write_3cyc_1V0], (instregex "^[SU]CVTF_ZPmZ_Dto[HSD]", + "^[SU]CVTF_ZPmZ_StoD")>; // Convert to floating point, 32b to single or half -def : InstRW<[N2Write_4cyc_2V0], (instregex "^[SU]CVTF_ZPmZ_Sto[HS]$")>; - -// Convert to floating point, 32b to double -def : InstRW<[N2Write_3cyc_1V0], (instregex "^[SU]CVTF_ZPmZ_StoD$")>; +def : InstRW<[N2Write_4cyc_2V0], (instregex "^[SU]CVTF_ZPmZ_Sto[HS]")>; // Convert to floating point, 16b to half -def : InstRW<[N2Write_6cyc_4V0], (instregex "^[SU]CVTF_ZPmZ_HtoH$")>; +def : InstRW<[N2Write_6cyc_4V0], (instregex "^[SU]CVTF_ZPmZ_HtoH")>; // Copy, scalar def : InstRW<[N2Write_5cyc_1M0_1V], (instregex "^CPY_ZPmR_[BHSD]$")>; @@ -1717,10 +1715,12 @@ def : InstRW<[N2Write_2cyc_1V], (instregex "^CPY_ZPm[IV]_[BHSD]$", "^CPY_ZPzI_[BHSD]$")>; // Divides, 32 bit -def : InstRW<[N2Write_12cyc_1V0], (instregex "^[SU]DIVR?_ZPmZ_S$")>; +def : InstRW<[N2Write_12cyc_1V0], (instregex "^[SU]DIVR?_ZPmZ_S", + "^[SU]DIV_ZPZZ_S")>; // Divides, 64 bit -def : InstRW<[N2Write_20cyc_1V0], (instregex "^[SU]DIVR?_ZPmZ_D$")>; +def : InstRW<[N2Write_20cyc_1V0], (instregex "^[SU]DIVR?_ZPmZ_D", + "^[SU]DIV_ZPZZ_D")>; // Dot product, 8 bit def : InstRW<[N2Write_3cyc_1V], (instregex "^[SU]DOT_ZZZI?_S$")>; @@ -1739,9 +1739,9 @@ def : InstRW<[N2Write_2cyc_1V], (instregex "^DUP_ZI_[BHSD]$", def : InstRW<[N2Write_3cyc_1M0], (instregex "^DUP_ZR_[BHSD]$")>; // Extend, sign or zero -def : InstRW<[N2Write_2cyc_1V1], (instregex "^[SU]XTB_ZPmZ_[HSD]$", - "^[SU]XTH_ZPmZ_[SD]$", - "^[SU]XTW_ZPmZ_[D]$")>; +def : InstRW<[N2Write_2cyc_1V1], (instregex "^[SU]XTB_ZPmZ_[HSD]", + "^[SU]XTH_ZPmZ_[SD]", + "^[SU]XTW_ZPmZ_[D]")>; // Extract def : InstRW<[N2Write_2cyc_1V], (instrs EXT_ZZI, EXT_ZZI_B)>; @@ -1778,14 +1778,16 @@ def : InstRW<[N2Write_8cyc_2M0_2V0], (instregex "^INDEX_(IR|RI|RR)_D$")>; // Logical def : InstRW<[N2Write_2cyc_1V], - (instregex "^(AND|EOR|ORR)_ZI$", - "^(AND|BIC|EOR|EOR(BT|TB)?|ORR)_ZZZ$", - "^EOR(BT|TB)_ZZZ_[BHSD]$", - "^(AND|BIC|EOR|NOT|ORR)_ZPmZ_[BHSD]$")>; + (instregex "^(AND|EOR|ORR)_ZI", + "^(AND|BIC|EOR|ORR)_ZZZ", + "^EOR(BT|TB)_ZZZ_[BHSD]", + "^(AND|BIC|EOR|NOT|ORR)_(ZPmZ|ZPZZ)_[BHSD]", + "^NOT_ZPmZ_[BHSD]")>; // Max/min, basic and pairwise -def : InstRW<[N2Write_2cyc_1V], (instregex "^[SU](MAX|MIN)_ZI_[BHSD]$", - "^[SU](MAX|MIN)P?_ZPmZ_[BHSD]$")>; +def : InstRW<[N2Write_2cyc_1V], (instregex "^[SU](MAX|MIN)_ZI_[BHSD]", + "^[SU](MAX|MIN)P?_ZPmZ_[BHSD]", + "^[SU](MAX|MIN)_ZPZZ_[BHSD]")>; // Matching operations def : InstRW<[N2Write_2cyc_1V0_1M], (instregex "^N?MATCH_PPzZZ_[BH]$")>; @@ -1798,12 +1800,16 @@ def : InstRW<[N2Write_2cyc_1V], (instregex "^MOVPRFX_ZP[mz]Z_[BHSD]$", "^MOVPRFX_ZZ$")>; // Multiply, B, H, S element size -def : InstRW<[N2Write_4cyc_1V0], (instregex "^MUL_(ZI|ZPmZ|ZZZI|ZZZ)_[BHS]$", - "^[SU]MULH_(ZPmZ|ZZZ)_[BHS]$")>; +def : InstRW<[N2Write_4cyc_1V0], (instregex "^MUL_(ZI|ZPmZ|ZZZI|ZZZ)_[BHS]", + "^MUL_ZPZZ_[BHS]", + "^[SU]MULH_(ZPmZ|ZZZ)_[BHS]", + "^[SU]MULH_ZPZZ_[BHS]")>; // Multiply, D element size -def : InstRW<[N2Write_5cyc_2V0], (instregex "^MUL_(ZI|ZPmZ|ZZZI|ZZZ)_D$", - "^[SU]MULH_(ZPmZ|ZZZ)_D$")>; +def : InstRW<[N2Write_5cyc_2V0], (instregex "^MUL_(ZI|ZPmZ|ZZZI|ZZZ)_D", + "^MUL_ZPZZ_D", + "^[SU]MULH_(ZPmZ|ZZZ)_D", + "^[SU]MULH_ZPZZ_D")>; // Multiply long def : InstRW<[N2Write_4cyc_1V0], (instregex "^[SU]MULL[BT]_ZZZI_[SD]$", @@ -1811,11 +1817,11 @@ def : InstRW<[N2Write_4cyc_1V0], (instregex "^[SU]MULL[BT]_ZZZI_[SD]$", // Multiply accumulate, B, H, S element size def : InstRW<[N2Write_4cyc_1V0], (instregex "^ML[AS]_ZZZI_[BHS]$", - "^(ML[AS]|MAD|MSB)_ZPmZZ_[BHS]$")>; + "^(ML[AS]|MAD|MSB)_(ZPmZZ|ZPZZZ)_[BHS]")>; // Multiply accumulate, D element size def : InstRW<[N2Write_5cyc_2V0], (instregex "^ML[AS]_ZZZI_D$", - "^(ML[AS]|MAD|MSB)_ZPmZZ_D$")>; + "^(ML[AS]|MAD|MSB)_(ZPmZZ|ZPZZZ)_D")>; // Multiply accumulate long def : InstRW<[N2Write_4cyc_1V0], (instregex "^[SU]ML[AS]L[BT]_ZZZ_[HSD]$", @@ -1864,7 +1870,7 @@ def : InstRW<[N2Write_2cyc_1V0], (instregex "^(DEC|INC|SQDEC|SQINC|UQDEC|UQINC)[HWD]_ZPiI$")>; // Reciprocal estimate -def : InstRW<[N2Write_4cyc_2V0], (instrs URECPE_ZPmZ_S, URSQRTE_ZPmZ_S)>; +def : InstRW<[N2Write_4cyc_2V0], (instregex "^URECPE_ZPmZ_S", "^URSQRTE_ZPmZ_S")>; // Reduction, arithmetic, B form def : InstRW<[N2Write_11cyc_2V_2V1], (instregex "^[SU](ADD|MAX|MIN)V_VPZ_B")>; @@ -1909,13 +1915,17 @@ def : InstRW<[N2Write_2cyc_1V], (instregex "^(UZP|ZIP)[12]_ZZZ_[BHSDQ]$")>; // ----------------------------------------------------------------------------- // Floating point absolute value/difference -def : InstRW<[N2Write_2cyc_1V], (instregex "^FAB[SD]_ZPmZ_[HSD]$")>; +def : InstRW<[N2Write_2cyc_1V], (instregex "^FAB[SD]_ZPmZ_[HSD]", + "^FABD_ZPZZ_[HSD]", + "^FABS_ZPmZ_[HSD]")>; // Floating point arithmetic -def : InstRW<[N2Write_2cyc_1V], (instregex "^F(ADD|SUB)_(ZPm[IZ]|ZZZ)_[HSD]$", - "^FADDP_ZPmZZ_[HSD]$", - "^FNEG_ZPmZ_[HSD]$", - "^FSUBR_ZPm[IZ]_[HSD]$")>; +def : InstRW<[N2Write_2cyc_1V], (instregex "^F(ADD|SUB)_(ZPm[IZ]|ZZZ)_[HSD]", + "^F(ADD|SUB)_ZPZ[IZ]_[HSD]", + "^FADDP_ZPmZZ_[HSD]", + "^FNEG_ZPmZ_[HSD]", + "^FSUBR_ZPm[IZ]_[HSD]", + "^FSUBR_(ZPZI|ZPZZ)_[HSD]")>; // Floating point associative add, F16 def : InstRW<[N2Write_10cyc_1V1], (instrs FADDA_VPZ_H)>; @@ -1940,80 +1950,80 @@ def : InstRW<[N2Write_5cyc_1V], (instregex "^FCMLA_ZPmZZ_[HSD]$", "^FCMLA_ZZZI_[HS]$")>; // Floating point convert, long or narrow (F16 to F32 or F32 to F16) -def : InstRW<[N2Write_4cyc_2V0], (instregex "^FCVT_ZPmZ_(HtoS|StoH)$", - "^FCVTLT_ZPmZ_HtoS$", - "^FCVTNT_ZPmZ_StoH$")>; +def : InstRW<[N2Write_4cyc_2V0], (instregex "^FCVT_ZPmZ_(HtoS|StoH)", + "^FCVTLT_ZPmZ_HtoS", + "^FCVTNT_ZPmZ_StoH")>; // Floating point convert, long or narrow (F16 to F64, F32 to F64, F64 to F32 // or F64 to F16) -def : InstRW<[N2Write_3cyc_1V0], (instregex "^FCVT_ZPmZ_(HtoD|StoD|DtoS|DtoH)$", - "^FCVTLT_ZPmZ_StoD$", - "^FCVTNT_ZPmZ_DtoS$")>; +def : InstRW<[N2Write_3cyc_1V0], (instregex "^FCVT_ZPmZ_(HtoD|StoD|DtoS|DtoH)", + "^FCVTLT_ZPmZ_StoD", + "^FCVTNT_ZPmZ_DtoS")>; // Floating point convert, round to odd def : InstRW<[N2Write_3cyc_1V0], (instrs FCVTX_ZPmZ_DtoS, FCVTXNT_ZPmZ_DtoS)>; // Floating point base2 log, F16 -def : InstRW<[N2Write_6cyc_4V0], (instrs FLOGB_ZPmZ_H)>; +def : InstRW<[N2Write_6cyc_4V0], (instregex "^FLOGB_(ZPmZ|ZPZZ)_H")>; // Floating point base2 log, F32 -def : InstRW<[N2Write_4cyc_2V0], (instrs FLOGB_ZPmZ_S)>; +def : InstRW<[N2Write_4cyc_2V0], (instregex "^FLOGB_(ZPmZ|ZPZZ)_S")>; // Floating point base2 log, F64 -def : InstRW<[N2Write_3cyc_1V0], (instrs FLOGB_ZPmZ_D)>; +def : InstRW<[N2Write_3cyc_1V0], (instregex "^FLOGB_(ZPmZ|ZPZZ)_D")>; // Floating point convert to integer, F16 -def : InstRW<[N2Write_6cyc_4V0], (instregex "^FCVTZ[SU]_ZPmZ_HtoH$")>; +def : InstRW<[N2Write_6cyc_4V0], (instregex "^FCVTZ[SU]_ZPmZ_HtoH")>; // Floating point convert to integer, F32 -def : InstRW<[N2Write_4cyc_2V0], (instregex "^FCVTZ[SU]_ZPmZ_(HtoS|StoS)$")>; +def : InstRW<[N2Write_4cyc_2V0], (instregex "^FCVTZ[SU]_ZPmZ_(HtoS|StoS)")>; // Floating point convert to integer, F64 def : InstRW<[N2Write_3cyc_1V0], - (instregex "^FCVTZ[SU]_ZPmZ_(HtoD|StoD|DtoS|DtoD)$")>; + (instregex "^FCVTZ[SU]_ZPmZ_(HtoD|StoD|DtoS|DtoD)")>; // Floating point copy def : InstRW<[N2Write_2cyc_1V], (instregex "^FCPY_ZPmI_[HSD]$", "^FDUP_ZI_[HSD]$")>; // Floating point divide, F16 -def : InstRW<[N2Write_13cyc_1V0], (instregex "^FDIVR?_ZPmZ_H$")>; +def : InstRW<[N2Write_13cyc_1V0], (instregex "^FDIVR?_(ZPmZ|ZPZZ)_H")>; // Floating point divide, F32 -def : InstRW<[N2Write_10cyc_1V0], (instregex "^FDIVR?_ZPmZ_S$")>; +def : InstRW<[N2Write_10cyc_1V0], (instregex "^FDIVR?_(ZPmZ|ZPZZ)_S")>; // Floating point divide, F64 -def : InstRW<[N2Write_15cyc_1V0], (instregex "^FDIVR?_ZPmZ_D$")>; +def : InstRW<[N2Write_15cyc_1V0], (instregex "^FDIVR?_(ZPmZ|ZPZZ)_D")>; // Floating point min/max pairwise -def : InstRW<[N2Write_2cyc_1V], (instregex "^F(MAX|MIN)(NM)?P_ZPmZZ_[HSD]$")>; +def : InstRW<[N2Write_2cyc_1V], (instregex "^F(MAX|MIN)(NM)?P_ZPmZZ_[HSD]")>; // Floating point min/max -def : InstRW<[N2Write_2cyc_1V], (instregex "^F(MAX|MIN)(NM)?_ZPm[IZ]_[HSD]$")>; +def : InstRW<[N2Write_2cyc_1V], (instregex "^F(MAX|MIN)(NM)?_ZPm[IZ]_[HSD]", + "^F(MAX|MIN)(NM)?_ZPZ[IZ]_[HSD]")>; // Floating point multiply -def : InstRW<[N2Write_3cyc_1V], (instregex "^(FSCALE|FMULX)_ZPmZ_[HSD]$", - "^FMUL_(ZPm[IZ]|ZZZI?)_[HSD]$")>; +def : InstRW<[N2Write_3cyc_1V], (instregex "^(FSCALE|FMULX)_ZPmZ_[HSD]", + "^FMULX_ZPZZ_[HSD]", + "^FMUL_(ZPm[IZ]|ZZZI?)_[HSD]", + "^FMUL_ZPZ[IZ]_[HSD]")>; // Floating point multiply accumulate -def : InstRW<[N2Write_4cyc_1V], - (instregex "^FML[AS]_(ZPmZZ|ZZZI)_[HSD]$", - "^(FMAD|FNMAD|FNML[AS]|FN?MSB)_ZPmZZ_[HSD]$")>; +def : InstRW<[N2Write_4cyc_1V], (instregex "^F(N?M(AD|SB)|N?ML[AS])_ZPmZZ_[HSD]$", + "^FN?ML[AS]_ZPZZZ_[HSD]", + "^FML[AS]_ZZZI_[HSD]$")>; // Floating point multiply add/sub accumulate long def : InstRW<[N2Write_4cyc_1V], (instregex "^FML[AS]L[BT]_ZZZI?_SHH$")>; // Floating point reciprocal estimate, F16 -def : InstRW<[N2Write_6cyc_4V0], (instrs FRECPE_ZZ_H, FRECPX_ZPmZ_H, - FRSQRTE_ZZ_H)>; +def : InstRW<[N2Write_6cyc_4V0], (instregex "^FR(ECP|SQRT)E_ZZ_H", "^FRECPX_ZPmZ_H")>; // Floating point reciprocal estimate, F32 -def : InstRW<[N2Write_4cyc_2V0], (instrs FRECPE_ZZ_S, FRECPX_ZPmZ_S, - FRSQRTE_ZZ_S)>; +def : InstRW<[N2Write_4cyc_2V0], (instregex "^FR(ECP|SQRT)E_ZZ_S", "^FRECPX_ZPmZ_S")>; // Floating point reciprocal estimate, F64 -def : InstRW<[N2Write_3cyc_1V0], (instrs FRECPE_ZZ_D, FRECPX_ZPmZ_D, - FRSQRTE_ZZ_D)>; +def : InstRW<[N2Write_3cyc_1V0], (instregex "^FR(ECP|SQRT)E_ZZ_D", "^FRECPX_ZPmZ_D")>; // Floating point reciprocal step def : InstRW<[N2Write_4cyc_1V0], (instregex "^F(RECPS|RSQRTS)_ZZZ_[HSD]$")>; @@ -2031,22 +2041,22 @@ def : InstRW<[N2Write_2cyc_1V], (instregex "^(FADDV|FMAXNMV|FMAXV|FMINNMV|FMINV)_VPZ_D$")>; // Floating point round to integral, F16 -def : InstRW<[N2Write_6cyc_4V0], (instregex "^FRINT[AIMNPXZ]_ZPmZ_H$")>; +def : InstRW<[N2Write_6cyc_4V0], (instregex "^FRINT[AIMNPXZ]_ZPmZ_H")>; // Floating point round to integral, F32 -def : InstRW<[N2Write_4cyc_2V0], (instregex "^FRINT[AIMNPXZ]_ZPmZ_S$")>; +def : InstRW<[N2Write_4cyc_2V0], (instregex "^FRINT[AIMNPXZ]_ZPmZ_S")>; // Floating point round to integral, F64 -def : InstRW<[N2Write_3cyc_1V0], (instregex "^FRINT[AIMNPXZ]_ZPmZ_D$")>; +def : InstRW<[N2Write_3cyc_1V0], (instregex "^FRINT[AIMNPXZ]_ZPmZ_D")>; // Floating point square root, F16 -def : InstRW<[N2Write_13cyc_1V0], (instrs FSQRT_ZPmZ_H)>; +def : InstRW<[N2Write_13cyc_1V0], (instregex "^FSQRT_ZPmZ_H")>; // Floating point square root, F32 -def : InstRW<[N2Write_10cyc_1V0], (instrs FSQRT_ZPmZ_S)>; +def : InstRW<[N2Write_10cyc_1V0], (instregex "^FSQRT_ZPmZ_S")>; // Floating point square root, F64 -def : InstRW<[N2Write_16cyc_1V0], (instrs FSQRT_ZPmZ_D)>; +def : InstRW<[N2Write_16cyc_1V0], (instregex "^FSQRT_ZPmZ_D")>; // Floating point trigonometric exponentiation def : InstRW<[N2Write_3cyc_1V1], (instregex "^FEXPA_ZZ_[HSD]$")>; diff --git a/llvm/unittests/Target/AArch64/AArch64SVESchedPseudoTest.cpp b/llvm/unittests/Target/AArch64/AArch64SVESchedPseudoTest.cpp index 6098d4e62392..a527bdeef724 100644 --- a/llvm/unittests/Target/AArch64/AArch64SVESchedPseudoTest.cpp +++ b/llvm/unittests/Target/AArch64/AArch64SVESchedPseudoTest.cpp @@ -107,6 +107,10 @@ TEST(AArch64SVESchedPseudoTesta510, IsCorrect) { runSVEPseudoTestForCPU("cortex-a510"); } +TEST(AArch64SVESchedPseudoTestn1, IsCorrect) { + runSVEPseudoTestForCPU("neoverse-n2"); +} + TEST(AArch64SVESchedPseudoTestv1, IsCorrect) { runSVEPseudoTestForCPU("neoverse-v1"); } -- GitLab From 3a62edcf527bfc927f148c99b9d37d3d199db611 Mon Sep 17 00:00:00 2001 From: zhongyunde 00443407 Date: Wed, 6 Mar 2024 14:09:18 -0500 Subject: [PATCH 0003/1674] [AArch64] MachineCombiner msub matching Pattern should be sorted in priority order since the pattern evalutor stops checking as soon as it finds a faster sequence. so for a * b - c * d, we prefer to match the 2nd operands of sub, which can be use msub to fold them. Refer to https://www.slideshare.net/chimerawang/instruction-combine-in-llvm Fix https://github.com/llvm/llvm-project/issues/84152 --- llvm/lib/Target/AArch64/AArch64InstrInfo.cpp | 2 +- llvm/test/CodeGen/AArch64/scalar-mla-mls.ll | 31 ++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 llvm/test/CodeGen/AArch64/scalar-mla-mls.ll diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp index 5df691f35275..5893f76dbd55 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp @@ -6110,8 +6110,8 @@ static bool getMaddPatterns(MachineInstr &Root, setFound(AArch64::MADDXrrr, 2, AArch64::XZR, MCP::MULADDX_OP2); break; case AArch64::SUBWrr: - setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULSUBW_OP1); setFound(AArch64::MADDWrrr, 2, AArch64::WZR, MCP::MULSUBW_OP2); + setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULSUBW_OP1); break; case AArch64::SUBXrr: setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULSUBX_OP1); diff --git a/llvm/test/CodeGen/AArch64/scalar-mla-mls.ll b/llvm/test/CodeGen/AArch64/scalar-mla-mls.ll new file mode 100644 index 000000000000..36ac36701fa8 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/scalar-mla-mls.ll @@ -0,0 +1,31 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -verify-machineinstrs -mtriple=aarch64-none-linux-gnu -mattr=+neon | FileCheck %s + +define ptr @test_scalar_msub(ptr %a, ptr %b) { +; CHECK-LABEL: test_scalar_msub: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: ldp w8, w11, [x1] +; CHECK-NEXT: ldp w9, w10, [x0] +; CHECK-NEXT: mul w12, w8, w9 +; CHECK-NEXT: mul w8, w10, w8 +; CHECK-NEXT: madd w8, w11, w9, w8 +; CHECK-NEXT: msub w9, w11, w10, w12 +; CHECK-NEXT: stp w9, w8, [x0] +; CHECK-NEXT: ret +entry: + %0 = load i32, ptr %a, align 4 + %1 = load i32, ptr %b, align 4 + %mul = mul nsw i32 %1, %0 + %_M_imag = getelementptr inbounds i8, ptr %a, i64 4 + %2 = load i32, ptr %_M_imag, align 4 + %_M_imag.i = getelementptr inbounds i8, ptr %b, i64 4 + %3 = load i32, ptr %_M_imag.i, align 4 + %mul3 = mul nsw i32 %3, %2 + %sub = sub nsw i32 %mul, %mul3 + %mul6 = mul nsw i32 %3, %0 + %mul9 = mul nsw i32 %2, %1 + %add = add nsw i32 %mul6, %mul9 + store i32 %add, ptr %_M_imag, align 4 + store i32 %sub, ptr %a, align 4 + ret ptr %a +} -- GitLab From a110a1c0ed9850be168cd0e29f05179e80941b04 Mon Sep 17 00:00:00 2001 From: zhongyunde 00443407 Date: Fri, 8 Mar 2024 11:27:48 -0500 Subject: [PATCH 0004/1674] [AArch64] MachineCombiner msub matching for i64 --- llvm/lib/Target/AArch64/AArch64InstrInfo.cpp | 2 +- llvm/test/CodeGen/AArch64/scalar-mla-mls.ll | 32 ++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp index 5893f76dbd55..02943b8a4ab1 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp @@ -6114,8 +6114,8 @@ static bool getMaddPatterns(MachineInstr &Root, setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULSUBW_OP1); break; case AArch64::SUBXrr: - setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULSUBX_OP1); setFound(AArch64::MADDXrrr, 2, AArch64::XZR, MCP::MULSUBX_OP2); + setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULSUBX_OP1); break; case AArch64::ADDWri: setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULADDWI_OP1); diff --git a/llvm/test/CodeGen/AArch64/scalar-mla-mls.ll b/llvm/test/CodeGen/AArch64/scalar-mla-mls.ll index 36ac36701fa8..c8ba50ae0cc3 100644 --- a/llvm/test/CodeGen/AArch64/scalar-mla-mls.ll +++ b/llvm/test/CodeGen/AArch64/scalar-mla-mls.ll @@ -29,3 +29,35 @@ entry: store i32 %sub, ptr %a, align 4 ret ptr %a } + +define ptr @test_scalar_msub_i64(ptr %a, ptr %b) { +; CHECK-LABEL: test_scalar_msub_i64: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: ldr x8, [x1] +; CHECK-NEXT: ldur x9, [x0, #4] +; CHECK-NEXT: ldr x10, [x0] +; CHECK-NEXT: ldur x12, [x1, #4] +; CHECK-NEXT: mul x11, x9, x8 +; CHECK-NEXT: mul x8, x8, x10 +; CHECK-NEXT: madd x10, x12, x10, x11 +; CHECK-NEXT: msub x8, x12, x9, x8 +; CHECK-NEXT: stur x10, [x0, #4] +; CHECK-NEXT: str x8, [x0] +; CHECK-NEXT: ret +entry: + %0 = load i64, ptr %a, align 8 + %1 = load i64, ptr %b, align 8 + %mul = mul nsw i64 %1, %0 + %_M_imag = getelementptr inbounds i8, ptr %a, i64 4 + %2 = load i64, ptr %_M_imag, align 8 + %_M_imag.i = getelementptr inbounds i8, ptr %b, i64 4 + %3 = load i64, ptr %_M_imag.i, align 8 + %mul3 = mul nsw i64 %3, %2 + %sub = sub nsw i64 %mul, %mul3 + %mul6 = mul nsw i64 %3, %0 + %mul9 = mul nsw i64 %2, %1 + %add = add nsw i64 %mul6, %mul9 + store i64 %add, ptr %_M_imag, align 8 + store i64 %sub, ptr %a, align 8 + ret ptr %a +} -- GitLab From 860a40057da2fbb626fdf17feec18070d95b687b Mon Sep 17 00:00:00 2001 From: Tom Eccles Date: Fri, 8 Mar 2024 10:33:43 +0000 Subject: [PATCH 0005/1674] [flang][NFC] move loadIfRef to FIRBuilder (#84306) This will be useful for OpenMP too. I changed the definition slightly to use `fir::isa_ref_type` (which also includes llvm pointers) because I think it reads better using the common type helpers. There shouldn't be any llvm pointers in lowering so this isn't a functional change. --- flang/include/flang/Optimizer/Builder/FIRBuilder.h | 4 ++++ flang/lib/Lower/OpenACC.cpp | 12 ++---------- flang/lib/Optimizer/Builder/FIRBuilder.cpp | 7 +++++++ 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/flang/include/flang/Optimizer/Builder/FIRBuilder.h b/flang/include/flang/Optimizer/Builder/FIRBuilder.h index bd9b67b14b96..d61bf681be61 100644 --- a/flang/include/flang/Optimizer/Builder/FIRBuilder.h +++ b/flang/include/flang/Optimizer/Builder/FIRBuilder.h @@ -309,6 +309,10 @@ public: void createStoreWithConvert(mlir::Location loc, mlir::Value val, mlir::Value addr); + /// Create a fir.load if \p val is a reference or pointer type. Return the + /// result of the load if it was created, otherwise return \p val + mlir::Value loadIfRef(mlir::Location loc, mlir::Value val); + /// Create a new FuncOp. If the function may have already been created, use /// `addNamedFunction` instead. mlir::func::FuncOp createFunction(mlir::Location loc, llvm::StringRef name, diff --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp index 151077d81ba1..d2c6006ecf91 100644 --- a/flang/lib/Lower/OpenACC.cpp +++ b/flang/lib/Lower/OpenACC.cpp @@ -1041,14 +1041,6 @@ static mlir::Value genLogicalCombiner(fir::FirOpBuilder &builder, return builder.create(loc, value1.getType(), combined); } -static mlir::Value loadIfRef(fir::FirOpBuilder &builder, mlir::Location loc, - mlir::Value value) { - if (mlir::isa( - value.getType())) - return builder.create(loc, value); - return value; -} - static mlir::Value genComparisonCombiner(fir::FirOpBuilder &builder, mlir::Location loc, mlir::arith::CmpIPredicate pred, @@ -1066,8 +1058,8 @@ static mlir::Value genScalarCombiner(fir::FirOpBuilder &builder, mlir::acc::ReductionOperator op, mlir::Type ty, mlir::Value value1, mlir::Value value2) { - value1 = loadIfRef(builder, loc, value1); - value2 = loadIfRef(builder, loc, value2); + value1 = builder.loadIfRef(loc, value1); + value2 = builder.loadIfRef(loc, value2); if (op == mlir::acc::ReductionOperator::AccAdd) { if (ty.isIntOrIndex()) return builder.create(loc, value1, value2); diff --git a/flang/lib/Optimizer/Builder/FIRBuilder.cpp b/flang/lib/Optimizer/Builder/FIRBuilder.cpp index 788c99e40105..12da7412888a 100644 --- a/flang/lib/Optimizer/Builder/FIRBuilder.cpp +++ b/flang/lib/Optimizer/Builder/FIRBuilder.cpp @@ -16,6 +16,7 @@ #include "flang/Optimizer/Builder/Todo.h" #include "flang/Optimizer/Dialect/FIRAttr.h" #include "flang/Optimizer/Dialect/FIROpsSupport.h" +#include "flang/Optimizer/Dialect/FIRType.h" #include "flang/Optimizer/Support/FatalError.h" #include "flang/Optimizer/Support/InternalNames.h" #include "mlir/Dialect/LLVMIR/LLVMDialect.h" @@ -404,6 +405,12 @@ void fir::FirOpBuilder::createStoreWithConvert(mlir::Location loc, create(loc, cast, addr); } +mlir::Value fir::FirOpBuilder::loadIfRef(mlir::Location loc, mlir::Value val) { + if (fir::isa_ref_type(val.getType())) + return create(loc, val); + return val; +} + fir::StringLitOp fir::FirOpBuilder::createStringLitOp(mlir::Location loc, llvm::StringRef data) { auto type = fir::CharacterType::get(getContext(), 1, data.size()); -- GitLab From c76d853c176231b991cfc744e7cb69443ddc2268 Mon Sep 17 00:00:00 2001 From: Tom Eccles Date: Fri, 8 Mar 2024 10:34:00 +0000 Subject: [PATCH 0006/1674] [flang][TBAABuilder] not all loads and stores are inside of functions (#84305) TBAA builder assumed that all loads/stores are inside of functions and hit an assertion once it found loads and stores inside of an omp::ReductionDeclareOp. For now just don't add TBAA tags to those loads and stores. They would end up in a different TBAA tree to the host function after OpenMPIRBuilder inlines them anyway so there isn't an easy way of making this work. --- flang/lib/Optimizer/CodeGen/TBAABuilder.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/flang/lib/Optimizer/CodeGen/TBAABuilder.cpp b/flang/lib/Optimizer/CodeGen/TBAABuilder.cpp index 8e7f59f76383..b1b0e9b766a6 100644 --- a/flang/lib/Optimizer/CodeGen/TBAABuilder.cpp +++ b/flang/lib/Optimizer/CodeGen/TBAABuilder.cpp @@ -102,7 +102,8 @@ void TBAABuilder::attachTBAATag(AliasAnalysisOpInterface op, Type baseFIRType, return; mlir::LLVM::LLVMFuncOp func = op->getParentOfType(); - assert(func && "func.func should have already been converted to llvm.func"); + if (!func) + return; ++tagAttachmentCounter; if (tagAttachmentLimit != kTagAttachmentUnlimited && -- GitLab From 419e8da2dadc7eaa1499935f55303c3f132948c4 Mon Sep 17 00:00:00 2001 From: Christian Sigg Date: Fri, 8 Mar 2024 11:37:10 +0100 Subject: [PATCH 0007/1674] [mlir][bazel] Fix BUILD after 5669660f37ef1800f4a7852577364b024d75e3d8 --- .../llvm-project-overlay/mlir/BUILD.bazel | 101 ++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel index a2f170222684..cca548d51348 100644 --- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel @@ -3637,6 +3637,107 @@ cc_library( ], ) +##---------------------------------------------------------------------------## +# XeGPU dialect. +##---------------------------------------------------------------------------## + +td_library( + name = "XeGPUTdFiles", + srcs = glob(["include/mlir/Dialect/XeGPU/IR/*.td"]), + includes = ["include"], + deps = [ + ":BuiltinDialectTdFiles", + ":OpBaseTdFiles", + ], +) + +gentbl_cc_library( + name = "XeGPUIncGen", + tbl_outs = [ + ( + [ + "-gen-dialect-decls", + "-dialect=xegpu", + ], + "include/mlir/Dialect/XeGPU/IR/XeGPUDialect.h.inc", + ), + ( + [ + "-gen-dialect-defs", + "-dialect=xegpu", + ], + "include/mlir/Dialect/XeGPU/IR/XeGPUDialect.cpp.inc", + ), + ( + ["-gen-op-decls"], + "include/mlir/Dialect/XeGPU/IR/XeGPU.h.inc", + ), + ( + ["-gen-op-defs"], + "include/mlir/Dialect/XeGPU/IR/XeGPU.cpp.inc", + ), + ( + ["-gen-op-doc"], + "g3doc/Dialects/XeGPU/XeGPU.md", + ), + ( + [ + "-gen-typedef-decls", + "-typedefs-dialect=xegpu", + ], + "include/mlir/Dialect/XeGPU/IR/XeGPUTypes.h.inc", + ), + ( + [ + "-gen-typedef-defs", + "-typedefs-dialect=xegpu", + ], + "include/mlir/Dialect/XeGPU/IR/XeGPUTypes.cpp.inc", + ), + ( + ["-gen-enum-decls"], + "include/mlir/Dialect/XeGPU/IR/XeGPUEnums.h.inc", + ), + ( + ["-gen-enum-defs"], + "include/mlir/Dialect/XeGPU/IR/XeGPUEnums.cpp.inc", + ), + ( + [ + "-gen-attrdef-decls", + "-attrdefs-dialect=xegpu", + ], + "include/mlir/Dialect/XeGPU/IR/XeGPUAttrs.h.inc", + ), + ( + [ + "-gen-attrdef-defs", + "-attrdefs-dialect=xegpu", + ], + "include/mlir/Dialect/XeGPU/IR/XeGPUAttrs.cpp.inc", + ), + ], + tblgen = ":mlir-tblgen", + td_file = "include/mlir/Dialect/XeGPU/IR/XeGPU.td", + deps = [":XeGPUTdFiles"], +) + +cc_library( + name = "XeGPUDialect", + srcs = [ + "lib/Dialect/XeGPU/IR/XeGPUDialect.cpp", + "lib/Dialect/XeGPU/IR/XeGPUOps.cpp", + ], + hdrs = ["include/mlir/Dialect/XeGPU/IR/XeGPU.h"], + includes = ["include"], + deps = [ + ":IR", + ":XeGPUIncGen", + "//llvm:Core", + "//llvm:Support", + ], +) + td_library( name = "FuncTdFiles", srcs = [ -- GitLab From dd547d5cca64c1f97d00af0c3a555c2b341161b0 Mon Sep 17 00:00:00 2001 From: Christian Sigg Date: Fri, 8 Mar 2024 11:39:59 +0100 Subject: [PATCH 0008/1674] [mlir][bazel] Fix BUILD after 5669660f37ef1800f4a7852577364b024d75e3d8 Second attempt. --- utils/bazel/llvm-project-overlay/mlir/BUILD.bazel | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel index cca548d51348..cadcbcc5a092 100644 --- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel @@ -9269,6 +9269,7 @@ cc_library( ":VectorTransforms", ":X86VectorDialect", ":X86VectorTransforms", + ":XeGPUDialect", ":config", ], ) -- GitLab From 0f8cb6818d788c34d70d586f701469a5adabfce1 Mon Sep 17 00:00:00 2001 From: David Spickett Date: Fri, 8 Mar 2024 10:42:31 +0000 Subject: [PATCH 0009/1674] [lldb][Docs] Update the build guide * gmake is needed on FreeBSD * pkg can install libxml2 on FreeBSD * Make the swig note into an RST note box. --- lldb/docs/resources/build.rst | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/lldb/docs/resources/build.rst b/lldb/docs/resources/build.rst index 55fe73c52f61..fe8e293db642 100644 --- a/lldb/docs/resources/build.rst +++ b/lldb/docs/resources/build.rst @@ -33,6 +33,9 @@ scripting support. * `Python `_ * `SWIG `_ 4 or later. +If you are on FreeBSD or NetBSD, you will need to install ``gmake`` for building +the test programs. On other platforms ``make`` is used. + .. _Optional Dependencies: Optional Dependencies @@ -71,16 +74,16 @@ commands below. $ yum install libedit-devel libxml2-devel ncurses-devel python-devel swig $ sudo apt-get install build-essential swig python3-dev libedit-dev libncurses5-dev - $ pkg install swig python + $ pkg install swig python libxml2 $ pkgin install swig python36 cmake ninja-build $ brew install swig cmake ninja -Note that there's an `incompatibility -`_ between Python version 3.7 and later -and swig versions older than 4.0.0 which makes builds of LLDB using debug -versions of python unusable. This primarily affects Windows, as debug builds of -LLDB must use debug python as well. - +.. note:: + There is an `incompatibility + `_ between Python version 3.7 and later + and swig versions older than 4.0.0 which makes builds of LLDB using debug + versions of python unusable. This primarily affects Windows, as debug builds of + LLDB must use debug python as well. Windows ******* -- GitLab From 50633868538d4fd6875a7de57804b3dde5c86108 Mon Sep 17 00:00:00 2001 From: Christian Sigg Date: Fri, 8 Mar 2024 11:47:38 +0100 Subject: [PATCH 0010/1674] [clang][bazel] Fix BUILD after da00c60dae0040185dc45039c4397f6e746548e9 --- utils/bazel/llvm-project-overlay/clang/BUILD.bazel | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/bazel/llvm-project-overlay/clang/BUILD.bazel b/utils/bazel/llvm-project-overlay/clang/BUILD.bazel index 6d3904f84673..339706077b52 100644 --- a/utils/bazel/llvm-project-overlay/clang/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/clang/BUILD.bazel @@ -2092,6 +2092,7 @@ cc_library( deps = [ ":ast", ":basic", + ":frontend", ":lex", ":sema", ":serialization_attr_gen", -- GitLab From 005e182ecbf0099005f7d3c33c2b7556abb515e4 Mon Sep 17 00:00:00 2001 From: Christian Sigg Date: Fri, 8 Mar 2024 11:55:19 +0100 Subject: [PATCH 0011/1674] [clang][bazel] Fix BUILD after da00c60dae0040185dc45039c4397f6e746548e9 Second attempt. This also reverts 50633868538d4fd6875a7de57804b3dde5c86108. --- utils/bazel/llvm-project-overlay/clang/BUILD.bazel | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/utils/bazel/llvm-project-overlay/clang/BUILD.bazel b/utils/bazel/llvm-project-overlay/clang/BUILD.bazel index 339706077b52..d6b124f9d8e4 100644 --- a/utils/bazel/llvm-project-overlay/clang/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/clang/BUILD.bazel @@ -2079,6 +2079,7 @@ cc_library( "include/clang/Serialization/AttrPCHRead.inc", "include/clang/Serialization/AttrPCHWrite.inc", ] + glob([ + "include/clang/Frontend/*.h", "lib/Serialization/*.cpp", "lib/Serialization/*.h", ]), @@ -2090,12 +2091,14 @@ cc_library( "include/clang/Serialization/*.def", ]), deps = [ + ":apinotes", ":ast", ":basic", - ":frontend", + ":driver", ":lex", ":sema", ":serialization_attr_gen", + ":static_analyzer_core_options", ":type_nodes_gen", "//llvm:BitReader", "//llvm:BitWriter", -- GitLab From ee94bd20ba09975de493675dd6a0b7fc7dd5cece Mon Sep 17 00:00:00 2001 From: Krystian Stasiowski Date: Fri, 8 Mar 2024 05:57:04 -0500 Subject: [PATCH 0012/1674] Revert "[Clang][Sema] Fix crash when using name of UnresolvedUsingValueDecl with template arguments (#83842)" (#84457) This reverts commit a642eb89bdaf10c6b4994fc1187de27b441236ed (see #83842) --- clang/docs/ReleaseNotes.rst | 2 -- clang/lib/Sema/SemaDecl.cpp | 5 +--- clang/lib/Sema/SemaTemplate.cpp | 10 +++---- .../unqual-unresolved-using-value.cpp | 30 ------------------- 4 files changed, 5 insertions(+), 42 deletions(-) delete mode 100644 clang/test/SemaTemplate/unqual-unresolved-using-value.cpp diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index 225ca85e1811..0ee2801766a9 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -332,8 +332,6 @@ Bug Fixes to C++ Support our attention by an attempt to fix in (#GH77703). Fixes (#GH83385). - Fix evaluation of some immediate calls in default arguments. Fixes (#GH80630) -- Fix a crash when an explicit template argument list is used with a name for which lookup - finds a non-template function and a dependent using declarator. - Fixed an issue where the ``RequiresExprBody`` was involved in the lambda dependency calculation. (#GH56556), (#GH82849). - Fix a bug where overload resolution falsely reported an ambiguity when it was comparing diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp index 910d0dfbf9f6..1f4a041e88df 100644 --- a/clang/lib/Sema/SemaDecl.cpp +++ b/clang/lib/Sema/SemaDecl.cpp @@ -1110,9 +1110,7 @@ Corrected: // unqualified-id followed by a < and name lookup finds either one // or more functions or finds nothing. if (!IsFilteredTemplateName) - FilterAcceptableTemplateNames(Result, - /*AllowFunctionTemplates=*/true, - /*AllowDependent=*/true); + FilterAcceptableTemplateNames(Result); bool IsFunctionTemplate; bool IsVarTemplate; @@ -1122,7 +1120,6 @@ Corrected: Template = Context.getOverloadedTemplateName(Result.begin(), Result.end()); } else if (!Result.empty()) { - assert(!Result.isUnresolvableResult()); auto *TD = cast(getAsTemplateNameDecl( *Result.begin(), /*AllowFunctionTemplates=*/true, /*AllowDependent=*/false)); diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp index 7e91815c2d52..83eee83aa6ca 100644 --- a/clang/lib/Sema/SemaTemplate.cpp +++ b/clang/lib/Sema/SemaTemplate.cpp @@ -491,20 +491,18 @@ bool Sema::LookupTemplateName(LookupResult &Found, // To keep our behavior consistent, we apply the "finds nothing" part in // all language modes, and diagnose the empty lookup in ActOnCallExpr if we // successfully form a call to an undeclared template-id. - bool AnyFunctions = - getLangOpts().CPlusPlus20 && llvm::any_of(Found, [](NamedDecl *ND) { + bool AllFunctions = + getLangOpts().CPlusPlus20 && llvm::all_of(Found, [](NamedDecl *ND) { return isa(ND->getUnderlyingDecl()); }); - if (AnyFunctions || (Found.empty() && !IsDependent)) { + if (AllFunctions || (Found.empty() && !IsDependent)) { // If lookup found any functions, or if this is a name that can only be // used for a function, then strongly assume this is a function // template-id. *ATK = (Found.empty() && Found.getLookupName().isIdentifier()) ? AssumedTemplateKind::FoundNothing : AssumedTemplateKind::FoundFunctions; - FilterAcceptableTemplateNames(Found, - /*AllowFunctionTemplates*/ true, - /*AllowDependent*/ true); + Found.clear(); return false; } } diff --git a/clang/test/SemaTemplate/unqual-unresolved-using-value.cpp b/clang/test/SemaTemplate/unqual-unresolved-using-value.cpp deleted file mode 100644 index 688e7a0a10b7..000000000000 --- a/clang/test/SemaTemplate/unqual-unresolved-using-value.cpp +++ /dev/null @@ -1,30 +0,0 @@ -// RUN: %clang_cc1 -std=c++20 -fsyntax-only -verify %s - -template -struct A : T { - using T::f; - using T::g; - using T::h; - - void f(); - void g(); - - void i() { - f(); - g(); // expected-error{{no member named 'g' in 'A'}} - h(); // expected-error{{expected '(' for function-style cast or type construction}} - // expected-error@-1{{expected expression}} - } -}; - -struct B { - template - void f(); - - void g(); - - template - void h(); -}; - -template struct A; // expected-note{{in instantiation of member function 'A::i' requested here}} -- GitLab From 6a0618a0289cb0c23ef3e5c820418650cc1d0fdc Mon Sep 17 00:00:00 2001 From: Amirreza Ashouri Date: Fri, 8 Mar 2024 14:27:35 +0330 Subject: [PATCH 0013/1674] [clang][NFC] Refactor `clang/test/SemaCXX/type-traits.cpp` to use modern `static_assert` (#77584) static_assert is supported in all modes, so the T/F macros aren't needed anymore. --- clang/test/SemaCXX/type-traits.cpp | 4745 ++++++++++++++-------------- 1 file changed, 2364 insertions(+), 2381 deletions(-) diff --git a/clang/test/SemaCXX/type-traits.cpp b/clang/test/SemaCXX/type-traits.cpp index 831de2589dcb..f50f51cc7098 100644 --- a/clang/test/SemaCXX/type-traits.cpp +++ b/clang/test/SemaCXX/type-traits.cpp @@ -1,10 +1,8 @@ -// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -verify -std=gnu++11 -fblocks -Wno-deprecated-builtins -Wno-defaulted-function-deleted %s -// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -verify -std=gnu++14 -fblocks -Wno-deprecated-builtins -Wno-defaulted-function-deleted %s -// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -verify -std=gnu++17 -fblocks -Wno-deprecated-builtins -Wno-defaulted-function-deleted %s -// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -verify -std=gnu++20 -fblocks -Wno-deprecated-builtins -Wno-defaulted-function-deleted %s +// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -verify -std=gnu++11 -fblocks -Wno-deprecated-builtins -Wno-defaulted-function-deleted -Wno-c++17-extensions %s +// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -verify -std=gnu++14 -fblocks -Wno-deprecated-builtins -Wno-defaulted-function-deleted -Wno-c++17-extensions %s +// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -verify -std=gnu++17 -fblocks -Wno-deprecated-builtins -Wno-defaulted-function-deleted %s +// RUN: %clang_cc1 -triple x86_64-apple-darwin10 -fsyntax-only -verify -std=gnu++20 -fblocks -Wno-deprecated-builtins -Wno-defaulted-function-deleted %s -#define T(b) (b) ? 1 : -1 -#define F(b) (b) ? -1 : 1 struct NonPOD { NonPOD(int); }; typedef NonPOD NonPODAr[10]; @@ -214,56 +212,56 @@ struct HasVirtBase : virtual ACompleteType {}; void is_pod() { - { int arr[T(__is_pod(int))]; } - { int arr[T(__is_pod(Enum))]; } - { int arr[T(__is_pod(POD))]; } - { int arr[T(__is_pod(Int))]; } - { int arr[T(__is_pod(IntAr))]; } - { int arr[T(__is_pod(Statics))]; } - { int arr[T(__is_pod(Empty))]; } - { int arr[T(__is_pod(EmptyUnion))]; } - { int arr[T(__is_pod(Union))]; } - { int arr[T(__is_pod(HasFunc))]; } - { int arr[T(__is_pod(HasOp))]; } - { int arr[T(__is_pod(HasConv))]; } - { int arr[T(__is_pod(HasAssign))]; } - { int arr[T(__is_pod(IntArNB))]; } - { int arr[T(__is_pod(HasAnonymousUnion))]; } - { int arr[T(__is_pod(Vector))]; } - { int arr[T(__is_pod(VectorExt))]; } - { int arr[T(__is_pod(Derives))]; } - { int arr[T(__is_pod(DerivesAr))]; } - { int arr[T(__is_pod(DerivesArNB))]; } - { int arr[T(__is_pod(DerivesEmpty))]; } - { int arr[T(__is_pod(HasPriv))]; } - { int arr[T(__is_pod(HasProt))]; } - { int arr[T(__is_pod(DerivesHasPriv))]; } - { int arr[T(__is_pod(DerivesHasProt))]; } - - { int arr[F(__is_pod(HasCons))]; } - { int arr[F(__is_pod(HasCopyAssign))]; } - { int arr[F(__is_pod(HasMoveAssign))]; } - { int arr[F(__is_pod(HasDest))]; } - { int arr[F(__is_pod(HasRef))]; } - { int arr[F(__is_pod(HasVirt))]; } - { int arr[F(__is_pod(DerivesHasCons))]; } - { int arr[F(__is_pod(DerivesHasCopyAssign))]; } - { int arr[F(__is_pod(DerivesHasMoveAssign))]; } - { int arr[F(__is_pod(DerivesHasDest))]; } - { int arr[F(__is_pod(DerivesHasRef))]; } - { int arr[F(__is_pod(DerivesHasVirt))]; } - { int arr[F(__is_pod(NonPOD))]; } - { int arr[F(__is_pod(HasNonPOD))]; } - { int arr[F(__is_pod(NonPODAr))]; } - { int arr[F(__is_pod(NonPODArNB))]; } - { int arr[F(__is_pod(void))]; } - { int arr[F(__is_pod(cvoid))]; } -// { int arr[F(__is_pod(NonPODUnion))]; } - - { int arr[T(__is_pod(ACompleteType))]; } - { int arr[F(__is_pod(AnIncompleteType))]; } // expected-error {{incomplete type}} - { int arr[F(__is_pod(AnIncompleteType[]))]; } // expected-error {{incomplete type}} - { int arr[F(__is_pod(AnIncompleteType[1]))]; } // expected-error {{incomplete type}} + static_assert(__is_pod(int)); + static_assert(__is_pod(Enum)); + static_assert(__is_pod(POD)); + static_assert(__is_pod(Int)); + static_assert(__is_pod(IntAr)); + static_assert(__is_pod(Statics)); + static_assert(__is_pod(Empty)); + static_assert(__is_pod(EmptyUnion)); + static_assert(__is_pod(Union)); + static_assert(__is_pod(HasFunc)); + static_assert(__is_pod(HasOp)); + static_assert(__is_pod(HasConv)); + static_assert(__is_pod(HasAssign)); + static_assert(__is_pod(IntArNB)); + static_assert(__is_pod(HasAnonymousUnion)); + static_assert(__is_pod(Vector)); + static_assert(__is_pod(VectorExt)); + static_assert(__is_pod(Derives)); + static_assert(__is_pod(DerivesAr)); + static_assert(__is_pod(DerivesArNB)); + static_assert(__is_pod(DerivesEmpty)); + static_assert(__is_pod(HasPriv)); + static_assert(__is_pod(HasProt)); + static_assert(__is_pod(DerivesHasPriv)); + static_assert(__is_pod(DerivesHasProt)); + + static_assert(!__is_pod(HasCons)); + static_assert(!__is_pod(HasCopyAssign)); + static_assert(!__is_pod(HasMoveAssign)); + static_assert(!__is_pod(HasDest)); + static_assert(!__is_pod(HasRef)); + static_assert(!__is_pod(HasVirt)); + static_assert(!__is_pod(DerivesHasCons)); + static_assert(!__is_pod(DerivesHasCopyAssign)); + static_assert(!__is_pod(DerivesHasMoveAssign)); + static_assert(!__is_pod(DerivesHasDest)); + static_assert(!__is_pod(DerivesHasRef)); + static_assert(!__is_pod(DerivesHasVirt)); + static_assert(!__is_pod(NonPOD)); + static_assert(!__is_pod(HasNonPOD)); + static_assert(!__is_pod(NonPODAr)); + static_assert(!__is_pod(NonPODArNB)); + static_assert(!__is_pod(void)); + static_assert(!__is_pod(cvoid)); +// static_assert(!__is_pod(NonPODUnion)); + + static_assert(__is_pod(ACompleteType)); + static_assert(!__is_pod(AnIncompleteType)); // expected-error {{incomplete type}} + static_assert(!__is_pod(AnIncompleteType[])); // expected-error {{incomplete type}} + static_assert(!__is_pod(AnIncompleteType[1])); // expected-error {{incomplete type}} } typedef Empty EmptyAr[10]; @@ -275,56 +273,56 @@ struct DerivesVirt : virtual POD {}; void is_empty() { - { int arr[T(__is_empty(Empty))]; } - { int arr[T(__is_empty(DerivesEmpty))]; } - { int arr[T(__is_empty(HasCons))]; } - { int arr[T(__is_empty(HasCopyAssign))]; } - { int arr[T(__is_empty(HasMoveAssign))]; } - { int arr[T(__is_empty(HasDest))]; } - { int arr[T(__is_empty(HasFunc))]; } - { int arr[T(__is_empty(HasOp))]; } - { int arr[T(__is_empty(HasConv))]; } - { int arr[T(__is_empty(HasAssign))]; } - { int arr[T(__is_empty(Bit0))]; } - { int arr[T(__is_empty(Bit0Cons))]; } - - { int arr[F(__is_empty(Int))]; } - { int arr[F(__is_empty(POD))]; } - { int arr[F(__is_empty(EmptyUnion))]; } - { int arr[F(__is_empty(IncompleteUnion))]; } - { int arr[F(__is_empty(EmptyAr))]; } - { int arr[F(__is_empty(HasRef))]; } - { int arr[F(__is_empty(HasVirt))]; } - { int arr[F(__is_empty(AnonBitOnly))]; } - { int arr[F(__is_empty(BitOnly))]; } - { int arr[F(__is_empty(void))]; } - { int arr[F(__is_empty(IntArNB))]; } - { int arr[F(__is_empty(HasAnonymousUnion))]; } -// { int arr[F(__is_empty(DerivesVirt))]; } - - { int arr[T(__is_empty(ACompleteType))]; } - { int arr[F(__is_empty(AnIncompleteType))]; } // expected-error {{incomplete type}} - { int arr[F(__is_empty(AnIncompleteType[]))]; } - { int arr[F(__is_empty(AnIncompleteType[1]))]; } + static_assert(__is_empty(Empty)); + static_assert(__is_empty(DerivesEmpty)); + static_assert(__is_empty(HasCons)); + static_assert(__is_empty(HasCopyAssign)); + static_assert(__is_empty(HasMoveAssign)); + static_assert(__is_empty(HasDest)); + static_assert(__is_empty(HasFunc)); + static_assert(__is_empty(HasOp)); + static_assert(__is_empty(HasConv)); + static_assert(__is_empty(HasAssign)); + static_assert(__is_empty(Bit0)); + static_assert(__is_empty(Bit0Cons)); + + static_assert(!__is_empty(Int)); + static_assert(!__is_empty(POD)); + static_assert(!__is_empty(EmptyUnion)); + static_assert(!__is_empty(IncompleteUnion)); + static_assert(!__is_empty(EmptyAr)); + static_assert(!__is_empty(HasRef)); + static_assert(!__is_empty(HasVirt)); + static_assert(!__is_empty(AnonBitOnly)); + static_assert(!__is_empty(BitOnly)); + static_assert(!__is_empty(void)); + static_assert(!__is_empty(IntArNB)); + static_assert(!__is_empty(HasAnonymousUnion)); +// static_assert(!__is_empty(DerivesVirt)); + + static_assert(__is_empty(ACompleteType)); + static_assert(!__is_empty(AnIncompleteType)); // expected-error {{incomplete type}} + static_assert(!__is_empty(AnIncompleteType[])); + static_assert(!__is_empty(AnIncompleteType[1])); } typedef Derives ClassType; void is_class() { - { int arr[T(__is_class(Derives))]; } - { int arr[T(__is_class(HasPriv))]; } - { int arr[T(__is_class(ClassType))]; } - { int arr[T(__is_class(HasAnonymousUnion))]; } - - { int arr[F(__is_class(int))]; } - { int arr[F(__is_class(Enum))]; } - { int arr[F(__is_class(Int))]; } - { int arr[F(__is_class(IntAr))]; } - { int arr[F(__is_class(DerivesAr))]; } - { int arr[F(__is_class(Union))]; } - { int arr[F(__is_class(cvoid))]; } - { int arr[F(__is_class(IntArNB))]; } + static_assert(__is_class(Derives)); + static_assert(__is_class(HasPriv)); + static_assert(__is_class(ClassType)); + static_assert(__is_class(HasAnonymousUnion)); + + static_assert(!__is_class(int)); + static_assert(!__is_class(Enum)); + static_assert(!__is_class(Int)); + static_assert(!__is_class(IntAr)); + static_assert(!__is_class(DerivesAr)); + static_assert(!__is_class(Union)); + static_assert(!__is_class(cvoid)); + static_assert(!__is_class(IntArNB)); } typedef Union UnionAr[10]; @@ -332,17 +330,17 @@ typedef Union UnionType; void is_union() { - { int arr[T(__is_union(Union))]; } - { int arr[T(__is_union(UnionType))]; } - - { int arr[F(__is_union(int))]; } - { int arr[F(__is_union(Enum))]; } - { int arr[F(__is_union(Int))]; } - { int arr[F(__is_union(IntAr))]; } - { int arr[F(__is_union(UnionAr))]; } - { int arr[F(__is_union(cvoid))]; } - { int arr[F(__is_union(IntArNB))]; } - { int arr[F(__is_union(HasAnonymousUnion))]; } + static_assert(__is_union(Union)); + static_assert(__is_union(UnionType)); + + static_assert(!__is_union(int)); + static_assert(!__is_union(Enum)); + static_assert(!__is_union(Int)); + static_assert(!__is_union(IntAr)); + static_assert(!__is_union(UnionAr)); + static_assert(!__is_union(cvoid)); + static_assert(!__is_union(IntArNB)); + static_assert(!__is_union(HasAnonymousUnion)); } typedef Enum EnumType; @@ -350,57 +348,57 @@ typedef EnumClass EnumClassType; void is_enum() { - { int arr[T(__is_enum(Enum))]; } - { int arr[T(__is_enum(EnumType))]; } - { int arr[T(__is_enum(SignedEnum))]; } - { int arr[T(__is_enum(UnsignedEnum))]; } - - { int arr[T(__is_enum(EnumClass))]; } - { int arr[T(__is_enum(EnumClassType))]; } - { int arr[T(__is_enum(SignedEnumClass))]; } - { int arr[T(__is_enum(UnsignedEnumClass))]; } - - { int arr[F(__is_enum(int))]; } - { int arr[F(__is_enum(Union))]; } - { int arr[F(__is_enum(Int))]; } - { int arr[F(__is_enum(IntAr))]; } - { int arr[F(__is_enum(UnionAr))]; } - { int arr[F(__is_enum(Derives))]; } - { int arr[F(__is_enum(ClassType))]; } - { int arr[F(__is_enum(cvoid))]; } - { int arr[F(__is_enum(IntArNB))]; } - { int arr[F(__is_enum(HasAnonymousUnion))]; } - { int arr[F(__is_enum(AnIncompleteType))]; } - { int arr[F(__is_enum(AnIncompleteTypeAr))]; } - { int arr[F(__is_enum(AnIncompleteTypeArMB))]; } - { int arr[F(__is_enum(AnIncompleteTypeArNB))]; } + static_assert(__is_enum(Enum)); + static_assert(__is_enum(EnumType)); + static_assert(__is_enum(SignedEnum)); + static_assert(__is_enum(UnsignedEnum)); + + static_assert(__is_enum(EnumClass)); + static_assert(__is_enum(EnumClassType)); + static_assert(__is_enum(SignedEnumClass)); + static_assert(__is_enum(UnsignedEnumClass)); + + static_assert(!__is_enum(int)); + static_assert(!__is_enum(Union)); + static_assert(!__is_enum(Int)); + static_assert(!__is_enum(IntAr)); + static_assert(!__is_enum(UnionAr)); + static_assert(!__is_enum(Derives)); + static_assert(!__is_enum(ClassType)); + static_assert(!__is_enum(cvoid)); + static_assert(!__is_enum(IntArNB)); + static_assert(!__is_enum(HasAnonymousUnion)); + static_assert(!__is_enum(AnIncompleteType)); + static_assert(!__is_enum(AnIncompleteTypeAr)); + static_assert(!__is_enum(AnIncompleteTypeArMB)); + static_assert(!__is_enum(AnIncompleteTypeArNB)); } void is_scoped_enum() { - static_assert(!__is_scoped_enum(Enum), ""); - static_assert(!__is_scoped_enum(EnumType), ""); - static_assert(!__is_scoped_enum(SignedEnum), ""); - static_assert(!__is_scoped_enum(UnsignedEnum), ""); - - static_assert(__is_scoped_enum(EnumClass), ""); - static_assert(__is_scoped_enum(EnumClassType), ""); - static_assert(__is_scoped_enum(SignedEnumClass), ""); - static_assert(__is_scoped_enum(UnsignedEnumClass), ""); - - static_assert(!__is_scoped_enum(int), ""); - static_assert(!__is_scoped_enum(Union), ""); - static_assert(!__is_scoped_enum(Int), ""); - static_assert(!__is_scoped_enum(IntAr), ""); - static_assert(!__is_scoped_enum(UnionAr), ""); - static_assert(!__is_scoped_enum(Derives), ""); - static_assert(!__is_scoped_enum(ClassType), ""); - static_assert(!__is_scoped_enum(cvoid), ""); - static_assert(!__is_scoped_enum(IntArNB), ""); - static_assert(!__is_scoped_enum(HasAnonymousUnion), ""); - static_assert(!__is_scoped_enum(AnIncompleteType), ""); - static_assert(!__is_scoped_enum(AnIncompleteTypeAr), ""); - static_assert(!__is_scoped_enum(AnIncompleteTypeArMB), ""); - static_assert(!__is_scoped_enum(AnIncompleteTypeArNB), ""); + static_assert(!__is_scoped_enum(Enum)); + static_assert(!__is_scoped_enum(EnumType)); + static_assert(!__is_scoped_enum(SignedEnum)); + static_assert(!__is_scoped_enum(UnsignedEnum)); + + static_assert(__is_scoped_enum(EnumClass)); + static_assert(__is_scoped_enum(EnumClassType)); + static_assert(__is_scoped_enum(SignedEnumClass)); + static_assert(__is_scoped_enum(UnsignedEnumClass)); + + static_assert(!__is_scoped_enum(int)); + static_assert(!__is_scoped_enum(Union)); + static_assert(!__is_scoped_enum(Int)); + static_assert(!__is_scoped_enum(IntAr)); + static_assert(!__is_scoped_enum(UnionAr)); + static_assert(!__is_scoped_enum(Derives)); + static_assert(!__is_scoped_enum(ClassType)); + static_assert(!__is_scoped_enum(cvoid)); + static_assert(!__is_scoped_enum(IntArNB)); + static_assert(!__is_scoped_enum(HasAnonymousUnion)); + static_assert(!__is_scoped_enum(AnIncompleteType)); + static_assert(!__is_scoped_enum(AnIncompleteTypeAr)); + static_assert(!__is_scoped_enum(AnIncompleteTypeArMB)); + static_assert(!__is_scoped_enum(AnIncompleteTypeArNB)); } struct FinalClass final { @@ -421,20 +419,20 @@ struct PotentiallyFinal final { }; void is_final() { - { int arr[T(__is_final(FinalClass))]; } - { int arr[T(__is_final(PotentiallyFinal))]; } - { int arr[T(__is_final(PotentiallyFinal))]; } - - { int arr[F(__is_final(int))]; } - { int arr[F(__is_final(Union))]; } - { int arr[F(__is_final(Int))]; } - { int arr[F(__is_final(IntAr))]; } - { int arr[F(__is_final(UnionAr))]; } - { int arr[F(__is_final(Derives))]; } - { int arr[F(__is_final(ClassType))]; } - { int arr[F(__is_final(cvoid))]; } - { int arr[F(__is_final(IntArNB))]; } - { int arr[F(__is_final(HasAnonymousUnion))]; } + static_assert(__is_final(FinalClass)); + static_assert(__is_final(PotentiallyFinal)); + static_assert(__is_final(PotentiallyFinal)); + + static_assert(!__is_final(int)); + static_assert(!__is_final(Union)); + static_assert(!__is_final(Int)); + static_assert(!__is_final(IntAr)); + static_assert(!__is_final(UnionAr)); + static_assert(!__is_final(Derives)); + static_assert(!__is_final(ClassType)); + static_assert(!__is_final(cvoid)); + static_assert(!__is_final(IntArNB)); + static_assert(!__is_final(HasAnonymousUnion)); } @@ -443,78 +441,78 @@ struct InheritPolymorph : Polymorph {}; void is_polymorphic() { - { int arr[T(__is_polymorphic(Polymorph))]; } - { int arr[T(__is_polymorphic(InheritPolymorph))]; } - - { int arr[F(__is_polymorphic(int))]; } - { int arr[F(__is_polymorphic(Union))]; } - { int arr[F(__is_polymorphic(IncompleteUnion))]; } - { int arr[F(__is_polymorphic(Int))]; } - { int arr[F(__is_polymorphic(IntAr))]; } - { int arr[F(__is_polymorphic(UnionAr))]; } - { int arr[F(__is_polymorphic(Derives))]; } - { int arr[F(__is_polymorphic(ClassType))]; } - { int arr[F(__is_polymorphic(Enum))]; } - { int arr[F(__is_polymorphic(cvoid))]; } - { int arr[F(__is_polymorphic(IntArNB))]; } + static_assert(__is_polymorphic(Polymorph)); + static_assert(__is_polymorphic(InheritPolymorph)); + + static_assert(!__is_polymorphic(int)); + static_assert(!__is_polymorphic(Union)); + static_assert(!__is_polymorphic(IncompleteUnion)); + static_assert(!__is_polymorphic(Int)); + static_assert(!__is_polymorphic(IntAr)); + static_assert(!__is_polymorphic(UnionAr)); + static_assert(!__is_polymorphic(Derives)); + static_assert(!__is_polymorphic(ClassType)); + static_assert(!__is_polymorphic(Enum)); + static_assert(!__is_polymorphic(cvoid)); + static_assert(!__is_polymorphic(IntArNB)); } void is_integral() { - int t01[T(__is_integral(bool))]; - int t02[T(__is_integral(char))]; - int t03[T(__is_integral(signed char))]; - int t04[T(__is_integral(unsigned char))]; - //int t05[T(__is_integral(char16_t))]; - //int t06[T(__is_integral(char32_t))]; - int t07[T(__is_integral(wchar_t))]; - int t08[T(__is_integral(short))]; - int t09[T(__is_integral(unsigned short))]; - int t10[T(__is_integral(int))]; - int t11[T(__is_integral(unsigned int))]; - int t12[T(__is_integral(long))]; - int t13[T(__is_integral(unsigned long))]; - - int t21[F(__is_integral(float))]; - int t22[F(__is_integral(double))]; - int t23[F(__is_integral(long double))]; - int t24[F(__is_integral(Union))]; - int t25[F(__is_integral(UnionAr))]; - int t26[F(__is_integral(Derives))]; - int t27[F(__is_integral(ClassType))]; - int t28[F(__is_integral(Enum))]; - int t29[F(__is_integral(void))]; - int t30[F(__is_integral(cvoid))]; - int t31[F(__is_integral(IntArNB))]; + static_assert(__is_integral(bool)); + static_assert(__is_integral(char)); + static_assert(__is_integral(signed char)); + static_assert(__is_integral(unsigned char)); + //static_assert(__is_integral(char16_t)); + //static_assert(__is_integral(char32_t)); + static_assert(__is_integral(wchar_t)); + static_assert(__is_integral(short)); + static_assert(__is_integral(unsigned short)); + static_assert(__is_integral(int)); + static_assert(__is_integral(unsigned int)); + static_assert(__is_integral(long)); + static_assert(__is_integral(unsigned long)); + + static_assert(!__is_integral(float)); + static_assert(!__is_integral(double)); + static_assert(!__is_integral(long double)); + static_assert(!__is_integral(Union)); + static_assert(!__is_integral(UnionAr)); + static_assert(!__is_integral(Derives)); + static_assert(!__is_integral(ClassType)); + static_assert(!__is_integral(Enum)); + static_assert(!__is_integral(void)); + static_assert(!__is_integral(cvoid)); + static_assert(!__is_integral(IntArNB)); } void is_floating_point() { - int t01[T(__is_floating_point(float))]; - int t02[T(__is_floating_point(double))]; - int t03[T(__is_floating_point(long double))]; - - int t11[F(__is_floating_point(bool))]; - int t12[F(__is_floating_point(char))]; - int t13[F(__is_floating_point(signed char))]; - int t14[F(__is_floating_point(unsigned char))]; - //int t15[F(__is_floating_point(char16_t))]; - //int t16[F(__is_floating_point(char32_t))]; - int t17[F(__is_floating_point(wchar_t))]; - int t18[F(__is_floating_point(short))]; - int t19[F(__is_floating_point(unsigned short))]; - int t20[F(__is_floating_point(int))]; - int t21[F(__is_floating_point(unsigned int))]; - int t22[F(__is_floating_point(long))]; - int t23[F(__is_floating_point(unsigned long))]; - int t24[F(__is_floating_point(Union))]; - int t25[F(__is_floating_point(UnionAr))]; - int t26[F(__is_floating_point(Derives))]; - int t27[F(__is_floating_point(ClassType))]; - int t28[F(__is_floating_point(Enum))]; - int t29[F(__is_floating_point(void))]; - int t30[F(__is_floating_point(cvoid))]; - int t31[F(__is_floating_point(IntArNB))]; + static_assert(__is_floating_point(float)); + static_assert(__is_floating_point(double)); + static_assert(__is_floating_point(long double)); + + static_assert(!__is_floating_point(bool)); + static_assert(!__is_floating_point(char)); + static_assert(!__is_floating_point(signed char)); + static_assert(!__is_floating_point(unsigned char)); + //static_assert(!__is_floating_point(char16_t)); + //static_assert(!__is_floating_point(char32_t)); + static_assert(!__is_floating_point(wchar_t)); + static_assert(!__is_floating_point(short)); + static_assert(!__is_floating_point(unsigned short)); + static_assert(!__is_floating_point(int)); + static_assert(!__is_floating_point(unsigned int)); + static_assert(!__is_floating_point(long)); + static_assert(!__is_floating_point(unsigned long)); + static_assert(!__is_floating_point(Union)); + static_assert(!__is_floating_point(UnionAr)); + static_assert(!__is_floating_point(Derives)); + static_assert(!__is_floating_point(ClassType)); + static_assert(!__is_floating_point(Enum)); + static_assert(!__is_floating_point(void)); + static_assert(!__is_floating_point(cvoid)); + static_assert(!__is_floating_point(IntArNB)); } template @@ -537,274 +535,274 @@ void is_aggregate() __is_aggregate(IncompleteUnion); // expected-error {{incomplete type}} // Valid since LWG3823 - static_assert(__is_aggregate(AnIncompleteType[]), ""); - static_assert(__is_aggregate(AnIncompleteType[1]), ""); - static_assert(__is_aggregate(AnIncompleteTypeAr), ""); - static_assert(__is_aggregate(AnIncompleteTypeArNB), ""); - static_assert(__is_aggregate(AnIncompleteTypeArMB), ""); - - static_assert(!__is_aggregate(NonPOD), ""); - static_assert(__is_aggregate(NonPODAr), ""); - static_assert(__is_aggregate(NonPODArNB), ""); - static_assert(__is_aggregate(NonPODArMB), ""); - - static_assert(!__is_aggregate(Enum), ""); - static_assert(__is_aggregate(POD), ""); - static_assert(__is_aggregate(Empty), ""); - static_assert(__is_aggregate(EmptyAr), ""); - static_assert(__is_aggregate(EmptyArNB), ""); - static_assert(__is_aggregate(EmptyArMB), ""); - static_assert(!__is_aggregate(void), ""); - static_assert(!__is_aggregate(const volatile void), ""); - static_assert(!__is_aggregate(int), ""); - static_assert(__is_aggregate(IntAr), ""); - static_assert(__is_aggregate(IntArNB), ""); - static_assert(__is_aggregate(EmptyUnion), ""); - static_assert(__is_aggregate(Union), ""); - static_assert(__is_aggregate(Statics), ""); - static_assert(__is_aggregate(HasFunc), ""); - static_assert(__is_aggregate(HasOp), ""); - static_assert(__is_aggregate(HasAssign), ""); - static_assert(__is_aggregate(HasAnonymousUnion), ""); - - static_assert(__is_aggregate(Derives) == TrueAfterCpp14, ""); - static_assert(__is_aggregate(DerivesAr), ""); - static_assert(__is_aggregate(DerivesArNB), ""); - static_assert(!__is_aggregate(HasCons), ""); + static_assert(__is_aggregate(AnIncompleteType[])); + static_assert(__is_aggregate(AnIncompleteType[1])); + static_assert(__is_aggregate(AnIncompleteTypeAr)); + static_assert(__is_aggregate(AnIncompleteTypeArNB)); + static_assert(__is_aggregate(AnIncompleteTypeArMB)); + + static_assert(!__is_aggregate(NonPOD)); + static_assert(__is_aggregate(NonPODAr)); + static_assert(__is_aggregate(NonPODArNB)); + static_assert(__is_aggregate(NonPODArMB)); + + static_assert(!__is_aggregate(Enum)); + static_assert(__is_aggregate(POD)); + static_assert(__is_aggregate(Empty)); + static_assert(__is_aggregate(EmptyAr)); + static_assert(__is_aggregate(EmptyArNB)); + static_assert(__is_aggregate(EmptyArMB)); + static_assert(!__is_aggregate(void)); + static_assert(!__is_aggregate(const volatile void)); + static_assert(!__is_aggregate(int)); + static_assert(__is_aggregate(IntAr)); + static_assert(__is_aggregate(IntArNB)); + static_assert(__is_aggregate(EmptyUnion)); + static_assert(__is_aggregate(Union)); + static_assert(__is_aggregate(Statics)); + static_assert(__is_aggregate(HasFunc)); + static_assert(__is_aggregate(HasOp)); + static_assert(__is_aggregate(HasAssign)); + static_assert(__is_aggregate(HasAnonymousUnion)); + + static_assert(__is_aggregate(Derives) == TrueAfterCpp14); + static_assert(__is_aggregate(DerivesAr)); + static_assert(__is_aggregate(DerivesArNB)); + static_assert(!__is_aggregate(HasCons)); #if __cplusplus >= 202002L - static_assert(!__is_aggregate(HasDefaultCons), ""); + static_assert(!__is_aggregate(HasDefaultCons)); #else - static_assert(__is_aggregate(HasDefaultCons), ""); + static_assert(__is_aggregate(HasDefaultCons)); #endif - static_assert(!__is_aggregate(HasExplicitDefaultCons), ""); - static_assert(!__is_aggregate(HasInheritedCons), ""); - static_assert(__is_aggregate(HasNoInheritedCons) == TrueAfterCpp14, ""); - static_assert(__is_aggregate(HasCopyAssign), ""); - static_assert(!__is_aggregate(NonTrivialDefault), ""); - static_assert(__is_aggregate(HasDest), ""); - static_assert(!__is_aggregate(HasPriv), ""); - static_assert(!__is_aggregate(HasProt), ""); - static_assert(__is_aggregate(HasRefAggregate), ""); - static_assert(__is_aggregate(HasNonPOD), ""); - static_assert(!__is_aggregate(HasVirt), ""); - static_assert(__is_aggregate(VirtAr), ""); - static_assert(__is_aggregate(HasInClassInit) == TrueAfterCpp11, ""); - static_assert(!__is_aggregate(HasPrivateBase), ""); - static_assert(!__is_aggregate(HasProtectedBase), ""); - static_assert(!__is_aggregate(HasVirtBase), ""); - - static_assert(__is_aggregate(AggregateTemplate), ""); - static_assert(!__is_aggregate(NonAggregateTemplate), ""); - - static_assert(__is_aggregate(Vector), ""); // Extension supported by GCC and Clang - static_assert(__is_aggregate(VectorExt), ""); - static_assert(__is_aggregate(ComplexInt), ""); - static_assert(__is_aggregate(ComplexFloat), ""); + static_assert(!__is_aggregate(HasExplicitDefaultCons)); + static_assert(!__is_aggregate(HasInheritedCons)); + static_assert(__is_aggregate(HasNoInheritedCons) == TrueAfterCpp14); + static_assert(__is_aggregate(HasCopyAssign)); + static_assert(!__is_aggregate(NonTrivialDefault)); + static_assert(__is_aggregate(HasDest)); + static_assert(!__is_aggregate(HasPriv)); + static_assert(!__is_aggregate(HasProt)); + static_assert(__is_aggregate(HasRefAggregate)); + static_assert(__is_aggregate(HasNonPOD)); + static_assert(!__is_aggregate(HasVirt)); + static_assert(__is_aggregate(VirtAr)); + static_assert(__is_aggregate(HasInClassInit) == TrueAfterCpp11); + static_assert(!__is_aggregate(HasPrivateBase)); + static_assert(!__is_aggregate(HasProtectedBase)); + static_assert(!__is_aggregate(HasVirtBase)); + + static_assert(__is_aggregate(AggregateTemplate)); + static_assert(!__is_aggregate(NonAggregateTemplate)); + + static_assert(__is_aggregate(Vector)); // Extension supported by GCC and Clang + static_assert(__is_aggregate(VectorExt)); + static_assert(__is_aggregate(ComplexInt)); + static_assert(__is_aggregate(ComplexFloat)); } void is_arithmetic() { - int t01[T(__is_arithmetic(float))]; - int t02[T(__is_arithmetic(double))]; - int t03[T(__is_arithmetic(long double))]; - int t11[T(__is_arithmetic(bool))]; - int t12[T(__is_arithmetic(char))]; - int t13[T(__is_arithmetic(signed char))]; - int t14[T(__is_arithmetic(unsigned char))]; - //int t15[T(__is_arithmetic(char16_t))]; - //int t16[T(__is_arithmetic(char32_t))]; - int t17[T(__is_arithmetic(wchar_t))]; - int t18[T(__is_arithmetic(short))]; - int t19[T(__is_arithmetic(unsigned short))]; - int t20[T(__is_arithmetic(int))]; - int t21[T(__is_arithmetic(unsigned int))]; - int t22[T(__is_arithmetic(long))]; - int t23[T(__is_arithmetic(unsigned long))]; - - int t24[F(__is_arithmetic(Union))]; - int t25[F(__is_arithmetic(UnionAr))]; - int t26[F(__is_arithmetic(Derives))]; - int t27[F(__is_arithmetic(ClassType))]; - int t28[F(__is_arithmetic(Enum))]; - int t29[F(__is_arithmetic(void))]; - int t30[F(__is_arithmetic(cvoid))]; - int t31[F(__is_arithmetic(IntArNB))]; + static_assert(__is_arithmetic(float)); + static_assert(__is_arithmetic(double)); + static_assert(__is_arithmetic(long double)); + static_assert(__is_arithmetic(bool)); + static_assert(__is_arithmetic(char)); + static_assert(__is_arithmetic(signed char)); + static_assert(__is_arithmetic(unsigned char)); + //static_assert(__is_arithmetic(char16_t)); + //static_assert(__is_arithmetic(char32_t)); + static_assert(__is_arithmetic(wchar_t)); + static_assert(__is_arithmetic(short)); + static_assert(__is_arithmetic(unsigned short)); + static_assert(__is_arithmetic(int)); + static_assert(__is_arithmetic(unsigned int)); + static_assert(__is_arithmetic(long)); + static_assert(__is_arithmetic(unsigned long)); + + static_assert(!__is_arithmetic(Union)); + static_assert(!__is_arithmetic(UnionAr)); + static_assert(!__is_arithmetic(Derives)); + static_assert(!__is_arithmetic(ClassType)); + static_assert(!__is_arithmetic(Enum)); + static_assert(!__is_arithmetic(void)); + static_assert(!__is_arithmetic(cvoid)); + static_assert(!__is_arithmetic(IntArNB)); } void is_complete_type() { - int t01[T(__is_complete_type(float))]; - int t02[T(__is_complete_type(double))]; - int t03[T(__is_complete_type(long double))]; - int t11[T(__is_complete_type(bool))]; - int t12[T(__is_complete_type(char))]; - int t13[T(__is_complete_type(signed char))]; - int t14[T(__is_complete_type(unsigned char))]; - //int t15[T(__is_complete_type(char16_t))]; - //int t16[T(__is_complete_type(char32_t))]; - int t17[T(__is_complete_type(wchar_t))]; - int t18[T(__is_complete_type(short))]; - int t19[T(__is_complete_type(unsigned short))]; - int t20[T(__is_complete_type(int))]; - int t21[T(__is_complete_type(unsigned int))]; - int t22[T(__is_complete_type(long))]; - int t23[T(__is_complete_type(unsigned long))]; - int t24[T(__is_complete_type(ACompleteType))]; - - int t30[F(__is_complete_type(AnIncompleteType))]; + static_assert(__is_complete_type(float)); + static_assert(__is_complete_type(double)); + static_assert(__is_complete_type(long double)); + static_assert(__is_complete_type(bool)); + static_assert(__is_complete_type(char)); + static_assert(__is_complete_type(signed char)); + static_assert(__is_complete_type(unsigned char)); + //static_assert(__is_complete_type(char16_t)); + //static_assert(__is_complete_type(char32_t)); + static_assert(__is_complete_type(wchar_t)); + static_assert(__is_complete_type(short)); + static_assert(__is_complete_type(unsigned short)); + static_assert(__is_complete_type(int)); + static_assert(__is_complete_type(unsigned int)); + static_assert(__is_complete_type(long)); + static_assert(__is_complete_type(unsigned long)); + static_assert(__is_complete_type(ACompleteType)); + + static_assert(!__is_complete_type(AnIncompleteType)); } void is_void() { - int t01[T(__is_void(void))]; - int t02[T(__is_void(cvoid))]; - - int t10[F(__is_void(float))]; - int t11[F(__is_void(double))]; - int t12[F(__is_void(long double))]; - int t13[F(__is_void(bool))]; - int t14[F(__is_void(char))]; - int t15[F(__is_void(signed char))]; - int t16[F(__is_void(unsigned char))]; - int t17[F(__is_void(wchar_t))]; - int t18[F(__is_void(short))]; - int t19[F(__is_void(unsigned short))]; - int t20[F(__is_void(int))]; - int t21[F(__is_void(unsigned int))]; - int t22[F(__is_void(long))]; - int t23[F(__is_void(unsigned long))]; - int t24[F(__is_void(Union))]; - int t25[F(__is_void(UnionAr))]; - int t26[F(__is_void(Derives))]; - int t27[F(__is_void(ClassType))]; - int t28[F(__is_void(Enum))]; - int t29[F(__is_void(IntArNB))]; - int t30[F(__is_void(void*))]; - int t31[F(__is_void(cvoid*))]; + static_assert(__is_void(void)); + static_assert(__is_void(cvoid)); + + static_assert(!__is_void(float)); + static_assert(!__is_void(double)); + static_assert(!__is_void(long double)); + static_assert(!__is_void(bool)); + static_assert(!__is_void(char)); + static_assert(!__is_void(signed char)); + static_assert(!__is_void(unsigned char)); + static_assert(!__is_void(wchar_t)); + static_assert(!__is_void(short)); + static_assert(!__is_void(unsigned short)); + static_assert(!__is_void(int)); + static_assert(!__is_void(unsigned int)); + static_assert(!__is_void(long)); + static_assert(!__is_void(unsigned long)); + static_assert(!__is_void(Union)); + static_assert(!__is_void(UnionAr)); + static_assert(!__is_void(Derives)); + static_assert(!__is_void(ClassType)); + static_assert(!__is_void(Enum)); + static_assert(!__is_void(IntArNB)); + static_assert(!__is_void(void*)); + static_assert(!__is_void(cvoid*)); } void is_array() { - int t01[T(__is_array(IntAr))]; - int t02[T(__is_array(IntArNB))]; - int t03[T(__is_array(UnionAr))]; - - int t10[F(__is_array(void))]; - int t11[F(__is_array(cvoid))]; - int t12[F(__is_array(float))]; - int t13[F(__is_array(double))]; - int t14[F(__is_array(long double))]; - int t15[F(__is_array(bool))]; - int t16[F(__is_array(char))]; - int t17[F(__is_array(signed char))]; - int t18[F(__is_array(unsigned char))]; - int t19[F(__is_array(wchar_t))]; - int t20[F(__is_array(short))]; - int t21[F(__is_array(unsigned short))]; - int t22[F(__is_array(int))]; - int t23[F(__is_array(unsigned int))]; - int t24[F(__is_array(long))]; - int t25[F(__is_array(unsigned long))]; - int t26[F(__is_array(Union))]; - int t27[F(__is_array(Derives))]; - int t28[F(__is_array(ClassType))]; - int t29[F(__is_array(Enum))]; - int t30[F(__is_array(void*))]; - int t31[F(__is_array(cvoid*))]; + static_assert(__is_array(IntAr)); + static_assert(__is_array(IntArNB)); + static_assert(__is_array(UnionAr)); + + static_assert(!__is_array(void)); + static_assert(!__is_array(cvoid)); + static_assert(!__is_array(float)); + static_assert(!__is_array(double)); + static_assert(!__is_array(long double)); + static_assert(!__is_array(bool)); + static_assert(!__is_array(char)); + static_assert(!__is_array(signed char)); + static_assert(!__is_array(unsigned char)); + static_assert(!__is_array(wchar_t)); + static_assert(!__is_array(short)); + static_assert(!__is_array(unsigned short)); + static_assert(!__is_array(int)); + static_assert(!__is_array(unsigned int)); + static_assert(!__is_array(long)); + static_assert(!__is_array(unsigned long)); + static_assert(!__is_array(Union)); + static_assert(!__is_array(Derives)); + static_assert(!__is_array(ClassType)); + static_assert(!__is_array(Enum)); + static_assert(!__is_array(void*)); + static_assert(!__is_array(cvoid*)); } void is_bounded_array(int n) { - static_assert(__is_bounded_array(IntAr), ""); - static_assert(!__is_bounded_array(IntArNB), ""); - static_assert(__is_bounded_array(UnionAr), ""); - - static_assert(!__is_bounded_array(void), ""); - static_assert(!__is_bounded_array(cvoid), ""); - static_assert(!__is_bounded_array(float), ""); - static_assert(!__is_bounded_array(double), ""); - static_assert(!__is_bounded_array(long double), ""); - static_assert(!__is_bounded_array(bool), ""); - static_assert(!__is_bounded_array(char), ""); - static_assert(!__is_bounded_array(signed char), ""); - static_assert(!__is_bounded_array(unsigned char), ""); - static_assert(!__is_bounded_array(wchar_t), ""); - static_assert(!__is_bounded_array(short), ""); - static_assert(!__is_bounded_array(unsigned short), ""); - static_assert(!__is_bounded_array(int), ""); - static_assert(!__is_bounded_array(unsigned int), ""); - static_assert(!__is_bounded_array(long), ""); - static_assert(!__is_bounded_array(unsigned long), ""); - static_assert(!__is_bounded_array(Union), ""); - static_assert(!__is_bounded_array(Derives), ""); - static_assert(!__is_bounded_array(ClassType), ""); - static_assert(!__is_bounded_array(Enum), ""); - static_assert(!__is_bounded_array(void *), ""); - static_assert(!__is_bounded_array(cvoid *), ""); + static_assert(__is_bounded_array(IntAr)); + static_assert(!__is_bounded_array(IntArNB)); + static_assert(__is_bounded_array(UnionAr)); + + static_assert(!__is_bounded_array(void)); + static_assert(!__is_bounded_array(cvoid)); + static_assert(!__is_bounded_array(float)); + static_assert(!__is_bounded_array(double)); + static_assert(!__is_bounded_array(long double)); + static_assert(!__is_bounded_array(bool)); + static_assert(!__is_bounded_array(char)); + static_assert(!__is_bounded_array(signed char)); + static_assert(!__is_bounded_array(unsigned char)); + static_assert(!__is_bounded_array(wchar_t)); + static_assert(!__is_bounded_array(short)); + static_assert(!__is_bounded_array(unsigned short)); + static_assert(!__is_bounded_array(int)); + static_assert(!__is_bounded_array(unsigned int)); + static_assert(!__is_bounded_array(long)); + static_assert(!__is_bounded_array(unsigned long)); + static_assert(!__is_bounded_array(Union)); + static_assert(!__is_bounded_array(Derives)); + static_assert(!__is_bounded_array(ClassType)); + static_assert(!__is_bounded_array(Enum)); + static_assert(!__is_bounded_array(void *)); + static_assert(!__is_bounded_array(cvoid *)); int t32[n]; (void)__is_bounded_array(decltype(t32)); // expected-error{{variable length arrays are not supported for '__is_bounded_array'}} } void is_unbounded_array(int n) { - static_assert(!__is_unbounded_array(IntAr), ""); - static_assert(__is_unbounded_array(IntArNB), ""); - static_assert(!__is_unbounded_array(UnionAr), ""); - - static_assert(!__is_unbounded_array(void), ""); - static_assert(!__is_unbounded_array(cvoid), ""); - static_assert(!__is_unbounded_array(float), ""); - static_assert(!__is_unbounded_array(double), ""); - static_assert(!__is_unbounded_array(long double), ""); - static_assert(!__is_unbounded_array(bool), ""); - static_assert(!__is_unbounded_array(char), ""); - static_assert(!__is_unbounded_array(signed char), ""); - static_assert(!__is_unbounded_array(unsigned char), ""); - static_assert(!__is_unbounded_array(wchar_t), ""); - static_assert(!__is_unbounded_array(short), ""); - static_assert(!__is_unbounded_array(unsigned short), ""); - static_assert(!__is_unbounded_array(int), ""); - static_assert(!__is_unbounded_array(unsigned int), ""); - static_assert(!__is_unbounded_array(long), ""); - static_assert(!__is_unbounded_array(unsigned long), ""); - static_assert(!__is_unbounded_array(Union), ""); - static_assert(!__is_unbounded_array(Derives), ""); - static_assert(!__is_unbounded_array(ClassType), ""); - static_assert(!__is_unbounded_array(Enum), ""); - static_assert(!__is_unbounded_array(void *), ""); - static_assert(!__is_unbounded_array(cvoid *), ""); + static_assert(!__is_unbounded_array(IntAr)); + static_assert(__is_unbounded_array(IntArNB)); + static_assert(!__is_unbounded_array(UnionAr)); + + static_assert(!__is_unbounded_array(void)); + static_assert(!__is_unbounded_array(cvoid)); + static_assert(!__is_unbounded_array(float)); + static_assert(!__is_unbounded_array(double)); + static_assert(!__is_unbounded_array(long double)); + static_assert(!__is_unbounded_array(bool)); + static_assert(!__is_unbounded_array(char)); + static_assert(!__is_unbounded_array(signed char)); + static_assert(!__is_unbounded_array(unsigned char)); + static_assert(!__is_unbounded_array(wchar_t)); + static_assert(!__is_unbounded_array(short)); + static_assert(!__is_unbounded_array(unsigned short)); + static_assert(!__is_unbounded_array(int)); + static_assert(!__is_unbounded_array(unsigned int)); + static_assert(!__is_unbounded_array(long)); + static_assert(!__is_unbounded_array(unsigned long)); + static_assert(!__is_unbounded_array(Union)); + static_assert(!__is_unbounded_array(Derives)); + static_assert(!__is_unbounded_array(ClassType)); + static_assert(!__is_unbounded_array(Enum)); + static_assert(!__is_unbounded_array(void *)); + static_assert(!__is_unbounded_array(cvoid *)); int t32[n]; (void)__is_unbounded_array(decltype(t32)); // expected-error{{variable length arrays are not supported for '__is_unbounded_array'}} } void is_referenceable() { - static_assert(__is_referenceable(int), ""); - static_assert(__is_referenceable(const int), ""); - static_assert(__is_referenceable(volatile int), ""); - static_assert(__is_referenceable(const volatile int), ""); - static_assert(__is_referenceable(int *), ""); - static_assert(__is_referenceable(int &), ""); - static_assert(__is_referenceable(int &&), ""); - static_assert(__is_referenceable(int (*)()), ""); - static_assert(__is_referenceable(int (&)()), ""); - static_assert(__is_referenceable(int(&&)()), ""); - static_assert(__is_referenceable(IntAr), ""); - static_assert(__is_referenceable(IntArNB), ""); - static_assert(__is_referenceable(decltype(nullptr)), ""); - static_assert(__is_referenceable(Empty), ""); - static_assert(__is_referenceable(Union), ""); - static_assert(__is_referenceable(Derives), ""); - static_assert(__is_referenceable(Enum), ""); - static_assert(__is_referenceable(EnumClass), ""); - static_assert(__is_referenceable(int Empty::*), ""); - static_assert(__is_referenceable(int(Empty::*)()), ""); - static_assert(__is_referenceable(AnIncompleteType), ""); - static_assert(__is_referenceable(struct AnIncompleteType), ""); + static_assert(__is_referenceable(int)); + static_assert(__is_referenceable(const int)); + static_assert(__is_referenceable(volatile int)); + static_assert(__is_referenceable(const volatile int)); + static_assert(__is_referenceable(int *)); + static_assert(__is_referenceable(int &)); + static_assert(__is_referenceable(int &&)); + static_assert(__is_referenceable(int (*)())); + static_assert(__is_referenceable(int (&)())); + static_assert(__is_referenceable(int(&&)())); + static_assert(__is_referenceable(IntAr)); + static_assert(__is_referenceable(IntArNB)); + static_assert(__is_referenceable(decltype(nullptr))); + static_assert(__is_referenceable(Empty)); + static_assert(__is_referenceable(Union)); + static_assert(__is_referenceable(Derives)); + static_assert(__is_referenceable(Enum)); + static_assert(__is_referenceable(EnumClass)); + static_assert(__is_referenceable(int Empty::*)); + static_assert(__is_referenceable(int(Empty::*)())); + static_assert(__is_referenceable(AnIncompleteType)); + static_assert(__is_referenceable(struct AnIncompleteType)); using function_type = void(int); - static_assert(__is_referenceable(function_type), ""); + static_assert(__is_referenceable(function_type)); - static_assert(!__is_referenceable(void), ""); + static_assert(!__is_referenceable(void)); } template void tmpl_func(T&) {} @@ -817,152 +815,152 @@ template struct type_wrapper { void is_function() { - int t01[T(__is_function(type_wrapper::type))]; - int t02[T(__is_function(typeof(tmpl_func)))]; + static_assert(__is_function(type_wrapper::type)); + static_assert(__is_function(typeof(tmpl_func))); typedef void (*ptr_to_func_type)(void); - int t10[F(__is_function(void))]; - int t11[F(__is_function(cvoid))]; - int t12[F(__is_function(float))]; - int t13[F(__is_function(double))]; - int t14[F(__is_function(long double))]; - int t15[F(__is_function(bool))]; - int t16[F(__is_function(char))]; - int t17[F(__is_function(signed char))]; - int t18[F(__is_function(unsigned char))]; - int t19[F(__is_function(wchar_t))]; - int t20[F(__is_function(short))]; - int t21[F(__is_function(unsigned short))]; - int t22[F(__is_function(int))]; - int t23[F(__is_function(unsigned int))]; - int t24[F(__is_function(long))]; - int t25[F(__is_function(unsigned long))]; - int t26[F(__is_function(Union))]; - int t27[F(__is_function(Derives))]; - int t28[F(__is_function(ClassType))]; - int t29[F(__is_function(Enum))]; - int t30[F(__is_function(void*))]; - int t31[F(__is_function(cvoid*))]; - int t32[F(__is_function(void(*)()))]; - int t33[F(__is_function(ptr_to_func_type))]; - int t34[F(__is_function(type_wrapper::ptrtype))]; - int t35[F(__is_function(type_wrapper::reftype))]; + static_assert(!__is_function(void)); + static_assert(!__is_function(cvoid)); + static_assert(!__is_function(float)); + static_assert(!__is_function(double)); + static_assert(!__is_function(long double)); + static_assert(!__is_function(bool)); + static_assert(!__is_function(char)); + static_assert(!__is_function(signed char)); + static_assert(!__is_function(unsigned char)); + static_assert(!__is_function(wchar_t)); + static_assert(!__is_function(short)); + static_assert(!__is_function(unsigned short)); + static_assert(!__is_function(int)); + static_assert(!__is_function(unsigned int)); + static_assert(!__is_function(long)); + static_assert(!__is_function(unsigned long)); + static_assert(!__is_function(Union)); + static_assert(!__is_function(Derives)); + static_assert(!__is_function(ClassType)); + static_assert(!__is_function(Enum)); + static_assert(!__is_function(void*)); + static_assert(!__is_function(cvoid*)); + static_assert(!__is_function(void(*)())); + static_assert(!__is_function(ptr_to_func_type)); + static_assert(!__is_function(type_wrapper::ptrtype)); + static_assert(!__is_function(type_wrapper::reftype)); } void is_reference() { - int t01[T(__is_reference(int&))]; - int t02[T(__is_reference(const int&))]; - int t03[T(__is_reference(void *&))]; + static_assert(__is_reference(int&)); + static_assert(__is_reference(const int&)); + static_assert(__is_reference(void *&)); - int t10[F(__is_reference(int))]; - int t11[F(__is_reference(const int))]; - int t12[F(__is_reference(void *))]; + static_assert(!__is_reference(int)); + static_assert(!__is_reference(const int)); + static_assert(!__is_reference(void *)); } void is_lvalue_reference() { - int t01[T(__is_lvalue_reference(int&))]; - int t02[T(__is_lvalue_reference(void *&))]; - int t03[T(__is_lvalue_reference(const int&))]; - int t04[T(__is_lvalue_reference(void * const &))]; - - int t10[F(__is_lvalue_reference(int))]; - int t11[F(__is_lvalue_reference(const int))]; - int t12[F(__is_lvalue_reference(void *))]; + static_assert(__is_lvalue_reference(int&)); + static_assert(__is_lvalue_reference(void *&)); + static_assert(__is_lvalue_reference(const int&)); + static_assert(__is_lvalue_reference(void * const &)); + + static_assert(!__is_lvalue_reference(int)); + static_assert(!__is_lvalue_reference(const int)); + static_assert(!__is_lvalue_reference(void *)); } #if __has_feature(cxx_rvalue_references) void is_rvalue_reference() { - int t01[T(__is_rvalue_reference(const int&&))]; - int t02[T(__is_rvalue_reference(void * const &&))]; - - int t10[F(__is_rvalue_reference(int&))]; - int t11[F(__is_rvalue_reference(void *&))]; - int t12[F(__is_rvalue_reference(const int&))]; - int t13[F(__is_rvalue_reference(void * const &))]; - int t14[F(__is_rvalue_reference(int))]; - int t15[F(__is_rvalue_reference(const int))]; - int t16[F(__is_rvalue_reference(void *))]; + static_assert(__is_rvalue_reference(const int&&)); + static_assert(__is_rvalue_reference(void * const &&)); + + static_assert(!__is_rvalue_reference(int&)); + static_assert(!__is_rvalue_reference(void *&)); + static_assert(!__is_rvalue_reference(const int&)); + static_assert(!__is_rvalue_reference(void * const &)); + static_assert(!__is_rvalue_reference(int)); + static_assert(!__is_rvalue_reference(const int)); + static_assert(!__is_rvalue_reference(void *)); } #endif void is_fundamental() { - int t01[T(__is_fundamental(float))]; - int t02[T(__is_fundamental(double))]; - int t03[T(__is_fundamental(long double))]; - int t11[T(__is_fundamental(bool))]; - int t12[T(__is_fundamental(char))]; - int t13[T(__is_fundamental(signed char))]; - int t14[T(__is_fundamental(unsigned char))]; - //int t15[T(__is_fundamental(char16_t))]; - //int t16[T(__is_fundamental(char32_t))]; - int t17[T(__is_fundamental(wchar_t))]; - int t18[T(__is_fundamental(short))]; - int t19[T(__is_fundamental(unsigned short))]; - int t20[T(__is_fundamental(int))]; - int t21[T(__is_fundamental(unsigned int))]; - int t22[T(__is_fundamental(long))]; - int t23[T(__is_fundamental(unsigned long))]; - int t24[T(__is_fundamental(void))]; - int t25[T(__is_fundamental(cvoid))]; - int t26[T(__is_fundamental(decltype(nullptr)))]; - - int t30[F(__is_fundamental(Union))]; - int t31[F(__is_fundamental(UnionAr))]; - int t32[F(__is_fundamental(Derives))]; - int t33[F(__is_fundamental(ClassType))]; - int t34[F(__is_fundamental(Enum))]; - int t35[F(__is_fundamental(IntArNB))]; + static_assert(__is_fundamental(float)); + static_assert(__is_fundamental(double)); + static_assert(__is_fundamental(long double)); + static_assert(__is_fundamental(bool)); + static_assert(__is_fundamental(char)); + static_assert(__is_fundamental(signed char)); + static_assert(__is_fundamental(unsigned char)); + //static_assert(__is_fundamental(char16_t)); + //static_assert(__is_fundamental(char32_t)); + static_assert(__is_fundamental(wchar_t)); + static_assert(__is_fundamental(short)); + static_assert(__is_fundamental(unsigned short)); + static_assert(__is_fundamental(int)); + static_assert(__is_fundamental(unsigned int)); + static_assert(__is_fundamental(long)); + static_assert(__is_fundamental(unsigned long)); + static_assert(__is_fundamental(void)); + static_assert(__is_fundamental(cvoid)); + static_assert(__is_fundamental(decltype(nullptr))); + + static_assert(!__is_fundamental(Union)); + static_assert(!__is_fundamental(UnionAr)); + static_assert(!__is_fundamental(Derives)); + static_assert(!__is_fundamental(ClassType)); + static_assert(!__is_fundamental(Enum)); + static_assert(!__is_fundamental(IntArNB)); } void is_object() { - int t01[T(__is_object(int))]; - int t02[T(__is_object(int *))]; - int t03[T(__is_object(void *))]; - int t04[T(__is_object(Union))]; - int t05[T(__is_object(UnionAr))]; - int t06[T(__is_object(ClassType))]; - int t07[T(__is_object(Enum))]; - - int t10[F(__is_object(type_wrapper::type))]; - int t11[F(__is_object(int&))]; - int t12[F(__is_object(void))]; + static_assert(__is_object(int)); + static_assert(__is_object(int *)); + static_assert(__is_object(void *)); + static_assert(__is_object(Union)); + static_assert(__is_object(UnionAr)); + static_assert(__is_object(ClassType)); + static_assert(__is_object(Enum)); + + static_assert(!__is_object(type_wrapper::type)); + static_assert(!__is_object(int&)); + static_assert(!__is_object(void)); } void is_scalar() { - int t01[T(__is_scalar(float))]; - int t02[T(__is_scalar(double))]; - int t03[T(__is_scalar(long double))]; - int t04[T(__is_scalar(bool))]; - int t05[T(__is_scalar(char))]; - int t06[T(__is_scalar(signed char))]; - int t07[T(__is_scalar(unsigned char))]; - int t08[T(__is_scalar(wchar_t))]; - int t09[T(__is_scalar(short))]; - int t10[T(__is_scalar(unsigned short))]; - int t11[T(__is_scalar(int))]; - int t12[T(__is_scalar(unsigned int))]; - int t13[T(__is_scalar(long))]; - int t14[T(__is_scalar(unsigned long))]; - int t15[T(__is_scalar(Enum))]; - int t16[T(__is_scalar(void*))]; - int t17[T(__is_scalar(cvoid*))]; - - int t20[F(__is_scalar(void))]; - int t21[F(__is_scalar(cvoid))]; - int t22[F(__is_scalar(Union))]; - int t23[F(__is_scalar(UnionAr))]; - int t24[F(__is_scalar(Derives))]; - int t25[F(__is_scalar(ClassType))]; - int t26[F(__is_scalar(IntArNB))]; + static_assert(__is_scalar(float)); + static_assert(__is_scalar(double)); + static_assert(__is_scalar(long double)); + static_assert(__is_scalar(bool)); + static_assert(__is_scalar(char)); + static_assert(__is_scalar(signed char)); + static_assert(__is_scalar(unsigned char)); + static_assert(__is_scalar(wchar_t)); + static_assert(__is_scalar(short)); + static_assert(__is_scalar(unsigned short)); + static_assert(__is_scalar(int)); + static_assert(__is_scalar(unsigned int)); + static_assert(__is_scalar(long)); + static_assert(__is_scalar(unsigned long)); + static_assert(__is_scalar(Enum)); + static_assert(__is_scalar(void*)); + static_assert(__is_scalar(cvoid*)); + + static_assert(!__is_scalar(void)); + static_assert(!__is_scalar(cvoid)); + static_assert(!__is_scalar(Union)); + static_assert(!__is_scalar(UnionAr)); + static_assert(!__is_scalar(Derives)); + static_assert(!__is_scalar(ClassType)); + static_assert(!__is_scalar(IntArNB)); } struct StructWithMembers { @@ -972,314 +970,314 @@ struct StructWithMembers { void is_compound() { - int t01[T(__is_compound(void*))]; - int t02[T(__is_compound(cvoid*))]; - int t03[T(__is_compound(void (*)()))]; - int t04[T(__is_compound(int StructWithMembers::*))]; - int t05[T(__is_compound(void (StructWithMembers::*)()))]; - int t06[T(__is_compound(int&))]; - int t07[T(__is_compound(Union))]; - int t08[T(__is_compound(UnionAr))]; - int t09[T(__is_compound(Derives))]; - int t10[T(__is_compound(ClassType))]; - int t11[T(__is_compound(IntArNB))]; - int t12[T(__is_compound(Enum))]; - - int t20[F(__is_compound(float))]; - int t21[F(__is_compound(double))]; - int t22[F(__is_compound(long double))]; - int t23[F(__is_compound(bool))]; - int t24[F(__is_compound(char))]; - int t25[F(__is_compound(signed char))]; - int t26[F(__is_compound(unsigned char))]; - int t27[F(__is_compound(wchar_t))]; - int t28[F(__is_compound(short))]; - int t29[F(__is_compound(unsigned short))]; - int t30[F(__is_compound(int))]; - int t31[F(__is_compound(unsigned int))]; - int t32[F(__is_compound(long))]; - int t33[F(__is_compound(unsigned long))]; - int t34[F(__is_compound(void))]; - int t35[F(__is_compound(cvoid))]; + static_assert(__is_compound(void*)); + static_assert(__is_compound(cvoid*)); + static_assert(__is_compound(void (*)())); + static_assert(__is_compound(int StructWithMembers::*)); + static_assert(__is_compound(void (StructWithMembers::*)())); + static_assert(__is_compound(int&)); + static_assert(__is_compound(Union)); + static_assert(__is_compound(UnionAr)); + static_assert(__is_compound(Derives)); + static_assert(__is_compound(ClassType)); + static_assert(__is_compound(IntArNB)); + static_assert(__is_compound(Enum)); + + static_assert(!__is_compound(float)); + static_assert(!__is_compound(double)); + static_assert(!__is_compound(long double)); + static_assert(!__is_compound(bool)); + static_assert(!__is_compound(char)); + static_assert(!__is_compound(signed char)); + static_assert(!__is_compound(unsigned char)); + static_assert(!__is_compound(wchar_t)); + static_assert(!__is_compound(short)); + static_assert(!__is_compound(unsigned short)); + static_assert(!__is_compound(int)); + static_assert(!__is_compound(unsigned int)); + static_assert(!__is_compound(long)); + static_assert(!__is_compound(unsigned long)); + static_assert(!__is_compound(void)); + static_assert(!__is_compound(cvoid)); } void is_pointer() { StructWithMembers x; - int t01[T(__is_pointer(void*))]; - int t02[T(__is_pointer(cvoid*))]; - int t03[T(__is_pointer(cvoid*))]; - int t04[T(__is_pointer(char*))]; - int t05[T(__is_pointer(int*))]; - int t06[T(__is_pointer(int**))]; - int t07[T(__is_pointer(ClassType*))]; - int t08[T(__is_pointer(Derives*))]; - int t09[T(__is_pointer(Enum*))]; - int t10[T(__is_pointer(IntArNB*))]; - int t11[T(__is_pointer(Union*))]; - int t12[T(__is_pointer(UnionAr*))]; - int t13[T(__is_pointer(StructWithMembers*))]; - int t14[T(__is_pointer(void (*)()))]; - - int t20[F(__is_pointer(void))]; - int t21[F(__is_pointer(cvoid))]; - int t22[F(__is_pointer(cvoid))]; - int t23[F(__is_pointer(char))]; - int t24[F(__is_pointer(int))]; - int t25[F(__is_pointer(int))]; - int t26[F(__is_pointer(ClassType))]; - int t27[F(__is_pointer(Derives))]; - int t28[F(__is_pointer(Enum))]; - int t29[F(__is_pointer(IntArNB))]; - int t30[F(__is_pointer(Union))]; - int t31[F(__is_pointer(UnionAr))]; - int t32[F(__is_pointer(StructWithMembers))]; - int t33[F(__is_pointer(int StructWithMembers::*))]; - int t34[F(__is_pointer(void (StructWithMembers::*) ()))]; + static_assert(__is_pointer(void*)); + static_assert(__is_pointer(cvoid*)); + static_assert(__is_pointer(cvoid*)); + static_assert(__is_pointer(char*)); + static_assert(__is_pointer(int*)); + static_assert(__is_pointer(int**)); + static_assert(__is_pointer(ClassType*)); + static_assert(__is_pointer(Derives*)); + static_assert(__is_pointer(Enum*)); + static_assert(__is_pointer(IntArNB*)); + static_assert(__is_pointer(Union*)); + static_assert(__is_pointer(UnionAr*)); + static_assert(__is_pointer(StructWithMembers*)); + static_assert(__is_pointer(void (*)())); + + static_assert(!__is_pointer(void)); + static_assert(!__is_pointer(cvoid)); + static_assert(!__is_pointer(cvoid)); + static_assert(!__is_pointer(char)); + static_assert(!__is_pointer(int)); + static_assert(!__is_pointer(int)); + static_assert(!__is_pointer(ClassType)); + static_assert(!__is_pointer(Derives)); + static_assert(!__is_pointer(Enum)); + static_assert(!__is_pointer(IntArNB)); + static_assert(!__is_pointer(Union)); + static_assert(!__is_pointer(UnionAr)); + static_assert(!__is_pointer(StructWithMembers)); + static_assert(!__is_pointer(int StructWithMembers::*)); + static_assert(!__is_pointer(void (StructWithMembers::*) ())); } void is_null_pointer() { StructWithMembers x; - static_assert(__is_nullptr(decltype(nullptr)), ""); - static_assert(!__is_nullptr(void *), ""); - static_assert(!__is_nullptr(cvoid *), ""); - static_assert(!__is_nullptr(cvoid *), ""); - static_assert(!__is_nullptr(char *), ""); - static_assert(!__is_nullptr(int *), ""); - static_assert(!__is_nullptr(int **), ""); - static_assert(!__is_nullptr(ClassType *), ""); - static_assert(!__is_nullptr(Derives *), ""); - static_assert(!__is_nullptr(Enum *), ""); - static_assert(!__is_nullptr(IntArNB *), ""); - static_assert(!__is_nullptr(Union *), ""); - static_assert(!__is_nullptr(UnionAr *), ""); - static_assert(!__is_nullptr(StructWithMembers *), ""); - static_assert(!__is_nullptr(void (*)()), ""); - - static_assert(!__is_nullptr(void), ""); - static_assert(!__is_nullptr(cvoid), ""); - static_assert(!__is_nullptr(cvoid), ""); - static_assert(!__is_nullptr(char), ""); - static_assert(!__is_nullptr(int), ""); - static_assert(!__is_nullptr(int), ""); - static_assert(!__is_nullptr(ClassType), ""); - static_assert(!__is_nullptr(Derives), ""); - static_assert(!__is_nullptr(Enum), ""); - static_assert(!__is_nullptr(IntArNB), ""); - static_assert(!__is_nullptr(Union), ""); - static_assert(!__is_nullptr(UnionAr), ""); - static_assert(!__is_nullptr(StructWithMembers), ""); - static_assert(!__is_nullptr(int StructWithMembers::*), ""); - static_assert(!__is_nullptr(void(StructWithMembers::*)()), ""); + static_assert(__is_nullptr(decltype(nullptr))); + static_assert(!__is_nullptr(void *)); + static_assert(!__is_nullptr(cvoid *)); + static_assert(!__is_nullptr(cvoid *)); + static_assert(!__is_nullptr(char *)); + static_assert(!__is_nullptr(int *)); + static_assert(!__is_nullptr(int **)); + static_assert(!__is_nullptr(ClassType *)); + static_assert(!__is_nullptr(Derives *)); + static_assert(!__is_nullptr(Enum *)); + static_assert(!__is_nullptr(IntArNB *)); + static_assert(!__is_nullptr(Union *)); + static_assert(!__is_nullptr(UnionAr *)); + static_assert(!__is_nullptr(StructWithMembers *)); + static_assert(!__is_nullptr(void (*)())); + + static_assert(!__is_nullptr(void)); + static_assert(!__is_nullptr(cvoid)); + static_assert(!__is_nullptr(cvoid)); + static_assert(!__is_nullptr(char)); + static_assert(!__is_nullptr(int)); + static_assert(!__is_nullptr(int)); + static_assert(!__is_nullptr(ClassType)); + static_assert(!__is_nullptr(Derives)); + static_assert(!__is_nullptr(Enum)); + static_assert(!__is_nullptr(IntArNB)); + static_assert(!__is_nullptr(Union)); + static_assert(!__is_nullptr(UnionAr)); + static_assert(!__is_nullptr(StructWithMembers)); + static_assert(!__is_nullptr(int StructWithMembers::*)); + static_assert(!__is_nullptr(void(StructWithMembers::*)())); } void is_member_object_pointer() { StructWithMembers x; - int t01[T(__is_member_object_pointer(int StructWithMembers::*))]; - - int t10[F(__is_member_object_pointer(void (StructWithMembers::*) ()))]; - int t11[F(__is_member_object_pointer(void*))]; - int t12[F(__is_member_object_pointer(cvoid*))]; - int t13[F(__is_member_object_pointer(cvoid*))]; - int t14[F(__is_member_object_pointer(char*))]; - int t15[F(__is_member_object_pointer(int*))]; - int t16[F(__is_member_object_pointer(int**))]; - int t17[F(__is_member_object_pointer(ClassType*))]; - int t18[F(__is_member_object_pointer(Derives*))]; - int t19[F(__is_member_object_pointer(Enum*))]; - int t20[F(__is_member_object_pointer(IntArNB*))]; - int t21[F(__is_member_object_pointer(Union*))]; - int t22[F(__is_member_object_pointer(UnionAr*))]; - int t23[F(__is_member_object_pointer(StructWithMembers*))]; - int t24[F(__is_member_object_pointer(void))]; - int t25[F(__is_member_object_pointer(cvoid))]; - int t26[F(__is_member_object_pointer(cvoid))]; - int t27[F(__is_member_object_pointer(char))]; - int t28[F(__is_member_object_pointer(int))]; - int t29[F(__is_member_object_pointer(int))]; - int t30[F(__is_member_object_pointer(ClassType))]; - int t31[F(__is_member_object_pointer(Derives))]; - int t32[F(__is_member_object_pointer(Enum))]; - int t33[F(__is_member_object_pointer(IntArNB))]; - int t34[F(__is_member_object_pointer(Union))]; - int t35[F(__is_member_object_pointer(UnionAr))]; - int t36[F(__is_member_object_pointer(StructWithMembers))]; - int t37[F(__is_member_object_pointer(void (*)()))]; + static_assert(__is_member_object_pointer(int StructWithMembers::*)); + + static_assert(!__is_member_object_pointer(void (StructWithMembers::*) ())); + static_assert(!__is_member_object_pointer(void*)); + static_assert(!__is_member_object_pointer(cvoid*)); + static_assert(!__is_member_object_pointer(cvoid*)); + static_assert(!__is_member_object_pointer(char*)); + static_assert(!__is_member_object_pointer(int*)); + static_assert(!__is_member_object_pointer(int**)); + static_assert(!__is_member_object_pointer(ClassType*)); + static_assert(!__is_member_object_pointer(Derives*)); + static_assert(!__is_member_object_pointer(Enum*)); + static_assert(!__is_member_object_pointer(IntArNB*)); + static_assert(!__is_member_object_pointer(Union*)); + static_assert(!__is_member_object_pointer(UnionAr*)); + static_assert(!__is_member_object_pointer(StructWithMembers*)); + static_assert(!__is_member_object_pointer(void)); + static_assert(!__is_member_object_pointer(cvoid)); + static_assert(!__is_member_object_pointer(cvoid)); + static_assert(!__is_member_object_pointer(char)); + static_assert(!__is_member_object_pointer(int)); + static_assert(!__is_member_object_pointer(int)); + static_assert(!__is_member_object_pointer(ClassType)); + static_assert(!__is_member_object_pointer(Derives)); + static_assert(!__is_member_object_pointer(Enum)); + static_assert(!__is_member_object_pointer(IntArNB)); + static_assert(!__is_member_object_pointer(Union)); + static_assert(!__is_member_object_pointer(UnionAr)); + static_assert(!__is_member_object_pointer(StructWithMembers)); + static_assert(!__is_member_object_pointer(void (*)())); } void is_member_function_pointer() { StructWithMembers x; - int t01[T(__is_member_function_pointer(void (StructWithMembers::*) ()))]; - - int t10[F(__is_member_function_pointer(int StructWithMembers::*))]; - int t11[F(__is_member_function_pointer(void*))]; - int t12[F(__is_member_function_pointer(cvoid*))]; - int t13[F(__is_member_function_pointer(cvoid*))]; - int t14[F(__is_member_function_pointer(char*))]; - int t15[F(__is_member_function_pointer(int*))]; - int t16[F(__is_member_function_pointer(int**))]; - int t17[F(__is_member_function_pointer(ClassType*))]; - int t18[F(__is_member_function_pointer(Derives*))]; - int t19[F(__is_member_function_pointer(Enum*))]; - int t20[F(__is_member_function_pointer(IntArNB*))]; - int t21[F(__is_member_function_pointer(Union*))]; - int t22[F(__is_member_function_pointer(UnionAr*))]; - int t23[F(__is_member_function_pointer(StructWithMembers*))]; - int t24[F(__is_member_function_pointer(void))]; - int t25[F(__is_member_function_pointer(cvoid))]; - int t26[F(__is_member_function_pointer(cvoid))]; - int t27[F(__is_member_function_pointer(char))]; - int t28[F(__is_member_function_pointer(int))]; - int t29[F(__is_member_function_pointer(int))]; - int t30[F(__is_member_function_pointer(ClassType))]; - int t31[F(__is_member_function_pointer(Derives))]; - int t32[F(__is_member_function_pointer(Enum))]; - int t33[F(__is_member_function_pointer(IntArNB))]; - int t34[F(__is_member_function_pointer(Union))]; - int t35[F(__is_member_function_pointer(UnionAr))]; - int t36[F(__is_member_function_pointer(StructWithMembers))]; - int t37[F(__is_member_function_pointer(void (*)()))]; + static_assert(__is_member_function_pointer(void (StructWithMembers::*) ())); + + static_assert(!__is_member_function_pointer(int StructWithMembers::*)); + static_assert(!__is_member_function_pointer(void*)); + static_assert(!__is_member_function_pointer(cvoid*)); + static_assert(!__is_member_function_pointer(cvoid*)); + static_assert(!__is_member_function_pointer(char*)); + static_assert(!__is_member_function_pointer(int*)); + static_assert(!__is_member_function_pointer(int**)); + static_assert(!__is_member_function_pointer(ClassType*)); + static_assert(!__is_member_function_pointer(Derives*)); + static_assert(!__is_member_function_pointer(Enum*)); + static_assert(!__is_member_function_pointer(IntArNB*)); + static_assert(!__is_member_function_pointer(Union*)); + static_assert(!__is_member_function_pointer(UnionAr*)); + static_assert(!__is_member_function_pointer(StructWithMembers*)); + static_assert(!__is_member_function_pointer(void)); + static_assert(!__is_member_function_pointer(cvoid)); + static_assert(!__is_member_function_pointer(cvoid)); + static_assert(!__is_member_function_pointer(char)); + static_assert(!__is_member_function_pointer(int)); + static_assert(!__is_member_function_pointer(int)); + static_assert(!__is_member_function_pointer(ClassType)); + static_assert(!__is_member_function_pointer(Derives)); + static_assert(!__is_member_function_pointer(Enum)); + static_assert(!__is_member_function_pointer(IntArNB)); + static_assert(!__is_member_function_pointer(Union)); + static_assert(!__is_member_function_pointer(UnionAr)); + static_assert(!__is_member_function_pointer(StructWithMembers)); + static_assert(!__is_member_function_pointer(void (*)())); } void is_member_pointer() { StructWithMembers x; - int t01[T(__is_member_pointer(int StructWithMembers::*))]; - int t02[T(__is_member_pointer(void (StructWithMembers::*) ()))]; - - int t10[F(__is_member_pointer(void*))]; - int t11[F(__is_member_pointer(cvoid*))]; - int t12[F(__is_member_pointer(cvoid*))]; - int t13[F(__is_member_pointer(char*))]; - int t14[F(__is_member_pointer(int*))]; - int t15[F(__is_member_pointer(int**))]; - int t16[F(__is_member_pointer(ClassType*))]; - int t17[F(__is_member_pointer(Derives*))]; - int t18[F(__is_member_pointer(Enum*))]; - int t19[F(__is_member_pointer(IntArNB*))]; - int t20[F(__is_member_pointer(Union*))]; - int t21[F(__is_member_pointer(UnionAr*))]; - int t22[F(__is_member_pointer(StructWithMembers*))]; - int t23[F(__is_member_pointer(void))]; - int t24[F(__is_member_pointer(cvoid))]; - int t25[F(__is_member_pointer(cvoid))]; - int t26[F(__is_member_pointer(char))]; - int t27[F(__is_member_pointer(int))]; - int t28[F(__is_member_pointer(int))]; - int t29[F(__is_member_pointer(ClassType))]; - int t30[F(__is_member_pointer(Derives))]; - int t31[F(__is_member_pointer(Enum))]; - int t32[F(__is_member_pointer(IntArNB))]; - int t33[F(__is_member_pointer(Union))]; - int t34[F(__is_member_pointer(UnionAr))]; - int t35[F(__is_member_pointer(StructWithMembers))]; - int t36[F(__is_member_pointer(void (*)()))]; + static_assert(__is_member_pointer(int StructWithMembers::*)); + static_assert(__is_member_pointer(void (StructWithMembers::*) ())); + + static_assert(!__is_member_pointer(void*)); + static_assert(!__is_member_pointer(cvoid*)); + static_assert(!__is_member_pointer(cvoid*)); + static_assert(!__is_member_pointer(char*)); + static_assert(!__is_member_pointer(int*)); + static_assert(!__is_member_pointer(int**)); + static_assert(!__is_member_pointer(ClassType*)); + static_assert(!__is_member_pointer(Derives*)); + static_assert(!__is_member_pointer(Enum*)); + static_assert(!__is_member_pointer(IntArNB*)); + static_assert(!__is_member_pointer(Union*)); + static_assert(!__is_member_pointer(UnionAr*)); + static_assert(!__is_member_pointer(StructWithMembers*)); + static_assert(!__is_member_pointer(void)); + static_assert(!__is_member_pointer(cvoid)); + static_assert(!__is_member_pointer(cvoid)); + static_assert(!__is_member_pointer(char)); + static_assert(!__is_member_pointer(int)); + static_assert(!__is_member_pointer(int)); + static_assert(!__is_member_pointer(ClassType)); + static_assert(!__is_member_pointer(Derives)); + static_assert(!__is_member_pointer(Enum)); + static_assert(!__is_member_pointer(IntArNB)); + static_assert(!__is_member_pointer(Union)); + static_assert(!__is_member_pointer(UnionAr)); + static_assert(!__is_member_pointer(StructWithMembers)); + static_assert(!__is_member_pointer(void (*)())); } void is_const() { - int t01[T(__is_const(cvoid))]; - int t02[T(__is_const(const char))]; - int t03[T(__is_const(const int))]; - int t04[T(__is_const(const long))]; - int t05[T(__is_const(const short))]; - int t06[T(__is_const(const signed char))]; - int t07[T(__is_const(const wchar_t))]; - int t08[T(__is_const(const bool))]; - int t09[T(__is_const(const float))]; - int t10[T(__is_const(const double))]; - int t11[T(__is_const(const long double))]; - int t12[T(__is_const(const unsigned char))]; - int t13[T(__is_const(const unsigned int))]; - int t14[T(__is_const(const unsigned long long))]; - int t15[T(__is_const(const unsigned long))]; - int t16[T(__is_const(const unsigned short))]; - int t17[T(__is_const(const void))]; - int t18[T(__is_const(const ClassType))]; - int t19[T(__is_const(const Derives))]; - int t20[T(__is_const(const Enum))]; - int t21[T(__is_const(const IntArNB))]; - int t22[T(__is_const(const Union))]; - int t23[T(__is_const(const UnionAr))]; - - int t30[F(__is_const(char))]; - int t31[F(__is_const(int))]; - int t32[F(__is_const(long))]; - int t33[F(__is_const(short))]; - int t34[F(__is_const(signed char))]; - int t35[F(__is_const(wchar_t))]; - int t36[F(__is_const(bool))]; - int t37[F(__is_const(float))]; - int t38[F(__is_const(double))]; - int t39[F(__is_const(long double))]; - int t40[F(__is_const(unsigned char))]; - int t41[F(__is_const(unsigned int))]; - int t42[F(__is_const(unsigned long long))]; - int t43[F(__is_const(unsigned long))]; - int t44[F(__is_const(unsigned short))]; - int t45[F(__is_const(void))]; - int t46[F(__is_const(ClassType))]; - int t47[F(__is_const(Derives))]; - int t48[F(__is_const(Enum))]; - int t49[F(__is_const(IntArNB))]; - int t50[F(__is_const(Union))]; - int t51[F(__is_const(UnionAr))]; + static_assert(__is_const(cvoid)); + static_assert(__is_const(const char)); + static_assert(__is_const(const int)); + static_assert(__is_const(const long)); + static_assert(__is_const(const short)); + static_assert(__is_const(const signed char)); + static_assert(__is_const(const wchar_t)); + static_assert(__is_const(const bool)); + static_assert(__is_const(const float)); + static_assert(__is_const(const double)); + static_assert(__is_const(const long double)); + static_assert(__is_const(const unsigned char)); + static_assert(__is_const(const unsigned int)); + static_assert(__is_const(const unsigned long long)); + static_assert(__is_const(const unsigned long)); + static_assert(__is_const(const unsigned short)); + static_assert(__is_const(const void)); + static_assert(__is_const(const ClassType)); + static_assert(__is_const(const Derives)); + static_assert(__is_const(const Enum)); + static_assert(__is_const(const IntArNB)); + static_assert(__is_const(const Union)); + static_assert(__is_const(const UnionAr)); + + static_assert(!__is_const(char)); + static_assert(!__is_const(int)); + static_assert(!__is_const(long)); + static_assert(!__is_const(short)); + static_assert(!__is_const(signed char)); + static_assert(!__is_const(wchar_t)); + static_assert(!__is_const(bool)); + static_assert(!__is_const(float)); + static_assert(!__is_const(double)); + static_assert(!__is_const(long double)); + static_assert(!__is_const(unsigned char)); + static_assert(!__is_const(unsigned int)); + static_assert(!__is_const(unsigned long long)); + static_assert(!__is_const(unsigned long)); + static_assert(!__is_const(unsigned short)); + static_assert(!__is_const(void)); + static_assert(!__is_const(ClassType)); + static_assert(!__is_const(Derives)); + static_assert(!__is_const(Enum)); + static_assert(!__is_const(IntArNB)); + static_assert(!__is_const(Union)); + static_assert(!__is_const(UnionAr)); } void is_volatile() { - int t02[T(__is_volatile(volatile char))]; - int t03[T(__is_volatile(volatile int))]; - int t04[T(__is_volatile(volatile long))]; - int t05[T(__is_volatile(volatile short))]; - int t06[T(__is_volatile(volatile signed char))]; - int t07[T(__is_volatile(volatile wchar_t))]; - int t08[T(__is_volatile(volatile bool))]; - int t09[T(__is_volatile(volatile float))]; - int t10[T(__is_volatile(volatile double))]; - int t11[T(__is_volatile(volatile long double))]; - int t12[T(__is_volatile(volatile unsigned char))]; - int t13[T(__is_volatile(volatile unsigned int))]; - int t14[T(__is_volatile(volatile unsigned long long))]; - int t15[T(__is_volatile(volatile unsigned long))]; - int t16[T(__is_volatile(volatile unsigned short))]; - int t17[T(__is_volatile(volatile void))]; - int t18[T(__is_volatile(volatile ClassType))]; - int t19[T(__is_volatile(volatile Derives))]; - int t20[T(__is_volatile(volatile Enum))]; - int t21[T(__is_volatile(volatile IntArNB))]; - int t22[T(__is_volatile(volatile Union))]; - int t23[T(__is_volatile(volatile UnionAr))]; - - int t30[F(__is_volatile(char))]; - int t31[F(__is_volatile(int))]; - int t32[F(__is_volatile(long))]; - int t33[F(__is_volatile(short))]; - int t34[F(__is_volatile(signed char))]; - int t35[F(__is_volatile(wchar_t))]; - int t36[F(__is_volatile(bool))]; - int t37[F(__is_volatile(float))]; - int t38[F(__is_volatile(double))]; - int t39[F(__is_volatile(long double))]; - int t40[F(__is_volatile(unsigned char))]; - int t41[F(__is_volatile(unsigned int))]; - int t42[F(__is_volatile(unsigned long long))]; - int t43[F(__is_volatile(unsigned long))]; - int t44[F(__is_volatile(unsigned short))]; - int t45[F(__is_volatile(void))]; - int t46[F(__is_volatile(ClassType))]; - int t47[F(__is_volatile(Derives))]; - int t48[F(__is_volatile(Enum))]; - int t49[F(__is_volatile(IntArNB))]; - int t50[F(__is_volatile(Union))]; - int t51[F(__is_volatile(UnionAr))]; + static_assert(__is_volatile(volatile char)); + static_assert(__is_volatile(volatile int)); + static_assert(__is_volatile(volatile long)); + static_assert(__is_volatile(volatile short)); + static_assert(__is_volatile(volatile signed char)); + static_assert(__is_volatile(volatile wchar_t)); + static_assert(__is_volatile(volatile bool)); + static_assert(__is_volatile(volatile float)); + static_assert(__is_volatile(volatile double)); + static_assert(__is_volatile(volatile long double)); + static_assert(__is_volatile(volatile unsigned char)); + static_assert(__is_volatile(volatile unsigned int)); + static_assert(__is_volatile(volatile unsigned long long)); + static_assert(__is_volatile(volatile unsigned long)); + static_assert(__is_volatile(volatile unsigned short)); + static_assert(__is_volatile(volatile void)); + static_assert(__is_volatile(volatile ClassType)); + static_assert(__is_volatile(volatile Derives)); + static_assert(__is_volatile(volatile Enum)); + static_assert(__is_volatile(volatile IntArNB)); + static_assert(__is_volatile(volatile Union)); + static_assert(__is_volatile(volatile UnionAr)); + + static_assert(!__is_volatile(char)); + static_assert(!__is_volatile(int)); + static_assert(!__is_volatile(long)); + static_assert(!__is_volatile(short)); + static_assert(!__is_volatile(signed char)); + static_assert(!__is_volatile(wchar_t)); + static_assert(!__is_volatile(bool)); + static_assert(!__is_volatile(float)); + static_assert(!__is_volatile(double)); + static_assert(!__is_volatile(long double)); + static_assert(!__is_volatile(unsigned char)); + static_assert(!__is_volatile(unsigned int)); + static_assert(!__is_volatile(unsigned long long)); + static_assert(!__is_volatile(unsigned long)); + static_assert(!__is_volatile(unsigned short)); + static_assert(!__is_volatile(void)); + static_assert(!__is_volatile(ClassType)); + static_assert(!__is_volatile(Derives)); + static_assert(!__is_volatile(Enum)); + static_assert(!__is_volatile(IntArNB)); + static_assert(!__is_volatile(Union)); + static_assert(!__is_volatile(UnionAr)); } struct TrivialStruct { @@ -1352,87 +1350,87 @@ ExtDefaulted::~ExtDefaulted() = default; void is_trivial2() { - int t01[T(__is_trivial(char))]; - int t02[T(__is_trivial(int))]; - int t03[T(__is_trivial(long))]; - int t04[T(__is_trivial(short))]; - int t05[T(__is_trivial(signed char))]; - int t06[T(__is_trivial(wchar_t))]; - int t07[T(__is_trivial(bool))]; - int t08[T(__is_trivial(float))]; - int t09[T(__is_trivial(double))]; - int t10[T(__is_trivial(long double))]; - int t11[T(__is_trivial(unsigned char))]; - int t12[T(__is_trivial(unsigned int))]; - int t13[T(__is_trivial(unsigned long long))]; - int t14[T(__is_trivial(unsigned long))]; - int t15[T(__is_trivial(unsigned short))]; - int t16[T(__is_trivial(ClassType))]; - int t17[T(__is_trivial(Derives))]; - int t18[T(__is_trivial(Enum))]; - int t19[T(__is_trivial(IntAr))]; - int t20[T(__is_trivial(Union))]; - int t21[T(__is_trivial(UnionAr))]; - int t22[T(__is_trivial(TrivialStruct))]; - int t23[T(__is_trivial(AllDefaulted))]; - int t24[T(__is_trivial(AllDeleted))]; - - int t30[F(__is_trivial(void))]; - int t31[F(__is_trivial(NonTrivialStruct))]; - int t32[F(__is_trivial(SuperNonTrivialStruct))]; - int t33[F(__is_trivial(NonTCStruct))]; - int t34[F(__is_trivial(ExtDefaulted))]; - - int t40[T(__is_trivial(ACompleteType))]; - int t41[F(__is_trivial(AnIncompleteType))]; // expected-error {{incomplete type}} - int t42[F(__is_trivial(AnIncompleteType[]))]; // expected-error {{incomplete type}} - int t43[F(__is_trivial(AnIncompleteType[1]))]; // expected-error {{incomplete type}} - int t44[F(__is_trivial(void))]; - int t45[F(__is_trivial(const volatile void))]; + static_assert(__is_trivial(char)); + static_assert(__is_trivial(int)); + static_assert(__is_trivial(long)); + static_assert(__is_trivial(short)); + static_assert(__is_trivial(signed char)); + static_assert(__is_trivial(wchar_t)); + static_assert(__is_trivial(bool)); + static_assert(__is_trivial(float)); + static_assert(__is_trivial(double)); + static_assert(__is_trivial(long double)); + static_assert(__is_trivial(unsigned char)); + static_assert(__is_trivial(unsigned int)); + static_assert(__is_trivial(unsigned long long)); + static_assert(__is_trivial(unsigned long)); + static_assert(__is_trivial(unsigned short)); + static_assert(__is_trivial(ClassType)); + static_assert(__is_trivial(Derives)); + static_assert(__is_trivial(Enum)); + static_assert(__is_trivial(IntAr)); + static_assert(__is_trivial(Union)); + static_assert(__is_trivial(UnionAr)); + static_assert(__is_trivial(TrivialStruct)); + static_assert(__is_trivial(AllDefaulted)); + static_assert(__is_trivial(AllDeleted)); + + static_assert(!__is_trivial(void)); + static_assert(!__is_trivial(NonTrivialStruct)); + static_assert(!__is_trivial(SuperNonTrivialStruct)); + static_assert(!__is_trivial(NonTCStruct)); + static_assert(!__is_trivial(ExtDefaulted)); + + static_assert(__is_trivial(ACompleteType)); + static_assert(!__is_trivial(AnIncompleteType)); // expected-error {{incomplete type}} + static_assert(!__is_trivial(AnIncompleteType[])); // expected-error {{incomplete type}} + static_assert(!__is_trivial(AnIncompleteType[1])); // expected-error {{incomplete type}} + static_assert(!__is_trivial(void)); + static_assert(!__is_trivial(const volatile void)); } void is_trivially_copyable2() { - int t01[T(__is_trivially_copyable(char))]; - int t02[T(__is_trivially_copyable(int))]; - int t03[T(__is_trivially_copyable(long))]; - int t04[T(__is_trivially_copyable(short))]; - int t05[T(__is_trivially_copyable(signed char))]; - int t06[T(__is_trivially_copyable(wchar_t))]; - int t07[T(__is_trivially_copyable(bool))]; - int t08[T(__is_trivially_copyable(float))]; - int t09[T(__is_trivially_copyable(double))]; - int t10[T(__is_trivially_copyable(long double))]; - int t11[T(__is_trivially_copyable(unsigned char))]; - int t12[T(__is_trivially_copyable(unsigned int))]; - int t13[T(__is_trivially_copyable(unsigned long long))]; - int t14[T(__is_trivially_copyable(unsigned long))]; - int t15[T(__is_trivially_copyable(unsigned short))]; - int t16[T(__is_trivially_copyable(ClassType))]; - int t17[T(__is_trivially_copyable(Derives))]; - int t18[T(__is_trivially_copyable(Enum))]; - int t19[T(__is_trivially_copyable(IntAr))]; - int t20[T(__is_trivially_copyable(Union))]; - int t21[T(__is_trivially_copyable(UnionAr))]; - int t22[T(__is_trivially_copyable(TrivialStruct))]; - int t23[T(__is_trivially_copyable(NonTrivialStruct))]; - int t24[T(__is_trivially_copyable(AllDefaulted))]; - int t25[T(__is_trivially_copyable(AllDeleted))]; - - int t30[F(__is_trivially_copyable(void))]; - int t31[F(__is_trivially_copyable(SuperNonTrivialStruct))]; - int t32[F(__is_trivially_copyable(NonTCStruct))]; - int t33[F(__is_trivially_copyable(ExtDefaulted))]; - - int t34[T(__is_trivially_copyable(const int))]; - int t35[T(__is_trivially_copyable(volatile int))]; - - int t40[T(__is_trivially_copyable(ACompleteType))]; - int t41[F(__is_trivially_copyable(AnIncompleteType))]; // expected-error {{incomplete type}} - int t42[F(__is_trivially_copyable(AnIncompleteType[]))]; // expected-error {{incomplete type}} - int t43[F(__is_trivially_copyable(AnIncompleteType[1]))]; // expected-error {{incomplete type}} - int t44[F(__is_trivially_copyable(void))]; - int t45[F(__is_trivially_copyable(const volatile void))]; + static_assert(__is_trivially_copyable(char)); + static_assert(__is_trivially_copyable(int)); + static_assert(__is_trivially_copyable(long)); + static_assert(__is_trivially_copyable(short)); + static_assert(__is_trivially_copyable(signed char)); + static_assert(__is_trivially_copyable(wchar_t)); + static_assert(__is_trivially_copyable(bool)); + static_assert(__is_trivially_copyable(float)); + static_assert(__is_trivially_copyable(double)); + static_assert(__is_trivially_copyable(long double)); + static_assert(__is_trivially_copyable(unsigned char)); + static_assert(__is_trivially_copyable(unsigned int)); + static_assert(__is_trivially_copyable(unsigned long long)); + static_assert(__is_trivially_copyable(unsigned long)); + static_assert(__is_trivially_copyable(unsigned short)); + static_assert(__is_trivially_copyable(ClassType)); + static_assert(__is_trivially_copyable(Derives)); + static_assert(__is_trivially_copyable(Enum)); + static_assert(__is_trivially_copyable(IntAr)); + static_assert(__is_trivially_copyable(Union)); + static_assert(__is_trivially_copyable(UnionAr)); + static_assert(__is_trivially_copyable(TrivialStruct)); + static_assert(__is_trivially_copyable(NonTrivialStruct)); + static_assert(__is_trivially_copyable(AllDefaulted)); + static_assert(__is_trivially_copyable(AllDeleted)); + + static_assert(!__is_trivially_copyable(void)); + static_assert(!__is_trivially_copyable(SuperNonTrivialStruct)); + static_assert(!__is_trivially_copyable(NonTCStruct)); + static_assert(!__is_trivially_copyable(ExtDefaulted)); + + static_assert(__is_trivially_copyable(const int)); + static_assert(__is_trivially_copyable(volatile int)); + + static_assert(__is_trivially_copyable(ACompleteType)); + static_assert(!__is_trivially_copyable(AnIncompleteType)); // expected-error {{incomplete type}} + static_assert(!__is_trivially_copyable(AnIncompleteType[])); // expected-error {{incomplete type}} + static_assert(!__is_trivially_copyable(AnIncompleteType[1])); // expected-error {{incomplete type}} + static_assert(!__is_trivially_copyable(void)); + static_assert(!__is_trivially_copyable(const volatile void)); } struct CStruct { @@ -1477,32 +1475,32 @@ void is_standard_layout() typedef ConstInt ConstIntAr[4]; typedef CppStructStandard CppStructStandardAr[4]; - int t01[T(__is_standard_layout(int))]; - int t02[T(__is_standard_layout(ConstInt))]; - int t03[T(__is_standard_layout(ConstIntAr))]; - int t04[T(__is_standard_layout(CStruct))]; - int t05[T(__is_standard_layout(CppStructStandard))]; - int t06[T(__is_standard_layout(CppStructStandardAr))]; - int t07[T(__is_standard_layout(Vector))]; - int t08[T(__is_standard_layout(VectorExt))]; + static_assert(__is_standard_layout(int)); + static_assert(__is_standard_layout(ConstInt)); + static_assert(__is_standard_layout(ConstIntAr)); + static_assert(__is_standard_layout(CStruct)); + static_assert(__is_standard_layout(CppStructStandard)); + static_assert(__is_standard_layout(CppStructStandardAr)); + static_assert(__is_standard_layout(Vector)); + static_assert(__is_standard_layout(VectorExt)); typedef CppStructNonStandardByBase CppStructNonStandardByBaseAr[4]; - int t10[F(__is_standard_layout(CppStructNonStandardByVirt))]; - int t11[F(__is_standard_layout(CppStructNonStandardByMemb))]; - int t12[F(__is_standard_layout(CppStructNonStandardByProt))]; - int t13[F(__is_standard_layout(CppStructNonStandardByVirtBase))]; - int t14[F(__is_standard_layout(CppStructNonStandardByBase))]; - int t15[F(__is_standard_layout(CppStructNonStandardByBaseAr))]; - int t16[F(__is_standard_layout(CppStructNonStandardBySameBase))]; - int t17[F(__is_standard_layout(CppStructNonStandardBy2ndVirtBase))]; - - int t40[T(__is_standard_layout(ACompleteType))]; - int t41[F(__is_standard_layout(AnIncompleteType))]; // expected-error {{incomplete type}} - int t42[F(__is_standard_layout(AnIncompleteType[]))]; // expected-error {{incomplete type}} - int t43[F(__is_standard_layout(AnIncompleteType[1]))]; // expected-error {{incomplete type}} - int t44[F(__is_standard_layout(void))]; - int t45[F(__is_standard_layout(const volatile void))]; + static_assert(!__is_standard_layout(CppStructNonStandardByVirt)); + static_assert(!__is_standard_layout(CppStructNonStandardByMemb)); + static_assert(!__is_standard_layout(CppStructNonStandardByProt)); + static_assert(!__is_standard_layout(CppStructNonStandardByVirtBase)); + static_assert(!__is_standard_layout(CppStructNonStandardByBase)); + static_assert(!__is_standard_layout(CppStructNonStandardByBaseAr)); + static_assert(!__is_standard_layout(CppStructNonStandardBySameBase)); + static_assert(!__is_standard_layout(CppStructNonStandardBy2ndVirtBase)); + + static_assert(__is_standard_layout(ACompleteType)); + static_assert(!__is_standard_layout(AnIncompleteType)); // expected-error {{incomplete type}} + static_assert(!__is_standard_layout(AnIncompleteType[])); // expected-error {{incomplete type}} + static_assert(!__is_standard_layout(AnIncompleteType[1])); // expected-error {{incomplete type}} + static_assert(!__is_standard_layout(void)); + static_assert(!__is_standard_layout(const volatile void)); struct HasAnonEmptyBitfield { int : 0; }; struct HasAnonBitfield { int : 4; }; @@ -1510,11 +1508,11 @@ void is_standard_layout() struct DerivesFromBitfieldWithBitfield : HasAnonBitfield { int : 5; }; struct DerivesFromBitfieldTwice : DerivesFromBitfield, HasAnonEmptyBitfield {}; - int t50[T(__is_standard_layout(HasAnonEmptyBitfield))]; - int t51[T(__is_standard_layout(HasAnonBitfield))]; - int t52[T(__is_standard_layout(DerivesFromBitfield))]; - int t53[F(__is_standard_layout(DerivesFromBitfieldWithBitfield))]; - int t54[F(__is_standard_layout(DerivesFromBitfieldTwice))]; + static_assert(__is_standard_layout(HasAnonEmptyBitfield)); + static_assert(__is_standard_layout(HasAnonBitfield)); + static_assert(__is_standard_layout(DerivesFromBitfield)); + static_assert(!__is_standard_layout(DerivesFromBitfieldWithBitfield)); + static_assert(!__is_standard_layout(DerivesFromBitfieldTwice)); struct Empty {}; struct HasEmptyBase : Empty {}; @@ -1528,16 +1526,16 @@ void is_standard_layout() struct HasEmptyIndirectBaseAsSecondMember : HasEmptyBase { int n; Empty e; }; struct HasEmptyIndirectBaseAfterBitfield : HasEmptyBase { int : 4; Empty e; }; - int t60[T(__is_standard_layout(Empty))]; - int t61[T(__is_standard_layout(HasEmptyBase))]; - int t62[F(__is_standard_layout(HasRepeatedEmptyBase))]; - int t63[F(__is_standard_layout(HasEmptyBaseAsMember))]; - int t64[F(__is_standard_layout(HasEmptyBaseAsSubobjectOfMember1))]; - int t65[T(__is_standard_layout(HasEmptyBaseAsSubobjectOfMember2))]; // FIXME: standard bug? - int t66[F(__is_standard_layout(HasEmptyBaseAsSubobjectOfMember3))]; - int t67[F(__is_standard_layout(HasEmptyIndirectBaseAsMember))]; - int t68[T(__is_standard_layout(HasEmptyIndirectBaseAsSecondMember))]; - int t69[F(__is_standard_layout(HasEmptyIndirectBaseAfterBitfield))]; // FIXME: standard bug? + static_assert(__is_standard_layout(Empty)); + static_assert(__is_standard_layout(HasEmptyBase)); + static_assert(!__is_standard_layout(HasRepeatedEmptyBase)); + static_assert(!__is_standard_layout(HasEmptyBaseAsMember)); + static_assert(!__is_standard_layout(HasEmptyBaseAsSubobjectOfMember1)); + static_assert(__is_standard_layout(HasEmptyBaseAsSubobjectOfMember2)); // FIXME: standard bug? + static_assert(!__is_standard_layout(HasEmptyBaseAsSubobjectOfMember3)); + static_assert(!__is_standard_layout(HasEmptyIndirectBaseAsMember)); + static_assert(__is_standard_layout(HasEmptyIndirectBaseAsSecondMember)); + static_assert(!__is_standard_layout(HasEmptyIndirectBaseAfterBitfield)); // FIXME: standard bug? struct StructWithEmptyFields { int n; @@ -1554,8 +1552,8 @@ void is_standard_layout() UnionWithEmptyFields u; }; - int t70[T(__is_standard_layout(HasEmptyIndirectBaseAsSecondStructMember))]; - int t71[F(__is_standard_layout(HasEmptyIndirectBaseAsSecondUnionMember))]; + static_assert(__is_standard_layout(HasEmptyIndirectBaseAsSecondStructMember)); + static_assert(!__is_standard_layout(HasEmptyIndirectBaseAsSecondUnionMember)); } struct CStruct2 { @@ -1813,62 +1811,62 @@ void is_layout_compatible(int n) void is_signed() { - //int t01[T(__is_signed(char))]; - int t02[T(__is_signed(int))]; - int t03[T(__is_signed(long))]; - int t04[T(__is_signed(short))]; - int t05[T(__is_signed(signed char))]; - int t06[T(__is_signed(wchar_t))]; - int t07[T(__is_signed(float))]; - int t08[T(__is_signed(double))]; - int t09[T(__is_signed(long double))]; - - int t13[F(__is_signed(bool))]; - int t14[F(__is_signed(cvoid))]; - int t15[F(__is_signed(unsigned char))]; - int t16[F(__is_signed(unsigned int))]; - int t17[F(__is_signed(unsigned long long))]; - int t18[F(__is_signed(unsigned long))]; - int t19[F(__is_signed(unsigned short))]; - int t20[F(__is_signed(void))]; - int t21[F(__is_signed(ClassType))]; - int t22[F(__is_signed(Derives))]; - int t23[F(__is_signed(Enum))]; - int t24[F(__is_signed(SignedEnum))]; - int t25[F(__is_signed(IntArNB))]; - int t26[F(__is_signed(Union))]; - int t27[F(__is_signed(UnionAr))]; - int t28[F(__is_signed(UnsignedEnum))]; + //static_assert(__is_signed(char)); + static_assert(__is_signed(int)); + static_assert(__is_signed(long)); + static_assert(__is_signed(short)); + static_assert(__is_signed(signed char)); + static_assert(__is_signed(wchar_t)); + static_assert(__is_signed(float)); + static_assert(__is_signed(double)); + static_assert(__is_signed(long double)); + + static_assert(!__is_signed(bool)); + static_assert(!__is_signed(cvoid)); + static_assert(!__is_signed(unsigned char)); + static_assert(!__is_signed(unsigned int)); + static_assert(!__is_signed(unsigned long long)); + static_assert(!__is_signed(unsigned long)); + static_assert(!__is_signed(unsigned short)); + static_assert(!__is_signed(void)); + static_assert(!__is_signed(ClassType)); + static_assert(!__is_signed(Derives)); + static_assert(!__is_signed(Enum)); + static_assert(!__is_signed(SignedEnum)); + static_assert(!__is_signed(IntArNB)); + static_assert(!__is_signed(Union)); + static_assert(!__is_signed(UnionAr)); + static_assert(!__is_signed(UnsignedEnum)); } void is_unsigned() { - int t01[T(__is_unsigned(bool))]; - int t02[T(__is_unsigned(unsigned char))]; - int t03[T(__is_unsigned(unsigned short))]; - int t04[T(__is_unsigned(unsigned int))]; - int t05[T(__is_unsigned(unsigned long))]; - int t06[T(__is_unsigned(unsigned long long))]; - - int t10[F(__is_unsigned(void))]; - int t11[F(__is_unsigned(cvoid))]; - int t12[F(__is_unsigned(float))]; - int t13[F(__is_unsigned(double))]; - int t14[F(__is_unsigned(long double))]; - int t16[F(__is_unsigned(char))]; - int t17[F(__is_unsigned(signed char))]; - int t18[F(__is_unsigned(wchar_t))]; - int t19[F(__is_unsigned(short))]; - int t20[F(__is_unsigned(int))]; - int t21[F(__is_unsigned(long))]; - int t22[F(__is_unsigned(Union))]; - int t23[F(__is_unsigned(UnionAr))]; - int t24[F(__is_unsigned(Derives))]; - int t25[F(__is_unsigned(ClassType))]; - int t26[F(__is_unsigned(IntArNB))]; - int t27[F(__is_unsigned(Enum))]; - int t28[F(__is_unsigned(UnsignedEnum))]; - int t29[F(__is_unsigned(SignedEnum))]; + static_assert(__is_unsigned(bool)); + static_assert(__is_unsigned(unsigned char)); + static_assert(__is_unsigned(unsigned short)); + static_assert(__is_unsigned(unsigned int)); + static_assert(__is_unsigned(unsigned long)); + static_assert(__is_unsigned(unsigned long long)); + + static_assert(!__is_unsigned(void)); + static_assert(!__is_unsigned(cvoid)); + static_assert(!__is_unsigned(float)); + static_assert(!__is_unsigned(double)); + static_assert(!__is_unsigned(long double)); + static_assert(!__is_unsigned(char)); + static_assert(!__is_unsigned(signed char)); + static_assert(!__is_unsigned(wchar_t)); + static_assert(!__is_unsigned(short)); + static_assert(!__is_unsigned(int)); + static_assert(!__is_unsigned(long)); + static_assert(!__is_unsigned(Union)); + static_assert(!__is_unsigned(UnionAr)); + static_assert(!__is_unsigned(Derives)); + static_assert(!__is_unsigned(ClassType)); + static_assert(!__is_unsigned(IntArNB)); + static_assert(!__is_unsigned(Enum)); + static_assert(!__is_unsigned(UnsignedEnum)); + static_assert(!__is_unsigned(SignedEnum)); } typedef Int& IntRef; @@ -1891,35 +1889,35 @@ struct HasTemplateCons { }; void has_trivial_default_constructor() { - { int arr[T(__has_trivial_constructor(Int))]; } - { int arr[T(__has_trivial_constructor(IntAr))]; } - { int arr[T(__has_trivial_constructor(Union))]; } - { int arr[T(__has_trivial_constructor(UnionAr))]; } - { int arr[T(__has_trivial_constructor(POD))]; } - { int arr[T(__has_trivial_constructor(Derives))]; } - { int arr[T(__has_trivial_constructor(DerivesAr))]; } - { int arr[T(__has_trivial_constructor(ConstIntAr))]; } - { int arr[T(__has_trivial_constructor(ConstIntArAr))]; } - { int arr[T(__has_trivial_constructor(HasDest))]; } - { int arr[T(__has_trivial_constructor(HasPriv))]; } - { int arr[T(__has_trivial_constructor(HasCopyAssign))]; } - { int arr[T(__has_trivial_constructor(HasMoveAssign))]; } - { int arr[T(__has_trivial_constructor(const Int))]; } - { int arr[T(__has_trivial_constructor(AllDefaulted))]; } - { int arr[T(__has_trivial_constructor(AllDeleted))]; } - { int arr[T(__has_trivial_constructor(ACompleteType[]))]; } - - { int arr[F(__has_trivial_constructor(AnIncompleteType[]))]; } // expected-error {{incomplete type}} - { int arr[F(__has_trivial_constructor(HasCons))]; } - { int arr[F(__has_trivial_constructor(HasRef))]; } - { int arr[F(__has_trivial_constructor(HasCopy))]; } - { int arr[F(__has_trivial_constructor(IntRef))]; } - { int arr[F(__has_trivial_constructor(VirtAr))]; } - { int arr[F(__has_trivial_constructor(void))]; } - { int arr[F(__has_trivial_constructor(cvoid))]; } - { int arr[F(__has_trivial_constructor(HasTemplateCons))]; } - { int arr[F(__has_trivial_constructor(AllPrivate))]; } - { int arr[F(__has_trivial_constructor(ExtDefaulted))]; } + static_assert(__has_trivial_constructor(Int)); + static_assert(__has_trivial_constructor(IntAr)); + static_assert(__has_trivial_constructor(Union)); + static_assert(__has_trivial_constructor(UnionAr)); + static_assert(__has_trivial_constructor(POD)); + static_assert(__has_trivial_constructor(Derives)); + static_assert(__has_trivial_constructor(DerivesAr)); + static_assert(__has_trivial_constructor(ConstIntAr)); + static_assert(__has_trivial_constructor(ConstIntArAr)); + static_assert(__has_trivial_constructor(HasDest)); + static_assert(__has_trivial_constructor(HasPriv)); + static_assert(__has_trivial_constructor(HasCopyAssign)); + static_assert(__has_trivial_constructor(HasMoveAssign)); + static_assert(__has_trivial_constructor(const Int)); + static_assert(__has_trivial_constructor(AllDefaulted)); + static_assert(__has_trivial_constructor(AllDeleted)); + static_assert(__has_trivial_constructor(ACompleteType[])); + + static_assert(!__has_trivial_constructor(AnIncompleteType[])); // expected-error {{incomplete type}} + static_assert(!__has_trivial_constructor(HasCons)); + static_assert(!__has_trivial_constructor(HasRef)); + static_assert(!__has_trivial_constructor(HasCopy)); + static_assert(!__has_trivial_constructor(IntRef)); + static_assert(!__has_trivial_constructor(VirtAr)); + static_assert(!__has_trivial_constructor(void)); + static_assert(!__has_trivial_constructor(cvoid)); + static_assert(!__has_trivial_constructor(HasTemplateCons)); + static_assert(!__has_trivial_constructor(AllPrivate)); + static_assert(!__has_trivial_constructor(ExtDefaulted)); } void has_trivial_move_constructor() { @@ -1935,127 +1933,127 @@ void has_trivial_move_constructor() { // type (or array thereof), the constructor selected // to copy/move that member is trivial; // otherwise the copy/move constructor is non-trivial. - { int arr[T(__has_trivial_move_constructor(POD))]; } - { int arr[T(__has_trivial_move_constructor(Union))]; } - { int arr[T(__has_trivial_move_constructor(HasCons))]; } - { int arr[T(__has_trivial_move_constructor(HasStaticMemberMoveCtor))]; } - { int arr[T(__has_trivial_move_constructor(AllDeleted))]; } - { int arr[T(__has_trivial_move_constructor(ACompleteType[]))]; } - - { int arr[F(__has_trivial_move_constructor(AnIncompleteType[]))]; } // expected-error {{incomplete type}} - { int arr[F(__has_trivial_move_constructor(HasVirt))]; } - { int arr[F(__has_trivial_move_constructor(DerivesVirt))]; } - { int arr[F(__has_trivial_move_constructor(HasMoveCtor))]; } - { int arr[F(__has_trivial_move_constructor(DerivesHasMoveCtor))]; } - { int arr[F(__has_trivial_move_constructor(HasMemberMoveCtor))]; } + static_assert(__has_trivial_move_constructor(POD)); + static_assert(__has_trivial_move_constructor(Union)); + static_assert(__has_trivial_move_constructor(HasCons)); + static_assert(__has_trivial_move_constructor(HasStaticMemberMoveCtor)); + static_assert(__has_trivial_move_constructor(AllDeleted)); + static_assert(__has_trivial_move_constructor(ACompleteType[])); + + static_assert(!__has_trivial_move_constructor(AnIncompleteType[])); // expected-error {{incomplete type}} + static_assert(!__has_trivial_move_constructor(HasVirt)); + static_assert(!__has_trivial_move_constructor(DerivesVirt)); + static_assert(!__has_trivial_move_constructor(HasMoveCtor)); + static_assert(!__has_trivial_move_constructor(DerivesHasMoveCtor)); + static_assert(!__has_trivial_move_constructor(HasMemberMoveCtor)); } void has_trivial_copy_constructor() { - { int arr[T(__has_trivial_copy(Int))]; } - { int arr[T(__has_trivial_copy(IntAr))]; } - { int arr[T(__has_trivial_copy(Union))]; } - { int arr[T(__has_trivial_copy(UnionAr))]; } - { int arr[T(__has_trivial_copy(POD))]; } - { int arr[T(__has_trivial_copy(Derives))]; } - { int arr[T(__has_trivial_copy(ConstIntAr))]; } - { int arr[T(__has_trivial_copy(ConstIntArAr))]; } - { int arr[T(__has_trivial_copy(HasDest))]; } - { int arr[T(__has_trivial_copy(HasPriv))]; } - { int arr[T(__has_trivial_copy(HasCons))]; } - { int arr[T(__has_trivial_copy(HasRef))]; } - { int arr[T(__has_trivial_copy(HasMove))]; } - { int arr[T(__has_trivial_copy(IntRef))]; } - { int arr[T(__has_trivial_copy(HasCopyAssign))]; } - { int arr[T(__has_trivial_copy(HasMoveAssign))]; } - { int arr[T(__has_trivial_copy(const Int))]; } - { int arr[T(__has_trivial_copy(AllDefaulted))]; } - { int arr[T(__has_trivial_copy(AllDeleted))]; } - { int arr[T(__has_trivial_copy(DerivesAr))]; } - { int arr[T(__has_trivial_copy(DerivesHasRef))]; } - { int arr[T(__has_trivial_copy(ACompleteType[]))]; } - - { int arr[F(__has_trivial_copy(AnIncompleteType[]))]; } // expected-error {{incomplete type}} - { int arr[F(__has_trivial_copy(HasCopy))]; } - { int arr[F(__has_trivial_copy(HasTemplateCons))]; } - { int arr[F(__has_trivial_copy(VirtAr))]; } - { int arr[F(__has_trivial_copy(void))]; } - { int arr[F(__has_trivial_copy(cvoid))]; } - { int arr[F(__has_trivial_copy(AllPrivate))]; } - { int arr[F(__has_trivial_copy(ExtDefaulted))]; } + static_assert(__has_trivial_copy(Int)); + static_assert(__has_trivial_copy(IntAr)); + static_assert(__has_trivial_copy(Union)); + static_assert(__has_trivial_copy(UnionAr)); + static_assert(__has_trivial_copy(POD)); + static_assert(__has_trivial_copy(Derives)); + static_assert(__has_trivial_copy(ConstIntAr)); + static_assert(__has_trivial_copy(ConstIntArAr)); + static_assert(__has_trivial_copy(HasDest)); + static_assert(__has_trivial_copy(HasPriv)); + static_assert(__has_trivial_copy(HasCons)); + static_assert(__has_trivial_copy(HasRef)); + static_assert(__has_trivial_copy(HasMove)); + static_assert(__has_trivial_copy(IntRef)); + static_assert(__has_trivial_copy(HasCopyAssign)); + static_assert(__has_trivial_copy(HasMoveAssign)); + static_assert(__has_trivial_copy(const Int)); + static_assert(__has_trivial_copy(AllDefaulted)); + static_assert(__has_trivial_copy(AllDeleted)); + static_assert(__has_trivial_copy(DerivesAr)); + static_assert(__has_trivial_copy(DerivesHasRef)); + static_assert(__has_trivial_copy(ACompleteType[])); + + static_assert(!__has_trivial_copy(AnIncompleteType[])); // expected-error {{incomplete type}} + static_assert(!__has_trivial_copy(HasCopy)); + static_assert(!__has_trivial_copy(HasTemplateCons)); + static_assert(!__has_trivial_copy(VirtAr)); + static_assert(!__has_trivial_copy(void)); + static_assert(!__has_trivial_copy(cvoid)); + static_assert(!__has_trivial_copy(AllPrivate)); + static_assert(!__has_trivial_copy(ExtDefaulted)); } void has_trivial_copy_assignment() { - { int arr[T(__has_trivial_assign(Int))]; } - { int arr[T(__has_trivial_assign(IntAr))]; } - { int arr[T(__has_trivial_assign(Union))]; } - { int arr[T(__has_trivial_assign(UnionAr))]; } - { int arr[T(__has_trivial_assign(POD))]; } - { int arr[T(__has_trivial_assign(Derives))]; } - { int arr[T(__has_trivial_assign(HasDest))]; } - { int arr[T(__has_trivial_assign(HasPriv))]; } - { int arr[T(__has_trivial_assign(HasCons))]; } - { int arr[T(__has_trivial_assign(HasRef))]; } - { int arr[T(__has_trivial_assign(HasCopy))]; } - { int arr[T(__has_trivial_assign(HasMove))]; } - { int arr[T(__has_trivial_assign(HasMoveAssign))]; } - { int arr[T(__has_trivial_assign(AllDefaulted))]; } - { int arr[T(__has_trivial_assign(AllDeleted))]; } - { int arr[T(__has_trivial_assign(DerivesAr))]; } - { int arr[T(__has_trivial_assign(DerivesHasRef))]; } - { int arr[T(__has_trivial_assign(ACompleteType[]))]; } - - { int arr[F(__has_trivial_assign(AnIncompleteType[]))]; } // expected-error {{incomplete type}} - { int arr[F(__has_trivial_assign(IntRef))]; } - { int arr[F(__has_trivial_assign(HasCopyAssign))]; } - { int arr[F(__has_trivial_assign(const Int))]; } - { int arr[F(__has_trivial_assign(ConstIntAr))]; } - { int arr[F(__has_trivial_assign(ConstIntArAr))]; } - { int arr[F(__has_trivial_assign(VirtAr))]; } - { int arr[F(__has_trivial_assign(void))]; } - { int arr[F(__has_trivial_assign(cvoid))]; } - { int arr[F(__has_trivial_assign(AllPrivate))]; } - { int arr[F(__has_trivial_assign(ExtDefaulted))]; } + static_assert(__has_trivial_assign(Int)); + static_assert(__has_trivial_assign(IntAr)); + static_assert(__has_trivial_assign(Union)); + static_assert(__has_trivial_assign(UnionAr)); + static_assert(__has_trivial_assign(POD)); + static_assert(__has_trivial_assign(Derives)); + static_assert(__has_trivial_assign(HasDest)); + static_assert(__has_trivial_assign(HasPriv)); + static_assert(__has_trivial_assign(HasCons)); + static_assert(__has_trivial_assign(HasRef)); + static_assert(__has_trivial_assign(HasCopy)); + static_assert(__has_trivial_assign(HasMove)); + static_assert(__has_trivial_assign(HasMoveAssign)); + static_assert(__has_trivial_assign(AllDefaulted)); + static_assert(__has_trivial_assign(AllDeleted)); + static_assert(__has_trivial_assign(DerivesAr)); + static_assert(__has_trivial_assign(DerivesHasRef)); + static_assert(__has_trivial_assign(ACompleteType[])); + + static_assert(!__has_trivial_assign(AnIncompleteType[])); // expected-error {{incomplete type}} + static_assert(!__has_trivial_assign(IntRef)); + static_assert(!__has_trivial_assign(HasCopyAssign)); + static_assert(!__has_trivial_assign(const Int)); + static_assert(!__has_trivial_assign(ConstIntAr)); + static_assert(!__has_trivial_assign(ConstIntArAr)); + static_assert(!__has_trivial_assign(VirtAr)); + static_assert(!__has_trivial_assign(void)); + static_assert(!__has_trivial_assign(cvoid)); + static_assert(!__has_trivial_assign(AllPrivate)); + static_assert(!__has_trivial_assign(ExtDefaulted)); } void has_trivial_destructor() { - { int arr[T(__has_trivial_destructor(Int))]; } - { int arr[T(__has_trivial_destructor(IntAr))]; } - { int arr[T(__has_trivial_destructor(Union))]; } - { int arr[T(__has_trivial_destructor(UnionAr))]; } - { int arr[T(__has_trivial_destructor(POD))]; } - { int arr[T(__has_trivial_destructor(Derives))]; } - { int arr[T(__has_trivial_destructor(ConstIntAr))]; } - { int arr[T(__has_trivial_destructor(ConstIntArAr))]; } - { int arr[T(__has_trivial_destructor(HasPriv))]; } - { int arr[T(__has_trivial_destructor(HasCons))]; } - { int arr[T(__has_trivial_destructor(HasRef))]; } - { int arr[T(__has_trivial_destructor(HasCopy))]; } - { int arr[T(__has_trivial_destructor(HasMove))]; } - { int arr[T(__has_trivial_destructor(IntRef))]; } - { int arr[T(__has_trivial_destructor(HasCopyAssign))]; } - { int arr[T(__has_trivial_destructor(HasMoveAssign))]; } - { int arr[T(__has_trivial_destructor(const Int))]; } - { int arr[T(__has_trivial_destructor(DerivesAr))]; } - { int arr[T(__has_trivial_destructor(VirtAr))]; } - { int arr[T(__has_trivial_destructor(AllDefaulted))]; } - { int arr[T(__has_trivial_destructor(AllDeleted))]; } - { int arr[T(__has_trivial_destructor(DerivesHasRef))]; } - { int arr[T(__has_trivial_destructor(ACompleteType[]))]; } - - { int arr[F(__has_trivial_destructor(HasDest))]; } - { int arr[F(__has_trivial_destructor(AnIncompleteType[]))]; } // expected-error {{incomplete type}} - { int arr[F(__has_trivial_destructor(void))]; } - { int arr[F(__has_trivial_destructor(cvoid))]; } - { int arr[F(__has_trivial_destructor(AllPrivate))]; } - { int arr[F(__has_trivial_destructor(ExtDefaulted))]; } + static_assert(__has_trivial_destructor(Int)); + static_assert(__has_trivial_destructor(IntAr)); + static_assert(__has_trivial_destructor(Union)); + static_assert(__has_trivial_destructor(UnionAr)); + static_assert(__has_trivial_destructor(POD)); + static_assert(__has_trivial_destructor(Derives)); + static_assert(__has_trivial_destructor(ConstIntAr)); + static_assert(__has_trivial_destructor(ConstIntArAr)); + static_assert(__has_trivial_destructor(HasPriv)); + static_assert(__has_trivial_destructor(HasCons)); + static_assert(__has_trivial_destructor(HasRef)); + static_assert(__has_trivial_destructor(HasCopy)); + static_assert(__has_trivial_destructor(HasMove)); + static_assert(__has_trivial_destructor(IntRef)); + static_assert(__has_trivial_destructor(HasCopyAssign)); + static_assert(__has_trivial_destructor(HasMoveAssign)); + static_assert(__has_trivial_destructor(const Int)); + static_assert(__has_trivial_destructor(DerivesAr)); + static_assert(__has_trivial_destructor(VirtAr)); + static_assert(__has_trivial_destructor(AllDefaulted)); + static_assert(__has_trivial_destructor(AllDeleted)); + static_assert(__has_trivial_destructor(DerivesHasRef)); + static_assert(__has_trivial_destructor(ACompleteType[])); + + static_assert(!__has_trivial_destructor(HasDest)); + static_assert(!__has_trivial_destructor(AnIncompleteType[])); // expected-error {{incomplete type}} + static_assert(!__has_trivial_destructor(void)); + static_assert(!__has_trivial_destructor(cvoid)); + static_assert(!__has_trivial_destructor(AllPrivate)); + static_assert(!__has_trivial_destructor(ExtDefaulted)); } struct A { ~A() {} }; template struct B : A { }; void f() { - { int arr[F(__has_trivial_destructor(A))]; } - { int arr[F(__has_trivial_destructor(B))]; } + static_assert(!__has_trivial_destructor(A)); + static_assert(!__has_trivial_destructor(B)); } class PR11110 { @@ -2076,69 +2074,69 @@ public: }; void has_nothrow_assign() { - { int arr[T(__has_nothrow_assign(Int))]; } - { int arr[T(__has_nothrow_assign(IntAr))]; } - { int arr[T(__has_nothrow_assign(Union))]; } - { int arr[T(__has_nothrow_assign(UnionAr))]; } - { int arr[T(__has_nothrow_assign(POD))]; } - { int arr[T(__has_nothrow_assign(Derives))]; } - { int arr[T(__has_nothrow_assign(HasDest))]; } - { int arr[T(__has_nothrow_assign(HasPriv))]; } - { int arr[T(__has_nothrow_assign(HasCons))]; } - { int arr[T(__has_nothrow_assign(HasRef))]; } - { int arr[T(__has_nothrow_assign(HasCopy))]; } - { int arr[T(__has_nothrow_assign(HasMove))]; } - { int arr[T(__has_nothrow_assign(HasMoveAssign))]; } - { int arr[T(__has_nothrow_assign(HasNoThrowCopyAssign))]; } - { int arr[T(__has_nothrow_assign(HasMultipleNoThrowCopyAssign))]; } - { int arr[T(__has_nothrow_assign(HasVirtDest))]; } - { int arr[T(__has_nothrow_assign(AllPrivate))]; } - { int arr[T(__has_nothrow_assign(UsingAssign))]; } - { int arr[T(__has_nothrow_assign(DerivesAr))]; } - { int arr[T(__has_nothrow_assign(ACompleteType[]))]; } - - { int arr[F(__has_nothrow_assign(AnIncompleteType[]))]; } // expected-error {{incomplete type}} - { int arr[F(__has_nothrow_assign(IntRef))]; } - { int arr[F(__has_nothrow_assign(HasCopyAssign))]; } - { int arr[F(__has_nothrow_assign(HasMultipleCopyAssign))]; } - { int arr[F(__has_nothrow_assign(const Int))]; } - { int arr[F(__has_nothrow_assign(ConstIntAr))]; } - { int arr[F(__has_nothrow_assign(ConstIntArAr))]; } - { int arr[F(__has_nothrow_assign(VirtAr))]; } - { int arr[F(__has_nothrow_assign(void))]; } - { int arr[F(__has_nothrow_assign(cvoid))]; } - { int arr[F(__has_nothrow_assign(PR11110))]; } + static_assert(__has_nothrow_assign(Int)); + static_assert(__has_nothrow_assign(IntAr)); + static_assert(__has_nothrow_assign(Union)); + static_assert(__has_nothrow_assign(UnionAr)); + static_assert(__has_nothrow_assign(POD)); + static_assert(__has_nothrow_assign(Derives)); + static_assert(__has_nothrow_assign(HasDest)); + static_assert(__has_nothrow_assign(HasPriv)); + static_assert(__has_nothrow_assign(HasCons)); + static_assert(__has_nothrow_assign(HasRef)); + static_assert(__has_nothrow_assign(HasCopy)); + static_assert(__has_nothrow_assign(HasMove)); + static_assert(__has_nothrow_assign(HasMoveAssign)); + static_assert(__has_nothrow_assign(HasNoThrowCopyAssign)); + static_assert(__has_nothrow_assign(HasMultipleNoThrowCopyAssign)); + static_assert(__has_nothrow_assign(HasVirtDest)); + static_assert(__has_nothrow_assign(AllPrivate)); + static_assert(__has_nothrow_assign(UsingAssign)); + static_assert(__has_nothrow_assign(DerivesAr)); + static_assert(__has_nothrow_assign(ACompleteType[])); + + static_assert(!__has_nothrow_assign(AnIncompleteType[])); // expected-error {{incomplete type}} + static_assert(!__has_nothrow_assign(IntRef)); + static_assert(!__has_nothrow_assign(HasCopyAssign)); + static_assert(!__has_nothrow_assign(HasMultipleCopyAssign)); + static_assert(!__has_nothrow_assign(const Int)); + static_assert(!__has_nothrow_assign(ConstIntAr)); + static_assert(!__has_nothrow_assign(ConstIntArAr)); + static_assert(!__has_nothrow_assign(VirtAr)); + static_assert(!__has_nothrow_assign(void)); + static_assert(!__has_nothrow_assign(cvoid)); + static_assert(!__has_nothrow_assign(PR11110)); } void has_nothrow_move_assign() { - { int arr[T(__has_nothrow_move_assign(Int))]; } - { int arr[T(__has_nothrow_move_assign(Enum))]; } - { int arr[T(__has_nothrow_move_assign(Int*))]; } - { int arr[T(__has_nothrow_move_assign(Enum POD::*))]; } - { int arr[T(__has_nothrow_move_assign(POD))]; } - { int arr[T(__has_nothrow_move_assign(HasPriv))]; } - { int arr[T(__has_nothrow_move_assign(HasNoThrowMoveAssign))]; } - { int arr[T(__has_nothrow_move_assign(HasNoExceptNoThrowMoveAssign))]; } - { int arr[T(__has_nothrow_move_assign(HasMemberNoThrowMoveAssign))]; } - { int arr[T(__has_nothrow_move_assign(HasMemberNoExceptNoThrowMoveAssign))]; } - { int arr[T(__has_nothrow_move_assign(AllDeleted))]; } - { int arr[T(__has_nothrow_move_assign(ACompleteType[]))]; } - - { int arr[F(__has_nothrow_move_assign(AnIncompleteType[]))]; } // expected-error {{incomplete type}} - { int arr[F(__has_nothrow_move_assign(HasThrowMoveAssign))]; } - { int arr[F(__has_nothrow_move_assign(HasNoExceptFalseMoveAssign))]; } - { int arr[F(__has_nothrow_move_assign(HasMemberThrowMoveAssign))]; } - { int arr[F(__has_nothrow_move_assign(HasMemberNoExceptFalseMoveAssign))]; } - { int arr[F(__has_nothrow_move_assign(NoDefaultMoveAssignDueToUDCopyCtor))]; } - { int arr[F(__has_nothrow_move_assign(NoDefaultMoveAssignDueToUDCopyAssign))]; } - { int arr[F(__has_nothrow_move_assign(NoDefaultMoveAssignDueToDtor))]; } - - - { int arr[T(__is_nothrow_assignable(HasNoThrowMoveAssign, HasNoThrowMoveAssign))]; } - { int arr[F(__is_nothrow_assignable(HasThrowMoveAssign, HasThrowMoveAssign))]; } - - { int arr[T(__is_assignable(HasNoThrowMoveAssign, HasNoThrowMoveAssign))]; } - { int arr[T(__is_assignable(HasThrowMoveAssign, HasThrowMoveAssign))]; } + static_assert(__has_nothrow_move_assign(Int)); + static_assert(__has_nothrow_move_assign(Enum)); + static_assert(__has_nothrow_move_assign(Int*)); + static_assert(__has_nothrow_move_assign(Enum POD::*)); + static_assert(__has_nothrow_move_assign(POD)); + static_assert(__has_nothrow_move_assign(HasPriv)); + static_assert(__has_nothrow_move_assign(HasNoThrowMoveAssign)); + static_assert(__has_nothrow_move_assign(HasNoExceptNoThrowMoveAssign)); + static_assert(__has_nothrow_move_assign(HasMemberNoThrowMoveAssign)); + static_assert(__has_nothrow_move_assign(HasMemberNoExceptNoThrowMoveAssign)); + static_assert(__has_nothrow_move_assign(AllDeleted)); + static_assert(__has_nothrow_move_assign(ACompleteType[])); + + static_assert(!__has_nothrow_move_assign(AnIncompleteType[])); // expected-error {{incomplete type}} + static_assert(!__has_nothrow_move_assign(HasThrowMoveAssign)); + static_assert(!__has_nothrow_move_assign(HasNoExceptFalseMoveAssign)); + static_assert(!__has_nothrow_move_assign(HasMemberThrowMoveAssign)); + static_assert(!__has_nothrow_move_assign(HasMemberNoExceptFalseMoveAssign)); + static_assert(!__has_nothrow_move_assign(NoDefaultMoveAssignDueToUDCopyCtor)); + static_assert(!__has_nothrow_move_assign(NoDefaultMoveAssignDueToUDCopyAssign)); + static_assert(!__has_nothrow_move_assign(NoDefaultMoveAssignDueToDtor)); + + + static_assert(__is_nothrow_assignable(HasNoThrowMoveAssign, HasNoThrowMoveAssign)); + static_assert(!__is_nothrow_assignable(HasThrowMoveAssign, HasThrowMoveAssign)); + + static_assert(__is_assignable(HasNoThrowMoveAssign, HasNoThrowMoveAssign)); + static_assert(__is_assignable(HasThrowMoveAssign, HasThrowMoveAssign)); } void has_trivial_move_assign() { @@ -2153,120 +2151,120 @@ void has_trivial_move_assign() { // - for each non-static data member of X that is of class type // (or array thereof), the assignment operator // selected to copy/move that member is trivial; - { int arr[T(__has_trivial_move_assign(Int))]; } - { int arr[T(__has_trivial_move_assign(HasStaticMemberMoveAssign))]; } - { int arr[T(__has_trivial_move_assign(AllDeleted))]; } - { int arr[T(__has_trivial_move_assign(ACompleteType[]))]; } - - { int arr[F(__has_trivial_move_assign(AnIncompleteType[]))]; } // expected-error {{incomplete type}} - { int arr[F(__has_trivial_move_assign(HasVirt))]; } - { int arr[F(__has_trivial_move_assign(DerivesVirt))]; } - { int arr[F(__has_trivial_move_assign(HasMoveAssign))]; } - { int arr[F(__has_trivial_move_assign(DerivesHasMoveAssign))]; } - { int arr[F(__has_trivial_move_assign(HasMemberMoveAssign))]; } - { int arr[F(__has_nothrow_move_assign(NoDefaultMoveAssignDueToUDCopyCtor))]; } - { int arr[F(__has_nothrow_move_assign(NoDefaultMoveAssignDueToUDCopyAssign))]; } + static_assert(__has_trivial_move_assign(Int)); + static_assert(__has_trivial_move_assign(HasStaticMemberMoveAssign)); + static_assert(__has_trivial_move_assign(AllDeleted)); + static_assert(__has_trivial_move_assign(ACompleteType[])); + + static_assert(!__has_trivial_move_assign(AnIncompleteType[])); // expected-error {{incomplete type}} + static_assert(!__has_trivial_move_assign(HasVirt)); + static_assert(!__has_trivial_move_assign(DerivesVirt)); + static_assert(!__has_trivial_move_assign(HasMoveAssign)); + static_assert(!__has_trivial_move_assign(DerivesHasMoveAssign)); + static_assert(!__has_trivial_move_assign(HasMemberMoveAssign)); + static_assert(!__has_nothrow_move_assign(NoDefaultMoveAssignDueToUDCopyCtor)); + static_assert(!__has_nothrow_move_assign(NoDefaultMoveAssignDueToUDCopyAssign)); } void has_nothrow_copy() { - { int arr[T(__has_nothrow_copy(Int))]; } - { int arr[T(__has_nothrow_copy(IntAr))]; } - { int arr[T(__has_nothrow_copy(Union))]; } - { int arr[T(__has_nothrow_copy(UnionAr))]; } - { int arr[T(__has_nothrow_copy(POD))]; } - { int arr[T(__has_nothrow_copy(const Int))]; } - { int arr[T(__has_nothrow_copy(ConstIntAr))]; } - { int arr[T(__has_nothrow_copy(ConstIntArAr))]; } - { int arr[T(__has_nothrow_copy(Derives))]; } - { int arr[T(__has_nothrow_copy(IntRef))]; } - { int arr[T(__has_nothrow_copy(HasDest))]; } - { int arr[T(__has_nothrow_copy(HasPriv))]; } - { int arr[T(__has_nothrow_copy(HasCons))]; } - { int arr[T(__has_nothrow_copy(HasRef))]; } - { int arr[T(__has_nothrow_copy(HasMove))]; } - { int arr[T(__has_nothrow_copy(HasCopyAssign))]; } - { int arr[T(__has_nothrow_copy(HasMoveAssign))]; } - { int arr[T(__has_nothrow_copy(HasNoThrowCopy))]; } - { int arr[T(__has_nothrow_copy(HasMultipleNoThrowCopy))]; } - { int arr[T(__has_nothrow_copy(HasVirtDest))]; } - { int arr[T(__has_nothrow_copy(HasTemplateCons))]; } - { int arr[T(__has_nothrow_copy(AllPrivate))]; } - { int arr[T(__has_nothrow_copy(DerivesAr))]; } - { int arr[T(__has_nothrow_copy(ACompleteType[]))]; } - - { int arr[F(__has_nothrow_copy(AnIncompleteType[]))]; } // expected-error {{incomplete type}} - { int arr[F(__has_nothrow_copy(HasCopy))]; } - { int arr[F(__has_nothrow_copy(HasMultipleCopy))]; } - { int arr[F(__has_nothrow_copy(VirtAr))]; } - { int arr[F(__has_nothrow_copy(void))]; } - { int arr[F(__has_nothrow_copy(cvoid))]; } + static_assert(__has_nothrow_copy(Int)); + static_assert(__has_nothrow_copy(IntAr)); + static_assert(__has_nothrow_copy(Union)); + static_assert(__has_nothrow_copy(UnionAr)); + static_assert(__has_nothrow_copy(POD)); + static_assert(__has_nothrow_copy(const Int)); + static_assert(__has_nothrow_copy(ConstIntAr)); + static_assert(__has_nothrow_copy(ConstIntArAr)); + static_assert(__has_nothrow_copy(Derives)); + static_assert(__has_nothrow_copy(IntRef)); + static_assert(__has_nothrow_copy(HasDest)); + static_assert(__has_nothrow_copy(HasPriv)); + static_assert(__has_nothrow_copy(HasCons)); + static_assert(__has_nothrow_copy(HasRef)); + static_assert(__has_nothrow_copy(HasMove)); + static_assert(__has_nothrow_copy(HasCopyAssign)); + static_assert(__has_nothrow_copy(HasMoveAssign)); + static_assert(__has_nothrow_copy(HasNoThrowCopy)); + static_assert(__has_nothrow_copy(HasMultipleNoThrowCopy)); + static_assert(__has_nothrow_copy(HasVirtDest)); + static_assert(__has_nothrow_copy(HasTemplateCons)); + static_assert(__has_nothrow_copy(AllPrivate)); + static_assert(__has_nothrow_copy(DerivesAr)); + static_assert(__has_nothrow_copy(ACompleteType[])); + + static_assert(!__has_nothrow_copy(AnIncompleteType[])); // expected-error {{incomplete type}} + static_assert(!__has_nothrow_copy(HasCopy)); + static_assert(!__has_nothrow_copy(HasMultipleCopy)); + static_assert(!__has_nothrow_copy(VirtAr)); + static_assert(!__has_nothrow_copy(void)); + static_assert(!__has_nothrow_copy(cvoid)); } void has_nothrow_constructor() { - { int arr[T(__has_nothrow_constructor(Int))]; } - { int arr[T(__has_nothrow_constructor(IntAr))]; } - { int arr[T(__has_nothrow_constructor(Union))]; } - { int arr[T(__has_nothrow_constructor(UnionAr))]; } - { int arr[T(__has_nothrow_constructor(POD))]; } - { int arr[T(__has_nothrow_constructor(Derives))]; } - { int arr[T(__has_nothrow_constructor(DerivesAr))]; } - { int arr[T(__has_nothrow_constructor(ConstIntAr))]; } - { int arr[T(__has_nothrow_constructor(ConstIntArAr))]; } - { int arr[T(__has_nothrow_constructor(HasDest))]; } - { int arr[T(__has_nothrow_constructor(HasPriv))]; } - { int arr[T(__has_nothrow_constructor(HasCopyAssign))]; } - { int arr[T(__has_nothrow_constructor(const Int))]; } - { int arr[T(__has_nothrow_constructor(HasNoThrowConstructor))]; } - { int arr[T(__has_nothrow_constructor(HasVirtDest))]; } - // { int arr[T(__has_nothrow_constructor(VirtAr))]; } // not implemented - { int arr[T(__has_nothrow_constructor(AllPrivate))]; } - { int arr[T(__has_nothrow_constructor(ACompleteType[]))]; } - - { int arr[F(__has_nothrow_constructor(AnIncompleteType[]))]; } // expected-error {{incomplete type}} - { int arr[F(__has_nothrow_constructor(HasCons))]; } - { int arr[F(__has_nothrow_constructor(HasRef))]; } - { int arr[F(__has_nothrow_constructor(HasCopy))]; } - { int arr[F(__has_nothrow_constructor(HasMove))]; } - { int arr[F(__has_nothrow_constructor(HasNoThrowConstructorWithArgs))]; } - { int arr[F(__has_nothrow_constructor(IntRef))]; } - { int arr[F(__has_nothrow_constructor(void))]; } - { int arr[F(__has_nothrow_constructor(cvoid))]; } - { int arr[F(__has_nothrow_constructor(HasTemplateCons))]; } - - { int arr[F(__has_nothrow_constructor(HasMultipleDefaultConstructor1))]; } - { int arr[F(__has_nothrow_constructor(HasMultipleDefaultConstructor2))]; } + static_assert(__has_nothrow_constructor(Int)); + static_assert(__has_nothrow_constructor(IntAr)); + static_assert(__has_nothrow_constructor(Union)); + static_assert(__has_nothrow_constructor(UnionAr)); + static_assert(__has_nothrow_constructor(POD)); + static_assert(__has_nothrow_constructor(Derives)); + static_assert(__has_nothrow_constructor(DerivesAr)); + static_assert(__has_nothrow_constructor(ConstIntAr)); + static_assert(__has_nothrow_constructor(ConstIntArAr)); + static_assert(__has_nothrow_constructor(HasDest)); + static_assert(__has_nothrow_constructor(HasPriv)); + static_assert(__has_nothrow_constructor(HasCopyAssign)); + static_assert(__has_nothrow_constructor(const Int)); + static_assert(__has_nothrow_constructor(HasNoThrowConstructor)); + static_assert(__has_nothrow_constructor(HasVirtDest)); + // static_assert(__has_nothrow_constructor(VirtAr)); // not implemented + static_assert(__has_nothrow_constructor(AllPrivate)); + static_assert(__has_nothrow_constructor(ACompleteType[])); + + static_assert(!__has_nothrow_constructor(AnIncompleteType[])); // expected-error {{incomplete type}} + static_assert(!__has_nothrow_constructor(HasCons)); + static_assert(!__has_nothrow_constructor(HasRef)); + static_assert(!__has_nothrow_constructor(HasCopy)); + static_assert(!__has_nothrow_constructor(HasMove)); + static_assert(!__has_nothrow_constructor(HasNoThrowConstructorWithArgs)); + static_assert(!__has_nothrow_constructor(IntRef)); + static_assert(!__has_nothrow_constructor(void)); + static_assert(!__has_nothrow_constructor(cvoid)); + static_assert(!__has_nothrow_constructor(HasTemplateCons)); + + static_assert(!__has_nothrow_constructor(HasMultipleDefaultConstructor1)); + static_assert(!__has_nothrow_constructor(HasMultipleDefaultConstructor2)); } void has_virtual_destructor() { - { int arr[F(__has_virtual_destructor(Int))]; } - { int arr[F(__has_virtual_destructor(IntAr))]; } - { int arr[F(__has_virtual_destructor(Union))]; } - { int arr[F(__has_virtual_destructor(UnionAr))]; } - { int arr[F(__has_virtual_destructor(POD))]; } - { int arr[F(__has_virtual_destructor(Derives))]; } - { int arr[F(__has_virtual_destructor(DerivesAr))]; } - { int arr[F(__has_virtual_destructor(const Int))]; } - { int arr[F(__has_virtual_destructor(ConstIntAr))]; } - { int arr[F(__has_virtual_destructor(ConstIntArAr))]; } - { int arr[F(__has_virtual_destructor(HasDest))]; } - { int arr[F(__has_virtual_destructor(HasPriv))]; } - { int arr[F(__has_virtual_destructor(HasCons))]; } - { int arr[F(__has_virtual_destructor(HasRef))]; } - { int arr[F(__has_virtual_destructor(HasCopy))]; } - { int arr[F(__has_virtual_destructor(HasMove))]; } - { int arr[F(__has_virtual_destructor(HasCopyAssign))]; } - { int arr[F(__has_virtual_destructor(HasMoveAssign))]; } - { int arr[F(__has_virtual_destructor(IntRef))]; } - { int arr[F(__has_virtual_destructor(VirtAr))]; } - { int arr[F(__has_virtual_destructor(ACompleteType[]))]; } - - { int arr[F(__has_virtual_destructor(AnIncompleteType[]))]; } // expected-error {{incomplete type}} - { int arr[T(__has_virtual_destructor(HasVirtDest))]; } - { int arr[T(__has_virtual_destructor(DerivedVirtDest))]; } - { int arr[F(__has_virtual_destructor(VirtDestAr))]; } - { int arr[F(__has_virtual_destructor(void))]; } - { int arr[F(__has_virtual_destructor(cvoid))]; } - { int arr[F(__has_virtual_destructor(AllPrivate))]; } + static_assert(!__has_virtual_destructor(Int)); + static_assert(!__has_virtual_destructor(IntAr)); + static_assert(!__has_virtual_destructor(Union)); + static_assert(!__has_virtual_destructor(UnionAr)); + static_assert(!__has_virtual_destructor(POD)); + static_assert(!__has_virtual_destructor(Derives)); + static_assert(!__has_virtual_destructor(DerivesAr)); + static_assert(!__has_virtual_destructor(const Int)); + static_assert(!__has_virtual_destructor(ConstIntAr)); + static_assert(!__has_virtual_destructor(ConstIntArAr)); + static_assert(!__has_virtual_destructor(HasDest)); + static_assert(!__has_virtual_destructor(HasPriv)); + static_assert(!__has_virtual_destructor(HasCons)); + static_assert(!__has_virtual_destructor(HasRef)); + static_assert(!__has_virtual_destructor(HasCopy)); + static_assert(!__has_virtual_destructor(HasMove)); + static_assert(!__has_virtual_destructor(HasCopyAssign)); + static_assert(!__has_virtual_destructor(HasMoveAssign)); + static_assert(!__has_virtual_destructor(IntRef)); + static_assert(!__has_virtual_destructor(VirtAr)); + static_assert(!__has_virtual_destructor(ACompleteType[])); + + static_assert(!__has_virtual_destructor(AnIncompleteType[])); // expected-error {{incomplete type}} + static_assert(__has_virtual_destructor(HasVirtDest)); + static_assert(__has_virtual_destructor(DerivedVirtDest)); + static_assert(!__has_virtual_destructor(VirtDestAr)); + static_assert(!__has_virtual_destructor(void)); + static_assert(!__has_virtual_destructor(cvoid)); + static_assert(!__has_virtual_destructor(AllPrivate)); } @@ -2282,66 +2280,57 @@ template struct CrazyDerived : T { }; class class_forward; // expected-note 2 {{forward declaration of 'class_forward'}} -template -void isBaseOfT() { - int t[T(__is_base_of(Base, Derived))]; -}; -template -void isBaseOfF() { - int t[F(__is_base_of(Base, Derived))]; -}; - template class DerivedTemp : Base {}; template class NonderivedTemp {}; template class UndefinedTemp; // expected-note {{declared here}} void is_base_of() { - { int arr[T(__is_base_of(Base, Derived))]; } - { int arr[T(__is_base_of(const Base, Derived))]; } - { int arr[F(__is_base_of(Derived, Base))]; } - { int arr[F(__is_base_of(Derived, int))]; } - { int arr[T(__is_base_of(Base, Base))]; } - { int arr[T(__is_base_of(Base, Derived3))]; } - { int arr[T(__is_base_of(Derived, Derived3))]; } - { int arr[T(__is_base_of(Derived2b, Derived3))]; } - { int arr[T(__is_base_of(Derived2a, Derived3))]; } - { int arr[T(__is_base_of(BaseA, DerivedB))]; } - { int arr[F(__is_base_of(DerivedB, BaseA))]; } - { int arr[T(__is_base_of(Base, CrazyDerived))]; } - { int arr[F(__is_base_of(Union, Union))]; } - { int arr[T(__is_base_of(Empty, Empty))]; } - { int arr[T(__is_base_of(class_forward, class_forward))]; } - { int arr[F(__is_base_of(Empty, class_forward))]; } // expected-error {{incomplete type 'class_forward' used in type trait expression}} - { int arr[F(__is_base_of(Base&, Derived&))]; } - int t18[F(__is_base_of(Base[10], Derived[10]))]; - { int arr[F(__is_base_of(int, int))]; } - { int arr[F(__is_base_of(long, int))]; } - { int arr[T(__is_base_of(Base, DerivedTemp))]; } - { int arr[F(__is_base_of(Base, NonderivedTemp))]; } - { int arr[F(__is_base_of(Base, UndefinedTemp))]; } // expected-error {{implicit instantiation of undefined template 'UndefinedTemp'}} - - { int arr[F(__is_base_of(IncompleteUnion, IncompleteUnion))]; } - { int arr[F(__is_base_of(Union, IncompleteUnion))]; } - { int arr[F(__is_base_of(IncompleteUnion, Union))]; } - { int arr[F(__is_base_of(IncompleteStruct, IncompleteUnion))]; } - { int arr[F(__is_base_of(IncompleteUnion, IncompleteStruct))]; } - { int arr[F(__is_base_of(Empty, IncompleteUnion))]; } - { int arr[F(__is_base_of(IncompleteUnion, Empty))]; } - { int arr[F(__is_base_of(int, IncompleteUnion))]; } - { int arr[F(__is_base_of(IncompleteUnion, int))]; } - { int arr[F(__is_base_of(Empty, Union))]; } - { int arr[F(__is_base_of(Union, Empty))]; } - { int arr[F(__is_base_of(int, Empty))]; } - { int arr[F(__is_base_of(Union, int))]; } - - isBaseOfT(); - isBaseOfF(); - - isBaseOfT >(); - isBaseOfF, Base>(); - - isBaseOfT, DerivedB >(); - isBaseOfF, BaseA >(); + static_assert(__is_base_of(Base, Derived)); + static_assert(__is_base_of(const Base, Derived)); + static_assert(!__is_base_of(Derived, Base)); + static_assert(!__is_base_of(Derived, int)); + static_assert(__is_base_of(Base, Base)); + static_assert(__is_base_of(Base, Derived3)); + static_assert(__is_base_of(Derived, Derived3)); + static_assert(__is_base_of(Derived2b, Derived3)); + static_assert(__is_base_of(Derived2a, Derived3)); + static_assert(__is_base_of(BaseA, DerivedB)); + static_assert(!__is_base_of(DerivedB, BaseA)); + static_assert(__is_base_of(Base, CrazyDerived)); + static_assert(!__is_base_of(Union, Union)); + static_assert(__is_base_of(Empty, Empty)); + static_assert(__is_base_of(class_forward, class_forward)); + static_assert(!__is_base_of(Empty, class_forward)); // expected-error {{incomplete type 'class_forward' used in type trait expression}} + static_assert(!__is_base_of(Base&, Derived&)); + static_assert(!__is_base_of(Base[10], Derived[10])); + static_assert(!__is_base_of(int, int)); + static_assert(!__is_base_of(long, int)); + static_assert(__is_base_of(Base, DerivedTemp)); + static_assert(!__is_base_of(Base, NonderivedTemp)); + static_assert(!__is_base_of(Base, UndefinedTemp)); // expected-error {{implicit instantiation of undefined template 'UndefinedTemp'}} + + static_assert(!__is_base_of(IncompleteUnion, IncompleteUnion)); + static_assert(!__is_base_of(Union, IncompleteUnion)); + static_assert(!__is_base_of(IncompleteUnion, Union)); + static_assert(!__is_base_of(IncompleteStruct, IncompleteUnion)); + static_assert(!__is_base_of(IncompleteUnion, IncompleteStruct)); + static_assert(!__is_base_of(Empty, IncompleteUnion)); + static_assert(!__is_base_of(IncompleteUnion, Empty)); + static_assert(!__is_base_of(int, IncompleteUnion)); + static_assert(!__is_base_of(IncompleteUnion, int)); + static_assert(!__is_base_of(Empty, Union)); + static_assert(!__is_base_of(Union, Empty)); + static_assert(!__is_base_of(int, Empty)); + static_assert(!__is_base_of(Union, int)); + + static_assert(__is_base_of(Base, Derived)); + static_assert(!__is_base_of(Derived, Base)); + + static_assert(__is_base_of(Base, CrazyDerived)); + static_assert(!__is_base_of(CrazyDerived, Base)); + + static_assert(__is_base_of(BaseA, DerivedB)); + static_assert(!__is_base_of(DerivedB, BaseA)); } template @@ -2354,17 +2343,17 @@ typedef class Base BaseTypedef; void is_same() { - int t01[T(__is_same(Base, Base))]; - int t02[T(__is_same(Base, BaseTypedef))]; - int t03[T(__is_same(TemplateClass, TemplateAlias))]; + static_assert(__is_same(Base, Base)); + static_assert(__is_same(Base, BaseTypedef)); + static_assert(__is_same(TemplateClass, TemplateAlias)); - int t10[F(__is_same(Base, const Base))]; - int t11[F(__is_same(Base, Base&))]; - int t12[F(__is_same(Base, Derived))]; + static_assert(!__is_same(Base, const Base)); + static_assert(!__is_same(Base, Base&)); + static_assert(!__is_same(Base, Derived)); // __is_same_as is a GCC compatibility synonym for __is_same. - int t20[T(__is_same_as(int, int))]; - int t21[F(__is_same_as(int, float))]; + static_assert(__is_same_as(int, int)); + static_assert(!__is_same_as(int, float)); } struct IntWrapper @@ -2392,26 +2381,26 @@ struct FloatWrapper void is_convertible() { - int t01[T(__is_convertible(IntWrapper, IntWrapper))]; - int t02[T(__is_convertible(IntWrapper, const IntWrapper))]; - int t03[T(__is_convertible(IntWrapper, int))]; - int t04[T(__is_convertible(int, IntWrapper))]; - int t05[T(__is_convertible(IntWrapper, FloatWrapper))]; - int t06[T(__is_convertible(FloatWrapper, IntWrapper))]; - int t07[T(__is_convertible(FloatWrapper, float))]; - int t08[T(__is_convertible(float, FloatWrapper))]; + static_assert(__is_convertible(IntWrapper, IntWrapper)); + static_assert(__is_convertible(IntWrapper, const IntWrapper)); + static_assert(__is_convertible(IntWrapper, int)); + static_assert(__is_convertible(int, IntWrapper)); + static_assert(__is_convertible(IntWrapper, FloatWrapper)); + static_assert(__is_convertible(FloatWrapper, IntWrapper)); + static_assert(__is_convertible(FloatWrapper, float)); + static_assert(__is_convertible(float, FloatWrapper)); } void is_nothrow_convertible() { - int t01[T(__is_nothrow_convertible(IntWrapper, IntWrapper))]; - int t02[T(__is_nothrow_convertible(IntWrapper, const IntWrapper))]; - int t03[T(__is_nothrow_convertible(IntWrapper, int))]; - int t04[F(__is_nothrow_convertible(int, IntWrapper))]; - int t05[F(__is_nothrow_convertible(IntWrapper, FloatWrapper))]; - int t06[F(__is_nothrow_convertible(FloatWrapper, IntWrapper))]; - int t07[F(__is_nothrow_convertible(FloatWrapper, float))]; - int t08[T(__is_nothrow_convertible(float, FloatWrapper))]; + static_assert(__is_nothrow_convertible(IntWrapper, IntWrapper)); + static_assert(__is_nothrow_convertible(IntWrapper, const IntWrapper)); + static_assert(__is_nothrow_convertible(IntWrapper, int)); + static_assert(!__is_nothrow_convertible(int, IntWrapper)); + static_assert(!__is_nothrow_convertible(IntWrapper, FloatWrapper)); + static_assert(!__is_nothrow_convertible(FloatWrapper, IntWrapper)); + static_assert(!__is_nothrow_convertible(FloatWrapper, float)); + static_assert(__is_nothrow_convertible(float, FloatWrapper)); } struct FromInt { FromInt(int); }; @@ -2432,30 +2421,30 @@ struct X0 { struct Abstract { virtual void f() = 0; }; void is_convertible_to() { - { int arr[T(__is_convertible_to(Int, Int))]; } - { int arr[F(__is_convertible_to(Int, IntAr))]; } - { int arr[F(__is_convertible_to(IntAr, IntAr))]; } - { int arr[T(__is_convertible_to(void, void))]; } - { int arr[T(__is_convertible_to(cvoid, void))]; } - { int arr[T(__is_convertible_to(void, cvoid))]; } - { int arr[T(__is_convertible_to(cvoid, cvoid))]; } - { int arr[T(__is_convertible_to(int, FromInt))]; } - { int arr[T(__is_convertible_to(long, FromInt))]; } - { int arr[T(__is_convertible_to(double, FromInt))]; } - { int arr[T(__is_convertible_to(const int, FromInt))]; } - { int arr[T(__is_convertible_to(const int&, FromInt))]; } - { int arr[T(__is_convertible_to(ToInt, int))]; } - { int arr[T(__is_convertible_to(ToInt, const int&))]; } - { int arr[T(__is_convertible_to(ToInt, long))]; } - { int arr[F(__is_convertible_to(ToInt, int&))]; } - { int arr[F(__is_convertible_to(ToInt, FromInt))]; } - { int arr[T(__is_convertible_to(IntAr&, IntAr&))]; } - { int arr[T(__is_convertible_to(IntAr&, const IntAr&))]; } - { int arr[F(__is_convertible_to(const IntAr&, IntAr&))]; } - { int arr[F(__is_convertible_to(Function, Function))]; } - { int arr[F(__is_convertible_to(PrivateCopy, PrivateCopy))]; } - { int arr[T(__is_convertible_to(X0, X0))]; } - { int arr[F(__is_convertible_to(Abstract, Abstract))]; } + static_assert(__is_convertible_to(Int, Int)); + static_assert(!__is_convertible_to(Int, IntAr)); + static_assert(!__is_convertible_to(IntAr, IntAr)); + static_assert(__is_convertible_to(void, void)); + static_assert(__is_convertible_to(cvoid, void)); + static_assert(__is_convertible_to(void, cvoid)); + static_assert(__is_convertible_to(cvoid, cvoid)); + static_assert(__is_convertible_to(int, FromInt)); + static_assert(__is_convertible_to(long, FromInt)); + static_assert(__is_convertible_to(double, FromInt)); + static_assert(__is_convertible_to(const int, FromInt)); + static_assert(__is_convertible_to(const int&, FromInt)); + static_assert(__is_convertible_to(ToInt, int)); + static_assert(__is_convertible_to(ToInt, const int&)); + static_assert(__is_convertible_to(ToInt, long)); + static_assert(!__is_convertible_to(ToInt, int&)); + static_assert(!__is_convertible_to(ToInt, FromInt)); + static_assert(__is_convertible_to(IntAr&, IntAr&)); + static_assert(__is_convertible_to(IntAr&, const IntAr&)); + static_assert(!__is_convertible_to(const IntAr&, IntAr&)); + static_assert(!__is_convertible_to(Function, Function)); + static_assert(!__is_convertible_to(PrivateCopy, PrivateCopy)); + static_assert(__is_convertible_to(X0, X0)); + static_assert(!__is_convertible_to(Abstract, Abstract)); } namespace is_convertible_to_instantiate { @@ -2466,269 +2455,269 @@ namespace is_convertible_to_instantiate { void is_trivial() { - { int arr[T(__is_trivial(int))]; } - { int arr[T(__is_trivial(Enum))]; } - { int arr[T(__is_trivial(POD))]; } - { int arr[T(__is_trivial(Int))]; } - { int arr[T(__is_trivial(IntAr))]; } - { int arr[T(__is_trivial(IntArNB))]; } - { int arr[T(__is_trivial(Statics))]; } - { int arr[T(__is_trivial(Empty))]; } - { int arr[T(__is_trivial(EmptyUnion))]; } - { int arr[T(__is_trivial(Union))]; } - { int arr[T(__is_trivial(Derives))]; } - { int arr[T(__is_trivial(DerivesAr))]; } - { int arr[T(__is_trivial(DerivesArNB))]; } - { int arr[T(__is_trivial(DerivesEmpty))]; } - { int arr[T(__is_trivial(HasFunc))]; } - { int arr[T(__is_trivial(HasOp))]; } - { int arr[T(__is_trivial(HasConv))]; } - { int arr[T(__is_trivial(HasAssign))]; } - { int arr[T(__is_trivial(HasAnonymousUnion))]; } - { int arr[T(__is_trivial(HasPriv))]; } - { int arr[T(__is_trivial(HasProt))]; } - { int arr[T(__is_trivial(DerivesHasPriv))]; } - { int arr[T(__is_trivial(DerivesHasProt))]; } - { int arr[T(__is_trivial(Vector))]; } - { int arr[T(__is_trivial(VectorExt))]; } - - { int arr[F(__is_trivial(HasCons))]; } - { int arr[F(__is_trivial(HasCopyAssign))]; } - { int arr[F(__is_trivial(HasMoveAssign))]; } - { int arr[F(__is_trivial(HasDest))]; } - { int arr[F(__is_trivial(HasRef))]; } - { int arr[F(__is_trivial(HasNonPOD))]; } - { int arr[F(__is_trivial(HasVirt))]; } - { int arr[F(__is_trivial(DerivesHasCons))]; } - { int arr[F(__is_trivial(DerivesHasCopyAssign))]; } - { int arr[F(__is_trivial(DerivesHasMoveAssign))]; } - { int arr[F(__is_trivial(DerivesHasDest))]; } - { int arr[F(__is_trivial(DerivesHasRef))]; } - { int arr[F(__is_trivial(DerivesHasVirt))]; } - { int arr[F(__is_trivial(void))]; } - { int arr[F(__is_trivial(cvoid))]; } + static_assert(__is_trivial(int)); + static_assert(__is_trivial(Enum)); + static_assert(__is_trivial(POD)); + static_assert(__is_trivial(Int)); + static_assert(__is_trivial(IntAr)); + static_assert(__is_trivial(IntArNB)); + static_assert(__is_trivial(Statics)); + static_assert(__is_trivial(Empty)); + static_assert(__is_trivial(EmptyUnion)); + static_assert(__is_trivial(Union)); + static_assert(__is_trivial(Derives)); + static_assert(__is_trivial(DerivesAr)); + static_assert(__is_trivial(DerivesArNB)); + static_assert(__is_trivial(DerivesEmpty)); + static_assert(__is_trivial(HasFunc)); + static_assert(__is_trivial(HasOp)); + static_assert(__is_trivial(HasConv)); + static_assert(__is_trivial(HasAssign)); + static_assert(__is_trivial(HasAnonymousUnion)); + static_assert(__is_trivial(HasPriv)); + static_assert(__is_trivial(HasProt)); + static_assert(__is_trivial(DerivesHasPriv)); + static_assert(__is_trivial(DerivesHasProt)); + static_assert(__is_trivial(Vector)); + static_assert(__is_trivial(VectorExt)); + + static_assert(!__is_trivial(HasCons)); + static_assert(!__is_trivial(HasCopyAssign)); + static_assert(!__is_trivial(HasMoveAssign)); + static_assert(!__is_trivial(HasDest)); + static_assert(!__is_trivial(HasRef)); + static_assert(!__is_trivial(HasNonPOD)); + static_assert(!__is_trivial(HasVirt)); + static_assert(!__is_trivial(DerivesHasCons)); + static_assert(!__is_trivial(DerivesHasCopyAssign)); + static_assert(!__is_trivial(DerivesHasMoveAssign)); + static_assert(!__is_trivial(DerivesHasDest)); + static_assert(!__is_trivial(DerivesHasRef)); + static_assert(!__is_trivial(DerivesHasVirt)); + static_assert(!__is_trivial(void)); + static_assert(!__is_trivial(cvoid)); } template struct TriviallyConstructibleTemplate {}; void trivial_checks() { - { int arr[T(__is_trivially_copyable(int))]; } - { int arr[T(__is_trivially_copyable(Enum))]; } - { int arr[T(__is_trivially_copyable(POD))]; } - { int arr[T(__is_trivially_copyable(Int))]; } - { int arr[T(__is_trivially_copyable(IntAr))]; } - { int arr[T(__is_trivially_copyable(IntArNB))]; } - { int arr[T(__is_trivially_copyable(Statics))]; } - { int arr[T(__is_trivially_copyable(Empty))]; } - { int arr[T(__is_trivially_copyable(EmptyUnion))]; } - { int arr[T(__is_trivially_copyable(Union))]; } - { int arr[T(__is_trivially_copyable(Derives))]; } - { int arr[T(__is_trivially_copyable(DerivesAr))]; } - { int arr[T(__is_trivially_copyable(DerivesArNB))]; } - { int arr[T(__is_trivially_copyable(DerivesEmpty))]; } - { int arr[T(__is_trivially_copyable(HasFunc))]; } - { int arr[T(__is_trivially_copyable(HasOp))]; } - { int arr[T(__is_trivially_copyable(HasConv))]; } - { int arr[T(__is_trivially_copyable(HasAssign))]; } - { int arr[T(__is_trivially_copyable(HasAnonymousUnion))]; } - { int arr[T(__is_trivially_copyable(HasPriv))]; } - { int arr[T(__is_trivially_copyable(HasProt))]; } - { int arr[T(__is_trivially_copyable(DerivesHasPriv))]; } - { int arr[T(__is_trivially_copyable(DerivesHasProt))]; } - { int arr[T(__is_trivially_copyable(Vector))]; } - { int arr[T(__is_trivially_copyable(VectorExt))]; } - { int arr[T(__is_trivially_copyable(HasCons))]; } - { int arr[T(__is_trivially_copyable(HasRef))]; } - { int arr[T(__is_trivially_copyable(HasNonPOD))]; } - { int arr[T(__is_trivially_copyable(DerivesHasCons))]; } - { int arr[T(__is_trivially_copyable(DerivesHasRef))]; } - { int arr[T(__is_trivially_copyable(NonTrivialDefault))]; } - { int arr[T(__is_trivially_copyable(NonTrivialDefault[]))]; } - { int arr[T(__is_trivially_copyable(NonTrivialDefault[3]))]; } - - { int arr[F(__is_trivially_copyable(HasCopyAssign))]; } - { int arr[F(__is_trivially_copyable(HasMoveAssign))]; } - { int arr[F(__is_trivially_copyable(HasDest))]; } - { int arr[F(__is_trivially_copyable(HasVirt))]; } - { int arr[F(__is_trivially_copyable(DerivesHasCopyAssign))]; } - { int arr[F(__is_trivially_copyable(DerivesHasMoveAssign))]; } - { int arr[F(__is_trivially_copyable(DerivesHasDest))]; } - { int arr[F(__is_trivially_copyable(DerivesHasVirt))]; } - { int arr[F(__is_trivially_copyable(void))]; } - { int arr[F(__is_trivially_copyable(cvoid))]; } - - { int arr[T((__is_trivially_constructible(int)))]; } - { int arr[T((__is_trivially_constructible(int, int)))]; } - { int arr[T((__is_trivially_constructible(int, float)))]; } - { int arr[T((__is_trivially_constructible(int, int&)))]; } - { int arr[T((__is_trivially_constructible(int, const int&)))]; } - { int arr[T((__is_trivially_constructible(int, int)))]; } - { int arr[T((__is_trivially_constructible(HasCopyAssign, HasCopyAssign)))]; } - { int arr[T((__is_trivially_constructible(HasCopyAssign, const HasCopyAssign&)))]; } - { int arr[T((__is_trivially_constructible(HasCopyAssign, HasCopyAssign&&)))]; } - { int arr[T((__is_trivially_constructible(HasCopyAssign)))]; } - { int arr[T((__is_trivially_constructible(NonTrivialDefault, - const NonTrivialDefault&)))]; } - { int arr[T((__is_trivially_constructible(NonTrivialDefault, - NonTrivialDefault&&)))]; } - { int arr[T((__is_trivially_constructible(AllDefaulted)))]; } - { int arr[T((__is_trivially_constructible(AllDefaulted, - const AllDefaulted &)))]; } - { int arr[T((__is_trivially_constructible(AllDefaulted, - AllDefaulted &&)))]; } - - { int arr[F((__is_trivially_constructible(int, int*)))]; } - { int arr[F((__is_trivially_constructible(NonTrivialDefault)))]; } - { int arr[F((__is_trivially_constructible(ThreeArgCtor, int*, char*, int&)))]; } - { int arr[F((__is_trivially_constructible(AllDeleted)))]; } - { int arr[F((__is_trivially_constructible(AllDeleted, - const AllDeleted &)))]; } - { int arr[F((__is_trivially_constructible(AllDeleted, - AllDeleted &&)))]; } - { int arr[F((__is_trivially_constructible(ExtDefaulted)))]; } - { int arr[F((__is_trivially_constructible(ExtDefaulted, - const ExtDefaulted &)))]; } - { int arr[F((__is_trivially_constructible(ExtDefaulted, - ExtDefaulted &&)))]; } - - { int arr[T((__is_trivially_constructible(TriviallyConstructibleTemplate)))]; } - { int arr[F((__is_trivially_constructible(class_forward)))]; } // expected-error {{incomplete type 'class_forward' used in type trait expression}} - { int arr[F((__is_trivially_constructible(class_forward[])))]; } - { int arr[F((__is_trivially_constructible(void)))]; } - - { int arr[T((__is_trivially_assignable(int&, int)))]; } - { int arr[T((__is_trivially_assignable(int&, int&)))]; } - { int arr[T((__is_trivially_assignable(int&, int&&)))]; } - { int arr[T((__is_trivially_assignable(int&, const int&)))]; } - { int arr[T((__is_trivially_assignable(POD&, POD)))]; } - { int arr[T((__is_trivially_assignable(POD&, POD&)))]; } - { int arr[T((__is_trivially_assignable(POD&, POD&&)))]; } - { int arr[T((__is_trivially_assignable(POD&, const POD&)))]; } - { int arr[T((__is_trivially_assignable(int*&, int*)))]; } - { int arr[T((__is_trivially_assignable(AllDefaulted, - const AllDefaulted &)))]; } - { int arr[T((__is_trivially_assignable(AllDefaulted, - AllDefaulted &&)))]; } - - { int arr[F((__is_trivially_assignable(int*&, float*)))]; } - { int arr[F((__is_trivially_assignable(HasCopyAssign&, HasCopyAssign)))]; } - { int arr[F((__is_trivially_assignable(HasCopyAssign&, HasCopyAssign&)))]; } - { int arr[F((__is_trivially_assignable(HasCopyAssign&, const HasCopyAssign&)))]; } - { int arr[F((__is_trivially_assignable(HasCopyAssign&, HasCopyAssign&&)))]; } - { int arr[F((__is_trivially_assignable(TrivialMoveButNotCopy&, - TrivialMoveButNotCopy&)))]; } - { int arr[F((__is_trivially_assignable(TrivialMoveButNotCopy&, - const TrivialMoveButNotCopy&)))]; } - { int arr[F((__is_trivially_assignable(AllDeleted, - const AllDeleted &)))]; } - { int arr[F((__is_trivially_assignable(AllDeleted, - AllDeleted &&)))]; } - { int arr[F((__is_trivially_assignable(ExtDefaulted, - const ExtDefaulted &)))]; } - { int arr[F((__is_trivially_assignable(ExtDefaulted, - ExtDefaulted &&)))]; } - - { int arr[T((__is_trivially_assignable(HasDefaultTrivialCopyAssign&, - HasDefaultTrivialCopyAssign&)))]; } - { int arr[T((__is_trivially_assignable(HasDefaultTrivialCopyAssign&, - const HasDefaultTrivialCopyAssign&)))]; } - { int arr[T((__is_trivially_assignable(TrivialMoveButNotCopy&, - TrivialMoveButNotCopy)))]; } - { int arr[T((__is_trivially_assignable(TrivialMoveButNotCopy&, - TrivialMoveButNotCopy&&)))]; } - { int arr[T((__is_trivially_assignable(int&, int)))]; } - { int arr[T((__is_trivially_assignable(int&, int&)))]; } - { int arr[T((__is_trivially_assignable(int&, int&&)))]; } - { int arr[T((__is_trivially_assignable(int&, const int&)))]; } - { int arr[T((__is_trivially_assignable(POD&, POD)))]; } - { int arr[T((__is_trivially_assignable(POD&, POD&)))]; } - { int arr[T((__is_trivially_assignable(POD&, POD&&)))]; } - { int arr[T((__is_trivially_assignable(POD&, const POD&)))]; } - { int arr[T((__is_trivially_assignable(int*&, int*)))]; } - { int arr[T((__is_trivially_assignable(AllDefaulted, - const AllDefaulted &)))]; } - { int arr[T((__is_trivially_assignable(AllDefaulted, - AllDefaulted &&)))]; } - - { int arr[F((__is_assignable(int *&, float *)))]; } - { int arr[T((__is_assignable(HasCopyAssign &, HasCopyAssign)))]; } - { int arr[T((__is_assignable(HasCopyAssign &, HasCopyAssign &)))]; } - { int arr[T((__is_assignable(HasCopyAssign &, const HasCopyAssign &)))]; } - { int arr[T((__is_assignable(HasCopyAssign &, HasCopyAssign &&)))]; } - { int arr[T((__is_assignable(TrivialMoveButNotCopy &, - TrivialMoveButNotCopy &)))]; } - { int arr[T((__is_assignable(TrivialMoveButNotCopy &, - const TrivialMoveButNotCopy &)))]; } - { int arr[F((__is_assignable(AllDeleted, - const AllDeleted &)))]; } - { int arr[F((__is_assignable(AllDeleted, - AllDeleted &&)))]; } - { int arr[T((__is_assignable(ExtDefaulted, - const ExtDefaulted &)))]; } - { int arr[T((__is_assignable(ExtDefaulted, - ExtDefaulted &&)))]; } - - { int arr[T((__is_assignable(HasDefaultTrivialCopyAssign &, - HasDefaultTrivialCopyAssign &)))]; } - { int arr[T((__is_assignable(HasDefaultTrivialCopyAssign &, - const HasDefaultTrivialCopyAssign &)))]; } - { int arr[T((__is_assignable(TrivialMoveButNotCopy &, - TrivialMoveButNotCopy)))]; } - { int arr[T((__is_assignable(TrivialMoveButNotCopy &, - TrivialMoveButNotCopy &&)))]; } - - { int arr[T(__is_assignable(ACompleteType, ACompleteType))]; } - { int arr[F(__is_assignable(AnIncompleteType, AnIncompleteType))]; } // expected-error {{incomplete type}} - { int arr[F(__is_assignable(AnIncompleteType[], AnIncompleteType[]))]; } - { int arr[F(__is_assignable(AnIncompleteType[1], AnIncompleteType[1]))]; } // expected-error {{incomplete type}} - { int arr[F(__is_assignable(void, void))]; } - { int arr[F(__is_assignable(const volatile void, const volatile void))]; } + static_assert(__is_trivially_copyable(int)); + static_assert(__is_trivially_copyable(Enum)); + static_assert(__is_trivially_copyable(POD)); + static_assert(__is_trivially_copyable(Int)); + static_assert(__is_trivially_copyable(IntAr)); + static_assert(__is_trivially_copyable(IntArNB)); + static_assert(__is_trivially_copyable(Statics)); + static_assert(__is_trivially_copyable(Empty)); + static_assert(__is_trivially_copyable(EmptyUnion)); + static_assert(__is_trivially_copyable(Union)); + static_assert(__is_trivially_copyable(Derives)); + static_assert(__is_trivially_copyable(DerivesAr)); + static_assert(__is_trivially_copyable(DerivesArNB)); + static_assert(__is_trivially_copyable(DerivesEmpty)); + static_assert(__is_trivially_copyable(HasFunc)); + static_assert(__is_trivially_copyable(HasOp)); + static_assert(__is_trivially_copyable(HasConv)); + static_assert(__is_trivially_copyable(HasAssign)); + static_assert(__is_trivially_copyable(HasAnonymousUnion)); + static_assert(__is_trivially_copyable(HasPriv)); + static_assert(__is_trivially_copyable(HasProt)); + static_assert(__is_trivially_copyable(DerivesHasPriv)); + static_assert(__is_trivially_copyable(DerivesHasProt)); + static_assert(__is_trivially_copyable(Vector)); + static_assert(__is_trivially_copyable(VectorExt)); + static_assert(__is_trivially_copyable(HasCons)); + static_assert(__is_trivially_copyable(HasRef)); + static_assert(__is_trivially_copyable(HasNonPOD)); + static_assert(__is_trivially_copyable(DerivesHasCons)); + static_assert(__is_trivially_copyable(DerivesHasRef)); + static_assert(__is_trivially_copyable(NonTrivialDefault)); + static_assert(__is_trivially_copyable(NonTrivialDefault[])); + static_assert(__is_trivially_copyable(NonTrivialDefault[3])); + + static_assert(!__is_trivially_copyable(HasCopyAssign)); + static_assert(!__is_trivially_copyable(HasMoveAssign)); + static_assert(!__is_trivially_copyable(HasDest)); + static_assert(!__is_trivially_copyable(HasVirt)); + static_assert(!__is_trivially_copyable(DerivesHasCopyAssign)); + static_assert(!__is_trivially_copyable(DerivesHasMoveAssign)); + static_assert(!__is_trivially_copyable(DerivesHasDest)); + static_assert(!__is_trivially_copyable(DerivesHasVirt)); + static_assert(!__is_trivially_copyable(void)); + static_assert(!__is_trivially_copyable(cvoid)); + + static_assert((__is_trivially_constructible(int))); + static_assert((__is_trivially_constructible(int, int))); + static_assert((__is_trivially_constructible(int, float))); + static_assert((__is_trivially_constructible(int, int&))); + static_assert((__is_trivially_constructible(int, const int&))); + static_assert((__is_trivially_constructible(int, int))); + static_assert((__is_trivially_constructible(HasCopyAssign, HasCopyAssign))); + static_assert((__is_trivially_constructible(HasCopyAssign, const HasCopyAssign&))); + static_assert((__is_trivially_constructible(HasCopyAssign, HasCopyAssign&&))); + static_assert((__is_trivially_constructible(HasCopyAssign))); + static_assert((__is_trivially_constructible(NonTrivialDefault, + const NonTrivialDefault&))); + static_assert((__is_trivially_constructible(NonTrivialDefault, + NonTrivialDefault&&))); + static_assert((__is_trivially_constructible(AllDefaulted))); + static_assert((__is_trivially_constructible(AllDefaulted, + const AllDefaulted &))); + static_assert((__is_trivially_constructible(AllDefaulted, + AllDefaulted &&))); + + static_assert(!(__is_trivially_constructible(int, int*))); + static_assert(!(__is_trivially_constructible(NonTrivialDefault))); + static_assert(!(__is_trivially_constructible(ThreeArgCtor, int*, char*, int&))); + static_assert(!(__is_trivially_constructible(AllDeleted))); + static_assert(!(__is_trivially_constructible(AllDeleted, + const AllDeleted &))); + static_assert(!(__is_trivially_constructible(AllDeleted, + AllDeleted &&))); + static_assert(!(__is_trivially_constructible(ExtDefaulted))); + static_assert(!(__is_trivially_constructible(ExtDefaulted, + const ExtDefaulted &))); + static_assert(!(__is_trivially_constructible(ExtDefaulted, + ExtDefaulted &&))); + + static_assert((__is_trivially_constructible(TriviallyConstructibleTemplate))); + static_assert(!(__is_trivially_constructible(class_forward))); // expected-error {{incomplete type 'class_forward' used in type trait expression}} + static_assert(!(__is_trivially_constructible(class_forward[]))); + static_assert(!(__is_trivially_constructible(void))); + + static_assert((__is_trivially_assignable(int&, int))); + static_assert((__is_trivially_assignable(int&, int&))); + static_assert((__is_trivially_assignable(int&, int&&))); + static_assert((__is_trivially_assignable(int&, const int&))); + static_assert((__is_trivially_assignable(POD&, POD))); + static_assert((__is_trivially_assignable(POD&, POD&))); + static_assert((__is_trivially_assignable(POD&, POD&&))); + static_assert((__is_trivially_assignable(POD&, const POD&))); + static_assert((__is_trivially_assignable(int*&, int*))); + static_assert((__is_trivially_assignable(AllDefaulted, + const AllDefaulted &))); + static_assert((__is_trivially_assignable(AllDefaulted, + AllDefaulted &&))); + + static_assert(!(__is_trivially_assignable(int*&, float*))); + static_assert(!(__is_trivially_assignable(HasCopyAssign&, HasCopyAssign))); + static_assert(!(__is_trivially_assignable(HasCopyAssign&, HasCopyAssign&))); + static_assert(!(__is_trivially_assignable(HasCopyAssign&, const HasCopyAssign&))); + static_assert(!(__is_trivially_assignable(HasCopyAssign&, HasCopyAssign&&))); + static_assert(!(__is_trivially_assignable(TrivialMoveButNotCopy&, + TrivialMoveButNotCopy&))); + static_assert(!(__is_trivially_assignable(TrivialMoveButNotCopy&, + const TrivialMoveButNotCopy&))); + static_assert(!(__is_trivially_assignable(AllDeleted, + const AllDeleted &))); + static_assert(!(__is_trivially_assignable(AllDeleted, + AllDeleted &&))); + static_assert(!(__is_trivially_assignable(ExtDefaulted, + const ExtDefaulted &))); + static_assert(!(__is_trivially_assignable(ExtDefaulted, + ExtDefaulted &&))); + + static_assert((__is_trivially_assignable(HasDefaultTrivialCopyAssign&, + HasDefaultTrivialCopyAssign&))); + static_assert((__is_trivially_assignable(HasDefaultTrivialCopyAssign&, + const HasDefaultTrivialCopyAssign&))); + static_assert((__is_trivially_assignable(TrivialMoveButNotCopy&, + TrivialMoveButNotCopy))); + static_assert((__is_trivially_assignable(TrivialMoveButNotCopy&, + TrivialMoveButNotCopy&&))); + static_assert((__is_trivially_assignable(int&, int))); + static_assert((__is_trivially_assignable(int&, int&))); + static_assert((__is_trivially_assignable(int&, int&&))); + static_assert((__is_trivially_assignable(int&, const int&))); + static_assert((__is_trivially_assignable(POD&, POD))); + static_assert((__is_trivially_assignable(POD&, POD&))); + static_assert((__is_trivially_assignable(POD&, POD&&))); + static_assert((__is_trivially_assignable(POD&, const POD&))); + static_assert((__is_trivially_assignable(int*&, int*))); + static_assert((__is_trivially_assignable(AllDefaulted, + const AllDefaulted &))); + static_assert((__is_trivially_assignable(AllDefaulted, + AllDefaulted &&))); + + static_assert(!(__is_assignable(int *&, float *))); + static_assert((__is_assignable(HasCopyAssign &, HasCopyAssign))); + static_assert((__is_assignable(HasCopyAssign &, HasCopyAssign &))); + static_assert((__is_assignable(HasCopyAssign &, const HasCopyAssign &))); + static_assert((__is_assignable(HasCopyAssign &, HasCopyAssign &&))); + static_assert((__is_assignable(TrivialMoveButNotCopy &, + TrivialMoveButNotCopy &))); + static_assert((__is_assignable(TrivialMoveButNotCopy &, + const TrivialMoveButNotCopy &))); + static_assert(!(__is_assignable(AllDeleted, + const AllDeleted &))); + static_assert(!(__is_assignable(AllDeleted, + AllDeleted &&))); + static_assert((__is_assignable(ExtDefaulted, + const ExtDefaulted &))); + static_assert((__is_assignable(ExtDefaulted, + ExtDefaulted &&))); + + static_assert((__is_assignable(HasDefaultTrivialCopyAssign &, + HasDefaultTrivialCopyAssign &))); + static_assert((__is_assignable(HasDefaultTrivialCopyAssign &, + const HasDefaultTrivialCopyAssign &))); + static_assert((__is_assignable(TrivialMoveButNotCopy &, + TrivialMoveButNotCopy))); + static_assert((__is_assignable(TrivialMoveButNotCopy &, + TrivialMoveButNotCopy &&))); + + static_assert(__is_assignable(ACompleteType, ACompleteType)); + static_assert(!__is_assignable(AnIncompleteType, AnIncompleteType)); // expected-error {{incomplete type}} + static_assert(!__is_assignable(AnIncompleteType[], AnIncompleteType[])); + static_assert(!__is_assignable(AnIncompleteType[1], AnIncompleteType[1])); // expected-error {{incomplete type}} + static_assert(!__is_assignable(void, void)); + static_assert(!__is_assignable(const volatile void, const volatile void)); } void constructible_checks() { - { int arr[T(__is_constructible(HasNoThrowConstructorWithArgs))]; } - { int arr[F(__is_nothrow_constructible(HasNoThrowConstructorWithArgs))]; } // MSVC doesn't look into default args and gets this wrong. + static_assert(__is_constructible(HasNoThrowConstructorWithArgs)); + static_assert(!__is_nothrow_constructible(HasNoThrowConstructorWithArgs)); // MSVC doesn't look into default args and gets this wrong. - { int arr[T(__is_constructible(HasNoThrowConstructorWithArgs, HasCons))]; } - { int arr[T(__is_nothrow_constructible(HasNoThrowConstructorWithArgs, HasCons))]; } + static_assert(__is_constructible(HasNoThrowConstructorWithArgs, HasCons)); + static_assert(__is_nothrow_constructible(HasNoThrowConstructorWithArgs, HasCons)); - { int arr[T(__is_constructible(NonTrivialDefault))]; } - { int arr[F(__is_nothrow_constructible(NonTrivialDefault))]; } + static_assert(__is_constructible(NonTrivialDefault)); + static_assert(!__is_nothrow_constructible(NonTrivialDefault)); - { int arr[T(__is_constructible(int))]; } - { int arr[T(__is_nothrow_constructible(int))]; } + static_assert(__is_constructible(int)); + static_assert(__is_nothrow_constructible(int)); - { int arr[F(__is_constructible(NonPOD))]; } - { int arr[F(__is_nothrow_constructible(NonPOD))]; } + static_assert(!__is_constructible(NonPOD)); + static_assert(!__is_nothrow_constructible(NonPOD)); - { int arr[T(__is_constructible(NonPOD, int))]; } - { int arr[F(__is_nothrow_constructible(NonPOD, int))]; } + static_assert(__is_constructible(NonPOD, int)); + static_assert(!__is_nothrow_constructible(NonPOD, int)); // PR19178 - { int arr[F(__is_constructible(Abstract))]; } - { int arr[F(__is_nothrow_constructible(Abstract))]; } + static_assert(!__is_constructible(Abstract)); + static_assert(!__is_nothrow_constructible(Abstract)); // PR20228 - { int arr[T(__is_constructible(VariadicCtor, - int, int, int, int, int, int, int, int, int))]; } + static_assert(__is_constructible(VariadicCtor, + int, int, int, int, int, int, int, int, int)); // PR25513 - { int arr[F(__is_constructible(int(int)))]; } - { int arr[T(__is_constructible(int const &, long))]; } - - { int arr[T(__is_constructible(ACompleteType))]; } - { int arr[T(__is_nothrow_constructible(ACompleteType))]; } - { int arr[F(__is_constructible(AnIncompleteType))]; } // expected-error {{incomplete type}} - { int arr[F(__is_nothrow_constructible(AnIncompleteType))]; } // expected-error {{incomplete type}} - { int arr[F(__is_constructible(AnIncompleteType[]))]; } - { int arr[F(__is_nothrow_constructible(AnIncompleteType[]))]; } - { int arr[F(__is_constructible(AnIncompleteType[1]))]; } // expected-error {{incomplete type}} - { int arr[F(__is_nothrow_constructible(AnIncompleteType[1]))]; } // expected-error {{incomplete type}} - { int arr[F(__is_constructible(void))]; } - { int arr[F(__is_nothrow_constructible(void))]; } - { int arr[F(__is_constructible(const volatile void))]; } - { int arr[F(__is_nothrow_constructible(const volatile void))]; } + static_assert(!__is_constructible(int(int))); + static_assert(__is_constructible(int const &, long)); + + static_assert(__is_constructible(ACompleteType)); + static_assert(__is_nothrow_constructible(ACompleteType)); + static_assert(!__is_constructible(AnIncompleteType)); // expected-error {{incomplete type}} + static_assert(!__is_nothrow_constructible(AnIncompleteType)); // expected-error {{incomplete type}} + static_assert(!__is_constructible(AnIncompleteType[])); + static_assert(!__is_nothrow_constructible(AnIncompleteType[])); + static_assert(!__is_constructible(AnIncompleteType[1])); // expected-error {{incomplete type}} + static_assert(!__is_nothrow_constructible(AnIncompleteType[1])); // expected-error {{incomplete type}} + static_assert(!__is_constructible(void)); + static_assert(!__is_nothrow_constructible(void)); + static_assert(!__is_constructible(const volatile void)); + static_assert(!__is_nothrow_constructible(const volatile void)); } // Instantiation of __is_trivially_constructible @@ -2738,32 +2727,32 @@ struct is_trivially_constructible { }; void is_trivially_constructible_test() { - { int arr[T((is_trivially_constructible::value))]; } - { int arr[T((is_trivially_constructible::value))]; } - { int arr[T((is_trivially_constructible::value))]; } - { int arr[T((is_trivially_constructible::value))]; } - { int arr[T((is_trivially_constructible::value))]; } - { int arr[T((is_trivially_constructible::value))]; } - { int arr[T((is_trivially_constructible::value))]; } - { int arr[T((is_trivially_constructible::value))]; } - { int arr[T((is_trivially_constructible::value))]; } - { int arr[T((is_trivially_constructible::value))]; } - { int arr[T((is_trivially_constructible::value))]; } - { int arr[T((is_trivially_constructible::value))]; } - - { int arr[F((is_trivially_constructible::value))]; } - { int arr[F((is_trivially_constructible::value))]; } - { int arr[F((is_trivially_constructible::value))]; } - { int arr[F((is_trivially_constructible::value))]; } // PR19178 - - { int arr[T(__is_trivially_constructible(ACompleteType))]; } - { int arr[F(__is_trivially_constructible(AnIncompleteType))]; } // expected-error {{incomplete type}} - { int arr[F(__is_trivially_constructible(AnIncompleteType[]))]; } - { int arr[F(__is_trivially_constructible(AnIncompleteType[1]))]; } // expected-error {{incomplete type}} - { int arr[F(__is_trivially_constructible(void))]; } - { int arr[F(__is_trivially_constructible(const volatile void))]; } + static_assert((is_trivially_constructible::value)); + static_assert((is_trivially_constructible::value)); + static_assert((is_trivially_constructible::value)); + static_assert((is_trivially_constructible::value)); + static_assert((is_trivially_constructible::value)); + static_assert((is_trivially_constructible::value)); + static_assert((is_trivially_constructible::value)); + static_assert((is_trivially_constructible::value)); + static_assert((is_trivially_constructible::value)); + static_assert((is_trivially_constructible::value)); + static_assert((is_trivially_constructible::value)); + static_assert((is_trivially_constructible::value)); + + static_assert(!(is_trivially_constructible::value)); + static_assert(!(is_trivially_constructible::value)); + static_assert(!(is_trivially_constructible::value)); + static_assert(!(is_trivially_constructible::value)); // PR19178 + + static_assert(__is_trivially_constructible(ACompleteType)); + static_assert(!__is_trivially_constructible(AnIncompleteType)); // expected-error {{incomplete type}} + static_assert(!__is_trivially_constructible(AnIncompleteType[])); + static_assert(!__is_trivially_constructible(AnIncompleteType[1])); // expected-error {{incomplete type}} + static_assert(!__is_trivially_constructible(void)); + static_assert(!__is_trivially_constructible(const volatile void)); } template @@ -2773,365 +2762,359 @@ struct ConvertsToRef { }; void reference_binds_to_temporary_checks() { - { int arr[F((__reference_binds_to_temporary(int &, int &)))]; } - { int arr[F((__reference_binds_to_temporary(int &, int &&)))]; } + static_assert(!(__reference_binds_to_temporary(int &, int &))); + static_assert(!(__reference_binds_to_temporary(int &, int &&))); - { int arr[F((__reference_binds_to_temporary(int const &, int &)))]; } - { int arr[F((__reference_binds_to_temporary(int const &, int const &)))]; } - { int arr[F((__reference_binds_to_temporary(int const &, int &&)))]; } + static_assert(!(__reference_binds_to_temporary(int const &, int &))); + static_assert(!(__reference_binds_to_temporary(int const &, int const &))); + static_assert(!(__reference_binds_to_temporary(int const &, int &&))); - { int arr[F((__reference_binds_to_temporary(int &, long &)))]; } // doesn't construct - { int arr[T((__reference_binds_to_temporary(int const &, long &)))]; } - { int arr[T((__reference_binds_to_temporary(int const &, long &&)))]; } - { int arr[T((__reference_binds_to_temporary(int &&, long &)))]; } + static_assert(!(__reference_binds_to_temporary(int &, long &))); // doesn't construct + static_assert((__reference_binds_to_temporary(int const &, long &))); + static_assert((__reference_binds_to_temporary(int const &, long &&))); + static_assert((__reference_binds_to_temporary(int &&, long &))); using LRef = ConvertsToRef; using RRef = ConvertsToRef; using CLRef = ConvertsToRef; using LongRef = ConvertsToRef; - { int arr[T((__is_constructible(int &, LRef)))]; } - { int arr[F((__reference_binds_to_temporary(int &, LRef)))]; } + static_assert((__is_constructible(int &, LRef))); + static_assert(!(__reference_binds_to_temporary(int &, LRef))); - { int arr[T((__is_constructible(int &&, RRef)))]; } - { int arr[F((__reference_binds_to_temporary(int &&, RRef)))]; } + static_assert((__is_constructible(int &&, RRef))); + static_assert(!(__reference_binds_to_temporary(int &&, RRef))); - { int arr[T((__is_constructible(int const &, CLRef)))]; } - { int arr[F((__reference_binds_to_temporary(int &&, CLRef)))]; } + static_assert((__is_constructible(int const &, CLRef))); + static_assert(!(__reference_binds_to_temporary(int &&, CLRef))); - { int arr[T((__is_constructible(int const &, LongRef)))]; } - { int arr[T((__reference_binds_to_temporary(int const &, LongRef)))]; } + static_assert((__is_constructible(int const &, LongRef))); + static_assert((__reference_binds_to_temporary(int const &, LongRef))); // Test that it doesn't accept non-reference types as input. - { int arr[F((__reference_binds_to_temporary(int, long)))]; } + static_assert(!(__reference_binds_to_temporary(int, long))); - { int arr[T((__reference_binds_to_temporary(const int &, long)))]; } + static_assert((__reference_binds_to_temporary(const int &, long))); } void reference_constructs_from_temporary_checks() { - static_assert(!__reference_constructs_from_temporary(int &, int &), ""); - static_assert(!__reference_constructs_from_temporary(int &, int &&), ""); + static_assert(!__reference_constructs_from_temporary(int &, int &)); + static_assert(!__reference_constructs_from_temporary(int &, int &&)); - static_assert(!__reference_constructs_from_temporary(int const &, int &), ""); - static_assert(!__reference_constructs_from_temporary(int const &, int const &), ""); - static_assert(!__reference_constructs_from_temporary(int const &, int &&), ""); + static_assert(!__reference_constructs_from_temporary(int const &, int &)); + static_assert(!__reference_constructs_from_temporary(int const &, int const &)); + static_assert(!__reference_constructs_from_temporary(int const &, int &&)); - static_assert(!__reference_constructs_from_temporary(int &, long &), ""); // doesn't construct + static_assert(!__reference_constructs_from_temporary(int &, long &)); // doesn't construct - static_assert(__reference_constructs_from_temporary(int const &, long &), ""); - static_assert(__reference_constructs_from_temporary(int const &, long &&), ""); - static_assert(__reference_constructs_from_temporary(int &&, long &), ""); + static_assert(__reference_constructs_from_temporary(int const &, long &)); + static_assert(__reference_constructs_from_temporary(int const &, long &&)); + static_assert(__reference_constructs_from_temporary(int &&, long &)); using LRef = ConvertsToRef; using RRef = ConvertsToRef; using CLRef = ConvertsToRef; using LongRef = ConvertsToRef; - static_assert(__is_constructible(int &, LRef), ""); - static_assert(!__reference_constructs_from_temporary(int &, LRef), ""); + static_assert(__is_constructible(int &, LRef)); + static_assert(!__reference_constructs_from_temporary(int &, LRef)); - static_assert(__is_constructible(int &&, RRef), ""); - static_assert(!__reference_constructs_from_temporary(int &&, RRef), ""); + static_assert(__is_constructible(int &&, RRef)); + static_assert(!__reference_constructs_from_temporary(int &&, RRef)); - static_assert(__is_constructible(int const &, CLRef), ""); - static_assert(!__reference_constructs_from_temporary(int &&, CLRef), ""); + static_assert(__is_constructible(int const &, CLRef)); + static_assert(!__reference_constructs_from_temporary(int &&, CLRef)); - static_assert(__is_constructible(int const &, LongRef), ""); - static_assert(__reference_constructs_from_temporary(int const &, LongRef), ""); + static_assert(__is_constructible(int const &, LongRef)); + static_assert(__reference_constructs_from_temporary(int const &, LongRef)); // Test that it doesn't accept non-reference types as input. - static_assert(!__reference_constructs_from_temporary(int, long), ""); + static_assert(!__reference_constructs_from_temporary(int, long)); - static_assert(__reference_constructs_from_temporary(const int &, long), ""); + static_assert(__reference_constructs_from_temporary(const int &, long)); // Additional checks - static_assert(__reference_constructs_from_temporary(POD const&, Derives), ""); - static_assert(__reference_constructs_from_temporary(int&&, int), ""); - static_assert(__reference_constructs_from_temporary(const int&, int), ""); - static_assert(!__reference_constructs_from_temporary(int&&, int&&), ""); - static_assert(!__reference_constructs_from_temporary(const int&, int&&), ""); - static_assert(__reference_constructs_from_temporary(int&&, long&&), ""); - static_assert(__reference_constructs_from_temporary(int&&, long), ""); + static_assert(__reference_constructs_from_temporary(POD const&, Derives)); + static_assert(__reference_constructs_from_temporary(int&&, int)); + static_assert(__reference_constructs_from_temporary(const int&, int)); + static_assert(!__reference_constructs_from_temporary(int&&, int&&)); + static_assert(!__reference_constructs_from_temporary(const int&, int&&)); + static_assert(__reference_constructs_from_temporary(int&&, long&&)); + static_assert(__reference_constructs_from_temporary(int&&, long)); } void array_rank() { - int t01[T(__array_rank(IntAr) == 1)]; - int t02[T(__array_rank(ConstIntArAr) == 2)]; + static_assert(__array_rank(IntAr) == 1); + static_assert(__array_rank(ConstIntArAr) == 2); } void array_extent() { - int t01[T(__array_extent(IntAr, 0) == 10)]; - int t02[T(__array_extent(ConstIntArAr, 0) == 4)]; - int t03[T(__array_extent(ConstIntArAr, 1) == 10)]; + static_assert(__array_extent(IntAr, 0) == 10); + static_assert(__array_extent(ConstIntArAr, 0) == 4); + static_assert(__array_extent(ConstIntArAr, 1) == 10); } void is_destructible_test() { - { int arr[T(__is_destructible(int))]; } - { int arr[T(__is_destructible(int[2]))]; } - { int arr[F(__is_destructible(int[]))]; } - { int arr[F(__is_destructible(void))]; } - { int arr[T(__is_destructible(int &))]; } - { int arr[T(__is_destructible(HasDest))]; } - { int arr[F(__is_destructible(AllPrivate))]; } - { int arr[T(__is_destructible(SuperNonTrivialStruct))]; } - { int arr[T(__is_destructible(AllDefaulted))]; } - { int arr[F(__is_destructible(AllDeleted))]; } - { int arr[T(__is_destructible(ThrowingDtor))]; } - { int arr[T(__is_destructible(NoThrowDtor))]; } - - { int arr[T(__is_destructible(ACompleteType))]; } - { int arr[F(__is_destructible(AnIncompleteType))]; } // expected-error {{incomplete type}} - { int arr[F(__is_destructible(AnIncompleteType[]))]; } - { int arr[F(__is_destructible(AnIncompleteType[1]))]; } // expected-error {{incomplete type}} - { int arr[F(__is_destructible(void))]; } - { int arr[F(__is_destructible(const volatile void))]; } + static_assert(__is_destructible(int)); + static_assert(__is_destructible(int[2])); + static_assert(!__is_destructible(int[])); + static_assert(!__is_destructible(void)); + static_assert(__is_destructible(int &)); + static_assert(__is_destructible(HasDest)); + static_assert(!__is_destructible(AllPrivate)); + static_assert(__is_destructible(SuperNonTrivialStruct)); + static_assert(__is_destructible(AllDefaulted)); + static_assert(!__is_destructible(AllDeleted)); + static_assert(__is_destructible(ThrowingDtor)); + static_assert(__is_destructible(NoThrowDtor)); + + static_assert(__is_destructible(ACompleteType)); + static_assert(!__is_destructible(AnIncompleteType)); // expected-error {{incomplete type}} + static_assert(!__is_destructible(AnIncompleteType[])); + static_assert(!__is_destructible(AnIncompleteType[1])); // expected-error {{incomplete type}} + static_assert(!__is_destructible(void)); + static_assert(!__is_destructible(const volatile void)); } void is_nothrow_destructible_test() { - { int arr[T(__is_nothrow_destructible(int))]; } - { int arr[T(__is_nothrow_destructible(int[2]))]; } - { int arr[F(__is_nothrow_destructible(int[]))]; } - { int arr[F(__is_nothrow_destructible(void))]; } - { int arr[T(__is_nothrow_destructible(int &))]; } - { int arr[T(__is_nothrow_destructible(HasDest))]; } - { int arr[F(__is_nothrow_destructible(AllPrivate))]; } - { int arr[T(__is_nothrow_destructible(SuperNonTrivialStruct))]; } - { int arr[T(__is_nothrow_destructible(AllDefaulted))]; } - { int arr[F(__is_nothrow_destructible(AllDeleted))]; } - { int arr[F(__is_nothrow_destructible(ThrowingDtor))]; } - { int arr[T(__is_nothrow_destructible(NoExceptDtor))]; } - { int arr[T(__is_nothrow_destructible(NoThrowDtor))]; } - - { int arr[T(__is_nothrow_destructible(ACompleteType))]; } - { int arr[F(__is_nothrow_destructible(AnIncompleteType))]; } // expected-error {{incomplete type}} - { int arr[F(__is_nothrow_destructible(AnIncompleteType[]))]; } - { int arr[F(__is_nothrow_destructible(AnIncompleteType[1]))]; } // expected-error {{incomplete type}} - { int arr[F(__is_nothrow_destructible(void))]; } - { int arr[F(__is_nothrow_destructible(const volatile void))]; } + static_assert(__is_nothrow_destructible(int)); + static_assert(__is_nothrow_destructible(int[2])); + static_assert(!__is_nothrow_destructible(int[])); + static_assert(!__is_nothrow_destructible(void)); + static_assert(__is_nothrow_destructible(int &)); + static_assert(__is_nothrow_destructible(HasDest)); + static_assert(!__is_nothrow_destructible(AllPrivate)); + static_assert(__is_nothrow_destructible(SuperNonTrivialStruct)); + static_assert(__is_nothrow_destructible(AllDefaulted)); + static_assert(!__is_nothrow_destructible(AllDeleted)); + static_assert(!__is_nothrow_destructible(ThrowingDtor)); + static_assert(__is_nothrow_destructible(NoExceptDtor)); + static_assert(__is_nothrow_destructible(NoThrowDtor)); + + static_assert(__is_nothrow_destructible(ACompleteType)); + static_assert(!__is_nothrow_destructible(AnIncompleteType)); // expected-error {{incomplete type}} + static_assert(!__is_nothrow_destructible(AnIncompleteType[])); + static_assert(!__is_nothrow_destructible(AnIncompleteType[1])); // expected-error {{incomplete type}} + static_assert(!__is_nothrow_destructible(void)); + static_assert(!__is_nothrow_destructible(const volatile void)); } void is_trivially_destructible_test() { - { int arr[T(__is_trivially_destructible(int))]; } - { int arr[T(__is_trivially_destructible(int[2]))]; } - { int arr[F(__is_trivially_destructible(int[]))]; } - { int arr[F(__is_trivially_destructible(void))]; } - { int arr[T(__is_trivially_destructible(int &))]; } - { int arr[F(__is_trivially_destructible(HasDest))]; } - { int arr[F(__is_trivially_destructible(AllPrivate))]; } - { int arr[F(__is_trivially_destructible(SuperNonTrivialStruct))]; } - { int arr[T(__is_trivially_destructible(AllDefaulted))]; } - { int arr[F(__is_trivially_destructible(AllDeleted))]; } - { int arr[F(__is_trivially_destructible(ThrowingDtor))]; } - { int arr[F(__is_trivially_destructible(NoThrowDtor))]; } - - { int arr[T(__is_trivially_destructible(ACompleteType))]; } - { int arr[F(__is_trivially_destructible(AnIncompleteType))]; } // expected-error {{incomplete type}} - { int arr[F(__is_trivially_destructible(AnIncompleteType[]))]; } - { int arr[F(__is_trivially_destructible(AnIncompleteType[1]))]; } // expected-error {{incomplete type}} - { int arr[F(__is_trivially_destructible(void))]; } - { int arr[F(__is_trivially_destructible(const volatile void))]; } + static_assert(__is_trivially_destructible(int)); + static_assert(__is_trivially_destructible(int[2])); + static_assert(!__is_trivially_destructible(int[])); + static_assert(!__is_trivially_destructible(void)); + static_assert(__is_trivially_destructible(int &)); + static_assert(!__is_trivially_destructible(HasDest)); + static_assert(!__is_trivially_destructible(AllPrivate)); + static_assert(!__is_trivially_destructible(SuperNonTrivialStruct)); + static_assert(__is_trivially_destructible(AllDefaulted)); + static_assert(!__is_trivially_destructible(AllDeleted)); + static_assert(!__is_trivially_destructible(ThrowingDtor)); + static_assert(!__is_trivially_destructible(NoThrowDtor)); + + static_assert(__is_trivially_destructible(ACompleteType)); + static_assert(!__is_trivially_destructible(AnIncompleteType)); // expected-error {{incomplete type}} + static_assert(!__is_trivially_destructible(AnIncompleteType[])); + static_assert(!__is_trivially_destructible(AnIncompleteType[1])); // expected-error {{incomplete type}} + static_assert(!__is_trivially_destructible(void)); + static_assert(!__is_trivially_destructible(const volatile void)); } -// Instantiation of __has_unique_object_representations -template -struct has_unique_object_representations { - static const bool value = __has_unique_object_representations(T); -}; - -static_assert(!has_unique_object_representations::value, "void is never unique"); -static_assert(!has_unique_object_representations::value, "void is never unique"); -static_assert(!has_unique_object_representations::value, "void is never unique"); -static_assert(!has_unique_object_representations::value, "void is never unique"); +static_assert(!__has_unique_object_representations(void), "void is never unique"); +static_assert(!__has_unique_object_representations(const void), "void is never unique"); +static_assert(!__has_unique_object_representations(volatile void), "void is never unique"); +static_assert(!__has_unique_object_representations(const volatile void), "void is never unique"); -static_assert(has_unique_object_representations::value, "integrals are"); -static_assert(has_unique_object_representations::value, "integrals are"); -static_assert(has_unique_object_representations::value, "integrals are"); -static_assert(has_unique_object_representations::value, "integrals are"); +static_assert(__has_unique_object_representations(int), "integrals are"); +static_assert(__has_unique_object_representations(const int), "integrals are"); +static_assert(__has_unique_object_representations(volatile int), "integrals are"); +static_assert(__has_unique_object_representations(const volatile int), "integrals are"); -static_assert(has_unique_object_representations::value, "as are pointers"); -static_assert(has_unique_object_representations::value, "as are pointers"); -static_assert(has_unique_object_representations::value, "are pointers"); -static_assert(has_unique_object_representations::value, "as are pointers"); +static_assert(__has_unique_object_representations(void *), "as are pointers"); +static_assert(__has_unique_object_representations(const void *), "as are pointers"); +static_assert(__has_unique_object_representations(volatile void *), "are pointers"); +static_assert(__has_unique_object_representations(const volatile void *), "as are pointers"); -static_assert(has_unique_object_representations::value, "as are pointers"); -static_assert(has_unique_object_representations::value, "as are pointers"); -static_assert(has_unique_object_representations::value, "as are pointers"); -static_assert(has_unique_object_representations::value, "as are pointers"); +static_assert(__has_unique_object_representations(int *), "as are pointers"); +static_assert(__has_unique_object_representations(const int *), "as are pointers"); +static_assert(__has_unique_object_representations(volatile int *), "as are pointers"); +static_assert(__has_unique_object_representations(const volatile int *), "as are pointers"); class C {}; using FP = int (*)(int); using PMF = int (C::*)(int); using PMD = int C::*; -static_assert(has_unique_object_representations::value, "even function pointers"); -static_assert(has_unique_object_representations::value, "even function pointers"); -static_assert(has_unique_object_representations::value, "even function pointers"); -static_assert(has_unique_object_representations::value, "even function pointers"); - -static_assert(has_unique_object_representations::value, "and pointer to members"); -static_assert(has_unique_object_representations::value, "and pointer to members"); -static_assert(has_unique_object_representations::value, "and pointer to members"); -static_assert(has_unique_object_representations::value, "and pointer to members"); - -static_assert(has_unique_object_representations::value, "and pointer to members"); -static_assert(has_unique_object_representations::value, "and pointer to members"); -static_assert(has_unique_object_representations::value, "and pointer to members"); -static_assert(has_unique_object_representations::value, "and pointer to members"); - -static_assert(has_unique_object_representations::value, "yes, all integral types"); -static_assert(has_unique_object_representations::value, "yes, all integral types"); -static_assert(has_unique_object_representations::value, "yes, all integral types"); -static_assert(has_unique_object_representations::value, "yes, all integral types"); -static_assert(has_unique_object_representations::value, "yes, all integral types"); -static_assert(has_unique_object_representations::value, "yes, all integral types"); -static_assert(has_unique_object_representations::value, "yes, all integral types"); -static_assert(has_unique_object_representations::value, "yes, all integral types"); -static_assert(has_unique_object_representations::value, "yes, all integral types"); -static_assert(has_unique_object_representations::value, "yes, all integral types"); -static_assert(has_unique_object_representations::value, "yes, all integral types"); -static_assert(has_unique_object_representations::value, "yes, all integral types"); -static_assert(has_unique_object_representations::value, "yes, all integral types"); -static_assert(has_unique_object_representations::value, "yes, all integral types"); -static_assert(has_unique_object_representations::value, "yes, all integral types"); - -static_assert(!has_unique_object_representations::value, "but not void!"); -static_assert(!has_unique_object_representations::value, "or nullptr_t"); -static_assert(!has_unique_object_representations::value, "definitely not Floating Point"); -static_assert(!has_unique_object_representations::value, "definitely not Floating Point"); -static_assert(!has_unique_object_representations::value, "definitely not Floating Point"); +static_assert(__has_unique_object_representations(FP), "even function pointers"); +static_assert(__has_unique_object_representations(const FP), "even function pointers"); +static_assert(__has_unique_object_representations(volatile FP), "even function pointers"); +static_assert(__has_unique_object_representations(const volatile FP), "even function pointers"); + +static_assert(__has_unique_object_representations(PMF), "and pointer to members"); +static_assert(__has_unique_object_representations(const PMF), "and pointer to members"); +static_assert(__has_unique_object_representations(volatile PMF), "and pointer to members"); +static_assert(__has_unique_object_representations(const volatile PMF), "and pointer to members"); + +static_assert(__has_unique_object_representations(PMD), "and pointer to members"); +static_assert(__has_unique_object_representations(const PMD), "and pointer to members"); +static_assert(__has_unique_object_representations(volatile PMD), "and pointer to members"); +static_assert(__has_unique_object_representations(const volatile PMD), "and pointer to members"); + +static_assert(__has_unique_object_representations(bool), "yes, all integral types"); +static_assert(__has_unique_object_representations(char), "yes, all integral types"); +static_assert(__has_unique_object_representations(signed char), "yes, all integral types"); +static_assert(__has_unique_object_representations(unsigned char), "yes, all integral types"); +static_assert(__has_unique_object_representations(short), "yes, all integral types"); +static_assert(__has_unique_object_representations(unsigned short), "yes, all integral types"); +static_assert(__has_unique_object_representations(int), "yes, all integral types"); +static_assert(__has_unique_object_representations(unsigned int), "yes, all integral types"); +static_assert(__has_unique_object_representations(long), "yes, all integral types"); +static_assert(__has_unique_object_representations(unsigned long), "yes, all integral types"); +static_assert(__has_unique_object_representations(long long), "yes, all integral types"); +static_assert(__has_unique_object_representations(unsigned long long), "yes, all integral types"); +static_assert(__has_unique_object_representations(wchar_t), "yes, all integral types"); +static_assert(__has_unique_object_representations(char16_t), "yes, all integral types"); +static_assert(__has_unique_object_representations(char32_t), "yes, all integral types"); + +static_assert(!__has_unique_object_representations(void), "but not void!"); +static_assert(!__has_unique_object_representations(decltype(nullptr)), "or nullptr_t"); +static_assert(!__has_unique_object_representations(float), "definitely not Floating Point"); +static_assert(!__has_unique_object_representations(double), "definitely not Floating Point"); +static_assert(!__has_unique_object_representations(long double), "definitely not Floating Point"); struct NoPadding { int a; int b; }; -static_assert(has_unique_object_representations::value, "types without padding are"); +static_assert(__has_unique_object_representations(NoPadding), "types without padding are"); struct InheritsFromNoPadding : NoPadding { int c; int d; }; -static_assert(has_unique_object_representations::value, "types without padding are"); +static_assert(__has_unique_object_representations(InheritsFromNoPadding), "types without padding are"); struct VirtuallyInheritsFromNoPadding : virtual NoPadding { int c; int d; }; -static_assert(!has_unique_object_representations::value, "No virtual inheritance"); +static_assert(!__has_unique_object_representations(VirtuallyInheritsFromNoPadding), "No virtual inheritance"); struct Padding { char a; int b; }; -//static_assert(!has_unique_object_representations::value, "but not with padding"); +//static_assert(!__has_unique_object_representations(Padding), "but not with padding"); struct InheritsFromPadding : Padding { int c; int d; }; -static_assert(!has_unique_object_representations::value, "or its subclasses"); +static_assert(!__has_unique_object_representations(InheritsFromPadding), "or its subclasses"); struct TailPadding { int a; char b; }; -static_assert(!has_unique_object_representations::value, "even at the end"); +static_assert(!__has_unique_object_representations(TailPadding), "even at the end"); struct TinyStruct { char a; }; -static_assert(has_unique_object_representations::value, "Should be no padding"); +static_assert(__has_unique_object_representations(TinyStruct), "Should be no padding"); struct InheritsFromTinyStruct : TinyStruct { int b; }; -static_assert(!has_unique_object_representations::value, "Inherit causes padding"); +static_assert(!__has_unique_object_representations(InheritsFromTinyStruct), "Inherit causes padding"); union NoPaddingUnion { int a; unsigned int b; }; -static_assert(has_unique_object_representations::value, "unions follow the same rules as structs"); +static_assert(__has_unique_object_representations(NoPaddingUnion), "unions follow the same rules as structs"); union PaddingUnion { int a; long long b; }; -static_assert(!has_unique_object_representations::value, "unions follow the same rules as structs"); +static_assert(!__has_unique_object_representations(PaddingUnion), "unions follow the same rules as structs"); struct NotTriviallyCopyable { int x; NotTriviallyCopyable(const NotTriviallyCopyable &) {} }; -static_assert(!has_unique_object_representations::value, "must be trivially copyable"); +static_assert(!__has_unique_object_representations(NotTriviallyCopyable), "must be trivially copyable"); struct HasNonUniqueMember { float x; }; -static_assert(!has_unique_object_representations::value, "all members must be unique"); +static_assert(!__has_unique_object_representations(HasNonUniqueMember), "all members must be unique"); enum ExampleEnum { xExample, yExample }; enum LLEnum : long long { xLongExample, yLongExample }; -static_assert(has_unique_object_representations::value, "Enums are integrals, so unique!"); -static_assert(has_unique_object_representations::value, "Enums are integrals, so unique!"); +static_assert(__has_unique_object_representations(ExampleEnum), "Enums are integrals, so unique!"); +static_assert(__has_unique_object_representations(LLEnum), "Enums are integrals, so unique!"); enum class ExampleEnumClass { xExample, yExample }; enum class LLEnumClass : long long { xLongExample, yLongExample }; -static_assert(has_unique_object_representations::value, "Enums are integrals, so unique!"); -static_assert(has_unique_object_representations::value, "Enums are integrals, so unique!"); +static_assert(__has_unique_object_representations(ExampleEnumClass), "Enums are integrals, so unique!"); +static_assert(__has_unique_object_representations(LLEnumClass), "Enums are integrals, so unique!"); // because references aren't trivially copyable. -static_assert(!has_unique_object_representations::value, "No references!"); -static_assert(!has_unique_object_representations::value, "No references!"); -static_assert(!has_unique_object_representations::value, "No references!"); -static_assert(!has_unique_object_representations::value, "No references!"); -static_assert(!has_unique_object_representations::value, "No empty types!"); -static_assert(!has_unique_object_representations::value, "No empty types!"); +static_assert(!__has_unique_object_representations(int &), "No references!"); +static_assert(!__has_unique_object_representations(const int &), "No references!"); +static_assert(!__has_unique_object_representations(volatile int &), "No references!"); +static_assert(!__has_unique_object_representations(const volatile int &), "No references!"); +static_assert(!__has_unique_object_representations(Empty), "No empty types!"); +static_assert(!__has_unique_object_representations(EmptyUnion), "No empty types!"); class Compressed : Empty { int x; }; -static_assert(has_unique_object_representations::value, "But inheriting from one is ok"); +static_assert(__has_unique_object_representations(Compressed), "But inheriting from one is ok"); class EmptyInheritor : Compressed {}; -static_assert(has_unique_object_representations::value, "As long as the base has items, empty is ok"); +static_assert(__has_unique_object_representations(EmptyInheritor), "As long as the base has items, empty is ok"); class Dynamic { virtual void A(); int i; }; -static_assert(!has_unique_object_representations::value, "Dynamic types are not valid"); +static_assert(!__has_unique_object_representations(Dynamic), "Dynamic types are not valid"); class InheritsDynamic : Dynamic { int j; }; -static_assert(!has_unique_object_representations::value, "Dynamic types are not valid"); +static_assert(!__has_unique_object_representations(InheritsDynamic), "Dynamic types are not valid"); -static_assert(has_unique_object_representations::value, "Arrays are fine, as long as their value type is"); -static_assert(has_unique_object_representations::value, "Arrays are fine, as long as their value type is"); -static_assert(has_unique_object_representations::value, "Arrays are fine, as long as their value type is"); -static_assert(!has_unique_object_representations::value, "So no array of doubles!"); -static_assert(!has_unique_object_representations::value, "So no array of doubles!"); -static_assert(!has_unique_object_representations::value, "So no array of doubles!"); +static_assert(__has_unique_object_representations(int[42]), "Arrays are fine, as long as their value type is"); +static_assert(__has_unique_object_representations(int[]), "Arrays are fine, as long as their value type is"); +static_assert(__has_unique_object_representations(int[][42]), "Arrays are fine, as long as their value type is"); +static_assert(!__has_unique_object_representations(double[42]), "So no array of doubles!"); +static_assert(!__has_unique_object_representations(double[]), "So no array of doubles!"); +static_assert(!__has_unique_object_representations(double[][42]), "So no array of doubles!"); struct __attribute__((aligned(16))) WeirdAlignment { int i; @@ -3139,9 +3122,9 @@ struct __attribute__((aligned(16))) WeirdAlignment { union __attribute__((aligned(16))) WeirdAlignmentUnion { int i; }; -static_assert(!has_unique_object_representations::value, "Alignment causes padding"); -static_assert(!has_unique_object_representations::value, "Alignment causes padding"); -static_assert(!has_unique_object_representations::value, "Also no arrays that have padding"); +static_assert(!__has_unique_object_representations(WeirdAlignment), "Alignment causes padding"); +static_assert(!__has_unique_object_representations(WeirdAlignmentUnion), "Alignment causes padding"); +static_assert(!__has_unique_object_representations(WeirdAlignment[42]), "Also no arrays that have padding"); struct __attribute__((packed)) PackedNoPadding1 { short i; @@ -3151,41 +3134,41 @@ struct __attribute__((packed)) PackedNoPadding2 { int j; short i; }; -static_assert(has_unique_object_representations::value, "Packed structs have no padding"); -static_assert(has_unique_object_representations::value, "Packed structs have no padding"); - -static_assert(!has_unique_object_representations::value, "Functions are not unique"); -static_assert(!has_unique_object_representations::value, "Functions are not unique"); -static_assert(!has_unique_object_representations::value, "Functions are not unique"); -static_assert(!has_unique_object_representations::value, "Functions are not unique"); -static_assert(!has_unique_object_representations::value, "Functions are not unique"); -static_assert(!has_unique_object_representations::value, "Functions are not unique"); -static_assert(!has_unique_object_representations::value, "Functions are not unique"); -static_assert(!has_unique_object_representations::value, "Functions are not unique"); -static_assert(!has_unique_object_representations::value, "Functions are not unique"); -static_assert(!has_unique_object_representations::value, "Functions are not unique"); -static_assert(!has_unique_object_representations::value, "Functions are not unique"); -static_assert(!has_unique_object_representations::value, "Functions are not unique"); - -static_assert(!has_unique_object_representations::value, "Functions are not unique"); -static_assert(!has_unique_object_representations::value, "Functions are not unique"); -static_assert(!has_unique_object_representations::value, "Functions are not unique"); -static_assert(!has_unique_object_representations::value, "Functions are not unique"); -static_assert(!has_unique_object_representations::value, "Functions are not unique"); -static_assert(!has_unique_object_representations::value, "Functions are not unique"); -static_assert(!has_unique_object_representations::value, "Functions are not unique"); -static_assert(!has_unique_object_representations::value, "Functions are not unique"); -static_assert(!has_unique_object_representations::value, "Functions are not unique"); -static_assert(!has_unique_object_representations::value, "Functions are not unique"); -static_assert(!has_unique_object_representations::value, "Functions are not unique"); -static_assert(!has_unique_object_representations::value, "Functions are not unique"); +static_assert(__has_unique_object_representations(PackedNoPadding1), "Packed structs have no padding"); +static_assert(__has_unique_object_representations(PackedNoPadding2), "Packed structs have no padding"); + +static_assert(!__has_unique_object_representations(int(int)), "Functions are not unique"); +static_assert(!__has_unique_object_representations(int(int) const), "Functions are not unique"); +static_assert(!__has_unique_object_representations(int(int) volatile), "Functions are not unique"); +static_assert(!__has_unique_object_representations(int(int) const volatile), "Functions are not unique"); +static_assert(!__has_unique_object_representations(int(int) &), "Functions are not unique"); +static_assert(!__has_unique_object_representations(int(int) const &), "Functions are not unique"); +static_assert(!__has_unique_object_representations(int(int) volatile &), "Functions are not unique"); +static_assert(!__has_unique_object_representations(int(int) const volatile &), "Functions are not unique"); +static_assert(!__has_unique_object_representations(int(int) &&), "Functions are not unique"); +static_assert(!__has_unique_object_representations(int(int) const &&), "Functions are not unique"); +static_assert(!__has_unique_object_representations(int(int) volatile &&), "Functions are not unique"); +static_assert(!__has_unique_object_representations(int(int) const volatile &&), "Functions are not unique"); + +static_assert(!__has_unique_object_representations(int(int, ...)), "Functions are not unique"); +static_assert(!__has_unique_object_representations(int(int, ...) const), "Functions are not unique"); +static_assert(!__has_unique_object_representations(int(int, ...) volatile), "Functions are not unique"); +static_assert(!__has_unique_object_representations(int(int, ...) const volatile), "Functions are not unique"); +static_assert(!__has_unique_object_representations(int(int, ...) &), "Functions are not unique"); +static_assert(!__has_unique_object_representations(int(int, ...) const &), "Functions are not unique"); +static_assert(!__has_unique_object_representations(int(int, ...) volatile &), "Functions are not unique"); +static_assert(!__has_unique_object_representations(int(int, ...) const volatile &), "Functions are not unique"); +static_assert(!__has_unique_object_representations(int(int, ...) &&), "Functions are not unique"); +static_assert(!__has_unique_object_representations(int(int, ...) const &&), "Functions are not unique"); +static_assert(!__has_unique_object_representations(int(int, ...) volatile &&), "Functions are not unique"); +static_assert(!__has_unique_object_representations(int(int, ...) const volatile &&), "Functions are not unique"); void foo(){ static auto lambda = []() {}; - static_assert(!has_unique_object_representations::value, "Lambdas follow struct rules"); + static_assert(!__has_unique_object_representations(decltype(lambda)), "Lambdas follow struct rules"); int i; static auto lambda2 = [i]() {}; - static_assert(has_unique_object_representations::value, "Lambdas follow struct rules"); + static_assert(__has_unique_object_representations(decltype(lambda2)), "Lambdas follow struct rules"); } struct PaddedBitfield { @@ -3225,26 +3208,26 @@ struct UnnamedEmptyBitfieldSplit { short also_named; }; -static_assert(!has_unique_object_representations::value, "Bitfield padding"); -static_assert(has_unique_object_representations::value, "Bitfield padding"); -static_assert(!has_unique_object_representations::value, "Bitfield padding"); -static_assert(!has_unique_object_representations::value, "Bitfield padding"); -static_assert(!has_unique_object_representations::value, "Bitfield padding"); -static_assert(has_unique_object_representations::value, "Bitfield padding"); +static_assert(!__has_unique_object_representations(PaddedBitfield), "Bitfield padding"); +static_assert(__has_unique_object_representations(UnPaddedBitfield), "Bitfield padding"); +static_assert(!__has_unique_object_representations(AlignedPaddedBitfield), "Bitfield padding"); +static_assert(!__has_unique_object_representations(UnnamedBitfield), "Bitfield padding"); +static_assert(!__has_unique_object_representations(UnnamedBitfieldPacked), "Bitfield padding"); +static_assert(__has_unique_object_representations(UnnamedEmptyBitfield), "Bitfield padding"); static_assert(sizeof(UnnamedEmptyBitfieldSplit) != (sizeof(short) * 2), "Wrong size"); -static_assert(!has_unique_object_representations::value, "Bitfield padding"); +static_assert(!__has_unique_object_representations(UnnamedEmptyBitfieldSplit), "Bitfield padding"); struct BoolBitfield { bool b : 8; }; -static_assert(has_unique_object_representations::value, "Bitfield bool"); +static_assert(__has_unique_object_representations(BoolBitfield), "Bitfield bool"); struct BoolBitfield2 { bool b : 16; }; -static_assert(!has_unique_object_representations::value, "Bitfield bool"); +static_assert(!__has_unique_object_representations(BoolBitfield2), "Bitfield bool"); struct GreaterSizeBitfield { //expected-warning@+1 {{width of bit-field 'n'}} @@ -3252,13 +3235,13 @@ struct GreaterSizeBitfield { }; static_assert(sizeof(GreaterSizeBitfield) == 128, "Bitfield Size"); -static_assert(!has_unique_object_representations::value, "Bitfield padding"); +static_assert(!__has_unique_object_representations(GreaterSizeBitfield), "Bitfield padding"); struct StructWithRef { int &I; }; -static_assert(has_unique_object_representations::value, "References are still unique"); +static_assert(__has_unique_object_representations(StructWithRef), "References are still unique"); struct NotUniqueBecauseTailPadding { int &r; @@ -3268,12 +3251,12 @@ struct CanBeUniqueIfNoPadding : NotUniqueBecauseTailPadding { char b[7]; }; -static_assert(!has_unique_object_representations::value, +static_assert(!__has_unique_object_representations(NotUniqueBecauseTailPadding), "non trivial"); // Can be unique on Itanium, since the is child class' data is 'folded' into the // parent's tail padding. static_assert(sizeof(CanBeUniqueIfNoPadding) != 16 || - has_unique_object_representations::value, + __has_unique_object_representations(CanBeUniqueIfNoPadding), "inherit from std layout"); namespace ErrorType { @@ -3285,10 +3268,10 @@ namespace ErrorType { bool b = __has_unique_object_representations(T); }; -static_assert(!has_unique_object_representations<_BitInt(7)>::value, "BitInt:"); -static_assert(has_unique_object_representations<_BitInt(8)>::value, "BitInt:"); -static_assert(!has_unique_object_representations<_BitInt(127)>::value, "BitInt:"); -static_assert(has_unique_object_representations<_BitInt(128)>::value, "BitInt:"); +static_assert(!__has_unique_object_representations(_BitInt(7)), "BitInt:"); +static_assert(__has_unique_object_representations(_BitInt(8)), "BitInt:"); +static_assert(!__has_unique_object_representations(_BitInt(127)), "BitInt:"); +static_assert(__has_unique_object_representations(_BitInt(128)), "BitInt:"); namespace PR46209 { @@ -3303,8 +3286,8 @@ namespace PR46209 { Foo foo; }; - static_assert(!__is_trivially_assignable(Foo &, const Foo &), ""); - static_assert(!__is_trivially_assignable(Bar &, const Bar &), ""); + static_assert(!__is_trivially_assignable(Foo &, const Foo &)); + static_assert(!__is_trivially_assignable(Bar &, const Bar &)); // Foo2 has both a trivial assignment operator and a non-trivial one. struct Foo2 { @@ -3317,8 +3300,8 @@ namespace PR46209 { Foo2 foo; }; - static_assert(__is_trivially_assignable(Foo2 &, const Foo2 &), ""); - static_assert(__is_trivially_assignable(Bar2 &, const Bar2 &), ""); + static_assert(__is_trivially_assignable(Foo2 &, const Foo2 &)); + static_assert(__is_trivially_assignable(Bar2 &, const Bar2 &)); } namespace ConstClass { @@ -3328,7 +3311,7 @@ namespace ConstClass { struct B { const A a; }; - static_assert(!__is_trivially_assignable(B&, const B&), ""); + static_assert(!__is_trivially_assignable(B&, const B&)); } namespace type_trait_expr_numargs_overflow { @@ -3354,21 +3337,21 @@ void test() { (void) __is_constructible(int, T32768(int)); } namespace is_trivially_relocatable { -static_assert(!__is_trivially_relocatable(void), ""); -static_assert(__is_trivially_relocatable(int), ""); -static_assert(__is_trivially_relocatable(int[]), ""); +static_assert(!__is_trivially_relocatable(void)); +static_assert(__is_trivially_relocatable(int)); +static_assert(__is_trivially_relocatable(int[])); enum Enum {}; -static_assert(__is_trivially_relocatable(Enum), ""); -static_assert(__is_trivially_relocatable(Enum[]), ""); +static_assert(__is_trivially_relocatable(Enum)); +static_assert(__is_trivially_relocatable(Enum[])); union Union {int x;}; -static_assert(__is_trivially_relocatable(Union), ""); -static_assert(__is_trivially_relocatable(Union[]), ""); +static_assert(__is_trivially_relocatable(Union)); +static_assert(__is_trivially_relocatable(Union[])); struct Trivial {}; -static_assert(__is_trivially_relocatable(Trivial), ""); -static_assert(__is_trivially_relocatable(Trivial[]), ""); +static_assert(__is_trivially_relocatable(Trivial)); +static_assert(__is_trivially_relocatable(Trivial[])); struct Incomplete; // expected-note {{forward declaration of 'is_trivially_relocatable::Incomplete'}} bool unused = __is_trivially_relocatable(Incomplete); // expected-error {{incomplete type}} @@ -3376,67 +3359,67 @@ bool unused = __is_trivially_relocatable(Incomplete); // expected-error {{incomp struct NontrivialDtor { ~NontrivialDtor() {} }; -static_assert(!__is_trivially_relocatable(NontrivialDtor), ""); -static_assert(!__is_trivially_relocatable(NontrivialDtor[]), ""); +static_assert(!__is_trivially_relocatable(NontrivialDtor)); +static_assert(!__is_trivially_relocatable(NontrivialDtor[])); struct NontrivialCopyCtor { NontrivialCopyCtor(const NontrivialCopyCtor&) {} }; -static_assert(!__is_trivially_relocatable(NontrivialCopyCtor), ""); -static_assert(!__is_trivially_relocatable(NontrivialCopyCtor[]), ""); +static_assert(!__is_trivially_relocatable(NontrivialCopyCtor)); +static_assert(!__is_trivially_relocatable(NontrivialCopyCtor[])); struct NontrivialMoveCtor { NontrivialMoveCtor(NontrivialMoveCtor&&) {} }; -static_assert(!__is_trivially_relocatable(NontrivialMoveCtor), ""); -static_assert(!__is_trivially_relocatable(NontrivialMoveCtor[]), ""); +static_assert(!__is_trivially_relocatable(NontrivialMoveCtor)); +static_assert(!__is_trivially_relocatable(NontrivialMoveCtor[])); struct [[clang::trivial_abi]] TrivialAbiNontrivialDtor { ~TrivialAbiNontrivialDtor() {} }; -static_assert(__is_trivially_relocatable(TrivialAbiNontrivialDtor), ""); -static_assert(__is_trivially_relocatable(TrivialAbiNontrivialDtor[]), ""); +static_assert(__is_trivially_relocatable(TrivialAbiNontrivialDtor)); +static_assert(__is_trivially_relocatable(TrivialAbiNontrivialDtor[])); struct [[clang::trivial_abi]] TrivialAbiNontrivialCopyCtor { TrivialAbiNontrivialCopyCtor(const TrivialAbiNontrivialCopyCtor&) {} }; -static_assert(__is_trivially_relocatable(TrivialAbiNontrivialCopyCtor), ""); -static_assert(__is_trivially_relocatable(TrivialAbiNontrivialCopyCtor[]), ""); +static_assert(__is_trivially_relocatable(TrivialAbiNontrivialCopyCtor)); +static_assert(__is_trivially_relocatable(TrivialAbiNontrivialCopyCtor[])); // A more complete set of tests for the behavior of trivial_abi can be found in // clang/test/SemaCXX/attr-trivial-abi.cpp struct [[clang::trivial_abi]] TrivialAbiNontrivialMoveCtor { TrivialAbiNontrivialMoveCtor(TrivialAbiNontrivialMoveCtor&&) {} }; -static_assert(__is_trivially_relocatable(TrivialAbiNontrivialMoveCtor), ""); -static_assert(__is_trivially_relocatable(TrivialAbiNontrivialMoveCtor[]), ""); +static_assert(__is_trivially_relocatable(TrivialAbiNontrivialMoveCtor)); +static_assert(__is_trivially_relocatable(TrivialAbiNontrivialMoveCtor[])); } // namespace is_trivially_relocatable namespace is_trivially_equality_comparable { struct ForwardDeclared; // expected-note {{forward declaration of 'is_trivially_equality_comparable::ForwardDeclared'}} -static_assert(!__is_trivially_equality_comparable(ForwardDeclared), ""); // expected-error {{incomplete type 'ForwardDeclared' used in type trait expression}} +static_assert(!__is_trivially_equality_comparable(ForwardDeclared)); // expected-error {{incomplete type 'ForwardDeclared' used in type trait expression}} -static_assert(!__is_trivially_equality_comparable(void), ""); -static_assert(__is_trivially_equality_comparable(int), ""); -static_assert(!__is_trivially_equality_comparable(int[]), ""); -static_assert(!__is_trivially_equality_comparable(int[3]), ""); -static_assert(!__is_trivially_equality_comparable(float), ""); -static_assert(!__is_trivially_equality_comparable(double), ""); -static_assert(!__is_trivially_equality_comparable(long double), ""); +static_assert(!__is_trivially_equality_comparable(void)); +static_assert(__is_trivially_equality_comparable(int)); +static_assert(!__is_trivially_equality_comparable(int[])); +static_assert(!__is_trivially_equality_comparable(int[3])); +static_assert(!__is_trivially_equality_comparable(float)); +static_assert(!__is_trivially_equality_comparable(double)); +static_assert(!__is_trivially_equality_comparable(long double)); struct NonTriviallyEqualityComparableNoComparator { int i; int j; }; -static_assert(!__is_trivially_equality_comparable(NonTriviallyEqualityComparableNoComparator), ""); +static_assert(!__is_trivially_equality_comparable(NonTriviallyEqualityComparableNoComparator)); struct NonTriviallyEqualityComparableNonDefaultedComparator { int i; int j; bool operator==(const NonTriviallyEqualityComparableNonDefaultedComparator&); }; -static_assert(!__is_trivially_equality_comparable(NonTriviallyEqualityComparableNonDefaultedComparator), ""); +static_assert(!__is_trivially_equality_comparable(NonTriviallyEqualityComparableNonDefaultedComparator)); #if __cplusplus >= 202002L @@ -3490,7 +3473,7 @@ struct NotTriviallyEqualityComparableHasPadding { bool operator==(const NotTriviallyEqualityComparableHasPadding&) const = default; }; -static_assert(!__is_trivially_equality_comparable(NotTriviallyEqualityComparableHasPadding), ""); +static_assert(!__is_trivially_equality_comparable(NotTriviallyEqualityComparableHasPadding)); struct NotTriviallyEqualityComparableHasFloat { float i; @@ -3498,7 +3481,7 @@ struct NotTriviallyEqualityComparableHasFloat { bool operator==(const NotTriviallyEqualityComparableHasFloat&) const = default; }; -static_assert(!__is_trivially_equality_comparable(NotTriviallyEqualityComparableHasFloat), ""); +static_assert(!__is_trivially_equality_comparable(NotTriviallyEqualityComparableHasFloat)); struct NotTriviallyEqualityComparableHasTailPadding { int i; @@ -3506,14 +3489,14 @@ struct NotTriviallyEqualityComparableHasTailPadding { bool operator==(const NotTriviallyEqualityComparableHasTailPadding&) const = default; }; -static_assert(!__is_trivially_equality_comparable(NotTriviallyEqualityComparableHasTailPadding), ""); +static_assert(!__is_trivially_equality_comparable(NotTriviallyEqualityComparableHasTailPadding)); struct NotTriviallyEqualityComparableBase : NotTriviallyEqualityComparableHasTailPadding { char j; bool operator==(const NotTriviallyEqualityComparableBase&) const = default; }; -static_assert(!__is_trivially_equality_comparable(NotTriviallyEqualityComparableBase), ""); +static_assert(!__is_trivially_equality_comparable(NotTriviallyEqualityComparableBase)); class TriviallyEqualityComparablePaddedOutBase { int i; @@ -3522,14 +3505,14 @@ class TriviallyEqualityComparablePaddedOutBase { public: bool operator==(const TriviallyEqualityComparablePaddedOutBase&) const = default; }; -static_assert(!__is_trivially_equality_comparable(TriviallyEqualityComparablePaddedOutBase), ""); +static_assert(!__is_trivially_equality_comparable(TriviallyEqualityComparablePaddedOutBase)); struct TriviallyEqualityComparablePaddedOut : TriviallyEqualityComparablePaddedOutBase { char j[3]; bool operator==(const TriviallyEqualityComparablePaddedOut&) const = default; }; -static_assert(__is_trivially_equality_comparable(TriviallyEqualityComparablePaddedOut), ""); +static_assert(__is_trivially_equality_comparable(TriviallyEqualityComparablePaddedOut)); struct TriviallyEqualityComparable1 { char i; @@ -3652,7 +3635,7 @@ struct TriviallyEqualityComparable { friend bool operator==(const TriviallyEqualityComparable&, const TriviallyEqualityComparable&) = default; }; -static_assert(__is_trivially_equality_comparable(TriviallyEqualityComparable), ""); +static_assert(__is_trivially_equality_comparable(TriviallyEqualityComparable)); struct TriviallyEqualityComparableNonTriviallyCopyable { TriviallyEqualityComparableNonTriviallyCopyable(const TriviallyEqualityComparableNonTriviallyCopyable&); @@ -3668,7 +3651,7 @@ struct NotTriviallyEqualityComparableHasPadding { friend bool operator==(const NotTriviallyEqualityComparableHasPadding&, const NotTriviallyEqualityComparableHasPadding&) = default; }; -static_assert(!__is_trivially_equality_comparable(NotTriviallyEqualityComparableHasPadding), ""); +static_assert(!__is_trivially_equality_comparable(NotTriviallyEqualityComparableHasPadding)); struct NotTriviallyEqualityComparableHasFloat { float i; @@ -3676,7 +3659,7 @@ struct NotTriviallyEqualityComparableHasFloat { friend bool operator==(const NotTriviallyEqualityComparableHasFloat&, const NotTriviallyEqualityComparableHasFloat&) = default; }; -static_assert(!__is_trivially_equality_comparable(NotTriviallyEqualityComparableHasFloat), ""); +static_assert(!__is_trivially_equality_comparable(NotTriviallyEqualityComparableHasFloat)); struct NotTriviallyEqualityComparableHasTailPadding { int i; @@ -3684,14 +3667,14 @@ struct NotTriviallyEqualityComparableHasTailPadding { friend bool operator==(const NotTriviallyEqualityComparableHasTailPadding&, const NotTriviallyEqualityComparableHasTailPadding&) = default; }; -static_assert(!__is_trivially_equality_comparable(NotTriviallyEqualityComparableHasTailPadding), ""); +static_assert(!__is_trivially_equality_comparable(NotTriviallyEqualityComparableHasTailPadding)); struct NotTriviallyEqualityComparableBase : NotTriviallyEqualityComparableHasTailPadding { char j; friend bool operator==(const NotTriviallyEqualityComparableBase&, const NotTriviallyEqualityComparableBase&) = default; }; -static_assert(!__is_trivially_equality_comparable(NotTriviallyEqualityComparableBase), ""); +static_assert(!__is_trivially_equality_comparable(NotTriviallyEqualityComparableBase)); class TriviallyEqualityComparablePaddedOutBase { int i; @@ -3700,14 +3683,14 @@ class TriviallyEqualityComparablePaddedOutBase { public: friend bool operator==(const TriviallyEqualityComparablePaddedOutBase&, const TriviallyEqualityComparablePaddedOutBase&) = default; }; -static_assert(!__is_trivially_equality_comparable(TriviallyEqualityComparablePaddedOutBase), ""); +static_assert(!__is_trivially_equality_comparable(TriviallyEqualityComparablePaddedOutBase)); struct TriviallyEqualityComparablePaddedOut : TriviallyEqualityComparablePaddedOutBase { char j[3]; friend bool operator==(const TriviallyEqualityComparablePaddedOut&, const TriviallyEqualityComparablePaddedOut&) = default; }; -static_assert(__is_trivially_equality_comparable(TriviallyEqualityComparablePaddedOut), ""); +static_assert(__is_trivially_equality_comparable(TriviallyEqualityComparablePaddedOut)); struct TriviallyEqualityComparable1 { char i; @@ -3824,10 +3807,10 @@ union D { int x; }; -static_assert(__can_pass_in_regs(A), ""); -static_assert(__can_pass_in_regs(A), ""); -static_assert(!__can_pass_in_regs(B), ""); -static_assert(__can_pass_in_regs(D), ""); +static_assert(__can_pass_in_regs(A)); +static_assert(__can_pass_in_regs(A)); +static_assert(!__can_pass_in_regs(B)); +static_assert(__can_pass_in_regs(D)); void test_errors() { (void)__can_pass_in_regs(const A); // expected-error {{not an unqualified class type}} @@ -3844,379 +3827,379 @@ struct S {}; template using remove_const_t = __remove_const(T); void check_remove_const() { - static_assert(__is_same(remove_const_t, void), ""); - static_assert(__is_same(remove_const_t, void), ""); - static_assert(__is_same(remove_const_t, int), ""); - static_assert(__is_same(remove_const_t, int), ""); - static_assert(__is_same(remove_const_t, volatile int), ""); - static_assert(__is_same(remove_const_t, volatile int), ""); - static_assert(__is_same(remove_const_t, int *), ""); - static_assert(__is_same(remove_const_t, int *), ""); - static_assert(__is_same(remove_const_t, int const *), ""); - static_assert(__is_same(remove_const_t, int const *__restrict), ""); - static_assert(__is_same(remove_const_t, int &), ""); - static_assert(__is_same(remove_const_t, int const &), ""); - static_assert(__is_same(remove_const_t, int &&), ""); - static_assert(__is_same(remove_const_t, int const &&), ""); - static_assert(__is_same(remove_const_t, int()), ""); - static_assert(__is_same(remove_const_t, int (*)()), ""); - static_assert(__is_same(remove_const_t, int (&)()), ""); - - static_assert(__is_same(remove_const_t, S), ""); - static_assert(__is_same(remove_const_t, S), ""); - static_assert(__is_same(remove_const_t, volatile S), ""); - static_assert(__is_same(remove_const_t, S *__restrict), ""); - static_assert(__is_same(remove_const_t, volatile S), ""); - static_assert(__is_same(remove_const_t, S *volatile __restrict), ""); - static_assert(__is_same(remove_const_t, int S::*), ""); - static_assert(__is_same(remove_const_t, int(S::*)()), ""); + static_assert(__is_same(remove_const_t, void)); + static_assert(__is_same(remove_const_t, void)); + static_assert(__is_same(remove_const_t, int)); + static_assert(__is_same(remove_const_t, int)); + static_assert(__is_same(remove_const_t, volatile int)); + static_assert(__is_same(remove_const_t, volatile int)); + static_assert(__is_same(remove_const_t, int *)); + static_assert(__is_same(remove_const_t, int *)); + static_assert(__is_same(remove_const_t, int const *)); + static_assert(__is_same(remove_const_t, int const *__restrict)); + static_assert(__is_same(remove_const_t, int &)); + static_assert(__is_same(remove_const_t, int const &)); + static_assert(__is_same(remove_const_t, int &&)); + static_assert(__is_same(remove_const_t, int const &&)); + static_assert(__is_same(remove_const_t, int())); + static_assert(__is_same(remove_const_t, int (*)())); + static_assert(__is_same(remove_const_t, int (&)())); + + static_assert(__is_same(remove_const_t, S)); + static_assert(__is_same(remove_const_t, S)); + static_assert(__is_same(remove_const_t, volatile S)); + static_assert(__is_same(remove_const_t, S *__restrict)); + static_assert(__is_same(remove_const_t, volatile S)); + static_assert(__is_same(remove_const_t, S *volatile __restrict)); + static_assert(__is_same(remove_const_t, int S::*)); + static_assert(__is_same(remove_const_t, int(S::*)())); } template using remove_restrict_t = __remove_restrict(T); void check_remove_restrict() { - static_assert(__is_same(remove_restrict_t, void), ""); - static_assert(__is_same(remove_restrict_t, int), ""); - static_assert(__is_same(remove_restrict_t, const int), ""); - static_assert(__is_same(remove_restrict_t, volatile int), ""); - static_assert(__is_same(remove_restrict_t, int *), ""); - static_assert(__is_same(remove_restrict_t, int *const volatile), ""); - static_assert(__is_same(remove_restrict_t, int *), ""); - static_assert(__is_same(remove_restrict_t, int *), ""); - static_assert(__is_same(remove_restrict_t, int &), ""); - static_assert(__is_same(remove_restrict_t, int &), ""); - static_assert(__is_same(remove_restrict_t, int &&), ""); - static_assert(__is_same(remove_restrict_t, int &&), ""); - static_assert(__is_same(remove_restrict_t, int()), ""); - static_assert(__is_same(remove_restrict_t, int (*const volatile)()), ""); - static_assert(__is_same(remove_restrict_t, int (&)()), ""); - - static_assert(__is_same(remove_restrict_t, S), ""); - static_assert(__is_same(remove_restrict_t, const S), ""); - static_assert(__is_same(remove_restrict_t, volatile S), ""); - static_assert(__is_same(remove_restrict_t, S *), ""); - static_assert(__is_same(remove_restrict_t, S *const volatile), ""); - static_assert(__is_same(remove_restrict_t, int S::*), ""); - static_assert(__is_same(remove_restrict_t, int(S::*const volatile)()), ""); + static_assert(__is_same(remove_restrict_t, void)); + static_assert(__is_same(remove_restrict_t, int)); + static_assert(__is_same(remove_restrict_t, const int)); + static_assert(__is_same(remove_restrict_t, volatile int)); + static_assert(__is_same(remove_restrict_t, int *)); + static_assert(__is_same(remove_restrict_t, int *const volatile)); + static_assert(__is_same(remove_restrict_t, int *)); + static_assert(__is_same(remove_restrict_t, int *)); + static_assert(__is_same(remove_restrict_t, int &)); + static_assert(__is_same(remove_restrict_t, int &)); + static_assert(__is_same(remove_restrict_t, int &&)); + static_assert(__is_same(remove_restrict_t, int &&)); + static_assert(__is_same(remove_restrict_t, int())); + static_assert(__is_same(remove_restrict_t, int (*const volatile)())); + static_assert(__is_same(remove_restrict_t, int (&)())); + + static_assert(__is_same(remove_restrict_t, S)); + static_assert(__is_same(remove_restrict_t, const S)); + static_assert(__is_same(remove_restrict_t, volatile S)); + static_assert(__is_same(remove_restrict_t, S *)); + static_assert(__is_same(remove_restrict_t, S *const volatile)); + static_assert(__is_same(remove_restrict_t, int S::*)); + static_assert(__is_same(remove_restrict_t, int(S::*const volatile)())); } template using remove_volatile_t = __remove_volatile(T); void check_remove_volatile() { - static_assert(__is_same(remove_volatile_t, void), ""); - static_assert(__is_same(remove_volatile_t, void), ""); - static_assert(__is_same(remove_volatile_t, int), ""); - static_assert(__is_same(remove_volatile_t, const int), ""); - static_assert(__is_same(remove_volatile_t, int), ""); - static_assert(__is_same(remove_volatile_t, int *__restrict), ""); - static_assert(__is_same(remove_volatile_t, const int), ""); - static_assert(__is_same(remove_volatile_t, int *const __restrict), ""); - static_assert(__is_same(remove_volatile_t, int *), ""); - static_assert(__is_same(remove_volatile_t, int *), ""); - static_assert(__is_same(remove_volatile_t, int volatile *), ""); - static_assert(__is_same(remove_volatile_t, int &), ""); - static_assert(__is_same(remove_volatile_t, int volatile &), ""); - static_assert(__is_same(remove_volatile_t, int &&), ""); - static_assert(__is_same(remove_volatile_t, int volatile &&), ""); - static_assert(__is_same(remove_volatile_t, int()), ""); - static_assert(__is_same(remove_volatile_t, int (*)()), ""); - static_assert(__is_same(remove_volatile_t, int (&)()), ""); - - static_assert(__is_same(remove_volatile_t, S), ""); - static_assert(__is_same(remove_volatile_t, const S), ""); - static_assert(__is_same(remove_volatile_t, S), ""); - static_assert(__is_same(remove_volatile_t, const S), ""); - static_assert(__is_same(remove_volatile_t, int S::*), ""); - static_assert(__is_same(remove_volatile_t, int(S::*)()), ""); + static_assert(__is_same(remove_volatile_t, void)); + static_assert(__is_same(remove_volatile_t, void)); + static_assert(__is_same(remove_volatile_t, int)); + static_assert(__is_same(remove_volatile_t, const int)); + static_assert(__is_same(remove_volatile_t, int)); + static_assert(__is_same(remove_volatile_t, int *__restrict)); + static_assert(__is_same(remove_volatile_t, const int)); + static_assert(__is_same(remove_volatile_t, int *const __restrict)); + static_assert(__is_same(remove_volatile_t, int *)); + static_assert(__is_same(remove_volatile_t, int *)); + static_assert(__is_same(remove_volatile_t, int volatile *)); + static_assert(__is_same(remove_volatile_t, int &)); + static_assert(__is_same(remove_volatile_t, int volatile &)); + static_assert(__is_same(remove_volatile_t, int &&)); + static_assert(__is_same(remove_volatile_t, int volatile &&)); + static_assert(__is_same(remove_volatile_t, int())); + static_assert(__is_same(remove_volatile_t, int (*)())); + static_assert(__is_same(remove_volatile_t, int (&)())); + + static_assert(__is_same(remove_volatile_t, S)); + static_assert(__is_same(remove_volatile_t, const S)); + static_assert(__is_same(remove_volatile_t, S)); + static_assert(__is_same(remove_volatile_t, const S)); + static_assert(__is_same(remove_volatile_t, int S::*)); + static_assert(__is_same(remove_volatile_t, int(S::*)())); } template using remove_cv_t = __remove_cv(T); void check_remove_cv() { - static_assert(__is_same(remove_cv_t, void), ""); - static_assert(__is_same(remove_cv_t, void), ""); - static_assert(__is_same(remove_cv_t, int), ""); - static_assert(__is_same(remove_cv_t, int), ""); - static_assert(__is_same(remove_cv_t, int), ""); - static_assert(__is_same(remove_cv_t, int), ""); - static_assert(__is_same(remove_cv_t, int *), ""); - static_assert(__is_same(remove_cv_t, int *), ""); - static_assert(__is_same(remove_cv_t, int const *), ""); - static_assert(__is_same(remove_cv_t, int const *__restrict), ""); - static_assert(__is_same(remove_cv_t, int const *_Nonnull), ""); - static_assert(__is_same(remove_cv_t, int &), ""); - static_assert(__is_same(remove_cv_t, int const volatile &), ""); - static_assert(__is_same(remove_cv_t, int &&), ""); - static_assert(__is_same(remove_cv_t, int const volatile &&), ""); - static_assert(__is_same(remove_cv_t, int()), ""); - static_assert(__is_same(remove_cv_t, int (*)()), ""); - static_assert(__is_same(remove_cv_t, int (&)()), ""); - - static_assert(__is_same(remove_cv_t, S), ""); - static_assert(__is_same(remove_cv_t, S), ""); - static_assert(__is_same(remove_cv_t, S), ""); - static_assert(__is_same(remove_cv_t, S), ""); - static_assert(__is_same(remove_cv_t, int S::*), ""); - static_assert(__is_same(remove_cv_t, int(S::*)()), ""); + static_assert(__is_same(remove_cv_t, void)); + static_assert(__is_same(remove_cv_t, void)); + static_assert(__is_same(remove_cv_t, int)); + static_assert(__is_same(remove_cv_t, int)); + static_assert(__is_same(remove_cv_t, int)); + static_assert(__is_same(remove_cv_t, int)); + static_assert(__is_same(remove_cv_t, int *)); + static_assert(__is_same(remove_cv_t, int *)); + static_assert(__is_same(remove_cv_t, int const *)); + static_assert(__is_same(remove_cv_t, int const *__restrict)); + static_assert(__is_same(remove_cv_t, int const *_Nonnull)); + static_assert(__is_same(remove_cv_t, int &)); + static_assert(__is_same(remove_cv_t, int const volatile &)); + static_assert(__is_same(remove_cv_t, int &&)); + static_assert(__is_same(remove_cv_t, int const volatile &&)); + static_assert(__is_same(remove_cv_t, int())); + static_assert(__is_same(remove_cv_t, int (*)())); + static_assert(__is_same(remove_cv_t, int (&)())); + + static_assert(__is_same(remove_cv_t, S)); + static_assert(__is_same(remove_cv_t, S)); + static_assert(__is_same(remove_cv_t, S)); + static_assert(__is_same(remove_cv_t, S)); + static_assert(__is_same(remove_cv_t, int S::*)); + static_assert(__is_same(remove_cv_t, int(S::*)())); } template using add_pointer_t = __add_pointer(T); void add_pointer() { - static_assert(__is_same(add_pointer_t, void *), ""); - static_assert(__is_same(add_pointer_t, const void *), ""); - static_assert(__is_same(add_pointer_t, volatile void *), ""); - static_assert(__is_same(add_pointer_t, const volatile void *), ""); - static_assert(__is_same(add_pointer_t, int *), ""); - static_assert(__is_same(add_pointer_t, const int *), ""); - static_assert(__is_same(add_pointer_t, volatile int *), ""); - static_assert(__is_same(add_pointer_t, const volatile int *), ""); - static_assert(__is_same(add_pointer_t, int **), ""); - static_assert(__is_same(add_pointer_t, int *), ""); - static_assert(__is_same(add_pointer_t, int *), ""); - static_assert(__is_same(add_pointer_t, int (*)()), ""); - static_assert(__is_same(add_pointer_t, int (**)()), ""); - static_assert(__is_same(add_pointer_t, int (*)()), ""); - - static_assert(__is_same(add_pointer_t, S *), ""); - static_assert(__is_same(add_pointer_t, const S *), ""); - static_assert(__is_same(add_pointer_t, volatile S *), ""); - static_assert(__is_same(add_pointer_t, const volatile S *), ""); - static_assert(__is_same(add_pointer_t, int S::**), ""); - static_assert(__is_same(add_pointer_t, int(S::**)()), ""); - - static_assert(__is_same(add_pointer_t, int __attribute__((address_space(1))) *), ""); - static_assert(__is_same(add_pointer_t, S __attribute__((address_space(2))) *), ""); + static_assert(__is_same(add_pointer_t, void *)); + static_assert(__is_same(add_pointer_t, const void *)); + static_assert(__is_same(add_pointer_t, volatile void *)); + static_assert(__is_same(add_pointer_t, const volatile void *)); + static_assert(__is_same(add_pointer_t, int *)); + static_assert(__is_same(add_pointer_t, const int *)); + static_assert(__is_same(add_pointer_t, volatile int *)); + static_assert(__is_same(add_pointer_t, const volatile int *)); + static_assert(__is_same(add_pointer_t, int **)); + static_assert(__is_same(add_pointer_t, int *)); + static_assert(__is_same(add_pointer_t, int *)); + static_assert(__is_same(add_pointer_t, int (*)())); + static_assert(__is_same(add_pointer_t, int (**)())); + static_assert(__is_same(add_pointer_t, int (*)())); + + static_assert(__is_same(add_pointer_t, S *)); + static_assert(__is_same(add_pointer_t, const S *)); + static_assert(__is_same(add_pointer_t, volatile S *)); + static_assert(__is_same(add_pointer_t, const volatile S *)); + static_assert(__is_same(add_pointer_t, int S::**)); + static_assert(__is_same(add_pointer_t, int(S::**)())); + + static_assert(__is_same(add_pointer_t, int __attribute__((address_space(1))) *)); + static_assert(__is_same(add_pointer_t, S __attribute__((address_space(2))) *)); } template using remove_pointer_t = __remove_pointer(T); void remove_pointer() { - static_assert(__is_same(remove_pointer_t, void), ""); - static_assert(__is_same(remove_pointer_t, const void), ""); - static_assert(__is_same(remove_pointer_t, volatile void), ""); - static_assert(__is_same(remove_pointer_t, const volatile void), ""); - static_assert(__is_same(remove_pointer_t, int), ""); - static_assert(__is_same(remove_pointer_t, const int), ""); - static_assert(__is_same(remove_pointer_t, volatile int), ""); - static_assert(__is_same(remove_pointer_t, const volatile int), ""); - static_assert(__is_same(remove_pointer_t, int), ""); - static_assert(__is_same(remove_pointer_t, const int), ""); - static_assert(__is_same(remove_pointer_t, volatile int), ""); - static_assert(__is_same(remove_pointer_t, const volatile int), ""); - static_assert(__is_same(remove_pointer_t, int), ""); - static_assert(__is_same(remove_pointer_t, int), ""); - static_assert(__is_same(remove_pointer_t, int), ""); - static_assert(__is_same(remove_pointer_t, int &), ""); - static_assert(__is_same(remove_pointer_t, int &&), ""); - static_assert(__is_same(remove_pointer_t, int()), ""); - static_assert(__is_same(remove_pointer_t, int()), ""); - static_assert(__is_same(remove_pointer_t, int (&)()), ""); - - static_assert(__is_same(remove_pointer_t, S), ""); - static_assert(__is_same(remove_pointer_t, const S), ""); - static_assert(__is_same(remove_pointer_t, volatile S), ""); - static_assert(__is_same(remove_pointer_t, const volatile S), ""); - static_assert(__is_same(remove_pointer_t, int S::*), ""); - static_assert(__is_same(remove_pointer_t, int(S::*)()), ""); - - static_assert(__is_same(remove_pointer_t, int __attribute__((address_space(1)))), ""); - static_assert(__is_same(remove_pointer_t, S __attribute__((address_space(2)))), ""); - - static_assert(__is_same(remove_pointer_t, int (^)(char)), ""); + static_assert(__is_same(remove_pointer_t, void)); + static_assert(__is_same(remove_pointer_t, const void)); + static_assert(__is_same(remove_pointer_t, volatile void)); + static_assert(__is_same(remove_pointer_t, const volatile void)); + static_assert(__is_same(remove_pointer_t, int)); + static_assert(__is_same(remove_pointer_t, const int)); + static_assert(__is_same(remove_pointer_t, volatile int)); + static_assert(__is_same(remove_pointer_t, const volatile int)); + static_assert(__is_same(remove_pointer_t, int)); + static_assert(__is_same(remove_pointer_t, const int)); + static_assert(__is_same(remove_pointer_t, volatile int)); + static_assert(__is_same(remove_pointer_t, const volatile int)); + static_assert(__is_same(remove_pointer_t, int)); + static_assert(__is_same(remove_pointer_t, int)); + static_assert(__is_same(remove_pointer_t, int)); + static_assert(__is_same(remove_pointer_t, int &)); + static_assert(__is_same(remove_pointer_t, int &&)); + static_assert(__is_same(remove_pointer_t, int())); + static_assert(__is_same(remove_pointer_t, int())); + static_assert(__is_same(remove_pointer_t, int (&)())); + + static_assert(__is_same(remove_pointer_t, S)); + static_assert(__is_same(remove_pointer_t, const S)); + static_assert(__is_same(remove_pointer_t, volatile S)); + static_assert(__is_same(remove_pointer_t, const volatile S)); + static_assert(__is_same(remove_pointer_t, int S::*)); + static_assert(__is_same(remove_pointer_t, int(S::*)())); + + static_assert(__is_same(remove_pointer_t, int __attribute__((address_space(1))))); + static_assert(__is_same(remove_pointer_t, S __attribute__((address_space(2))))); + + static_assert(__is_same(remove_pointer_t, int (^)(char))); } template using add_lvalue_reference_t = __add_lvalue_reference(T); void add_lvalue_reference() { - static_assert(__is_same(add_lvalue_reference_t, void), ""); - static_assert(__is_same(add_lvalue_reference_t, const void), ""); - static_assert(__is_same(add_lvalue_reference_t, volatile void), ""); - static_assert(__is_same(add_lvalue_reference_t, const volatile void), ""); - static_assert(__is_same(add_lvalue_reference_t, int &), ""); - static_assert(__is_same(add_lvalue_reference_t, const int &), ""); - static_assert(__is_same(add_lvalue_reference_t, volatile int &), ""); - static_assert(__is_same(add_lvalue_reference_t, const volatile int &), ""); - static_assert(__is_same(add_lvalue_reference_t, int *&), ""); - static_assert(__is_same(add_lvalue_reference_t, int &), ""); - static_assert(__is_same(add_lvalue_reference_t, int &), ""); // reference collapsing - static_assert(__is_same(add_lvalue_reference_t, int (&)()), ""); - static_assert(__is_same(add_lvalue_reference_t, int (*&)()), ""); - static_assert(__is_same(add_lvalue_reference_t, int (&)()), ""); - - static_assert(__is_same(add_lvalue_reference_t, S &), ""); - static_assert(__is_same(add_lvalue_reference_t, const S &), ""); - static_assert(__is_same(add_lvalue_reference_t, volatile S &), ""); - static_assert(__is_same(add_lvalue_reference_t, const volatile S &), ""); - static_assert(__is_same(add_lvalue_reference_t, int S::*&), ""); - static_assert(__is_same(add_lvalue_reference_t, int(S::*&)()), ""); + static_assert(__is_same(add_lvalue_reference_t, void)); + static_assert(__is_same(add_lvalue_reference_t, const void)); + static_assert(__is_same(add_lvalue_reference_t, volatile void)); + static_assert(__is_same(add_lvalue_reference_t, const volatile void)); + static_assert(__is_same(add_lvalue_reference_t, int &)); + static_assert(__is_same(add_lvalue_reference_t, const int &)); + static_assert(__is_same(add_lvalue_reference_t, volatile int &)); + static_assert(__is_same(add_lvalue_reference_t, const volatile int &)); + static_assert(__is_same(add_lvalue_reference_t, int *&)); + static_assert(__is_same(add_lvalue_reference_t, int &)); + static_assert(__is_same(add_lvalue_reference_t, int &)); // reference collapsing + static_assert(__is_same(add_lvalue_reference_t, int (&)())); + static_assert(__is_same(add_lvalue_reference_t, int (*&)())); + static_assert(__is_same(add_lvalue_reference_t, int (&)())); + + static_assert(__is_same(add_lvalue_reference_t, S &)); + static_assert(__is_same(add_lvalue_reference_t, const S &)); + static_assert(__is_same(add_lvalue_reference_t, volatile S &)); + static_assert(__is_same(add_lvalue_reference_t, const volatile S &)); + static_assert(__is_same(add_lvalue_reference_t, int S::*&)); + static_assert(__is_same(add_lvalue_reference_t, int(S::*&)())); } template using add_rvalue_reference_t = __add_rvalue_reference(T); void add_rvalue_reference() { - static_assert(__is_same(add_rvalue_reference_t, void), ""); - static_assert(__is_same(add_rvalue_reference_t, const void), ""); - static_assert(__is_same(add_rvalue_reference_t, volatile void), ""); - static_assert(__is_same(add_rvalue_reference_t, const volatile void), ""); - static_assert(__is_same(add_rvalue_reference_t, int &&), ""); - static_assert(__is_same(add_rvalue_reference_t, const int &&), ""); - static_assert(__is_same(add_rvalue_reference_t, volatile int &&), ""); - static_assert(__is_same(add_rvalue_reference_t, const volatile int &&), ""); - static_assert(__is_same(add_rvalue_reference_t, int *&&), ""); - static_assert(__is_same(add_rvalue_reference_t, int &), ""); // reference collapsing - static_assert(__is_same(add_rvalue_reference_t, int &&), ""); - static_assert(__is_same(add_rvalue_reference_t, int(&&)()), ""); - static_assert(__is_same(add_rvalue_reference_t, int (*&&)()), ""); - static_assert(__is_same(add_rvalue_reference_t, int (&)()), ""); // reference collapsing - - static_assert(__is_same(add_rvalue_reference_t, S &&), ""); - static_assert(__is_same(add_rvalue_reference_t, const S &&), ""); - static_assert(__is_same(add_rvalue_reference_t, volatile S &&), ""); - static_assert(__is_same(add_rvalue_reference_t, const volatile S &&), ""); - static_assert(__is_same(add_rvalue_reference_t, int S::*&&), ""); - static_assert(__is_same(add_rvalue_reference_t, int(S::* &&)()), ""); + static_assert(__is_same(add_rvalue_reference_t, void)); + static_assert(__is_same(add_rvalue_reference_t, const void)); + static_assert(__is_same(add_rvalue_reference_t, volatile void)); + static_assert(__is_same(add_rvalue_reference_t, const volatile void)); + static_assert(__is_same(add_rvalue_reference_t, int &&)); + static_assert(__is_same(add_rvalue_reference_t, const int &&)); + static_assert(__is_same(add_rvalue_reference_t, volatile int &&)); + static_assert(__is_same(add_rvalue_reference_t, const volatile int &&)); + static_assert(__is_same(add_rvalue_reference_t, int *&&)); + static_assert(__is_same(add_rvalue_reference_t, int &)); // reference collapsing + static_assert(__is_same(add_rvalue_reference_t, int &&)); + static_assert(__is_same(add_rvalue_reference_t, int(&&)())); + static_assert(__is_same(add_rvalue_reference_t, int (*&&)())); + static_assert(__is_same(add_rvalue_reference_t, int (&)())); // reference collapsing + + static_assert(__is_same(add_rvalue_reference_t, S &&)); + static_assert(__is_same(add_rvalue_reference_t, const S &&)); + static_assert(__is_same(add_rvalue_reference_t, volatile S &&)); + static_assert(__is_same(add_rvalue_reference_t, const volatile S &&)); + static_assert(__is_same(add_rvalue_reference_t, int S::*&&)); + static_assert(__is_same(add_rvalue_reference_t, int(S::* &&)())); } template using remove_reference_t = __remove_reference_t(T); void check_remove_reference() { - static_assert(__is_same(remove_reference_t, void), ""); - static_assert(__is_same(remove_reference_t, const volatile void), ""); - static_assert(__is_same(remove_reference_t, int), ""); - static_assert(__is_same(remove_reference_t, const int), ""); - static_assert(__is_same(remove_reference_t, volatile int), ""); - static_assert(__is_same(remove_reference_t, const volatile int), ""); - static_assert(__is_same(remove_reference_t, int *), ""); - static_assert(__is_same(remove_reference_t, int *const volatile), ""); - static_assert(__is_same(remove_reference_t, int const *const volatile), ""); - static_assert(__is_same(remove_reference_t, int), ""); - static_assert(__is_same(remove_reference_t, int const volatile), ""); - static_assert(__is_same(remove_reference_t, int), ""); - static_assert(__is_same(remove_reference_t, int const volatile), ""); - static_assert(__is_same(remove_reference_t, int()), ""); - static_assert(__is_same(remove_reference_t, int (*const volatile)()), ""); - static_assert(__is_same(remove_reference_t, int()), ""); - - static_assert(__is_same(remove_reference_t, S), ""); - static_assert(__is_same(remove_reference_t, S), ""); - static_assert(__is_same(remove_reference_t, S), ""); - static_assert(__is_same(remove_reference_t, const S), ""); - static_assert(__is_same(remove_reference_t, const S), ""); - static_assert(__is_same(remove_reference_t, const S), ""); - static_assert(__is_same(remove_reference_t, volatile S), ""); - static_assert(__is_same(remove_reference_t, volatile S), ""); - static_assert(__is_same(remove_reference_t, volatile S), ""); - static_assert(__is_same(remove_reference_t, const volatile S), ""); - static_assert(__is_same(remove_reference_t, const volatile S), ""); - static_assert(__is_same(remove_reference_t, const volatile S), ""); - static_assert(__is_same(remove_reference_t, int S::*const volatile), ""); - static_assert(__is_same(remove_reference_t, int(S::*const volatile)()), ""); - static_assert(__is_same(remove_reference_t, int(S::*const volatile)() &), ""); + static_assert(__is_same(remove_reference_t, void)); + static_assert(__is_same(remove_reference_t, const volatile void)); + static_assert(__is_same(remove_reference_t, int)); + static_assert(__is_same(remove_reference_t, const int)); + static_assert(__is_same(remove_reference_t, volatile int)); + static_assert(__is_same(remove_reference_t, const volatile int)); + static_assert(__is_same(remove_reference_t, int *)); + static_assert(__is_same(remove_reference_t, int *const volatile)); + static_assert(__is_same(remove_reference_t, int const *const volatile)); + static_assert(__is_same(remove_reference_t, int)); + static_assert(__is_same(remove_reference_t, int const volatile)); + static_assert(__is_same(remove_reference_t, int)); + static_assert(__is_same(remove_reference_t, int const volatile)); + static_assert(__is_same(remove_reference_t, int())); + static_assert(__is_same(remove_reference_t, int (*const volatile)())); + static_assert(__is_same(remove_reference_t, int())); + + static_assert(__is_same(remove_reference_t, S)); + static_assert(__is_same(remove_reference_t, S)); + static_assert(__is_same(remove_reference_t, S)); + static_assert(__is_same(remove_reference_t, const S)); + static_assert(__is_same(remove_reference_t, const S)); + static_assert(__is_same(remove_reference_t, const S)); + static_assert(__is_same(remove_reference_t, volatile S)); + static_assert(__is_same(remove_reference_t, volatile S)); + static_assert(__is_same(remove_reference_t, volatile S)); + static_assert(__is_same(remove_reference_t, const volatile S)); + static_assert(__is_same(remove_reference_t, const volatile S)); + static_assert(__is_same(remove_reference_t, const volatile S)); + static_assert(__is_same(remove_reference_t, int S::*const volatile)); + static_assert(__is_same(remove_reference_t, int(S::*const volatile)())); + static_assert(__is_same(remove_reference_t, int(S::*const volatile)() &)); } template using remove_cvref_t = __remove_cvref(T); void check_remove_cvref() { - static_assert(__is_same(remove_cvref_t, void), ""); - static_assert(__is_same(remove_cvref_t, void), ""); - static_assert(__is_same(remove_cvref_t, int), ""); - static_assert(__is_same(remove_cvref_t, int), ""); - static_assert(__is_same(remove_cvref_t, int), ""); - static_assert(__is_same(remove_cvref_t, int), ""); - static_assert(__is_same(remove_cvref_t, int *), ""); - static_assert(__is_same(remove_cvref_t, int *), ""); - static_assert(__is_same(remove_cvref_t, int const *), ""); - static_assert(__is_same(remove_cvref_t, int const *__restrict), ""); - static_assert(__is_same(remove_cvref_t, int const *_Nonnull), ""); - static_assert(__is_same(remove_cvref_t, int), ""); - static_assert(__is_same(remove_cvref_t, int), ""); - static_assert(__is_same(remove_cvref_t, int), ""); - static_assert(__is_same(remove_cvref_t, int), ""); - static_assert(__is_same(remove_cvref_t, int()), ""); - static_assert(__is_same(remove_cvref_t, int (*)()), ""); - static_assert(__is_same(remove_cvref_t, int()), ""); - - static_assert(__is_same(remove_cvref_t, S), ""); - static_assert(__is_same(remove_cvref_t, S), ""); - static_assert(__is_same(remove_cvref_t, S), ""); - static_assert(__is_same(remove_cvref_t, S), ""); - static_assert(__is_same(remove_cvref_t, S), ""); - static_assert(__is_same(remove_cvref_t, S), ""); - static_assert(__is_same(remove_cvref_t, S), ""); - static_assert(__is_same(remove_cvref_t, S), ""); - static_assert(__is_same(remove_cvref_t, S), ""); - static_assert(__is_same(remove_cvref_t, S), ""); - static_assert(__is_same(remove_cvref_t, S), ""); - static_assert(__is_same(remove_cvref_t, S), ""); - static_assert(__is_same(remove_cvref_t, int S::*), ""); - static_assert(__is_same(remove_cvref_t, int(S::*)()), ""); - static_assert(__is_same(remove_cvref_t, int(S::*)() &), ""); - static_assert(__is_same(remove_cvref_t, int(S::*)() &&), ""); + static_assert(__is_same(remove_cvref_t, void)); + static_assert(__is_same(remove_cvref_t, void)); + static_assert(__is_same(remove_cvref_t, int)); + static_assert(__is_same(remove_cvref_t, int)); + static_assert(__is_same(remove_cvref_t, int)); + static_assert(__is_same(remove_cvref_t, int)); + static_assert(__is_same(remove_cvref_t, int *)); + static_assert(__is_same(remove_cvref_t, int *)); + static_assert(__is_same(remove_cvref_t, int const *)); + static_assert(__is_same(remove_cvref_t, int const *__restrict)); + static_assert(__is_same(remove_cvref_t, int const *_Nonnull)); + static_assert(__is_same(remove_cvref_t, int)); + static_assert(__is_same(remove_cvref_t, int)); + static_assert(__is_same(remove_cvref_t, int)); + static_assert(__is_same(remove_cvref_t, int)); + static_assert(__is_same(remove_cvref_t, int())); + static_assert(__is_same(remove_cvref_t, int (*)())); + static_assert(__is_same(remove_cvref_t, int())); + + static_assert(__is_same(remove_cvref_t, S)); + static_assert(__is_same(remove_cvref_t, S)); + static_assert(__is_same(remove_cvref_t, S)); + static_assert(__is_same(remove_cvref_t, S)); + static_assert(__is_same(remove_cvref_t, S)); + static_assert(__is_same(remove_cvref_t, S)); + static_assert(__is_same(remove_cvref_t, S)); + static_assert(__is_same(remove_cvref_t, S)); + static_assert(__is_same(remove_cvref_t, S)); + static_assert(__is_same(remove_cvref_t, S)); + static_assert(__is_same(remove_cvref_t, S)); + static_assert(__is_same(remove_cvref_t, S)); + static_assert(__is_same(remove_cvref_t, int S::*)); + static_assert(__is_same(remove_cvref_t, int(S::*)())); + static_assert(__is_same(remove_cvref_t, int(S::*)() &)); + static_assert(__is_same(remove_cvref_t, int(S::*)() &&)); } template using decay_t = __decay(T); void check_decay() { - static_assert(__is_same(decay_t, void), ""); - static_assert(__is_same(decay_t, void), ""); - static_assert(__is_same(decay_t, int), ""); - static_assert(__is_same(decay_t, int), ""); - static_assert(__is_same(decay_t, int), ""); - static_assert(__is_same(decay_t, int), ""); - static_assert(__is_same(decay_t, int *), ""); - static_assert(__is_same(decay_t, int *), ""); - static_assert(__is_same(decay_t, int *), ""); - static_assert(__is_same(decay_t, int const *), ""); - static_assert(__is_same(decay_t, int const *), ""); - static_assert(__is_same(decay_t, int), ""); - static_assert(__is_same(decay_t, int), ""); - static_assert(__is_same(decay_t, int), ""); - static_assert(__is_same(decay_t, int), ""); - static_assert(__is_same(decay_t, int (*)()), ""); - static_assert(__is_same(decay_t, int (*)()), ""); - static_assert(__is_same(decay_t, int (*)()), ""); - static_assert(__is_same(decay_t, int (*)()), ""); - static_assert(__is_same(decay_t, int (*)()), ""); - static_assert(__is_same(decay_t, int (*)()), ""); - static_assert(__is_same(decay_t, int *), ""); - static_assert(__is_same(decay_t, int *), ""); - - static_assert(__is_same(decay_t, S), ""); - static_assert(__is_same(decay_t, S), ""); - static_assert(__is_same(decay_t, S), ""); - static_assert(__is_same(decay_t, S), ""); - static_assert(__is_same(decay_t, S), ""); - static_assert(__is_same(decay_t, S), ""); - static_assert(__is_same(decay_t, S), ""); - static_assert(__is_same(decay_t, S), ""); - static_assert(__is_same(decay_t, S), ""); - static_assert(__is_same(decay_t, S), ""); - static_assert(__is_same(decay_t, S), ""); - static_assert(__is_same(decay_t, S), ""); - static_assert(__is_same(decay_t, int S::*), ""); - static_assert(__is_same(decay_t, int(S::*)()), ""); - static_assert(__is_same(decay_t, int S::*), ""); - static_assert(__is_same(decay_t, int(S::*)()), ""); - static_assert(__is_same(decay_t, int S::*), ""); + static_assert(__is_same(decay_t, void)); + static_assert(__is_same(decay_t, void)); + static_assert(__is_same(decay_t, int)); + static_assert(__is_same(decay_t, int)); + static_assert(__is_same(decay_t, int)); + static_assert(__is_same(decay_t, int)); + static_assert(__is_same(decay_t, int *)); + static_assert(__is_same(decay_t, int *)); + static_assert(__is_same(decay_t, int *)); + static_assert(__is_same(decay_t, int const *)); + static_assert(__is_same(decay_t, int const *)); + static_assert(__is_same(decay_t, int)); + static_assert(__is_same(decay_t, int)); + static_assert(__is_same(decay_t, int)); + static_assert(__is_same(decay_t, int)); + static_assert(__is_same(decay_t, int (*)())); + static_assert(__is_same(decay_t, int (*)())); + static_assert(__is_same(decay_t, int (*)())); + static_assert(__is_same(decay_t, int (*)())); + static_assert(__is_same(decay_t, int (*)())); + static_assert(__is_same(decay_t, int (*)())); + static_assert(__is_same(decay_t, int *)); + static_assert(__is_same(decay_t, int *)); + + static_assert(__is_same(decay_t, S)); + static_assert(__is_same(decay_t, S)); + static_assert(__is_same(decay_t, S)); + static_assert(__is_same(decay_t, S)); + static_assert(__is_same(decay_t, S)); + static_assert(__is_same(decay_t, S)); + static_assert(__is_same(decay_t, S)); + static_assert(__is_same(decay_t, S)); + static_assert(__is_same(decay_t, S)); + static_assert(__is_same(decay_t, S)); + static_assert(__is_same(decay_t, S)); + static_assert(__is_same(decay_t, S)); + static_assert(__is_same(decay_t, int S::*)); + static_assert(__is_same(decay_t, int(S::*)())); + static_assert(__is_same(decay_t, int S::*)); + static_assert(__is_same(decay_t, int(S::*)())); + static_assert(__is_same(decay_t, int S::*)); } template struct CheckAbominableFunction {}; template struct CheckAbominableFunction { static void checks() { - static_assert(__is_same(add_lvalue_reference_t, M), ""); - static_assert(__is_same(add_pointer_t, M), ""); - static_assert(__is_same(add_rvalue_reference_t, M), ""); - static_assert(__is_same(decay_t, M), ""); - static_assert(__is_same(remove_const_t, M), ""); - static_assert(__is_same(remove_volatile_t, M), ""); - static_assert(__is_same(remove_cv_t, M), ""); - static_assert(__is_same(remove_cvref_t, M), ""); - static_assert(__is_same(remove_pointer_t, M), ""); - static_assert(__is_same(remove_reference_t, M), ""); - - static_assert(!__is_referenceable(M), ""); + static_assert(__is_same(add_lvalue_reference_t, M)); + static_assert(__is_same(add_pointer_t, M)); + static_assert(__is_same(add_rvalue_reference_t, M)); + static_assert(__is_same(decay_t, M)); + static_assert(__is_same(remove_const_t, M)); + static_assert(__is_same(remove_volatile_t, M)); + static_assert(__is_same(remove_cv_t, M)); + static_assert(__is_same(remove_cvref_t, M)); + static_assert(__is_same(remove_pointer_t, M)); + static_assert(__is_same(remove_reference_t, M)); + + static_assert(!__is_referenceable(M)); } }; @@ -4237,10 +4220,10 @@ void check_abominable_function() { template using make_signed_t = __make_signed(T); template void check_make_signed() { - static_assert(__is_same(make_signed_t, Expected), ""); - static_assert(__is_same(make_signed_t, const Expected), ""); - static_assert(__is_same(make_signed_t, volatile Expected), ""); - static_assert(__is_same(make_signed_t, const volatile Expected), ""); + static_assert(__is_same(make_signed_t, Expected)); + static_assert(__is_same(make_signed_t, const Expected)); + static_assert(__is_same(make_signed_t, volatile Expected)); + static_assert(__is_same(make_signed_t, const volatile Expected)); } #if defined(__ILP32__) || defined(__LLP64__) @@ -4350,10 +4333,10 @@ using make_unsigned_t = __make_unsigned(T); template void check_make_unsigned() { - static_assert(__is_same(make_unsigned_t, Expected), ""); - static_assert(__is_same(make_unsigned_t, const Expected), ""); - static_assert(__is_same(make_unsigned_t, volatile Expected), ""); - static_assert(__is_same(make_unsigned_t, const volatile Expected), ""); + static_assert(__is_same(make_unsigned_t, Expected)); + static_assert(__is_same(make_unsigned_t, const Expected)); + static_assert(__is_same(make_unsigned_t, volatile Expected)); + static_assert(__is_same(make_unsigned_t, const volatile Expected)); } void make_unsigned() { @@ -4436,74 +4419,74 @@ void make_unsigned() { template using remove_extent_t = __remove_extent(T); void remove_extent() { - static_assert(__is_same(remove_extent_t, void), ""); - static_assert(__is_same(remove_extent_t, int), ""); - static_assert(__is_same(remove_extent_t, int), ""); - static_assert(__is_same(remove_extent_t, int), ""); - static_assert(__is_same(remove_extent_t, int[2]), ""); - static_assert(__is_same(remove_extent_t, int[2]), ""); - static_assert(__is_same(remove_extent_t, const int), ""); - static_assert(__is_same(remove_extent_t, const int), ""); - static_assert(__is_same(remove_extent_t, const int[2]), ""); - static_assert(__is_same(remove_extent_t, const int[2]), ""); - static_assert(__is_same(remove_extent_t, volatile int), ""); - static_assert(__is_same(remove_extent_t, volatile int), ""); - static_assert(__is_same(remove_extent_t, volatile int[2]), ""); - static_assert(__is_same(remove_extent_t, volatile int[2]), ""); - static_assert(__is_same(remove_extent_t, const volatile int), ""); - static_assert(__is_same(remove_extent_t, const volatile int), ""); - static_assert(__is_same(remove_extent_t, const volatile int[2]), ""); - static_assert(__is_same(remove_extent_t, const volatile int[2]), ""); - static_assert(__is_same(remove_extent_t, int *), ""); - static_assert(__is_same(remove_extent_t, int &), ""); - static_assert(__is_same(remove_extent_t, int &&), ""); - static_assert(__is_same(remove_extent_t, int()), ""); - static_assert(__is_same(remove_extent_t, int (*)()), ""); - static_assert(__is_same(remove_extent_t, int (&)()), ""); - - static_assert(__is_same(remove_extent_t, S), ""); - static_assert(__is_same(remove_extent_t, int S::*), ""); - static_assert(__is_same(remove_extent_t, int(S::*)()), ""); + static_assert(__is_same(remove_extent_t, void)); + static_assert(__is_same(remove_extent_t, int)); + static_assert(__is_same(remove_extent_t, int)); + static_assert(__is_same(remove_extent_t, int)); + static_assert(__is_same(remove_extent_t, int[2])); + static_assert(__is_same(remove_extent_t, int[2])); + static_assert(__is_same(remove_extent_t, const int)); + static_assert(__is_same(remove_extent_t, const int)); + static_assert(__is_same(remove_extent_t, const int[2])); + static_assert(__is_same(remove_extent_t, const int[2])); + static_assert(__is_same(remove_extent_t, volatile int)); + static_assert(__is_same(remove_extent_t, volatile int)); + static_assert(__is_same(remove_extent_t, volatile int[2])); + static_assert(__is_same(remove_extent_t, volatile int[2])); + static_assert(__is_same(remove_extent_t, const volatile int)); + static_assert(__is_same(remove_extent_t, const volatile int)); + static_assert(__is_same(remove_extent_t, const volatile int[2])); + static_assert(__is_same(remove_extent_t, const volatile int[2])); + static_assert(__is_same(remove_extent_t, int *)); + static_assert(__is_same(remove_extent_t, int &)); + static_assert(__is_same(remove_extent_t, int &&)); + static_assert(__is_same(remove_extent_t, int())); + static_assert(__is_same(remove_extent_t, int (*)())); + static_assert(__is_same(remove_extent_t, int (&)())); + + static_assert(__is_same(remove_extent_t, S)); + static_assert(__is_same(remove_extent_t, int S::*)); + static_assert(__is_same(remove_extent_t, int(S::*)())); using SomeArray = int[1][2]; - static_assert(__is_same(remove_extent_t, const int[2]), ""); + static_assert(__is_same(remove_extent_t, const int[2])); } template using remove_all_extents_t = __remove_all_extents(T); void remove_all_extents() { - static_assert(__is_same(remove_all_extents_t, void), ""); - static_assert(__is_same(remove_all_extents_t, int), ""); - static_assert(__is_same(remove_all_extents_t, const int), ""); - static_assert(__is_same(remove_all_extents_t, volatile int), ""); - static_assert(__is_same(remove_all_extents_t, const volatile int), ""); - static_assert(__is_same(remove_all_extents_t, int), ""); - static_assert(__is_same(remove_all_extents_t, int), ""); - static_assert(__is_same(remove_all_extents_t, int), ""); - static_assert(__is_same(remove_all_extents_t, int), ""); - static_assert(__is_same(remove_all_extents_t, const int), ""); - static_assert(__is_same(remove_all_extents_t, const int), ""); - static_assert(__is_same(remove_all_extents_t, const int), ""); - static_assert(__is_same(remove_all_extents_t, const int), ""); - static_assert(__is_same(remove_all_extents_t, volatile int), ""); - static_assert(__is_same(remove_all_extents_t, volatile int), ""); - static_assert(__is_same(remove_all_extents_t, volatile int), ""); - static_assert(__is_same(remove_all_extents_t, volatile int), ""); - static_assert(__is_same(remove_all_extents_t, const volatile int), ""); - static_assert(__is_same(remove_all_extents_t, const volatile int), ""); - static_assert(__is_same(remove_all_extents_t, const volatile int), ""); - static_assert(__is_same(remove_all_extents_t, const volatile int), ""); - static_assert(__is_same(remove_all_extents_t, int *), ""); - static_assert(__is_same(remove_all_extents_t, int &), ""); - static_assert(__is_same(remove_all_extents_t, int &&), ""); - static_assert(__is_same(remove_all_extents_t, int()), ""); - static_assert(__is_same(remove_all_extents_t, int (*)()), ""); - static_assert(__is_same(remove_all_extents_t, int (&)()), ""); - - static_assert(__is_same(remove_all_extents_t, S), ""); - static_assert(__is_same(remove_all_extents_t, int S::*), ""); - static_assert(__is_same(remove_all_extents_t, int(S::*)()), ""); + static_assert(__is_same(remove_all_extents_t, void)); + static_assert(__is_same(remove_all_extents_t, int)); + static_assert(__is_same(remove_all_extents_t, const int)); + static_assert(__is_same(remove_all_extents_t, volatile int)); + static_assert(__is_same(remove_all_extents_t, const volatile int)); + static_assert(__is_same(remove_all_extents_t, int)); + static_assert(__is_same(remove_all_extents_t, int)); + static_assert(__is_same(remove_all_extents_t, int)); + static_assert(__is_same(remove_all_extents_t, int)); + static_assert(__is_same(remove_all_extents_t, const int)); + static_assert(__is_same(remove_all_extents_t, const int)); + static_assert(__is_same(remove_all_extents_t, const int)); + static_assert(__is_same(remove_all_extents_t, const int)); + static_assert(__is_same(remove_all_extents_t, volatile int)); + static_assert(__is_same(remove_all_extents_t, volatile int)); + static_assert(__is_same(remove_all_extents_t, volatile int)); + static_assert(__is_same(remove_all_extents_t, volatile int)); + static_assert(__is_same(remove_all_extents_t, const volatile int)); + static_assert(__is_same(remove_all_extents_t, const volatile int)); + static_assert(__is_same(remove_all_extents_t, const volatile int)); + static_assert(__is_same(remove_all_extents_t, const volatile int)); + static_assert(__is_same(remove_all_extents_t, int *)); + static_assert(__is_same(remove_all_extents_t, int &)); + static_assert(__is_same(remove_all_extents_t, int &&)); + static_assert(__is_same(remove_all_extents_t, int())); + static_assert(__is_same(remove_all_extents_t, int (*)())); + static_assert(__is_same(remove_all_extents_t, int (&)())); + + static_assert(__is_same(remove_all_extents_t, S)); + static_assert(__is_same(remove_all_extents_t, int S::*)); + static_assert(__is_same(remove_all_extents_t, int(S::*)())); using SomeArray = int[1][2]; - static_assert(__is_same(remove_all_extents_t, const int), ""); + static_assert(__is_same(remove_all_extents_t, const int)); } -- GitLab From 235332150d52d11b340a10be1bb88432d2cf4179 Mon Sep 17 00:00:00 2001 From: David Spickett Date: Fri, 8 Mar 2024 11:27:54 +0000 Subject: [PATCH 0014/1674] [lldb][Docs] Add Curses version note to build page This explains a thing that hit me on FreeBSD because the base system has an ncursesw at one version and I installed from pkg another version that was simply ncurses (no wide char support). For whatever reason, when we pass -lcurses to the linker it ends up picking bits of both installs. This led to lldb crashing immediately if you tried to use the `gui` command. In a way that gave little information but I stumbled onto https://github.com/vifm/vifm/issues/325 which is very similar. ``` ec2-user@freebsd:~/build-llvm $ ldd ./bin/lldb | grep curses libncursesw.so.9 => /lib/libncursesw.so.9 (0x6a515206e000) libncurses.so.6 => /usr/local/lib/libncurses.so.6 (0x6a5158e86000) ``` We should only see one version, and it and libpanel etc should all have "w" or not have "w". This was not the case for my build. What I can see from the CMake side seemed fine, it found the pkg installed ncurses in /usr/local. Something else must decide that -lcurses should pull in the other one. Regardless, I don't know how to fix that but the solution for most people is just not to add another ncurses if they already have one. So I've added a note saying so, and how to check what your lldb is using. --- lldb/docs/resources/build.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lldb/docs/resources/build.rst b/lldb/docs/resources/build.rst index fe8e293db642..5f4d35ced623 100644 --- a/lldb/docs/resources/build.rst +++ b/lldb/docs/resources/build.rst @@ -85,6 +85,13 @@ commands below. versions of python unusable. This primarily affects Windows, as debug builds of LLDB must use debug python as well. +.. note:: + Installing multiple versions of Curses, particularly when only one is built with + wide character support, can cause lldb to be linked with an incorrect set of + libraries. If your system already has Curses, we recommend you use that version. + If you do install another one, use a tool like ``ldd`` to ensure only one version + of Curses is being used in the final ``lldb`` executable. + Windows ******* -- GitLab From 4d478bcb4fc4fcf6532b725a0d070cb0c0cf9430 Mon Sep 17 00:00:00 2001 From: Dmitry Chernenkov Date: Fri, 8 Mar 2024 11:53:15 +0000 Subject: [PATCH 0015/1674] [bazel] Add support_uint dep in libc unittest --- utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel index 44692947af7c..37eb21c77e49 100644 --- a/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel @@ -126,5 +126,6 @@ libc_support_library( deps = [ "//libc:__support_cpp_string", "//libc:__support_cpp_type_traits", + "//libc:__support_uint", ], ) -- GitLab From bd6eb54886ad12fb523e924e7291abfa2b010e3c Mon Sep 17 00:00:00 2001 From: Paul Walker Date: Fri, 8 Mar 2024 12:09:05 +0000 Subject: [PATCH 0016/1674] [LLVM][CodeGen] Teach SelectionDAG how to expand FREM to a vector math call. (#83859) This removes, at least when a vector library is available, a failure case for scalable vectors. Doing so means we can confidently cost vector FREM instructions without making an assumption that later passes will transform the IR before it gets to the code generator. NOTE: Whilst only FREM has been implemented the same mechanism can be used for the other libm related ISD nodes. --- .../SelectionDAG/LegalizeVectorOps.cpp | 128 ++++++++++++++++++ llvm/lib/CodeGen/TargetPassConfig.cpp | 6 +- .../CodeGen/AArch64/fp-veclib-expansion.ll | 116 ++++++++++++++++ 3 files changed, 249 insertions(+), 1 deletion(-) create mode 100644 llvm/test/CodeGen/AArch64/fp-veclib-expansion.ll diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp index 6074498d9144..57a3f6a65e00 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp @@ -28,6 +28,8 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/Analysis/TargetLibraryInfo.h" +#include "llvm/Analysis/VectorUtils.h" #include "llvm/CodeGen/ISDOpcodes.h" #include "llvm/CodeGen/SelectionDAG.h" #include "llvm/CodeGen/SelectionDAGNodes.h" @@ -147,6 +149,14 @@ class VectorLegalizer { void ExpandStrictFPOp(SDNode *Node, SmallVectorImpl &Results); void ExpandREM(SDNode *Node, SmallVectorImpl &Results); + bool tryExpandVecMathCall(SDNode *Node, RTLIB::Libcall LC, + SmallVectorImpl &Results); + bool tryExpandVecMathCall(SDNode *Node, RTLIB::Libcall Call_F32, + RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80, + RTLIB::Libcall Call_F128, + RTLIB::Libcall Call_PPCF128, + SmallVectorImpl &Results); + void UnrollStrictFPOp(SDNode *Node, SmallVectorImpl &Results); /// Implements vector promotion. @@ -1139,6 +1149,13 @@ void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl &Results) { case ISD::VP_MERGE: Results.push_back(ExpandVP_MERGE(Node)); return; + case ISD::FREM: + if (tryExpandVecMathCall(Node, RTLIB::REM_F32, RTLIB::REM_F64, + RTLIB::REM_F80, RTLIB::REM_F128, + RTLIB::REM_PPCF128, Results)) + return; + + break; } SDValue Unrolled = DAG.UnrollVectorOp(Node); @@ -1842,6 +1859,117 @@ void VectorLegalizer::ExpandREM(SDNode *Node, Results.push_back(Result); } +// Try to expand libm nodes into vector math routine calls. Callers provide the +// LibFunc equivalent of the passed in Node, which is used to lookup mappings +// within TargetLibraryInfo. The only mappings considered are those where the +// result and all operands are the same vector type. While predicated nodes are +// not supported, we will emit calls to masked routines by passing in an all +// true mask. +bool VectorLegalizer::tryExpandVecMathCall(SDNode *Node, RTLIB::Libcall LC, + SmallVectorImpl &Results) { + // Chain must be propagated but currently strict fp operations are down + // converted to their none strict counterpart. + assert(!Node->isStrictFPOpcode() && "Unexpected strict fp operation!"); + + const char *LCName = TLI.getLibcallName(LC); + if (!LCName) + return false; + LLVM_DEBUG(dbgs() << "Looking for vector variant of " << LCName << "\n"); + + EVT VT = Node->getValueType(0); + ElementCount VL = VT.getVectorElementCount(); + + // Lookup a vector function equivalent to the specified libcall. Prefer + // unmasked variants but we will generate a mask if need be. + const TargetLibraryInfo &TLibInfo = DAG.getLibInfo(); + const VecDesc *VD = TLibInfo.getVectorMappingInfo(LCName, VL, false); + if (!VD) + VD = TLibInfo.getVectorMappingInfo(LCName, VL, /*Masked=*/true); + if (!VD) + return false; + + LLVMContext *Ctx = DAG.getContext(); + Type *Ty = VT.getTypeForEVT(*Ctx); + Type *ScalarTy = Ty->getScalarType(); + + // Construct a scalar function type based on Node's operands. + SmallVector ArgTys; + for (unsigned i = 0; i < Node->getNumOperands(); ++i) { + assert(Node->getOperand(i).getValueType() == VT && + "Expected matching vector types!"); + ArgTys.push_back(ScalarTy); + } + FunctionType *ScalarFTy = FunctionType::get(ScalarTy, ArgTys, false); + + // Generate call information for the vector function. + const std::string MangledName = VD->getVectorFunctionABIVariantString(); + auto OptVFInfo = VFABI::tryDemangleForVFABI(MangledName, ScalarFTy); + if (!OptVFInfo) + return false; + + LLVM_DEBUG(dbgs() << "Found vector variant " << VD->getVectorFnName() + << "\n"); + + // Sanity check just in case OptVFInfo has unexpected parameters. + if (OptVFInfo->Shape.Parameters.size() != + Node->getNumOperands() + VD->isMasked()) + return false; + + // Collect vector call operands. + + SDLoc DL(Node); + TargetLowering::ArgListTy Args; + TargetLowering::ArgListEntry Entry; + Entry.IsSExt = false; + Entry.IsZExt = false; + + unsigned OpNum = 0; + for (auto &VFParam : OptVFInfo->Shape.Parameters) { + if (VFParam.ParamKind == VFParamKind::GlobalPredicate) { + EVT MaskVT = TLI.getSetCCResultType(DAG.getDataLayout(), *Ctx, VT); + Entry.Node = DAG.getBoolConstant(true, DL, MaskVT, VT); + Entry.Ty = MaskVT.getTypeForEVT(*Ctx); + Args.push_back(Entry); + continue; + } + + // Only vector operands are supported. + if (VFParam.ParamKind != VFParamKind::Vector) + return false; + + Entry.Node = Node->getOperand(OpNum++); + Entry.Ty = Ty; + Args.push_back(Entry); + } + + // Emit a call to the vector function. + SDValue Callee = DAG.getExternalSymbol(VD->getVectorFnName().data(), + TLI.getPointerTy(DAG.getDataLayout())); + TargetLowering::CallLoweringInfo CLI(DAG); + CLI.setDebugLoc(DL) + .setChain(DAG.getEntryNode()) + .setLibCallee(CallingConv::C, Ty, Callee, std::move(Args)); + + std::pair CallResult = TLI.LowerCallTo(CLI); + Results.push_back(CallResult.first); + return true; +} + +/// Try to expand the node to a vector libcall based on the result type. +bool VectorLegalizer::tryExpandVecMathCall( + SDNode *Node, RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64, + RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128, + RTLIB::Libcall Call_PPCF128, SmallVectorImpl &Results) { + RTLIB::Libcall LC = RTLIB::getFPLibCall( + Node->getValueType(0).getVectorElementType(), Call_F32, Call_F64, + Call_F80, Call_F128, Call_PPCF128); + + if (LC == RTLIB::UNKNOWN_LIBCALL) + return false; + + return tryExpandVecMathCall(Node, LC, Results); +} + void VectorLegalizer::UnrollStrictFPOp(SDNode *Node, SmallVectorImpl &Results) { EVT VT = Node->getValueType(0); diff --git a/llvm/lib/CodeGen/TargetPassConfig.cpp b/llvm/lib/CodeGen/TargetPassConfig.cpp index cf068ece8d4c..8832b51333d9 100644 --- a/llvm/lib/CodeGen/TargetPassConfig.cpp +++ b/llvm/lib/CodeGen/TargetPassConfig.cpp @@ -205,6 +205,10 @@ static cl::opt MISchedPostRA( static cl::opt EarlyLiveIntervals("early-live-intervals", cl::Hidden, cl::desc("Run live interval analysis earlier in the pipeline")); +static cl::opt DisableReplaceWithVecLib( + "disable-replace-with-vec-lib", cl::Hidden, + cl::desc("Disable replace with vector math call pass")); + /// Option names for limiting the codegen pipeline. /// Those are used in error reporting and we didn't want /// to duplicate their names all over the place. @@ -856,7 +860,7 @@ void TargetPassConfig::addIRPasses() { if (getOptLevel() != CodeGenOptLevel::None && !DisableConstantHoisting) addPass(createConstantHoistingPass()); - if (getOptLevel() != CodeGenOptLevel::None) + if (getOptLevel() != CodeGenOptLevel::None && !DisableReplaceWithVecLib) addPass(createReplaceWithVeclibLegacyPass()); if (getOptLevel() != CodeGenOptLevel::None && !DisablePartialLibcallInlining) diff --git a/llvm/test/CodeGen/AArch64/fp-veclib-expansion.ll b/llvm/test/CodeGen/AArch64/fp-veclib-expansion.ll new file mode 100644 index 000000000000..67c056c780cc --- /dev/null +++ b/llvm/test/CodeGen/AArch64/fp-veclib-expansion.ll @@ -0,0 +1,116 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +; RUN: llc --disable-replace-with-vec-lib --vector-library=ArmPL < %s -o - | FileCheck --check-prefix=ARMPL %s +; RUN: llc --disable-replace-with-vec-lib --vector-library=sleefgnuabi < %s -o - | FileCheck --check-prefix=SLEEF %s + +target triple = "aarch64-unknown-linux-gnu" + +define <2 x double> @frem_v2f64(<2 x double> %unused, <2 x double> %a, <2 x double> %b) #0 { +; ARMPL-LABEL: frem_v2f64: +; ARMPL: // %bb.0: +; ARMPL-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; ARMPL-NEXT: .cfi_def_cfa_offset 16 +; ARMPL-NEXT: .cfi_offset w30, -16 +; ARMPL-NEXT: mov v0.16b, v1.16b +; ARMPL-NEXT: mov v1.16b, v2.16b +; ARMPL-NEXT: bl armpl_vfmodq_f64 +; ARMPL-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; ARMPL-NEXT: ret +; +; SLEEF-LABEL: frem_v2f64: +; SLEEF: // %bb.0: +; SLEEF-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; SLEEF-NEXT: .cfi_def_cfa_offset 16 +; SLEEF-NEXT: .cfi_offset w30, -16 +; SLEEF-NEXT: mov v0.16b, v1.16b +; SLEEF-NEXT: mov v1.16b, v2.16b +; SLEEF-NEXT: bl _ZGVnN2vv_fmod +; SLEEF-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; SLEEF-NEXT: ret + %res = frem <2 x double> %a, %b + ret <2 x double> %res +} + +define <4 x float> @frem_strict_v4f32(<4 x float> %unused, <4 x float> %a, <4 x float> %b) #1 { +; ARMPL-LABEL: frem_strict_v4f32: +; ARMPL: // %bb.0: +; ARMPL-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; ARMPL-NEXT: .cfi_def_cfa_offset 16 +; ARMPL-NEXT: .cfi_offset w30, -16 +; ARMPL-NEXT: mov v0.16b, v1.16b +; ARMPL-NEXT: mov v1.16b, v2.16b +; ARMPL-NEXT: bl armpl_vfmodq_f32 +; ARMPL-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; ARMPL-NEXT: ret +; +; SLEEF-LABEL: frem_strict_v4f32: +; SLEEF: // %bb.0: +; SLEEF-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; SLEEF-NEXT: .cfi_def_cfa_offset 16 +; SLEEF-NEXT: .cfi_offset w30, -16 +; SLEEF-NEXT: mov v0.16b, v1.16b +; SLEEF-NEXT: mov v1.16b, v2.16b +; SLEEF-NEXT: bl _ZGVnN4vv_fmodf +; SLEEF-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; SLEEF-NEXT: ret + %res = frem <4 x float> %a, %b + ret <4 x float> %res +} + +define @frem_nxv4f32( %unused, %a, %b) #0 { +; ARMPL-LABEL: frem_nxv4f32: +; ARMPL: // %bb.0: +; ARMPL-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; ARMPL-NEXT: .cfi_def_cfa_offset 16 +; ARMPL-NEXT: .cfi_offset w30, -16 +; ARMPL-NEXT: ptrue p0.s +; ARMPL-NEXT: mov z0.d, z1.d +; ARMPL-NEXT: mov z1.d, z2.d +; ARMPL-NEXT: bl armpl_svfmod_f32_x +; ARMPL-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; ARMPL-NEXT: ret +; +; SLEEF-LABEL: frem_nxv4f32: +; SLEEF: // %bb.0: +; SLEEF-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; SLEEF-NEXT: .cfi_def_cfa_offset 16 +; SLEEF-NEXT: .cfi_offset w30, -16 +; SLEEF-NEXT: ptrue p0.s +; SLEEF-NEXT: mov z0.d, z1.d +; SLEEF-NEXT: mov z1.d, z2.d +; SLEEF-NEXT: bl _ZGVsMxvv_fmodf +; SLEEF-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; SLEEF-NEXT: ret + %res = frem %a, %b + ret %res +} + +define @frem_strict_nxv2f64( %unused, %a, %b) #1 { +; ARMPL-LABEL: frem_strict_nxv2f64: +; ARMPL: // %bb.0: +; ARMPL-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; ARMPL-NEXT: .cfi_def_cfa_offset 16 +; ARMPL-NEXT: .cfi_offset w30, -16 +; ARMPL-NEXT: ptrue p0.d +; ARMPL-NEXT: mov z0.d, z1.d +; ARMPL-NEXT: mov z1.d, z2.d +; ARMPL-NEXT: bl armpl_svfmod_f64_x +; ARMPL-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; ARMPL-NEXT: ret +; +; SLEEF-LABEL: frem_strict_nxv2f64: +; SLEEF: // %bb.0: +; SLEEF-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; SLEEF-NEXT: .cfi_def_cfa_offset 16 +; SLEEF-NEXT: .cfi_offset w30, -16 +; SLEEF-NEXT: ptrue p0.d +; SLEEF-NEXT: mov z0.d, z1.d +; SLEEF-NEXT: mov z1.d, z2.d +; SLEEF-NEXT: bl _ZGVsMxvv_fmod +; SLEEF-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; SLEEF-NEXT: ret + %res = frem %a, %b + ret %res +} + +attributes #0 = { "target-features"="+sve" } +attributes #1 = { "target-features"="+sve" strictfp } -- GitLab From 5d98d8822c69bc9d32c25190beaf1eaccf9ddd37 Mon Sep 17 00:00:00 2001 From: Younan Zhang Date: Fri, 8 Mar 2024 20:26:31 +0800 Subject: [PATCH 0017/1674] [Sema] Avoid unnecessary copy on MultiLevelTemplateArgumentList. NFC (#84459) We don't modify the MLTAL parameter in `SetupConstraintScope`, and it is better if we don't copy the 120-byte object each time we call the function. --- clang/include/clang/Sema/Sema.h | 10 ++++++---- clang/lib/Sema/SemaConcept.cpp | 3 ++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h index 592c7871a4a5..2cac7e6ca08f 100644 --- a/clang/include/clang/Sema/Sema.h +++ b/clang/include/clang/Sema/Sema.h @@ -11607,12 +11607,14 @@ private: LocalInstantiationScope &Scope, const MultiLevelTemplateArgumentList &TemplateArgs); - /// used by SetupConstraintCheckingTemplateArgumentsAndScope to recursively(in + /// Used by SetupConstraintCheckingTemplateArgumentsAndScope to recursively(in /// the case of lambdas) set up the LocalInstantiationScope of the current /// function. - bool SetupConstraintScope( - FunctionDecl *FD, std::optional> TemplateArgs, - MultiLevelTemplateArgumentList MLTAL, LocalInstantiationScope &Scope); + bool + SetupConstraintScope(FunctionDecl *FD, + std::optional> TemplateArgs, + const MultiLevelTemplateArgumentList &MLTAL, + LocalInstantiationScope &Scope); /// Used during constraint checking, sets up the constraint template argument /// lists, and calls SetupConstraintScope to set up the diff --git a/clang/lib/Sema/SemaConcept.cpp b/clang/lib/Sema/SemaConcept.cpp index 2878e4d31ee8..a8e387e35fb4 100644 --- a/clang/lib/Sema/SemaConcept.cpp +++ b/clang/lib/Sema/SemaConcept.cpp @@ -586,7 +586,8 @@ bool Sema::addInstantiatedCapturesToScope( bool Sema::SetupConstraintScope( FunctionDecl *FD, std::optional> TemplateArgs, - MultiLevelTemplateArgumentList MLTAL, LocalInstantiationScope &Scope) { + const MultiLevelTemplateArgumentList &MLTAL, + LocalInstantiationScope &Scope) { if (FD->isTemplateInstantiation() && FD->getPrimaryTemplate()) { FunctionTemplateDecl *PrimaryTemplate = FD->getPrimaryTemplate(); InstantiatingTemplate Inst( -- GitLab From 430de48a61369ea41efe1cbb98654a86568ae95f Mon Sep 17 00:00:00 2001 From: Jay Foad Date: Fri, 8 Mar 2024 12:26:30 +0000 Subject: [PATCH 0018/1674] [GISel] Simplify getConstantVRegValWithLookThrough. NFC. --- llvm/lib/CodeGen/GlobalISel/Utils.cpp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/llvm/lib/CodeGen/GlobalISel/Utils.cpp b/llvm/lib/CodeGen/GlobalISel/Utils.cpp index 23ad68b331c9..a9fa73b60a09 100644 --- a/llvm/lib/CodeGen/GlobalISel/Utils.cpp +++ b/llvm/lib/CodeGen/GlobalISel/Utils.cpp @@ -355,18 +355,17 @@ std::optional getConstantVRegValWithLookThrough( if (!MaybeVal) return std::nullopt; APInt &Val = *MaybeVal; - while (!SeenOpcodes.empty()) { - std::pair OpcodeAndSize = SeenOpcodes.pop_back_val(); - switch (OpcodeAndSize.first) { + for (auto [Opcode, Size] : reverse(SeenOpcodes)) { + switch (Opcode) { case TargetOpcode::G_TRUNC: - Val = Val.trunc(OpcodeAndSize.second); + Val = Val.trunc(Size); break; case TargetOpcode::G_ANYEXT: case TargetOpcode::G_SEXT: - Val = Val.sext(OpcodeAndSize.second); + Val = Val.sext(Size); break; case TargetOpcode::G_ZEXT: - Val = Val.zext(OpcodeAndSize.second); + Val = Val.zext(Size); break; } } -- GitLab From d74287226ac8c144a2700ba589d34a8bebd85d0d Mon Sep 17 00:00:00 2001 From: Kiran Chandramohan Date: Fri, 8 Mar 2024 12:43:39 +0000 Subject: [PATCH 0019/1674] [Flang][AArch64] Add support for complex16 params/returns (#84217) Fixes #84088 --- flang/lib/Optimizer/CodeGen/Target.cpp | 6 ++++-- flang/test/Fir/target-complex16.f90 | 29 ++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 flang/test/Fir/target-complex16.f90 diff --git a/flang/lib/Optimizer/CodeGen/Target.cpp b/flang/lib/Optimizer/CodeGen/Target.cpp index 7c77bdd79008..cea7a1f97f41 100644 --- a/flang/lib/Optimizer/CodeGen/Target.cpp +++ b/flang/lib/Optimizer/CodeGen/Target.cpp @@ -737,7 +737,8 @@ struct TargetAArch64 : public GenericTarget { CodeGenSpecifics::Marshalling marshal; const auto *sem = &floatToSemantics(kindMap, eleTy); if (sem == &llvm::APFloat::IEEEsingle() || - sem == &llvm::APFloat::IEEEdouble()) { + sem == &llvm::APFloat::IEEEdouble() || + sem == &llvm::APFloat::IEEEquad()) { // [2 x t] array of 2 eleTy marshal.emplace_back(fir::SequenceType::get({2}, eleTy), AT{}); } else { @@ -751,7 +752,8 @@ struct TargetAArch64 : public GenericTarget { CodeGenSpecifics::Marshalling marshal; const auto *sem = &floatToSemantics(kindMap, eleTy); if (sem == &llvm::APFloat::IEEEsingle() || - sem == &llvm::APFloat::IEEEdouble()) { + sem == &llvm::APFloat::IEEEdouble() || + sem == &llvm::APFloat::IEEEquad()) { // Use a type that will be translated into LLVM as: // { t, t } struct of 2 eleTy marshal.emplace_back(mlir::TupleType::get(eleTy.getContext(), diff --git a/flang/test/Fir/target-complex16.f90 b/flang/test/Fir/target-complex16.f90 new file mode 100644 index 000000000000..9245b205f68a --- /dev/null +++ b/flang/test/Fir/target-complex16.f90 @@ -0,0 +1,29 @@ +// RUN: tco --target=aarch64-unknown-linux-gnu %s | FileCheck %s --check-prefix=AARCH64 + +// AARCH64-LABEL: define { fp128, fp128 } @gen16() +func.func @gen16() -> !fir.complex<16> { + // AARCH64: %[[VAL1:.*]] = alloca { fp128, fp128 }, i64 1, align 16 + %1 = fir.undefined !fir.complex<16> + %2 = arith.constant 1.0 : f128 + %3 = arith.constant -4.0 : f128 + %c0 = arith.constant 0 : i32 + // AARCH64: store { fp128, fp128 } { fp128 0xL0000000000000000C001000000000000, fp128 0xL00000000000000003FFF000000000000 }, ptr %[[VAL1]], align 16 + %4 = fir.insert_value %1, %3, [0 : index] : (!fir.complex<16>, f128) -> !fir.complex<16> + %c1 = arith.constant 1 : i32 + %5 = fir.insert_value %4, %2, [1 : index] : (!fir.complex<16>, f128) -> !fir.complex<16> + // AARCH64: %[[VAL2:.*]] = load { fp128, fp128 }, ptr %[[VAL1]], align 16 + // AARCH64: ret { fp128, fp128 } %[[VAL2]] + return %5 : !fir.complex<16> +} + +// AARCH64: declare void @sink16([2 x fp128]) +func.func private @sink16(!fir.complex<16>) -> () + +// AARCH64-LABEL: define void @call16() +func.func @call16() { + // AARCH64: = call { fp128, fp128 } @gen16() + %1 = fir.call @gen16() : () -> !fir.complex<16> + // AARCH64: call void @sink16([2 x fp128] % + fir.call @sink16(%1) : (!fir.complex<16>) -> () + return +} -- GitLab From 1ca8092e87aea58f1c3752d03c20bdfc4259e409 Mon Sep 17 00:00:00 2001 From: Shengchen Kan Date: Fri, 8 Mar 2024 20:54:33 +0800 Subject: [PATCH 0020/1674] [X86][MC] Support encoding/decoding for APX CCMP/CTEST (#83863) APX assembly syntax recommendations: https://cdrdv2.intel.com/v1/dl/getContent/817241 NOTE: The change in llvm/tools/llvm-exegesis/lib/X86/Target.cpp is for test LLVM :: tools/llvm-exegesis/X86/latency/latency-SETCCr-cond-codes-sweep.s For `SETcc`, llvm-exegesis would randomly choose 1 other instruction to test with `SETcc`, after selecting the instruction, llvm-exegesis would check if the operand is initialized and valid, if not `randomizeTargetMCOperand` would choose a value for invalid operand, it misses support for condition code operand, which cause the flaky failure after `CCMP` supported. llvm-exegesis can choose `CCMP` without specifying ccmp feature b/c it use `MCSubtarget` and only16/32/64 bit is considered. llvm-exegesis doesn't choose other instructions b/c requirement in `hasAliasingRegistersThrough`: the instruction should use GPR (defined by `SETcc`) and define `EFLAGS` (used by `SETcc`). --- .../Support/X86DisassemblerDecoderCommon.h | 1 + .../lib/Target/X86/AsmParser/X86AsmParser.cpp | 67 + .../X86/Disassembler/X86Disassembler.cpp | 43 +- .../X86/Disassembler/X86DisassemblerDecoder.h | 8 +- .../lib/Target/X86/MCTargetDesc/X86BaseInfo.h | 5 +- .../X86/MCTargetDesc/X86InstPrinterCommon.cpp | 29 +- .../X86/MCTargetDesc/X86InstPrinterCommon.h | 1 + .../X86/MCTargetDesc/X86MCCodeEmitter.cpp | 51 +- llvm/lib/Target/X86/X86InstrAsmAlias.td | 128 ++ .../Target/X86/X86InstrConditionalCompare.td | 110 ++ llvm/lib/Target/X86/X86InstrFormats.td | 2 + llvm/lib/Target/X86/X86InstrInfo.td | 1 + llvm/lib/Target/X86/X86InstrOperands.td | 6 + llvm/test/MC/Disassembler/X86/apx/ccmp.txt | 1598 +++++++++++++++++ .../Disassembler/X86/apx/reverse-encoding.txt | 18 + llvm/test/MC/X86/apx/ccmp-att-error.s | 29 + llvm/test/MC/X86/apx/ccmp-att.s | 1213 +++++++++++++ llvm/test/MC/X86/apx/ccmp-intel-error.s | 29 + llvm/test/MC/X86/apx/ccmp-intel.s | 1210 +++++++++++++ llvm/test/TableGen/x86-fold-tables.inc | 23 + llvm/tools/llvm-exegesis/lib/X86/Target.cpp | 4 + llvm/utils/TableGen/X86RecognizableInstr.cpp | 11 + llvm/utils/TableGen/X86RecognizableInstr.h | 2 + 23 files changed, 4574 insertions(+), 15 deletions(-) create mode 100644 llvm/lib/Target/X86/X86InstrConditionalCompare.td create mode 100644 llvm/test/MC/Disassembler/X86/apx/ccmp.txt create mode 100644 llvm/test/MC/X86/apx/ccmp-att-error.s create mode 100644 llvm/test/MC/X86/apx/ccmp-att.s create mode 100644 llvm/test/MC/X86/apx/ccmp-intel-error.s create mode 100644 llvm/test/MC/X86/apx/ccmp-intel.s diff --git a/llvm/include/llvm/Support/X86DisassemblerDecoderCommon.h b/llvm/include/llvm/Support/X86DisassemblerDecoderCommon.h index 0dc974ea9efd..5daae45df2f8 100644 --- a/llvm/include/llvm/Support/X86DisassemblerDecoderCommon.h +++ b/llvm/include/llvm/Support/X86DisassemblerDecoderCommon.h @@ -434,6 +434,7 @@ enum ModRMDecisionType { MODRMTYPES MODRM_max }; ENUM_ENTRY(ENCODING_Rv, \ "Register code of operand size added to the opcode byte") \ ENUM_ENTRY(ENCODING_CC, "Condition code encoded in opcode") \ + ENUM_ENTRY(ENCODING_CF, "Condition flags encoded in EVEX.VVVV") \ ENUM_ENTRY(ENCODING_DUP, \ "Duplicate of another operand; ID is encoded in type") \ ENUM_ENTRY(ENCODING_SI, "Source index; encoded in OpSize/Adsize prefix") \ diff --git a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp index 051f6caa8c04..e3701d69934c 100644 --- a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp +++ b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp @@ -1122,6 +1122,7 @@ private: unsigned IdentifyMasmOperator(StringRef Name); bool ParseMasmOperator(unsigned OpKind, int64_t &Val); bool ParseRoundingModeOp(SMLoc Start, OperandVector &Operands); + bool parseCFlagsOp(OperandVector &Operands); bool ParseIntelNamedOperator(StringRef Name, IntelExprStateMachine &SM, bool &ParseError, SMLoc &End); bool ParseMasmNamedOperator(StringRef Name, IntelExprStateMachine &SM, @@ -2306,6 +2307,67 @@ bool X86AsmParser::ParseRoundingModeOp(SMLoc Start, OperandVector &Operands) { return Error(Tok.getLoc(), "unknown token in expression"); } +/// Parse condtional flags for CCMP/CTEST, e.g {dfv=of,sf,zf,cf} right after +/// mnemonic. +bool X86AsmParser::parseCFlagsOp(OperandVector &Operands) { + MCAsmParser &Parser = getParser(); + AsmToken Tok = Parser.getTok(); + const SMLoc Start = Tok.getLoc(); + if (!Tok.is(AsmToken::LCurly)) + return Error(Tok.getLoc(), "Expected { at this point"); + Parser.Lex(); // Eat "{" + Tok = Parser.getTok(); + if (Tok.getIdentifier() != "dfv") + return Error(Tok.getLoc(), "Expected dfv at this point"); + Parser.Lex(); // Eat "dfv" + Tok = Parser.getTok(); + if (!Tok.is(AsmToken::Equal)) + return Error(Tok.getLoc(), "Expected = at this point"); + Parser.Lex(); // Eat "=" + + Tok = Parser.getTok(); + SMLoc End; + if (Tok.is(AsmToken::RCurly)) { + End = Tok.getEndLoc(); + Operands.push_back(X86Operand::CreateImm( + MCConstantExpr::create(0, Parser.getContext()), Start, End)); + Parser.Lex(); // Eat "}" + return false; + } + unsigned CFlags = 0; + for (unsigned I = 0; I < 4; ++I) { + Tok = Parser.getTok(); + unsigned CFlag = StringSwitch(Tok.getIdentifier()) + .Case("of", 0x8) + .Case("sf", 0x4) + .Case("zf", 0x2) + .Case("cf", 0x1) + .Default(~0U); + if (CFlag == ~0U) + return Error(Tok.getLoc(), "Invalid conditional flags"); + + if (CFlags & CFlag) + return Error(Tok.getLoc(), "Duplicated conditional flag"); + CFlags |= CFlag; + + Parser.Lex(); // Eat one conditional flag + Tok = Parser.getTok(); + if (Tok.is(AsmToken::RCurly)) { + End = Tok.getEndLoc(); + Operands.push_back(X86Operand::CreateImm( + MCConstantExpr::create(CFlags, Parser.getContext()), Start, End)); + Parser.Lex(); // Eat "}" + return false; + } else if (I == 3) { + return Error(Tok.getLoc(), "Expected } at this point"); + } else if (Tok.isNot(AsmToken::Comma)) { + return Error(Tok.getLoc(), "Expected } or , at this point"); + } + Parser.Lex(); // Eat "," + } + llvm_unreachable("Unexpected control flow"); +} + /// Parse the '.' operator. bool X86AsmParser::ParseIntelDotOperator(IntelExprStateMachine &SM, SMLoc &End) { @@ -3461,6 +3523,11 @@ bool X86AsmParser::ParseInstruction(ParseInstructionInfo &Info, StringRef Name, Operands.push_back(X86Operand::CreateImm(ImmOp, NameLoc, NameLoc)); } + // Parse condtional flags after mnemonic. + if ((Name.starts_with("ccmp") || Name.starts_with("ctest")) && + parseCFlagsOp(Operands)) + return true; + // This does the actual operand parsing. Don't parse any more if we have a // prefix juxtaposed with an operation like "lock incl 4(%rax)", because we // just want to parse the "lock" as the first instruction and the "incl" as diff --git a/llvm/lib/Target/X86/Disassembler/X86Disassembler.cpp b/llvm/lib/Target/X86/Disassembler/X86Disassembler.cpp index dbc2cef39d86..0ff440bdbe0d 100644 --- a/llvm/lib/Target/X86/Disassembler/X86Disassembler.cpp +++ b/llvm/lib/Target/X86/Disassembler/X86Disassembler.cpp @@ -1141,6 +1141,25 @@ static int getInstructionIDWithAttrMask(uint16_t *instructionID, return 0; } +static bool isCCMPOrCTEST(InternalInstruction *insn) { + if (insn->opcodeType != MAP4) + return false; + if (insn->opcode == 0x83 && regFromModRM(insn->modRM) == 7) + return true; + switch (insn->opcode & 0xfe) { + default: + return false; + case 0x38: + case 0x3a: + case 0x84: + return true; + case 0x80: + return regFromModRM(insn->modRM) == 7; + case 0xf6: + return regFromModRM(insn->modRM) == 0; + } +} + static bool isNF(InternalInstruction *insn) { if (!nfFromEVEX4of4(insn->vectorExtensionPrefix[3])) return false; @@ -1197,9 +1216,12 @@ static int getInstructionID(struct InternalInstruction *insn, attrMask |= ATTR_EVEXKZ; if (bFromEVEX4of4(insn->vectorExtensionPrefix[3])) attrMask |= ATTR_EVEXB; - if (isNF(insn)) // NF bit is the MSB of aaa. + if (isNF(insn) && !readModRM(insn) && + !isCCMPOrCTEST(insn)) // NF bit is the MSB of aaa. attrMask |= ATTR_EVEXNF; - else if (aaaFromEVEX4of4(insn->vectorExtensionPrefix[3])) + // aaa is not used a opmask in MAP4 + else if (aaaFromEVEX4of4(insn->vectorExtensionPrefix[3]) && + (insn->opcodeType != MAP4)) attrMask |= ATTR_EVEXK; if (lFromEVEX4of4(insn->vectorExtensionPrefix[3])) attrMask |= ATTR_VEXL; @@ -1732,8 +1754,15 @@ static int readOperands(struct InternalInstruction *insn) { if (readOpcodeRegister(insn, 0)) return -1; break; + case ENCODING_CF: + insn->immediates[1] = oszcFromEVEX3of4(insn->vectorExtensionPrefix[2]); + needVVVV = false; // oszc shares the same bits with VVVV + break; case ENCODING_CC: - insn->immediates[1] = insn->opcode & 0xf; + if (isCCMPOrCTEST(insn)) + insn->immediates[2] = scFromEVEX4of4(insn->vectorExtensionPrefix[3]); + else + insn->immediates[1] = insn->opcode & 0xf; break; case ENCODING_FP: break; @@ -2371,9 +2400,15 @@ static bool translateOperand(MCInst &mcInst, const OperandSpecifier &operand, case ENCODING_Rv: translateRegister(mcInst, insn.opcodeRegister); return false; - case ENCODING_CC: + case ENCODING_CF: mcInst.addOperand(MCOperand::createImm(insn.immediates[1])); return false; + case ENCODING_CC: + if (isCCMPOrCTEST(&insn)) + mcInst.addOperand(MCOperand::createImm(insn.immediates[2])); + else + mcInst.addOperand(MCOperand::createImm(insn.immediates[1])); + return false; case ENCODING_FP: translateFPRegister(mcInst, insn.modRM & 7); return false; diff --git a/llvm/lib/Target/X86/Disassembler/X86DisassemblerDecoder.h b/llvm/lib/Target/X86/Disassembler/X86DisassemblerDecoder.h index 9cae0f02926f..7cd01c151ed2 100644 --- a/llvm/lib/Target/X86/Disassembler/X86DisassemblerDecoder.h +++ b/llvm/lib/Target/X86/Disassembler/X86DisassemblerDecoder.h @@ -33,6 +33,8 @@ namespace X86Disassembler { #define twoBitsFromOffset6(val) (((val) >> 6) & 0x3) #define threeBitsFromOffset0(val) ((val) & 0x7) #define threeBitsFromOffset3(val) (((val) >> 3) & 0x7) +#define fourBitsFromOffset0(val) ((val) & 0xf) +#define fourBitsFromOffset3(val) (((val) >> 3) & 0xf) #define fiveBitsFromOffset0(val) ((val) & 0x1f) #define invertedBitFromOffset2(val) (((~(val)) >> 2) & 0x1) #define invertedBitFromOffset3(val) (((~(val)) >> 3) & 0x1) @@ -97,6 +99,7 @@ namespace X86Disassembler { #define vvvvFromEVEX3of4(evex) invertedFourBitsFromOffset3(evex) #define x2FromEVEX3of4(evex) invertedBitFromOffset2(evex) #define ppFromEVEX3of4(evex) twoBitsFromOffset0(evex) +#define oszcFromEVEX3of4(evex) fourBitsFromOffset3(evex) #define zFromEVEX4of4(evex) bitFromOffset7(evex) #define l2FromEVEX4of4(evex) bitFromOffset6(evex) #define lFromEVEX4of4(evex) bitFromOffset5(evex) @@ -104,6 +107,7 @@ namespace X86Disassembler { #define v2FromEVEX4of4(evex) invertedBitFromOffset3(evex) #define aaaFromEVEX4of4(evex) threeBitsFromOffset0(evex) #define nfFromEVEX4of4(evex) bitFromOffset2(evex) +#define scFromEVEX4of4(evex) fourBitsFromOffset0(evex) // These enums represent Intel registers for use by the decoder. #define REGS_8BIT \ @@ -755,10 +759,10 @@ struct InternalInstruction { // The displacement, used for memory operands int32_t displacement; - // Immediates. There can be two in some cases + // Immediates. There can be three in some cases uint8_t numImmediatesConsumed; uint8_t numImmediatesTranslated; - uint64_t immediates[2]; + uint64_t immediates[3]; // A register or immediate operand encoded into the opcode Reg opcodeRegister; diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86BaseInfo.h b/llvm/lib/Target/X86/MCTargetDesc/X86BaseInfo.h index 28a067d525e0..65b8ebc0b9b9 100644 --- a/llvm/lib/Target/X86/MCTargetDesc/X86BaseInfo.h +++ b/llvm/lib/Target/X86/MCTargetDesc/X86BaseInfo.h @@ -875,7 +875,10 @@ enum : uint64_t { ExplicitOpPrefixMask = 3ULL << ExplicitOpPrefixShift, /// EVEX_NF - Set if this instruction has EVEX.NF field set. EVEX_NFShift = ExplicitOpPrefixShift + 2, - EVEX_NF = 1ULL << EVEX_NFShift + EVEX_NF = 1ULL << EVEX_NFShift, + // TwoConditionalOps - Set if this instruction has two conditional operands + TwoConditionalOps_Shift = EVEX_NFShift + 1, + TwoConditionalOps = 1ULL << TwoConditionalOps_Shift }; /// \returns true if the instruction with given opcode is a prefix. diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86InstPrinterCommon.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86InstPrinterCommon.cpp index 7422a9897346..fd46e4e1df82 100644 --- a/llvm/lib/Target/X86/MCTargetDesc/X86InstPrinterCommon.cpp +++ b/llvm/lib/Target/X86/MCTargetDesc/X86InstPrinterCommon.cpp @@ -29,7 +29,9 @@ using namespace llvm; void X86InstPrinterCommon::printCondCode(const MCInst *MI, unsigned Op, raw_ostream &O) { int64_t Imm = MI->getOperand(Op).getImm(); - bool IsCMPCCXADD = X86::isCMPCCXADD(MI->getOpcode()); + unsigned Opc = MI->getOpcode(); + bool IsCMPCCXADD = X86::isCMPCCXADD(Opc); + bool IsCCMPOrCTEST = X86::isCCMPCC(Opc) || X86::isCTESTCC(Opc); // clang-format off switch (Imm) { @@ -44,8 +46,8 @@ void X86InstPrinterCommon::printCondCode(const MCInst *MI, unsigned Op, case 7: O << (IsCMPCCXADD ? "nbe" : "a"); break; case 8: O << "s"; break; case 9: O << "ns"; break; - case 0xa: O << "p"; break; - case 0xb: O << "np"; break; + case 0xa: O << (IsCCMPOrCTEST ? "t" : "p"); break; + case 0xb: O << (IsCCMPOrCTEST ? "f" : "np"); break; case 0xc: O << "l"; break; case 0xd: O << (IsCMPCCXADD ? "nl" : "ge"); break; case 0xe: O << "le"; break; @@ -54,6 +56,27 @@ void X86InstPrinterCommon::printCondCode(const MCInst *MI, unsigned Op, // clang-format on } +void X86InstPrinterCommon::printCondFlags(const MCInst *MI, unsigned Op, + raw_ostream &O) { + // +----+----+----+----+ + // | OF | SF | ZF | CF | + // +----+----+----+----+ + int64_t Imm = MI->getOperand(Op).getImm(); + assert(Imm >= 0 && Imm < 16 && "Invalid condition flags"); + O << "{dfv="; + std::string Flags; + if (Imm & 0x8) + Flags += "of,"; + if (Imm & 0x4) + Flags += "sf,"; + if (Imm & 0x2) + Flags += "zf,"; + if (Imm & 0x1) + Flags += "cf,"; + StringRef SimplifiedFlags = StringRef(Flags).rtrim(","); + O << SimplifiedFlags << "}"; +} + void X86InstPrinterCommon::printSSEAVXCC(const MCInst *MI, unsigned Op, raw_ostream &O) { int64_t Imm = MI->getOperand(Op).getImm(); diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86InstPrinterCommon.h b/llvm/lib/Target/X86/MCTargetDesc/X86InstPrinterCommon.h index 0cb5bf014b20..221102e17c65 100644 --- a/llvm/lib/Target/X86/MCTargetDesc/X86InstPrinterCommon.h +++ b/llvm/lib/Target/X86/MCTargetDesc/X86InstPrinterCommon.h @@ -24,6 +24,7 @@ public: virtual void printOperand(const MCInst *MI, unsigned OpNo, raw_ostream &O) = 0; void printCondCode(const MCInst *MI, unsigned Op, raw_ostream &OS); + void printCondFlags(const MCInst *MI, unsigned Op, raw_ostream &OS); void printSSEAVXCC(const MCInst *MI, unsigned Op, raw_ostream &OS); void printVPCOMMnemonic(const MCInst *MI, raw_ostream &OS); void printVPCMPMnemonic(const MCInst *MI, raw_ostream &OS); diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp index 1fa676eeb79b..dcd0551b7b76 100644 --- a/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp +++ b/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp @@ -200,8 +200,12 @@ public: void setB(const MCInst &MI, unsigned OpNum) { B = getRegEncoding(MI, OpNum) >> 3 & 1; } - void set4V(const MCInst &MI, unsigned OpNum) { - set4V(getRegEncoding(MI, OpNum)); + void set4V(const MCInst &MI, unsigned OpNum, bool IsImm = false) { + // OF, SF, ZF and CF reuse VEX_4V bits but are not reversed + if (IsImm) + set4V(~(MI.getOperand(OpNum).getImm())); + else + set4V(getRegEncoding(MI, OpNum)); } void setL(bool V) { VEX_L = V; } void setPP(unsigned V) { VEX_PP = V; } @@ -252,6 +256,11 @@ public: EVEX_aaa = getRegEncoding(MI, OpNum); } void setNF(bool V) { EVEX_aaa |= V << 2; } + void setSC(const MCInst &MI, unsigned OpNum) { + unsigned Encoding = MI.getOperand(OpNum).getImm(); + EVEX_V2 = ~(Encoding >> 3) & 0x1; + EVEX_aaa = Encoding & 0x7; + } X86OpcodePrefixHelper(const MCRegisterInfo &MRI) : W(0), R(0), X(0), B(0), M(0), R2(0), X2(0), B2(0), VEX_4V(0), VEX_L(0), @@ -1045,6 +1054,7 @@ X86MCCodeEmitter::emitVEXOpcodePrefix(int MemOperand, const MCInst &MI, uint8_t EVEX_rc = 0; unsigned CurOp = X86II::getOperandBias(Desc); + bool HasTwoConditionalOps = TSFlags & X86II::TwoConditionalOps; switch (TSFlags & X86II::FormMask) { default: @@ -1086,6 +1096,10 @@ X86MCCodeEmitter::emitVEXOpcodePrefix(int MemOperand, const MCInst &MI, Prefix.set4VV2(MI, CurOp++); Prefix.setRR2(MI, CurOp++); + if (HasTwoConditionalOps) { + Prefix.set4V(MI, CurOp++, /*IsImm=*/true); + Prefix.setSC(MI, CurOp++); + } break; } case X86II::MRMSrcMemFSIB: @@ -1115,7 +1129,11 @@ X86MCCodeEmitter::emitVEXOpcodePrefix(int MemOperand, const MCInst &MI, Prefix.setBB2(MI, MemOperand + X86::AddrBaseReg); Prefix.setXX2(MI, MemOperand + X86::AddrIndexReg); Prefix.setV2(MI, MemOperand + X86::AddrIndexReg, HasVEX_4V); - + CurOp += X86::AddrNumOperands; + if (HasTwoConditionalOps) { + Prefix.set4V(MI, CurOp++, /*IsImm=*/true); + Prefix.setSC(MI, CurOp++); + } break; } case X86II::MRMSrcMem4VOp3: { @@ -1155,7 +1173,11 @@ X86MCCodeEmitter::emitVEXOpcodePrefix(int MemOperand, const MCInst &MI, Prefix.setBB2(MI, MemOperand + X86::AddrBaseReg); Prefix.setXX2(MI, MemOperand + X86::AddrIndexReg); Prefix.setV2(MI, MemOperand + X86::AddrIndexReg, HasVEX_4V); - + CurOp += X86::AddrNumOperands + 1; // Skip first imm. + if (HasTwoConditionalOps) { + Prefix.set4V(MI, CurOp++, /*IsImm=*/true); + Prefix.setSC(MI, CurOp++); + } break; } case X86II::MRMSrcReg: { @@ -1183,6 +1205,11 @@ X86MCCodeEmitter::emitVEXOpcodePrefix(int MemOperand, const MCInst &MI, Prefix.setX(MI, CurOp, 4); ++CurOp; + if (HasTwoConditionalOps) { + Prefix.set4V(MI, CurOp++, /*IsImm=*/true); + Prefix.setSC(MI, CurOp++); + } + if (TSFlags & X86II::EVEX_B) { if (HasEVEX_RC) { unsigned NumOps = Desc.getNumOperands(); @@ -1236,6 +1263,10 @@ X86MCCodeEmitter::emitVEXOpcodePrefix(int MemOperand, const MCInst &MI, Prefix.set4VV2(MI, CurOp++); Prefix.setRR2(MI, CurOp++); + if (HasTwoConditionalOps) { + Prefix.set4V(MI, CurOp++, /*IsImm=*/true); + Prefix.setSC(MI, CurOp++); + } if (TSFlags & X86II::EVEX_B) EncodeRC = true; break; @@ -1266,6 +1297,10 @@ X86MCCodeEmitter::emitVEXOpcodePrefix(int MemOperand, const MCInst &MI, Prefix.setBB2(MI, CurOp); Prefix.setX(MI, CurOp, 4); ++CurOp; + if (HasTwoConditionalOps) { + Prefix.set4V(MI, ++CurOp, /*IsImm=*/true); + Prefix.setSC(MI, ++CurOp); + } break; } } @@ -1525,6 +1560,7 @@ void X86MCCodeEmitter::encodeInstruction(const MCInst &MI, unsigned OpcodeOffset = 0; bool IsND = X86II::hasNewDataDest(TSFlags); + bool HasTwoConditionalOps = TSFlags & X86II::TwoConditionalOps; uint64_t Form = TSFlags & X86II::FormMask; switch (Form) { @@ -1914,11 +1950,16 @@ void X86MCCodeEmitter::encodeInstruction(const MCInst &MI, // If there is a remaining operand, it must be a trailing immediate. Emit it // according to the right size for the instruction. Some instructions // (SSE4a extrq and insertq) have two trailing immediates. - while (CurOp != NumOps && NumOps - CurOp <= 2) { + + // Skip two trainling conditional operands encoded in EVEX prefix + unsigned RemaningOps = NumOps - CurOp - 2 * HasTwoConditionalOps; + while (RemaningOps) { emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags), StartByte, CB, Fixups); + --RemaningOps; } + CurOp += 2 * HasTwoConditionalOps; } if ((TSFlags & X86II::OpMapMask) == X86II::ThreeDNow) diff --git a/llvm/lib/Target/X86/X86InstrAsmAlias.td b/llvm/lib/Target/X86/X86InstrAsmAlias.td index 2590be8651d5..fab058dc390b 100644 --- a/llvm/lib/Target/X86/X86InstrAsmAlias.td +++ b/llvm/lib/Target/X86/X86InstrAsmAlias.td @@ -62,6 +62,134 @@ multiclass CMPCCXADD_Aliases { (CMPCCXADDmr64_EVEX GR64:$dst, i64mem:$dstsrc2, GR64:$src3, CC), 0>; } +// CCMP Instructions Alias +multiclass CCMP_Aliases { +let Predicates = [In64BitMode] in { +def : InstAlias<"ccmp"#Cond#"{b} $dcf\t{$src2, $src1|$src1, $src2}", + (CCMP8rr GR8:$src1, GR8:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ccmp"#Cond#"{w} $dcf\t{$src2, $src1|$src1, $src2}", + (CCMP16rr GR16:$src1, GR16:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ccmp"#Cond#"{l} $dcf\t{$src2, $src1|$src1, $src2}", + (CCMP32rr GR32:$src1, GR32:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ccmp"#Cond#"{q} $dcf\t{$src2, $src1|$src1, $src2}", + (CCMP64rr GR64:$src1, GR64:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ccmp"#Cond#"{b} $dcf\t{$src2, $src1|$src1, $src2}", + (CCMP8rm GR8:$src1, i8mem:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ccmp"#Cond#"{w} $dcf\t{$src2, $src1|$src1, $src2}", + (CCMP16rm GR16:$src1, i16mem:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ccmp"#Cond#"{l} $dcf\t{$src2, $src1|$src1, $src2}", + (CCMP32rm GR32:$src1, i32mem:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ccmp"#Cond#"{q} $dcf\t{$src2, $src1|$src1, $src2}", + (CCMP64rm GR64:$src1, i64mem:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ccmp"#Cond#"{b} $dcf\t{$src2, $src1|$src1, $src2}", + (CCMP8mr i8mem:$src1, GR8:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ccmp"#Cond#"{w} $dcf\t{$src2, $src1|$src1, $src2}", + (CCMP16mr i16mem:$src1, GR16:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ccmp"#Cond#"{l} $dcf\t{$src2, $src1|$src1, $src2}", + (CCMP32mr i32mem:$src1, GR32:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ccmp"#Cond#"{q} $dcf\t{$src2, $src1|$src1, $src2}", + (CCMP64mr i64mem:$src1, GR64:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ccmp"#Cond#"{b} $dcf\t{$src2, $src1|$src1, $src2}", + (CCMP8ri GR8:$src1, i8imm:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ccmp"#Cond#"{w} $dcf\t{$src2, $src1|$src1, $src2}", + (CCMP16ri GR16:$src1, i16imm:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ccmp"#Cond#"{l} $dcf\t{$src2, $src1|$src1, $src2}", + (CCMP32ri GR32:$src1, i32imm:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ccmp"#Cond#"{q} $dcf\t{$src2, $src1|$src1, $src2}", + (CCMP64ri32 GR64:$src1, i64i32imm:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ccmp"#Cond#"{w} $dcf\t{$src2, $src1|$src1, $src2}", + (CCMP16ri8 GR16:$src1, i16i8imm:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ccmp"#Cond#"{l} $dcf\t{$src2, $src1|$src1, $src2}", + (CCMP32ri8 GR32:$src1, i32i8imm:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ccmp"#Cond#"{q} $dcf\t{$src2, $src1|$src1, $src2}", + (CCMP64ri8 GR64:$src1, i64i8imm:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ccmp"#Cond#"{b} $dcf\t{$src2, $src1|$src1, $src2}", + (CCMP8mi i8mem:$src1, i8imm:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ccmp"#Cond#"{w} $dcf\t{$src2, $src1|$src1, $src2}", + (CCMP16mi i16mem:$src1, i16imm:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ccmp"#Cond#"{l} $dcf\t{$src2, $src1|$src1, $src2}", + (CCMP32mi i32mem:$src1, i32imm:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ccmp"#Cond#"{q} $dcf\t{$src2, $src1|$src1, $src2}", + (CCMP64mi32 i64mem:$src1, i64i32imm:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ccmp"#Cond#"{w} $dcf\t{$src2, $src1|$src1, $src2}", + (CCMP16mi8 i16mem:$src1, i16i8imm:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ccmp"#Cond#"{l} $dcf\t{$src2, $src1|$src1, $src2}", + (CCMP32mi8 i32mem:$src1, i32i8imm:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ccmp"#Cond#"{q} $dcf\t{$src2, $src1|$src1, $src2}", + (CCMP64mi8 i64mem:$src1, i64i8imm:$src2, cflags:$dcf, CC), 0>; +} +} +defm : CCMP_Aliases<"o" , 0>; +defm : CCMP_Aliases<"no", 1>; +defm : CCMP_Aliases<"b" , 2>; +defm : CCMP_Aliases<"ae", 3>; +defm : CCMP_Aliases<"e" , 4>; +defm : CCMP_Aliases<"ne", 5>; +defm : CCMP_Aliases<"be", 6>; +defm : CCMP_Aliases<"a" , 7>; +defm : CCMP_Aliases<"s" , 8>; +defm : CCMP_Aliases<"ns", 9>; +defm : CCMP_Aliases<"t" , 10>; +defm : CCMP_Aliases<"f", 11>; +defm : CCMP_Aliases<"l" , 12>; +defm : CCMP_Aliases<"ge", 13>; +defm : CCMP_Aliases<"le", 14>; +defm : CCMP_Aliases<"g" , 15>; + +// CTEST Instructions Alias +multiclass CTEST_Aliases { +let Predicates = [In64BitMode] in { +def : InstAlias<"ctest"#Cond#"{b} $dcf\t{$src2, $src1|$src1, $src2}", + (CTEST8rr GR8:$src1, GR8:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ctest"#Cond#"{w} $dcf\t{$src2, $src1|$src1, $src2}", + (CTEST16rr GR16:$src1, GR16:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ctest"#Cond#"{l} $dcf\t{$src2, $src1|$src1, $src2}", + (CTEST32rr GR32:$src1, GR32:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ctest"#Cond#"{q} $dcf\t{$src2, $src1|$src1, $src2}", + (CTEST64rr GR64:$src1, GR64:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ctest"#Cond#"{b} $dcf\t{$src2, $src1|$src1, $src2}", + (CTEST8mr i8mem:$src1, GR8:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ctest"#Cond#"{w} $dcf\t{$src2, $src1|$src1, $src2}", + (CTEST16mr i16mem:$src1, GR16:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ctest"#Cond#"{l} $dcf\t{$src2, $src1|$src1, $src2}", + (CTEST32mr i32mem:$src1, GR32:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ctest"#Cond#"{q} $dcf\t{$src2, $src1|$src1, $src2}", + (CTEST64mr i64mem:$src1, GR64:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ctest"#Cond#"{b} $dcf\t{$src2, $src1|$src1, $src2}", + (CTEST8ri GR8:$src1, i8imm:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ctest"#Cond#"{w} $dcf\t{$src2, $src1|$src1, $src2}", + (CTEST16ri GR16:$src1, i16imm:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ctest"#Cond#"{l} $dcf\t{$src2, $src1|$src1, $src2}", + (CTEST32ri GR32:$src1, i32imm:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ctest"#Cond#"{q} $dcf\t{$src2, $src1|$src1, $src2}", + (CTEST64ri32 GR64:$src1, i64i32imm:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ctest"#Cond#"{b} $dcf\t{$src2, $src1|$src1, $src2}", + (CTEST8mi i8mem:$src1, i8imm:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ctest"#Cond#"{w} $dcf\t{$src2, $src1|$src1, $src2}", + (CTEST16mi i16mem:$src1, i16imm:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ctest"#Cond#"{l} $dcf\t{$src2, $src1|$src1, $src2}", + (CTEST32mi i32mem:$src1, i32imm:$src2, cflags:$dcf, CC), 0>; +def : InstAlias<"ctest"#Cond#"{q} $dcf\t{$src2, $src1|$src1, $src2}", + (CTEST64mi32 i64mem:$src1, i64i32imm:$src2, cflags:$dcf, CC), 0>; +} +} +defm : CTEST_Aliases<"o" , 0>; +defm : CTEST_Aliases<"no", 1>; +defm : CTEST_Aliases<"b" , 2>; +defm : CTEST_Aliases<"ae", 3>; +defm : CTEST_Aliases<"e" , 4>; +defm : CTEST_Aliases<"ne", 5>; +defm : CTEST_Aliases<"be", 6>; +defm : CTEST_Aliases<"a" , 7>; +defm : CTEST_Aliases<"s" , 8>; +defm : CTEST_Aliases<"ns", 9>; +defm : CTEST_Aliases<"t" , 10>; +defm : CTEST_Aliases<"f", 11>; +defm : CTEST_Aliases<"l" , 12>; +defm : CTEST_Aliases<"ge", 13>; +defm : CTEST_Aliases<"le", 14>; +defm : CTEST_Aliases<"g" , 15>; + //===----------------------------------------------------------------------===// // Assembler Mnemonic Aliases //===----------------------------------------------------------------------===// diff --git a/llvm/lib/Target/X86/X86InstrConditionalCompare.td b/llvm/lib/Target/X86/X86InstrConditionalCompare.td new file mode 100644 index 000000000000..e5c1143eba87 --- /dev/null +++ b/llvm/lib/Target/X86/X86InstrConditionalCompare.td @@ -0,0 +1,110 @@ +//===-- X86InstrConditionalCompare.td - Conditional Compare --*- tablegen -*-==// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file describes the X86 conditional compare instructions. +// +//===----------------------------------------------------------------------===// + +class BinCondOp o, Format f, X86TypeInfo t, DAGOperand op1, DAGOperand op2, string m> + : ITy, T_MAP4, EVEX, Requires<[In64BitMode]> { + let isCodeGenOnly = 1; + let ForceDisassemble = 1; + let Uses = [EFLAGS]; + let Defs = [EFLAGS]; + let hasTwoConditionalOps = 1; + let ImmT = !if(!eq(op2, i16i8imm), Imm8, + !if(!eq(op2, i32i8imm), Imm8, + !if(!eq(op2, i64i8imm), Imm8, + !if(!eq(op2, i8imm), Imm8, + !if(!eq(op2, i16imm), Imm16, + !if(!eq(op2, i32imm), Imm32, + !if(!eq(op2, i64i32imm), Imm32S, NoImm))))))); +} + +class Ccmp o, Format f, X86TypeInfo t, DAGOperand op1, DAGOperand op2>: + BinCondOp; + +class Ctest o, Format f, X86TypeInfo t, DAGOperand op1, DAGOperand op2>: + BinCondOp; + +//===----------------------------------------------------------------------===// +// CCMP Instructions +// +let SchedRW = [WriteALU] in { + def CCMP8rr : Ccmp<0x38, MRMDestReg, Xi8, GR8, GR8>; + def CCMP16rr: Ccmp<0x39, MRMDestReg, Xi16, GR16, GR16>, PD; + def CCMP32rr: Ccmp<0x39, MRMDestReg, Xi32, GR32, GR32>; + def CCMP64rr: Ccmp<0x39, MRMDestReg, Xi64, GR64, GR64>; + def CCMP8rr_REV : Ccmp<0x3a, MRMSrcReg, Xi8, GR8, GR8>; + def CCMP16rr_REV: Ccmp<0x3b, MRMSrcReg, Xi16, GR16, GR16>, PD; + def CCMP32rr_REV: Ccmp<0x3b, MRMSrcReg, Xi32, GR32, GR32>; + def CCMP64rr_REV: Ccmp<0x3b, MRMSrcReg, Xi64, GR64, GR64>; + def CCMP16ri8: Ccmp<0x83, MRM7r, Xi16, GR16, i16i8imm>, PD; + def CCMP32ri8: Ccmp<0x83, MRM7r, Xi32, GR32, i32i8imm>; + def CCMP64ri8: Ccmp<0x83, MRM7r, Xi64, GR64, i64i8imm>; + + def CCMP8ri : Ccmp<0x80, MRM7r, Xi8, GR8, i8imm>; + def CCMP16ri: Ccmp<0x81, MRM7r, Xi16, GR16, i16imm>, PD; + def CCMP32ri: Ccmp<0x81, MRM7r, Xi32, GR32, i32imm>; + def CCMP64ri32: Ccmp<0x81, MRM7r, Xi64, GR64, i64i32imm>; +} + +let mayLoad = 1 in { + let SchedRW = [WriteALU.Folded] in { + def CCMP16mi8: Ccmp<0x83, MRM7m, Xi16, i16mem, i16i8imm>, PD; + def CCMP32mi8: Ccmp<0x83, MRM7m, Xi32, i32mem, i32i8imm>; + def CCMP64mi8: Ccmp<0x83, MRM7m, Xi64, i64mem, i64i8imm>; + def CCMP8mi : Ccmp<0x80, MRM7m, Xi8, i8mem, i8imm>; + def CCMP16mi: Ccmp<0x81, MRM7m, Xi16, i16mem, i16imm>, PD; + def CCMP32mi: Ccmp<0x81, MRM7m, Xi32, i32mem, i32imm>; + def CCMP64mi32: Ccmp<0x81, MRM7m, Xi64, i64mem, i64i32imm>; + } + let SchedRW = [WriteALU.Folded, WriteALU.ReadAfterFold] in { + def CCMP8rm : Ccmp<0x3a, MRMSrcMem, Xi8, GR8, i8mem>; + def CCMP16rm: Ccmp<0x3b, MRMSrcMem, Xi16, GR16, i16mem>, PD; + def CCMP32rm: Ccmp<0x3b, MRMSrcMem, Xi32, GR32, i32mem>; + def CCMP64rm: Ccmp<0x3b, MRMSrcMem, Xi64, GR64, i64mem>; + + def CCMP8mr : Ccmp<0x38, MRMDestMem, Xi8, i8mem, GR8>; + def CCMP16mr: Ccmp<0x39, MRMDestMem, Xi16, i16mem, GR16>, PD; + def CCMP32mr: Ccmp<0x39, MRMDestMem, Xi32, i32mem, GR32>; + def CCMP64mr: Ccmp<0x39, MRMDestMem, Xi64, i64mem, GR64>; + } +} + +//===----------------------------------------------------------------------===// +// CTEST Instructions +// +let SchedRW = [WriteALU] in { + let isCommutable = 1 in { + def CTEST8rr : Ctest<0x84, MRMDestReg, Xi8, GR8, GR8>; + def CTEST16rr: Ctest<0x85, MRMDestReg, Xi16, GR16, GR16>, PD; + def CTEST32rr: Ctest<0x85, MRMDestReg, Xi32, GR32, GR32>; + def CTEST64rr: Ctest<0x85, MRMDestReg, Xi64, GR64, GR64>; + } + def CTEST8ri : Ctest<0xF6, MRM0r, Xi8, GR8, i8imm>; + def CTEST16ri: Ctest<0xF7, MRM0r, Xi16, GR16, i16imm>, PD; + def CTEST32ri: Ctest<0xF7, MRM0r, Xi32, GR32, i32imm>; + def CTEST64ri32: Ctest<0xF7, MRM0r, Xi64, GR64, i64i32imm>; +} + +let mayLoad = 1 in { + let SchedRW = [WriteALU.Folded] in { + def CTEST8mi : Ctest<0xF6, MRM0m, Xi8, i8mem, i8imm>; + def CTEST16mi: Ctest<0xF7, MRM0m, Xi16, i16mem, i16imm>, PD; + def CTEST32mi: Ctest<0xF7, MRM0m, Xi32, i32mem, i32imm>; + def CTEST64mi32: Ctest<0xF7, MRM0m, Xi64, i64mem, i64i32imm>; + } + let SchedRW = [WriteALU.Folded, WriteALU.ReadAfterFold] in { + def CTEST8mr : Ctest<0x84, MRMDestMem, Xi8, i8mem, GR8>; + def CTEST16mr: Ctest<0x85, MRMDestMem, Xi16, i16mem, GR16>, PD; + def CTEST32mr: Ctest<0x85, MRMDestMem, Xi32, i32mem, GR32>; + def CTEST64mr: Ctest<0x85, MRMDestMem, Xi64, i64mem, GR64>; + } +} diff --git a/llvm/lib/Target/X86/X86InstrFormats.td b/llvm/lib/Target/X86/X86InstrFormats.td index 8798b13a1761..13014293267b 100644 --- a/llvm/lib/Target/X86/X86InstrFormats.td +++ b/llvm/lib/Target/X86/X86InstrFormats.td @@ -255,6 +255,7 @@ class X86Inst opcod, Format f, ImmType i, dag outs, dag ins, bit hasEVEX_L2 = 0; // Does this inst set the EVEX_L2 field? bit hasEVEX_B = 0; // Does this inst set the EVEX_B field? bit hasEVEX_NF = 0; // Does this inst set the EVEX_NF field? + bit hasTwoConditionalOps = 0; // Does this inst have two conditional operands? bits<3> CD8_Form = 0; // Compressed disp8 form - vector-width. // Declare it int rather than bits<4> so that all bits are defined when // assigning to bits<7>. @@ -305,4 +306,5 @@ class X86Inst opcod, Format f, ImmType i, dag outs, dag ins, let TSFlags{49} = hasNoTrackPrefix; let TSFlags{51-50} = explicitOpPrefixBits; let TSFlags{52} = hasEVEX_NF; + let TSFlags{53} = hasTwoConditionalOps; } diff --git a/llvm/lib/Target/X86/X86InstrInfo.td b/llvm/lib/Target/X86/X86InstrInfo.td index ee54796323b8..479278433610 100644 --- a/llvm/lib/Target/X86/X86InstrInfo.td +++ b/llvm/lib/Target/X86/X86InstrInfo.td @@ -45,6 +45,7 @@ include "X86InstrMisc.td" include "X86InstrTBM.td" include "X86InstrArithmetic.td" include "X86InstrCMovSetCC.td" +include "X86InstrConditionalCompare.td" include "X86InstrExtension.td" include "X86InstrControl.td" include "X86InstrShiftRotate.td" diff --git a/llvm/lib/Target/X86/X86InstrOperands.td b/llvm/lib/Target/X86/X86InstrOperands.td index 761458f9cffc..f8f5cd83166e 100644 --- a/llvm/lib/Target/X86/X86InstrOperands.td +++ b/llvm/lib/Target/X86/X86InstrOperands.td @@ -424,6 +424,12 @@ def u4imm : Operand { let OperandType = "OPERAND_IMMEDIATE"; } +def cflags : Operand { + let PrintMethod = "printCondFlags"; + let ParserMatchClass = ImmUnsignedi4AsmOperand; + let OperandType = "OPERAND_IMMEDIATE"; +} + // Unsigned 8-bit immediate used by SSE/AVX instructions. def u8imm : Operand { let PrintMethod = "printU8Imm"; diff --git a/llvm/test/MC/Disassembler/X86/apx/ccmp.txt b/llvm/test/MC/Disassembler/X86/apx/ccmp.txt new file mode 100644 index 000000000000..d2656d62972d --- /dev/null +++ b/llvm/test/MC/Disassembler/X86/apx/ccmp.txt @@ -0,0 +1,1598 @@ +# RUN: llvm-mc -triple x86_64 -disassemble %s | FileCheck %s --check-prefix=ATT +# RUN: llvm-mc -triple x86_64 -disassemble -output-asm-variant=1 %s | FileCheck %s --check-prefix=INTEL + +## Condition flags + +# ATT: ccmpoq {dfv=} %rax, %rbx +# INTEL: ccmpo {dfv=} rbx, rax +0x62,0xf4,0x84,0x00,0x39,0xc3 + +# ATT: ccmpoq {dfv=of} %rax, %rbx +# INTEL: ccmpo {dfv=of} rbx, rax +0x62,0xf4,0xc4,0x00,0x39,0xc3 + +# ATT: ccmpoq {dfv=sf} %rax, %rbx +# INTEL: ccmpo {dfv=sf} rbx, rax +0x62,0xf4,0xa4,0x00,0x39,0xc3 + +# ATT: ccmpoq {dfv=zf} %rax, %rbx +# INTEL: ccmpo {dfv=zf} rbx, rax +0x62,0xf4,0x94,0x00,0x39,0xc3 + +# ATT: ccmpoq {dfv=cf} %rax, %rbx +# INTEL: ccmpo {dfv=cf} rbx, rax +0x62,0xf4,0x8c,0x00,0x39,0xc3 + +# ATT: ccmpoq {dfv=of,sf} %rax, %rbx +# INTEL: ccmpo {dfv=of,sf} rbx, rax +0x62,0xf4,0xe4,0x00,0x39,0xc3 + +# ATT: ccmpoq {dfv=of,sf,zf} %rax, %rbx +# INTEL: ccmpo {dfv=of,sf,zf} rbx, rax +0x62,0xf4,0xf4,0x00,0x39,0xc3 + +# ATT: ccmpoq {dfv=of,sf,zf,cf} %rax, %rbx +# INTEL: ccmpo {dfv=of,sf,zf,cf} rbx, rax +0x62,0xf4,0xfc,0x00,0x39,0xc3 + +## Condition code + +# ATT: ccmpbb {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpb {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x02,0x80,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpbw {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpb {dfv=of} word ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x45,0x02,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpbw {dfv=of} $1234, 123(%r8,%rax,4) +# INTEL: ccmpb {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +0x62,0xd4,0x45,0x02,0x81,0x7c,0x80,0x7b,0xd2,0x04 + +# ATT: ccmpbl {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpb {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x02,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpbl {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmpb {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0x44,0x02,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmpbq {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpb {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0xc4,0x02,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpbq {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmpb {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0xc4,0x02,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmpbb {dfv=of} %bl, 123(%r8,%rax,4) +# INTEL: ccmpb {dfv=of} byte ptr [r8 + 4*rax + 123], bl +0x62,0xd4,0x44,0x02,0x38,0x5c,0x80,0x7b + +# ATT: ccmpbw {dfv=of} %dx, 123(%r8,%rax,4) +# INTEL: ccmpb {dfv=of} word ptr [r8 + 4*rax + 123], dx +0x62,0xd4,0x45,0x02,0x39,0x54,0x80,0x7b + +# ATT: ccmpbl {dfv=of} %ecx, 123(%r8,%rax,4) +# INTEL: ccmpb {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +0x62,0xd4,0x44,0x02,0x39,0x4c,0x80,0x7b + +# ATT: ccmpbq {dfv=of} %r9, 123(%r8,%rax,4) +# INTEL: ccmpb {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +0x62,0x54,0xc4,0x02,0x39,0x4c,0x80,0x7b + +# ATT: ccmpbb {dfv=of} 123(%r8,%rax,4), %bl +# INTEL: ccmpb {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x02,0x3a,0x5c,0x80,0x7b + +# ATT: ccmpbw {dfv=of} 123(%r8,%rax,4), %dx +# INTEL: ccmpb {dfv=of} dx, word ptr [r8 + 4*rax + 123] +0x62,0xd4,0x45,0x02,0x3b,0x54,0x80,0x7b + +# ATT: ccmpbl {dfv=of} 123(%r8,%rax,4), %ecx +# INTEL: ccmpb {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x02,0x3b,0x4c,0x80,0x7b + +# ATT: ccmpbq {dfv=of} 123(%r8,%rax,4), %r9 +# INTEL: ccmpb {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +0x62,0x54,0xc4,0x02,0x3b,0x4c,0x80,0x7b + +# ATT: ccmpbb {dfv=of} $123, %bl +# INTEL: ccmpb {dfv=of} bl, 123 +0x62,0xf4,0x44,0x02,0x80,0xfb,0x7b + +# ATT: ccmpbw {dfv=of} $123, %dx +# INTEL: ccmpb {dfv=of} dx, 123 +0x62,0xf4,0x45,0x02,0x83,0xfa,0x7b + +# ATT: ccmpbl {dfv=of} $123, %ecx +# INTEL: ccmpb {dfv=of} ecx, 123 +0x62,0xf4,0x44,0x02,0x83,0xf9,0x7b + +# ATT: ccmpbq {dfv=of} $123, %r9 +# INTEL: ccmpb {dfv=of} r9, 123 +0x62,0xd4,0xc4,0x02,0x83,0xf9,0x7b + +# ATT: ccmpbw {dfv=of} $1234, %dx +# INTEL: ccmpb {dfv=of} dx, 1234 +0x62,0xf4,0x45,0x02,0x81,0xfa,0xd2,0x04 + +# ATT: ccmpbl {dfv=of} $123456, %ecx +# INTEL: ccmpb {dfv=of} ecx, 123456 +0x62,0xf4,0x44,0x02,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmpbq {dfv=of} $123456, %r9 +# INTEL: ccmpb {dfv=of} r9, 123456 +0x62,0xd4,0xc4,0x02,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmpbb {dfv=of} %bl, %dl +# INTEL: ccmpb {dfv=of} dl, bl +0x62,0xf4,0x44,0x02,0x3a,0xd3 + +# ATT: ccmpbw {dfv=of} %dx, %ax +# INTEL: ccmpb {dfv=of} ax, dx +0x62,0xf4,0x45,0x02,0x3b,0xc2 + +# ATT: ccmpbl {dfv=of} %ecx, %edx +# INTEL: ccmpb {dfv=of} edx, ecx +0x62,0xf4,0x44,0x02,0x3b,0xd1 + +# ATT: ccmpbq {dfv=of} %r9, %r15 +# INTEL: ccmpb {dfv=of} r15, r9 +0x62,0x54,0xc4,0x02,0x3b,0xf9 + +# ATT: ccmpfb {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpf {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x0b,0x80,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpfw {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpf {dfv=of} word ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x45,0x0b,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpfw {dfv=of} $1234, 123(%r8,%rax,4) +# INTEL: ccmpf {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +0x62,0xd4,0x45,0x0b,0x81,0x7c,0x80,0x7b,0xd2,0x04 + +# ATT: ccmpfl {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpf {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x0b,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpfl {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmpf {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0x44,0x0b,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmpfq {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpf {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0xc4,0x0b,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpfq {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmpf {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0xc4,0x0b,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmpfb {dfv=of} %bl, 123(%r8,%rax,4) +# INTEL: ccmpf {dfv=of} byte ptr [r8 + 4*rax + 123], bl +0x62,0xd4,0x44,0x0b,0x38,0x5c,0x80,0x7b + +# ATT: ccmpfw {dfv=of} %dx, 123(%r8,%rax,4) +# INTEL: ccmpf {dfv=of} word ptr [r8 + 4*rax + 123], dx +0x62,0xd4,0x45,0x0b,0x39,0x54,0x80,0x7b + +# ATT: ccmpfl {dfv=of} %ecx, 123(%r8,%rax,4) +# INTEL: ccmpf {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +0x62,0xd4,0x44,0x0b,0x39,0x4c,0x80,0x7b + +# ATT: ccmpfq {dfv=of} %r9, 123(%r8,%rax,4) +# INTEL: ccmpf {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +0x62,0x54,0xc4,0x0b,0x39,0x4c,0x80,0x7b + +# ATT: ccmpfb {dfv=of} 123(%r8,%rax,4), %bl +# INTEL: ccmpf {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x0b,0x3a,0x5c,0x80,0x7b + +# ATT: ccmpfw {dfv=of} 123(%r8,%rax,4), %dx +# INTEL: ccmpf {dfv=of} dx, word ptr [r8 + 4*rax + 123] +0x62,0xd4,0x45,0x0b,0x3b,0x54,0x80,0x7b + +# ATT: ccmpfl {dfv=of} 123(%r8,%rax,4), %ecx +# INTEL: ccmpf {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x0b,0x3b,0x4c,0x80,0x7b + +# ATT: ccmpfq {dfv=of} 123(%r8,%rax,4), %r9 +# INTEL: ccmpf {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +0x62,0x54,0xc4,0x0b,0x3b,0x4c,0x80,0x7b + +# ATT: ccmpfb {dfv=of} $123, %bl +# INTEL: ccmpf {dfv=of} bl, 123 +0x62,0xf4,0x44,0x0b,0x80,0xfb,0x7b + +# ATT: ccmpfw {dfv=of} $123, %dx +# INTEL: ccmpf {dfv=of} dx, 123 +0x62,0xf4,0x45,0x0b,0x83,0xfa,0x7b + +# ATT: ccmpfl {dfv=of} $123, %ecx +# INTEL: ccmpf {dfv=of} ecx, 123 +0x62,0xf4,0x44,0x0b,0x83,0xf9,0x7b + +# ATT: ccmpfq {dfv=of} $123, %r9 +# INTEL: ccmpf {dfv=of} r9, 123 +0x62,0xd4,0xc4,0x0b,0x83,0xf9,0x7b + +# ATT: ccmpfw {dfv=of} $1234, %dx +# INTEL: ccmpf {dfv=of} dx, 1234 +0x62,0xf4,0x45,0x0b,0x81,0xfa,0xd2,0x04 + +# ATT: ccmpfl {dfv=of} $123456, %ecx +# INTEL: ccmpf {dfv=of} ecx, 123456 +0x62,0xf4,0x44,0x0b,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmpfq {dfv=of} $123456, %r9 +# INTEL: ccmpf {dfv=of} r9, 123456 +0x62,0xd4,0xc4,0x0b,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmpfb {dfv=of} %bl, %dl +# INTEL: ccmpf {dfv=of} dl, bl +0x62,0xf4,0x44,0x0b,0x3a,0xd3 + +# ATT: ccmpfw {dfv=of} %dx, %ax +# INTEL: ccmpf {dfv=of} ax, dx +0x62,0xf4,0x45,0x0b,0x3b,0xc2 + +# ATT: ccmpfl {dfv=of} %ecx, %edx +# INTEL: ccmpf {dfv=of} edx, ecx +0x62,0xf4,0x44,0x0b,0x3b,0xd1 + +# ATT: ccmpfq {dfv=of} %r9, %r15 +# INTEL: ccmpf {dfv=of} r15, r9 +0x62,0x54,0xc4,0x0b,0x3b,0xf9 + +# ATT: ccmplb {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpl {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x0c,0x80,0x7c,0x80,0x7b,0x7b + +# ATT: ccmplw {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpl {dfv=of} word ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x45,0x0c,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmplw {dfv=of} $1234, 123(%r8,%rax,4) +# INTEL: ccmpl {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +0x62,0xd4,0x45,0x0c,0x81,0x7c,0x80,0x7b,0xd2,0x04 + +# ATT: ccmpll {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpl {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x0c,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpll {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmpl {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0x44,0x0c,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmplq {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpl {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0xc4,0x0c,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmplq {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmpl {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0xc4,0x0c,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmplb {dfv=of} %bl, 123(%r8,%rax,4) +# INTEL: ccmpl {dfv=of} byte ptr [r8 + 4*rax + 123], bl +0x62,0xd4,0x44,0x0c,0x38,0x5c,0x80,0x7b + +# ATT: ccmplw {dfv=of} %dx, 123(%r8,%rax,4) +# INTEL: ccmpl {dfv=of} word ptr [r8 + 4*rax + 123], dx +0x62,0xd4,0x45,0x0c,0x39,0x54,0x80,0x7b + +# ATT: ccmpll {dfv=of} %ecx, 123(%r8,%rax,4) +# INTEL: ccmpl {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +0x62,0xd4,0x44,0x0c,0x39,0x4c,0x80,0x7b + +# ATT: ccmplq {dfv=of} %r9, 123(%r8,%rax,4) +# INTEL: ccmpl {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +0x62,0x54,0xc4,0x0c,0x39,0x4c,0x80,0x7b + +# ATT: ccmplb {dfv=of} 123(%r8,%rax,4), %bl +# INTEL: ccmpl {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x0c,0x3a,0x5c,0x80,0x7b + +# ATT: ccmplw {dfv=of} 123(%r8,%rax,4), %dx +# INTEL: ccmpl {dfv=of} dx, word ptr [r8 + 4*rax + 123] +0x62,0xd4,0x45,0x0c,0x3b,0x54,0x80,0x7b + +# ATT: ccmpll {dfv=of} 123(%r8,%rax,4), %ecx +# INTEL: ccmpl {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x0c,0x3b,0x4c,0x80,0x7b + +# ATT: ccmplq {dfv=of} 123(%r8,%rax,4), %r9 +# INTEL: ccmpl {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +0x62,0x54,0xc4,0x0c,0x3b,0x4c,0x80,0x7b + +# ATT: ccmplb {dfv=of} $123, %bl +# INTEL: ccmpl {dfv=of} bl, 123 +0x62,0xf4,0x44,0x0c,0x80,0xfb,0x7b + +# ATT: ccmplw {dfv=of} $123, %dx +# INTEL: ccmpl {dfv=of} dx, 123 +0x62,0xf4,0x45,0x0c,0x83,0xfa,0x7b + +# ATT: ccmpll {dfv=of} $123, %ecx +# INTEL: ccmpl {dfv=of} ecx, 123 +0x62,0xf4,0x44,0x0c,0x83,0xf9,0x7b + +# ATT: ccmplq {dfv=of} $123, %r9 +# INTEL: ccmpl {dfv=of} r9, 123 +0x62,0xd4,0xc4,0x0c,0x83,0xf9,0x7b + +# ATT: ccmplw {dfv=of} $1234, %dx +# INTEL: ccmpl {dfv=of} dx, 1234 +0x62,0xf4,0x45,0x0c,0x81,0xfa,0xd2,0x04 + +# ATT: ccmpll {dfv=of} $123456, %ecx +# INTEL: ccmpl {dfv=of} ecx, 123456 +0x62,0xf4,0x44,0x0c,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmplq {dfv=of} $123456, %r9 +# INTEL: ccmpl {dfv=of} r9, 123456 +0x62,0xd4,0xc4,0x0c,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmplb {dfv=of} %bl, %dl +# INTEL: ccmpl {dfv=of} dl, bl +0x62,0xf4,0x44,0x0c,0x3a,0xd3 + +# ATT: ccmplw {dfv=of} %dx, %ax +# INTEL: ccmpl {dfv=of} ax, dx +0x62,0xf4,0x45,0x0c,0x3b,0xc2 + +# ATT: ccmpll {dfv=of} %ecx, %edx +# INTEL: ccmpl {dfv=of} edx, ecx +0x62,0xf4,0x44,0x0c,0x3b,0xd1 + +# ATT: ccmplq {dfv=of} %r9, %r15 +# INTEL: ccmpl {dfv=of} r15, r9 +0x62,0x54,0xc4,0x0c,0x3b,0xf9 + +# ATT: ccmpleb {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmple {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x0e,0x80,0x7c,0x80,0x7b,0x7b + +# ATT: ccmplew {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmple {dfv=of} word ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x45,0x0e,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmplew {dfv=of} $1234, 123(%r8,%rax,4) +# INTEL: ccmple {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +0x62,0xd4,0x45,0x0e,0x81,0x7c,0x80,0x7b,0xd2,0x04 + +# ATT: ccmplel {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmple {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x0e,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmplel {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmple {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0x44,0x0e,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmpleq {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmple {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0xc4,0x0e,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpleq {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmple {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0xc4,0x0e,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmpleb {dfv=of} %bl, 123(%r8,%rax,4) +# INTEL: ccmple {dfv=of} byte ptr [r8 + 4*rax + 123], bl +0x62,0xd4,0x44,0x0e,0x38,0x5c,0x80,0x7b + +# ATT: ccmplew {dfv=of} %dx, 123(%r8,%rax,4) +# INTEL: ccmple {dfv=of} word ptr [r8 + 4*rax + 123], dx +0x62,0xd4,0x45,0x0e,0x39,0x54,0x80,0x7b + +# ATT: ccmplel {dfv=of} %ecx, 123(%r8,%rax,4) +# INTEL: ccmple {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +0x62,0xd4,0x44,0x0e,0x39,0x4c,0x80,0x7b + +# ATT: ccmpleq {dfv=of} %r9, 123(%r8,%rax,4) +# INTEL: ccmple {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +0x62,0x54,0xc4,0x0e,0x39,0x4c,0x80,0x7b + +# ATT: ccmpleb {dfv=of} 123(%r8,%rax,4), %bl +# INTEL: ccmple {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x0e,0x3a,0x5c,0x80,0x7b + +# ATT: ccmplew {dfv=of} 123(%r8,%rax,4), %dx +# INTEL: ccmple {dfv=of} dx, word ptr [r8 + 4*rax + 123] +0x62,0xd4,0x45,0x0e,0x3b,0x54,0x80,0x7b + +# ATT: ccmplel {dfv=of} 123(%r8,%rax,4), %ecx +# INTEL: ccmple {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x0e,0x3b,0x4c,0x80,0x7b + +# ATT: ccmpleq {dfv=of} 123(%r8,%rax,4), %r9 +# INTEL: ccmple {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +0x62,0x54,0xc4,0x0e,0x3b,0x4c,0x80,0x7b + +# ATT: ccmpleb {dfv=of} $123, %bl +# INTEL: ccmple {dfv=of} bl, 123 +0x62,0xf4,0x44,0x0e,0x80,0xfb,0x7b + +# ATT: ccmplew {dfv=of} $123, %dx +# INTEL: ccmple {dfv=of} dx, 123 +0x62,0xf4,0x45,0x0e,0x83,0xfa,0x7b + +# ATT: ccmplel {dfv=of} $123, %ecx +# INTEL: ccmple {dfv=of} ecx, 123 +0x62,0xf4,0x44,0x0e,0x83,0xf9,0x7b + +# ATT: ccmpleq {dfv=of} $123, %r9 +# INTEL: ccmple {dfv=of} r9, 123 +0x62,0xd4,0xc4,0x0e,0x83,0xf9,0x7b + +# ATT: ccmplew {dfv=of} $1234, %dx +# INTEL: ccmple {dfv=of} dx, 1234 +0x62,0xf4,0x45,0x0e,0x81,0xfa,0xd2,0x04 + +# ATT: ccmplel {dfv=of} $123456, %ecx +# INTEL: ccmple {dfv=of} ecx, 123456 +0x62,0xf4,0x44,0x0e,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmpleq {dfv=of} $123456, %r9 +# INTEL: ccmple {dfv=of} r9, 123456 +0x62,0xd4,0xc4,0x0e,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmpleb {dfv=of} %bl, %dl +# INTEL: ccmple {dfv=of} dl, bl +0x62,0xf4,0x44,0x0e,0x3a,0xd3 + +# ATT: ccmplew {dfv=of} %dx, %ax +# INTEL: ccmple {dfv=of} ax, dx +0x62,0xf4,0x45,0x0e,0x3b,0xc2 + +# ATT: ccmplel {dfv=of} %ecx, %edx +# INTEL: ccmple {dfv=of} edx, ecx +0x62,0xf4,0x44,0x0e,0x3b,0xd1 + +# ATT: ccmpleq {dfv=of} %r9, %r15 +# INTEL: ccmple {dfv=of} r15, r9 +0x62,0x54,0xc4,0x0e,0x3b,0xf9 + +# ATT: ccmpaeb {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpae {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x03,0x80,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpaew {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpae {dfv=of} word ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x45,0x03,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpaew {dfv=of} $1234, 123(%r8,%rax,4) +# INTEL: ccmpae {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +0x62,0xd4,0x45,0x03,0x81,0x7c,0x80,0x7b,0xd2,0x04 + +# ATT: ccmpael {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpae {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x03,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpael {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmpae {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0x44,0x03,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmpaeq {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpae {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0xc4,0x03,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpaeq {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmpae {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0xc4,0x03,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmpaeb {dfv=of} %bl, 123(%r8,%rax,4) +# INTEL: ccmpae {dfv=of} byte ptr [r8 + 4*rax + 123], bl +0x62,0xd4,0x44,0x03,0x38,0x5c,0x80,0x7b + +# ATT: ccmpaew {dfv=of} %dx, 123(%r8,%rax,4) +# INTEL: ccmpae {dfv=of} word ptr [r8 + 4*rax + 123], dx +0x62,0xd4,0x45,0x03,0x39,0x54,0x80,0x7b + +# ATT: ccmpael {dfv=of} %ecx, 123(%r8,%rax,4) +# INTEL: ccmpae {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +0x62,0xd4,0x44,0x03,0x39,0x4c,0x80,0x7b + +# ATT: ccmpaeq {dfv=of} %r9, 123(%r8,%rax,4) +# INTEL: ccmpae {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +0x62,0x54,0xc4,0x03,0x39,0x4c,0x80,0x7b + +# ATT: ccmpaeb {dfv=of} 123(%r8,%rax,4), %bl +# INTEL: ccmpae {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x03,0x3a,0x5c,0x80,0x7b + +# ATT: ccmpaew {dfv=of} 123(%r8,%rax,4), %dx +# INTEL: ccmpae {dfv=of} dx, word ptr [r8 + 4*rax + 123] +0x62,0xd4,0x45,0x03,0x3b,0x54,0x80,0x7b + +# ATT: ccmpael {dfv=of} 123(%r8,%rax,4), %ecx +# INTEL: ccmpae {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x03,0x3b,0x4c,0x80,0x7b + +# ATT: ccmpaeq {dfv=of} 123(%r8,%rax,4), %r9 +# INTEL: ccmpae {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +0x62,0x54,0xc4,0x03,0x3b,0x4c,0x80,0x7b + +# ATT: ccmpaeb {dfv=of} $123, %bl +# INTEL: ccmpae {dfv=of} bl, 123 +0x62,0xf4,0x44,0x03,0x80,0xfb,0x7b + +# ATT: ccmpaew {dfv=of} $123, %dx +# INTEL: ccmpae {dfv=of} dx, 123 +0x62,0xf4,0x45,0x03,0x83,0xfa,0x7b + +# ATT: ccmpael {dfv=of} $123, %ecx +# INTEL: ccmpae {dfv=of} ecx, 123 +0x62,0xf4,0x44,0x03,0x83,0xf9,0x7b + +# ATT: ccmpaeq {dfv=of} $123, %r9 +# INTEL: ccmpae {dfv=of} r9, 123 +0x62,0xd4,0xc4,0x03,0x83,0xf9,0x7b + +# ATT: ccmpaew {dfv=of} $1234, %dx +# INTEL: ccmpae {dfv=of} dx, 1234 +0x62,0xf4,0x45,0x03,0x81,0xfa,0xd2,0x04 + +# ATT: ccmpael {dfv=of} $123456, %ecx +# INTEL: ccmpae {dfv=of} ecx, 123456 +0x62,0xf4,0x44,0x03,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmpaeq {dfv=of} $123456, %r9 +# INTEL: ccmpae {dfv=of} r9, 123456 +0x62,0xd4,0xc4,0x03,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmpaeb {dfv=of} %bl, %dl +# INTEL: ccmpae {dfv=of} dl, bl +0x62,0xf4,0x44,0x03,0x3a,0xd3 + +# ATT: ccmpaew {dfv=of} %dx, %ax +# INTEL: ccmpae {dfv=of} ax, dx +0x62,0xf4,0x45,0x03,0x3b,0xc2 + +# ATT: ccmpael {dfv=of} %ecx, %edx +# INTEL: ccmpae {dfv=of} edx, ecx +0x62,0xf4,0x44,0x03,0x3b,0xd1 + +# ATT: ccmpaeq {dfv=of} %r9, %r15 +# INTEL: ccmpae {dfv=of} r15, r9 +0x62,0x54,0xc4,0x03,0x3b,0xf9 + +# ATT: ccmpab {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpa {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x07,0x80,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpaw {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpa {dfv=of} word ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x45,0x07,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpaw {dfv=of} $1234, 123(%r8,%rax,4) +# INTEL: ccmpa {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +0x62,0xd4,0x45,0x07,0x81,0x7c,0x80,0x7b,0xd2,0x04 + +# ATT: ccmpal {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpa {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x07,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpal {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmpa {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0x44,0x07,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmpaq {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpa {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0xc4,0x07,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpaq {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmpa {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0xc4,0x07,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmpab {dfv=of} %bl, 123(%r8,%rax,4) +# INTEL: ccmpa {dfv=of} byte ptr [r8 + 4*rax + 123], bl +0x62,0xd4,0x44,0x07,0x38,0x5c,0x80,0x7b + +# ATT: ccmpaw {dfv=of} %dx, 123(%r8,%rax,4) +# INTEL: ccmpa {dfv=of} word ptr [r8 + 4*rax + 123], dx +0x62,0xd4,0x45,0x07,0x39,0x54,0x80,0x7b + +# ATT: ccmpal {dfv=of} %ecx, 123(%r8,%rax,4) +# INTEL: ccmpa {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +0x62,0xd4,0x44,0x07,0x39,0x4c,0x80,0x7b + +# ATT: ccmpaq {dfv=of} %r9, 123(%r8,%rax,4) +# INTEL: ccmpa {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +0x62,0x54,0xc4,0x07,0x39,0x4c,0x80,0x7b + +# ATT: ccmpab {dfv=of} 123(%r8,%rax,4), %bl +# INTEL: ccmpa {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x07,0x3a,0x5c,0x80,0x7b + +# ATT: ccmpaw {dfv=of} 123(%r8,%rax,4), %dx +# INTEL: ccmpa {dfv=of} dx, word ptr [r8 + 4*rax + 123] +0x62,0xd4,0x45,0x07,0x3b,0x54,0x80,0x7b + +# ATT: ccmpal {dfv=of} 123(%r8,%rax,4), %ecx +# INTEL: ccmpa {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x07,0x3b,0x4c,0x80,0x7b + +# ATT: ccmpaq {dfv=of} 123(%r8,%rax,4), %r9 +# INTEL: ccmpa {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +0x62,0x54,0xc4,0x07,0x3b,0x4c,0x80,0x7b + +# ATT: ccmpab {dfv=of} $123, %bl +# INTEL: ccmpa {dfv=of} bl, 123 +0x62,0xf4,0x44,0x07,0x80,0xfb,0x7b + +# ATT: ccmpaw {dfv=of} $123, %dx +# INTEL: ccmpa {dfv=of} dx, 123 +0x62,0xf4,0x45,0x07,0x83,0xfa,0x7b + +# ATT: ccmpal {dfv=of} $123, %ecx +# INTEL: ccmpa {dfv=of} ecx, 123 +0x62,0xf4,0x44,0x07,0x83,0xf9,0x7b + +# ATT: ccmpaq {dfv=of} $123, %r9 +# INTEL: ccmpa {dfv=of} r9, 123 +0x62,0xd4,0xc4,0x07,0x83,0xf9,0x7b + +# ATT: ccmpaw {dfv=of} $1234, %dx +# INTEL: ccmpa {dfv=of} dx, 1234 +0x62,0xf4,0x45,0x07,0x81,0xfa,0xd2,0x04 + +# ATT: ccmpal {dfv=of} $123456, %ecx +# INTEL: ccmpa {dfv=of} ecx, 123456 +0x62,0xf4,0x44,0x07,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmpaq {dfv=of} $123456, %r9 +# INTEL: ccmpa {dfv=of} r9, 123456 +0x62,0xd4,0xc4,0x07,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmpab {dfv=of} %bl, %dl +# INTEL: ccmpa {dfv=of} dl, bl +0x62,0xf4,0x44,0x07,0x3a,0xd3 + +# ATT: ccmpaw {dfv=of} %dx, %ax +# INTEL: ccmpa {dfv=of} ax, dx +0x62,0xf4,0x45,0x07,0x3b,0xc2 + +# ATT: ccmpal {dfv=of} %ecx, %edx +# INTEL: ccmpa {dfv=of} edx, ecx +0x62,0xf4,0x44,0x07,0x3b,0xd1 + +# ATT: ccmpaq {dfv=of} %r9, %r15 +# INTEL: ccmpa {dfv=of} r15, r9 +0x62,0x54,0xc4,0x07,0x3b,0xf9 + +# ATT: ccmpgeb {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpge {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x0d,0x80,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpgew {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpge {dfv=of} word ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x45,0x0d,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpgew {dfv=of} $1234, 123(%r8,%rax,4) +# INTEL: ccmpge {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +0x62,0xd4,0x45,0x0d,0x81,0x7c,0x80,0x7b,0xd2,0x04 + +# ATT: ccmpgel {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpge {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x0d,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpgel {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmpge {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0x44,0x0d,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmpgeq {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpge {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0xc4,0x0d,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpgeq {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmpge {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0xc4,0x0d,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmpgeb {dfv=of} %bl, 123(%r8,%rax,4) +# INTEL: ccmpge {dfv=of} byte ptr [r8 + 4*rax + 123], bl +0x62,0xd4,0x44,0x0d,0x38,0x5c,0x80,0x7b + +# ATT: ccmpgew {dfv=of} %dx, 123(%r8,%rax,4) +# INTEL: ccmpge {dfv=of} word ptr [r8 + 4*rax + 123], dx +0x62,0xd4,0x45,0x0d,0x39,0x54,0x80,0x7b + +# ATT: ccmpgel {dfv=of} %ecx, 123(%r8,%rax,4) +# INTEL: ccmpge {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +0x62,0xd4,0x44,0x0d,0x39,0x4c,0x80,0x7b + +# ATT: ccmpgeq {dfv=of} %r9, 123(%r8,%rax,4) +# INTEL: ccmpge {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +0x62,0x54,0xc4,0x0d,0x39,0x4c,0x80,0x7b + +# ATT: ccmpgeb {dfv=of} 123(%r8,%rax,4), %bl +# INTEL: ccmpge {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x0d,0x3a,0x5c,0x80,0x7b + +# ATT: ccmpgew {dfv=of} 123(%r8,%rax,4), %dx +# INTEL: ccmpge {dfv=of} dx, word ptr [r8 + 4*rax + 123] +0x62,0xd4,0x45,0x0d,0x3b,0x54,0x80,0x7b + +# ATT: ccmpgel {dfv=of} 123(%r8,%rax,4), %ecx +# INTEL: ccmpge {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x0d,0x3b,0x4c,0x80,0x7b + +# ATT: ccmpgeq {dfv=of} 123(%r8,%rax,4), %r9 +# INTEL: ccmpge {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +0x62,0x54,0xc4,0x0d,0x3b,0x4c,0x80,0x7b + +# ATT: ccmpgeb {dfv=of} $123, %bl +# INTEL: ccmpge {dfv=of} bl, 123 +0x62,0xf4,0x44,0x0d,0x80,0xfb,0x7b + +# ATT: ccmpgew {dfv=of} $123, %dx +# INTEL: ccmpge {dfv=of} dx, 123 +0x62,0xf4,0x45,0x0d,0x83,0xfa,0x7b + +# ATT: ccmpgel {dfv=of} $123, %ecx +# INTEL: ccmpge {dfv=of} ecx, 123 +0x62,0xf4,0x44,0x0d,0x83,0xf9,0x7b + +# ATT: ccmpgeq {dfv=of} $123, %r9 +# INTEL: ccmpge {dfv=of} r9, 123 +0x62,0xd4,0xc4,0x0d,0x83,0xf9,0x7b + +# ATT: ccmpgew {dfv=of} $1234, %dx +# INTEL: ccmpge {dfv=of} dx, 1234 +0x62,0xf4,0x45,0x0d,0x81,0xfa,0xd2,0x04 + +# ATT: ccmpgel {dfv=of} $123456, %ecx +# INTEL: ccmpge {dfv=of} ecx, 123456 +0x62,0xf4,0x44,0x0d,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmpgeq {dfv=of} $123456, %r9 +# INTEL: ccmpge {dfv=of} r9, 123456 +0x62,0xd4,0xc4,0x0d,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmpgeb {dfv=of} %bl, %dl +# INTEL: ccmpge {dfv=of} dl, bl +0x62,0xf4,0x44,0x0d,0x3a,0xd3 + +# ATT: ccmpgew {dfv=of} %dx, %ax +# INTEL: ccmpge {dfv=of} ax, dx +0x62,0xf4,0x45,0x0d,0x3b,0xc2 + +# ATT: ccmpgel {dfv=of} %ecx, %edx +# INTEL: ccmpge {dfv=of} edx, ecx +0x62,0xf4,0x44,0x0d,0x3b,0xd1 + +# ATT: ccmpgeq {dfv=of} %r9, %r15 +# INTEL: ccmpge {dfv=of} r15, r9 +0x62,0x54,0xc4,0x0d,0x3b,0xf9 + +# ATT: ccmpgb {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpg {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x0f,0x80,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpgw {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpg {dfv=of} word ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x45,0x0f,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpgw {dfv=of} $1234, 123(%r8,%rax,4) +# INTEL: ccmpg {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +0x62,0xd4,0x45,0x0f,0x81,0x7c,0x80,0x7b,0xd2,0x04 + +# ATT: ccmpgl {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpg {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x0f,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpgl {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmpg {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0x44,0x0f,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmpgq {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpg {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0xc4,0x0f,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpgq {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmpg {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0xc4,0x0f,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmpgb {dfv=of} %bl, 123(%r8,%rax,4) +# INTEL: ccmpg {dfv=of} byte ptr [r8 + 4*rax + 123], bl +0x62,0xd4,0x44,0x0f,0x38,0x5c,0x80,0x7b + +# ATT: ccmpgw {dfv=of} %dx, 123(%r8,%rax,4) +# INTEL: ccmpg {dfv=of} word ptr [r8 + 4*rax + 123], dx +0x62,0xd4,0x45,0x0f,0x39,0x54,0x80,0x7b + +# ATT: ccmpgl {dfv=of} %ecx, 123(%r8,%rax,4) +# INTEL: ccmpg {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +0x62,0xd4,0x44,0x0f,0x39,0x4c,0x80,0x7b + +# ATT: ccmpgq {dfv=of} %r9, 123(%r8,%rax,4) +# INTEL: ccmpg {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +0x62,0x54,0xc4,0x0f,0x39,0x4c,0x80,0x7b + +# ATT: ccmpgb {dfv=of} 123(%r8,%rax,4), %bl +# INTEL: ccmpg {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x0f,0x3a,0x5c,0x80,0x7b + +# ATT: ccmpgw {dfv=of} 123(%r8,%rax,4), %dx +# INTEL: ccmpg {dfv=of} dx, word ptr [r8 + 4*rax + 123] +0x62,0xd4,0x45,0x0f,0x3b,0x54,0x80,0x7b + +# ATT: ccmpgl {dfv=of} 123(%r8,%rax,4), %ecx +# INTEL: ccmpg {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x0f,0x3b,0x4c,0x80,0x7b + +# ATT: ccmpgq {dfv=of} 123(%r8,%rax,4), %r9 +# INTEL: ccmpg {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +0x62,0x54,0xc4,0x0f,0x3b,0x4c,0x80,0x7b + +# ATT: ccmpgb {dfv=of} $123, %bl +# INTEL: ccmpg {dfv=of} bl, 123 +0x62,0xf4,0x44,0x0f,0x80,0xfb,0x7b + +# ATT: ccmpgw {dfv=of} $123, %dx +# INTEL: ccmpg {dfv=of} dx, 123 +0x62,0xf4,0x45,0x0f,0x83,0xfa,0x7b + +# ATT: ccmpgl {dfv=of} $123, %ecx +# INTEL: ccmpg {dfv=of} ecx, 123 +0x62,0xf4,0x44,0x0f,0x83,0xf9,0x7b + +# ATT: ccmpgq {dfv=of} $123, %r9 +# INTEL: ccmpg {dfv=of} r9, 123 +0x62,0xd4,0xc4,0x0f,0x83,0xf9,0x7b + +# ATT: ccmpgw {dfv=of} $1234, %dx +# INTEL: ccmpg {dfv=of} dx, 1234 +0x62,0xf4,0x45,0x0f,0x81,0xfa,0xd2,0x04 + +# ATT: ccmpgl {dfv=of} $123456, %ecx +# INTEL: ccmpg {dfv=of} ecx, 123456 +0x62,0xf4,0x44,0x0f,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmpgq {dfv=of} $123456, %r9 +# INTEL: ccmpg {dfv=of} r9, 123456 +0x62,0xd4,0xc4,0x0f,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmpgb {dfv=of} %bl, %dl +# INTEL: ccmpg {dfv=of} dl, bl +0x62,0xf4,0x44,0x0f,0x3a,0xd3 + +# ATT: ccmpgw {dfv=of} %dx, %ax +# INTEL: ccmpg {dfv=of} ax, dx +0x62,0xf4,0x45,0x0f,0x3b,0xc2 + +# ATT: ccmpgl {dfv=of} %ecx, %edx +# INTEL: ccmpg {dfv=of} edx, ecx +0x62,0xf4,0x44,0x0f,0x3b,0xd1 + +# ATT: ccmpgq {dfv=of} %r9, %r15 +# INTEL: ccmpg {dfv=of} r15, r9 +0x62,0x54,0xc4,0x0f,0x3b,0xf9 + +# ATT: ccmpnob {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpno {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x01,0x80,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpnow {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpno {dfv=of} word ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x45,0x01,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpnow {dfv=of} $1234, 123(%r8,%rax,4) +# INTEL: ccmpno {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +0x62,0xd4,0x45,0x01,0x81,0x7c,0x80,0x7b,0xd2,0x04 + +# ATT: ccmpnol {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpno {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x01,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpnol {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmpno {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0x44,0x01,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmpnoq {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpno {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0xc4,0x01,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpnoq {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmpno {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0xc4,0x01,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmpnob {dfv=of} %bl, 123(%r8,%rax,4) +# INTEL: ccmpno {dfv=of} byte ptr [r8 + 4*rax + 123], bl +0x62,0xd4,0x44,0x01,0x38,0x5c,0x80,0x7b + +# ATT: ccmpnow {dfv=of} %dx, 123(%r8,%rax,4) +# INTEL: ccmpno {dfv=of} word ptr [r8 + 4*rax + 123], dx +0x62,0xd4,0x45,0x01,0x39,0x54,0x80,0x7b + +# ATT: ccmpnol {dfv=of} %ecx, 123(%r8,%rax,4) +# INTEL: ccmpno {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +0x62,0xd4,0x44,0x01,0x39,0x4c,0x80,0x7b + +# ATT: ccmpnoq {dfv=of} %r9, 123(%r8,%rax,4) +# INTEL: ccmpno {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +0x62,0x54,0xc4,0x01,0x39,0x4c,0x80,0x7b + +# ATT: ccmpnob {dfv=of} 123(%r8,%rax,4), %bl +# INTEL: ccmpno {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x01,0x3a,0x5c,0x80,0x7b + +# ATT: ccmpnow {dfv=of} 123(%r8,%rax,4), %dx +# INTEL: ccmpno {dfv=of} dx, word ptr [r8 + 4*rax + 123] +0x62,0xd4,0x45,0x01,0x3b,0x54,0x80,0x7b + +# ATT: ccmpnol {dfv=of} 123(%r8,%rax,4), %ecx +# INTEL: ccmpno {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x01,0x3b,0x4c,0x80,0x7b + +# ATT: ccmpnoq {dfv=of} 123(%r8,%rax,4), %r9 +# INTEL: ccmpno {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +0x62,0x54,0xc4,0x01,0x3b,0x4c,0x80,0x7b + +# ATT: ccmpnob {dfv=of} $123, %bl +# INTEL: ccmpno {dfv=of} bl, 123 +0x62,0xf4,0x44,0x01,0x80,0xfb,0x7b + +# ATT: ccmpnow {dfv=of} $123, %dx +# INTEL: ccmpno {dfv=of} dx, 123 +0x62,0xf4,0x45,0x01,0x83,0xfa,0x7b + +# ATT: ccmpnol {dfv=of} $123, %ecx +# INTEL: ccmpno {dfv=of} ecx, 123 +0x62,0xf4,0x44,0x01,0x83,0xf9,0x7b + +# ATT: ccmpnoq {dfv=of} $123, %r9 +# INTEL: ccmpno {dfv=of} r9, 123 +0x62,0xd4,0xc4,0x01,0x83,0xf9,0x7b + +# ATT: ccmpnow {dfv=of} $1234, %dx +# INTEL: ccmpno {dfv=of} dx, 1234 +0x62,0xf4,0x45,0x01,0x81,0xfa,0xd2,0x04 + +# ATT: ccmpnol {dfv=of} $123456, %ecx +# INTEL: ccmpno {dfv=of} ecx, 123456 +0x62,0xf4,0x44,0x01,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmpnoq {dfv=of} $123456, %r9 +# INTEL: ccmpno {dfv=of} r9, 123456 +0x62,0xd4,0xc4,0x01,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmpnob {dfv=of} %bl, %dl +# INTEL: ccmpno {dfv=of} dl, bl +0x62,0xf4,0x44,0x01,0x3a,0xd3 + +# ATT: ccmpnow {dfv=of} %dx, %ax +# INTEL: ccmpno {dfv=of} ax, dx +0x62,0xf4,0x45,0x01,0x3b,0xc2 + +# ATT: ccmpnol {dfv=of} %ecx, %edx +# INTEL: ccmpno {dfv=of} edx, ecx +0x62,0xf4,0x44,0x01,0x3b,0xd1 + +# ATT: ccmpnoq {dfv=of} %r9, %r15 +# INTEL: ccmpno {dfv=of} r15, r9 +0x62,0x54,0xc4,0x01,0x3b,0xf9 + +# ATT: ccmpnsb {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpns {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x09,0x80,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpnsw {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpns {dfv=of} word ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x45,0x09,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpnsw {dfv=of} $1234, 123(%r8,%rax,4) +# INTEL: ccmpns {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +0x62,0xd4,0x45,0x09,0x81,0x7c,0x80,0x7b,0xd2,0x04 + +# ATT: ccmpnsl {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpns {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x09,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpnsl {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmpns {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0x44,0x09,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmpnsq {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpns {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0xc4,0x09,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpnsq {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmpns {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0xc4,0x09,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmpnsb {dfv=of} %bl, 123(%r8,%rax,4) +# INTEL: ccmpns {dfv=of} byte ptr [r8 + 4*rax + 123], bl +0x62,0xd4,0x44,0x09,0x38,0x5c,0x80,0x7b + +# ATT: ccmpnsw {dfv=of} %dx, 123(%r8,%rax,4) +# INTEL: ccmpns {dfv=of} word ptr [r8 + 4*rax + 123], dx +0x62,0xd4,0x45,0x09,0x39,0x54,0x80,0x7b + +# ATT: ccmpnsl {dfv=of} %ecx, 123(%r8,%rax,4) +# INTEL: ccmpns {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +0x62,0xd4,0x44,0x09,0x39,0x4c,0x80,0x7b + +# ATT: ccmpnsq {dfv=of} %r9, 123(%r8,%rax,4) +# INTEL: ccmpns {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +0x62,0x54,0xc4,0x09,0x39,0x4c,0x80,0x7b + +# ATT: ccmpnsb {dfv=of} 123(%r8,%rax,4), %bl +# INTEL: ccmpns {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x09,0x3a,0x5c,0x80,0x7b + +# ATT: ccmpnsw {dfv=of} 123(%r8,%rax,4), %dx +# INTEL: ccmpns {dfv=of} dx, word ptr [r8 + 4*rax + 123] +0x62,0xd4,0x45,0x09,0x3b,0x54,0x80,0x7b + +# ATT: ccmpnsl {dfv=of} 123(%r8,%rax,4), %ecx +# INTEL: ccmpns {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x09,0x3b,0x4c,0x80,0x7b + +# ATT: ccmpnsq {dfv=of} 123(%r8,%rax,4), %r9 +# INTEL: ccmpns {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +0x62,0x54,0xc4,0x09,0x3b,0x4c,0x80,0x7b + +# ATT: ccmpnsb {dfv=of} $123, %bl +# INTEL: ccmpns {dfv=of} bl, 123 +0x62,0xf4,0x44,0x09,0x80,0xfb,0x7b + +# ATT: ccmpnsw {dfv=of} $123, %dx +# INTEL: ccmpns {dfv=of} dx, 123 +0x62,0xf4,0x45,0x09,0x83,0xfa,0x7b + +# ATT: ccmpnsl {dfv=of} $123, %ecx +# INTEL: ccmpns {dfv=of} ecx, 123 +0x62,0xf4,0x44,0x09,0x83,0xf9,0x7b + +# ATT: ccmpnsq {dfv=of} $123, %r9 +# INTEL: ccmpns {dfv=of} r9, 123 +0x62,0xd4,0xc4,0x09,0x83,0xf9,0x7b + +# ATT: ccmpnsw {dfv=of} $1234, %dx +# INTEL: ccmpns {dfv=of} dx, 1234 +0x62,0xf4,0x45,0x09,0x81,0xfa,0xd2,0x04 + +# ATT: ccmpnsl {dfv=of} $123456, %ecx +# INTEL: ccmpns {dfv=of} ecx, 123456 +0x62,0xf4,0x44,0x09,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmpnsq {dfv=of} $123456, %r9 +# INTEL: ccmpns {dfv=of} r9, 123456 +0x62,0xd4,0xc4,0x09,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmpnsb {dfv=of} %bl, %dl +# INTEL: ccmpns {dfv=of} dl, bl +0x62,0xf4,0x44,0x09,0x3a,0xd3 + +# ATT: ccmpnsw {dfv=of} %dx, %ax +# INTEL: ccmpns {dfv=of} ax, dx +0x62,0xf4,0x45,0x09,0x3b,0xc2 + +# ATT: ccmpnsl {dfv=of} %ecx, %edx +# INTEL: ccmpns {dfv=of} edx, ecx +0x62,0xf4,0x44,0x09,0x3b,0xd1 + +# ATT: ccmpnsq {dfv=of} %r9, %r15 +# INTEL: ccmpns {dfv=of} r15, r9 +0x62,0x54,0xc4,0x09,0x3b,0xf9 + +# ATT: ccmpneb {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpne {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x05,0x80,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpnew {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpne {dfv=of} word ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x45,0x05,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpnew {dfv=of} $1234, 123(%r8,%rax,4) +# INTEL: ccmpne {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +0x62,0xd4,0x45,0x05,0x81,0x7c,0x80,0x7b,0xd2,0x04 + +# ATT: ccmpnel {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpne {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x05,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpnel {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmpne {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0x44,0x05,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmpneq {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpne {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0xc4,0x05,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpneq {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmpne {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0xc4,0x05,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmpneb {dfv=of} %bl, 123(%r8,%rax,4) +# INTEL: ccmpne {dfv=of} byte ptr [r8 + 4*rax + 123], bl +0x62,0xd4,0x44,0x05,0x38,0x5c,0x80,0x7b + +# ATT: ccmpnew {dfv=of} %dx, 123(%r8,%rax,4) +# INTEL: ccmpne {dfv=of} word ptr [r8 + 4*rax + 123], dx +0x62,0xd4,0x45,0x05,0x39,0x54,0x80,0x7b + +# ATT: ccmpnel {dfv=of} %ecx, 123(%r8,%rax,4) +# INTEL: ccmpne {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +0x62,0xd4,0x44,0x05,0x39,0x4c,0x80,0x7b + +# ATT: ccmpneq {dfv=of} %r9, 123(%r8,%rax,4) +# INTEL: ccmpne {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +0x62,0x54,0xc4,0x05,0x39,0x4c,0x80,0x7b + +# ATT: ccmpneb {dfv=of} 123(%r8,%rax,4), %bl +# INTEL: ccmpne {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x05,0x3a,0x5c,0x80,0x7b + +# ATT: ccmpnew {dfv=of} 123(%r8,%rax,4), %dx +# INTEL: ccmpne {dfv=of} dx, word ptr [r8 + 4*rax + 123] +0x62,0xd4,0x45,0x05,0x3b,0x54,0x80,0x7b + +# ATT: ccmpnel {dfv=of} 123(%r8,%rax,4), %ecx +# INTEL: ccmpne {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x05,0x3b,0x4c,0x80,0x7b + +# ATT: ccmpneq {dfv=of} 123(%r8,%rax,4), %r9 +# INTEL: ccmpne {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +0x62,0x54,0xc4,0x05,0x3b,0x4c,0x80,0x7b + +# ATT: ccmpneb {dfv=of} $123, %bl +# INTEL: ccmpne {dfv=of} bl, 123 +0x62,0xf4,0x44,0x05,0x80,0xfb,0x7b + +# ATT: ccmpnew {dfv=of} $123, %dx +# INTEL: ccmpne {dfv=of} dx, 123 +0x62,0xf4,0x45,0x05,0x83,0xfa,0x7b + +# ATT: ccmpnel {dfv=of} $123, %ecx +# INTEL: ccmpne {dfv=of} ecx, 123 +0x62,0xf4,0x44,0x05,0x83,0xf9,0x7b + +# ATT: ccmpneq {dfv=of} $123, %r9 +# INTEL: ccmpne {dfv=of} r9, 123 +0x62,0xd4,0xc4,0x05,0x83,0xf9,0x7b + +# ATT: ccmpnew {dfv=of} $1234, %dx +# INTEL: ccmpne {dfv=of} dx, 1234 +0x62,0xf4,0x45,0x05,0x81,0xfa,0xd2,0x04 + +# ATT: ccmpnel {dfv=of} $123456, %ecx +# INTEL: ccmpne {dfv=of} ecx, 123456 +0x62,0xf4,0x44,0x05,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmpneq {dfv=of} $123456, %r9 +# INTEL: ccmpne {dfv=of} r9, 123456 +0x62,0xd4,0xc4,0x05,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmpneb {dfv=of} %bl, %dl +# INTEL: ccmpne {dfv=of} dl, bl +0x62,0xf4,0x44,0x05,0x3a,0xd3 + +# ATT: ccmpnew {dfv=of} %dx, %ax +# INTEL: ccmpne {dfv=of} ax, dx +0x62,0xf4,0x45,0x05,0x3b,0xc2 + +# ATT: ccmpnel {dfv=of} %ecx, %edx +# INTEL: ccmpne {dfv=of} edx, ecx +0x62,0xf4,0x44,0x05,0x3b,0xd1 + +# ATT: ccmpneq {dfv=of} %r9, %r15 +# INTEL: ccmpne {dfv=of} r15, r9 +0x62,0x54,0xc4,0x05,0x3b,0xf9 + +# ATT: ccmpob {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpo {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x00,0x80,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpow {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpo {dfv=of} word ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x45,0x00,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpow {dfv=of} $1234, 123(%r8,%rax,4) +# INTEL: ccmpo {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +0x62,0xd4,0x45,0x00,0x81,0x7c,0x80,0x7b,0xd2,0x04 + +# ATT: ccmpol {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpo {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x00,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpol {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmpo {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0x44,0x00,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmpoq {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpo {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0xc4,0x00,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpoq {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmpo {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0xc4,0x00,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmpob {dfv=of} %bl, 123(%r8,%rax,4) +# INTEL: ccmpo {dfv=of} byte ptr [r8 + 4*rax + 123], bl +0x62,0xd4,0x44,0x00,0x38,0x5c,0x80,0x7b + +# ATT: ccmpow {dfv=of} %dx, 123(%r8,%rax,4) +# INTEL: ccmpo {dfv=of} word ptr [r8 + 4*rax + 123], dx +0x62,0xd4,0x45,0x00,0x39,0x54,0x80,0x7b + +# ATT: ccmpol {dfv=of} %ecx, 123(%r8,%rax,4) +# INTEL: ccmpo {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +0x62,0xd4,0x44,0x00,0x39,0x4c,0x80,0x7b + +# ATT: ccmpoq {dfv=of} %r9, 123(%r8,%rax,4) +# INTEL: ccmpo {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +0x62,0x54,0xc4,0x00,0x39,0x4c,0x80,0x7b + +# ATT: ccmpob {dfv=of} 123(%r8,%rax,4), %bl +# INTEL: ccmpo {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x00,0x3a,0x5c,0x80,0x7b + +# ATT: ccmpow {dfv=of} 123(%r8,%rax,4), %dx +# INTEL: ccmpo {dfv=of} dx, word ptr [r8 + 4*rax + 123] +0x62,0xd4,0x45,0x00,0x3b,0x54,0x80,0x7b + +# ATT: ccmpol {dfv=of} 123(%r8,%rax,4), %ecx +# INTEL: ccmpo {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x00,0x3b,0x4c,0x80,0x7b + +# ATT: ccmpoq {dfv=of} 123(%r8,%rax,4), %r9 +# INTEL: ccmpo {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +0x62,0x54,0xc4,0x00,0x3b,0x4c,0x80,0x7b + +# ATT: ccmpob {dfv=of} $123, %bl +# INTEL: ccmpo {dfv=of} bl, 123 +0x62,0xf4,0x44,0x00,0x80,0xfb,0x7b + +# ATT: ccmpow {dfv=of} $123, %dx +# INTEL: ccmpo {dfv=of} dx, 123 +0x62,0xf4,0x45,0x00,0x83,0xfa,0x7b + +# ATT: ccmpol {dfv=of} $123, %ecx +# INTEL: ccmpo {dfv=of} ecx, 123 +0x62,0xf4,0x44,0x00,0x83,0xf9,0x7b + +# ATT: ccmpoq {dfv=of} $123, %r9 +# INTEL: ccmpo {dfv=of} r9, 123 +0x62,0xd4,0xc4,0x00,0x83,0xf9,0x7b + +# ATT: ccmpow {dfv=of} $1234, %dx +# INTEL: ccmpo {dfv=of} dx, 1234 +0x62,0xf4,0x45,0x00,0x81,0xfa,0xd2,0x04 + +# ATT: ccmpol {dfv=of} $123456, %ecx +# INTEL: ccmpo {dfv=of} ecx, 123456 +0x62,0xf4,0x44,0x00,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmpoq {dfv=of} $123456, %r9 +# INTEL: ccmpo {dfv=of} r9, 123456 +0x62,0xd4,0xc4,0x00,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmpob {dfv=of} %bl, %dl +# INTEL: ccmpo {dfv=of} dl, bl +0x62,0xf4,0x44,0x00,0x3a,0xd3 + +# ATT: ccmpow {dfv=of} %dx, %ax +# INTEL: ccmpo {dfv=of} ax, dx +0x62,0xf4,0x45,0x00,0x3b,0xc2 + +# ATT: ccmpol {dfv=of} %ecx, %edx +# INTEL: ccmpo {dfv=of} edx, ecx +0x62,0xf4,0x44,0x00,0x3b,0xd1 + +# ATT: ccmpoq {dfv=of} %r9, %r15 +# INTEL: ccmpo {dfv=of} r15, r9 +0x62,0x54,0xc4,0x00,0x3b,0xf9 + +# ATT: ccmpsb {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmps {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x08,0x80,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpsw {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmps {dfv=of} word ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x45,0x08,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpsw {dfv=of} $1234, 123(%r8,%rax,4) +# INTEL: ccmps {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +0x62,0xd4,0x45,0x08,0x81,0x7c,0x80,0x7b,0xd2,0x04 + +# ATT: ccmpsl {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmps {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x08,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpsl {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmps {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0x44,0x08,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmpsq {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmps {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0xc4,0x08,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpsq {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmps {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0xc4,0x08,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmpsb {dfv=of} %bl, 123(%r8,%rax,4) +# INTEL: ccmps {dfv=of} byte ptr [r8 + 4*rax + 123], bl +0x62,0xd4,0x44,0x08,0x38,0x5c,0x80,0x7b + +# ATT: ccmpsw {dfv=of} %dx, 123(%r8,%rax,4) +# INTEL: ccmps {dfv=of} word ptr [r8 + 4*rax + 123], dx +0x62,0xd4,0x45,0x08,0x39,0x54,0x80,0x7b + +# ATT: ccmpsl {dfv=of} %ecx, 123(%r8,%rax,4) +# INTEL: ccmps {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +0x62,0xd4,0x44,0x08,0x39,0x4c,0x80,0x7b + +# ATT: ccmpsq {dfv=of} %r9, 123(%r8,%rax,4) +# INTEL: ccmps {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +0x62,0x54,0xc4,0x08,0x39,0x4c,0x80,0x7b + +# ATT: ccmpsb {dfv=of} 123(%r8,%rax,4), %bl +# INTEL: ccmps {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x08,0x3a,0x5c,0x80,0x7b + +# ATT: ccmpsw {dfv=of} 123(%r8,%rax,4), %dx +# INTEL: ccmps {dfv=of} dx, word ptr [r8 + 4*rax + 123] +0x62,0xd4,0x45,0x08,0x3b,0x54,0x80,0x7b + +# ATT: ccmpsl {dfv=of} 123(%r8,%rax,4), %ecx +# INTEL: ccmps {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x08,0x3b,0x4c,0x80,0x7b + +# ATT: ccmpsq {dfv=of} 123(%r8,%rax,4), %r9 +# INTEL: ccmps {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +0x62,0x54,0xc4,0x08,0x3b,0x4c,0x80,0x7b + +# ATT: ccmpsb {dfv=of} $123, %bl +# INTEL: ccmps {dfv=of} bl, 123 +0x62,0xf4,0x44,0x08,0x80,0xfb,0x7b + +# ATT: ccmpsw {dfv=of} $123, %dx +# INTEL: ccmps {dfv=of} dx, 123 +0x62,0xf4,0x45,0x08,0x83,0xfa,0x7b + +# ATT: ccmpsl {dfv=of} $123, %ecx +# INTEL: ccmps {dfv=of} ecx, 123 +0x62,0xf4,0x44,0x08,0x83,0xf9,0x7b + +# ATT: ccmpsq {dfv=of} $123, %r9 +# INTEL: ccmps {dfv=of} r9, 123 +0x62,0xd4,0xc4,0x08,0x83,0xf9,0x7b + +# ATT: ccmpsw {dfv=of} $1234, %dx +# INTEL: ccmps {dfv=of} dx, 1234 +0x62,0xf4,0x45,0x08,0x81,0xfa,0xd2,0x04 + +# ATT: ccmpsl {dfv=of} $123456, %ecx +# INTEL: ccmps {dfv=of} ecx, 123456 +0x62,0xf4,0x44,0x08,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmpsq {dfv=of} $123456, %r9 +# INTEL: ccmps {dfv=of} r9, 123456 +0x62,0xd4,0xc4,0x08,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmpsb {dfv=of} %bl, %dl +# INTEL: ccmps {dfv=of} dl, bl +0x62,0xf4,0x44,0x08,0x3a,0xd3 + +# ATT: ccmpsw {dfv=of} %dx, %ax +# INTEL: ccmps {dfv=of} ax, dx +0x62,0xf4,0x45,0x08,0x3b,0xc2 + +# ATT: ccmpsl {dfv=of} %ecx, %edx +# INTEL: ccmps {dfv=of} edx, ecx +0x62,0xf4,0x44,0x08,0x3b,0xd1 + +# ATT: ccmpsq {dfv=of} %r9, %r15 +# INTEL: ccmps {dfv=of} r15, r9 +0x62,0x54,0xc4,0x08,0x3b,0xf9 + +# ATT: ccmptb {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpt {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x0a,0x80,0x7c,0x80,0x7b,0x7b + +# ATT: ccmptw {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpt {dfv=of} word ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x45,0x0a,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmptw {dfv=of} $1234, 123(%r8,%rax,4) +# INTEL: ccmpt {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +0x62,0xd4,0x45,0x0a,0x81,0x7c,0x80,0x7b,0xd2,0x04 + +# ATT: ccmptl {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpt {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x0a,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmptl {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmpt {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0x44,0x0a,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmptq {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpt {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0xc4,0x0a,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmptq {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmpt {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0xc4,0x0a,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmptb {dfv=of} %bl, 123(%r8,%rax,4) +# INTEL: ccmpt {dfv=of} byte ptr [r8 + 4*rax + 123], bl +0x62,0xd4,0x44,0x0a,0x38,0x5c,0x80,0x7b + +# ATT: ccmptw {dfv=of} %dx, 123(%r8,%rax,4) +# INTEL: ccmpt {dfv=of} word ptr [r8 + 4*rax + 123], dx +0x62,0xd4,0x45,0x0a,0x39,0x54,0x80,0x7b + +# ATT: ccmptl {dfv=of} %ecx, 123(%r8,%rax,4) +# INTEL: ccmpt {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +0x62,0xd4,0x44,0x0a,0x39,0x4c,0x80,0x7b + +# ATT: ccmptq {dfv=of} %r9, 123(%r8,%rax,4) +# INTEL: ccmpt {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +0x62,0x54,0xc4,0x0a,0x39,0x4c,0x80,0x7b + +# ATT: ccmptb {dfv=of} 123(%r8,%rax,4), %bl +# INTEL: ccmpt {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x0a,0x3a,0x5c,0x80,0x7b + +# ATT: ccmptw {dfv=of} 123(%r8,%rax,4), %dx +# INTEL: ccmpt {dfv=of} dx, word ptr [r8 + 4*rax + 123] +0x62,0xd4,0x45,0x0a,0x3b,0x54,0x80,0x7b + +# ATT: ccmptl {dfv=of} 123(%r8,%rax,4), %ecx +# INTEL: ccmpt {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x0a,0x3b,0x4c,0x80,0x7b + +# ATT: ccmptq {dfv=of} 123(%r8,%rax,4), %r9 +# INTEL: ccmpt {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +0x62,0x54,0xc4,0x0a,0x3b,0x4c,0x80,0x7b + +# ATT: ccmptb {dfv=of} $123, %bl +# INTEL: ccmpt {dfv=of} bl, 123 +0x62,0xf4,0x44,0x0a,0x80,0xfb,0x7b + +# ATT: ccmptw {dfv=of} $123, %dx +# INTEL: ccmpt {dfv=of} dx, 123 +0x62,0xf4,0x45,0x0a,0x83,0xfa,0x7b + +# ATT: ccmptl {dfv=of} $123, %ecx +# INTEL: ccmpt {dfv=of} ecx, 123 +0x62,0xf4,0x44,0x0a,0x83,0xf9,0x7b + +# ATT: ccmptq {dfv=of} $123, %r9 +# INTEL: ccmpt {dfv=of} r9, 123 +0x62,0xd4,0xc4,0x0a,0x83,0xf9,0x7b + +# ATT: ccmptw {dfv=of} $1234, %dx +# INTEL: ccmpt {dfv=of} dx, 1234 +0x62,0xf4,0x45,0x0a,0x81,0xfa,0xd2,0x04 + +# ATT: ccmptl {dfv=of} $123456, %ecx +# INTEL: ccmpt {dfv=of} ecx, 123456 +0x62,0xf4,0x44,0x0a,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmptq {dfv=of} $123456, %r9 +# INTEL: ccmpt {dfv=of} r9, 123456 +0x62,0xd4,0xc4,0x0a,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmptb {dfv=of} %bl, %dl +# INTEL: ccmpt {dfv=of} dl, bl +0x62,0xf4,0x44,0x0a,0x3a,0xd3 + +# ATT: ccmptw {dfv=of} %dx, %ax +# INTEL: ccmpt {dfv=of} ax, dx +0x62,0xf4,0x45,0x0a,0x3b,0xc2 + +# ATT: ccmptl {dfv=of} %ecx, %edx +# INTEL: ccmpt {dfv=of} edx, ecx +0x62,0xf4,0x44,0x0a,0x3b,0xd1 + +# ATT: ccmptq {dfv=of} %r9, %r15 +# INTEL: ccmpt {dfv=of} r15, r9 +0x62,0x54,0xc4,0x0a,0x3b,0xf9 + +# ATT: ccmpeb {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpe {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x04,0x80,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpew {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpe {dfv=of} word ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x45,0x04,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpew {dfv=of} $1234, 123(%r8,%rax,4) +# INTEL: ccmpe {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +0x62,0xd4,0x45,0x04,0x81,0x7c,0x80,0x7b,0xd2,0x04 + +# ATT: ccmpel {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpe {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0x44,0x04,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpel {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmpe {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0x44,0x04,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmpeq {dfv=of} $123, 123(%r8,%rax,4) +# INTEL: ccmpe {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +0x62,0xd4,0xc4,0x04,0x83,0x7c,0x80,0x7b,0x7b + +# ATT: ccmpeq {dfv=of} $123456, 123(%r8,%rax,4) +# INTEL: ccmpe {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +0x62,0xd4,0xc4,0x04,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00 + +# ATT: ccmpeb {dfv=of} %bl, 123(%r8,%rax,4) +# INTEL: ccmpe {dfv=of} byte ptr [r8 + 4*rax + 123], bl +0x62,0xd4,0x44,0x04,0x38,0x5c,0x80,0x7b + +# ATT: ccmpew {dfv=of} %dx, 123(%r8,%rax,4) +# INTEL: ccmpe {dfv=of} word ptr [r8 + 4*rax + 123], dx +0x62,0xd4,0x45,0x04,0x39,0x54,0x80,0x7b + +# ATT: ccmpel {dfv=of} %ecx, 123(%r8,%rax,4) +# INTEL: ccmpe {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +0x62,0xd4,0x44,0x04,0x39,0x4c,0x80,0x7b + +# ATT: ccmpeq {dfv=of} %r9, 123(%r8,%rax,4) +# INTEL: ccmpe {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +0x62,0x54,0xc4,0x04,0x39,0x4c,0x80,0x7b + +# ATT: ccmpeb {dfv=of} 123(%r8,%rax,4), %bl +# INTEL: ccmpe {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x04,0x3a,0x5c,0x80,0x7b + +# ATT: ccmpew {dfv=of} 123(%r8,%rax,4), %dx +# INTEL: ccmpe {dfv=of} dx, word ptr [r8 + 4*rax + 123] +0x62,0xd4,0x45,0x04,0x3b,0x54,0x80,0x7b + +# ATT: ccmpel {dfv=of} 123(%r8,%rax,4), %ecx +# INTEL: ccmpe {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +0x62,0xd4,0x44,0x04,0x3b,0x4c,0x80,0x7b + +# ATT: ccmpeq {dfv=of} 123(%r8,%rax,4), %r9 +# INTEL: ccmpe {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +0x62,0x54,0xc4,0x04,0x3b,0x4c,0x80,0x7b + +# ATT: ccmpeb {dfv=of} $123, %bl +# INTEL: ccmpe {dfv=of} bl, 123 +0x62,0xf4,0x44,0x04,0x80,0xfb,0x7b + +# ATT: ccmpew {dfv=of} $123, %dx +# INTEL: ccmpe {dfv=of} dx, 123 +0x62,0xf4,0x45,0x04,0x83,0xfa,0x7b + +# ATT: ccmpel {dfv=of} $123, %ecx +# INTEL: ccmpe {dfv=of} ecx, 123 +0x62,0xf4,0x44,0x04,0x83,0xf9,0x7b + +# ATT: ccmpeq {dfv=of} $123, %r9 +# INTEL: ccmpe {dfv=of} r9, 123 +0x62,0xd4,0xc4,0x04,0x83,0xf9,0x7b + +# ATT: ccmpew {dfv=of} $1234, %dx +# INTEL: ccmpe {dfv=of} dx, 1234 +0x62,0xf4,0x45,0x04,0x81,0xfa,0xd2,0x04 + +# ATT: ccmpel {dfv=of} $123456, %ecx +# INTEL: ccmpe {dfv=of} ecx, 123456 +0x62,0xf4,0x44,0x04,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmpeq {dfv=of} $123456, %r9 +# INTEL: ccmpe {dfv=of} r9, 123456 +0x62,0xd4,0xc4,0x04,0x81,0xf9,0x40,0xe2,0x01,0x00 + +# ATT: ccmpeb {dfv=of} %bl, %dl +# INTEL: ccmpe {dfv=of} dl, bl +0x62,0xf4,0x44,0x04,0x3a,0xd3 + +# ATT: ccmpew {dfv=of} %dx, %ax +# INTEL: ccmpe {dfv=of} ax, dx +0x62,0xf4,0x45,0x04,0x3b,0xc2 + +# ATT: ccmpel {dfv=of} %ecx, %edx +# INTEL: ccmpe {dfv=of} edx, ecx +0x62,0xf4,0x44,0x04,0x3b,0xd1 + +# ATT: ccmpeq {dfv=of} %r9, %r15 +# INTEL: ccmpe {dfv=of} r15, r9 +0x62,0x54,0xc4,0x04,0x3b,0xf9 diff --git a/llvm/test/MC/Disassembler/X86/apx/reverse-encoding.txt b/llvm/test/MC/Disassembler/X86/apx/reverse-encoding.txt index 94499299477d..9e812e370146 100644 --- a/llvm/test/MC/Disassembler/X86/apx/reverse-encoding.txt +++ b/llvm/test/MC/Disassembler/X86/apx/reverse-encoding.txt @@ -412,3 +412,21 @@ # ATT: movbeq %r16, %r17 # INTEL: movbe r17, r16 0x62,0xec,0xfc,0x08,0x60,0xc8 + +## ccmp + +# ATT: ccmpob {dfv=} %r16b, %r17b +# INTEL: ccmpo {dfv=} r17b, r16b +0x62,0xec,0x04,0x00,0x3a,0xc8 + +# ATT: ccmpow {dfv=} %r16w, %r17w +# INTEL: ccmpo {dfv=} r17w, r16w +0x62,0xec,0x05,0x00,0x3b,0xc8 + +# ATT: ccmpol {dfv=} %r16d, %r17d +# INTEL: ccmpo {dfv=} r17d, r16d +0x62,0xec,0x04,0x00,0x3b,0xc8 + +# ATT: ccmpoq {dfv=} %r16, %r17 +# INTEL: ccmpo {dfv=} r17, r16 +0x62,0xec,0x84,0x00,0x3b,0xc8 diff --git a/llvm/test/MC/X86/apx/ccmp-att-error.s b/llvm/test/MC/X86/apx/ccmp-att-error.s new file mode 100644 index 000000000000..5a1237195a22 --- /dev/null +++ b/llvm/test/MC/X86/apx/ccmp-att-error.s @@ -0,0 +1,29 @@ +# RUN: not llvm-mc -triple x86_64 -show-encoding %s 2>&1 | FileCheck --strict-whitespace %s + +# CHECK: [[#@LINE+2]]:8: error: Expected { at this point +# CHECK: ccmpeq $1 %rax, %rbx +ccmpeq $1 %rax, %rbx + +# CHECK: [[#@LINE+2]]:9: error: Expected dfv at this point +# CHECK: ccmpeq {sf} %rax, %rbx +ccmpeq {sf} %rax, %rbx + +# CHECK: [[#@LINE+2]]:12: error: Expected = at this point +# CHECK: ccmpeq {dfv:sf} %rax, %rbx +ccmpeq {dfv:sf} %rax, %rbx + +# CHECK: [[#@LINE+2]]:18: error: Expected } or , at this point +# CHECK: ccmpeq {dfv=sf,cf%rax, %rbx +ccmpeq {dfv=sf,cf%rax, %rbx + +# CHECK: [[#@LINE+2]]:13: error: Invalid conditional flags +# CHECK: ccmpeq {dfv=pf} %rax, %rbx +ccmpeq {dfv=pf} %rax, %rbx + +# CHECK: [[#@LINE+2]]:19: error: Duplicated conditional flag +# CHECK: ccmpeq {dfv=of,zf,of} %rax, %rbx +ccmpeq {dfv=of,zf,of} %rax, %rbx + +# CHECK: [[#@LINE+2]]:24: error: Expected } at this point +# CHECK: ccmpeq {dfv=of,sf,zf,cf,of} %rax, %rbx +ccmpeq {dfv=of,sf,zf,cf,of} %rax, %rbx diff --git a/llvm/test/MC/X86/apx/ccmp-att.s b/llvm/test/MC/X86/apx/ccmp-att.s new file mode 100644 index 000000000000..0b78562f32e1 --- /dev/null +++ b/llvm/test/MC/X86/apx/ccmp-att.s @@ -0,0 +1,1213 @@ +# RUN: llvm-mc -triple x86_64 -show-encoding %s | FileCheck %s +# RUN: not llvm-mc -triple i386 -show-encoding %s 2>&1 | FileCheck %s --check-prefix=ERROR + +# ERROR-COUNT-401: error: +# ERROR-NOT: error: +## Condition flags + +# CHECK: ccmpoq {dfv=} %rax, %rbx +# CHECK: encoding: [0x62,0xf4,0x84,0x00,0x39,0xc3] + ccmpoq {dfv=} %rax, %rbx +# CHECK: ccmpoq {dfv=of} %rax, %rbx +# CHECK: encoding: [0x62,0xf4,0xc4,0x00,0x39,0xc3] + ccmpoq {dfv=of} %rax, %rbx +# CHECK: ccmpoq {dfv=sf} %rax, %rbx +# CHECK: encoding: [0x62,0xf4,0xa4,0x00,0x39,0xc3] + ccmpoq {dfv=sf} %rax, %rbx +# CHECK: ccmpoq {dfv=zf} %rax, %rbx +# CHECK: encoding: [0x62,0xf4,0x94,0x00,0x39,0xc3] + ccmpoq {dfv=zf} %rax, %rbx +# CHECK: ccmpoq {dfv=cf} %rax, %rbx +# CHECK: encoding: [0x62,0xf4,0x8c,0x00,0x39,0xc3] + ccmpoq {dfv=cf} %rax, %rbx +# CHECK: ccmpoq {dfv=of,sf} %rax, %rbx +# CHECK: encoding: [0x62,0xf4,0xe4,0x00,0x39,0xc3] + ccmpoq {dfv=of,sf} %rax, %rbx +# CHECK: ccmpoq {dfv=of,sf} %rax, %rbx +# CHECK: encoding: [0x62,0xf4,0xe4,0x00,0x39,0xc3] + ccmpoq {dfv=sf,of} %rax, %rbx +# CHECK: ccmpoq {dfv=of,sf,zf} %rax, %rbx +# CHECK: encoding: [0x62,0xf4,0xf4,0x00,0x39,0xc3] + ccmpoq {dfv=of,sf,zf} %rax, %rbx +# CHECK: ccmpoq {dfv=of,sf,zf} %rax, %rbx +# CHECK: encoding: [0x62,0xf4,0xf4,0x00,0x39,0xc3] + ccmpoq {dfv=zf,of,sf} %rax, %rbx +# CHECK: ccmpoq {dfv=of,sf,zf,cf} %rax, %rbx +# CHECK: encoding: [0x62,0xf4,0xfc,0x00,0x39,0xc3] + ccmpoq {dfv=of,sf,zf,cf} %rax, %rbx +# CHECK: ccmpoq {dfv=of,sf,zf,cf} %rax, %rbx +# CHECK: encoding: [0x62,0xf4,0xfc,0x00,0x39,0xc3] + ccmpoq {dfv=cf,zf,sf,of} %rax, %rbx + +## Condition code + +# CHECK: ccmpbb {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x02,0x80,0x7c,0x80,0x7b,0x7b] + ccmpbb {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpbw {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x02,0x83,0x7c,0x80,0x7b,0x7b] + ccmpbw {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpbw {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x02,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmpbw {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: ccmpbl {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x02,0x83,0x7c,0x80,0x7b,0x7b] + ccmpbl {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpbl {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x02,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpbl {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmpbq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x02,0x83,0x7c,0x80,0x7b,0x7b] + ccmpbq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpbq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x02,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpbq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmpbb {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x02,0x38,0x5c,0x80,0x7b] + ccmpbb {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: ccmpbw {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x02,0x39,0x54,0x80,0x7b] + ccmpbw {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: ccmpbl {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x02,0x39,0x4c,0x80,0x7b] + ccmpbl {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: ccmpbq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0x54,0xc4,0x02,0x39,0x4c,0x80,0x7b] + ccmpbq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: ccmpbb {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: encoding: [0x62,0xd4,0x44,0x02,0x3a,0x5c,0x80,0x7b] + ccmpbb {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: ccmpbw {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: encoding: [0x62,0xd4,0x45,0x02,0x3b,0x54,0x80,0x7b] + ccmpbw {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: ccmpbl {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x02,0x3b,0x4c,0x80,0x7b] + ccmpbl {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: ccmpbq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x02,0x3b,0x4c,0x80,0x7b] + ccmpbq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: ccmpbb {dfv=of} $123, %bl +# CHECK: encoding: [0x62,0xf4,0x44,0x02,0x80,0xfb,0x7b] + ccmpbb {dfv=of} $123, %bl +# CHECK: ccmpbw {dfv=of} $123, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x02,0x83,0xfa,0x7b] + ccmpbw {dfv=of} $123, %dx +# CHECK: ccmpbl {dfv=of} $123, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x02,0x83,0xf9,0x7b] + ccmpbl {dfv=of} $123, %ecx +# CHECK: ccmpbq {dfv=of} $123, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x02,0x83,0xf9,0x7b] + ccmpbq {dfv=of} $123, %r9 +# CHECK: ccmpbw {dfv=of} $1234, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x02,0x81,0xfa,0xd2,0x04] + ccmpbw {dfv=of} $1234, %dx +# CHECK: ccmpbl {dfv=of} $123456, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x02,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpbl {dfv=of} $123456, %ecx +# CHECK: ccmpbq {dfv=of} $123456, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x02,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpbq {dfv=of} $123456, %r9 +# CHECK: ccmpbb {dfv=of} %bl, %dl +# CHECK: encoding: [0x62,0xf4,0x44,0x02,0x38,0xda] + ccmpbb {dfv=of} %bl, %dl +# CHECK: ccmpbw {dfv=of} %dx, %ax +# CHECK: encoding: [0x62,0xf4,0x45,0x02,0x39,0xd0] + ccmpbw {dfv=of} %dx, %ax +# CHECK: ccmpbl {dfv=of} %ecx, %edx +# CHECK: encoding: [0x62,0xf4,0x44,0x02,0x39,0xca] + ccmpbl {dfv=of} %ecx, %edx +# CHECK: ccmpbq {dfv=of} %r9, %r15 +# CHECK: encoding: [0x62,0x54,0xc4,0x02,0x39,0xcf] + ccmpbq {dfv=of} %r9, %r15 +# CHECK: ccmpfb {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0b,0x80,0x7c,0x80,0x7b,0x7b] + ccmpfb {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpfw {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x0b,0x83,0x7c,0x80,0x7b,0x7b] + ccmpfw {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpfw {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x0b,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmpfw {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: ccmpfl {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0b,0x83,0x7c,0x80,0x7b,0x7b] + ccmpfl {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpfl {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0b,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpfl {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmpfq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x0b,0x83,0x7c,0x80,0x7b,0x7b] + ccmpfq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpfq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x0b,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpfq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmpfb {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0b,0x38,0x5c,0x80,0x7b] + ccmpfb {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: ccmpfw {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x0b,0x39,0x54,0x80,0x7b] + ccmpfw {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: ccmpfl {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0b,0x39,0x4c,0x80,0x7b] + ccmpfl {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: ccmpfq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0x54,0xc4,0x0b,0x39,0x4c,0x80,0x7b] + ccmpfq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: ccmpfb {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: encoding: [0x62,0xd4,0x44,0x0b,0x3a,0x5c,0x80,0x7b] + ccmpfb {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: ccmpfw {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: encoding: [0x62,0xd4,0x45,0x0b,0x3b,0x54,0x80,0x7b] + ccmpfw {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: ccmpfl {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x0b,0x3b,0x4c,0x80,0x7b] + ccmpfl {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: ccmpfq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x0b,0x3b,0x4c,0x80,0x7b] + ccmpfq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: ccmpfb {dfv=of} $123, %bl +# CHECK: encoding: [0x62,0xf4,0x44,0x0b,0x80,0xfb,0x7b] + ccmpfb {dfv=of} $123, %bl +# CHECK: ccmpfw {dfv=of} $123, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x0b,0x83,0xfa,0x7b] + ccmpfw {dfv=of} $123, %dx +# CHECK: ccmpfl {dfv=of} $123, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x0b,0x83,0xf9,0x7b] + ccmpfl {dfv=of} $123, %ecx +# CHECK: ccmpfq {dfv=of} $123, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0b,0x83,0xf9,0x7b] + ccmpfq {dfv=of} $123, %r9 +# CHECK: ccmpfw {dfv=of} $1234, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x0b,0x81,0xfa,0xd2,0x04] + ccmpfw {dfv=of} $1234, %dx +# CHECK: ccmpfl {dfv=of} $123456, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x0b,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpfl {dfv=of} $123456, %ecx +# CHECK: ccmpfq {dfv=of} $123456, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0b,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpfq {dfv=of} $123456, %r9 +# CHECK: ccmpfb {dfv=of} %bl, %dl +# CHECK: encoding: [0x62,0xf4,0x44,0x0b,0x38,0xda] + ccmpfb {dfv=of} %bl, %dl +# CHECK: ccmpfw {dfv=of} %dx, %ax +# CHECK: encoding: [0x62,0xf4,0x45,0x0b,0x39,0xd0] + ccmpfw {dfv=of} %dx, %ax +# CHECK: ccmpfl {dfv=of} %ecx, %edx +# CHECK: encoding: [0x62,0xf4,0x44,0x0b,0x39,0xca] + ccmpfl {dfv=of} %ecx, %edx +# CHECK: ccmpfq {dfv=of} %r9, %r15 +# CHECK: encoding: [0x62,0x54,0xc4,0x0b,0x39,0xcf] + ccmpfq {dfv=of} %r9, %r15 +# CHECK: ccmplb {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0c,0x80,0x7c,0x80,0x7b,0x7b] + ccmplb {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmplw {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x0c,0x83,0x7c,0x80,0x7b,0x7b] + ccmplw {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmplw {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x0c,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmplw {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: ccmpll {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0c,0x83,0x7c,0x80,0x7b,0x7b] + ccmpll {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpll {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0c,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpll {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmplq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x0c,0x83,0x7c,0x80,0x7b,0x7b] + ccmplq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmplq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x0c,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmplq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmplb {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0c,0x38,0x5c,0x80,0x7b] + ccmplb {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: ccmplw {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x0c,0x39,0x54,0x80,0x7b] + ccmplw {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: ccmpll {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0c,0x39,0x4c,0x80,0x7b] + ccmpll {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: ccmplq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0x54,0xc4,0x0c,0x39,0x4c,0x80,0x7b] + ccmplq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: ccmplb {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: encoding: [0x62,0xd4,0x44,0x0c,0x3a,0x5c,0x80,0x7b] + ccmplb {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: ccmplw {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: encoding: [0x62,0xd4,0x45,0x0c,0x3b,0x54,0x80,0x7b] + ccmplw {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: ccmpll {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x0c,0x3b,0x4c,0x80,0x7b] + ccmpll {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: ccmplq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x0c,0x3b,0x4c,0x80,0x7b] + ccmplq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: ccmplb {dfv=of} $123, %bl +# CHECK: encoding: [0x62,0xf4,0x44,0x0c,0x80,0xfb,0x7b] + ccmplb {dfv=of} $123, %bl +# CHECK: ccmplw {dfv=of} $123, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x0c,0x83,0xfa,0x7b] + ccmplw {dfv=of} $123, %dx +# CHECK: ccmpll {dfv=of} $123, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x0c,0x83,0xf9,0x7b] + ccmpll {dfv=of} $123, %ecx +# CHECK: ccmplq {dfv=of} $123, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0c,0x83,0xf9,0x7b] + ccmplq {dfv=of} $123, %r9 +# CHECK: ccmplw {dfv=of} $1234, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x0c,0x81,0xfa,0xd2,0x04] + ccmplw {dfv=of} $1234, %dx +# CHECK: ccmpll {dfv=of} $123456, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x0c,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpll {dfv=of} $123456, %ecx +# CHECK: ccmplq {dfv=of} $123456, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0c,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmplq {dfv=of} $123456, %r9 +# CHECK: ccmplb {dfv=of} %bl, %dl +# CHECK: encoding: [0x62,0xf4,0x44,0x0c,0x38,0xda] + ccmplb {dfv=of} %bl, %dl +# CHECK: ccmplw {dfv=of} %dx, %ax +# CHECK: encoding: [0x62,0xf4,0x45,0x0c,0x39,0xd0] + ccmplw {dfv=of} %dx, %ax +# CHECK: ccmpll {dfv=of} %ecx, %edx +# CHECK: encoding: [0x62,0xf4,0x44,0x0c,0x39,0xca] + ccmpll {dfv=of} %ecx, %edx +# CHECK: ccmplq {dfv=of} %r9, %r15 +# CHECK: encoding: [0x62,0x54,0xc4,0x0c,0x39,0xcf] + ccmplq {dfv=of} %r9, %r15 +# CHECK: ccmpleb {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0e,0x80,0x7c,0x80,0x7b,0x7b] + ccmpleb {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmplew {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x0e,0x83,0x7c,0x80,0x7b,0x7b] + ccmplew {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmplew {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x0e,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmplew {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: ccmplel {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0e,0x83,0x7c,0x80,0x7b,0x7b] + ccmplel {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmplel {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0e,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmplel {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmpleq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x0e,0x83,0x7c,0x80,0x7b,0x7b] + ccmpleq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpleq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x0e,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpleq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmpleb {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0e,0x38,0x5c,0x80,0x7b] + ccmpleb {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: ccmplew {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x0e,0x39,0x54,0x80,0x7b] + ccmplew {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: ccmplel {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0e,0x39,0x4c,0x80,0x7b] + ccmplel {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: ccmpleq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0x54,0xc4,0x0e,0x39,0x4c,0x80,0x7b] + ccmpleq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: ccmpleb {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: encoding: [0x62,0xd4,0x44,0x0e,0x3a,0x5c,0x80,0x7b] + ccmpleb {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: ccmplew {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: encoding: [0x62,0xd4,0x45,0x0e,0x3b,0x54,0x80,0x7b] + ccmplew {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: ccmplel {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x0e,0x3b,0x4c,0x80,0x7b] + ccmplel {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: ccmpleq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x0e,0x3b,0x4c,0x80,0x7b] + ccmpleq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: ccmpleb {dfv=of} $123, %bl +# CHECK: encoding: [0x62,0xf4,0x44,0x0e,0x80,0xfb,0x7b] + ccmpleb {dfv=of} $123, %bl +# CHECK: ccmplew {dfv=of} $123, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x0e,0x83,0xfa,0x7b] + ccmplew {dfv=of} $123, %dx +# CHECK: ccmplel {dfv=of} $123, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x0e,0x83,0xf9,0x7b] + ccmplel {dfv=of} $123, %ecx +# CHECK: ccmpleq {dfv=of} $123, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0e,0x83,0xf9,0x7b] + ccmpleq {dfv=of} $123, %r9 +# CHECK: ccmplew {dfv=of} $1234, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x0e,0x81,0xfa,0xd2,0x04] + ccmplew {dfv=of} $1234, %dx +# CHECK: ccmplel {dfv=of} $123456, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x0e,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmplel {dfv=of} $123456, %ecx +# CHECK: ccmpleq {dfv=of} $123456, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0e,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpleq {dfv=of} $123456, %r9 +# CHECK: ccmpleb {dfv=of} %bl, %dl +# CHECK: encoding: [0x62,0xf4,0x44,0x0e,0x38,0xda] + ccmpleb {dfv=of} %bl, %dl +# CHECK: ccmplew {dfv=of} %dx, %ax +# CHECK: encoding: [0x62,0xf4,0x45,0x0e,0x39,0xd0] + ccmplew {dfv=of} %dx, %ax +# CHECK: ccmplel {dfv=of} %ecx, %edx +# CHECK: encoding: [0x62,0xf4,0x44,0x0e,0x39,0xca] + ccmplel {dfv=of} %ecx, %edx +# CHECK: ccmpleq {dfv=of} %r9, %r15 +# CHECK: encoding: [0x62,0x54,0xc4,0x0e,0x39,0xcf] + ccmpleq {dfv=of} %r9, %r15 +# CHECK: ccmpaeb {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x03,0x80,0x7c,0x80,0x7b,0x7b] + ccmpaeb {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpaew {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x03,0x83,0x7c,0x80,0x7b,0x7b] + ccmpaew {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpaew {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x03,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmpaew {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: ccmpael {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x03,0x83,0x7c,0x80,0x7b,0x7b] + ccmpael {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpael {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x03,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpael {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmpaeq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x03,0x83,0x7c,0x80,0x7b,0x7b] + ccmpaeq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpaeq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x03,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpaeq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmpaeb {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x03,0x38,0x5c,0x80,0x7b] + ccmpaeb {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: ccmpaew {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x03,0x39,0x54,0x80,0x7b] + ccmpaew {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: ccmpael {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x03,0x39,0x4c,0x80,0x7b] + ccmpael {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: ccmpaeq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0x54,0xc4,0x03,0x39,0x4c,0x80,0x7b] + ccmpaeq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: ccmpaeb {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: encoding: [0x62,0xd4,0x44,0x03,0x3a,0x5c,0x80,0x7b] + ccmpaeb {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: ccmpaew {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: encoding: [0x62,0xd4,0x45,0x03,0x3b,0x54,0x80,0x7b] + ccmpaew {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: ccmpael {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x03,0x3b,0x4c,0x80,0x7b] + ccmpael {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: ccmpaeq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x03,0x3b,0x4c,0x80,0x7b] + ccmpaeq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: ccmpaeb {dfv=of} $123, %bl +# CHECK: encoding: [0x62,0xf4,0x44,0x03,0x80,0xfb,0x7b] + ccmpaeb {dfv=of} $123, %bl +# CHECK: ccmpaew {dfv=of} $123, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x03,0x83,0xfa,0x7b] + ccmpaew {dfv=of} $123, %dx +# CHECK: ccmpael {dfv=of} $123, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x03,0x83,0xf9,0x7b] + ccmpael {dfv=of} $123, %ecx +# CHECK: ccmpaeq {dfv=of} $123, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x03,0x83,0xf9,0x7b] + ccmpaeq {dfv=of} $123, %r9 +# CHECK: ccmpaew {dfv=of} $1234, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x03,0x81,0xfa,0xd2,0x04] + ccmpaew {dfv=of} $1234, %dx +# CHECK: ccmpael {dfv=of} $123456, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x03,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpael {dfv=of} $123456, %ecx +# CHECK: ccmpaeq {dfv=of} $123456, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x03,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpaeq {dfv=of} $123456, %r9 +# CHECK: ccmpaeb {dfv=of} %bl, %dl +# CHECK: encoding: [0x62,0xf4,0x44,0x03,0x38,0xda] + ccmpaeb {dfv=of} %bl, %dl +# CHECK: ccmpaew {dfv=of} %dx, %ax +# CHECK: encoding: [0x62,0xf4,0x45,0x03,0x39,0xd0] + ccmpaew {dfv=of} %dx, %ax +# CHECK: ccmpael {dfv=of} %ecx, %edx +# CHECK: encoding: [0x62,0xf4,0x44,0x03,0x39,0xca] + ccmpael {dfv=of} %ecx, %edx +# CHECK: ccmpaeq {dfv=of} %r9, %r15 +# CHECK: encoding: [0x62,0x54,0xc4,0x03,0x39,0xcf] + ccmpaeq {dfv=of} %r9, %r15 +# CHECK: ccmpab {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x07,0x80,0x7c,0x80,0x7b,0x7b] + ccmpab {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpaw {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x07,0x83,0x7c,0x80,0x7b,0x7b] + ccmpaw {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpaw {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x07,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmpaw {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: ccmpal {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x07,0x83,0x7c,0x80,0x7b,0x7b] + ccmpal {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpal {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x07,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpal {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmpaq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x07,0x83,0x7c,0x80,0x7b,0x7b] + ccmpaq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpaq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x07,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpaq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmpab {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x07,0x38,0x5c,0x80,0x7b] + ccmpab {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: ccmpaw {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x07,0x39,0x54,0x80,0x7b] + ccmpaw {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: ccmpal {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x07,0x39,0x4c,0x80,0x7b] + ccmpal {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: ccmpaq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0x54,0xc4,0x07,0x39,0x4c,0x80,0x7b] + ccmpaq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: ccmpab {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: encoding: [0x62,0xd4,0x44,0x07,0x3a,0x5c,0x80,0x7b] + ccmpab {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: ccmpaw {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: encoding: [0x62,0xd4,0x45,0x07,0x3b,0x54,0x80,0x7b] + ccmpaw {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: ccmpal {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x07,0x3b,0x4c,0x80,0x7b] + ccmpal {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: ccmpaq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x07,0x3b,0x4c,0x80,0x7b] + ccmpaq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: ccmpab {dfv=of} $123, %bl +# CHECK: encoding: [0x62,0xf4,0x44,0x07,0x80,0xfb,0x7b] + ccmpab {dfv=of} $123, %bl +# CHECK: ccmpaw {dfv=of} $123, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x07,0x83,0xfa,0x7b] + ccmpaw {dfv=of} $123, %dx +# CHECK: ccmpal {dfv=of} $123, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x07,0x83,0xf9,0x7b] + ccmpal {dfv=of} $123, %ecx +# CHECK: ccmpaq {dfv=of} $123, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x07,0x83,0xf9,0x7b] + ccmpaq {dfv=of} $123, %r9 +# CHECK: ccmpaw {dfv=of} $1234, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x07,0x81,0xfa,0xd2,0x04] + ccmpaw {dfv=of} $1234, %dx +# CHECK: ccmpal {dfv=of} $123456, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x07,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpal {dfv=of} $123456, %ecx +# CHECK: ccmpaq {dfv=of} $123456, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x07,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpaq {dfv=of} $123456, %r9 +# CHECK: ccmpab {dfv=of} %bl, %dl +# CHECK: encoding: [0x62,0xf4,0x44,0x07,0x38,0xda] + ccmpab {dfv=of} %bl, %dl +# CHECK: ccmpaw {dfv=of} %dx, %ax +# CHECK: encoding: [0x62,0xf4,0x45,0x07,0x39,0xd0] + ccmpaw {dfv=of} %dx, %ax +# CHECK: ccmpal {dfv=of} %ecx, %edx +# CHECK: encoding: [0x62,0xf4,0x44,0x07,0x39,0xca] + ccmpal {dfv=of} %ecx, %edx +# CHECK: ccmpaq {dfv=of} %r9, %r15 +# CHECK: encoding: [0x62,0x54,0xc4,0x07,0x39,0xcf] + ccmpaq {dfv=of} %r9, %r15 +# CHECK: ccmpgeb {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0d,0x80,0x7c,0x80,0x7b,0x7b] + ccmpgeb {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpgew {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x0d,0x83,0x7c,0x80,0x7b,0x7b] + ccmpgew {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpgew {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x0d,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmpgew {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: ccmpgel {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0d,0x83,0x7c,0x80,0x7b,0x7b] + ccmpgel {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpgel {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0d,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpgel {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmpgeq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x0d,0x83,0x7c,0x80,0x7b,0x7b] + ccmpgeq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpgeq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x0d,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpgeq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmpgeb {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0d,0x38,0x5c,0x80,0x7b] + ccmpgeb {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: ccmpgew {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x0d,0x39,0x54,0x80,0x7b] + ccmpgew {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: ccmpgel {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0d,0x39,0x4c,0x80,0x7b] + ccmpgel {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: ccmpgeq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0x54,0xc4,0x0d,0x39,0x4c,0x80,0x7b] + ccmpgeq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: ccmpgeb {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: encoding: [0x62,0xd4,0x44,0x0d,0x3a,0x5c,0x80,0x7b] + ccmpgeb {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: ccmpgew {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: encoding: [0x62,0xd4,0x45,0x0d,0x3b,0x54,0x80,0x7b] + ccmpgew {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: ccmpgel {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x0d,0x3b,0x4c,0x80,0x7b] + ccmpgel {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: ccmpgeq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x0d,0x3b,0x4c,0x80,0x7b] + ccmpgeq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: ccmpgeb {dfv=of} $123, %bl +# CHECK: encoding: [0x62,0xf4,0x44,0x0d,0x80,0xfb,0x7b] + ccmpgeb {dfv=of} $123, %bl +# CHECK: ccmpgew {dfv=of} $123, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x0d,0x83,0xfa,0x7b] + ccmpgew {dfv=of} $123, %dx +# CHECK: ccmpgel {dfv=of} $123, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x0d,0x83,0xf9,0x7b] + ccmpgel {dfv=of} $123, %ecx +# CHECK: ccmpgeq {dfv=of} $123, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0d,0x83,0xf9,0x7b] + ccmpgeq {dfv=of} $123, %r9 +# CHECK: ccmpgew {dfv=of} $1234, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x0d,0x81,0xfa,0xd2,0x04] + ccmpgew {dfv=of} $1234, %dx +# CHECK: ccmpgel {dfv=of} $123456, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x0d,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpgel {dfv=of} $123456, %ecx +# CHECK: ccmpgeq {dfv=of} $123456, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0d,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpgeq {dfv=of} $123456, %r9 +# CHECK: ccmpgeb {dfv=of} %bl, %dl +# CHECK: encoding: [0x62,0xf4,0x44,0x0d,0x38,0xda] + ccmpgeb {dfv=of} %bl, %dl +# CHECK: ccmpgew {dfv=of} %dx, %ax +# CHECK: encoding: [0x62,0xf4,0x45,0x0d,0x39,0xd0] + ccmpgew {dfv=of} %dx, %ax +# CHECK: ccmpgel {dfv=of} %ecx, %edx +# CHECK: encoding: [0x62,0xf4,0x44,0x0d,0x39,0xca] + ccmpgel {dfv=of} %ecx, %edx +# CHECK: ccmpgeq {dfv=of} %r9, %r15 +# CHECK: encoding: [0x62,0x54,0xc4,0x0d,0x39,0xcf] + ccmpgeq {dfv=of} %r9, %r15 +# CHECK: ccmpgb {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0f,0x80,0x7c,0x80,0x7b,0x7b] + ccmpgb {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpgw {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x0f,0x83,0x7c,0x80,0x7b,0x7b] + ccmpgw {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpgw {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x0f,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmpgw {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: ccmpgl {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0f,0x83,0x7c,0x80,0x7b,0x7b] + ccmpgl {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpgl {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0f,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpgl {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmpgq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x0f,0x83,0x7c,0x80,0x7b,0x7b] + ccmpgq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpgq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x0f,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpgq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmpgb {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0f,0x38,0x5c,0x80,0x7b] + ccmpgb {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: ccmpgw {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x0f,0x39,0x54,0x80,0x7b] + ccmpgw {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: ccmpgl {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0f,0x39,0x4c,0x80,0x7b] + ccmpgl {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: ccmpgq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0x54,0xc4,0x0f,0x39,0x4c,0x80,0x7b] + ccmpgq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: ccmpgb {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: encoding: [0x62,0xd4,0x44,0x0f,0x3a,0x5c,0x80,0x7b] + ccmpgb {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: ccmpgw {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: encoding: [0x62,0xd4,0x45,0x0f,0x3b,0x54,0x80,0x7b] + ccmpgw {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: ccmpgl {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x0f,0x3b,0x4c,0x80,0x7b] + ccmpgl {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: ccmpgq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x0f,0x3b,0x4c,0x80,0x7b] + ccmpgq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: ccmpgb {dfv=of} $123, %bl +# CHECK: encoding: [0x62,0xf4,0x44,0x0f,0x80,0xfb,0x7b] + ccmpgb {dfv=of} $123, %bl +# CHECK: ccmpgw {dfv=of} $123, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x0f,0x83,0xfa,0x7b] + ccmpgw {dfv=of} $123, %dx +# CHECK: ccmpgl {dfv=of} $123, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x0f,0x83,0xf9,0x7b] + ccmpgl {dfv=of} $123, %ecx +# CHECK: ccmpgq {dfv=of} $123, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0f,0x83,0xf9,0x7b] + ccmpgq {dfv=of} $123, %r9 +# CHECK: ccmpgw {dfv=of} $1234, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x0f,0x81,0xfa,0xd2,0x04] + ccmpgw {dfv=of} $1234, %dx +# CHECK: ccmpgl {dfv=of} $123456, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x0f,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpgl {dfv=of} $123456, %ecx +# CHECK: ccmpgq {dfv=of} $123456, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0f,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpgq {dfv=of} $123456, %r9 +# CHECK: ccmpgb {dfv=of} %bl, %dl +# CHECK: encoding: [0x62,0xf4,0x44,0x0f,0x38,0xda] + ccmpgb {dfv=of} %bl, %dl +# CHECK: ccmpgw {dfv=of} %dx, %ax +# CHECK: encoding: [0x62,0xf4,0x45,0x0f,0x39,0xd0] + ccmpgw {dfv=of} %dx, %ax +# CHECK: ccmpgl {dfv=of} %ecx, %edx +# CHECK: encoding: [0x62,0xf4,0x44,0x0f,0x39,0xca] + ccmpgl {dfv=of} %ecx, %edx +# CHECK: ccmpgq {dfv=of} %r9, %r15 +# CHECK: encoding: [0x62,0x54,0xc4,0x0f,0x39,0xcf] + ccmpgq {dfv=of} %r9, %r15 +# CHECK: ccmpnob {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x01,0x80,0x7c,0x80,0x7b,0x7b] + ccmpnob {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpnow {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x01,0x83,0x7c,0x80,0x7b,0x7b] + ccmpnow {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpnow {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x01,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmpnow {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: ccmpnol {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x01,0x83,0x7c,0x80,0x7b,0x7b] + ccmpnol {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpnol {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x01,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpnol {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmpnoq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x01,0x83,0x7c,0x80,0x7b,0x7b] + ccmpnoq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpnoq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x01,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpnoq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmpnob {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x01,0x38,0x5c,0x80,0x7b] + ccmpnob {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: ccmpnow {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x01,0x39,0x54,0x80,0x7b] + ccmpnow {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: ccmpnol {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x01,0x39,0x4c,0x80,0x7b] + ccmpnol {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: ccmpnoq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0x54,0xc4,0x01,0x39,0x4c,0x80,0x7b] + ccmpnoq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: ccmpnob {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: encoding: [0x62,0xd4,0x44,0x01,0x3a,0x5c,0x80,0x7b] + ccmpnob {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: ccmpnow {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: encoding: [0x62,0xd4,0x45,0x01,0x3b,0x54,0x80,0x7b] + ccmpnow {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: ccmpnol {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x01,0x3b,0x4c,0x80,0x7b] + ccmpnol {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: ccmpnoq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x01,0x3b,0x4c,0x80,0x7b] + ccmpnoq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: ccmpnob {dfv=of} $123, %bl +# CHECK: encoding: [0x62,0xf4,0x44,0x01,0x80,0xfb,0x7b] + ccmpnob {dfv=of} $123, %bl +# CHECK: ccmpnow {dfv=of} $123, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x01,0x83,0xfa,0x7b] + ccmpnow {dfv=of} $123, %dx +# CHECK: ccmpnol {dfv=of} $123, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x01,0x83,0xf9,0x7b] + ccmpnol {dfv=of} $123, %ecx +# CHECK: ccmpnoq {dfv=of} $123, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x01,0x83,0xf9,0x7b] + ccmpnoq {dfv=of} $123, %r9 +# CHECK: ccmpnow {dfv=of} $1234, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x01,0x81,0xfa,0xd2,0x04] + ccmpnow {dfv=of} $1234, %dx +# CHECK: ccmpnol {dfv=of} $123456, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x01,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpnol {dfv=of} $123456, %ecx +# CHECK: ccmpnoq {dfv=of} $123456, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x01,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpnoq {dfv=of} $123456, %r9 +# CHECK: ccmpnob {dfv=of} %bl, %dl +# CHECK: encoding: [0x62,0xf4,0x44,0x01,0x38,0xda] + ccmpnob {dfv=of} %bl, %dl +# CHECK: ccmpnow {dfv=of} %dx, %ax +# CHECK: encoding: [0x62,0xf4,0x45,0x01,0x39,0xd0] + ccmpnow {dfv=of} %dx, %ax +# CHECK: ccmpnol {dfv=of} %ecx, %edx +# CHECK: encoding: [0x62,0xf4,0x44,0x01,0x39,0xca] + ccmpnol {dfv=of} %ecx, %edx +# CHECK: ccmpnoq {dfv=of} %r9, %r15 +# CHECK: encoding: [0x62,0x54,0xc4,0x01,0x39,0xcf] + ccmpnoq {dfv=of} %r9, %r15 +# CHECK: ccmpnsb {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x09,0x80,0x7c,0x80,0x7b,0x7b] + ccmpnsb {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpnsw {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x09,0x83,0x7c,0x80,0x7b,0x7b] + ccmpnsw {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpnsw {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x09,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmpnsw {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: ccmpnsl {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x09,0x83,0x7c,0x80,0x7b,0x7b] + ccmpnsl {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpnsl {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x09,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpnsl {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmpnsq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x09,0x83,0x7c,0x80,0x7b,0x7b] + ccmpnsq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpnsq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x09,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpnsq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmpnsb {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x09,0x38,0x5c,0x80,0x7b] + ccmpnsb {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: ccmpnsw {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x09,0x39,0x54,0x80,0x7b] + ccmpnsw {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: ccmpnsl {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x09,0x39,0x4c,0x80,0x7b] + ccmpnsl {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: ccmpnsq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0x54,0xc4,0x09,0x39,0x4c,0x80,0x7b] + ccmpnsq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: ccmpnsb {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: encoding: [0x62,0xd4,0x44,0x09,0x3a,0x5c,0x80,0x7b] + ccmpnsb {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: ccmpnsw {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: encoding: [0x62,0xd4,0x45,0x09,0x3b,0x54,0x80,0x7b] + ccmpnsw {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: ccmpnsl {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x09,0x3b,0x4c,0x80,0x7b] + ccmpnsl {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: ccmpnsq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x09,0x3b,0x4c,0x80,0x7b] + ccmpnsq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: ccmpnsb {dfv=of} $123, %bl +# CHECK: encoding: [0x62,0xf4,0x44,0x09,0x80,0xfb,0x7b] + ccmpnsb {dfv=of} $123, %bl +# CHECK: ccmpnsw {dfv=of} $123, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x09,0x83,0xfa,0x7b] + ccmpnsw {dfv=of} $123, %dx +# CHECK: ccmpnsl {dfv=of} $123, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x09,0x83,0xf9,0x7b] + ccmpnsl {dfv=of} $123, %ecx +# CHECK: ccmpnsq {dfv=of} $123, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x09,0x83,0xf9,0x7b] + ccmpnsq {dfv=of} $123, %r9 +# CHECK: ccmpnsw {dfv=of} $1234, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x09,0x81,0xfa,0xd2,0x04] + ccmpnsw {dfv=of} $1234, %dx +# CHECK: ccmpnsl {dfv=of} $123456, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x09,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpnsl {dfv=of} $123456, %ecx +# CHECK: ccmpnsq {dfv=of} $123456, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x09,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpnsq {dfv=of} $123456, %r9 +# CHECK: ccmpnsb {dfv=of} %bl, %dl +# CHECK: encoding: [0x62,0xf4,0x44,0x09,0x38,0xda] + ccmpnsb {dfv=of} %bl, %dl +# CHECK: ccmpnsw {dfv=of} %dx, %ax +# CHECK: encoding: [0x62,0xf4,0x45,0x09,0x39,0xd0] + ccmpnsw {dfv=of} %dx, %ax +# CHECK: ccmpnsl {dfv=of} %ecx, %edx +# CHECK: encoding: [0x62,0xf4,0x44,0x09,0x39,0xca] + ccmpnsl {dfv=of} %ecx, %edx +# CHECK: ccmpnsq {dfv=of} %r9, %r15 +# CHECK: encoding: [0x62,0x54,0xc4,0x09,0x39,0xcf] + ccmpnsq {dfv=of} %r9, %r15 +# CHECK: ccmpneb {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x05,0x80,0x7c,0x80,0x7b,0x7b] + ccmpneb {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpnew {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x05,0x83,0x7c,0x80,0x7b,0x7b] + ccmpnew {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpnew {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x05,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmpnew {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: ccmpnel {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x05,0x83,0x7c,0x80,0x7b,0x7b] + ccmpnel {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpnel {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x05,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpnel {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmpneq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x05,0x83,0x7c,0x80,0x7b,0x7b] + ccmpneq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpneq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x05,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpneq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmpneb {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x05,0x38,0x5c,0x80,0x7b] + ccmpneb {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: ccmpnew {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x05,0x39,0x54,0x80,0x7b] + ccmpnew {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: ccmpnel {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x05,0x39,0x4c,0x80,0x7b] + ccmpnel {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: ccmpneq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0x54,0xc4,0x05,0x39,0x4c,0x80,0x7b] + ccmpneq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: ccmpneb {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: encoding: [0x62,0xd4,0x44,0x05,0x3a,0x5c,0x80,0x7b] + ccmpneb {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: ccmpnew {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: encoding: [0x62,0xd4,0x45,0x05,0x3b,0x54,0x80,0x7b] + ccmpnew {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: ccmpnel {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x05,0x3b,0x4c,0x80,0x7b] + ccmpnel {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: ccmpneq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x05,0x3b,0x4c,0x80,0x7b] + ccmpneq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: ccmpneb {dfv=of} $123, %bl +# CHECK: encoding: [0x62,0xf4,0x44,0x05,0x80,0xfb,0x7b] + ccmpneb {dfv=of} $123, %bl +# CHECK: ccmpnew {dfv=of} $123, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x05,0x83,0xfa,0x7b] + ccmpnew {dfv=of} $123, %dx +# CHECK: ccmpnel {dfv=of} $123, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x05,0x83,0xf9,0x7b] + ccmpnel {dfv=of} $123, %ecx +# CHECK: ccmpneq {dfv=of} $123, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x05,0x83,0xf9,0x7b] + ccmpneq {dfv=of} $123, %r9 +# CHECK: ccmpnew {dfv=of} $1234, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x05,0x81,0xfa,0xd2,0x04] + ccmpnew {dfv=of} $1234, %dx +# CHECK: ccmpnel {dfv=of} $123456, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x05,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpnel {dfv=of} $123456, %ecx +# CHECK: ccmpneq {dfv=of} $123456, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x05,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpneq {dfv=of} $123456, %r9 +# CHECK: ccmpneb {dfv=of} %bl, %dl +# CHECK: encoding: [0x62,0xf4,0x44,0x05,0x38,0xda] + ccmpneb {dfv=of} %bl, %dl +# CHECK: ccmpnew {dfv=of} %dx, %ax +# CHECK: encoding: [0x62,0xf4,0x45,0x05,0x39,0xd0] + ccmpnew {dfv=of} %dx, %ax +# CHECK: ccmpnel {dfv=of} %ecx, %edx +# CHECK: encoding: [0x62,0xf4,0x44,0x05,0x39,0xca] + ccmpnel {dfv=of} %ecx, %edx +# CHECK: ccmpneq {dfv=of} %r9, %r15 +# CHECK: encoding: [0x62,0x54,0xc4,0x05,0x39,0xcf] + ccmpneq {dfv=of} %r9, %r15 +# CHECK: ccmpob {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x00,0x80,0x7c,0x80,0x7b,0x7b] + ccmpob {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpow {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x00,0x83,0x7c,0x80,0x7b,0x7b] + ccmpow {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpow {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x00,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmpow {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: ccmpol {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x00,0x83,0x7c,0x80,0x7b,0x7b] + ccmpol {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpol {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x00,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpol {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmpoq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x00,0x83,0x7c,0x80,0x7b,0x7b] + ccmpoq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpoq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x00,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpoq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmpob {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x00,0x38,0x5c,0x80,0x7b] + ccmpob {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: ccmpow {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x00,0x39,0x54,0x80,0x7b] + ccmpow {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: ccmpol {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x00,0x39,0x4c,0x80,0x7b] + ccmpol {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: ccmpoq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0x54,0xc4,0x00,0x39,0x4c,0x80,0x7b] + ccmpoq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: ccmpob {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: encoding: [0x62,0xd4,0x44,0x00,0x3a,0x5c,0x80,0x7b] + ccmpob {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: ccmpow {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: encoding: [0x62,0xd4,0x45,0x00,0x3b,0x54,0x80,0x7b] + ccmpow {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: ccmpol {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x00,0x3b,0x4c,0x80,0x7b] + ccmpol {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: ccmpoq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x00,0x3b,0x4c,0x80,0x7b] + ccmpoq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: ccmpob {dfv=of} $123, %bl +# CHECK: encoding: [0x62,0xf4,0x44,0x00,0x80,0xfb,0x7b] + ccmpob {dfv=of} $123, %bl +# CHECK: ccmpow {dfv=of} $123, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x00,0x83,0xfa,0x7b] + ccmpow {dfv=of} $123, %dx +# CHECK: ccmpol {dfv=of} $123, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x00,0x83,0xf9,0x7b] + ccmpol {dfv=of} $123, %ecx +# CHECK: ccmpoq {dfv=of} $123, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x00,0x83,0xf9,0x7b] + ccmpoq {dfv=of} $123, %r9 +# CHECK: ccmpow {dfv=of} $1234, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x00,0x81,0xfa,0xd2,0x04] + ccmpow {dfv=of} $1234, %dx +# CHECK: ccmpol {dfv=of} $123456, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x00,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpol {dfv=of} $123456, %ecx +# CHECK: ccmpoq {dfv=of} $123456, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x00,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpoq {dfv=of} $123456, %r9 +# CHECK: ccmpob {dfv=of} %bl, %dl +# CHECK: encoding: [0x62,0xf4,0x44,0x00,0x38,0xda] + ccmpob {dfv=of} %bl, %dl +# CHECK: ccmpow {dfv=of} %dx, %ax +# CHECK: encoding: [0x62,0xf4,0x45,0x00,0x39,0xd0] + ccmpow {dfv=of} %dx, %ax +# CHECK: ccmpol {dfv=of} %ecx, %edx +# CHECK: encoding: [0x62,0xf4,0x44,0x00,0x39,0xca] + ccmpol {dfv=of} %ecx, %edx +# CHECK: ccmpoq {dfv=of} %r9, %r15 +# CHECK: encoding: [0x62,0x54,0xc4,0x00,0x39,0xcf] + ccmpoq {dfv=of} %r9, %r15 +# CHECK: ccmpsb {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x08,0x80,0x7c,0x80,0x7b,0x7b] + ccmpsb {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpsw {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x08,0x83,0x7c,0x80,0x7b,0x7b] + ccmpsw {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpsw {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x08,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmpsw {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: ccmpsl {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x08,0x83,0x7c,0x80,0x7b,0x7b] + ccmpsl {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpsl {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x08,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpsl {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmpsq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x08,0x83,0x7c,0x80,0x7b,0x7b] + ccmpsq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpsq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x08,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpsq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmpsb {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x08,0x38,0x5c,0x80,0x7b] + ccmpsb {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: ccmpsw {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x08,0x39,0x54,0x80,0x7b] + ccmpsw {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: ccmpsl {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x08,0x39,0x4c,0x80,0x7b] + ccmpsl {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: ccmpsq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0x54,0xc4,0x08,0x39,0x4c,0x80,0x7b] + ccmpsq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: ccmpsb {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: encoding: [0x62,0xd4,0x44,0x08,0x3a,0x5c,0x80,0x7b] + ccmpsb {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: ccmpsw {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: encoding: [0x62,0xd4,0x45,0x08,0x3b,0x54,0x80,0x7b] + ccmpsw {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: ccmpsl {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x08,0x3b,0x4c,0x80,0x7b] + ccmpsl {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: ccmpsq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x08,0x3b,0x4c,0x80,0x7b] + ccmpsq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: ccmpsb {dfv=of} $123, %bl +# CHECK: encoding: [0x62,0xf4,0x44,0x08,0x80,0xfb,0x7b] + ccmpsb {dfv=of} $123, %bl +# CHECK: ccmpsw {dfv=of} $123, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x08,0x83,0xfa,0x7b] + ccmpsw {dfv=of} $123, %dx +# CHECK: ccmpsl {dfv=of} $123, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x08,0x83,0xf9,0x7b] + ccmpsl {dfv=of} $123, %ecx +# CHECK: ccmpsq {dfv=of} $123, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x08,0x83,0xf9,0x7b] + ccmpsq {dfv=of} $123, %r9 +# CHECK: ccmpsw {dfv=of} $1234, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x08,0x81,0xfa,0xd2,0x04] + ccmpsw {dfv=of} $1234, %dx +# CHECK: ccmpsl {dfv=of} $123456, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x08,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpsl {dfv=of} $123456, %ecx +# CHECK: ccmpsq {dfv=of} $123456, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x08,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpsq {dfv=of} $123456, %r9 +# CHECK: ccmpsb {dfv=of} %bl, %dl +# CHECK: encoding: [0x62,0xf4,0x44,0x08,0x38,0xda] + ccmpsb {dfv=of} %bl, %dl +# CHECK: ccmpsw {dfv=of} %dx, %ax +# CHECK: encoding: [0x62,0xf4,0x45,0x08,0x39,0xd0] + ccmpsw {dfv=of} %dx, %ax +# CHECK: ccmpsl {dfv=of} %ecx, %edx +# CHECK: encoding: [0x62,0xf4,0x44,0x08,0x39,0xca] + ccmpsl {dfv=of} %ecx, %edx +# CHECK: ccmpsq {dfv=of} %r9, %r15 +# CHECK: encoding: [0x62,0x54,0xc4,0x08,0x39,0xcf] + ccmpsq {dfv=of} %r9, %r15 +# CHECK: ccmptb {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0a,0x80,0x7c,0x80,0x7b,0x7b] + ccmptb {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmptw {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x0a,0x83,0x7c,0x80,0x7b,0x7b] + ccmptw {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmptw {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x0a,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmptw {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: ccmptl {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0a,0x83,0x7c,0x80,0x7b,0x7b] + ccmptl {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmptl {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0a,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmptl {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmptq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x0a,0x83,0x7c,0x80,0x7b,0x7b] + ccmptq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmptq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x0a,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmptq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmptb {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0a,0x38,0x5c,0x80,0x7b] + ccmptb {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: ccmptw {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x0a,0x39,0x54,0x80,0x7b] + ccmptw {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: ccmptl {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x0a,0x39,0x4c,0x80,0x7b] + ccmptl {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: ccmptq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0x54,0xc4,0x0a,0x39,0x4c,0x80,0x7b] + ccmptq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: ccmptb {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: encoding: [0x62,0xd4,0x44,0x0a,0x3a,0x5c,0x80,0x7b] + ccmptb {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: ccmptw {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: encoding: [0x62,0xd4,0x45,0x0a,0x3b,0x54,0x80,0x7b] + ccmptw {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: ccmptl {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x0a,0x3b,0x4c,0x80,0x7b] + ccmptl {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: ccmptq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x0a,0x3b,0x4c,0x80,0x7b] + ccmptq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: ccmptb {dfv=of} $123, %bl +# CHECK: encoding: [0x62,0xf4,0x44,0x0a,0x80,0xfb,0x7b] + ccmptb {dfv=of} $123, %bl +# CHECK: ccmptw {dfv=of} $123, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x0a,0x83,0xfa,0x7b] + ccmptw {dfv=of} $123, %dx +# CHECK: ccmptl {dfv=of} $123, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x0a,0x83,0xf9,0x7b] + ccmptl {dfv=of} $123, %ecx +# CHECK: ccmptq {dfv=of} $123, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0a,0x83,0xf9,0x7b] + ccmptq {dfv=of} $123, %r9 +# CHECK: ccmptw {dfv=of} $1234, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x0a,0x81,0xfa,0xd2,0x04] + ccmptw {dfv=of} $1234, %dx +# CHECK: ccmptl {dfv=of} $123456, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x0a,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmptl {dfv=of} $123456, %ecx +# CHECK: ccmptq {dfv=of} $123456, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0a,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmptq {dfv=of} $123456, %r9 +# CHECK: ccmptb {dfv=of} %bl, %dl +# CHECK: encoding: [0x62,0xf4,0x44,0x0a,0x38,0xda] + ccmptb {dfv=of} %bl, %dl +# CHECK: ccmptw {dfv=of} %dx, %ax +# CHECK: encoding: [0x62,0xf4,0x45,0x0a,0x39,0xd0] + ccmptw {dfv=of} %dx, %ax +# CHECK: ccmptl {dfv=of} %ecx, %edx +# CHECK: encoding: [0x62,0xf4,0x44,0x0a,0x39,0xca] + ccmptl {dfv=of} %ecx, %edx +# CHECK: ccmptq {dfv=of} %r9, %r15 +# CHECK: encoding: [0x62,0x54,0xc4,0x0a,0x39,0xcf] + ccmptq {dfv=of} %r9, %r15 +# CHECK: ccmpeb {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x04,0x80,0x7c,0x80,0x7b,0x7b] + ccmpeb {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpew {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x04,0x83,0x7c,0x80,0x7b,0x7b] + ccmpew {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpew {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x04,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmpew {dfv=of} $1234, 123(%r8,%rax,4) +# CHECK: ccmpel {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x04,0x83,0x7c,0x80,0x7b,0x7b] + ccmpel {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpel {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x04,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpel {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmpeq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x04,0x83,0x7c,0x80,0x7b,0x7b] + ccmpeq {dfv=of} $123, 123(%r8,%rax,4) +# CHECK: ccmpeq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0xc4,0x04,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpeq {dfv=of} $123456, 123(%r8,%rax,4) +# CHECK: ccmpeb {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x04,0x38,0x5c,0x80,0x7b] + ccmpeb {dfv=of} %bl, 123(%r8,%rax,4) +# CHECK: ccmpew {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x45,0x04,0x39,0x54,0x80,0x7b] + ccmpew {dfv=of} %dx, 123(%r8,%rax,4) +# CHECK: ccmpel {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0xd4,0x44,0x04,0x39,0x4c,0x80,0x7b] + ccmpel {dfv=of} %ecx, 123(%r8,%rax,4) +# CHECK: ccmpeq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: encoding: [0x62,0x54,0xc4,0x04,0x39,0x4c,0x80,0x7b] + ccmpeq {dfv=of} %r9, 123(%r8,%rax,4) +# CHECK: ccmpeb {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: encoding: [0x62,0xd4,0x44,0x04,0x3a,0x5c,0x80,0x7b] + ccmpeb {dfv=of} 123(%r8,%rax,4), %bl +# CHECK: ccmpew {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: encoding: [0x62,0xd4,0x45,0x04,0x3b,0x54,0x80,0x7b] + ccmpew {dfv=of} 123(%r8,%rax,4), %dx +# CHECK: ccmpel {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x04,0x3b,0x4c,0x80,0x7b] + ccmpel {dfv=of} 123(%r8,%rax,4), %ecx +# CHECK: ccmpeq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x04,0x3b,0x4c,0x80,0x7b] + ccmpeq {dfv=of} 123(%r8,%rax,4), %r9 +# CHECK: ccmpeb {dfv=of} $123, %bl +# CHECK: encoding: [0x62,0xf4,0x44,0x04,0x80,0xfb,0x7b] + ccmpeb {dfv=of} $123, %bl +# CHECK: ccmpew {dfv=of} $123, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x04,0x83,0xfa,0x7b] + ccmpew {dfv=of} $123, %dx +# CHECK: ccmpel {dfv=of} $123, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x04,0x83,0xf9,0x7b] + ccmpel {dfv=of} $123, %ecx +# CHECK: ccmpeq {dfv=of} $123, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x04,0x83,0xf9,0x7b] + ccmpeq {dfv=of} $123, %r9 +# CHECK: ccmpew {dfv=of} $1234, %dx +# CHECK: encoding: [0x62,0xf4,0x45,0x04,0x81,0xfa,0xd2,0x04] + ccmpew {dfv=of} $1234, %dx +# CHECK: ccmpel {dfv=of} $123456, %ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x04,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpel {dfv=of} $123456, %ecx +# CHECK: ccmpeq {dfv=of} $123456, %r9 +# CHECK: encoding: [0x62,0xd4,0xc4,0x04,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpeq {dfv=of} $123456, %r9 +# CHECK: ccmpeb {dfv=of} %bl, %dl +# CHECK: encoding: [0x62,0xf4,0x44,0x04,0x38,0xda] + ccmpeb {dfv=of} %bl, %dl +# CHECK: ccmpew {dfv=of} %dx, %ax +# CHECK: encoding: [0x62,0xf4,0x45,0x04,0x39,0xd0] + ccmpew {dfv=of} %dx, %ax +# CHECK: ccmpel {dfv=of} %ecx, %edx +# CHECK: encoding: [0x62,0xf4,0x44,0x04,0x39,0xca] + ccmpel {dfv=of} %ecx, %edx +# CHECK: ccmpeq {dfv=of} %r9, %r15 +# CHECK: encoding: [0x62,0x54,0xc4,0x04,0x39,0xcf] + ccmpeq {dfv=of} %r9, %r15 diff --git a/llvm/test/MC/X86/apx/ccmp-intel-error.s b/llvm/test/MC/X86/apx/ccmp-intel-error.s new file mode 100644 index 000000000000..e37cc53548ae --- /dev/null +++ b/llvm/test/MC/X86/apx/ccmp-intel-error.s @@ -0,0 +1,29 @@ +# RUN: not llvm-mc -triple x86_64 -show-encoding -x86-asm-syntax=intel -output-asm-variant=1 %s 2>&1 | FileCheck --strict-whitespace %s + +# CHECK: [[#@LINE+2]]:7: error: Expected { at this point +# CHECK: ccmpe 1 rbx, rax +ccmpe 1 rbx, rax + +# CHECK: [[#@LINE+2]]:8: error: Expected dfv at this point +# CHECK: ccmpe {sf} rbx, rax +ccmpe {sf} rbx, rax + +# CHECK: [[#@LINE+2]]:11: error: Expected = at this point +# CHECK: ccmpe {dfv:sf} rbx, rax +ccmpe {dfv:sf} rbx, rax + +# CHECK: [[#@LINE+2]]:18: error: Expected } or , at this point +# CHECK: ccmpe {dfv=sf,cf rbx, rax +ccmpe {dfv=sf,cf rbx, rax + +# CHECK: [[#@LINE+2]]:12: error: Invalid conditional flags +# CHECK: ccmpe {dfv=pf} rbx, rax +ccmpe {dfv=pf} rbx, rax + +# CHECK: [[#@LINE+2]]:19: error: Duplicated conditional flag +# CHECK: ccmpeq {dfv=of,zf,of} rbx, rax +ccmpeq {dfv=of,zf,of} rbx, rax + +# CHECK: [[#@LINE+2]]:24: error: Expected } at this point +# CHECK: ccmpeq {dfv=of,sf,zf,cf,of} rbx, rax +ccmpeq {dfv=of,sf,zf,cf,of} rbx, rax diff --git a/llvm/test/MC/X86/apx/ccmp-intel.s b/llvm/test/MC/X86/apx/ccmp-intel.s new file mode 100644 index 000000000000..095f7fb71b7c --- /dev/null +++ b/llvm/test/MC/X86/apx/ccmp-intel.s @@ -0,0 +1,1210 @@ +# RUN: llvm-mc -triple x86_64 -show-encoding -x86-asm-syntax=intel -output-asm-variant=1 %s | FileCheck %s + +## Condition flags + +# CHECK: ccmpo {dfv=} rbx, rax +# CHECK: encoding: [0x62,0xf4,0x84,0x00,0x39,0xc3] + ccmpo {dfv=} rbx, rax +# CHECK: ccmpo {dfv=of} rbx, rax +# CHECK: encoding: [0x62,0xf4,0xc4,0x00,0x39,0xc3] + ccmpo {dfv=of} rbx, rax +# CHECK: ccmpo {dfv=sf} rbx, rax +# CHECK: encoding: [0x62,0xf4,0xa4,0x00,0x39,0xc3] + ccmpo {dfv=sf} rbx, rax +# CHECK: ccmpo {dfv=zf} rbx, rax +# CHECK: encoding: [0x62,0xf4,0x94,0x00,0x39,0xc3] + ccmpo {dfv=zf} rbx, rax +# CHECK: ccmpo {dfv=cf} rbx, rax +# CHECK: encoding: [0x62,0xf4,0x8c,0x00,0x39,0xc3] + ccmpo {dfv=cf} rbx, rax +# CHECK: ccmpo {dfv=of,sf} rbx, rax +# CHECK: encoding: [0x62,0xf4,0xe4,0x00,0x39,0xc3] + ccmpo {dfv=of,sf} rbx, rax +# CHECK: ccmpo {dfv=of,sf} rbx, rax +# CHECK: encoding: [0x62,0xf4,0xe4,0x00,0x39,0xc3] + ccmpo {dfv=sf,of} rbx, rax +# CHECK: ccmpo {dfv=of,sf,zf} rbx, rax +# CHECK: encoding: [0x62,0xf4,0xf4,0x00,0x39,0xc3] + ccmpo {dfv=of,sf,zf} rbx, rax +# CHECK: ccmpo {dfv=of,sf,zf} rbx, rax +# CHECK: encoding: [0x62,0xf4,0xf4,0x00,0x39,0xc3] + ccmpo {dfv=zf,of,sf} rbx, rax +# CHECK: ccmpo {dfv=of,sf,zf,cf} rbx, rax +# CHECK: encoding: [0x62,0xf4,0xfc,0x00,0x39,0xc3] + ccmpo {dfv=of,sf,zf,cf} rbx, rax +# CHECK: ccmpo {dfv=of,sf,zf,cf} rbx, rax +# CHECK: encoding: [0x62,0xf4,0xfc,0x00,0x39,0xc3] + ccmpo {dfv=cf,zf,sf,of} rbx, rax + +## Condition code + +# CHECK: ccmpb {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x02,0x80,0x7c,0x80,0x7b,0x7b] + ccmpb {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpb {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x45,0x02,0x83,0x7c,0x80,0x7b,0x7b] + ccmpb {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpb {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: encoding: [0x62,0xd4,0x45,0x02,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmpb {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: ccmpb {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x02,0x83,0x7c,0x80,0x7b,0x7b] + ccmpb {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpb {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0x44,0x02,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpb {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmpb {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x02,0x83,0x7c,0x80,0x7b,0x7b] + ccmpb {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpb {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x02,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpb {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmpb {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: encoding: [0x62,0xd4,0x44,0x02,0x38,0x5c,0x80,0x7b] + ccmpb {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: ccmpb {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: encoding: [0x62,0xd4,0x45,0x02,0x39,0x54,0x80,0x7b] + ccmpb {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: ccmpb {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x02,0x39,0x4c,0x80,0x7b] + ccmpb {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: ccmpb {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x02,0x39,0x4c,0x80,0x7b] + ccmpb {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: ccmpb {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x02,0x3a,0x5c,0x80,0x7b] + ccmpb {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: ccmpb {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x45,0x02,0x3b,0x54,0x80,0x7b] + ccmpb {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: ccmpb {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x02,0x3b,0x4c,0x80,0x7b] + ccmpb {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: ccmpb {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0x54,0xc4,0x02,0x3b,0x4c,0x80,0x7b] + ccmpb {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: ccmpb {dfv=of} bl, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x02,0x80,0xfb,0x7b] + ccmpb {dfv=of} bl, 123 +# CHECK: ccmpb {dfv=of} dx, 123 +# CHECK: encoding: [0x62,0xf4,0x45,0x02,0x83,0xfa,0x7b] + ccmpb {dfv=of} dx, 123 +# CHECK: ccmpb {dfv=of} ecx, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x02,0x83,0xf9,0x7b] + ccmpb {dfv=of} ecx, 123 +# CHECK: ccmpb {dfv=of} r9, 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x02,0x83,0xf9,0x7b] + ccmpb {dfv=of} r9, 123 +# CHECK: ccmpb {dfv=of} dx, 1234 +# CHECK: encoding: [0x62,0xf4,0x45,0x02,0x81,0xfa,0xd2,0x04] + ccmpb {dfv=of} dx, 1234 +# CHECK: ccmpb {dfv=of} ecx, 123456 +# CHECK: encoding: [0x62,0xf4,0x44,0x02,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpb {dfv=of} ecx, 123456 +# CHECK: ccmpb {dfv=of} r9, 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x02,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpb {dfv=of} r9, 123456 +# CHECK: ccmpb {dfv=of} dl, bl +# CHECK: encoding: [0x62,0xf4,0x44,0x02,0x38,0xda] + ccmpb {dfv=of} dl, bl +# CHECK: ccmpb {dfv=of} ax, dx +# CHECK: encoding: [0x62,0xf4,0x45,0x02,0x39,0xd0] + ccmpb {dfv=of} ax, dx +# CHECK: ccmpb {dfv=of} edx, ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x02,0x39,0xca] + ccmpb {dfv=of} edx, ecx +# CHECK: ccmpb {dfv=of} r15, r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x02,0x39,0xcf] + ccmpb {dfv=of} r15, r9 +# CHECK: ccmpf {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x0b,0x80,0x7c,0x80,0x7b,0x7b] + ccmpf {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpf {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x45,0x0b,0x83,0x7c,0x80,0x7b,0x7b] + ccmpf {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpf {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: encoding: [0x62,0xd4,0x45,0x0b,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmpf {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: ccmpf {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x0b,0x83,0x7c,0x80,0x7b,0x7b] + ccmpf {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpf {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0x44,0x0b,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpf {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmpf {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0b,0x83,0x7c,0x80,0x7b,0x7b] + ccmpf {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpf {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0b,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpf {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmpf {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: encoding: [0x62,0xd4,0x44,0x0b,0x38,0x5c,0x80,0x7b] + ccmpf {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: ccmpf {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: encoding: [0x62,0xd4,0x45,0x0b,0x39,0x54,0x80,0x7b] + ccmpf {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: ccmpf {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x0b,0x39,0x4c,0x80,0x7b] + ccmpf {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: ccmpf {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x0b,0x39,0x4c,0x80,0x7b] + ccmpf {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: ccmpf {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x0b,0x3a,0x5c,0x80,0x7b] + ccmpf {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: ccmpf {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x45,0x0b,0x3b,0x54,0x80,0x7b] + ccmpf {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: ccmpf {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x0b,0x3b,0x4c,0x80,0x7b] + ccmpf {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: ccmpf {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0x54,0xc4,0x0b,0x3b,0x4c,0x80,0x7b] + ccmpf {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: ccmpf {dfv=of} bl, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x0b,0x80,0xfb,0x7b] + ccmpf {dfv=of} bl, 123 +# CHECK: ccmpf {dfv=of} dx, 123 +# CHECK: encoding: [0x62,0xf4,0x45,0x0b,0x83,0xfa,0x7b] + ccmpf {dfv=of} dx, 123 +# CHECK: ccmpf {dfv=of} ecx, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x0b,0x83,0xf9,0x7b] + ccmpf {dfv=of} ecx, 123 +# CHECK: ccmpf {dfv=of} r9, 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0b,0x83,0xf9,0x7b] + ccmpf {dfv=of} r9, 123 +# CHECK: ccmpf {dfv=of} dx, 1234 +# CHECK: encoding: [0x62,0xf4,0x45,0x0b,0x81,0xfa,0xd2,0x04] + ccmpf {dfv=of} dx, 1234 +# CHECK: ccmpf {dfv=of} ecx, 123456 +# CHECK: encoding: [0x62,0xf4,0x44,0x0b,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpf {dfv=of} ecx, 123456 +# CHECK: ccmpf {dfv=of} r9, 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0b,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpf {dfv=of} r9, 123456 +# CHECK: ccmpf {dfv=of} dl, bl +# CHECK: encoding: [0x62,0xf4,0x44,0x0b,0x38,0xda] + ccmpf {dfv=of} dl, bl +# CHECK: ccmpf {dfv=of} ax, dx +# CHECK: encoding: [0x62,0xf4,0x45,0x0b,0x39,0xd0] + ccmpf {dfv=of} ax, dx +# CHECK: ccmpf {dfv=of} edx, ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x0b,0x39,0xca] + ccmpf {dfv=of} edx, ecx +# CHECK: ccmpf {dfv=of} r15, r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x0b,0x39,0xcf] + ccmpf {dfv=of} r15, r9 +# CHECK: ccmpl {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x0c,0x80,0x7c,0x80,0x7b,0x7b] + ccmpl {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpl {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x45,0x0c,0x83,0x7c,0x80,0x7b,0x7b] + ccmpl {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpl {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: encoding: [0x62,0xd4,0x45,0x0c,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmpl {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: ccmpl {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x0c,0x83,0x7c,0x80,0x7b,0x7b] + ccmpl {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpl {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0x44,0x0c,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpl {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmpl {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0c,0x83,0x7c,0x80,0x7b,0x7b] + ccmpl {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpl {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0c,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpl {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmpl {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: encoding: [0x62,0xd4,0x44,0x0c,0x38,0x5c,0x80,0x7b] + ccmpl {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: ccmpl {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: encoding: [0x62,0xd4,0x45,0x0c,0x39,0x54,0x80,0x7b] + ccmpl {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: ccmpl {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x0c,0x39,0x4c,0x80,0x7b] + ccmpl {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: ccmpl {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x0c,0x39,0x4c,0x80,0x7b] + ccmpl {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: ccmpl {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x0c,0x3a,0x5c,0x80,0x7b] + ccmpl {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: ccmpl {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x45,0x0c,0x3b,0x54,0x80,0x7b] + ccmpl {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: ccmpl {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x0c,0x3b,0x4c,0x80,0x7b] + ccmpl {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: ccmpl {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0x54,0xc4,0x0c,0x3b,0x4c,0x80,0x7b] + ccmpl {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: ccmpl {dfv=of} bl, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x0c,0x80,0xfb,0x7b] + ccmpl {dfv=of} bl, 123 +# CHECK: ccmpl {dfv=of} dx, 123 +# CHECK: encoding: [0x62,0xf4,0x45,0x0c,0x83,0xfa,0x7b] + ccmpl {dfv=of} dx, 123 +# CHECK: ccmpl {dfv=of} ecx, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x0c,0x83,0xf9,0x7b] + ccmpl {dfv=of} ecx, 123 +# CHECK: ccmpl {dfv=of} r9, 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0c,0x83,0xf9,0x7b] + ccmpl {dfv=of} r9, 123 +# CHECK: ccmpl {dfv=of} dx, 1234 +# CHECK: encoding: [0x62,0xf4,0x45,0x0c,0x81,0xfa,0xd2,0x04] + ccmpl {dfv=of} dx, 1234 +# CHECK: ccmpl {dfv=of} ecx, 123456 +# CHECK: encoding: [0x62,0xf4,0x44,0x0c,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpl {dfv=of} ecx, 123456 +# CHECK: ccmpl {dfv=of} r9, 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0c,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpl {dfv=of} r9, 123456 +# CHECK: ccmpl {dfv=of} dl, bl +# CHECK: encoding: [0x62,0xf4,0x44,0x0c,0x38,0xda] + ccmpl {dfv=of} dl, bl +# CHECK: ccmpl {dfv=of} ax, dx +# CHECK: encoding: [0x62,0xf4,0x45,0x0c,0x39,0xd0] + ccmpl {dfv=of} ax, dx +# CHECK: ccmpl {dfv=of} edx, ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x0c,0x39,0xca] + ccmpl {dfv=of} edx, ecx +# CHECK: ccmpl {dfv=of} r15, r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x0c,0x39,0xcf] + ccmpl {dfv=of} r15, r9 +# CHECK: ccmple {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x0e,0x80,0x7c,0x80,0x7b,0x7b] + ccmple {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmple {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x45,0x0e,0x83,0x7c,0x80,0x7b,0x7b] + ccmple {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmple {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: encoding: [0x62,0xd4,0x45,0x0e,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmple {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: ccmple {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x0e,0x83,0x7c,0x80,0x7b,0x7b] + ccmple {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmple {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0x44,0x0e,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmple {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmple {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0e,0x83,0x7c,0x80,0x7b,0x7b] + ccmple {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmple {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0e,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmple {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmple {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: encoding: [0x62,0xd4,0x44,0x0e,0x38,0x5c,0x80,0x7b] + ccmple {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: ccmple {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: encoding: [0x62,0xd4,0x45,0x0e,0x39,0x54,0x80,0x7b] + ccmple {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: ccmple {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x0e,0x39,0x4c,0x80,0x7b] + ccmple {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: ccmple {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x0e,0x39,0x4c,0x80,0x7b] + ccmple {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: ccmple {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x0e,0x3a,0x5c,0x80,0x7b] + ccmple {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: ccmple {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x45,0x0e,0x3b,0x54,0x80,0x7b] + ccmple {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: ccmple {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x0e,0x3b,0x4c,0x80,0x7b] + ccmple {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: ccmple {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0x54,0xc4,0x0e,0x3b,0x4c,0x80,0x7b] + ccmple {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: ccmple {dfv=of} bl, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x0e,0x80,0xfb,0x7b] + ccmple {dfv=of} bl, 123 +# CHECK: ccmple {dfv=of} dx, 123 +# CHECK: encoding: [0x62,0xf4,0x45,0x0e,0x83,0xfa,0x7b] + ccmple {dfv=of} dx, 123 +# CHECK: ccmple {dfv=of} ecx, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x0e,0x83,0xf9,0x7b] + ccmple {dfv=of} ecx, 123 +# CHECK: ccmple {dfv=of} r9, 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0e,0x83,0xf9,0x7b] + ccmple {dfv=of} r9, 123 +# CHECK: ccmple {dfv=of} dx, 1234 +# CHECK: encoding: [0x62,0xf4,0x45,0x0e,0x81,0xfa,0xd2,0x04] + ccmple {dfv=of} dx, 1234 +# CHECK: ccmple {dfv=of} ecx, 123456 +# CHECK: encoding: [0x62,0xf4,0x44,0x0e,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmple {dfv=of} ecx, 123456 +# CHECK: ccmple {dfv=of} r9, 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0e,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmple {dfv=of} r9, 123456 +# CHECK: ccmple {dfv=of} dl, bl +# CHECK: encoding: [0x62,0xf4,0x44,0x0e,0x38,0xda] + ccmple {dfv=of} dl, bl +# CHECK: ccmple {dfv=of} ax, dx +# CHECK: encoding: [0x62,0xf4,0x45,0x0e,0x39,0xd0] + ccmple {dfv=of} ax, dx +# CHECK: ccmple {dfv=of} edx, ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x0e,0x39,0xca] + ccmple {dfv=of} edx, ecx +# CHECK: ccmple {dfv=of} r15, r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x0e,0x39,0xcf] + ccmple {dfv=of} r15, r9 +# CHECK: ccmpae {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x03,0x80,0x7c,0x80,0x7b,0x7b] + ccmpae {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpae {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x45,0x03,0x83,0x7c,0x80,0x7b,0x7b] + ccmpae {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpae {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: encoding: [0x62,0xd4,0x45,0x03,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmpae {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: ccmpae {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x03,0x83,0x7c,0x80,0x7b,0x7b] + ccmpae {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpae {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0x44,0x03,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpae {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmpae {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x03,0x83,0x7c,0x80,0x7b,0x7b] + ccmpae {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpae {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x03,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpae {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmpae {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: encoding: [0x62,0xd4,0x44,0x03,0x38,0x5c,0x80,0x7b] + ccmpae {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: ccmpae {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: encoding: [0x62,0xd4,0x45,0x03,0x39,0x54,0x80,0x7b] + ccmpae {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: ccmpae {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x03,0x39,0x4c,0x80,0x7b] + ccmpae {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: ccmpae {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x03,0x39,0x4c,0x80,0x7b] + ccmpae {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: ccmpae {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x03,0x3a,0x5c,0x80,0x7b] + ccmpae {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: ccmpae {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x45,0x03,0x3b,0x54,0x80,0x7b] + ccmpae {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: ccmpae {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x03,0x3b,0x4c,0x80,0x7b] + ccmpae {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: ccmpae {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0x54,0xc4,0x03,0x3b,0x4c,0x80,0x7b] + ccmpae {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: ccmpae {dfv=of} bl, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x03,0x80,0xfb,0x7b] + ccmpae {dfv=of} bl, 123 +# CHECK: ccmpae {dfv=of} dx, 123 +# CHECK: encoding: [0x62,0xf4,0x45,0x03,0x83,0xfa,0x7b] + ccmpae {dfv=of} dx, 123 +# CHECK: ccmpae {dfv=of} ecx, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x03,0x83,0xf9,0x7b] + ccmpae {dfv=of} ecx, 123 +# CHECK: ccmpae {dfv=of} r9, 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x03,0x83,0xf9,0x7b] + ccmpae {dfv=of} r9, 123 +# CHECK: ccmpae {dfv=of} dx, 1234 +# CHECK: encoding: [0x62,0xf4,0x45,0x03,0x81,0xfa,0xd2,0x04] + ccmpae {dfv=of} dx, 1234 +# CHECK: ccmpae {dfv=of} ecx, 123456 +# CHECK: encoding: [0x62,0xf4,0x44,0x03,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpae {dfv=of} ecx, 123456 +# CHECK: ccmpae {dfv=of} r9, 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x03,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpae {dfv=of} r9, 123456 +# CHECK: ccmpae {dfv=of} dl, bl +# CHECK: encoding: [0x62,0xf4,0x44,0x03,0x38,0xda] + ccmpae {dfv=of} dl, bl +# CHECK: ccmpae {dfv=of} ax, dx +# CHECK: encoding: [0x62,0xf4,0x45,0x03,0x39,0xd0] + ccmpae {dfv=of} ax, dx +# CHECK: ccmpae {dfv=of} edx, ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x03,0x39,0xca] + ccmpae {dfv=of} edx, ecx +# CHECK: ccmpae {dfv=of} r15, r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x03,0x39,0xcf] + ccmpae {dfv=of} r15, r9 +# CHECK: ccmpa {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x07,0x80,0x7c,0x80,0x7b,0x7b] + ccmpa {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpa {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x45,0x07,0x83,0x7c,0x80,0x7b,0x7b] + ccmpa {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpa {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: encoding: [0x62,0xd4,0x45,0x07,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmpa {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: ccmpa {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x07,0x83,0x7c,0x80,0x7b,0x7b] + ccmpa {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpa {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0x44,0x07,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpa {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmpa {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x07,0x83,0x7c,0x80,0x7b,0x7b] + ccmpa {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpa {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x07,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpa {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmpa {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: encoding: [0x62,0xd4,0x44,0x07,0x38,0x5c,0x80,0x7b] + ccmpa {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: ccmpa {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: encoding: [0x62,0xd4,0x45,0x07,0x39,0x54,0x80,0x7b] + ccmpa {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: ccmpa {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x07,0x39,0x4c,0x80,0x7b] + ccmpa {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: ccmpa {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x07,0x39,0x4c,0x80,0x7b] + ccmpa {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: ccmpa {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x07,0x3a,0x5c,0x80,0x7b] + ccmpa {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: ccmpa {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x45,0x07,0x3b,0x54,0x80,0x7b] + ccmpa {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: ccmpa {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x07,0x3b,0x4c,0x80,0x7b] + ccmpa {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: ccmpa {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0x54,0xc4,0x07,0x3b,0x4c,0x80,0x7b] + ccmpa {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: ccmpa {dfv=of} bl, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x07,0x80,0xfb,0x7b] + ccmpa {dfv=of} bl, 123 +# CHECK: ccmpa {dfv=of} dx, 123 +# CHECK: encoding: [0x62,0xf4,0x45,0x07,0x83,0xfa,0x7b] + ccmpa {dfv=of} dx, 123 +# CHECK: ccmpa {dfv=of} ecx, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x07,0x83,0xf9,0x7b] + ccmpa {dfv=of} ecx, 123 +# CHECK: ccmpa {dfv=of} r9, 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x07,0x83,0xf9,0x7b] + ccmpa {dfv=of} r9, 123 +# CHECK: ccmpa {dfv=of} dx, 1234 +# CHECK: encoding: [0x62,0xf4,0x45,0x07,0x81,0xfa,0xd2,0x04] + ccmpa {dfv=of} dx, 1234 +# CHECK: ccmpa {dfv=of} ecx, 123456 +# CHECK: encoding: [0x62,0xf4,0x44,0x07,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpa {dfv=of} ecx, 123456 +# CHECK: ccmpa {dfv=of} r9, 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x07,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpa {dfv=of} r9, 123456 +# CHECK: ccmpa {dfv=of} dl, bl +# CHECK: encoding: [0x62,0xf4,0x44,0x07,0x38,0xda] + ccmpa {dfv=of} dl, bl +# CHECK: ccmpa {dfv=of} ax, dx +# CHECK: encoding: [0x62,0xf4,0x45,0x07,0x39,0xd0] + ccmpa {dfv=of} ax, dx +# CHECK: ccmpa {dfv=of} edx, ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x07,0x39,0xca] + ccmpa {dfv=of} edx, ecx +# CHECK: ccmpa {dfv=of} r15, r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x07,0x39,0xcf] + ccmpa {dfv=of} r15, r9 +# CHECK: ccmpge {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x0d,0x80,0x7c,0x80,0x7b,0x7b] + ccmpge {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpge {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x45,0x0d,0x83,0x7c,0x80,0x7b,0x7b] + ccmpge {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpge {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: encoding: [0x62,0xd4,0x45,0x0d,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmpge {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: ccmpge {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x0d,0x83,0x7c,0x80,0x7b,0x7b] + ccmpge {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpge {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0x44,0x0d,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpge {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmpge {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0d,0x83,0x7c,0x80,0x7b,0x7b] + ccmpge {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpge {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0d,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpge {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmpge {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: encoding: [0x62,0xd4,0x44,0x0d,0x38,0x5c,0x80,0x7b] + ccmpge {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: ccmpge {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: encoding: [0x62,0xd4,0x45,0x0d,0x39,0x54,0x80,0x7b] + ccmpge {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: ccmpge {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x0d,0x39,0x4c,0x80,0x7b] + ccmpge {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: ccmpge {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x0d,0x39,0x4c,0x80,0x7b] + ccmpge {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: ccmpge {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x0d,0x3a,0x5c,0x80,0x7b] + ccmpge {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: ccmpge {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x45,0x0d,0x3b,0x54,0x80,0x7b] + ccmpge {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: ccmpge {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x0d,0x3b,0x4c,0x80,0x7b] + ccmpge {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: ccmpge {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0x54,0xc4,0x0d,0x3b,0x4c,0x80,0x7b] + ccmpge {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: ccmpge {dfv=of} bl, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x0d,0x80,0xfb,0x7b] + ccmpge {dfv=of} bl, 123 +# CHECK: ccmpge {dfv=of} dx, 123 +# CHECK: encoding: [0x62,0xf4,0x45,0x0d,0x83,0xfa,0x7b] + ccmpge {dfv=of} dx, 123 +# CHECK: ccmpge {dfv=of} ecx, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x0d,0x83,0xf9,0x7b] + ccmpge {dfv=of} ecx, 123 +# CHECK: ccmpge {dfv=of} r9, 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0d,0x83,0xf9,0x7b] + ccmpge {dfv=of} r9, 123 +# CHECK: ccmpge {dfv=of} dx, 1234 +# CHECK: encoding: [0x62,0xf4,0x45,0x0d,0x81,0xfa,0xd2,0x04] + ccmpge {dfv=of} dx, 1234 +# CHECK: ccmpge {dfv=of} ecx, 123456 +# CHECK: encoding: [0x62,0xf4,0x44,0x0d,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpge {dfv=of} ecx, 123456 +# CHECK: ccmpge {dfv=of} r9, 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0d,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpge {dfv=of} r9, 123456 +# CHECK: ccmpge {dfv=of} dl, bl +# CHECK: encoding: [0x62,0xf4,0x44,0x0d,0x38,0xda] + ccmpge {dfv=of} dl, bl +# CHECK: ccmpge {dfv=of} ax, dx +# CHECK: encoding: [0x62,0xf4,0x45,0x0d,0x39,0xd0] + ccmpge {dfv=of} ax, dx +# CHECK: ccmpge {dfv=of} edx, ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x0d,0x39,0xca] + ccmpge {dfv=of} edx, ecx +# CHECK: ccmpge {dfv=of} r15, r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x0d,0x39,0xcf] + ccmpge {dfv=of} r15, r9 +# CHECK: ccmpg {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x0f,0x80,0x7c,0x80,0x7b,0x7b] + ccmpg {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpg {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x45,0x0f,0x83,0x7c,0x80,0x7b,0x7b] + ccmpg {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpg {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: encoding: [0x62,0xd4,0x45,0x0f,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmpg {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: ccmpg {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x0f,0x83,0x7c,0x80,0x7b,0x7b] + ccmpg {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpg {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0x44,0x0f,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpg {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmpg {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0f,0x83,0x7c,0x80,0x7b,0x7b] + ccmpg {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpg {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0f,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpg {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmpg {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: encoding: [0x62,0xd4,0x44,0x0f,0x38,0x5c,0x80,0x7b] + ccmpg {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: ccmpg {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: encoding: [0x62,0xd4,0x45,0x0f,0x39,0x54,0x80,0x7b] + ccmpg {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: ccmpg {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x0f,0x39,0x4c,0x80,0x7b] + ccmpg {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: ccmpg {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x0f,0x39,0x4c,0x80,0x7b] + ccmpg {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: ccmpg {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x0f,0x3a,0x5c,0x80,0x7b] + ccmpg {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: ccmpg {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x45,0x0f,0x3b,0x54,0x80,0x7b] + ccmpg {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: ccmpg {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x0f,0x3b,0x4c,0x80,0x7b] + ccmpg {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: ccmpg {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0x54,0xc4,0x0f,0x3b,0x4c,0x80,0x7b] + ccmpg {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: ccmpg {dfv=of} bl, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x0f,0x80,0xfb,0x7b] + ccmpg {dfv=of} bl, 123 +# CHECK: ccmpg {dfv=of} dx, 123 +# CHECK: encoding: [0x62,0xf4,0x45,0x0f,0x83,0xfa,0x7b] + ccmpg {dfv=of} dx, 123 +# CHECK: ccmpg {dfv=of} ecx, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x0f,0x83,0xf9,0x7b] + ccmpg {dfv=of} ecx, 123 +# CHECK: ccmpg {dfv=of} r9, 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0f,0x83,0xf9,0x7b] + ccmpg {dfv=of} r9, 123 +# CHECK: ccmpg {dfv=of} dx, 1234 +# CHECK: encoding: [0x62,0xf4,0x45,0x0f,0x81,0xfa,0xd2,0x04] + ccmpg {dfv=of} dx, 1234 +# CHECK: ccmpg {dfv=of} ecx, 123456 +# CHECK: encoding: [0x62,0xf4,0x44,0x0f,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpg {dfv=of} ecx, 123456 +# CHECK: ccmpg {dfv=of} r9, 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0f,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpg {dfv=of} r9, 123456 +# CHECK: ccmpg {dfv=of} dl, bl +# CHECK: encoding: [0x62,0xf4,0x44,0x0f,0x38,0xda] + ccmpg {dfv=of} dl, bl +# CHECK: ccmpg {dfv=of} ax, dx +# CHECK: encoding: [0x62,0xf4,0x45,0x0f,0x39,0xd0] + ccmpg {dfv=of} ax, dx +# CHECK: ccmpg {dfv=of} edx, ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x0f,0x39,0xca] + ccmpg {dfv=of} edx, ecx +# CHECK: ccmpg {dfv=of} r15, r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x0f,0x39,0xcf] + ccmpg {dfv=of} r15, r9 +# CHECK: ccmpno {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x01,0x80,0x7c,0x80,0x7b,0x7b] + ccmpno {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpno {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x45,0x01,0x83,0x7c,0x80,0x7b,0x7b] + ccmpno {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpno {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: encoding: [0x62,0xd4,0x45,0x01,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmpno {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: ccmpno {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x01,0x83,0x7c,0x80,0x7b,0x7b] + ccmpno {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpno {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0x44,0x01,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpno {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmpno {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x01,0x83,0x7c,0x80,0x7b,0x7b] + ccmpno {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpno {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x01,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpno {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmpno {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: encoding: [0x62,0xd4,0x44,0x01,0x38,0x5c,0x80,0x7b] + ccmpno {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: ccmpno {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: encoding: [0x62,0xd4,0x45,0x01,0x39,0x54,0x80,0x7b] + ccmpno {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: ccmpno {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x01,0x39,0x4c,0x80,0x7b] + ccmpno {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: ccmpno {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x01,0x39,0x4c,0x80,0x7b] + ccmpno {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: ccmpno {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x01,0x3a,0x5c,0x80,0x7b] + ccmpno {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: ccmpno {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x45,0x01,0x3b,0x54,0x80,0x7b] + ccmpno {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: ccmpno {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x01,0x3b,0x4c,0x80,0x7b] + ccmpno {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: ccmpno {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0x54,0xc4,0x01,0x3b,0x4c,0x80,0x7b] + ccmpno {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: ccmpno {dfv=of} bl, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x01,0x80,0xfb,0x7b] + ccmpno {dfv=of} bl, 123 +# CHECK: ccmpno {dfv=of} dx, 123 +# CHECK: encoding: [0x62,0xf4,0x45,0x01,0x83,0xfa,0x7b] + ccmpno {dfv=of} dx, 123 +# CHECK: ccmpno {dfv=of} ecx, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x01,0x83,0xf9,0x7b] + ccmpno {dfv=of} ecx, 123 +# CHECK: ccmpno {dfv=of} r9, 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x01,0x83,0xf9,0x7b] + ccmpno {dfv=of} r9, 123 +# CHECK: ccmpno {dfv=of} dx, 1234 +# CHECK: encoding: [0x62,0xf4,0x45,0x01,0x81,0xfa,0xd2,0x04] + ccmpno {dfv=of} dx, 1234 +# CHECK: ccmpno {dfv=of} ecx, 123456 +# CHECK: encoding: [0x62,0xf4,0x44,0x01,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpno {dfv=of} ecx, 123456 +# CHECK: ccmpno {dfv=of} r9, 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x01,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpno {dfv=of} r9, 123456 +# CHECK: ccmpno {dfv=of} dl, bl +# CHECK: encoding: [0x62,0xf4,0x44,0x01,0x38,0xda] + ccmpno {dfv=of} dl, bl +# CHECK: ccmpno {dfv=of} ax, dx +# CHECK: encoding: [0x62,0xf4,0x45,0x01,0x39,0xd0] + ccmpno {dfv=of} ax, dx +# CHECK: ccmpno {dfv=of} edx, ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x01,0x39,0xca] + ccmpno {dfv=of} edx, ecx +# CHECK: ccmpno {dfv=of} r15, r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x01,0x39,0xcf] + ccmpno {dfv=of} r15, r9 +# CHECK: ccmpns {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x09,0x80,0x7c,0x80,0x7b,0x7b] + ccmpns {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpns {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x45,0x09,0x83,0x7c,0x80,0x7b,0x7b] + ccmpns {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpns {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: encoding: [0x62,0xd4,0x45,0x09,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmpns {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: ccmpns {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x09,0x83,0x7c,0x80,0x7b,0x7b] + ccmpns {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpns {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0x44,0x09,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpns {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmpns {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x09,0x83,0x7c,0x80,0x7b,0x7b] + ccmpns {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpns {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x09,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpns {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmpns {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: encoding: [0x62,0xd4,0x44,0x09,0x38,0x5c,0x80,0x7b] + ccmpns {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: ccmpns {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: encoding: [0x62,0xd4,0x45,0x09,0x39,0x54,0x80,0x7b] + ccmpns {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: ccmpns {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x09,0x39,0x4c,0x80,0x7b] + ccmpns {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: ccmpns {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x09,0x39,0x4c,0x80,0x7b] + ccmpns {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: ccmpns {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x09,0x3a,0x5c,0x80,0x7b] + ccmpns {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: ccmpns {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x45,0x09,0x3b,0x54,0x80,0x7b] + ccmpns {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: ccmpns {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x09,0x3b,0x4c,0x80,0x7b] + ccmpns {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: ccmpns {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0x54,0xc4,0x09,0x3b,0x4c,0x80,0x7b] + ccmpns {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: ccmpns {dfv=of} bl, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x09,0x80,0xfb,0x7b] + ccmpns {dfv=of} bl, 123 +# CHECK: ccmpns {dfv=of} dx, 123 +# CHECK: encoding: [0x62,0xf4,0x45,0x09,0x83,0xfa,0x7b] + ccmpns {dfv=of} dx, 123 +# CHECK: ccmpns {dfv=of} ecx, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x09,0x83,0xf9,0x7b] + ccmpns {dfv=of} ecx, 123 +# CHECK: ccmpns {dfv=of} r9, 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x09,0x83,0xf9,0x7b] + ccmpns {dfv=of} r9, 123 +# CHECK: ccmpns {dfv=of} dx, 1234 +# CHECK: encoding: [0x62,0xf4,0x45,0x09,0x81,0xfa,0xd2,0x04] + ccmpns {dfv=of} dx, 1234 +# CHECK: ccmpns {dfv=of} ecx, 123456 +# CHECK: encoding: [0x62,0xf4,0x44,0x09,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpns {dfv=of} ecx, 123456 +# CHECK: ccmpns {dfv=of} r9, 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x09,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpns {dfv=of} r9, 123456 +# CHECK: ccmpns {dfv=of} dl, bl +# CHECK: encoding: [0x62,0xf4,0x44,0x09,0x38,0xda] + ccmpns {dfv=of} dl, bl +# CHECK: ccmpns {dfv=of} ax, dx +# CHECK: encoding: [0x62,0xf4,0x45,0x09,0x39,0xd0] + ccmpns {dfv=of} ax, dx +# CHECK: ccmpns {dfv=of} edx, ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x09,0x39,0xca] + ccmpns {dfv=of} edx, ecx +# CHECK: ccmpns {dfv=of} r15, r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x09,0x39,0xcf] + ccmpns {dfv=of} r15, r9 +# CHECK: ccmpne {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x05,0x80,0x7c,0x80,0x7b,0x7b] + ccmpne {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpne {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x45,0x05,0x83,0x7c,0x80,0x7b,0x7b] + ccmpne {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpne {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: encoding: [0x62,0xd4,0x45,0x05,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmpne {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: ccmpne {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x05,0x83,0x7c,0x80,0x7b,0x7b] + ccmpne {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpne {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0x44,0x05,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpne {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmpne {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x05,0x83,0x7c,0x80,0x7b,0x7b] + ccmpne {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpne {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x05,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpne {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmpne {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: encoding: [0x62,0xd4,0x44,0x05,0x38,0x5c,0x80,0x7b] + ccmpne {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: ccmpne {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: encoding: [0x62,0xd4,0x45,0x05,0x39,0x54,0x80,0x7b] + ccmpne {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: ccmpne {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x05,0x39,0x4c,0x80,0x7b] + ccmpne {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: ccmpne {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x05,0x39,0x4c,0x80,0x7b] + ccmpne {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: ccmpne {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x05,0x3a,0x5c,0x80,0x7b] + ccmpne {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: ccmpne {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x45,0x05,0x3b,0x54,0x80,0x7b] + ccmpne {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: ccmpne {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x05,0x3b,0x4c,0x80,0x7b] + ccmpne {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: ccmpne {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0x54,0xc4,0x05,0x3b,0x4c,0x80,0x7b] + ccmpne {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: ccmpne {dfv=of} bl, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x05,0x80,0xfb,0x7b] + ccmpne {dfv=of} bl, 123 +# CHECK: ccmpne {dfv=of} dx, 123 +# CHECK: encoding: [0x62,0xf4,0x45,0x05,0x83,0xfa,0x7b] + ccmpne {dfv=of} dx, 123 +# CHECK: ccmpne {dfv=of} ecx, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x05,0x83,0xf9,0x7b] + ccmpne {dfv=of} ecx, 123 +# CHECK: ccmpne {dfv=of} r9, 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x05,0x83,0xf9,0x7b] + ccmpne {dfv=of} r9, 123 +# CHECK: ccmpne {dfv=of} dx, 1234 +# CHECK: encoding: [0x62,0xf4,0x45,0x05,0x81,0xfa,0xd2,0x04] + ccmpne {dfv=of} dx, 1234 +# CHECK: ccmpne {dfv=of} ecx, 123456 +# CHECK: encoding: [0x62,0xf4,0x44,0x05,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpne {dfv=of} ecx, 123456 +# CHECK: ccmpne {dfv=of} r9, 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x05,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpne {dfv=of} r9, 123456 +# CHECK: ccmpne {dfv=of} dl, bl +# CHECK: encoding: [0x62,0xf4,0x44,0x05,0x38,0xda] + ccmpne {dfv=of} dl, bl +# CHECK: ccmpne {dfv=of} ax, dx +# CHECK: encoding: [0x62,0xf4,0x45,0x05,0x39,0xd0] + ccmpne {dfv=of} ax, dx +# CHECK: ccmpne {dfv=of} edx, ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x05,0x39,0xca] + ccmpne {dfv=of} edx, ecx +# CHECK: ccmpne {dfv=of} r15, r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x05,0x39,0xcf] + ccmpne {dfv=of} r15, r9 +# CHECK: ccmpo {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x00,0x80,0x7c,0x80,0x7b,0x7b] + ccmpo {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpo {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x45,0x00,0x83,0x7c,0x80,0x7b,0x7b] + ccmpo {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpo {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: encoding: [0x62,0xd4,0x45,0x00,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmpo {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: ccmpo {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x00,0x83,0x7c,0x80,0x7b,0x7b] + ccmpo {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpo {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0x44,0x00,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpo {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmpo {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x00,0x83,0x7c,0x80,0x7b,0x7b] + ccmpo {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpo {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x00,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpo {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmpo {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: encoding: [0x62,0xd4,0x44,0x00,0x38,0x5c,0x80,0x7b] + ccmpo {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: ccmpo {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: encoding: [0x62,0xd4,0x45,0x00,0x39,0x54,0x80,0x7b] + ccmpo {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: ccmpo {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x00,0x39,0x4c,0x80,0x7b] + ccmpo {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: ccmpo {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x00,0x39,0x4c,0x80,0x7b] + ccmpo {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: ccmpo {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x00,0x3a,0x5c,0x80,0x7b] + ccmpo {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: ccmpo {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x45,0x00,0x3b,0x54,0x80,0x7b] + ccmpo {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: ccmpo {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x00,0x3b,0x4c,0x80,0x7b] + ccmpo {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: ccmpo {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0x54,0xc4,0x00,0x3b,0x4c,0x80,0x7b] + ccmpo {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: ccmpo {dfv=of} bl, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x00,0x80,0xfb,0x7b] + ccmpo {dfv=of} bl, 123 +# CHECK: ccmpo {dfv=of} dx, 123 +# CHECK: encoding: [0x62,0xf4,0x45,0x00,0x83,0xfa,0x7b] + ccmpo {dfv=of} dx, 123 +# CHECK: ccmpo {dfv=of} ecx, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x00,0x83,0xf9,0x7b] + ccmpo {dfv=of} ecx, 123 +# CHECK: ccmpo {dfv=of} r9, 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x00,0x83,0xf9,0x7b] + ccmpo {dfv=of} r9, 123 +# CHECK: ccmpo {dfv=of} dx, 1234 +# CHECK: encoding: [0x62,0xf4,0x45,0x00,0x81,0xfa,0xd2,0x04] + ccmpo {dfv=of} dx, 1234 +# CHECK: ccmpo {dfv=of} ecx, 123456 +# CHECK: encoding: [0x62,0xf4,0x44,0x00,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpo {dfv=of} ecx, 123456 +# CHECK: ccmpo {dfv=of} r9, 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x00,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpo {dfv=of} r9, 123456 +# CHECK: ccmpo {dfv=of} dl, bl +# CHECK: encoding: [0x62,0xf4,0x44,0x00,0x38,0xda] + ccmpo {dfv=of} dl, bl +# CHECK: ccmpo {dfv=of} ax, dx +# CHECK: encoding: [0x62,0xf4,0x45,0x00,0x39,0xd0] + ccmpo {dfv=of} ax, dx +# CHECK: ccmpo {dfv=of} edx, ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x00,0x39,0xca] + ccmpo {dfv=of} edx, ecx +# CHECK: ccmpo {dfv=of} r15, r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x00,0x39,0xcf] + ccmpo {dfv=of} r15, r9 +# CHECK: ccmps {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x08,0x80,0x7c,0x80,0x7b,0x7b] + ccmps {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmps {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x45,0x08,0x83,0x7c,0x80,0x7b,0x7b] + ccmps {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmps {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: encoding: [0x62,0xd4,0x45,0x08,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmps {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: ccmps {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x08,0x83,0x7c,0x80,0x7b,0x7b] + ccmps {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmps {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0x44,0x08,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmps {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmps {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x08,0x83,0x7c,0x80,0x7b,0x7b] + ccmps {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmps {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x08,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmps {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmps {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: encoding: [0x62,0xd4,0x44,0x08,0x38,0x5c,0x80,0x7b] + ccmps {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: ccmps {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: encoding: [0x62,0xd4,0x45,0x08,0x39,0x54,0x80,0x7b] + ccmps {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: ccmps {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x08,0x39,0x4c,0x80,0x7b] + ccmps {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: ccmps {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x08,0x39,0x4c,0x80,0x7b] + ccmps {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: ccmps {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x08,0x3a,0x5c,0x80,0x7b] + ccmps {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: ccmps {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x45,0x08,0x3b,0x54,0x80,0x7b] + ccmps {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: ccmps {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x08,0x3b,0x4c,0x80,0x7b] + ccmps {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: ccmps {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0x54,0xc4,0x08,0x3b,0x4c,0x80,0x7b] + ccmps {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: ccmps {dfv=of} bl, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x08,0x80,0xfb,0x7b] + ccmps {dfv=of} bl, 123 +# CHECK: ccmps {dfv=of} dx, 123 +# CHECK: encoding: [0x62,0xf4,0x45,0x08,0x83,0xfa,0x7b] + ccmps {dfv=of} dx, 123 +# CHECK: ccmps {dfv=of} ecx, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x08,0x83,0xf9,0x7b] + ccmps {dfv=of} ecx, 123 +# CHECK: ccmps {dfv=of} r9, 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x08,0x83,0xf9,0x7b] + ccmps {dfv=of} r9, 123 +# CHECK: ccmps {dfv=of} dx, 1234 +# CHECK: encoding: [0x62,0xf4,0x45,0x08,0x81,0xfa,0xd2,0x04] + ccmps {dfv=of} dx, 1234 +# CHECK: ccmps {dfv=of} ecx, 123456 +# CHECK: encoding: [0x62,0xf4,0x44,0x08,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmps {dfv=of} ecx, 123456 +# CHECK: ccmps {dfv=of} r9, 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x08,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmps {dfv=of} r9, 123456 +# CHECK: ccmps {dfv=of} dl, bl +# CHECK: encoding: [0x62,0xf4,0x44,0x08,0x38,0xda] + ccmps {dfv=of} dl, bl +# CHECK: ccmps {dfv=of} ax, dx +# CHECK: encoding: [0x62,0xf4,0x45,0x08,0x39,0xd0] + ccmps {dfv=of} ax, dx +# CHECK: ccmps {dfv=of} edx, ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x08,0x39,0xca] + ccmps {dfv=of} edx, ecx +# CHECK: ccmps {dfv=of} r15, r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x08,0x39,0xcf] + ccmps {dfv=of} r15, r9 +# CHECK: ccmpt {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x0a,0x80,0x7c,0x80,0x7b,0x7b] + ccmpt {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpt {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x45,0x0a,0x83,0x7c,0x80,0x7b,0x7b] + ccmpt {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpt {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: encoding: [0x62,0xd4,0x45,0x0a,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmpt {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: ccmpt {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x0a,0x83,0x7c,0x80,0x7b,0x7b] + ccmpt {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpt {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0x44,0x0a,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpt {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmpt {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0a,0x83,0x7c,0x80,0x7b,0x7b] + ccmpt {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpt {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0a,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpt {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmpt {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: encoding: [0x62,0xd4,0x44,0x0a,0x38,0x5c,0x80,0x7b] + ccmpt {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: ccmpt {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: encoding: [0x62,0xd4,0x45,0x0a,0x39,0x54,0x80,0x7b] + ccmpt {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: ccmpt {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x0a,0x39,0x4c,0x80,0x7b] + ccmpt {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: ccmpt {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x0a,0x39,0x4c,0x80,0x7b] + ccmpt {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: ccmpt {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x0a,0x3a,0x5c,0x80,0x7b] + ccmpt {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: ccmpt {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x45,0x0a,0x3b,0x54,0x80,0x7b] + ccmpt {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: ccmpt {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x0a,0x3b,0x4c,0x80,0x7b] + ccmpt {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: ccmpt {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0x54,0xc4,0x0a,0x3b,0x4c,0x80,0x7b] + ccmpt {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: ccmpt {dfv=of} bl, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x0a,0x80,0xfb,0x7b] + ccmpt {dfv=of} bl, 123 +# CHECK: ccmpt {dfv=of} dx, 123 +# CHECK: encoding: [0x62,0xf4,0x45,0x0a,0x83,0xfa,0x7b] + ccmpt {dfv=of} dx, 123 +# CHECK: ccmpt {dfv=of} ecx, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x0a,0x83,0xf9,0x7b] + ccmpt {dfv=of} ecx, 123 +# CHECK: ccmpt {dfv=of} r9, 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0a,0x83,0xf9,0x7b] + ccmpt {dfv=of} r9, 123 +# CHECK: ccmpt {dfv=of} dx, 1234 +# CHECK: encoding: [0x62,0xf4,0x45,0x0a,0x81,0xfa,0xd2,0x04] + ccmpt {dfv=of} dx, 1234 +# CHECK: ccmpt {dfv=of} ecx, 123456 +# CHECK: encoding: [0x62,0xf4,0x44,0x0a,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpt {dfv=of} ecx, 123456 +# CHECK: ccmpt {dfv=of} r9, 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x0a,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpt {dfv=of} r9, 123456 +# CHECK: ccmpt {dfv=of} dl, bl +# CHECK: encoding: [0x62,0xf4,0x44,0x0a,0x38,0xda] + ccmpt {dfv=of} dl, bl +# CHECK: ccmpt {dfv=of} ax, dx +# CHECK: encoding: [0x62,0xf4,0x45,0x0a,0x39,0xd0] + ccmpt {dfv=of} ax, dx +# CHECK: ccmpt {dfv=of} edx, ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x0a,0x39,0xca] + ccmpt {dfv=of} edx, ecx +# CHECK: ccmpt {dfv=of} r15, r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x0a,0x39,0xcf] + ccmpt {dfv=of} r15, r9 +# CHECK: ccmpe {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x04,0x80,0x7c,0x80,0x7b,0x7b] + ccmpe {dfv=of} byte ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpe {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x45,0x04,0x83,0x7c,0x80,0x7b,0x7b] + ccmpe {dfv=of} word ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpe {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: encoding: [0x62,0xd4,0x45,0x04,0x81,0x7c,0x80,0x7b,0xd2,0x04] + ccmpe {dfv=of} word ptr [r8 + 4*rax + 123], 1234 +# CHECK: ccmpe {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0x44,0x04,0x83,0x7c,0x80,0x7b,0x7b] + ccmpe {dfv=of} dword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpe {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0x44,0x04,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpe {dfv=of} dword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmpe {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x04,0x83,0x7c,0x80,0x7b,0x7b] + ccmpe {dfv=of} qword ptr [r8 + 4*rax + 123], 123 +# CHECK: ccmpe {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x04,0x81,0x7c,0x80,0x7b,0x40,0xe2,0x01,0x00] + ccmpe {dfv=of} qword ptr [r8 + 4*rax + 123], 123456 +# CHECK: ccmpe {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: encoding: [0x62,0xd4,0x44,0x04,0x38,0x5c,0x80,0x7b] + ccmpe {dfv=of} byte ptr [r8 + 4*rax + 123], bl +# CHECK: ccmpe {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: encoding: [0x62,0xd4,0x45,0x04,0x39,0x54,0x80,0x7b] + ccmpe {dfv=of} word ptr [r8 + 4*rax + 123], dx +# CHECK: ccmpe {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: encoding: [0x62,0xd4,0x44,0x04,0x39,0x4c,0x80,0x7b] + ccmpe {dfv=of} dword ptr [r8 + 4*rax + 123], ecx +# CHECK: ccmpe {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x04,0x39,0x4c,0x80,0x7b] + ccmpe {dfv=of} qword ptr [r8 + 4*rax + 123], r9 +# CHECK: ccmpe {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x04,0x3a,0x5c,0x80,0x7b] + ccmpe {dfv=of} bl, byte ptr [r8 + 4*rax + 123] +# CHECK: ccmpe {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x45,0x04,0x3b,0x54,0x80,0x7b] + ccmpe {dfv=of} dx, word ptr [r8 + 4*rax + 123] +# CHECK: ccmpe {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0xd4,0x44,0x04,0x3b,0x4c,0x80,0x7b] + ccmpe {dfv=of} ecx, dword ptr [r8 + 4*rax + 123] +# CHECK: ccmpe {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: encoding: [0x62,0x54,0xc4,0x04,0x3b,0x4c,0x80,0x7b] + ccmpe {dfv=of} r9, qword ptr [r8 + 4*rax + 123] +# CHECK: ccmpe {dfv=of} bl, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x04,0x80,0xfb,0x7b] + ccmpe {dfv=of} bl, 123 +# CHECK: ccmpe {dfv=of} dx, 123 +# CHECK: encoding: [0x62,0xf4,0x45,0x04,0x83,0xfa,0x7b] + ccmpe {dfv=of} dx, 123 +# CHECK: ccmpe {dfv=of} ecx, 123 +# CHECK: encoding: [0x62,0xf4,0x44,0x04,0x83,0xf9,0x7b] + ccmpe {dfv=of} ecx, 123 +# CHECK: ccmpe {dfv=of} r9, 123 +# CHECK: encoding: [0x62,0xd4,0xc4,0x04,0x83,0xf9,0x7b] + ccmpe {dfv=of} r9, 123 +# CHECK: ccmpe {dfv=of} dx, 1234 +# CHECK: encoding: [0x62,0xf4,0x45,0x04,0x81,0xfa,0xd2,0x04] + ccmpe {dfv=of} dx, 1234 +# CHECK: ccmpe {dfv=of} ecx, 123456 +# CHECK: encoding: [0x62,0xf4,0x44,0x04,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpe {dfv=of} ecx, 123456 +# CHECK: ccmpe {dfv=of} r9, 123456 +# CHECK: encoding: [0x62,0xd4,0xc4,0x04,0x81,0xf9,0x40,0xe2,0x01,0x00] + ccmpe {dfv=of} r9, 123456 +# CHECK: ccmpe {dfv=of} dl, bl +# CHECK: encoding: [0x62,0xf4,0x44,0x04,0x38,0xda] + ccmpe {dfv=of} dl, bl +# CHECK: ccmpe {dfv=of} ax, dx +# CHECK: encoding: [0x62,0xf4,0x45,0x04,0x39,0xd0] + ccmpe {dfv=of} ax, dx +# CHECK: ccmpe {dfv=of} edx, ecx +# CHECK: encoding: [0x62,0xf4,0x44,0x04,0x39,0xca] + ccmpe {dfv=of} edx, ecx +# CHECK: ccmpe {dfv=of} r15, r9 +# CHECK: encoding: [0x62,0x54,0xc4,0x04,0x39,0xcf] + ccmpe {dfv=of} r15, r9 diff --git a/llvm/test/TableGen/x86-fold-tables.inc b/llvm/test/TableGen/x86-fold-tables.inc index abeef29c80ed..c35f22ff36de 100644 --- a/llvm/test/TableGen/x86-fold-tables.inc +++ b/llvm/test/TableGen/x86-fold-tables.inc @@ -363,6 +363,17 @@ static const X86FoldTableEntry Table0[] = { {X86::CALL32r_NT, X86::CALL32m_NT, TB_FOLDED_LOAD}, {X86::CALL64r, X86::CALL64m, TB_FOLDED_LOAD}, {X86::CALL64r_NT, X86::CALL64m_NT, TB_FOLDED_LOAD}, + {X86::CCMP16ri, X86::CCMP16mi, TB_FOLDED_LOAD}, + {X86::CCMP16ri8, X86::CCMP16mi8, TB_FOLDED_LOAD}, + {X86::CCMP16rr, X86::CCMP16mr, TB_FOLDED_LOAD}, + {X86::CCMP32ri, X86::CCMP32mi, TB_FOLDED_LOAD}, + {X86::CCMP32ri8, X86::CCMP32mi8, TB_FOLDED_LOAD}, + {X86::CCMP32rr, X86::CCMP32mr, TB_FOLDED_LOAD}, + {X86::CCMP64ri32, X86::CCMP64mi32, TB_FOLDED_LOAD}, + {X86::CCMP64ri8, X86::CCMP64mi8, TB_FOLDED_LOAD}, + {X86::CCMP64rr, X86::CCMP64mr, TB_FOLDED_LOAD}, + {X86::CCMP8ri, X86::CCMP8mi, TB_FOLDED_LOAD}, + {X86::CCMP8rr, X86::CCMP8mr, TB_FOLDED_LOAD}, {X86::CMP16ri, X86::CMP16mi, TB_FOLDED_LOAD}, {X86::CMP16ri8, X86::CMP16mi8, TB_FOLDED_LOAD}, {X86::CMP16rr, X86::CMP16mr, TB_FOLDED_LOAD}, @@ -375,6 +386,14 @@ static const X86FoldTableEntry Table0[] = { {X86::CMP8ri, X86::CMP8mi, TB_FOLDED_LOAD}, {X86::CMP8ri8, X86::CMP8mi8, TB_FOLDED_LOAD}, {X86::CMP8rr, X86::CMP8mr, TB_FOLDED_LOAD}, + {X86::CTEST16ri, X86::CTEST16mi, TB_FOLDED_LOAD}, + {X86::CTEST16rr, X86::CTEST16mr, TB_FOLDED_LOAD}, + {X86::CTEST32ri, X86::CTEST32mi, TB_FOLDED_LOAD}, + {X86::CTEST32rr, X86::CTEST32mr, TB_FOLDED_LOAD}, + {X86::CTEST64ri32, X86::CTEST64mi32, TB_FOLDED_LOAD}, + {X86::CTEST64rr, X86::CTEST64mr, TB_FOLDED_LOAD}, + {X86::CTEST8ri, X86::CTEST8mi, TB_FOLDED_LOAD}, + {X86::CTEST8rr, X86::CTEST8mr, TB_FOLDED_LOAD}, {X86::DIV16r, X86::DIV16m, TB_FOLDED_LOAD}, {X86::DIV16r_NF, X86::DIV16m_NF, TB_FOLDED_LOAD}, {X86::DIV32r, X86::DIV32m, TB_FOLDED_LOAD}, @@ -675,6 +694,10 @@ static const X86FoldTableEntry Table1[] = { {X86::BZHI64rr, X86::BZHI64rm, 0}, {X86::BZHI64rr_EVEX, X86::BZHI64rm_EVEX, 0}, {X86::BZHI64rr_NF, X86::BZHI64rm_NF, 0}, + {X86::CCMP16rr, X86::CCMP16rm, 0}, + {X86::CCMP32rr, X86::CCMP32rm, 0}, + {X86::CCMP64rr, X86::CCMP64rm, 0}, + {X86::CCMP8rr, X86::CCMP8rm, 0}, {X86::CMP16rr, X86::CMP16rm, 0}, {X86::CMP32rr, X86::CMP32rm, 0}, {X86::CMP64rr, X86::CMP64rm, 0}, diff --git a/llvm/tools/llvm-exegesis/lib/X86/Target.cpp b/llvm/tools/llvm-exegesis/lib/X86/Target.cpp index a41a995f5560..31bec2545926 100644 --- a/llvm/tools/llvm-exegesis/lib/X86/Target.cpp +++ b/llvm/tools/llvm-exegesis/lib/X86/Target.cpp @@ -884,6 +884,10 @@ Error ExegesisX86Target::randomizeTargetMCOperand( const BitVector &ForbiddenRegs) const { const Operand &Op = Instr.getPrimaryOperand(Var); switch (Op.getExplicitOperandInfo().OperandType) { + case X86::OperandType::OPERAND_COND_CODE: + AssignedValue = + MCOperand::createImm(randomIndex(X86::CondCode::LAST_VALID_COND)); + return Error::success(); case X86::OperandType::OPERAND_ROUNDING_CONTROL: AssignedValue = MCOperand::createImm(randomIndex(X86::STATIC_ROUNDING::TO_ZERO)); diff --git a/llvm/utils/TableGen/X86RecognizableInstr.cpp b/llvm/utils/TableGen/X86RecognizableInstr.cpp index 5aa2e51c78ba..39006a40daeb 100644 --- a/llvm/utils/TableGen/X86RecognizableInstr.cpp +++ b/llvm/utils/TableGen/X86RecognizableInstr.cpp @@ -127,6 +127,7 @@ RecognizableInstrBase::RecognizableInstrBase(const CodeGenInstruction &insn) { HasEVEX_KZ = Rec->getValueAsBit("hasEVEX_Z"); HasEVEX_B = Rec->getValueAsBit("hasEVEX_B"); HasEVEX_NF = Rec->getValueAsBit("hasEVEX_NF"); + HasTwoConditionalOps = Rec->getValueAsBit("hasTwoConditionalOps"); IsCodeGenOnly = Rec->getValueAsBit("isCodeGenOnly"); IsAsmParserOnly = Rec->getValueAsBit("isAsmParserOnly"); ForceDisassemble = Rec->getValueAsBit("ForceDisassemble"); @@ -494,6 +495,8 @@ void RecognizableInstr::emitInstructionSpecifier() { ++additionalOperands; if (HasEVEX_K) ++additionalOperands; + if (HasTwoConditionalOps) + additionalOperands += 2; #endif bool IsND = OpMap == X86Local::T_MAP4 && HasEVEX_B && HasVEX_4V; @@ -561,6 +564,7 @@ void RecognizableInstr::emitInstructionSpecifier() { HANDLE_OPERAND(roRegister) HANDLE_OPTIONAL(immediate) + HANDLE_OPTIONAL(immediate) break; case X86Local::MRMDestMem4VOp3CC: // Operand 1 is a register operand in the Reg/Opcode field. @@ -599,6 +603,7 @@ void RecognizableInstr::emitInstructionSpecifier() { HANDLE_OPERAND(roRegister) HANDLE_OPTIONAL(immediate) + HANDLE_OPTIONAL(immediate) break; case X86Local::MRMSrcReg: // Operand 1 is a register operand in the Reg/Opcode field. @@ -735,6 +740,7 @@ void RecognizableInstr::emitInstructionSpecifier() { HANDLE_OPTIONAL(rmRegister) HANDLE_OPTIONAL(relocation) HANDLE_OPTIONAL(immediate) + HANDLE_OPTIONAL(immediate) break; case X86Local::MRMXmCC: assert(numPhysicalOperands == 2 && @@ -763,6 +769,8 @@ void RecognizableInstr::emitInstructionSpecifier() { HANDLE_OPERAND(writemaskRegister) HANDLE_OPERAND(memory) HANDLE_OPTIONAL(relocation) + HANDLE_OPTIONAL(immediate) + HANDLE_OPTIONAL(immediate) break; case X86Local::RawFrmImm8: // operand 1 is a 16-bit immediate @@ -1032,6 +1040,7 @@ OperandType RecognizableInstr::typeFromString(const std::string &s, TYPE("i16imm_brtarget", TYPE_REL) TYPE("i32imm_brtarget", TYPE_REL) TYPE("ccode", TYPE_IMM) + TYPE("cflags", TYPE_IMM) TYPE("AVX512RC", TYPE_IMM) TYPE("brtarget32", TYPE_REL) TYPE("brtarget16", TYPE_REL) @@ -1127,6 +1136,8 @@ RecognizableInstr::immediateEncodingFromString(const std::string &s, ENCODING("i64i32imm", ENCODING_ID) ENCODING("i64i8imm", ENCODING_IB) ENCODING("i8imm", ENCODING_IB) + ENCODING("ccode", ENCODING_CC) + ENCODING("cflags", ENCODING_CF) ENCODING("u4imm", ENCODING_IB) ENCODING("u8imm", ENCODING_IB) ENCODING("i16u8imm", ENCODING_IB) diff --git a/llvm/utils/TableGen/X86RecognizableInstr.h b/llvm/utils/TableGen/X86RecognizableInstr.h index 549fc5b4d543..ad319b3f28b7 100644 --- a/llvm/utils/TableGen/X86RecognizableInstr.h +++ b/llvm/utils/TableGen/X86RecognizableInstr.h @@ -214,6 +214,8 @@ struct RecognizableInstrBase { bool HasEVEX_B; /// The hasEVEX_NF field from the record bool HasEVEX_NF; + /// The hasTwoConditionalOps field from the record + bool HasTwoConditionalOps; /// Indicates that the instruction uses the L and L' fields for RC. bool EncodeRC; /// The isCodeGenOnly field from the record -- GitLab From 4d1d1271b8612c72e0020c9d9f42d2ef70f717e7 Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Fri, 8 Mar 2024 06:54:52 -0600 Subject: [PATCH 0021/1674] [HIP] Make the HIP default architecture use the enum value (#84400) Summary: This default enum is used in other places, we should keep it consistent. --- clang/include/clang/Basic/Cuda.h | 2 +- clang/lib/Driver/Driver.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/clang/include/clang/Basic/Cuda.h b/clang/include/clang/Basic/Cuda.h index 916cb4b7ef34..3e77a74c7c00 100644 --- a/clang/include/clang/Basic/Cuda.h +++ b/clang/include/clang/Basic/Cuda.h @@ -123,7 +123,7 @@ enum class CudaArch { LAST, CudaDefault = CudaArch::SM_52, - HIPDefault = CudaArch::GFX803, + HIPDefault = CudaArch::GFX906, }; static inline bool IsNVIDIAGpuArch(CudaArch A) { diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp index 96e6ad77f5e5..fce43430a913 100644 --- a/clang/lib/Driver/Driver.cpp +++ b/clang/lib/Driver/Driver.cpp @@ -3373,7 +3373,7 @@ class OffloadingActionBuilder final { const Driver::InputList &Inputs) : CudaActionBuilderBase(C, Args, Inputs, Action::OFK_HIP) { - DefaultCudaArch = CudaArch::GFX906; + DefaultCudaArch = CudaArch::HIPDefault; if (Args.hasArg(options::OPT_fhip_emit_relocatable, options::OPT_fno_hip_emit_relocatable)) { -- GitLab From 7457e2c1535acd548d2619dfb34eb93d27d15908 Mon Sep 17 00:00:00 2001 From: Balazs Benics Date: Fri, 8 Mar 2024 13:57:27 +0100 Subject: [PATCH 0022/1674] [clang][ASTMatcher] Add matchers for isExplicitObjectMemberFunction() (#84446) Note that this patch will be necessary to fix `forEachArgumentWithParam()` and `forEachArgumentWithParamType()` matchers for deducing "this"; which is my true motivation. There the bug is that with explicit obj params, one should not adjust the number of arguments in presence of `CXXMethodDecls`, and this causes a mismatch there mapping the argument to the wrong param. But, I'll come back there once we have this matcher. --- clang/docs/LibASTMatchersReference.html | 15 +++++++++++++++ clang/docs/ReleaseNotes.rst | 1 + clang/include/clang/ASTMatchers/ASTMatchers.h | 19 +++++++++++++++++++ clang/include/clang/Testing/CommandLineArgs.h | 1 + clang/include/clang/Testing/TestClangConfig.h | 16 +++++++++++----- clang/lib/ASTMatchers/Dynamic/Registry.cpp | 1 + clang/lib/Testing/CommandLineArgs.cpp | 7 +++++++ .../ASTMatchers/ASTMatchersNarrowingTest.cpp | 14 ++++++++++++++ .../ASTMatchers/ASTMatchersNodeTest.cpp | 2 +- clang/unittests/ASTMatchers/ASTMatchersTest.h | 14 ++++++++++---- 10 files changed, 80 insertions(+), 10 deletions(-) diff --git a/clang/docs/LibASTMatchersReference.html b/clang/docs/LibASTMatchersReference.html index 8a06084955aa..bb1b68f6671b 100644 --- a/clang/docs/LibASTMatchersReference.html +++ b/clang/docs/LibASTMatchersReference.html @@ -3546,6 +3546,21 @@ cxxMethodDecl(isConst()) matches A::foo() but not A::bar() +Matcher<CXXMethodDecl>isExplicitObjectMemberFunction +
Matches if the given method declaration declares a member function with an explicit object parameter.
+
+Given
+struct A {
+  int operator-(this A, int);
+  void fun(this A &&self);
+  static int operator()(int);
+  int operator+(int);
+};
+
+cxxMethodDecl(isExplicitObjectMemberFunction()) matches the first two methods but not the last two.
+
+ + Matcher<CXXMethodDecl>isCopyAssignmentOperator 
Matches if the given method declaration declares a copy assignment
 operator.
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 0ee2801766a9..73901a14631b 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -433,6 +433,7 @@ AST Matchers
 ------------
 
 - ``isInStdNamespace`` now supports Decl declared with ``extern "C++"``.
+- Add ``isExplicitObjectMemberFunction``.
 
 clang-format
 ------------
diff --git a/clang/include/clang/ASTMatchers/ASTMatchers.h b/clang/include/clang/ASTMatchers/ASTMatchers.h
index ced89ff127ab..96dbcdc344e1 100644
--- a/clang/include/clang/ASTMatchers/ASTMatchers.h
+++ b/clang/include/clang/ASTMatchers/ASTMatchers.h
@@ -6366,6 +6366,25 @@ AST_MATCHER(CXXMethodDecl, isConst) {
   return Node.isConst();
 }
 
+/// Matches if the given method declaration declares a member function with an
+/// explicit object parameter.
+///
+/// Given
+/// \code
+/// struct A {
+///  int operator-(this A, int);
+///  void fun(this A &&self);
+///  static int operator()(int);
+///  int operator+(int);
+/// };
+/// \endcode
+///
+/// cxxMethodDecl(isExplicitObjectMemberFunction()) matches the first two
+/// methods but not the last two.
+AST_MATCHER(CXXMethodDecl, isExplicitObjectMemberFunction) {
+  return Node.isExplicitObjectMemberFunction();
+}
+
 /// Matches if the given method declaration declares a copy assignment
 /// operator.
 ///
diff --git a/clang/include/clang/Testing/CommandLineArgs.h b/clang/include/clang/Testing/CommandLineArgs.h
index 4dd28718dfa6..e71907e8bbd0 100644
--- a/clang/include/clang/Testing/CommandLineArgs.h
+++ b/clang/include/clang/Testing/CommandLineArgs.h
@@ -28,6 +28,7 @@ enum TestLanguage {
   Lang_CXX14,
   Lang_CXX17,
   Lang_CXX20,
+  Lang_CXX23,
   Lang_OpenCL,
   Lang_OBJC,
   Lang_OBJCXX
diff --git a/clang/include/clang/Testing/TestClangConfig.h b/clang/include/clang/Testing/TestClangConfig.h
index 92d5cc3cff99..1b4efca80e9d 100644
--- a/clang/include/clang/Testing/TestClangConfig.h
+++ b/clang/include/clang/Testing/TestClangConfig.h
@@ -34,24 +34,30 @@ struct TestClangConfig {
   bool isCXX() const {
     return Language == Lang_CXX03 || Language == Lang_CXX11 ||
            Language == Lang_CXX14 || Language == Lang_CXX17 ||
-           Language == Lang_CXX20;
+           Language == Lang_CXX20 || Language == Lang_CXX23;
   }
 
   bool isCXX11OrLater() const {
     return Language == Lang_CXX11 || Language == Lang_CXX14 ||
-           Language == Lang_CXX17 || Language == Lang_CXX20;
+           Language == Lang_CXX17 || Language == Lang_CXX20 ||
+           Language == Lang_CXX23;
   }
 
   bool isCXX14OrLater() const {
     return Language == Lang_CXX14 || Language == Lang_CXX17 ||
-           Language == Lang_CXX20;
+           Language == Lang_CXX20 || Language == Lang_CXX23;
   }
 
   bool isCXX17OrLater() const {
-    return Language == Lang_CXX17 || Language == Lang_CXX20;
+    return Language == Lang_CXX17 || Language == Lang_CXX20 ||
+           Language == Lang_CXX23;
   }
 
-  bool isCXX20OrLater() const { return Language == Lang_CXX20; }
+  bool isCXX20OrLater() const {
+    return Language == Lang_CXX20 || Language == Lang_CXX23;
+  }
+
+  bool isCXX23OrLater() const { return Language == Lang_CXX23; }
 
   bool supportsCXXDynamicExceptionSpecification() const {
     return Language == Lang_CXX03 || Language == Lang_CXX11 ||
diff --git a/clang/lib/ASTMatchers/Dynamic/Registry.cpp b/clang/lib/ASTMatchers/Dynamic/Registry.cpp
index 15dad022df5f..2c75e6beb743 100644
--- a/clang/lib/ASTMatchers/Dynamic/Registry.cpp
+++ b/clang/lib/ASTMatchers/Dynamic/Registry.cpp
@@ -432,6 +432,7 @@ RegistryMaps::RegistryMaps() {
   REGISTER_MATCHER(isExpansionInMainFile);
   REGISTER_MATCHER(isExpansionInSystemHeader);
   REGISTER_MATCHER(isExplicit);
+  REGISTER_MATCHER(isExplicitObjectMemberFunction);
   REGISTER_MATCHER(isExplicitTemplateSpecialization);
   REGISTER_MATCHER(isExpr);
   REGISTER_MATCHER(isExternC);
diff --git a/clang/lib/Testing/CommandLineArgs.cpp b/clang/lib/Testing/CommandLineArgs.cpp
index 0da087c33e3f..3abc689b93e8 100644
--- a/clang/lib/Testing/CommandLineArgs.cpp
+++ b/clang/lib/Testing/CommandLineArgs.cpp
@@ -37,6 +37,9 @@ std::vector getCommandLineArgsForTesting(TestLanguage Lang) {
   case Lang_CXX20:
     Args = {"-std=c++20", "-frtti"};
     break;
+  case Lang_CXX23:
+    Args = {"-std=c++23", "-frtti"};
+    break;
   case Lang_OBJC:
     Args = {"-x", "objective-c", "-frtti", "-fobjc-nonfragile-abi"};
     break;
@@ -73,6 +76,9 @@ std::vector getCC1ArgsForTesting(TestLanguage Lang) {
   case Lang_CXX20:
     Args = {"-std=c++20"};
     break;
+  case Lang_CXX23:
+    Args = {"-std=c++23"};
+    break;
   case Lang_OBJC:
     Args = {"-xobjective-c"};
     break;
@@ -96,6 +102,7 @@ StringRef getFilenameForTesting(TestLanguage Lang) {
   case Lang_CXX14:
   case Lang_CXX17:
   case Lang_CXX20:
+  case Lang_CXX23:
     return "input.cc";
 
   case Lang_OpenCL:
diff --git a/clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp b/clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
index b75da7bc1ed0..87774b00956a 100644
--- a/clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
+++ b/clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
@@ -2107,6 +2107,20 @@ TEST_P(ASTMatchersTest, IsPure) {
   EXPECT_TRUE(notMatches("class X { int f(); };", cxxMethodDecl(isPure())));
 }
 
+TEST_P(ASTMatchersTest, IsExplicitObjectMemberFunction) {
+  if (!GetParam().isCXX23OrLater()) {
+    return;
+  }
+
+  auto ExpObjParamFn = cxxMethodDecl(isExplicitObjectMemberFunction());
+  EXPECT_TRUE(
+      notMatches("struct A { static int operator()(int); };", ExpObjParamFn));
+  EXPECT_TRUE(notMatches("struct A { int operator+(int); };", ExpObjParamFn));
+  EXPECT_TRUE(
+      matches("struct A { int operator-(this A, int); };", ExpObjParamFn));
+  EXPECT_TRUE(matches("struct A { void fun(this A &&self); };", ExpObjParamFn));
+}
+
 TEST_P(ASTMatchersTest, IsCopyAssignmentOperator) {
   if (!GetParam().isCXX()) {
     return;
diff --git a/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp b/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp
index ae30c03126d7..0edc65162fbe 100644
--- a/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp
+++ b/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp
@@ -2754,7 +2754,7 @@ TEST(MatchFinderAPI, MatchesDynamic) {
 static std::vector allTestClangConfigs() {
   std::vector all_configs;
   for (TestLanguage lang : {Lang_C89, Lang_C99, Lang_CXX03, Lang_CXX11,
-                            Lang_CXX14, Lang_CXX17, Lang_CXX20}) {
+                            Lang_CXX14, Lang_CXX17, Lang_CXX20, Lang_CXX23}) {
     TestClangConfig config;
     config.Language = lang;
 
diff --git a/clang/unittests/ASTMatchers/ASTMatchersTest.h b/clang/unittests/ASTMatchers/ASTMatchersTest.h
index 79c618605483..1ed1b5958a8b 100644
--- a/clang/unittests/ASTMatchers/ASTMatchersTest.h
+++ b/clang/unittests/ASTMatchers/ASTMatchersTest.h
@@ -62,22 +62,28 @@ private:
 
 inline ArrayRef langCxx11OrLater() {
   static const TestLanguage Result[] = {Lang_CXX11, Lang_CXX14, Lang_CXX17,
-                                        Lang_CXX20};
+                                        Lang_CXX20, Lang_CXX23};
   return ArrayRef(Result);
 }
 
 inline ArrayRef langCxx14OrLater() {
-  static const TestLanguage Result[] = {Lang_CXX14, Lang_CXX17, Lang_CXX20};
+  static const TestLanguage Result[] = {Lang_CXX14, Lang_CXX17, Lang_CXX20,
+                                        Lang_CXX23};
   return ArrayRef(Result);
 }
 
 inline ArrayRef langCxx17OrLater() {
-  static const TestLanguage Result[] = {Lang_CXX17, Lang_CXX20};
+  static const TestLanguage Result[] = {Lang_CXX17, Lang_CXX20, Lang_CXX23};
   return ArrayRef(Result);
 }
 
 inline ArrayRef langCxx20OrLater() {
-  static const TestLanguage Result[] = {Lang_CXX20};
+  static const TestLanguage Result[] = {Lang_CXX20, Lang_CXX23};
+  return ArrayRef(Result);
+}
+
+inline ArrayRef langCxx23OrLater() {
+  static const TestLanguage Result[] = {Lang_CXX23};
   return ArrayRef(Result);
 }
 
-- 
GitLab


From 9baa4140f2b633a8d075f859690139ef2d79b78b Mon Sep 17 00:00:00 2001
From: Guillaume Chatelet 
Date: Fri, 8 Mar 2024 14:16:07 +0100
Subject: [PATCH 0023/1674] [libc][NFC] Simplify `BigInt::mul` (#84468)

---
 libc/src/__support/UInt.h | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/libc/src/__support/UInt.h b/libc/src/__support/UInt.h
index cf0c5a669ae8..fadccf545006 100644
--- a/libc/src/__support/UInt.h
+++ b/libc/src/__support/UInt.h
@@ -262,15 +262,13 @@ struct BigInt {
   // Returns the carry value produced by the multiplication operation.
   LIBC_INLINE constexpr WordType mul(WordType x) {
     BigInt<2 * WORD_SIZE, Signed, WordType> partial_sum(0);
-    WordType carry = 0;
     for (size_t i = 0; i < WORD_COUNT; ++i) {
       NumberPair prod = full_mul(val[i], x);
       BigInt<2 * WORD_SIZE, Signed, WordType> tmp({prod.lo, prod.hi});
-      carry += partial_sum.add(tmp);
+      const WordType carry = partial_sum.add(tmp);
       val[i] = partial_sum.val[0];
       partial_sum.val[0] = partial_sum.val[1];
       partial_sum.val[1] = carry;
-      carry = 0;
     }
     return partial_sum.val[1];
   }
-- 
GitLab


From aa6e7a61c56823afb4db65e42af1e6ce26de6186 Mon Sep 17 00:00:00 2001
From: Guillaume Chatelet 
Date: Fri, 8 Mar 2024 14:16:42 +0100
Subject: [PATCH 0024/1674] [libc][NFC] Remove integer_utils.h (#84466)

Its sole user is `BigInt` so moving `full_mul` inside UInt.h.
---
 libc/src/__support/CMakeLists.txt             | 13 ----
 libc/src/__support/UInt.h                     | 60 ++++++++++++++--
 libc/src/__support/integer_utils.h            | 69 -------------------
 .../llvm-project-overlay/libc/BUILD.bazel     | 12 ----
 4 files changed, 54 insertions(+), 100 deletions(-)
 delete mode 100644 libc/src/__support/integer_utils.h

diff --git a/libc/src/__support/CMakeLists.txt b/libc/src/__support/CMakeLists.txt
index 17c04aa57e6f..66e4d808333f 100644
--- a/libc/src/__support/CMakeLists.txt
+++ b/libc/src/__support/CMakeLists.txt
@@ -194,24 +194,11 @@ add_header_library(
     libc.src.__support.CPP.type_traits
 )
 
-add_header_library(
-  integer_utils
-  HDRS
-    integer_utils.h
-  DEPENDS
-    .math_extras
-    .number_pair
-    libc.src.__support.common
-    libc.src.__support.CPP.bit
-    libc.src.__support.CPP.type_traits
-)
-
 add_header_library(
   uint
   HDRS
     UInt.h
   DEPENDS
-    .integer_utils
     .math_extras
     .number_pair
     libc.src.__support.CPP.array
diff --git a/libc/src/__support/UInt.h b/libc/src/__support/UInt.h
index fadccf545006..94c8d86fd764 100644
--- a/libc/src/__support/UInt.h
+++ b/libc/src/__support/UInt.h
@@ -14,7 +14,6 @@
 #include "src/__support/CPP/limits.h"
 #include "src/__support/CPP/optional.h"
 #include "src/__support/CPP/type_traits.h"
-#include "src/__support/integer_utils.h"
 #include "src/__support/macros/attributes.h"   // LIBC_INLINE
 #include "src/__support/macros/optimization.h" // LIBC_UNLIKELY
 #include "src/__support/math_extras.h"         // SumCarry, DiffBorrow
@@ -36,6 +35,52 @@ template <> struct half_width<__uint128_t> : cpp::type_identity {};
 #endif // __SIZEOF_INT128__
 
 template  using half_width_t = typename half_width::type;
+
+template  constexpr NumberPair full_mul(T a, T b) {
+  NumberPair pa = split(a);
+  NumberPair pb = split(b);
+  NumberPair prod;
+
+  prod.lo = pa.lo * pb.lo;                    // exact
+  prod.hi = pa.hi * pb.hi;                    // exact
+  NumberPair lo_hi = split(pa.lo * pb.hi); // exact
+  NumberPair hi_lo = split(pa.hi * pb.lo); // exact
+
+  constexpr size_t HALF_BIT_WIDTH = sizeof(T) * CHAR_BIT / 2;
+
+  auto r1 = add_with_carry(prod.lo, lo_hi.lo << HALF_BIT_WIDTH, T(0));
+  prod.lo = r1.sum;
+  prod.hi = add_with_carry(prod.hi, lo_hi.hi, r1.carry).sum;
+
+  auto r2 = add_with_carry(prod.lo, hi_lo.lo << HALF_BIT_WIDTH, T(0));
+  prod.lo = r2.sum;
+  prod.hi = add_with_carry(prod.hi, hi_lo.hi, r2.carry).sum;
+
+  return prod;
+}
+
+template <>
+LIBC_INLINE constexpr NumberPair full_mul(uint32_t a,
+                                                              uint32_t b) {
+  uint64_t prod = uint64_t(a) * uint64_t(b);
+  NumberPair result;
+  result.lo = uint32_t(prod);
+  result.hi = uint32_t(prod >> 32);
+  return result;
+}
+
+#ifdef __SIZEOF_INT128__
+template <>
+LIBC_INLINE constexpr NumberPair full_mul(uint64_t a,
+                                                              uint64_t b) {
+  __uint128_t prod = __uint128_t(a) * __uint128_t(b);
+  NumberPair result;
+  result.lo = uint64_t(prod);
+  result.hi = uint64_t(prod >> 64);
+  return result;
+}
+#endif // __SIZEOF_INT128__
+
 } // namespace internal
 
 template 
@@ -263,7 +308,7 @@ struct BigInt {
   LIBC_INLINE constexpr WordType mul(WordType x) {
     BigInt<2 * WORD_SIZE, Signed, WordType> partial_sum(0);
     for (size_t i = 0; i < WORD_COUNT; ++i) {
-      NumberPair prod = full_mul(val[i], x);
+      NumberPair prod = internal::full_mul(val[i], x);
       BigInt<2 * WORD_SIZE, Signed, WordType> tmp({prod.lo, prod.hi});
       const WordType carry = partial_sum.add(tmp);
       val[i] = partial_sum.val[0];
@@ -296,7 +341,8 @@ struct BigInt {
         WordType carry = 0;
         for (size_t i = 0; i < WORD_COUNT; ++i) {
           for (size_t j = 0; j <= i; j++) {
-            NumberPair prod = full_mul(val[j], other.val[i - j]);
+            NumberPair prod =
+                internal::full_mul(val[j], other.val[i - j]);
             BigInt<2 * WORD_SIZE, Signed, WordType> tmp({prod.lo, prod.hi});
             carry += partial_sum.add(tmp);
           }
@@ -324,7 +370,8 @@ struct BigInt {
           i < OTHER_WORDCOUNT ? 0 : i - OTHER_WORDCOUNT + 1;
       const size_t upper_idx = i < WORD_COUNT ? i : WORD_COUNT - 1;
       for (size_t j = lower_idx; j <= upper_idx; ++j) {
-        NumberPair prod = full_mul(val[j], other.val[i - j]);
+        NumberPair prod =
+            internal::full_mul(val[j], other.val[i - j]);
         BigInt<2 * WORD_SIZE, Signed, WordType> tmp({prod.lo, prod.hi});
         carry += partial_sum.add(tmp);
       }
@@ -366,7 +413,7 @@ struct BigInt {
     // product.
     for (size_t i = 0; i < WORD_COUNT; ++i) {
       NumberPair prod =
-          full_mul(val[i], other.val[WORD_COUNT - 1 - i]);
+          internal::full_mul(val[i], other.val[WORD_COUNT - 1 - i]);
       BigInt<2 * WORD_SIZE, Signed, WordType> tmp({prod.lo, prod.hi});
       carry += partial_sum.add(tmp);
     }
@@ -375,7 +422,8 @@ struct BigInt {
       partial_sum.val[1] = carry;
       carry = 0;
       for (size_t j = i - WORD_COUNT + 1; j < WORD_COUNT; ++j) {
-        NumberPair prod = full_mul(val[j], other.val[i - j]);
+        NumberPair prod =
+            internal::full_mul(val[j], other.val[i - j]);
         BigInt<2 * WORD_SIZE, Signed, WordType> tmp({prod.lo, prod.hi});
         carry += partial_sum.add(tmp);
       }
diff --git a/libc/src/__support/integer_utils.h b/libc/src/__support/integer_utils.h
deleted file mode 100644
index 15e04bda8082..000000000000
--- a/libc/src/__support/integer_utils.h
+++ /dev/null
@@ -1,69 +0,0 @@
-//===-- Utilities for integers. ---------------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_SRC___SUPPORT_INTEGER_UTILS_H
-#define LLVM_LIBC_SRC___SUPPORT_INTEGER_UTILS_H
-
-#include "src/__support/CPP/type_traits.h"
-#include "src/__support/common.h"
-
-#include "math_extras.h"
-#include "number_pair.h"
-
-#include 
-
-namespace LIBC_NAMESPACE {
-
-template  constexpr NumberPair full_mul(T a, T b) {
-  NumberPair pa = split(a);
-  NumberPair pb = split(b);
-  NumberPair prod;
-
-  prod.lo = pa.lo * pb.lo;                    // exact
-  prod.hi = pa.hi * pb.hi;                    // exact
-  NumberPair lo_hi = split(pa.lo * pb.hi); // exact
-  NumberPair hi_lo = split(pa.hi * pb.lo); // exact
-
-  constexpr size_t HALF_BIT_WIDTH = sizeof(T) * CHAR_BIT / 2;
-
-  auto r1 = add_with_carry(prod.lo, lo_hi.lo << HALF_BIT_WIDTH, T(0));
-  prod.lo = r1.sum;
-  prod.hi = add_with_carry(prod.hi, lo_hi.hi, r1.carry).sum;
-
-  auto r2 = add_with_carry(prod.lo, hi_lo.lo << HALF_BIT_WIDTH, T(0));
-  prod.lo = r2.sum;
-  prod.hi = add_with_carry(prod.hi, hi_lo.hi, r2.carry).sum;
-
-  return prod;
-}
-
-template <>
-LIBC_INLINE constexpr NumberPair full_mul(uint32_t a,
-                                                              uint32_t b) {
-  uint64_t prod = uint64_t(a) * uint64_t(b);
-  NumberPair result;
-  result.lo = uint32_t(prod);
-  result.hi = uint32_t(prod >> 32);
-  return result;
-}
-
-#ifdef __SIZEOF_INT128__
-template <>
-LIBC_INLINE constexpr NumberPair full_mul(uint64_t a,
-                                                              uint64_t b) {
-  __uint128_t prod = __uint128_t(a) * __uint128_t(b);
-  NumberPair result;
-  result.lo = uint64_t(prod);
-  result.hi = uint64_t(prod >> 64);
-  return result;
-}
-#endif // __SIZEOF_INT128__
-
-} // namespace LIBC_NAMESPACE
-
-#endif // LLVM_LIBC_SRC___SUPPORT_INTEGER_UTILS_H
diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
index 5c6cf761ebe7..127059c529ec 100644
--- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
@@ -438,17 +438,6 @@ libc_support_library(
     ],
 )
 
-libc_support_library(
-    name = "__support_integer_utils",
-    hdrs = ["src/__support/integer_utils.h"],
-    deps = [
-        ":__support_common",
-        ":__support_cpp_type_traits",
-        ":__support_math_extras",
-        ":__support_number_pair",
-    ],
-)
-
 libc_support_library(
     name = "__support_uint",
     hdrs = ["src/__support/UInt.h"],
@@ -458,7 +447,6 @@ libc_support_library(
         ":__support_cpp_limits",
         ":__support_cpp_optional",
         ":__support_cpp_type_traits",
-        ":__support_integer_utils",
         ":__support_macros_attributes",
         ":__support_macros_optimization",
         ":__support_math_extras",
-- 
GitLab


From 912ea6e335ca6174b3433fd960ed8b592794361b Mon Sep 17 00:00:00 2001
From: Congcong Cai 
Date: Fri, 8 Mar 2024 21:18:58 +0800
Subject: [PATCH 0025/1674] [clang-tidy]avoid bugprone-unused-return-value
 false positive for function with the same prefix as the default argument
 (#84333)

---
 .../bugprone/UnusedReturnValueCheck.cpp       | 184 +++++++++---------
 clang-tools-extra/docs/ReleaseNotes.rst       |   4 +-
 .../checks/bugprone/unused-return-value.rst   |  33 ++--
 .../checkers/bugprone/unused-return-value.cpp |  14 ++
 4 files changed, 126 insertions(+), 109 deletions(-)

diff --git a/clang-tools-extra/clang-tidy/bugprone/UnusedReturnValueCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/UnusedReturnValueCheck.cpp
index b4bf85c912c3..1252b2f23805 100644
--- a/clang-tools-extra/clang-tidy/bugprone/UnusedReturnValueCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/UnusedReturnValueCheck.cpp
@@ -34,102 +34,102 @@ UnusedReturnValueCheck::UnusedReturnValueCheck(llvm::StringRef Name,
                                                ClangTidyContext *Context)
     : ClangTidyCheck(Name, Context),
       CheckedFunctions(utils::options::parseStringList(
-          Options.get("CheckedFunctions", "::std::async;"
-                                          "::std::launder;"
-                                          "::std::remove;"
-                                          "::std::remove_if;"
-                                          "::std::unique;"
-                                          "::std::unique_ptr::release;"
-                                          "::std::basic_string::empty;"
-                                          "::std::vector::empty;"
-                                          "::std::back_inserter;"
-                                          "::std::distance;"
-                                          "::std::find;"
-                                          "::std::find_if;"
-                                          "::std::inserter;"
-                                          "::std::lower_bound;"
-                                          "::std::make_pair;"
-                                          "::std::map::count;"
-                                          "::std::map::find;"
-                                          "::std::map::lower_bound;"
-                                          "::std::multimap::equal_range;"
-                                          "::std::multimap::upper_bound;"
-                                          "::std::set::count;"
-                                          "::std::set::find;"
-                                          "::std::setfill;"
-                                          "::std::setprecision;"
-                                          "::std::setw;"
-                                          "::std::upper_bound;"
-                                          "::std::vector::at;"
+          Options.get("CheckedFunctions", "::std::async$;"
+                                          "::std::launder$;"
+                                          "::std::remove$;"
+                                          "::std::remove_if$;"
+                                          "::std::unique$;"
+                                          "::std::unique_ptr::release$;"
+                                          "::std::basic_string::empty$;"
+                                          "::std::vector::empty$;"
+                                          "::std::back_inserter$;"
+                                          "::std::distance$;"
+                                          "::std::find$;"
+                                          "::std::find_if$;"
+                                          "::std::inserter$;"
+                                          "::std::lower_bound$;"
+                                          "::std::make_pair$;"
+                                          "::std::map::count$;"
+                                          "::std::map::find$;"
+                                          "::std::map::lower_bound$;"
+                                          "::std::multimap::equal_range$;"
+                                          "::std::multimap::upper_bound$;"
+                                          "::std::set::count$;"
+                                          "::std::set::find$;"
+                                          "::std::setfill$;"
+                                          "::std::setprecision$;"
+                                          "::std::setw$;"
+                                          "::std::upper_bound$;"
+                                          "::std::vector::at$;"
                                           // C standard library
-                                          "::bsearch;"
-                                          "::ferror;"
-                                          "::feof;"
-                                          "::isalnum;"
-                                          "::isalpha;"
-                                          "::isblank;"
-                                          "::iscntrl;"
-                                          "::isdigit;"
-                                          "::isgraph;"
-                                          "::islower;"
-                                          "::isprint;"
-                                          "::ispunct;"
-                                          "::isspace;"
-                                          "::isupper;"
-                                          "::iswalnum;"
-                                          "::iswprint;"
-                                          "::iswspace;"
-                                          "::isxdigit;"
-                                          "::memchr;"
-                                          "::memcmp;"
-                                          "::strcmp;"
-                                          "::strcoll;"
-                                          "::strncmp;"
-                                          "::strpbrk;"
-                                          "::strrchr;"
-                                          "::strspn;"
-                                          "::strstr;"
-                                          "::wcscmp;"
+                                          "::bsearch$;"
+                                          "::ferror$;"
+                                          "::feof$;"
+                                          "::isalnum$;"
+                                          "::isalpha$;"
+                                          "::isblank$;"
+                                          "::iscntrl$;"
+                                          "::isdigit$;"
+                                          "::isgraph$;"
+                                          "::islower$;"
+                                          "::isprint$;"
+                                          "::ispunct$;"
+                                          "::isspace$;"
+                                          "::isupper$;"
+                                          "::iswalnum$;"
+                                          "::iswprint$;"
+                                          "::iswspace$;"
+                                          "::isxdigit$;"
+                                          "::memchr$;"
+                                          "::memcmp$;"
+                                          "::strcmp$;"
+                                          "::strcoll$;"
+                                          "::strncmp$;"
+                                          "::strpbrk$;"
+                                          "::strrchr$;"
+                                          "::strspn$;"
+                                          "::strstr$;"
+                                          "::wcscmp$;"
                                           // POSIX
-                                          "::access;"
-                                          "::bind;"
-                                          "::connect;"
-                                          "::difftime;"
-                                          "::dlsym;"
-                                          "::fnmatch;"
-                                          "::getaddrinfo;"
-                                          "::getopt;"
-                                          "::htonl;"
-                                          "::htons;"
-                                          "::iconv_open;"
-                                          "::inet_addr;"
-                                          "::isascii;"
-                                          "::isatty;"
-                                          "::mmap;"
-                                          "::newlocale;"
-                                          "::openat;"
-                                          "::pathconf;"
-                                          "::pthread_equal;"
-                                          "::pthread_getspecific;"
-                                          "::pthread_mutex_trylock;"
-                                          "::readdir;"
-                                          "::readlink;"
-                                          "::recvmsg;"
-                                          "::regexec;"
-                                          "::scandir;"
-                                          "::semget;"
-                                          "::setjmp;"
-                                          "::shm_open;"
-                                          "::shmget;"
-                                          "::sigismember;"
-                                          "::strcasecmp;"
-                                          "::strsignal;"
+                                          "::access$;"
+                                          "::bind$;"
+                                          "::connect$;"
+                                          "::difftime$;"
+                                          "::dlsym$;"
+                                          "::fnmatch$;"
+                                          "::getaddrinfo$;"
+                                          "::getopt$;"
+                                          "::htonl$;"
+                                          "::htons$;"
+                                          "::iconv_open$;"
+                                          "::inet_addr$;"
+                                          "::isascii$;"
+                                          "::isatty$;"
+                                          "::mmap$;"
+                                          "::newlocale$;"
+                                          "::openat$;"
+                                          "::pathconf$;"
+                                          "::pthread_equal$;"
+                                          "::pthread_getspecific$;"
+                                          "::pthread_mutex_trylock$;"
+                                          "::readdir$;"
+                                          "::readlink$;"
+                                          "::recvmsg$;"
+                                          "::regexec$;"
+                                          "::scandir$;"
+                                          "::semget$;"
+                                          "::setjmp$;"
+                                          "::shm_open$;"
+                                          "::shmget$;"
+                                          "::sigismember$;"
+                                          "::strcasecmp$;"
+                                          "::strsignal$;"
                                           "::ttyname"))),
       CheckedReturnTypes(utils::options::parseStringList(
-          Options.get("CheckedReturnTypes", "::std::error_code;"
-                                            "::std::error_condition;"
-                                            "::std::errc;"
-                                            "::std::expected;"
+          Options.get("CheckedReturnTypes", "::std::error_code$;"
+                                            "::std::error_condition$;"
+                                            "::std::errc$;"
+                                            "::std::expected$;"
                                             "::boost::system::error_code"))),
       AllowCastToVoid(Options.get("AllowCastToVoid", false)) {}
 
diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index d98c4ff9a750..b5f025ce467a 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -152,7 +152,9 @@ Changes in existing checks
 
 - Improved :doc:`bugprone-unused-return-value
   ` check by updating the
-  parameter `CheckedFunctions` to support regexp.
+  parameter `CheckedFunctions` to support regexp and avoiding false postive for
+  function with the same prefix as the default argument, e.g. ``std::unique_ptr``
+  and ``std::unique``.
 
 - Improved :doc:`bugprone-use-after-move
   ` check to also handle
diff --git a/clang-tools-extra/docs/clang-tidy/checks/bugprone/unused-return-value.rst b/clang-tools-extra/docs/clang-tidy/checks/bugprone/unused-return-value.rst
index 8d5eddbe215c..9c01ef50b538 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/bugprone/unused-return-value.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/bugprone/unused-return-value.rst
@@ -14,22 +14,23 @@ Options
    This parameter supports regexp. The function is checked if the name
    and scope matches, with any arguments.
    By default the following functions are checked:
-   ``std::async, std::launder, std::remove, std::remove_if, std::unique,
-   std::unique_ptr::release, std::basic_string::empty, std::vector::empty,
-   std::back_inserter, std::distance, std::find, std::find_if, std::inserter,
-   std::lower_bound, std::make_pair, std::map::count, std::map::find,
-   std::map::lower_bound, std::multimap::equal_range,
-   std::multimap::upper_bound, std::set::count, std::set::find, std::setfill,
-   std::setprecision, std::setw, std::upper_bound, std::vector::at,
-   bsearch, ferror, feof, isalnum, isalpha, isblank, iscntrl, isdigit, isgraph,
-   islower, isprint, ispunct, isspace, isupper, iswalnum, iswprint, iswspace,
-   isxdigit, memchr, memcmp, strcmp, strcoll, strncmp, strpbrk, strrchr,
-   strspn, strstr, wcscmp, access, bind, connect, difftime, dlsym, fnmatch,
-   getaddrinfo, getopt, htonl, htons, iconv_open, inet_addr, isascii, isatty,
-   mmap, newlocale, openat, pathconf, pthread_equal, pthread_getspecific,
-   pthread_mutex_trylock, readdir, readlink, recvmsg, regexec, scandir,
-   semget, setjmp, shm_open, shmget, sigismember, strcasecmp, strsignal,
-   ttyname``
+   ``::std::async$, ::std::launder$, ::std::remove$, ::std::remove_if$, ::std::unique$,
+   ::std::unique_ptr::release$, ::std::basic_string::empty$, ::std::vector::empty$,
+   ::std::back_inserter$, ::std::distance$, ::std::find$, ::std::find_if$, ::std::inserter$,
+   ::std::lower_bound$, ::std::make_pair$, ::std::map::count$, ::std::map::find$,
+   ::std::map::lower_bound$, ::std::multimap::equal_range$, ::std::multimap::upper_bound$,
+   ::std::set::count$, ::std::set::find$, ::std::setfill$, ::std::setprecision$,
+   ::std::setw$, ::std::upper_bound$, ::std::vector::at$, ::bsearch$, ::ferror$,
+   ::feof$, ::isalnum$, ::isalpha$, ::isblank$, ::iscntrl$, ::isdigit$, ::isgraph$,
+   ::islower$, ::isprint$, ::ispunct$, ::isspace$, ::isupper$, ::iswalnum$, ::iswprint$,
+   ::iswspace$, ::isxdigit$, ::memchr$, ::memcmp$, ::strcmp$, ::strcoll$, ::strncmp$,
+   ::strpbrk$, ::strrchr$, ::strspn$, ::strstr$, ::wcscmp$, ::access$, ::bind$,
+   ::connect$, ::difftime$, ::dlsym$, ::fnmatch$, ::getaddrinfo$, ::getopt$,
+   ::htonl$, ::htons$, ::iconv_open$, ::inet_addr$, isascii$, isatty$, ::mmap$,
+   ::newlocale$, ::openat$, ::pathconf$, ::pthread_equal$, ::pthread_getspecific$,
+   ::pthread_mutex_trylock$, ::readdir$, ::readlink$, ::recvmsg$, ::regexec$, ::scandir$,
+   ::semget$, ::setjmp$, ::shm_open$, ::shmget$, ::sigismember$, ::strcasecmp$, ::strsignal$,
+   ::ttyname$``
 
    - ``std::async()``. Not using the return value makes the call synchronous.
    - ``std::launder()``. Not using the return value usually means that the
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/unused-return-value.cpp b/clang-tools-extra/test/clang-tidy/checkers/bugprone/unused-return-value.cpp
index 5c6ce1e4bf1f..e784c9b85172 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/bugprone/unused-return-value.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/unused-return-value.cpp
@@ -30,6 +30,11 @@ struct default_delete;
 
 template >
 struct unique_ptr {
+  unique_ptr();
+  unique_ptr(unique_ptr const&);
+  unique_ptr(unique_ptr &&);
+  unique_ptr& operator=(unique_ptr const&);
+  unique_ptr& operator=(unique_ptr &&);
   T *release() noexcept;
 };
 
@@ -254,3 +259,12 @@ void noWarning() {
   ({ std::async(increment, 42); });
   auto StmtExprRetval = ({ std::async(increment, 42); });
 }
+
+namespace gh84314 {
+
+extern std::unique_ptr alloc();
+void f1(std::unique_ptr &foo) {
+    foo = alloc();
+}
+
+} // namespace gh84314
\ No newline at end of file
-- 
GitLab


From 10b01563da1d7c98d5fff6aa8b4a78be56a8099d Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek 
Date: Fri, 8 Mar 2024 07:20:20 -0600
Subject: [PATCH 0026/1674] [flang][HLFIR] Fix use-after-free when rewriting
 users in `canonicalize` (#84371)

Rewriting an op can invalidate the operator range being iterated on.
Store the users in a separate list, and iterate over the list instead.

This was detected by address sanitizer.
---
 flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp b/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp
index 8bc92a991a69..3568fe202caf 100644
--- a/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp
+++ b/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp
@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "flang/Optimizer/HLFIR/HLFIROps.h"
+
 #include "flang/Optimizer/Dialect/FIROpsSupport.h"
 #include "flang/Optimizer/Dialect/FIRType.h"
 #include "flang/Optimizer/Dialect/Support/FIRContext.h"
@@ -1152,7 +1153,9 @@ hlfir::MatmulOp::canonicalize(MatmulOp matmulOp,
 
       // but we do need to get rid of the hlfir.destroy for the hlfir.transpose
       // result (which is entirely removed)
-      for (mlir::Operation *user : transposeOp->getResult(0).getUsers())
+      llvm::SmallVector users(
+          transposeOp->getResult(0).getUsers());
+      for (mlir::Operation *user : users)
         if (auto destroyOp = mlir::dyn_cast_or_null(user))
           rewriter.eraseOp(destroyOp);
       rewriter.eraseOp(transposeOp);
@@ -1864,7 +1867,8 @@ hlfir::ForallIndexOp::canonicalize(hlfir::ForallIndexOp indexOp,
       return mlir::failure();
 
   auto insertPt = rewriter.saveInsertionPoint();
-  for (mlir::Operation *user : indexOp->getResult(0).getUsers())
+  llvm::SmallVector users(indexOp->getResult(0).getUsers());
+  for (mlir::Operation *user : users)
     if (auto loadOp = mlir::dyn_cast(user)) {
       rewriter.setInsertionPoint(loadOp);
       rewriter.replaceOpWithNewOp(
-- 
GitLab


From 65524fcb5d4e5f4f5450c38d3e075cce192586bb Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek 
Date: Fri, 8 Mar 2024 07:20:55 -0600
Subject: [PATCH 0027/1674] [flang][CodeGen] Replace correct op in
 BoxedProcedurePass (#84375)

---
 flang/lib/Optimizer/CodeGen/BoxedProcedure.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flang/lib/Optimizer/CodeGen/BoxedProcedure.cpp b/flang/lib/Optimizer/CodeGen/BoxedProcedure.cpp
index 846a78931acb..4cf39716a737 100644
--- a/flang/lib/Optimizer/CodeGen/BoxedProcedure.cpp
+++ b/flang/lib/Optimizer/CodeGen/BoxedProcedure.cpp
@@ -342,7 +342,7 @@ public:
             auto toTy = typeConverter.convertType(ty);
             auto toOnTy = typeConverter.convertType(onTy);
             rewriter.replaceOpWithNewOp(
-                mem, toTy, index.getFieldId(), toOnTy, index.getTypeparams());
+                index, toTy, index.getFieldId(), toOnTy, index.getTypeparams());
           }
         } else if (op->getDialect() == firDialect) {
           rewriter.startOpModification(op);
-- 
GitLab


From aa26faf05ecd7ee77487772fe0406f2de1df0dac Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek 
Date: Fri, 8 Mar 2024 07:21:21 -0600
Subject: [PATCH 0028/1674] [flang][unittests] Fix buffer overrun in
 FrontendActionTest (#84381)

When` SmallVector` is used as a backing storage, it can't be
assumed to end with a \x0. When creating a `StringRef` from it, pass the
length explicitly.

This was detected by address sanitizer.
---
 flang/unittests/Frontend/FrontendActionTest.cpp | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/flang/unittests/Frontend/FrontendActionTest.cpp b/flang/unittests/Frontend/FrontendActionTest.cpp
index 6ec15832d96d..123f428cc8b4 100644
--- a/flang/unittests/Frontend/FrontendActionTest.cpp
+++ b/flang/unittests/Frontend/FrontendActionTest.cpp
@@ -198,7 +198,7 @@ TEST_F(FrontendActionTest, EmitLLVM) {
   EXPECT_TRUE(success);
   EXPECT_TRUE(!outputFileBuffer.empty());
 
-  EXPECT_TRUE(llvm::StringRef(outputFileBuffer.data())
+  EXPECT_TRUE(llvm::StringRef(outputFileBuffer.begin(), outputFileBuffer.size())
                   .contains("define void @_QQmain()"));
 }
 
@@ -227,6 +227,7 @@ TEST_F(FrontendActionTest, EmitAsm) {
   EXPECT_TRUE(success);
   EXPECT_TRUE(!outputFileBuffer.empty());
 
-  EXPECT_TRUE(llvm::StringRef(outputFileBuffer.data()).contains("_QQmain"));
+  EXPECT_TRUE(llvm::StringRef(outputFileBuffer.begin(), outputFileBuffer.size())
+                  .contains("_QQmain"));
 }
 } // namespace
-- 
GitLab


From 141ebdd242f35bfebc576c51d8b60182ae4eac2d Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek 
Date: Fri, 8 Mar 2024 07:21:42 -0600
Subject: [PATCH 0029/1674] =?UTF-8?q?[Frontend][OpenMP]=20introduce=20OMP.?=
 =?UTF-8?q?h=20header=20file,=20use=20it=20instead=20of=20OMP=E2=80=A6=20(?=
 =?UTF-8?q?#84188)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

….h.inc

The consumers of OpenMP-related definitions include a TableGen-generated
file OMP.h.inc. Having a separate OMP.h allows putting additional
declarations in there that are not auto-generated.

This patch is NFC.
---
 flang/lib/Semantics/rewrite-directives.cpp     |  2 +-
 llvm/include/llvm/Frontend/OpenMP/OMP.h        | 18 ++++++++++++++++++
 .../llvm/Frontend/OpenMP/OMPConstants.h        |  3 +--
 llvm/lib/Frontend/OpenMP/OMP.cpp               |  2 +-
 4 files changed, 21 insertions(+), 4 deletions(-)
 create mode 100644 llvm/include/llvm/Frontend/OpenMP/OMP.h

diff --git a/flang/lib/Semantics/rewrite-directives.cpp b/flang/lib/Semantics/rewrite-directives.cpp
index bab91d253082..2c3c87f2546a 100644
--- a/flang/lib/Semantics/rewrite-directives.cpp
+++ b/flang/lib/Semantics/rewrite-directives.cpp
@@ -11,7 +11,7 @@
 #include "flang/Parser/parse-tree.h"
 #include "flang/Semantics/semantics.h"
 #include "flang/Semantics/symbol.h"
-#include "llvm/Frontend/OpenMP/OMP.h.inc"
+#include "llvm/Frontend/OpenMP/OMP.h"
 #include 
 
 namespace Fortran::semantics {
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.h b/llvm/include/llvm/Frontend/OpenMP/OMP.h
new file mode 100644
index 000000000000..a85cd9d344c6
--- /dev/null
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.h
@@ -0,0 +1,18 @@
+//===-- OMP.h - Core OpenMP definitions and declarations ---------- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains the core set of OpenMP definitions and declarations.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_FRONTEND_OPENMP_OMP_H
+#define LLVM_FRONTEND_OPENMP_OMP_H
+
+#include "llvm/Frontend/OpenMP/OMP.h.inc"
+
+#endif // LLVM_FRONTEND_OPENMP_OMP_H
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPConstants.h b/llvm/include/llvm/Frontend/OpenMP/OMPConstants.h
index f8812e7955b8..1bfaf3718552 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPConstants.h
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPConstants.h
@@ -15,9 +15,8 @@
 #define LLVM_FRONTEND_OPENMP_OMPCONSTANTS_H
 
 #include "llvm/ADT/BitmaskEnum.h"
-
 #include "llvm/ADT/StringRef.h"
-#include "llvm/Frontend/OpenMP/OMP.h.inc"
+#include "llvm/Frontend/OpenMP/OMP.h"
 
 namespace llvm {
 namespace omp {
diff --git a/llvm/lib/Frontend/OpenMP/OMP.cpp b/llvm/lib/Frontend/OpenMP/OMP.cpp
index eac5d4168c5c..4f2f95392648 100644
--- a/llvm/lib/Frontend/OpenMP/OMP.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMP.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/Frontend/OpenMP/OMP.h.inc"
+#include "llvm/Frontend/OpenMP/OMP.h"
 
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringSwitch.h"
-- 
GitLab


From 7415524b45392651969374c067041daa82dc89e7 Mon Sep 17 00:00:00 2001
From: Haojian Wu 
Date: Fri, 8 Mar 2024 14:24:03 +0100
Subject: [PATCH 0030/1674] [clang] Implement CTAD for type alias template.
 (#77890)

Fixes #54051

This patch implements the C++20 feature -- CTAD for alias templates (P1814R0, specified in https://eel.is/c++draft/over.match.class.deduct#3). It is an initial patch:
- it cover major pieces, thus it works for most cases;
- the big missing piece is to implement the associated constraints (over.match.class.deduct#3.3) for the synthesized deduction guides, see the FIXME in code and tests;
- Some enhancements on the TreeTransform&TemplateInstantiator to allow performing instantiation on `BuildingDeductionGuides` mode;
---
 clang/docs/ReleaseNotes.rst                   |   4 +
 .../clang/Basic/DiagnosticSemaKinds.td        |  10 +-
 clang/include/clang/Sema/Sema.h               |  17 +-
 clang/lib/Sema/SemaInit.cpp                   |  37 +-
 clang/lib/Sema/SemaTemplate.cpp               | 498 +++++++++++++++---
 clang/lib/Sema/SemaTemplateDeduction.cpp      |   9 +
 clang/lib/Sema/SemaTemplateInstantiate.cpp    |  69 ++-
 .../lib/Sema/SemaTemplateInstantiateDecl.cpp  |  29 +-
 clang/lib/Sema/TreeTransform.h                |   8 +
 clang/test/SemaCXX/cxx17-compat.cpp           |  11 +
 ...xx1z-class-template-argument-deduction.cpp |   6 +-
 clang/test/SemaCXX/cxx20-ctad-type-alias.cpp  | 232 ++++++++
 clang/www/cxx_status.html                     |   8 +-
 13 files changed, 827 insertions(+), 111 deletions(-)
 create mode 100644 clang/test/SemaCXX/cxx20-ctad-type-alias.cpp

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 73901a14631b..24b452bd487f 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -91,6 +91,10 @@ C++20 Feature Support
   current module units.
   Fixes `#84002 `_.
 
+- Initial support for class template argument deduction (CTAD) for type alias
+  templates (`P1814R0 `_).
+  (#GH54051).
+
 C++23 Feature Support
 ^^^^^^^^^^^^^^^^^^^^^
 
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index a80c8b75e863..6da49facd27e 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -2515,10 +2515,11 @@ def err_deduced_class_template_compound_type : Error<
   "cannot %select{form pointer to|form reference to|form array of|"
   "form function returning|use parentheses when declaring variable with}0 "
   "deduced class template specialization type">;
-def err_deduced_non_class_template_specialization_type : Error<
+def err_deduced_non_class_or_alias_template_specialization_type : Error<
   "%select{|function template|variable template|alias template|"
   "template template parameter|concept|template}0 %1 requires template "
-  "arguments; argument deduction only allowed for class templates">;
+  "arguments; argument deduction only allowed for class templates or alias "
+  "templates">;
 def err_deduced_class_template_ctor_ambiguous : Error<
   "ambiguous deduction for template arguments of %0">;
 def err_deduced_class_template_ctor_no_viable : Error<
@@ -8201,6 +8202,11 @@ let CategoryName = "Lambda Issue" in {
   def warn_cxx17_compat_lambda_def_ctor_assign : Warning<
     "%select{default construction|assignment}0 of lambda is incompatible with "
     "C++ standards before C++20">, InGroup, DefaultIgnore;
+
+  // C++20 class template argument deduction for alias templates.
+  def warn_cxx17_compat_ctad_for_alias_templates : Warning<
+  "class template argument deduction for alias templates is incompatible with "
+  "C++ standards before C++20">, InGroup, DefaultIgnore;
 }
 
 def err_return_in_captured_stmt : Error<
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 2cac7e6ca08f..5b4d67494f2a 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -9827,6 +9827,12 @@ public:
                           ArrayRef TemplateArgs,
                           sema::TemplateDeductionInfo &Info);
 
+  TemplateDeductionResult DeduceTemplateArguments(
+      TemplateParameterList *TemplateParams, ArrayRef Ps,
+      ArrayRef As, sema::TemplateDeductionInfo &Info,
+      SmallVectorImpl &Deduced,
+      bool NumberOfArgumentsMustMatch);
+
   TemplateDeductionResult SubstituteExplicitTemplateArguments(
       FunctionTemplateDecl *FunctionTemplate,
       TemplateArgumentListInfo &ExplicitTemplateArgs,
@@ -10378,6 +10384,9 @@ public:
     InstantiatingTemplate &operator=(const InstantiatingTemplate &) = delete;
   };
 
+  bool SubstTemplateArgument(const TemplateArgumentLoc &Input,
+                             const MultiLevelTemplateArgumentList &TemplateArgs,
+                             TemplateArgumentLoc &Output);
   bool
   SubstTemplateArguments(ArrayRef Args,
                          const MultiLevelTemplateArgumentList &TemplateArgs,
@@ -10862,9 +10871,11 @@ public:
                                   ParmVarDecl *Param);
   void InstantiateExceptionSpec(SourceLocation PointOfInstantiation,
                                 FunctionDecl *Function);
-  FunctionDecl *InstantiateFunctionDeclaration(FunctionTemplateDecl *FTD,
-                                               const TemplateArgumentList *Args,
-                                               SourceLocation Loc);
+  FunctionDecl *InstantiateFunctionDeclaration(
+      FunctionTemplateDecl *FTD, const TemplateArgumentList *Args,
+      SourceLocation Loc,
+      CodeSynthesisContext::SynthesisKind CSC =
+          CodeSynthesisContext::ExplicitTemplateArgumentSubstitution);
   void InstantiateFunctionDefinition(SourceLocation PointOfInstantiation,
                                      FunctionDecl *Function,
                                      bool Recursive = false,
diff --git a/clang/lib/Sema/SemaInit.cpp b/clang/lib/Sema/SemaInit.cpp
index 011deed7a9a9..aa470adb30b4 100644
--- a/clang/lib/Sema/SemaInit.cpp
+++ b/clang/lib/Sema/SemaInit.cpp
@@ -10720,13 +10720,40 @@ QualType Sema::DeduceTemplateSpecializationFromInitializer(
   if (TemplateName.isDependent())
     return SubstAutoTypeDependent(TSInfo->getType());
 
-  // We can only perform deduction for class templates.
+  // We can only perform deduction for class templates or alias templates.
   auto *Template =
       dyn_cast_or_null(TemplateName.getAsTemplateDecl());
+  TemplateDecl *LookupTemplateDecl = Template;
+  if (!Template) {
+    if (auto *AliasTemplate = dyn_cast_or_null(
+            TemplateName.getAsTemplateDecl())) {
+      Diag(Kind.getLocation(),
+           diag::warn_cxx17_compat_ctad_for_alias_templates);
+      LookupTemplateDecl = AliasTemplate;
+      auto UnderlyingType = AliasTemplate->getTemplatedDecl()
+                                ->getUnderlyingType()
+                                .getCanonicalType();
+      // C++ [over.match.class.deduct#3]: ..., the defining-type-id of A must be
+      // of the form
+      //   [typename] [nested-name-specifier] [template] simple-template-id
+      if (const auto *TST =
+              UnderlyingType->getAs()) {
+        Template = dyn_cast_or_null(
+            TST->getTemplateName().getAsTemplateDecl());
+      } else if (const auto *RT = UnderlyingType->getAs()) {
+        // Cases where template arguments in the RHS of the alias are not
+        // dependent. e.g.
+        //   using AliasFoo = Foo;
+        if (const auto *CTSD = llvm::dyn_cast(
+                RT->getAsCXXRecordDecl()))
+          Template = CTSD->getSpecializedTemplate();
+      }
+    }
+  }
   if (!Template) {
     Diag(Kind.getLocation(),
-         diag::err_deduced_non_class_template_specialization_type)
-      << (int)getTemplateNameKindForDiagnostics(TemplateName) << TemplateName;
+         diag::err_deduced_non_class_or_alias_template_specialization_type)
+        << (int)getTemplateNameKindForDiagnostics(TemplateName) << TemplateName;
     if (auto *TD = TemplateName.getAsTemplateDecl())
       NoteTemplateLocation(*TD);
     return QualType();
@@ -10753,10 +10780,10 @@ QualType Sema::DeduceTemplateSpecializationFromInitializer(
   //     template-name, a function template [...]
   //  - For each deduction-guide, a function or function template [...]
   DeclarationNameInfo NameInfo(
-      Context.DeclarationNames.getCXXDeductionGuideName(Template),
+      Context.DeclarationNames.getCXXDeductionGuideName(LookupTemplateDecl),
       TSInfo->getTypeLoc().getEndLoc());
   LookupResult Guides(*this, NameInfo, LookupOrdinaryName);
-  LookupQualifiedName(Guides, Template->getDeclContext());
+  LookupQualifiedName(Guides, LookupTemplateDecl->getDeclContext());
 
   // FIXME: Do not diagnose inaccessible deduction guides. The standard isn't
   // clear on this, but they're not found by name so access does not apply.
diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp
index 83eee83aa6ca..d62095558d0f 100644
--- a/clang/lib/Sema/SemaTemplate.cpp
+++ b/clang/lib/Sema/SemaTemplate.cpp
@@ -2262,6 +2262,92 @@ public:
   }
 };
 
+// Build a deduction guide with the specified parameter types.
+FunctionTemplateDecl *buildDeductionGuide(
+    Sema &SemaRef, TemplateDecl *OriginalTemplate,
+    TemplateParameterList *TemplateParams, CXXConstructorDecl *Ctor,
+    ExplicitSpecifier ES, TypeSourceInfo *TInfo, SourceLocation LocStart,
+    SourceLocation Loc, SourceLocation LocEnd, bool IsImplicit,
+    llvm::ArrayRef MaterializedTypedefs = {}) {
+  DeclContext *DC = OriginalTemplate->getDeclContext();
+  auto DeductionGuideName =
+      SemaRef.Context.DeclarationNames.getCXXDeductionGuideName(
+          OriginalTemplate);
+
+  DeclarationNameInfo Name(DeductionGuideName, Loc);
+  ArrayRef Params =
+      TInfo->getTypeLoc().castAs().getParams();
+
+  // Build the implicit deduction guide template.
+  auto *Guide =
+      CXXDeductionGuideDecl::Create(SemaRef.Context, DC, LocStart, ES, Name,
+                                    TInfo->getType(), TInfo, LocEnd, Ctor);
+  Guide->setImplicit(IsImplicit);
+  Guide->setParams(Params);
+
+  for (auto *Param : Params)
+    Param->setDeclContext(Guide);
+  for (auto *TD : MaterializedTypedefs)
+    TD->setDeclContext(Guide);
+
+  auto *GuideTemplate = FunctionTemplateDecl::Create(
+      SemaRef.Context, DC, Loc, DeductionGuideName, TemplateParams, Guide);
+  GuideTemplate->setImplicit(IsImplicit);
+  Guide->setDescribedFunctionTemplate(GuideTemplate);
+
+  if (isa(DC)) {
+    Guide->setAccess(AS_public);
+    GuideTemplate->setAccess(AS_public);
+  }
+
+  DC->addDecl(GuideTemplate);
+  return GuideTemplate;
+}
+
+// Transform a given template type parameter `TTP`.
+TemplateTypeParmDecl *
+transformTemplateTypeParam(Sema &SemaRef, DeclContext *DC,
+                           TemplateTypeParmDecl *TTP,
+                           MultiLevelTemplateArgumentList &Args,
+                           unsigned NewDepth, unsigned NewIndex) {
+  // TemplateTypeParmDecl's index cannot be changed after creation, so
+  // substitute it directly.
+  auto *NewTTP = TemplateTypeParmDecl::Create(
+      SemaRef.Context, DC, TTP->getBeginLoc(), TTP->getLocation(), NewDepth,
+      NewIndex, TTP->getIdentifier(), TTP->wasDeclaredWithTypename(),
+      TTP->isParameterPack(), TTP->hasTypeConstraint(),
+      TTP->isExpandedParameterPack()
+          ? std::optional(TTP->getNumExpansionParameters())
+          : std::nullopt);
+  if (const auto *TC = TTP->getTypeConstraint())
+    SemaRef.SubstTypeConstraint(NewTTP, TC, Args,
+                                /*EvaluateConstraint=*/true);
+  if (TTP->hasDefaultArgument()) {
+    TypeSourceInfo *InstantiatedDefaultArg =
+        SemaRef.SubstType(TTP->getDefaultArgumentInfo(), Args,
+                          TTP->getDefaultArgumentLoc(), TTP->getDeclName());
+    if (InstantiatedDefaultArg)
+      NewTTP->setDefaultArgument(InstantiatedDefaultArg);
+  }
+  SemaRef.CurrentInstantiationScope->InstantiatedLocal(TTP, NewTTP);
+  return NewTTP;
+}
+// Similar to above, but for non-type template or template template parameters.
+template 
+NonTypeTemplateOrTemplateTemplateParmDecl *
+transformTemplateParam(Sema &SemaRef, DeclContext *DC,
+                       NonTypeTemplateOrTemplateTemplateParmDecl *OldParam,
+                       MultiLevelTemplateArgumentList &Args, unsigned NewIndex,
+                       unsigned NewDepth) {
+  // Ask the template instantiator to do the heavy lifting for us, then adjust
+  // the index of the parameter once it's done.
+  auto *NewParam = cast(
+      SemaRef.SubstDecl(OldParam, DC, Args));
+  NewParam->setPosition(NewIndex);
+  NewParam->setDepth(NewDepth);
+  return NewParam;
+}
+
 /// Transform to convert portions of a constructor declaration into the
 /// corresponding deduction guide, per C++1z [over.match.class.deduct]p1.
 struct ConvertConstructorToDeductionGuideTransform {
@@ -2338,7 +2424,6 @@ struct ConvertConstructorToDeductionGuideTransform {
         NamedDecl *NewParam = transformTemplateParameter(Param, Args);
         if (!NewParam)
           return nullptr;
-
         // Constraints require that we substitute depth-1 arguments
         // to match depths when substituted for evaluation later
         Depth1Args.push_back(SemaRef.Context.getCanonicalTemplateArgument(
@@ -2411,9 +2496,10 @@ struct ConvertConstructorToDeductionGuideTransform {
       return nullptr;
     TypeSourceInfo *NewTInfo = TLB.getTypeSourceInfo(SemaRef.Context, NewType);
 
-    return buildDeductionGuide(TemplateParams, CD, CD->getExplicitSpecifier(),
-                               NewTInfo, CD->getBeginLoc(), CD->getLocation(),
-                               CD->getEndLoc(), MaterializedTypedefs);
+    return buildDeductionGuide(
+        SemaRef, Template, TemplateParams, CD, CD->getExplicitSpecifier(),
+        NewTInfo, CD->getBeginLoc(), CD->getLocation(), CD->getEndLoc(),
+        /*IsImplicit=*/true, MaterializedTypedefs);
   }
 
   /// Build a deduction guide with the specified parameter types.
@@ -2448,8 +2534,9 @@ struct ConvertConstructorToDeductionGuideTransform {
       Params.push_back(NewParam);
     }
 
-    return buildDeductionGuide(GetTemplateParameterList(Template), nullptr,
-                               ExplicitSpecifier(), TSI, Loc, Loc, Loc);
+    return buildDeductionGuide(
+        SemaRef, Template, GetTemplateParameterList(Template), nullptr,
+        ExplicitSpecifier(), TSI, Loc, Loc, Loc, /*IsImplicit=*/true);
   }
 
 private:
@@ -2458,50 +2545,18 @@ private:
   /// renumbering as we go.
   NamedDecl *transformTemplateParameter(NamedDecl *TemplateParam,
                                         MultiLevelTemplateArgumentList &Args) {
-    if (auto *TTP = dyn_cast(TemplateParam)) {
-      // TemplateTypeParmDecl's index cannot be changed after creation, so
-      // substitute it directly.
-      auto *NewTTP = TemplateTypeParmDecl::Create(
-          SemaRef.Context, DC, TTP->getBeginLoc(), TTP->getLocation(),
-          TTP->getDepth() - 1, Depth1IndexAdjustment + TTP->getIndex(),
-          TTP->getIdentifier(), TTP->wasDeclaredWithTypename(),
-          TTP->isParameterPack(), TTP->hasTypeConstraint(),
-          TTP->isExpandedParameterPack()
-              ? std::optional(TTP->getNumExpansionParameters())
-              : std::nullopt);
-      if (const auto *TC = TTP->getTypeConstraint())
-        SemaRef.SubstTypeConstraint(NewTTP, TC, Args,
-                                    /*EvaluateConstraint*/ true);
-      if (TTP->hasDefaultArgument()) {
-        TypeSourceInfo *InstantiatedDefaultArg =
-            SemaRef.SubstType(TTP->getDefaultArgumentInfo(), Args,
-                              TTP->getDefaultArgumentLoc(), TTP->getDeclName());
-        if (InstantiatedDefaultArg)
-          NewTTP->setDefaultArgument(InstantiatedDefaultArg);
-      }
-      SemaRef.CurrentInstantiationScope->InstantiatedLocal(TemplateParam,
-                                                           NewTTP);
-      return NewTTP;
-    }
-
+    if (auto *TTP = dyn_cast(TemplateParam))
+      return transformTemplateTypeParam(
+          SemaRef, DC, TTP, Args, TTP->getDepth() - 1,
+          Depth1IndexAdjustment + TTP->getIndex());
     if (auto *TTP = dyn_cast(TemplateParam))
-      return transformTemplateParameterImpl(TTP, Args);
-
-    return transformTemplateParameterImpl(
-        cast(TemplateParam), Args);
-  }
-  template
-  TemplateParmDecl *
-  transformTemplateParameterImpl(TemplateParmDecl *OldParam,
-                                 MultiLevelTemplateArgumentList &Args) {
-    // Ask the template instantiator to do the heavy lifting for us, then adjust
-    // the index of the parameter once it's done.
-    auto *NewParam =
-        cast(SemaRef.SubstDecl(OldParam, DC, Args));
-    assert(NewParam->getDepth() == OldParam->getDepth() - 1 &&
-           "unexpected template param depth");
-    NewParam->setPosition(NewParam->getPosition() + Depth1IndexAdjustment);
-    return NewParam;
+      return transformTemplateParam(SemaRef, DC, TTP, Args,
+                                    Depth1IndexAdjustment + TTP->getIndex(),
+                                    TTP->getDepth() - 1);
+    auto *NTTP = cast(TemplateParam);
+    return transformTemplateParam(SemaRef, DC, NTTP, Args,
+                                  Depth1IndexAdjustment + NTTP->getIndex(),
+                                  NTTP->getDepth() - 1);
   }
 
   QualType transformFunctionProtoType(
@@ -2596,7 +2651,7 @@ private:
       // placeholder to indicate there is a default argument.
       QualType ParamTy = NewDI->getType();
       NewDefArg = new (SemaRef.Context)
-          OpaqueValueExpr(OldParam->getDefaultArgRange().getBegin(),
+          OpaqueValueExpr(OldParam->getDefaultArg()->getBeginLoc(),
                           ParamTy.getNonLValueExprType(SemaRef.Context),
                           ParamTy->isLValueReferenceType()   ? VK_LValue
                           : ParamTy->isRValueReferenceType() ? VK_XValue
@@ -2616,44 +2671,310 @@ private:
     SemaRef.CurrentInstantiationScope->InstantiatedLocal(OldParam, NewParam);
     return NewParam;
   }
+};
+
+// Find all template parameters that appear in the given DeducedArgs.
+// Return the indices of the template parameters in the TemplateParams.
+SmallVector TemplateParamsReferencedInTemplateArgumentList(
+    ArrayRef TemplateParams,
+    ArrayRef DeducedArgs) {
+  struct TemplateParamsReferencedFinder
+      : public RecursiveASTVisitor {
+    llvm::DenseSet TemplateParams;
+    llvm::DenseSet ReferencedTemplateParams;
+
+    TemplateParamsReferencedFinder(ArrayRef TemplateParams)
+        : TemplateParams(TemplateParams.begin(), TemplateParams.end()) {}
+
+    bool VisitTemplateTypeParmType(TemplateTypeParmType *TTP) {
+      TTP->getIndex();
+      MarkAppeared(TTP->getDecl());
+      return true;
+    }
+    bool VisitDeclRefExpr(DeclRefExpr *DRE) {
+      MarkAppeared(DRE->getFoundDecl());
+      return true;
+    }
+
+    void MarkAppeared(NamedDecl *ND) {
+      if (TemplateParams.contains(ND))
+        ReferencedTemplateParams.insert(ND);
+    }
+  };
+  TemplateParamsReferencedFinder Finder(TemplateParams);
+  Finder.TraverseTemplateArguments(DeducedArgs);
 
-  FunctionTemplateDecl *buildDeductionGuide(
-      TemplateParameterList *TemplateParams, CXXConstructorDecl *Ctor,
-      ExplicitSpecifier ES, TypeSourceInfo *TInfo, SourceLocation LocStart,
-      SourceLocation Loc, SourceLocation LocEnd,
-      llvm::ArrayRef MaterializedTypedefs = {}) {
-    DeclarationNameInfo Name(DeductionGuideName, Loc);
-    ArrayRef Params =
-        TInfo->getTypeLoc().castAs().getParams();
+  SmallVector Results;
+  for (unsigned Index = 0; Index < TemplateParams.size(); ++Index) {
+    if (Finder.ReferencedTemplateParams.contains(TemplateParams[Index]))
+      Results.push_back(Index);
+  }
+  return Results;
+}
 
-    // Build the implicit deduction guide template.
-    auto *Guide =
-        CXXDeductionGuideDecl::Create(SemaRef.Context, DC, LocStart, ES, Name,
-                                      TInfo->getType(), TInfo, LocEnd, Ctor);
-    Guide->setImplicit();
-    Guide->setParams(Params);
+bool hasDeclaredDeductionGuides(DeclarationName Name, DeclContext *DC) {
+  // Check whether we've already declared deduction guides for this template.
+  // FIXME: Consider storing a flag on the template to indicate this.
+  assert(Name.getNameKind() ==
+             DeclarationName::NameKind::CXXDeductionGuideName &&
+         "name must be a deduction guide name");
+  auto Existing = DC->lookup(Name);
+  for (auto *D : Existing)
+    if (D->isImplicit())
+      return true;
+  return false;
+}
 
-    for (auto *Param : Params)
-      Param->setDeclContext(Guide);
-    for (auto *TD : MaterializedTypedefs)
-      TD->setDeclContext(Guide);
+// Build deduction guides for a type alias template.
+void DeclareImplicitDeductionGuidesForTypeAlias(
+    Sema &SemaRef, TypeAliasTemplateDecl *AliasTemplate, SourceLocation Loc) {
+  auto &Context = SemaRef.Context;
+  // FIXME: if there is an explicit deduction guide after the first use of the
+  // type alias usage, we will not cover this explicit deduction guide. fix this
+  // case.
+  if (hasDeclaredDeductionGuides(
+          Context.DeclarationNames.getCXXDeductionGuideName(AliasTemplate),
+          AliasTemplate->getDeclContext()))
+    return;
+  // Unwrap the sugared ElaboratedType.
+  auto RhsType = AliasTemplate->getTemplatedDecl()
+                     ->getUnderlyingType()
+                     .getSingleStepDesugaredType(Context);
+  TemplateDecl *Template = nullptr;
+  llvm::ArrayRef AliasRhsTemplateArgs;
+  if (const auto *TST = RhsType->getAs()) {
+    // Cases where the RHS of the alias is dependent. e.g.
+    //   template
+    //   using AliasFoo1 = Foo; // a class/type alias template specialization
+    Template = TST->getTemplateName().getAsTemplateDecl();
+    AliasRhsTemplateArgs = TST->template_arguments();
+  } else if (const auto *RT = RhsType->getAs()) {
+    // Cases where template arguments in the RHS of the alias are not
+    // dependent. e.g.
+    //   using AliasFoo = Foo;
+    if (const auto *CTSD = llvm::dyn_cast(
+            RT->getAsCXXRecordDecl())) {
+      Template = CTSD->getSpecializedTemplate();
+      AliasRhsTemplateArgs = CTSD->getTemplateArgs().asArray();
+    }
+  } else {
+    assert(false && "unhandled RHS type of the alias");
+  }
+  if (!Template)
+    return;
+  DeclarationNameInfo NameInfo(
+      Context.DeclarationNames.getCXXDeductionGuideName(Template), Loc);
+  LookupResult Guides(SemaRef, NameInfo, clang::Sema::LookupOrdinaryName);
+  SemaRef.LookupQualifiedName(Guides, Template->getDeclContext());
+  Guides.suppressDiagnostics();
+
+  for (auto *G : Guides) {
+    FunctionTemplateDecl *F = dyn_cast(G);
+    if (!F)
+      continue;
+    auto RType = F->getTemplatedDecl()->getReturnType();
+    // The (trailing) return type of the deduction guide.
+    const TemplateSpecializationType *FReturnType =
+        RType->getAs();
+    if (const auto *InjectedCNT = RType->getAs())
+      // implicitly-generated deduction guide.
+      FReturnType = InjectedCNT->getInjectedTST();
+    else if (const auto *ET = RType->getAs())
+      // explicit deduction guide.
+      FReturnType = ET->getNamedType()->getAs();
+    assert(FReturnType && "expected to see a return type");
+    // Deduce template arguments of the deduction guide f from the RHS of
+    // the alias.
+    //
+    // C++ [over.match.class.deduct]p3: ...For each function or function
+    // template f in the guides of the template named by the
+    // simple-template-id of the defining-type-id, the template arguments
+    // of the return type of f are deduced from the defining-type-id of A
+    // according to the process in [temp.deduct.type] with the exception
+    // that deduction does not fail if not all template arguments are
+    // deduced.
+    //
+    //
+    //  template
+    //  f(X, Y) -> f;
+    //
+    //  template
+    //  using alias = f;
+    //
+    // The RHS of alias is f, we deduced the template arguments of
+    // the return type of the deduction guide from it: Y->int, X->U
+    sema::TemplateDeductionInfo TDeduceInfo(Loc);
+    // Must initialize n elements, this is required by DeduceTemplateArguments.
+    SmallVector DeduceResults(
+        F->getTemplateParameters()->size());
+
+    // FIXME: DeduceTemplateArguments stops immediately at the first
+    // non-deducible template argument. However, this doesn't seem to casue
+    // issues for practice cases, we probably need to extend it to continue
+    // performing deduction for rest of arguments to align with the C++
+    // standard.
+    SemaRef.DeduceTemplateArguments(
+        F->getTemplateParameters(), FReturnType->template_arguments(),
+        AliasRhsTemplateArgs, TDeduceInfo, DeduceResults,
+        /*NumberOfArgumentsMustMatch=*/false);
+
+    SmallVector DeducedArgs;
+    SmallVector NonDeducedTemplateParamsInFIndex;
+    // !!NOTE: DeduceResults respects the sequence of template parameters of
+    // the deduction guide f.
+    for (unsigned Index = 0; Index < DeduceResults.size(); ++Index) {
+      if (const auto &D = DeduceResults[Index]; !D.isNull()) // Deduced
+        DeducedArgs.push_back(D);
+      else
+        NonDeducedTemplateParamsInFIndex.push_back(Index);
+    }
+    auto DeducedAliasTemplateParams =
+        TemplateParamsReferencedInTemplateArgumentList(
+            AliasTemplate->getTemplateParameters()->asArray(), DeducedArgs);
+    // All template arguments null by default.
+    SmallVector TemplateArgsForBuildingFPrime(
+        F->getTemplateParameters()->size());
+
+    Sema::InstantiatingTemplate BuildingDeductionGuides(
+        SemaRef, AliasTemplate->getLocation(), F,
+        Sema::InstantiatingTemplate::BuildingDeductionGuidesTag{});
+    if (BuildingDeductionGuides.isInvalid())
+      return;
+    LocalInstantiationScope Scope(SemaRef);
 
-    auto *GuideTemplate = FunctionTemplateDecl::Create(
-        SemaRef.Context, DC, Loc, DeductionGuideName, TemplateParams, Guide);
-    GuideTemplate->setImplicit();
-    Guide->setDescribedFunctionTemplate(GuideTemplate);
+    // Create a template parameter list for the synthesized deduction guide f'.
+    //
+    // C++ [over.match.class.deduct]p3.2:
+    //   If f is a function template, f' is a function template whose template
+    //   parameter list consists of all the template parameters of A
+    //   (including their default template arguments) that appear in the above
+    //   deductions or (recursively) in their default template arguments
+    SmallVector FPrimeTemplateParams;
+    // Store template arguments that refer to the newly-created template
+    // parameters, used for building `TemplateArgsForBuildingFPrime`.
+    SmallVector TransformedDeducedAliasArgs(
+        AliasTemplate->getTemplateParameters()->size());
+    auto TransformTemplateParameter =
+        [&SemaRef](DeclContext *DC, NamedDecl *TemplateParam,
+                   MultiLevelTemplateArgumentList &Args,
+                   unsigned NewIndex) -> NamedDecl * {
+      if (auto *TTP = dyn_cast(TemplateParam))
+        return transformTemplateTypeParam(SemaRef, DC, TTP, Args,
+                                          TTP->getDepth(), NewIndex);
+      if (auto *TTP = dyn_cast(TemplateParam))
+        return transformTemplateParam(SemaRef, DC, TTP, Args, NewIndex,
+                                      TTP->getDepth());
+      if (auto *NTTP = dyn_cast(TemplateParam))
+        return transformTemplateParam(SemaRef, DC, NTTP, Args, NewIndex,
+                                      NTTP->getDepth());
+      return nullptr;
+    };
 
-    if (isa(DC)) {
-      Guide->setAccess(AS_public);
-      GuideTemplate->setAccess(AS_public);
+    for (unsigned AliasTemplateParamIdx : DeducedAliasTemplateParams) {
+      auto *TP = AliasTemplate->getTemplateParameters()->getParam(
+          AliasTemplateParamIdx);
+      // Rebuild any internal references to earlier parameters and reindex as
+      // we go.
+      MultiLevelTemplateArgumentList Args;
+      Args.setKind(TemplateSubstitutionKind::Rewrite);
+      Args.addOuterTemplateArguments(TransformedDeducedAliasArgs);
+      NamedDecl *NewParam =
+          TransformTemplateParameter(AliasTemplate->getDeclContext(), TP, Args,
+                                     /*NewIndex*/ FPrimeTemplateParams.size());
+      FPrimeTemplateParams.push_back(NewParam);
+
+      auto NewTemplateArgument = Context.getCanonicalTemplateArgument(
+          Context.getInjectedTemplateArg(NewParam));
+      TransformedDeducedAliasArgs[AliasTemplateParamIdx] = NewTemplateArgument;
+    }
+    //   ...followed by the template parameters of f that were not deduced
+    //   (including their default template arguments)
+    for (unsigned FTemplateParamIdx : NonDeducedTemplateParamsInFIndex) {
+      auto *TP = F->getTemplateParameters()->getParam(FTemplateParamIdx);
+      MultiLevelTemplateArgumentList Args;
+      Args.setKind(TemplateSubstitutionKind::Rewrite);
+      // We take a shortcut here, it is ok to reuse the
+      // TemplateArgsForBuildingFPrime.
+      Args.addOuterTemplateArguments(TemplateArgsForBuildingFPrime);
+      NamedDecl *NewParam = TransformTemplateParameter(
+          F->getDeclContext(), TP, Args, FPrimeTemplateParams.size());
+      FPrimeTemplateParams.push_back(NewParam);
+
+      assert(TemplateArgsForBuildingFPrime[FTemplateParamIdx].isNull() &&
+             "The argument must be null before setting");
+      TemplateArgsForBuildingFPrime[FTemplateParamIdx] =
+          Context.getCanonicalTemplateArgument(
+              Context.getInjectedTemplateArg(NewParam));
+    }
+    // FIXME: implement the associated constraint per C++
+    // [over.match.class.deduct]p3.3:
+    //    The associated constraints ([temp.constr.decl]) are the
+    //    conjunction of the associated constraints of g and a
+    //    constraint that is satisfied if and only if the arguments
+    //    of A are deducible (see below) from the return type.
+    auto *FPrimeTemplateParamList = TemplateParameterList::Create(
+        Context, AliasTemplate->getTemplateParameters()->getTemplateLoc(),
+        AliasTemplate->getTemplateParameters()->getLAngleLoc(),
+        FPrimeTemplateParams,
+        AliasTemplate->getTemplateParameters()->getRAngleLoc(),
+        /*RequiresClause=*/nullptr);
+
+    // To form a deduction guide f' from f, we leverage clang's instantiation
+    // mechanism, we construct a template argument list where the template
+    // arguments refer to the newly-created template parameters of f', and
+    // then apply instantiation on this template argument list to instantiate
+    // f, this ensures all template parameter occurrences are updated
+    // correctly.
+    //
+    // The template argument list is formed from the `DeducedArgs`, two parts:
+    //  1) appeared template parameters of alias: transfrom the deduced
+    //  template argument;
+    //  2) non-deduced template parameters of f: rebuild a
+    //  template argument;
+    //
+    // 2) has been built already (when rebuilding the new template
+    // parameters), we now perform 1).
+    MultiLevelTemplateArgumentList Args;
+    Args.setKind(TemplateSubstitutionKind::Rewrite);
+    Args.addOuterTemplateArguments(TransformedDeducedAliasArgs);
+    for (unsigned Index = 0; Index < DeduceResults.size(); ++Index) {
+      const auto &D = DeduceResults[Index];
+      if (D.isNull()) {
+        // 2): Non-deduced template parameter has been built already.
+        assert(!TemplateArgsForBuildingFPrime[Index].isNull() &&
+               "template arguments for non-deduced template parameters should "
+               "be been set!");
+        continue;
+      }
+      TemplateArgumentLoc Input = SemaRef.getTrivialTemplateArgumentLoc(
+          D, QualType(), SourceLocation{});
+      TemplateArgumentLoc Output;
+      if (!SemaRef.SubstTemplateArgument(Input, Args, Output)) {
+        assert(TemplateArgsForBuildingFPrime[Index].isNull() &&
+               "InstantiatedArgs must be null before setting");
+        TemplateArgsForBuildingFPrime[Index] = (Output.getArgument());
+      }
     }
 
-    DC->addDecl(GuideTemplate);
-    return GuideTemplate;
+    auto *TemplateArgListForBuildingFPrime = TemplateArgumentList::CreateCopy(
+        Context, TemplateArgsForBuildingFPrime);
+    // Form the f' by substituting the template arguments into f.
+    if (auto *FPrime = SemaRef.InstantiateFunctionDeclaration(
+            F, TemplateArgListForBuildingFPrime, AliasTemplate->getLocation(),
+            Sema::CodeSynthesisContext::BuildingDeductionGuides)) {
+      auto *GG = dyn_cast(FPrime);
+      buildDeductionGuide(SemaRef, AliasTemplate, FPrimeTemplateParamList,
+                          GG->getCorrespondingConstructor(),
+                          GG->getExplicitSpecifier(), GG->getTypeSourceInfo(),
+                          AliasTemplate->getBeginLoc(),
+                          AliasTemplate->getLocation(),
+                          AliasTemplate->getEndLoc(), F->isImplicit());
+    }
   }
-};
 }
 
+} // namespace
+
 FunctionTemplateDecl *Sema::DeclareImplicitDeductionGuideFromInitList(
     TemplateDecl *Template, MutableArrayRef ParamTypes,
     SourceLocation Loc) {
@@ -2697,6 +3018,10 @@ FunctionTemplateDecl *Sema::DeclareImplicitDeductionGuideFromInitList(
 
 void Sema::DeclareImplicitDeductionGuides(TemplateDecl *Template,
                                           SourceLocation Loc) {
+  if (auto *AliasTemplate = llvm::dyn_cast(Template)) {
+    DeclareImplicitDeductionGuidesForTypeAlias(*this, AliasTemplate, Loc);
+    return;
+  }
   if (CXXRecordDecl *DefRecord =
           cast(Template->getTemplatedDecl())->getDefinition()) {
     if (TemplateDecl *DescribedTemplate = DefRecord->getDescribedClassTemplate())
@@ -2712,12 +3037,8 @@ void Sema::DeclareImplicitDeductionGuides(TemplateDecl *Template,
   if (!isCompleteType(Loc, Transform.DeducedType))
     return;
 
-  // Check whether we've already declared deduction guides for this template.
-  // FIXME: Consider storing a flag on the template to indicate this.
-  auto Existing = DC->lookup(Transform.DeductionGuideName);
-  for (auto *D : Existing)
-    if (D->isImplicit())
-      return;
+  if (hasDeclaredDeductionGuides(Transform.DeductionGuideName, DC))
+    return;
 
   // In case we were expanding a pack when we attempted to declare deduction
   // guides, turn off pack expansion for everything we're about to do.
@@ -5368,6 +5689,15 @@ bool Sema::CheckTemplateTypeArgument(
     [[fallthrough]];
   }
   default: {
+    // We allow instantiateing a template with template argument packs when
+    // building deduction guides.
+    if (Arg.getKind() == TemplateArgument::Pack &&
+        CodeSynthesisContexts.back().Kind ==
+            Sema::CodeSynthesisContext::BuildingDeductionGuides) {
+      SugaredConverted.push_back(Arg);
+      CanonicalConverted.push_back(Arg);
+      return false;
+    }
     // We have a template type parameter but the template argument
     // is not a type.
     SourceRange SR = AL.getSourceRange();
diff --git a/clang/lib/Sema/SemaTemplateDeduction.cpp b/clang/lib/Sema/SemaTemplateDeduction.cpp
index 65f7fa15b20d..97f8445bf819 100644
--- a/clang/lib/Sema/SemaTemplateDeduction.cpp
+++ b/clang/lib/Sema/SemaTemplateDeduction.cpp
@@ -2531,6 +2531,15 @@ DeduceTemplateArguments(Sema &S, TemplateParameterList *TemplateParams,
   return TemplateDeductionResult::Success;
 }
 
+TemplateDeductionResult Sema::DeduceTemplateArguments(
+    TemplateParameterList *TemplateParams, ArrayRef Ps,
+    ArrayRef As, sema::TemplateDeductionInfo &Info,
+    SmallVectorImpl &Deduced,
+    bool NumberOfArgumentsMustMatch) {
+  return ::DeduceTemplateArguments(*this, TemplateParams, Ps, As, Info, Deduced,
+                                   NumberOfArgumentsMustMatch);
+}
+
 /// Determine whether two template arguments are the same.
 static bool isSameTemplateArg(ASTContext &Context,
                               TemplateArgument X,
diff --git a/clang/lib/Sema/SemaTemplateInstantiate.cpp b/clang/lib/Sema/SemaTemplateInstantiate.cpp
index 371378485626..d9994d7fd37a 100644
--- a/clang/lib/Sema/SemaTemplateInstantiate.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiate.cpp
@@ -21,6 +21,7 @@
 #include "clang/AST/ExprConcepts.h"
 #include "clang/AST/PrettyDeclStackTrace.h"
 #include "clang/AST/Type.h"
+#include "clang/AST/TypeLoc.h"
 #include "clang/AST/TypeVisitor.h"
 #include "clang/Basic/LangOptions.h"
 #include "clang/Basic/Stack.h"
@@ -547,9 +548,9 @@ Sema::InstantiatingTemplate::InstantiatingTemplate(
     : InstantiatingTemplate(SemaRef, Kind, PointOfInstantiation,
                             InstantiationRange, FunctionTemplate, nullptr,
                             TemplateArgs, &DeductionInfo) {
-  assert(
-    Kind == CodeSynthesisContext::ExplicitTemplateArgumentSubstitution ||
-    Kind == CodeSynthesisContext::DeducedTemplateArgumentSubstitution);
+  assert(Kind == CodeSynthesisContext::ExplicitTemplateArgumentSubstitution ||
+         Kind == CodeSynthesisContext::DeducedTemplateArgumentSubstitution ||
+         Kind == CodeSynthesisContext::BuildingDeductionGuides);
 }
 
 Sema::InstantiatingTemplate::InstantiatingTemplate(
@@ -1446,6 +1447,59 @@ namespace {
       return inherited::TransformFunctionProtoType(TLB, TL);
     }
 
+    QualType TransformInjectedClassNameType(TypeLocBuilder &TLB,
+                                            InjectedClassNameTypeLoc TL) {
+      auto Type = inherited::TransformInjectedClassNameType(TLB, TL);
+      // Special case for transforming a deduction guide, we return a
+      // transformed TemplateSpecializationType.
+      if (Type.isNull() &&
+          SemaRef.CodeSynthesisContexts.back().Kind ==
+              Sema::CodeSynthesisContext::BuildingDeductionGuides) {
+        // Return a TemplateSpecializationType for transforming a deduction
+        // guide.
+        if (auto *ICT = TL.getType()->getAs()) {
+          auto Type =
+              inherited::TransformType(ICT->getInjectedSpecializationType());
+          TLB.pushTrivial(SemaRef.Context, Type, TL.getNameLoc());
+          return Type;
+        }
+      }
+      return Type;
+    }
+    // Override the default version to handle a rewrite-template-arg-pack case
+    // for building a deduction guide.
+    bool TransformTemplateArgument(const TemplateArgumentLoc &Input,
+                                   TemplateArgumentLoc &Output,
+                                   bool Uneval = false) {
+      const TemplateArgument &Arg = Input.getArgument();
+      std::vector TArgs;
+      switch (Arg.getKind()) {
+      case TemplateArgument::Pack:
+        // Literally rewrite the template argument pack, instead of unpacking
+        // it.
+        assert(
+            SemaRef.CodeSynthesisContexts.back().Kind ==
+                Sema::CodeSynthesisContext::BuildingDeductionGuides &&
+            "Transforming a template argument pack is only allowed in building "
+            "deduction guide");
+        for (auto &pack : Arg.getPackAsArray()) {
+          TemplateArgumentLoc Input = SemaRef.getTrivialTemplateArgumentLoc(
+              pack, QualType(), SourceLocation{});
+          TemplateArgumentLoc Output;
+          if (SemaRef.SubstTemplateArgument(Input, TemplateArgs, Output))
+            return true; // fails
+          TArgs.push_back(Output.getArgument());
+        }
+        Output = SemaRef.getTrivialTemplateArgumentLoc(
+            TemplateArgument(llvm::ArrayRef(TArgs).copy(SemaRef.Context)),
+            QualType(), SourceLocation{});
+        return false;
+      default:
+        break;
+      }
+      return inherited::TransformTemplateArgument(Input, Output, Uneval);
+    }
+
     template
     QualType TransformFunctionProtoType(TypeLocBuilder &TLB,
                                         FunctionProtoTypeLoc TL,
@@ -4138,6 +4192,15 @@ Sema::SubstStmt(Stmt *S, const MultiLevelTemplateArgumentList &TemplateArgs) {
   return Instantiator.TransformStmt(S);
 }
 
+bool Sema::SubstTemplateArgument(
+    const TemplateArgumentLoc &Input,
+    const MultiLevelTemplateArgumentList &TemplateArgs,
+    TemplateArgumentLoc &Output) {
+  TemplateInstantiator Instantiator(*this, TemplateArgs, SourceLocation(),
+                                    DeclarationName());
+  return Instantiator.TransformTemplateArgument(Input, Output);
+}
+
 bool Sema::SubstTemplateArguments(
     ArrayRef Args,
     const MultiLevelTemplateArgumentList &TemplateArgs,
diff --git a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
index 9c696e072ba4..20c2c93ac9c7 100644
--- a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
@@ -2219,7 +2219,9 @@ Decl *TemplateDeclInstantiator::VisitFunctionDecl(
       FunctionTemplate->setInstantiatedFromMemberTemplate(
                                            D->getDescribedFunctionTemplate());
     }
-  } else if (FunctionTemplate) {
+  } else if (FunctionTemplate &&
+             SemaRef.CodeSynthesisContexts.back().Kind !=
+                 Sema::CodeSynthesisContext::BuildingDeductionGuides) {
     // Record this function template specialization.
     ArrayRef Innermost = TemplateArgs.getInnermost();
     Function->setFunctionTemplateSpecialization(FunctionTemplate,
@@ -4853,16 +4855,13 @@ bool TemplateDeclInstantiator::SubstDefaultedFunction(FunctionDecl *New,
 ///
 /// Usually this should not be used, and template argument deduction should be
 /// used in its place.
-FunctionDecl *
-Sema::InstantiateFunctionDeclaration(FunctionTemplateDecl *FTD,
-                                     const TemplateArgumentList *Args,
-                                     SourceLocation Loc) {
+FunctionDecl *Sema::InstantiateFunctionDeclaration(
+    FunctionTemplateDecl *FTD, const TemplateArgumentList *Args,
+    SourceLocation Loc, CodeSynthesisContext::SynthesisKind CSC) {
   FunctionDecl *FD = FTD->getTemplatedDecl();
 
   sema::TemplateDeductionInfo Info(Loc);
-  InstantiatingTemplate Inst(
-      *this, Loc, FTD, Args->asArray(),
-      CodeSynthesisContext::ExplicitTemplateArgumentSubstitution, Info);
+  InstantiatingTemplate Inst(*this, Loc, FTD, Args->asArray(), CSC, Info);
   if (Inst.isInvalid())
     return nullptr;
 
@@ -6286,8 +6285,18 @@ NamedDecl *Sema::FindInstantiatedDecl(SourceLocation Loc, NamedDecl *D,
           QualType T = CheckTemplateIdType(TemplateName(TD), Loc, Args);
           if (T.isNull())
             return nullptr;
-          auto *SubstRecord = T->getAsCXXRecordDecl();
-          assert(SubstRecord && "class template id not a class type?");
+          CXXRecordDecl *SubstRecord = T->getAsCXXRecordDecl();
+
+          if (!SubstRecord) {
+            // T can be a dependent TemplateSpecializationType when performing a
+            // substitution for building a deduction guide.
+            assert(CodeSynthesisContexts.back().Kind ==
+                   CodeSynthesisContext::BuildingDeductionGuides);
+            // Return a nullptr as a sentinel value, we handle it properly in
+            // the TemplateInstantiator::TransformInjectedClassNameType
+            // override, which we transform it to a TemplateSpecializationType.
+            return nullptr;
+          }
           // Check that this template-id names the primary template and not a
           // partial or explicit specialization. (In the latter cases, it's
           // meaningless to attempt to find an instantiation of D within the
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 409aee73d960..2d22692f3ab7 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -4785,6 +4785,14 @@ bool TreeTransform::TransformTemplateArguments(
     TemplateArgumentLoc In = *First;
 
     if (In.getArgument().getKind() == TemplateArgument::Pack) {
+      // When building the deduction guides, we rewrite the argument packs
+      // instead of unpacking.
+      if (getSema().CodeSynthesisContexts.back().Kind ==
+          Sema::CodeSynthesisContext::BuildingDeductionGuides) {
+        if (getDerived().TransformTemplateArgument(In, Out, Uneval))
+          return true;
+        continue;
+      }
       // Unpack argument packs, which we translate them into separate
       // arguments.
       // FIXME: We could do much better if we could guarantee that the
diff --git a/clang/test/SemaCXX/cxx17-compat.cpp b/clang/test/SemaCXX/cxx17-compat.cpp
index d53d80f0d42c..54ea3384022d 100644
--- a/clang/test/SemaCXX/cxx17-compat.cpp
+++ b/clang/test/SemaCXX/cxx17-compat.cpp
@@ -131,3 +131,14 @@ namespace NTTP {
   // expected-warning@-4 {{non-type template parameter of type 'A' is incompatible with C++ standards before C++20}}
 #endif
 }
+
+namespace CTADForAliasTemplate {
+template struct A { A(T); };
+template using B = A;
+B b = {1};
+#if __cplusplus <= 201703L
+  // FIXME: diagnose as well
+#else
+  // expected-warning@-4 {{class template argument deduction for alias templates is incompatible with C++ standards before C++20}}
+#endif
+}
diff --git a/clang/test/SemaCXX/cxx1z-class-template-argument-deduction.cpp b/clang/test/SemaCXX/cxx1z-class-template-argument-deduction.cpp
index 33ed4295c2e4..2f067ea53a50 100644
--- a/clang/test/SemaCXX/cxx1z-class-template-argument-deduction.cpp
+++ b/clang/test/SemaCXX/cxx1z-class-template-argument-deduction.cpp
@@ -101,13 +101,13 @@ namespace dependent {
   struct B {
     template struct X { X(T); };
     X(int) -> X;
-    template using Y = X; // expected-note {{template}}
+    template using Y = X;
   };
   template void f() {
     typename T::X tx = 0;
-    typename T::Y ty = 0; // expected-error {{alias template 'Y' requires template arguments; argument deduction only allowed for class templates}}
+    typename T::Y ty = 0;
   }
-  template void f(); // expected-note {{in instantiation of}}
+  template void f();
 
   template struct C { C(T); };
   template C(T) -> C;
diff --git a/clang/test/SemaCXX/cxx20-ctad-type-alias.cpp b/clang/test/SemaCXX/cxx20-ctad-type-alias.cpp
new file mode 100644
index 000000000000..794496ed4184
--- /dev/null
+++ b/clang/test/SemaCXX/cxx20-ctad-type-alias.cpp
@@ -0,0 +1,232 @@
+// RUN: %clang_cc1 -fsyntax-only -Wno-c++11-narrowing -Wno-literal-conversion -std=c++20 -verify %s
+
+namespace test1 {
+template 
+struct Foo { T t; };
+template 
+using Bar = Foo;
+
+Bar s = {1};
+}  // namespace test1
+
+namespace test2 {
+template 
+struct XYpair {
+  X x;
+  Y y;
+};
+// A tricky explicit deduction guide that swapping X and Y.
+template 
+XYpair(X, Y) -> XYpair;
+template 
+using AliasXYpair = XYpair;
+
+AliasXYpair xy = {1.1, 2};  // XYpair
+static_assert(__is_same(decltype(xy.x), int));
+static_assert(__is_same(decltype(xy.y), double));
+}  // namespace test2
+
+namespace test3 {
+template 
+struct container {
+  // test with default arguments.
+  container(T a, T b = T());
+};
+
+template 
+using vector = container;
+vector v(0, 0);
+}  // namespace test3
+
+namespace test4 {
+// Explicit deduction guide.
+template 
+struct X {
+  T t;
+  X(T);
+};
+
+template 
+X(T) -> X;
+
+template 
+using AX = X;
+
+AX s = {1};
+static_assert(__is_same(decltype(s.t), double)); // explicit one is picked.
+}  // namespace test4
+
+namespace test5 {
+template 
+struct Foo {};
+// Template parameter pack
+template 
+using AF = Foo<1>;
+auto a = AF{};
+}  // namespace test5
+
+namespace test6 {
+// non-type template argument.
+template 
+struct Foo {
+  Foo(T);
+};
+template 
+using AF = Foo;
+
+AF b{0}; 
+}  // namespace test6
+
+namespace test7 {
+template 
+struct Foo {
+  Foo(T);
+};
+// using alias chain.
+template 
+using AF1 = Foo;
+template 
+using AF2 = AF1;  
+AF2 b = 1;  
+}  // namespace test7
+
+namespace test8 {
+template 
+struct Foo {
+  Foo(T const (&)[N]);
+};
+
+template 
+using Bar = Foo;
+
+Bar s = {{1}};
+}  // namespace test8
+
+namespace test9 {
+template 
+struct Foo {
+  Foo(T const (&)[N]);
+};
+
+template 
+using Bar = Foo;
+
+// FIXME: we should reject this case? GCC rejects it, MSVC accepts it.
+Bar s = {{1}};
+}  // namespace test9
+
+namespace test10 {
+template 
+struct Foo {
+  template 
+  Foo(U);
+};
+
+template 
+Foo(U) -> Foo;
+
+template 
+using A = Foo;
+A a(2);  // Foo
+}  // namespace test10
+
+namespace test11 {
+struct A {};
+template struct Foo { T c; };
+template using AFoo = Foo;
+
+AFoo s = {1};
+} // namespace test11
+
+namespace test12 {
+// no crash on null access attribute
+template
+struct Foo {
+  template
+  struct Bar { 
+    Bar(K);
+  };
+
+  template
+  using ABar = Bar;
+  void test() { ABar k = 2; }
+};
+
+void func(Foo s) {
+  s.test();
+}
+} // namespace test12
+
+namespace test13 {
+template 
+struct Foo {
+  Foo(Ts...);
+};
+
+template 
+using AFoo = Foo;
+
+auto b = AFoo{};
+} // namespace test13
+
+namespace test14 {
+template
+concept IsInt = __is_same(decltype(T()), int);
+
+template
+struct Foo {
+  Foo(T const (&)[N]);
+};
+
+template 
+using Bar = Foo; // expected-note {{constraints not satisfied for class template 'Foo'}}
+// expected-note@-1 {{candidate template ignored: could not match}}
+double abc[3];
+Bar s2 = {abc}; // expected-error {{no viable constructor or deduction guide for deduction }}
+} // namespace test14
+
+namespace test15 {
+template  struct Foo { Foo(T); };
+
+template using AFoo = Foo;
+template concept False = false;
+template using BFoo = AFoo;
+int i = 0;
+AFoo a1(&i); // OK, deduce Foo
+
+// FIXME: we should reject this case as the W is not deduced from the deduced
+// type Foo.
+BFoo b2(&i); 
+} // namespace test15
+
+namespace test16 {
+struct X { X(int); X(const X&); };
+template
+struct Foo {
+  T t;
+  Foo(T t) : t(t) {}
+};
+template
+using AFoo = Foo;
+int i = 0;
+AFoo s{i};
+static_assert(__is_same(decltype(s.t), int));
+
+// explicit deduction guide.
+Foo(int) -> Foo;
+AFoo s2{i};
+// FIXME: the type should be X because of the above explicit deduction guide.
+static_assert(__is_same(decltype(s2.t), int));
+} // namespace test16
+
+namespace test17 {
+template 
+struct Foo { T t; };
+
+// CTAD for alias templates only works for the RHS of the alias of form of
+//  [typename] [nested-name-specifier] [template] simple-template-id
+template 
+using AFoo = Foo*; // expected-note {{template is declared here}}
+
+AFoo s = {1}; // expected-error {{alias template 'AFoo' requires template arguments; argument deduction only allowed for}}
+} // namespace test17
diff --git a/clang/www/cxx_status.html b/clang/www/cxx_status.html
index fa00e7685610..a3090adb5d47 100755
--- a/clang/www/cxx_status.html
+++ b/clang/www/cxx_status.html
@@ -836,7 +836,13 @@ C++23, informally referred to as C++26.

     
       Class template argument deduction for alias templates
       P1814R0
-      No
+      
+        

+          Clang 19 (Partial)
+          The associated constraints (over.match.class.deduct#3.3) for the
+          synthesized deduction guides are not yet implemented.
+        

+      
     
     
       Permit conversions to arrays of unknown bound
-- 
GitLab


From d2dff43170858459e03f9c8c764bf472ae7a59fa Mon Sep 17 00:00:00 2001
From: paperchalice 
Date: Fri, 8 Mar 2024 21:26:10 +0800
Subject: [PATCH 0031/1674] [CodeGen] Skip declaration in
 ModuleToMachineFunctionPassAdaptor (#84417)

Like ModuleToFunctionPassAdaptor.

---------

Co-authored-by: Matt Arsenault 
---
 llvm/lib/CodeGen/MachinePassManager.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/CodeGen/MachinePassManager.cpp b/llvm/lib/CodeGen/MachinePassManager.cpp
index 9a750b5bed43..2763193b2c30 100644
--- a/llvm/lib/CodeGen/MachinePassManager.cpp
+++ b/llvm/lib/CodeGen/MachinePassManager.cpp
@@ -79,7 +79,7 @@ ModuleToMachineFunctionPassAdaptor::run(Module &M, ModuleAnalysisManager &AM) {
   for (Function &F : M) {
     // Do not codegen any 'available_externally' functions at all, they have
     // definitions outside the translation unit.
-    if (F.hasAvailableExternallyLinkage())
+    if (F.isDeclaration() || F.hasAvailableExternallyLinkage())
       continue;
 
     MachineFunction &MF = MMI.getOrCreateMachineFunction(F);
-- 
GitLab


From 7c21495fee927359d642793d2eeb24696a301262 Mon Sep 17 00:00:00 2001
From: AtariDreams <83477269+AtariDreams@users.noreply.github.com>
Date: Fri, 8 Mar 2024 08:35:00 -0500
Subject: [PATCH 0032/1674] Reapply "Convert many LivePhysRegs uses to
 LiveRegUnits" (#84338)

This only converts the instances where all that is needed is to change
the variable type name.

Basically, anything that involves a function that LiveRegUnits does not
directly have was skipped to play it safe.

Reverts
https://github.com/llvm/llvm-project/commit/7a0e222a17058a311b69153d0b6f1b4459414778
---
 llvm/lib/CodeGen/ReachingDefAnalysis.cpp      | 24 +++++++++----------
 .../Target/AMDGPU/SIOptimizeExecMasking.cpp   |  8 +++----
 llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp |  6 ++---
 llvm/lib/Target/Hexagon/HexagonGenMux.cpp     | 12 +++-------
 .../lib/Target/SystemZ/SystemZElimCompare.cpp |  6 ++---
 llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp  |  6 ++---
 .../lib/Target/SystemZ/SystemZShortenInst.cpp |  8 +++----
 llvm/lib/Target/X86/X86FloatingPoint.cpp      |  8 +++----
 8 files changed, 36 insertions(+), 42 deletions(-)

diff --git a/llvm/lib/CodeGen/ReachingDefAnalysis.cpp b/llvm/lib/CodeGen/ReachingDefAnalysis.cpp
index 61a668907be7..07fa92889d88 100644
--- a/llvm/lib/CodeGen/ReachingDefAnalysis.cpp
+++ b/llvm/lib/CodeGen/ReachingDefAnalysis.cpp
@@ -6,10 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/ADT/SmallSet.h"
-#include "llvm/ADT/SetOperations.h"
-#include "llvm/CodeGen/LivePhysRegs.h"
 #include "llvm/CodeGen/ReachingDefAnalysis.h"
+#include "llvm/ADT/SetOperations.h"
+#include "llvm/ADT/SmallSet.h"
+#include "llvm/CodeGen/LiveRegUnits.h"
 #include "llvm/CodeGen/TargetRegisterInfo.h"
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
 #include "llvm/Support/Debug.h"
@@ -421,9 +421,9 @@ void ReachingDefAnalysis::getLiveOuts(MachineBasicBlock *MBB,
     return;
 
   VisitedBBs.insert(MBB);
-  LivePhysRegs LiveRegs(*TRI);
+  LiveRegUnits LiveRegs(*TRI);
   LiveRegs.addLiveOuts(*MBB);
-  if (LiveRegs.available(MBB->getParent()->getRegInfo(), PhysReg))
+  if (LiveRegs.available(PhysReg))
     return;
 
   if (auto *Def = getLocalLiveOutMIDef(MBB, PhysReg))
@@ -469,11 +469,11 @@ MachineInstr *ReachingDefAnalysis::getMIOperand(MachineInstr *MI,
 bool ReachingDefAnalysis::isRegUsedAfter(MachineInstr *MI,
                                          MCRegister PhysReg) const {
   MachineBasicBlock *MBB = MI->getParent();
-  LivePhysRegs LiveRegs(*TRI);
+  LiveRegUnits LiveRegs(*TRI);
   LiveRegs.addLiveOuts(*MBB);
 
   // Yes if the register is live out of the basic block.
-  if (!LiveRegs.available(MBB->getParent()->getRegInfo(), PhysReg))
+  if (!LiveRegs.available(PhysReg))
     return true;
 
   // Walk backwards through the block to see if the register is live at some
@@ -481,7 +481,7 @@ bool ReachingDefAnalysis::isRegUsedAfter(MachineInstr *MI,
   for (MachineInstr &Last :
        instructionsWithoutDebug(MBB->instr_rbegin(), MBB->instr_rend())) {
     LiveRegs.stepBackward(Last);
-    if (!LiveRegs.available(MBB->getParent()->getRegInfo(), PhysReg))
+    if (!LiveRegs.available(PhysReg))
       return InstIds.lookup(&Last) > InstIds.lookup(MI);
   }
   return false;
@@ -504,9 +504,9 @@ bool ReachingDefAnalysis::isRegDefinedAfter(MachineInstr *MI,
 bool ReachingDefAnalysis::isReachingDefLiveOut(MachineInstr *MI,
                                                MCRegister PhysReg) const {
   MachineBasicBlock *MBB = MI->getParent();
-  LivePhysRegs LiveRegs(*TRI);
+  LiveRegUnits LiveRegs(*TRI);
   LiveRegs.addLiveOuts(*MBB);
-  if (LiveRegs.available(MBB->getParent()->getRegInfo(), PhysReg))
+  if (LiveRegs.available(PhysReg))
     return false;
 
   auto Last = MBB->getLastNonDebugInstr();
@@ -525,9 +525,9 @@ bool ReachingDefAnalysis::isReachingDefLiveOut(MachineInstr *MI,
 MachineInstr *
 ReachingDefAnalysis::getLocalLiveOutMIDef(MachineBasicBlock *MBB,
                                           MCRegister PhysReg) const {
-  LivePhysRegs LiveRegs(*TRI);
+  LiveRegUnits LiveRegs(*TRI);
   LiveRegs.addLiveOuts(*MBB);
-  if (LiveRegs.available(MBB->getParent()->getRegInfo(), PhysReg))
+  if (LiveRegs.available(PhysReg))
     return nullptr;
 
   auto Last = MBB->getLastNonDebugInstr();
diff --git a/llvm/lib/Target/AMDGPU/SIOptimizeExecMasking.cpp b/llvm/lib/Target/AMDGPU/SIOptimizeExecMasking.cpp
index e3f54d01eb22..3c60459e54e8 100644
--- a/llvm/lib/Target/AMDGPU/SIOptimizeExecMasking.cpp
+++ b/llvm/lib/Target/AMDGPU/SIOptimizeExecMasking.cpp
@@ -11,7 +11,7 @@
 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
 #include "SIRegisterInfo.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/CodeGen/LivePhysRegs.h"
+#include "llvm/CodeGen/LiveRegUnits.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/MachineOperand.h"
 #include "llvm/CodeGen/TargetRegisterInfo.h"
@@ -313,7 +313,7 @@ MachineBasicBlock::reverse_iterator SIOptimizeExecMasking::findExecCopy(
   return E;
 }
 
-// XXX - Seems LivePhysRegs doesn't work correctly since it will incorrectly
+// XXX - Seems LiveRegUnits doesn't work correctly since it will incorrectly
 // report the register as unavailable because a super-register with a lane mask
 // is unavailable.
 static bool isLiveOut(const MachineBasicBlock &MBB, unsigned Reg) {
@@ -383,7 +383,7 @@ bool SIOptimizeExecMasking::isRegisterInUseBetween(MachineInstr &Stop,
                                                    MCRegister Reg,
                                                    bool UseLiveOuts,
                                                    bool IgnoreStart) const {
-  LivePhysRegs LR(*TRI);
+  LiveRegUnits LR(*TRI);
   if (UseLiveOuts)
     LR.addLiveOuts(*Stop.getParent());
 
@@ -396,7 +396,7 @@ bool SIOptimizeExecMasking::isRegisterInUseBetween(MachineInstr &Stop,
     LR.stepBackward(*A);
   }
 
-  return !LR.available(*MRI, Reg);
+  return !LR.available(Reg) || MRI->isReserved(Reg);
 }
 
 // Determine if a register Reg is not re-defined and still in use
diff --git a/llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp b/llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp
index 6121055eb021..469340784284 100644
--- a/llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp
+++ b/llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp
@@ -31,7 +31,7 @@
 #include "llvm/ADT/Statistic.h"
 #include "llvm/ADT/iterator_range.h"
 #include "llvm/Analysis/AliasAnalysis.h"
-#include "llvm/CodeGen/LivePhysRegs.h"
+#include "llvm/CodeGen/LiveRegUnits.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineDominators.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
@@ -109,7 +109,7 @@ namespace {
     const ARMSubtarget *STI;
     const TargetLowering *TL;
     ARMFunctionInfo *AFI;
-    LivePhysRegs LiveRegs;
+    LiveRegUnits LiveRegs;
     RegisterClassInfo RegClassInfo;
     MachineBasicBlock::const_iterator LiveRegPos;
     bool LiveRegsValid;
@@ -589,7 +589,7 @@ unsigned ARMLoadStoreOpt::findFreeReg(const TargetRegisterClass &RegClass) {
   }
 
   for (unsigned Reg : RegClassInfo.getOrder(&RegClass))
-    if (LiveRegs.available(MF->getRegInfo(), Reg))
+    if (LiveRegs.available(Reg) && !MF->getRegInfo().isReserved(Reg))
       return Reg;
   return 0;
 }
diff --git a/llvm/lib/Target/Hexagon/HexagonGenMux.cpp b/llvm/lib/Target/Hexagon/HexagonGenMux.cpp
index 509144ceb6a0..74ab58e02966 100644
--- a/llvm/lib/Target/Hexagon/HexagonGenMux.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonGenMux.cpp
@@ -26,7 +26,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
-#include "llvm/CodeGen/LivePhysRegs.h"
+#include "llvm/CodeGen/LiveRegUnits.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
@@ -346,14 +346,8 @@ bool HexagonGenMux::genMuxInBlock(MachineBasicBlock &B) {
 
   // Fix up kill flags.
 
-  LivePhysRegs LPR(*HRI);
+  LiveRegUnits LPR(*HRI);
   LPR.addLiveOuts(B);
-  auto IsLive = [&LPR, this](unsigned Reg) -> bool {
-    for (MCPhysReg S : HRI->subregs_inclusive(Reg))
-      if (LPR.contains(S))
-        return true;
-    return false;
-  };
   for (MachineInstr &I : llvm::reverse(B)) {
     if (I.isDebugInstr())
       continue;
@@ -365,7 +359,7 @@ bool HexagonGenMux::genMuxInBlock(MachineBasicBlock &B) {
       if (!Op.isReg() || !Op.isUse())
         continue;
       assert(Op.getSubReg() == 0 && "Should have physical registers only");
-      bool Live = IsLive(Op.getReg());
+      bool Live = !LPR.available(Op.getReg());
       Op.setIsKill(!Live);
     }
     LPR.stepBackward(I);
diff --git a/llvm/lib/Target/SystemZ/SystemZElimCompare.cpp b/llvm/lib/Target/SystemZ/SystemZElimCompare.cpp
index 7423ed429ffb..e58f50e471fc 100644
--- a/llvm/lib/Target/SystemZ/SystemZElimCompare.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZElimCompare.cpp
@@ -18,7 +18,7 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/ADT/StringRef.h"
-#include "llvm/CodeGen/LivePhysRegs.h"
+#include "llvm/CodeGen/LiveRegUnits.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
@@ -690,9 +690,9 @@ bool SystemZElimCompare::processBlock(MachineBasicBlock &MBB) {
   // Walk backwards through the block looking for comparisons, recording
   // all CC users as we go.  The subroutines can delete Compare and
   // instructions before it.
-  LivePhysRegs LiveRegs(*TRI);
+  LiveRegUnits LiveRegs(*TRI);
   LiveRegs.addLiveOuts(MBB);
-  bool CompleteCCUsers = !LiveRegs.contains(SystemZ::CC);
+  bool CompleteCCUsers = LiveRegs.available(SystemZ::CC);
   SmallVector CCUsers;
   MachineBasicBlock::iterator MBBI = MBB.end();
   while (MBBI != MBB.begin()) {
diff --git a/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp b/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp
index 046a12208467..53e9bf9a9d1b 100644
--- a/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp
@@ -18,7 +18,7 @@
 #include "llvm/ADT/Statistic.h"
 #include "llvm/CodeGen/LiveInterval.h"
 #include "llvm/CodeGen/LiveIntervals.h"
-#include "llvm/CodeGen/LivePhysRegs.h"
+#include "llvm/CodeGen/LiveRegUnits.h"
 #include "llvm/CodeGen/LiveVariables.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
@@ -1874,9 +1874,9 @@ prepareCompareSwapOperands(MachineBasicBlock::iterator const MBBI) const {
     }
   }
   if (CCLive) {
-    LivePhysRegs LiveRegs(*MBB->getParent()->getSubtarget().getRegisterInfo());
+    LiveRegUnits LiveRegs(*MBB->getParent()->getSubtarget().getRegisterInfo());
     LiveRegs.addLiveOuts(*MBB);
-    if (LiveRegs.contains(SystemZ::CC))
+    if (!LiveRegs.available(SystemZ::CC))
       return false;
   }
 
diff --git a/llvm/lib/Target/SystemZ/SystemZShortenInst.cpp b/llvm/lib/Target/SystemZ/SystemZShortenInst.cpp
index 30b22fa1ce92..c0adfdbf120b 100644
--- a/llvm/lib/Target/SystemZ/SystemZShortenInst.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZShortenInst.cpp
@@ -13,7 +13,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "SystemZTargetMachine.h"
-#include "llvm/CodeGen/LivePhysRegs.h"
+#include "llvm/CodeGen/LiveRegUnits.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/TargetRegisterInfo.h"
@@ -46,7 +46,7 @@ private:
 
   const SystemZInstrInfo *TII;
   const TargetRegisterInfo *TRI;
-  LivePhysRegs LiveRegs;
+  LiveRegUnits LiveRegs;
 };
 
 char SystemZShortenInst::ID = 0;
@@ -88,7 +88,7 @@ bool SystemZShortenInst::shortenIIF(MachineInstr &MI, unsigned LLIxL,
   unsigned GR64BitReg =
       TRI->getMatchingSuperReg(Reg, thisSubRegIdx, &SystemZ::GR64BitRegClass);
   Register OtherReg = TRI->getSubReg(GR64BitReg, otherSubRegIdx);
-  if (LiveRegs.contains(OtherReg))
+  if (!LiveRegs.available(OtherReg))
     return false;
 
   uint64_t Imm = MI.getOperand(1).getImm();
@@ -143,7 +143,7 @@ bool SystemZShortenInst::shortenOn001(MachineInstr &MI, unsigned Opcode) {
 // Calls shortenOn001 if CCLive is false. CC def operand is added in
 // case of success.
 bool SystemZShortenInst::shortenOn001AddCC(MachineInstr &MI, unsigned Opcode) {
-  if (!LiveRegs.contains(SystemZ::CC) && shortenOn001(MI, Opcode)) {
+  if (LiveRegs.available(SystemZ::CC) && shortenOn001(MI, Opcode)) {
     MachineInstrBuilder(*MI.getParent()->getParent(), &MI)
       .addReg(SystemZ::CC, RegState::ImplicitDefine | RegState::Dead);
     return true;
diff --git a/llvm/lib/Target/X86/X86FloatingPoint.cpp b/llvm/lib/Target/X86/X86FloatingPoint.cpp
index ca4d03913d09..260879ffaa4f 100644
--- a/llvm/lib/Target/X86/X86FloatingPoint.cpp
+++ b/llvm/lib/Target/X86/X86FloatingPoint.cpp
@@ -30,7 +30,7 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/CodeGen/EdgeBundles.h"
-#include "llvm/CodeGen/LivePhysRegs.h"
+#include "llvm/CodeGen/LiveRegUnits.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
@@ -1751,7 +1751,7 @@ void FPS::handleSpecialFP(MachineBasicBlock::iterator &Inst) {
 void FPS::setKillFlags(MachineBasicBlock &MBB) const {
   const TargetRegisterInfo &TRI =
       *MBB.getParent()->getSubtarget().getRegisterInfo();
-  LivePhysRegs LPR(TRI);
+  LiveRegUnits LPR(TRI);
 
   LPR.addLiveOuts(MBB);
 
@@ -1773,14 +1773,14 @@ void FPS::setKillFlags(MachineBasicBlock &MBB) const {
 
       if (MO.isDef()) {
         Defs.set(Reg);
-        if (!LPR.contains(MO.getReg()))
+        if (LPR.available(MO.getReg()))
           MO.setIsDead();
       } else
         Uses.push_back(&MO);
     }
 
     for (auto *MO : Uses)
-      if (Defs.test(getFPReg(*MO)) || !LPR.contains(MO->getReg()))
+      if (Defs.test(getFPReg(*MO)) || LPR.available(MO->getReg()))
         MO->setIsKill();
 
     LPR.stepBackward(MI);
-- 
GitLab


From f07157eb2e8212068cb9469d77f9bc3779ef2ea4 Mon Sep 17 00:00:00 2001
From: Balazs Benics 
Date: Fri, 8 Mar 2024 14:52:19 +0100
Subject: [PATCH 0034/1674] Fixup 7457e2c1535acd54 by adding a switch case

This fixups 7457e2c1535acd548d2619dfb34eb93d27d15908 as I forgot one
switch to update, which then broke this build bot:
https://lab.llvm.org/buildbot/#/builders/57/builds/33272
---
 clang/unittests/AST/MatchVerifier.h | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/clang/unittests/AST/MatchVerifier.h b/clang/unittests/AST/MatchVerifier.h
index 8bcf05642cb5..da1e351da4a0 100644
--- a/clang/unittests/AST/MatchVerifier.h
+++ b/clang/unittests/AST/MatchVerifier.h
@@ -116,6 +116,10 @@ MatchVerifier::match(const std::string &Code,
     Args.push_back("-std=c++20");
     FileName = "input.cc";
     break;
+  case Lang_CXX23:
+    Args.push_back("-std=c++23");
+    FileName = "input.cc";
+    break;
   case Lang_OpenCL:
     Args.push_back("-cl-no-stdinc");
     FileName = "input.cl";
-- 
GitLab


From ba2236d3000645d3127f972aa7ac1844c47e299c Mon Sep 17 00:00:00 2001
From: Will Hawkins 
Date: Fri, 8 Mar 2024 09:25:51 -0500
Subject: [PATCH 0035/1674] [libcxx][NFC] Consolidate testing concept
 CanBePiped (#80154)

Almost every test needed a CanBePiped concept and each implemented it
separately, but identically. Consolidate all implementations into
test_range.h.
---
 .../test/std/ranges/range.adaptors/range.all/all.pass.cpp | 8 ++------
 .../ranges/range.adaptors/range.chunk.by/adaptor.pass.cpp | 6 +-----
 .../range.adaptors/range.common.view/adaptor.pass.cpp     | 6 +-----
 .../range.adaptors/range.drop.while/adaptor.pass.cpp      | 8 ++------
 .../std/ranges/range.adaptors/range.drop/adaptor.pass.cpp | 7 ++-----
 .../ranges/range.adaptors/range.elements/adaptor.pass.cpp | 8 ++------
 .../ranges/range.adaptors/range.filter/adaptor.pass.cpp   | 6 +-----
 .../std/ranges/range.adaptors/range.join/adaptor.pass.cpp | 6 +-----
 .../range.adaptors/range.lazy.split/adaptor.pass.cpp      | 6 +-----
 .../ranges/range.adaptors/range.reverse/adaptor.pass.cpp  | 6 +-----
 .../ranges/range.adaptors/range.split/adaptor.pass.cpp    | 6 +-----
 .../range.adaptors/range.take.while/adaptor.pass.cpp      | 7 +------
 .../std/ranges/range.adaptors/range.take/adaptor.pass.cpp | 7 ++-----
 .../range.adaptors/range.transform/adaptor.pass.cpp       | 6 +-----
 libcxx/test/support/test_range.h                          | 5 +++++
 15 files changed, 24 insertions(+), 74 deletions(-)

diff --git a/libcxx/test/std/ranges/range.adaptors/range.all/all.pass.cpp b/libcxx/test/std/ranges/range.adaptors/range.all/all.pass.cpp
index d90a25d93e34..d31373fcad78 100644
--- a/libcxx/test/std/ranges/range.adaptors/range.all/all.pass.cpp
+++ b/libcxx/test/std/ranges/range.adaptors/range.all/all.pass.cpp
@@ -17,8 +17,9 @@
 #include 
 #include 
 
-#include "test_macros.h"
 #include "test_iterators.h"
+#include "test_macros.h"
+#include "test_range.h"
 
 int globalBuff[8];
 
@@ -82,11 +83,6 @@ struct RandomAccessRange {
 template<>
 inline constexpr bool std::ranges::enable_borrowed_range = true;
 
-template 
-concept CanBePiped = requires (View&& view, T&& t) {
-  { std::forward(view) | std::forward(t) };
-};
-
 constexpr bool test() {
   {
     ASSERT_SAME_TYPE(decltype(std::views::all(View())), View);
diff --git a/libcxx/test/std/ranges/range.adaptors/range.chunk.by/adaptor.pass.cpp b/libcxx/test/std/ranges/range.adaptors/range.chunk.by/adaptor.pass.cpp
index 423c481127e2..697085a1d5c5 100644
--- a/libcxx/test/std/ranges/range.adaptors/range.chunk.by/adaptor.pass.cpp
+++ b/libcxx/test/std/ranges/range.adaptors/range.chunk.by/adaptor.pass.cpp
@@ -22,11 +22,7 @@
 #include 
 
 #include "test_iterators.h"
-
-template 
-concept CanBePiped = requires(View&& view, T&& t) {
-  { std::forward(view) | std::forward(t) };
-};
+#include "test_range.h"
 
 struct Pred {
   constexpr bool operator()(int x, int y) const { return x != -y; }
diff --git a/libcxx/test/std/ranges/range.adaptors/range.common.view/adaptor.pass.cpp b/libcxx/test/std/ranges/range.adaptors/range.common.view/adaptor.pass.cpp
index 143f7df1d372..b0e72e9e7914 100644
--- a/libcxx/test/std/ranges/range.adaptors/range.common.view/adaptor.pass.cpp
+++ b/libcxx/test/std/ranges/range.adaptors/range.common.view/adaptor.pass.cpp
@@ -18,13 +18,9 @@
 #include 
 
 #include "test_iterators.h"
+#include "test_range.h"
 #include "types.h"
 
-template 
-concept CanBePiped = requires (View&& view, T&& t) {
-  { std::forward(view) | std::forward(t) };
-};
-
 constexpr bool test() {
   int buf[] = {1, 2, 3};
 
diff --git a/libcxx/test/std/ranges/range.adaptors/range.drop.while/adaptor.pass.cpp b/libcxx/test/std/ranges/range.adaptors/range.drop.while/adaptor.pass.cpp
index 409b400f7f87..c41d4172f199 100644
--- a/libcxx/test/std/ranges/range.adaptors/range.drop.while/adaptor.pass.cpp
+++ b/libcxx/test/std/ranges/range.adaptors/range.drop.while/adaptor.pass.cpp
@@ -16,6 +16,8 @@
 #include 
 #include 
 
+#include "test_range.h"
+
 struct Pred {
   constexpr bool operator()(int i) const { return i < 3; }
 };
@@ -51,12 +53,6 @@ static_assert(std::is_invocable_v);
 static_assert(std::is_invocable_v);
 
-template 
-concept CanBePiped =
-    requires(View&& view, T&& t) {
-      { std::forward(view) | std::forward(t) };
-    };
-
 static_assert(!CanBePiped);
 static_assert(CanBePiped);
 static_assert(!CanBePiped);
diff --git a/libcxx/test/std/ranges/range.adaptors/range.drop/adaptor.pass.cpp b/libcxx/test/std/ranges/range.adaptors/range.drop/adaptor.pass.cpp
index 457b137c8045..d07fe9ac0ea7 100644
--- a/libcxx/test/std/ranges/range.adaptors/range.drop/adaptor.pass.cpp
+++ b/libcxx/test/std/ranges/range.adaptors/range.drop/adaptor.pass.cpp
@@ -18,12 +18,9 @@
 #include 
 #include 
 #include 
-#include "test_iterators.h"
 
-template 
-concept CanBePiped = requires (View&& view, T&& t) {
-  { std::forward(view) | std::forward(t) };
-};
+#include "test_iterators.h"
+#include "test_range.h"
 
 struct SizedView : std::ranges::view_base {
   int* begin_ = nullptr;
diff --git a/libcxx/test/std/ranges/range.adaptors/range.elements/adaptor.pass.cpp b/libcxx/test/std/ranges/range.adaptors/range.elements/adaptor.pass.cpp
index d68d6e57e2ed..35589c96e4b7 100644
--- a/libcxx/test/std/ranges/range.adaptors/range.elements/adaptor.pass.cpp
+++ b/libcxx/test/std/ranges/range.adaptors/range.elements/adaptor.pass.cpp
@@ -19,6 +19,8 @@
 #include 
 #include 
 
+#include "test_range.h"
+
 template 
 struct View : std::ranges::view_base {
   T* begin() const;
@@ -41,12 +43,6 @@ static_assert(!std::is_invocable_v>);
 static_assert(std::is_invocable_v>>);
 static_assert(!std::is_invocable_v>>);
 
-template 
-concept CanBePiped =
-    requires(View&& view, T&& t) {
-      { std::forward(view) | std::forward(t) };
-    };
-
 static_assert(!CanBePiped, decltype((std::views::elements<0>))>);
 static_assert(CanBePiped>, decltype((std::views::elements<0>))>);
 static_assert(CanBePiped>, decltype((std::views::elements<0>))>);
diff --git a/libcxx/test/std/ranges/range.adaptors/range.filter/adaptor.pass.cpp b/libcxx/test/std/ranges/range.adaptors/range.filter/adaptor.pass.cpp
index 1adf1b4630b8..a44e96e1d4bd 100644
--- a/libcxx/test/std/ranges/range.adaptors/range.filter/adaptor.pass.cpp
+++ b/libcxx/test/std/ranges/range.adaptors/range.filter/adaptor.pass.cpp
@@ -19,11 +19,7 @@
 #include 
 
 #include "test_iterators.h"
-
-template 
-concept CanBePiped = requires (View&& view, T&& t) {
-  { std::forward(view) | std::forward(t) };
-};
+#include "test_range.h"
 
 struct NonCopyablePredicate {
   NonCopyablePredicate(NonCopyablePredicate const&) = delete;
diff --git a/libcxx/test/std/ranges/range.adaptors/range.join/adaptor.pass.cpp b/libcxx/test/std/ranges/range.adaptors/range.join/adaptor.pass.cpp
index 9beb3d282a27..68fdfb5b8475 100644
--- a/libcxx/test/std/ranges/range.adaptors/range.join/adaptor.pass.cpp
+++ b/libcxx/test/std/ranges/range.adaptors/range.join/adaptor.pass.cpp
@@ -15,6 +15,7 @@
 #include 
 #include 
 
+#include "test_range.h"
 #include "types.h"
 
 struct MoveOnlyOuter : SimpleForwardCommonOuter {
@@ -32,11 +33,6 @@ struct Foo {
   constexpr Foo(int ii) : i(ii) {}
 };
 
-template 
-concept CanBePiped = requires(View&& view, T&& t) {
-  { std::forward(view) | std::forward(t) };
-};
-
 constexpr bool test() {
   int buffer1[3] = {1, 2, 3};
   int buffer2[2] = {4, 5};
diff --git a/libcxx/test/std/ranges/range.adaptors/range.lazy.split/adaptor.pass.cpp b/libcxx/test/std/ranges/range.adaptors/range.lazy.split/adaptor.pass.cpp
index 6bfa0ab487ba..7bb7e95b02da 100644
--- a/libcxx/test/std/ranges/range.adaptors/range.lazy.split/adaptor.pass.cpp
+++ b/libcxx/test/std/ranges/range.adaptors/range.lazy.split/adaptor.pass.cpp
@@ -19,13 +19,9 @@
 #include 
 
 #include "test_iterators.h"
+#include "test_range.h"
 #include "types.h"
 
-template 
-concept CanBePiped = requires (View&& view, T&& t) {
-  { std::forward(view) | std::forward(t) };
-};
-
 struct SomeView : std::ranges::view_base {
   const std::string_view* v_;
   constexpr SomeView(const std::string_view& v) : v_(&v) {}
diff --git a/libcxx/test/std/ranges/range.adaptors/range.reverse/adaptor.pass.cpp b/libcxx/test/std/ranges/range.adaptors/range.reverse/adaptor.pass.cpp
index 5e0a0a3ce3fb..26c9ac08d338 100644
--- a/libcxx/test/std/ranges/range.adaptors/range.reverse/adaptor.pass.cpp
+++ b/libcxx/test/std/ranges/range.adaptors/range.reverse/adaptor.pass.cpp
@@ -17,13 +17,9 @@
 #include 
 #include 
 
+#include "test_range.h"
 #include "types.h"
 
-template 
-concept CanBePiped = requires (View&& view, T&& t) {
-  { std::forward(view) | std::forward(t) };
-};
-
 constexpr bool test() {
   int buf[] = {1, 2, 3};
 
diff --git a/libcxx/test/std/ranges/range.adaptors/range.split/adaptor.pass.cpp b/libcxx/test/std/ranges/range.adaptors/range.split/adaptor.pass.cpp
index 85d13ac5c29d..34c0f15ce346 100644
--- a/libcxx/test/std/ranges/range.adaptors/range.split/adaptor.pass.cpp
+++ b/libcxx/test/std/ranges/range.adaptors/range.split/adaptor.pass.cpp
@@ -19,11 +19,7 @@
 #include 
 
 #include "test_iterators.h"
-
-template 
-concept CanBePiped = requires (View&& view, T&& t) {
-  { std::forward(view) | std::forward(t) };
-};
+#include "test_range.h"
 
 struct SomeView : std::ranges::view_base {
   const std::string_view* v_;
diff --git a/libcxx/test/std/ranges/range.adaptors/range.take.while/adaptor.pass.cpp b/libcxx/test/std/ranges/range.adaptors/range.take.while/adaptor.pass.cpp
index 8796f9df63ce..c224d198d16b 100644
--- a/libcxx/test/std/ranges/range.adaptors/range.take.while/adaptor.pass.cpp
+++ b/libcxx/test/std/ranges/range.adaptors/range.take.while/adaptor.pass.cpp
@@ -16,6 +16,7 @@
 #include 
 #include 
 
+#include "test_range.h"
 #include "types.h"
 
 struct Pred {
@@ -42,12 +43,6 @@ static_assert(std::is_invocable_v);
 static_assert(std::is_invocable_v);
 
-template 
-concept CanBePiped =
-    requires(View&& view, T&& t) {
-      { std::forward(view) | std::forward(t) };
-    };
-
 static_assert(!CanBePiped);
 static_assert(CanBePiped);
 static_assert(!CanBePiped);
diff --git a/libcxx/test/std/ranges/range.adaptors/range.take/adaptor.pass.cpp b/libcxx/test/std/ranges/range.adaptors/range.take/adaptor.pass.cpp
index bb5b5f5ff490..b30c4fbe3fd9 100644
--- a/libcxx/test/std/ranges/range.adaptors/range.take/adaptor.pass.cpp
+++ b/libcxx/test/std/ranges/range.adaptors/range.take/adaptor.pass.cpp
@@ -17,12 +17,9 @@
 #include 
 #include 
 #include 
-#include "test_iterators.h"
 
-template 
-concept CanBePiped = requires (View&& view, T&& t) {
-  { std::forward(view) | std::forward(t) };
-};
+#include "test_iterators.h"
+#include "test_range.h"
 
 struct SizedView : std::ranges::view_base {
   int* begin_ = nullptr;
diff --git a/libcxx/test/std/ranges/range.adaptors/range.transform/adaptor.pass.cpp b/libcxx/test/std/ranges/range.adaptors/range.transform/adaptor.pass.cpp
index 59dd85bf28bc..91d6f2eab18b 100644
--- a/libcxx/test/std/ranges/range.adaptors/range.transform/adaptor.pass.cpp
+++ b/libcxx/test/std/ranges/range.adaptors/range.transform/adaptor.pass.cpp
@@ -18,13 +18,9 @@
 #include 
 
 #include "test_macros.h"
+#include "test_range.h"
 #include "types.h"
 
-template 
-concept CanBePiped = requires (View&& view, T&& t) {
-  { std::forward(view) | std::forward(t) };
-};
-
 struct NonCopyableFunction {
   NonCopyableFunction(NonCopyableFunction const&) = delete;
   template 
diff --git a/libcxx/test/support/test_range.h b/libcxx/test/support/test_range.h
index 6061f710a263..c5eeb25bb59e 100644
--- a/libcxx/test/support/test_range.h
+++ b/libcxx/test/support/test_range.h
@@ -89,4 +89,9 @@ concept simple_view =
     std::same_as, std::ranges::iterator_t> &&
     std::same_as, std::ranges::sentinel_t>;
 
+template 
+concept CanBePiped = requires(View&& view, T&& t) {
+  { std::forward(view) | std::forward(t) };
+};
+
 #endif // LIBCXX_TEST_SUPPORT_TEST_RANGE_H
-- 
GitLab


From 01e5d4609bed5db52890cb66a91c65e9bb6a122f Mon Sep 17 00:00:00 2001
From: Stephen Tozer 
Date: Fri, 8 Mar 2024 15:03:15 +0000
Subject: [PATCH 0036/1674] [RemoveDIs] Add documentation for IR debug records
 (#81156)

This patch adds minimal documentation for the IR representation of the
RemoveDIs model of debug info. This patch assumes
that the default for all cases is still debug intrinsic functions, and
so does not update all existing text to refer to debug records, but only
adds a section in the LangRef and SourceLevelDebugging documents to
explain the new format.
---
 llvm/docs/LangRef.rst              | 31 +++++++++++++-
 llvm/docs/SourceLevelDebugging.rst | 66 ++++++++++++++++++++++++++++++
 2 files changed, 95 insertions(+), 2 deletions(-)

diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index e0f301aa06dc..a7b77d6f776a 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -908,7 +908,8 @@ Syntax::
 
 A function definition contains a list of basic blocks, forming the CFG (Control
 Flow Graph) for the function. Each basic block may optionally start with a label
-(giving the basic block a symbol table entry), contains a list of instructions,
+(giving the basic block a symbol table entry), contains a list of instructions
+and :ref:`debug records `,
 and ends with a :ref:`terminator ` instruction (such as a branch or
 function return). If an explicit label name is not provided, a block is assigned
 an implicit numbered label, using the next value from the same counter as used
@@ -8541,7 +8542,10 @@ The LLVM instruction set consists of several different classifications
 of instructions: :ref:`terminator instructions `, :ref:`binary
 instructions `, :ref:`bitwise binary
 instructions `, :ref:`memory instructions `, and
-:ref:`other instructions `.
+:ref:`other instructions `. There are also :ref:`debug records
+`, which are not instructions themselves but are printed
+interleaved with instructions to describe changes in the state of the program's
+debug information at each position in the program's execution.
 
 .. _terminators:
 
@@ -12695,6 +12699,29 @@ Example:
 
       %tok = cleanuppad within %cs []
 
+.. _debugrecords:
+
+Debug Records
+-----------------------
+
+Debug records appear interleaved with instructions, but are not instructions;
+they are used only to define debug information, and have no effect on generated
+code. They are distinguished from instructions by the use of a leading `#` and
+an extra level of indentation. As an example:
+
+.. code-block:: llvm
+
+  %inst1 = op1 %a, %b
+    #dbg_value(%inst1, !10, !DIExpression(), !11)
+  %inst2 = op2 %inst1, %c
+
+These debug records are an optional replacement for
+:ref:`debug intrinsics`. Debug records will be output if the
+``--write-experimental-debuginfo`` flag is passed to LLVM; it is an error for both
+records and intrinsics to appear in the same module. More information about
+debug records can be found in the `LLVM Source Level Debugging
+`_ document.
+
 .. _intrinsics:
 
 Intrinsic Functions
diff --git a/llvm/docs/SourceLevelDebugging.rst b/llvm/docs/SourceLevelDebugging.rst
index e1df7c355ee0..7f7e595eb14d 100644
--- a/llvm/docs/SourceLevelDebugging.rst
+++ b/llvm/docs/SourceLevelDebugging.rst
@@ -167,6 +167,17 @@ conventions used by the C and C++ front-ends.
 Debug information descriptors are `specialized metadata nodes
 `_, first-class subclasses of ``Metadata``.
 
+There are two models for defining the values of source variables at different
+states of the program and tracking these values through optimization and code
+generation: :ref:`intrinsic function calls `, the
+current default, and :ref:`debug records `, which are a new
+non-instruction-based model
+(for an explanation of how this works and why it is desirable, see the
+`RemoveDIs `_ document). Each module must use one or
+the other; they may never be mixed within an IR module. To enable writing debug
+records instead of intrinsic calls, use the flag
+``--write-experimental-debuginfo``.
+
 .. _format_common_intrinsics:
 
 Debugger intrinsic functions
@@ -268,6 +279,61 @@ The formal LLVM-IR signature is:
 
 See :doc:`AssignmentTracking` for more info.
 
+.. _debug_records:
+
+Debug Records
+----------------------------
+
+LLVM also has an alternative to intrinsic functions, debug records, which
+function similarly but are not instructions. The basic syntax for debug records
+is:
+
+.. code-block:: llvm
+
+    #dbg_([, ]* )
+  ; Using the intrinsic model, the above is equivalent to:
+  call void llvm.dbg.([metadata , ]*), !dbg 
+
+A debug intrinsic function can be converted to a debug record with the
+following steps:
+
+1. Add an extra level of indentation.
+2. Replace everything prior to the intrinsic kind (declare/value/assign) with
+   ``#dbg_``.
+3. Remove the leading ``metadata`` from the intrinsic's arguments.
+4. Transfer the ``!dbg`` attachment to be an argument, dropping the leading
+   ``!dbg``.
+
+For each kind of intrinsic function, there is an equivalent debug record.
+
+``#dbg_declare``
+^^^^^^^^^^^^^^^^
+
+.. code-block:: llvm
+
+    #dbg_declare([Value|MDNode], DILocalVariable, DIExpression, DILocation)
+
+Equivalent to the ``llvm.dbg.declare`` intrinsic.
+
+``#dbg_value``
+^^^^^^^^^^^^^^
+
+.. code-block:: llvm
+
+    #dbg_value([Value|DIArgList|MDNode], DILocalVariable, DIExpression, DILocation)
+
+Equivalent to the ``llvm.dbg.value`` intrinsic.
+
+``#dbg_assign``
+^^^^^^^^^^^^^^^
+
+.. code-block:: llvm
+
+    #dbg_assign([Value|DIArgList|MDNode], DILocalVariable, DIExpression,
+                DIAssignID, [Value|MDNode], DIExpression, DILocation)
+
+Equivalent to the ``llvm.dbg.assign`` intrinsic.
+
 Object lifetimes and scoping
 ============================
 
-- 
GitLab


From 23c397c7c9407deea3398e787f2901f86207dc11 Mon Sep 17 00:00:00 2001
From: Guillaume Chatelet 
Date: Fri, 8 Mar 2024 16:06:56 +0100
Subject: [PATCH 0037/1674] [libc] Provide `LIBC_TYPES_HAS_INT128` (#84149)

Umbrella bug #83182
---
 libc/src/__support/CMakeLists.txt             |  2 ++
 libc/src/__support/CPP/CMakeLists.txt         |  1 +
 libc/src/__support/CPP/limits.h               |  5 ++--
 .../__support/CPP/type_traits/is_integral.h   |  3 ++-
 .../__support/CPP/type_traits/make_signed.h   |  3 ++-
 .../__support/CPP/type_traits/make_unsigned.h |  3 ++-
 libc/src/__support/UInt.h                     | 23 ++++++++++---------
 libc/src/__support/UInt128.h                  |  5 ++--
 libc/src/__support/macros/properties/types.h  |  7 ++++++
 libc/test/UnitTest/CMakeLists.txt             |  1 +
 libc/test/UnitTest/LibcTest.cpp               |  5 ++--
 libc/test/UnitTest/TestLogger.cpp             |  5 ++--
 libc/test/src/__support/CMakeLists.txt        |  6 +++--
 libc/test/src/__support/CPP/CMakeLists.txt    |  2 ++
 libc/test/src/__support/CPP/bit_test.cpp      |  9 ++++----
 libc/test/src/__support/CPP/limits_test.cpp   |  5 ++--
 .../src/__support/integer_literals_test.cpp   |  5 ++--
 libc/test/src/__support/uint_test.cpp         |  9 ++++----
 .../llvm-project-overlay/libc/BUILD.bazel     |  3 +++
 .../libc/test/UnitTest/BUILD.bazel            |  2 ++
 .../libc/test/src/__support/BUILD.bazel       |  2 ++
 .../libc/test/src/__support/CPP/BUILD.bazel   |  2 ++
 22 files changed, 72 insertions(+), 36 deletions(-)

diff --git a/libc/src/__support/CMakeLists.txt b/libc/src/__support/CMakeLists.txt
index 66e4d808333f..2e5a026bf423 100644
--- a/libc/src/__support/CMakeLists.txt
+++ b/libc/src/__support/CMakeLists.txt
@@ -205,6 +205,7 @@ add_header_library(
     libc.src.__support.CPP.bit
     libc.src.__support.CPP.type_traits
     libc.src.__support.macros.optimization
+    libc.src.__support.macros.properties.types
 )
 
 add_header_library(
@@ -213,6 +214,7 @@ add_header_library(
     UInt128.h
   DEPENDS
     .uint
+    libc.src.__support.macros.properties.types
 )
 
 add_header_library(
diff --git a/libc/src/__support/CPP/CMakeLists.txt b/libc/src/__support/CPP/CMakeLists.txt
index 6c35bc709081..6216505eae23 100644
--- a/libc/src/__support/CPP/CMakeLists.txt
+++ b/libc/src/__support/CPP/CMakeLists.txt
@@ -49,6 +49,7 @@ add_header_library(
   DEPENDS
     .type_traits
     libc.include.llvm-libc-macros.limits_macros
+    libc.src.__support.macros.properties.types
 )
 
 add_header_library(
diff --git a/libc/src/__support/CPP/limits.h b/libc/src/__support/CPP/limits.h
index 1ffde5f9556f..5b9b3e755c72 100644
--- a/libc/src/__support/CPP/limits.h
+++ b/libc/src/__support/CPP/limits.h
@@ -12,7 +12,8 @@
 #include "include/llvm-libc-macros/limits-macros.h" // CHAR_BIT
 #include "src/__support/CPP/type_traits/is_integral.h"
 #include "src/__support/CPP/type_traits/is_signed.h"
-#include "src/__support/macros/attributes.h" // LIBC_INLINE
+#include "src/__support/macros/attributes.h"       // LIBC_INLINE
+#include "src/__support/macros/properties/types.h" // LIBC_TYPES_HAS_INT128
 
 namespace LIBC_NAMESPACE {
 namespace cpp {
@@ -76,7 +77,7 @@ template <>
 struct numeric_limits
     : public internal::integer_impl {};
 
-#ifdef __SIZEOF_INT128__
+#ifdef LIBC_TYPES_HAS_INT128
 // On platform where UInt128 resolves to __uint128_t, this specialization
 // provides the limits of UInt128.
 template <>
diff --git a/libc/src/__support/CPP/type_traits/is_integral.h b/libc/src/__support/CPP/type_traits/is_integral.h
index 2808be594b20..68e16ff84183 100644
--- a/libc/src/__support/CPP/type_traits/is_integral.h
+++ b/libc/src/__support/CPP/type_traits/is_integral.h
@@ -11,6 +11,7 @@
 #include "src/__support/CPP/type_traits/is_same.h"
 #include "src/__support/CPP/type_traits/remove_cv.h"
 #include "src/__support/macros/attributes.h"
+#include "src/__support/macros/properties/types.h" // LIBC_TYPES_HAS_INT128
 
 namespace LIBC_NAMESPACE::cpp {
 
@@ -25,7 +26,7 @@ private:
 public:
   LIBC_INLINE_VAR static constexpr bool value = __is_unqualified_any_of<
       T,
-#ifdef __SIZEOF_INT128__
+#ifdef LIBC_TYPES_HAS_INT128
       __int128_t, __uint128_t,
 #endif
       char, signed char, unsigned char, short, unsigned short, int,
diff --git a/libc/src/__support/CPP/type_traits/make_signed.h b/libc/src/__support/CPP/type_traits/make_signed.h
index 21302850bfd4..4652d8b6bfa5 100644
--- a/libc/src/__support/CPP/type_traits/make_signed.h
+++ b/libc/src/__support/CPP/type_traits/make_signed.h
@@ -9,6 +9,7 @@
 #define LLVM_LIBC_SRC___SUPPORT_CPP_TYPE_TRAITS_MAKE_SIGNED_H
 
 #include "src/__support/CPP/type_traits/type_identity.h"
+#include "src/__support/macros/properties/types.h" // LIBC_TYPES_HAS_INT128
 
 namespace LIBC_NAMESPACE::cpp {
 
@@ -26,7 +27,7 @@ template <> struct make_signed : type_identity {};
 template <> struct make_signed : type_identity {};
 template <>
 struct make_signed : type_identity {};
-#ifdef __SIZEOF_INT128__
+#ifdef LIBC_TYPES_HAS_INT128
 template <> struct make_signed<__int128_t> : type_identity<__int128_t> {};
 template <> struct make_signed<__uint128_t> : type_identity<__int128_t> {};
 #endif
diff --git a/libc/src/__support/CPP/type_traits/make_unsigned.h b/libc/src/__support/CPP/type_traits/make_unsigned.h
index 20948014a665..1e814ae002a7 100644
--- a/libc/src/__support/CPP/type_traits/make_unsigned.h
+++ b/libc/src/__support/CPP/type_traits/make_unsigned.h
@@ -9,6 +9,7 @@
 #define LLVM_LIBC_SRC___SUPPORT_CPP_TYPE_TRAITS_MAKE_UNSIGNED_H
 
 #include "src/__support/CPP/type_traits/type_identity.h"
+#include "src/__support/macros/properties/types.h" // LIBC_TYPES_HAS_INT128
 
 namespace LIBC_NAMESPACE::cpp {
 
@@ -31,7 +32,7 @@ template <>
 struct make_unsigned : type_identity {};
 template <>
 struct make_unsigned : type_identity {};
-#ifdef __SIZEOF_INT128__
+#ifdef LIBC_TYPES_HAS_INT128
 template <> struct make_unsigned<__int128_t> : type_identity<__uint128_t> {};
 template <> struct make_unsigned<__uint128_t> : type_identity<__uint128_t> {};
 #endif
diff --git a/libc/src/__support/UInt.h b/libc/src/__support/UInt.h
index 94c8d86fd764..c49c8314cd49 100644
--- a/libc/src/__support/UInt.h
+++ b/libc/src/__support/UInt.h
@@ -14,9 +14,10 @@
 #include "src/__support/CPP/limits.h"
 #include "src/__support/CPP/optional.h"
 #include "src/__support/CPP/type_traits.h"
-#include "src/__support/macros/attributes.h"   // LIBC_INLINE
-#include "src/__support/macros/optimization.h" // LIBC_UNLIKELY
-#include "src/__support/math_extras.h"         // SumCarry, DiffBorrow
+#include "src/__support/macros/attributes.h"       // LIBC_INLINE
+#include "src/__support/macros/optimization.h"     // LIBC_UNLIKELY
+#include "src/__support/macros/properties/types.h" // LIBC_TYPES_HAS_INT128
+#include "src/__support/math_extras.h"             // SumCarry, DiffBorrow
 #include "src/__support/number_pair.h"
 
 #include  // For size_t
@@ -30,9 +31,9 @@ template  struct half_width;
 template <> struct half_width : cpp::type_identity {};
 template <> struct half_width : cpp::type_identity {};
 template <> struct half_width : cpp::type_identity {};
-#ifdef __SIZEOF_INT128__
+#ifdef LIBC_TYPES_HAS_INT128
 template <> struct half_width<__uint128_t> : cpp::type_identity {};
-#endif // __SIZEOF_INT128__
+#endif // LIBC_TYPES_HAS_INT128
 
 template  using half_width_t = typename half_width::type;
 
@@ -69,7 +70,7 @@ LIBC_INLINE constexpr NumberPair full_mul(uint32_t a,
   return result;
 }
 
-#ifdef __SIZEOF_INT128__
+#ifdef LIBC_TYPES_HAS_INT128
 template <>
 LIBC_INLINE constexpr NumberPair full_mul(uint64_t a,
                                                               uint64_t b) {
@@ -79,7 +80,7 @@ LIBC_INLINE constexpr NumberPair full_mul(uint64_t a,
   result.hi = uint64_t(prod >> 64);
   return result;
 }
-#endif // __SIZEOF_INT128__
+#endif // LIBC_TYPES_HAS_INT128
 
 } // namespace internal
 
@@ -682,7 +683,7 @@ struct BigInt {
       val[1] = uint32_t(tmp >> 32);
       return;
     }
-#ifdef __SIZEOF_INT128__
+#ifdef LIBC_TYPES_HAS_INT128
     if constexpr ((Bits == 128) && (WORD_SIZE == 64)) {
       // Use builtin 128 bits if available;
       if (s >= 128) {
@@ -696,7 +697,7 @@ struct BigInt {
       val[1] = uint64_t(tmp >> 64);
       return;
     }
-#endif // __SIZEOF_INT128__
+#endif // LIBC_TYPES_HAS_INT128
     if (LIBC_UNLIKELY(s == 0))
       return;
 
@@ -753,7 +754,7 @@ struct BigInt {
       val[1] = uint32_t(tmp >> 32);
       return;
     }
-#ifdef __SIZEOF_INT128__
+#ifdef LIBC_TYPES_HAS_INT128
     if constexpr ((Bits == 128) && (WORD_SIZE == 64)) {
       // Use builtin 128 bits if available;
       if (s >= 128) {
@@ -771,7 +772,7 @@ struct BigInt {
       val[1] = uint64_t(tmp >> 64);
       return;
     }
-#endif // __SIZEOF_INT128__
+#endif // LIBC_TYPES_HAS_INT128
 
     if (LIBC_UNLIKELY(s == 0))
       return;
diff --git a/libc/src/__support/UInt128.h b/libc/src/__support/UInt128.h
index 06696b7a6158..b6ef9ca18eb0 100644
--- a/libc/src/__support/UInt128.h
+++ b/libc/src/__support/UInt128.h
@@ -10,13 +10,14 @@
 #define LLVM_LIBC_SRC___SUPPORT_UINT128_H
 
 #include "UInt.h"
+#include "src/__support/macros/properties/types.h" // LIBC_TYPES_HAS_INT128
 
-#if defined(__SIZEOF_INT128__)
+#ifdef LIBC_TYPES_HAS_INT128
 using UInt128 = __uint128_t;
 using Int128 = __int128_t;
 #else
 using UInt128 = LIBC_NAMESPACE::UInt<128>;
 using Int128 = LIBC_NAMESPACE::Int<128>;
-#endif
+#endif // LIBC_TYPES_HAS_INT128
 
 #endif // LLVM_LIBC_SRC___SUPPORT_UINT128_H
diff --git a/libc/src/__support/macros/properties/types.h b/libc/src/__support/macros/properties/types.h
index 8760f78875c4..42345e4743ce 100644
--- a/libc/src/__support/macros/properties/types.h
+++ b/libc/src/__support/macros/properties/types.h
@@ -17,6 +17,8 @@
 #include "src/__support/macros/properties/cpu_features.h"
 #include "src/__support/macros/properties/os.h"
 
+#include  // __SIZEOF_INT128__
+
 // 'long double' properties.
 #if (LDBL_MANT_DIG == 53)
 #define LIBC_TYPES_LONG_DOUBLE_IS_FLOAT64
@@ -26,6 +28,11 @@
 #define LIBC_TYPES_LONG_DOUBLE_IS_FLOAT128
 #endif
 
+// int128 / uint128 support
+#if defined(__SIZEOF_INT128__)
+#define LIBC_TYPES_HAS_INT128
+#endif // defined(__SIZEOF_INT128__)
+
 // -- float16 support ---------------------------------------------------------
 // TODO: move this logic to "llvm-libc-types/float16.h"
 #if defined(LIBC_TARGET_ARCH_IS_X86_64) && defined(LIBC_TARGET_CPU_HAS_SSE2)
diff --git a/libc/test/UnitTest/CMakeLists.txt b/libc/test/UnitTest/CMakeLists.txt
index 8a35f1204eb5..f7a6f4a91fab 100644
--- a/libc/test/UnitTest/CMakeLists.txt
+++ b/libc/test/UnitTest/CMakeLists.txt
@@ -73,6 +73,7 @@ add_unittest_framework_library(
     libc.src.__support.CPP.string_view
     libc.src.__support.CPP.type_traits
     libc.src.__support.fixed_point.fx_rep
+    libc.src.__support.macros.properties.types
     libc.src.__support.OSUtil.osutil
     libc.src.__support.uint
     libc.src.__support.uint128
diff --git a/libc/test/UnitTest/LibcTest.cpp b/libc/test/UnitTest/LibcTest.cpp
index 7dc85d0d4e2d..03cd25191ecd 100644
--- a/libc/test/UnitTest/LibcTest.cpp
+++ b/libc/test/UnitTest/LibcTest.cpp
@@ -13,6 +13,7 @@
 #include "src/__support/CPP/string_view.h"
 #include "src/__support/UInt128.h"
 #include "src/__support/fixed_point/fx_rep.h"
+#include "src/__support/macros/properties/types.h" // LIBC_TYPES_HAS_INT128
 #include "test/UnitTest/TestLogger.h"
 
 #if __STDC_HOSTED__
@@ -215,11 +216,11 @@ TEST_SPECIALIZATION(bool);
 
 // We cannot just use a single UInt128 specialization as that resolves to only
 // one type, UInt<128> or __uint128_t. We want both overloads as we want to
-#ifdef __SIZEOF_INT128__
+#ifdef LIBC_TYPES_HAS_INT128
 // When builtin __uint128_t type is available, include its specialization
 // also.
 TEST_SPECIALIZATION(__uint128_t);
-#endif
+#endif // LIBC_TYPES_HAS_INT128
 
 TEST_SPECIALIZATION(LIBC_NAMESPACE::Int<128>);
 
diff --git a/libc/test/UnitTest/TestLogger.cpp b/libc/test/UnitTest/TestLogger.cpp
index 32f0ec52041f..4756188b46cb 100644
--- a/libc/test/UnitTest/TestLogger.cpp
+++ b/libc/test/UnitTest/TestLogger.cpp
@@ -4,6 +4,7 @@
 #include "src/__support/OSUtil/io.h" // write_to_stderr
 #include "src/__support/UInt.h"      // is_big_int
 #include "src/__support/UInt128.h"
+#include "src/__support/macros/properties/types.h" // LIBC_TYPES_HAS_INT128
 
 #include 
 
@@ -72,9 +73,9 @@ template TestLogger &TestLogger::operator<< (unsigned long);
 template TestLogger &
     TestLogger::operator<< (unsigned long long);
 
-#ifdef __SIZEOF_INT128__
+#ifdef LIBC_TYPES_HAS_INT128
 template TestLogger &TestLogger::operator<< <__uint128_t>(__uint128_t);
-#endif
+#endif // LIBC_TYPES_HAS_INT128
 template TestLogger &TestLogger::operator<< >(UInt<128>);
 template TestLogger &TestLogger::operator<< >(UInt<192>);
 template TestLogger &TestLogger::operator<< >(UInt<256>);
diff --git a/libc/test/src/__support/CMakeLists.txt b/libc/test/src/__support/CMakeLists.txt
index adbacb9728cc..91dd0dc4decf 100644
--- a/libc/test/src/__support/CMakeLists.txt
+++ b/libc/test/src/__support/CMakeLists.txt
@@ -106,8 +106,9 @@ if(NOT LIBC_TARGET_ARCHITECTURE_IS_NVPTX)
     SRCS
       uint_test.cpp
     DEPENDS
-      libc.src.__support.uint
       libc.src.__support.CPP.optional
+      libc.src.__support.macros.properties.types
+      libc.src.__support.uint
   )
 endif()
 
@@ -118,8 +119,9 @@ add_libc_test(
   SRCS
     integer_literals_test.cpp
   DEPENDS
-    libc.src.__support.integer_literals
     libc.src.__support.CPP.optional
+    libc.src.__support.integer_literals
+    libc.src.__support.macros.properties.types
 )
 
 add_libc_test(
diff --git a/libc/test/src/__support/CPP/CMakeLists.txt b/libc/test/src/__support/CPP/CMakeLists.txt
index d7f332f5b0fb..f94429e03b3c 100644
--- a/libc/test/src/__support/CPP/CMakeLists.txt
+++ b/libc/test/src/__support/CPP/CMakeLists.txt
@@ -8,6 +8,7 @@ add_libc_test(
     bit_test.cpp
   DEPENDS
     libc.src.__support.CPP.bit
+    libc.src.__support.macros.properties.types
     libc.src.__support.uint
 )
 
@@ -49,6 +50,7 @@ add_libc_test(
     limits_test.cpp
   DEPENDS
     libc.src.__support.CPP.limits
+    libc.src.__support.macros.properties.types
     libc.src.__support.uint
 )
 
diff --git a/libc/test/src/__support/CPP/bit_test.cpp b/libc/test/src/__support/CPP/bit_test.cpp
index da7bb3389b2f..3deb1f41dcf8 100644
--- a/libc/test/src/__support/CPP/bit_test.cpp
+++ b/libc/test/src/__support/CPP/bit_test.cpp
@@ -8,6 +8,7 @@
 
 #include "src/__support/CPP/bit.h"
 #include "src/__support/UInt.h"
+#include "src/__support/macros/properties/types.h" // LIBC_TYPES_HAS_INT128
 #include "test/UnitTest/Test.h"
 
 #include 
@@ -15,16 +16,16 @@
 namespace LIBC_NAMESPACE::cpp {
 
 using UnsignedTypesNoBigInt = testing::TypeList<
-#if defined(__SIZEOF_INT128__)
+#if defined(LIBC_TYPES_HAS_INT128)
     __uint128_t,
-#endif
+#endif // LIBC_TYPES_HAS_INT128
     unsigned char, unsigned short, unsigned int, unsigned long,
     unsigned long long>;
 
 using UnsignedTypes = testing::TypeList<
-#if defined(__SIZEOF_INT128__)
+#if defined(LIBC_TYPES_HAS_INT128)
     __uint128_t,
-#endif
+#endif // LIBC_TYPES_HAS_INT128
     unsigned char, unsigned short, unsigned int, unsigned long,
     unsigned long long, UInt<128>>;
 
diff --git a/libc/test/src/__support/CPP/limits_test.cpp b/libc/test/src/__support/CPP/limits_test.cpp
index d83e307fb196..efcd6839d073 100644
--- a/libc/test/src/__support/CPP/limits_test.cpp
+++ b/libc/test/src/__support/CPP/limits_test.cpp
@@ -8,6 +8,7 @@
 
 #include "src/__support/CPP/limits.h"
 #include "src/__support/UInt.h"
+#include "src/__support/macros/properties/types.h" // LIBC_TYPES_HAS_INT128
 #include "test/UnitTest/Test.h"
 
 namespace LIBC_NAMESPACE {
@@ -36,9 +37,9 @@ TEST(LlvmLibcLimitsTest, UInt128Limits) {
   auto umax64 = LIBC_NAMESPACE::UInt<128>(cpp::numeric_limits::max());
   EXPECT_GT(umax128, umax64);
   ASSERT_EQ(~LIBC_NAMESPACE::UInt<128>(0), umax128);
-#ifdef __SIZEOF_INT128__
+#ifdef LIBC_TYPES_HAS_INT128
   ASSERT_EQ(~__uint128_t(0), cpp::numeric_limits<__uint128_t>::max());
-#endif
+#endif // LIBC_TYPES_HAS_INT128
 }
 
 } // namespace LIBC_NAMESPACE
diff --git a/libc/test/src/__support/integer_literals_test.cpp b/libc/test/src/__support/integer_literals_test.cpp
index b8b399d9cf40..5298cf30156e 100644
--- a/libc/test/src/__support/integer_literals_test.cpp
+++ b/libc/test/src/__support/integer_literals_test.cpp
@@ -8,6 +8,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/__support/integer_literals.h"
+#include "src/__support/macros/properties/types.h" // LIBC_TYPES_HAS_INT128
 #include "test/UnitTest/Test.h"
 
 using LIBC_NAMESPACE::operator""_u8;
@@ -66,7 +67,7 @@ TEST(LlvmLibcIntegerLiteralTest, u64) {
 }
 
 TEST(LlvmLibcIntegerLiteralTest, u128) {
-#if defined(__SIZEOF_INT128__)
+#ifdef LIBC_TYPES_HAS_INT128
   const __uint128_t ZERO = 0;
   const __uint128_t U8_MAX = UINT8_MAX;
   const __uint128_t U16_MAX = UINT16_MAX;
@@ -80,7 +81,7 @@ TEST(LlvmLibcIntegerLiteralTest, u128) {
   const UInt128 U32_MAX = UINT32_MAX;
   const UInt128 U64_MAX = UINT64_MAX;
   const UInt128 U128_MAX = (U64_MAX << 64) | U64_MAX;
-#endif
+#endif // LIBC_TYPES_HAS_INT128
   EXPECT_EQ(ZERO, 0_u128);
   EXPECT_EQ(U8_MAX, 255_u128);
   EXPECT_EQ(U8_MAX, 0xFF_u128);
diff --git a/libc/test/src/__support/uint_test.cpp b/libc/test/src/__support/uint_test.cpp
index eb1db9729cf4..851656e9fcbb 100644
--- a/libc/test/src/__support/uint_test.cpp
+++ b/libc/test/src/__support/uint_test.cpp
@@ -8,6 +8,7 @@
 
 #include "src/__support/CPP/optional.h"
 #include "src/__support/UInt.h"
+#include "src/__support/macros/properties/types.h" // LIBC_TYPES_HAS_INT128
 
 #include "test/UnitTest/Test.h"
 #include  // HUGE_VALF, HUGE_VALF
@@ -41,7 +42,7 @@ TEST(LlvmLibcUIntClassTest, BitCastToFromDouble) {
   }
 }
 
-#ifdef __SIZEOF_INT128__
+#ifdef LIBC_TYPES_HAS_INT128
 TEST(LlvmLibcUIntClassTest, BitCastToFromNativeUint128) {
   static_assert(cpp::is_trivially_copyable::value);
   static_assert(sizeof(LL_UInt128) == sizeof(__uint128_t));
@@ -52,7 +53,7 @@ TEST(LlvmLibcUIntClassTest, BitCastToFromNativeUint128) {
     EXPECT_TRUE(value == forth);
   }
 }
-#endif
+#endif // LIBC_TYPES_HAS_INT128
 
 #ifdef LIBC_TYPES_HAS_FLOAT128
 TEST(LlvmLibcUIntClassTest, BitCastToFromNativeFloat128) {
@@ -652,7 +653,7 @@ TEST(LlvmLibcUIntClassTest, BasicArithmeticInt128Tests) {
   ASSERT_EQ(c * b, b);
 }
 
-#ifdef __SIZEOF_INT128__
+#ifdef LIBC_TYPES_HAS_INT128
 
 TEST(LlvmLibcUIntClassTest, ConstructorFromUInt128Tests) {
   __uint128_t a = (__uint128_t(123) << 64) + 1;
@@ -707,7 +708,7 @@ TEST(LlvmLibcUIntClassTest, WordTypeUInt128Tests) {
   EXPECT_TRUE(f == r);
 }
 
-#endif // __SIZEOF_INT128__
+#endif // LIBC_TYPES_HAS_INT128
 
 TEST(LlvmLibcUIntClassTest, OtherWordTypeTests) {
   using LL_UInt96 = BigInt<96, false, uint32_t>;
diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
index 127059c529ec..5f6c43cd6af7 100644
--- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
@@ -211,6 +211,7 @@ libc_support_library(
     deps = [
         "__support_cpp_type_traits",
         "__support_macros_attributes",
+        ":__support_macros_properties_types",
         ":llvm_libc_macros_limits_macros",
     ],
 )
@@ -449,6 +450,7 @@ libc_support_library(
         ":__support_cpp_type_traits",
         ":__support_macros_attributes",
         ":__support_macros_optimization",
+        ":__support_macros_properties_types",
         ":__support_math_extras",
         ":__support_number_pair",
     ],
@@ -458,6 +460,7 @@ libc_support_library(
     name = "__support_uint128",
     hdrs = ["src/__support/UInt128.h"],
     deps = [
+        ":__support_macros_properties_types",
         ":__support_uint",
     ],
 )
diff --git a/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel
index 37eb21c77e49..4a94916e1205 100644
--- a/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel
@@ -17,6 +17,7 @@ libc_support_library(
     deps = [
         "//libc:__support_cpp_string",
         "//libc:__support_cpp_string_view",
+        "//libc:__support_macros_properties_types",
         "//libc:__support_osutil_io",
         "//libc:__support_uint",
         "//libc:__support_uint128",
@@ -52,6 +53,7 @@ libc_support_library(
         "//libc:__support_fputil_fpbits_str",
         "//libc:__support_fputil_rounding_mode",
         "//libc:__support_macros_properties_architectures",
+        "//libc:__support_macros_properties_types",
         "//libc:__support_stringutil",
         "//libc:__support_uint128",
         "//libc:errno",
diff --git a/utils/bazel/llvm-project-overlay/libc/test/src/__support/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/src/__support/BUILD.bazel
index 19d4c7869799..6837b9880d5a 100644
--- a/utils/bazel/llvm-project-overlay/libc/test/src/__support/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/test/src/__support/BUILD.bazel
@@ -86,6 +86,7 @@ libc_test(
     srcs = ["uint_test.cpp"],
     deps = [
         "//libc:__support_cpp_optional",
+        "//libc:__support_macros_properties_types",
         "//libc:__support_uint",
     ],
 )
@@ -111,5 +112,6 @@ libc_test(
     srcs = ["integer_literals_test.cpp"],
     deps = [
         "//libc:__support_integer_literals",
+        "//libc:__support_macros_properties_types",
     ],
 )
diff --git a/utils/bazel/llvm-project-overlay/libc/test/src/__support/CPP/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/src/__support/CPP/BUILD.bazel
index 07d9ac1d5008..dad1c7708e44 100644
--- a/utils/bazel/llvm-project-overlay/libc/test/src/__support/CPP/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/test/src/__support/CPP/BUILD.bazel
@@ -27,6 +27,7 @@ libc_test(
     srcs = ["bit_test.cpp"],
     deps = [
         "//libc:__support_cpp_bit",
+        "//libc:__support_macros_properties_types",
         "//libc:__support_uint",
     ],
 )
@@ -48,6 +49,7 @@ libc_test(
     srcs = ["limits_test.cpp"],
     deps = [
         "//libc:__support_cpp_limits",
+        "//libc:__support_macros_properties_types",
         "//libc:__support_uint",
     ],
 )
-- 
GitLab


From cb6f657a70f7a8d6ecd4fcc2101550a7400f94a7 Mon Sep 17 00:00:00 2001
From: Joseph Huber 
Date: Fri, 8 Mar 2024 09:06:12 -0600
Subject: [PATCH 0038/1674] [CUDA][HIP][NFC] Precommit new driver changes to
 tests

Summary:
A lot of these tests implicitly rely on `--no-offload-new-driver` a
patch in the future will change the default so this patch makes it
explicit in locations where it matters.
---
 clang/test/Driver/cuda-bad-arch.cu            |   4 +-
 clang/test/Driver/cuda-external-tools.cu      |  14 +--
 clang/test/Driver/cuda-phases.cu              |  24 ++--
 clang/test/Driver/hip-binding.hip             |  10 +-
 clang/test/Driver/hip-cuid-hash.hip           |   8 +-
 clang/test/Driver/hip-cuid.hip                |   5 +
 clang/test/Driver/hip-dependent-options.hip   |   4 +-
 clang/test/Driver/hip-device-compile.hip      |  22 ++--
 clang/test/Driver/hip-link-bc-to-bc.hip       |   4 +-
 clang/test/Driver/hip-link-bundle-archive.hip |  22 ++--
 clang/test/Driver/hip-link-save-temps.hip     |   8 +-
 clang/test/Driver/hip-link-shared-library.hip |   3 +-
 clang/test/Driver/hip-link-static-library.hip |   6 +-
 .../test/Driver/hip-offload-compress-zlib.hip |   6 +-
 .../test/Driver/hip-offload-compress-zstd.hip |   6 +-
 clang/test/Driver/hip-output-file-name.hip    |  26 ++---
 clang/test/Driver/hip-partial-link.hip        |  14 +--
 clang/test/Driver/hip-phases.hip              | 106 +++++++++---------
 clang/test/Driver/hip-rdc-device-only.hip     |  14 +--
 clang/test/Driver/hip-target-id.hip           |  10 +-
 clang/test/Driver/hip-toolchain-features.hip  |  24 ++--
 .../Driver/hip-toolchain-rdc-separate.hip     |   8 +-
 .../Driver/hip-toolchain-rdc-static-lib.hip   |   2 +-
 clang/test/Driver/hip-toolchain-rdc.hip       |   4 +-
 clang/test/Driver/hip-unbundle-preproc.hipi   |   6 +-
 clang/test/Driver/hipspv-toolchain-rdc.hip    |   2 +-
 clang/test/Driver/hipspv-toolchain.hip        |   2 +-
 clang/test/Driver/lto.cu                      |  22 ++--
 clang/test/Driver/thinlto.cu                  |   4 +-
 29 files changed, 197 insertions(+), 193 deletions(-)

diff --git a/clang/test/Driver/cuda-bad-arch.cu b/clang/test/Driver/cuda-bad-arch.cu
index 877b20bc9351..35a56a8bef0f 100644
--- a/clang/test/Driver/cuda-bad-arch.cu
+++ b/clang/test/Driver/cuda-bad-arch.cu
@@ -30,9 +30,9 @@
 // RUN: | FileCheck -check-prefix OK %s
 
 // We don't allow using NVPTX/AMDGCN for host compilation.
-// RUN: not %clang -### --cuda-host-only --target=nvptx-nvidia-cuda -nogpulib -nogpuinc -c %s 2>&1 \
+// RUN: not %clang -### --no-offload-new-driver --cuda-host-only --target=nvptx-nvidia-cuda -nogpulib -nogpuinc -c %s 2>&1 \
 // RUN: | FileCheck -check-prefix HOST_NVPTX %s
-// RUN: not %clang -### --cuda-host-only --target=amdgcn-amd-amdhsa -nogpulib -nogpuinc -c %s 2>&1 \
+// RUN: not %clang -### --no-offload-new-driver --cuda-host-only --target=amdgcn-amd-amdhsa -nogpulib -nogpuinc -c %s 2>&1 \
 // RUN: | FileCheck -check-prefix HOST_AMDGCN %s
 
 // OK-NOT: error: Unsupported CUDA gpu architecture
diff --git a/clang/test/Driver/cuda-external-tools.cu b/clang/test/Driver/cuda-external-tools.cu
index 1aa87cc09982..946e144fce38 100644
--- a/clang/test/Driver/cuda-external-tools.cu
+++ b/clang/test/Driver/cuda-external-tools.cu
@@ -25,7 +25,7 @@
 // RUN: | FileCheck -check-prefixes=CHECK,ARCH64,SM35,OPT3 %s
 // Generating relocatable device code
 // RUN: %clang -### --target=x86_64-linux-gnu -fgpu-rdc -c %s 2>&1 \
-// RUN:   --offload-arch=sm_35 --cuda-path=%S/Inputs/CUDA/usr/local/cuda \
+// RUN:   --no-offload-new-driver --offload-arch=sm_35 --cuda-path=%S/Inputs/CUDA/usr/local/cuda \
 // RUN: | FileCheck -check-prefixes=CHECK,ARCH64,SM35,RDC %s
 
 // With debugging enabled, ptxas should be run with with no ptxas optimizations.
@@ -59,7 +59,7 @@
 // RUN: | FileCheck -check-prefixes=CHECK,ARCH64,SM35 %s
 // Separate compilation targeting sm_35.
 // RUN: %clang -### --target=x86_64-linux-gnu --cuda-gpu-arch=sm_35 -fgpu-rdc -c %s 2>&1 \
-// RUN:   --offload-arch=sm_35 --cuda-path=%S/Inputs/CUDA/usr/local/cuda \
+// RUN:   --no-offload-new-driver --offload-arch=sm_35 --cuda-path=%S/Inputs/CUDA/usr/local/cuda \
 // RUN: | FileCheck -check-prefixes=CHECK,ARCH64,SM35,RDC %s
 
 // 32-bit compile.
@@ -68,7 +68,7 @@
 // RUN: | FileCheck -check-prefixes=CHECK,ARCH32,SM35 %s
 // 32-bit compile when generating relocatable device code.
 // RUN: %clang -### --target=i386-linux-gnu -fgpu-rdc -c %s 2>&1 \
-// RUN:   --offload-arch=sm_35 --cuda-path=%S/Inputs/CUDA/usr/local/cuda \
+// RUN:   --no-offload-new-driver --offload-arch=sm_35 --cuda-path=%S/Inputs/CUDA/usr/local/cuda \
 // RUN: | FileCheck -check-prefixes=CHECK,ARCH32,SM35,RDC %s
 
 // Compile with -fintegrated-as.  This should still cause us to invoke ptxas.
@@ -77,7 +77,7 @@
 // RUN: | FileCheck -check-prefixes=CHECK,ARCH64,SM35,OPT0 %s
 // Check that we still pass -c when generating relocatable device code.
 // RUN: %clang -### --target=x86_64-linux-gnu -fintegrated-as -fgpu-rdc -c %s 2>&1 \
-// RUN:   --offload-arch=sm_35 --cuda-path=%S/Inputs/CUDA/usr/local/cuda \
+// RUN:   --no-offload-new-driver --offload-arch=sm_35 --cuda-path=%S/Inputs/CUDA/usr/local/cuda \
 // RUN: | FileCheck -check-prefixes=CHECK,ARCH64,SM35,RDC %s
 
 // Check -Xcuda-ptxas and -Xcuda-fatbinary
@@ -99,13 +99,13 @@
 
 // Check relocatable device code generation on MacOS.
 // RUN: %clang -### --target=x86_64-apple-macosx -O0 -fgpu-rdc -c %s 2>&1 \
-// RUN:   --offload-arch=sm_35 --cuda-path=%S/Inputs/CUDA/usr/local/cuda \
+// RUN:   --no-offload-new-driver --offload-arch=sm_35 --cuda-path=%S/Inputs/CUDA/usr/local/cuda \
 // RUN: | FileCheck -check-prefixes=CHECK,ARCH64,SM35,RDC %s
 // RUN: %clang -### --target=x86_64-apple-macosx --cuda-gpu-arch=sm_35 -fgpu-rdc -c %s 2>&1 \
-// RUN:   --offload-arch=sm_35 --cuda-path=%S/Inputs/CUDA/usr/local/cuda \
+// RUN:   --no-offload-new-driver --offload-arch=sm_35 --cuda-path=%S/Inputs/CUDA/usr/local/cuda \
 // RUN: | FileCheck -check-prefixes=CHECK,ARCH64,SM35,RDC %s
 // RUN: %clang -### --target=i386-apple-macosx -fgpu-rdc -c %s 2>&1 \
-// RUN:   --offload-arch=sm_35 --cuda-path=%S/Inputs/CUDA/usr/local/cuda \
+// RUN:   --no-offload-new-driver --offload-arch=sm_35 --cuda-path=%S/Inputs/CUDA/usr/local/cuda \
 // RUN: | FileCheck -check-prefixes=CHECK,ARCH32,SM35,RDC %s
 
 // Check that CLANG forwards the -v flag to PTXAS.
diff --git a/clang/test/Driver/cuda-phases.cu b/clang/test/Driver/cuda-phases.cu
index a1c3c9b51b1e..85b1a550524d 100644
--- a/clang/test/Driver/cuda-phases.cu
+++ b/clang/test/Driver/cuda-phases.cu
@@ -11,7 +11,7 @@
 //
 // Test CUDA NVPTX phases.
 // RUN: %clang -target powerpc64le-ibm-linux-gnu -ccc-print-phases \
-// RUN: --cuda-gpu-arch=sm_30 %s 2>&1 \
+// RUN:   --no-offload-new-driver --cuda-gpu-arch=sm_30 %s 2>&1 \
 // RUN: | FileCheck -check-prefixes=BIN %s
 //
 // BIN-DAG: [[P0:[0-9]+]]: input, "{{.*}}cuda-phases.cu", [[T:cuda]], (host-[[T]])
@@ -34,7 +34,7 @@
 // Test single gpu architecture up to the assemble phase.
 //
 // RUN: %clang -target powerpc64le-ibm-linux-gnu -ccc-print-phases \
-// RUN: --cuda-gpu-arch=sm_30 %s -S 2>&1 \
+// RUN:   --no-offload-new-driver --cuda-gpu-arch=sm_30 %s -S 2>&1 \
 // RUN: | FileCheck -check-prefixes=ASM %s
 // ASM-DAG: [[P0:[0-9]+]]: input, "{{.*}}cuda-phases.cu", [[T:cuda]], (device-[[T]], [[ARCH:sm_30]])
 // ASM-DAG: [[P1:[0-9]+]]: preprocessor, {[[P0]]}, [[T]]-cpp-output, (device-[[T]], [[ARCH]])
@@ -50,7 +50,7 @@
 // Test two gpu architectures with complete compilation.
 //
 // RUN: %clang -target powerpc64le-ibm-linux-gnu -ccc-print-phases \
-// RUN: --cuda-gpu-arch=sm_30 --cuda-gpu-arch=sm_35 %s 2>&1 \
+// RUN:   --no-offload-new-driver --cuda-gpu-arch=sm_30 --cuda-gpu-arch=sm_35 %s 2>&1 \
 // RUN: | FileCheck -check-prefixes=BIN2 %s
 // BIN2-DAG: [[P0:[0-9]+]]: input, "{{.*}}cuda-phases.cu", [[T:cuda]], (host-[[T]])
 // BIN2-DAG: [[P1:[0-9]+]]: preprocessor, {[[P0]]}, [[T]]-cpp-output, (host-[[T]])
@@ -79,7 +79,7 @@
 // Test two gpu architecturess up to the assemble phase.
 //
 // RUN: %clang -target powerpc64le-ibm-linux-gnu -ccc-print-phases \
-// RUN: --cuda-gpu-arch=sm_30 --cuda-gpu-arch=sm_35 %s -S 2>&1 \
+// RUN:   --no-offload-new-driver --cuda-gpu-arch=sm_30 --cuda-gpu-arch=sm_35 %s -S 2>&1 \
 // RUN: | FileCheck -check-prefixes=ASM2 %s
 // ASM2-DAG: [[P0:[0-9]+]]: input, "{{.*}}cuda-phases.cu", [[T:cuda]], (device-[[T]], [[ARCH1:sm_30]])
 // ASM2-DAG: [[P1:[0-9]+]]: preprocessor, {[[P0]]}, [[T]]-cpp-output, (device-[[T]], [[ARCH1]])
@@ -101,7 +101,7 @@
 // compilation mode.
 //
 // RUN: %clang -target powerpc64le-ibm-linux-gnu -ccc-print-phases \
-// RUN: --cuda-gpu-arch=sm_30 %s --cuda-host-only 2>&1 \
+// RUN:   --no-offload-new-driver --cuda-gpu-arch=sm_30 %s --cuda-host-only 2>&1 \
 // RUN: | FileCheck -check-prefixes=HBIN %s
 // HBIN-DAG: [[P0:[0-9]+]]: input, "{{.*}}cuda-phases.cu", [[T:cuda]], (host-[[T]])
 // HBIN-DAG: [[P1:[0-9]+]]: preprocessor, {[[P0]]}, [[T]]-cpp-output, (host-[[T]])
@@ -115,7 +115,7 @@
 // compilation mode.
 //
 // RUN: %clang -target powerpc64le-ibm-linux-gnu -ccc-print-phases \
-// RUN: --cuda-gpu-arch=sm_30 %s --cuda-host-only -S 2>&1 \
+// RUN:   --no-offload-new-driver --cuda-gpu-arch=sm_30 %s --cuda-host-only -S 2>&1 \
 // RUN: | FileCheck -check-prefixes=HASM %s
 // HASM-DAG: [[P0:[0-9]+]]: input, "{{.*}}cuda-phases.cu", [[T:cuda]], (host-[[T]])
 // HASM-DAG: [[P1:[0-9]+]]: preprocessor, {[[P0]]}, [[T]]-cpp-output, (host-[[T]])
@@ -128,7 +128,7 @@
 // compilation mode.
 //
 // RUN: %clang -target powerpc64le-ibm-linux-gnu -ccc-print-phases \
-// RUN: --cuda-gpu-arch=sm_30 --cuda-gpu-arch=sm_35 %s --cuda-host-only 2>&1 \
+// RUN:   --no-offload-new-driver --cuda-gpu-arch=sm_30 --cuda-gpu-arch=sm_35 %s --cuda-host-only 2>&1 \
 // RUN: | FileCheck -check-prefixes=HBIN2 %s
 // HBIN2-DAG: [[P0:[0-9]+]]: input, "{{.*}}cuda-phases.cu", [[T:cuda]], (host-[[T]])
 // HBIN2-DAG: [[P1:[0-9]+]]: preprocessor, {[[P0]]}, [[T]]-cpp-output, (host-[[T]])
@@ -143,7 +143,7 @@
 // compilation mode.
 //
 // RUN: %clang -target powerpc64le-ibm-linux-gnu -ccc-print-phases \
-// RUN: --cuda-gpu-arch=sm_30 --cuda-gpu-arch=sm_35 %s --cuda-host-only -S \
+// RUN:   --no-offload-new-driver --cuda-gpu-arch=sm_30 --cuda-gpu-arch=sm_35 %s --cuda-host-only -S \
 // RUN: 2>&1 | FileCheck -check-prefixes=HASM2 %s
 // HASM2-DAG: [[P0:[0-9]+]]: input, "{{.*}}cuda-phases.cu", [[T:cuda]], (host-[[T]])
 // HASM2-DAG: [[P1:[0-9]+]]: preprocessor, {[[P0]]}, [[T]]-cpp-output, (host-[[T]])
@@ -156,7 +156,7 @@
 // compilation mode.
 //
 // RUN: %clang -target powerpc64le-ibm-linux-gnu -ccc-print-phases \
-// RUN: --cuda-gpu-arch=sm_30 %s --cuda-device-only 2>&1 \
+// RUN:   --no-offload-new-driver --cuda-gpu-arch=sm_30 %s --cuda-device-only 2>&1 \
 // RUN: | FileCheck -check-prefixes=DBIN %s
 // DBIN-DAG: [[P0:[0-9]+]]: input, "{{.*}}cuda-phases.cu", [[T:cuda]], (device-[[T]], [[ARCH:sm_30]])
 // DBIN-DAG: [[P1:[0-9]+]]: preprocessor, {[[P0]]}, [[T]]-cpp-output, (device-[[T]], [[ARCH]])
@@ -170,7 +170,7 @@
 // compilation mode.
 //
 // RUN: %clang -target powerpc64le-ibm-linux-gnu -ccc-print-phases \
-// RUN: --cuda-gpu-arch=sm_30 %s --cuda-device-only -S 2>&1 \
+// RUN:   --no-offload-new-driver --cuda-gpu-arch=sm_30 %s --cuda-device-only -S 2>&1 \
 // RUN: | FileCheck -check-prefixes=DASM %s
 // DASM-DAG: [[P0:[0-9]+]]: input, "{{.*}}cuda-phases.cu", [[T:cuda]], (device-[[T]], [[ARCH:sm_30]])
 // DASM-DAG: [[P1:[0-9]+]]: preprocessor, {[[P0]]}, [[T]]-cpp-output, (device-[[T]], [[ARCH]])
@@ -184,7 +184,7 @@
 // compilation mode.
 //
 // RUN: %clang -target powerpc64le-ibm-linux-gnu -ccc-print-phases \
-// RUN: --cuda-gpu-arch=sm_30 --cuda-gpu-arch=sm_35 %s --cuda-device-only 2>&1 \
+// RUN:   --no-offload-new-driver --cuda-gpu-arch=sm_30 --cuda-gpu-arch=sm_35 %s --cuda-device-only 2>&1 \
 // RUN: | FileCheck -check-prefixes=DBIN2 %s
 // DBIN2-DAG: [[P0:[0-9]+]]: input, "{{.*}}cuda-phases.cu", [[T:cuda]], (device-[[T]], [[ARCH:sm_30]])
 // DBIN2-DAG: [[P1:[0-9]+]]: preprocessor, {[[P0]]}, [[T]]-cpp-output, (device-[[T]], [[ARCH]])
@@ -204,7 +204,7 @@
 // compilation mode.
 //
 // RUN: %clang -target powerpc64le-ibm-linux-gnu -ccc-print-phases \
-// RUN: --cuda-gpu-arch=sm_30 --cuda-gpu-arch=sm_35 %s --cuda-device-only -S \
+// RUN:   --no-offload-new-driver --cuda-gpu-arch=sm_30 --cuda-gpu-arch=sm_35 %s --cuda-device-only -S \
 // RUN: 2>&1 | FileCheck -check-prefixes=DASM2 %s
 // DASM2-DAG: [[P0:[0-9]+]]: input, "{{.*}}cuda-phases.cu", [[T:cuda]], (device-[[T]], [[ARCH:sm_30]])
 // DASM2-DAG: [[P1:[0-9]+]]: preprocessor, {[[P0]]}, [[T]]-cpp-output, (device-[[T]], [[ARCH]])
diff --git a/clang/test/Driver/hip-binding.hip b/clang/test/Driver/hip-binding.hip
index c48397168a60..79ec2039edb7 100644
--- a/clang/test/Driver/hip-binding.hip
+++ b/clang/test/Driver/hip-binding.hip
@@ -3,10 +3,10 @@
 
 // RUN: %clang -ccc-print-bindings --target=x86_64-linux-gnu \
 // RUN:   --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s \
-// RUN:   -c 2>&1 | FileCheck -check-prefix=NRDCS %s
+// RUN:   --no-offload-new-driver -c 2>&1 | FileCheck -check-prefix=NRDCS %s
 // RUN: %clang -ccc-print-bindings --target=x86_64-linux-gnu --offload-new-driver \
 // RUN:   --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s \
-// RUN:   -c 2>&1 | FileCheck -check-prefix=NRDCS %s
+// RUN:   --no-offload-new-driver -c 2>&1 | FileCheck -check-prefix=NRDCS %s
 // NRDCS: # "amdgcn-amd-amdhsa" - "clang", inputs: ["[[IN:.*hip-binding.hip]]"], output: "[[OBJ1:.*o]]"
 // NRDCS: # "amdgcn-amd-amdhsa" - "AMDGCN::Linker", inputs: ["[[OBJ1]]"], output: "[[IMG1:.*]]"
 // NRDCS: # "amdgcn-amd-amdhsa" - "clang", inputs: ["[[IN:.*hip-binding.hip]]"], output: "[[OBJ2:.*o]]"
@@ -16,7 +16,7 @@
 
 // RUN: %clang -ccc-print-bindings --target=x86_64-linux-gnu \
 // RUN:   --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s \
-// RUN:   -c -fgpu-rdc 2>&1 | FileCheck -check-prefix=RDCS %s
+// RUN:   --no-offload-new-driver -c -fgpu-rdc 2>&1 | FileCheck -check-prefix=RDCS %s
 // RDCS: # "amdgcn-amd-amdhsa" - "clang", inputs: ["[[IN:.*hip-binding.hip]]"], output: "[[BC1:.*bc]]"
 // RDCS: # "amdgcn-amd-amdhsa" - "clang", inputs: ["[[IN:.*hip-binding.hip]]"], output: "[[BC2:.*bc]]"
 // RDCS: # "x86_64-unknown-linux-gnu" - "clang", inputs: ["[[IN]]"], output: "[[HOSTOBJ:.*o]]"
@@ -32,7 +32,7 @@
 
 // RUN: touch %t.o
 // RUN: %clang --hip-link -ccc-print-bindings --target=x86_64-linux-gnu \
-// RUN:   --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 -fgpu-rdc %t.o\
+// RUN:   --no-offload-new-driver --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 -fgpu-rdc %t.o\
 // RUN: 2>&1 | FileCheck %s
 
 // CHECK: # "x86_64-unknown-linux-gnu" - "offload bundler", inputs: ["[[IN:.*o]]"], outputs: ["[[HOSTOBJ:.*o]]", "{{.*o}}", "{{.*o}}"]
@@ -46,7 +46,7 @@
 // CHECK: # "x86_64-unknown-linux-gnu" - "GNU::Linker", inputs: ["[[HOSTOBJ]]", "[[FATBINOBJ]]"], output: "a.out"
 
 // RUN: %clang --hip-link -ccc-print-bindings --target=x86_64-linux-gnu \
-// RUN:   --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %t.o\
+// RUN:   --no-offload-new-driver --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %t.o\
 // RUN: 2>&1 | FileCheck -check-prefix=NORDC %s
 
 // NORDC-NOT: offload bundler
diff --git a/clang/test/Driver/hip-cuid-hash.hip b/clang/test/Driver/hip-cuid-hash.hip
index 1b4d26c471c1..ef2a32a69c8f 100644
--- a/clang/test/Driver/hip-cuid-hash.hip
+++ b/clang/test/Driver/hip-cuid-hash.hip
@@ -4,11 +4,11 @@
 // Check CUID generated by hash.
 // The same CUID is generated for the same file with the same options.
 
-// RUN: %clang -### -x hip --target=x86_64-unknown-linux-gnu \
+// RUN: %clang -### -x hip --target=x86_64-unknown-linux-gnu --no-offload-new-driver \
 // RUN:   --offload-arch=gfx906 -c -nogpuinc -nogpulib -fuse-cuid=hash \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu >%t.out 2>&1
 
-// RUN: %clang -### -x hip --target=x86_64-unknown-linux-gnu \
+// RUN: %clang -### -x hip --target=x86_64-unknown-linux-gnu --no-offload-new-driver \
 // RUN:   --offload-arch=gfx906 -c -nogpuinc -nogpulib -fuse-cuid=hash \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu >>%t.out 2>&1
 
@@ -17,11 +17,11 @@
 // Check CUID generated by hash.
 // Different CUID's are generated for the same file with different options.
 
-// RUN: %clang -### -x hip --target=x86_64-unknown-linux-gnu -DX=1 \
+// RUN: %clang -### -x hip --target=x86_64-unknown-linux-gnu -DX=1 --no-offload-new-driver \
 // RUN:   --offload-arch=gfx906 -c -nogpuinc -nogpulib -fuse-cuid=hash \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu >%t.out 2>&1
 
-// RUN: %clang -### -x hip --target=x86_64-unknown-linux-gnu -DX=2 \
+// RUN: %clang -### -x hip --target=x86_64-unknown-linux-gnu -DX=2 --no-offload-new-driver \
 // RUN:   --offload-arch=gfx906 -c -nogpuinc -nogpulib -fuse-cuid=hash \
 // RUN:   %S/Inputs/../Inputs/hip_multiple_inputs/a.cu >>%t.out 2>&1
 
diff --git a/clang/test/Driver/hip-cuid.hip b/clang/test/Driver/hip-cuid.hip
index 421810b824fd..ce3d2de3501e 100644
--- a/clang/test/Driver/hip-cuid.hip
+++ b/clang/test/Driver/hip-cuid.hip
@@ -5,6 +5,7 @@
 
 // RUN: not %clang -### -x hip \
 // RUN:   --target=x86_64-unknown-linux-gnu \
+// RUN:   --no-offload-new-driver \
 // RUN:   --offload-arch=gfx900 \
 // RUN:   --offload-arch=gfx906 \
 // RUN:   -c -nogpuinc -nogpulib -fuse-cuid=invalid \
@@ -16,6 +17,7 @@
 
 // RUN: %clang -### -x hip \
 // RUN:   --target=x86_64-unknown-linux-gnu \
+// RUN:   --no-offload-new-driver \
 // RUN:   --offload-arch=gfx900 \
 // RUN:   --offload-arch=gfx906 \
 // RUN:   -c -nogpuinc -nogpulib -fuse-cuid=random \
@@ -27,6 +29,7 @@
 
 // RUN: %clang -### -x hip \
 // RUN:   --target=x86_64-unknown-linux-gnu \
+// RUN:   --no-offload-new-driver \
 // RUN:   --offload-arch=gfx900 \
 // RUN:   --offload-arch=gfx906 \
 // RUN:   -c -nogpuinc -nogpulib -cuid=xyz_123 \
@@ -38,6 +41,7 @@
 
 // RUN: %clang -### -x hip \
 // RUN:   --target=x86_64-unknown-linux-gnu \
+// RUN:   --no-offload-new-driver \
 // RUN:   --offload-arch=gfx900 \
 // RUN:   --offload-arch=gfx906 \
 // RUN:   -c -nogpuinc -nogpulib -fuse-cuid=random -cuid=xyz_123 \
@@ -49,6 +53,7 @@
 
 // RUN: %clang -### -x hip \
 // RUN:   --target=x86_64-unknown-linux-gnu \
+// RUN:   --no-offload-new-driver \
 // RUN:   --offload-arch=gfx900 \
 // RUN:   --offload-arch=gfx906 \
 // RUN:   -c -nogpuinc -nogpulib -fuse-cuid=hash \
diff --git a/clang/test/Driver/hip-dependent-options.hip b/clang/test/Driver/hip-dependent-options.hip
index 405b092d0f83..b0dc7f289e81 100644
--- a/clang/test/Driver/hip-dependent-options.hip
+++ b/clang/test/Driver/hip-dependent-options.hip
@@ -1,4 +1,4 @@
-// RUN: not %clang -### --target=x86_64-linux-gnu \
+// RUN: not %clang -### --target=x86_64-linux-gnu --no-offload-new-driver \
 // RUN:   -x hip --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
 // RUN:   -c -fhip-emit-relocatable -nogpuinc -nogpulib --cuda-device-only -fgpu-rdc \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu \
@@ -7,7 +7,7 @@
 
 // RELOCRDC: error: option '-fhip-emit-relocatable' cannot be specified with '-fgpu-rdc'
 
-// RUN: not %clang -### --target=x86_64-linux-gnu \
+// RUN: not %clang -### --target=x86_64-linux-gnu --no-offload-new-driver \
 // RUN:   -x hip --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
 // RUN:   -c -fhip-emit-relocatable -nogpuinc -nogpulib \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu \
diff --git a/clang/test/Driver/hip-device-compile.hip b/clang/test/Driver/hip-device-compile.hip
index 3c3e38785624..74be9c6cf2ee 100644
--- a/clang/test/Driver/hip-device-compile.hip
+++ b/clang/test/Driver/hip-device-compile.hip
@@ -8,7 +8,7 @@
 // Output unbundled bitcode.
 // RUN: %clang -c -emit-llvm --cuda-device-only -### --target=x86_64-linux-gnu \
 // RUN:   --rocm-path=%S/Inputs/rocm -o a.bc -x hip --cuda-gpu-arch=gfx900 --no-gpu-bundle-output \
-// RUN:   --hip-device-lib=lib1.bc \
+// RUN:   --no-offload-new-driver --hip-device-lib=lib1.bc \
 // RUN:   --hip-device-lib-path=%S/Inputs/hip_multiple_inputs/lib1 \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu \
 // RUN: 2>&1 | FileCheck -check-prefixes=CHECK,BC,NBUN %s
@@ -16,7 +16,7 @@
 // Output bundled bitcode.
 // RUN: %clang -c -emit-llvm --cuda-device-only -### --target=x86_64-linux-gnu \
 // RUN:   --rocm-path=%S/Inputs/rocm -o a.bc -x hip --cuda-gpu-arch=gfx900 --no-gpu-bundle-output \
-// RUN:   --hip-device-lib=lib1.bc \
+// RUN:   --no-offload-new-driver --hip-device-lib=lib1.bc \
 // RUN:   --hip-device-lib-path=%S/Inputs/hip_multiple_inputs/lib1 \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu --gpu-bundle-output \
 // RUN: 2>&1 | FileCheck -check-prefixes=CHECK,BCBUN %s
@@ -24,7 +24,7 @@
 // Output unbundled LLVM IR.
 // RUN: %clang -c -S -emit-llvm --cuda-device-only -### --target=x86_64-linux-gnu \
 // RUN:   --rocm-path=%S/Inputs/rocm -o a.ll -x hip --cuda-gpu-arch=gfx900 --no-gpu-bundle-output \
-// RUN:   --hip-device-lib=lib1.bc \
+// RUN:   --no-offload-new-driver --hip-device-lib=lib1.bc \
 // RUN:   --hip-device-lib-path=%S/Inputs/hip_multiple_inputs/lib1 \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu \
 // RUN: 2>&1 | FileCheck -check-prefixes=CHECK,LL,NBUN %s
@@ -32,7 +32,7 @@
 // Output bundled LLVM IR.
 // RUN: %clang -c -S -emit-llvm --cuda-device-only -### --target=x86_64-linux-gnu \
 // RUN:   --rocm-path=%S/Inputs/rocm -o a.ll -x hip --cuda-gpu-arch=gfx900 --no-gpu-bundle-output \
-// RUN:   --hip-device-lib=lib1.bc \
+// RUN:   --no-offload-new-driver --hip-device-lib=lib1.bc \
 // RUN:   --hip-device-lib-path=%S/Inputs/hip_multiple_inputs/lib1 \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu --gpu-bundle-output \
 // RUN: 2>&1 | FileCheck -check-prefixes=CHECK,LLBUN %s
@@ -40,7 +40,7 @@
 // Output unbundled assembly.
 // RUN: %clang -c -S --cuda-device-only -### --target=x86_64-linux-gnu \
 // RUN:   --rocm-path=%S/Inputs/rocm -o a.s -x hip --cuda-gpu-arch=gfx900 --no-gpu-bundle-output \
-// RUN:   --hip-device-lib=lib1.bc \
+// RUN:   --no-offload-new-driver --hip-device-lib=lib1.bc \
 // RUN:   --hip-device-lib-path=%S/Inputs/hip_multiple_inputs/lib1 \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu \
 // RUN: 2>&1 | FileCheck -check-prefixes=CHECK,ASM,NBUN %s
@@ -48,7 +48,7 @@
 // Output relocatable.
 // RUN: %clang -c --cuda-device-only -### --target=x86_64-linux-gnu \
 // RUN:   --rocm-path=%S/Inputs/rocm -o a.o -x hip --cuda-gpu-arch=gfx900 -fhip-emit-relocatable \
-// RUN:   --hip-device-lib=lib1.bc \
+// RUN:   --no-offload-new-driver --hip-device-lib=lib1.bc \
 // RUN:   --hip-device-lib-path=%S/Inputs/hip_multiple_inputs/lib1 \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu \
 // RUN: 2>&1 | FileCheck -check-prefixes=CHECK,NBUN,RELOC %s
@@ -56,7 +56,7 @@
 // Output bundled assembly.
 // RUN: %clang -c -S --cuda-device-only -### --target=x86_64-linux-gnu \
 // RUN:   --rocm-path=%S/Inputs/rocm -o a.s -x hip --cuda-gpu-arch=gfx900 --no-gpu-bundle-output \
-// RUN:   --hip-device-lib=lib1.bc \
+// RUN:   --no-offload-new-driver --hip-device-lib=lib1.bc \
 // RUN:   --hip-device-lib-path=%S/Inputs/hip_multiple_inputs/lib1 \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu --gpu-bundle-output \
 // RUN: 2>&1 | FileCheck -check-prefixes=CHECK,ASMBUN %s
@@ -96,7 +96,7 @@
 // Output bundled code objects.
 // RUN: %clang -c --cuda-device-only -### --target=x86_64-linux-gnu \
 // RUN:   --rocm-path=%S/Inputs/rocm -o a.o -x hip --cuda-gpu-arch=gfx900 \
-// RUN:   --hip-device-lib=lib1.bc \
+// RUN:   --no-offload-new-driver --hip-device-lib=lib1.bc \
 // RUN:   --hip-device-lib-path=%S/Inputs/hip_multiple_inputs/lib1 \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu \
 // RUN: 2>&1 | FileCheck -check-prefixes=OBJ,OBJ-BUN %s
@@ -104,7 +104,7 @@
 // Output unbundled code objects.
 // RUN: %clang -c --cuda-device-only -### --target=x86_64-linux-gnu \
 // RUN:   --rocm-path=%S/Inputs/rocm -o a.o -x hip --cuda-gpu-arch=gfx900 \
-// RUN:   --hip-device-lib=lib1.bc \
+// RUN:   --no-offload-new-driver --hip-device-lib=lib1.bc \
 // RUN:   --hip-device-lib-path=%S/Inputs/hip_multiple_inputs/lib1 \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu --no-gpu-bundle-output \
 // RUN: 2>&1 | FileCheck -check-prefixes=OBJ,OBJ-UBUN %s
@@ -112,7 +112,7 @@
 // Output bundled code objects.
 // RUN: %clang --cuda-device-only -### --target=x86_64-linux-gnu \
 // RUN:   --rocm-path=%S/Inputs/rocm -o a.o -x hip --cuda-gpu-arch=gfx900 \
-// RUN:   --hip-device-lib=lib1.bc \
+// RUN:   --no-offload-new-driver --hip-device-lib=lib1.bc \
 // RUN:   --hip-device-lib-path=%S/Inputs/hip_multiple_inputs/lib1 \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu \
 // RUN: 2>&1 | FileCheck -check-prefixes=OBJ,OBJ-BUN %s
@@ -120,7 +120,7 @@
 // Output unbundled code objects.
 // RUN: %clang --cuda-device-only -### --target=x86_64-linux-gnu \
 // RUN:   --rocm-path=%S/Inputs/rocm -o a.o -x hip --cuda-gpu-arch=gfx900 \
-// RUN:   --hip-device-lib=lib1.bc \
+// RUN:   --no-offload-new-driver --hip-device-lib=lib1.bc \
 // RUN:   --hip-device-lib-path=%S/Inputs/hip_multiple_inputs/lib1 \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu --no-gpu-bundle-output \
 // RUN: 2>&1 | FileCheck -check-prefixes=OBJ,OBJ-UBUN %s
diff --git a/clang/test/Driver/hip-link-bc-to-bc.hip b/clang/test/Driver/hip-link-bc-to-bc.hip
index 52eab97bcebb..7b232f2656a9 100644
--- a/clang/test/Driver/hip-link-bc-to-bc.hip
+++ b/clang/test/Driver/hip-link-bc-to-bc.hip
@@ -7,7 +7,7 @@
 
 // RUN: %clang -### --target=x86_64-unknown-linux-gnu --offload-arch=gfx906 --hip-link \
 // RUN:   -nogpulib -nogpuinc -emit-llvm -fgpu-rdc --cuda-device-only \
-// RUN:   %t/bundle1.bc %t/bundle2.bc \
+// RUN:   --no-offload-new-driver %t/bundle1.bc %t/bundle2.bc \
 // RUN:   2>&1 | FileCheck -check-prefix=BITCODE %s
 
 // BITCODE: "{{.*}}clang-offload-bundler" "-type=bc" "-targets=host-x86_64-unknown-linux-gnu,hip-amdgcn-amd-amdhsa-gfx906" "-input={{.*}}bundle1.bc" "-output=[[B1HOST:.*\.bc]]" "-output=[[B1DEV1:.*\.bc]]" "-unbundle" "-allow-missing-bundles"
@@ -24,7 +24,7 @@
 
 // RUN: %clang -### --target=x86_64-unknown-linux-gnu --offload-arch=gfx906 --hip-link \
 // RUN:   -nogpulib -nogpuinc -emit-llvm -fgpu-rdc --cuda-device-only \
-// RUN:   %t/bundle.bc -L%t -lhipbundle \
+// RUN:   --no-offload-new-driver %t/bundle.bc -L%t -lhipbundle \
 // RUN:   2>&1 | FileCheck -check-prefix=ARCHIVE %s
 
 // ARCHIVE: "{{.*}}clang-offload-bundler" "-type=bc" "-targets=host-x86_64-unknown-linux-gnu,hip-amdgcn-amd-amdhsa-gfx906" "-input={{.*}}bundle.bc" "-output=[[HOST:.*\.bc]]" "-output=[[DEV1:.*\.bc]]" "-unbundle" "-allow-missing-bundles"
diff --git a/clang/test/Driver/hip-link-bundle-archive.hip b/clang/test/Driver/hip-link-bundle-archive.hip
index dd1d779fe19a..cfbf71372260 100644
--- a/clang/test/Driver/hip-link-bundle-archive.hip
+++ b/clang/test/Driver/hip-link-bundle-archive.hip
@@ -9,23 +9,23 @@
 // RUN: touch %t/dummy.bc
 // RUN: llvm-ar cr %t/libhipBundled.a %t/dummy.bc
 // RUN: %clang -### --offload-arch=gfx906 --offload-arch=gfx1030 \
-// RUN:   --target=x86_64-unknown-linux-gnu \
+// RUN:   --no-offload-new-driver --target=x86_64-unknown-linux-gnu \
 // RUN:   -nogpuinc -nogpulib %s -fgpu-rdc -L%t -lhipBundled \
 // RUN:   2>&1 | FileCheck -check-prefixes=GNU,GNU1,GNU-L %s
 
 // RUN: %clang -### --offload-arch=gfx906 --offload-arch=gfx1030 -nogpuinc \
-// RUN:   --target=x86_64-unknown-linux-gnu \
+// RUN:   --no-offload-new-driver --target=x86_64-unknown-linux-gnu \
 // RUN:   -nogpuinc -nogpulib %s -fgpu-rdc -L%t -l:libhipBundled.a \
 // RUN:   2>&1 | FileCheck -check-prefixes=GNU,GNU1,GNU-LA %s
 
 // RUN: %clang -### --offload-arch=gfx906 --offload-arch=gfx1030 \
-// RUN:   --target=x86_64-unknown-linux-gnu \
+// RUN:   --no-offload-new-driver --target=x86_64-unknown-linux-gnu \
 // RUN:   -nogpuinc -nogpulib %s -fgpu-rdc %t/libhipBundled.a \
 // RUN:   2>&1 | FileCheck -check-prefixes=GNU,GNU1,GNU-A %s
 
 // RUN: llvm-ar cr %t/libhipBundled.a.5.2 %t/dummy.bc
 // RUN: %clang -### --offload-arch=gfx906 --offload-arch=gfx1030 \
-// RUN:   --target=x86_64-unknown-linux-gnu \
+// RUN:   --no-offload-new-driver --target=x86_64-unknown-linux-gnu \
 // RUN:   -nogpuinc -nogpulib %s -fgpu-rdc %t/libhipBundled.a.5.2 \
 // RUN:   2>&1 | FileCheck -check-prefixes=GNU,GNU2,GNU-A %s
 
@@ -33,22 +33,22 @@
 
 // RUN: touch %t/libNonArchive.a
 // RUN: %clang -### --offload-arch=gfx906 --offload-arch=gfx1030 \
-// RUN:   --target=x86_64-unknown-linux-gnu \
+// RUN:   --no-offload-new-driver --target=x86_64-unknown-linux-gnu \
 // RUN:   -nogpuinc -nogpulib %s -fgpu-rdc -L%t -lNonArchive \
 // RUN:   2>&1 | FileCheck -check-prefixes=NONARCHIVE %s
 // RUN: %clang -### --offload-arch=gfx906 --offload-arch=gfx1030 \
-// RUN:   --target=x86_64-unknown-linux-gnu \
+// RUN:   --no-offload-new-driver --target=x86_64-unknown-linux-gnu \
 // RUN:   -nogpuinc -nogpulib %s -fgpu-rdc -L%t -l:libNonArchive.a \
 // RUN:   2>&1 | FileCheck -check-prefixes=NONARCHIVE %s
 // RUN: not %clang -### --offload-arch=gfx906 --offload-arch=gfx1030 \
-// RUN:   --target=x86_64-unknown-linux-gnu \
+// RUN:   --no-offload-new-driver --target=x86_64-unknown-linux-gnu \
 // RUN:   -nogpuinc -nogpulib %s -fgpu-rdc -L%t libNonArchive.a \
 // RUN:   2>&1 | FileCheck -check-prefixes=NONARCHIVE %s
 
 // Check if a file does not exist, it is not unbundled.
 
 // RUN: not %clang -### --offload-arch=gfx906 --offload-arch=gfx1030 \
-// RUN:   --target=x86_64-unknown-linux-gnu \
+// RUN:   --no-offload-new-driver --target=x86_64-unknown-linux-gnu \
 // RUN:   -nogpuinc -nogpulib %s -fgpu-rdc %t/NoneExist.a \
 // RUN:   2>&1 | FileCheck -check-prefixes=NONE %s
 
@@ -56,17 +56,17 @@
 
 // RUN: llvm-ar cr %t/hipBundled2.lib %t/dummy.bc
 // RUN: %clang -### --offload-arch=gfx906 --offload-arch=gfx1030 \
-// RUN:   --target=x86_64-pc-windows-msvc -fuse-ld= \
+// RUN:   --no-offload-new-driver --target=x86_64-pc-windows-msvc -fuse-ld= \
 // RUN:   -nogpuinc -nogpulib %s -fgpu-rdc -L%t -lhipBundled2 \
 // RUN:   2>&1 | FileCheck -check-prefix=MSVC %s
 
 // RUN: %clang -### --offload-arch=gfx906 --offload-arch=gfx1030 \
-// RUN:   --target=x86_64-pc-windows-msvc -fuse-ld= \
+// RUN:   --no-offload-new-driver --target=x86_64-pc-windows-msvc -fuse-ld= \
 // RUN:   -nogpuinc -nogpulib %s -fgpu-rdc -L%t -l:hipBundled2.lib \
 // RUN:   2>&1 | FileCheck -check-prefix=MSVC %s
 
 // RUN: %clang -### --offload-arch=gfx906 --offload-arch=gfx1030 \
-// RUN:   --target=x86_64-pc-windows-msvc -fuse-ld= \
+// RUN:   --no-offload-new-driver --target=x86_64-pc-windows-msvc -fuse-ld= \
 // RUN:   -nogpuinc -nogpulib %s -fgpu-rdc %t/hipBundled2.lib \
 // RUN:   2>&1 | FileCheck -check-prefix=MSVC %s
 
diff --git a/clang/test/Driver/hip-link-save-temps.hip b/clang/test/Driver/hip-link-save-temps.hip
index b8a5dcefb8cf..e54be63c578b 100644
--- a/clang/test/Driver/hip-link-save-temps.hip
+++ b/clang/test/Driver/hip-link-save-temps.hip
@@ -7,14 +7,14 @@
 // RUN: touch %t/obj2.o
 // RUN: %clang -### --target=x86_64-linux-gnu -nogpulib -save-temps \
 // RUN:   --hip-link -o executable -fgpu-rdc --cuda-gpu-arch=gfx900 \
-// RUN:   --offload-arch=gfx906  %t/obj1.o %t/obj2.o 2>&1 | \
+// RUN:   --no-offload-new-driver --offload-arch=gfx906  %t/obj1.o %t/obj2.o 2>&1 | \
 // RUN:   FileCheck -check-prefixes=CHECK,OUT %s
 
 // -fgpu-rdc link without output
 // RUN: touch %t/obj1.o
 // RUN: touch %t/obj2.o
 // RUN: %clang -### --target=x86_64-linux-gnu -nogpulib -save-temps \
-// RUN:   --hip-link -fgpu-rdc --cuda-gpu-arch=gfx900 \
+// RUN:   --no-offload-new-driver --hip-link -fgpu-rdc --cuda-gpu-arch=gfx900 \
 // RUN:   --offload-arch=gfx906  %t/obj1.o %t/obj2.o 2>&1 | \
 // RUN:   FileCheck -check-prefixes=CHECK,NOUT %s
 
@@ -23,7 +23,7 @@
 // RUN: touch %t/obj2.o
 // RUN: %clang -### --target=x86_64-linux-gnu -nogpulib -save-temps \
 // RUN:   --hip-link -o libTest.a -fgpu-rdc --cuda-gpu-arch=gfx900 \
-// RUN:   --emit-static-lib \
+// RUN:   --no-offload-new-driver --emit-static-lib \
 // RUN:   --offload-arch=gfx906  %t/obj1.o %t/obj2.o 2>&1 | \
 // RUN:   FileCheck -check-prefixes=CHECK,SLO %s
 
@@ -32,7 +32,7 @@
 // RUN: touch %t/obj2.o
 // RUN: %clang -### --target=x86_64-linux-gnu -nogpulib -save-temps \
 // RUN:   --hip-link -fgpu-rdc --cuda-gpu-arch=gfx900 \
-// RUN:   --emit-static-lib \
+// RUN:   --no-offload-new-driver --emit-static-lib \
 // RUN:   --offload-arch=gfx906  %t/obj1.o %t/obj2.o 2>&1 | \
 // RUN:   FileCheck -check-prefixes=CHECK,SLNO %s
 
diff --git a/clang/test/Driver/hip-link-shared-library.hip b/clang/test/Driver/hip-link-shared-library.hip
index fc1f9f53b954..73643682dda8 100644
--- a/clang/test/Driver/hip-link-shared-library.hip
+++ b/clang/test/Driver/hip-link-shared-library.hip
@@ -1,7 +1,7 @@
 // RUN: touch %t.o
 // RUN: %clang --hip-link -ccc-print-bindings --target=x86_64-linux-gnu \
 // RUN:   --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %t.o %S/Inputs/in.so \
-// RUN:   -fgpu-rdc 2>&1 | FileCheck %s
+// RUN:   --no-offload-new-driver -fgpu-rdc 2>&1 | FileCheck %s
 
 // CHECK: # "x86_64-unknown-linux-gnu" - "offload bundler", inputs: ["[[IN:.*o]]"], outputs: ["[[HOSTOBJ:.*o]]", "{{.*o}}", "{{.*o}}"]
 // CHECK: # "amdgcn-amd-amdhsa" - "offload bundler", inputs: ["[[IN]]"], outputs: ["{{.*o}}", "[[DOBJ1:.*o]]", "[[DOBJ2:.*o]]"]
@@ -12,4 +12,3 @@
 // CHECK: # "amdgcn-amd-amdhsa" - "AMDGCN::Linker", inputs: ["[[IMG1]]", "[[IMG2]]"], output: "[[FATBINOBJ:.*o]]"
 // CHECK-NOT: offload bundler
 // CHECK: # "x86_64-unknown-linux-gnu" - "GNU::Linker", inputs: ["[[HOSTOBJ]]", "{{.*}}/Inputs/in.so", "[[FATBINOBJ]]"], output: "a.out"
-
diff --git a/clang/test/Driver/hip-link-static-library.hip b/clang/test/Driver/hip-link-static-library.hip
index 63675ffd6262..3159b5f3984f 100644
--- a/clang/test/Driver/hip-link-static-library.hip
+++ b/clang/test/Driver/hip-link-static-library.hip
@@ -3,7 +3,7 @@
 
 // RUN: touch %t.o
 // RUN: %clang --hip-link -ccc-print-bindings --target=x86_64-linux-gnu \
-// RUN:   --emit-static-lib \
+// RUN:   --no-offload-new-driver --emit-static-lib \
 // RUN:   --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 -fgpu-rdc %t.o\
 // RUN: 2>&1 | FileCheck %s
 
@@ -18,7 +18,7 @@
 // CHECK: # "x86_64-unknown-linux-gnu" - "GNU::StaticLibTool", inputs: ["[[HOSTOBJ]]", "[[FATBINOBJ]]"], output: "a.out"
 
 // RUN: %clang --hip-link -ccc-print-bindings --target=x86_64-linux-gnu \
-// RUN:   --emit-static-lib \
+// RUN:   --no-offload-new-driver --emit-static-lib \
 // RUN:   --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %t.o\
 // RUN: 2>&1 | FileCheck -check-prefix=NORDC %s
 
@@ -26,7 +26,7 @@
 // NORDC: # "x86_64-unknown-linux-gnu" - "GNU::StaticLibTool", inputs: ["{{.*o}}"], output: "a.out"
 
 // RUN: %clang --hip-link -### --target=x86_64-linux-gnu \
-// RUN:   --emit-static-lib -lgcc \
+// RUN:   --no-offload-new-driver --emit-static-lib -lgcc \
 // RUN:   -Wl,--enable-new-dtags -Wl,--rpath=/opt \
 // RUN:   --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 -fgpu-rdc %t.o\
 // RUN: 2>&1 | FileCheck -check-prefix=NOFLAG %s
diff --git a/clang/test/Driver/hip-offload-compress-zlib.hip b/clang/test/Driver/hip-offload-compress-zlib.hip
index 7557fdde8786..7a269c566bb9 100644
--- a/clang/test/Driver/hip-offload-compress-zlib.hip
+++ b/clang/test/Driver/hip-offload-compress-zlib.hip
@@ -7,7 +7,7 @@
 // RUN: rm -rf %t.bc
 // RUN: %clang -c -v --target=x86_64-linux-gnu \
 // RUN:   -x hip --offload-arch=gfx1100 --offload-arch=gfx1101 \
-// RUN:   -fgpu-rdc -nogpuinc -nogpulib \
+// RUN:   --no-offload-new-driver -fgpu-rdc -nogpuinc -nogpulib \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu \
 // RUN:   --offload-compress --offload-device-only --gpu-bundle-output \
 // RUN:   -o %t.bc \
@@ -22,7 +22,7 @@
 
 // RUN: %clang --hip-link -### -v --target=x86_64-linux-gnu \
 // RUN:   --offload-arch=gfx1100 --offload-arch=gfx1101 \
-// RUN:   -fgpu-rdc -nogpulib \
+// RUN:   --no-offload-new-driver -fgpu-rdc -nogpulib \
 // RUN:   %t.bc --offload-device-only \
 // RUN: 2>&1 | FileCheck -check-prefix=UNBUNDLE %s
 
@@ -35,7 +35,7 @@
 
 // RUN: %clang -c -### -v --target=x86_64-linux-gnu \
 // RUN:   -x hip --offload-arch=gfx1100 --offload-arch=gfx1101 \
-// RUN:   -nogpuinc -nogpulib \
+// RUN:   --no-offload-new-driver -nogpuinc -nogpulib \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu \
 // RUN:   --offload-compress \
 // RUN: 2>&1 | FileCheck -check-prefix=CO %s
diff --git a/clang/test/Driver/hip-offload-compress-zstd.hip b/clang/test/Driver/hip-offload-compress-zstd.hip
index 3680ae47974a..fa7fb3b6d5b5 100644
--- a/clang/test/Driver/hip-offload-compress-zstd.hip
+++ b/clang/test/Driver/hip-offload-compress-zstd.hip
@@ -7,7 +7,7 @@
 // RUN: rm -rf %t.bc
 // RUN: %clang -c -v --target=x86_64-linux-gnu \
 // RUN:   -x hip --offload-arch=gfx1100 --offload-arch=gfx1101 \
-// RUN:   -fgpu-rdc -nogpuinc -nogpulib \
+// RUN:   --no-offload-new-driver -fgpu-rdc -nogpuinc -nogpulib \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu \
 // RUN:   --offload-compress --offload-device-only --gpu-bundle-output \
 // RUN:   -o %t.bc \
@@ -22,7 +22,7 @@
 
 // RUN: %clang --hip-link -### -v --target=x86_64-linux-gnu \
 // RUN:   --offload-arch=gfx1100 --offload-arch=gfx1101 \
-// RUN:   -fgpu-rdc -nogpulib \
+// RUN:   --no-offload-new-driver -fgpu-rdc -nogpulib \
 // RUN:   %t.bc --offload-device-only \
 // RUN: 2>&1 | FileCheck -check-prefix=UNBUNDLE %s
 
@@ -35,7 +35,7 @@
 
 // RUN: %clang -c -### -v --target=x86_64-linux-gnu \
 // RUN:   -x hip --offload-arch=gfx1100 --offload-arch=gfx1101 \
-// RUN:   -nogpuinc -nogpulib \
+// RUN:   --no-offload-new-driver -nogpuinc -nogpulib \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu \
 // RUN:   --offload-compress \
 // RUN: 2>&1 | FileCheck -check-prefix=CO %s
diff --git a/clang/test/Driver/hip-output-file-name.hip b/clang/test/Driver/hip-output-file-name.hip
index 746678b81e25..aca64346e0c4 100644
--- a/clang/test/Driver/hip-output-file-name.hip
+++ b/clang/test/Driver/hip-output-file-name.hip
@@ -2,7 +2,7 @@
 // REQUIRES: amdgpu-registered-target
 
 // Output bundled code objects for combined compilation.
-// RUN: %clang -### -c --target=x86_64-linux-gnu -fgpu-rdc \
+// RUN: %clang -### -c --target=x86_64-linux-gnu -fgpu-rdc --no-offload-new-driver \
 // RUN:   -nogpulib -nogpuinc --offload-arch=gfx803 --offload-arch=gfx900 %s \
 // RUN: 2>&1 | FileCheck %s
 
@@ -13,42 +13,42 @@
 // is used to bundle the final output.
 
 // Output bundled PPE for one GPU for mixed compliation.
-// RUN: %clang -### -E --target=x86_64-linux-gnu \
+// RUN: %clang -### -E --target=x86_64-linux-gnu --no-offload-new-driver \
 // RUN:   -nogpulib -nogpuinc --offload-arch=gfx803 %s \
 // RUN: 2>&1 | FileCheck -check-prefixes=DASH %s
 
 // Output unbundled PPE for one GPU for device only compilation.
 // RUN: %clang -### -E --offload-device-only --target=x86_64-linux-gnu \
-// RUN:   -nogpulib -nogpuinc --offload-arch=gfx803 %s \
+// RUN:   --no-offload-new-driver -nogpulib -nogpuinc --offload-arch=gfx803 %s \
 // RUN: 2>&1 | FileCheck -check-prefixes=CLANG-DASH %s
 
 // Output bundled PPE for two GPUs for mixed compilation.
-// RUN: %clang -### -E --target=x86_64-linux-gnu \
+// RUN: %clang -### -E --target=x86_64-linux-gnu --no-offload-new-driver \
 // RUN:   -nogpulib -nogpuinc --offload-arch=gfx803 --offload-arch=gfx900 %s \
 // RUN: 2>&1 | FileCheck -check-prefixes=DASH %s
 
 // Output bundled PPE for two GPUs for mixed compilation with -save-temps.
-// RUN: %clang -### -E -save-temps --target=x86_64-linux-gnu \
+// RUN: %clang -### -E -save-temps --target=x86_64-linux-gnu --no-offload-new-driver \
 // RUN:   -nogpulib -nogpuinc --offload-arch=gfx803 --offload-arch=gfx900 %s \
 // RUN: 2>&1 | FileCheck -check-prefixes=DASH %s
 
 // Output unbundled PPE for two GPUs for device only compilation.
-// RUN: %clang -### -E --offload-device-only --target=x86_64-linux-gnu \
+// RUN: %clang -### -E --offload-device-only --target=x86_64-linux-gnu --no-offload-new-driver \
 // RUN:   -nogpulib -nogpuinc --offload-arch=gfx803 --offload-arch=gfx900 %s \
 // RUN: 2>&1 | FileCheck -check-prefixes=CLANG-DASH %s
 
 // Output bundled PPE for two GPUs for device only compilation with --gpu-bundle-output.
-// RUN: %clang -### -E --offload-device-only --target=x86_64-linux-gnu \
+// RUN: %clang -### -E --offload-device-only --target=x86_64-linux-gnu --no-offload-new-driver \
 // RUN:   -nogpulib -nogpuinc --offload-arch=gfx803 --offload-arch=gfx900 %s --gpu-bundle-output \
 // RUN: 2>&1 | FileCheck -check-prefixes=DASH %s
 
 // Output unbundled PPE for two GPUs for device only compilation with --no-gpu-bundle-output.
-// RUN: %clang -### -E --offload-device-only --target=x86_64-linux-gnu \
+// RUN: %clang -### -E --offload-device-only --target=x86_64-linux-gnu --no-offload-new-driver \
 // RUN:   -nogpulib -nogpuinc --offload-arch=gfx803 --offload-arch=gfx900 %s --no-gpu-bundle-output \
 // RUN: 2>&1 | FileCheck -check-prefixes=CLANG-DASH %s
 
 // Output unbundled PPE for host only compilation.
-// RUN: %clang -### -E --offload-host-only --target=x86_64-linux-gnu \
+// RUN: %clang -### -E --offload-host-only --target=x86_64-linux-gnu --no-offload-new-driver \
 // RUN:   -nogpulib -nogpuinc --offload-arch=gfx803 --offload-arch=gfx900 %s \
 // RUN: 2>&1 | FileCheck -check-prefixes=CLANG-DASH %s
 
@@ -60,22 +60,22 @@
 // Check -E with -o.
 
 // Output bundled PPE for two GPUs for mixed compilation.
-// RUN: %clang -### -E -o test.cui --target=x86_64-linux-gnu \
+// RUN: %clang -### -E -o test.cui --target=x86_64-linux-gnu --no-offload-new-driver \
 // RUN:   -nogpulib -nogpuinc --offload-arch=gfx803 --offload-arch=gfx900 %s \
 // RUN: 2>&1 | FileCheck -check-prefixes=OUT %s
 
 // Output bundled PPE for two GPUs for mixed compilation.
-// RUN: %clang -### -E -o test.cui -save-temps --target=x86_64-linux-gnu \
+// RUN: %clang -### -E -o test.cui -save-temps --target=x86_64-linux-gnu --no-offload-new-driver \
 // RUN:   -nogpulib -nogpuinc --offload-arch=gfx803 --offload-arch=gfx900 %s \
 // RUN: 2>&1 | FileCheck -check-prefixes=OUT %s
 
 // Output bundled PPE for two GPUs for device only compilation with --gpu-bundle-output.
-// RUN: %clang -### -E -o test.cui --offload-device-only --target=x86_64-linux-gnu \
+// RUN: %clang -### -E -o test.cui --offload-device-only --target=x86_64-linux-gnu --no-offload-new-driver \
 // RUN:   -nogpulib -nogpuinc --offload-arch=gfx803 --offload-arch=gfx900 --gpu-bundle-output %s \
 // RUN: 2>&1 | FileCheck -check-prefixes=OUT %s
 
 // Output unbundled PPE for two GPUs for device only compilation.
-// RUN: %clang -### -E -o test.cui --offload-host-only --target=x86_64-linux-gnu \
+// RUN: %clang -### -E -o test.cui --offload-host-only --target=x86_64-linux-gnu --no-offload-new-driver \
 // RUN:   -nogpulib -nogpuinc --offload-arch=gfx803 --offload-arch=gfx900 %s \
 // RUN: 2>&1 | FileCheck -check-prefixes=CLANG-OUT %s
 
diff --git a/clang/test/Driver/hip-partial-link.hip b/clang/test/Driver/hip-partial-link.hip
index a1d31f9a6519..faa185972abc 100644
--- a/clang/test/Driver/hip-partial-link.hip
+++ b/clang/test/Driver/hip-partial-link.hip
@@ -1,14 +1,14 @@
 // REQUIRES: x86-registered-target, amdgpu-registered-target, lld, system-linux
 
-// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu \
+// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu --no-offload-new-driver \
 // RUN:   --offload-arch=gfx906 -c -nostdinc -nogpuinc -nohipwrapperinc \
 // RUN:   -nogpulib -fgpu-rdc -I%S/Inputs %s -o %t.1.o
 
-// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -DLIB \
+// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -DLIB --no-offload-new-driver \
 // RUN:   --offload-arch=gfx906 -c -nostdinc -nogpuinc -nohipwrapperinc \
 // RUN:   -nogpulib -fgpu-rdc -I%S/Inputs %s -o %t.2.o
 
-// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -DMAIN \
+// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -DMAIN --no-offload-new-driver \
 // RUN:   --offload-arch=gfx906 -c -nostdinc -nogpuinc -nohipwrapperinc \
 // RUN:   -nogpulib -fgpu-rdc -I%S/Inputs %s -o %t.main.o
 
@@ -24,7 +24,7 @@
 
 // Link %t.1.o and %t.2.o by -r and then link with %t.main.o
 
-// RUN: %clang -v --target=x86_64-unknown-linux-gnu \
+// RUN: %clang -v --target=x86_64-unknown-linux-gnu --no-offload-new-driver \
 // RUN:   --hip-link -fgpu-rdc --offload-arch=gfx906 \
 // RUN:   -r -fuse-ld=lld -nostdlib %t.1.o %t.2.o -o %t.lib.o \
 // RUN:   2>&1 | FileCheck -check-prefix=LD-R %s
@@ -46,7 +46,7 @@
 // OBJ:  D __hip_gpubin_handle_[[ID1]]
 // OBJ:  D __hip_gpubin_handle_[[ID2]]
 
-// RUN: %clang -v --target=x86_64-unknown-linux-gnu \
+// RUN: %clang -v --target=x86_64-unknown-linux-gnu --no-offload-new-driver \
 // RUN:   --hip-link -no-hip-rt -fgpu-rdc --offload-arch=gfx906 \
 // RUN:   -fuse-ld=lld -nostdlib -r %t.main.o %t.lib.o -o %t.final.o \
 // RUN:   2>&1 | FileCheck -check-prefix=LINK-O %s
@@ -54,7 +54,7 @@
 
 // Generate a static lib with %t.1.o and %t.2.o then link with %t.main.o
 
-// RUN: %clang -v --target=x86_64-unknown-linux-gnu \
+// RUN: %clang -v --target=x86_64-unknown-linux-gnu --no-offload-new-driver \
 // RUN:   --hip-link -fgpu-rdc --offload-arch=gfx906 \
 // RUN:   --emit-static-lib -fuse-ld=lld -nostdlib %t.1.o %t.2.o -o %t.a \
 // RUN:   2>&1 | FileCheck -check-prefix=STATIC %s
@@ -68,7 +68,7 @@
 // STATIC: "{{.*}}/llvm-mc" -triple x86_64-unknown-linux-gnu
 // STATIC: "{{.*}}/llvm-ar"
 
-// RUN: %clang -v --target=x86_64-unknown-linux-gnu \
+// RUN: %clang -v --target=x86_64-unknown-linux-gnu --no-offload-new-driver \
 // RUN:   --hip-link -no-hip-rt -fgpu-rdc --offload-arch=gfx906 \
 // RUN:   -fuse-ld=lld -nostdlib -r %t.main.o %t.a -o %t.final.o \
 // RUN:   2>&1 | FileCheck -check-prefix=LINK-A %s
diff --git a/clang/test/Driver/hip-phases.hip b/clang/test/Driver/hip-phases.hip
index e976583820cc..ca63d4304d39 100644
--- a/clang/test/Driver/hip-phases.hip
+++ b/clang/test/Driver/hip-phases.hip
@@ -10,14 +10,14 @@
 // Test single gpu architecture with complete compilation.
 //
 // RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases \
-// RUN: --cuda-gpu-arch=gfx803 %s 2>&1 \
+// RUN: --no-offload-new-driver --cuda-gpu-arch=gfx803 %s 2>&1 \
 // RUN: | FileCheck -check-prefixes=BIN,NRD,OLD %s
 // RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases \
 // RUN: --offload-new-driver --cuda-gpu-arch=gfx803 %s 2>&1 \
 // RUN: | FileCheck -check-prefixes=BIN,NRD,NEW %s
 //
 // RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases \
-// RUN: --cuda-gpu-arch=gfx803 -fgpu-rdc %s 2>&1 \
+// RUN: --no-offload-new-driver --cuda-gpu-arch=gfx803 -fgpu-rdc %s 2>&1 \
 // RUN: | FileCheck -check-prefixes=BIN,RDC %s
 //
 // BIN-DAG: [[P0:[0-9]+]]: input, "{{.*}}hip-phases.hip", [[T:hip]], (host-[[T]])
@@ -49,7 +49,7 @@
 // Test single gpu architecture up to the assemble phase.
 //
 // RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases \
-// RUN: --cuda-gpu-arch=gfx803 %s -S 2>&1 \
+// RUN: --no-offload-new-driver --cuda-gpu-arch=gfx803 %s -S 2>&1 \
 // RUN: | FileCheck -check-prefixes=ASM %s
 // ASM-DAG: [[P0:[0-9]+]]: input, "{{.*}}hip-phases.hip", [[T:hip]], (device-[[T]], [[ARCH:gfx803]])
 // ASM-DAG: [[P1:[0-9]+]]: preprocessor, {[[P0]]}, [[T]]-cpp-output, (device-[[T]], [[ARCH]])
@@ -64,11 +64,11 @@
 // Test two gpu architectures with complete compilation with -fno-gpu-rdc.
 //
 // RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases \
-// RUN: --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s 2>&1 \
+// RUN: --no-offload-new-driver --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s 2>&1 \
 // RUN: | FileCheck -check-prefixes=NRD2,NCL2 %s
 
 // RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases \
-// RUN: --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s -c 2>&1 \
+// RUN: --no-offload-new-driver --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s -c 2>&1 \
 // RUN: | FileCheck -check-prefixes=NRD2 %s
 
 // NRD2-DAG: [[P0:[0-9]+]]: input, "{{.*}}hip-phases.hip", [[T:hip]], (host-[[T]])
@@ -100,11 +100,11 @@
 // Test two gpu architectures with complete compilation with -fgpu-rdc.
 //
 // RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases \
-// RUN: --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s -fgpu-rdc 2>&1 \
+// RUN: --no-offload-new-driver --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s -fgpu-rdc 2>&1 \
 // RUN: | FileCheck -check-prefixes=RDC2,RCL2 %s
 
 // RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases \
-// RUN: --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s -fgpu-rdc -c 2>&1 \
+// RUN: --no-offload-new-driver --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s -fgpu-rdc -c 2>&1 \
 // RUN: | FileCheck -check-prefixes=RDC2,RC2 %s
 
 // RCL2-DAG: [[P0:[0-9]+]]: input, "{{.*}}hip-phases.hip", [[T:hip]], (host-[[T]])
@@ -144,7 +144,7 @@
 // Test two gpu architecturess up to the assemble phase.
 //
 // RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases \
-// RUN: --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s -S 2>&1 \
+// RUN: --no-offload-new-driver --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s -S 2>&1 \
 // RUN: | FileCheck -check-prefixes=ASM2 %s
 // ASM2-DAG: [[P0:[0-9]+]]: input, "{{.*}}hip-phases.hip", [[T:hip]], (device-[[T]], [[ARCH1:gfx803]])
 // ASM2-DAG: [[P1:[0-9]+]]: preprocessor, {[[P0]]}, [[T]]-cpp-output, (device-[[T]], [[ARCH1]])
@@ -162,7 +162,7 @@
 // compilation mode.
 //
 // RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases \
-// RUN: --cuda-gpu-arch=gfx803 %s --cuda-host-only 2>&1 \
+// RUN: --no-offload-new-driver --cuda-gpu-arch=gfx803 %s --cuda-host-only 2>&1 \
 // RUN: | FileCheck -check-prefixes=HBIN %s
 // HBIN-DAG: [[P0:[0-9]+]]: input, "{{.*}}hip-phases.hip", [[T:hip]], (host-[[T]])
 // HBIN-DAG: [[P1:[0-9]+]]: preprocessor, {[[P0]]}, [[T]]-cpp-output, (host-[[T]])
@@ -176,7 +176,7 @@
 // compilation mode.
 //
 // RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases \
-// RUN: --cuda-gpu-arch=gfx803 %s --cuda-host-only -S 2>&1 \
+// RUN: --no-offload-new-driver --cuda-gpu-arch=gfx803 %s --cuda-host-only -S 2>&1 \
 // RUN: | FileCheck -check-prefixes=HASM %s
 // HASM-DAG: [[P0:[0-9]+]]: input, "{{.*}}hip-phases.hip", [[T:hip]], (host-[[T]])
 // HASM-DAG: [[P1:[0-9]+]]: preprocessor, {[[P0]]}, [[T]]-cpp-output, (host-[[T]])
@@ -189,7 +189,7 @@
 // compilation mode.
 //
 // RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases \
-// RUN: --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s --cuda-host-only 2>&1 \
+// RUN: --no-offload-new-driver --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s --cuda-host-only 2>&1 \
 // RUN: | FileCheck -check-prefixes=HBIN2 %s
 // HBIN2-DAG: [[P0:[0-9]+]]: input, "{{.*}}hip-phases.hip", [[T:hip]], (host-[[T]])
 // HBIN2-DAG: [[P1:[0-9]+]]: preprocessor, {[[P0]]}, [[T]]-cpp-output, (host-[[T]])
@@ -204,7 +204,7 @@
 // compilation mode.
 //
 // RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases \
-// RUN: --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s --cuda-host-only -S \
+// RUN: --no-offload-new-driver --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s --cuda-host-only -S \
 // RUN: 2>&1 | FileCheck -check-prefixes=HASM2 %s
 // HASM2-DAG: [[P0:[0-9]+]]: input, "{{.*}}hip-phases.hip", [[T:hip]], (host-[[T]])
 // HASM2-DAG: [[P1:[0-9]+]]: preprocessor, {[[P0]]}, [[T]]-cpp-output, (host-[[T]])
@@ -217,14 +217,14 @@
 // compilation mode.
 //
 // RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases \
-// RUN: --cuda-gpu-arch=gfx803 %s --cuda-device-only 2>&1 \
+// RUN: --no-offload-new-driver --cuda-gpu-arch=gfx803 %s --cuda-device-only 2>&1 \
 // RUN: | FileCheck -check-prefixes=DBIN %s
 //
 // Test single gpu architecture with complete compilation in device-only
 // compilation mode with an unused host linker flag.
 //
 // RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases \
-// RUN: --cuda-gpu-arch=gfx803 %s --cuda-device-only -Wl,--disable-new-dtags 2>&1 \
+// RUN: --no-offload-new-driver --cuda-gpu-arch=gfx803 %s --cuda-device-only -Wl,--disable-new-dtags 2>&1 \
 // RUN: | FileCheck -check-prefixes=DBIN %s
 
 // DBIN-DAG: [[P0:[0-9]+]]: input, "{{.*}}hip-phases.hip", [[T:hip]], (device-[[T]], [[ARCH:gfx803]])
@@ -242,7 +242,7 @@
 // Test single gpu architecture up to the assemble phase in device-only
 // compilation mode.
 //
-// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases \
+// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases --no-offload-new-driver \
 // RUN: --cuda-gpu-arch=gfx803 %s --cuda-device-only -S --no-gpu-bundle-output 2>&1 \
 // RUN: | FileCheck -check-prefixes=DASM %s
 // DASM-DAG: [[P0:[0-9]+]]: input, "{{.*}}hip-phases.hip", [[T:hip]], (device-[[T]], [[ARCH:gfx803]])
@@ -257,11 +257,11 @@
 // Test single gpu architecture with compile to relocatable in device-only
 // compilation mode.
 //
-// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases \
+// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases --no-offload-new-driver \
 // RUN: --cuda-gpu-arch=gfx803 %s --cuda-device-only -fhip-emit-relocatable 2>&1 \
 // RUN: | FileCheck -check-prefixes=RELOC %s
 //
-// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases \
+// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases --no-offload-new-driver \
 // RUN: --cuda-gpu-arch=gfx803 %s --cuda-device-only -fhip-emit-relocatable -Wl,--disable-new-dtags \
 // RUN: 2>&1 | FileCheck -check-prefixes=RELOC %s
 //
@@ -278,11 +278,11 @@
 // Test two gpu architectures with compile to relocatable in device-only
 // compilation mode.
 //
-// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases \
+// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases --no-offload-new-driver \
 // RUN: --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s --cuda-device-only -fhip-emit-relocatable 2>&1 \
 // RUN: | FileCheck -check-prefixes=RELOC2 %s
 //
-// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases \
+// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases --no-offload-new-driver \
 // RUN: --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s --cuda-device-only -fhip-emit-relocatable \
 // RUN: -Wl,--disable-new-dtags 2>&1 | FileCheck -check-prefixes=RELOC2 %s
 //
@@ -306,14 +306,14 @@
 // Test two gpu architectures with complete compilation in device-only
 // compilation mode.
 //
-// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases \
+// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases --no-offload-new-driver \
 // RUN: --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s --cuda-device-only \
 // RUN: 2>&1 | FileCheck -check-prefixes=DBIN2 %s
 //
 // Test two gpu architectures with complete compilation in device-only
 // compilation mode with an unused host linker flag.
 //
-// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases \
+// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu -ccc-print-phases --no-offload-new-driver \
 // RUN: --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s --cuda-device-only \
 // RUN: -Wl,--disable-new-dtags 2>&1 | FileCheck -check-prefixes=DBIN2 %s
 
@@ -339,19 +339,19 @@
 // Test two gpu architectures up to the assemble phase in device-only
 // compilation mode.
 //
-// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu \
+// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu --no-offload-new-driver \
 // RUN: -ccc-print-phases --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s \
 // RUN: --cuda-device-only -S -o %t.s 2>&1 \
 // RUN: | FileCheck -check-prefixes=DASM2,DASM2-NOBUNDLE %s
-// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu \
+// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu --no-offload-new-driver \
 // RUN: -ccc-print-phases --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s \
 // RUN: --cuda-device-only -S -o %t.s --no-gpu-bundle-output 2>&1 \
 // RUN: | FileCheck -check-prefixes=DASM2,DASM2-NOBUNDLE %s
-// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu \
+// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu --no-offload-new-driver \
 // RUN: -ccc-print-phases --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s \
 // RUN: --cuda-device-only -S 2>&1 \
 // RUN: | FileCheck -check-prefixes=DASM2,DASM2-NOBUNDLE %s
-// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu \
+// RUN: %clang -x hip --target=x86_64-unknown-linux-gnu --no-offload-new-driver \
 // RUN: -ccc-print-phases --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s \
 // RUN: --cuda-device-only -S --gpu-bundle-output 2>&1 \
 // RUN: | FileCheck -check-prefixes=DASM2,DASM2-BUNDLE %s
@@ -376,29 +376,29 @@
 // RUN: touch %t/obj1.o %t/obj2.o
 
 // RUN: %clang --target=x86_64-unknown-linux-gnu -ccc-print-phases --hip-link \
-// RUN: --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %t/obj1.o %t/obj2.o 2>&1 \
+// RUN: --no-offload-new-driver --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %t/obj1.o %t/obj2.o 2>&1 \
 // RUN: | FileCheck -check-prefixes=L2,NL2 %s
 //
 // RUN: %clang --target=x86_64-unknown-linux-gnu -ccc-print-phases --hip-link \
-// RUN: --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %t/obj1.o %t/obj2.o \
+// RUN: --no-offload-new-driver --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %t/obj1.o %t/obj2.o \
 // RUN: -fgpu-rdc 2>&1 | FileCheck -check-prefixes=L2,RL2,RL2-EM %s
 //
 // RUN: %clang --target=x86_64-unknown-linux-gnu -ccc-print-phases --hip-link \
-// RUN: --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %t/obj1.o %t/obj2.o \
+// RUN: --no-offload-new-driver --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %t/obj1.o %t/obj2.o \
 // RUN: -fgpu-rdc --cuda-device-only 2>&1 | FileCheck -check-prefixes=L2,RL2,RL2-DEV %s
 
 // RUN: %clang --target=x86_64-unknown-linux-gnu -ccc-print-phases --hip-link \
-// RUN: --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %t/obj1.o %t/obj2.o \
+// RUN: --no-offload-new-driver --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %t/obj1.o %t/obj2.o \
 // RUN: -fgpu-rdc --cuda-device-only -Wl,--disable-new-dtags 2>&1 \
 // RUN: | FileCheck -check-prefixes=L2,RL2,RL2-DEV %s
 
 // RUN: %clang --target=x86_64-unknown-linux-gnu -ccc-print-phases --hip-link \
-// RUN: --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %t/obj1.o %t/obj2.o \
+// RUN: --no-offload-new-driver --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %t/obj1.o %t/obj2.o \
 // RUN: -fgpu-rdc --cuda-device-only --no-gpu-bundle-output 2>&1 \
 // RUN: | FileCheck -check-prefixes=L2,RL2,RL2-NB %s
 
 // RUN: %clang --target=x86_64-unknown-linux-gnu -ccc-print-phases --hip-link \
-// RUN: --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %t/obj1.o %t/obj2.o \
+// RUN: --no-offload-new-driver --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %t/obj1.o %t/obj2.o \
 // RUN: -fgpu-rdc --cuda-device-only --no-gpu-bundle-output -Wl,--disable-new-dtags 2>&1 \
 // RUN: | FileCheck -check-prefixes=L2,RL2,RL2-NB %s
 
@@ -428,12 +428,12 @@
 //
 // RUN: %clang -x hip --target=x86_64-unknown-linux-gnu \
 // RUN: -ccc-print-phases --cuda-gpu-arch=gfx803 %s \
-// RUN: --cuda-device-only -E 2>&1 \
+// RUN: --no-offload-new-driver --cuda-device-only -E 2>&1 \
 // RUN: | FileCheck -check-prefixes=PPE,PPEN %s
 
 // RUN: %clang -x hip --target=x86_64-unknown-linux-gnu \
 // RUN: -ccc-print-phases --cuda-gpu-arch=gfx803 %s \
-// RUN: --cuda-device-only -E --no-gpu-bundle-output 2>&1 \
+// RUN: --no-offload-new-driver --cuda-device-only -E --no-gpu-bundle-output 2>&1 \
 // RUN: | FileCheck -check-prefixes=PPE,PPEN %s
 
 // Test one gpu architectures up to the preprocessor expansion output phase in device-only
@@ -441,7 +441,7 @@
 
 // RUN: %clang -x hip --target=x86_64-unknown-linux-gnu \
 // RUN: -ccc-print-phases --cuda-gpu-arch=gfx803 %s \
-// RUN: --cuda-device-only -E --gpu-bundle-output 2>&1 \
+// RUN: --no-offload-new-driver --cuda-device-only -E --gpu-bundle-output 2>&1 \
 // RUN: | FileCheck -check-prefixes=PPE,PPEB %s
 
 // Test two gpu architectures up to the preprocessor expansion output phase in device-only
@@ -449,12 +449,12 @@
 
 // RUN: %clang -x hip --target=x86_64-unknown-linux-gnu \
 // RUN: -ccc-print-phases --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s \
-// RUN: --cuda-device-only -E 2>&1 \
+// RUN: --no-offload-new-driver --cuda-device-only -E 2>&1 \
 // RUN: | FileCheck -check-prefixes=PPE2,PPE2N %s
 
 // RUN: %clang -x hip --target=x86_64-unknown-linux-gnu \
 // RUN: -ccc-print-phases --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s \
-// RUN: --cuda-device-only -E --no-gpu-bundle-output 2>&1 \
+// RUN: --no-offload-new-driver --cuda-device-only -E --no-gpu-bundle-output 2>&1 \
 // RUN: | FileCheck -check-prefixes=PPE2,PPE2N %s
 
 // Test two gpu architectures up to the preprocessor expansion output phase in device-only
@@ -462,7 +462,7 @@
 
 // RUN: %clang -x hip --target=x86_64-unknown-linux-gnu \
 // RUN: -ccc-print-phases --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s \
-// RUN: --cuda-device-only -E --gpu-bundle-output 2>&1 \
+// RUN: --no-offload-new-driver --cuda-device-only -E --gpu-bundle-output 2>&1 \
 // RUN: | FileCheck -check-prefixes=PPE2,PPE2B %s
 
 // Test one gpu architectures up to the LLVM IR output phase in device-only
@@ -470,7 +470,7 @@
 //
 // RUN: %clang -x hip --target=x86_64-unknown-linux-gnu \
 // RUN: -ccc-print-phases --cuda-gpu-arch=gfx803 %s \
-// RUN: --cuda-device-only -c -emit-llvm 2>&1 \
+// RUN: --no-offload-new-driver --cuda-device-only -c -emit-llvm 2>&1 \
 // RUN: | FileCheck -check-prefixes=LLVM %s
 
 // Test two gpu architectures up to the LLVM IR output phase in device-only
@@ -478,7 +478,7 @@
 //
 // RUN: %clang -x hip --target=x86_64-unknown-linux-gnu \
 // RUN: -ccc-print-phases --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s \
-// RUN: --cuda-device-only -c -emit-llvm -o %t.bc --gpu-bundle-output 2>&1 \
+// RUN: --no-offload-new-driver --cuda-device-only -c -emit-llvm -o %t.bc --gpu-bundle-output 2>&1 \
 // RUN: | FileCheck -check-prefixes=LLVM2 %s
 
 // Test two gpu architectures up to the LLVM IR output phase in device-only
@@ -486,7 +486,7 @@
 //
 // RUN: %clang -x hip-cpp-output --target=x86_64-unknown-linux-gnu \
 // RUN: -ccc-print-phases --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 %s \
-// RUN: --cuda-device-only -c -emit-llvm -o %t.bc --gpu-bundle-output 2>&1 \
+// RUN: --no-offload-new-driver --cuda-device-only -c -emit-llvm -o %t.bc --gpu-bundle-output 2>&1 \
 // RUN: | FileCheck -check-prefixes=PPELLVM2 %s
 
 // PPE-DAG: [[P0:[0-9]+]]: input, "{{.*}}hip-phases.hip", [[T:hip]], (device-[[T]], [[ARCH:gfx803]])
@@ -541,50 +541,50 @@
 // C++ program should have no offload kind.
 
 // Test compile empty.hip and empty.cpp.
-// RUN: %clang --target=x86_64-unknown-linux-gnu \
+// RUN: %clang --target=x86_64-unknown-linux-gnu --no-offload-new-driver \
 // RUN: -ccc-print-phases --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
 // RUN: -c %S/Inputs/empty.hip %S/Inputs/empty.cpp 2>&1 | FileCheck -check-prefixes=MIXED %s
-// RUN: %clang --target=x86_64-unknown-linux-gnu \
+// RUN: %clang --target=x86_64-unknown-linux-gnu --no-offload-new-driver \
 // RUN: -ccc-print-phases --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
 // RUN: -c %S/Inputs/empty.hip %S/Inputs/empty.cpp 2>&1 | FileCheck -check-prefixes=MIXED-NEG %s
 
 // Test compile and link empty.hip and empty.cpp.
-// RUN: %clang --target=x86_64-unknown-linux-gnu \
+// RUN: %clang --target=x86_64-unknown-linux-gnu --no-offload-new-driver \
 // RUN: -ccc-print-phases --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
 // RUN: %S/Inputs/empty.hip %S/Inputs/empty.cpp 2>&1 | FileCheck -check-prefixes=MIXED %s
-// RUN: %clang --target=x86_64-unknown-linux-gnu \
+// RUN: %clang --target=x86_64-unknown-linux-gnu --no-offload-new-driver \
 // RUN: -ccc-print-phases --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
 // RUN: %S/Inputs/empty.hip %S/Inputs/empty.cpp 2>&1 | FileCheck -check-prefixes=MIXED-NEG %s
 
 // Test compile and link empty.hip and empty.cpp with --hip-link -fgpu-rdc.
-// RUN: %clang --target=x86_64-unknown-linux-gnu --hip-link -fgpu-rdc \
+// RUN: %clang --target=x86_64-unknown-linux-gnu --hip-link -fgpu-rdc --no-offload-new-driver \
 // RUN: -ccc-print-phases --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
 // RUN: %S/Inputs/empty.hip %S/Inputs/empty.cpp 2>&1 | FileCheck -check-prefixes=MIXED %s
-// RUN: %clang --target=x86_64-unknown-linux-gnu --hip-link -fgpu-rdc \
+// RUN: %clang --target=x86_64-unknown-linux-gnu --hip-link -fgpu-rdc --no-offload-new-driver \
 // RUN: -ccc-print-phases --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
 // RUN: %S/Inputs/empty.hip %S/Inputs/empty.cpp 2>&1 | FileCheck -check-prefixes=MIXED-NEG %s
 
 // Test compile and link -x hip empty.hip and -x c++ empty.cpp.
-// RUN: %clang --target=x86_64-unknown-linux-gnu \
+// RUN: %clang --target=x86_64-unknown-linux-gnu --no-offload-new-driver \
 // RUN: -ccc-print-phases --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
 // RUN: -x hip %S/Inputs/empty.hip -x c++ %S/Inputs/empty.cpp 2>&1 | FileCheck -check-prefixes=MIXED %s
-// RUN: %clang --target=x86_64-unknown-linux-gnu \
+// RUN: %clang --target=x86_64-unknown-linux-gnu --no-offload-new-driver \
 // RUN: -ccc-print-phases --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
 // RUN: -x hip %S/Inputs/empty.hip -x c++ %S/Inputs/empty.cpp 2>&1 | FileCheck -check-prefixes=MIXED-NEG %s
 
 // Test compile and link -x hip empty.hip and empty.cpp.
-// RUN: %clang --target=x86_64-unknown-linux-gnu \
+// RUN: %clang --target=x86_64-unknown-linux-gnu --no-offload-new-driver \
 // RUN: -ccc-print-phases --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
 // RUN: -x hip %S/Inputs/empty.hip %S/Inputs/empty.cpp 2>&1 | FileCheck -check-prefixes=MIXED2 %s
-// RUN: %clang --target=x86_64-unknown-linux-gnu \
+// RUN: %clang --target=x86_64-unknown-linux-gnu --no-offload-new-driver \
 // RUN: -ccc-print-phases --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
 // RUN: -x hip %S/Inputs/empty.hip %S/Inputs/empty.cpp 2>&1 | FileCheck -check-prefixes=MIXED2-NEG %s
 
 // Test compile and link empty.hip and -x hip empty.cpp.
-// RUN: %clang --target=x86_64-unknown-linux-gnu \
+// RUN: %clang --target=x86_64-unknown-linux-gnu --no-offload-new-driver \
 // RUN: -ccc-print-phases --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
 // RUN: %S/Inputs/empty.hip -x hip %S/Inputs/empty.cpp 2>&1 | FileCheck -check-prefixes=MIXED2 %s
-// RUN: %clang --target=x86_64-unknown-linux-gnu \
+// RUN: %clang --target=x86_64-unknown-linux-gnu --no-offload-new-driver \
 // RUN: -ccc-print-phases --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
 // RUN: -x hip %S/Inputs/empty.hip %S/Inputs/empty.cpp 2>&1 | FileCheck -check-prefixes=MIXED2-NEG %s
 
@@ -609,7 +609,7 @@
 // RUN: touch %t/bitcodeA.bc
 // RUN: touch %t/bitcodeB.bc
 // RUN: %clang -ccc-print-phases --hip-link -emit-llvm --cuda-device-only \
-// RUN: --offload-arch=gfx906 %t/bitcodeA.bc %t/bitcodeB.bc 2>&1 \
+// RUN: --no-offload-new-driver --offload-arch=gfx906 %t/bitcodeA.bc %t/bitcodeB.bc 2>&1 \
 // RUN: | FileCheck -check-prefixes=CHECK %s
 
 // CHECK: [[A0:[0-9]+]]: input, "{{.*}}bitcodeA.bc", ir
diff --git a/clang/test/Driver/hip-rdc-device-only.hip b/clang/test/Driver/hip-rdc-device-only.hip
index d972927ff7a3..d79cc1febf3d 100644
--- a/clang/test/Driver/hip-rdc-device-only.hip
+++ b/clang/test/Driver/hip-rdc-device-only.hip
@@ -1,7 +1,7 @@
 // REQUIRES: x86-registered-target
 // REQUIRES: amdgpu-registered-target
 
-// RUN: %clang -### --target=x86_64-linux-gnu \
+// RUN: %clang -### --target=x86_64-linux-gnu --no-offload-new-driver \
 // RUN:   -x hip --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
 // RUN:   -c -nogpuinc -nogpulib --cuda-device-only -fgpu-rdc \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu \
@@ -11,7 +11,7 @@
 // With `-emit-llvm`, the output should be the same as the aforementioned line
 // as `-fgpu-rdc` in HIP implies `-emit-llvm`.
 
-// RUN: %clang -### --target=x86_64-linux-gnu \
+// RUN: %clang -### --target=x86_64-linux-gnu --no-offload-new-driver \
 // RUN:   -x hip --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
 // RUN:   -c -emit-llvm -nogpuinc -nogpulib --cuda-device-only -fgpu-rdc \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu \
@@ -21,14 +21,14 @@
 // With `-fno-hip-emit-relocatable`, the output should be the same as the aforementioned line
 // as `-fgpu-rdc` in HIP implies `-fno-hip-emit-relocatable`.
 
-// RUN: %clang -### --target=x86_64-linux-gnu \
+// RUN: %clang -### --target=x86_64-linux-gnu --no-offload-new-driver \
 // RUN:   -x hip --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
 // RUN:   -c -fno-hip-emit-relocatable -nogpuinc -nogpulib --cuda-device-only -fgpu-rdc \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu \
 // RUN:   %S/Inputs/hip_multiple_inputs/b.hip --gpu-bundle-output \
 // RUN: 2>&1 | FileCheck -check-prefixes=COMMON,EMITBC %s
 
-// RUN: %clang -### --target=x86_64-linux-gnu \
+// RUN: %clang -### --target=x86_64-linux-gnu --no-offload-new-driver \
 // RUN:   -x hip --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
 // RUN:   -S -nogpuinc -nogpulib --cuda-device-only -fgpu-rdc \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu \
@@ -38,7 +38,7 @@
 // With `-emit-llvm`, the output should be the same as the aforementioned line
 // as `-fgpu-rdc` in HIP implies `-emit-llvm`.
 
-// RUN: %clang -### --target=x86_64-linux-gnu \
+// RUN: %clang -### --target=x86_64-linux-gnu --no-offload-new-driver \
 // RUN:   -x hip --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
 // RUN:   -S -emit-llvm -nogpuinc -nogpulib --cuda-device-only -fgpu-rdc \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu \
@@ -49,7 +49,7 @@
 // output, there should 3 steps (preprocessor, compile, and backend) per source
 // and per target, totally 12 steps.
 
-// RUN: %clang -### -save-temps --target=x86_64-linux-gnu \
+// RUN: %clang -### -save-temps --target=x86_64-linux-gnu --no-offload-new-driver \
 // RUN:   -x hip --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
 // RUN:   -S -nogpuinc -nogpulib --cuda-device-only -fgpu-rdc \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu \
@@ -58,7 +58,7 @@
 
 // Check output one file without bundling cause error.
 
-// RUN: not %clang -### --target=x86_64-linux-gnu \
+// RUN: not %clang -### --target=x86_64-linux-gnu --no-offload-new-driver \
 // RUN:   -x hip --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
 // RUN:   -S -nogpuinc -nogpulib --cuda-device-only -fgpu-rdc \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu -o %t.s --no-gpu-bundle-output \
diff --git a/clang/test/Driver/hip-target-id.hip b/clang/test/Driver/hip-target-id.hip
index 703d6b50be2e..e7ba456565c0 100644
--- a/clang/test/Driver/hip-target-id.hip
+++ b/clang/test/Driver/hip-target-id.hip
@@ -5,7 +5,7 @@
 // RUN:   -x hip \
 // RUN:   --offload-arch=gfx908:xnack+:sramecc+ \
 // RUN:   --offload-arch=gfx908:xnack+:sramecc- \
-// RUN:   --rocm-path=%S/Inputs/rocm \
+// RUN:   --no-offload-new-driver --rocm-path=%S/Inputs/rocm \
 // RUN:   %s 2>&1 | FileCheck %s
 
 // RUN: %clang -### --target=x86_64-linux-gnu \
@@ -13,7 +13,7 @@
 // RUN:   --offload-arch=gfx908:xnack+:sramecc+ \
 // RUN:   --offload-arch=gfx908:xnack+:sramecc- \
 // RUN:   --rocm-path=%S/Inputs/rocm \
-// RUN:   -save-temps \
+// RUN:   --no-offload-new-driver -save-temps \
 // RUN:   %s 2>&1 | FileCheck --check-prefixes=CHECK,TMP %s
 
 // RUN: %clang -### --target=x86_64-linux-gnu \
@@ -21,7 +21,7 @@
 // RUN:   --offload-arch=gfx908:xnack+:sramecc+ \
 // RUN:   --offload-arch=gfx908:xnack+:sramecc- \
 // RUN:   --rocm-path=%S/Inputs/rocm \
-// RUN:   -fgpu-rdc \
+// RUN:   --no-offload-new-driver -fgpu-rdc \
 // RUN:   %s 2>&1 | FileCheck --check-prefixes=CHECK %s
 
 // CHECK: [[CLANG:"[^"]*clang[^"]*"]] "-cc1" "-triple" "amdgcn-amd-amdhsa"
@@ -55,7 +55,7 @@
 // RUN:   --offload-arch=fiji \
 // RUN:   --offload-arch=gfx803 \
 // RUN:   --offload-arch=fiji \
-// RUN:   --rocm-path=%S/Inputs/rocm \
+// RUN:   --no-offload-new-driver --rocm-path=%S/Inputs/rocm \
 // RUN:   %s 2>&1 | FileCheck -check-prefix=FIJI %s
 // FIJI: "-targets=host-x86_64-unknown-linux,hipv4-amdgcn-amd-amdhsa--gfx803"
 
@@ -66,6 +66,6 @@
 // RUN:   --offload-arch=gfx908:sramecc+ \
 // RUN:   --offload-arch=gfx908:sramecc- \
 // RUN:   --offload-arch=gfx906 \
-// RUN:   --rocm-path=%S/Inputs/rocm \
+// RUN:   --no-offload-new-driver --rocm-path=%S/Inputs/rocm \
 // RUN:   %s 2>&1 | FileCheck -check-prefix=MULTI %s
 // MULTI: "-targets=host-x86_64-unknown-linux,hipv4-amdgcn-amd-amdhsa--gfx900:xnack+,hipv4-amdgcn-amd-amdhsa--gfx900:xnack-,hipv4-amdgcn-amd-amdhsa--gfx906,hipv4-amdgcn-amd-amdhsa--gfx908:sramecc+,hipv4-amdgcn-amd-amdhsa--gfx908:sramecc-"
diff --git a/clang/test/Driver/hip-toolchain-features.hip b/clang/test/Driver/hip-toolchain-features.hip
index 2e11ce38403e..551d8ef42e02 100644
--- a/clang/test/Driver/hip-toolchain-features.hip
+++ b/clang/test/Driver/hip-toolchain-features.hip
@@ -1,10 +1,10 @@
 // REQUIRES: x86-registered-target
 // REQUIRES: amdgpu-registered-target
 
-// RUN: %clang -### --target=x86_64-linux-gnu -fgpu-rdc -nogpulib \
+// RUN: %clang -### --target=x86_64-linux-gnu -fgpu-rdc -nogpulib --no-offload-new-driver \
 // RUN:   -nogpuinc --offload-arch=gfx906:xnack+ --offload-arch=gfx900:xnack+ %s \
 // RUN:   2>&1 | FileCheck %s -check-prefix=XNACK
-// RUN: %clang -### --target=x86_64-linux-gnu -fgpu-rdc -nogpulib \
+// RUN: %clang -### --target=x86_64-linux-gnu -fgpu-rdc -nogpulib --no-offload-new-driver \
 // RUN:   -nogpuinc --offload-arch=gfx906:xnack- --offload-arch=gfx900:xnack- %s \
 // RUN:   2>&1 | FileCheck %s -check-prefix=NOXNACK
 
@@ -14,10 +14,10 @@
 // NOXNACK: {{.*}}lld{{.*}} "-plugin-opt=-mattr=-xnack"
 
 // RUN: %clang -### --target=x86_64-linux-gnu -fgpu-rdc -nogpulib \
-// RUN:   -nogpuinc --offload-arch=gfx908:sramecc+ %s \
+// RUN:   -nogpuinc --offload-arch=gfx908:sramecc+ --no-offload-new-driver %s \
 // RUN:   2>&1 | FileCheck %s -check-prefix=SRAM
 // RUN: %clang -### --target=x86_64-linux-gnu -fgpu-rdc -nogpulib \
-// RUN:   -nogpuinc --offload-arch=gfx908:sramecc- %s \
+// RUN:   -nogpuinc --offload-arch=gfx908:sramecc- --no-offload-new-driver %s \
 // RUN:   2>&1 | FileCheck %s -check-prefix=NOSRAM
 
 // SRAM: {{.*}}clang{{.*}}"-target-feature" "+sramecc"
@@ -26,10 +26,10 @@
 // NOTSRAM: {{.*}}lld{{.*}} "-plugin-opt=-mattr=-sramecc"
 
 // RUN: %clang -### --target=x86_64-linux-gnu -fgpu-rdc -nogpulib \
-// RUN:   -nogpuinc --offload-arch=gfx1010 %s \
+// RUN:   -nogpuinc --offload-arch=gfx1010 --no-offload-new-driver %s \
 // RUN:   -mcumode  2>&1 | FileCheck %s -check-prefix=CUMODE
 // RUN: %clang -### --target=x86_64-linux-gnu -fgpu-rdc -nogpulib \
-// RUN:   -nogpuinc --offload-arch=gfx1010 %s \
+// RUN:   -nogpuinc --offload-arch=gfx1010 --no-offload-new-driver %s \
 // RUN:   -mno-cumode  2>&1 | FileCheck %s -check-prefix=NOTCUMODE
 
 // CUMODE: {{.*}}clang{{.*}}"-target-feature" "+cumode"
@@ -38,20 +38,20 @@
 // NOTCUMODE: {{.*}}lld{{.*}} "-plugin-opt=-mattr=-cumode"
 
 // RUN: %clang -### --target=x86_64-linux-gnu -fgpu-rdc -nogpulib \
-// RUN:   -nogpuinc --offload-arch=gfx908:xnack+:sramecc+ %s \
+// RUN:   -nogpuinc --offload-arch=gfx908:xnack+:sramecc+ --no-offload-new-driver %s \
 // RUN:   2>&1 | FileCheck %s -check-prefix=ALL3
 // RUN: %clang -### --target=x86_64-linux-gnu -fgpu-rdc -nogpulib \
-// RUN:   -nogpuinc --offload-arch=gfx908:xnack-:sramecc- %s \
+// RUN:   -nogpuinc --offload-arch=gfx908:xnack-:sramecc- --no-offload-new-driver %s \
 // RUN:   2>&1 | FileCheck %s -check-prefix=NOALL3
 
 // ALL3: {{.*}}clang{{.*}}"-target-feature" "+sramecc" "-target-feature" "+xnack"
 // NOALL3: {{.*}}clang{{.*}}"-target-feature" "-sramecc" "-target-feature" "-xnack"
 
 // RUN: %clang -### --target=x86_64-linux-gnu -fgpu-rdc -nogpulib \
-// RUN:   -nogpuinc --offload-arch=gfx1010 %s \
+// RUN:   -nogpuinc --offload-arch=gfx1010 --no-offload-new-driver %s \
 // RUN:   -mtgsplit  2>&1 | FileCheck %s -check-prefix=TGSPLIT
 // RUN: %clang -### --target=x86_64-linux-gnu -fgpu-rdc -nogpulib \
-// RUN:   -nogpuinc --offload-arch=gfx1010 %s \
+// RUN:   -nogpuinc --offload-arch=gfx1010 --no-offload-new-driver %s \
 // RUN:   -mno-tgsplit  2>&1 | FileCheck %s -check-prefix=NOTTGSPLIT
 
 // TGSPLIT: {{.*}}clang{{.*}}"-target-feature" "+tgsplit"
@@ -60,7 +60,7 @@
 // NOTTGSPLIT: {{.*}}lld{{.*}} "-plugin-opt=-mattr=-tgsplit"
 
 // RUN: %clang -### --target=x86_64-linux-gnu -fgpu-rdc -nogpulib \
-// RUN:   -nogpuinc --offload-arch=gfx1010 %s \
+// RUN:   -nogpuinc --offload-arch=gfx1010 --no-offload-new-driver %s \
 // RUN:   -mcumode -mcumode -mno-cumode -mwavefrontsize64 -mcumode \
 // RUN:   -mwavefrontsize64 -mno-wavefrontsize64 2>&1 \
 // RUN:   | FileCheck %s -check-prefix=DUP
@@ -71,7 +71,7 @@
 // DUP: {{.*}}lld{{.*}} "-plugin-opt=-mattr=+cumode"
 
 // RUN: %clang -### --target=x86_64-linux-gnu -fgpu-rdc -nogpulib \
-// RUN:   -nogpuinc --offload-arch=gfx1010 %s \
+// RUN:   -nogpuinc --offload-arch=gfx1010 --no-offload-new-driver %s \
 // RUN:   -mno-wavefrontsize64 -mwavefrontsize64 2>&1 \
 // RUN:   | FileCheck %s -check-prefix=WAVE64
 // WAVE64: {{.*}}clang{{.*}} "-target-feature" "+wavefrontsize64"
diff --git a/clang/test/Driver/hip-toolchain-rdc-separate.hip b/clang/test/Driver/hip-toolchain-rdc-separate.hip
index e52184fdeacf..6efca87dc0db 100644
--- a/clang/test/Driver/hip-toolchain-rdc-separate.hip
+++ b/clang/test/Driver/hip-toolchain-rdc-separate.hip
@@ -7,7 +7,7 @@
 // RUN:   --hip-device-lib=lib1.bc --hip-device-lib=lib2.bc \
 // RUN:   --hip-device-lib-path=%S/Inputs/hip_multiple_inputs/lib1 \
 // RUN:   --hip-device-lib-path=%S/Inputs/hip_multiple_inputs/lib2 \
-// RUN:   -fuse-ld=lld -B%S/Inputs/lld -fgpu-rdc -nogpuinc \
+// RUN:   --no-offload-new-driver -fuse-ld=lld -B%S/Inputs/lld -fgpu-rdc -nogpuinc \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu \
 // RUN:   %S/Inputs/hip_multiple_inputs/b.hip \
 // RUN: 2>&1 | FileCheck %s
@@ -84,19 +84,19 @@
 
 // RUN: touch %t/a.o %t/b.o
 // RUN: %clang --hip-link -### --target=x86_64-linux-gnu \
-// RUN:   --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
+// RUN:   --no-offload-new-driver --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
 // RUN:   -fuse-ld=lld -B%S/Inputs/lld -fgpu-rdc -nogpuinc \
 // RUN:   %t/a.o %t/b.o \
 // RUN: 2>&1 | FileCheck -check-prefixes=LINK,LINK-HOST-UNBUNDLE,LLD-TMP,LINK-BUNDLE,LINK-EMBED %s
 
 // RUN: %clang --hip-link -### --target=x86_64-linux-gnu \
-// RUN:   --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
+// RUN:   --no-offload-new-driver --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
 // RUN:   -fuse-ld=lld -B%S/Inputs/lld -fgpu-rdc -nogpuinc \
 // RUN:   %t/a.o %t/b.o --cuda-device-only \
 // RUN: 2>&1 | FileCheck -check-prefixes=LINK,LLD-TMP,LINK-BUNDLE,LINK-NOEMBED %s
 
 // RUN: %clang --hip-link -### --target=x86_64-linux-gnu \
-// RUN:   --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
+// RUN:   --no-offload-new-driver --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
 // RUN:   -fuse-ld=lld -B%S/Inputs/lld -fgpu-rdc -nogpuinc \
 // RUN:   %t/a.o %t/b.o --cuda-device-only --no-gpu-bundle-output \
 // RUN: 2>&1 | FileCheck -check-prefixes=LINK,LLD-FIN,LINK-NOBUNDLE,LINK-NOEMBED %s
diff --git a/clang/test/Driver/hip-toolchain-rdc-static-lib.hip b/clang/test/Driver/hip-toolchain-rdc-static-lib.hip
index fd12d48a2b82..2cfb24852386 100644
--- a/clang/test/Driver/hip-toolchain-rdc-static-lib.hip
+++ b/clang/test/Driver/hip-toolchain-rdc-static-lib.hip
@@ -3,7 +3,7 @@
 
 // RUN: %clang -### --target=x86_64-linux-gnu \
 // RUN:   -x hip --cuda-gpu-arch=gfx803 --cuda-gpu-arch=gfx900 \
-// RUN:   --emit-static-lib -nogpulib \
+// RUN:   --no-offload-new-driver --emit-static-lib -nogpulib \
 // RUN:   -fuse-ld=lld -B%S/Inputs/lld -fgpu-rdc -nogpuinc \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu \
 // RUN:   %S/Inputs/hip_multiple_inputs/b.hip \
diff --git a/clang/test/Driver/hip-toolchain-rdc.hip b/clang/test/Driver/hip-toolchain-rdc.hip
index d19d8ccd6cb2..49acc40ec6f9 100644
--- a/clang/test/Driver/hip-toolchain-rdc.hip
+++ b/clang/test/Driver/hip-toolchain-rdc.hip
@@ -7,7 +7,7 @@
 // RUN:   --hip-device-lib-path=%S/Inputs/hip_multiple_inputs/lib1 \
 // RUN:   --hip-device-lib-path=%S/Inputs/hip_multiple_inputs/lib2 \
 // RUN:   -fuse-ld=lld -B%S/Inputs/lld -fgpu-rdc -nogpuinc \
-// RUN:   -fhip-dump-offload-linker-script \
+// RUN:   --no-offload-new-driver -fhip-dump-offload-linker-script \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu \
 // RUN:   %S/Inputs/hip_multiple_inputs/b.hip \
 // RUN: 2>&1 | FileCheck -check-prefixes=CHECK,LNX %s
@@ -18,7 +18,7 @@
 // RUN:   --hip-device-lib-path=%S/Inputs/hip_multiple_inputs/lib1 \
 // RUN:   --hip-device-lib-path=%S/Inputs/hip_multiple_inputs/lib2 \
 // RUN:   -fuse-ld=lld -B%S/Inputs/lld -fgpu-rdc -nogpuinc \
-// RUN:   -fhip-dump-offload-linker-script \
+// RUN:   --no-offload-new-driver -fhip-dump-offload-linker-script \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu \
 // RUN:   %S/Inputs/hip_multiple_inputs/b.hip \
 // RUN: 2>&1 | FileCheck -check-prefixes=CHECK,MSVC %s
diff --git a/clang/test/Driver/hip-unbundle-preproc.hipi b/clang/test/Driver/hip-unbundle-preproc.hipi
index fecdfdbe7d14..6d92d2381973 100644
--- a/clang/test/Driver/hip-unbundle-preproc.hipi
+++ b/clang/test/Driver/hip-unbundle-preproc.hipi
@@ -1,11 +1,11 @@
 // REQUIRES: amdgpu-registered-target
 
 // RUN: %clang -### --target=x86_64-unknown-linux-gnu \
-// RUN:   --offload-arch=gfx803 -nogpulib \
+// RUN:   --no-offload-new-driver --offload-arch=gfx803 -nogpulib \
 // RUN:   -x hip-cpp-output %s 2>&1 | FileCheck %s
 
 // RUN: %clang -### --target=x86_64-unknown-linux-gnu \
-// RUN:   --offload-arch=gfx803 -nogpulib \
+// RUN:   --no-offload-new-driver --offload-arch=gfx803 -nogpulib \
 // RUN:   %s 2>&1 | FileCheck %s
 
 // CHECK: {{".*clang-offload-bundler.*"}} {{.*}}"-output=[[HOST_PP:.*hipi]]" "-output=[[DEV_PP:.*hipi]]" "-unbundle"
@@ -16,7 +16,7 @@
 // CHECK: {{".*ld.*"}} {{.*}}"[[HOST_O]]"
 
 // RUN: %clang -### --target=x86_64-unknown-linux-gnu \
-// RUN:   --offload-arch=gfx803 -nogpulib -fgpu-rdc \
+// RUN:   --no-offload-new-driver --offload-arch=gfx803 -nogpulib -fgpu-rdc \
 // RUN:   %s 2>&1 | FileCheck -check-prefix=RDC %s
 
 // RDC: {{".*clang-offload-bundler.*"}} {{.*}}"-output=[[HOST_PP:.*hipi]]" "-output=[[DEV_PP:.*hipi]]" "-unbundle"
diff --git a/clang/test/Driver/hipspv-toolchain-rdc.hip b/clang/test/Driver/hipspv-toolchain-rdc.hip
index 2bfcec977d39..d4e612cc5437 100644
--- a/clang/test/Driver/hipspv-toolchain-rdc.hip
+++ b/clang/test/Driver/hipspv-toolchain-rdc.hip
@@ -2,7 +2,7 @@
 // UNSUPPORTED: system-windows
 
 // RUN: %clang -### -x hip -target x86_64-linux-gnu --offload=spirv64 \
-// RUN:   -fgpu-rdc --hip-path=%S/Inputs/hipspv -nohipwrapperinc \
+// RUN:   --no-offload-new-driver -fgpu-rdc --hip-path=%S/Inputs/hipspv -nohipwrapperinc \
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu \
 // RUN:   %S/Inputs/hip_multiple_inputs/b.hip \
 // RUN: 2>&1 | FileCheck %s
diff --git a/clang/test/Driver/hipspv-toolchain.hip b/clang/test/Driver/hipspv-toolchain.hip
index bfae41049ba4..4602cd3fc8d6 100644
--- a/clang/test/Driver/hipspv-toolchain.hip
+++ b/clang/test/Driver/hipspv-toolchain.hip
@@ -2,7 +2,7 @@
 // UNSUPPORTED: system-windows
 
 // RUN: %clang -### -target x86_64-linux-gnu --offload=spirv64 \
-// RUN:   --hip-path=%S/Inputs/hipspv -nohipwrapperinc %s \
+// RUN:   --no-offload-new-driver --hip-path=%S/Inputs/hipspv -nohipwrapperinc %s \
 // RUN: 2>&1 | FileCheck %s
 
 // CHECK: [[CLANG:".*clang.*"]] "-cc1" "-triple" "spirv64"
diff --git a/clang/test/Driver/lto.cu b/clang/test/Driver/lto.cu
index dadf35d8e6c9..fb8cc5cba393 100644
--- a/clang/test/Driver/lto.cu
+++ b/clang/test/Driver/lto.cu
@@ -2,14 +2,14 @@
 // REQUIRES: nvptx-registered-target
 
 // -flto causes a switch to llvm-bc object files.
-// RUN: %clangxx --target=x86_64-unknown-linux-gnu -nocudainc -nocudalib -ccc-print-phases -c %s -flto 2> %t
+// RUN: %clangxx --target=x86_64-unknown-linux-gnu --no-offload-new-driver -nocudainc -nocudalib -ccc-print-phases -c %s -flto 2> %t
 // RUN: FileCheck -check-prefix=CHECK-COMPILE-ACTIONS < %t %s
 //
 // CHECK-COMPILE-ACTIONS: 2: compiler, {1}, ir, (host-cuda)
 // CHECK-COMPILE-ACTIONS-NOT: lto-bc
 // CHECK-COMPILE-ACTIONS: 12: backend, {11}, lto-bc, (host-cuda)
 
-// RUN: %clangxx --target=x86_64-unknown-linux-gnu -nocudainc -nocudalib -ccc-print-phases %s -flto 2> %t
+// RUN: %clangxx --target=x86_64-unknown-linux-gnu --no-offload-new-driver -nocudainc -nocudalib -ccc-print-phases %s -flto 2> %t
 // RUN: FileCheck -check-prefix=CHECK-COMPILELINK-ACTIONS < %t %s
 //
 // CHECK-COMPILELINK-ACTIONS: 0: input, "{{.*}}lto.cu", cuda, (host-cuda)
@@ -29,7 +29,7 @@
 
 // llvm-bc and llvm-ll outputs need to match regular suffixes
 // (unfortunately).
-// RUN: %clangxx %s --target=x86_64-unknown-linux-gnu -nocudainc -nocudalib -flto -save-temps --cuda-path=%S/Inputs/CUDA_80/usr/local/cuda -### 2> %t
+// RUN: %clangxx %s --target=x86_64-unknown-linux-gnu --no-offload-new-driver -nocudainc -nocudalib -flto -save-temps --cuda-path=%S/Inputs/CUDA_80/usr/local/cuda -### 2> %t
 // RUN: FileCheck -check-prefix=CHECK-COMPILELINK-SUFFIXES < %t %s
 //
 // CHECK-COMPILELINK-SUFFIXES: "-o" "[[CPP:.*lto-host.*\.cui]]" "-x" "cuda" "{{.*}}lto.cu"
@@ -37,36 +37,36 @@
 // CHECK-COMPILELINK-SUFFIXES: "-o" "[[OBJ:.*lto-host.*\.o]]" {{.*}}[[BC]]"
 // CHECK-COMPILELINK-SUFFIXES: "{{.*}}a.{{(out|exe)}}" {{.*}}[[OBJ]]"
 
-// RUN: %clangxx --target=x86_64-unknown-linux-gnu %s -nocudainc -nocudalib -flto -S -### 2> %t
+// RUN: %clangxx --target=x86_64-unknown-linux-gnu %s --no-offload-new-driver -nocudainc -nocudalib -flto -S -### 2> %t
 // RUN: FileCheck -check-prefix=CHECK-COMPILE-SUFFIXES < %t %s
 //
 // CHECK-COMPILE-SUFFIXES: "-o" "{{.*}}lto.s" "-x" "cuda" "{{.*}}lto.cu"
 
-// RUN: not %clangxx --target=x86_64-unknown-linux-gnu -nocudainc -nocudalib %s -emit-llvm 2>&1 \
+// RUN: not %clangxx --target=x86_64-unknown-linux-gnu --no-offload-new-driver -nocudainc -nocudalib %s -emit-llvm 2>&1 \
 // RUN:    | FileCheck --check-prefix=LLVM-LINK %s
 // LLVM-LINK: -emit-llvm cannot be used when linking
 
 /// With ld.bfd or gold, link against LLVMgold.
 // RUN: %clangxx -nocudainc -nocudalib --target=x86_64-unknown-linux-gnu --offload-arch=sm_52 --sysroot=%S/Inputs/basic_cross_linux_tree %s \
-// RUN:   -fuse-ld=bfd -flto=thin -### 2>&1 | FileCheck --check-prefix=LLVMGOLD %s
+// RUN:   --no-offload-new-driver -fuse-ld=bfd -flto=thin -### 2>&1 | FileCheck --check-prefix=LLVMGOLD %s
 // RUN: %clangxx -nocudainc -nocudalib --target=x86_64-unknown-linux-gnu --offload-arch=sm_52 --sysroot=%S/Inputs/basic_cross_linux_tree %s \
-// RUN:   -fuse-ld=gold -flto=full -### 2>&1 | FileCheck --check-prefix=LLVMGOLD %s
+// RUN:   --no-offload-new-driver -fuse-ld=gold -flto=full -### 2>&1 | FileCheck --check-prefix=LLVMGOLD %s
 //
 // LLVMGOLD: "-plugin" "{{.*}}{{[/\\]}}LLVMgold.{{dll|dylib|so}}"
 
 /// lld does not need LLVMgold.
 // RUN: %clangxx -nocudainc -nocudalib --target=x86_64-unknown-linux-gnu --offload-arch=sm_52 --sysroot=%S/Inputs/basic_cross_linux_tree %s \
-// RUN:   -fuse-ld=lld -flto=full -### 2>&1 | FileCheck --check-prefix=NO-LLVMGOLD %s
+// RUN:   --no-offload-new-driver -fuse-ld=lld -flto=full -### 2>&1 | FileCheck --check-prefix=NO-LLVMGOLD %s
 // RUN: %clangxx -nocudainc -nocudalib --target=x86_64-unknown-linux-gnu --offload-arch=sm_52 --sysroot=%S/Inputs/basic_cross_linux_tree %s \
-// RUN:   -fuse-ld=gold -flto=full -fno-lto -### 2>&1 | FileCheck --check-prefix=NO-LLVMGOLD %s
+// RUN:   --no-offload-new-driver -fuse-ld=gold -flto=full -fno-lto -### 2>&1 | FileCheck --check-prefix=NO-LLVMGOLD %s
 //
 // NO-LLVMGOLD-NOT: "-plugin" "{{.*}}{{[/\\]}}LLVMgold.{{dll|dylib|so}}"
 
 // -flto passes along an explicit debugger tuning argument.
-// RUN: %clangxx -nocudainc -nocudalib \
+// RUN: %clangxx -nocudainc -nocudalib --no-offload-new-driver \
 // RUN:          --target=x86_64-unknown-linux -### %s -flto -glldb --offload-arch=sm_52 --cuda-path=%S/Inputs/CUDA_80/usr/local/cuda 2> %t
 // RUN: FileCheck -check-prefix=CHECK-TUNING-LLDB < %t %s
-// RUN: %clangxx -nocudainc -nocudalib \
+// RUN: %clangxx -nocudainc -nocudalib --no-offload-new-driver \
 // RUN:          --target=x86_64-unknown-linux -### %s -flto -g --offload-arch=sm_52 --cuda-path=%S/Inputs/CUDA_80/usr/local/cuda 2> %t
 // RUN: FileCheck -check-prefix=CHECK-NO-TUNING < %t %s
 //
diff --git a/clang/test/Driver/thinlto.cu b/clang/test/Driver/thinlto.cu
index d4f265baf370..7c51a5194e0b 100644
--- a/clang/test/Driver/thinlto.cu
+++ b/clang/test/Driver/thinlto.cu
@@ -2,14 +2,14 @@
 // REQUIRES: nvptx-registered-target
 
 // -flto=thin causes a switch to llvm-bc object files.
-// RUN: %clangxx -ccc-print-phases -nocudainc -nocudalib -c %s -flto=thin 2> %t
+// RUN: %clangxx -ccc-print-phases --no-offload-new-driver -nocudainc -nocudalib -c %s -flto=thin 2> %t
 // RUN: FileCheck -check-prefix=CHECK-COMPILE-ACTIONS < %t %s
 //
 // CHECK-COMPILE-ACTIONS: 2: compiler, {1}, ir, (host-cuda)
 // CHECK-COMPILE-ACTIONS-NOT: lto-bc
 // CHECK-COMPILE-ACTIONS: 12: backend, {11}, lto-bc, (host-cuda)
 
-// RUN: %clangxx -ccc-print-phases -nocudainc -nocudalib %s -flto=thin 2> %t
+// RUN: %clangxx -ccc-print-phases --no-offload-new-driver -nocudainc -nocudalib %s -flto=thin 2> %t
 // RUN: FileCheck -check-prefix=CHECK-COMPILELINK-ACTIONS < %t %s
 //
 // CHECK-COMPILELINK-ACTIONS: 0: input, "{{.*}}thinlto.cu", cuda, (host-cuda)
-- 
GitLab


From ce683273aa3dbdd7bce2168c29b1a7652ce73608 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim 
Date: Fri, 8 Mar 2024 13:50:06 +0000
Subject: [PATCH 0039/1674] [X86] PBLENDW instructions can run on Port1 or
 Port5

When we copied the SkylakeServer model we missed this diff

Confirmed with uops.info and Agner
---
 llvm/lib/Target/X86/X86SchedIceLake.td                 |  6 +++---
 .../tools/llvm-mca/X86/IceLakeServer/resources-avx1.s  | 10 +++++-----
 .../tools/llvm-mca/X86/IceLakeServer/resources-avx2.s  | 10 +++++-----
 .../tools/llvm-mca/X86/IceLakeServer/resources-sse41.s | 10 +++++-----
 4 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/llvm/lib/Target/X86/X86SchedIceLake.td b/llvm/lib/Target/X86/X86SchedIceLake.td
index 3144327670c7..c9ae9901ed5b 100644
--- a/llvm/lib/Target/X86/X86SchedIceLake.td
+++ b/llvm/lib/Target/X86/X86SchedIceLake.td
@@ -396,9 +396,9 @@ defm : ICXWriteResPair; // Vector v
 defm : ICXWriteResPair;
 defm : ICXWriteResPair;
 defm : ICXWriteResPair;
-defm : ICXWriteResPair; // Vector blends.
-defm : ICXWriteResPair;
-defm : ICXWriteResPair;
+defm : ICXWriteResPair; // Vector blends.
+defm : ICXWriteResPair;
+defm : ICXWriteResPair;
 defm : ICXWriteResPair; // Vector variable blends.
 defm : ICXWriteResPair;
 defm : ICXWriteResPair;
diff --git a/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx1.s b/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx1.s
index ad7bc8d27e21..e467c4e48ebd 100644
--- a/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx1.s
+++ b/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx1.s
@@ -1401,8 +1401,8 @@ vzeroupper
 # CHECK-NEXT:  2      7     0.50    *                   vpavgw	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  2      2     0.67                        vpblendvb	%xmm3, %xmm0, %xmm1, %xmm2
 # CHECK-NEXT:  3      8     0.67    *                   vpblendvb	%xmm3, (%rax), %xmm1, %xmm2
-# CHECK-NEXT:  1      1     1.00                        vpblendw	$11, %xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  2      7     1.00    *                   vpblendw	$11, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpblendw	$11, %xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      7     0.50    *                   vpblendw	$11, (%rax), %xmm1, %xmm2
 # CHECK-NEXT:  1      6     1.00                        vpclmulqdq	$11, %xmm0, %xmm1, %xmm2
 # CHECK-NEXT:  2      12    1.00    *                   vpclmulqdq	$11, (%rax), %xmm1, %xmm2
 # CHECK-NEXT:  1      1     0.50                        vpcmpeqb	%xmm0, %xmm1, %xmm2
@@ -1738,7 +1738,7 @@ vzeroupper
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]
-# CHECK-NEXT:  -     126.00 322.92 231.92 160.50 160.50 19.00  297.92 6.25   19.00  19.00  19.00
+# CHECK-NEXT:  -     126.00 322.92 232.92 160.50 160.50 19.00  296.92 6.25   19.00  19.00  19.00
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   Instructions:
@@ -2113,8 +2113,8 @@ vzeroupper
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -     vpavgw	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  -      -     0.67   0.67    -      -      -     0.67    -      -      -      -     vpblendvb	%xmm3, %xmm0, %xmm1, %xmm2
 # CHECK-NEXT:  -      -     0.67   0.67   0.50   0.50    -     0.67    -      -      -      -     vpblendvb	%xmm3, (%rax), %xmm1, %xmm2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     vpblendw	$11, %xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -      -      -     vpblendw	$11, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -     0.50    -      -      -     0.50    -      -      -      -     vpblendw	$11, %xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -     0.50   0.50   0.50    -     0.50    -      -      -      -     vpblendw	$11, (%rax), %xmm1, %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     vpclmulqdq	$11, %xmm0, %xmm1, %xmm2
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -      -      -     vpclmulqdq	$11, (%rax), %xmm1, %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -      -      -     vpcmpeqb	%xmm0, %xmm1, %xmm2
diff --git a/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx2.s b/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx2.s
index e65504f79ff1..97f0d052f455 100644
--- a/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx2.s
+++ b/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-avx2.s
@@ -524,8 +524,8 @@ vpxor           (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  2      8     0.50    *                   vpblendd	$11, (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  2      2     0.67                        vpblendvb	%ymm3, %ymm0, %ymm1, %ymm2
 # CHECK-NEXT:  3      8     0.67    *                   vpblendvb	%ymm3, (%rax), %ymm1, %ymm2
-# CHECK-NEXT:  1      1     1.00                        vpblendw	$11, %ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  2      8     1.00    *                   vpblendw	$11, (%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpblendw	$11, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      8     0.50    *                   vpblendw	$11, (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  1      3     1.00                        vpbroadcastb	%xmm0, %xmm0
 # CHECK-NEXT:  2      7     1.00    *                   vpbroadcastb	(%rax), %xmm0
 # CHECK-NEXT:  1      3     1.00                        vpbroadcastb	%xmm0, %ymm0
@@ -778,7 +778,7 @@ vpxor           (%rax), %ymm1, %ymm2
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]
-# CHECK-NEXT:  -      -     110.33 102.33 98.00  98.00  2.50   151.33  -     2.50   2.50   2.50
+# CHECK-NEXT:  -      -     110.33 103.33 98.00  98.00  2.50   150.33  -     2.50   2.50   2.50
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   Instructions:
@@ -846,8 +846,8 @@ vpxor           (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -      -      -     vpblendd	$11, (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -     0.67   0.67    -      -      -     0.67    -      -      -      -     vpblendvb	%ymm3, %ymm0, %ymm1, %ymm2
 # CHECK-NEXT:  -      -     0.67   0.67   0.50   0.50    -     0.67    -      -      -      -     vpblendvb	%ymm3, (%rax), %ymm1, %ymm2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     vpblendw	$11, %ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -      -      -     vpblendw	$11, (%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -     0.50    -      -      -     0.50    -      -      -      -     vpblendw	$11, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -     0.50   0.50   0.50    -     0.50    -      -      -      -     vpblendw	$11, (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     vpbroadcastb	%xmm0, %xmm0
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -      -      -     vpbroadcastb	(%rax), %xmm0
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     vpbroadcastb	%xmm0, %ymm0
diff --git a/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-sse41.s b/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-sse41.s
index 4b4c6b90541a..554d7aad54ba 100644
--- a/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-sse41.s
+++ b/llvm/test/tools/llvm-mca/X86/IceLakeServer/resources-sse41.s
@@ -178,8 +178,8 @@ roundss     $1, (%rax), %xmm2
 # CHECK-NEXT:  2      10    1.00    *                   packusdw	(%rax), %xmm2
 # CHECK-NEXT:  2      2     0.67                        pblendvb	%xmm0, %xmm0, %xmm2
 # CHECK-NEXT:  3      8     0.67    *                   pblendvb	%xmm0, (%rax), %xmm2
-# CHECK-NEXT:  1      1     1.00                        pblendw	$11, %xmm0, %xmm2
-# CHECK-NEXT:  2      7     1.00    *                   pblendw	$11, (%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        pblendw	$11, %xmm0, %xmm2
+# CHECK-NEXT:  2      7     0.50    *                   pblendw	$11, (%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        pcmpeqq	%xmm0, %xmm2
 # CHECK-NEXT:  2      7     0.50    *                   pcmpeqq	(%rax), %xmm2
 # CHECK-NEXT:  2      3     1.00                        pextrb	$1, %xmm0, %ecx
@@ -268,7 +268,7 @@ roundss     $1, (%rax), %xmm2
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]
-# CHECK-NEXT:  -      -     36.67  40.67  22.00  22.00  2.50   54.67   -     2.50   2.50   2.50
+# CHECK-NEXT:  -      -     36.67  41.67  22.00  22.00  2.50   53.67   -     2.50   2.50   2.50
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   Instructions:
@@ -295,8 +295,8 @@ roundss     $1, (%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -      -      -     packusdw	(%rax), %xmm2
 # CHECK-NEXT:  -      -     0.67   0.67    -      -      -     0.67    -      -      -      -     pblendvb	%xmm0, %xmm0, %xmm2
 # CHECK-NEXT:  -      -     0.67   0.67   0.50   0.50    -     0.67    -      -      -      -     pblendvb	%xmm0, (%rax), %xmm2
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     pblendw	$11, %xmm0, %xmm2
-# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -      -      -     pblendw	$11, (%rax), %xmm2
+# CHECK-NEXT:  -      -      -     0.50    -      -      -     0.50    -      -      -      -     pblendw	$11, %xmm0, %xmm2
+# CHECK-NEXT:  -      -      -     0.50   0.50   0.50    -     0.50    -      -      -      -     pblendw	$11, (%rax), %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -      -      -     pcmpeqq	%xmm0, %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -     pcmpeqq	(%rax), %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -     1.00    -      -      -      -     pextrb	$1, %xmm0, %ecx
-- 
GitLab


From c4a89f1538bbcac19628570b6b489cd9ce4265e2 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek 
Date: Fri, 8 Mar 2024 09:15:59 -0600
Subject: [PATCH 0040/1674] [flang][OpenMP] Fix use-after-free in
 OMPFunctionFiltering (#84373)

When walking over functions (in pre-order), if the function being
visited needs to be erased, skip visiting its regions.

This was detected by address sanitizer.
---
 flang/lib/Optimizer/Transforms/OMPFunctionFiltering.cpp | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/flang/lib/Optimizer/Transforms/OMPFunctionFiltering.cpp b/flang/lib/Optimizer/Transforms/OMPFunctionFiltering.cpp
index 466bf53e8dbd..959099d039a5 100644
--- a/flang/lib/Optimizer/Transforms/OMPFunctionFiltering.cpp
+++ b/flang/lib/Optimizer/Transforms/OMPFunctionFiltering.cpp
@@ -79,12 +79,15 @@ public:
           // Remove the callOp
           callOp->erase();
         }
-        if (!hasTargetRegion)
+        if (!hasTargetRegion) {
           funcOp.erase();
-        else if (declareTargetOp)
+          return WalkResult::skip();
+        }
+        if (declareTargetOp)
           declareTargetOp.setDeclareTarget(declareType,
                                            omp::DeclareTargetCaptureClause::to);
       }
+      return WalkResult::advance();
     });
   }
 };
-- 
GitLab


From e0d49066c1acfa4ae0f0e8ea49b0f0b6bb3f1a25 Mon Sep 17 00:00:00 2001
From: Andreas Jonson 
Date: Fri, 8 Mar 2024 16:20:04 +0100
Subject: [PATCH 0041/1674] [IR] Add new Range attribute using new
 ConstantRange Attribute type (#83171)

implementation as discussed in
https://discourse.llvm.org/t/rfc-metadata-attachments-for-function-arguments/76420
---
 llvm/docs/LangRef.rst                         | 16 ++++
 llvm/include/llvm/ADT/FoldingSet.h            |  7 ++
 llvm/include/llvm/AsmParser/LLParser.h        |  1 +
 llvm/include/llvm/Bitcode/LLVMBitCodes.h      |  1 +
 llvm/include/llvm/IR/Attributes.h             | 23 ++++++
 llvm/include/llvm/IR/Attributes.td            |  6 ++
 llvm/lib/AsmParser/LLParser.cpp               | 43 ++++++++++
 llvm/lib/Bitcode/Reader/BitcodeReader.cpp     | 41 ++++++++++
 llvm/lib/Bitcode/Writer/BitcodeWriter.cpp     | 61 +++++++++-----
 llvm/lib/IR/AttributeImpl.h                   | 28 ++++++-
 llvm/lib/IR/Attributes.cpp                    | 80 ++++++++++++++++++-
 llvm/lib/IR/Verifier.cpp                      |  5 ++
 llvm/lib/Transforms/Utils/CodeExtractor.cpp   |  1 +
 .../range-attribute-invalid-range.ll          |  6 ++
 .../Assembler/range-attribute-invalid-type.ll |  6 ++
 llvm/test/Bitcode/attributes.ll               | 10 +++
 llvm/test/Verifier/range-attr.ll              | 19 +++++
 llvm/utils/TableGen/Attributes.cpp            |  9 ++-
 18 files changed, 337 insertions(+), 26 deletions(-)
 create mode 100644 llvm/test/Assembler/range-attribute-invalid-range.ll
 create mode 100644 llvm/test/Assembler/range-attribute-invalid-type.ll
 create mode 100644 llvm/test/Verifier/range-attr.ll

diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index a7b77d6f776a..b70220dec926 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -1635,6 +1635,22 @@ Currently, only the following parameter attributes are defined:
 
     This attribute cannot be applied to return values.
 
+``range( , )``
+    This attribute expresses the possible range of the parameter or return value.
+    If the value is not in the specified range, it is converted to poison.
+    The arguments passed to ``range`` have the following properties:
+
+    -  The type must match the scalar type of the parameter or return value.
+    -  The pair ``a,b`` represents the range ``[a,b)``.
+    -  Both ``a`` and ``b`` are constants.
+    -  The range is allowed to wrap.
+    -  The range should not represent the full or empty set. That is, ``a!=b``.
+    
+    This attribute may only be applied to parameters or return values with integer 
+    or vector of integer types.
+    
+    For vector-typed parameters, the range is applied element-wise.
+
 .. _gc:
 
 Garbage Collector Strategy Names
diff --git a/llvm/include/llvm/ADT/FoldingSet.h b/llvm/include/llvm/ADT/FoldingSet.h
index f82eabd5044b..ddc3e52255d6 100644
--- a/llvm/include/llvm/ADT/FoldingSet.h
+++ b/llvm/include/llvm/ADT/FoldingSet.h
@@ -16,6 +16,7 @@
 #ifndef LLVM_ADT_FOLDINGSET_H
 #define LLVM_ADT_FOLDINGSET_H
 
+#include "llvm/ADT/APInt.h"
 #include "llvm/ADT/Hashing.h"
 #include "llvm/ADT/STLForwardCompat.h"
 #include "llvm/ADT/SmallVector.h"
@@ -354,6 +355,12 @@ public:
     AddInteger(unsigned(I));
     AddInteger(unsigned(I >> 32));
   }
+  void AddInteger(const APInt &Int) {
+    const auto *Parts = Int.getRawData();
+    for (int i = 0, N = Int.getNumWords(); i < N; ++i) {
+      AddInteger(Parts[i]);
+    }
+  }
 
   void AddBoolean(bool B) { AddInteger(B ? 1U : 0U); }
   void AddString(StringRef String);
diff --git a/llvm/include/llvm/AsmParser/LLParser.h b/llvm/include/llvm/AsmParser/LLParser.h
index e5e1ade8b38b..e85728aa3c0d 100644
--- a/llvm/include/llvm/AsmParser/LLParser.h
+++ b/llvm/include/llvm/AsmParser/LLParser.h
@@ -369,6 +369,7 @@ namespace llvm {
     bool parseFnAttributeValuePairs(AttrBuilder &B,
                                     std::vector &FwdRefAttrGrps,
                                     bool inAttrGrp, LocTy &BuiltinLoc);
+    bool parseRangeAttr(AttrBuilder &B);
     bool parseRequiredTypeAttr(AttrBuilder &B, lltok::Kind AttrToken,
                                Attribute::AttrKind AttrKind);
 
diff --git a/llvm/include/llvm/Bitcode/LLVMBitCodes.h b/llvm/include/llvm/Bitcode/LLVMBitCodes.h
index c6f0ddf29a6d..c0a52d64a101 100644
--- a/llvm/include/llvm/Bitcode/LLVMBitCodes.h
+++ b/llvm/include/llvm/Bitcode/LLVMBitCodes.h
@@ -724,6 +724,7 @@ enum AttributeKindCodes {
   ATTR_KIND_WRITABLE = 89,
   ATTR_KIND_CORO_ONLY_DESTROY_WHEN_COMPLETE = 90,
   ATTR_KIND_DEAD_ON_UNWIND = 91,
+  ATTR_KIND_RANGE = 92,
 };
 
 enum ComdatSelectionKindCodes {
diff --git a/llvm/include/llvm/IR/Attributes.h b/llvm/include/llvm/IR/Attributes.h
index a4ebe5d732f5..0c2a02514ba0 100644
--- a/llvm/include/llvm/IR/Attributes.h
+++ b/llvm/include/llvm/IR/Attributes.h
@@ -37,6 +37,7 @@ class AttributeMask;
 class AttributeImpl;
 class AttributeListImpl;
 class AttributeSetNode;
+class ConstantRange;
 class FoldingSetNodeID;
 class Function;
 class LLVMContext;
@@ -103,6 +104,9 @@ public:
   static bool isTypeAttrKind(AttrKind Kind) {
     return Kind >= FirstTypeAttr && Kind <= LastTypeAttr;
   }
+  static bool isConstantRangeAttrKind(AttrKind Kind) {
+    return Kind >= FirstConstantRangeAttr && Kind <= LastConstantRangeAttr;
+  }
 
   static bool canUseAsFnAttr(AttrKind Kind);
   static bool canUseAsParamAttr(AttrKind Kind);
@@ -125,6 +129,8 @@ public:
   static Attribute get(LLVMContext &Context, StringRef Kind,
                        StringRef Val = StringRef());
   static Attribute get(LLVMContext &Context, AttrKind Kind, Type *Ty);
+  static Attribute get(LLVMContext &Context, AttrKind Kind,
+                       const ConstantRange &CR);
 
   /// Return a uniquified Attribute object that has the specific
   /// alignment set.
@@ -180,6 +186,9 @@ public:
   /// Return true if the attribute is a type attribute.
   bool isTypeAttribute() const;
 
+  /// Return true if the attribute is a ConstantRange attribute.
+  bool isConstantRangeAttribute() const;
+
   /// Return true if the attribute is any kind of attribute.
   bool isValid() const { return pImpl; }
 
@@ -213,6 +222,10 @@ public:
   /// a type attribute.
   Type *getValueAsType() const;
 
+  /// Return the attribute's value as a ConstantRange. This requires the
+  /// attribute to be a ConstantRange attribute.
+  ConstantRange getValueAsConstantRange() const;
+
   /// Returns the alignment field of an attribute as a byte alignment
   /// value.
   MaybeAlign getAlignment() const;
@@ -251,6 +264,9 @@ public:
   /// Return the FPClassTest for nofpclass
   FPClassTest getNoFPClass() const;
 
+  /// Returns the value of the range attribute.
+  ConstantRange getRange() const;
+
   /// The Attribute is converted to a string of equivalent mnemonic. This
   /// is, presumably, for writing out the mnemonics for the assembly writer.
   std::string getAsString(bool InAttrGrp = false) const;
@@ -1189,6 +1205,13 @@ public:
   // Add nofpclass attribute
   AttrBuilder &addNoFPClassAttr(FPClassTest NoFPClassMask);
 
+  /// Add a ConstantRange attribute with the given range.
+  AttrBuilder &addConstantRangeAttr(Attribute::AttrKind Kind,
+                                    const ConstantRange &CR);
+
+  /// Add range attribute.
+  AttrBuilder &addRangeAttr(const ConstantRange &CR);
+
   ArrayRef attrs() const { return Attrs; }
 
   bool operator==(const AttrBuilder &B) const;
diff --git a/llvm/include/llvm/IR/Attributes.td b/llvm/include/llvm/IR/Attributes.td
index 08afecf32015..cef8b17769f0 100644
--- a/llvm/include/llvm/IR/Attributes.td
+++ b/llvm/include/llvm/IR/Attributes.td
@@ -44,6 +44,9 @@ class StrBoolAttr : Attr;
 /// Arbitrary string attribute.
 class ComplexStrAttr P> : Attr;
 
+/// ConstantRange attribute.
+class ConstantRangeAttr P> : Attr;
+
 /// Target-independent enum attributes.
 
 /// Alignment of parameter (5 bits) stored as log2 of alignment with +1 bias.
@@ -218,6 +221,9 @@ def OptimizeNone : EnumAttr<"optnone", [FnAttr]>;
 /// Similar to byval but without a copy.
 def Preallocated : TypeAttr<"preallocated", [FnAttr, ParamAttr]>;
 
+/// Parameter or return value is within the specified range.
+def Range : ConstantRangeAttr<"range", [ParamAttr, RetAttr]>;
+
 /// Function does not access memory.
 def ReadNone : EnumAttr<"readnone", [ParamAttr]>;
 
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index e140c9419520..78bcd94e23fa 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -1596,6 +1596,8 @@ bool LLParser::parseEnumAttribute(Attribute::AttrKind Attr, AttrBuilder &B,
 
     return true;
   }
+  case Attribute::Range:
+    return parseRangeAttr(B);
   default:
     B.addAttribute(Attr);
     Lex.Lex();
@@ -3008,6 +3010,47 @@ bool LLParser::parseRequiredTypeAttr(AttrBuilder &B, lltok::Kind AttrToken,
   return false;
 }
 
+/// parseRangeAttr
+///   ::= range( ,)
+bool LLParser::parseRangeAttr(AttrBuilder &B) {
+  Lex.Lex();
+
+  APInt Lower;
+  APInt Upper;
+  Type *Ty = nullptr;
+  LocTy TyLoc;
+
+  auto ParseAPSInt = [&](unsigned BitWidth, APInt &Val) {
+    if (Lex.getKind() != lltok::APSInt)
+      return tokError("expected integer");
+    if (Lex.getAPSIntVal().getBitWidth() > BitWidth)
+      return tokError(
+          "integer is too large for the bit width of specified type");
+    Val = Lex.getAPSIntVal().extend(BitWidth);
+    Lex.Lex();
+    return false;
+  };
+
+  if (parseToken(lltok::lparen, "expected '('") || parseType(Ty, TyLoc))
+    return true;
+  if (!Ty->isIntegerTy())
+    return error(TyLoc, "the range must have integer type!");
+
+  unsigned BitWidth = Ty->getPrimitiveSizeInBits();
+
+  if (ParseAPSInt(BitWidth, Lower) ||
+      parseToken(lltok::comma, "expected ','") || ParseAPSInt(BitWidth, Upper))
+    return true;
+  if (Lower == Upper)
+    return tokError("the range should not represent the full or empty set!");
+
+  if (parseToken(lltok::rparen, "expected ')'"))
+    return true;
+
+  B.addRangeAttr(ConstantRange(Lower, Upper));
+  return false;
+}
+
 /// parseOptionalOperandBundles
 ///    ::= /*empty*/
 ///    ::= '[' OperandBundle [, OperandBundle ]* ']'
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index 832907a3f53f..9c63116114f3 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -815,6 +815,30 @@ private:
     return getFnValueByID(ValNo, Ty, TyID, ConstExprInsertBB);
   }
 
+  Expected readConstantRange(ArrayRef Record,
+                                            unsigned &OpNum) {
+    if (Record.size() - OpNum < 3)
+      return error("Too few records for range");
+    unsigned BitWidth = Record[OpNum++];
+    if (BitWidth > 64) {
+      unsigned LowerActiveWords = Record[OpNum];
+      unsigned UpperActiveWords = Record[OpNum++] >> 32;
+      if (Record.size() - OpNum < LowerActiveWords + UpperActiveWords)
+        return error("Too few records for range");
+      APInt Lower =
+          readWideAPInt(ArrayRef(&Record[OpNum], LowerActiveWords), BitWidth);
+      OpNum += LowerActiveWords;
+      APInt Upper =
+          readWideAPInt(ArrayRef(&Record[OpNum], UpperActiveWords), BitWidth);
+      OpNum += UpperActiveWords;
+      return ConstantRange(Lower, Upper);
+    } else {
+      int64_t Start = BitcodeReader::decodeSignRotatedValue(Record[OpNum++]);
+      int64_t End = BitcodeReader::decodeSignRotatedValue(Record[OpNum++]);
+      return ConstantRange(APInt(BitWidth, Start), APInt(BitWidth, End));
+    }
+  }
+
   /// Upgrades old-style typeless byval/sret/inalloca attributes by adding the
   /// corresponding argument's pointee type. Also upgrades intrinsics that now
   /// require an elementtype attribute.
@@ -2103,6 +2127,8 @@ static Attribute::AttrKind getAttrFromCode(uint64_t Code) {
     return Attribute::CoroDestroyOnlyWhenComplete;
   case bitc::ATTR_KIND_DEAD_ON_UNWIND:
     return Attribute::DeadOnUnwind;
+  case bitc::ATTR_KIND_RANGE:
+    return Attribute::Range;
   }
 }
 
@@ -2272,6 +2298,21 @@ Error BitcodeReader::parseAttributeGroupBlock() {
             return error("Not a type attribute");
 
           B.addTypeAttr(Kind, HasType ? getTypeByID(Record[++i]) : nullptr);
+        } else if (Record[i] == 7) {
+          Attribute::AttrKind Kind;
+
+          i++;
+          if (Error Err = parseAttrKind(Record[i++], &Kind))
+            return Err;
+          if (!Attribute::isConstantRangeAttrKind(Kind))
+            return error("Not a ConstantRange attribute");
+
+          Expected MaybeCR = readConstantRange(Record, i);
+          if (!MaybeCR)
+            return MaybeCR.takeError();
+          i--;
+
+          B.addConstantRangeAttr(Kind, MaybeCR.get());
         } else {
           return error("Invalid attribute group entry");
         }
diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
index 656f2a6ce870..597f49332fad 100644
--- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -844,6 +844,8 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) {
     return bitc::ATTR_KIND_CORO_ONLY_DESTROY_WHEN_COMPLETE;
   case Attribute::DeadOnUnwind:
     return bitc::ATTR_KIND_DEAD_ON_UNWIND;
+  case Attribute::Range:
+    return bitc::ATTR_KIND_RANGE;
   case Attribute::EndAttrKinds:
     llvm_unreachable("Can not encode end-attribute kinds marker.");
   case Attribute::None:
@@ -856,6 +858,39 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) {
   llvm_unreachable("Trying to encode unknown attribute");
 }
 
+static void emitSignedInt64(SmallVectorImpl &Vals, uint64_t V) {
+  if ((int64_t)V >= 0)
+    Vals.push_back(V << 1);
+  else
+    Vals.push_back((-V << 1) | 1);
+}
+
+static void emitWideAPInt(SmallVectorImpl &Vals, const APInt &A) {
+  // We have an arbitrary precision integer value to write whose
+  // bit width is > 64. However, in canonical unsigned integer
+  // format it is likely that the high bits are going to be zero.
+  // So, we only write the number of active words.
+  unsigned NumWords = A.getActiveWords();
+  const uint64_t *RawData = A.getRawData();
+  for (unsigned i = 0; i < NumWords; i++)
+    emitSignedInt64(Vals, RawData[i]);
+}
+
+static void emitConstantRange(SmallVectorImpl &Record,
+                              const ConstantRange &CR) {
+  unsigned BitWidth = CR.getBitWidth();
+  Record.push_back(BitWidth);
+  if (BitWidth > 64) {
+    Record.push_back(CR.getLower().getActiveWords() |
+                     (uint64_t(CR.getUpper().getActiveWords()) << 32));
+    emitWideAPInt(Record, CR.getLower());
+    emitWideAPInt(Record, CR.getUpper());
+  } else {
+    emitSignedInt64(Record, CR.getLower().getSExtValue());
+    emitSignedInt64(Record, CR.getUpper().getSExtValue());
+  }
+}
+
 void ModuleBitcodeWriter::writeAttributeGroupTable() {
   const std::vector &AttrGrps =
       VE.getAttributeGroups();
@@ -889,13 +924,17 @@ void ModuleBitcodeWriter::writeAttributeGroupTable() {
           Record.append(Val.begin(), Val.end());
           Record.push_back(0);
         }
-      } else {
-        assert(Attr.isTypeAttribute());
+      } else if (Attr.isTypeAttribute()) {
         Type *Ty = Attr.getValueAsType();
         Record.push_back(Ty ? 6 : 5);
         Record.push_back(getAttrKindEncoding(Attr.getKindAsEnum()));
         if (Ty)
           Record.push_back(VE.getTypeID(Attr.getValueAsType()));
+      } else {
+        assert(Attr.isConstantRangeAttribute());
+        Record.push_back(7);
+        Record.push_back(getAttrKindEncoding(Attr.getKindAsEnum()));
+        emitConstantRange(Record, Attr.getValueAsConstantRange());
       }
     }
 
@@ -1716,24 +1755,6 @@ void ModuleBitcodeWriter::writeDIGenericSubrange(
   Record.clear();
 }
 
-static void emitSignedInt64(SmallVectorImpl &Vals, uint64_t V) {
-  if ((int64_t)V >= 0)
-    Vals.push_back(V << 1);
-  else
-    Vals.push_back((-V << 1) | 1);
-}
-
-static void emitWideAPInt(SmallVectorImpl &Vals, const APInt &A) {
-  // We have an arbitrary precision integer value to write whose
-  // bit width is > 64. However, in canonical unsigned integer
-  // format it is likely that the high bits are going to be zero.
-  // So, we only write the number of active words.
-  unsigned NumWords = A.getActiveWords();
-  const uint64_t *RawData = A.getRawData();
-  for (unsigned i = 0; i < NumWords; i++)
-    emitSignedInt64(Vals, RawData[i]);
-}
-
 void ModuleBitcodeWriter::writeDIEnumerator(const DIEnumerator *N,
                                             SmallVectorImpl &Record,
                                             unsigned Abbrev) {
diff --git a/llvm/lib/IR/AttributeImpl.h b/llvm/lib/IR/AttributeImpl.h
index 78496786b0ae..9a6427bbc3d5 100644
--- a/llvm/lib/IR/AttributeImpl.h
+++ b/llvm/lib/IR/AttributeImpl.h
@@ -20,6 +20,7 @@
 #include "llvm/ADT/FoldingSet.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/IR/Attributes.h"
+#include "llvm/IR/ConstantRange.h"
 #include "llvm/Support/TrailingObjects.h"
 #include 
 #include 
@@ -46,6 +47,7 @@ protected:
     IntAttrEntry,
     StringAttrEntry,
     TypeAttrEntry,
+    ConstantRangeAttrEntry,
   };
 
   AttributeImpl(AttrEntryKind KindID) : KindID(KindID) {}
@@ -59,6 +61,9 @@ public:
   bool isIntAttribute() const { return KindID == IntAttrEntry; }
   bool isStringAttribute() const { return KindID == StringAttrEntry; }
   bool isTypeAttribute() const { return KindID == TypeAttrEntry; }
+  bool isConstantRangeAttribute() const {
+    return KindID == ConstantRangeAttrEntry;
+  }
 
   bool hasAttribute(Attribute::AttrKind A) const;
   bool hasAttribute(StringRef Kind) const;
@@ -72,6 +77,8 @@ public:
 
   Type *getValueAsType() const;
 
+  ConstantRange getValueAsConstantRange() const;
+
   /// Used when sorting the attributes.
   bool operator<(const AttributeImpl &AI) const;
 
@@ -82,8 +89,10 @@ public:
       Profile(ID, getKindAsEnum(), getValueAsInt());
     else if (isStringAttribute())
       Profile(ID, getKindAsString(), getValueAsString());
-    else
+    else if (isTypeAttribute())
       Profile(ID, getKindAsEnum(), getValueAsType());
+    else
+      Profile(ID, getKindAsEnum(), getValueAsConstantRange());
   }
 
   static void Profile(FoldingSetNodeID &ID, Attribute::AttrKind Kind) {
@@ -108,6 +117,13 @@ public:
     ID.AddInteger(Kind);
     ID.AddPointer(Ty);
   }
+
+  static void Profile(FoldingSetNodeID &ID, Attribute::AttrKind Kind,
+                      const ConstantRange &CR) {
+    ID.AddInteger(Kind);
+    ID.AddInteger(CR.getLower());
+    ID.AddInteger(CR.getUpper());
+  }
 };
 
 static_assert(std::is_trivially_destructible::value,
@@ -196,6 +212,16 @@ public:
   Type *getTypeValue() const { return Ty; }
 };
 
+class ConstantRangeAttributeImpl : public EnumAttributeImpl {
+  ConstantRange CR;
+
+public:
+  ConstantRangeAttributeImpl(Attribute::AttrKind Kind, const ConstantRange &CR)
+      : EnumAttributeImpl(ConstantRangeAttrEntry, Kind), CR(CR) {}
+
+  ConstantRange getConstantRangeValue() const { return CR; }
+};
+
 class AttributeBitSet {
   /// Bitset with a bit for each available attribute Attribute::AttrKind.
   uint8_t AvailableAttrs[12] = {};
diff --git a/llvm/lib/IR/Attributes.cpp b/llvm/lib/IR/Attributes.cpp
index 00acbbe7989d..fec34d0a4a81 100644
--- a/llvm/lib/IR/Attributes.cpp
+++ b/llvm/lib/IR/Attributes.cpp
@@ -24,6 +24,7 @@
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/Config/llvm-config.h"
 #include "llvm/IR/AttributeMask.h"
+#include "llvm/IR/ConstantRange.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Type.h"
@@ -165,6 +166,30 @@ Attribute Attribute::get(LLVMContext &Context, Attribute::AttrKind Kind,
   return Attribute(PA);
 }
 
+Attribute Attribute::get(LLVMContext &Context, Attribute::AttrKind Kind,
+                         const ConstantRange &CR) {
+  assert(Attribute::isConstantRangeAttrKind(Kind) &&
+         "Not a ConstantRange attribute");
+  LLVMContextImpl *pImpl = Context.pImpl;
+  FoldingSetNodeID ID;
+  ID.AddInteger(Kind);
+  ID.AddInteger(CR.getLower());
+  ID.AddInteger(CR.getUpper());
+
+  void *InsertPoint;
+  AttributeImpl *PA = pImpl->AttrsSet.FindNodeOrInsertPos(ID, InsertPoint);
+
+  if (!PA) {
+    // If we didn't find any existing attributes of the same shape then create a
+    // new one and insert it.
+    PA = new (pImpl->Alloc) ConstantRangeAttributeImpl(Kind, CR);
+    pImpl->AttrsSet.InsertNode(PA, InsertPoint);
+  }
+
+  // Return the Attribute that we found or created.
+  return Attribute(PA);
+}
+
 Attribute Attribute::getWithAlignment(LLVMContext &Context, Align A) {
   assert(A <= llvm::Value::MaximumAlignment && "Alignment too large.");
   return get(Context, Alignment, A.value());
@@ -287,9 +312,14 @@ bool Attribute::isTypeAttribute() const {
   return pImpl && pImpl->isTypeAttribute();
 }
 
+bool Attribute::isConstantRangeAttribute() const {
+  return pImpl && pImpl->isConstantRangeAttribute();
+}
+
 Attribute::AttrKind Attribute::getKindAsEnum() const {
   if (!pImpl) return None;
-  assert((isEnumAttribute() || isIntAttribute() || isTypeAttribute()) &&
+  assert((isEnumAttribute() || isIntAttribute() || isTypeAttribute() ||
+          isConstantRangeAttribute()) &&
          "Invalid attribute type to get the kind as an enum!");
   return pImpl->getKindAsEnum();
 }
@@ -329,6 +359,11 @@ Type *Attribute::getValueAsType() const {
   return pImpl->getValueAsType();
 }
 
+ConstantRange Attribute::getValueAsConstantRange() const {
+  assert(isConstantRangeAttribute() &&
+         "Invalid attribute type to get the value as a ConstantRange!");
+  return pImpl->getValueAsConstantRange();
+}
 
 bool Attribute::hasAttribute(AttrKind Kind) const {
   return (pImpl && pImpl->hasAttribute(Kind)) || (!pImpl && Kind == None);
@@ -408,6 +443,12 @@ FPClassTest Attribute::getNoFPClass() const {
   return static_cast(pImpl->getValueAsInt());
 }
 
+ConstantRange Attribute::getRange() const {
+  assert(hasAttribute(Attribute::Range) &&
+         "Trying to get range args from non-range attribute");
+  return pImpl->getValueAsConstantRange();
+}
+
 static const char *getModRefStr(ModRefInfo MR) {
   switch (MR) {
   case ModRefInfo::NoModRef:
@@ -562,6 +603,18 @@ std::string Attribute::getAsString(bool InAttrGrp) const {
     return Result;
   }
 
+  if (hasAttribute(Attribute::Range)) {
+    std::string Result;
+    raw_string_ostream OS(Result);
+    ConstantRange CR = getValueAsConstantRange();
+    OS << "range(";
+    OS << "i" << CR.getBitWidth() << " ";
+    OS << CR.getLower() << ", " << CR.getUpper();
+    OS << ")";
+    OS.flush();
+    return Result;
+  }
+
   // Convert target-dependent attributes to strings of the form:
   //
   //   "kind"
@@ -651,7 +704,8 @@ bool AttributeImpl::hasAttribute(StringRef Kind) const {
 }
 
 Attribute::AttrKind AttributeImpl::getKindAsEnum() const {
-  assert(isEnumAttribute() || isIntAttribute() || isTypeAttribute());
+  assert(isEnumAttribute() || isIntAttribute() || isTypeAttribute() ||
+         isConstantRangeAttribute());
   return static_cast(this)->getEnumKind();
 }
 
@@ -680,6 +734,12 @@ Type *AttributeImpl::getValueAsType() const {
   return static_cast(this)->getTypeValue();
 }
 
+ConstantRange AttributeImpl::getValueAsConstantRange() const {
+  assert(isConstantRangeAttribute());
+  return static_cast(this)
+      ->getConstantRangeValue();
+}
+
 bool AttributeImpl::operator<(const AttributeImpl &AI) const {
   if (this == &AI)
     return false;
@@ -693,6 +753,7 @@ bool AttributeImpl::operator<(const AttributeImpl &AI) const {
       return getKindAsEnum() < AI.getKindAsEnum();
     assert(!AI.isEnumAttribute() && "Non-unique attribute");
     assert(!AI.isTypeAttribute() && "Comparison of types would be unstable");
+    assert(!AI.isConstantRangeAttribute() && "Unclear how to compare ranges");
     // TODO: Is this actually needed?
     assert(AI.isIntAttribute() && "Only possibility left");
     return getValueAsInt() < AI.getValueAsInt();
@@ -1881,6 +1942,15 @@ AttrBuilder &AttrBuilder::addInAllocaAttr(Type *Ty) {
   return addTypeAttr(Attribute::InAlloca, Ty);
 }
 
+AttrBuilder &AttrBuilder::addConstantRangeAttr(Attribute::AttrKind Kind,
+                                               const ConstantRange &CR) {
+  return addAttribute(Attribute::get(Ctx, Kind, CR));
+}
+
+AttrBuilder &AttrBuilder::addRangeAttr(const ConstantRange &CR) {
+  return addConstantRangeAttr(Attribute::Range, CR);
+}
+
 AttrBuilder &AttrBuilder::merge(const AttrBuilder &B) {
   // TODO: Could make this O(n) as we're merging two sorted lists.
   for (const auto &I : B.attrs())
@@ -1952,6 +2022,12 @@ AttributeMask AttributeFuncs::typeIncompatible(Type *Ty,
       Incompatible.addAttribute(Attribute::SExt).addAttribute(Attribute::ZExt);
   }
 
+  if (!Ty->isIntOrIntVectorTy()) {
+    // Attributes that only apply to integers or vector of integers.
+    if (ASK & ASK_SAFE_TO_DROP)
+      Incompatible.addAttribute(Attribute::Range);
+  }
+
   if (!Ty->isPointerTy()) {
     // Attributes that only apply to pointers.
     if (ASK & ASK_SAFE_TO_DROP)
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index fd5f7d57c258..3cf5e81efb3b 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -2039,6 +2039,11 @@ void Verifier::verifyParameterAttrs(AttributeSet Attrs, Type *Ty,
     Check((Val & ~static_cast(fcAllFlags)) == 0,
           "Invalid value for 'nofpclass' test mask", V);
   }
+  if (Attrs.hasAttribute(Attribute::Range)) {
+    auto CR = Attrs.getAttribute(Attribute::Range).getValueAsConstantRange();
+    Check(Ty->isIntOrIntVectorTy(CR.getBitWidth()),
+          "Range bit width must match type bit width!", V);
+  }
 }
 
 void Verifier::checkUnsignedBaseTenFuncAttr(AttributeList Attrs, StringRef Attr,
diff --git a/llvm/lib/Transforms/Utils/CodeExtractor.cpp b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
index 3071ec0c9113..ab2d25c3f17c 100644
--- a/llvm/lib/Transforms/Utils/CodeExtractor.cpp
+++ b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
@@ -999,6 +999,7 @@ Function *CodeExtractor::constructFunction(const ValueSet &inputs,
       case Attribute::WriteOnly:
       case Attribute::Writable:
       case Attribute::DeadOnUnwind:
+      case Attribute::Range:
       //  These are not really attributes.
       case Attribute::None:
       case Attribute::EndAttrKinds:
diff --git a/llvm/test/Assembler/range-attribute-invalid-range.ll b/llvm/test/Assembler/range-attribute-invalid-range.ll
new file mode 100644
index 000000000000..cf6d3f080183
--- /dev/null
+++ b/llvm/test/Assembler/range-attribute-invalid-range.ll
@@ -0,0 +1,6 @@
+; RUN: not llvm-as < %s -o /dev/null 2>&1 | FileCheck %s
+
+; CHECK: the range should not represent the full or empty set!
+define void @range_empty(i8 range(i8 0, 0) %a) {
+  ret void
+}
diff --git a/llvm/test/Assembler/range-attribute-invalid-type.ll b/llvm/test/Assembler/range-attribute-invalid-type.ll
new file mode 100644
index 000000000000..cc09149a94dc
--- /dev/null
+++ b/llvm/test/Assembler/range-attribute-invalid-type.ll
@@ -0,0 +1,6 @@
+; RUN: not llvm-as < %s -o /dev/null 2>&1 | FileCheck %s
+
+; CHECK: the range must have integer type!
+define void @range_vector_type(i8 range(<4 x i32> 0, 0) %a) {
+  ret void
+}
diff --git a/llvm/test/Bitcode/attributes.ll b/llvm/test/Bitcode/attributes.ll
index 6921f11a352d..26163b4d38c8 100644
--- a/llvm/test/Bitcode/attributes.ll
+++ b/llvm/test/Bitcode/attributes.ll
@@ -526,6 +526,16 @@ define void @f91(ptr dead_on_unwind %p) {
   ret void
 }
 
+; CHECK: define range(i32 -1, 42) i32 @range_attribute(<4 x i32> range(i32 -1, 42) %a)
+define range(i32 -1, 42) i32 @range_attribute(<4 x i32> range(i32 -1, 42) %a) {
+  ret i32 0
+}
+
+; CHECK: define void @wide_range_attribute(i128 range(i128 618970019642690137449562111, 618970019642690137449562114) %a)
+define void @wide_range_attribute(i128 range(i128 618970019642690137449562111, 618970019642690137449562114) %a) {
+  ret void
+}
+
 ; CHECK: attributes #0 = { noreturn }
 ; CHECK: attributes #1 = { nounwind }
 ; CHECK: attributes #2 = { memory(none) }
diff --git a/llvm/test/Verifier/range-attr.ll b/llvm/test/Verifier/range-attr.ll
new file mode 100644
index 000000000000..f985ab696eac
--- /dev/null
+++ b/llvm/test/Verifier/range-attr.ll
@@ -0,0 +1,19 @@
+; RUN: not llvm-as %s -o /dev/null 2>&1 | FileCheck %s
+
+; CHECK: Range bit width must match type bit width!
+; CHECK-NEXT: ptr @bit_widths_do_not_match
+define void @bit_widths_do_not_match(i32 range(i8 1, 0) %a) {
+  ret void
+}
+
+; CHECK: Range bit width must match type bit width!
+; CHECK-NEXT: ptr @bit_widths_do_not_match_vector
+define void @bit_widths_do_not_match_vector(<4 x i32> range(i8 1, 0) %a) {
+  ret void
+}
+
+; CHECK: Attribute 'range(i8 1, 0)' applied to incompatible type!
+; CHECK-NEXT: ptr @not-integer-type
+define void @not-integer-type(ptr range(i8 1, 0) %a) {
+  ret void
+}
diff --git a/llvm/utils/TableGen/Attributes.cpp b/llvm/utils/TableGen/Attributes.cpp
index db3c4decccb4..d9fc7834416c 100644
--- a/llvm/utils/TableGen/Attributes.cpp
+++ b/llvm/utils/TableGen/Attributes.cpp
@@ -53,7 +53,8 @@ void Attributes::emitTargetIndependentNames(raw_ostream &OS) {
   };
 
   // Emit attribute enums in the same order llvm::Attribute::operator< expects.
-  Emit({"EnumAttr", "TypeAttr", "IntAttr"}, "ATTRIBUTE_ENUM");
+  Emit({"EnumAttr", "TypeAttr", "IntAttr", "ConstantRangeAttr"},
+       "ATTRIBUTE_ENUM");
   Emit({"StrBoolAttr"}, "ATTRIBUTE_STRBOOL");
   Emit({"ComplexStrAttr"}, "ATTRIBUTE_COMPLEXSTR");
 
@@ -63,7 +64,8 @@ void Attributes::emitTargetIndependentNames(raw_ostream &OS) {
   OS << "#ifdef GET_ATTR_ENUM\n";
   OS << "#undef GET_ATTR_ENUM\n";
   unsigned Value = 1; // Leave zero for AttrKind::None.
-  for (StringRef KindName : {"EnumAttr", "TypeAttr", "IntAttr"}) {
+  for (StringRef KindName :
+       {"EnumAttr", "TypeAttr", "IntAttr", "ConstantRangeAttr"}) {
     OS << "First" << KindName << " = " << Value << ",\n";
     for (auto *A : Records.getAllDerivedDefinitions(KindName)) {
       OS << A->getName() << " = " << Value << ",\n";
@@ -117,7 +119,8 @@ void Attributes::emitAttributeProperties(raw_ostream &OS) {
   OS << "#ifdef GET_ATTR_PROP_TABLE\n";
   OS << "#undef GET_ATTR_PROP_TABLE\n";
   OS << "static const uint8_t AttrPropTable[] = {\n";
-  for (StringRef KindName : {"EnumAttr", "TypeAttr", "IntAttr"}) {
+  for (StringRef KindName :
+       {"EnumAttr", "TypeAttr", "IntAttr", "ConstantRangeAttr"}) {
     for (auto *A : Records.getAllDerivedDefinitions(KindName)) {
       OS << "0";
       for (Init *P : *A->getValueAsListInit("Properties"))
-- 
GitLab


From 2b4d8188b263019477fa996b74b3da8a87a0e04b Mon Sep 17 00:00:00 2001
From: Lukacma 
Date: Fri, 8 Mar 2024 15:35:48 +0000
Subject: [PATCH 0042/1674] [Clang][LLVM][SVE2.1]  Created intrinsics for DUPQ
 instr. (#83260)

This patch adds clang and llvm support for following intrinsic and maps
it to DUPQ instruction:
```
   // Variants are also available for:
   // _s8, _u16, _s16, _u32, _s32, _u64, _s64
   // _bf16, _f16, _f32, _f64
   svuint8_t svdup_laneq[_u8](svuint8_t zn, uint64_t imm_idx);
```
---
 clang/include/clang/Basic/arm_sve.td          |   9 +
 .../acle_sve2p1_dupq.c                        | 213 ++++++++++++++++++
 .../acle_sve2p1_imm.cpp                       |  29 +++
 llvm/include/llvm/IR/IntrinsicsAArch64.td     |   7 +
 .../lib/Target/AArch64/AArch64SVEInstrInfo.td |   2 +-
 llvm/lib/Target/AArch64/SVEInstrFormats.td    |  20 +-
 .../CodeGen/AArch64/sve2p1-intrinsics-dupq.ll |  83 +++++++
 7 files changed, 357 insertions(+), 6 deletions(-)
 create mode 100644 clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_dupq.c
 create mode 100644 llvm/test/CodeGen/AArch64/sve2p1-intrinsics-dupq.ll

diff --git a/clang/include/clang/Basic/arm_sve.td b/clang/include/clang/Basic/arm_sve.td
index 6da30e08e752..6cc249837d3f 100644
--- a/clang/include/clang/Basic/arm_sve.td
+++ b/clang/include/clang/Basic/arm_sve.td
@@ -2215,6 +2215,15 @@ let TargetGuard = "sve2p1" in {
   def SVTBXQ : SInst<"svtbxq[_{d}]", "dddu", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_tbxq">;
   // EXTQ
   def EXTQ : SInst<"svextq[_{d}]", "dddk", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_extq", [], [ImmCheck<2, ImmCheck0_15>]>;
+  // DUPQ
+  def SVDUP_LANEQ_B  : SInst<"svdup_laneq[_{d}]", "ddi",  "cUc", MergeNone, "aarch64_sve_dup_laneq", [IsStreamingCompatible], [ImmCheck<1, ImmCheck0_15>]>;
+  def SVDUP_LANEQ_H  : SInst<"svdup_laneq[_{d}]", "ddi",  "sUsh", MergeNone, "aarch64_sve_dup_laneq", [IsStreamingCompatible], [ImmCheck<1, ImmCheck0_7>]>;
+  def SVDUP_LANEQ_S  : SInst<"svdup_laneq[_{d}]", "ddi",  "iUif", MergeNone, "aarch64_sve_dup_laneq", [IsStreamingCompatible], [ImmCheck<1, ImmCheck0_3>]>;
+  def SVDUP_LANEQ_D  : SInst<"svdup_laneq[_{d}]", "ddi",  "lUld", MergeNone, "aarch64_sve_dup_laneq", [IsStreamingCompatible], [ImmCheck<1, ImmCheck0_1>]>;
+
+  let TargetGuard = "bf16" in {
+    def SVDUP_LANEQ_BF16  : SInst<"svdup_laneq[_{d}]", "ddi",  "b", MergeNone, "aarch64_sve_dup_laneq", [IsStreamingCompatible], [ImmCheck<1, ImmCheck0_7>]>;
+  }
   // PMOV
   // Move to Pred
   multiclass PMOV_TO_PRED flags=[], ImmCheckType immCh > {
diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_dupq.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_dupq.c
new file mode 100644
index 000000000000..587a67aa6b7c
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_dupq.c
@@ -0,0 +1,213 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -target-feature +bf16\
+// RUN:   -S -Werror -emit-llvm -disable-O0-optnone -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -target-feature +bf16\
+// RUN:   -S -Werror -emit-llvm -disable-O0-optnone -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -target-feature +bf16\
+// RUN:   -S -Werror -emit-llvm -disable-O0-optnone -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -target-feature +bf16\
+// RUN:   -S -Werror -emit-llvm -disable-O0-optnone -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -target-feature +bf16 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s
+
+
+#include 
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1, A2_UNUSED) A1
+#else
+#define SVE_ACLE_FUNC(A1, A2) A1##A2
+#endif
+
+// CHECK-LABEL: define dso_local  @test_svdup_laneq_s8
+// CHECK-SAME: ( [[ZN:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call  @llvm.aarch64.sve.dup.laneq.nxv16i8( [[ZN]], i32 0)
+// CHECK-NEXT:    ret  [[TMP0]]
+//
+// CPP-CHECK-LABEL: define dso_local  @_Z19test_svdup_laneq_s8u10__SVInt8_t
+// CPP-CHECK-SAME: ( [[ZN:%.*]]) #[[ATTR0:[0-9]+]] {
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call  @llvm.aarch64.sve.dup.laneq.nxv16i8( [[ZN]], i32 0)
+// CPP-CHECK-NEXT:    ret  [[TMP0]]
+//
+svint8_t test_svdup_laneq_s8(svint8_t zn) {
+    return SVE_ACLE_FUNC(svdup_laneq, _s8)(zn, 0);
+}
+
+// CHECK-LABEL: define dso_local  @test_svdup_laneq_u8
+// CHECK-SAME: ( [[ZN:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call  @llvm.aarch64.sve.dup.laneq.nxv16i8( [[ZN]], i32 15)
+// CHECK-NEXT:    ret  [[TMP0]]
+//
+// CPP-CHECK-LABEL: define dso_local  @_Z19test_svdup_laneq_u8u11__SVUint8_t
+// CPP-CHECK-SAME: ( [[ZN:%.*]]) #[[ATTR0]] {
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call  @llvm.aarch64.sve.dup.laneq.nxv16i8( [[ZN]], i32 15)
+// CPP-CHECK-NEXT:    ret  [[TMP0]]
+//
+svuint8_t test_svdup_laneq_u8(svuint8_t zn) {
+    return SVE_ACLE_FUNC(svdup_laneq, _u8)(zn, 15);
+}
+
+// CHECK-LABEL: define dso_local  @test_svdup_laneq_s16
+// CHECK-SAME: ( [[ZN:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call  @llvm.aarch64.sve.dup.laneq.nxv8i16( [[ZN]], i32 1)
+// CHECK-NEXT:    ret  [[TMP0]]
+//
+// CPP-CHECK-LABEL: define dso_local  @_Z20test_svdup_laneq_s16u11__SVInt16_t
+// CPP-CHECK-SAME: ( [[ZN:%.*]]) #[[ATTR0]] {
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call  @llvm.aarch64.sve.dup.laneq.nxv8i16( [[ZN]], i32 1)
+// CPP-CHECK-NEXT:    ret  [[TMP0]]
+//
+svint16_t test_svdup_laneq_s16(svint16_t zn) {
+    return SVE_ACLE_FUNC(svdup_laneq, _s16)(zn, 1);
+}
+
+// CHECK-LABEL: define dso_local  @test_svdup_laneq_u16
+// CHECK-SAME: ( [[ZN:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call  @llvm.aarch64.sve.dup.laneq.nxv8i16( [[ZN]], i32 7)
+// CHECK-NEXT:    ret  [[TMP0]]
+//
+// CPP-CHECK-LABEL: define dso_local  @_Z20test_svdup_laneq_u16u12__SVUint16_t
+// CPP-CHECK-SAME: ( [[ZN:%.*]]) #[[ATTR0]] {
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call  @llvm.aarch64.sve.dup.laneq.nxv8i16( [[ZN]], i32 7)
+// CPP-CHECK-NEXT:    ret  [[TMP0]]
+//
+svuint16_t test_svdup_laneq_u16(svuint16_t zn) {
+    return SVE_ACLE_FUNC(svdup_laneq, _u16)(zn, 7);
+}
+
+// CHECK-LABEL: define dso_local  @test_svdup_laneq_s32
+// CHECK-SAME: ( [[ZN:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call  @llvm.aarch64.sve.dup.laneq.nxv4i32( [[ZN]], i32 2)
+// CHECK-NEXT:    ret  [[TMP0]]
+//
+// CPP-CHECK-LABEL: define dso_local  @_Z20test_svdup_laneq_s32u11__SVInt32_t
+// CPP-CHECK-SAME: ( [[ZN:%.*]]) #[[ATTR0]] {
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call  @llvm.aarch64.sve.dup.laneq.nxv4i32( [[ZN]], i32 2)
+// CPP-CHECK-NEXT:    ret  [[TMP0]]
+//
+svint32_t test_svdup_laneq_s32(svint32_t zn) {
+    return SVE_ACLE_FUNC(svdup_laneq, _s32)(zn, 2);
+}
+
+// CHECK-LABEL: define dso_local  @test_svdup_laneq_u32
+// CHECK-SAME: ( [[ZN:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call  @llvm.aarch64.sve.dup.laneq.nxv4i32( [[ZN]], i32 3)
+// CHECK-NEXT:    ret  [[TMP0]]
+//
+// CPP-CHECK-LABEL: define dso_local  @_Z20test_svdup_laneq_u32u12__SVUint32_t
+// CPP-CHECK-SAME: ( [[ZN:%.*]]) #[[ATTR0]] {
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call  @llvm.aarch64.sve.dup.laneq.nxv4i32( [[ZN]], i32 3)
+// CPP-CHECK-NEXT:    ret  [[TMP0]]
+//
+svuint32_t test_svdup_laneq_u32(svuint32_t zn) {
+    return SVE_ACLE_FUNC(svdup_laneq, _u32)(zn, 3);
+}
+
+// CHECK-LABEL: define dso_local  @test_svdup_laneq_s64
+// CHECK-SAME: ( [[ZN:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call  @llvm.aarch64.sve.dup.laneq.nxv2i64( [[ZN]], i32 0)
+// CHECK-NEXT:    ret  [[TMP0]]
+//
+// CPP-CHECK-LABEL: define dso_local  @_Z20test_svdup_laneq_s64u11__SVInt64_t
+// CPP-CHECK-SAME: ( [[ZN:%.*]]) #[[ATTR0]] {
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call  @llvm.aarch64.sve.dup.laneq.nxv2i64( [[ZN]], i32 0)
+// CPP-CHECK-NEXT:    ret  [[TMP0]]
+//
+svint64_t test_svdup_laneq_s64(svint64_t zn) {
+    return SVE_ACLE_FUNC(svdup_laneq, _s64)(zn, 0);
+}
+
+// CHECK-LABEL: define dso_local  @test_svdup_laneq_u64
+// CHECK-SAME: ( [[ZN:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call  @llvm.aarch64.sve.dup.laneq.nxv2i64( [[ZN]], i32 1)
+// CHECK-NEXT:    ret  [[TMP0]]
+//
+// CPP-CHECK-LABEL: define dso_local  @_Z20test_svdup_laneq_u64u12__SVUint64_t
+// CPP-CHECK-SAME: ( [[ZN:%.*]]) #[[ATTR0]] {
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call  @llvm.aarch64.sve.dup.laneq.nxv2i64( [[ZN]], i32 1)
+// CPP-CHECK-NEXT:    ret  [[TMP0]]
+//
+svuint64_t test_svdup_laneq_u64(svuint64_t zn) {
+    return SVE_ACLE_FUNC(svdup_laneq, _u64)(zn, 1);
+}
+
+// CHECK-LABEL: define dso_local  @test_svdup_laneq_f16
+// CHECK-SAME: ( [[ZN:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call  @llvm.aarch64.sve.dup.laneq.nxv8f16( [[ZN]], i32 4)
+// CHECK-NEXT:    ret  [[TMP0]]
+//
+// CPP-CHECK-LABEL: define dso_local  @_Z20test_svdup_laneq_f16u13__SVFloat16_t
+// CPP-CHECK-SAME: ( [[ZN:%.*]]) #[[ATTR0]] {
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call  @llvm.aarch64.sve.dup.laneq.nxv8f16( [[ZN]], i32 4)
+// CPP-CHECK-NEXT:    ret  [[TMP0]]
+//
+svfloat16_t test_svdup_laneq_f16(svfloat16_t zn) {
+    return SVE_ACLE_FUNC(svdup_laneq, _f16)(zn, 4);
+}
+
+// CHECK-LABEL: define dso_local  @test_svdup_laneq_f32
+// CHECK-SAME: ( [[ZN:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call  @llvm.aarch64.sve.dup.laneq.nxv4f32( [[ZN]], i32 1)
+// CHECK-NEXT:    ret  [[TMP0]]
+//
+// CPP-CHECK-LABEL: define dso_local  @_Z20test_svdup_laneq_f32u13__SVFloat32_t
+// CPP-CHECK-SAME: ( [[ZN:%.*]]) #[[ATTR0]] {
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call  @llvm.aarch64.sve.dup.laneq.nxv4f32( [[ZN]], i32 1)
+// CPP-CHECK-NEXT:    ret  [[TMP0]]
+//
+svfloat32_t test_svdup_laneq_f32(svfloat32_t zn) {
+    return SVE_ACLE_FUNC(svdup_laneq, _f32)(zn, 1);
+}
+
+// CHECK-LABEL: define dso_local  @test_svdup_laneq_f64
+// CHECK-SAME: ( [[ZN:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call  @llvm.aarch64.sve.dup.laneq.nxv2f64( [[ZN]], i32 1)
+// CHECK-NEXT:    ret  [[TMP0]]
+//
+// CPP-CHECK-LABEL: define dso_local  @_Z20test_svdup_laneq_f64u13__SVFloat64_t
+// CPP-CHECK-SAME: ( [[ZN:%.*]]) #[[ATTR0]] {
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call  @llvm.aarch64.sve.dup.laneq.nxv2f64( [[ZN]], i32 1)
+// CPP-CHECK-NEXT:    ret  [[TMP0]]
+//
+svfloat64_t test_svdup_laneq_f64(svfloat64_t zn) {
+    return SVE_ACLE_FUNC(svdup_laneq, _f64)(zn, 1);
+}
+
+// CHECK-LABEL: define dso_local  @test_svdup_laneq_bf16
+// CHECK-SAME: ( [[ZN:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call  @llvm.aarch64.sve.dup.laneq.nxv8bf16( [[ZN]], i32 3)
+// CHECK-NEXT:    ret  [[TMP0]]
+//
+// CPP-CHECK-LABEL: define dso_local  @_Z21test_svdup_laneq_bf16u14__SVBfloat16_t
+// CPP-CHECK-SAME: ( [[ZN:%.*]]) #[[ATTR0]] {
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call  @llvm.aarch64.sve.dup.laneq.nxv8bf16( [[ZN]], i32 3)
+// CPP-CHECK-NEXT:    ret  [[TMP0]]
+//
+svbfloat16_t test_svdup_laneq_bf16(svbfloat16_t zn) {
+    return SVE_ACLE_FUNC(svdup_laneq, _bf16)(zn, 3);
+}
diff --git a/clang/test/Sema/aarch64-sve2p1-intrinsics/acle_sve2p1_imm.cpp b/clang/test/Sema/aarch64-sve2p1-intrinsics/acle_sve2p1_imm.cpp
index d857608d4415..a6ec5150f0aa 100644
--- a/clang/test/Sema/aarch64-sve2p1-intrinsics/acle_sve2p1_imm.cpp
+++ b/clang/test/Sema/aarch64-sve2p1-intrinsics/acle_sve2p1_imm.cpp
@@ -188,3 +188,32 @@ void test_svget_svset_b(uint64_t idx, svboolx2_t tuple2, svboolx4_t tuple4, svbo
   svget2_b(tuple2, idx); // expected-error {{argument to 'svget2_b' must be a constant integer}}
   svget4_b(tuple4, idx); // expected-error {{argument to 'svget4_b' must be a constant integer}}
 }
+
+__attribute__((target("+sve2p1")))
+void test_svdup_laneq(){  
+  svuint8_t zn_u8;
+  svuint16_t zn_u16;
+  svuint32_t zn_u32;
+  svuint64_t zn_u64;
+  svint8_t zn_s8;
+  svint16_t zn_s16;
+  svint32_t zn_s32;
+  svint64_t zn_s64;
+  svfloat16_t zn_f16;
+  svfloat32_t zn_f32;
+  svfloat64_t zn_f64;
+  svbfloat16_t zn_bf16;
+
+  svdup_laneq_u8(zn_u8,-1); // expected-error {{argument value 18446744073709551615 is outside the valid range [0, 15]}}
+  svdup_laneq_u16(zn_u16,-1); // expected-error {{argument value 18446744073709551615 is outside the valid range [0, 7]}}
+  svdup_laneq_u32(zn_u32,-1); // expected-error {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
+  svdup_laneq_u64(zn_u64,-1); // expected-error {{argument value 18446744073709551615 is outside the valid range [0, 1]}}
+  svdup_laneq_s8(zn_s8,-1); // expected-error {{argument value 18446744073709551615 is outside the valid range [0, 15]}}
+  svdup_laneq_s16(zn_s16,-1); // expected-error {{argument value 18446744073709551615 is outside the valid range [0, 7]}}
+  svdup_laneq_s32(zn_s32,-1); // expected-error {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
+  svdup_laneq_s64(zn_s64,-1); // expected-error {{argument value 18446744073709551615 is outside the valid range [0, 1]}}
+  svdup_laneq_f16(zn_f16,-1); // expected-error {{argument value 18446744073709551615 is outside the valid range [0, 7]}}
+  svdup_laneq_f32(zn_f32,-1); // expected-error {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
+  svdup_laneq_f64(zn_f64,-1); // expected-error {{argument value 18446744073709551615 is outside the valid range [0, 1]}}
+  svdup_laneq_bf16(zn_bf16,-1); // expected-error {{argument value 18446744073709551615 is outside the valid range [0, 7]}}
+}
\ No newline at end of file
diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td b/llvm/include/llvm/IR/IntrinsicsAArch64.td
index 5a9a7c4b43a1..bcaa37de74b6 100644
--- a/llvm/include/llvm/IR/IntrinsicsAArch64.td
+++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td
@@ -1360,6 +1360,12 @@ let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
                  LLVMSubdivide2VectorType<0>,
                  llvm_i32_ty],
                 [IntrNoMem, ImmArg>]>;
+  
+  class SVE2_1VectorArgIndexed_Intrinsic
+    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
+                [LLVMMatchType<0>,
+                 llvm_i32_ty],
+                [IntrNoMem, ImmArg>]>;
 
   class AdvSIMD_SVE_CDOT_LANE_Intrinsic
     : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
@@ -1913,6 +1919,7 @@ def int_aarch64_sve_clastb    : AdvSIMD_Pred2VectorArg_Intrinsic;
 def int_aarch64_sve_clastb_n  : AdvSIMD_SVE_ReduceWithInit_Intrinsic;
 def int_aarch64_sve_compact   : AdvSIMD_Pred1VectorArg_Intrinsic;
 def int_aarch64_sve_dupq_lane : AdvSIMD_SVE_DUPQ_Intrinsic;
+def int_aarch64_sve_dup_laneq : SVE2_1VectorArgIndexed_Intrinsic;
 def int_aarch64_sve_ext       : AdvSIMD_2VectorArgIndexed_Intrinsic;
 def int_aarch64_sve_sel       : AdvSIMD_Pred2VectorArg_Intrinsic;
 def int_aarch64_sve_lasta     : AdvSIMD_SVE_Reduce_Intrinsic;
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 7c98f934a131..e0a010af4155 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -4101,7 +4101,7 @@ defm FMINNMQV : sve2p1_fp_reduction_q<0b101, "fminnmqv", int_aarch64_sve_fminnmq
 defm FMAXQV   : sve2p1_fp_reduction_q<0b110, "fmaxqv", int_aarch64_sve_fmaxqv>;
 defm FMINQV   : sve2p1_fp_reduction_q<0b111, "fminqv", int_aarch64_sve_fminqv>;
 
-defm DUPQ_ZZI : sve2p1_dupq<"dupq">;
+defm DUPQ_ZZI : sve2p1_dupq<"dupq", int_aarch64_sve_dup_laneq>;
 defm EXTQ_ZZI : sve2p1_extq<"extq", int_aarch64_sve_extq>;
 
 defm PMOV_PZI : sve2p1_vector_to_pred<"pmov", int_aarch64_sve_pmov_to_pred_lane, int_aarch64_sve_pmov_to_pred_lane_zero>;
diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td
index 8baf3a6d3d81..c19e02bb03d1 100644
--- a/llvm/lib/Target/AArch64/SVEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td
@@ -9893,23 +9893,33 @@ class sve2p1_dupq ind_tsz, string mnemonic, ZPRRegOp zprty, Operand ityp
   let hasSideEffects = 0;
 }
 
-multiclass sve2p1_dupq {
-  def _B : sve2p1_dupq<{?, ?, ?, ?, 1}, mnemonic, ZPR8, VectorIndexB32b> {
+multiclass sve2p1_dupq {
+  def _B : sve2p1_dupq<{?, ?, ?, ?, 1}, mnemonic, ZPR8, VectorIndexB32b_timm> {
     bits<4> index;
     let Inst{20-17} = index;
   }
-  def _H : sve2p1_dupq<{?, ?, ?, 1, 0}, mnemonic, ZPR16, VectorIndexH32b> {
+  def _H : sve2p1_dupq<{?, ?, ?, 1, 0}, mnemonic, ZPR16, VectorIndexH32b_timm> {
     bits<3> index;
     let Inst{20-18} = index;
   }
-  def _S : sve2p1_dupq<{?, ?, 1, 0, 0}, mnemonic, ZPR32, VectorIndexS32b> {
+  def _S : sve2p1_dupq<{?, ?, 1, 0, 0}, mnemonic, ZPR32, VectorIndexS32b_timm> {
     bits<2> index;
     let Inst{20-19} = index;
   }
-  def _D : sve2p1_dupq<{?, 1, 0, 0, 0}, mnemonic, ZPR64, VectorIndexD32b> {
+  def _D : sve2p1_dupq<{?, 1, 0, 0, 0}, mnemonic, ZPR64, VectorIndexD32b_timm> {
     bits<1> index;
     let Inst{20} = index;
   }
+
+  def : SVE_2_Op_Imm_Pat(NAME # _B)>;
+  def : SVE_2_Op_Imm_Pat(NAME # _H)>;
+  def : SVE_2_Op_Imm_Pat(NAME # _S)>;
+  def : SVE_2_Op_Imm_Pat(NAME # _D)>;
+
+  def : SVE_2_Op_Imm_Pat(NAME # _H)>;
+  def : SVE_2_Op_Imm_Pat(NAME # _S)>;
+  def : SVE_2_Op_Imm_Pat(NAME # _D)>;
+  def : SVE_2_Op_Imm_Pat(NAME # _H)>;
 }
 
 
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-dupq.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-dupq.ll
new file mode 100644
index 000000000000..f1a423b02ac2
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-dupq.ll
@@ -0,0 +1,83 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve2p1,+bf16 < %s | FileCheck %s
+
+define  @test_dupq_i8 ( %zn) {
+; CHECK-LABEL: test_dupq_i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    dupq z0.b, z0.b[15]
+; CHECK-NEXT:    ret
+  %res = call  @llvm.aarch64.sve.dup.laneq.nxv16i8( %zn, i32 15)
+  ret  %res
+}
+
+define  @test_dupq_i16 ( %zn) {
+; CHECK-LABEL: test_dupq_i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    dupq z0.h, z0.h[7]
+; CHECK-NEXT:    ret
+  %res = call  @llvm.aarch64.sve.dup.laneq.nxv8i16( %zn, i32 7)
+  ret  %res
+}
+
+define  @test_dupq__i32 ( %zn) {
+; CHECK-LABEL: test_dupq__i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    dupq z0.s, z0.s[3]
+; CHECK-NEXT:    ret
+  %res = call  @llvm.aarch64.sve.dup.laneq.nxv4i32( %zn, i32 3)
+  ret  %res
+}
+
+define  @test_dupq_i64 ( %zn) {
+; CHECK-LABEL: test_dupq_i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    dupq z0.d, z0.d[1]
+; CHECK-NEXT:    ret
+  %res = call  @llvm.aarch64.sve.dup.laneq.nxv2i64( %zn, i32 1)
+  ret  %res
+}
+
+define  @test_dupq_f16( %zn) {
+; CHECK-LABEL: test_dupq_f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    dupq z0.h, z0.h[4]
+; CHECK-NEXT:    ret
+  %res = call  @llvm.aarch64.sve.dup.laneq.nxv8f16( %zn, i32 4)
+  ret  %res
+}
+
+define  @test_dupq_f32( %zn) {
+; CHECK-LABEL: test_dupq_f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    dupq z0.s, z0.s[2]
+; CHECK-NEXT:    ret
+  %res = call  @llvm.aarch64.sve.dup.laneq.nxv4f32( %zn, i32 2)
+  ret  %res
+}
+
+define  @test_dupq_f64( %zn) {
+; CHECK-LABEL: test_dupq_f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    dupq z0.d, z0.d[0]
+; CHECK-NEXT:    ret
+  %res = call  @llvm.aarch64.sve.dup.laneq.nxv2f64( %zn, i32 0)
+  ret  %res
+}
+
+define  @test_dupq_bf16( %zn) {
+; CHECK-LABEL: test_dupq_bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    dupq z0.h, z0.h[1]
+; CHECK-NEXT:    ret
+  %res = call  @llvm.aarch64.sve.dup.laneq.nxv8bf16( %zn, i32 1)
+  ret  %res
+}
+
+declare  @llvm.aarch64.sve.dup.laneq.nxv16i8(, i32)
+declare  @llvm.aarch64.sve.dup.laneq.nxv8i16(, i32)
+declare  @llvm.aarch64.sve.dup.laneq.nxv4i32(, i32)
+declare  @llvm.aarch64.sve.dup.laneq.nxv2i64(, i32)
+declare  @llvm.aarch64.sve.dup.laneq.nxv8f16(, i32)
+declare  @llvm.aarch64.sve.dup.laneq.nxv4f32(, i32)
+declare  @llvm.aarch64.sve.dup.laneq.nxv2f64(, i32)
+declare  @llvm.aarch64.sve.dup.laneq.nxv8bf16(, i32)
-- 
GitLab


From 29afd64615c24af9d89f3e1e8d24e9827a34224c Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan 
Date: Fri, 8 Mar 2024 10:36:22 -0500
Subject: [PATCH 0043/1674] [libc][c23] make inttypes.h compat with c23
 (#84493)

---
 .../llvm-libc-macros/inttypes-macros.h        | 137 +++++++++++++++++-
 libc/spec/stdc.td                             |   4 +-
 2 files changed, 137 insertions(+), 4 deletions(-)

diff --git a/libc/include/llvm-libc-macros/inttypes-macros.h b/libc/include/llvm-libc-macros/inttypes-macros.h
index 8e7d4f558a63..9b554670271e 100644
--- a/libc/include/llvm-libc-macros/inttypes-macros.h
+++ b/libc/include/llvm-libc-macros/inttypes-macros.h
@@ -9,11 +9,9 @@
 #define LLVM_LIBC_MACROS_INTTYPES_MACROS_H
 
 // fprintf/scanf format macros.
-// POSIX.1-2008, Technical Corrigendum 1, XBD/TC1-2008/0050 [211] is applied.
+#define __STDC_VERSION_INTTYPES_H__ 202311L
 
 // clang provides these macros, so we don't need to define them.
-// TODO: ISO C23 will provide binary notations.
-
 #ifndef __clang__
 #if __UINTPTR_MAX__ == __UINT64_MAX__
 #define __PRI64 "l"
@@ -117,6 +115,94 @@
 #define __UINTPTR_FMTX__ __PRIPTR "X"
 #endif
 
+// only recent clang provides these macros, so we need to check if they are
+// available.
+#ifndef __UINT8_FMTb__
+#define __UINT8_FMTb__ "hhb"
+#endif
+#ifndef __UINT16_FMTb__
+#define __UINT16_FMTb__ "hb"
+#endif
+#ifndef __UINT32_FMTb__
+#define __UINT32_FMTb__ "b"
+#endif
+#ifndef __UINT64_FMTb__
+#define __UINT64_FMTb__ __PRI64 "b"
+#endif
+#ifndef __UINT_LEAST8_FMTb__
+#define __UINT_LEAST8_FMTb__ "hhb"
+#endif
+#ifndef __UINT_LEAST16_FMTb__
+#define __UINT_LEAST16_FMTb__ "hb"
+#endif
+#ifndef __UINT_LEAST32_FMTb__
+#define __UINT_LEAST32_FMTb__ "b"
+#endif
+#ifndef __UINT_LEAST64_FMTb__
+#define __UINT_LEAST64_FMTb__ __PRI64 "b"
+#endif
+#ifndef __UINT_FAST8_FMTb__
+#define __UINT_FAST8_FMTb__ "hhb"
+#endif
+#ifndef __UINT_FAST16_FMTb__
+#define __UINT_FAST16_FMTb__ "hb"
+#endif
+#ifndef __UINT_FAST32_FMTb__
+#define __UINT_FAST32_FMTb__ "b"
+#endif
+#ifndef __UINT_FAST64_FMTb__
+#define __UINT_FAST64_FMTb__ __PRI64 "b"
+#endif
+#ifndef __UINTMAX_FMTb__
+#define __UINTMAX_FMTb__ __PRI64 "b"
+#endif
+#ifndef __UINTPTR_FMTb__
+#define __UINTPTR_FMTb__ __PRIPTR "b"
+#endif
+
+#ifndef __UINT8_FMTB__
+#define __UINT8_FMTB__ "hhB"
+#endif
+#ifndef __UINT16_FMTB__
+#define __UINT16_FMTB__ "hB"
+#endif
+#ifndef __UINT32_FMTB__
+#define __UINT32_FMTB__ "B"
+#endif
+#ifndef __UINT64_FMTB__
+#define __UINT64_FMTB__ __PRI64 "B"
+#endif
+#ifndef __UINT_LEAST8_FMTB__
+#define __UINT_LEAST8_FMTB__ "hhB"
+#endif
+#ifndef __UINT_LEAST16_FMTB__
+#define __UINT_LEAST16_FMTB__ "hB"
+#endif
+#ifndef __UINT_LEAST32_FMTB__
+#define __UINT_LEAST32_FMTB__ "B"
+#endif
+#ifndef __UINT_LEAST64_FMTB__
+#define __UINT_LEAST64_FMTB__ __PRI64 "B"
+#endif
+#ifndef __UINT_FAST8_FMTB__
+#define __UINT_FAST8_FMTB__ "hhB"
+#endif
+#ifndef __UINT_FAST16_FMTB__
+#define __UINT_FAST16_FMTB__ "hB"
+#endif
+#ifndef __UINT_FAST32_FMTB__
+#define __UINT_FAST32_FMTB__ "B"
+#endif
+#ifndef __UINT_FAST64_FMTB__
+#define __UINT_FAST64_FMTB__ __PRI64 "B"
+#endif
+#ifndef __UINTMAX_FMTB__
+#define __UINTMAX_FMTB__ __PRI64 "B"
+#endif
+#ifndef __UINTPTR_FMTB__
+#define __UINTPTR_FMTB__ __PRIPTR "B"
+#endif
+
 // The fprintf() macros for signed integers.
 #define PRId8 __INT8_FMTd__
 #define PRId16 __INT16_FMTd__
@@ -209,6 +295,36 @@
 #define PRIXMAX __UINTMAX_FMTX__
 #define PRIXPTR __UINTPTR_FMTX__
 
+#define PRIb8 __UINT8_FMTb__
+#define PRIb16 __UINT16_FMTb__
+#define PRIb32 __UINT32_FMTb__
+#define PRIb64 __UINT64_FMTb__
+#define PRIbLEAST8 __UINT_LEAST8_FMTb__
+#define PRIbLEAST16 __UINT_LEAST16_FMTb__
+#define PRIbLEAST32 __UINT_LEAST32_FMTb__
+#define PRIbLEAST64 __UINT_LEAST64_FMTb__
+#define PRIbFAST8 __UINT_FAST8_FMTb__
+#define PRIbFAST16 __UINT_FAST16_FMTb__
+#define PRIbFAST32 __UINT_FAST32_FMTb__
+#define PRIbFAST64 __UINT_FAST64_FMTb__
+#define PRIbMAX __UINTMAX_FMTb__
+#define PRIbPTR __UINTPTR_FMTb__
+
+#define PRIB8 __UINT8_FMTB__
+#define PRIB16 __UINT16_FMTB__
+#define PRIB32 __UINT32_FMTB__
+#define PRIB64 __UINT64_FMTB__
+#define PRIBLEAST8 __UINT_LEAST8_FMTB__
+#define PRIBLEAST16 __UINT_LEAST16_FMTB__
+#define PRIBLEAST32 __UINT_LEAST32_FMTB__
+#define PRIBLEAST64 __UINT_LEAST64_FMTB__
+#define PRIBFAST8 __UINT_FAST8_FMTB__
+#define PRIBFAST16 __UINT_FAST16_FMTB__
+#define PRIBFAST32 __UINT_FAST32_FMTB__
+#define PRIBFAST64 __UINT_FAST64_FMTB__
+#define PRIBMAX __UINTMAX_FMTB__
+#define PRIBPTR __UINTPTR_FMTB__
+
 // The fscanf() macros for signed integers.
 #define SCNd8 __INT8_FMTd__
 #define SCNd16 __INT16_FMTd__
@@ -286,4 +402,19 @@
 #define SCNxMAX __UINTMAX_FMTx__
 #define SCNxPTR __UINTPTR_FMTx__
 
+#define SCNb8 __UINT8_FMTb__
+#define SCNb16 __UINT16_FMTb__
+#define SCNb32 __UINT32_FMTb__
+#define SCNb64 __UINT64_FMTb__
+#define SCNbLEAST8 __UINT_LEAST8_FMTb__
+#define SCNbLEAST16 __UINT_LEAST16_FMTb__
+#define SCNbLEAST32 __UINT_LEAST32_FMTb__
+#define SCNbLEAST64 __UINT_LEAST64_FMTb__
+#define SCNbFAST8 __UINT_FAST8_FMTb__
+#define SCNbFAST16 __UINT_FAST16_FMTb__
+#define SCNbFAST32 __UINT_FAST32_FMTb__
+#define SCNbFAST64 __UINT_FAST64_FMTb__
+#define SCNbMAX __UINTMAX_FMTb__
+#define SCNbPTR __UINTPTR_FMTb__
+
 #endif // LLVM_LIBC_MACROS_INTTYPES_MACROS_H
diff --git a/libc/spec/stdc.td b/libc/spec/stdc.td
index e09cce0efd9b..d84eb3570e5c 100644
--- a/libc/spec/stdc.td
+++ b/libc/spec/stdc.td
@@ -948,7 +948,9 @@ def StdC : StandardSpec<"stdc"> {
 
   HeaderSpec IntTypes = HeaderSpec<
       "inttypes.h",
-      [], // Macros
+      [
+        Macro<"__STDC_VERSION_INTTYPES_H__">,
+      ], // Macros
       [
         IMaxDivTType,
       ], // Types
-- 
GitLab


From b46f980454d5ceafc8dab37dbdb1011e333ae6de Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" 
Date: Fri, 8 Mar 2024 10:50:38 -0500
Subject: [PATCH 0044/1674] [HIP] fix host-used external kernel (#83870)

In -fgpu-rdc mode, when an external kernel is used by a host function
with weak_odr linkage (e.g. explicitly instantiated template function),
the kernel should not be marked as host-used external kernel, since the
host function may be dropped by the linker. Mark the external kernel as
host-used external kernel will force a reference to the external kernel,
which the user may not define in other TU.

Fixes: https://github.com/llvm/llvm-project/issues/83771
---
 clang/lib/Sema/SemaCUDA.cpp                |  5 ++++-
 clang/lib/Sema/SemaExpr.cpp                |  5 ++++-
 clang/test/CodeGenCUDA/host-used-extern.cu | 13 +++++++++++++
 3 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/clang/lib/Sema/SemaCUDA.cpp b/clang/lib/Sema/SemaCUDA.cpp
index 6a66ecf6f94c..4d4f4b6a2d4d 100644
--- a/clang/lib/Sema/SemaCUDA.cpp
+++ b/clang/lib/Sema/SemaCUDA.cpp
@@ -895,7 +895,10 @@ bool Sema::CheckCUDACall(SourceLocation Loc, FunctionDecl *Callee) {
   if (DiagKind == SemaDiagnosticBuilder::K_Nop) {
     // For -fgpu-rdc, keep track of external kernels used by host functions.
     if (LangOpts.CUDAIsDevice && LangOpts.GPURelocatableDeviceCode &&
-        Callee->hasAttr() && !Callee->isDefined())
+        Callee->hasAttr() && !Callee->isDefined() &&
+        (!Caller || (!Caller->getDescribedFunctionTemplate() &&
+                     getASTContext().GetGVALinkageForFunction(Caller) ==
+                         GVA_StrongExternal)))
       getASTContext().CUDAExternalDeviceDeclODRUsedByHost.insert(Callee);
     return true;
   }
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 47bb263f56aa..93f82e68ab64 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -19218,7 +19218,10 @@ MarkVarDeclODRUsed(ValueDecl *V, SourceLocation Loc, Sema &SemaRef,
       // externalize the static device side variable ODR-used by host code.
       if (!Var->hasExternalStorage())
         SemaRef.getASTContext().CUDADeviceVarODRUsedByHost.insert(Var);
-      else if (SemaRef.LangOpts.GPURelocatableDeviceCode)
+      else if (SemaRef.LangOpts.GPURelocatableDeviceCode &&
+               (!FD || (!FD->getDescribedFunctionTemplate() &&
+                        SemaRef.getASTContext().GetGVALinkageForFunction(FD) ==
+                            GVA_StrongExternal)))
         SemaRef.getASTContext().CUDAExternalDeviceDeclODRUsedByHost.insert(Var);
     }
   }
diff --git a/clang/test/CodeGenCUDA/host-used-extern.cu b/clang/test/CodeGenCUDA/host-used-extern.cu
index e8f8e12aad47..1ae644ae981a 100644
--- a/clang/test/CodeGenCUDA/host-used-extern.cu
+++ b/clang/test/CodeGenCUDA/host-used-extern.cu
@@ -24,6 +24,7 @@
 
 // NEG-NOT: @__clang_gpu_used_external = {{.*}} @_Z7kernel2v
 // NEG-NOT: @__clang_gpu_used_external = {{.*}} @_Z7kernel3v
+// NEG-NOT: @__clang_gpu_used_external = {{.*}} @_Z7kernel5v
 // NEG-NOT: @__clang_gpu_used_external = {{.*}} @var2
 // NEG-NOT: @__clang_gpu_used_external = {{.*}} @var3
 // NEG-NOT: @__clang_gpu_used_external = {{.*}} @ext_shvar
@@ -44,6 +45,10 @@ __global__ void kernel3();
 // kernel4 is marked as used even though it is not called.
 __global__ void kernel4();
 
+// kernel5 is not marked as used since it is called by host function
+// with weak_odr linkage, which may be dropped by linker.
+__global__ void kernel5();
+
 extern __device__ int var1;
 
 __device__ int var2;
@@ -67,3 +72,11 @@ __global__ void test_lambda_using_extern_shared() {
   };
   lambda();
 }
+
+template
+void template_caller() {
+  kernel5<<<1, 1>>>();
+  var1 = 1;
+}
+
+template void template_caller();
-- 
GitLab


From 5620542e4d19f794a0d6bd1bd1861bcedd7a788a Mon Sep 17 00:00:00 2001
From: Antonio Frighetto 
Date: Fri, 8 Mar 2024 16:32:03 +0100
Subject: [PATCH 0045/1674] [IR] Add `getNoWrapKind` method for
 `OverflowingBinaryOperator` (NFC)

---
 llvm/include/llvm/IR/Operator.h     | 12 ++++++++++++
 llvm/lib/Analysis/LazyValueInfo.cpp |  7 +------
 2 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/llvm/include/llvm/IR/Operator.h b/llvm/include/llvm/IR/Operator.h
index 7168128648d8..e52649833901 100644
--- a/llvm/include/llvm/IR/Operator.h
+++ b/llvm/include/llvm/IR/Operator.h
@@ -109,6 +109,18 @@ public:
     return (SubclassOptionalData & NoSignedWrap) != 0;
   }
 
+  /// Returns the no-wrap kind of the operation.
+  unsigned getNoWrapKind() const {
+    unsigned NoWrapKind = 0;
+    if (hasNoUnsignedWrap())
+      NoWrapKind |= NoUnsignedWrap;
+
+    if (hasNoSignedWrap())
+      NoWrapKind |= NoSignedWrap;
+
+    return NoWrapKind;
+  }
+
   static bool classof(const Instruction *I) {
     return I->getOpcode() == Instruction::Add ||
            I->getOpcode() == Instruction::Sub ||
diff --git a/llvm/lib/Analysis/LazyValueInfo.cpp b/llvm/lib/Analysis/LazyValueInfo.cpp
index b948eb6ebd12..9ae31d165235 100644
--- a/llvm/lib/Analysis/LazyValueInfo.cpp
+++ b/llvm/lib/Analysis/LazyValueInfo.cpp
@@ -997,12 +997,7 @@ LazyValueInfoImpl::solveBlockValueBinaryOp(BinaryOperator *BO, BasicBlock *BB) {
   assert(BO->getOperand(0)->getType()->isSized() &&
          "all operands to binary operators are sized");
   if (auto *OBO = dyn_cast(BO)) {
-    unsigned NoWrapKind = 0;
-    if (OBO->hasNoUnsignedWrap())
-      NoWrapKind |= OverflowingBinaryOperator::NoUnsignedWrap;
-    if (OBO->hasNoSignedWrap())
-      NoWrapKind |= OverflowingBinaryOperator::NoSignedWrap;
-
+    unsigned NoWrapKind = OBO->getNoWrapKind();
     return solveBlockValueBinaryOpImpl(
         BO, BB,
         [BO, NoWrapKind](const ConstantRange &CR1, const ConstantRange &CR2) {
-- 
GitLab


From 9d5fdad7e5002dbe087bbf0a78ffe9423b6aab56 Mon Sep 17 00:00:00 2001
From: Louis Dionne 
Date: Fri, 8 Mar 2024 11:03:09 -0500
Subject: [PATCH 0046/1674] [libc++] Disable module_std and module_std_compat
 tests

Those have been crippling the CI for over a week now. This is the only
solution I see until we gain a better understanding of why they fail.
Otherwise all other patches are blocked on these spurious failures in
the stage1 of the CI.
---
 libcxx/test/libcxx/module_std.gen.py        | 3 +++
 libcxx/test/libcxx/module_std_compat.gen.py | 3 +++
 2 files changed, 6 insertions(+)

diff --git a/libcxx/test/libcxx/module_std.gen.py b/libcxx/test/libcxx/module_std.gen.py
index fc23985caf30..932af742640c 100644
--- a/libcxx/test/libcxx/module_std.gen.py
+++ b/libcxx/test/libcxx/module_std.gen.py
@@ -18,6 +18,9 @@
 
 # RUN: %{python} %s %{libcxx-dir}/utils
 
+# TODO: Re-enable this test once we understand why it keeps timing out.
+# UNSUPPORTED: clang
+
 import sys
 
 sys.path.append(sys.argv[1])
diff --git a/libcxx/test/libcxx/module_std_compat.gen.py b/libcxx/test/libcxx/module_std_compat.gen.py
index 000aa2998612..d29f490ab135 100644
--- a/libcxx/test/libcxx/module_std_compat.gen.py
+++ b/libcxx/test/libcxx/module_std_compat.gen.py
@@ -18,6 +18,9 @@
 
 # RUN: %{python} %s %{libcxx-dir}/utils
 
+# TODO: Re-enable this test once we understand why it keeps timing out.
+# UNSUPPORTED: clang
+
 import sys
 
 sys.path.append(sys.argv[1])
-- 
GitLab


From 0456a32a2a622a73e01b01c65c21973ac5a689fc Mon Sep 17 00:00:00 2001
From: Jay Foad 
Date: Fri, 8 Mar 2024 16:08:09 +0000
Subject: [PATCH 0047/1674] [AMDGPU] Simplify renamed BUF instruction
 definitions. NFC. (#84503)

Use optional arguments instead of separate (multi)classes for renamed
instructions.
---
 llvm/lib/Target/AMDGPU/BUFInstructions.td | 239 +++++++++++-----------
 1 file changed, 114 insertions(+), 125 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/BUFInstructions.td b/llvm/lib/Target/AMDGPU/BUFInstructions.td
index 9c6934865bfa..dedf381afca8 100644
--- a/llvm/lib/Target/AMDGPU/BUFInstructions.td
+++ b/llvm/lib/Target/AMDGPU/BUFInstructions.td
@@ -2466,51 +2466,50 @@ class Mnem_gfx11  :
 class Mnem_gfx12  :
   MnemonicAlias, Requires<[isGFX12Plus]>;
 
-multiclass MUBUF_Real_AllAddr_gfx11_Renamed_Impl2 op, string real_name> {
+multiclass MUBUF_Real_AllAddr_gfx11_Impl2 op, string real_name> {
   defm _BOTHEN : MUBUF_Real_gfx11;
   defm _IDXEN  : MUBUF_Real_gfx11;
   defm _OFFEN  : MUBUF_Real_gfx11;
   defm _OFFSET : MUBUF_Real_gfx11;
 }
 
-multiclass MUBUF_Real_AllAddr_gfx12_Renamed_Impl2 op, string real_name> {
+multiclass MUBUF_Real_AllAddr_gfx12_Impl2 op, string real_name> {
   defm _VBUFFER_BOTHEN : VBUFFER_MUBUF_Real_gfx12;
   defm _VBUFFER_IDXEN  : VBUFFER_MUBUF_Real_gfx12;
   defm _VBUFFER_OFFEN  : VBUFFER_MUBUF_Real_gfx12;
   defm _VBUFFER_OFFSET : VBUFFER_MUBUF_Real_gfx12;
 }
 
-multiclass MUBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl2 op, string real_name> :
-  MUBUF_Real_AllAddr_gfx11_Renamed_Impl2,
-  MUBUF_Real_AllAddr_gfx12_Renamed_Impl2;
+multiclass MUBUF_Real_AllAddr_gfx11_gfx12_Impl2 op, string real_name> :
+  MUBUF_Real_AllAddr_gfx11_Impl2,
+  MUBUF_Real_AllAddr_gfx12_Impl2;
 
-multiclass MUBUF_Real_AllAddr_gfx11_Renamed_Impl op, string real_name,
+multiclass MUBUF_Real_AllAddr_gfx11_Impl op, string real_name,
                                                  bit hasTFE = 1> {
-  defm NAME : MUBUF_Real_AllAddr_gfx11_Renamed_Impl2;
+  defm NAME : MUBUF_Real_AllAddr_gfx11_Impl2;
   if hasTFE then
-    defm _TFE : MUBUF_Real_AllAddr_gfx11_Renamed_Impl2;
+    defm _TFE : MUBUF_Real_AllAddr_gfx11_Impl2;
 }
 
-multiclass MUBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl op, string real_name,
+multiclass MUBUF_Real_AllAddr_gfx11_gfx12_Impl op, string real_name,
                                                  bit hasTFE = 1> {
-  defm NAME : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl2;
+  defm NAME : MUBUF_Real_AllAddr_gfx11_gfx12_Impl2;
   if hasTFE then
-    defm _TFE : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl2;
+    defm _TFE : MUBUF_Real_AllAddr_gfx11_gfx12_Impl2;
 }
 
 // Non-renamed, non-atomic gfx11/gfx12 mubuf instructions.
 multiclass MUBUF_Real_AllAddr_gfx11 op, bit hasTFE = 1> :
-  MUBUF_Real_AllAddr_gfx11_Renamed_Impl.Mnemonic, hasTFE>;
+  MUBUF_Real_AllAddr_gfx11_Impl.Mnemonic, hasTFE>;
 
-multiclass MUBUF_Real_AllAddr_gfx11_gfx12 op, bit hasTFE = 1> :
-  MUBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl.Mnemonic, hasTFE>;
-
-multiclass MUBUF_Real_AllAddr_gfx11_gfx12_Renamed op, string real_name> :
-  MUBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl {
-  def : Mnem_gfx11_gfx12.Mnemonic, real_name>;
+multiclass MUBUF_Real_AllAddr_gfx11_gfx12 op, string real_name = !tolower(NAME)> :
+  MUBUF_Real_AllAddr_gfx11_gfx12_Impl {
+  defvar ps = get_BUF_ps;
+  if !ne(ps.Mnemonic, real_name) then
+    def : Mnem_gfx11_gfx12;
 }
 
-multiclass MUBUF_Real_Atomic_gfx11_Renamed_impl op, bit is_return,
+multiclass MUBUF_Real_Atomic_gfx11_impl op, bit is_return,
                                                 string real_name> {
   defvar Rtn = !if(is_return, "_RTN", "");
   defm _BOTHEN#Rtn : MUBUF_Real_gfx11;
@@ -2519,7 +2518,7 @@ multiclass MUBUF_Real_Atomic_gfx11_Renamed_impl op, bit is_return,
   defm _OFFSET#Rtn : MUBUF_Real_gfx11;
 }
 
-multiclass MUBUF_Real_Atomic_gfx12_Renamed_impl op, bit is_return,
+multiclass MUBUF_Real_Atomic_gfx12_impl op, bit is_return,
                                                 string real_name> {
   defvar Rtn = !if(is_return, "_RTN", "");
   defm _VBUFFER_BOTHEN#Rtn : VBUFFER_MUBUF_Real_gfx12;
@@ -2528,124 +2527,117 @@ multiclass MUBUF_Real_Atomic_gfx12_Renamed_impl op, bit is_return,
   defm _VBUFFER_OFFSET#Rtn : VBUFFER_MUBUF_Real_gfx12;
 }
 
-multiclass MUBUF_Real_Atomic_gfx11_gfx12_Renamed_impl op, bit is_return,
+multiclass MUBUF_Real_Atomic_gfx11_gfx12_impl op, bit is_return,
                                                 string real_name> :
-  MUBUF_Real_Atomic_gfx11_Renamed_impl,
-  MUBUF_Real_Atomic_gfx12_Renamed_impl;
-
-// Non-renamed gfx11/gfx12 mubuf atomic.
-multiclass MUBUF_Real_Atomic_gfx11_gfx12 op> :
-  MUBUF_Real_Atomic_gfx11_gfx12_Renamed_impl.Mnemonic>,
-  MUBUF_Real_Atomic_gfx11_gfx12_Renamed_impl.Mnemonic>;
+  MUBUF_Real_Atomic_gfx11_impl,
+  MUBUF_Real_Atomic_gfx12_impl;
 
 multiclass MUBUF_Real_Atomic_gfx12 op> :
-  MUBUF_Real_Atomic_gfx12_Renamed_impl.Mnemonic>,
-  MUBUF_Real_Atomic_gfx12_Renamed_impl.Mnemonic>;
-
-multiclass MUBUF_Real_Atomic_gfx11_Renamed op, string real_name> :
-  MUBUF_Real_Atomic_gfx11_Renamed_impl,
-  MUBUF_Real_Atomic_gfx11_Renamed_impl {
-  def : Mnem_gfx11_gfx12.Mnemonic, real_name>;
-}
+  MUBUF_Real_Atomic_gfx12_impl.Mnemonic>,
+  MUBUF_Real_Atomic_gfx12_impl.Mnemonic>;
 
-multiclass MUBUF_Real_Atomic_gfx11_gfx12_Renamed op, string real_name> :
-  MUBUF_Real_Atomic_gfx11_gfx12_Renamed_impl,
-  MUBUF_Real_Atomic_gfx11_gfx12_Renamed_impl  {
+multiclass MUBUF_Real_Atomic_gfx11 op, string real_name> :
+  MUBUF_Real_Atomic_gfx11_impl,
+  MUBUF_Real_Atomic_gfx11_impl {
   def : Mnem_gfx11_gfx12.Mnemonic, real_name>;
 }
 
-multiclass MUBUF_Real_Atomic_gfx11_gfx12_Renamed_gfx12_Renamed op, string gfx12_name, string gfx11_name> :
-  MUBUF_Real_Atomic_gfx11_Renamed_impl,
-  MUBUF_Real_Atomic_gfx11_Renamed_impl,
-  MUBUF_Real_Atomic_gfx12_Renamed_impl,
-  MUBUF_Real_Atomic_gfx12_Renamed_impl {
-  def : Mnem_gfx11.Mnemonic, gfx11_name>;
-  def : Mnem_gfx12.Mnemonic, gfx12_name>;
-  def : Mnem_gfx12;
+multiclass MUBUF_Real_Atomic_gfx11_gfx12 op, string gfx12_name = !tolower(NAME), string gfx11_name = gfx12_name> :
+  MUBUF_Real_Atomic_gfx11_impl,
+  MUBUF_Real_Atomic_gfx11_impl,
+  MUBUF_Real_Atomic_gfx12_impl,
+  MUBUF_Real_Atomic_gfx12_impl {
+  defvar ps = get_BUF_ps;
+  if !ne(ps.Mnemonic, gfx11_name) then
+    def : Mnem_gfx11;
+  if !ne(ps.Mnemonic, gfx12_name) then
+    def : Mnem_gfx12;
+  if !ne(gfx11_name, gfx12_name) then
+    def : Mnem_gfx12;
 }
 
 defm BUFFER_GL0_INV               : MUBUF_Real_gfx11<0x02B>;
 defm BUFFER_GL1_INV               : MUBUF_Real_gfx11<0x02C>;
 
-defm BUFFER_LOAD_DWORD            : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x014, "buffer_load_b32">;
-defm BUFFER_LOAD_DWORDX2          : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x015, "buffer_load_b64">;
-defm BUFFER_LOAD_DWORDX3          : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x016, "buffer_load_b96">;
-defm BUFFER_LOAD_DWORDX4          : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x017, "buffer_load_b128">;
-defm BUFFER_LOAD_SHORT_D16        : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x020, "buffer_load_d16_b16">;
-defm BUFFER_LOAD_FORMAT_D16_X     : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x008, "buffer_load_d16_format_x">;
-defm BUFFER_LOAD_FORMAT_D16_XY    : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x009, "buffer_load_d16_format_xy">;
-defm BUFFER_LOAD_FORMAT_D16_XYZ   : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00a, "buffer_load_d16_format_xyz">;
-defm BUFFER_LOAD_FORMAT_D16_XYZW  : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00b, "buffer_load_d16_format_xyzw">;
-defm BUFFER_LOAD_SHORT_D16_HI     : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x023, "buffer_load_d16_hi_b16">;
-defm BUFFER_LOAD_FORMAT_D16_HI_X  : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x026, "buffer_load_d16_hi_format_x">;
-defm BUFFER_LOAD_SBYTE_D16_HI     : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x022, "buffer_load_d16_hi_i8">;
-defm BUFFER_LOAD_UBYTE_D16_HI     : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x021, "buffer_load_d16_hi_u8">;
-defm BUFFER_LOAD_SBYTE_D16        : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x01f, "buffer_load_d16_i8">;
-defm BUFFER_LOAD_UBYTE_D16        : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x01e, "buffer_load_d16_u8">;
+defm BUFFER_LOAD_DWORD            : MUBUF_Real_AllAddr_gfx11_gfx12<0x014, "buffer_load_b32">;
+defm BUFFER_LOAD_DWORDX2          : MUBUF_Real_AllAddr_gfx11_gfx12<0x015, "buffer_load_b64">;
+defm BUFFER_LOAD_DWORDX3          : MUBUF_Real_AllAddr_gfx11_gfx12<0x016, "buffer_load_b96">;
+defm BUFFER_LOAD_DWORDX4          : MUBUF_Real_AllAddr_gfx11_gfx12<0x017, "buffer_load_b128">;
+defm BUFFER_LOAD_SHORT_D16        : MUBUF_Real_AllAddr_gfx11_gfx12<0x020, "buffer_load_d16_b16">;
+defm BUFFER_LOAD_FORMAT_D16_X     : MUBUF_Real_AllAddr_gfx11_gfx12<0x008, "buffer_load_d16_format_x">;
+defm BUFFER_LOAD_FORMAT_D16_XY    : MUBUF_Real_AllAddr_gfx11_gfx12<0x009, "buffer_load_d16_format_xy">;
+defm BUFFER_LOAD_FORMAT_D16_XYZ   : MUBUF_Real_AllAddr_gfx11_gfx12<0x00a, "buffer_load_d16_format_xyz">;
+defm BUFFER_LOAD_FORMAT_D16_XYZW  : MUBUF_Real_AllAddr_gfx11_gfx12<0x00b, "buffer_load_d16_format_xyzw">;
+defm BUFFER_LOAD_SHORT_D16_HI     : MUBUF_Real_AllAddr_gfx11_gfx12<0x023, "buffer_load_d16_hi_b16">;
+defm BUFFER_LOAD_FORMAT_D16_HI_X  : MUBUF_Real_AllAddr_gfx11_gfx12<0x026, "buffer_load_d16_hi_format_x">;
+defm BUFFER_LOAD_SBYTE_D16_HI     : MUBUF_Real_AllAddr_gfx11_gfx12<0x022, "buffer_load_d16_hi_i8">;
+defm BUFFER_LOAD_UBYTE_D16_HI     : MUBUF_Real_AllAddr_gfx11_gfx12<0x021, "buffer_load_d16_hi_u8">;
+defm BUFFER_LOAD_SBYTE_D16        : MUBUF_Real_AllAddr_gfx11_gfx12<0x01f, "buffer_load_d16_i8">;
+defm BUFFER_LOAD_UBYTE_D16        : MUBUF_Real_AllAddr_gfx11_gfx12<0x01e, "buffer_load_d16_u8">;
 defm BUFFER_LOAD_FORMAT_X         : MUBUF_Real_AllAddr_gfx11_gfx12<0x000>;
 defm BUFFER_LOAD_FORMAT_XY        : MUBUF_Real_AllAddr_gfx11_gfx12<0x001>;
 defm BUFFER_LOAD_FORMAT_XYZ       : MUBUF_Real_AllAddr_gfx11_gfx12<0x002>;
 defm BUFFER_LOAD_FORMAT_XYZW      : MUBUF_Real_AllAddr_gfx11_gfx12<0x003>;
-defm BUFFER_LOAD_SBYTE            : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x011, "buffer_load_i8">;
-defm BUFFER_LOAD_SSHORT           : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x013, "buffer_load_i16">;
-defm BUFFER_LOAD_UBYTE            : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x010, "buffer_load_u8">;
-defm BUFFER_LOAD_USHORT           : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x012, "buffer_load_u16">;
+defm BUFFER_LOAD_SBYTE            : MUBUF_Real_AllAddr_gfx11_gfx12<0x011, "buffer_load_i8">;
+defm BUFFER_LOAD_SSHORT           : MUBUF_Real_AllAddr_gfx11_gfx12<0x013, "buffer_load_i16">;
+defm BUFFER_LOAD_UBYTE            : MUBUF_Real_AllAddr_gfx11_gfx12<0x010, "buffer_load_u8">;
+defm BUFFER_LOAD_USHORT           : MUBUF_Real_AllAddr_gfx11_gfx12<0x012, "buffer_load_u16">;
 defm BUFFER_LOAD_LDS_B32          : MUBUF_Real_AllAddr_gfx11<0x031, 0>;
 defm BUFFER_LOAD_LDS_FORMAT_X     : MUBUF_Real_AllAddr_gfx11<0x032, 0>;
 defm BUFFER_LOAD_LDS_I8           : MUBUF_Real_AllAddr_gfx11<0x02e, 0>;
 defm BUFFER_LOAD_LDS_I16          : MUBUF_Real_AllAddr_gfx11<0x030, 0>;
 defm BUFFER_LOAD_LDS_U8           : MUBUF_Real_AllAddr_gfx11<0x02d, 0>;
 defm BUFFER_LOAD_LDS_U16          : MUBUF_Real_AllAddr_gfx11<0x02f, 0>;
-defm BUFFER_STORE_BYTE            : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x018, "buffer_store_b8">;
-defm BUFFER_STORE_SHORT           : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x019, "buffer_store_b16">;
-defm BUFFER_STORE_DWORD           : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x01A, "buffer_store_b32">;
-defm BUFFER_STORE_DWORDX2         : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x01B, "buffer_store_b64">;
-defm BUFFER_STORE_DWORDX3         : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x01C, "buffer_store_b96">;
-defm BUFFER_STORE_DWORDX4         : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x01D, "buffer_store_b128">;
-defm BUFFER_STORE_FORMAT_D16_X    : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00C, "buffer_store_d16_format_x">;
-defm BUFFER_STORE_FORMAT_D16_XY   : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00D, "buffer_store_d16_format_xy">;
-defm BUFFER_STORE_FORMAT_D16_XYZ  : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00E, "buffer_store_d16_format_xyz">;
-defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00F, "buffer_store_d16_format_xyzw">;
-defm BUFFER_STORE_BYTE_D16_HI     : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x024, "buffer_store_d16_hi_b8">;
-defm BUFFER_STORE_SHORT_D16_HI    : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x025, "buffer_store_d16_hi_b16">;
-defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x027, "buffer_store_d16_hi_format_x">;
+defm BUFFER_STORE_BYTE            : MUBUF_Real_AllAddr_gfx11_gfx12<0x018, "buffer_store_b8">;
+defm BUFFER_STORE_SHORT           : MUBUF_Real_AllAddr_gfx11_gfx12<0x019, "buffer_store_b16">;
+defm BUFFER_STORE_DWORD           : MUBUF_Real_AllAddr_gfx11_gfx12<0x01A, "buffer_store_b32">;
+defm BUFFER_STORE_DWORDX2         : MUBUF_Real_AllAddr_gfx11_gfx12<0x01B, "buffer_store_b64">;
+defm BUFFER_STORE_DWORDX3         : MUBUF_Real_AllAddr_gfx11_gfx12<0x01C, "buffer_store_b96">;
+defm BUFFER_STORE_DWORDX4         : MUBUF_Real_AllAddr_gfx11_gfx12<0x01D, "buffer_store_b128">;
+defm BUFFER_STORE_FORMAT_D16_X    : MUBUF_Real_AllAddr_gfx11_gfx12<0x00C, "buffer_store_d16_format_x">;
+defm BUFFER_STORE_FORMAT_D16_XY   : MUBUF_Real_AllAddr_gfx11_gfx12<0x00D, "buffer_store_d16_format_xy">;
+defm BUFFER_STORE_FORMAT_D16_XYZ  : MUBUF_Real_AllAddr_gfx11_gfx12<0x00E, "buffer_store_d16_format_xyz">;
+defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx11_gfx12<0x00F, "buffer_store_d16_format_xyzw">;
+defm BUFFER_STORE_BYTE_D16_HI     : MUBUF_Real_AllAddr_gfx11_gfx12<0x024, "buffer_store_d16_hi_b8">;
+defm BUFFER_STORE_SHORT_D16_HI    : MUBUF_Real_AllAddr_gfx11_gfx12<0x025, "buffer_store_d16_hi_b16">;
+defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx11_gfx12<0x027, "buffer_store_d16_hi_format_x">;
 defm BUFFER_STORE_FORMAT_X        : MUBUF_Real_AllAddr_gfx11_gfx12<0x004>;
 defm BUFFER_STORE_FORMAT_XY       : MUBUF_Real_AllAddr_gfx11_gfx12<0x005>;
 defm BUFFER_STORE_FORMAT_XYZ      : MUBUF_Real_AllAddr_gfx11_gfx12<0x006>;
 defm BUFFER_STORE_FORMAT_XYZW     : MUBUF_Real_AllAddr_gfx11_gfx12<0x007>;
 defm BUFFER_ATOMIC_ADD_F32        : MUBUF_Real_Atomic_gfx11_gfx12<0x056>;
-defm BUFFER_ATOMIC_ADD            : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x035, "buffer_atomic_add_u32">;
-defm BUFFER_ATOMIC_ADD_X2         : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x043, "buffer_atomic_add_u64">;
-defm BUFFER_ATOMIC_AND            : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x03C, "buffer_atomic_and_b32">;
-defm BUFFER_ATOMIC_AND_X2         : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x049, "buffer_atomic_and_b64">;
-defm BUFFER_ATOMIC_CMPSWAP        : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x034, "buffer_atomic_cmpswap_b32">;
-defm BUFFER_ATOMIC_CMPSWAP_X2     : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x042, "buffer_atomic_cmpswap_b64">;
-defm BUFFER_ATOMIC_FCMPSWAP       : MUBUF_Real_Atomic_gfx11_Renamed<0x050, "buffer_atomic_cmpswap_f32">;
+defm BUFFER_ATOMIC_ADD            : MUBUF_Real_Atomic_gfx11_gfx12<0x035, "buffer_atomic_add_u32">;
+defm BUFFER_ATOMIC_ADD_X2         : MUBUF_Real_Atomic_gfx11_gfx12<0x043, "buffer_atomic_add_u64">;
+defm BUFFER_ATOMIC_AND            : MUBUF_Real_Atomic_gfx11_gfx12<0x03C, "buffer_atomic_and_b32">;
+defm BUFFER_ATOMIC_AND_X2         : MUBUF_Real_Atomic_gfx11_gfx12<0x049, "buffer_atomic_and_b64">;
+defm BUFFER_ATOMIC_CMPSWAP        : MUBUF_Real_Atomic_gfx11_gfx12<0x034, "buffer_atomic_cmpswap_b32">;
+defm BUFFER_ATOMIC_CMPSWAP_X2     : MUBUF_Real_Atomic_gfx11_gfx12<0x042, "buffer_atomic_cmpswap_b64">;
+defm BUFFER_ATOMIC_FCMPSWAP       : MUBUF_Real_Atomic_gfx11<0x050, "buffer_atomic_cmpswap_f32">;
 defm BUFFER_ATOMIC_COND_SUB_U32   : MUBUF_Real_Atomic_gfx12<0x050>;
-defm BUFFER_ATOMIC_CSUB           : MUBUF_Real_Atomic_gfx11_gfx12_Renamed_gfx12_Renamed<0x037, "buffer_atomic_sub_clamp_u32", "buffer_atomic_csub_u32">;
+defm BUFFER_ATOMIC_CSUB           : MUBUF_Real_Atomic_gfx11_gfx12<0x037, "buffer_atomic_sub_clamp_u32", "buffer_atomic_csub_u32">;
 def : Mnem_gfx11_gfx12<"buffer_atomic_csub", "buffer_atomic_csub_u32">;
-defm BUFFER_ATOMIC_DEC            : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x040, "buffer_atomic_dec_u32">;
-defm BUFFER_ATOMIC_DEC_X2         : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x04D, "buffer_atomic_dec_u64">;
-defm BUFFER_ATOMIC_INC            : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x03F, "buffer_atomic_inc_u32">;
-defm BUFFER_ATOMIC_INC_X2         : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x04C, "buffer_atomic_inc_u64">;
-defm BUFFER_ATOMIC_FMAX           : MUBUF_Real_Atomic_gfx11_gfx12_Renamed_gfx12_Renamed<0x052, "buffer_atomic_max_num_f32", "buffer_atomic_max_f32">;
-defm BUFFER_ATOMIC_SMAX           : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x03A, "buffer_atomic_max_i32">;
-defm BUFFER_ATOMIC_SMAX_X2        : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x047, "buffer_atomic_max_i64">;
-defm BUFFER_ATOMIC_UMAX           : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x03B, "buffer_atomic_max_u32">;
-defm BUFFER_ATOMIC_UMAX_X2        : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x048, "buffer_atomic_max_u64">;
-defm BUFFER_ATOMIC_FMIN           : MUBUF_Real_Atomic_gfx11_gfx12_Renamed_gfx12_Renamed<0x051, "buffer_atomic_min_num_f32", "buffer_atomic_min_f32">;
-defm BUFFER_ATOMIC_SMIN           : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x038, "buffer_atomic_min_i32">;
-defm BUFFER_ATOMIC_SMIN_X2        : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x045, "buffer_atomic_min_i64">;
-defm BUFFER_ATOMIC_UMIN           : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x039, "buffer_atomic_min_u32">;
-defm BUFFER_ATOMIC_UMIN_X2        : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x046, "buffer_atomic_min_u64">;
-defm BUFFER_ATOMIC_OR             : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x03D, "buffer_atomic_or_b32">;
-defm BUFFER_ATOMIC_OR_X2          : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x04A, "buffer_atomic_or_b64">;
-defm BUFFER_ATOMIC_SUB            : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x036, "buffer_atomic_sub_u32">;
-defm BUFFER_ATOMIC_SUB_X2         : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x044, "buffer_atomic_sub_u64">;
-defm BUFFER_ATOMIC_SWAP           : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x033, "buffer_atomic_swap_b32">;
-defm BUFFER_ATOMIC_SWAP_X2        : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x041, "buffer_atomic_swap_b64">;
-defm BUFFER_ATOMIC_XOR            : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x03E, "buffer_atomic_xor_b32">;
-defm BUFFER_ATOMIC_XOR_X2         : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x04B, "buffer_atomic_xor_b64">;
+defm BUFFER_ATOMIC_DEC            : MUBUF_Real_Atomic_gfx11_gfx12<0x040, "buffer_atomic_dec_u32">;
+defm BUFFER_ATOMIC_DEC_X2         : MUBUF_Real_Atomic_gfx11_gfx12<0x04D, "buffer_atomic_dec_u64">;
+defm BUFFER_ATOMIC_INC            : MUBUF_Real_Atomic_gfx11_gfx12<0x03F, "buffer_atomic_inc_u32">;
+defm BUFFER_ATOMIC_INC_X2         : MUBUF_Real_Atomic_gfx11_gfx12<0x04C, "buffer_atomic_inc_u64">;
+defm BUFFER_ATOMIC_FMAX           : MUBUF_Real_Atomic_gfx11_gfx12<0x052, "buffer_atomic_max_num_f32", "buffer_atomic_max_f32">;
+defm BUFFER_ATOMIC_SMAX           : MUBUF_Real_Atomic_gfx11_gfx12<0x03A, "buffer_atomic_max_i32">;
+defm BUFFER_ATOMIC_SMAX_X2        : MUBUF_Real_Atomic_gfx11_gfx12<0x047, "buffer_atomic_max_i64">;
+defm BUFFER_ATOMIC_UMAX           : MUBUF_Real_Atomic_gfx11_gfx12<0x03B, "buffer_atomic_max_u32">;
+defm BUFFER_ATOMIC_UMAX_X2        : MUBUF_Real_Atomic_gfx11_gfx12<0x048, "buffer_atomic_max_u64">;
+defm BUFFER_ATOMIC_FMIN           : MUBUF_Real_Atomic_gfx11_gfx12<0x051, "buffer_atomic_min_num_f32", "buffer_atomic_min_f32">;
+defm BUFFER_ATOMIC_SMIN           : MUBUF_Real_Atomic_gfx11_gfx12<0x038, "buffer_atomic_min_i32">;
+defm BUFFER_ATOMIC_SMIN_X2        : MUBUF_Real_Atomic_gfx11_gfx12<0x045, "buffer_atomic_min_i64">;
+defm BUFFER_ATOMIC_UMIN           : MUBUF_Real_Atomic_gfx11_gfx12<0x039, "buffer_atomic_min_u32">;
+defm BUFFER_ATOMIC_UMIN_X2        : MUBUF_Real_Atomic_gfx11_gfx12<0x046, "buffer_atomic_min_u64">;
+defm BUFFER_ATOMIC_OR             : MUBUF_Real_Atomic_gfx11_gfx12<0x03D, "buffer_atomic_or_b32">;
+defm BUFFER_ATOMIC_OR_X2          : MUBUF_Real_Atomic_gfx11_gfx12<0x04A, "buffer_atomic_or_b64">;
+defm BUFFER_ATOMIC_SUB            : MUBUF_Real_Atomic_gfx11_gfx12<0x036, "buffer_atomic_sub_u32">;
+defm BUFFER_ATOMIC_SUB_X2         : MUBUF_Real_Atomic_gfx11_gfx12<0x044, "buffer_atomic_sub_u64">;
+defm BUFFER_ATOMIC_SWAP           : MUBUF_Real_Atomic_gfx11_gfx12<0x033, "buffer_atomic_swap_b32">;
+defm BUFFER_ATOMIC_SWAP_X2        : MUBUF_Real_Atomic_gfx11_gfx12<0x041, "buffer_atomic_swap_b64">;
+defm BUFFER_ATOMIC_XOR            : MUBUF_Real_Atomic_gfx11_gfx12<0x03E, "buffer_atomic_xor_b32">;
+defm BUFFER_ATOMIC_XOR_X2         : MUBUF_Real_Atomic_gfx11_gfx12<0x04B, "buffer_atomic_xor_b64">;
 defm BUFFER_ATOMIC_PK_ADD_F16     : MUBUF_Real_Atomic_gfx12<0x059>;
 defm BUFFER_ATOMIC_PK_ADD_BF16    : MUBUF_Real_Atomic_gfx12<0x05a>;
 
@@ -2883,7 +2875,7 @@ class Base_MTBUF_Real_gfx6_gfx7_gfx10 op, MTBUF_Pseudo ps, int ef> :
 // MTBUF - GFX11.
 //===----------------------------------------------------------------------===//
 
-multiclass MTBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl op, string real_name> {
+multiclass MTBUF_Real_AllAddr_gfx11_gfx12_Impl op, string real_name> {
   defm _BOTHEN : MTBUF_Real_gfx11;
   defm _IDXEN  : MTBUF_Real_gfx11;
   defm _OFFEN  : MTBUF_Real_gfx11;
@@ -2895,28 +2887,25 @@ multiclass MTBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl op, string real_n
   defm _VBUFFER_OFFSET : VBUFFER_MTBUF_Real_gfx12;
 }
 
-multiclass MTBUF_Real_AllAddr_gfx11_gfx12 op>
- : MTBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl.Mnemonic>;
-
-
-multiclass MTBUF_Real_AllAddr_gfx11_gfx12_Renamed op, string real_name>
-  : MTBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl {
+multiclass MTBUF_Real_AllAddr_gfx11_gfx12 op, string real_name = !tolower(NAME)>
+  : MTBUF_Real_AllAddr_gfx11_gfx12_Impl {
   defvar ps = get_BUF_ps;
-  def : Mnem_gfx11_gfx12;
+  if !ne(ps.Mnemonic, real_name) then
+    def : Mnem_gfx11_gfx12;
 }
 
-defm TBUFFER_LOAD_FORMAT_D16_X     : MTBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x008, "tbuffer_load_d16_format_x">;
-defm TBUFFER_LOAD_FORMAT_D16_XY    : MTBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x009, "tbuffer_load_d16_format_xy">;
-defm TBUFFER_LOAD_FORMAT_D16_XYZ   : MTBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00a, "tbuffer_load_d16_format_xyz">;
-defm TBUFFER_LOAD_FORMAT_D16_XYZW  : MTBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00b, "tbuffer_load_d16_format_xyzw">;
+defm TBUFFER_LOAD_FORMAT_D16_X     : MTBUF_Real_AllAddr_gfx11_gfx12<0x008, "tbuffer_load_d16_format_x">;
+defm TBUFFER_LOAD_FORMAT_D16_XY    : MTBUF_Real_AllAddr_gfx11_gfx12<0x009, "tbuffer_load_d16_format_xy">;
+defm TBUFFER_LOAD_FORMAT_D16_XYZ   : MTBUF_Real_AllAddr_gfx11_gfx12<0x00a, "tbuffer_load_d16_format_xyz">;
+defm TBUFFER_LOAD_FORMAT_D16_XYZW  : MTBUF_Real_AllAddr_gfx11_gfx12<0x00b, "tbuffer_load_d16_format_xyzw">;
 defm TBUFFER_LOAD_FORMAT_X         : MTBUF_Real_AllAddr_gfx11_gfx12<0x000>;
 defm TBUFFER_LOAD_FORMAT_XY        : MTBUF_Real_AllAddr_gfx11_gfx12<0x001>;
 defm TBUFFER_LOAD_FORMAT_XYZ       : MTBUF_Real_AllAddr_gfx11_gfx12<0x002>;
 defm TBUFFER_LOAD_FORMAT_XYZW      : MTBUF_Real_AllAddr_gfx11_gfx12<0x003>;
-defm TBUFFER_STORE_FORMAT_D16_X    : MTBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00c, "tbuffer_store_d16_format_x">;
-defm TBUFFER_STORE_FORMAT_D16_XY   : MTBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00d, "tbuffer_store_d16_format_xy">;
-defm TBUFFER_STORE_FORMAT_D16_XYZ  : MTBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00e, "tbuffer_store_d16_format_xyz">;
-defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00f, "tbuffer_store_d16_format_xyzw">;
+defm TBUFFER_STORE_FORMAT_D16_X    : MTBUF_Real_AllAddr_gfx11_gfx12<0x00c, "tbuffer_store_d16_format_x">;
+defm TBUFFER_STORE_FORMAT_D16_XY   : MTBUF_Real_AllAddr_gfx11_gfx12<0x00d, "tbuffer_store_d16_format_xy">;
+defm TBUFFER_STORE_FORMAT_D16_XYZ  : MTBUF_Real_AllAddr_gfx11_gfx12<0x00e, "tbuffer_store_d16_format_xyz">;
+defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx11_gfx12<0x00f, "tbuffer_store_d16_format_xyzw">;
 defm TBUFFER_STORE_FORMAT_X        : MTBUF_Real_AllAddr_gfx11_gfx12<0x004>;
 defm TBUFFER_STORE_FORMAT_XY       : MTBUF_Real_AllAddr_gfx11_gfx12<0x005>;
 defm TBUFFER_STORE_FORMAT_XYZ      : MTBUF_Real_AllAddr_gfx11_gfx12<0x006>;
-- 
GitLab


From 35d3b33ba5c9b90443ac985f2521b78f84b611fe Mon Sep 17 00:00:00 2001
From: Andreas Fertig 
Date: Fri, 8 Mar 2024 17:10:20 +0100
Subject: [PATCH 0048/1674] [C++20][Coroutines] Lambda-coroutine with operator
 new in promise_type (#84193)

Fix #84064

According to http://eel.is/c++draft/dcl.fct.def.coroutine#9 the first
parameter for overload resolution of `operator new` is `size_t` followed
by the arguments of the coroutine function.
http://eel.is/c++draft/dcl.fct.def.coroutine#4 states that the first
argument is the lvalue of `*this` if the coroutine is a member function.

Before this patch, Clang handled class types correctly but ignored
lambdas. This patch adds support for lambda coroutines with a
`promise_type` that implements a custom `operator new`.

The patch does consider C++23 `static operator()`, which already worked
as there is no `this` parameter.
---
 clang/include/clang/Sema/Sema.h  | 12 ++++-
 clang/lib/Sema/SemaCoroutine.cpp | 18 +++++++-
 clang/lib/Sema/SemaExprCXX.cpp   | 16 +++++--
 clang/test/SemaCXX/gh84064-1.cpp | 79 ++++++++++++++++++++++++++++++++
 clang/test/SemaCXX/gh84064-2.cpp | 53 +++++++++++++++++++++
 5 files changed, 169 insertions(+), 9 deletions(-)
 create mode 100644 clang/test/SemaCXX/gh84064-1.cpp
 create mode 100644 clang/test/SemaCXX/gh84064-2.cpp

diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 5b4d67494f2a..cfc1c3b34947 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -6752,10 +6752,18 @@ public:
                             SourceLocation RParenLoc);
 
   //// ActOnCXXThis -  Parse 'this' pointer.
-  ExprResult ActOnCXXThis(SourceLocation loc);
+  ///
+  /// \param ThisRefersToClosureObject Whether to skip the 'this' check for a
+  /// lambda because 'this' refers to the closure object.
+  ExprResult ActOnCXXThis(SourceLocation loc,
+                          bool ThisRefersToClosureObject = false);
 
   /// Build a CXXThisExpr and mark it referenced in the current context.
-  Expr *BuildCXXThisExpr(SourceLocation Loc, QualType Type, bool IsImplicit);
+  ///
+  /// \param ThisRefersToClosureObject Whether to skip the 'this' check for a
+  /// lambda because 'this' refers to the closure object.
+  Expr *BuildCXXThisExpr(SourceLocation Loc, QualType Type, bool IsImplicit,
+                         bool ThisRefersToClosureObject = false);
   void MarkThisReferenced(CXXThisExpr *This);
 
   /// Try to retrieve the type of the 'this' pointer.
diff --git a/clang/lib/Sema/SemaCoroutine.cpp b/clang/lib/Sema/SemaCoroutine.cpp
index a969b9383563..301a5ff72a3b 100644
--- a/clang/lib/Sema/SemaCoroutine.cpp
+++ b/clang/lib/Sema/SemaCoroutine.cpp
@@ -25,6 +25,7 @@
 #include "clang/Sema/Initialization.h"
 #include "clang/Sema/Overload.h"
 #include "clang/Sema/ScopeInfo.h"
+#include "clang/Sema/Sema.h"
 #include "clang/Sema/SemaInternal.h"
 #include "llvm/ADT/SmallSet.h"
 
@@ -1378,8 +1379,21 @@ bool CoroutineStmtBuilder::makeReturnOnAllocFailure() {
 static bool collectPlacementArgs(Sema &S, FunctionDecl &FD, SourceLocation Loc,
                                  SmallVectorImpl &PlacementArgs) {
   if (auto *MD = dyn_cast(&FD)) {
-    if (MD->isImplicitObjectMemberFunction() && !isLambdaCallOperator(MD)) {
-      ExprResult ThisExpr = S.ActOnCXXThis(Loc);
+    if (MD->isImplicitObjectMemberFunction()) {
+      ExprResult ThisExpr{};
+
+      if (isLambdaCallOperator(MD) && !MD->isStatic()) {
+        Qualifiers ThisQuals = MD->getMethodQualifiers();
+        CXXRecordDecl *Record = MD->getParent();
+
+        Sema::CXXThisScopeRAII ThisScope(S, Record, ThisQuals,
+                                         Record != nullptr);
+
+        ThisExpr = S.ActOnCXXThis(Loc, /*ThisRefersToClosureObject=*/true);
+      } else {
+        ThisExpr = S.ActOnCXXThis(Loc);
+      }
+
       if (ThisExpr.isInvalid())
         return false;
       ThisExpr = S.CreateBuiltinUnaryOp(Loc, UO_Deref, ThisExpr.get());
diff --git a/clang/lib/Sema/SemaExprCXX.cpp b/clang/lib/Sema/SemaExprCXX.cpp
index c34a40fa7c81..88e3d9ced044 100644
--- a/clang/lib/Sema/SemaExprCXX.cpp
+++ b/clang/lib/Sema/SemaExprCXX.cpp
@@ -1414,7 +1414,8 @@ bool Sema::CheckCXXThisCapture(SourceLocation Loc, const bool Explicit,
   return false;
 }
 
-ExprResult Sema::ActOnCXXThis(SourceLocation Loc) {
+ExprResult Sema::ActOnCXXThis(SourceLocation Loc,
+                              bool ThisRefersToClosureObject) {
   /// C++ 9.3.2: In the body of a non-static member function, the keyword this
   /// is a non-lvalue expression whose value is the address of the object for
   /// which the function is called.
@@ -1434,13 +1435,18 @@ ExprResult Sema::ActOnCXXThis(SourceLocation Loc) {
     return Diag(Loc, diag::err_invalid_this_use) << 0;
   }
 
-  return BuildCXXThisExpr(Loc, ThisTy, /*IsImplicit=*/false);
+  return BuildCXXThisExpr(Loc, ThisTy, /*IsImplicit=*/false,
+                          ThisRefersToClosureObject);
 }
 
-Expr *Sema::BuildCXXThisExpr(SourceLocation Loc, QualType Type,
-                             bool IsImplicit) {
+Expr *Sema::BuildCXXThisExpr(SourceLocation Loc, QualType Type, bool IsImplicit,
+                             bool ThisRefersToClosureObject) {
   auto *This = CXXThisExpr::Create(Context, Loc, Type, IsImplicit);
-  MarkThisReferenced(This);
+
+  if (!ThisRefersToClosureObject) {
+    MarkThisReferenced(This);
+  }
+
   return This;
 }
 
diff --git a/clang/test/SemaCXX/gh84064-1.cpp b/clang/test/SemaCXX/gh84064-1.cpp
new file mode 100644
index 000000000000..d9c2738a002b
--- /dev/null
+++ b/clang/test/SemaCXX/gh84064-1.cpp
@@ -0,0 +1,79 @@
+// RUN: %clang_cc1 -fsyntax-only -verify -I%S/Inputs -std=c++20 %s
+
+// expected-no-diagnostics
+
+#include "std-coroutine.h"
+
+using size_t = decltype(sizeof(0));
+
+struct Generator {
+  struct promise_type {
+    int _val{};
+
+    Generator get_return_object() noexcept
+    {
+      return {};
+    }
+
+    std::suspend_never initial_suspend() noexcept
+    {
+      return {};
+    }
+
+    std::suspend_always final_suspend() noexcept
+    {
+      return {};
+    }
+
+    void return_void() noexcept {}
+    void unhandled_exception() noexcept {}
+
+    template
+    static void*
+    operator new(size_t size,
+                 This&,
+                 TheRest&&...) noexcept
+    {
+        return nullptr;
+    }
+
+    static void operator delete(void*, size_t)
+    {
+    }
+  };
+};
+
+struct CapturingThisTest
+{
+    int x{};
+
+    void AsPointer()
+    {
+      auto lamb = [=,this]() -> Generator {
+        int y = x;
+        co_return;
+      };
+
+      static_assert(sizeof(decltype(lamb)) == sizeof(void*));
+    }
+
+    void AsStarThis()
+    {
+      auto lamb = [*this]() -> Generator {
+        int y = x;
+        co_return;
+      };
+
+      static_assert(sizeof(decltype(lamb)) == sizeof(int));
+    }
+};
+
+int main()
+{
+  auto lamb = []() -> Generator {
+    co_return;
+  };
+
+  static_assert(sizeof(decltype(lamb)) == 1);
+}
+
diff --git a/clang/test/SemaCXX/gh84064-2.cpp b/clang/test/SemaCXX/gh84064-2.cpp
new file mode 100644
index 000000000000..457de43eab6d
--- /dev/null
+++ b/clang/test/SemaCXX/gh84064-2.cpp
@@ -0,0 +1,53 @@
+// RUN: %clang_cc1 -fsyntax-only -verify -I%S/Inputs -std=c++23 %s
+
+// expected-no-diagnostics
+
+#include "std-coroutine.h"
+
+using size_t = decltype(sizeof(0));
+
+struct GeneratorStatic {
+  struct promise_type {
+    int _val{};
+
+    GeneratorStatic get_return_object() noexcept
+    {
+      return {};
+    }
+
+    std::suspend_never initial_suspend() noexcept
+    {
+      return {};
+    }
+
+    std::suspend_always final_suspend() noexcept
+    {
+      return {};
+    }
+
+    void return_void() noexcept {}
+    void unhandled_exception() noexcept {}
+
+    template
+    static void*
+    operator new(size_t  size,
+                 TheRest&&...) noexcept
+    {
+        return nullptr;
+    }
+
+    static void operator delete(void*, size_t)
+    {
+    }
+  };
+};
+
+
+int main()
+{
+  auto lambCpp23 = []() static -> GeneratorStatic {
+    co_return;
+  };
+
+  static_assert(sizeof(decltype(lambCpp23)) == 1);
+}
-- 
GitLab


From 81601391369c267216199db3f7fcb9864ccf6fec Mon Sep 17 00:00:00 2001
From: Jonathan Thackray 
Date: Fri, 8 Mar 2024 16:11:36 +0000
Subject: [PATCH 0049/1674] Add support for Arm Cortex A78AE CPU (#84485)

Add support for Arm Cortex A78AE CPU

Technical Reference Manual for Arm Cortex A78AE:
   https://developer.arm.com/documentation/101779/0003

Fixes #84450
---
 clang/docs/ReleaseNotes.rst                   |  2 ++
 clang/test/Driver/aarch64-mcpu.c              |  2 ++
 clang/test/Driver/arm-cortex-cpus-2.c         |  7 +++++++
 clang/test/Misc/target-invalid-cpu-note.c     |  6 +++---
 .../llvm/TargetParser/AArch64TargetParser.h   |  5 +++++
 .../llvm/TargetParser/ARMTargetParser.def     |  2 ++
 llvm/lib/Target/AArch64/AArch64.td            | 18 ++++++++++++++++++
 llvm/lib/Target/AArch64/AArch64Subtarget.cpp  |  1 +
 llvm/lib/Target/AArch64/AArch64Subtarget.h    |  1 +
 llvm/lib/Target/ARM/ARM.td                    | 10 ++++++++++
 llvm/lib/Target/ARM/ARMSubtarget.cpp          |  1 +
 llvm/lib/Target/ARM/ARMSubtarget.h            |  1 +
 llvm/lib/TargetParser/Host.cpp                |  1 +
 .../TargetParser/TargetParserTest.cpp         | 19 +++++++++++++++++--
 14 files changed, 71 insertions(+), 5 deletions(-)

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 24b452bd487f..690fc7ed271a 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -388,6 +388,8 @@ Arm and AArch64 Support
   instructions (rdm). The identifier is available on the command line as
   a feature modifier for -march and -mcpu as well as via target attributes
   like ``target_version`` or ``target_clones``.
+- Support has been added for the following processors (-mcpu identifiers in parenthesis):
+    * Arm Cortex-A78AE (cortex-a78ae).
 
 Android Support
 ^^^^^^^^^^^^^^^
diff --git a/clang/test/Driver/aarch64-mcpu.c b/clang/test/Driver/aarch64-mcpu.c
index 3e07f3597f34..cacfc691058d 100644
--- a/clang/test/Driver/aarch64-mcpu.c
+++ b/clang/test/Driver/aarch64-mcpu.c
@@ -50,6 +50,8 @@
 // CORTEXA78: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-cpu" "cortex-a78"
 // RUN: %clang --target=aarch64 -mcpu=cortex-a78c  -### -c %s 2>&1 | FileCheck -check-prefix=CORTEX-A78C %s
 // CORTEX-A78C: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-cpu" "cortex-a78c"
+// RUN: %clang --target=aarch64 -mcpu=cortex-a78ae  -### -c %s 2>&1 | FileCheck -check-prefix=CORTEX-A78AE %s
+// CORTEX-A78AE: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-cpu" "cortex-a78ae"
 // RUN: %clang --target=aarch64 -mcpu=cortex-a715  -### -c %s 2>&1 | FileCheck -check-prefix=CORTEX-A715 %s
 // CORTEX-A715: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-cpu" "cortex-a715"
 // RUN: %clang --target=aarch64 -mcpu=cortex-a720  -### -c %s 2>&1 | FileCheck -check-prefix=CORTEX-A720 %s
diff --git a/clang/test/Driver/arm-cortex-cpus-2.c b/clang/test/Driver/arm-cortex-cpus-2.c
index c322303d2278..5ce3758655b5 100644
--- a/clang/test/Driver/arm-cortex-cpus-2.c
+++ b/clang/test/Driver/arm-cortex-cpus-2.c
@@ -537,6 +537,13 @@
 // CHECK-CORTEX-A78C-MFPU: "-target-feature" "+sha2"
 // CHECK-CORTEX-A78C-MFPU: "-target-feature" "+aes"
 
+// RUN: %clang -target armv8a-arm-none-eabi -mcpu=cortex-a78ae -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-CORTEX-A78AE %s
+// RUN: %clang -target armv8a-arm-none-eabi -mcpu=cortex-a78ae -mfpu=crypto-neon-fp-armv8 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-CORTEX-A78AE-MFPU %s
+// CHECK-CORTEX-A78AE: "-cc1"{{.*}} "-triple" "armv8.2a-{{.*}} "-target-cpu" "cortex-a78ae"
+// CHECK-CORTEX-A78AE-MFPU: "-cc1"{{.*}} "-target-feature" "+fp-armv8"
+// CHECK-CORTEX-A78AE-MFPU: "-target-feature" "+sha2"
+// CHECK-CORTEX-A78AE-MFPU: "-target-feature" "+aes"
+
 // RUN: %clang -target armv8a-arm-none-eabi -mcpu=cortex-a710 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-CORTEX-A710 %s
 // RUN: %clang -target armv8a-arm-none-eabi -mcpu=cortex-a710 -mfpu=crypto-neon-fp-armv8 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-CORTEX-A710-MFPU %s
 // CHECK-CORTEX-A710: "-cc1"{{.*}} "-triple" "armv9a-{{.*}} "-target-cpu" "cortex-a710"
diff --git a/clang/test/Misc/target-invalid-cpu-note.c b/clang/test/Misc/target-invalid-cpu-note.c
index ef2c1c0cfd3d..b65a8fb057ee 100644
--- a/clang/test/Misc/target-invalid-cpu-note.c
+++ b/clang/test/Misc/target-invalid-cpu-note.c
@@ -1,15 +1,15 @@
 // Use CHECK-NEXT instead of multiple CHECK-SAME to ensure we will fail if there is anything extra in the output.
 // RUN: not %clang_cc1 -triple armv5--- -target-cpu not-a-cpu -fsyntax-only %s 2>&1 | FileCheck %s --check-prefix ARM
 // ARM: error: unknown target CPU 'not-a-cpu'
-// ARM-NEXT: note: valid target CPU values are: arm8, arm810, strongarm, strongarm110, strongarm1100, strongarm1110, arm7tdmi, arm7tdmi-s, arm710t, arm720t, arm9, arm9tdmi, arm920, arm920t, arm922t, arm940t, ep9312, arm10tdmi, arm1020t, arm9e, arm946e-s, arm966e-s, arm968e-s, arm10e, arm1020e, arm1022e, arm926ej-s, arm1136j-s, arm1136jf-s, mpcore, mpcorenovfp, arm1176jz-s, arm1176jzf-s, arm1156t2-s, arm1156t2f-s, cortex-m0, cortex-m0plus, cortex-m1, sc000, cortex-a5, cortex-a7, cortex-a8, cortex-a9, cortex-a12, cortex-a15, cortex-a17, krait, cortex-r4, cortex-r4f, cortex-r5, cortex-r7, cortex-r8, cortex-r52, sc300, cortex-m3, cortex-m4, cortex-m7, cortex-m23, cortex-m33, cortex-m35p, cortex-m55, cortex-m85, cortex-m52, cortex-a32, cortex-a35, cortex-a53, cortex-a55, cortex-a57, cortex-a72, cortex-a73, cortex-a75, cortex-a76, cortex-a76ae, cortex-a77, cortex-a78, cortex-a78c, cortex-a710, cortex-x1, cortex-x1c, neoverse-n1, neoverse-n2, neoverse-v1, cyclone, exynos-m3, exynos-m4, exynos-m5, kryo, iwmmxt, xscale, swift{{$}}
+// ARM-NEXT: note: valid target CPU values are: arm8, arm810, strongarm, strongarm110, strongarm1100, strongarm1110, arm7tdmi, arm7tdmi-s, arm710t, arm720t, arm9, arm9tdmi, arm920, arm920t, arm922t, arm940t, ep9312, arm10tdmi, arm1020t, arm9e, arm946e-s, arm966e-s, arm968e-s, arm10e, arm1020e, arm1022e, arm926ej-s, arm1136j-s, arm1136jf-s, mpcore, mpcorenovfp, arm1176jz-s, arm1176jzf-s, arm1156t2-s, arm1156t2f-s, cortex-m0, cortex-m0plus, cortex-m1, sc000, cortex-a5, cortex-a7, cortex-a8, cortex-a9, cortex-a12, cortex-a15, cortex-a17, krait, cortex-r4, cortex-r4f, cortex-r5, cortex-r7, cortex-r8, cortex-r52, sc300, cortex-m3, cortex-m4, cortex-m7, cortex-m23, cortex-m33, cortex-m35p, cortex-m55, cortex-m85, cortex-m52, cortex-a32, cortex-a35, cortex-a53, cortex-a55, cortex-a57, cortex-a72, cortex-a73, cortex-a75, cortex-a76, cortex-a76ae, cortex-a77, cortex-a78, cortex-a78ae, cortex-a78c, cortex-a710, cortex-x1, cortex-x1c, neoverse-n1, neoverse-n2, neoverse-v1, cyclone, exynos-m3, exynos-m4, exynos-m5, kryo, iwmmxt, xscale, swift{{$}}
 
 // RUN: not %clang_cc1 -triple arm64--- -target-cpu not-a-cpu -fsyntax-only %s 2>&1 | FileCheck %s --check-prefix AARCH64
 // AARCH64: error: unknown target CPU 'not-a-cpu'
-// AARCH64-NEXT: note: valid target CPU values are: cortex-a34, cortex-a35, cortex-a53, cortex-a55, cortex-a510, cortex-a520, cortex-a57, cortex-a65, cortex-a65ae, cortex-a72, cortex-a73, cortex-a75, cortex-a76, cortex-a76ae, cortex-a77, cortex-a78, cortex-a78c, cortex-a710, cortex-a715, cortex-a720, cortex-r82, cortex-x1, cortex-x1c, cortex-x2, cortex-x3, cortex-x4, neoverse-e1, neoverse-n1, neoverse-n2, neoverse-512tvb, neoverse-v1, neoverse-v2, cyclone, apple-a7, apple-a8, apple-a9, apple-a10, apple-a11, apple-a12, apple-a13, apple-a14, apple-a15, apple-a16, apple-a17, apple-m1, apple-m2, apple-m3, apple-s4, apple-s5, exynos-m3, exynos-m4, exynos-m5, falkor, saphira, kryo, thunderx2t99, thunderx3t110, thunderx, thunderxt88, thunderxt81, thunderxt83, tsv110, a64fx, carmel, ampere1, ampere1a, ampere1b, cobalt-100, grace{{$}}
+// AARCH64-NEXT: note: valid target CPU values are: cortex-a34, cortex-a35, cortex-a53, cortex-a55, cortex-a510, cortex-a520, cortex-a57, cortex-a65, cortex-a65ae, cortex-a72, cortex-a73, cortex-a75, cortex-a76, cortex-a76ae, cortex-a77, cortex-a78, cortex-a78ae, cortex-a78c, cortex-a710, cortex-a715, cortex-a720, cortex-r82, cortex-x1, cortex-x1c, cortex-x2, cortex-x3, cortex-x4, neoverse-e1, neoverse-n1, neoverse-n2, neoverse-512tvb, neoverse-v1, neoverse-v2, cyclone, apple-a7, apple-a8, apple-a9, apple-a10, apple-a11, apple-a12, apple-a13, apple-a14, apple-a15, apple-a16, apple-a17, apple-m1, apple-m2, apple-m3, apple-s4, apple-s5, exynos-m3, exynos-m4, exynos-m5, falkor, saphira, kryo, thunderx2t99, thunderx3t110, thunderx, thunderxt88, thunderxt81, thunderxt83, tsv110, a64fx, carmel, ampere1, ampere1a, ampere1b, cobalt-100, grace{{$}}
 
 // RUN: not %clang_cc1 -triple arm64--- -tune-cpu not-a-cpu -fsyntax-only %s 2>&1 | FileCheck %s --check-prefix TUNE_AARCH64
 // TUNE_AARCH64: error: unknown target CPU 'not-a-cpu'
-// TUNE_AARCH64-NEXT: note: valid target CPU values are: cortex-a34, cortex-a35, cortex-a53, cortex-a55, cortex-a510, cortex-a520, cortex-a57, cortex-a65, cortex-a65ae, cortex-a72, cortex-a73, cortex-a75, cortex-a76, cortex-a76ae, cortex-a77, cortex-a78, cortex-a78c, cortex-a710, cortex-a715, cortex-a720, cortex-r82, cortex-x1, cortex-x1c, cortex-x2, cortex-x3, cortex-x4, neoverse-e1, neoverse-n1, neoverse-n2, neoverse-512tvb, neoverse-v1, neoverse-v2, cyclone, apple-a7, apple-a8, apple-a9, apple-a10, apple-a11, apple-a12, apple-a13, apple-a14, apple-a15, apple-a16, apple-a17, apple-m1, apple-m2, apple-m3, apple-s4, apple-s5, exynos-m3, exynos-m4, exynos-m5, falkor, saphira, kryo, thunderx2t99, thunderx3t110, thunderx, thunderxt88, thunderxt81, thunderxt83, tsv110, a64fx, carmel, ampere1, ampere1a, ampere1b, cobalt-100, grace{{$}}
+// TUNE_AARCH64-NEXT: note: valid target CPU values are: cortex-a34, cortex-a35, cortex-a53, cortex-a55, cortex-a510, cortex-a520, cortex-a57, cortex-a65, cortex-a65ae, cortex-a72, cortex-a73, cortex-a75, cortex-a76, cortex-a76ae, cortex-a77, cortex-a78, cortex-a78ae, cortex-a78c, cortex-a710, cortex-a715, cortex-a720, cortex-r82, cortex-x1, cortex-x1c, cortex-x2, cortex-x3, cortex-x4, neoverse-e1, neoverse-n1, neoverse-n2, neoverse-512tvb, neoverse-v1, neoverse-v2, cyclone, apple-a7, apple-a8, apple-a9, apple-a10, apple-a11, apple-a12, apple-a13, apple-a14, apple-a15, apple-a16, apple-a17, apple-m1, apple-m2, apple-m3, apple-s4, apple-s5, exynos-m3, exynos-m4, exynos-m5, falkor, saphira, kryo, thunderx2t99, thunderx3t110, thunderx, thunderxt88, thunderxt81, thunderxt83, tsv110, a64fx, carmel, ampere1, ampere1a, ampere1b, cobalt-100, grace{{$}}
 
 // RUN: not %clang_cc1 -triple i386--- -target-cpu not-a-cpu -fsyntax-only %s 2>&1 | FileCheck %s --check-prefix X86
 // X86: error: unknown target CPU 'not-a-cpu'
diff --git a/llvm/include/llvm/TargetParser/AArch64TargetParser.h b/llvm/include/llvm/TargetParser/AArch64TargetParser.h
index e74e8fa32a62..f4bb94f98bcd 100644
--- a/llvm/include/llvm/TargetParser/AArch64TargetParser.h
+++ b/llvm/include/llvm/TargetParser/AArch64TargetParser.h
@@ -592,6 +592,11 @@ inline constexpr CpuInfo CpuInfos[] = {
                                AArch64::AEK_FP16, AArch64::AEK_DOTPROD,
                                AArch64::AEK_RCPC, AArch64::AEK_SSBS,
                                AArch64::AEK_PROFILE})},
+    {"cortex-a78ae", ARMV8_2A,
+     AArch64::ExtensionBitset({AArch64::AEK_AES, AArch64::AEK_SHA2,
+                               AArch64::AEK_FP16, AArch64::AEK_DOTPROD,
+                               AArch64::AEK_RCPC, AArch64::AEK_SSBS,
+                               AArch64::AEK_PROFILE})},
     {"cortex-a78c", ARMV8_2A,
      AArch64::ExtensionBitset(
          {AArch64::AEK_AES, AArch64::AEK_SHA2, AArch64::AEK_FP16,
diff --git a/llvm/include/llvm/TargetParser/ARMTargetParser.def b/llvm/include/llvm/TargetParser/ARMTargetParser.def
index fa6cfdfeda28..b821d224d7a8 100644
--- a/llvm/include/llvm/TargetParser/ARMTargetParser.def
+++ b/llvm/include/llvm/TargetParser/ARMTargetParser.def
@@ -363,6 +363,8 @@ ARM_CPU_NAME("cortex-a77", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false,
              (ARM::AEK_FP16 | ARM::AEK_DOTPROD))
 ARM_CPU_NAME("cortex-a78", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false,
              (ARM::AEK_FP16 | ARM::AEK_DOTPROD))
+ARM_CPU_NAME("cortex-a78ae", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false,
+             (ARM::AEK_RAS | ARM::AEK_DOTPROD))
 ARM_CPU_NAME("cortex-a78c", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false,
              ARM::AEK_FP16 | ARM::AEK_DOTPROD)
 ARM_CPU_NAME("cortex-a710", ARMV9A, FK_NEON_FP_ARMV8, false,
diff --git a/llvm/lib/Target/AArch64/AArch64.td b/llvm/lib/Target/AArch64/AArch64.td
index b837066554f3..402c7292d7f8 100644
--- a/llvm/lib/Target/AArch64/AArch64.td
+++ b/llvm/lib/Target/AArch64/AArch64.td
@@ -944,6 +944,18 @@ def TuneA78 : SubtargetFeature<"a78", "ARMProcFamily", "CortexA78",
                                FeatureEnableSelectOptimize,
                                FeaturePredictableSelectIsExpensive]>;
 
+def TuneA78AE : SubtargetFeature<"a78ae", "ARMProcFamily",
+                                 "CortexA78AE",
+                                 "Cortex-A78AE ARM processors", [
+                                 FeatureCmpBccFusion,
+                                 FeatureFuseAES,
+                                 FeatureFuseAdrpAdd,
+                                 FeatureAddrLSLFast,
+                                 FeatureALULSLFast,
+                                 FeaturePostRAScheduler,
+                                 FeatureEnableSelectOptimize,
+                                 FeaturePredictableSelectIsExpensive]>;
+
 def TuneA78C : SubtargetFeature<"a78c", "ARMProcFamily",
                                 "CortexA78C",
                                 "Cortex-A78C ARM processors", [
@@ -1425,6 +1437,10 @@ def ProcessorFeatures {
                                  FeatureNEON, FeatureFullFP16, FeatureDotProd,
                                  FeatureRCPC, FeaturePerfMon, FeatureSPE,
                                  FeatureSSBS];
+  list A78AE = [HasV8_2aOps, FeatureCrypto, FeatureFPARMv8,
+                                  FeatureNEON, FeatureFullFP16, FeatureDotProd,
+                                  FeatureRCPC, FeaturePerfMon, FeatureSPE,
+                                  FeatureSSBS];
   list A78C = [HasV8_2aOps, FeatureCrypto, FeatureFPARMv8,
                                  FeatureNEON, FeatureFullFP16, FeatureDotProd,
                                  FeatureFlagM, FeaturePAuth,
@@ -1602,6 +1618,8 @@ def : ProcessorModel<"cortex-a77", CortexA57Model, ProcessorFeatures.A77,
                      [TuneA77]>;
 def : ProcessorModel<"cortex-a78", CortexA57Model, ProcessorFeatures.A78,
                      [TuneA78]>;
+def : ProcessorModel<"cortex-a78ae", CortexA57Model, ProcessorFeatures.A78AE,
+                     [TuneA78AE]>;
 def : ProcessorModel<"cortex-a78c", CortexA57Model, ProcessorFeatures.A78C,
                      [TuneA78C]>;
 def : ProcessorModel<"cortex-a710", NeoverseN2Model, ProcessorFeatures.A710,
diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.cpp b/llvm/lib/Target/AArch64/AArch64Subtarget.cpp
index 2fa87f6afc7a..23b1deb3697f 100644
--- a/llvm/lib/Target/AArch64/AArch64Subtarget.cpp
+++ b/llvm/lib/Target/AArch64/AArch64Subtarget.cpp
@@ -140,6 +140,7 @@ void AArch64Subtarget::initializeProperties(bool HasMinSize) {
   case CortexA76:
   case CortexA77:
   case CortexA78:
+  case CortexA78AE:
   case CortexA78C:
   case CortexR82:
   case CortexX1:
diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.h b/llvm/lib/Target/AArch64/AArch64Subtarget.h
index 4dae3ab28d3e..95bef7a76bca 100644
--- a/llvm/lib/Target/AArch64/AArch64Subtarget.h
+++ b/llvm/lib/Target/AArch64/AArch64Subtarget.h
@@ -66,6 +66,7 @@ public:
     CortexA76,
     CortexA77,
     CortexA78,
+    CortexA78AE,
     CortexA78C,
     CortexA710,
     CortexA715,
diff --git a/llvm/lib/Target/ARM/ARM.td b/llvm/lib/Target/ARM/ARM.td
index b62e1a032631..4510c7cf4f42 100644
--- a/llvm/lib/Target/ARM/ARM.td
+++ b/llvm/lib/Target/ARM/ARM.td
@@ -809,6 +809,8 @@ def ProcA77     : SubtargetFeature<"a77", "ARMProcFamily", "CortexA77",
                                    "Cortex-A77 ARM processors", []>;
 def ProcA78     : SubtargetFeature<"cortex-a78", "ARMProcFamily", "CortexA78",
                                    "Cortex-A78 ARM processors", []>;
+def ProcA78AE   : SubtargetFeature<"cortex-a78ae", "ARMProcFamily", "CortexA78AE",
+                                   "Cortex-A78AE ARM processors", []>;
 def ProcA78C    : SubtargetFeature<"a78c", "ARMProcFamily", "CortexA78C",
                                    "Cortex-A78C ARM processors", []>;
 def ProcA710    : SubtargetFeature<"cortex-a710", "ARMProcFamily",
@@ -1632,6 +1634,14 @@ def : ProcNoItin<"cortex-a78",                          [ARMv82a, ProcA78,
                                                          FeatureFullFP16,
                                                          FeatureDotProd]>;
 
+def : ProcNoItin<"cortex-a78ae",                        [ARMv82a, ProcA78AE,
+                                                         FeatureHWDivThumb,
+                                                         FeatureHWDivARM,
+                                                         FeatureCrypto,
+                                                         FeatureCRC,
+                                                         FeatureFullFP16,
+                                                         FeatureDotProd]>;
+
 def : ProcNoItin<"cortex-a78c",                         [ARMv82a, ProcA78C,
                                                          FeatureHWDivThumb,
                                                          FeatureHWDivARM,
diff --git a/llvm/lib/Target/ARM/ARMSubtarget.cpp b/llvm/lib/Target/ARM/ARMSubtarget.cpp
index 691715dc2963..717e61518c6e 100644
--- a/llvm/lib/Target/ARM/ARMSubtarget.cpp
+++ b/llvm/lib/Target/ARM/ARMSubtarget.cpp
@@ -289,6 +289,7 @@ void ARMSubtarget::initSubtargetFeatures(StringRef CPU, StringRef FS) {
   case CortexA76:
   case CortexA77:
   case CortexA78:
+  case CortexA78AE:
   case CortexA78C:
   case CortexA710:
   case CortexR4:
diff --git a/llvm/lib/Target/ARM/ARMSubtarget.h b/llvm/lib/Target/ARM/ARMSubtarget.h
index 044b1c4c54e0..497ae160fde2 100644
--- a/llvm/lib/Target/ARM/ARMSubtarget.h
+++ b/llvm/lib/Target/ARM/ARMSubtarget.h
@@ -66,6 +66,7 @@ protected:
     CortexA76,
     CortexA77,
     CortexA78,
+    CortexA78AE,
     CortexA78C,
     CortexA710,
     CortexA8,
diff --git a/llvm/lib/TargetParser/Host.cpp b/llvm/lib/TargetParser/Host.cpp
index a4cc757a9214..ae58ddf03f86 100644
--- a/llvm/lib/TargetParser/Host.cpp
+++ b/llvm/lib/TargetParser/Host.cpp
@@ -217,6 +217,7 @@ StringRef sys::detail::getHostCPUNameForARM(StringRef ProcCpuinfoContent) {
         .Case("0xd0b", "cortex-a76")
         .Case("0xd0d", "cortex-a77")
         .Case("0xd41", "cortex-a78")
+        .Case("0xd42", "cortex-a78ae")
         .Case("0xd47", "cortex-a710")
         .Case("0xd4d", "cortex-a715")
         .Case("0xd81", "cortex-a720")
diff --git a/llvm/unittests/TargetParser/TargetParserTest.cpp b/llvm/unittests/TargetParser/TargetParserTest.cpp
index 7e57c4fc378a..3773f59a3c5a 100644
--- a/llvm/unittests/TargetParser/TargetParserTest.cpp
+++ b/llvm/unittests/TargetParser/TargetParserTest.cpp
@@ -454,6 +454,12 @@ INSTANTIATE_TEST_SUITE_P(
                 ARM::AEK_VIRT | ARM::AEK_HWDIVARM | ARM::AEK_HWDIVTHUMB |
                 ARM::AEK_DSP | ARM::AEK_CRC | ARM::AEK_RAS,
             "8.2-A"),
+        ARMCPUTestParams(
+            "cortex-a78ae", "armv8.2-a", "crypto-neon-fp-armv8",
+            ARM::AEK_RAS | ARM::AEK_DOTPROD | ARM::AEK_SEC | ARM::AEK_MP |
+                ARM::AEK_VIRT | ARM::AEK_HWDIVARM | ARM::AEK_HWDIVTHUMB |
+                ARM::AEK_DSP | ARM::AEK_CRC | ARM::AEK_RAS,
+            "8.2-A"),
         ARMCPUTestParams(
             "cortex-x1", "armv8.2-a", "crypto-neon-fp-armv8",
             ARM::AEK_RAS | ARM::AEK_FP16 | ARM::AEK_DOTPROD | ARM::AEK_SEC |
@@ -542,7 +548,7 @@ INSTANTIATE_TEST_SUITE_P(
                                    "7-S")),
     ARMCPUTestParams::PrintToStringParamName);
 
-static constexpr unsigned NumARMCPUArchs = 90;
+static constexpr unsigned NumARMCPUArchs = 91;
 
 TEST(TargetParserTest, testARMCPUArchList) {
   SmallVector List;
@@ -1209,6 +1215,15 @@ INSTANTIATE_TEST_SUITE_P(
                  AArch64::AEK_DOTPROD, AArch64::AEK_RCPC, AArch64::AEK_SSBS,
                  AArch64::AEK_PROFILE}),
             "8.2-A"),
+        ARMCPUTestParams(
+            "cortex-a78ae", "armv8.2-a", "crypto-neon-fp-armv8",
+            AArch64::ExtensionBitset(
+                {AArch64::AEK_CRC, AArch64::AEK_AES, AArch64::AEK_SHA2,
+                 AArch64::AEK_FP, AArch64::AEK_RDM, AArch64::AEK_SIMD,
+                 AArch64::AEK_RAS, AArch64::AEK_LSE, AArch64::AEK_FP16,
+                 AArch64::AEK_DOTPROD, AArch64::AEK_RCPC, AArch64::AEK_SSBS,
+                 AArch64::AEK_PROFILE}),
+            "8.2-A"),
         ARMCPUTestParams(
             "cortex-a78c", "armv8.2-a", "crypto-neon-fp-armv8",
             AArch64::ExtensionBitset(
@@ -1702,7 +1717,7 @@ INSTANTIATE_TEST_SUITE_P(
     ARMCPUTestParams::PrintToStringParamName);
 
 // Note: number of CPUs includes aliases.
-static constexpr unsigned NumAArch64CPUArchs = 69;
+static constexpr unsigned NumAArch64CPUArchs = 70;
 
 TEST(TargetParserTest, testAArch64CPUArchList) {
   SmallVector List;
-- 
GitLab


From abfac563f5b5a123e4bf773c3a09777e6fc4f50c Mon Sep 17 00:00:00 2001
From: Boian Petkantchin 
Date: Fri, 8 Mar 2024 08:14:36 -0800
Subject: [PATCH 0050/1674] [mlir][mesh] Make sharding propagation and
 spmdization work on FuncOpInterface (#84415)

Make them more general instead of only supporting `func::FuncOp`.
---
 .../mlir/Dialect/Mesh/Transforms/Passes.td     |  4 ++--
 .../Mesh/Transforms/ShardingPropagation.cpp    |  5 +++--
 .../Dialect/Mesh/Transforms/Spmdization.cpp    | 18 ++++++++++--------
 mlir/test/Dialect/Linalg/mesh-spmdization.mlir |  3 +--
 .../Dialect/Mesh/sharding-propagation.mlir     |  2 +-
 mlir/test/Dialect/Mesh/spmdization.mlir        |  4 +++-
 6 files changed, 20 insertions(+), 16 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Mesh/Transforms/Passes.td b/mlir/include/mlir/Dialect/Mesh/Transforms/Passes.td
index 7fb6631574b4..06ebf151e7d6 100644
--- a/mlir/include/mlir/Dialect/Mesh/Transforms/Passes.td
+++ b/mlir/include/mlir/Dialect/Mesh/Transforms/Passes.td
@@ -16,7 +16,7 @@ include "mlir/Pass/PassBase.td"
 // ShardingPropagation
 //===----------------------------------------------------------------------===//
 
-def ShardingPropagation : Pass<"sharding-propagation", "mlir::func::FuncOp"> {
+def ShardingPropagation : InterfacePass<"sharding-propagation", "mlir::FunctionOpInterface"> {
   let summary = "sharding propagation";
   let description = [{
     Propagates sharding information throughout the graph. After this pass, each
@@ -29,7 +29,7 @@ def ShardingPropagation : Pass<"sharding-propagation", "mlir::func::FuncOp"> {
   ];
 }
 
-def Spmdization : Pass<"mesh-spmdization", "mlir::func::FuncOp"> {
+def Spmdization : InterfacePass<"mesh-spmdization", "mlir::FunctionOpInterface"> {
   let summary = "Partition a function into SPMD form.";
   let description = [{
     This pass fits in right after a pass that annotates the function with
diff --git a/mlir/lib/Dialect/Mesh/Transforms/ShardingPropagation.cpp b/mlir/lib/Dialect/Mesh/Transforms/ShardingPropagation.cpp
index 9f2647b21cbf..29320f1e339f 100644
--- a/mlir/lib/Dialect/Mesh/Transforms/ShardingPropagation.cpp
+++ b/mlir/lib/Dialect/Mesh/Transforms/ShardingPropagation.cpp
@@ -12,6 +12,7 @@
 #include "mlir/Dialect/Mesh/IR/MeshDialect.h"
 #include "mlir/Dialect/Mesh/IR/MeshOps.h"
 #include "mlir/Dialect/Mesh/Interfaces/ShardingInterface.h"
+#include "mlir/Interfaces/FunctionInterfaces.h"
 #include "mlir/Pass/Pass.h"
 #include "llvm/Support/Debug.h"
 #include 
@@ -172,9 +173,9 @@ static LogicalResult visitOp(Operation *op, OpBuilder &builder) {
 struct ShardingPropagation
     : public mesh::impl::ShardingPropagationBase {
   void runOnOperation() override {
-    func::FuncOp funcOp = getOperation();
+    FunctionOpInterface funcOp = getOperation();
     MLIRContext *ctx = funcOp.getContext();
-    Region ®ion = funcOp.getBody();
+    Region ®ion = funcOp.getFunctionBody();
     OpBuilder builder(ctx);
     if (!region.hasOneBlock()) {
       funcOp.emitOpError() << "only one block is supported!";
diff --git a/mlir/lib/Dialect/Mesh/Transforms/Spmdization.cpp b/mlir/lib/Dialect/Mesh/Transforms/Spmdization.cpp
index c4d8b0b15e46..e4868435135e 100644
--- a/mlir/lib/Dialect/Mesh/Transforms/Spmdization.cpp
+++ b/mlir/lib/Dialect/Mesh/Transforms/Spmdization.cpp
@@ -24,6 +24,8 @@
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/IR/SymbolTable.h"
 #include "mlir/IR/Value.h"
+#include "mlir/Interfaces/ControlFlowInterfaces.h"
+#include "mlir/Interfaces/FunctionInterfaces.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Support/LLVM.h"
 #include "mlir/Support/LogicalResult.h"
@@ -694,7 +696,7 @@ static LogicalResult spmdizeBlock(Block &block, IRMapping &spmdizationMap,
 }
 
 static LogicalResult
-spmdizeFuncOp(func::FuncOp op, IRMapping &spmdizationMap,
+spmdizeFuncOp(FunctionOpInterface op, IRMapping &spmdizationMap,
               SymbolTableCollection &symbolTableCollection) {
   OpBuilder builder(op.getFunctionBody());
 
@@ -717,21 +719,21 @@ spmdizeFuncOp(func::FuncOp op, IRMapping &spmdizationMap,
 
   // Find a return op and change the function results signature to its operands
   // signature.
-  func::ReturnOp returnOp;
-  for (Block &block : op.getBody()) {
+  Operation *returnOp = nullptr;
+  for (Block &block : op.getFunctionBody()) {
     if (block.empty()) {
       continue;
     }
 
-    returnOp = llvm::cast(block.back());
-    if (returnOp) {
+    if (block.back().hasTrait()) {
+      returnOp = &block.back();
       break;
     }
   }
   assert(returnOp);
-  op.setFunctionType(FunctionType::get(op->getContext(),
-                                       op.getBody().front().getArgumentTypes(),
-                                       returnOp->getOperandTypes()));
+  op.setType(FunctionType::get(op->getContext(),
+                               op.getFunctionBody().front().getArgumentTypes(),
+                               returnOp->getOperandTypes()));
 
   return success();
 }
diff --git a/mlir/test/Dialect/Linalg/mesh-spmdization.mlir b/mlir/test/Dialect/Linalg/mesh-spmdization.mlir
index 6d21def8de27..bd56c801283b 100644
--- a/mlir/test/Dialect/Linalg/mesh-spmdization.mlir
+++ b/mlir/test/Dialect/Linalg/mesh-spmdization.mlir
@@ -1,6 +1,5 @@
 // RUN: mlir-opt \
-// RUN:  --mesh-spmdization \
-// RUN:  --test-constant-fold \
+// RUN:  --pass-pipeline="builtin.module(func.func(mesh-spmdization,test-constant-fold))" \
 // RUN:  --split-input-file \
 // RUN:  %s | FileCheck %s
 
diff --git a/mlir/test/Dialect/Mesh/sharding-propagation.mlir b/mlir/test/Dialect/Mesh/sharding-propagation.mlir
index 94f8d94073c5..270787ab5188 100644
--- a/mlir/test/Dialect/Mesh/sharding-propagation.mlir
+++ b/mlir/test/Dialect/Mesh/sharding-propagation.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt -sharding-propagation %s | FileCheck %s
+// RUN: mlir-opt --pass-pipeline="builtin.module(func.func(sharding-propagation))" %s | FileCheck %s
 
 mesh.mesh @mesh_1d(shape = ?)
 mesh.mesh @mesh_2d(shape = 2x4)
diff --git a/mlir/test/Dialect/Mesh/spmdization.mlir b/mlir/test/Dialect/Mesh/spmdization.mlir
index 572d3eb55eaa..2df247aba351 100644
--- a/mlir/test/Dialect/Mesh/spmdization.mlir
+++ b/mlir/test/Dialect/Mesh/spmdization.mlir
@@ -1,4 +1,6 @@
-// RUN: mlir-opt --mesh-spmdization --test-constant-fold %s | FileCheck %s
+// RUN: mlir-opt \
+// RUN:   --pass-pipeline="builtin.module(func.func(mesh-spmdization,test-constant-fold))" \
+// RUN:   %s | FileCheck %s
 
 mesh.mesh @mesh_1d(shape = 2)
 
-- 
GitLab


From c54e0524eeffcf2c5428cd2bc0449a1989e82cd8 Mon Sep 17 00:00:00 2001
From: erichkeane 
Date: Wed, 6 Mar 2024 12:28:58 -0800
Subject: [PATCH 0051/1674] [OpenACC] Add Compute Context Serialization test,
 fix a bug

While working on a followup patch, I discovered we didn't
serialize/deserialize the Compute Context properly, so this patch fixes
it, then adds a PCH mode to the ast test.
---
 clang/lib/Serialization/ASTReaderStmt.cpp        | 2 +-
 clang/lib/Serialization/ASTWriterStmt.cpp        | 2 +-
 clang/test/SemaOpenACC/compute-construct-ast.cpp | 9 +++++++++
 3 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/clang/lib/Serialization/ASTReaderStmt.cpp b/clang/lib/Serialization/ASTReaderStmt.cpp
index 3da44ffccc38..674ed47581df 100644
--- a/clang/lib/Serialization/ASTReaderStmt.cpp
+++ b/clang/lib/Serialization/ASTReaderStmt.cpp
@@ -2805,7 +2805,7 @@ void ASTStmtReader::VisitOpenACCAssociatedStmtConstruct(
 
 void ASTStmtReader::VisitOpenACCComputeConstruct(OpenACCComputeConstruct *S) {
   VisitStmt(S);
-  VisitOpenACCConstructStmt(S);
+  VisitOpenACCAssociatedStmtConstruct(S);
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/clang/lib/Serialization/ASTWriterStmt.cpp b/clang/lib/Serialization/ASTWriterStmt.cpp
index 484621ae8130..7ce48fede383 100644
--- a/clang/lib/Serialization/ASTWriterStmt.cpp
+++ b/clang/lib/Serialization/ASTWriterStmt.cpp
@@ -2855,7 +2855,7 @@ void ASTStmtWriter::VisitOpenACCAssociatedStmtConstruct(
 
 void ASTStmtWriter::VisitOpenACCComputeConstruct(OpenACCComputeConstruct *S) {
   VisitStmt(S);
-  VisitOpenACCConstructStmt(S);
+  VisitOpenACCAssociatedStmtConstruct(S);
   Code = serialization::STMT_OPENACC_COMPUTE_CONSTRUCT;
 }
 
diff --git a/clang/test/SemaOpenACC/compute-construct-ast.cpp b/clang/test/SemaOpenACC/compute-construct-ast.cpp
index 0d1d0b266573..55c080838a18 100644
--- a/clang/test/SemaOpenACC/compute-construct-ast.cpp
+++ b/clang/test/SemaOpenACC/compute-construct-ast.cpp
@@ -1,5 +1,12 @@
 // RUN: %clang_cc1 %s -fopenacc -ast-dump | FileCheck %s
 
+// Test this with PCH.
+// RUN: %clang_cc1 %s -fopenacc -emit-pch -o %t %s
+// RUN: %clang_cc1 %s -fopenacc -include-pch %t -ast-dump-all | FileCheck %s
+
+#ifndef PCH_HELPER
+#define PCH_HELPER
+
 void NormalFunc() {
   // FIXME: Add a test once we have clauses for this.
   // CHECK-LABEL: NormalFunc
@@ -104,3 +111,5 @@ struct S {
 void use() {
   TemplFunc();
 }
+#endif
+
-- 
GitLab


From 37f3661f4e8d180dc702ab42fa015bc3c21ceb51 Mon Sep 17 00:00:00 2001
From: Nick Desaulniers 
Date: Fri, 8 Mar 2024 08:50:41 -0800
Subject: [PATCH 0052/1674] [libc][docs] add slides link for Tue's 2023 talk
 (#84516)

---
 libc/docs/talks.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libc/docs/talks.rst b/libc/docs/talks.rst
index 6daae9f1e7b6..174de0f56e7d 100644
--- a/libc/docs/talks.rst
+++ b/libc/docs/talks.rst
@@ -6,6 +6,7 @@ Talks
 ----
 * Math functions in LLVM libc or yet another correctly rounded libm - Tue Ly
 
+  * `slides `__
   * `video `__
 * The LLVM C Library for GPUs - Joseph Huber
 
-- 
GitLab


From 599ab6d63e5b12fd9c421140653f0510f0d29eab Mon Sep 17 00:00:00 2001
From: Louis Dionne 
Date: Fri, 8 Mar 2024 12:09:35 -0500
Subject: [PATCH 0053/1674] Revert "[libc++] Disable module_std and
 module_std_compat tests"

It looks like that broke the CI even more. Reverting until I can
investigate.
---
 libcxx/test/libcxx/module_std.gen.py        | 3 ---
 libcxx/test/libcxx/module_std_compat.gen.py | 3 ---
 2 files changed, 6 deletions(-)

diff --git a/libcxx/test/libcxx/module_std.gen.py b/libcxx/test/libcxx/module_std.gen.py
index 932af742640c..fc23985caf30 100644
--- a/libcxx/test/libcxx/module_std.gen.py
+++ b/libcxx/test/libcxx/module_std.gen.py
@@ -18,9 +18,6 @@
 
 # RUN: %{python} %s %{libcxx-dir}/utils
 
-# TODO: Re-enable this test once we understand why it keeps timing out.
-# UNSUPPORTED: clang
-
 import sys
 
 sys.path.append(sys.argv[1])
diff --git a/libcxx/test/libcxx/module_std_compat.gen.py b/libcxx/test/libcxx/module_std_compat.gen.py
index d29f490ab135..000aa2998612 100644
--- a/libcxx/test/libcxx/module_std_compat.gen.py
+++ b/libcxx/test/libcxx/module_std_compat.gen.py
@@ -18,9 +18,6 @@
 
 # RUN: %{python} %s %{libcxx-dir}/utils
 
-# TODO: Re-enable this test once we understand why it keeps timing out.
-# UNSUPPORTED: clang
-
 import sys
 
 sys.path.append(sys.argv[1])
-- 
GitLab


From 60d7bf3dbe6c32ed67ed65cb15727cddd9531979 Mon Sep 17 00:00:00 2001
From: lntue <35648136+lntue@users.noreply.github.com>
Date: Fri, 8 Mar 2024 12:15:02 -0500
Subject: [PATCH 0054/1674] [libc][math][c23] Add (l|ll)rintf128 and
 (l|ll)roundf128 math functions. (#84504)

---
 libc/config/linux/aarch64/entrypoints.txt     |   5 +
 libc/config/linux/riscv/entrypoints.txt       |   5 +
 libc/config/linux/x86_64/entrypoints.txt      |   5 +
 libc/docs/math/index.rst                      |  10 ++
 libc/spec/gnu_ext.td                          |   2 -
 libc/spec/stdc.td                             |   8 ++
 libc/src/math/CMakeLists.txt                  |   5 +
 libc/src/math/generic/CMakeLists.txt          | 115 ++++++++++++++----
 libc/src/math/generic/llrintf128.cpp          |  21 ++++
 libc/src/math/generic/llroundf128.cpp         |  19 +++
 libc/src/math/generic/lrintf128.cpp           |  20 +++
 libc/src/math/generic/lroundf128.cpp          |  19 +++
 libc/src/math/generic/rintf128.cpp            |  19 +++
 libc/src/math/llrintf128.h                    |  20 +++
 libc/src/math/llroundf128.h                   |  20 +++
 libc/src/math/lrintf128.h                     |  20 +++
 libc/src/math/lroundf128.h                    |  20 +++
 libc/src/math/rintf128.h                      |  20 +++
 libc/test/src/math/smoke/CMakeLists.txt       |  81 ++++++++++++
 libc/test/src/math/smoke/llrintf128_test.cpp  |  14 +++
 libc/test/src/math/smoke/llroundf128_test.cpp |  13 ++
 libc/test/src/math/smoke/lrintf128_test.cpp   |  14 +++
 libc/test/src/math/smoke/lroundf128_test.cpp  |  13 ++
 libc/test/src/math/smoke/rintf128_test.cpp    |  13 ++
 24 files changed, 474 insertions(+), 27 deletions(-)
 create mode 100644 libc/src/math/generic/llrintf128.cpp
 create mode 100644 libc/src/math/generic/llroundf128.cpp
 create mode 100644 libc/src/math/generic/lrintf128.cpp
 create mode 100644 libc/src/math/generic/lroundf128.cpp
 create mode 100644 libc/src/math/generic/rintf128.cpp
 create mode 100644 libc/src/math/llrintf128.h
 create mode 100644 libc/src/math/llroundf128.h
 create mode 100644 libc/src/math/lrintf128.h
 create mode 100644 libc/src/math/lroundf128.h
 create mode 100644 libc/src/math/rintf128.h
 create mode 100644 libc/test/src/math/smoke/llrintf128_test.cpp
 create mode 100644 libc/test/src/math/smoke/llroundf128_test.cpp
 create mode 100644 libc/test/src/math/smoke/lrintf128_test.cpp
 create mode 100644 libc/test/src/math/smoke/lroundf128_test.cpp
 create mode 100644 libc/test/src/math/smoke/rintf128_test.cpp

diff --git a/libc/config/linux/aarch64/entrypoints.txt b/libc/config/linux/aarch64/entrypoints.txt
index c32773f67cda..fa15ddd17aef 100644
--- a/libc/config/linux/aarch64/entrypoints.txt
+++ b/libc/config/linux/aarch64/entrypoints.txt
@@ -430,6 +430,11 @@ if(LIBC_TYPES_HAS_FLOAT128)
     libc.src.math.ldexpf128
     libc.src.math.llogbf128
     libc.src.math.logbf128
+    libc.src.math.llrintf128
+    libc.src.math.llroundf128
+    libc.src.math.lrintf128
+    libc.src.math.lroundf128
+    libc.src.math.rintf128
     libc.src.math.roundf128
     libc.src.math.sqrtf128
     libc.src.math.truncf128
diff --git a/libc/config/linux/riscv/entrypoints.txt b/libc/config/linux/riscv/entrypoints.txt
index bf518083b51f..924cf2f1d68b 100644
--- a/libc/config/linux/riscv/entrypoints.txt
+++ b/libc/config/linux/riscv/entrypoints.txt
@@ -438,6 +438,11 @@ if(LIBC_TYPES_HAS_FLOAT128)
     libc.src.math.ldexpf128
     libc.src.math.llogbf128
     libc.src.math.logbf128
+    libc.src.math.llrintf128
+    libc.src.math.llroundf128
+    libc.src.math.lrintf128
+    libc.src.math.lroundf128
+    libc.src.math.rintf128
     libc.src.math.roundf128
     libc.src.math.sqrtf128
     libc.src.math.truncf128
diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt
index 0b77a9e170aa..0880c372b373 100644
--- a/libc/config/linux/x86_64/entrypoints.txt
+++ b/libc/config/linux/x86_64/entrypoints.txt
@@ -468,6 +468,11 @@ if(LIBC_TYPES_HAS_FLOAT128)
     libc.src.math.ldexpf128
     libc.src.math.llogbf128
     libc.src.math.logbf128
+    libc.src.math.llrintf128
+    libc.src.math.llroundf128
+    libc.src.math.lrintf128
+    libc.src.math.lroundf128
+    libc.src.math.rintf128
     libc.src.math.roundf128
     libc.src.math.sqrtf128
     libc.src.math.truncf128
diff --git a/libc/docs/math/index.rst b/libc/docs/math/index.rst
index 80d12718edcc..81d95d9b6cfa 100644
--- a/libc/docs/math/index.rst
+++ b/libc/docs/math/index.rst
@@ -209,12 +209,16 @@ Basic Operations
 +--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
 | llrintl      | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
 +--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| llrintf128   | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
++--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
 | llround      | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
 +--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
 | llroundf     | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
 +--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
 | llroundl     | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
 +--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| llroundf128  | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
++--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
 | logb         | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
 +--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
 | logbf        | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
@@ -229,12 +233,16 @@ Basic Operations
 +--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
 | lrintl       | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
 +--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| lrintf128    | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
++--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
 | lround       | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
 +--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
 | lroundf      | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
 +--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
 | lroundl      | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
 +--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| lroundf128   | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
++--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
 | modf         | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
 +--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
 | modff        | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
@@ -283,6 +291,8 @@ Basic Operations
 +--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
 | rintl        | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
 +--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
+| rintf128     | |check| | |check| |         | |check| |         |         |         |         |         |         |         |         |
++--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
 | round        | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
 +--------------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+
 | roundf       | |check| | |check| | |check| | |check| | |check| |         |         | |check| | |check| | |check| |         |         |
diff --git a/libc/spec/gnu_ext.td b/libc/spec/gnu_ext.td
index add07d75050d..161bb4e4a0d9 100644
--- a/libc/spec/gnu_ext.td
+++ b/libc/spec/gnu_ext.td
@@ -33,8 +33,6 @@ def GnuExtensions : StandardSpec<"GNUExtensions"> {
             RetValSpec,
             [ArgSpec, ArgSpec, ArgSpec]
         >,
-        FunctionSpec<"exp10", RetValSpec, [ArgSpec]>,
-        FunctionSpec<"exp10f", RetValSpec, [ArgSpec]>,
       ]
   >;
 
diff --git a/libc/spec/stdc.td b/libc/spec/stdc.td
index d84eb3570e5c..cc845a93a333 100644
--- a/libc/spec/stdc.td
+++ b/libc/spec/stdc.td
@@ -470,6 +470,9 @@ def StdC : StandardSpec<"stdc"> {
           FunctionSpec<"expm1", RetValSpec, [ArgSpec]>,
           FunctionSpec<"expm1f", RetValSpec, [ArgSpec]>,
 
+          FunctionSpec<"exp10", RetValSpec, [ArgSpec]>,
+          FunctionSpec<"exp10f", RetValSpec, [ArgSpec]>,
+
           FunctionSpec<"remainderf", RetValSpec, [ArgSpec, ArgSpec]>,
           FunctionSpec<"remainder", RetValSpec, [ArgSpec, ArgSpec]>,
           FunctionSpec<"remainderl", RetValSpec, [ArgSpec, ArgSpec]>,
@@ -486,22 +489,27 @@ def StdC : StandardSpec<"stdc"> {
           FunctionSpec<"lround", RetValSpec, [ArgSpec]>,
           FunctionSpec<"lroundf", RetValSpec, [ArgSpec]>,
           FunctionSpec<"lroundl", RetValSpec, [ArgSpec]>,
+          GuardedFunctionSpec<"lroundf128", RetValSpec, [ArgSpec], "LIBC_TYPES_HAS_FLOAT128">,
 
           FunctionSpec<"llround", RetValSpec, [ArgSpec]>,
           FunctionSpec<"llroundf", RetValSpec, [ArgSpec]>,
           FunctionSpec<"llroundl", RetValSpec, [ArgSpec]>,
+          GuardedFunctionSpec<"llroundf128", RetValSpec, [ArgSpec], "LIBC_TYPES_HAS_FLOAT128">,
 
           FunctionSpec<"rint", RetValSpec, [ArgSpec]>,
           FunctionSpec<"rintf", RetValSpec, [ArgSpec]>,
           FunctionSpec<"rintl", RetValSpec, [ArgSpec]>,
+          GuardedFunctionSpec<"rintf128", RetValSpec, [ArgSpec], "LIBC_TYPES_HAS_FLOAT128">,
 
           FunctionSpec<"lrint", RetValSpec, [ArgSpec]>,
           FunctionSpec<"lrintf", RetValSpec, [ArgSpec]>,
           FunctionSpec<"lrintl", RetValSpec, [ArgSpec]>,
+          GuardedFunctionSpec<"lrintf128", RetValSpec, [ArgSpec], "LIBC_TYPES_HAS_FLOAT128">,
 
           FunctionSpec<"llrint", RetValSpec, [ArgSpec]>,
           FunctionSpec<"llrintf", RetValSpec, [ArgSpec]>,
           FunctionSpec<"llrintl", RetValSpec, [ArgSpec]>,
+          GuardedFunctionSpec<"llrintf128", RetValSpec, [ArgSpec], "LIBC_TYPES_HAS_FLOAT128">,
 
           FunctionSpec<"sqrt", RetValSpec, [ArgSpec]>,
           FunctionSpec<"sqrtf", RetValSpec, [ArgSpec]>,
diff --git a/libc/src/math/CMakeLists.txt b/libc/src/math/CMakeLists.txt
index 882befd9f7e7..035eefd82d36 100644
--- a/libc/src/math/CMakeLists.txt
+++ b/libc/src/math/CMakeLists.txt
@@ -163,18 +163,22 @@ add_math_entrypoint_object(logbf128)
 add_math_entrypoint_object(llrint)
 add_math_entrypoint_object(llrintf)
 add_math_entrypoint_object(llrintl)
+add_math_entrypoint_object(llrintf128)
 
 add_math_entrypoint_object(llround)
 add_math_entrypoint_object(llroundf)
 add_math_entrypoint_object(llroundl)
+add_math_entrypoint_object(llroundf128)
 
 add_math_entrypoint_object(lrint)
 add_math_entrypoint_object(lrintf)
 add_math_entrypoint_object(lrintl)
+add_math_entrypoint_object(lrintf128)
 
 add_math_entrypoint_object(lround)
 add_math_entrypoint_object(lroundf)
 add_math_entrypoint_object(lroundl)
+add_math_entrypoint_object(lroundf128)
 
 add_math_entrypoint_object(modf)
 add_math_entrypoint_object(modff)
@@ -210,6 +214,7 @@ add_math_entrypoint_object(remquol)
 add_math_entrypoint_object(rint)
 add_math_entrypoint_object(rintf)
 add_math_entrypoint_object(rintl)
+add_math_entrypoint_object(rintf128)
 
 add_math_entrypoint_object(round)
 add_math_entrypoint_object(roundf)
diff --git a/libc/src/math/generic/CMakeLists.txt b/libc/src/math/generic/CMakeLists.txt
index 82d2a5e66af7..a7b7065980b1 100644
--- a/libc/src/math/generic/CMakeLists.txt
+++ b/libc/src/math/generic/CMakeLists.txt
@@ -375,10 +375,10 @@ add_entrypoint_object(
     lround.cpp
   HDRS
     ../lround.h
+  COMPILE_OPTIONS
+    -O3
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
-  COMPILE_OPTIONS
-    -O2
 )
 
 add_entrypoint_object(
@@ -387,10 +387,10 @@ add_entrypoint_object(
     lroundf.cpp
   HDRS
     ../lroundf.h
+  COMPILE_OPTIONS
+    -O3
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
-  COMPILE_OPTIONS
-    -O2
 )
 
 add_entrypoint_object(
@@ -399,10 +399,23 @@ add_entrypoint_object(
     lroundl.cpp
   HDRS
     ../lroundl.h
+  COMPILE_OPTIONS
+    -O3
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
+)
+
+add_entrypoint_object(
+  lroundf128
+  SRCS
+    lroundf128.cpp
+  HDRS
+    ../lroundf128.h
   COMPILE_OPTIONS
-    -O2
+    -O3
+  DEPENDS
+    libc.src.__support.macros.properties.types
+    libc.src.__support.FPUtil.nearest_integer_operations
 )
 
 add_entrypoint_object(
@@ -411,10 +424,10 @@ add_entrypoint_object(
     llround.cpp
   HDRS
     ../llround.h
+  COMPILE_OPTIONS
+    -O3
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
-  COMPILE_OPTIONS
-    -O2
 )
 
 add_entrypoint_object(
@@ -423,10 +436,10 @@ add_entrypoint_object(
     llroundf.cpp
   HDRS
     ../llroundf.h
+  COMPILE_OPTIONS
+    -O3
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
-  COMPILE_OPTIONS
-    -O2
 )
 
 add_entrypoint_object(
@@ -435,10 +448,23 @@ add_entrypoint_object(
     llroundl.cpp
   HDRS
     ../llroundl.h
+  COMPILE_OPTIONS
+    -O3
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
+)
+
+add_entrypoint_object(
+  llroundf128
+  SRCS
+    llroundf128.cpp
+  HDRS
+    ../llroundf128.h
   COMPILE_OPTIONS
-    -O2
+    -O3
+  DEPENDS
+    libc.src.__support.macros.properties.types
+    libc.src.__support.FPUtil.nearest_integer_operations
 )
 
 add_entrypoint_object(
@@ -447,10 +473,10 @@ add_entrypoint_object(
     rint.cpp
   HDRS
     ../rint.h
+  COMPILE_OPTIONS
+    -O3
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
-  COMPILE_OPTIONS
-    -O2
 )
 
 add_entrypoint_object(
@@ -459,10 +485,10 @@ add_entrypoint_object(
     rintf.cpp
   HDRS
     ../rintf.h
+  COMPILE_OPTIONS
+    -O3
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
-  COMPILE_OPTIONS
-    -O2
 )
 
 add_entrypoint_object(
@@ -471,10 +497,23 @@ add_entrypoint_object(
     rintl.cpp
   HDRS
     ../rintl.h
+  COMPILE_OPTIONS
+    -O3
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
+)
+
+add_entrypoint_object(
+  rintf128
+  SRCS
+    rintf128.cpp
+  HDRS
+    ../rintf128.h
   COMPILE_OPTIONS
-    -O2
+    -O3
+  DEPENDS
+    libc.src.__support.macros.properties.types
+    libc.src.__support.FPUtil.nearest_integer_operations
 )
 
 add_entrypoint_object(
@@ -483,10 +522,10 @@ add_entrypoint_object(
     lrint.cpp
   HDRS
     ../lrint.h
+  COMPILE_OPTIONS
+    -O3
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
-  COMPILE_OPTIONS
-    -O2
 )
 
 add_entrypoint_object(
@@ -495,10 +534,10 @@ add_entrypoint_object(
     lrintf.cpp
   HDRS
     ../lrintf.h
+  COMPILE_OPTIONS
+    -O3
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
-  COMPILE_OPTIONS
-    -O2
 )
 
 add_entrypoint_object(
@@ -507,10 +546,23 @@ add_entrypoint_object(
     lrintl.cpp
   HDRS
     ../lrintl.h
+  COMPILE_OPTIONS
+    -O3
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
+)
+
+add_entrypoint_object(
+  lrintf128
+  SRCS
+    lrintf128.cpp
+  HDRS
+    ../lrintf128.h
   COMPILE_OPTIONS
-    -O2
+    -O3
+  DEPENDS
+    libc.src.__support.macros.properties.types
+    libc.src.__support.FPUtil.nearest_integer_operations
 )
 
 add_entrypoint_object(
@@ -519,10 +571,10 @@ add_entrypoint_object(
     llrint.cpp
   HDRS
     ../llrint.h
+  COMPILE_OPTIONS
+    -O3
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
-  COMPILE_OPTIONS
-    -O2
 )
 
 add_entrypoint_object(
@@ -531,10 +583,10 @@ add_entrypoint_object(
     llrintf.cpp
   HDRS
     ../llrintf.h
+  COMPILE_OPTIONS
+    -O3
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
-  COMPILE_OPTIONS
-    -O2
 )
 
 add_entrypoint_object(
@@ -543,10 +595,23 @@ add_entrypoint_object(
     llrintl.cpp
   HDRS
     ../llrintl.h
+  COMPILE_OPTIONS
+    -O3
   DEPENDS
     libc.src.__support.FPUtil.nearest_integer_operations
+)
+
+add_entrypoint_object(
+  llrintf128
+  SRCS
+    llrintf128.cpp
+  HDRS
+    ../llrintf128.h
   COMPILE_OPTIONS
-    -O2
+    -O3
+  DEPENDS
+    libc.src.__support.macros.properties.types
+    libc.src.__support.FPUtil.nearest_integer_operations
 )
 
 add_entrypoint_object(
diff --git a/libc/src/math/generic/llrintf128.cpp b/libc/src/math/generic/llrintf128.cpp
new file mode 100644
index 000000000000..e5a4c50a26e8
--- /dev/null
+++ b/libc/src/math/generic/llrintf128.cpp
@@ -0,0 +1,21 @@
+//===-- Implementation of llrintf128 function -----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/llrintf128.h"
+#include "src/__support/FPUtil/NearestIntegerOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(long long, llrintf128, (float128 x)) {
+  return fputil::round_to_signed_integer_using_current_rounding_mode(
+      x);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/llroundf128.cpp b/libc/src/math/generic/llroundf128.cpp
new file mode 100644
index 000000000000..25791631dd7e
--- /dev/null
+++ b/libc/src/math/generic/llroundf128.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of llroundf128 function ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/llroundf128.h"
+#include "src/__support/FPUtil/NearestIntegerOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(long long, llroundf128, (float128 x)) {
+  return fputil::round_to_signed_integer(x);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/lrintf128.cpp b/libc/src/math/generic/lrintf128.cpp
new file mode 100644
index 000000000000..8e06062fc580
--- /dev/null
+++ b/libc/src/math/generic/lrintf128.cpp
@@ -0,0 +1,20 @@
+//===-- Implementation of lrintf128 function ------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/lrintf128.h"
+#include "src/__support/FPUtil/NearestIntegerOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(long, lrintf128, (float128 x)) {
+  return fputil::round_to_signed_integer_using_current_rounding_mode(x);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/lroundf128.cpp b/libc/src/math/generic/lroundf128.cpp
new file mode 100644
index 000000000000..f93c47503825
--- /dev/null
+++ b/libc/src/math/generic/lroundf128.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of lroundf128 function -----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/lroundf128.h"
+#include "src/__support/FPUtil/NearestIntegerOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(long, lroundf128, (float128 x)) {
+  return fputil::round_to_signed_integer(x);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/generic/rintf128.cpp b/libc/src/math/generic/rintf128.cpp
new file mode 100644
index 000000000000..ba9912d6f853
--- /dev/null
+++ b/libc/src/math/generic/rintf128.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of rintf128 function -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/rintf128.h"
+#include "src/__support/FPUtil/NearestIntegerOperations.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(float128, rintf128, (float128 x)) {
+  return fputil::round_using_current_rounding_mode(x);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/llrintf128.h b/libc/src/math/llrintf128.h
new file mode 100644
index 000000000000..ac9c249342cc
--- /dev/null
+++ b/libc/src/math/llrintf128.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for llrintf128 --------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_LLRINTF128_H
+#define LLVM_LIBC_SRC_MATH_LLRINTF128_H
+
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE {
+
+long long llrintf128(float128 x);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_LLRINTF128_H
diff --git a/libc/src/math/llroundf128.h b/libc/src/math/llroundf128.h
new file mode 100644
index 000000000000..3245dfafc4d5
--- /dev/null
+++ b/libc/src/math/llroundf128.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for llroundf128 -------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_LLROUNDF128_H
+#define LLVM_LIBC_SRC_MATH_LLROUNDF128_H
+
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE {
+
+long long llroundf128(float128 x);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_LLROUNDF128_H
diff --git a/libc/src/math/lrintf128.h b/libc/src/math/lrintf128.h
new file mode 100644
index 000000000000..8f3f5ceabd3c
--- /dev/null
+++ b/libc/src/math/lrintf128.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for lrintf128 ---------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_LRINTF128_H
+#define LLVM_LIBC_SRC_MATH_LRINTF128_H
+
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE {
+
+long lrintf128(float128 x);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_LRINTF128_H
diff --git a/libc/src/math/lroundf128.h b/libc/src/math/lroundf128.h
new file mode 100644
index 000000000000..663b3732655b
--- /dev/null
+++ b/libc/src/math/lroundf128.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for lroundf128 --------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_LROUNDF128_H
+#define LLVM_LIBC_SRC_MATH_LROUNDF128_H
+
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE {
+
+long lroundf128(float128 x);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_LROUNDF128_H
diff --git a/libc/src/math/rintf128.h b/libc/src/math/rintf128.h
new file mode 100644
index 000000000000..2d9248974f02
--- /dev/null
+++ b/libc/src/math/rintf128.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for rintf128 ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_RINTF128_H
+#define LLVM_LIBC_SRC_MATH_RINTF128_H
+
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE {
+
+float128 rintf128(float128 x);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_RINTF128_H
diff --git a/libc/test/src/math/smoke/CMakeLists.txt b/libc/test/src/math/smoke/CMakeLists.txt
index be1810944495..63faaa9d4e4c 100644
--- a/libc/test/src/math/smoke/CMakeLists.txt
+++ b/libc/test/src/math/smoke/CMakeLists.txt
@@ -387,6 +387,24 @@ add_fp_unittest(
     libc.src.__support.FPUtil.fp_bits
 )
 
+add_fp_unittest(
+  lroundf128_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    lroundf128_test.cpp
+  HDRS
+    RoundToIntegerTest.h
+  DEPENDS
+    libc.include.math
+    libc.src.errno.errno
+    libc.src.fenv.feclearexcept
+    libc.src.fenv.feraiseexcept
+    libc.src.fenv.fetestexcept
+    libc.src.math.lroundf128
+    libc.src.__support.FPUtil.fp_bits
+)
+
 add_fp_unittest(
   llround_test
   SUITE
@@ -441,6 +459,24 @@ add_fp_unittest(
     libc.src.__support.FPUtil.fp_bits
 )
 
+add_fp_unittest(
+  llroundf128_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    llroundf128_test.cpp
+  HDRS
+    RoundToIntegerTest.h
+  DEPENDS
+    libc.include.math
+    libc.src.errno.errno
+    libc.src.fenv.feclearexcept
+    libc.src.fenv.feraiseexcept
+    libc.src.fenv.fetestexcept
+    libc.src.math.llroundf128
+    libc.src.__support.FPUtil.fp_bits
+)
+
 add_fp_unittest(
   rint_test
   SUITE
@@ -486,6 +522,21 @@ add_fp_unittest(
     libc.src.__support.FPUtil.fp_bits
 )
 
+add_fp_unittest(
+  rintf128_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    rintf128_test.cpp
+  HDRS
+    RIntTest.h
+  DEPENDS
+    libc.include.math
+    libc.src.math.rintf128
+    libc.src.__support.FPUtil.fenv_impl
+    libc.src.__support.FPUtil.fp_bits
+)
+
 add_fp_unittest(
   lrint_test
   SUITE
@@ -531,6 +582,21 @@ add_fp_unittest(
     libc.src.__support.FPUtil.fp_bits
 )
 
+add_fp_unittest(
+  lrintf128_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    lrintf128_test.cpp
+  HDRS
+    RoundToIntegerTest.h
+  DEPENDS
+    libc.include.math
+    libc.src.math.lrintf128
+    libc.src.__support.FPUtil.fenv_impl
+    libc.src.__support.FPUtil.fp_bits
+)
+
 add_fp_unittest(
   llrint_test
   SUITE
@@ -576,6 +642,21 @@ add_fp_unittest(
     libc.src.__support.FPUtil.fp_bits
 )
 
+add_fp_unittest(
+  llrintf128_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    llrintf128_test.cpp
+  HDRS
+    RoundToIntegerTest.h
+  DEPENDS
+    libc.include.math
+    libc.src.math.llrintf128
+    libc.src.__support.FPUtil.fenv_impl
+    libc.src.__support.FPUtil.fp_bits
+)
+
 add_fp_unittest(
   expf_test
   SUITE
diff --git a/libc/test/src/math/smoke/llrintf128_test.cpp b/libc/test/src/math/smoke/llrintf128_test.cpp
new file mode 100644
index 000000000000..8847b2918e63
--- /dev/null
+++ b/libc/test/src/math/smoke/llrintf128_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for llrintf128 ------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "RoundToIntegerTest.h"
+
+#include "src/math/llrintf128.h"
+
+LIST_ROUND_TO_INTEGER_TESTS_WITH_MODES(float128, long long,
+                                       LIBC_NAMESPACE::llrintf128)
diff --git a/libc/test/src/math/smoke/llroundf128_test.cpp b/libc/test/src/math/smoke/llroundf128_test.cpp
new file mode 100644
index 000000000000..b00055e759f8
--- /dev/null
+++ b/libc/test/src/math/smoke/llroundf128_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for llroundf128 -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "RoundToIntegerTest.h"
+
+#include "src/math/llroundf128.h"
+
+LIST_ROUND_TO_INTEGER_TESTS(float128, long long, LIBC_NAMESPACE::llroundf128)
diff --git a/libc/test/src/math/smoke/lrintf128_test.cpp b/libc/test/src/math/smoke/lrintf128_test.cpp
new file mode 100644
index 000000000000..a559ccf0eda7
--- /dev/null
+++ b/libc/test/src/math/smoke/lrintf128_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for lrintf128 -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "RoundToIntegerTest.h"
+
+#include "src/math/lrintf128.h"
+
+LIST_ROUND_TO_INTEGER_TESTS_WITH_MODES(float128, long,
+                                       LIBC_NAMESPACE::lrintf128)
diff --git a/libc/test/src/math/smoke/lroundf128_test.cpp b/libc/test/src/math/smoke/lroundf128_test.cpp
new file mode 100644
index 000000000000..ed87c9af0489
--- /dev/null
+++ b/libc/test/src/math/smoke/lroundf128_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for lroundf128 ------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "RoundToIntegerTest.h"
+
+#include "src/math/lroundf128.h"
+
+LIST_ROUND_TO_INTEGER_TESTS(float128, long, LIBC_NAMESPACE::lroundf128)
diff --git a/libc/test/src/math/smoke/rintf128_test.cpp b/libc/test/src/math/smoke/rintf128_test.cpp
new file mode 100644
index 000000000000..11078d795bac
--- /dev/null
+++ b/libc/test/src/math/smoke/rintf128_test.cpp
@@ -0,0 +1,13 @@
+//===-- Unittests for rintf128 --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "RIntTest.h"
+
+#include "src/math/rintf128.h"
+
+LIST_RINT_TESTS(float128, LIBC_NAMESPACE::rintf128)
-- 
GitLab


From 73b2d672c1c8318cd16a02812c39ae3997b9dbcd Mon Sep 17 00:00:00 2001
From: David Spickett 
Date: Fri, 8 Mar 2024 16:54:01 +0000
Subject: [PATCH 0055/1674] [lldb][FreeBSD] Fix crash when execve fails with
 asserts enabled

535da10842c7309e9eeaf9828cf6bb034fecaf16 introduced a check, when execve fails,
to see if we are allowed to trace programs at all.

Unfortunately because we like to call Status vars "error" and "status"
in various combinations, one got misnamed. This lead to lldb-server trying
to make an error value out of a success value when you did the following:

```
$ ./bin/lldb-server gdbserver 127.0.0.1:1234 -- is_not_a_file
Assertion failed: (Err && "Cannot create Expected from Error success value."), function Expected...
```

This happened because the execve fails, but the check whether we can
trace says yes we can trace, but then we use the Status from the check
to create the return value. That Status is in fact a success value not
the failed Status we got from the execve attempt.

With the name corrected you now get:
```
$ ./bin/lldb-server gdbserver 127.0.0.1:1234 -- is_not_a_file
error: failed to launch 'is_not_a_file': execve failed: No such file or directory
```

Which is what we expect to see. This also fixes the test `TestGDBRemoteLaunch.py`
when asserts are enabled.
---
 lldb/source/Plugins/Process/FreeBSD/NativeProcessFreeBSD.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lldb/source/Plugins/Process/FreeBSD/NativeProcessFreeBSD.cpp b/lldb/source/Plugins/Process/FreeBSD/NativeProcessFreeBSD.cpp
index 9c620e4807e3..064bddd37052 100644
--- a/lldb/source/Plugins/Process/FreeBSD/NativeProcessFreeBSD.cpp
+++ b/lldb/source/Plugins/Process/FreeBSD/NativeProcessFreeBSD.cpp
@@ -76,9 +76,9 @@ NativeProcessFreeBSD::Manager::Launch(ProcessLaunchInfo &launch_info,
                     .GetProcessId();
   LLDB_LOG(log, "pid = {0:x}", pid);
   if (status.Fail()) {
-    auto error = CanTrace();
     LLDB_LOG(log, "failed to launch process: {0}", status);
-    if (status.Fail())
+    auto error = CanTrace();
+    if (error.Fail())
       return error.ToError();
     return status.ToError();
   }
-- 
GitLab


From 0bbada93a559b604797fe57978f3eca5e41edaeb Mon Sep 17 00:00:00 2001
From: Eric Fiselier 
Date: Fri, 8 Mar 2024 12:17:07 -0500
Subject: [PATCH 0056/1674] Actually disable the module generation tests.

LIT was never really meant to generate tests during discovery,
and we probably shouldn't be doing this.

This hack is even worse than the initial attempt because it buries the
"UNSUPPORTED" at the bottom of the test.
---
 libcxx/test/libcxx/module_std.gen.py        | 5 +++++
 libcxx/test/libcxx/module_std_compat.gen.py | 5 +++++
 2 files changed, 10 insertions(+)

diff --git a/libcxx/test/libcxx/module_std.gen.py b/libcxx/test/libcxx/module_std.gen.py
index fc23985caf30..5acaa837d37e 100644
--- a/libcxx/test/libcxx/module_std.gen.py
+++ b/libcxx/test/libcxx/module_std.gen.py
@@ -16,7 +16,11 @@
 # to be one monolitic test. Since the test doesn't take very long it's
 # not a huge issue.
 
+# WARNING: Disabled at the bottom. Fix this test and remove the UNSUPPORTED line
+# TODO: Re-enable this test once we understand why it keeps timing out.
+
 # RUN: %{python} %s %{libcxx-dir}/utils
+# END.
 
 import sys
 
@@ -35,4 +39,5 @@ generator = module_test_generator(
 
 
 print("//--- module_std.sh.cpp")
+print('// UNSUPPORTED: clang')
 generator.write_test("std")
diff --git a/libcxx/test/libcxx/module_std_compat.gen.py b/libcxx/test/libcxx/module_std_compat.gen.py
index 000aa2998612..a502276a1ccf 100644
--- a/libcxx/test/libcxx/module_std_compat.gen.py
+++ b/libcxx/test/libcxx/module_std_compat.gen.py
@@ -16,7 +16,11 @@
 # to be one monolitic test. Since the test doesn't take very long it's
 # not a huge issue.
 
+# WARNING: Disabled at the bottom. Fix this test and remove the UNSUPPORTED line
+# TODO: Re-enable this test once we understand why it keeps timing out.
+
 # RUN: %{python} %s %{libcxx-dir}/utils
+# END.
 
 import sys
 
@@ -36,6 +40,7 @@ generator = module_test_generator(
 
 
 print("//--- module_std_compat.sh.cpp")
+print("// UNSUPPORTED: clang")
 generator.write_test(
     "std.compat",
     module_c_headers,
-- 
GitLab


From dbca8aa1472b9140a68918b2615519dc7dad593f Mon Sep 17 00:00:00 2001
From: Derek Schuff 
Date: Fri, 8 Mar 2024 09:20:44 -0800
Subject: [PATCH 0057/1674] [WebAssembly] Use SetVector instead of SmallPtrSet
 in FixBrTableDefaults (#84418)

This pass inserts all the MBBs into a set and then iterates over them.
But when
the number of elements gets large enough, SmallPtrSet expands into a
hash table
which wouldn't have a deterministic iteration order since the elements
are
pointers. This results in nondeterministic jump table layouts.
Use SetVector instead for a deterministic iteration order.
---
 .../Target/WebAssembly/WebAssemblyFixBrTableDefaults.cpp  | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyFixBrTableDefaults.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyFixBrTableDefaults.cpp
index 495f19a7ccde..4252fc1f55fc 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyFixBrTableDefaults.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyFixBrTableDefaults.cpp
@@ -159,19 +159,21 @@ bool WebAssemblyFixBrTableDefaults::runOnMachineFunction(MachineFunction &MF) {
                     << MF.getName() << '\n');
 
   bool Changed = false;
-  SmallPtrSet MBBSet;
+  SetVector,
+            DenseSet, 16>
+      MBBSet;
   for (auto &MBB : MF)
     MBBSet.insert(&MBB);
 
   while (!MBBSet.empty()) {
     MachineBasicBlock *MBB = *MBBSet.begin();
-    MBBSet.erase(MBB);
+    MBBSet.remove(MBB);
     for (auto &MI : *MBB) {
       if (WebAssembly::isBrTable(MI.getOpcode())) {
         fixBrTableIndex(MI, MBB, MF);
         auto *Fixed = fixBrTableDefault(MI, MBB, MF);
         if (Fixed != nullptr) {
-          MBBSet.erase(Fixed);
+          MBBSet.remove(Fixed);
           Changed = true;
         }
         break;
-- 
GitLab


From ea12c1fa15093e24818785b2ca6e06588372a3bf Mon Sep 17 00:00:00 2001
From: sylvain-audi <62035306+sylvain-audi@users.noreply.github.com>
Date: Fri, 8 Mar 2024 12:29:15 -0500
Subject: [PATCH 0058/1674] [Asan] Add "funclet" OpBundle to generated runtime
 calls if required by EH personality (#82533)

Previously, runtime calls introduced by ASan instrumentation into EH
pads were missing the funclet token expected by WinEHPrepare.
WinEHPrepare would then identify the containing BB as invalid and
discard it, causing invalid code generation that most likely crashes.

Also fixed localescape test, switching its EH personality to match code
without funclets.

This PR is based on the Phabricator patch
https://reviews.llvm.org/D143108

Fixes https://github.com/llvm/llvm-project/issues/64990
---
 .../asan/TestCases/Windows/issue64990.cpp     |  18 +
 .../Instrumentation/AddressSanitizer.cpp      |  62 ++-
 .../AddressSanitizer/asan-funclet.ll          | 527 +++++++++++-------
 .../AddressSanitizer/localescape.ll           |   4 +-
 4 files changed, 402 insertions(+), 209 deletions(-)
 create mode 100644 compiler-rt/test/asan/TestCases/Windows/issue64990.cpp

diff --git a/compiler-rt/test/asan/TestCases/Windows/issue64990.cpp b/compiler-rt/test/asan/TestCases/Windows/issue64990.cpp
new file mode 100644
index 000000000000..a5a46b5a81dd
--- /dev/null
+++ b/compiler-rt/test/asan/TestCases/Windows/issue64990.cpp
@@ -0,0 +1,18 @@
+// Repro for the issue #64990: Asan with Windows EH generates __asan_xxx runtime calls without required funclet tokens
+// RUN: %clang_cl_asan %Od %s -EHsc %Fe%t
+// RUN: not %run %t 2>&1 | FileCheck %s
+
+char buff1[6] = "hello";
+char buff2[6] = "hello";
+
+int main(int argc, char **argv) {
+  try {
+    throw 1;
+  } catch (...) {
+    // Make asan generate call to __asan_memcpy inside the EH pad.
+    __builtin_memcpy(buff1, buff2 + 3, 6);
+  }
+  return 0;
+}
+
+// CHECK: SUMMARY: AddressSanitizer: global-buffer-overflow {{.*}} in __asan_memcpy
diff --git a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
index db75eec21a37..26fedbfd65dd 100644
--- a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
@@ -43,6 +43,7 @@
 #include "llvm/IR/DebugInfoMetadata.h"
 #include "llvm/IR/DebugLoc.h"
 #include "llvm/IR/DerivedTypes.h"
+#include "llvm/IR/EHPersonalities.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/GlobalAlias.h"
 #include "llvm/IR/GlobalValue.h"
@@ -643,21 +644,70 @@ static uint64_t GetCtorAndDtorPriority(Triple &TargetTriple) {
 
 namespace {
 /// Helper RAII class to post-process inserted asan runtime calls during a
-/// pass on a single Function. This is a no-op implementation, for a first NFC
-/// commit. Coming up: detect and add "funclet" opBundle to function calls that
-/// need them.
+/// pass on a single Function. Upon end of scope, detects and applies the
+/// required funclet OpBundle.
 class RuntimeCallInserter {
   Function *OwnerFn = nullptr;
+  bool TrackInsertedCalls = false;
+  SmallVector InsertedCalls;
 
 public:
-  RuntimeCallInserter(Function &Fn) : OwnerFn(&Fn) {}
+  RuntimeCallInserter(Function &Fn) : OwnerFn(&Fn) {
+    if (Fn.hasPersonalityFn()) {
+      auto Personality = classifyEHPersonality(Fn.getPersonalityFn());
+      if (isScopedEHPersonality(Personality))
+        TrackInsertedCalls = true;
+    }
+  }
+
+  ~RuntimeCallInserter() {
+    if (InsertedCalls.empty())
+      return;
+    assert(TrackInsertedCalls && "Calls were wrongly tracked");
+
+    DenseMap BlockColors = colorEHFunclets(*OwnerFn);
+    for (CallInst *CI : InsertedCalls) {
+      BasicBlock *BB = CI->getParent();
+      assert(BB && "Instruction doesn't belong to a BasicBlock");
+      assert(BB->getParent() == OwnerFn &&
+             "Instruction doesn't belong to the expected Function!");
+
+      ColorVector &Colors = BlockColors[BB];
+      // funclet opbundles are only valid in monochromatic BBs.
+      // Note that unreachable BBs are seen as colorless by colorEHFunclets()
+      // and will be DCE'ed later.
+      if (Colors.empty())
+        continue;
+      if (Colors.size() != 1) {
+        OwnerFn->getContext().emitError(
+            "Instruction's BasicBlock is not monochromatic");
+        continue;
+      }
+
+      BasicBlock *Color = Colors.front();
+      Instruction *EHPad = Color->getFirstNonPHI();
+
+      if (EHPad && EHPad->isEHPad()) {
+        // Replace CI with a clone with an added funclet OperandBundle
+        OperandBundleDef OB("funclet", EHPad);
+        auto *NewCall =
+            CallBase::addOperandBundle(CI, LLVMContext::OB_funclet, OB, CI);
+        NewCall->copyMetadata(*CI);
+        CI->replaceAllUsesWith(NewCall);
+        CI->eraseFromParent();
+      }
+    }
+  }
 
   CallInst *createRuntimeCall(IRBuilder<> &IRB, FunctionCallee Callee,
                               ArrayRef Args = {},
                               const Twine &Name = "") {
     assert(IRB.GetInsertBlock()->getParent() == OwnerFn);
-    (void)OwnerFn;
-    return IRB.CreateCall(Callee, Args, Name, nullptr);
+
+    CallInst *Inst = IRB.CreateCall(Callee, Args, Name, nullptr);
+    if (TrackInsertedCalls)
+      InsertedCalls.push_back(Inst);
+    return Inst;
   }
 };
 
diff --git a/llvm/test/Instrumentation/AddressSanitizer/asan-funclet.ll b/llvm/test/Instrumentation/AddressSanitizer/asan-funclet.ll
index f0a5c67365ab..6d40e2b7eaf9 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/asan-funclet.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/asan-funclet.ll
@@ -1,8 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
 
-; Warning! The output of this test is currently invalid.
-; It serves as a base for the bugfix patch to highlight the modified generated code.
-
 ; Test appropriate tagging of funclet for function calls generated by asan.
 ; RUN: opt -S -passes=asan,win-eh-prepare -asan-use-stack-safety=0 -asan-max-inline-poisoning-size=0 \
 ; RUN:   -asan-detect-invalid-pointer-cmp -asan-detect-invalid-pointer-sub -asan-use-after-scope < %s | FileCheck %s --check-prefixes=CHECK,CHECK-INLINE
@@ -53,137 +50,221 @@ define void @FuncletPersonality(ptr %ptrParam) sanitize_address personality ptr
 ; CHECK-INLINE-NEXT:    store i64 [[TMP12]], ptr [[ASAN_LOCAL_STACK_BASE]], align 8
 ; CHECK-INLINE-NEXT:    [[TMP13:%.*]] = add i64 [[TMP12]], 32
 ; CHECK-INLINE-NEXT:    [[TMP14:%.*]] = inttoptr i64 [[TMP13]] to ptr
-; CHECK-INLINE-NEXT:    [[TMP15:%.*]] = add i64 [[TMP12]], 8528
+; CHECK-INLINE-NEXT:    [[TMP15:%.*]] = add i64 [[TMP12]], 8480
 ; CHECK-INLINE-NEXT:    [[TMP16:%.*]] = inttoptr i64 [[TMP15]] to ptr
-; CHECK-INLINE-NEXT:    [[TMP17:%.*]] = inttoptr i64 [[TMP12]] to ptr
-; CHECK-INLINE-NEXT:    store i64 1102416563, ptr [[TMP17]], align 8
-; CHECK-INLINE-NEXT:    [[TMP18:%.*]] = add i64 [[TMP12]], 8
-; CHECK-INLINE-NEXT:    [[TMP19:%.*]] = inttoptr i64 [[TMP18]] to ptr
-; CHECK-INLINE-NEXT:    store i64 ptrtoint (ptr @___asan_gen_ to i64), ptr [[TMP19]], align 8
-; CHECK-INLINE-NEXT:    [[TMP20:%.*]] = add i64 [[TMP12]], 16
-; CHECK-INLINE-NEXT:    [[TMP21:%.*]] = inttoptr i64 [[TMP20]] to ptr
-; CHECK-INLINE-NEXT:    store i64 ptrtoint (ptr @FuncletPersonality to i64), ptr [[TMP21]], align 8
-; CHECK-INLINE-NEXT:    [[TMP22:%.*]] = lshr i64 [[TMP12]], 3
-; CHECK-INLINE-NEXT:    [[TMP23:%.*]] = add i64 [[TMP22]], [[TMP1]]
-; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_f1(i64 [[TMP23]], i64 4)
-; CHECK-INLINE-NEXT:    [[TMP24:%.*]] = add i64 [[TMP23]], 1028
-; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_f2(i64 [[TMP24]], i64 32)
-; CHECK-INLINE-NEXT:    [[TMP25:%.*]] = add i64 [[TMP23]], 1060
-; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_04(i64 [[TMP25]], i64 1)
-; CHECK-INLINE-NEXT:    [[TMP26:%.*]] = add i64 [[TMP23]], 1061
-; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_f2(i64 [[TMP26]], i64 1)
-; CHECK-INLINE-NEXT:    [[TMP27:%.*]] = add i64 [[TMP23]], 1062
-; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_04(i64 [[TMP27]], i64 1)
-; CHECK-INLINE-NEXT:    [[TMP28:%.*]] = add i64 [[TMP23]], 1063
-; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_f2(i64 [[TMP28]], i64 1)
-; CHECK-INLINE-NEXT:    [[TMP29:%.*]] = add i64 [[TMP23]], 1064
-; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_04(i64 [[TMP29]], i64 1)
-; CHECK-INLINE-NEXT:    [[TMP30:%.*]] = add i64 [[TMP23]], 1065
-; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_f2(i64 [[TMP30]], i64 1)
-; CHECK-INLINE-NEXT:    [[TMP31:%.*]] = add i64 [[TMP23]], 1066
-; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_f8(i64 [[TMP31]], i64 1)
-; CHECK-INLINE-NEXT:    [[TMP32:%.*]] = add i64 [[TMP23]], 1067
-; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_f3(i64 [[TMP32]], i64 1)
-; CHECK-INLINE-NEXT:    [[TMP33:%.*]] = add i64 [[TMP23]], 1066
+; CHECK-INLINE-NEXT:    [[TMP17:%.*]] = add i64 [[TMP12]], 8496
+; CHECK-INLINE-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; CHECK-INLINE-NEXT:    [[TMP19:%.*]] = add i64 [[TMP12]], 8512
+; CHECK-INLINE-NEXT:    [[TMP20:%.*]] = inttoptr i64 [[TMP19]] to ptr
+; CHECK-INLINE-NEXT:    [[TMP21:%.*]] = add i64 [[TMP12]], 8528
+; CHECK-INLINE-NEXT:    [[TMP22:%.*]] = inttoptr i64 [[TMP21]] to ptr
+; CHECK-INLINE-NEXT:    [[TMP23:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-INLINE-NEXT:    store i64 1102416563, ptr [[TMP23]], align 8
+; CHECK-INLINE-NEXT:    [[TMP24:%.*]] = add i64 [[TMP12]], 8
+; CHECK-INLINE-NEXT:    [[TMP25:%.*]] = inttoptr i64 [[TMP24]] to ptr
+; CHECK-INLINE-NEXT:    store i64 ptrtoint (ptr @___asan_gen_ to i64), ptr [[TMP25]], align 8
+; CHECK-INLINE-NEXT:    [[TMP26:%.*]] = add i64 [[TMP12]], 16
+; CHECK-INLINE-NEXT:    [[TMP27:%.*]] = inttoptr i64 [[TMP26]] to ptr
+; CHECK-INLINE-NEXT:    store i64 ptrtoint (ptr @FuncletPersonality to i64), ptr [[TMP27]], align 8
+; CHECK-INLINE-NEXT:    [[TMP28:%.*]] = lshr i64 [[TMP12]], 3
+; CHECK-INLINE-NEXT:    [[TMP29:%.*]] = add i64 [[TMP28]], [[TMP1]]
+; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_f1(i64 [[TMP29]], i64 4)
+; CHECK-INLINE-NEXT:    [[TMP30:%.*]] = add i64 [[TMP29]], 1028
+; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_f2(i64 [[TMP30]], i64 32)
+; CHECK-INLINE-NEXT:    [[TMP31:%.*]] = add i64 [[TMP29]], 1060
+; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_04(i64 [[TMP31]], i64 1)
+; CHECK-INLINE-NEXT:    [[TMP32:%.*]] = add i64 [[TMP29]], 1061
+; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_f2(i64 [[TMP32]], i64 1)
+; CHECK-INLINE-NEXT:    [[TMP33:%.*]] = add i64 [[TMP29]], 1062
 ; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_04(i64 [[TMP33]], i64 1)
-; CHECK-INLINE-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[TMP16]])
-; CHECK-INLINE-NEXT:    [[TMP34:%.*]] = lshr i64 [[TMP15]], 3
-; CHECK-INLINE-NEXT:    [[TMP35:%.*]] = add i64 [[TMP34]], [[TMP1]]
-; CHECK-INLINE-NEXT:    [[TMP36:%.*]] = inttoptr i64 [[TMP35]] to ptr
-; CHECK-INLINE-NEXT:    [[TMP37:%.*]] = load i8, ptr [[TMP36]], align 1
-; CHECK-INLINE-NEXT:    [[TMP38:%.*]] = icmp ne i8 [[TMP37]], 0
-; CHECK-INLINE-NEXT:    br i1 [[TMP38]], label [[TMP39:%.*]], label [[TMP44:%.*]], !prof [[PROF0:![0-9]+]]
-; CHECK-INLINE:       39:
-; CHECK-INLINE-NEXT:    [[TMP40:%.*]] = and i64 [[TMP15]], 7
-; CHECK-INLINE-NEXT:    [[TMP41:%.*]] = trunc i64 [[TMP40]] to i8
-; CHECK-INLINE-NEXT:    [[TMP42:%.*]] = icmp sge i8 [[TMP41]], [[TMP37]]
-; CHECK-INLINE-NEXT:    br i1 [[TMP42]], label [[TMP43:%.*]], label [[TMP44]]
-; CHECK-INLINE:       43:
-; CHECK-INLINE-NEXT:    call void @__asan_report_store1(i64 [[TMP15]]) #[[ATTR8:[0-9]+]]
+; CHECK-INLINE-NEXT:    [[TMP34:%.*]] = add i64 [[TMP29]], 1063
+; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_f2(i64 [[TMP34]], i64 1)
+; CHECK-INLINE-NEXT:    [[TMP35:%.*]] = add i64 [[TMP29]], 1064
+; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_04(i64 [[TMP35]], i64 1)
+; CHECK-INLINE-NEXT:    [[TMP36:%.*]] = add i64 [[TMP29]], 1065
+; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_f2(i64 [[TMP36]], i64 1)
+; CHECK-INLINE-NEXT:    [[TMP37:%.*]] = add i64 [[TMP29]], 1066
+; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_f8(i64 [[TMP37]], i64 1)
+; CHECK-INLINE-NEXT:    [[TMP38:%.*]] = add i64 [[TMP29]], 1067
+; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_f3(i64 [[TMP38]], i64 1)
+; CHECK-INLINE-NEXT:    [[TMP39:%.*]] = add i64 [[TMP29]], 1066
+; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_04(i64 [[TMP39]], i64 1)
+; CHECK-INLINE-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[TMP22]])
+; CHECK-INLINE-NEXT:    [[TMP40:%.*]] = lshr i64 [[TMP21]], 3
+; CHECK-INLINE-NEXT:    [[TMP41:%.*]] = add i64 [[TMP40]], [[TMP1]]
+; CHECK-INLINE-NEXT:    [[TMP42:%.*]] = inttoptr i64 [[TMP41]] to ptr
+; CHECK-INLINE-NEXT:    [[TMP43:%.*]] = load i8, ptr [[TMP42]], align 1
+; CHECK-INLINE-NEXT:    [[TMP44:%.*]] = icmp ne i8 [[TMP43]], 0
+; CHECK-INLINE-NEXT:    br i1 [[TMP44]], label [[TMP45:%.*]], label [[TMP50:%.*]], !prof [[PROF0:![0-9]+]]
+; CHECK-INLINE:       45:
+; CHECK-INLINE-NEXT:    [[TMP46:%.*]] = and i64 [[TMP21]], 7
+; CHECK-INLINE-NEXT:    [[TMP47:%.*]] = trunc i64 [[TMP46]] to i8
+; CHECK-INLINE-NEXT:    [[TMP48:%.*]] = icmp sge i8 [[TMP47]], [[TMP43]]
+; CHECK-INLINE-NEXT:    br i1 [[TMP48]], label [[TMP49:%.*]], label [[TMP50]]
+; CHECK-INLINE:       49:
+; CHECK-INLINE-NEXT:    call void @__asan_report_store1(i64 [[TMP21]]) #[[ATTR8:[0-9]+]]
 ; CHECK-INLINE-NEXT:    unreachable
-; CHECK-INLINE:       44:
-; CHECK-INLINE-NEXT:    store volatile i8 0, ptr [[TMP16]], align 1
-; CHECK-INLINE-NEXT:    [[TMP45:%.*]] = add i64 [[TMP23]], 1066
-; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_f8(i64 [[TMP45]], i64 1)
-; CHECK-INLINE-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[TMP16]])
-; CHECK-INLINE-NEXT:    [[TMP46:%.*]] = alloca i8, i64 96, align 32
-; CHECK-INLINE-NEXT:    [[TMP47:%.*]] = ptrtoint ptr [[TMP46]] to i64
-; CHECK-INLINE-NEXT:    [[TMP48:%.*]] = add i64 [[TMP47]], 32
-; CHECK-INLINE-NEXT:    call void @__asan_alloca_poison(i64 [[TMP48]], i64 4)
-; CHECK-INLINE-NEXT:    [[TMP49:%.*]] = ptrtoint ptr [[TMP46]] to i64
-; CHECK-INLINE-NEXT:    store i64 [[TMP49]], ptr [[TMP0]], align 8
-; CHECK-INLINE-NEXT:    [[TMP50:%.*]] = inttoptr i64 [[TMP48]] to ptr
-; CHECK-INLINE-NEXT:    [[TMP51:%.*]] = alloca i8, i64 96, align 32
-; CHECK-INLINE-NEXT:    [[TMP52:%.*]] = ptrtoint ptr [[TMP51]] to i64
-; CHECK-INLINE-NEXT:    [[TMP53:%.*]] = add i64 [[TMP52]], 32
-; CHECK-INLINE-NEXT:    call void @__asan_alloca_poison(i64 [[TMP53]], i64 8)
-; CHECK-INLINE-NEXT:    [[TMP54:%.*]] = ptrtoint ptr [[TMP51]] to i64
-; CHECK-INLINE-NEXT:    store i64 [[TMP54]], ptr [[TMP0]], align 8
-; CHECK-INLINE-NEXT:    [[TMP55:%.*]] = inttoptr i64 [[TMP53]] to ptr
-; CHECK-INLINE-NEXT:    [[TMP56:%.*]] = lshr i64 [[TMP53]], 3
-; CHECK-INLINE-NEXT:    [[TMP57:%.*]] = add i64 [[TMP56]], [[TMP1]]
-; CHECK-INLINE-NEXT:    [[TMP58:%.*]] = inttoptr i64 [[TMP57]] to ptr
-; CHECK-INLINE-NEXT:    [[TMP59:%.*]] = load i8, ptr [[TMP58]], align 1
-; CHECK-INLINE-NEXT:    [[TMP60:%.*]] = icmp ne i8 [[TMP59]], 0
-; CHECK-INLINE-NEXT:    br i1 [[TMP60]], label [[TMP61:%.*]], label [[TMP62:%.*]]
-; CHECK-INLINE:       61:
-; CHECK-INLINE-NEXT:    call void @__asan_report_store8(i64 [[TMP53]]) #[[ATTR8]]
+; CHECK-INLINE:       50:
+; CHECK-INLINE-NEXT:    store volatile i8 0, ptr [[TMP22]], align 1
+; CHECK-INLINE-NEXT:    [[TMP51:%.*]] = add i64 [[TMP29]], 1066
+; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_f8(i64 [[TMP51]], i64 1)
+; CHECK-INLINE-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[TMP22]])
+; CHECK-INLINE-NEXT:    [[TMP52:%.*]] = alloca i8, i64 96, align 32
+; CHECK-INLINE-NEXT:    [[TMP53:%.*]] = ptrtoint ptr [[TMP52]] to i64
+; CHECK-INLINE-NEXT:    [[TMP54:%.*]] = add i64 [[TMP53]], 32
+; CHECK-INLINE-NEXT:    call void @__asan_alloca_poison(i64 [[TMP54]], i64 4)
+; CHECK-INLINE-NEXT:    [[TMP55:%.*]] = ptrtoint ptr [[TMP52]] to i64
+; CHECK-INLINE-NEXT:    store i64 [[TMP55]], ptr [[TMP0]], align 8
+; CHECK-INLINE-NEXT:    [[TMP56:%.*]] = inttoptr i64 [[TMP54]] to ptr
+; CHECK-INLINE-NEXT:    [[TMP57:%.*]] = alloca i8, i64 96, align 32
+; CHECK-INLINE-NEXT:    [[TMP58:%.*]] = ptrtoint ptr [[TMP57]] to i64
+; CHECK-INLINE-NEXT:    [[TMP59:%.*]] = add i64 [[TMP58]], 32
+; CHECK-INLINE-NEXT:    call void @__asan_alloca_poison(i64 [[TMP59]], i64 8)
+; CHECK-INLINE-NEXT:    [[TMP60:%.*]] = ptrtoint ptr [[TMP57]] to i64
+; CHECK-INLINE-NEXT:    store i64 [[TMP60]], ptr [[TMP0]], align 8
+; CHECK-INLINE-NEXT:    [[TMP61:%.*]] = inttoptr i64 [[TMP59]] to ptr
+; CHECK-INLINE-NEXT:    [[TMP62:%.*]] = lshr i64 [[TMP59]], 3
+; CHECK-INLINE-NEXT:    [[TMP63:%.*]] = add i64 [[TMP62]], [[TMP1]]
+; CHECK-INLINE-NEXT:    [[TMP64:%.*]] = inttoptr i64 [[TMP63]] to ptr
+; CHECK-INLINE-NEXT:    [[TMP65:%.*]] = load i8, ptr [[TMP64]], align 1
+; CHECK-INLINE-NEXT:    [[TMP66:%.*]] = icmp ne i8 [[TMP65]], 0
+; CHECK-INLINE-NEXT:    br i1 [[TMP66]], label [[TMP67:%.*]], label [[TMP68:%.*]]
+; CHECK-INLINE:       67:
+; CHECK-INLINE-NEXT:    call void @__asan_report_store8(i64 [[TMP59]]) #[[ATTR8]]
 ; CHECK-INLINE-NEXT:    unreachable
-; CHECK-INLINE:       62:
-; CHECK-INLINE-NEXT:    store volatile i64 0, ptr [[TMP55]], align 8
-; CHECK-INLINE-NEXT:    [[TMPCOPYI64:%.*]] = load i64, ptr [[TMP55]], align 8
-; CHECK-INLINE-NEXT:    [[TMP63:%.*]] = and i64 [[TMPCOPYI64]], 31
-; CHECK-INLINE-NEXT:    [[TMP64:%.*]] = sub i64 32, [[TMP63]]
-; CHECK-INLINE-NEXT:    [[TMP65:%.*]] = icmp ne i64 [[TMP64]], 32
-; CHECK-INLINE-NEXT:    [[TMP66:%.*]] = select i1 [[TMP65]], i64 [[TMP64]], i64 0
-; CHECK-INLINE-NEXT:    [[TMP67:%.*]] = add i64 64, [[TMP66]]
-; CHECK-INLINE-NEXT:    [[TMP68:%.*]] = add i64 [[TMPCOPYI64]], [[TMP67]]
-; CHECK-INLINE-NEXT:    [[TMP69:%.*]] = alloca i8, i64 [[TMP68]], align 32
-; CHECK-INLINE-NEXT:    [[TMP70:%.*]] = ptrtoint ptr [[TMP69]] to i64
-; CHECK-INLINE-NEXT:    [[TMP71:%.*]] = add i64 [[TMP70]], 32
-; CHECK-INLINE-NEXT:    call void @__asan_alloca_poison(i64 [[TMP71]], i64 [[TMPCOPYI64]])
-; CHECK-INLINE-NEXT:    [[TMP72:%.*]] = ptrtoint ptr [[TMP69]] to i64
-; CHECK-INLINE-NEXT:    store i64 [[TMP72]], ptr [[TMP0]], align 8
-; CHECK-INLINE-NEXT:    [[TMP73:%.*]] = inttoptr i64 [[TMP71]] to ptr
-; CHECK-INLINE-NEXT:    [[TMP74:%.*]] = lshr i64 [[TMP71]], 3
-; CHECK-INLINE-NEXT:    [[TMP75:%.*]] = add i64 [[TMP74]], [[TMP1]]
-; CHECK-INLINE-NEXT:    [[TMP76:%.*]] = inttoptr i64 [[TMP75]] to ptr
-; CHECK-INLINE-NEXT:    [[TMP77:%.*]] = load i8, ptr [[TMP76]], align 1
-; CHECK-INLINE-NEXT:    [[TMP78:%.*]] = icmp ne i8 [[TMP77]], 0
-; CHECK-INLINE-NEXT:    br i1 [[TMP78]], label [[TMP79:%.*]], label [[TMP84:%.*]], !prof [[PROF0]]
-; CHECK-INLINE:       79:
-; CHECK-INLINE-NEXT:    [[TMP80:%.*]] = and i64 [[TMP71]], 7
-; CHECK-INLINE-NEXT:    [[TMP81:%.*]] = trunc i64 [[TMP80]] to i8
-; CHECK-INLINE-NEXT:    [[TMP82:%.*]] = icmp sge i8 [[TMP81]], [[TMP77]]
-; CHECK-INLINE-NEXT:    br i1 [[TMP82]], label [[TMP83:%.*]], label [[TMP84]]
-; CHECK-INLINE:       83:
-; CHECK-INLINE-NEXT:    call void @__asan_report_store1(i64 [[TMP71]]) #[[ATTR8]]
+; CHECK-INLINE:       68:
+; CHECK-INLINE-NEXT:    store volatile i64 0, ptr [[TMP61]], align 8
+; CHECK-INLINE-NEXT:    [[TMPCOPYI64:%.*]] = load i64, ptr [[TMP61]], align 8
+; CHECK-INLINE-NEXT:    [[TMP69:%.*]] = and i64 [[TMPCOPYI64]], 31
+; CHECK-INLINE-NEXT:    [[TMP70:%.*]] = sub i64 32, [[TMP69]]
+; CHECK-INLINE-NEXT:    [[TMP71:%.*]] = icmp ne i64 [[TMP70]], 32
+; CHECK-INLINE-NEXT:    [[TMP72:%.*]] = select i1 [[TMP71]], i64 [[TMP70]], i64 0
+; CHECK-INLINE-NEXT:    [[TMP73:%.*]] = add i64 64, [[TMP72]]
+; CHECK-INLINE-NEXT:    [[TMP74:%.*]] = add i64 [[TMPCOPYI64]], [[TMP73]]
+; CHECK-INLINE-NEXT:    [[TMP75:%.*]] = alloca i8, i64 [[TMP74]], align 32
+; CHECK-INLINE-NEXT:    [[TMP76:%.*]] = ptrtoint ptr [[TMP75]] to i64
+; CHECK-INLINE-NEXT:    [[TMP77:%.*]] = add i64 [[TMP76]], 32
+; CHECK-INLINE-NEXT:    call void @__asan_alloca_poison(i64 [[TMP77]], i64 [[TMPCOPYI64]])
+; CHECK-INLINE-NEXT:    [[TMP78:%.*]] = ptrtoint ptr [[TMP75]] to i64
+; CHECK-INLINE-NEXT:    store i64 [[TMP78]], ptr [[TMP0]], align 8
+; CHECK-INLINE-NEXT:    [[TMP79:%.*]] = inttoptr i64 [[TMP77]] to ptr
+; CHECK-INLINE-NEXT:    [[TMP80:%.*]] = lshr i64 [[TMP77]], 3
+; CHECK-INLINE-NEXT:    [[TMP81:%.*]] = add i64 [[TMP80]], [[TMP1]]
+; CHECK-INLINE-NEXT:    [[TMP82:%.*]] = inttoptr i64 [[TMP81]] to ptr
+; CHECK-INLINE-NEXT:    [[TMP83:%.*]] = load i8, ptr [[TMP82]], align 1
+; CHECK-INLINE-NEXT:    [[TMP84:%.*]] = icmp ne i8 [[TMP83]], 0
+; CHECK-INLINE-NEXT:    br i1 [[TMP84]], label [[TMP85:%.*]], label [[TMP90:%.*]], !prof [[PROF0]]
+; CHECK-INLINE:       85:
+; CHECK-INLINE-NEXT:    [[TMP86:%.*]] = and i64 [[TMP77]], 7
+; CHECK-INLINE-NEXT:    [[TMP87:%.*]] = trunc i64 [[TMP86]] to i8
+; CHECK-INLINE-NEXT:    [[TMP88:%.*]] = icmp sge i8 [[TMP87]], [[TMP83]]
+; CHECK-INLINE-NEXT:    br i1 [[TMP88]], label [[TMP89:%.*]], label [[TMP90]]
+; CHECK-INLINE:       89:
+; CHECK-INLINE-NEXT:    call void @__asan_report_store1(i64 [[TMP77]]) #[[ATTR8]]
 ; CHECK-INLINE-NEXT:    unreachable
-; CHECK-INLINE:       84:
-; CHECK-INLINE-NEXT:    store volatile i8 0, ptr [[TMP73]], align 1
+; CHECK-INLINE:       90:
+; CHECK-INLINE-NEXT:    store volatile i8 0, ptr [[TMP79]], align 1
 ; CHECK-INLINE-NEXT:    invoke void @MayThrowFunc()
 ; CHECK-INLINE-NEXT:            to label [[INVOKE_CONT:%.*]] unwind label [[EHCLEANUP:%.*]]
 ; CHECK-INLINE:       invoke.cont:
 ; CHECK-INLINE-NEXT:    call void @DeInit(ptr [[TMP14]])
-; CHECK-INLINE-NEXT:    [[TMP85:%.*]] = ptrtoint ptr [[TMP0]] to i64
-; CHECK-INLINE-NEXT:    [[TMP86:%.*]] = load i64, ptr [[TMP0]], align 8
-; CHECK-INLINE-NEXT:    call void @__asan_allocas_unpoison(i64 [[TMP86]], i64 [[TMP85]])
-; CHECK-INLINE-NEXT:    store i64 1172321806, ptr [[TMP17]], align 8
-; CHECK-INLINE-NEXT:    [[TMP87:%.*]] = icmp ne i64 [[TMP7]], 0
-; CHECK-INLINE-NEXT:    br i1 [[TMP87]], label [[TMP88:%.*]], label [[TMP89:%.*]]
-; CHECK-INLINE:       88:
+; CHECK-INLINE-NEXT:    [[TMP91:%.*]] = ptrtoint ptr [[TMP0]] to i64
+; CHECK-INLINE-NEXT:    [[TMP92:%.*]] = load i64, ptr [[TMP0]], align 8
+; CHECK-INLINE-NEXT:    call void @__asan_allocas_unpoison(i64 [[TMP92]], i64 [[TMP91]])
+; CHECK-INLINE-NEXT:    store i64 1172321806, ptr [[TMP23]], align 8
+; CHECK-INLINE-NEXT:    [[TMP93:%.*]] = icmp ne i64 [[TMP7]], 0
+; CHECK-INLINE-NEXT:    br i1 [[TMP93]], label [[TMP94:%.*]], label [[TMP95:%.*]]
+; CHECK-INLINE:       94:
 ; CHECK-INLINE-NEXT:    call void @__asan_stack_free_8(i64 [[TMP7]], i64 8544)
-; CHECK-INLINE-NEXT:    br label [[TMP91:%.*]]
-; CHECK-INLINE:       89:
-; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_00(i64 [[TMP23]], i64 4)
-; CHECK-INLINE-NEXT:    [[TMP90:%.*]] = add i64 [[TMP23]], 1028
-; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_00(i64 [[TMP90]], i64 40)
-; CHECK-INLINE-NEXT:    br label [[TMP91]]
-; CHECK-INLINE:       91:
+; CHECK-INLINE-NEXT:    br label [[TMP97:%.*]]
+; CHECK-INLINE:       95:
+; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_00(i64 [[TMP29]], i64 4)
+; CHECK-INLINE-NEXT:    [[TMP96:%.*]] = add i64 [[TMP29]], 1028
+; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_00(i64 [[TMP96]], i64 40)
+; CHECK-INLINE-NEXT:    br label [[TMP97]]
+; CHECK-INLINE:       97:
 ; CHECK-INLINE-NEXT:    ret void
 ; CHECK-INLINE:       ehcleanup:
-; CHECK-INLINE-NEXT:    [[TMP92:%.*]] = cleanuppad within none []
+; CHECK-INLINE-NEXT:    [[TMP98:%.*]] = cleanuppad within none []
+; CHECK-INLINE-NEXT:    call void @__asan_unpoison_stack_memory(i64 [[TMP54]], i64 4) [ "funclet"(token [[TMP98]]) ]
+; CHECK-INLINE-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[TMP56]])
+; CHECK-INLINE-NEXT:    [[TMP99:%.*]] = lshr i64 [[TMP54]], 3
+; CHECK-INLINE-NEXT:    [[TMP100:%.*]] = add i64 [[TMP99]], [[TMP1]]
+; CHECK-INLINE-NEXT:    [[TMP101:%.*]] = inttoptr i64 [[TMP100]] to ptr
+; CHECK-INLINE-NEXT:    [[TMP102:%.*]] = load i8, ptr [[TMP101]], align 1
+; CHECK-INLINE-NEXT:    [[TMP103:%.*]] = icmp ne i8 [[TMP102]], 0
+; CHECK-INLINE-NEXT:    br i1 [[TMP103]], label [[TMP104:%.*]], label [[TMP109:%.*]], !prof [[PROF0]]
+; CHECK-INLINE:       104:
+; CHECK-INLINE-NEXT:    [[TMP105:%.*]] = and i64 [[TMP54]], 7
+; CHECK-INLINE-NEXT:    [[TMP106:%.*]] = trunc i64 [[TMP105]] to i8
+; CHECK-INLINE-NEXT:    [[TMP107:%.*]] = icmp sge i8 [[TMP106]], [[TMP102]]
+; CHECK-INLINE-NEXT:    br i1 [[TMP107]], label [[TMP108:%.*]], label [[TMP109]]
+; CHECK-INLINE:       108:
+; CHECK-INLINE-NEXT:    call void @__asan_report_store1(i64 [[TMP54]]) #[[ATTR8]] [ "funclet"(token [[TMP98]]) ]
+; CHECK-INLINE-NEXT:    unreachable
+; CHECK-INLINE:       109:
+; CHECK-INLINE-NEXT:    store volatile i8 0, ptr [[TMP56]], align 1
+; CHECK-INLINE-NEXT:    call void @__asan_poison_stack_memory(i64 [[TMP54]], i64 4) [ "funclet"(token [[TMP98]]) ]
+; CHECK-INLINE-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[TMP56]])
+; CHECK-INLINE-NEXT:    call void @DeInit(ptr [[TMP14]]) [ "funclet"(token [[TMP98]]) ]
+; CHECK-INLINE-NEXT:    [[TMP110:%.*]] = call ptr @__asan_memset(ptr [[TMP16]], i32 0, i64 4) [ "funclet"(token [[TMP98]]) ]
+; CHECK-INLINE-NEXT:    [[TMP111:%.*]] = call ptr @__asan_memcpy(ptr [[TMP18]], ptr [[TMP16]], i64 4) [ "funclet"(token [[TMP98]]) ]
+; CHECK-INLINE-NEXT:    [[TMP112:%.*]] = call ptr @__asan_memmove(ptr [[TMP20]], ptr [[TMP16]], i64 4) [ "funclet"(token [[TMP98]]) ]
+; CHECK-INLINE-NEXT:    call void @__sanitizer_ptr_cmp(i64 [[TMP15]], i64 [[TMP17]]) [ "funclet"(token [[TMP98]]) ]
+; CHECK-INLINE-NEXT:    call void @__sanitizer_ptr_sub(i64 [[TMP15]], i64 [[TMP17]]) [ "funclet"(token [[TMP98]]) ]
+; CHECK-INLINE-NEXT:    [[TMP113:%.*]] = ptrtoint ptr [[PTRPARAM]] to i64
+; CHECK-INLINE-NEXT:    [[TMP114:%.*]] = add i64 [[TMP113]], 7
+; CHECK-INLINE-NEXT:    [[TMP115:%.*]] = inttoptr i64 [[TMP114]] to ptr
+; CHECK-INLINE-NEXT:    [[TMP116:%.*]] = ptrtoint ptr [[PTRPARAM]] to i64
+; CHECK-INLINE-NEXT:    [[TMP117:%.*]] = lshr i64 [[TMP116]], 3
+; CHECK-INLINE-NEXT:    [[TMP118:%.*]] = add i64 [[TMP117]], [[TMP1]]
+; CHECK-INLINE-NEXT:    [[TMP119:%.*]] = inttoptr i64 [[TMP118]] to ptr
+; CHECK-INLINE-NEXT:    [[TMP120:%.*]] = load i8, ptr [[TMP119]], align 1
+; CHECK-INLINE-NEXT:    [[TMP121:%.*]] = icmp ne i8 [[TMP120]], 0
+; CHECK-INLINE-NEXT:    br i1 [[TMP121]], label [[TMP122:%.*]], label [[TMP127:%.*]], !prof [[PROF0]]
+; CHECK-INLINE:       122:
+; CHECK-INLINE-NEXT:    [[TMP123:%.*]] = and i64 [[TMP116]], 7
+; CHECK-INLINE-NEXT:    [[TMP124:%.*]] = trunc i64 [[TMP123]] to i8
+; CHECK-INLINE-NEXT:    [[TMP125:%.*]] = icmp sge i8 [[TMP124]], [[TMP120]]
+; CHECK-INLINE-NEXT:    br i1 [[TMP125]], label [[TMP126:%.*]], label [[TMP127]]
+; CHECK-INLINE:       126:
+; CHECK-INLINE-NEXT:    call void @__asan_report_store_n(i64 [[TMP116]], i64 8) #[[ATTR8]] [ "funclet"(token [[TMP98]]) ]
+; CHECK-INLINE-NEXT:    unreachable
+; CHECK-INLINE:       127:
+; CHECK-INLINE-NEXT:    [[TMP128:%.*]] = lshr i64 [[TMP114]], 3
+; CHECK-INLINE-NEXT:    [[TMP129:%.*]] = add i64 [[TMP128]], [[TMP1]]
+; CHECK-INLINE-NEXT:    [[TMP130:%.*]] = inttoptr i64 [[TMP129]] to ptr
+; CHECK-INLINE-NEXT:    [[TMP131:%.*]] = load i8, ptr [[TMP130]], align 1
+; CHECK-INLINE-NEXT:    [[TMP132:%.*]] = icmp ne i8 [[TMP131]], 0
+; CHECK-INLINE-NEXT:    br i1 [[TMP132]], label [[TMP133:%.*]], label [[EHEXIT:%.*]], !prof [[PROF0]]
+; CHECK-INLINE:       133:
+; CHECK-INLINE-NEXT:    [[TMP134:%.*]] = and i64 [[TMP114]], 7
+; CHECK-INLINE-NEXT:    [[TMP135:%.*]] = trunc i64 [[TMP134]] to i8
+; CHECK-INLINE-NEXT:    [[TMP136:%.*]] = icmp sge i8 [[TMP135]], [[TMP131]]
+; CHECK-INLINE-NEXT:    br i1 [[TMP136]], label [[TMP137:%.*]], label [[EHEXIT]]
+; CHECK-INLINE:       137:
+; CHECK-INLINE-NEXT:    call void @__asan_report_store_n(i64 [[TMP114]], i64 8) #[[ATTR8]] [ "funclet"(token [[TMP98]]) ]
 ; CHECK-INLINE-NEXT:    unreachable
+; CHECK-INLINE:       ehexit:
+; CHECK-INLINE-NEXT:    store i64 0, ptr [[PTRPARAM]], align 1
+; CHECK-INLINE-NEXT:    [[TMP138:%.*]] = call i64 @llvm.get.dynamic.area.offset.i64()
+; CHECK-INLINE-NEXT:    [[TMP139:%.*]] = ptrtoint ptr [[TMP0]] to i64
+; CHECK-INLINE-NEXT:    [[TMP140:%.*]] = add i64 [[TMP139]], [[TMP138]]
+; CHECK-INLINE-NEXT:    [[TMP141:%.*]] = load i64, ptr [[TMP0]], align 8
+; CHECK-INLINE-NEXT:    call void @__asan_allocas_unpoison(i64 [[TMP141]], i64 [[TMP140]]) [ "funclet"(token [[TMP98]]) ]
+; CHECK-INLINE-NEXT:    store i64 1172321806, ptr [[TMP23]], align 8
+; CHECK-INLINE-NEXT:    [[TMP142:%.*]] = icmp ne i64 [[TMP7]], 0
+; CHECK-INLINE-NEXT:    br i1 [[TMP142]], label [[TMP143:%.*]], label [[TMP144:%.*]]
+; CHECK-INLINE:       143:
+; CHECK-INLINE-NEXT:    call void @__asan_stack_free_8(i64 [[TMP7]], i64 8544) [ "funclet"(token [[TMP98]]) ]
+; CHECK-INLINE-NEXT:    br label [[TMP146:%.*]]
+; CHECK-INLINE:       144:
+; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_00(i64 [[TMP29]], i64 4) [ "funclet"(token [[TMP98]]) ]
+; CHECK-INLINE-NEXT:    [[TMP145:%.*]] = add i64 [[TMP29]], 1028
+; CHECK-INLINE-NEXT:    call void @__asan_set_shadow_00(i64 [[TMP145]], i64 40) [ "funclet"(token [[TMP98]]) ]
+; CHECK-INLINE-NEXT:    br label [[TMP146]]
+; CHECK-INLINE:       146:
+; CHECK-INLINE-NEXT:    cleanupret from [[TMP98]] unwind to caller
 ;
 ; CHECK-OUTLINE-LABEL: define void @FuncletPersonality(
 ; CHECK-OUTLINE-SAME: ptr [[PTRPARAM:%.*]]) #[[ATTR4:[0-9]+]] personality ptr @__CxxFrameHandler3 {
@@ -211,96 +292,140 @@ define void @FuncletPersonality(ptr %ptrParam) sanitize_address personality ptr
 ; CHECK-OUTLINE-NEXT:    store i64 [[TMP12]], ptr [[ASAN_LOCAL_STACK_BASE]], align 8
 ; CHECK-OUTLINE-NEXT:    [[TMP13:%.*]] = add i64 [[TMP12]], 32
 ; CHECK-OUTLINE-NEXT:    [[TMP14:%.*]] = inttoptr i64 [[TMP13]] to ptr
-; CHECK-OUTLINE-NEXT:    [[TMP15:%.*]] = add i64 [[TMP12]], 8528
+; CHECK-OUTLINE-NEXT:    [[TMP15:%.*]] = add i64 [[TMP12]], 8480
 ; CHECK-OUTLINE-NEXT:    [[TMP16:%.*]] = inttoptr i64 [[TMP15]] to ptr
-; CHECK-OUTLINE-NEXT:    [[TMP17:%.*]] = add i64 [[TMP12]], 8560
+; CHECK-OUTLINE-NEXT:    [[TMP17:%.*]] = add i64 [[TMP12]], 8496
 ; CHECK-OUTLINE-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
-; CHECK-OUTLINE-NEXT:    [[TMP19:%.*]] = inttoptr i64 [[TMP12]] to ptr
-; CHECK-OUTLINE-NEXT:    store i64 1102416563, ptr [[TMP19]], align 8
-; CHECK-OUTLINE-NEXT:    [[TMP20:%.*]] = add i64 [[TMP12]], 8
-; CHECK-OUTLINE-NEXT:    [[TMP21:%.*]] = inttoptr i64 [[TMP20]] to ptr
-; CHECK-OUTLINE-NEXT:    store i64 ptrtoint (ptr @___asan_gen_ to i64), ptr [[TMP21]], align 8
-; CHECK-OUTLINE-NEXT:    [[TMP22:%.*]] = add i64 [[TMP12]], 16
-; CHECK-OUTLINE-NEXT:    [[TMP23:%.*]] = inttoptr i64 [[TMP22]] to ptr
-; CHECK-OUTLINE-NEXT:    store i64 ptrtoint (ptr @FuncletPersonality to i64), ptr [[TMP23]], align 8
-; CHECK-OUTLINE-NEXT:    [[TMP24:%.*]] = lshr i64 [[TMP12]], 3
-; CHECK-OUTLINE-NEXT:    [[TMP25:%.*]] = add i64 [[TMP24]], [[TMP1]]
-; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_f1(i64 [[TMP25]], i64 4)
-; CHECK-OUTLINE-NEXT:    [[TMP26:%.*]] = add i64 [[TMP25]], 1028
-; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_f2(i64 [[TMP26]], i64 32)
-; CHECK-OUTLINE-NEXT:    [[TMP27:%.*]] = add i64 [[TMP25]], 1060
-; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_04(i64 [[TMP27]], i64 1)
-; CHECK-OUTLINE-NEXT:    [[TMP28:%.*]] = add i64 [[TMP25]], 1061
-; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_f2(i64 [[TMP28]], i64 1)
-; CHECK-OUTLINE-NEXT:    [[TMP29:%.*]] = add i64 [[TMP25]], 1062
-; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_04(i64 [[TMP29]], i64 1)
-; CHECK-OUTLINE-NEXT:    [[TMP30:%.*]] = add i64 [[TMP25]], 1063
-; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_f2(i64 [[TMP30]], i64 1)
-; CHECK-OUTLINE-NEXT:    [[TMP31:%.*]] = add i64 [[TMP25]], 1064
-; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_04(i64 [[TMP31]], i64 1)
-; CHECK-OUTLINE-NEXT:    [[TMP32:%.*]] = add i64 [[TMP25]], 1065
-; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_f2(i64 [[TMP32]], i64 1)
-; CHECK-OUTLINE-NEXT:    [[TMP33:%.*]] = add i64 [[TMP25]], 1066
-; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_f8(i64 [[TMP33]], i64 1)
-; CHECK-OUTLINE-NEXT:    [[TMP34:%.*]] = add i64 [[TMP25]], 1067
-; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_f2(i64 [[TMP34]], i64 1)
-; CHECK-OUTLINE-NEXT:    [[TMP35:%.*]] = add i64 [[TMP25]], 1068
-; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_f8(i64 [[TMP35]], i64 1)
-; CHECK-OUTLINE-NEXT:    [[TMP36:%.*]] = add i64 [[TMP25]], 1069
+; CHECK-OUTLINE-NEXT:    [[TMP19:%.*]] = add i64 [[TMP12]], 8512
+; CHECK-OUTLINE-NEXT:    [[TMP20:%.*]] = inttoptr i64 [[TMP19]] to ptr
+; CHECK-OUTLINE-NEXT:    [[TMP21:%.*]] = add i64 [[TMP12]], 8528
+; CHECK-OUTLINE-NEXT:    [[TMP22:%.*]] = inttoptr i64 [[TMP21]] to ptr
+; CHECK-OUTLINE-NEXT:    [[TMP23:%.*]] = add i64 [[TMP12]], 8544
+; CHECK-OUTLINE-NEXT:    [[TMP24:%.*]] = inttoptr i64 [[TMP23]] to ptr
+; CHECK-OUTLINE-NEXT:    [[TMP25:%.*]] = add i64 [[TMP12]], 8560
+; CHECK-OUTLINE-NEXT:    [[TMP26:%.*]] = inttoptr i64 [[TMP25]] to ptr
+; CHECK-OUTLINE-NEXT:    [[TMP27:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-OUTLINE-NEXT:    store i64 1102416563, ptr [[TMP27]], align 8
+; CHECK-OUTLINE-NEXT:    [[TMP28:%.*]] = add i64 [[TMP12]], 8
+; CHECK-OUTLINE-NEXT:    [[TMP29:%.*]] = inttoptr i64 [[TMP28]] to ptr
+; CHECK-OUTLINE-NEXT:    store i64 ptrtoint (ptr @___asan_gen_ to i64), ptr [[TMP29]], align 8
+; CHECK-OUTLINE-NEXT:    [[TMP30:%.*]] = add i64 [[TMP12]], 16
+; CHECK-OUTLINE-NEXT:    [[TMP31:%.*]] = inttoptr i64 [[TMP30]] to ptr
+; CHECK-OUTLINE-NEXT:    store i64 ptrtoint (ptr @FuncletPersonality to i64), ptr [[TMP31]], align 8
+; CHECK-OUTLINE-NEXT:    [[TMP32:%.*]] = lshr i64 [[TMP12]], 3
+; CHECK-OUTLINE-NEXT:    [[TMP33:%.*]] = add i64 [[TMP32]], [[TMP1]]
+; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_f1(i64 [[TMP33]], i64 4)
+; CHECK-OUTLINE-NEXT:    [[TMP34:%.*]] = add i64 [[TMP33]], 1028
+; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_f2(i64 [[TMP34]], i64 32)
+; CHECK-OUTLINE-NEXT:    [[TMP35:%.*]] = add i64 [[TMP33]], 1060
+; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_04(i64 [[TMP35]], i64 1)
+; CHECK-OUTLINE-NEXT:    [[TMP36:%.*]] = add i64 [[TMP33]], 1061
 ; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_f2(i64 [[TMP36]], i64 1)
-; CHECK-OUTLINE-NEXT:    [[TMP37:%.*]] = add i64 [[TMP25]], 1071
-; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_f3(i64 [[TMP37]], i64 5)
-; CHECK-OUTLINE-NEXT:    [[TMP38:%.*]] = add i64 [[TMP25]], 1066
-; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_04(i64 [[TMP38]], i64 1)
-; CHECK-OUTLINE-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[TMP16]])
-; CHECK-OUTLINE-NEXT:    call void @__asan_store1(i64 [[TMP15]])
-; CHECK-OUTLINE-NEXT:    store volatile i8 0, ptr [[TMP16]], align 1
-; CHECK-OUTLINE-NEXT:    [[TMP39:%.*]] = add i64 [[TMP25]], 1066
-; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_f8(i64 [[TMP39]], i64 1)
-; CHECK-OUTLINE-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[TMP16]])
-; CHECK-OUTLINE-NEXT:    call void @__asan_store8(i64 [[TMP17]])
-; CHECK-OUTLINE-NEXT:    store volatile i64 0, ptr [[TMP18]], align 8
-; CHECK-OUTLINE-NEXT:    [[TMPCOPYI64:%.*]] = load i64, ptr [[TMP18]], align 8
-; CHECK-OUTLINE-NEXT:    [[TMP40:%.*]] = and i64 [[TMPCOPYI64]], 31
-; CHECK-OUTLINE-NEXT:    [[TMP41:%.*]] = sub i64 32, [[TMP40]]
-; CHECK-OUTLINE-NEXT:    [[TMP42:%.*]] = icmp ne i64 [[TMP41]], 32
-; CHECK-OUTLINE-NEXT:    [[TMP43:%.*]] = select i1 [[TMP42]], i64 [[TMP41]], i64 0
-; CHECK-OUTLINE-NEXT:    [[TMP44:%.*]] = add i64 64, [[TMP43]]
-; CHECK-OUTLINE-NEXT:    [[TMP45:%.*]] = add i64 [[TMPCOPYI64]], [[TMP44]]
-; CHECK-OUTLINE-NEXT:    [[TMP46:%.*]] = alloca i8, i64 [[TMP45]], align 32
-; CHECK-OUTLINE-NEXT:    [[TMP47:%.*]] = ptrtoint ptr [[TMP46]] to i64
-; CHECK-OUTLINE-NEXT:    [[TMP48:%.*]] = add i64 [[TMP47]], 32
-; CHECK-OUTLINE-NEXT:    call void @__asan_alloca_poison(i64 [[TMP48]], i64 [[TMPCOPYI64]])
-; CHECK-OUTLINE-NEXT:    [[TMP49:%.*]] = ptrtoint ptr [[TMP46]] to i64
-; CHECK-OUTLINE-NEXT:    store i64 [[TMP49]], ptr [[TMP0]], align 8
-; CHECK-OUTLINE-NEXT:    [[TMP50:%.*]] = inttoptr i64 [[TMP48]] to ptr
-; CHECK-OUTLINE-NEXT:    call void @__asan_store1(i64 [[TMP48]])
-; CHECK-OUTLINE-NEXT:    store volatile i8 0, ptr [[TMP50]], align 1
+; CHECK-OUTLINE-NEXT:    [[TMP37:%.*]] = add i64 [[TMP33]], 1062
+; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_04(i64 [[TMP37]], i64 1)
+; CHECK-OUTLINE-NEXT:    [[TMP38:%.*]] = add i64 [[TMP33]], 1063
+; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_f2(i64 [[TMP38]], i64 1)
+; CHECK-OUTLINE-NEXT:    [[TMP39:%.*]] = add i64 [[TMP33]], 1064
+; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_04(i64 [[TMP39]], i64 1)
+; CHECK-OUTLINE-NEXT:    [[TMP40:%.*]] = add i64 [[TMP33]], 1065
+; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_f2(i64 [[TMP40]], i64 1)
+; CHECK-OUTLINE-NEXT:    [[TMP41:%.*]] = add i64 [[TMP33]], 1066
+; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_f8(i64 [[TMP41]], i64 1)
+; CHECK-OUTLINE-NEXT:    [[TMP42:%.*]] = add i64 [[TMP33]], 1067
+; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_f2(i64 [[TMP42]], i64 1)
+; CHECK-OUTLINE-NEXT:    [[TMP43:%.*]] = add i64 [[TMP33]], 1068
+; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_f8(i64 [[TMP43]], i64 1)
+; CHECK-OUTLINE-NEXT:    [[TMP44:%.*]] = add i64 [[TMP33]], 1069
+; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_f2(i64 [[TMP44]], i64 1)
+; CHECK-OUTLINE-NEXT:    [[TMP45:%.*]] = add i64 [[TMP33]], 1071
+; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_f3(i64 [[TMP45]], i64 5)
+; CHECK-OUTLINE-NEXT:    [[TMP46:%.*]] = add i64 [[TMP33]], 1066
+; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_04(i64 [[TMP46]], i64 1)
+; CHECK-OUTLINE-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[TMP22]])
+; CHECK-OUTLINE-NEXT:    call void @__asan_store1(i64 [[TMP21]])
+; CHECK-OUTLINE-NEXT:    store volatile i8 0, ptr [[TMP22]], align 1
+; CHECK-OUTLINE-NEXT:    [[TMP47:%.*]] = add i64 [[TMP33]], 1066
+; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_f8(i64 [[TMP47]], i64 1)
+; CHECK-OUTLINE-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[TMP22]])
+; CHECK-OUTLINE-NEXT:    call void @__asan_store8(i64 [[TMP25]])
+; CHECK-OUTLINE-NEXT:    store volatile i64 0, ptr [[TMP26]], align 8
+; CHECK-OUTLINE-NEXT:    [[TMPCOPYI64:%.*]] = load i64, ptr [[TMP26]], align 8
+; CHECK-OUTLINE-NEXT:    [[TMP48:%.*]] = and i64 [[TMPCOPYI64]], 31
+; CHECK-OUTLINE-NEXT:    [[TMP49:%.*]] = sub i64 32, [[TMP48]]
+; CHECK-OUTLINE-NEXT:    [[TMP50:%.*]] = icmp ne i64 [[TMP49]], 32
+; CHECK-OUTLINE-NEXT:    [[TMP51:%.*]] = select i1 [[TMP50]], i64 [[TMP49]], i64 0
+; CHECK-OUTLINE-NEXT:    [[TMP52:%.*]] = add i64 64, [[TMP51]]
+; CHECK-OUTLINE-NEXT:    [[TMP53:%.*]] = add i64 [[TMPCOPYI64]], [[TMP52]]
+; CHECK-OUTLINE-NEXT:    [[TMP54:%.*]] = alloca i8, i64 [[TMP53]], align 32
+; CHECK-OUTLINE-NEXT:    [[TMP55:%.*]] = ptrtoint ptr [[TMP54]] to i64
+; CHECK-OUTLINE-NEXT:    [[TMP56:%.*]] = add i64 [[TMP55]], 32
+; CHECK-OUTLINE-NEXT:    call void @__asan_alloca_poison(i64 [[TMP56]], i64 [[TMPCOPYI64]])
+; CHECK-OUTLINE-NEXT:    [[TMP57:%.*]] = ptrtoint ptr [[TMP54]] to i64
+; CHECK-OUTLINE-NEXT:    store i64 [[TMP57]], ptr [[TMP0]], align 8
+; CHECK-OUTLINE-NEXT:    [[TMP58:%.*]] = inttoptr i64 [[TMP56]] to ptr
+; CHECK-OUTLINE-NEXT:    call void @__asan_store1(i64 [[TMP56]])
+; CHECK-OUTLINE-NEXT:    store volatile i8 0, ptr [[TMP58]], align 1
 ; CHECK-OUTLINE-NEXT:    invoke void @MayThrowFunc()
 ; CHECK-OUTLINE-NEXT:            to label [[INVOKE_CONT:%.*]] unwind label [[EHCLEANUP:%.*]]
 ; CHECK-OUTLINE:       invoke.cont:
 ; CHECK-OUTLINE-NEXT:    call void @DeInit(ptr [[TMP14]])
-; CHECK-OUTLINE-NEXT:    [[TMP51:%.*]] = ptrtoint ptr [[TMP0]] to i64
-; CHECK-OUTLINE-NEXT:    [[TMP52:%.*]] = load i64, ptr [[TMP0]], align 8
-; CHECK-OUTLINE-NEXT:    call void @__asan_allocas_unpoison(i64 [[TMP52]], i64 [[TMP51]])
-; CHECK-OUTLINE-NEXT:    store i64 1172321806, ptr [[TMP19]], align 8
-; CHECK-OUTLINE-NEXT:    [[TMP53:%.*]] = icmp ne i64 [[TMP7]], 0
-; CHECK-OUTLINE-NEXT:    br i1 [[TMP53]], label [[TMP54:%.*]], label [[TMP55:%.*]]
-; CHECK-OUTLINE:       54:
+; CHECK-OUTLINE-NEXT:    [[TMP59:%.*]] = ptrtoint ptr [[TMP0]] to i64
+; CHECK-OUTLINE-NEXT:    [[TMP60:%.*]] = load i64, ptr [[TMP0]], align 8
+; CHECK-OUTLINE-NEXT:    call void @__asan_allocas_unpoison(i64 [[TMP60]], i64 [[TMP59]])
+; CHECK-OUTLINE-NEXT:    store i64 1172321806, ptr [[TMP27]], align 8
+; CHECK-OUTLINE-NEXT:    [[TMP61:%.*]] = icmp ne i64 [[TMP7]], 0
+; CHECK-OUTLINE-NEXT:    br i1 [[TMP61]], label [[TMP62:%.*]], label [[TMP63:%.*]]
+; CHECK-OUTLINE:       62:
 ; CHECK-OUTLINE-NEXT:    call void @__asan_stack_free_8(i64 [[TMP7]], i64 8608)
-; CHECK-OUTLINE-NEXT:    br label [[TMP58:%.*]]
-; CHECK-OUTLINE:       55:
-; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_00(i64 [[TMP25]], i64 4)
-; CHECK-OUTLINE-NEXT:    [[TMP56:%.*]] = add i64 [[TMP25]], 1028
-; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_00(i64 [[TMP56]], i64 42)
-; CHECK-OUTLINE-NEXT:    [[TMP57:%.*]] = add i64 [[TMP25]], 1071
-; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_00(i64 [[TMP57]], i64 5)
-; CHECK-OUTLINE-NEXT:    br label [[TMP58]]
-; CHECK-OUTLINE:       58:
+; CHECK-OUTLINE-NEXT:    br label [[TMP66:%.*]]
+; CHECK-OUTLINE:       63:
+; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_00(i64 [[TMP33]], i64 4)
+; CHECK-OUTLINE-NEXT:    [[TMP64:%.*]] = add i64 [[TMP33]], 1028
+; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_00(i64 [[TMP64]], i64 42)
+; CHECK-OUTLINE-NEXT:    [[TMP65:%.*]] = add i64 [[TMP33]], 1071
+; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_00(i64 [[TMP65]], i64 5)
+; CHECK-OUTLINE-NEXT:    br label [[TMP66]]
+; CHECK-OUTLINE:       66:
 ; CHECK-OUTLINE-NEXT:    ret void
 ; CHECK-OUTLINE:       ehcleanup:
-; CHECK-OUTLINE-NEXT:    [[TMP59:%.*]] = cleanuppad within none []
-; CHECK-OUTLINE-NEXT:    unreachable
+; CHECK-OUTLINE-NEXT:    [[TMP67:%.*]] = cleanuppad within none []
+; CHECK-OUTLINE-NEXT:    [[TMP68:%.*]] = add i64 [[TMP33]], 1068
+; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_04(i64 [[TMP68]], i64 1) [ "funclet"(token [[TMP67]]) ]
+; CHECK-OUTLINE-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[TMP24]])
+; CHECK-OUTLINE-NEXT:    call void @__asan_store1(i64 [[TMP23]]) [ "funclet"(token [[TMP67]]) ]
+; CHECK-OUTLINE-NEXT:    store volatile i8 0, ptr [[TMP24]], align 1
+; CHECK-OUTLINE-NEXT:    [[TMP69:%.*]] = add i64 [[TMP33]], 1068
+; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_f8(i64 [[TMP69]], i64 1) [ "funclet"(token [[TMP67]]) ]
+; CHECK-OUTLINE-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[TMP24]])
+; CHECK-OUTLINE-NEXT:    call void @DeInit(ptr [[TMP14]]) [ "funclet"(token [[TMP67]]) ]
+; CHECK-OUTLINE-NEXT:    [[TMP70:%.*]] = call ptr @__asan_memset(ptr [[TMP16]], i32 0, i64 4) [ "funclet"(token [[TMP67]]) ]
+; CHECK-OUTLINE-NEXT:    [[TMP71:%.*]] = call ptr @__asan_memcpy(ptr [[TMP18]], ptr [[TMP16]], i64 4) [ "funclet"(token [[TMP67]]) ]
+; CHECK-OUTLINE-NEXT:    [[TMP72:%.*]] = call ptr @__asan_memmove(ptr [[TMP20]], ptr [[TMP16]], i64 4) [ "funclet"(token [[TMP67]]) ]
+; CHECK-OUTLINE-NEXT:    call void @__sanitizer_ptr_cmp(i64 [[TMP15]], i64 [[TMP17]]) [ "funclet"(token [[TMP67]]) ]
+; CHECK-OUTLINE-NEXT:    call void @__sanitizer_ptr_sub(i64 [[TMP15]], i64 [[TMP17]]) [ "funclet"(token [[TMP67]]) ]
+; CHECK-OUTLINE-NEXT:    [[TMP73:%.*]] = ptrtoint ptr [[PTRPARAM]] to i64
+; CHECK-OUTLINE-NEXT:    call void @__asan_storeN(i64 [[TMP73]], i64 8) [ "funclet"(token [[TMP67]]) ]
+; CHECK-OUTLINE-NEXT:    store i64 0, ptr [[PTRPARAM]], align 1
+; CHECK-OUTLINE-NEXT:    [[TMP74:%.*]] = call i64 @llvm.get.dynamic.area.offset.i64()
+; CHECK-OUTLINE-NEXT:    [[TMP75:%.*]] = ptrtoint ptr [[TMP0]] to i64
+; CHECK-OUTLINE-NEXT:    [[TMP76:%.*]] = add i64 [[TMP75]], [[TMP74]]
+; CHECK-OUTLINE-NEXT:    [[TMP77:%.*]] = load i64, ptr [[TMP0]], align 8
+; CHECK-OUTLINE-NEXT:    call void @__asan_allocas_unpoison(i64 [[TMP77]], i64 [[TMP76]]) [ "funclet"(token [[TMP67]]) ]
+; CHECK-OUTLINE-NEXT:    store i64 1172321806, ptr [[TMP27]], align 8
+; CHECK-OUTLINE-NEXT:    [[TMP78:%.*]] = icmp ne i64 [[TMP7]], 0
+; CHECK-OUTLINE-NEXT:    br i1 [[TMP78]], label [[TMP79:%.*]], label [[TMP80:%.*]]
+; CHECK-OUTLINE:       79:
+; CHECK-OUTLINE-NEXT:    call void @__asan_stack_free_8(i64 [[TMP7]], i64 8608) [ "funclet"(token [[TMP67]]) ]
+; CHECK-OUTLINE-NEXT:    br label [[TMP83:%.*]]
+; CHECK-OUTLINE:       80:
+; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_00(i64 [[TMP33]], i64 4) [ "funclet"(token [[TMP67]]) ]
+; CHECK-OUTLINE-NEXT:    [[TMP81:%.*]] = add i64 [[TMP33]], 1028
+; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_00(i64 [[TMP81]], i64 42) [ "funclet"(token [[TMP67]]) ]
+; CHECK-OUTLINE-NEXT:    [[TMP82:%.*]] = add i64 [[TMP33]], 1071
+; CHECK-OUTLINE-NEXT:    call void @__asan_set_shadow_00(i64 [[TMP82]], i64 5) [ "funclet"(token [[TMP67]]) ]
+; CHECK-OUTLINE-NEXT:    br label [[TMP83]]
+; CHECK-OUTLINE:       83:
+; CHECK-OUTLINE-NEXT:    cleanupret from [[TMP67]] unwind to caller
 ;
 
 
diff --git a/llvm/test/Instrumentation/AddressSanitizer/localescape.ll b/llvm/test/Instrumentation/AddressSanitizer/localescape.ll
index 4bce2203b3ee..9806dec0fd94 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/localescape.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/localescape.ll
@@ -14,10 +14,10 @@ declare ptr @llvm.eh.recoverfp(ptr, ptr)
 declare ptr @llvm.localrecover(ptr, ptr, i32)
 declare void @llvm.localescape(...) #1
 
-declare i32 @_except_handler3(...)
+declare i32 @__gcc_personality_v0(...)
 declare void @may_throw(ptr %r)
 
-define i32 @main() sanitize_address personality ptr @_except_handler3 {
+define i32 @main() sanitize_address personality ptr @__gcc_personality_v0 {
 entry:
   %r = alloca i32, align 4
   %__exception_code = alloca i32, align 4
-- 
GitLab


From 9fb552812c65a13cd19eedea1cc2ef739ad652e9 Mon Sep 17 00:00:00 2001
From: Piotr Zegar 
Date: Fri, 8 Mar 2024 18:34:26 +0100
Subject: [PATCH 0059/1674] [clang-tidy] Fix Hungarian Prefix in
 readability-identifier-naming (#84236)

Fix handling of Hungarian Prefix when configured to LowerCase.
Warnings no longer will be emited for names that already match
this style.

Fixes: #80268
---
 .../readability/IdentifierNamingCheck.cpp     |  10 +-
 clang-tools-extra/docs/ReleaseNotes.rst       |   3 +-
 .../checks/readability/identifier-naming.rst  |  28 +-
 .../hungarian-notation3/.clang-tidy           |  60 ++
 ...g-hungarian-notation-lower-case-prefix.cpp | 678 ++++++++++++++++++
 5 files changed, 766 insertions(+), 13 deletions(-)
 create mode 100644 clang-tools-extra/test/clang-tidy/checkers/readability/Inputs/identifier-naming/hungarian-notation3/.clang-tidy
 create mode 100644 clang-tools-extra/test/clang-tidy/checkers/readability/identifier-naming-hungarian-notation-lower-case-prefix.cpp

diff --git a/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.cpp b/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.cpp
index 335c3de25b86..dc30531ebda0 100644
--- a/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.cpp
@@ -888,8 +888,14 @@ bool IdentifierNamingCheck::matchesStyle(
     return false;
   if (IdentifierNamingCheck::HungarianPrefixType::HPT_Off != Style.HPType) {
     std::string HNPrefix = HungarianNotation.getPrefix(Decl, HNOption);
-    if (!Name.consume_front(HNPrefix))
-      return false;
+    if (!HNPrefix.empty()) {
+      if (!Name.consume_front(HNPrefix))
+        return false;
+      if (Style.HPType ==
+              IdentifierNamingCheck::HungarianPrefixType::HPT_LowerCase &&
+          !Name.consume_front("_"))
+        return false;
+    }
   }
 
   // Ensure the name doesn't have any extra underscores beyond those specified
diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index b5f025ce467a..a005adf76b8b 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -235,7 +235,8 @@ Changes in existing checks
 
 - Improved :doc:`readability-identifier-naming
   ` check in `GetConfigPerFile`
-  mode by resolving symbolic links to header files.
+  mode by resolving symbolic links to header files. Fixed handling of Hungarian
+  Prefix when configured to `LowerCase`.
 
 Removed checks
 ^^^^^^^^^^^^^^
diff --git a/clang-tools-extra/docs/clang-tidy/checks/readability/identifier-naming.rst b/clang-tools-extra/docs/clang-tidy/checks/readability/identifier-naming.rst
index 2affb55cfa9a..8a54687a2570 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/readability/identifier-naming.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/readability/identifier-naming.rst
@@ -11,14 +11,14 @@ another if a mismatch is detected
 
 Casing types include:
 
- - ``lower_case``,
- - ``UPPER_CASE``,
- - ``camelBack``,
- - ``CamelCase``,
- - ``camel_Snake_Back``,
- - ``Camel_Snake_Case``,
- - ``aNy_CasE``,
- - ``Leading_upper_snake_case``.
+ - ``lower_case``
+ - ``UPPER_CASE``
+ - ``camelBack``
+ - ``CamelCase``
+ - ``camel_Snake_Back``
+ - ``Camel_Snake_Case``
+ - ``aNy_CasE``
+ - ``Leading_upper_snake_case``
 
 It also supports a fixed prefix and suffix that will be prepended or appended
 to the identifiers, regardless of the casing.
@@ -32,8 +32,16 @@ but not where they are overridden, as it can't be fixed locally there.
 This also applies for pseudo-override patterns like CRTP.
 
 ``Leading_upper_snake_case`` is a naming convention where the first word is capitalized
-followed by lower case word(s) seperated by underscore(s) '_'. Examples include:
-Cap_snake_case, Cobra_case, Foo_bar_baz, and Master_copy_8gb.
+followed by lower case word(s) separated by underscore(s) '_'. Examples include:
+`Cap_snake_case`, `Cobra_case`, `Foo_bar_baz`, and `Master_copy_8gb`.
+
+Hungarian notation can be customized using different *HungarianPrefix* settings.
+The options and their corresponding values are:
+
+ - ``Off`` - the default setting
+ - ``On`` - example: ``int iVariable``
+ - ``LowerCase`` - example: ``int i_Variable``
+ - ``CamelCase`` - example: ``int IVariable``
 
 Options
 -------
diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability/Inputs/identifier-naming/hungarian-notation3/.clang-tidy b/clang-tools-extra/test/clang-tidy/checkers/readability/Inputs/identifier-naming/hungarian-notation3/.clang-tidy
new file mode 100644
index 000000000000..45b4cd46c249
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/readability/Inputs/identifier-naming/hungarian-notation3/.clang-tidy
@@ -0,0 +1,60 @@
+Checks: readability-identifier-naming
+CheckOptions:
+  readability-identifier-naming.AbstractClassCase: CamelCase
+  readability-identifier-naming.StructCase: CamelCase
+  readability-identifier-naming.UnionCase: camelBack
+  readability-identifier-naming.ClassCase: CamelCase
+  readability-identifier-naming.ClassConstantCase: CamelCase
+  readability-identifier-naming.ClassMemberCase: CamelCase
+  readability-identifier-naming.ConstantCase: CamelCase
+  readability-identifier-naming.ConstantMemberCase: CamelCase
+  readability-identifier-naming.ConstantParameterCase: CamelCase
+  readability-identifier-naming.ConstantPointerParameterCase: CamelCase
+  readability-identifier-naming.ConstexprVariableCase: CamelCase
+  readability-identifier-naming.EnumConstantCase: CamelCase
+  readability-identifier-naming.GlobalConstantCase: CamelCase
+  readability-identifier-naming.GlobalConstantPointerCase: CamelCase
+  readability-identifier-naming.GlobalPointerCase: CamelCase
+  readability-identifier-naming.GlobalVariableCase: CamelCase
+  readability-identifier-naming.LocalConstantCase: CamelCase
+  readability-identifier-naming.LocalConstantPointerCase: CamelCase
+  readability-identifier-naming.LocalPointerCase: CamelCase
+  readability-identifier-naming.LocalVariableCase: CamelCase
+  readability-identifier-naming.MemberCase: CamelCase
+  readability-identifier-naming.ParameterCase: CamelCase
+  readability-identifier-naming.PointerParameterCase: CamelCase
+  readability-identifier-naming.PrivateMemberCase: CamelCase
+  readability-identifier-naming.ProtectedMemberCase: CamelCase
+  readability-identifier-naming.PublicMemberCase: CamelCase
+  readability-identifier-naming.ScopedEnumConstantCase: CamelCase
+  readability-identifier-naming.StaticConstantCase: CamelCase
+  readability-identifier-naming.StaticVariableCase: CamelCase
+  readability-identifier-naming.VariableCase: CamelCase
+  readability-identifier-naming.AbstractClassHungarianPrefix: LowerCase
+  readability-identifier-naming.ClassHungarianPrefix: LowerCase
+  readability-identifier-naming.ClassConstantHungarianPrefix: LowerCase
+  readability-identifier-naming.ClassMemberHungarianPrefix: LowerCase
+  readability-identifier-naming.ConstantHungarianPrefix: LowerCase
+  readability-identifier-naming.ConstantMemberHungarianPrefix: LowerCase
+  readability-identifier-naming.ConstantParameterHungarianPrefix: LowerCase
+  readability-identifier-naming.ConstantPointerParameterHungarianPrefix: LowerCase
+  readability-identifier-naming.ConstexprVariableHungarianPrefix: LowerCase
+  readability-identifier-naming.EnumConstantHungarianPrefix: LowerCase
+  readability-identifier-naming.GlobalConstantHungarianPrefix: LowerCase
+  readability-identifier-naming.GlobalConstantPointerHungarianPrefix: LowerCase
+  readability-identifier-naming.GlobalPointerHungarianPrefix: LowerCase
+  readability-identifier-naming.GlobalVariableHungarianPrefix: LowerCase
+  readability-identifier-naming.LocalConstantHungarianPrefix: LowerCase
+  readability-identifier-naming.LocalConstantPointerHungarianPrefix: LowerCase
+  readability-identifier-naming.LocalPointerHungarianPrefix: LowerCase
+  readability-identifier-naming.LocalVariableHungarianPrefix: LowerCase
+  readability-identifier-naming.MemberHungarianPrefix: LowerCase
+  readability-identifier-naming.ParameterHungarianPrefix: LowerCase
+  readability-identifier-naming.PointerParameterHungarianPrefix: LowerCase
+  readability-identifier-naming.PrivateMemberHungarianPrefix: LowerCase
+  readability-identifier-naming.ProtectedMemberHungarianPrefix: LowerCase
+  readability-identifier-naming.PublicMemberHungarianPrefix: LowerCase
+  readability-identifier-naming.ScopedEnumConstantHungarianPrefix: LowerCase
+  readability-identifier-naming.StaticConstantHungarianPrefix: LowerCase
+  readability-identifier-naming.StaticVariableHungarianPrefix: LowerCase
+  readability-identifier-naming.VariableHungarianPrefix: LowerCase
diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability/identifier-naming-hungarian-notation-lower-case-prefix.cpp b/clang-tools-extra/test/clang-tidy/checkers/readability/identifier-naming-hungarian-notation-lower-case-prefix.cpp
new file mode 100644
index 000000000000..65aee7e9f6ce
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/readability/identifier-naming-hungarian-notation-lower-case-prefix.cpp
@@ -0,0 +1,678 @@
+// RUN: %check_clang_tidy %s readability-identifier-naming %t -- \
+// RUN:   --config-file=%S/Inputs/identifier-naming/hungarian-notation3/.clang-tidy -- -I %S
+
+#include "identifier-naming-standard-types.h"
+
+// clang-format off
+//===----------------------------------------------------------------------===//
+// Cases to CheckOptions
+//===----------------------------------------------------------------------===//
+class C_MyClass1 {
+public:
+  static int ClassMemberCase;
+  // CHECK-MESSAGES: :[[@LINE-1]]:14: warning: invalid case style for class member 'ClassMemberCase' [readability-identifier-naming]
+  // CHECK-FIXES: {{^}}  static int i_ClassMemberCase;
+
+  char const ConstantMemberCase = 0;
+  // CHECK-MESSAGES: :[[@LINE-1]]:14: warning: invalid case style for constant member 'ConstantMemberCase' [readability-identifier-naming]
+  // CHECK-FIXES: {{^}}  char const c_ConstantMemberCase = 0;
+
+  void MyFunc1(const int ConstantParameterCase);
+  // CHECK-MESSAGES: :[[@LINE-1]]:26: warning: invalid case style for constant parameter 'ConstantParameterCase' [readability-identifier-naming]
+  // CHECK-FIXES: {{^}}  void MyFunc1(const int i_ConstantParameterCase);
+
+  void MyFunc2(const int* ConstantPointerParameterCase);
+  // CHECK-MESSAGES: :[[@LINE-1]]:27: warning: invalid case style for pointer parameter 'ConstantPointerParameterCase' [readability-identifier-naming]
+  // CHECK-FIXES: {{^}}  void MyFunc2(const int* pi_ConstantPointerParameterCase);
+
+  static constexpr int ConstexprVariableCase = 123;
+  // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: invalid case style for constexpr variable 'ConstexprVariableCase' [readability-identifier-naming]
+  // CHECK-FIXES: {{^}}  static constexpr int i_ConstexprVariableCase = 123;
+};
+
+const int GlobalConstantCase = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:11: warning: invalid case style for global constant 'GlobalConstantCase' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}const int i_GlobalConstantCase = 0;
+
+const int* GlobalConstantPointerCase = nullptr;
+// CHECK-MESSAGES: :[[@LINE-1]]:12: warning: invalid case style for global pointer 'GlobalConstantPointerCase' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}const int* pi_GlobalConstantPointerCase = nullptr;
+
+int* GlobalPointerCase = nullptr;
+// CHECK-MESSAGES: :[[@LINE-1]]:6: warning: invalid case style for global pointer 'GlobalPointerCase' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int* pi_GlobalPointerCase = nullptr;
+
+int GlobalVariableCase = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'GlobalVariableCase' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int i_GlobalVariableCase = 0;
+
+void Func1(){
+  int const LocalConstantCase = 3;
+  // CHECK-MESSAGES: :[[@LINE-1]]:13: warning: invalid case style for local constant 'LocalConstantCase' [readability-identifier-naming]
+  // CHECK-FIXES: {{^}}  int const i_LocalConstantCase = 3;
+
+  unsigned const ConstantCase = 1;
+  // CHECK-MESSAGES: :[[@LINE-1]]:18: warning: invalid case style for local constant 'ConstantCase' [readability-identifier-naming]
+  // CHECK-FIXES: {{^}}  unsigned const u_ConstantCase = 1;
+
+  int* const LocalConstantPointerCase = nullptr;
+  // CHECK-MESSAGES: :[[@LINE-1]]:14: warning: invalid case style for local constant pointer 'LocalConstantPointerCase' [readability-identifier-naming]
+  // CHECK-FIXES: {{^}}  int* const pi_LocalConstantPointerCase = nullptr;
+
+  int *LocalPointerCase = nullptr;
+  // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: invalid case style for local pointer 'LocalPointerCase' [readability-identifier-naming]
+  // CHECK-FIXES: {{^}}  int *pi_LocalPointerCase = nullptr;
+
+  int LocalVariableCase = 0;
+  // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for local variable 'LocalVariableCase' [readability-identifier-naming]
+  // CHECK-FIXES: {{^}}  int i_LocalVariableCase = 0;
+}
+
+class C_MyClass2 {
+  char MemberCase;
+  // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: invalid case style for private member 'MemberCase' [readability-identifier-naming]
+  // CHECK-FIXES: {{^}}  char c_MemberCase;
+
+  void Func1(int ParameterCase);
+  // CHECK-MESSAGES: :[[@LINE-1]]:18: warning: invalid case style for parameter 'ParameterCase' [readability-identifier-naming]
+  // CHECK-FIXES: {{^}}  void Func1(int i_ParameterCase);
+
+  void Func2(const int ParameterCase);
+  // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: invalid case style for constant parameter 'ParameterCase' [readability-identifier-naming]
+  // CHECK-FIXES: {{^}}  void Func2(const int i_ParameterCase);
+
+  void Func3(const int *PointerParameterCase);
+  // CHECK-MESSAGES: :[[@LINE-1]]:25: warning: invalid case style for pointer parameter 'PointerParameterCase' [readability-identifier-naming]
+  // CHECK-FIXES: {{^}}  void Func3(const int *pi_PointerParameterCase);
+};
+
+class C_MyClass3 {
+private:
+  char PrivateMemberCase;
+  // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: invalid case style for private member 'PrivateMemberCase' [readability-identifier-naming]
+  // CHECK-FIXES: {{^}}  char c_PrivateMemberCase;
+
+protected:
+  char ProtectedMemberCase;
+  // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: invalid case style for protected member 'ProtectedMemberCase' [readability-identifier-naming]
+  // CHECK-FIXES: {{^}}  char c_ProtectedMemberCase;
+
+public:
+  char PublicMemberCase;
+  // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: invalid case style for public member 'PublicMemberCase' [readability-identifier-naming]
+  // CHECK-FIXES: {{^}}  char c_PublicMemberCase;
+};
+
+static const int StaticConstantCase = 3;
+// CHECK-MESSAGES: :[[@LINE-1]]:18: warning: invalid case style for global constant 'StaticConstantCase' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}static const int i_StaticConstantCase = 3;
+
+static int StaticVariableCase = 3;
+// CHECK-MESSAGES: :[[@LINE-1]]:12: warning: invalid case style for global variable 'StaticVariableCase' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}static int i_StaticVariableCase = 3;
+
+struct MyStruct { int StructCase; };
+// CHECK-MESSAGES: :[[@LINE-1]]:23: warning: invalid case style for public member 'StructCase' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}struct MyStruct { int i_StructCase; };
+
+struct shouldBeCamelCaseStruct { int i_Field; };
+// CHECK-MESSAGES: :[[@LINE-1]]:8: warning: invalid case style for struct 'shouldBeCamelCaseStruct' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}struct ShouldBeCamelCaseStruct { int i_Field; };
+
+union MyUnion { int UnionCase; long l_UnionCase; };
+// CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for union 'MyUnion' [readability-identifier-naming]
+// CHECK-MESSAGES: :[[@LINE-2]]:21: warning: invalid case style for public member 'UnionCase' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}union myUnion { int i_UnionCase; long l_UnionCase; };
+
+//===----------------------------------------------------------------------===//
+// C string
+//===----------------------------------------------------------------------===//
+const char *NamePtr = "Name";
+// CHECK-MESSAGES: :[[@LINE-1]]:13: warning: invalid case style for global pointer 'NamePtr' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}const char *sz_NamePtr = "Name";
+
+const char NameArray[] = "Name";
+// CHECK-MESSAGES: :[[@LINE-1]]:12: warning: invalid case style for global constant 'NameArray' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}const char sz_NameArray[] = "Name";
+
+const char *NamePtrArray[] = {"AA", "BB"};
+// CHECK-MESSAGES: :[[@LINE-1]]:13: warning: invalid case style for global variable 'NamePtrArray' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}const char *psz_NamePtrArray[] = {"AA", "BB"};
+
+const wchar_t *WideNamePtr = L"Name";
+// CHECK-MESSAGES: :[[@LINE-1]]:16: warning: invalid case style for global pointer 'WideNamePtr' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}const wchar_t *wsz_WideNamePtr = L"Name";
+
+const wchar_t WideNameArray[] = L"Name";
+// CHECK-MESSAGES: :[[@LINE-1]]:15: warning: invalid case style for global constant 'WideNameArray' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}const wchar_t wsz_WideNameArray[] = L"Name";
+
+const wchar_t *WideNamePtrArray[] = {L"AA", L"BB"};
+// CHECK-MESSAGES: :[[@LINE-1]]:16: warning: invalid case style for global variable 'WideNamePtrArray' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}const wchar_t *pwsz_WideNamePtrArray[] = {L"AA", L"BB"};
+
+class C_MyClass4 {
+private:
+  char *Name = "Text";
+  // CHECK-MESSAGES: :[[@LINE-1]]:9: warning: invalid case style for private member 'Name' [readability-identifier-naming]
+  // CHECK-FIXES: {{^}}  char *sz_Name = "Text";
+
+  const char *ConstName = "Text";
+  // CHECK-MESSAGES: :[[@LINE-1]]:15: warning: invalid case style for private member 'ConstName' [readability-identifier-naming]
+  // CHECK-FIXES: {{^}}  const char *sz_ConstName = "Text";
+
+public:
+  const char* DuplicateString(const char* Input, size_t n_RequiredSize);
+  // CHECK-MESSAGES: :[[@LINE-1]]:43: warning: invalid case style for pointer parameter 'Input' [readability-identifier-naming]
+  // CHECK-FIXES: {{^}}  const char* DuplicateString(const char* sz_Input, size_t n_RequiredSize);
+
+  size_t UpdateText(const char* Buffer, size_t n_BufferSize);
+  // CHECK-MESSAGES: :[[@LINE-1]]:33: warning: invalid case style for pointer parameter 'Buffer' [readability-identifier-naming]
+  // CHECK-FIXES: {{^}}  size_t UpdateText(const char* sz_Buffer, size_t n_BufferSize);
+};
+
+
+//===----------------------------------------------------------------------===//
+// Microsoft Windows data types
+//===----------------------------------------------------------------------===//
+DWORD MsDword = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for global variable 'MsDword' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}DWORD dw_MsDword = 0;
+
+BYTE MsByte = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:6: warning: invalid case style for global variable 'MsByte' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}BYTE by_MsByte = 0;
+
+WORD MsWord = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:6: warning: invalid case style for global variable 'MsWord' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}WORD w_MsWord = 0;
+
+BOOL MsBool = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:6: warning: invalid case style for global variable 'MsBool' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}BOOL b_MsBool = 0;
+
+BOOLEAN MsBoolean = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:9: warning: invalid case style for global variable 'MsBoolean' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}BOOLEAN b_MsBoolean = 0;
+
+CHAR MsValueChar = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:6: warning: invalid case style for global variable 'MsValueChar' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}CHAR c_MsValueChar = 0;
+
+UCHAR MsValueUchar = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for global variable 'MsValueUchar' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}UCHAR uc_MsValueUchar = 0;
+
+SHORT MsValueShort = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for global variable 'MsValueShort' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}SHORT s_MsValueShort = 0;
+
+USHORT MsValueUshort = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:8: warning: invalid case style for global variable 'MsValueUshort' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}USHORT us_MsValueUshort = 0;
+
+WORD MsValueWord = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:6: warning: invalid case style for global variable 'MsValueWord' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}WORD w_MsValueWord = 0;
+
+DWORD MsValueDword = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for global variable 'MsValueDword' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}DWORD dw_MsValueDword = 0;
+
+DWORD32 MsValueDword32 = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:9: warning: invalid case style for global variable 'MsValueDword32' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}DWORD32 dw32_MsValueDword32 = 0;
+
+DWORD64 MsValueDword64 = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:9: warning: invalid case style for global variable 'MsValueDword64' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}DWORD64 dw64_MsValueDword64 = 0;
+
+LONG MsValueLong = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:6: warning: invalid case style for global variable 'MsValueLong' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}LONG l_MsValueLong = 0;
+
+ULONG MsValueUlong = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for global variable 'MsValueUlong' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}ULONG ul_MsValueUlong = 0;
+
+ULONG32 MsValueUlong32 = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:9: warning: invalid case style for global variable 'MsValueUlong32' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}ULONG32 ul32_MsValueUlong32 = 0;
+
+ULONG64 MsValueUlong64 = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:9: warning: invalid case style for global variable 'MsValueUlong64' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}ULONG64 ul64_MsValueUlong64 = 0;
+
+ULONGLONG MsValueUlongLong = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:11: warning: invalid case style for global variable 'MsValueUlongLong' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}ULONGLONG ull_MsValueUlongLong = 0;
+
+HANDLE MsValueHandle = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:8: warning: invalid case style for global pointer 'MsValueHandle' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}HANDLE h_MsValueHandle = 0;
+
+INT MsValueInt = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'MsValueInt' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}INT i_MsValueInt = 0;
+
+INT8 MsValueInt8 = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:6: warning: invalid case style for global variable 'MsValueInt8' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}INT8 i8_MsValueInt8 = 0;
+
+INT16 MsValueInt16 = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for global variable 'MsValueInt16' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}INT16 i16_MsValueInt16 = 0;
+
+INT32 MsValueInt32 = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for global variable 'MsValueInt32' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}INT32 i32_MsValueInt32 = 0;
+
+INT64 MsValueINt64 = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for global variable 'MsValueINt64' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}INT64 i64_MsValueINt64 = 0;
+
+UINT MsValueUint = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:6: warning: invalid case style for global variable 'MsValueUint' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}UINT ui_MsValueUint = 0;
+
+UINT8 MsValueUint8 = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for global variable 'MsValueUint8' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}UINT8 u8_MsValueUint8 = 0;
+
+UINT16 MsValueUint16 = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:8: warning: invalid case style for global variable 'MsValueUint16' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}UINT16 u16_MsValueUint16 = 0;
+
+UINT32 MsValueUint32 = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:8: warning: invalid case style for global variable 'MsValueUint32' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}UINT32 u32_MsValueUint32 = 0;
+
+UINT64 MsValueUint64 = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:8: warning: invalid case style for global variable 'MsValueUint64' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}UINT64 u64_MsValueUint64 = 0;
+
+PVOID MsValuePvoid = NULL;
+// CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for global pointer 'MsValuePvoid' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}PVOID p_MsValuePvoid = NULL;
+
+
+//===----------------------------------------------------------------------===//
+// Array
+//===----------------------------------------------------------------------===//
+unsigned GlobalUnsignedArray[] = {1, 2, 3};
+// CHECK-MESSAGES: :[[@LINE-1]]:10: warning: invalid case style for global variable 'GlobalUnsignedArray' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}unsigned a_GlobalUnsignedArray[] = {1, 2, 3};
+
+int GlobalIntArray[] = {1, 2, 3};
+// CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'GlobalIntArray' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int a_GlobalIntArray[] = {1, 2, 3};
+
+int DataInt[1] = {0};
+// CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'DataInt' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int a_DataInt[1] = {0};
+
+int DataArray[2] = {0};
+// CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'DataArray' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int a_DataArray[2] = {0};
+
+
+//===----------------------------------------------------------------------===//
+// Pointer
+//===----------------------------------------------------------------------===//
+int *DataIntPtr[1] = {0};
+// CHECK-MESSAGES: :[[@LINE-1]]:6: warning: invalid case style for global variable 'DataIntPtr' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int *pa_DataIntPtr[1] = {0};
+
+void *BufferPtr1;
+// CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for global pointer 'BufferPtr1' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}void *p_BufferPtr1;
+
+void **BufferPtr2;
+// CHECK-MESSAGES: :[[@LINE-1]]:8: warning: invalid case style for global pointer 'BufferPtr2' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}void **pp_BufferPtr2;
+
+void **pBufferPtr3;
+// CHECK-MESSAGES: :[[@LINE-1]]:8: warning: invalid case style for global pointer 'pBufferPtr3' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}void **pp_BufferPtr3;
+
+int *pBufferPtr4;
+// CHECK-MESSAGES: :[[@LINE-1]]:6: warning: invalid case style for global pointer 'pBufferPtr4' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int *pi_BufferPtr4;
+
+typedef void (*FUNC_PTR_HELLO)();
+FUNC_PTR_HELLO Hello = NULL;
+// CHECK-MESSAGES: :[[@LINE-1]]:16: warning: invalid case style for global pointer 'Hello' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}FUNC_PTR_HELLO fn_Hello = NULL;
+
+void *ValueVoidPtr = NULL;
+// CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for global pointer 'ValueVoidPtr' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}void *p_ValueVoidPtr = NULL;
+
+ptrdiff_t PtrDiff = NULL;
+// CHECK-MESSAGES: :[[@LINE-1]]:11: warning: invalid case style for global variable 'PtrDiff' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}ptrdiff_t p_PtrDiff = NULL;
+
+int8_t *ValueI8Ptr;
+// CHECK-MESSAGES: :[[@LINE-1]]:9: warning: invalid case style for global pointer 'ValueI8Ptr' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int8_t *pi8_ValueI8Ptr;
+
+uint8_t *ValueU8Ptr;
+// CHECK-MESSAGES: :[[@LINE-1]]:10: warning: invalid case style for global pointer 'ValueU8Ptr' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}uint8_t *pu8_ValueU8Ptr;
+
+unsigned char *ValueUcPtr;
+// CHECK-MESSAGES: :[[@LINE-1]]:16: warning: invalid case style for global pointer 'ValueUcPtr' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}unsigned char *puc_ValueUcPtr;
+
+unsigned char **ValueUcPtr2;
+// CHECK-MESSAGES: :[[@LINE-1]]:17: warning: invalid case style for global pointer 'ValueUcPtr2' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}unsigned char **ppuc_ValueUcPtr2;
+
+void MyFunc2(void* Val){}
+// CHECK-MESSAGES: :[[@LINE-1]]:20: warning: invalid case style for pointer parameter 'Val' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}void MyFunc2(void* p_Val){}
+
+
+//===----------------------------------------------------------------------===//
+// Reference
+//===----------------------------------------------------------------------===//
+int i_ValueIndex = 1;
+int &RefValueIndex = i_ValueIndex;
+// CHECK-MESSAGES: :[[@LINE-1]]:6: warning: invalid case style for global variable 'RefValueIndex' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int &i_RefValueIndex = i_ValueIndex;
+
+const int &ConstRefValue = i_ValueIndex;
+// CHECK-MESSAGES: :[[@LINE-1]]:12: warning: invalid case style for global variable 'ConstRefValue' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}const int &i_ConstRefValue = i_ValueIndex;
+
+long long ll_ValueLongLong = 2;
+long long &RefValueLongLong = ll_ValueLongLong;
+// CHECK-MESSAGES: :[[@LINE-1]]:12: warning: invalid case style for global variable 'RefValueLongLong' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}long long &ll_RefValueLongLong = ll_ValueLongLong;
+
+
+//===----------------------------------------------------------------------===//
+// Various types
+//===----------------------------------------------------------------------===//
+int8_t ValueI8;
+// CHECK-MESSAGES: :[[@LINE-1]]:8: warning: invalid case style for global variable 'ValueI8' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int8_t i8_ValueI8;
+
+int16_t ValueI16 = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:9: warning: invalid case style for global variable 'ValueI16' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int16_t i16_ValueI16 = 0;
+
+int32_t ValueI32 = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:9: warning: invalid case style for global variable 'ValueI32' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int32_t i32_ValueI32 = 0;
+
+int64_t ValueI64 = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:9: warning: invalid case style for global variable 'ValueI64' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int64_t i64_ValueI64 = 0;
+
+uint8_t ValueU8 = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:9: warning: invalid case style for global variable 'ValueU8' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}uint8_t u8_ValueU8 = 0;
+
+uint16_t ValueU16 = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:10: warning: invalid case style for global variable 'ValueU16' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}uint16_t u16_ValueU16 = 0;
+
+uint32_t ValueU32 = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:10: warning: invalid case style for global variable 'ValueU32' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}uint32_t u32_ValueU32 = 0;
+
+uint64_t ValueU64 = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:10: warning: invalid case style for global variable 'ValueU64' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}uint64_t u64_ValueU64 = 0;
+
+float ValueFloat = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for global variable 'ValueFloat' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}float f_ValueFloat = 0;
+
+double ValueDouble = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:8: warning: invalid case style for global variable 'ValueDouble' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}double d_ValueDouble = 0;
+
+char ValueChar = 'c';
+// CHECK-MESSAGES: :[[@LINE-1]]:6: warning: invalid case style for global variable 'ValueChar' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}char c_ValueChar = 'c';
+
+bool ValueBool = true;
+// CHECK-MESSAGES: :[[@LINE-1]]:6: warning: invalid case style for global variable 'ValueBool' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}bool b_ValueBool = true;
+
+int ValueInt = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'ValueInt' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int i_ValueInt = 0;
+
+size_t ValueSize = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:8: warning: invalid case style for global variable 'ValueSize' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}size_t n_ValueSize = 0;
+
+wchar_t ValueWchar = 'w';
+// CHECK-MESSAGES: :[[@LINE-1]]:9: warning: invalid case style for global variable 'ValueWchar' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}wchar_t wc_ValueWchar = 'w';
+
+short ValueShort = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for global variable 'ValueShort' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}short s_ValueShort = 0;
+
+unsigned ValueUnsigned = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:10: warning: invalid case style for global variable 'ValueUnsigned' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}unsigned u_ValueUnsigned = 0;
+
+signed ValueSigned = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:8: warning: invalid case style for global variable 'ValueSigned' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}signed s_ValueSigned = 0;
+
+long ValueLong = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:6: warning: invalid case style for global variable 'ValueLong' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}long l_ValueLong = 0;
+
+long long ValueLongLong = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:11: warning: invalid case style for global variable 'ValueLongLong' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}long long ll_ValueLongLong = 0;
+
+long long int ValueLongLongInt = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:15: warning: invalid case style for global variable 'ValueLongLongInt' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}long long int lli_ValueLongLongInt = 0;
+
+long double ValueLongDouble = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:13: warning: invalid case style for global variable 'ValueLongDouble' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}long double ld_ValueLongDouble = 0;
+
+signed int ValueSignedInt = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:12: warning: invalid case style for global variable 'ValueSignedInt' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}signed int si_ValueSignedInt = 0;
+
+signed short ValueSignedShort = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:14: warning: invalid case style for global variable 'ValueSignedShort' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}signed short ss_ValueSignedShort = 0;
+
+signed short int ValueSignedShortInt = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:18: warning: invalid case style for global variable 'ValueSignedShortInt' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}signed short int ssi_ValueSignedShortInt = 0;
+
+signed long long ValueSignedLongLong = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:18: warning: invalid case style for global variable 'ValueSignedLongLong' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}signed long long sll_ValueSignedLongLong = 0;
+
+signed long int ValueSignedLongInt = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:17: warning: invalid case style for global variable 'ValueSignedLongInt' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}signed long int sli_ValueSignedLongInt = 0;
+
+signed long ValueSignedLong = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:13: warning: invalid case style for global variable 'ValueSignedLong' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}signed long sl_ValueSignedLong = 0;
+
+unsigned long long int ValueUnsignedLongLongInt = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:24: warning: invalid case style for global variable 'ValueUnsignedLongLongInt' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}unsigned long long int ulli_ValueUnsignedLongLongInt = 0;
+
+unsigned long long ValueUnsignedLongLong = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:20: warning: invalid case style for global variable 'ValueUnsignedLongLong' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}unsigned long long ull_ValueUnsignedLongLong = 0;
+
+unsigned long int ValueUnsignedLongInt = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:19: warning: invalid case style for global variable 'ValueUnsignedLongInt' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}unsigned long int uli_ValueUnsignedLongInt = 0;
+
+unsigned long ValueUnsignedLong = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:15: warning: invalid case style for global variable 'ValueUnsignedLong' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}unsigned long ul_ValueUnsignedLong = 0;
+
+unsigned short int ValueUnsignedShortInt = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:20: warning: invalid case style for global variable 'ValueUnsignedShortInt' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}unsigned short int usi_ValueUnsignedShortInt = 0;
+
+unsigned short ValueUnsignedShort = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:16: warning: invalid case style for global variable 'ValueUnsignedShort' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}unsigned short us_ValueUnsignedShort = 0;
+
+unsigned int ValueUnsignedInt = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:14: warning: invalid case style for global variable 'ValueUnsignedInt' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}unsigned int ui_ValueUnsignedInt = 0;
+
+unsigned char ValueUnsignedChar = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:15: warning: invalid case style for global variable 'ValueUnsignedChar' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}unsigned char uc_ValueUnsignedChar = 0;
+
+long int ValueLongInt = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:10: warning: invalid case style for global variable 'ValueLongInt' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}long int li_ValueLongInt = 0;
+
+
+//===----------------------------------------------------------------------===//
+// Specifier, Qualifier, Other keywords
+//===----------------------------------------------------------------------===//
+volatile int VolatileInt = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:14: warning: invalid case style for global variable 'VolatileInt' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}volatile int i_VolatileInt = 0;
+
+thread_local int ThreadLocalValueInt = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:18: warning: invalid case style for global variable 'ThreadLocalValueInt' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}thread_local int i_ThreadLocalValueInt = 0;
+
+extern int ExternValueInt;
+// CHECK-MESSAGES: :[[@LINE-1]]:12: warning: invalid case style for global variable 'ExternValueInt' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}extern int i_ExternValueInt;
+
+struct DataBuffer {
+    mutable size_t Size;
+};
+// CHECK-MESSAGES: :[[@LINE-2]]:20: warning: invalid case style for public member 'Size' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}    mutable size_t n_Size;
+
+static constexpr int const &ConstExprInt = 42;
+// CHECK-MESSAGES: :[[@LINE-1]]:29: warning: invalid case style for constexpr variable 'ConstExprInt' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}static constexpr int const &i_ConstExprInt = 42;
+
+
+//===----------------------------------------------------------------------===//
+// Redefined types
+//===----------------------------------------------------------------------===//
+typedef int INDEX;
+INDEX iIndex = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for global variable 'iIndex' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}INDEX Index = 0;
+
+
+//===----------------------------------------------------------------------===//
+// Class
+//===----------------------------------------------------------------------===//
+class ClassCase { int Func(); };
+// CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for class 'ClassCase' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}class C_ClassCase { int Func(); };
+
+class AbstractClassCase { virtual int Func() = 0; };
+// CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for abstract class 'AbstractClassCase' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}class I_AbstractClassCase { virtual int Func() = 0; };
+
+class AbstractClassCase1 { virtual int Func1() = 0; int Func2(); };
+// CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for abstract class 'AbstractClassCase1' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}class I_AbstractClassCase1 { virtual int Func1() = 0; int Func2(); };
+
+class ClassConstantCase { public: static const int i_ConstantCase; };
+// CHECK-MESSAGES: :[[@LINE-1]]:7: warning: invalid case style for class 'ClassConstantCase' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}class C_ClassConstantCase { public: static const int i_ConstantCase; };
+
+//===----------------------------------------------------------------------===//
+// Other Cases
+//===----------------------------------------------------------------------===//
+int lower_case = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'lower_case' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int i_LowerCase = 0;
+
+int lower_case1 = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'lower_case1' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int i_LowerCase1 = 0;
+
+int lower_case_2 = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'lower_case_2' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int i_LowerCase2 = 0;
+
+int UPPER_CASE = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'UPPER_CASE' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int i_UpperCase = 0;
+
+int UPPER_CASE_1 = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'UPPER_CASE_1' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int i_UpperCase1 = 0;
+
+int camelBack = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'camelBack' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int i_CamelBack = 0;
+
+int camelBack_1 = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'camelBack_1' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int i_CamelBack1 = 0;
+
+int camelBack2 = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'camelBack2' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int i_CamelBack2 = 0;
+
+int CamelCase = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'CamelCase' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int i_CamelCase = 0;
+
+int CamelCase_1 = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'CamelCase_1' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int i_CamelCase1 = 0;
+
+int CamelCase2 = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'CamelCase2' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int i_CamelCase2 = 0;
+
+int camel_Snake_Back = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'camel_Snake_Back' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int i_CamelSnakeBack = 0;
+
+int camel_Snake_Back_1 = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'camel_Snake_Back_1' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int i_CamelSnakeBack1 = 0;
+
+int Camel_Snake_Case = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'Camel_Snake_Case' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int i_CamelSnakeCase = 0;
+
+int Camel_Snake_Case_1 = 0;
+// CHECK-MESSAGES: :[[@LINE-1]]:5: warning: invalid case style for global variable 'Camel_Snake_Case_1' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}int i_CamelSnakeCase1 = 0;
+
+//===----------------------------------------------------------------------===//
+// Enum
+//===----------------------------------------------------------------------===//
+enum REV_TYPE { RevValid };
+// CHECK-MESSAGES: :[[@LINE-1]]:17: warning: invalid case style for enum constant 'RevValid' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}enum REV_TYPE { rt_RevValid };
+
+enum EnumConstantCase { OneByte, TwoByte };
+// CHECK-MESSAGES: :[[@LINE-1]]:25: warning: invalid case style for enum constant 'OneByte' [readability-identifier-naming]
+// CHECK-MESSAGES: :[[@LINE-2]]:34: warning: invalid case style for enum constant 'TwoByte' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}enum EnumConstantCase { ecc_OneByte, ecc_TwoByte };
+
+enum class ScopedEnumConstantCase { Case1 };
+// CHECK-MESSAGES: :[[@LINE-1]]:37: warning: invalid case style for scoped enum constant 'Case1' [readability-identifier-naming]
+// CHECK-FIXES: {{^}}enum class ScopedEnumConstantCase { secc_Case1 };
+// clang-format on
-- 
GitLab


From 81798d516067a58c8e4ba51072dae7351d445064 Mon Sep 17 00:00:00 2001
From: Fangrui Song 
Date: Fri, 8 Mar 2024 09:42:59 -0800
Subject: [PATCH 0060/1674] [ELF] Sort sections in order. NFC

---
 lld/ELF/DWARF.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lld/ELF/DWARF.cpp b/lld/ELF/DWARF.cpp
index a56454c53a6d..ac28aa8c7ca6 100644
--- a/lld/ELF/DWARF.cpp
+++ b/lld/ELF/DWARF.cpp
@@ -38,11 +38,11 @@ template  LLDDwarfObj::LLDDwarfObj(ObjFile *obj) {
                 .Case(".debug_addr", &addrSection)
                 .Case(".debug_gnu_pubnames", &gnuPubnamesSection)
                 .Case(".debug_gnu_pubtypes", &gnuPubtypesSection)
+                .Case(".debug_line", &lineSection)
                 .Case(".debug_loclists", &loclistsSection)
                 .Case(".debug_ranges", &rangesSection)
                 .Case(".debug_rnglists", &rnglistsSection)
                 .Case(".debug_str_offsets", &strOffsetsSection)
-                .Case(".debug_line", &lineSection)
                 .Default(nullptr)) {
       m->Data = toStringRef(sec->contentMaybeDecompress());
       m->sec = sec;
-- 
GitLab


From 2e0ddfc1638597e0d08fb028fdd6754387732107 Mon Sep 17 00:00:00 2001
From: Chris B 
Date: Fri, 8 Mar 2024 11:44:31 -0600
Subject: [PATCH 0061/1674] [DirectX][docs] Expand DXILArchitecture
 documentation (#84077)

This just seeks to give a high level overview of the current state of
the DirectX backend. It doesn't go in too deep, but should be a starting
point for further documentation.

---------

Co-authored-by: S. Bharadwaj Yadavalli 
---
 llvm/docs/DirectX/DXILArchitecture.rst | 46 ++++++++++++++++++++++++++
 1 file changed, 46 insertions(+)

diff --git a/llvm/docs/DirectX/DXILArchitecture.rst b/llvm/docs/DirectX/DXILArchitecture.rst
index f47db5413631..d6712bea4f77 100644
--- a/llvm/docs/DirectX/DXILArchitecture.rst
+++ b/llvm/docs/DirectX/DXILArchitecture.rst
@@ -89,6 +89,52 @@ leverage LLVM's current bitcode libraries to do a lot of the work, but
 it's possible that at some point in the future it will need to be
 completely separate as modern LLVM bitcode evolves.
 
+DirectX Backend Flow
+--------------------
+
+The code generation flow for DXIL is broken into a series of passes. The passes
+are grouped into two flows:
+
+#. Generating DXIL IR.
+#. Generating DXIL Binary.
+
+The passes to generate DXIL IR follow the flow:
+
+  DXILOpLowering -> DXILPrepare -> DXILTranslateMetadata
+
+Each of these passes has a defined responsibility:
+
+#. DXILOpLowering translates LLVM intrinsic calls to dx.op calls.
+#. DXILPrepare transforms the DXIL IR to be compatible with LLVM 3.7, and
+   inserts bitcasts to allow typed pointers to be inserted.
+#. DXILTranslateMetadata emits the DXIL Metadata structures.
+
+The passes to encode DXIL to binary in the DX Container follow the flow:
+
+  DXILEmbedder -> DXContainerGlobals -> AsmPrinter
+
+Each of these passes have the following defined responsibilities:
+
+#. DXILEmbedder runs the DXIL bitcode writer to generate a bitcode stream and
+   embeds the binary data inside a global in the original module.
+#. DXContainerGlobals generates binary data globals for the other DX Container
+   parts based on computed analysis passes.
+#. AsmPrinter is the standard LLVM infrastructure for emitting object files.
+
+When emitting DXIL into a DX Container file the MC layer is used in a similar
+way to how the Clang ``-fembed-bitcode`` option operates. The DX Container
+object writer knows how to construct the headers and structural fields of the
+container, and reads global variables from the module to fill in the remaining
+part data.
+
+DirectX Container
+-----------------
+
+The DirectX container format is treated in LLVM as an object file format.
+Reading is implemented between the BinaryFormat and Object libraries, and
+writing is implemented in the MC layer. Additional testing and inspection
+support are implemented in the ObjectYAML library and tools.
+
 Testing
 =======
 
-- 
GitLab


From 5a95378659506b0ce94ceb79a43477e73c9756f4 Mon Sep 17 00:00:00 2001
From: erichkeane 
Date: Fri, 8 Mar 2024 10:15:41 -0800
Subject: [PATCH 0062/1674] [NFC] Remove unnecessary 'Builtins.def' file.

This was replaced with tablegen anyway, so it just contained a stale
comment.  This patch removes it.
---
 clang/include/clang/Basic/Builtins.def | 102 -------------------------
 1 file changed, 102 deletions(-)
 delete mode 100644 clang/include/clang/Basic/Builtins.def

diff --git a/clang/include/clang/Basic/Builtins.def b/clang/include/clang/Basic/Builtins.def
deleted file mode 100644
index f356f881d5ef..000000000000
--- a/clang/include/clang/Basic/Builtins.def
+++ /dev/null
@@ -1,102 +0,0 @@
-//===--- Builtins.def - Builtin function info database ----------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// This is only documentation for the database layout. This will be removed once
-// all builtin databases are converted to tablegen files
-
-// The second value provided to the macro specifies the type of the function
-// (result value, then each argument) as follows:
-//  v -> void
-//  b -> boolean
-//  c -> char
-//  s -> short
-//  i -> int
-//  h -> half (__fp16, OpenCL)
-//  x -> half (_Float16)
-//  y -> half (__bf16)
-//  f -> float
-//  d -> double
-//  z -> size_t
-//  w -> wchar_t
-//  F -> constant CFString
-//  G -> id
-//  H -> SEL
-//  M -> struct objc_super
-//  a -> __builtin_va_list
-//  A -> "reference" to __builtin_va_list
-//  V -> Vector, followed by the number of elements and the base type.
-//  q -> Scalable vector, followed by the number of elements and the base type.
-//  Q -> target builtin type, followed by a character to distinguish the builtin type
-//    Qa -> AArch64 svcount_t builtin type.
-//  E -> ext_vector, followed by the number of elements and the base type.
-//  X -> _Complex, followed by the base type.
-//  Y -> ptrdiff_t
-//  P -> FILE
-//  J -> jmp_buf
-//  SJ -> sigjmp_buf
-//  K -> ucontext_t
-//  p -> pid_t
-//  . -> "...".  This may only occur at the end of the function list.
-//
-// Types may be prefixed with the following modifiers:
-//  L   -> long (e.g. Li for 'long int', Ld for 'long double')
-//  LL  -> long long (e.g. LLi for 'long long int', LLd for __float128)
-//  LLL -> __int128_t (e.g. LLLi)
-//  Z   -> int32_t (require a native 32-bit integer type on the target)
-//  W   -> int64_t (require a native 64-bit integer type on the target)
-//  N   -> 'int' size if target is LP64, 'L' otherwise.
-//  O   -> long for OpenCL targets, long long otherwise.
-//  S   -> signed
-//  U   -> unsigned
-//  I   -> Required to constant fold to an integer constant expression.
-//
-// Types may be postfixed with the following modifiers:
-// * -> pointer (optionally followed by an address space number, if no address
-//               space is specified than any address space will be accepted)
-// & -> reference (optionally followed by an address space number)
-// C -> const
-// D -> volatile
-// R -> restrict
-
-// The third value provided to the macro specifies information about attributes
-// of the function.  These must be kept in sync with the predicates in the
-// Builtin::Context class.  Currently we have:
-//  n -> nothrow
-//  r -> noreturn
-//  U -> pure
-//  c -> const
-//  t -> signature is meaningless, use custom typechecking
-//  T -> type is not important to semantic analysis and codegen; recognize as
-//       builtin even if type doesn't match signature, and don't warn if we
-//       can't be sure the type is right
-//  F -> this is a libc/libm function with a '__builtin_' prefix added.
-//  f -> this is a libc/libm function without a '__builtin_' prefix, or with
-//       'z', a C++ standard library function in namespace std::. This builtin
-//       is disableable by '-fno-builtin-foo' / '-fno-builtin-std-foo'.
-//  h -> this function requires a specific header or an explicit declaration.
-//  i -> this is a runtime library implemented function without the
-//       '__builtin_' prefix. It will be implemented in compiler-rt or libgcc.
-//  p:N: -> this is a printf-like function whose Nth argument is the format
-//          string.
-//  P:N: -> similar to the p:N: attribute, but the function is like vprintf
-//          in that it accepts its arguments as a va_list rather than
-//          through an ellipsis
-//  s:N: -> this is a scanf-like function whose Nth argument is the format
-//          string.
-//  S:N: -> similar to the s:N: attribute, but the function is like vscanf
-//          in that it accepts its arguments as a va_list rather than
-//          through an ellipsis
-//  e -> const, but only when -fno-math-errno and FP exceptions are ignored
-//  g -> const when FP exceptions are ignored
-//  j -> returns_twice (like setjmp)
-//  u -> arguments are not evaluated for their side-effects
-//  V:N: -> requires vectors of at least N bits to be legal
-//  C -> callback behavior: argument N is called with argument
-//                      M_0, ..., M_k as payload
-//  z -> this is a function in (possibly-versioned) namespace std
-//  E -> this function can be constant evaluated by Clang frontend
-- 
GitLab


From a456885efc95eb141d051ade517432c531d427f7 Mon Sep 17 00:00:00 2001
From: Craig Topper 
Date: Fri, 8 Mar 2024 10:17:54 -0800
Subject: [PATCH 0063/1674] [SelectionDAG] Allow FREEZE to be hoisted before
 integer SETCC. (#84241)

Teach canCreateUndefOrPoison that ISD::SETCC with integer operands can
never create undef/poison. FP SETCC is more complicated and will be
handled in a future patch.

Teach isGuaranteedNotToBeUndefOrPoison that ISD::CONDCODE is not
poison/undef. Its a special constant only used by setcc/select_cc like
nodes. This is needed since the hoisting will only hoist if exactly one
operand might be poison. setcc has 3 operand including the condition
code.

Recovers some regression from #84232.
---
 .../lib/CodeGen/SelectionDAG/SelectionDAG.cpp |   6 +
 llvm/test/CodeGen/RISCV/alu64.ll              |   3 +-
 llvm/test/CodeGen/RISCV/double-convert.ll     |  40 +++---
 llvm/test/CodeGen/RISCV/forced-atomics.ll     |   7 +-
 llvm/test/CodeGen/RISCV/fpclamptosat.ll       | 114 +++++++-----------
 .../CodeGen/RISCV/rvv/fpclamptosat_vec.ll     |  96 +++++++--------
 .../CodeGen/RISCV/signed-truncation-check.ll  |   9 +-
 7 files changed, 128 insertions(+), 147 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index f7ace79e8c51..50f53bbb04b6 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -4998,6 +4998,7 @@ bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op,
     return true;
 
   switch (Opcode) {
+  case ISD::CONDCODE:
   case ISD::VALUETYPE:
   case ISD::FrameIndex:
   case ISD::TargetFrameIndex:
@@ -5090,6 +5091,11 @@ bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts,
   case ISD::BUILD_PAIR:
     return false;
 
+  case ISD::SETCC:
+    // Integer setcc cannot create undef or poison.
+    // FIXME: Support FP.
+    return !Op.getOperand(0).getValueType().isInteger();
+
   // Matches hasPoisonGeneratingFlags().
   case ISD::ZERO_EXTEND:
     return ConsiderFlags && Op->getFlags().hasNonNeg();
diff --git a/llvm/test/CodeGen/RISCV/alu64.ll b/llvm/test/CodeGen/RISCV/alu64.ll
index e16f6abcca24..d2ee80e6aa95 100644
--- a/llvm/test/CodeGen/RISCV/alu64.ll
+++ b/llvm/test/CodeGen/RISCV/alu64.ll
@@ -57,9 +57,8 @@ define i64 @sltiu(i64 %a) nounwind {
 ;
 ; RV32I-LABEL: sltiu:
 ; RV32I:       # %bb.0:
+; RV32I-NEXT:    seqz a1, a1
 ; RV32I-NEXT:    sltiu a0, a0, 3
-; RV32I-NEXT:    snez a1, a1
-; RV32I-NEXT:    addi a1, a1, -1
 ; RV32I-NEXT:    and a0, a1, a0
 ; RV32I-NEXT:    li a1, 0
 ; RV32I-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/double-convert.ll b/llvm/test/CodeGen/RISCV/double-convert.ll
index f2e37f55521b..93cc32e76af4 100644
--- a/llvm/test/CodeGen/RISCV/double-convert.ll
+++ b/llvm/test/CodeGen/RISCV/double-convert.ll
@@ -868,32 +868,33 @@ define i64 @fcvt_l_d_sat(double %a) nounwind {
 ; RV32I-NEXT:    sw s3, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw s4, 8(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw s5, 4(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s6, 0(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a1
 ; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lui a3, 278016
-; RV32I-NEXT:    addi a3, a3, -1
-; RV32I-NEXT:    li a2, -1
-; RV32I-NEXT:    call __gtdf2
-; RV32I-NEXT:    mv s2, a0
 ; RV32I-NEXT:    lui a3, 802304
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s0
 ; RV32I-NEXT:    li a2, 0
 ; RV32I-NEXT:    call __gedf2
-; RV32I-NEXT:    mv s3, a0
+; RV32I-NEXT:    mv s2, a0
 ; RV32I-NEXT:    mv a0, s1
 ; RV32I-NEXT:    mv a1, s0
 ; RV32I-NEXT:    call __fixdfdi
-; RV32I-NEXT:    mv s4, a0
-; RV32I-NEXT:    mv s5, a1
-; RV32I-NEXT:    lui a0, 524288
-; RV32I-NEXT:    bgez s3, .LBB12_2
+; RV32I-NEXT:    mv s3, a0
+; RV32I-NEXT:    mv s4, a1
+; RV32I-NEXT:    lui s6, 524288
+; RV32I-NEXT:    bgez s2, .LBB12_2
 ; RV32I-NEXT:  # %bb.1: # %start
-; RV32I-NEXT:    lui s5, 524288
+; RV32I-NEXT:    lui s4, 524288
 ; RV32I-NEXT:  .LBB12_2: # %start
-; RV32I-NEXT:    blez s2, .LBB12_4
+; RV32I-NEXT:    lui a3, 278016
+; RV32I-NEXT:    addi a3, a3, -1
+; RV32I-NEXT:    li a2, -1
+; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    mv a1, s0
+; RV32I-NEXT:    call __gtdf2
+; RV32I-NEXT:    mv s5, a0
+; RV32I-NEXT:    blez a0, .LBB12_4
 ; RV32I-NEXT:  # %bb.3: # %start
-; RV32I-NEXT:    addi s5, a0, -1
+; RV32I-NEXT:    addi s4, s6, -1
 ; RV32I-NEXT:  .LBB12_4: # %start
 ; RV32I-NEXT:    mv a0, s1
 ; RV32I-NEXT:    mv a1, s0
@@ -902,11 +903,11 @@ define i64 @fcvt_l_d_sat(double %a) nounwind {
 ; RV32I-NEXT:    call __unorddf2
 ; RV32I-NEXT:    snez a0, a0
 ; RV32I-NEXT:    addi a0, a0, -1
-; RV32I-NEXT:    and a1, a0, s5
-; RV32I-NEXT:    slti a2, s3, 0
+; RV32I-NEXT:    and a1, a0, s4
+; RV32I-NEXT:    slti a2, s2, 0
 ; RV32I-NEXT:    addi a2, a2, -1
-; RV32I-NEXT:    and a2, a2, s4
-; RV32I-NEXT:    sgtz a3, s2
+; RV32I-NEXT:    and a2, a2, s3
+; RV32I-NEXT:    sgtz a3, s5
 ; RV32I-NEXT:    neg a3, a3
 ; RV32I-NEXT:    or a2, a3, a2
 ; RV32I-NEXT:    and a0, a0, a2
@@ -917,6 +918,7 @@ define i64 @fcvt_l_d_sat(double %a) nounwind {
 ; RV32I-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s4, 8(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s5, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s6, 0(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/RISCV/forced-atomics.ll b/llvm/test/CodeGen/RISCV/forced-atomics.ll
index 659e0748dd53..c303690aadff 100644
--- a/llvm/test/CodeGen/RISCV/forced-atomics.ll
+++ b/llvm/test/CodeGen/RISCV/forced-atomics.ll
@@ -3567,8 +3567,8 @@ define i64 @rmw64_umax_seq_cst(ptr %p) nounwind {
 ; RV32-NEXT:    # in Loop: Header=BB51_2 Depth=1
 ; RV32-NEXT:    neg a3, a0
 ; RV32-NEXT:    and a3, a3, a1
-; RV32-NEXT:    sw a4, 0(sp)
 ; RV32-NEXT:    sw a1, 4(sp)
+; RV32-NEXT:    sw a4, 0(sp)
 ; RV32-NEXT:    mv a1, sp
 ; RV32-NEXT:    li a4, 5
 ; RV32-NEXT:    li a5, 5
@@ -3659,8 +3659,8 @@ define i64 @rmw64_umin_seq_cst(ptr %p) nounwind {
 ; RV32-NEXT:    # in Loop: Header=BB52_2 Depth=1
 ; RV32-NEXT:    neg a3, a0
 ; RV32-NEXT:    and a3, a3, a1
-; RV32-NEXT:    sw a4, 0(sp)
 ; RV32-NEXT:    sw a1, 4(sp)
+; RV32-NEXT:    sw a4, 0(sp)
 ; RV32-NEXT:    mv a1, sp
 ; RV32-NEXT:    li a4, 5
 ; RV32-NEXT:    li a5, 5
@@ -3672,8 +3672,7 @@ define i64 @rmw64_umin_seq_cst(ptr %p) nounwind {
 ; RV32-NEXT:  .LBB52_2: # %atomicrmw.start
 ; RV32-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32-NEXT:    sltiu a0, a4, 2
-; RV32-NEXT:    snez a2, a1
-; RV32-NEXT:    addi a2, a2, -1
+; RV32-NEXT:    seqz a2, a1
 ; RV32-NEXT:    and a0, a2, a0
 ; RV32-NEXT:    mv a2, a4
 ; RV32-NEXT:    bnez a0, .LBB52_1
diff --git a/llvm/test/CodeGen/RISCV/fpclamptosat.ll b/llvm/test/CodeGen/RISCV/fpclamptosat.ll
index 630d16e7c888..06ab813faf02 100644
--- a/llvm/test/CodeGen/RISCV/fpclamptosat.ll
+++ b/llvm/test/CodeGen/RISCV/fpclamptosat.ll
@@ -114,9 +114,8 @@ define i32 @utest_f64i32(double %x) {
 ; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IF-NEXT:    .cfi_offset ra, -4
 ; RV32IF-NEXT:    call __fixunsdfdi
+; RV32IF-NEXT:    seqz a1, a1
 ; RV32IF-NEXT:    sltiu a2, a0, -1
-; RV32IF-NEXT:    snez a1, a1
-; RV32IF-NEXT:    addi a1, a1, -1
 ; RV32IF-NEXT:    and a1, a1, a2
 ; RV32IF-NEXT:    addi a1, a1, -1
 ; RV32IF-NEXT:    or a0, a1, a0
@@ -430,9 +429,8 @@ define i32 @utesth_f16i32(half %x) {
 ; RV32-NEXT:    .cfi_offset ra, -4
 ; RV32-NEXT:    call __extendhfsf2
 ; RV32-NEXT:    call __fixunssfdi
+; RV32-NEXT:    seqz a1, a1
 ; RV32-NEXT:    sltiu a2, a0, -1
-; RV32-NEXT:    snez a1, a1
-; RV32-NEXT:    addi a1, a1, -1
 ; RV32-NEXT:    and a1, a1, a2
 ; RV32-NEXT:    addi a1, a1, -1
 ; RV32-NEXT:    or a0, a1, a0
@@ -1229,10 +1227,8 @@ define i64 @utest_f64i64(double %x) {
 ; RV32IF-NEXT:    lw a1, 20(sp)
 ; RV32IF-NEXT:    lw a2, 12(sp)
 ; RV32IF-NEXT:    lw a3, 8(sp)
-; RV32IF-NEXT:    seqz a4, a0
-; RV32IF-NEXT:    snez a5, a1
-; RV32IF-NEXT:    addi a5, a5, -1
-; RV32IF-NEXT:    and a4, a5, a4
+; RV32IF-NEXT:    or a4, a1, a0
+; RV32IF-NEXT:    seqz a4, a4
 ; RV32IF-NEXT:    xori a0, a0, 1
 ; RV32IF-NEXT:    or a0, a0, a1
 ; RV32IF-NEXT:    seqz a0, a0
@@ -1271,10 +1267,8 @@ define i64 @utest_f64i64(double %x) {
 ; RV32IFD-NEXT:    lw a1, 20(sp)
 ; RV32IFD-NEXT:    lw a2, 12(sp)
 ; RV32IFD-NEXT:    lw a3, 8(sp)
-; RV32IFD-NEXT:    seqz a4, a0
-; RV32IFD-NEXT:    snez a5, a1
-; RV32IFD-NEXT:    addi a5, a5, -1
-; RV32IFD-NEXT:    and a4, a5, a4
+; RV32IFD-NEXT:    or a4, a1, a0
+; RV32IFD-NEXT:    seqz a4, a4
 ; RV32IFD-NEXT:    xori a0, a0, 1
 ; RV32IFD-NEXT:    or a0, a0, a1
 ; RV32IFD-NEXT:    seqz a0, a0
@@ -1324,8 +1318,8 @@ define i64 @ustest_f64i64(double %x) {
 ; RV32IF-NEXT:  # %bb.4: # %entry
 ; RV32IF-NEXT:    li a0, 1
 ; RV32IF-NEXT:  .LBB20_5: # %entry
-; RV32IF-NEXT:    lw a4, 8(sp)
-; RV32IF-NEXT:    lw a3, 12(sp)
+; RV32IF-NEXT:    lw a3, 8(sp)
+; RV32IF-NEXT:    lw a4, 12(sp)
 ; RV32IF-NEXT:    and a5, a2, a1
 ; RV32IF-NEXT:    beqz a5, .LBB20_7
 ; RV32IF-NEXT:  # %bb.6: # %entry
@@ -1334,18 +1328,17 @@ define i64 @ustest_f64i64(double %x) {
 ; RV32IF-NEXT:  .LBB20_7:
 ; RV32IF-NEXT:    snez a1, a0
 ; RV32IF-NEXT:  .LBB20_8: # %entry
-; RV32IF-NEXT:    and a3, a2, a3
+; RV32IF-NEXT:    and a4, a2, a4
 ; RV32IF-NEXT:    or a0, a0, a5
-; RV32IF-NEXT:    and a2, a2, a4
+; RV32IF-NEXT:    and a2, a2, a3
 ; RV32IF-NEXT:    bnez a0, .LBB20_10
 ; RV32IF-NEXT:  # %bb.9:
-; RV32IF-NEXT:    snez a0, a3
-; RV32IF-NEXT:    snez a1, a2
-; RV32IF-NEXT:    or a1, a1, a0
+; RV32IF-NEXT:    or a0, a2, a4
+; RV32IF-NEXT:    snez a1, a0
 ; RV32IF-NEXT:  .LBB20_10: # %entry
 ; RV32IF-NEXT:    neg a1, a1
 ; RV32IF-NEXT:    and a0, a1, a2
-; RV32IF-NEXT:    and a1, a1, a3
+; RV32IF-NEXT:    and a1, a1, a4
 ; RV32IF-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 32
 ; RV32IF-NEXT:    ret
@@ -1404,8 +1397,8 @@ define i64 @ustest_f64i64(double %x) {
 ; RV32IFD-NEXT:  # %bb.4: # %entry
 ; RV32IFD-NEXT:    li a0, 1
 ; RV32IFD-NEXT:  .LBB20_5: # %entry
-; RV32IFD-NEXT:    lw a4, 8(sp)
-; RV32IFD-NEXT:    lw a3, 12(sp)
+; RV32IFD-NEXT:    lw a3, 8(sp)
+; RV32IFD-NEXT:    lw a4, 12(sp)
 ; RV32IFD-NEXT:    and a5, a2, a1
 ; RV32IFD-NEXT:    beqz a5, .LBB20_7
 ; RV32IFD-NEXT:  # %bb.6: # %entry
@@ -1414,18 +1407,17 @@ define i64 @ustest_f64i64(double %x) {
 ; RV32IFD-NEXT:  .LBB20_7:
 ; RV32IFD-NEXT:    snez a1, a0
 ; RV32IFD-NEXT:  .LBB20_8: # %entry
-; RV32IFD-NEXT:    and a3, a2, a3
+; RV32IFD-NEXT:    and a4, a2, a4
 ; RV32IFD-NEXT:    or a0, a0, a5
-; RV32IFD-NEXT:    and a2, a2, a4
+; RV32IFD-NEXT:    and a2, a2, a3
 ; RV32IFD-NEXT:    bnez a0, .LBB20_10
 ; RV32IFD-NEXT:  # %bb.9:
-; RV32IFD-NEXT:    snez a0, a3
-; RV32IFD-NEXT:    snez a1, a2
-; RV32IFD-NEXT:    or a1, a1, a0
+; RV32IFD-NEXT:    or a0, a2, a4
+; RV32IFD-NEXT:    snez a1, a0
 ; RV32IFD-NEXT:  .LBB20_10: # %entry
 ; RV32IFD-NEXT:    neg a1, a1
 ; RV32IFD-NEXT:    and a0, a1, a2
-; RV32IFD-NEXT:    and a1, a1, a3
+; RV32IFD-NEXT:    and a1, a1, a4
 ; RV32IFD-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 32
 ; RV32IFD-NEXT:    ret
@@ -1531,10 +1523,8 @@ define i64 @utest_f32i64(float %x) {
 ; RV32-NEXT:    lw a1, 20(sp)
 ; RV32-NEXT:    lw a2, 12(sp)
 ; RV32-NEXT:    lw a3, 8(sp)
-; RV32-NEXT:    seqz a4, a0
-; RV32-NEXT:    snez a5, a1
-; RV32-NEXT:    addi a5, a5, -1
-; RV32-NEXT:    and a4, a5, a4
+; RV32-NEXT:    or a4, a1, a0
+; RV32-NEXT:    seqz a4, a4
 ; RV32-NEXT:    xori a0, a0, 1
 ; RV32-NEXT:    or a0, a0, a1
 ; RV32-NEXT:    seqz a0, a0
@@ -1596,8 +1586,8 @@ define i64 @ustest_f32i64(float %x) {
 ; RV32-NEXT:  # %bb.4: # %entry
 ; RV32-NEXT:    li a0, 1
 ; RV32-NEXT:  .LBB23_5: # %entry
-; RV32-NEXT:    lw a4, 8(sp)
-; RV32-NEXT:    lw a3, 12(sp)
+; RV32-NEXT:    lw a3, 8(sp)
+; RV32-NEXT:    lw a4, 12(sp)
 ; RV32-NEXT:    and a5, a2, a1
 ; RV32-NEXT:    beqz a5, .LBB23_7
 ; RV32-NEXT:  # %bb.6: # %entry
@@ -1606,18 +1596,17 @@ define i64 @ustest_f32i64(float %x) {
 ; RV32-NEXT:  .LBB23_7:
 ; RV32-NEXT:    snez a1, a0
 ; RV32-NEXT:  .LBB23_8: # %entry
-; RV32-NEXT:    and a3, a2, a3
+; RV32-NEXT:    and a4, a2, a4
 ; RV32-NEXT:    or a0, a0, a5
-; RV32-NEXT:    and a2, a2, a4
+; RV32-NEXT:    and a2, a2, a3
 ; RV32-NEXT:    bnez a0, .LBB23_10
 ; RV32-NEXT:  # %bb.9:
-; RV32-NEXT:    snez a0, a3
-; RV32-NEXT:    snez a1, a2
-; RV32-NEXT:    or a1, a1, a0
+; RV32-NEXT:    or a0, a2, a4
+; RV32-NEXT:    snez a1, a0
 ; RV32-NEXT:  .LBB23_10: # %entry
 ; RV32-NEXT:    neg a1, a1
 ; RV32-NEXT:    and a0, a1, a2
-; RV32-NEXT:    and a1, a1, a3
+; RV32-NEXT:    and a1, a1, a4
 ; RV32-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32-NEXT:    addi sp, sp, 32
 ; RV32-NEXT:    ret
@@ -1783,10 +1772,8 @@ define i64 @utesth_f16i64(half %x) {
 ; RV32-NEXT:    lw a1, 20(sp)
 ; RV32-NEXT:    lw a2, 12(sp)
 ; RV32-NEXT:    lw a3, 8(sp)
-; RV32-NEXT:    seqz a4, a0
-; RV32-NEXT:    snez a5, a1
-; RV32-NEXT:    addi a5, a5, -1
-; RV32-NEXT:    and a4, a5, a4
+; RV32-NEXT:    or a4, a1, a0
+; RV32-NEXT:    seqz a4, a4
 ; RV32-NEXT:    xori a0, a0, 1
 ; RV32-NEXT:    or a0, a0, a1
 ; RV32-NEXT:    seqz a0, a0
@@ -1850,8 +1837,8 @@ define i64 @ustest_f16i64(half %x) {
 ; RV32-NEXT:  # %bb.4: # %entry
 ; RV32-NEXT:    li a0, 1
 ; RV32-NEXT:  .LBB26_5: # %entry
-; RV32-NEXT:    lw a4, 8(sp)
-; RV32-NEXT:    lw a3, 12(sp)
+; RV32-NEXT:    lw a3, 8(sp)
+; RV32-NEXT:    lw a4, 12(sp)
 ; RV32-NEXT:    and a5, a2, a1
 ; RV32-NEXT:    beqz a5, .LBB26_7
 ; RV32-NEXT:  # %bb.6: # %entry
@@ -1860,18 +1847,17 @@ define i64 @ustest_f16i64(half %x) {
 ; RV32-NEXT:  .LBB26_7:
 ; RV32-NEXT:    snez a1, a0
 ; RV32-NEXT:  .LBB26_8: # %entry
-; RV32-NEXT:    and a3, a2, a3
+; RV32-NEXT:    and a4, a2, a4
 ; RV32-NEXT:    or a0, a0, a5
-; RV32-NEXT:    and a2, a2, a4
+; RV32-NEXT:    and a2, a2, a3
 ; RV32-NEXT:    bnez a0, .LBB26_10
 ; RV32-NEXT:  # %bb.9:
-; RV32-NEXT:    snez a0, a3
-; RV32-NEXT:    snez a1, a2
-; RV32-NEXT:    or a1, a1, a0
+; RV32-NEXT:    or a0, a2, a4
+; RV32-NEXT:    snez a1, a0
 ; RV32-NEXT:  .LBB26_10: # %entry
 ; RV32-NEXT:    neg a1, a1
 ; RV32-NEXT:    and a0, a1, a2
-; RV32-NEXT:    and a1, a1, a3
+; RV32-NEXT:    and a1, a1, a4
 ; RV32-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32-NEXT:    addi sp, sp, 32
 ; RV32-NEXT:    ret
@@ -3087,10 +3073,8 @@ define i64 @utest_f64i64_mm(double %x) {
 ; RV32IF-NEXT:    lw a1, 20(sp)
 ; RV32IF-NEXT:    lw a2, 12(sp)
 ; RV32IF-NEXT:    lw a3, 8(sp)
-; RV32IF-NEXT:    seqz a4, a0
-; RV32IF-NEXT:    snez a5, a1
-; RV32IF-NEXT:    addi a5, a5, -1
-; RV32IF-NEXT:    and a4, a5, a4
+; RV32IF-NEXT:    or a4, a1, a0
+; RV32IF-NEXT:    seqz a4, a4
 ; RV32IF-NEXT:    xori a0, a0, 1
 ; RV32IF-NEXT:    or a0, a0, a1
 ; RV32IF-NEXT:    seqz a0, a0
@@ -3129,10 +3113,8 @@ define i64 @utest_f64i64_mm(double %x) {
 ; RV32IFD-NEXT:    lw a1, 20(sp)
 ; RV32IFD-NEXT:    lw a2, 12(sp)
 ; RV32IFD-NEXT:    lw a3, 8(sp)
-; RV32IFD-NEXT:    seqz a4, a0
-; RV32IFD-NEXT:    snez a5, a1
-; RV32IFD-NEXT:    addi a5, a5, -1
-; RV32IFD-NEXT:    and a4, a5, a4
+; RV32IFD-NEXT:    or a4, a1, a0
+; RV32IFD-NEXT:    seqz a4, a4
 ; RV32IFD-NEXT:    xori a0, a0, 1
 ; RV32IFD-NEXT:    or a0, a0, a1
 ; RV32IFD-NEXT:    seqz a0, a0
@@ -3345,10 +3327,8 @@ define i64 @utest_f32i64_mm(float %x) {
 ; RV32-NEXT:    lw a1, 20(sp)
 ; RV32-NEXT:    lw a2, 12(sp)
 ; RV32-NEXT:    lw a3, 8(sp)
-; RV32-NEXT:    seqz a4, a0
-; RV32-NEXT:    snez a5, a1
-; RV32-NEXT:    addi a5, a5, -1
-; RV32-NEXT:    and a4, a5, a4
+; RV32-NEXT:    or a4, a1, a0
+; RV32-NEXT:    seqz a4, a4
 ; RV32-NEXT:    xori a0, a0, 1
 ; RV32-NEXT:    or a0, a0, a1
 ; RV32-NEXT:    seqz a0, a0
@@ -3570,10 +3550,8 @@ define i64 @utesth_f16i64_mm(half %x) {
 ; RV32-NEXT:    lw a1, 20(sp)
 ; RV32-NEXT:    lw a2, 12(sp)
 ; RV32-NEXT:    lw a3, 8(sp)
-; RV32-NEXT:    seqz a4, a0
-; RV32-NEXT:    snez a5, a1
-; RV32-NEXT:    addi a5, a5, -1
-; RV32-NEXT:    and a4, a5, a4
+; RV32-NEXT:    or a4, a1, a0
+; RV32-NEXT:    seqz a4, a4
 ; RV32-NEXT:    xori a0, a0, 1
 ; RV32-NEXT:    or a0, a0, a1
 ; RV32-NEXT:    seqz a0, a0
diff --git a/llvm/test/CodeGen/RISCV/rvv/fpclamptosat_vec.ll b/llvm/test/CodeGen/RISCV/rvv/fpclamptosat_vec.ll
index 3ada24bd9846..b3bda5973eb8 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fpclamptosat_vec.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fpclamptosat_vec.ll
@@ -5811,15 +5811,15 @@ define <2 x i64> @ustest_f64i64_mm(<2 x double> %x) {
 ; CHECK-NOV-NEXT:    mv s1, a1
 ; CHECK-NOV-NEXT:    fmv.d fa0, fs0
 ; CHECK-NOV-NEXT:    call __fixdfti
-; CHECK-NOV-NEXT:    mv a2, a1
+; CHECK-NOV-NEXT:    mv a2, s1
+; CHECK-NOV-NEXT:    mv a3, a1
 ; CHECK-NOV-NEXT:    blez a1, .LBB47_2
 ; CHECK-NOV-NEXT:  # %bb.1: # %entry
-; CHECK-NOV-NEXT:    li a2, 1
+; CHECK-NOV-NEXT:    li a3, 1
 ; CHECK-NOV-NEXT:  .LBB47_2: # %entry
-; CHECK-NOV-NEXT:    mv a3, s1
-; CHECK-NOV-NEXT:    blez s1, .LBB47_4
+; CHECK-NOV-NEXT:    blez a2, .LBB47_4
 ; CHECK-NOV-NEXT:  # %bb.3: # %entry
-; CHECK-NOV-NEXT:    li a3, 1
+; CHECK-NOV-NEXT:    li a2, 1
 ; CHECK-NOV-NEXT:  .LBB47_4: # %entry
 ; CHECK-NOV-NEXT:    slti a1, a1, 1
 ; CHECK-NOV-NEXT:    neg a1, a1
@@ -5827,11 +5827,11 @@ define <2 x i64> @ustest_f64i64_mm(<2 x double> %x) {
 ; CHECK-NOV-NEXT:    slti a0, s1, 1
 ; CHECK-NOV-NEXT:    neg a0, a0
 ; CHECK-NOV-NEXT:    and a0, a0, s0
-; CHECK-NOV-NEXT:    slti a3, a3, 0
-; CHECK-NOV-NEXT:    addi a3, a3, -1
-; CHECK-NOV-NEXT:    and a0, a3, a0
 ; CHECK-NOV-NEXT:    slti a2, a2, 0
 ; CHECK-NOV-NEXT:    addi a2, a2, -1
+; CHECK-NOV-NEXT:    and a0, a2, a0
+; CHECK-NOV-NEXT:    slti a2, a3, 0
+; CHECK-NOV-NEXT:    addi a2, a2, -1
 ; CHECK-NOV-NEXT:    and a1, a2, a1
 ; CHECK-NOV-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; CHECK-NOV-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
@@ -5867,15 +5867,15 @@ define <2 x i64> @ustest_f64i64_mm(<2 x double> %x) {
 ; CHECK-V-NEXT:    vl1r.v v8, (a0) # Unknown-size Folded Reload
 ; CHECK-V-NEXT:    vfmv.f.s fa0, v8
 ; CHECK-V-NEXT:    call __fixdfti
-; CHECK-V-NEXT:    mv a2, a1
+; CHECK-V-NEXT:    mv a2, s1
+; CHECK-V-NEXT:    mv a3, a1
 ; CHECK-V-NEXT:    blez a1, .LBB47_2
 ; CHECK-V-NEXT:  # %bb.1: # %entry
-; CHECK-V-NEXT:    li a2, 1
+; CHECK-V-NEXT:    li a3, 1
 ; CHECK-V-NEXT:  .LBB47_2: # %entry
-; CHECK-V-NEXT:    mv a3, s1
-; CHECK-V-NEXT:    blez s1, .LBB47_4
+; CHECK-V-NEXT:    blez a2, .LBB47_4
 ; CHECK-V-NEXT:  # %bb.3: # %entry
-; CHECK-V-NEXT:    li a3, 1
+; CHECK-V-NEXT:    li a2, 1
 ; CHECK-V-NEXT:  .LBB47_4: # %entry
 ; CHECK-V-NEXT:    slti a1, a1, 1
 ; CHECK-V-NEXT:    neg a1, a1
@@ -5883,11 +5883,11 @@ define <2 x i64> @ustest_f64i64_mm(<2 x double> %x) {
 ; CHECK-V-NEXT:    slti a1, s1, 1
 ; CHECK-V-NEXT:    neg a1, a1
 ; CHECK-V-NEXT:    and a1, a1, s0
-; CHECK-V-NEXT:    slti a3, a3, 0
-; CHECK-V-NEXT:    addi a3, a3, -1
-; CHECK-V-NEXT:    and a1, a3, a1
 ; CHECK-V-NEXT:    slti a2, a2, 0
 ; CHECK-V-NEXT:    addi a2, a2, -1
+; CHECK-V-NEXT:    and a1, a2, a1
+; CHECK-V-NEXT:    slti a2, a3, 0
+; CHECK-V-NEXT:    addi a2, a2, -1
 ; CHECK-V-NEXT:    and a0, a2, a0
 ; CHECK-V-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; CHECK-V-NEXT:    vmv.s.x v8, a0
@@ -6197,15 +6197,15 @@ define <2 x i64> @ustest_f32i64_mm(<2 x float> %x) {
 ; CHECK-NOV-NEXT:    mv s1, a1
 ; CHECK-NOV-NEXT:    fmv.s fa0, fs0
 ; CHECK-NOV-NEXT:    call __fixsfti
-; CHECK-NOV-NEXT:    mv a2, a1
+; CHECK-NOV-NEXT:    mv a2, s1
+; CHECK-NOV-NEXT:    mv a3, a1
 ; CHECK-NOV-NEXT:    blez a1, .LBB50_2
 ; CHECK-NOV-NEXT:  # %bb.1: # %entry
-; CHECK-NOV-NEXT:    li a2, 1
+; CHECK-NOV-NEXT:    li a3, 1
 ; CHECK-NOV-NEXT:  .LBB50_2: # %entry
-; CHECK-NOV-NEXT:    mv a3, s1
-; CHECK-NOV-NEXT:    blez s1, .LBB50_4
+; CHECK-NOV-NEXT:    blez a2, .LBB50_4
 ; CHECK-NOV-NEXT:  # %bb.3: # %entry
-; CHECK-NOV-NEXT:    li a3, 1
+; CHECK-NOV-NEXT:    li a2, 1
 ; CHECK-NOV-NEXT:  .LBB50_4: # %entry
 ; CHECK-NOV-NEXT:    slti a1, a1, 1
 ; CHECK-NOV-NEXT:    neg a1, a1
@@ -6213,11 +6213,11 @@ define <2 x i64> @ustest_f32i64_mm(<2 x float> %x) {
 ; CHECK-NOV-NEXT:    slti a0, s1, 1
 ; CHECK-NOV-NEXT:    neg a0, a0
 ; CHECK-NOV-NEXT:    and a0, a0, s0
-; CHECK-NOV-NEXT:    slti a3, a3, 0
-; CHECK-NOV-NEXT:    addi a3, a3, -1
-; CHECK-NOV-NEXT:    and a0, a3, a0
 ; CHECK-NOV-NEXT:    slti a2, a2, 0
 ; CHECK-NOV-NEXT:    addi a2, a2, -1
+; CHECK-NOV-NEXT:    and a0, a2, a0
+; CHECK-NOV-NEXT:    slti a2, a3, 0
+; CHECK-NOV-NEXT:    addi a2, a2, -1
 ; CHECK-NOV-NEXT:    and a1, a2, a1
 ; CHECK-NOV-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; CHECK-NOV-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
@@ -6253,15 +6253,15 @@ define <2 x i64> @ustest_f32i64_mm(<2 x float> %x) {
 ; CHECK-V-NEXT:    vl1r.v v8, (a0) # Unknown-size Folded Reload
 ; CHECK-V-NEXT:    vfmv.f.s fa0, v8
 ; CHECK-V-NEXT:    call __fixsfti
-; CHECK-V-NEXT:    mv a2, a1
+; CHECK-V-NEXT:    mv a2, s1
+; CHECK-V-NEXT:    mv a3, a1
 ; CHECK-V-NEXT:    blez a1, .LBB50_2
 ; CHECK-V-NEXT:  # %bb.1: # %entry
-; CHECK-V-NEXT:    li a2, 1
+; CHECK-V-NEXT:    li a3, 1
 ; CHECK-V-NEXT:  .LBB50_2: # %entry
-; CHECK-V-NEXT:    mv a3, s1
-; CHECK-V-NEXT:    blez s1, .LBB50_4
+; CHECK-V-NEXT:    blez a2, .LBB50_4
 ; CHECK-V-NEXT:  # %bb.3: # %entry
-; CHECK-V-NEXT:    li a3, 1
+; CHECK-V-NEXT:    li a2, 1
 ; CHECK-V-NEXT:  .LBB50_4: # %entry
 ; CHECK-V-NEXT:    slti a1, a1, 1
 ; CHECK-V-NEXT:    neg a1, a1
@@ -6269,11 +6269,11 @@ define <2 x i64> @ustest_f32i64_mm(<2 x float> %x) {
 ; CHECK-V-NEXT:    slti a1, s1, 1
 ; CHECK-V-NEXT:    neg a1, a1
 ; CHECK-V-NEXT:    and a1, a1, s0
-; CHECK-V-NEXT:    slti a3, a3, 0
-; CHECK-V-NEXT:    addi a3, a3, -1
-; CHECK-V-NEXT:    and a1, a3, a1
 ; CHECK-V-NEXT:    slti a2, a2, 0
 ; CHECK-V-NEXT:    addi a2, a2, -1
+; CHECK-V-NEXT:    and a1, a2, a1
+; CHECK-V-NEXT:    slti a2, a3, 0
+; CHECK-V-NEXT:    addi a2, a2, -1
 ; CHECK-V-NEXT:    and a0, a2, a0
 ; CHECK-V-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; CHECK-V-NEXT:    vmv.s.x v8, a0
@@ -6575,15 +6575,15 @@ define <2 x i64> @ustest_f16i64_mm(<2 x half> %x) {
 ; CHECK-NOV-NEXT:    fmv.w.x fa0, s2
 ; CHECK-NOV-NEXT:    call __extendhfsf2
 ; CHECK-NOV-NEXT:    call __fixsfti
-; CHECK-NOV-NEXT:    mv a2, a1
+; CHECK-NOV-NEXT:    mv a2, s1
+; CHECK-NOV-NEXT:    mv a3, a1
 ; CHECK-NOV-NEXT:    blez a1, .LBB53_2
 ; CHECK-NOV-NEXT:  # %bb.1: # %entry
-; CHECK-NOV-NEXT:    li a2, 1
+; CHECK-NOV-NEXT:    li a3, 1
 ; CHECK-NOV-NEXT:  .LBB53_2: # %entry
-; CHECK-NOV-NEXT:    mv a3, s1
-; CHECK-NOV-NEXT:    blez s1, .LBB53_4
+; CHECK-NOV-NEXT:    blez a2, .LBB53_4
 ; CHECK-NOV-NEXT:  # %bb.3: # %entry
-; CHECK-NOV-NEXT:    li a3, 1
+; CHECK-NOV-NEXT:    li a2, 1
 ; CHECK-NOV-NEXT:  .LBB53_4: # %entry
 ; CHECK-NOV-NEXT:    slti a1, a1, 1
 ; CHECK-NOV-NEXT:    neg a1, a1
@@ -6591,11 +6591,11 @@ define <2 x i64> @ustest_f16i64_mm(<2 x half> %x) {
 ; CHECK-NOV-NEXT:    slti a0, s1, 1
 ; CHECK-NOV-NEXT:    neg a0, a0
 ; CHECK-NOV-NEXT:    and a0, a0, s0
-; CHECK-NOV-NEXT:    slti a3, a3, 0
-; CHECK-NOV-NEXT:    addi a3, a3, -1
-; CHECK-NOV-NEXT:    and a0, a3, a0
 ; CHECK-NOV-NEXT:    slti a2, a2, 0
 ; CHECK-NOV-NEXT:    addi a2, a2, -1
+; CHECK-NOV-NEXT:    and a0, a2, a0
+; CHECK-NOV-NEXT:    slti a2, a3, 0
+; CHECK-NOV-NEXT:    addi a2, a2, -1
 ; CHECK-NOV-NEXT:    and a1, a2, a1
 ; CHECK-NOV-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; CHECK-NOV-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
@@ -6625,15 +6625,15 @@ define <2 x i64> @ustest_f16i64_mm(<2 x half> %x) {
 ; CHECK-V-NEXT:    fmv.w.x fa0, s2
 ; CHECK-V-NEXT:    call __extendhfsf2
 ; CHECK-V-NEXT:    call __fixsfti
-; CHECK-V-NEXT:    mv a2, a1
+; CHECK-V-NEXT:    mv a2, s1
+; CHECK-V-NEXT:    mv a3, a1
 ; CHECK-V-NEXT:    blez a1, .LBB53_2
 ; CHECK-V-NEXT:  # %bb.1: # %entry
-; CHECK-V-NEXT:    li a2, 1
+; CHECK-V-NEXT:    li a3, 1
 ; CHECK-V-NEXT:  .LBB53_2: # %entry
-; CHECK-V-NEXT:    mv a3, s1
-; CHECK-V-NEXT:    blez s1, .LBB53_4
+; CHECK-V-NEXT:    blez a2, .LBB53_4
 ; CHECK-V-NEXT:  # %bb.3: # %entry
-; CHECK-V-NEXT:    li a3, 1
+; CHECK-V-NEXT:    li a2, 1
 ; CHECK-V-NEXT:  .LBB53_4: # %entry
 ; CHECK-V-NEXT:    slti a1, a1, 1
 ; CHECK-V-NEXT:    neg a1, a1
@@ -6641,11 +6641,11 @@ define <2 x i64> @ustest_f16i64_mm(<2 x half> %x) {
 ; CHECK-V-NEXT:    slti a1, s1, 1
 ; CHECK-V-NEXT:    neg a1, a1
 ; CHECK-V-NEXT:    and a1, a1, s0
-; CHECK-V-NEXT:    slti a3, a3, 0
-; CHECK-V-NEXT:    addi a3, a3, -1
-; CHECK-V-NEXT:    and a1, a3, a1
 ; CHECK-V-NEXT:    slti a2, a2, 0
 ; CHECK-V-NEXT:    addi a2, a2, -1
+; CHECK-V-NEXT:    and a1, a2, a1
+; CHECK-V-NEXT:    slti a2, a3, 0
+; CHECK-V-NEXT:    addi a2, a2, -1
 ; CHECK-V-NEXT:    and a0, a2, a0
 ; CHECK-V-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; CHECK-V-NEXT:    vmv.s.x v9, a0
diff --git a/llvm/test/CodeGen/RISCV/signed-truncation-check.ll b/llvm/test/CodeGen/RISCV/signed-truncation-check.ll
index 069b2febc334..de36bcdb9106 100644
--- a/llvm/test/CodeGen/RISCV/signed-truncation-check.ll
+++ b/llvm/test/CodeGen/RISCV/signed-truncation-check.ll
@@ -422,8 +422,7 @@ define i1 @add_ugecmp_i64_i16(i64 %x) nounwind {
 ; RV32I-NEXT:    lui a1, 1048560
 ; RV32I-NEXT:    addi a1, a1, -1
 ; RV32I-NEXT:    sltu a1, a1, a2
-; RV32I-NEXT:    snez a0, a0
-; RV32I-NEXT:    addi a0, a0, -1
+; RV32I-NEXT:    seqz a0, a0
 ; RV32I-NEXT:    and a0, a0, a1
 ; RV32I-NEXT:    ret
 ;
@@ -463,8 +462,7 @@ define i1 @add_ugecmp_i64_i8(i64 %x) nounwind {
 ; RV32I-NEXT:    addi a2, a0, -128
 ; RV32I-NEXT:    sltu a0, a2, a0
 ; RV32I-NEXT:    add a0, a1, a0
-; RV32I-NEXT:    snez a0, a0
-; RV32I-NEXT:    addi a0, a0, -1
+; RV32I-NEXT:    seqz a0, a0
 ; RV32I-NEXT:    sltiu a1, a2, -256
 ; RV32I-NEXT:    xori a1, a1, 1
 ; RV32I-NEXT:    and a0, a0, a1
@@ -693,8 +691,7 @@ define i1 @add_ultcmp_i64_i8(i64 %x) nounwind {
 ; RV32I-NEXT:    addi a2, a0, 128
 ; RV32I-NEXT:    sltu a0, a2, a0
 ; RV32I-NEXT:    add a0, a1, a0
-; RV32I-NEXT:    snez a0, a0
-; RV32I-NEXT:    addi a0, a0, -1
+; RV32I-NEXT:    seqz a0, a0
 ; RV32I-NEXT:    sltiu a1, a2, 256
 ; RV32I-NEXT:    and a0, a0, a1
 ; RV32I-NEXT:    ret
-- 
GitLab


From 839a8fecb4c5dfe1b4484d5fc942a9490867c47a Mon Sep 17 00:00:00 2001
From: Changpeng Fang 
Date: Fri, 8 Mar 2024 10:30:01 -0800
Subject: [PATCH 0064/1674] AMDGPU: Copy SubtargetPredicate from pseudo to real
 for dpp16 and dpp8 (#84517)

We usually expect to copy SubtargetPredicate (and OtherPredicates) from
pseudo to real. However, in dpp16 and dpp8, there are assignments like
SubtargetPredicate = HasDPP/HasDPP16/HasDpp8. These assignments override
predicates copied from pseudo, and thus the predicates used to define
pseudo get lost.

Losing predicates is a subtle issue usually not easy to be found. It may
result in instructions being generated on GPUs that do not support the
features to generate them.
https://github.com/llvm/llvm-project/pull/84354 addressed one of such
issues, and inspired this work.

Fortunately, we found that the assignment of SubtargetPredicate usually
comes together with assignment of AssemblerPredicate, and with the same
value. For example:
  let AssemblerPredicate = HasDPP16;
  let SubtargetPredicate = HasDPP16;
One of them is redundant and can be removed.

In this work, we remove the redundant assignment of SubtargetPredicate,
and then copy it from pseudo for VOP*_DPP and VOP*_DPP8. With this
change, we can safely use SubtargetPredicate to define pseudo
instructions.
---
 llvm/lib/Target/AMDGPU/VOP1Instructions.td  | 3 ++-
 llvm/lib/Target/AMDGPU/VOP2Instructions.td  | 3 ++-
 llvm/lib/Target/AMDGPU/VOP3PInstructions.td | 3 ++-
 llvm/lib/Target/AMDGPU/VOPInstructions.td   | 6 ++----
 4 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/VOP1Instructions.td b/llvm/lib/Target/AMDGPU/VOP1Instructions.td
index dbb1977183d1..2341e0d9d32b 100644
--- a/llvm/lib/Target/AMDGPU/VOP1Instructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP1Instructions.td
@@ -730,6 +730,7 @@ class VOP1_DPP op, VOP1_DPP_Pseudo ps, VOPProfile p = ps.Pfl, bit isDPP1
   let SchedRW = ps.SchedRW;
   let Uses = ps.Uses;
   let TRANS = ps.TRANS;
+  let SubtargetPredicate = ps.SubtargetPredicate;
   let OtherPredicates = ps.OtherPredicates;
 
   bits<8> vdst;
@@ -743,7 +744,6 @@ class VOP1_DPP16 op, VOP1_DPP_Pseudo ps, int subtarget, VOPProfile p = p
     VOP1_DPP,
     SIMCInstr  {
   let AssemblerPredicate = HasDPP16;
-  let SubtargetPredicate = HasDPP16;
 }
 
 class VOP1_DPP16_Gen op, VOP1_DPP_Pseudo ps, GFXGen Gen, VOPProfile p = ps.Pfl> :
@@ -758,6 +758,7 @@ class VOP1_DPP8 op, VOP1_Pseudo ps, VOPProfile p = ps.Pfl> :
   let Defs = ps.Defs;
   let SchedRW = ps.SchedRW;
   let Uses = ps.Uses;
+  let SubtargetPredicate = ps.SubtargetPredicate;
   let OtherPredicates = ps.OtherPredicates;
 
   bits<8> vdst;
diff --git a/llvm/lib/Target/AMDGPU/VOP2Instructions.td b/llvm/lib/Target/AMDGPU/VOP2Instructions.td
index 53578682e002..8a92aa8228f1 100644
--- a/llvm/lib/Target/AMDGPU/VOP2Instructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP2Instructions.td
@@ -1259,7 +1259,7 @@ class Base_VOP2_DPP16 op, VOP2_DPP_Pseudo ps,
                  string opName = ps.OpName, VOPProfile p = ps.Pfl> :
     VOP2_DPP {
   let AssemblerPredicate = HasDPP16;
-  let SubtargetPredicate = HasDPP16;
+  let SubtargetPredicate = ps.SubtargetPredicate;
   let OtherPredicates = ps.OtherPredicates;
 }
 
@@ -1294,6 +1294,7 @@ class VOP2_DPP8 op, VOP2_Pseudo ps,
   let Inst{30-25} = op;
   let Inst{31}    = 0x0;
 
+  let SubtargetPredicate = ps.SubtargetPredicate;
   let OtherPredicates = ps.OtherPredicates;
 }
 
diff --git a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
index e1131bbb78d3..4ca2835eea58 100644
--- a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
@@ -1353,7 +1353,7 @@ class VOP3P_DPP16 op, VOP_DPP_Pseudo ps, int subtarget,
   let SchedRW = ps.SchedRW;
   let Uses = ps.Uses;
   let AssemblerPredicate = HasDPP16;
-  let SubtargetPredicate = HasDPP16;
+  let SubtargetPredicate = ps.SubtargetPredicate;
   let OtherPredicates = ps.OtherPredicates;
   let IsPacked = ps.IsPacked;
 }
@@ -1364,6 +1364,7 @@ class VOP3P_DPP8_Base op, VOP_Pseudo ps, string opName = ps.OpName>
   let Defs = ps.Defs;
   let SchedRW = ps.SchedRW;
   let Uses = ps.Uses;
+  let SubtargetPredicate = ps.SubtargetPredicate;
   let OtherPredicates = ps.OtherPredicates;
   let IsPacked = ps.IsPacked;
 }
diff --git a/llvm/lib/Target/AMDGPU/VOPInstructions.td b/llvm/lib/Target/AMDGPU/VOPInstructions.td
index fa8d46608f5d..a6272e946c51 100644
--- a/llvm/lib/Target/AMDGPU/VOPInstructions.td
+++ b/llvm/lib/Target/AMDGPU/VOPInstructions.td
@@ -832,7 +832,6 @@ class VOP_DPP_Pseudo  pattern=[],
   string AsmOperands = asmOps;
 
   let AsmMatchConverter = !if(P.HasModifiers, "cvtDPP", "");
-  let SubtargetPredicate = !if(P.HasExt64BitDPP, HasDPALU_DPP, HasDPP);
   let AssemblerPredicate = !if(P.HasExt64BitDPP, HasDPALU_DPP, HasDPP);
   let AsmVariantName = !if(P.HasExtDPP, AMDGPUAsmVariants.DPP,
                                         AMDGPUAsmVariants.Disable);
@@ -903,7 +902,6 @@ class VOP_DPP_Base  op, VOP_DPP_Pseudo ps, string opName = ps.OpName>
   let SchedRW = ps.SchedRW;
   let Uses = ps.Uses;
   let AssemblerPredicate = HasDPP16;
-  let SubtargetPredicate = HasDPP16;
+  let SubtargetPredicate = ps.SubtargetPredicate;
   let OtherPredicates = ps.OtherPredicates;
 }
 
@@ -1366,6 +1363,7 @@ class Base_VOP3_DPP8 op, VOP_Pseudo ps, string opName = ps.OpName>
   let SchedRW = ps.SchedRW;
   let Uses = ps.Uses;
 
+  let SubtargetPredicate = ps.SubtargetPredicate;
   let OtherPredicates = ps.OtherPredicates;
 }
 
-- 
GitLab


From 99118c809367d518ffe4de60c16da953744b68b9 Mon Sep 17 00:00:00 2001
From: Adrian Prantl 
Date: Fri, 8 Mar 2024 10:39:34 -0800
Subject: [PATCH 0065/1674] Change GetNumChildren()/CalculateNumChildren()
 methods return llvm::Expected (#84219)

Change GetNumChildren()/CalculateNumChildren() methods return
llvm::Expected

This is an NFC change that does not yet add any error handling or change
any code to return any errors.

This is the second big change in the patch series started with
https://github.com/llvm/llvm-project/pull/83501

A follow-up PR will wire up error handling.
---
 lldb/include/lldb/Core/ValueObject.h          | 11 +++-
 lldb/include/lldb/Core/ValueObjectCast.h      |  2 +-
 lldb/include/lldb/Core/ValueObjectChild.h     |  2 +-
 .../lldb/Core/ValueObjectConstResult.h        |  2 +-
 .../lldb/Core/ValueObjectDynamicValue.h       |  2 +-
 lldb/include/lldb/Core/ValueObjectMemory.h    |  2 +-
 lldb/include/lldb/Core/ValueObjectRegister.h  |  4 +-
 .../lldb/Core/ValueObjectSyntheticFilter.h    |  2 +-
 lldb/include/lldb/Core/ValueObjectVTable.h    |  2 +-
 lldb/include/lldb/Core/ValueObjectVariable.h  |  2 +-
 .../lldb/DataFormatters/TypeSynthetic.h       | 20 ++++--
 .../lldb/DataFormatters/VectorIterator.h      |  2 +-
 lldb/include/lldb/Symbol/CompilerType.h       |  5 +-
 lldb/include/lldb/Symbol/Type.h               |  2 +-
 lldb/include/lldb/Symbol/TypeSystem.h         |  7 +-
 .../lldb/Target/StackFrameRecognizer.h        |  3 +-
 lldb/include/lldb/Utility/Log.h               | 14 ++++
 lldb/source/API/SBValue.cpp                   |  2 +-
 lldb/source/Core/FormatEntity.cpp             |  2 +-
 lldb/source/Core/IOHandlerCursesGUI.cpp       |  2 +-
 lldb/source/Core/ValueObject.cpp              | 31 ++++++---
 lldb/source/Core/ValueObjectCast.cpp          |  6 +-
 lldb/source/Core/ValueObjectChild.cpp         |  6 +-
 lldb/source/Core/ValueObjectConstResult.cpp   |  7 +-
 lldb/source/Core/ValueObjectDynamicValue.cpp  |  7 +-
 lldb/source/Core/ValueObjectMemory.cpp        | 10 ++-
 lldb/source/Core/ValueObjectRegister.cpp      | 12 ++--
 .../Core/ValueObjectSyntheticFilter.cpp       | 24 ++++---
 lldb/source/Core/ValueObjectVTable.cpp        |  6 +-
 lldb/source/Core/ValueObjectVariable.cpp      |  7 +-
 lldb/source/DataFormatters/FormatManager.cpp  | 11 +++-
 lldb/source/DataFormatters/TypeSynthetic.cpp  | 17 ++++-
 .../DataFormatters/ValueObjectPrinter.cpp     |  2 +-
 lldb/source/DataFormatters/VectorType.cpp     | 16 +++--
 .../Clang/ClangExpressionSourceCode.cpp       |  2 +-
 .../TSan/InstrumentationRuntimeTSan.cpp       |  4 +-
 .../Language/CPlusPlus/BlockPointer.cpp       |  4 +-
 .../Plugins/Language/CPlusPlus/Coroutines.cpp |  8 ++-
 .../Plugins/Language/CPlusPlus/Coroutines.h   |  2 +-
 .../Language/CPlusPlus/GenericBitset.cpp      |  4 +-
 .../Language/CPlusPlus/GenericOptional.cpp    |  6 +-
 .../Plugins/Language/CPlusPlus/LibCxx.cpp     | 18 +++---
 .../Plugins/Language/CPlusPlus/LibCxx.h       |  8 +--
 .../Language/CPlusPlus/LibCxxAtomic.cpp       |  6 +-
 .../CPlusPlus/LibCxxInitializerList.cpp       |  6 +-
 .../Plugins/Language/CPlusPlus/LibCxxList.cpp | 12 ++--
 .../Plugins/Language/CPlusPlus/LibCxxMap.cpp  | 14 ++--
 .../Language/CPlusPlus/LibCxxQueue.cpp        |  2 +-
 .../CPlusPlus/LibCxxRangesRefView.cpp         |  2 +-
 .../Plugins/Language/CPlusPlus/LibCxxSpan.cpp |  6 +-
 .../Language/CPlusPlus/LibCxxTuple.cpp        |  4 +-
 .../Language/CPlusPlus/LibCxxUnorderedMap.cpp |  8 +--
 .../Language/CPlusPlus/LibCxxValarray.cpp     |  6 +-
 .../Language/CPlusPlus/LibCxxVariant.cpp      |  2 +-
 .../Language/CPlusPlus/LibCxxVector.cpp       | 14 ++--
 .../Plugins/Language/CPlusPlus/LibStdcpp.cpp  | 17 +++--
 .../Language/CPlusPlus/LibStdcppTuple.cpp     |  7 +-
 .../CPlusPlus/LibStdcppUniquePointer.cpp      |  5 +-
 lldb/source/Plugins/Language/ObjC/Cocoa.cpp   |  2 +-
 lldb/source/Plugins/Language/ObjC/NSArray.cpp | 30 ++++-----
 .../Plugins/Language/ObjC/NSDictionary.cpp    | 64 +++++++++----------
 lldb/source/Plugins/Language/ObjC/NSError.cpp |  2 +-
 .../Plugins/Language/ObjC/NSException.cpp     |  4 +-
 .../Plugins/Language/ObjC/NSIndexPath.cpp     |  6 +-
 lldb/source/Plugins/Language/ObjC/NSSet.cpp   | 32 +++++-----
 .../AppleObjCRuntime/AppleObjCRuntime.cpp     |  3 +-
 .../TypeSystem/Clang/TypeSystemClang.cpp      | 40 ++++++++----
 .../TypeSystem/Clang/TypeSystemClang.h        |  7 +-
 lldb/source/Symbol/CompilerType.cpp           |  5 +-
 lldb/source/Symbol/Type.cpp                   |  2 +-
 lldb/source/Symbol/Variable.cpp               |  6 +-
 lldb/source/Target/StackFrame.cpp             | 19 +++---
 72 files changed, 394 insertions(+), 247 deletions(-)

diff --git a/lldb/include/lldb/Core/ValueObject.h b/lldb/include/lldb/Core/ValueObject.h
index b4d2c8098edc..e7e35e2b2bff 100644
--- a/lldb/include/lldb/Core/ValueObject.h
+++ b/lldb/include/lldb/Core/ValueObject.h
@@ -476,7 +476,13 @@ public:
 
   virtual size_t GetIndexOfChildWithName(llvm::StringRef name);
 
-  uint32_t GetNumChildren(uint32_t max = UINT32_MAX);
+  llvm::Expected GetNumChildren(uint32_t max = UINT32_MAX);
+  /// Like \c GetNumChildren but returns 0 on error.  You probably
+  /// shouldn't be using this function. It exists primarily to ease the
+  /// transition to more pervasive error handling while not all APIs
+  /// have been updated.
+  uint32_t GetNumChildrenIgnoringErrors(uint32_t max = UINT32_MAX);
+  bool HasChildren() { return GetNumChildrenIgnoringErrors() > 0; }
 
   const Value &GetValue() const { return m_value; }
 
@@ -958,7 +964,8 @@ protected:
                                           int32_t synthetic_index);
 
   /// Should only be called by ValueObject::GetNumChildren().
-  virtual uint32_t CalculateNumChildren(uint32_t max = UINT32_MAX) = 0;
+  virtual llvm::Expected
+  CalculateNumChildren(uint32_t max = UINT32_MAX) = 0;
 
   void SetNumChildren(uint32_t num_children);
 
diff --git a/lldb/include/lldb/Core/ValueObjectCast.h b/lldb/include/lldb/Core/ValueObjectCast.h
index 51c647680d52..ba25e166f326 100644
--- a/lldb/include/lldb/Core/ValueObjectCast.h
+++ b/lldb/include/lldb/Core/ValueObjectCast.h
@@ -33,7 +33,7 @@ public:
 
   std::optional GetByteSize() override;
 
-  uint32_t CalculateNumChildren(uint32_t max) override;
+  llvm::Expected CalculateNumChildren(uint32_t max) override;
 
   lldb::ValueType GetValueType() const override;
 
diff --git a/lldb/include/lldb/Core/ValueObjectChild.h b/lldb/include/lldb/Core/ValueObjectChild.h
index 47a13be08bb8..1f88e607cb57 100644
--- a/lldb/include/lldb/Core/ValueObjectChild.h
+++ b/lldb/include/lldb/Core/ValueObjectChild.h
@@ -39,7 +39,7 @@ public:
 
   lldb::ValueType GetValueType() const override;
 
-  uint32_t CalculateNumChildren(uint32_t max) override;
+  llvm::Expected CalculateNumChildren(uint32_t max) override;
 
   ConstString GetTypeName() override;
 
diff --git a/lldb/include/lldb/Core/ValueObjectConstResult.h b/lldb/include/lldb/Core/ValueObjectConstResult.h
index 9f1246cf2a78..37dc0867f26c 100644
--- a/lldb/include/lldb/Core/ValueObjectConstResult.h
+++ b/lldb/include/lldb/Core/ValueObjectConstResult.h
@@ -67,7 +67,7 @@ public:
 
   lldb::ValueType GetValueType() const override;
 
-  uint32_t CalculateNumChildren(uint32_t max) override;
+  llvm::Expected CalculateNumChildren(uint32_t max) override;
 
   ConstString GetTypeName() override;
 
diff --git a/lldb/include/lldb/Core/ValueObjectDynamicValue.h b/lldb/include/lldb/Core/ValueObjectDynamicValue.h
index 21a9b409fd5b..82c20eee0cd4 100644
--- a/lldb/include/lldb/Core/ValueObjectDynamicValue.h
+++ b/lldb/include/lldb/Core/ValueObjectDynamicValue.h
@@ -43,7 +43,7 @@ public:
 
   ConstString GetDisplayTypeName() override;
 
-  uint32_t CalculateNumChildren(uint32_t max) override;
+  llvm::Expected CalculateNumChildren(uint32_t max) override;
 
   lldb::ValueType GetValueType() const override;
 
diff --git a/lldb/include/lldb/Core/ValueObjectMemory.h b/lldb/include/lldb/Core/ValueObjectMemory.h
index a74b325546b0..a8fb0353d601 100644
--- a/lldb/include/lldb/Core/ValueObjectMemory.h
+++ b/lldb/include/lldb/Core/ValueObjectMemory.h
@@ -47,7 +47,7 @@ public:
 
   ConstString GetDisplayTypeName() override;
 
-  uint32_t CalculateNumChildren(uint32_t max) override;
+  llvm::Expected CalculateNumChildren(uint32_t max) override;
 
   lldb::ValueType GetValueType() const override;
 
diff --git a/lldb/include/lldb/Core/ValueObjectRegister.h b/lldb/include/lldb/Core/ValueObjectRegister.h
index 6c470c1a6865..fec8566ba33d 100644
--- a/lldb/include/lldb/Core/ValueObjectRegister.h
+++ b/lldb/include/lldb/Core/ValueObjectRegister.h
@@ -47,7 +47,7 @@ public:
 
   ConstString GetQualifiedTypeName() override;
 
-  uint32_t CalculateNumChildren(uint32_t max) override;
+  llvm::Expected CalculateNumChildren(uint32_t max) override;
 
   ValueObject *CreateChildAtIndex(size_t idx, bool synthetic_array_member,
                                   int32_t synthetic_index) override;
@@ -95,7 +95,7 @@ public:
 
   ConstString GetTypeName() override;
 
-  uint32_t CalculateNumChildren(uint32_t max) override;
+  llvm::Expected CalculateNumChildren(uint32_t max) override;
 
   bool SetValueFromCString(const char *value_str, Status &error) override;
 
diff --git a/lldb/include/lldb/Core/ValueObjectSyntheticFilter.h b/lldb/include/lldb/Core/ValueObjectSyntheticFilter.h
index 1e54babc94f3..ca6d6c728005 100644
--- a/lldb/include/lldb/Core/ValueObjectSyntheticFilter.h
+++ b/lldb/include/lldb/Core/ValueObjectSyntheticFilter.h
@@ -47,7 +47,7 @@ public:
 
   bool MightHaveChildren() override;
 
-  uint32_t CalculateNumChildren(uint32_t max) override;
+  llvm::Expected CalculateNumChildren(uint32_t max) override;
 
   lldb::ValueType GetValueType() const override;
 
diff --git a/lldb/include/lldb/Core/ValueObjectVTable.h b/lldb/include/lldb/Core/ValueObjectVTable.h
index e7e14fc83d78..4662f395a4dd 100644
--- a/lldb/include/lldb/Core/ValueObjectVTable.h
+++ b/lldb/include/lldb/Core/ValueObjectVTable.h
@@ -64,7 +64,7 @@ public:
 
   std::optional GetByteSize() override;
 
-  uint32_t CalculateNumChildren(uint32_t max) override;
+  llvm::Expected CalculateNumChildren(uint32_t max) override;
 
   ValueObject *CreateChildAtIndex(size_t idx, bool synthetic_array_member,
                                   int32_t synthetic_index) override;
diff --git a/lldb/include/lldb/Core/ValueObjectVariable.h b/lldb/include/lldb/Core/ValueObjectVariable.h
index da270300df0b..db3847f14a0b 100644
--- a/lldb/include/lldb/Core/ValueObjectVariable.h
+++ b/lldb/include/lldb/Core/ValueObjectVariable.h
@@ -46,7 +46,7 @@ public:
 
   ConstString GetDisplayTypeName() override;
 
-  uint32_t CalculateNumChildren(uint32_t max) override;
+  llvm::Expected CalculateNumChildren(uint32_t max) override;
 
   lldb::ValueType GetValueType() const override;
 
diff --git a/lldb/include/lldb/DataFormatters/TypeSynthetic.h b/lldb/include/lldb/DataFormatters/TypeSynthetic.h
index 38f3ce0fa5f0..ede7442a02bf 100644
--- a/lldb/include/lldb/DataFormatters/TypeSynthetic.h
+++ b/lldb/include/lldb/DataFormatters/TypeSynthetic.h
@@ -38,13 +38,17 @@ public:
 
   virtual ~SyntheticChildrenFrontEnd() = default;
 
-  virtual uint32_t CalculateNumChildren() = 0;
+  virtual llvm::Expected CalculateNumChildren() = 0;
 
-  virtual uint32_t CalculateNumChildren(uint32_t max) {
+  virtual llvm::Expected CalculateNumChildren(uint32_t max) {
     auto count = CalculateNumChildren();
-    return count <= max ? count : max;
+    if (!count)
+      return count;
+    return *count <= max ? *count : max;
   }
 
+  uint32_t CalculateNumChildrenIgnoringErrors(uint32_t max = UINT32_MAX);
+
   virtual lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) = 0;
 
   virtual size_t GetIndexOfChildWithName(ConstString name) = 0;
@@ -109,7 +113,7 @@ public:
 
   ~SyntheticValueProviderFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override { return 0; }
+  llvm::Expected CalculateNumChildren() override { return 0; }
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override { return nullptr; }
 
@@ -322,7 +326,9 @@ public:
 
     ~FrontEnd() override = default;
 
-    uint32_t CalculateNumChildren() override { return filter->GetCount(); }
+    llvm::Expected CalculateNumChildren() override {
+      return filter->GetCount();
+    }
 
     lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override {
       if (idx >= filter->GetCount())
@@ -426,9 +432,9 @@ public:
 
     bool IsValid();
 
-    uint32_t CalculateNumChildren() override;
+    llvm::Expected CalculateNumChildren() override;
 
-    uint32_t CalculateNumChildren(uint32_t max) override;
+    llvm::Expected CalculateNumChildren(uint32_t max) override;
 
     lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
diff --git a/lldb/include/lldb/DataFormatters/VectorIterator.h b/lldb/include/lldb/DataFormatters/VectorIterator.h
index 7711b9de95db..70bcf50ca1b1 100644
--- a/lldb/include/lldb/DataFormatters/VectorIterator.h
+++ b/lldb/include/lldb/DataFormatters/VectorIterator.h
@@ -24,7 +24,7 @@ public:
   VectorIteratorSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp,
                                   llvm::ArrayRef item_names);
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
diff --git a/lldb/include/lldb/Symbol/CompilerType.h b/lldb/include/lldb/Symbol/CompilerType.h
index 414c44275aaa..c1dce4ccbf79 100644
--- a/lldb/include/lldb/Symbol/CompilerType.h
+++ b/lldb/include/lldb/Symbol/CompilerType.h
@@ -386,8 +386,9 @@ public:
 
   std::optional GetTypeBitAlign(ExecutionContextScope *exe_scope) const;
 
-  uint32_t GetNumChildren(bool omit_empty_base_classes,
-                          const ExecutionContext *exe_ctx) const;
+  llvm::Expected
+  GetNumChildren(bool omit_empty_base_classes,
+                 const ExecutionContext *exe_ctx) const;
 
   lldb::BasicType GetBasicTypeEnumeration() const;
 
diff --git a/lldb/include/lldb/Symbol/Type.h b/lldb/include/lldb/Symbol/Type.h
index acd1a769f13c..b5eac5fa732d 100644
--- a/lldb/include/lldb/Symbol/Type.h
+++ b/lldb/include/lldb/Symbol/Type.h
@@ -440,7 +440,7 @@ public:
 
   std::optional GetByteSize(ExecutionContextScope *exe_scope);
 
-  uint32_t GetNumChildren(bool omit_empty_base_classes);
+  llvm::Expected GetNumChildren(bool omit_empty_base_classes);
 
   bool IsAggregateType();
 
diff --git a/lldb/include/lldb/Symbol/TypeSystem.h b/lldb/include/lldb/Symbol/TypeSystem.h
index 63829131556e..f647fcbf1636 100644
--- a/lldb/include/lldb/Symbol/TypeSystem.h
+++ b/lldb/include/lldb/Symbol/TypeSystem.h
@@ -300,9 +300,10 @@ public:
 
   virtual lldb::Format GetFormat(lldb::opaque_compiler_type_t type) = 0;
 
-  virtual uint32_t GetNumChildren(lldb::opaque_compiler_type_t type,
-                                  bool omit_empty_base_classes,
-                                  const ExecutionContext *exe_ctx) = 0;
+  virtual llvm::Expected
+  GetNumChildren(lldb::opaque_compiler_type_t type,
+                 bool omit_empty_base_classes,
+                 const ExecutionContext *exe_ctx) = 0;
 
   virtual CompilerType GetBuiltinTypeByName(ConstString name);
 
diff --git a/lldb/include/lldb/Target/StackFrameRecognizer.h b/lldb/include/lldb/Target/StackFrameRecognizer.h
index e111f4a4dc70..5e8e12b2a4e9 100644
--- a/lldb/include/lldb/Target/StackFrameRecognizer.h
+++ b/lldb/include/lldb/Target/StackFrameRecognizer.h
@@ -164,7 +164,8 @@ class ValueObjectRecognizerSynthesizedValue : public ValueObject {
     m_value = m_parent->GetValue();
     return true;
   }
-  uint32_t CalculateNumChildren(uint32_t max = UINT32_MAX) override {
+  llvm::Expected
+  CalculateNumChildren(uint32_t max = UINT32_MAX) override {
     return m_parent->GetNumChildren(max);
   }
   CompilerType GetCompilerTypeImpl() override {
diff --git a/lldb/include/lldb/Utility/Log.h b/lldb/include/lldb/Utility/Log.h
index 1fe28d61b9da..01876ad732d4 100644
--- a/lldb/include/lldb/Utility/Log.h
+++ b/lldb/include/lldb/Utility/Log.h
@@ -373,4 +373,18 @@ template  Log *GetLog(Cat mask) {
       ::llvm::consumeError(::std::move(error_private));                        \
   } while (0)
 
+// Write message to the verbose log, if error is set. In the log
+// message refer to the error with {0}. Error is cleared regardless of
+// whether logging is enabled.
+#define LLDB_LOG_ERRORV(log, error, ...)                                       \
+  do {                                                                         \
+    ::lldb_private::Log *log_private = (log);                                  \
+    ::llvm::Error error_private = (error);                                     \
+    if (log_private && log_private->GetVerbose() && error_private) {           \
+      log_private->FormatError(::std::move(error_private), __FILE__, __func__, \
+                               __VA_ARGS__);                                   \
+    } else                                                                     \
+      ::llvm::consumeError(::std::move(error_private));                        \
+  } while (0)
+
 #endif // LLDB_UTILITY_LOG_H
diff --git a/lldb/source/API/SBValue.cpp b/lldb/source/API/SBValue.cpp
index 89d26a1fbe28..94a8f3ea319e 100644
--- a/lldb/source/API/SBValue.cpp
+++ b/lldb/source/API/SBValue.cpp
@@ -947,7 +947,7 @@ uint32_t SBValue::GetNumChildren(uint32_t max) {
   ValueLocker locker;
   lldb::ValueObjectSP value_sp(GetSP(locker));
   if (value_sp)
-    num_children = value_sp->GetNumChildren(max);
+    num_children = value_sp->GetNumChildrenIgnoringErrors(max);
 
   return num_children;
 }
diff --git a/lldb/source/Core/FormatEntity.cpp b/lldb/source/Core/FormatEntity.cpp
index fa5eadc6ff4e..cf82676bedda 100644
--- a/lldb/source/Core/FormatEntity.cpp
+++ b/lldb/source/Core/FormatEntity.cpp
@@ -926,7 +926,7 @@ static bool DumpValue(Stream &s, const SymbolContext *sc,
     s.PutChar('[');
 
     if (index_higher < 0)
-      index_higher = valobj->GetNumChildren() - 1;
+      index_higher = valobj->GetNumChildrenIgnoringErrors() - 1;
 
     uint32_t max_num_children =
         target->GetTargetSP()->GetMaximumNumberOfChildrenToDisplay();
diff --git a/lldb/source/Core/IOHandlerCursesGUI.cpp b/lldb/source/Core/IOHandlerCursesGUI.cpp
index 620e68a28510..f86dce247135 100644
--- a/lldb/source/Core/IOHandlerCursesGUI.cpp
+++ b/lldb/source/Core/IOHandlerCursesGUI.cpp
@@ -4519,7 +4519,7 @@ struct Row {
       calculated_children = true;
       ValueObjectSP valobj = value.GetSP();
       if (valobj) {
-        const size_t num_children = valobj->GetNumChildren();
+        const uint32_t num_children = valobj->GetNumChildrenIgnoringErrors();
         for (size_t i = 0; i < num_children; ++i) {
           children.push_back(Row(valobj->GetChildAtIndex(i), this));
         }
diff --git a/lldb/source/Core/ValueObject.cpp b/lldb/source/Core/ValueObject.cpp
index 0ed7f03be25c..d813044d02ff 100644
--- a/lldb/source/Core/ValueObject.cpp
+++ b/lldb/source/Core/ValueObject.cpp
@@ -377,7 +377,7 @@ ValueObjectSP ValueObject::GetChildAtIndex(uint32_t idx, bool can_create) {
   // We may need to update our value if we are dynamic
   if (IsPossibleDynamicType())
     UpdateValueIfNeeded(false);
-  if (idx < GetNumChildren()) {
+  if (idx < GetNumChildrenIgnoringErrors()) {
     // Check if we have already made the child value object?
     if (can_create && !m_children.HasChildAtIndex(idx)) {
       // No we haven't created the child at this index, so lets have our
@@ -440,7 +440,7 @@ ValueObjectSP ValueObject::GetChildMemberWithName(llvm::StringRef name,
   return child_sp;
 }
 
-uint32_t ValueObject::GetNumChildren(uint32_t max) {
+llvm::Expected ValueObject::GetNumChildren(uint32_t max) {
   UpdateValueIfNeeded();
 
   if (max < UINT32_MAX) {
@@ -452,11 +452,24 @@ uint32_t ValueObject::GetNumChildren(uint32_t max) {
   }
 
   if (!m_flags.m_children_count_valid) {
-    SetNumChildren(CalculateNumChildren());
+    auto num_children_or_err = CalculateNumChildren();
+    if (num_children_or_err)
+      SetNumChildren(*num_children_or_err);
+    else
+      return num_children_or_err;
   }
   return m_children.GetChildrenCount();
 }
 
+uint32_t ValueObject::GetNumChildrenIgnoringErrors(uint32_t max) {
+  auto value_or_err = GetNumChildren(max);
+  if (value_or_err)
+    return *value_or_err;
+  LLDB_LOG_ERRORV(GetLog(LLDBLog::DataFormatters), value_or_err.takeError(),
+                  "{0}");
+  return 0;
+}
+
 bool ValueObject::MightHaveChildren() {
   bool has_children = false;
   const uint32_t type_info = GetTypeInfo();
@@ -464,7 +477,7 @@ bool ValueObject::MightHaveChildren() {
     if (type_info & (eTypeHasChildren | eTypeIsPointer | eTypeIsReference))
       has_children = true;
   } else {
-    has_children = GetNumChildren() > 0;
+    has_children = GetNumChildrenIgnoringErrors() > 0;
   }
   return has_children;
 }
@@ -1176,7 +1189,7 @@ bool ValueObject::DumpPrintableRepresentation(
       if (flags.Test(eTypeIsArray)) {
         if ((custom_format == eFormatBytes) ||
             (custom_format == eFormatBytesWithASCII)) {
-          const size_t count = GetNumChildren();
+          const size_t count = GetNumChildrenIgnoringErrors();
 
           s << '[';
           for (size_t low = 0; low < count; low++) {
@@ -1215,7 +1228,7 @@ bool ValueObject::DumpPrintableRepresentation(
                                                      // format should be printed
                                                      // directly
         {
-          const size_t count = GetNumChildren();
+          const size_t count = GetNumChildrenIgnoringErrors();
 
           Format format = FormatManager::GetSingleItemFormat(custom_format);
 
@@ -1294,7 +1307,7 @@ bool ValueObject::DumpPrintableRepresentation(
       break;
 
     case eValueObjectRepresentationStyleChildrenCount:
-      strm.Printf("%" PRIu64 "", (uint64_t)GetNumChildren());
+      strm.Printf("%" PRIu64 "", (uint64_t)GetNumChildrenIgnoringErrors());
       str = strm.GetString();
       break;
 
@@ -2320,7 +2333,9 @@ ValueObjectSP ValueObject::GetValueForExpressionPath_Impl(
             child_valobj_sp = root->GetSyntheticArrayMember(index, true);
           if (!child_valobj_sp)
             if (root->HasSyntheticValue() &&
-                root->GetSyntheticValue()->GetNumChildren() > index)
+                llvm::expectedToStdOptional(
+                    root->GetSyntheticValue()->GetNumChildren())
+                        .value_or(0) > index)
               child_valobj_sp =
                   root->GetSyntheticValue()->GetChildAtIndex(index);
           if (child_valobj_sp) {
diff --git a/lldb/source/Core/ValueObjectCast.cpp b/lldb/source/Core/ValueObjectCast.cpp
index a5c555f86b13..c8e316415141 100644
--- a/lldb/source/Core/ValueObjectCast.cpp
+++ b/lldb/source/Core/ValueObjectCast.cpp
@@ -41,11 +41,13 @@ ValueObjectCast::~ValueObjectCast() = default;
 
 CompilerType ValueObjectCast::GetCompilerTypeImpl() { return m_cast_type; }
 
-uint32_t ValueObjectCast::CalculateNumChildren(uint32_t max) {
+llvm::Expected ValueObjectCast::CalculateNumChildren(uint32_t max) {
   ExecutionContext exe_ctx(GetExecutionContextRef());
   auto children_count = GetCompilerType().GetNumChildren(
       true, &exe_ctx);
-  return children_count <= max ? children_count : max;
+  if (!children_count)
+    return children_count;
+  return *children_count <= max ? *children_count : max;
 }
 
 std::optional ValueObjectCast::GetByteSize() {
diff --git a/lldb/source/Core/ValueObjectChild.cpp b/lldb/source/Core/ValueObjectChild.cpp
index 2e55dd7726bd..c6a97dd1a5cd 100644
--- a/lldb/source/Core/ValueObjectChild.cpp
+++ b/lldb/source/Core/ValueObjectChild.cpp
@@ -49,10 +49,12 @@ lldb::ValueType ValueObjectChild::GetValueType() const {
   return m_parent->GetValueType();
 }
 
-uint32_t ValueObjectChild::CalculateNumChildren(uint32_t max) {
+llvm::Expected ValueObjectChild::CalculateNumChildren(uint32_t max) {
   ExecutionContext exe_ctx(GetExecutionContextRef());
   auto children_count = GetCompilerType().GetNumChildren(true, &exe_ctx);
-  return children_count <= max ? children_count : max;
+  if (!children_count)
+    return children_count;
+  return *children_count <= max ? *children_count : max;
 }
 
 static void AdjustForBitfieldness(ConstString &name,
diff --git a/lldb/source/Core/ValueObjectConstResult.cpp b/lldb/source/Core/ValueObjectConstResult.cpp
index 5c7aa4452b70..8ac2c1cac2f6 100644
--- a/lldb/source/Core/ValueObjectConstResult.cpp
+++ b/lldb/source/Core/ValueObjectConstResult.cpp
@@ -216,10 +216,13 @@ std::optional ValueObjectConstResult::GetByteSize() {
 
 void ValueObjectConstResult::SetByteSize(size_t size) { m_byte_size = size; }
 
-uint32_t ValueObjectConstResult::CalculateNumChildren(uint32_t max) {
+llvm::Expected
+ValueObjectConstResult::CalculateNumChildren(uint32_t max) {
   ExecutionContext exe_ctx(GetExecutionContextRef());
   auto children_count = GetCompilerType().GetNumChildren(true, &exe_ctx);
-  return children_count <= max ? children_count : max;
+  if (!children_count)
+    return children_count;
+  return *children_count <= max ? *children_count : max;
 }
 
 ConstString ValueObjectConstResult::GetTypeName() {
diff --git a/lldb/source/Core/ValueObjectDynamicValue.cpp b/lldb/source/Core/ValueObjectDynamicValue.cpp
index 4e64760371ae..4695febdf8ca 100644
--- a/lldb/source/Core/ValueObjectDynamicValue.cpp
+++ b/lldb/source/Core/ValueObjectDynamicValue.cpp
@@ -85,12 +85,15 @@ ConstString ValueObjectDynamicValue::GetDisplayTypeName() {
   return m_parent->GetDisplayTypeName();
 }
 
-uint32_t ValueObjectDynamicValue::CalculateNumChildren(uint32_t max) {
+llvm::Expected
+ValueObjectDynamicValue::CalculateNumChildren(uint32_t max) {
   const bool success = UpdateValueIfNeeded(false);
   if (success && m_dynamic_type_info.HasType()) {
     ExecutionContext exe_ctx(GetExecutionContextRef());
     auto children_count = GetCompilerType().GetNumChildren(true, &exe_ctx);
-    return children_count <= max ? children_count : max;
+    if (!children_count)
+      return children_count;
+    return *children_count <= max ? *children_count : max;
   } else
     return m_parent->GetNumChildren(max);
 }
diff --git a/lldb/source/Core/ValueObjectMemory.cpp b/lldb/source/Core/ValueObjectMemory.cpp
index 7f68236c7884..f555ab82f441 100644
--- a/lldb/source/Core/ValueObjectMemory.cpp
+++ b/lldb/source/Core/ValueObjectMemory.cpp
@@ -126,17 +126,21 @@ ConstString ValueObjectMemory::GetDisplayTypeName() {
   return m_compiler_type.GetDisplayTypeName();
 }
 
-uint32_t ValueObjectMemory::CalculateNumChildren(uint32_t max) {
+llvm::Expected ValueObjectMemory::CalculateNumChildren(uint32_t max) {
   if (m_type_sp) {
     auto child_count = m_type_sp->GetNumChildren(true);
-    return child_count <= max ? child_count : max;
+    if (!child_count)
+      return child_count;
+    return *child_count <= max ? *child_count : max;
   }
 
   ExecutionContext exe_ctx(GetExecutionContextRef());
   const bool omit_empty_base_classes = true;
   auto child_count =
       m_compiler_type.GetNumChildren(omit_empty_base_classes, &exe_ctx);
-  return child_count <= max ? child_count : max;
+  if (!child_count)
+    return child_count;
+  return *child_count <= max ? *child_count : max;
 }
 
 std::optional ValueObjectMemory::GetByteSize() {
diff --git a/lldb/source/Core/ValueObjectRegister.cpp b/lldb/source/Core/ValueObjectRegister.cpp
index d4c144cc7edb..9d9da8bb119d 100644
--- a/lldb/source/Core/ValueObjectRegister.cpp
+++ b/lldb/source/Core/ValueObjectRegister.cpp
@@ -74,7 +74,8 @@ ConstString ValueObjectRegisterSet::GetQualifiedTypeName() {
   return ConstString();
 }
 
-uint32_t ValueObjectRegisterSet::CalculateNumChildren(uint32_t max) {
+llvm::Expected
+ValueObjectRegisterSet::CalculateNumChildren(uint32_t max) {
   const RegisterSet *reg_set = m_reg_ctx_sp->GetRegisterSet(m_reg_set_idx);
   if (reg_set) {
     auto reg_count = reg_set->num_registers;
@@ -118,7 +119,7 @@ ValueObject *ValueObjectRegisterSet::CreateChildAtIndex(
     size_t idx, bool synthetic_array_member, int32_t synthetic_index) {
   ValueObject *valobj = nullptr;
   if (m_reg_ctx_sp && m_reg_set) {
-    const size_t num_children = GetNumChildren();
+    uint32_t num_children = GetNumChildrenIgnoringErrors();
     if (idx < num_children)
       valobj = new ValueObjectRegister(
           *this, m_reg_ctx_sp,
@@ -220,10 +221,13 @@ ConstString ValueObjectRegister::GetTypeName() {
   return m_type_name;
 }
 
-uint32_t ValueObjectRegister::CalculateNumChildren(uint32_t max) {
+llvm::Expected
+ValueObjectRegister::CalculateNumChildren(uint32_t max) {
   ExecutionContext exe_ctx(GetExecutionContextRef());
   auto children_count = GetCompilerType().GetNumChildren(true, &exe_ctx);
-  return children_count <= max ? children_count : max;
+  if (!children_count)
+    return children_count;
+  return *children_count <= max ? *children_count : max;
 }
 
 std::optional ValueObjectRegister::GetByteSize() {
diff --git a/lldb/source/Core/ValueObjectSyntheticFilter.cpp b/lldb/source/Core/ValueObjectSyntheticFilter.cpp
index 7f8a9a34cb35..adac1b400705 100644
--- a/lldb/source/Core/ValueObjectSyntheticFilter.cpp
+++ b/lldb/source/Core/ValueObjectSyntheticFilter.cpp
@@ -31,7 +31,9 @@ public:
   DummySyntheticFrontEnd(ValueObject &backend)
       : SyntheticChildrenFrontEnd(backend) {}
 
-  uint32_t CalculateNumChildren() override { return m_backend.GetNumChildren(); }
+  llvm::Expected CalculateNumChildren() override {
+    return m_backend.GetNumChildren();
+  }
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override {
     return m_backend.GetChildAtIndex(idx);
@@ -84,7 +86,8 @@ ConstString ValueObjectSynthetic::GetDisplayTypeName() {
   return m_parent->GetDisplayTypeName();
 }
 
-uint32_t ValueObjectSynthetic::CalculateNumChildren(uint32_t max) {
+llvm::Expected
+ValueObjectSynthetic::CalculateNumChildren(uint32_t max) {
   Log *log = GetLog(LLDBLog::DataFormatters);
 
   UpdateValueIfNeeded();
@@ -92,18 +95,23 @@ uint32_t ValueObjectSynthetic::CalculateNumChildren(uint32_t max) {
     return m_synthetic_children_count <= max ? m_synthetic_children_count : max;
 
   if (max < UINT32_MAX) {
-    size_t num_children = m_synth_filter_up->CalculateNumChildren(max);
+    auto num_children = m_synth_filter_up->CalculateNumChildren(max);
     LLDB_LOGF(log,
               "[ValueObjectSynthetic::CalculateNumChildren] for VO of name "
-              "%s and type %s, the filter returned %zu child values",
-              GetName().AsCString(), GetTypeName().AsCString(), num_children);
+              "%s and type %s, the filter returned %u child values",
+              GetName().AsCString(), GetTypeName().AsCString(),
+              num_children ? *num_children : 0);
     return num_children;
   } else {
-    size_t num_children = (m_synthetic_children_count =
-                               m_synth_filter_up->CalculateNumChildren(max));
+    auto num_children_or_err = m_synth_filter_up->CalculateNumChildren(max);
+    if (!num_children_or_err) {
+      m_synthetic_children_count = 0;
+      return num_children_or_err;
+    }
+    auto num_children = (m_synthetic_children_count = *num_children_or_err);
     LLDB_LOGF(log,
               "[ValueObjectSynthetic::CalculateNumChildren] for VO of name "
-              "%s and type %s, the filter returned %zu child values",
+              "%s and type %s, the filter returned %u child values",
               GetName().AsCString(), GetTypeName().AsCString(), num_children);
     return num_children;
   }
diff --git a/lldb/source/Core/ValueObjectVTable.cpp b/lldb/source/Core/ValueObjectVTable.cpp
index 4d1cbb8d2f6f..9bfe8dc9700d 100644
--- a/lldb/source/Core/ValueObjectVTable.cpp
+++ b/lldb/source/Core/ValueObjectVTable.cpp
@@ -33,7 +33,9 @@ public:
 
   std::optional GetByteSize() override { return m_addr_size; };
 
-  uint32_t CalculateNumChildren(uint32_t max) override { return 0; };
+  llvm::Expected CalculateNumChildren(uint32_t max) override {
+    return 0;
+  };
 
   ValueType GetValueType() const override { return eValueTypeVTableEntry; };
 
@@ -159,7 +161,7 @@ std::optional ValueObjectVTable::GetByteSize() {
   return std::nullopt;
 }
 
-uint32_t ValueObjectVTable::CalculateNumChildren(uint32_t max) {
+llvm::Expected ValueObjectVTable::CalculateNumChildren(uint32_t max) {
   if (UpdateValueIfNeeded(false))
     return m_num_vtable_entries <= max ? m_num_vtable_entries : max;
   return 0;
diff --git a/lldb/source/Core/ValueObjectVariable.cpp b/lldb/source/Core/ValueObjectVariable.cpp
index dc62bb6358dc..fb29c22c0ab5 100644
--- a/lldb/source/Core/ValueObjectVariable.cpp
+++ b/lldb/source/Core/ValueObjectVariable.cpp
@@ -94,7 +94,8 @@ ConstString ValueObjectVariable::GetQualifiedTypeName() {
   return ConstString();
 }
 
-uint32_t ValueObjectVariable::CalculateNumChildren(uint32_t max) {
+llvm::Expected
+ValueObjectVariable::CalculateNumChildren(uint32_t max) {
   CompilerType type(GetCompilerType());
 
   if (!type.IsValid())
@@ -103,7 +104,9 @@ uint32_t ValueObjectVariable::CalculateNumChildren(uint32_t max) {
   ExecutionContext exe_ctx(GetExecutionContextRef());
   const bool omit_empty_base_classes = true;
   auto child_count = type.GetNumChildren(omit_empty_base_classes, &exe_ctx);
-  return child_count <= max ? child_count : max;
+  if (!child_count)
+    return child_count;
+  return *child_count <= max ? *child_count : max;
 }
 
 std::optional ValueObjectVariable::GetByteSize() {
diff --git a/lldb/source/DataFormatters/FormatManager.cpp b/lldb/source/DataFormatters/FormatManager.cpp
index 092fa3c8ce49..d7ba5b4b70c9 100644
--- a/lldb/source/DataFormatters/FormatManager.cpp
+++ b/lldb/source/DataFormatters/FormatManager.cpp
@@ -451,8 +451,13 @@ bool FormatManager::ShouldPrintAsOneLiner(ValueObject &valobj) {
   if (valobj.GetSummaryFormat().get() != nullptr)
     return valobj.GetSummaryFormat()->IsOneLiner();
 
+  auto num_children = valobj.GetNumChildren();
+  if (!num_children) {
+    llvm::consumeError(num_children.takeError());
+    return true;
+  }
   // no children, no party
-  if (valobj.GetNumChildren() == 0)
+  if (*num_children == 0)
     return false;
 
   // ask the type if it has any opinion about this eLazyBoolCalculate == no
@@ -471,7 +476,7 @@ bool FormatManager::ShouldPrintAsOneLiner(ValueObject &valobj) {
 
   size_t total_children_name_len = 0;
 
-  for (size_t idx = 0; idx < valobj.GetNumChildren(); idx++) {
+  for (size_t idx = 0; idx < *num_children; idx++) {
     bool is_synth_val = false;
     ValueObjectSP child_sp(valobj.GetChildAtIndex(idx));
     // something is wrong here - bail out
@@ -523,7 +528,7 @@ bool FormatManager::ShouldPrintAsOneLiner(ValueObject &valobj) {
     }
 
     // if this child has children..
-    if (child_sp->GetNumChildren()) {
+    if (child_sp->HasChildren()) {
       // ...and no summary...
       // (if it had a summary and the summary wanted children, we would have
       // bailed out anyway
diff --git a/lldb/source/DataFormatters/TypeSynthetic.cpp b/lldb/source/DataFormatters/TypeSynthetic.cpp
index 0ae38c4d31f2..e3f613606aa8 100644
--- a/lldb/source/DataFormatters/TypeSynthetic.cpp
+++ b/lldb/source/DataFormatters/TypeSynthetic.cpp
@@ -115,6 +115,17 @@ std::string CXXSyntheticChildren::GetDescription() {
   return std::string(sstr.GetString());
 }
 
+uint32_t
+SyntheticChildrenFrontEnd::CalculateNumChildrenIgnoringErrors(uint32_t max) {
+  auto value_or_err = CalculateNumChildren(max);
+  if (value_or_err)
+    return *value_or_err;
+  Log *log = GetLog(LLDBLog::DataFormatters);
+  if (log && log->GetVerbose())
+    LLDB_LOG_ERROR(log, value_or_err.takeError(), "{0}");
+  return 0;
+}
+
 lldb::ValueObjectSP SyntheticChildrenFrontEnd::CreateValueObjectFromExpression(
     llvm::StringRef name, llvm::StringRef expression,
     const ExecutionContext &exe_ctx) {
@@ -178,13 +189,15 @@ bool ScriptedSyntheticChildren::FrontEnd::IsValid() {
   return (m_wrapper_sp && m_wrapper_sp->IsValid() && m_interpreter);
 }
 
-uint32_t ScriptedSyntheticChildren::FrontEnd::CalculateNumChildren() {
+llvm::Expected
+ScriptedSyntheticChildren::FrontEnd::CalculateNumChildren() {
   if (!m_wrapper_sp || m_interpreter == nullptr)
     return 0;
   return m_interpreter->CalculateNumChildren(m_wrapper_sp, UINT32_MAX);
 }
 
-uint32_t ScriptedSyntheticChildren::FrontEnd::CalculateNumChildren(uint32_t max) {
+llvm::Expected
+ScriptedSyntheticChildren::FrontEnd::CalculateNumChildren(uint32_t max) {
   if (!m_wrapper_sp || m_interpreter == nullptr)
     return 0;
   return m_interpreter->CalculateNumChildren(m_wrapper_sp, max);
diff --git a/lldb/source/DataFormatters/ValueObjectPrinter.cpp b/lldb/source/DataFormatters/ValueObjectPrinter.cpp
index 46e50a8d421a..b853199e878c 100644
--- a/lldb/source/DataFormatters/ValueObjectPrinter.cpp
+++ b/lldb/source/DataFormatters/ValueObjectPrinter.cpp
@@ -627,7 +627,7 @@ uint32_t ValueObjectPrinter::GetMaxNumChildrenToPrint(bool &print_dotdotdot) {
   if (m_options.m_pointer_as_array)
     return m_options.m_pointer_as_array.m_element_count;
 
-  size_t num_children = synth_valobj.GetNumChildren();
+  uint32_t num_children = synth_valobj.GetNumChildrenIgnoringErrors();
   print_dotdotdot = false;
   if (num_children) {
     const size_t max_num_children = GetMostSpecializedValue()
diff --git a/lldb/source/DataFormatters/VectorType.cpp b/lldb/source/DataFormatters/VectorType.cpp
index a0626a8cba77..19de204c2435 100644
--- a/lldb/source/DataFormatters/VectorType.cpp
+++ b/lldb/source/DataFormatters/VectorType.cpp
@@ -9,6 +9,7 @@
 #include "lldb/DataFormatters/VectorType.h"
 
 #include "lldb/Core/ValueObject.h"
+#include "lldb/Core/ValueObjectConstResult.h"
 #include "lldb/DataFormatters/FormattersHelpers.h"
 #include "lldb/Symbol/CompilerType.h"
 #include "lldb/Symbol/TypeSystem.h"
@@ -224,10 +225,16 @@ public:
 
   ~VectorTypeSyntheticFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override { return m_num_children; }
+  llvm::Expected CalculateNumChildren() override {
+    return m_num_children;
+  }
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override {
-    if (idx >= CalculateNumChildren())
+    auto num_children_or_err = CalculateNumChildren();
+    if (!num_children_or_err)
+      return ValueObjectConstResult::Create(
+          nullptr, Status(num_children_or_err.takeError()));
+    if (idx >= *num_children_or_err)
       return {};
     std::optional size = m_child_type.GetByteSize(nullptr);
     if (!size)
@@ -266,7 +273,7 @@ public:
   size_t GetIndexOfChildWithName(ConstString name) override {
     const char *item_name = name.GetCString();
     uint32_t idx = ExtractIndexFromString(item_name);
-    if (idx < UINT32_MAX && idx >= CalculateNumChildren())
+    if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
       return UINT32_MAX;
     return idx;
   }
@@ -293,7 +300,8 @@ bool lldb_private::formatters::VectorTypeSummaryProvider(
   s.PutChar('(');
   bool first = true;
 
-  size_t idx = 0, len = synthetic_children->CalculateNumChildren();
+  size_t idx = 0,
+         len = synthetic_children->CalculateNumChildrenIgnoringErrors();
 
   for (; idx < len; idx++) {
     auto child_sp = synthetic_children->GetChildAtIndex(idx);
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionSourceCode.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionSourceCode.cpp
index b48bbbecc0cd..3d43ed3f99ff 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionSourceCode.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionSourceCode.cpp
@@ -229,7 +229,7 @@ void AddLambdaCaptureDecls(StreamString &stream, StackFrame *frame,
   assert(frame);
 
   if (auto thisValSP = ClangExpressionUtil::GetLambdaValueObject(frame)) {
-    uint32_t numChildren = thisValSP->GetNumChildren();
+    uint32_t numChildren = thisValSP->GetNumChildrenIgnoringErrors();
     for (uint32_t i = 0; i < numChildren; ++i) {
       auto childVal = thisValSP->GetChildAtIndex(i);
       ConstString childName(childVal ? childVal->GetName() : ConstString(""));
diff --git a/lldb/source/Plugins/InstrumentationRuntime/TSan/InstrumentationRuntimeTSan.cpp b/lldb/source/Plugins/InstrumentationRuntime/TSan/InstrumentationRuntimeTSan.cpp
index 72293c5331f4..b2781aa5e7db 100644
--- a/lldb/source/Plugins/InstrumentationRuntime/TSan/InstrumentationRuntimeTSan.cpp
+++ b/lldb/source/Plugins/InstrumentationRuntime/TSan/InstrumentationRuntimeTSan.cpp
@@ -27,6 +27,8 @@
 #include "lldb/Target/StopInfo.h"
 #include "lldb/Target/Target.h"
 #include "lldb/Target/Thread.h"
+#include "lldb/Utility/LLDBLog.h"
+#include "lldb/Utility/Log.h"
 #include "lldb/Utility/RegularExpression.h"
 #include "lldb/Utility/Stream.h"
 
@@ -211,7 +213,7 @@ CreateStackTrace(ValueObjectSP o,
   auto trace_sp = std::make_shared();
   ValueObjectSP trace_value_object =
       o->GetValueForExpressionPath(trace_item_name.c_str());
-  size_t count = trace_value_object->GetNumChildren();
+  size_t count = trace_value_object->GetNumChildrenIgnoringErrors();
   for (size_t j = 0; j < count; j++) {
     addr_t trace_addr =
         trace_value_object->GetChildAtIndex(j)->GetValueAsUnsigned(0);
diff --git a/lldb/source/Plugins/Language/CPlusPlus/BlockPointer.cpp b/lldb/source/Plugins/Language/CPlusPlus/BlockPointer.cpp
index ef0f67d1e9f9..9a6e135e0083 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/BlockPointer.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/BlockPointer.cpp
@@ -74,7 +74,7 @@ public:
 
   ~BlockPointerSyntheticFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override {
+  llvm::Expected CalculateNumChildren() override {
     const bool omit_empty_base_classes = false;
     return m_block_struct_type.GetNumChildren(omit_empty_base_classes, nullptr);
   }
@@ -84,7 +84,7 @@ public:
       return lldb::ValueObjectSP();
     }
 
-    if (idx >= CalculateNumChildren()) {
+    if (idx >= CalculateNumChildrenIgnoringErrors()) {
       return lldb::ValueObjectSP();
     }
 
diff --git a/lldb/source/Plugins/Language/CPlusPlus/Coroutines.cpp b/lldb/source/Plugins/Language/CPlusPlus/Coroutines.cpp
index 3827f9c21eff..5e63d1d7b214 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/Coroutines.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/Coroutines.cpp
@@ -11,6 +11,8 @@
 #include "Plugins/TypeSystem/Clang/TypeSystemClang.h"
 #include "lldb/Symbol/Function.h"
 #include "lldb/Symbol/VariableList.h"
+#include "lldb/Utility/LLDBLog.h"
+#include "lldb/Utility/Log.h"
 
 using namespace lldb;
 using namespace lldb_private;
@@ -22,7 +24,7 @@ static lldb::addr_t GetCoroFramePtrFromHandle(ValueObjectSP valobj_sp) {
 
   // We expect a single pointer in the `coroutine_handle` class.
   // We don't care about its name.
-  if (valobj_sp->GetNumChildren() != 1)
+  if (valobj_sp->GetNumChildrenIgnoringErrors() != 1)
     return LLDB_INVALID_ADDRESS;
   ValueObjectSP ptr_sp(valobj_sp->GetChildAtIndex(0));
   if (!ptr_sp)
@@ -104,8 +106,8 @@ lldb_private::formatters::StdlibCoroutineHandleSyntheticFrontEnd::
 lldb_private::formatters::StdlibCoroutineHandleSyntheticFrontEnd::
     ~StdlibCoroutineHandleSyntheticFrontEnd() = default;
 
-uint32_t lldb_private::formatters::StdlibCoroutineHandleSyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    StdlibCoroutineHandleSyntheticFrontEnd::CalculateNumChildren() {
   if (!m_resume_ptr_sp || !m_destroy_ptr_sp)
     return 0;
 
diff --git a/lldb/source/Plugins/Language/CPlusPlus/Coroutines.h b/lldb/source/Plugins/Language/CPlusPlus/Coroutines.h
index 5c6a80b57ff4..1d4bc65e2637 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/Coroutines.h
+++ b/lldb/source/Plugins/Language/CPlusPlus/Coroutines.h
@@ -34,7 +34,7 @@ public:
 
   ~StdlibCoroutineHandleSyntheticFrontEnd() override;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
diff --git a/lldb/source/Plugins/Language/CPlusPlus/GenericBitset.cpp b/lldb/source/Plugins/Language/CPlusPlus/GenericBitset.cpp
index 6a9da1d17c76..33955dccb6cc 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/GenericBitset.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/GenericBitset.cpp
@@ -34,7 +34,9 @@ public:
 
   bool MightHaveChildren() override { return true; }
   lldb::ChildCacheState Update() override;
-  uint32_t CalculateNumChildren() override { return m_elements.size(); }
+  llvm::Expected CalculateNumChildren() override {
+    return m_elements.size();
+  }
   ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
 private:
diff --git a/lldb/source/Plugins/Language/CPlusPlus/GenericOptional.cpp b/lldb/source/Plugins/Language/CPlusPlus/GenericOptional.cpp
index c06afb53eb8a..23756de7f1e6 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/GenericOptional.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/GenericOptional.cpp
@@ -19,7 +19,7 @@ using namespace lldb_private;
 bool lldb_private::formatters::GenericOptionalSummaryProvider(
     ValueObject &valobj, Stream &stream, const TypeSummaryOptions &options) {
   stream.Printf(" Has Value=%s ",
-                valobj.GetNumChildren() == 0 ? "false" : "true");
+                valobj.GetNumChildrenIgnoringErrors() == 0 ? "false" : "true");
 
   return true;
 }
@@ -41,7 +41,9 @@ public:
   }
 
   bool MightHaveChildren() override { return true; }
-  uint32_t CalculateNumChildren() override { return m_has_value ? 1U : 0U; }
+  llvm::Expected CalculateNumChildren() override {
+    return m_has_value ? 1U : 0U;
+  }
 
   ValueObjectSP GetChildAtIndex(uint32_t idx) override;
   lldb::ChildCacheState Update() override;
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp
index bba887fec3ac..10a1fe039189 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp
@@ -63,7 +63,7 @@ lldb::ValueObjectSP
 lldb_private::formatters::GetSecondValueOfLibCXXCompressedPair(
     ValueObject &pair) {
   ValueObjectSP value;
-  if (pair.GetNumChildren() > 1) {
+  if (pair.GetNumChildrenIgnoringErrors() > 1) {
     ValueObjectSP second_child = pair.GetChildAtIndex(1);
     if (second_child) {
       value = second_child->GetChildMemberWithName("__value_");
@@ -351,8 +351,8 @@ lldb_private::formatters::LibCxxMapIteratorSyntheticFrontEnd::Update() {
   return lldb::ChildCacheState::eRefetch;
 }
 
-uint32_t lldb_private::formatters::LibCxxMapIteratorSyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    LibCxxMapIteratorSyntheticFrontEnd::CalculateNumChildren() {
   return 2;
 }
 
@@ -509,8 +509,8 @@ lldb::ChildCacheState lldb_private::formatters::
   return lldb::ChildCacheState::eRefetch;
 }
 
-uint32_t lldb_private::formatters::LibCxxUnorderedMapIteratorSyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    LibCxxUnorderedMapIteratorSyntheticFrontEnd::CalculateNumChildren() {
   return 2;
 }
 
@@ -566,8 +566,8 @@ lldb_private::formatters::LibcxxSharedPtrSyntheticFrontEnd::
     Update();
 }
 
-uint32_t lldb_private::formatters::LibcxxSharedPtrSyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    LibcxxSharedPtrSyntheticFrontEnd::CalculateNumChildren() {
   return (m_cntrl ? 1 : 0);
 }
 
@@ -661,8 +661,8 @@ lldb_private::formatters::LibcxxUniquePtrSyntheticFrontEndCreator(
                     : nullptr);
 }
 
-uint32_t lldb_private::formatters::LibcxxUniquePtrSyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    LibcxxUniquePtrSyntheticFrontEnd::CalculateNumChildren() {
   if (m_value_ptr_sp)
     return m_deleter_sp ? 2 : 1;
   return 0;
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h
index ad2f58508ab7..a59f21841ec8 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h
@@ -87,7 +87,7 @@ class LibCxxMapIteratorSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   LibCxxMapIteratorSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -135,7 +135,7 @@ public:
 
   ~LibCxxUnorderedMapIteratorSyntheticFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -166,7 +166,7 @@ class LibcxxSharedPtrSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   LibcxxSharedPtrSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -186,7 +186,7 @@ class LibcxxUniquePtrSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   LibcxxUniquePtrSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxAtomic.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxAtomic.cpp
index 8e4c36103a74..7f30dc186291 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxAtomic.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxAtomic.cpp
@@ -90,7 +90,7 @@ public:
 
   ~LibcxxStdAtomicSyntheticFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -124,8 +124,8 @@ bool lldb_private::formatters::LibcxxStdAtomicSyntheticFrontEnd::
   return true;
 }
 
-uint32_t lldb_private::formatters::LibcxxStdAtomicSyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    LibcxxStdAtomicSyntheticFrontEnd::CalculateNumChildren() {
   return m_real_child ? 1 : 0;
 }
 
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxInitializerList.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxInitializerList.cpp
index 00012dfc056e..bd9c72497664 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxInitializerList.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxInitializerList.cpp
@@ -26,7 +26,7 @@ public:
 
   ~LibcxxInitializerListSyntheticFrontEnd() override;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -59,8 +59,8 @@ lldb_private::formatters::LibcxxInitializerListSyntheticFrontEnd::
   // delete m_start;
 }
 
-uint32_t lldb_private::formatters::LibcxxInitializerListSyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    LibcxxInitializerListSyntheticFrontEnd::CalculateNumChildren() {
   m_num_elements = 0;
   ValueObjectSP size_sp(m_backend.GetChildMemberWithName("__size_"));
   if (size_sp)
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxList.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxList.cpp
index 17f6b737d9f6..d7cfeb30557c 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxList.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxList.cpp
@@ -136,7 +136,7 @@ class ForwardListFrontEnd : public AbstractListFrontEnd {
 public:
   ForwardListFrontEnd(ValueObject &valobj);
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
   ValueObjectSP GetChildAtIndex(uint32_t idx) override;
   lldb::ChildCacheState Update() override;
 };
@@ -147,7 +147,7 @@ public:
 
   ~ListFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -240,7 +240,7 @@ ForwardListFrontEnd::ForwardListFrontEnd(ValueObject &valobj)
   Update();
 }
 
-uint32_t ForwardListFrontEnd::CalculateNumChildren() {
+llvm::Expected ForwardListFrontEnd::CalculateNumChildren() {
   if (m_count != UINT32_MAX)
     return m_count;
 
@@ -254,7 +254,7 @@ uint32_t ForwardListFrontEnd::CalculateNumChildren() {
 }
 
 ValueObjectSP ForwardListFrontEnd::GetChildAtIndex(uint32_t idx) {
-  if (idx >= CalculateNumChildren())
+  if (idx >= CalculateNumChildrenIgnoringErrors())
     return nullptr;
 
   if (!m_head)
@@ -308,7 +308,7 @@ ListFrontEnd::ListFrontEnd(lldb::ValueObjectSP valobj_sp)
     Update();
 }
 
-uint32_t ListFrontEnd::CalculateNumChildren() {
+llvm::Expected ListFrontEnd::CalculateNumChildren() {
   if (m_count != UINT32_MAX)
     return m_count;
   if (!m_head || !m_tail || m_node_address == 0)
@@ -347,7 +347,7 @@ lldb::ValueObjectSP ListFrontEnd::GetChildAtIndex(uint32_t idx) {
   static ConstString g_value("__value_");
   static ConstString g_next("__next_");
 
-  if (idx >= CalculateNumChildren())
+  if (idx >= CalculateNumChildrenIgnoringErrors())
     return lldb::ValueObjectSP();
 
   if (!m_head || !m_tail || m_node_address == 0)
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxMap.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxMap.cpp
index 6d24eb03779c..ec5b320e2218 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxMap.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxMap.cpp
@@ -177,7 +177,7 @@ public:
 
   ~LibcxxStdMapSyntheticFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -209,8 +209,8 @@ lldb_private::formatters::LibcxxStdMapSyntheticFrontEnd::
     Update();
 }
 
-uint32_t lldb_private::formatters::LibcxxStdMapSyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    LibcxxStdMapSyntheticFrontEnd::CalculateNumChildren() {
   if (m_count != UINT32_MAX)
     return m_count;
 
@@ -311,13 +311,13 @@ lldb_private::formatters::LibcxxStdMapSyntheticFrontEnd::GetChildAtIndex(
     uint32_t idx) {
   static ConstString g_cc_("__cc_"), g_cc("__cc");
   static ConstString g_nc("__nc");
-
-  if (idx >= CalculateNumChildren())
+  uint32_t num_children = CalculateNumChildrenIgnoringErrors();
+  if (idx >= num_children)
     return lldb::ValueObjectSP();
   if (m_tree == nullptr || m_root_node == nullptr)
     return lldb::ValueObjectSP();
 
-  MapIterator iterator(m_root_node, CalculateNumChildren());
+  MapIterator iterator(m_root_node, num_children);
 
   const bool need_to_skip = (idx > 0);
   size_t actual_advancde = idx;
@@ -382,7 +382,7 @@ lldb_private::formatters::LibcxxStdMapSyntheticFrontEnd::GetChildAtIndex(
   name.Printf("[%" PRIu64 "]", (uint64_t)idx);
   auto potential_child_sp = iterated_sp->Clone(ConstString(name.GetString()));
   if (potential_child_sp) {
-    switch (potential_child_sp->GetNumChildren()) {
+    switch (potential_child_sp->GetNumChildrenIgnoringErrors()) {
     case 1: {
       auto child0_sp = potential_child_sp->GetChildAtIndex(0);
       if (child0_sp &&
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxQueue.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxQueue.cpp
index fbadee89b7b7..5b459a17fe29 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxQueue.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxQueue.cpp
@@ -28,7 +28,7 @@ public:
   bool MightHaveChildren() override { return true; }
   lldb::ChildCacheState Update() override;
 
-  uint32_t CalculateNumChildren() override {
+  llvm::Expected CalculateNumChildren() override {
     return m_container_sp ? m_container_sp->GetNumChildren() : 0;
   }
 
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxRangesRefView.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxRangesRefView.cpp
index 74f54f767356..01a7b8f142ec 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxRangesRefView.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxRangesRefView.cpp
@@ -27,7 +27,7 @@ public:
 
   ~LibcxxStdRangesRefViewSyntheticFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override {
+  llvm::Expected CalculateNumChildren() override {
     // __range_ will be the sole child of this type
     return 1;
   }
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxSpan.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxSpan.cpp
index af2b51d2b540..9895f336bfd0 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxSpan.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxSpan.cpp
@@ -27,7 +27,7 @@ public:
 
   ~LibcxxStdSpanSyntheticFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -73,8 +73,8 @@ lldb_private::formatters::LibcxxStdSpanSyntheticFrontEnd::
     Update();
 }
 
-uint32_t lldb_private::formatters::LibcxxStdSpanSyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    LibcxxStdSpanSyntheticFrontEnd::CalculateNumChildren() {
   return m_num_elements;
 }
 
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxTuple.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxTuple.cpp
index 62bb7d619267..3e3259ab428d 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxTuple.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxTuple.cpp
@@ -26,7 +26,9 @@ public:
 
   bool MightHaveChildren() override { return true; }
   lldb::ChildCacheState Update() override;
-  uint32_t CalculateNumChildren() override { return m_elements.size(); }
+  llvm::Expected CalculateNumChildren() override {
+    return m_elements.size();
+  }
   ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
 private:
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxUnorderedMap.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxUnorderedMap.cpp
index b3c364294335..af29fdb6d001 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxUnorderedMap.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxUnorderedMap.cpp
@@ -33,7 +33,7 @@ public:
 
   ~LibcxxStdUnorderedMapSyntheticFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -62,8 +62,8 @@ lldb_private::formatters::LibcxxStdUnorderedMapSyntheticFrontEnd::
     Update();
 }
 
-uint32_t lldb_private::formatters::LibcxxStdUnorderedMapSyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    LibcxxStdUnorderedMapSyntheticFrontEnd::CalculateNumChildren() {
   return m_num_elements;
 }
 
@@ -94,7 +94,7 @@ static bool isUnorderedMap(ConstString type_name) {
 
 lldb::ValueObjectSP lldb_private::formatters::
     LibcxxStdUnorderedMapSyntheticFrontEnd::GetChildAtIndex(uint32_t idx) {
-  if (idx >= CalculateNumChildren())
+  if (idx >= CalculateNumChildrenIgnoringErrors())
     return lldb::ValueObjectSP();
   if (m_tree == nullptr)
     return lldb::ValueObjectSP();
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxValarray.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxValarray.cpp
index 463c7b8d7ce3..99f94406e99a 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxValarray.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxValarray.cpp
@@ -24,7 +24,7 @@ public:
 
   ~LibcxxStdValarraySyntheticFrontEnd() override;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -63,8 +63,8 @@ lldb_private::formatters::LibcxxStdValarraySyntheticFrontEnd::
   // delete m_finish;
 }
 
-uint32_t lldb_private::formatters::LibcxxStdValarraySyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    LibcxxStdValarraySyntheticFrontEnd::CalculateNumChildren() {
   if (!m_start || !m_finish)
     return 0;
   uint64_t start_val = m_start->GetValueAsUnsigned(0);
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxVariant.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxVariant.cpp
index 1f62062f09be..62794318e077 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxVariant.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxVariant.cpp
@@ -205,7 +205,7 @@ public:
 
   bool MightHaveChildren() override { return true; }
   lldb::ChildCacheState Update() override;
-  uint32_t CalculateNumChildren() override { return m_size; }
+  llvm::Expected CalculateNumChildren() override { return m_size; }
   ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
 private:
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxVector.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxVector.cpp
index fcf727ad2ea0..461fed35164b 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxVector.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxVector.cpp
@@ -25,7 +25,7 @@ public:
 
   ~LibcxxStdVectorSyntheticFrontEnd() override;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -46,7 +46,7 @@ class LibcxxVectorBoolSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   LibcxxVectorBoolSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -82,8 +82,8 @@ lldb_private::formatters::LibcxxStdVectorSyntheticFrontEnd::
   // delete m_finish;
 }
 
-uint32_t lldb_private::formatters::LibcxxStdVectorSyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    LibcxxStdVectorSyntheticFrontEnd::CalculateNumChildren() {
   if (!m_start || !m_finish)
     return 0;
   uint64_t start_val = m_start->GetValueAsUnsigned(0);
@@ -165,8 +165,8 @@ lldb_private::formatters::LibcxxVectorBoolSyntheticFrontEnd::
   }
 }
 
-uint32_t lldb_private::formatters::LibcxxVectorBoolSyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    LibcxxVectorBoolSyntheticFrontEnd::CalculateNumChildren() {
   return m_count;
 }
 
@@ -259,7 +259,7 @@ size_t lldb_private::formatters::LibcxxVectorBoolSyntheticFrontEnd::
     return UINT32_MAX;
   const char *item_name = name.GetCString();
   uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
     return UINT32_MAX;
   return idx;
 }
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.cpp
index 5abb3d50674b..86bb575af5ca 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.cpp
@@ -43,7 +43,7 @@ class LibstdcppMapIteratorSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   explicit LibstdcppMapIteratorSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -64,7 +64,7 @@ class LibStdcppSharedPtrSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   explicit LibStdcppSharedPtrSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -132,7 +132,8 @@ lldb::ChildCacheState LibstdcppMapIteratorSyntheticFrontEnd::Update() {
   return lldb::ChildCacheState::eReuse;
 }
 
-uint32_t LibstdcppMapIteratorSyntheticFrontEnd::CalculateNumChildren() {
+llvm::Expected
+LibstdcppMapIteratorSyntheticFrontEnd::CalculateNumChildren() {
   return 2;
 }
 
@@ -219,7 +220,10 @@ lldb::ChildCacheState VectorIteratorSyntheticFrontEnd::Update() {
   return lldb::ChildCacheState::eRefetch;
 }
 
-uint32_t VectorIteratorSyntheticFrontEnd::CalculateNumChildren() { return 1; }
+llvm::Expected
+VectorIteratorSyntheticFrontEnd::CalculateNumChildren() {
+  return 1;
+}
 
 lldb::ValueObjectSP
 VectorIteratorSyntheticFrontEnd::GetChildAtIndex(uint32_t idx) {
@@ -371,7 +375,10 @@ LibStdcppSharedPtrSyntheticFrontEnd::LibStdcppSharedPtrSyntheticFrontEnd(
     Update();
 }
 
-uint32_t LibStdcppSharedPtrSyntheticFrontEnd::CalculateNumChildren() { return 1; }
+llvm::Expected
+LibStdcppSharedPtrSyntheticFrontEnd::CalculateNumChildren() {
+  return 1;
+}
 
 lldb::ValueObjectSP
 LibStdcppSharedPtrSyntheticFrontEnd::GetChildAtIndex(uint32_t idx) {
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibStdcppTuple.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibStdcppTuple.cpp
index 64d2ec9d943a..05199ba35b9a 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibStdcppTuple.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibStdcppTuple.cpp
@@ -26,7 +26,7 @@ class LibStdcppTupleSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   explicit LibStdcppTupleSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -65,7 +65,7 @@ lldb::ChildCacheState LibStdcppTupleSyntheticFrontEnd::Update() {
     ValueObjectSP current_child = next_child_sp;
     next_child_sp = nullptr;
 
-    size_t child_count = current_child->GetNumChildren();
+    size_t child_count = current_child->GetNumChildrenIgnoringErrors();
     for (size_t i = 0; i < child_count; ++i) {
       ValueObjectSP child_sp = current_child->GetChildAtIndex(i);
       llvm::StringRef name_str = child_sp->GetName().GetStringRef();
@@ -95,7 +95,8 @@ LibStdcppTupleSyntheticFrontEnd::GetChildAtIndex(uint32_t idx) {
   return lldb::ValueObjectSP();
 }
 
-uint32_t LibStdcppTupleSyntheticFrontEnd::CalculateNumChildren() {
+llvm::Expected
+LibStdcppTupleSyntheticFrontEnd::CalculateNumChildren() {
   return m_members.size();
 }
 
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibStdcppUniquePointer.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibStdcppUniquePointer.cpp
index 3a48fe412e07..92f540d9ca52 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibStdcppUniquePointer.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibStdcppUniquePointer.cpp
@@ -26,7 +26,7 @@ class LibStdcppUniquePtrSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   explicit LibStdcppUniquePtrSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -135,7 +135,8 @@ LibStdcppUniquePtrSyntheticFrontEnd::GetChildAtIndex(uint32_t idx) {
   return lldb::ValueObjectSP();
 }
 
-uint32_t LibStdcppUniquePtrSyntheticFrontEnd::CalculateNumChildren() {
+llvm::Expected
+LibStdcppUniquePtrSyntheticFrontEnd::CalculateNumChildren() {
   if (m_del_obj)
     return 2;
   return 1;
diff --git a/lldb/source/Plugins/Language/ObjC/Cocoa.cpp b/lldb/source/Plugins/Language/ObjC/Cocoa.cpp
index cb740f8e71e1..96166657ceeb 100644
--- a/lldb/source/Plugins/Language/ObjC/Cocoa.cpp
+++ b/lldb/source/Plugins/Language/ObjC/Cocoa.cpp
@@ -1038,7 +1038,7 @@ public:
 
   ~ObjCClassSyntheticChildrenFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override { return 0; }
+  llvm::Expected CalculateNumChildren() override { return 0; }
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override {
     return lldb::ValueObjectSP();
diff --git a/lldb/source/Plugins/Language/ObjC/NSArray.cpp b/lldb/source/Plugins/Language/ObjC/NSArray.cpp
index 7f060b2613d6..67d0cd08f51a 100644
--- a/lldb/source/Plugins/Language/ObjC/NSArray.cpp
+++ b/lldb/source/Plugins/Language/ObjC/NSArray.cpp
@@ -50,7 +50,7 @@ public:
 
   ~NSArrayMSyntheticFrontEndBase() override = default;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -214,7 +214,7 @@ public:
 
   ~GenericNSArrayISyntheticFrontEnd() override;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -302,7 +302,7 @@ public:
 
   ~NSArray0SyntheticFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -319,7 +319,7 @@ public:
 
   ~NSArray1SyntheticFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -477,15 +477,15 @@ lldb_private::formatters::
     : NSArrayMSyntheticFrontEndBase(valobj_sp), m_data_32(nullptr),
       m_data_64(nullptr) {}
 
-uint32_t lldb_private::formatters::NSArrayMSyntheticFrontEndBase::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    NSArrayMSyntheticFrontEndBase::CalculateNumChildren() {
   return GetUsedCount();
 }
 
 lldb::ValueObjectSP
 lldb_private::formatters::NSArrayMSyntheticFrontEndBase::GetChildAtIndex(
     uint32_t idx) {
-  if (idx >= CalculateNumChildren())
+  if (idx >= CalculateNumChildrenIgnoringErrors())
     return lldb::ValueObjectSP();
   lldb::addr_t object_at_idx = GetDataAddress();
   size_t pyhs_idx = idx;
@@ -542,7 +542,7 @@ lldb_private::formatters::NSArrayMSyntheticFrontEndBase::GetIndexOfChildWithName
     ConstString name) {
   const char *item_name = name.GetCString();
   uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
     return UINT32_MAX;
   return idx;
 }
@@ -628,15 +628,15 @@ lldb_private::formatters::GenericNSArrayISyntheticFrontEnd::
   GetIndexOfChildWithName(ConstString name) {
   const char *item_name = name.GetCString();
   uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
     return UINT32_MAX;
   return idx;
 }
 
 template 
-uint32_t
-lldb_private::formatters::GenericNSArrayISyntheticFrontEnd::
-  CalculateNumChildren() {
+llvm::Expected
+lldb_private::formatters::GenericNSArrayISyntheticFrontEnd<
+    D32, D64, Inline>::CalculateNumChildren() {
   return m_data_32 ? m_data_32->used : m_data_64->used;
 }
 
@@ -685,7 +685,7 @@ template 
 lldb::ValueObjectSP
 lldb_private::formatters::GenericNSArrayISyntheticFrontEnd::
   GetChildAtIndex(uint32_t idx) {
-  if (idx >= CalculateNumChildren())
+  if (idx >= CalculateNumChildrenIgnoringErrors())
     return lldb::ValueObjectSP();
   lldb::addr_t object_at_idx;
   if (Inline) {
@@ -719,7 +719,7 @@ lldb_private::formatters::NSArray0SyntheticFrontEnd::GetIndexOfChildWithName(
   return UINT32_MAX;
 }
 
-uint32_t
+llvm::Expected
 lldb_private::formatters::NSArray0SyntheticFrontEnd::CalculateNumChildren() {
   return 0;
 }
@@ -754,7 +754,7 @@ lldb_private::formatters::NSArray1SyntheticFrontEnd::GetIndexOfChildWithName(
   return UINT32_MAX;
 }
 
-uint32_t
+llvm::Expected
 lldb_private::formatters::NSArray1SyntheticFrontEnd::CalculateNumChildren() {
   return 1;
 }
diff --git a/lldb/source/Plugins/Language/ObjC/NSDictionary.cpp b/lldb/source/Plugins/Language/ObjC/NSDictionary.cpp
index da94eda1529c..ec6fd756394a 100644
--- a/lldb/source/Plugins/Language/ObjC/NSDictionary.cpp
+++ b/lldb/source/Plugins/Language/ObjC/NSDictionary.cpp
@@ -103,7 +103,7 @@ public:
 
   ~NSDictionaryISyntheticFrontEnd() override;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -144,7 +144,7 @@ class NSConstantDictionarySyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   NSConstantDictionarySyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -176,7 +176,7 @@ class NSCFDictionarySyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   NSCFDictionarySyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -209,7 +209,7 @@ public:
 
   ~NSDictionary1SyntheticFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -230,7 +230,7 @@ public:
 
   ~GenericNSDictionaryMSyntheticFrontEnd() override;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -262,9 +262,9 @@ namespace Foundation1100 {
     NSDictionaryMSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
     
     ~NSDictionaryMSyntheticFrontEnd() override;
-    
-    uint32_t CalculateNumChildren() override;
-    
+
+    llvm::Expected CalculateNumChildren() override;
+
     lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
     lldb::ChildCacheState Update() override;
@@ -601,13 +601,13 @@ size_t lldb_private::formatters::NSDictionaryISyntheticFrontEnd::
     GetIndexOfChildWithName(ConstString name) {
   const char *item_name = name.GetCString();
   uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
     return UINT32_MAX;
   return idx;
 }
 
-uint32_t lldb_private::formatters::NSDictionaryISyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    NSDictionaryISyntheticFrontEnd::CalculateNumChildren() {
   if (!m_data_32 && !m_data_64)
     return 0;
   return (m_data_32 ? m_data_32->_used : m_data_64->_used);
@@ -656,7 +656,7 @@ bool lldb_private::formatters::NSDictionaryISyntheticFrontEnd::
 lldb::ValueObjectSP
 lldb_private::formatters::NSDictionaryISyntheticFrontEnd::GetChildAtIndex(
     uint32_t idx) {
-  uint32_t num_children = CalculateNumChildren();
+  uint32_t num_children = CalculateNumChildrenIgnoringErrors();
 
   if (idx >= num_children)
     return lldb::ValueObjectSP();
@@ -739,13 +739,13 @@ size_t lldb_private::formatters::NSCFDictionarySyntheticFrontEnd::
     GetIndexOfChildWithName(ConstString name) {
   const char *item_name = name.GetCString();
   const uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
     return UINT32_MAX;
   return idx;
 }
 
-uint32_t lldb_private::formatters::NSCFDictionarySyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    NSCFDictionarySyntheticFrontEnd::CalculateNumChildren() {
   if (!m_hashtable.IsValid())
     return 0;
   return m_hashtable.GetCount();
@@ -781,7 +781,7 @@ lldb_private::formatters::NSCFDictionarySyntheticFrontEnd::GetChildAtIndex(
   lldb::addr_t m_keys_ptr = m_hashtable.GetKeyPointer();
   lldb::addr_t m_values_ptr = m_hashtable.GetValuePointer();
 
-  const uint32_t num_children = CalculateNumChildren();
+  const uint32_t num_children = CalculateNumChildrenIgnoringErrors();
 
   if (idx >= num_children)
     return lldb::ValueObjectSP();
@@ -875,13 +875,13 @@ size_t lldb_private::formatters::NSConstantDictionarySyntheticFrontEnd::
     GetIndexOfChildWithName(ConstString name) {
   const char *item_name = name.GetCString();
   uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
     return UINT32_MAX;
   return idx;
 }
 
-uint32_t lldb_private::formatters::NSConstantDictionarySyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    NSConstantDictionarySyntheticFrontEnd::CalculateNumChildren() {
   return m_size;
 }
 
@@ -921,7 +921,7 @@ bool lldb_private::formatters::NSConstantDictionarySyntheticFrontEnd::
 
 lldb::ValueObjectSP lldb_private::formatters::
     NSConstantDictionarySyntheticFrontEnd::GetChildAtIndex(uint32_t idx) {
-  uint32_t num_children = CalculateNumChildren();
+  uint32_t num_children = CalculateNumChildrenIgnoringErrors();
 
   if (idx >= num_children)
     return lldb::ValueObjectSP();
@@ -994,8 +994,8 @@ size_t lldb_private::formatters::NSDictionary1SyntheticFrontEnd::
   return name == g_zero ? 0 : UINT32_MAX;
 }
 
-uint32_t lldb_private::formatters::NSDictionary1SyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    NSDictionary1SyntheticFrontEnd::CalculateNumChildren() {
   return 1;
 }
 
@@ -1081,17 +1081,18 @@ size_t lldb_private::formatters::GenericNSDictionaryMSyntheticFrontEnd<
     D32, D64>::GetIndexOfChildWithName(ConstString name) {
   const char *item_name = name.GetCString();
   uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
     return UINT32_MAX;
   return idx;
 }
 
 template 
-uint32_t
-lldb_private::formatters::GenericNSDictionaryMSyntheticFrontEnd::CalculateNumChildren() {
+llvm::Expected
+lldb_private::formatters::GenericNSDictionaryMSyntheticFrontEnd<
+    D32, D64>::CalculateNumChildren() {
   if (!m_data_32 && !m_data_64)
     return 0;
-  return (m_data_32 ? m_data_32->_used : m_data_64->_used);
+  return (m_data_32 ? (uint32_t)m_data_32->_used : (uint32_t)m_data_64->_used);
 }
 
 template 
@@ -1153,7 +1154,7 @@ lldb_private::formatters::GenericNSDictionaryMSyntheticFrontEnd<
     m_values_ptr = m_data_64->_buffer + (m_ptr_size * size);
   }
 
-  uint32_t num_children = CalculateNumChildren();
+  uint32_t num_children = CalculateNumChildrenIgnoringErrors();
 
   if (idx >= num_children)
     return lldb::ValueObjectSP();
@@ -1245,14 +1246,13 @@ lldb_private::formatters::Foundation1100::
   NSDictionaryMSyntheticFrontEnd::GetIndexOfChildWithName(ConstString name) {
   const char *item_name = name.GetCString();
   uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
     return UINT32_MAX;
   return idx;
 }
 
-uint32_t
-lldb_private::formatters::Foundation1100::
-  NSDictionaryMSyntheticFrontEnd::CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::Foundation1100::
+    NSDictionaryMSyntheticFrontEnd::CalculateNumChildren() {
   if (!m_data_32 && !m_data_64)
     return 0;
   return (m_data_32 ? m_data_32->_used : m_data_64->_used);
@@ -1306,7 +1306,7 @@ lldb_private::formatters::Foundation1100::
   lldb::addr_t m_values_ptr =
       (m_data_32 ? m_data_32->_objs_addr : m_data_64->_objs_addr);
 
-  uint32_t num_children = CalculateNumChildren();
+  uint32_t num_children = CalculateNumChildrenIgnoringErrors();
 
   if (idx >= num_children)
     return lldb::ValueObjectSP();
diff --git a/lldb/source/Plugins/Language/ObjC/NSError.cpp b/lldb/source/Plugins/Language/ObjC/NSError.cpp
index b034e799b716..5ef7edc7e80c 100644
--- a/lldb/source/Plugins/Language/ObjC/NSError.cpp
+++ b/lldb/source/Plugins/Language/ObjC/NSError.cpp
@@ -116,7 +116,7 @@ public:
   // no need to delete m_child_ptr - it's kept alive by the cluster manager on
   // our behalf
 
-  uint32_t CalculateNumChildren() override {
+  llvm::Expected CalculateNumChildren() override {
     if (m_child_ptr)
       return 1;
     if (m_child_sp)
diff --git a/lldb/source/Plugins/Language/ObjC/NSException.cpp b/lldb/source/Plugins/Language/ObjC/NSException.cpp
index 09d3a1b42b74..e7ce26ea4c6f 100644
--- a/lldb/source/Plugins/Language/ObjC/NSException.cpp
+++ b/lldb/source/Plugins/Language/ObjC/NSException.cpp
@@ -123,9 +123,7 @@ public:
 
   ~NSExceptionSyntheticFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override {
-    return 4;
-  }
+  llvm::Expected CalculateNumChildren() override { return 4; }
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override {
     switch (idx) {
diff --git a/lldb/source/Plugins/Language/ObjC/NSIndexPath.cpp b/lldb/source/Plugins/Language/ObjC/NSIndexPath.cpp
index 10bb907c58ed..a434cee09d38 100644
--- a/lldb/source/Plugins/Language/ObjC/NSIndexPath.cpp
+++ b/lldb/source/Plugins/Language/ObjC/NSIndexPath.cpp
@@ -40,7 +40,9 @@ public:
 
   ~NSIndexPathSyntheticFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override { return m_impl.GetNumIndexes(); }
+  llvm::Expected CalculateNumChildren() override {
+    return m_impl.GetNumIndexes();
+  }
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override {
     return m_impl.GetIndexAtIndex(idx, m_uint_star_type);
@@ -127,7 +129,7 @@ public:
   size_t GetIndexOfChildWithName(ConstString name) override {
     const char *item_name = name.GetCString();
     uint32_t idx = ExtractIndexFromString(item_name);
-    if (idx < UINT32_MAX && idx >= CalculateNumChildren())
+    if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
       return UINT32_MAX;
     return idx;
   }
diff --git a/lldb/source/Plugins/Language/ObjC/NSSet.cpp b/lldb/source/Plugins/Language/ObjC/NSSet.cpp
index c965a2a13400..7d0a6a507211 100644
--- a/lldb/source/Plugins/Language/ObjC/NSSet.cpp
+++ b/lldb/source/Plugins/Language/ObjC/NSSet.cpp
@@ -46,7 +46,7 @@ public:
 
   ~NSSetISyntheticFrontEnd() override;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -84,7 +84,7 @@ class NSCFSetSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   NSCFSetSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -117,7 +117,7 @@ public:
 
   ~GenericNSSetMSyntheticFrontEnd() override;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -233,7 +233,7 @@ public:
 
   ~NSSetCodeRunningSyntheticFrontEnd() override;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -414,12 +414,12 @@ lldb_private::formatters::NSSetISyntheticFrontEnd::GetIndexOfChildWithName(
     ConstString name) {
   const char *item_name = name.GetCString();
   uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
     return UINT32_MAX;
   return idx;
 }
 
-uint32_t
+llvm::Expected
 lldb_private::formatters::NSSetISyntheticFrontEnd::CalculateNumChildren() {
   if (!m_data_32 && !m_data_64)
     return 0;
@@ -468,7 +468,7 @@ bool lldb_private::formatters::NSSetISyntheticFrontEnd::MightHaveChildren() {
 lldb::ValueObjectSP
 lldb_private::formatters::NSSetISyntheticFrontEnd::GetChildAtIndex(
     uint32_t idx) {
-  uint32_t num_children = CalculateNumChildren();
+  uint32_t num_children = CalculateNumChildrenIgnoringErrors();
 
   if (idx >= num_children)
     return lldb::ValueObjectSP();
@@ -551,12 +551,12 @@ lldb_private::formatters::NSCFSetSyntheticFrontEnd::GetIndexOfChildWithName(
     ConstString name) {
   const char *item_name = name.GetCString();
   const uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
     return UINT32_MAX;
   return idx;
 }
 
-uint32_t
+llvm::Expected
 lldb_private::formatters::NSCFSetSyntheticFrontEnd::CalculateNumChildren() {
   if (!m_hashtable.IsValid())
     return 0;
@@ -591,7 +591,7 @@ lldb_private::formatters::NSCFSetSyntheticFrontEnd::GetChildAtIndex(
     uint32_t idx) {
   lldb::addr_t m_values_ptr = m_hashtable.GetValuePointer();
 
-  const uint32_t num_children = CalculateNumChildren();
+  const uint32_t num_children = CalculateNumChildrenIgnoringErrors();
 
   if (idx >= num_children)
     return lldb::ValueObjectSP();
@@ -691,18 +691,18 @@ lldb_private::formatters::
     ConstString name) {
   const char *item_name = name.GetCString();
   uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
     return UINT32_MAX;
   return idx;
 }
 
 template 
-uint32_t
-lldb_private::formatters::
-  GenericNSSetMSyntheticFrontEnd::CalculateNumChildren() {
+llvm::Expected
+lldb_private::formatters::GenericNSSetMSyntheticFrontEnd<
+    D32, D64>::CalculateNumChildren() {
   if (!m_data_32 && !m_data_64)
     return 0;
-  return (m_data_32 ? m_data_32->_used : m_data_64->_used);
+  return (m_data_32 ? (uint32_t)m_data_32->_used : (uint32_t)m_data_64->_used);
 }
 
 template 
@@ -753,7 +753,7 @@ lldb_private::formatters::
   lldb::addr_t m_objs_addr =
       (m_data_32 ? m_data_32->_objs_addr : m_data_64->_objs_addr);
 
-  uint32_t num_children = CalculateNumChildren();
+  uint32_t num_children = CalculateNumChildrenIgnoringErrors();
 
   if (idx >= num_children)
     return lldb::ValueObjectSP();
diff --git a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntime.cpp b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntime.cpp
index f08f9f0f815d..9434376f7d9e 100644
--- a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntime.cpp
+++ b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntime.cpp
@@ -539,7 +539,8 @@ ThreadSP AppleObjCRuntime::GetBacktraceThreadFromException(
     return object;
   };
 
-  for (size_t idx = 0; idx < reserved_dict->GetNumChildren(); idx++) {
+  for (size_t idx = 0; idx < reserved_dict->GetNumChildrenIgnoringErrors();
+       idx++) {
     ValueObjectSP dict_entry = reserved_dict->GetChildAtIndex(idx);
 
     DataExtractor data;
diff --git a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
index 51ab13108feb..c02b08cb4782 100644
--- a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
+++ b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
@@ -5263,9 +5263,10 @@ GetDynamicArrayInfo(TypeSystemClang &ast, SymbolFile *sym_file,
   return std::nullopt;
 }
 
-uint32_t TypeSystemClang::GetNumChildren(lldb::opaque_compiler_type_t type,
-                                         bool omit_empty_base_classes,
-                                         const ExecutionContext *exe_ctx) {
+llvm::Expected
+TypeSystemClang::GetNumChildren(lldb::opaque_compiler_type_t type,
+                                bool omit_empty_base_classes,
+                                const ExecutionContext *exe_ctx) {
   if (!type)
     return 0;
 
@@ -5361,9 +5362,13 @@ uint32_t TypeSystemClang::GetNumChildren(lldb::opaque_compiler_type_t type,
     CompilerType pointee_clang_type(GetPointeeType(type));
 
     uint32_t num_pointee_children = 0;
-    if (pointee_clang_type.IsAggregateType())
-      num_pointee_children =
+    if (pointee_clang_type.IsAggregateType()) {
+      auto num_children_or_err =
           pointee_clang_type.GetNumChildren(omit_empty_base_classes, exe_ctx);
+      if (!num_children_or_err)
+        return num_children_or_err;
+      num_pointee_children = *num_children_or_err;
+    }
     // If this type points to a simple type, then it has 1 child
     if (num_pointee_children == 0)
       num_children = 1;
@@ -5397,9 +5402,13 @@ uint32_t TypeSystemClang::GetNumChildren(lldb::opaque_compiler_type_t type,
     clang::QualType pointee_type(pointer_type->getPointeeType());
     CompilerType pointee_clang_type(GetType(pointee_type));
     uint32_t num_pointee_children = 0;
-    if (pointee_clang_type.IsAggregateType())
-      num_pointee_children =
+    if (pointee_clang_type.IsAggregateType()) {
+      auto num_children_or_err =
           pointee_clang_type.GetNumChildren(omit_empty_base_classes, exe_ctx);
+      if (!num_children_or_err)
+        return num_children_or_err;
+      num_pointee_children = *num_children_or_err;
+    }
     if (num_pointee_children == 0) {
       // We have a pointer to a pointee type that claims it has no children. We
       // will want to look at
@@ -6108,8 +6117,15 @@ CompilerType TypeSystemClang::GetChildCompilerTypeAtIndex(
   child_is_base_class = false;
   language_flags = 0;
 
-  const bool idx_is_valid =
-      idx < GetNumChildren(type, omit_empty_base_classes, exe_ctx);
+  auto num_children_or_err =
+      GetNumChildren(type, omit_empty_base_classes, exe_ctx);
+  if (!num_children_or_err) {
+    LLDB_LOG_ERRORV(GetLog(LLDBLog::Types), num_children_or_err.takeError(),
+                    "{0}");
+    return {};
+  }
+
+  const bool idx_is_valid = idx < *num_children_or_err;
   int32_t bit_offset;
   switch (parent_type_class) {
   case clang::Type::Builtin:
@@ -6265,8 +6281,10 @@ CompilerType TypeSystemClang::GetChildCompilerTypeAtIndex(
               CompilerType base_class_clang_type =
                   GetType(getASTContext().getObjCInterfaceType(
                       superclass_interface_decl));
-              if (base_class_clang_type.GetNumChildren(omit_empty_base_classes,
-                                                       exe_ctx) > 0) {
+              if (llvm::expectedToStdOptional(
+                      base_class_clang_type.GetNumChildren(
+                          omit_empty_base_classes, exe_ctx))
+                      .value_or(0) > 0) {
                 if (idx == 0) {
                   clang::QualType ivar_qual_type(
                       getASTContext().getObjCInterfaceType(
diff --git a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
index a73164895baa..05c303baa416 100644
--- a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
+++ b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
@@ -833,9 +833,10 @@ public:
   GetTypeBitAlign(lldb::opaque_compiler_type_t type,
                   ExecutionContextScope *exe_scope) override;
 
-  uint32_t GetNumChildren(lldb::opaque_compiler_type_t type,
-                          bool omit_empty_base_classes,
-                          const ExecutionContext *exe_ctx) override;
+  llvm::Expected
+  GetNumChildren(lldb::opaque_compiler_type_t type,
+                 bool omit_empty_base_classes,
+                 const ExecutionContext *exe_ctx) override;
 
   CompilerType GetBuiltinTypeByName(ConstString name) override;
 
diff --git a/lldb/source/Symbol/CompilerType.cpp b/lldb/source/Symbol/CompilerType.cpp
index 76b79daa6ac1..85dd2d841a5a 100644
--- a/lldb/source/Symbol/CompilerType.cpp
+++ b/lldb/source/Symbol/CompilerType.cpp
@@ -770,8 +770,9 @@ lldb::Format CompilerType::GetFormat() const {
   return lldb::eFormatDefault;
 }
 
-uint32_t CompilerType::GetNumChildren(bool omit_empty_base_classes,
-                                      const ExecutionContext *exe_ctx) const {
+llvm::Expected
+CompilerType::GetNumChildren(bool omit_empty_base_classes,
+                             const ExecutionContext *exe_ctx) const {
   if (IsValid())
     if (auto type_system_sp = GetTypeSystem())
       return type_system_sp->GetNumChildren(m_type, omit_empty_base_classes,
diff --git a/lldb/source/Symbol/Type.cpp b/lldb/source/Symbol/Type.cpp
index 6069d066eaf6..44a24d7178f5 100644
--- a/lldb/source/Symbol/Type.cpp
+++ b/lldb/source/Symbol/Type.cpp
@@ -488,7 +488,7 @@ std::optional Type::GetByteSize(ExecutionContextScope *exe_scope) {
   return {};
 }
 
-uint32_t Type::GetNumChildren(bool omit_empty_base_classes) {
+llvm::Expected Type::GetNumChildren(bool omit_empty_base_classes) {
   return GetForwardCompilerType().GetNumChildren(omit_empty_base_classes, nullptr);
 }
 
diff --git a/lldb/source/Symbol/Variable.cpp b/lldb/source/Symbol/Variable.cpp
index a33c3433d9e2..90edede0f065 100644
--- a/lldb/source/Symbol/Variable.cpp
+++ b/lldb/source/Symbol/Variable.cpp
@@ -27,6 +27,8 @@
 #include "lldb/Target/StackFrame.h"
 #include "lldb/Target/Target.h"
 #include "lldb/Target/Thread.h"
+#include "lldb/Utility/LLDBLog.h"
+#include "lldb/Utility/Log.h"
 #include "lldb/Utility/RegularExpression.h"
 #include "lldb/Utility/Stream.h"
 
@@ -568,7 +570,9 @@ static void PrivateAutoComplete(
       case eTypeClassObjCObjectPointer:
       case eTypeClassPointer: {
         bool omit_empty_base_classes = true;
-        if (compiler_type.GetNumChildren(omit_empty_base_classes, nullptr) > 0)
+        if (llvm::expectedToStdOptional(
+                compiler_type.GetNumChildren(omit_empty_base_classes, nullptr))
+                .value_or(0))
           request.AddCompletion((prefix_path + "->").str());
         else {
           request.AddCompletion(prefix_path.str());
diff --git a/lldb/source/Target/StackFrame.cpp b/lldb/source/Target/StackFrame.cpp
index 50cf01e63cd4..c29a71d92572 100644
--- a/lldb/source/Target/StackFrame.cpp
+++ b/lldb/source/Target/StackFrame.cpp
@@ -857,10 +857,11 @@ ValueObjectSP StackFrame::GetValueForVariableExpressionPath(
                   "\"(%s) %s\" is not an array type",
                   valobj_sp->GetTypeName().AsCString(""),
                   var_expr_path_strm.GetData());
-            } else if (
-                static_cast(child_index) >=
-                synthetic
-                    ->GetNumChildren() /* synthetic does not have that many values */) {
+            } else if (static_cast(child_index) >=
+                       synthetic
+                           ->GetNumChildrenIgnoringErrors() /* synthetic does
+                                                                not have that
+                                                                many values */) {
               valobj_sp->GetExpressionPath(var_expr_path_strm);
               error.SetErrorStringWithFormat(
                   "array index %ld is not valid for \"(%s) %s\"", child_index,
@@ -929,10 +930,9 @@ ValueObjectSP StackFrame::GetValueForVariableExpressionPath(
                 "\"(%s) %s\" is not an array type",
                 valobj_sp->GetTypeName().AsCString(""),
                 var_expr_path_strm.GetData());
-          } else if (
-              static_cast(child_index) >=
-              synthetic
-                  ->GetNumChildren() /* synthetic does not have that many values */) {
+          } else if (static_cast(child_index) >=
+                     synthetic->GetNumChildrenIgnoringErrors() /* synthetic
+                                     does not have that many values */) {
             valobj_sp->GetExpressionPath(var_expr_path_strm);
             error.SetErrorStringWithFormat(
                 "array index %ld is not valid for \"(%s) %s\"", child_index,
@@ -1397,7 +1397,8 @@ ValueObjectSP GetValueForOffset(StackFrame &frame, ValueObjectSP &parent,
     return parent;
   }
 
-  for (int ci = 0, ce = parent->GetNumChildren(); ci != ce; ++ci) {
+  for (int ci = 0, ce = parent->GetNumChildrenIgnoringErrors(); ci != ce;
+       ++ci) {
     ValueObjectSP child_sp = parent->GetChildAtIndex(ci);
 
     if (!child_sp) {
-- 
GitLab


From addda68f08191ecb91f22f48570f3defbd364b1d Mon Sep 17 00:00:00 2001
From: Changpeng Fang 
Date: Fri, 8 Mar 2024 10:53:28 -0800
Subject: [PATCH 0066/1674] AMDGPU: Rename HasVinterInsts to
 HasVINTERPEncoding, NFC (#84535)

---
 llvm/lib/Target/AMDGPU/AMDGPU.td              | 2 +-
 llvm/lib/Target/AMDGPU/GCNSubtarget.h         | 2 +-
 llvm/lib/Target/AMDGPU/VINTERPInstructions.td | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.td b/llvm/lib/Target/AMDGPU/AMDGPU.td
index 5d905f5cadc0..7183148e1310 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.td
@@ -1884,7 +1884,7 @@ def NotLDSRequiresM0Init : Predicate<"!Subtarget->ldsRequiresM0Init()">;
 def HasExportInsts : Predicate<"Subtarget->hasExportInsts()">,
   AssemblerPredicate<(all_of (not FeatureGFX90AInsts))>;
 
-def HasInterpInsts : Predicate<"Subtarget->hasInterpInsts()">,
+def HasVINTERPEncoding : Predicate<"Subtarget->hasVINTERPEncoding()">,
   AssemblerPredicate<(all_of FeatureGFX11Insts)>;
 
 def HasDSAddTid : Predicate<"Subtarget->getGeneration() >= AMDGPUSubtarget::GFX9">,
diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.h b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
index afe3af07dc37..ca51da659c33 100644
--- a/llvm/lib/Target/AMDGPU/GCNSubtarget.h
+++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
@@ -652,7 +652,7 @@ public:
     return !hasGFX940Insts();
   }
 
-  bool hasInterpInsts() const {
+  bool hasVINTERPEncoding() const {
     return GFX11Insts;
   }
 
diff --git a/llvm/lib/Target/AMDGPU/VINTERPInstructions.td b/llvm/lib/Target/AMDGPU/VINTERPInstructions.td
index 0303d1e23a0a..77063e2b70f6 100644
--- a/llvm/lib/Target/AMDGPU/VINTERPInstructions.td
+++ b/llvm/lib/Target/AMDGPU/VINTERPInstructions.td
@@ -105,7 +105,7 @@ class VOP3_VINTERP_F16  ArgVT> : VOPProfile {
 // VINTERP Pseudo Instructions
 //===----------------------------------------------------------------------===//
 
-let SubtargetPredicate = HasInterpInsts in {
+let SubtargetPredicate = HasVINTERPEncoding in {
 
 let Uses = [M0, EXEC, MODE] in {
 def V_INTERP_P10_F32_inreg : VINTERP_Pseudo <"v_interp_p10_f32", VOP3_VINTERP_F32>;
@@ -123,7 +123,7 @@ def V_INTERP_P2_RTZ_F16_F32_inreg :
   VINTERP_Pseudo <"v_interp_p2_rtz_f16_f32", VOP3_VINTERP_F16<[f16, f32, f32, f32]>>;
 } // Uses = [M0, EXEC]
 
-} // SubtargetPredicate = HasInterpInsts.
+} // SubtargetPredicate = HasVINTERPEncoding.
 
 class VInterpF32Pat  : GCNPat <
    (f32 (op
-- 
GitLab


From 2bd369b48dbf0bc3128becb7ef8f8a1b82514b87 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <5361294+alexey-bataev@users.noreply.github.com>
Date: Fri, 8 Mar 2024 13:57:02 -0500
Subject: [PATCH 0067/1674] [SLP]Improve minbitwidth analysis.

This improves overall analysis for minbitwidth in SLP. It allows to
analyze the trees with store/insertelement root nodes. Also, instead of
using single minbitwidth, detected from the very first analysis stage,
it tries to detect the best one for each trunc/ext subtree in the graph
and use it for the subtree.
Results in better code and less vector register pressure.

Metric: size..text

Program                                                                                                                                                size..text
                                                                                                                                                       results     results0    diff
                                                                      test-suite :: SingleSource/Benchmarks/Adobe-C++/simple_types_loop_invariant.test    92549.00    92609.00  0.1%
                                                                                  test-suite :: External/SPEC/CINT2017speed/625.x264_s/625.x264_s.test   663381.00   663493.00  0.0%
                                                                                   test-suite :: External/SPEC/CINT2017rate/525.x264_r/525.x264_r.test   663381.00   663493.00  0.0%
                                                                                               test-suite :: MultiSource/Benchmarks/Bullet/bullet.test   307182.00   307214.00  0.0%
                                                                             test-suite :: External/SPEC/CFP2017speed/638.imagick_s/638.imagick_s.test  1394420.00  1394484.00  0.0%
                                                                              test-suite :: External/SPEC/CFP2017rate/538.imagick_r/538.imagick_r.test  1394420.00  1394484.00  0.0%
                                                                                test-suite :: External/SPEC/CFP2017rate/510.parest_r/510.parest_r.test  2040257.00  2040273.00  0.0%

                                                                              test-suite :: External/SPEC/CFP2017rate/526.blender_r/526.blender_r.test 12396098.00 12395858.00 -0.0%
                                                                                         test-suite :: External/SPEC/CINT2006/445.gobmk/445.gobmk.test   909944.00   909768.00 -0.0%

SingleSource/Benchmarks/Adobe-C++/simple_types_loop_invariant - 4 scalar
instructions remain scalar (good).
Spec2017/x264 - the whole function idct4x4dc is vectorized using <16
x i16> instead of <16 x i32>, also zext/trunc are removed. In other
places last vector zext/sext removed and replaced by
extractelement + scalar zext/sext pair.
MultiSource/Benchmarks/Bullet/bullet - reduce or <4 x i32> replaced by
reduce or <4 x i8>
Spec2017/imagick - Removed extra zext from 2 packs of the operations.
Spec2017/parest - Removed extra zext, replaced by extractelement+scalar
zext
Spec2017/blender - the whole bunch of vector zext/sext replaced by
extractelement+scalar zext/sext, some extra code vectorized in smaller
types.
Spec2006/gobmk - fixed cost estimation, some small code remains scalar.

Original Pull Request: https://github.com/llvm/llvm-project/pull/84334

The patch has the same functionality (no test changes, no changes in
benchmarks) as the original patch, just has some compile time
improvements + fixes for xxhash unittest, discovered earlier in the
previous version of the patch.

Reviewers:

Pull Request: https://github.com/llvm/llvm-project/pull/84536
---
 .../Transforms/Vectorize/SLPVectorizer.cpp    | 676 +++++++++++++-----
 .../SLPVectorizer/AArch64/ext-trunc.ll        |   9 +-
 .../SLPVectorizer/AArch64/getelementptr2.ll   |   4 +-
 .../SLPVectorizer/AArch64/reduce-add-i64.ll   |  20 +-
 .../SLPVectorizer/RISCV/reductions.ll         |   7 +-
 .../Transforms/SLPVectorizer/X86/PR35777.ll   |   9 +-
 .../X86/int-bitcast-minbitwidth.ll            |   2 +-
 ...minbitwidth-multiuse-with-insertelement.ll |  17 +-
 .../X86/minbitwidth-transformed-operand.ll    |  21 +-
 .../SLPVectorizer/X86/minimum-sizes.ll        |  43 +-
 .../SLPVectorizer/X86/phi-undef-input.ll      |  24 +-
 .../Transforms/SLPVectorizer/X86/resched.ll   |  32 +-
 .../X86/reused-reductions-with-minbitwidth.ll |  10 +-
 .../X86/store-insertelement-minbitwidth.ll    |  22 +-
 .../SLPVectorizer/alt-cmp-vectorize.ll        |   4 +-
 15 files changed, 594 insertions(+), 306 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 36dc9094538a..a5c34bfbf9b4 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -1085,6 +1085,9 @@ public:
       BS->clear();
     }
     MinBWs.clear();
+    ReductionBitWidth = 0;
+    CastMaxMinBWSizes.reset();
+    TruncNodes.clear();
     InstrElementSize.clear();
     UserIgnoreList = nullptr;
     PostponedGathers.clear();
@@ -2287,6 +2290,7 @@ public:
   void clearReductionData() {
     AnalyzedReductionsRoots.clear();
     AnalyzedReductionVals.clear();
+    AnalyzedMinBWVals.clear();
   }
   /// Checks if the given value is gathered in one of the nodes.
   bool isAnyGathered(const SmallDenseSet &Vals) const {
@@ -2307,9 +2311,11 @@ private:
   /// constant and to be demoted. Required to correctly identify constant nodes
   /// to be demoted.
   bool collectValuesToDemote(
-      Value *V, SmallVectorImpl &ToDemote,
+      Value *V, bool IsProfitableToDemoteRoot, unsigned &BitWidth,
+      SmallVectorImpl &ToDemote,
       DenseMap> &DemotedConsts,
-      SmallVectorImpl &Roots, DenseSet &Visited) const;
+      DenseSet &Visited, unsigned &MaxDepthLevel,
+      bool &IsProfitableToDemote) const;
 
   /// Check if the operands on the edges \p Edges of the \p UserTE allows
   /// reordering (i.e. the operands can be reordered because they have only one
@@ -2375,6 +2381,10 @@ private:
   /// \ returns the graph entry for the \p Idx operand of the \p E entry.
   const TreeEntry *getOperandEntry(const TreeEntry *E, unsigned Idx) const;
 
+  /// \returns Cast context for the given graph node.
+  TargetTransformInfo::CastContextHint
+  getCastContextHint(const TreeEntry &TE) const;
+
   /// \returns the cost of the vectorizable entry.
   InstructionCost getEntryCost(const TreeEntry *E,
                                ArrayRef VectorizedVals,
@@ -2925,11 +2935,18 @@ private:
       }
       assert(!BundleMember && "Bundle and VL out of sync");
     } else {
-      MustGather.insert(VL.begin(), VL.end());
       // Build a map for gathered scalars to the nodes where they are used.
+      bool AllConstsOrCasts = true;
       for (Value *V : VL)
-        if (!isConstant(V))
+        if (!isConstant(V)) {
+          auto *I = dyn_cast(V);
+          AllConstsOrCasts &= I && I->getType()->isIntegerTy();
           ValueToGatherNodes.try_emplace(V).first->getSecond().insert(Last);
+        }
+      if (AllConstsOrCasts)
+        CastMaxMinBWSizes =
+            std::make_pair(std::numeric_limits::max(), 1);
+      MustGather.insert(VL.begin(), VL.end());
     }
 
     if (UserTreeIdx.UserTE)
@@ -3054,6 +3071,10 @@ private:
   /// Set of hashes for the list of reduction values already being analyzed.
   DenseSet AnalyzedReductionVals;
 
+  /// Values, already been analyzed for mininmal bitwidth and found to be
+  /// non-profitable.
+  DenseSet AnalyzedMinBWVals;
+
   /// A list of values that need to extracted out of the tree.
   /// This list holds pairs of (Internal Scalar : External User). External User
   /// can be nullptr, it means that this Internal Scalar will be used later,
@@ -3629,6 +3650,18 @@ private:
   /// value must be signed-extended, rather than zero-extended, back to its
   /// original width.
   DenseMap> MinBWs;
+
+  /// Final size of the reduced vector, if the current graph represents the
+  /// input for the reduction and it was possible to narrow the size of the
+  /// reduction.
+  unsigned ReductionBitWidth = 0;
+
+  /// If the tree contains any zext/sext/trunc nodes, contains max-min pair of
+  /// type sizes, used in the tree.
+  std::optional> CastMaxMinBWSizes;
+
+  /// Indices of the vectorized trunc nodes.
+  DenseSet TruncNodes;
 };
 
 } // end namespace slpvectorizer
@@ -6539,8 +6572,29 @@ void BoUpSLP::buildTree_rec(ArrayRef VL, unsigned Depth,
     case Instruction::Trunc:
     case Instruction::FPTrunc:
     case Instruction::BitCast: {
+      auto [PrevMaxBW, PrevMinBW] = CastMaxMinBWSizes.value_or(
+          std::make_pair(std::numeric_limits::min(),
+                         std::numeric_limits::max()));
+      if (ShuffleOrOp == Instruction::ZExt ||
+          ShuffleOrOp == Instruction::SExt) {
+        CastMaxMinBWSizes = std::make_pair(
+            std::max(DL->getTypeSizeInBits(VL0->getType()),
+                               PrevMaxBW),
+            std::min(
+                DL->getTypeSizeInBits(VL0->getOperand(0)->getType()),
+                PrevMinBW));
+      } else if (ShuffleOrOp == Instruction::Trunc) {
+        CastMaxMinBWSizes = std::make_pair(
+            std::max(
+                DL->getTypeSizeInBits(VL0->getOperand(0)->getType()),
+                PrevMaxBW),
+            std::min(DL->getTypeSizeInBits(VL0->getType()),
+                               PrevMinBW));
+        TruncNodes.insert(VectorizableTree.size());
+      }
       TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
                                    ReuseShuffleIndicies);
+
       LLVM_DEBUG(dbgs() << "SLP: added a vector of casts.\n");
 
       TE->setOperandsInOrder();
@@ -8362,6 +8416,22 @@ const BoUpSLP::TreeEntry *BoUpSLP::getOperandEntry(const TreeEntry *E,
   return It->get();
 }
 
+TTI::CastContextHint BoUpSLP::getCastContextHint(const TreeEntry &TE) const {
+  if (TE.State == TreeEntry::ScatterVectorize ||
+      TE.State == TreeEntry::StridedVectorize)
+    return TTI::CastContextHint::GatherScatter;
+  if (TE.State == TreeEntry::Vectorize && TE.getOpcode() == Instruction::Load &&
+      !TE.isAltShuffle()) {
+    if (TE.ReorderIndices.empty())
+      return TTI::CastContextHint::Normal;
+    SmallVector Mask;
+    inversePermutation(TE.ReorderIndices, Mask);
+    if (ShuffleVectorInst::isReverseMask(Mask, Mask.size()))
+      return TTI::CastContextHint::Reversed;
+  }
+  return TTI::CastContextHint::None;
+}
+
 InstructionCost
 BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals,
                       SmallPtrSetImpl &CheckedExtracts) {
@@ -8384,6 +8454,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals,
   // If we have computed a smaller type for the expression, update VecTy so
   // that the costs will be accurate.
   auto It = MinBWs.find(E);
+  Type *OrigScalarTy = ScalarTy;
   if (It != MinBWs.end()) {
     ScalarTy = IntegerType::get(F->getContext(), It->second.first);
     VecTy = FixedVectorType::get(ScalarTy, VL.size());
@@ -8441,24 +8512,11 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals,
     UsedScalars.set(I);
   }
   auto GetCastContextHint = [&](Value *V) {
-    if (const TreeEntry *OpTE = getTreeEntry(V)) {
-      if (OpTE->State == TreeEntry::ScatterVectorize ||
-          OpTE->State == TreeEntry::StridedVectorize)
-        return TTI::CastContextHint::GatherScatter;
-      if (OpTE->State == TreeEntry::Vectorize &&
-          OpTE->getOpcode() == Instruction::Load && !OpTE->isAltShuffle()) {
-        if (OpTE->ReorderIndices.empty())
-          return TTI::CastContextHint::Normal;
-        SmallVector Mask;
-        inversePermutation(OpTE->ReorderIndices, Mask);
-        if (ShuffleVectorInst::isReverseMask(Mask, Mask.size()))
-          return TTI::CastContextHint::Reversed;
-      }
-    } else {
-      InstructionsState SrcState = getSameOpcode(E->getOperand(0), *TLI);
-      if (SrcState.getOpcode() == Instruction::Load && !SrcState.isAltShuffle())
-        return TTI::CastContextHint::GatherScatter;
-    }
+    if (const TreeEntry *OpTE = getTreeEntry(V))
+      return getCastContextHint(*OpTE);
+    InstructionsState SrcState = getSameOpcode(E->getOperand(0), *TLI);
+    if (SrcState.getOpcode() == Instruction::Load && !SrcState.isAltShuffle())
+      return TTI::CastContextHint::GatherScatter;
     return TTI::CastContextHint::None;
   };
   auto GetCostDiff =
@@ -8507,8 +8565,6 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals,
               TTI::CastContextHint CCH = GetCastContextHint(VL0);
               VecCost += TTI->getCastInstrCost(VecOpcode, UserVecTy, VecTy, CCH,
                                                CostKind);
-              ScalarCost += Sz * TTI->getCastInstrCost(VecOpcode, UserScalarTy,
-                                                       ScalarTy, CCH, CostKind);
             }
           }
         }
@@ -8525,7 +8581,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals,
     InstructionCost ScalarCost = 0;
     InstructionCost VecCost = 0;
     std::tie(ScalarCost, VecCost) = getGEPCosts(
-        *TTI, Ptrs, BasePtr, E->getOpcode(), CostKind, ScalarTy, VecTy);
+        *TTI, Ptrs, BasePtr, E->getOpcode(), CostKind, OrigScalarTy, VecTy);
     LLVM_DEBUG(dumpTreeCosts(E, 0, VecCost, ScalarCost,
                              "Calculated GEPs cost for Tree"));
 
@@ -8572,7 +8628,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals,
           NumElts = ATy->getNumElements();
         else
           NumElts = AggregateTy->getStructNumElements();
-        SrcVecTy = FixedVectorType::get(ScalarTy, NumElts);
+        SrcVecTy = FixedVectorType::get(OrigScalarTy, NumElts);
       }
       if (I->hasOneUse()) {
         Instruction *Ext = I->user_back();
@@ -8740,13 +8796,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals,
       }
     }
     auto GetScalarCost = [&](unsigned Idx) -> InstructionCost {
-      // Do not count cost here if minimum bitwidth is in effect and it is just
-      // a bitcast (here it is just a noop).
-      if (VecOpcode != Opcode && VecOpcode == Instruction::BitCast)
-        return TTI::TCC_Free;
-      auto *VI = VL0->getOpcode() == Opcode
-                     ? cast(UniqueValues[Idx])
-                     : nullptr;
+      auto *VI = cast(UniqueValues[Idx]);
       return TTI->getCastInstrCost(Opcode, VL0->getType(),
                                    VL0->getOperand(0)->getType(),
                                    TTI::getCastContextHint(VI), CostKind, VI);
@@ -8789,7 +8839,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals,
                                        ? CmpInst::BAD_FCMP_PREDICATE
                                        : CmpInst::BAD_ICMP_PREDICATE;
 
-      return TTI->getCmpSelInstrCost(E->getOpcode(), ScalarTy,
+      return TTI->getCmpSelInstrCost(E->getOpcode(), OrigScalarTy,
                                      Builder.getInt1Ty(), CurrentPred, CostKind,
                                      VI);
     };
@@ -8844,7 +8894,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals,
       TTI::OperandValueInfo Op2Info =
           TTI::getOperandInfo(VI->getOperand(OpIdx));
       SmallVector Operands(VI->operand_values());
-      return TTI->getArithmeticInstrCost(ShuffleOrOp, ScalarTy, CostKind,
+      return TTI->getArithmeticInstrCost(ShuffleOrOp, OrigScalarTy, CostKind,
                                          Op1Info, Op2Info, Operands, VI);
     };
     auto GetVectorCost = [=](InstructionCost CommonCost) {
@@ -8863,9 +8913,9 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals,
   case Instruction::Load: {
     auto GetScalarCost = [&](unsigned Idx) {
       auto *VI = cast(UniqueValues[Idx]);
-      return TTI->getMemoryOpCost(Instruction::Load, ScalarTy, VI->getAlign(),
-                                  VI->getPointerAddressSpace(), CostKind,
-                                  TTI::OperandValueInfo(), VI);
+      return TTI->getMemoryOpCost(Instruction::Load, OrigScalarTy,
+                                  VI->getAlign(), VI->getPointerAddressSpace(),
+                                  CostKind, TTI::OperandValueInfo(), VI);
     };
     auto *LI0 = cast(VL0);
     auto GetVectorCost = [&](InstructionCost CommonCost) {
@@ -8908,9 +8958,9 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals,
     auto GetScalarCost = [=](unsigned Idx) {
       auto *VI = cast(VL[Idx]);
       TTI::OperandValueInfo OpInfo = TTI::getOperandInfo(VI->getValueOperand());
-      return TTI->getMemoryOpCost(Instruction::Store, ScalarTy, VI->getAlign(),
-                                  VI->getPointerAddressSpace(), CostKind,
-                                  OpInfo, VI);
+      return TTI->getMemoryOpCost(Instruction::Store, OrigScalarTy,
+                                  VI->getAlign(), VI->getPointerAddressSpace(),
+                                  CostKind, OpInfo, VI);
     };
     auto *BaseSI =
         cast(IsReorder ? VL[E->ReorderIndices.front()] : VL0);
@@ -9772,6 +9822,44 @@ InstructionCost BoUpSLP::getTreeCost(ArrayRef VectorizedVals) {
     Cost -= InsertCost;
   }
 
+  // Add the cost for reduced value resize (if required).
+  if (ReductionBitWidth != 0) {
+    assert(UserIgnoreList && "Expected reduction tree.");
+    const TreeEntry &E = *VectorizableTree.front().get();
+    auto It = MinBWs.find(&E);
+    if (It != MinBWs.end() && It->second.first != ReductionBitWidth) {
+      unsigned SrcSize = It->second.first;
+      unsigned DstSize = ReductionBitWidth;
+      unsigned Opcode = Instruction::Trunc;
+      if (SrcSize < DstSize)
+        Opcode = It->second.second ? Instruction::SExt : Instruction::ZExt;
+      auto *SrcVecTy =
+          FixedVectorType::get(Builder.getIntNTy(SrcSize), E.getVectorFactor());
+      auto *DstVecTy =
+          FixedVectorType::get(Builder.getIntNTy(DstSize), E.getVectorFactor());
+      TTI::CastContextHint CCH = getCastContextHint(E);
+      InstructionCost CastCost;
+      switch (E.getOpcode()) {
+      case Instruction::SExt:
+      case Instruction::ZExt:
+      case Instruction::Trunc: {
+        const TreeEntry *OpTE = getOperandEntry(&E, 0);
+        CCH = getCastContextHint(*OpTE);
+        break;
+      }
+      default:
+        break;
+      }
+      CastCost += TTI->getCastInstrCost(Opcode, DstVecTy, SrcVecTy, CCH,
+                                        TTI::TCK_RecipThroughput);
+      Cost += CastCost;
+      LLVM_DEBUG(dbgs() << "SLP: Adding cost " << CastCost
+                        << " for final resize for reduction from " << SrcVecTy
+                        << " to " << DstVecTy << "\n";
+                 dbgs() << "SLP: Current total cost = " << Cost << "\n");
+    }
+  }
+
 #ifndef NDEBUG
   SmallString<256> Str;
   {
@@ -10042,7 +10130,7 @@ BoUpSLP::isGatherShuffledSingleRegisterEntry(
             continue;
           VTE = *It->getSecond().begin();
           // Iterate through all vectorized nodes.
-          auto *MIt = find_if(It->getSecond(), [](const TreeEntry *MTE) {
+          auto *MIt = find_if(It->getSecond(), [&](const TreeEntry *MTE) {
             return MTE->State == TreeEntry::Vectorize;
           });
           if (MIt == It->getSecond().end())
@@ -10053,11 +10141,6 @@ BoUpSLP::isGatherShuffledSingleRegisterEntry(
       Instruction &LastBundleInst = getLastInstructionInBundle(VTE);
       if (&LastBundleInst == TEInsertPt || !CheckOrdering(&LastBundleInst))
         continue;
-      auto It = MinBWs.find(VTE);
-      // If vectorize node is demoted - do not match.
-      if (It != MinBWs.end() &&
-          It->second.first != DL->getTypeSizeInBits(V->getType()))
-        continue;
       VToTEs.insert(VTE);
     }
     if (VToTEs.empty())
@@ -10105,6 +10188,57 @@ BoUpSLP::isGatherShuffledSingleRegisterEntry(
     return std::nullopt;
   }
 
+  // Filter out entries with larger bitwidth of elements.
+  Type *ScalarTy = VL.front()->getType();
+  unsigned BitWidth = 0;
+  if (ScalarTy->isIntegerTy()) {
+    // Check if the used TEs supposed to be resized and choose the best
+    // candidates.
+    BitWidth = DL->getTypeStoreSize(ScalarTy);
+    if (TEUseEI.UserTE->getOpcode() != Instruction::Select ||
+        TEUseEI.EdgeIdx != 0) {
+      auto UserIt = MinBWs.find(TEUseEI.UserTE);
+      if (UserIt != MinBWs.end())
+        BitWidth = UserIt->second.second;
+    }
+    // Check if the used TEs supposed to be resized and choose the best
+    // candidates.
+    unsigned NodesBitWidth = 0;
+    auto CheckBitwidth = [&](const TreeEntry &TE) {
+      unsigned TEBitWidth = BitWidth;
+      auto UserIt = MinBWs.find(TEUseEI.UserTE);
+      if (UserIt != MinBWs.end())
+        TEBitWidth = UserIt->second.second;
+      if (BitWidth <= TEBitWidth) {
+        if (NodesBitWidth == 0)
+          NodesBitWidth = TEBitWidth;
+        return NodesBitWidth == TEBitWidth;
+      }
+      return false;
+    };
+    for (auto [Idx, Set] : enumerate(UsedTEs)) {
+      DenseSet ForRemoval;
+      for (const TreeEntry *TE : Set) {
+        if (!CheckBitwidth(*TE))
+          ForRemoval.insert(TE);
+      }
+      // All elements must be removed - remove the whole container.
+      if (ForRemoval.size() == Set.size()) {
+        Set.clear();
+        continue;
+      }
+      for (const TreeEntry *TE : ForRemoval)
+        Set.erase(TE);
+    }
+    for (auto *It = UsedTEs.begin(); It != UsedTEs.end();) {
+      if (It->empty()) {
+        UsedTEs.erase(It);
+        continue;
+      }
+      std::advance(It, 1);
+    }
+  }
+
   unsigned VF = 0;
   if (UsedTEs.size() == 1) {
     // Keep the order to avoid non-determinism.
@@ -12929,7 +13063,21 @@ Value *BoUpSLP::vectorizeTree(
   Builder.ClearInsertionPoint();
   InstrElementSize.clear();
 
-  return VectorizableTree[0]->VectorizedValue;
+  const TreeEntry &RootTE = *VectorizableTree.front().get();
+  Value *Vec = RootTE.VectorizedValue;
+  if (auto It = MinBWs.find(&RootTE); ReductionBitWidth != 0 &&
+                                      It != MinBWs.end() &&
+                                      ReductionBitWidth != It->second.first) {
+    IRBuilder<>::InsertPointGuard Guard(Builder);
+    Builder.SetInsertPoint(ReductionRoot->getParent(),
+                           ReductionRoot->getIterator());
+    Vec = Builder.CreateIntCast(
+        Vec,
+        VectorType::get(Builder.getIntNTy(ReductionBitWidth),
+                        cast(Vec->getType())->getElementCount()),
+        It->second.second);
+  }
+  return Vec;
 }
 
 void BoUpSLP::optimizeGatherSequence() {
@@ -13749,23 +13897,48 @@ unsigned BoUpSLP::getVectorElementSize(Value *V) {
 // smaller type with a truncation. We collect the values that will be demoted
 // in ToDemote and additional roots that require investigating in Roots.
 bool BoUpSLP::collectValuesToDemote(
-    Value *V, SmallVectorImpl &ToDemote,
+    Value *V, bool IsProfitableToDemoteRoot, unsigned &BitWidth,
+    SmallVectorImpl &ToDemote,
     DenseMap> &DemotedConsts,
-    SmallVectorImpl &Roots, DenseSet &Visited) const {
+    DenseSet &Visited, unsigned &MaxDepthLevel,
+    bool &IsProfitableToDemote) const {
   // We can always demote constants.
-  if (isa(V))
+  if (isa(V)) {
+    MaxDepthLevel = 1;
     return true;
+  }
+
+  if (DL->getTypeSizeInBits(V->getType()) == BitWidth) {
+    MaxDepthLevel = 1;
+    return true;
+  }
 
   // If the value is not a vectorized instruction in the expression and not used
   // by the insertelement instruction and not used in multiple vector nodes, it
   // cannot be demoted.
+  // TODO: improve handling of gathered values and others.
   auto *I = dyn_cast(V);
-  if (!I || !getTreeEntry(I) || MultiNodeScalars.contains(I) ||
-      !Visited.insert(I).second || all_of(I->users(), [&](User *U) {
+  const TreeEntry *ITE = I ? getTreeEntry(I) : nullptr;
+  if (!ITE || !Visited.insert(I).second || MultiNodeScalars.contains(I) ||
+      all_of(I->users(), [&](User *U) {
         return isa(U) && !getTreeEntry(U);
       }))
     return false;
 
+  auto IsPotentiallyTruncated = [&](Value *V, unsigned &BitWidth) -> bool {
+    if (MultiNodeScalars.contains(V))
+      return false;
+    uint32_t OrigBitWidth = DL->getTypeSizeInBits(V->getType());
+    APInt Mask = APInt::getBitsSetFrom(OrigBitWidth, BitWidth);
+    if (MaskedValueIsZero(V, Mask, SimplifyQuery(*DL)))
+      return true;
+    auto NumSignBits = ComputeNumSignBits(V, *DL, 0, AC, nullptr, DT);
+    unsigned BitWidth1 = OrigBitWidth - NumSignBits;
+    if (!isKnownNonNegative(V, SimplifyQuery(*DL)))
+      ++BitWidth1;
+    BitWidth = std::max(BitWidth, BitWidth1);
+    return BitWidth > 0 && OrigBitWidth >= (BitWidth * 2);
+  };
   unsigned Start = 0;
   unsigned End = I->getNumOperands();
   switch (I->getOpcode()) {
@@ -13773,12 +13946,14 @@ bool BoUpSLP::collectValuesToDemote(
   // We can always demote truncations and extensions. Since truncations can
   // seed additional demotion, we save the truncated value.
   case Instruction::Trunc:
-    Roots.push_back(I->getOperand(0));
+    MaxDepthLevel = 1;
+    if (IsProfitableToDemoteRoot)
+      IsProfitableToDemote = true;
     break;
   case Instruction::ZExt:
   case Instruction::SExt:
-    if (isa(I->getOperand(0)))
-      return false;
+    MaxDepthLevel = 1;
+    IsProfitableToDemote = true;
     break;
 
   // We can demote certain binary operations if we can demote both of their
@@ -13788,23 +13963,36 @@ bool BoUpSLP::collectValuesToDemote(
   case Instruction::Mul:
   case Instruction::And:
   case Instruction::Or:
-  case Instruction::Xor:
-    if (!collectValuesToDemote(I->getOperand(0), ToDemote, DemotedConsts, Roots,
-                               Visited) ||
-        !collectValuesToDemote(I->getOperand(1), ToDemote, DemotedConsts, Roots,
-                               Visited))
+  case Instruction::Xor: {
+    unsigned Level1, Level2;
+    if ((ITE->UserTreeIndices.size() > 1 &&
+         !IsPotentiallyTruncated(I, BitWidth)) ||
+        !collectValuesToDemote(I->getOperand(0), IsProfitableToDemoteRoot,
+                               BitWidth, ToDemote, DemotedConsts, Visited,
+                               Level1, IsProfitableToDemote) ||
+        !collectValuesToDemote(I->getOperand(1), IsProfitableToDemoteRoot,
+                               BitWidth, ToDemote, DemotedConsts, Visited,
+                               Level2, IsProfitableToDemote))
       return false;
+    MaxDepthLevel = std::max(Level1, Level2);
     break;
+  }
 
   // We can demote selects if we can demote their true and false values.
   case Instruction::Select: {
     Start = 1;
+    unsigned Level1, Level2;
     SelectInst *SI = cast(I);
-    if (!collectValuesToDemote(SI->getTrueValue(), ToDemote, DemotedConsts,
-                               Roots, Visited) ||
-        !collectValuesToDemote(SI->getFalseValue(), ToDemote, DemotedConsts,
-                               Roots, Visited))
+    if ((ITE->UserTreeIndices.size() > 1 &&
+         !IsPotentiallyTruncated(I, BitWidth)) ||
+        !collectValuesToDemote(SI->getTrueValue(), IsProfitableToDemoteRoot,
+                               BitWidth, ToDemote, DemotedConsts, Visited,
+                               Level1, IsProfitableToDemote) ||
+        !collectValuesToDemote(SI->getFalseValue(), IsProfitableToDemoteRoot,
+                               BitWidth, ToDemote, DemotedConsts, Visited,
+                               Level2, IsProfitableToDemote))
       return false;
+    MaxDepthLevel = std::max(Level1, Level2);
     break;
   }
 
@@ -13812,172 +14000,270 @@ bool BoUpSLP::collectValuesToDemote(
   // we don't need to worry about cycles since we ensure single use above.
   case Instruction::PHI: {
     PHINode *PN = cast(I);
-    for (Value *IncValue : PN->incoming_values())
-      if (!collectValuesToDemote(IncValue, ToDemote, DemotedConsts, Roots,
-                                 Visited))
+    MaxDepthLevel = 0;
+    if (ITE->UserTreeIndices.size() > 1 && !IsPotentiallyTruncated(I, BitWidth))
+      return false;
+    for (Value *IncValue : PN->incoming_values()) {
+      unsigned Level;
+      if (!collectValuesToDemote(IncValue, IsProfitableToDemoteRoot, BitWidth,
+                                 ToDemote, DemotedConsts, Visited, Level,
+                                 IsProfitableToDemote))
         return false;
+      MaxDepthLevel = std::max(MaxDepthLevel, Level);
+    }
     break;
   }
 
   // Otherwise, conservatively give up.
   default:
-    return false;
+    MaxDepthLevel = 1;
+    return IsProfitableToDemote && IsPotentiallyTruncated(I, BitWidth);
   }
 
+  ++MaxDepthLevel;
   // Gather demoted constant operands.
   for (unsigned Idx : seq(Start, End))
     if (isa(I->getOperand(Idx)))
       DemotedConsts.try_emplace(I).first->getSecond().push_back(Idx);
   // Record the value that we can demote.
   ToDemote.push_back(V);
-  return true;
+  return IsProfitableToDemote;
 }
 
 void BoUpSLP::computeMinimumValueSizes() {
   // We only attempt to truncate integer expressions.
-  auto &TreeRoot = VectorizableTree[0]->Scalars;
-  auto *TreeRootIT = dyn_cast(TreeRoot[0]->getType());
-  if (!TreeRootIT || VectorizableTree.front()->State == TreeEntry::NeedToGather)
+  bool IsStoreOrInsertElt =
+      VectorizableTree.front()->getOpcode() == Instruction::Store ||
+      VectorizableTree.front()->getOpcode() == Instruction::InsertElement;
+  if ((IsStoreOrInsertElt || UserIgnoreList) && TruncNodes.size() <= 1 &&
+      (!CastMaxMinBWSizes || CastMaxMinBWSizes->second == 0 ||
+       CastMaxMinBWSizes->first / CastMaxMinBWSizes->second <= 2))
     return;
 
+  unsigned NodeIdx = 0;
+  if (IsStoreOrInsertElt &&
+      VectorizableTree.front()->State != TreeEntry::NeedToGather)
+    NodeIdx = 1;
+
   // Ensure the roots of the vectorizable tree don't form a cycle.
-  if (!VectorizableTree.front()->UserTreeIndices.empty())
+  if (VectorizableTree[NodeIdx]->State == TreeEntry::NeedToGather ||
+      (NodeIdx == 0 && !VectorizableTree[NodeIdx]->UserTreeIndices.empty()) ||
+      (NodeIdx != 0 && any_of(VectorizableTree[NodeIdx]->UserTreeIndices,
+                              [NodeIdx](const EdgeInfo &EI) {
+                                return EI.UserTE->Idx >
+                                       static_cast(NodeIdx);
+                              })))
     return;
 
-  // Conservatively determine if we can actually truncate the roots of the
-  // expression. Collect the values that can be demoted in ToDemote and
-  // additional roots that require investigating in Roots.
-  SmallVector ToDemote;
-  DenseMap> DemotedConsts;
-  SmallVector Roots;
-  for (auto *Root : TreeRoot) {
-    DenseSet Visited;
-    if (!collectValuesToDemote(Root, ToDemote, DemotedConsts, Roots, Visited))
-      return;
-  }
-
-  // The maximum bit width required to represent all the values that can be
-  // demoted without loss of precision. It would be safe to truncate the roots
-  // of the expression to this width.
-  auto MaxBitWidth = 1u;
-
-  // We first check if all the bits of the roots are demanded. If they're not,
-  // we can truncate the roots to this narrower type.
-  for (auto *Root : TreeRoot) {
-    auto Mask = DB->getDemandedBits(cast(Root));
-    MaxBitWidth = std::max(Mask.getBitWidth() - Mask.countl_zero(),
-                                     MaxBitWidth);
-  }
-
-  // True if the roots can be zero-extended back to their original type, rather
-  // than sign-extended. We know that if the leading bits are not demanded, we
-  // can safely zero-extend. So we initialize IsKnownPositive to True.
-  bool IsKnownPositive = true;
-
-  // If all the bits of the roots are demanded, we can try a little harder to
-  // compute a narrower type. This can happen, for example, if the roots are
-  // getelementptr indices. InstCombine promotes these indices to the pointer
-  // width. Thus, all their bits are technically demanded even though the
-  // address computation might be vectorized in a smaller type.
-  //
-  // We start by looking at each entry that can be demoted. We compute the
-  // maximum bit width required to store the scalar by using ValueTracking to
-  // compute the number of high-order bits we can truncate.
-  if (MaxBitWidth == DL->getTypeSizeInBits(TreeRoot[0]->getType()) &&
-      all_of(TreeRoot, [](Value *V) {
-        return all_of(V->users(),
-                      [](User *U) { return isa(U); });
-      })) {
-    MaxBitWidth = 8u;
+  // The first value node for store/insertelement is sext/zext/trunc? Skip it,
+  // resize to the final type.
+  bool IsProfitableToDemoteRoot = !IsStoreOrInsertElt;
+  if (NodeIdx != 0 &&
+      VectorizableTree[NodeIdx]->State == TreeEntry::Vectorize &&
+      (VectorizableTree[NodeIdx]->getOpcode() == Instruction::ZExt ||
+       VectorizableTree[NodeIdx]->getOpcode() == Instruction::SExt ||
+       VectorizableTree[NodeIdx]->getOpcode() == Instruction::Trunc)) {
+    assert(IsStoreOrInsertElt && "Expected store/insertelement seeded graph.");
+    ++NodeIdx;
+    IsProfitableToDemoteRoot = true;
+  }
+
+  // Analyzed in reduction already and not profitable - exit.
+  if (AnalyzedMinBWVals.contains(VectorizableTree[NodeIdx]->Scalars.front()))
+    return;
 
+  SmallVector ToDemote;
+  DenseMap> DemotedConsts;
+  auto ComputeMaxBitWidth = [&](ArrayRef TreeRoot, unsigned VF,
+                                bool IsTopRoot, bool IsProfitableToDemoteRoot,
+                                unsigned Opcode, unsigned Limit) {
+    ToDemote.clear();
+    auto *TreeRootIT = dyn_cast(TreeRoot[0]->getType());
+    if (!TreeRootIT || !Opcode)
+      return 0u;
+
+    if (AnalyzedMinBWVals.contains(TreeRoot.front()))
+      return 0u;
+
+    unsigned NumParts = TTI->getNumberOfParts(
+        FixedVectorType::get(TreeRoot.front()->getType(), VF));
+
+    // The maximum bit width required to represent all the values that can be
+    // demoted without loss of precision. It would be safe to truncate the roots
+    // of the expression to this width.
+    unsigned MaxBitWidth = 1u;
+
+    // True if the roots can be zero-extended back to their original type,
+    // rather than sign-extended. We know that if the leading bits are not
+    // demanded, we can safely zero-extend. So we initialize IsKnownPositive to
+    // True.
     // Determine if the sign bit of all the roots is known to be zero. If not,
     // IsKnownPositive is set to False.
-    IsKnownPositive = llvm::all_of(TreeRoot, [&](Value *R) {
+    bool IsKnownPositive = all_of(TreeRoot, [&](Value *R) {
       KnownBits Known = computeKnownBits(R, *DL);
       return Known.isNonNegative();
     });
 
-    // Determine the maximum number of bits required to store the scalar
-    // values.
-    for (auto *Scalar : ToDemote) {
-      auto NumSignBits = ComputeNumSignBits(Scalar, *DL, 0, AC, nullptr, DT);
-      auto NumTypeBits = DL->getTypeSizeInBits(Scalar->getType());
-      MaxBitWidth = std::max(NumTypeBits - NumSignBits, MaxBitWidth);
-    }
-
-    // If we can't prove that the sign bit is zero, we must add one to the
-    // maximum bit width to account for the unknown sign bit. This preserves
-    // the existing sign bit so we can safely sign-extend the root back to the
-    // original type. Otherwise, if we know the sign bit is zero, we will
-    // zero-extend the root instead.
-    //
-    // FIXME: This is somewhat suboptimal, as there will be cases where adding
-    //        one to the maximum bit width will yield a larger-than-necessary
-    //        type. In general, we need to add an extra bit only if we can't
-    //        prove that the upper bit of the original type is equal to the
-    //        upper bit of the proposed smaller type. If these two bits are the
-    //        same (either zero or one) we know that sign-extending from the
-    //        smaller type will result in the same value. Here, since we can't
-    //        yet prove this, we are just making the proposed smaller type
-    //        larger to ensure correctness.
-    if (!IsKnownPositive)
-      ++MaxBitWidth;
-  }
-
-  // Round MaxBitWidth up to the next power-of-two.
-  MaxBitWidth = llvm::bit_ceil(MaxBitWidth);
-
-  // If the maximum bit width we compute is less than the with of the roots'
-  // type, we can proceed with the narrowing. Otherwise, do nothing.
-  if (MaxBitWidth >= TreeRootIT->getBitWidth())
-    return;
+    // We first check if all the bits of the roots are demanded. If they're not,
+    // we can truncate the roots to this narrower type.
+    for (auto *Root : TreeRoot) {
+      unsigned NumSignBits = ComputeNumSignBits(Root, *DL, 0, AC, nullptr, DT);
+      TypeSize NumTypeBits = DL->getTypeSizeInBits(Root->getType());
+      unsigned BitWidth1 = NumTypeBits - NumSignBits;
+      // If we can't prove that the sign bit is zero, we must add one to the
+      // maximum bit width to account for the unknown sign bit. This preserves
+      // the existing sign bit so we can safely sign-extend the root back to the
+      // original type. Otherwise, if we know the sign bit is zero, we will
+      // zero-extend the root instead.
+      //
+      // FIXME: This is somewhat suboptimal, as there will be cases where adding
+      //        one to the maximum bit width will yield a larger-than-necessary
+      //        type. In general, we need to add an extra bit only if we can't
+      //        prove that the upper bit of the original type is equal to the
+      //        upper bit of the proposed smaller type. If these two bits are
+      //        the same (either zero or one) we know that sign-extending from
+      //        the smaller type will result in the same value. Here, since we
+      //        can't yet prove this, we are just making the proposed smaller
+      //        type larger to ensure correctness.
+      if (!IsKnownPositive)
+        ++BitWidth1;
+
+      APInt Mask = DB->getDemandedBits(cast(Root));
+      unsigned BitWidth2 = Mask.getBitWidth() - Mask.countl_zero();
+      MaxBitWidth =
+          std::max(std::min(BitWidth1, BitWidth2), MaxBitWidth);
+    }
+
+    if (MaxBitWidth < 8 && MaxBitWidth > 1)
+      MaxBitWidth = 8;
+
+    // If the original type is large, but reduced type does not improve the reg
+    // use - ignore it.
+    if (NumParts > 1 &&
+        NumParts ==
+            TTI->getNumberOfParts(FixedVectorType::get(
+                IntegerType::get(F->getContext(), bit_ceil(MaxBitWidth)), VF)))
+      return 0u;
+
+    bool IsProfitableToDemote = Opcode == Instruction::Trunc ||
+                                Opcode == Instruction::SExt ||
+                                Opcode == Instruction::ZExt || NumParts > 1;
+    // Conservatively determine if we can actually truncate the roots of the
+    // expression. Collect the values that can be demoted in ToDemote and
+    // additional roots that require investigating in Roots.
+    for (auto *Root : TreeRoot) {
+      DenseSet Visited;
+      unsigned MaxDepthLevel = 0;
+      bool NeedToDemote = IsProfitableToDemote;
+
+      if (!collectValuesToDemote(Root, IsProfitableToDemoteRoot, MaxBitWidth,
+                                 ToDemote, DemotedConsts, Visited,
+                                 MaxDepthLevel, NeedToDemote) ||
+          (MaxDepthLevel <= Limit &&
+           !(((Opcode == Instruction::SExt || Opcode == Instruction::ZExt) &&
+              (!IsTopRoot || !(IsStoreOrInsertElt || UserIgnoreList) ||
+               DL->getTypeSizeInBits(Root->getType()) /
+                       DL->getTypeSizeInBits(
+                           cast(Root)->getOperand(0)->getType()) >
+                   2)) ||
+             (Opcode == Instruction::Trunc &&
+              (!IsTopRoot || !(IsStoreOrInsertElt || UserIgnoreList) ||
+               DL->getTypeSizeInBits(
+                   cast(Root)->getOperand(0)->getType()) /
+                       DL->getTypeSizeInBits(Root->getType()) >
+                   2)))))
+        return 0u;
+    }
+    // Round MaxBitWidth up to the next power-of-two.
+    MaxBitWidth = bit_ceil(MaxBitWidth);
+
+    return MaxBitWidth;
+  };
 
   // If we can truncate the root, we must collect additional values that might
   // be demoted as a result. That is, those seeded by truncations we will
   // modify.
-  while (!Roots.empty()) {
-    DenseSet Visited;
-    collectValuesToDemote(Roots.pop_back_val(), ToDemote, DemotedConsts, Roots,
-                          Visited);
-  }
-
-  // Check that all users are marked for demotion.
-  DenseSet Demoted(ToDemote.begin(), ToDemote.end());
-  DenseSet Visited;
-  for (Value *V: ToDemote) {
-    const TreeEntry *TE = getTreeEntry(V);
-    assert(TE && "Expected vectorized scalar.");
-    if (!Visited.insert(TE).second)
-      continue;
-    if (!all_of(TE->UserTreeIndices, [&](const EdgeInfo &EI) {
-          return all_of(EI.UserTE->Scalars,
-                        [&](Value *V) { return Demoted.contains(V); });
-        }))
-      return;
-  }
-  // Finally, map the values we can demote to the maximum bit with we computed.
-  for (auto *Scalar : ToDemote) {
-    auto *TE = getTreeEntry(Scalar);
-    assert(TE && "Expected vectorized scalar.");
-    if (MinBWs.contains(TE))
+  // Add reduction ops sizes, if any.
+  if (UserIgnoreList &&
+      isa(VectorizableTree.front()->Scalars.front()->getType())) {
+    for (Value *V : *UserIgnoreList) {
+      auto NumSignBits = ComputeNumSignBits(V, *DL, 0, AC, nullptr, DT);
+      auto NumTypeBits = DL->getTypeSizeInBits(V->getType());
+      unsigned BitWidth1 = NumTypeBits - NumSignBits;
+      if (!isKnownNonNegative(V, SimplifyQuery(*DL)))
+        ++BitWidth1;
+      auto Mask = DB->getDemandedBits(cast(V));
+      unsigned BitWidth2 = Mask.getBitWidth() - Mask.countl_zero();
+      ReductionBitWidth =
+          std::max(std::min(BitWidth1, BitWidth2), ReductionBitWidth);
+    }
+    if (ReductionBitWidth < 8 && ReductionBitWidth > 1)
+      ReductionBitWidth = 8;
+
+    ReductionBitWidth = bit_ceil(ReductionBitWidth);
+  }
+  bool IsTopRoot = NodeIdx == 0;
+  while (NodeIdx < VectorizableTree.size() &&
+         VectorizableTree[NodeIdx]->State == TreeEntry::Vectorize &&
+         VectorizableTree[NodeIdx]->getOpcode() == Instruction::Trunc)
+    ++NodeIdx;
+  while (NodeIdx < VectorizableTree.size()) {
+    ArrayRef TreeRoot = VectorizableTree[NodeIdx]->Scalars;
+    unsigned Limit = 2;
+    unsigned Opcode = VectorizableTree[NodeIdx]->getOpcode();
+    if (IsTopRoot &&
+        ReductionBitWidth ==
+            DL->getTypeSizeInBits(
+                VectorizableTree.front()->Scalars.front()->getType()))
+      Limit = 3;
+    unsigned MaxBitWidth = ComputeMaxBitWidth(
+        TreeRoot, VectorizableTree[NodeIdx]->getVectorFactor(), IsTopRoot,
+        IsProfitableToDemoteRoot, Opcode, Limit);
+    IsTopRoot = false;
+    IsProfitableToDemoteRoot = true;
+
+    if (TruncNodes.empty()) {
+      NodeIdx = VectorizableTree.size();
+    } else {
+      NodeIdx = *TruncNodes.begin() + 1;
+      TruncNodes.erase(TruncNodes.begin());
+    }
+
+    // If the maximum bit width we compute is less than the with of the roots'
+    // type, we can proceed with the narrowing. Otherwise, do nothing.
+    if (MaxBitWidth == 0 ||
+        MaxBitWidth >=
+            cast(TreeRoot.front()->getType())->getBitWidth()) {
+      if (UserIgnoreList)
+        AnalyzedMinBWVals.insert(TreeRoot.begin(), TreeRoot.end());
       continue;
-    bool IsSigned = any_of(TE->Scalars, [&](Value *R) {
-      KnownBits Known = computeKnownBits(R, *DL);
-      return !Known.isNonNegative();
-    });
-    MinBWs.try_emplace(TE, MaxBitWidth, IsSigned);
-    const auto *I = cast(Scalar);
-    auto DCIt = DemotedConsts.find(I);
-    if (DCIt != DemotedConsts.end()) {
-      for (unsigned Idx : DCIt->getSecond()) {
-        // Check that all instructions operands are demoted.
-        if (all_of(TE->Scalars, [&](Value *V) {
-              auto SIt = DemotedConsts.find(cast(V));
-              return SIt != DemotedConsts.end() &&
-                     is_contained(SIt->getSecond(), Idx);
-            })) {
+    }
+
+    // Finally, map the values we can demote to the maximum bit with we
+    // computed.
+    for (Value *Scalar : ToDemote) {
+      TreeEntry *TE = getTreeEntry(Scalar);
+      assert(TE && "Expected vectorized scalar.");
+      if (MinBWs.contains(TE))
+        continue;
+      bool IsSigned = any_of(TE->Scalars, [&](Value *R) {
+        return !isKnownNonNegative(R, SimplifyQuery(*DL));
+      });
+      MinBWs.try_emplace(TE, MaxBitWidth, IsSigned);
+      const auto *I = cast(Scalar);
+      auto DCIt = DemotedConsts.find(I);
+      if (DCIt != DemotedConsts.end()) {
+        for (unsigned Idx : DCIt->getSecond()) {
+          // Check that all instructions operands are demoted.
           const TreeEntry *CTE = getOperandEntry(TE, Idx);
-          MinBWs.try_emplace(CTE, MaxBitWidth, IsSigned);
+          if (all_of(TE->Scalars,
+                     [&](Value *V) {
+                       auto SIt = DemotedConsts.find(cast(V));
+                       return SIt != DemotedConsts.end() &&
+                              is_contained(SIt->getSecond(), Idx);
+                     }) ||
+              all_of(CTE->Scalars, Constant::classof))
+            MinBWs.try_emplace(CTE, MaxBitWidth, IsSigned);
         }
       }
     }
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/ext-trunc.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/ext-trunc.ll
index cef791633655..5e3fd156666f 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/ext-trunc.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/ext-trunc.ll
@@ -17,12 +17,13 @@ define void @test1(<4 x i16> %a, <4 x i16> %b, ptr %p) {
 ; CHECK-NEXT:    [[GEP0:%.*]] = getelementptr inbounds i64, ptr [[P:%.*]], i64 [[S0]]
 ; CHECK-NEXT:    [[LOAD0:%.*]] = load i64, ptr [[GEP0]], align 4
 ; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <4 x i32> [[SUB0]], <4 x i32> poison, <2 x i32> 
-; CHECK-NEXT:    [[TMP1:%.*]] = sext <2 x i32> [[TMP0]] to <2 x i64>
-; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x i32> [[TMP0]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = sext i32 [[TMP1]] to i64
 ; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds i64, ptr [[P]], i64 [[TMP2]]
 ; CHECK-NEXT:    [[LOAD1:%.*]] = load i64, ptr [[GEP1]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i64> [[TMP1]], i32 1
-; CHECK-NEXT:    [[GEP2:%.*]] = getelementptr inbounds i64, ptr [[P]], i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i32> [[TMP0]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = sext i32 [[TMP3]] to i64
+; CHECK-NEXT:    [[GEP2:%.*]] = getelementptr inbounds i64, ptr [[P]], i64 [[TMP4]]
 ; CHECK-NEXT:    [[LOAD2:%.*]] = load i64, ptr [[GEP2]], align 4
 ; CHECK-NEXT:    [[E3:%.*]] = extractelement <4 x i32> [[SUB0]], i32 3
 ; CHECK-NEXT:    [[S3:%.*]] = sext i32 [[E3]] to i64
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr2.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr2.ll
index 47485e514ec2..1cce52060c47 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr2.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr2.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ;test_i16_extend NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -mtriple=aarch64--linux-gnu -passes=slp-vectorizer,dce,instcombine -slp-threshold=-7 -pass-remarks-output=%t < %s | FileCheck %s
+; RUN: opt -S -mtriple=aarch64--linux-gnu -passes=slp-vectorizer,dce,instcombine -slp-threshold=-5 -pass-remarks-output=%t < %s | FileCheck %s
 ; RUN: cat %t | FileCheck -check-prefix=YAML %s
-; RUN: opt -S -mtriple=aarch64--linux-gnu -passes='slp-vectorizer,dce,instcombine' -slp-threshold=-7 -pass-remarks-output=%t < %s | FileCheck %s
+; RUN: opt -S -mtriple=aarch64--linux-gnu -passes='slp-vectorizer,dce,instcombine' -slp-threshold=-5 -pass-remarks-output=%t < %s | FileCheck %s
 ; RUN: cat %t | FileCheck -check-prefix=YAML %s
 
 
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/reduce-add-i64.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/reduce-add-i64.ll
index d67fdc1cd6aa..a7a7f642ced5 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/reduce-add-i64.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/reduce-add-i64.ll
@@ -28,21 +28,11 @@ entry:
 define i64 @red_zext_ld_4xi64(ptr %ptr) {
 ; CHECK-LABEL: @red_zext_ld_4xi64(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[LD0:%.*]] = load i8, ptr [[PTR:%.*]], align 1
-; CHECK-NEXT:    [[ZEXT:%.*]] = zext i8 [[LD0]] to i64
-; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 1
-; CHECK-NEXT:    [[LD1:%.*]] = load i8, ptr [[GEP]], align 1
-; CHECK-NEXT:    [[ZEXT_1:%.*]] = zext i8 [[LD1]] to i64
-; CHECK-NEXT:    [[ADD_1:%.*]] = add nuw nsw i64 [[ZEXT]], [[ZEXT_1]]
-; CHECK-NEXT:    [[GEP_1:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 2
-; CHECK-NEXT:    [[LD2:%.*]] = load i8, ptr [[GEP_1]], align 1
-; CHECK-NEXT:    [[ZEXT_2:%.*]] = zext i8 [[LD2]] to i64
-; CHECK-NEXT:    [[ADD_2:%.*]] = add nuw nsw i64 [[ADD_1]], [[ZEXT_2]]
-; CHECK-NEXT:    [[GEP_2:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 3
-; CHECK-NEXT:    [[LD3:%.*]] = load i8, ptr [[GEP_2]], align 1
-; CHECK-NEXT:    [[ZEXT_3:%.*]] = zext i8 [[LD3]] to i64
-; CHECK-NEXT:    [[ADD_3:%.*]] = add nuw nsw i64 [[ADD_2]], [[ZEXT_3]]
-; CHECK-NEXT:    ret i64 [[ADD_3]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i8>, ptr [[PTR:%.*]], align 1
+; CHECK-NEXT:    [[TMP1:%.*]] = zext <4 x i8> [[TMP0]] to <4 x i16>
+; CHECK-NEXT:    [[TMP2:%.*]] = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i16 [[TMP2]] to i64
+; CHECK-NEXT:    ret i64 [[TMP3]]
 ;
 entry:
   %ld0 = load i8, ptr %ptr
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/reductions.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/reductions.ll
index 000e7a56df37..500f10659f04 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/reductions.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/reductions.ll
@@ -802,9 +802,10 @@ define i64 @red_zext_ld_4xi64(ptr %ptr) {
 ; CHECK-LABEL: @red_zext_ld_4xi64(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i8>, ptr [[PTR:%.*]], align 1
-; CHECK-NEXT:    [[TMP1:%.*]] = zext <4 x i8> [[TMP0]] to <4 x i64>
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP1]])
-; CHECK-NEXT:    ret i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = zext <4 x i8> [[TMP0]] to <4 x i16>
+; CHECK-NEXT:    [[TMP2:%.*]] = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i16 [[TMP2]] to i64
+; CHECK-NEXT:    ret i64 [[TMP3]]
 ;
 entry:
   %ld0 = load i8, ptr %ptr
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/PR35777.ll b/llvm/test/Transforms/SLPVectorizer/X86/PR35777.ll
index 4565d4928ba4..05511f843a68 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/PR35777.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/PR35777.ll
@@ -15,11 +15,12 @@ define { i64, i64 } @patatino(double %arg) {
 ; CHECK-NEXT:    [[TMP6:%.*]] = load <2 x double>, ptr getelementptr inbounds ([6 x double], ptr @global, i64 0, i64 4), align 16
 ; CHECK-NEXT:    [[TMP7:%.*]] = fadd <2 x double> [[TMP6]], [[TMP5]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = fptosi <2 x double> [[TMP7]] to <2 x i32>
-; CHECK-NEXT:    [[TMP9:%.*]] = sext <2 x i32> [[TMP8]] to <2 x i64>
-; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <2 x i64> [[TMP9]], i32 0
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x i32> [[TMP8]], i32 0
+; CHECK-NEXT:    [[TMP10:%.*]] = sext i32 [[TMP9]] to i64
 ; CHECK-NEXT:    [[T16:%.*]] = insertvalue { i64, i64 } undef, i64 [[TMP10]], 0
-; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <2 x i64> [[TMP9]], i32 1
-; CHECK-NEXT:    [[T17:%.*]] = insertvalue { i64, i64 } [[T16]], i64 [[TMP11]], 1
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <2 x i32> [[TMP8]], i32 1
+; CHECK-NEXT:    [[TMP12:%.*]] = sext i32 [[TMP11]] to i64
+; CHECK-NEXT:    [[T17:%.*]] = insertvalue { i64, i64 } [[T16]], i64 [[TMP12]], 1
 ; CHECK-NEXT:    ret { i64, i64 } [[T17]]
 ;
 bb:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/int-bitcast-minbitwidth.ll b/llvm/test/Transforms/SLPVectorizer/X86/int-bitcast-minbitwidth.ll
index a0af8e36b36c..5ee801607653 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/int-bitcast-minbitwidth.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/int-bitcast-minbitwidth.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
-; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu -slp-threshold=-3 < %s | FileCheck %s
+; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu -slp-threshold=-6 < %s | FileCheck %s
 
 define void @t(i64 %v) {
 ; CHECK-LABEL: define void @t(
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-multiuse-with-insertelement.ll b/llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-multiuse-with-insertelement.ll
index 6e512fcbb739..6051638562b5 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-multiuse-with-insertelement.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-multiuse-with-insertelement.ll
@@ -6,18 +6,17 @@ define void @test(i8 %0) {
 ; CHECK-SAME: i8 [[TMP0:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i8> , i8 [[TMP0]], i32 1
-; CHECK-NEXT:    [[TMP2:%.*]] = sext <2 x i8> [[TMP1]] to <2 x i16>
-; CHECK-NEXT:    [[TMP3:%.*]] = sext <2 x i16> [[TMP2]] to <2 x i32>
-; CHECK-NEXT:    [[TMP4:%.*]] = mul <2 x i16> [[TMP2]], zeroinitializer
-; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i16> [[TMP4]], i32 0
-; CHECK-NEXT:    [[TMP6:%.*]] = zext i16 [[TMP5]] to i32
-; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i16> [[TMP4]], i32 1
-; CHECK-NEXT:    [[TMP8:%.*]] = zext i16 [[TMP7]] to i32
-; CHECK-NEXT:    [[ADD:%.*]] = or i32 [[TMP6]], [[TMP8]]
+; CHECK-NEXT:    [[TMP2:%.*]] = sext <2 x i8> [[TMP1]] to <2 x i32>
+; CHECK-NEXT:    [[TMP3:%.*]] = mul <2 x i8> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i8> [[TMP3]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = zext i8 [[TMP4]] to i32
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i8> [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP7:%.*]] = zext i8 [[TMP6]] to i32
+; CHECK-NEXT:    [[ADD:%.*]] = or i32 [[TMP5]], [[TMP7]]
 ; CHECK-NEXT:    [[SHR:%.*]] = lshr i32 [[ADD]], 1
 ; CHECK-NEXT:    [[CONV9:%.*]] = trunc i32 [[SHR]] to i8
 ; CHECK-NEXT:    store i8 [[CONV9]], ptr null, align 1
-; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> poison, <8 x i32> 
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> poison, <8 x i32> 
 ; CHECK-NEXT:    ret void
 ;
 entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-transformed-operand.ll b/llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-transformed-operand.ll
index 2c834616becc..4acd63078b82 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-transformed-operand.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-transformed-operand.ll
@@ -6,15 +6,20 @@ define void @test(i64 %d.promoted.i) {
 ; CHECK-SAME: i64 [[D_PROMOTED_I:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[AND_1_I:%.*]] = and i64 0, [[D_PROMOTED_I]]
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <8 x i64> , i64 [[AND_1_I]], i32 1
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <8 x i64> [[TMP0]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = mul <8 x i1> [[TMP1]], zeroinitializer
 ; CHECK-NEXT:    [[AND_1_I_1:%.*]] = and i64 0, 0
-; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <16 x i64> , i64 [[AND_1_I_1]], i32 1
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <16 x i64> [[TMP0]], i64 [[AND_1_I]], i32 9
-; CHECK-NEXT:    [[TMP2:%.*]] = trunc <16 x i64> [[TMP1]] to <16 x i1>
-; CHECK-NEXT:    [[TMP3:%.*]] = mul <16 x i1> [[TMP2]], zeroinitializer
-; CHECK-NEXT:    [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP3]])
-; CHECK-NEXT:    [[TMP5:%.*]] = zext i1 [[TMP4]] to i32
-; CHECK-NEXT:    [[TMP6:%.*]] = and i32 [[TMP5]], 0
-; CHECK-NEXT:    store i32 [[TMP6]], ptr null, align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <8 x i64> , i64 [[AND_1_I_1]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc <8 x i64> [[TMP3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = mul <8 x i1> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP5]])
+; CHECK-NEXT:    [[TMP7:%.*]] = zext i1 [[TMP6]] to i32
+; CHECK-NEXT:    [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP2]])
+; CHECK-NEXT:    [[TMP9:%.*]] = zext i1 [[TMP8]] to i32
+; CHECK-NEXT:    [[OP_RDX:%.*]] = or i32 [[TMP7]], [[TMP9]]
+; CHECK-NEXT:    [[TMP10:%.*]] = and i32 [[OP_RDX]], 0
+; CHECK-NEXT:    store i32 [[TMP10]], ptr null, align 4
 ; CHECK-NEXT:    ret void
 ;
 entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/minimum-sizes.ll b/llvm/test/Transforms/SLPVectorizer/X86/minimum-sizes.ll
index 651631de2c35..a316415dcc6b 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/minimum-sizes.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/minimum-sizes.ll
@@ -17,12 +17,15 @@ target triple = "x86_64-unknown-linux-gnu"
 define i8 @PR31243_zext(i8 %v0, i8 %v1, i8 %v2, i8 %v3, ptr %ptr) {
 ; SSE-LABEL: @PR31243_zext(
 ; SSE-NEXT:  entry:
-; SSE-NEXT:    [[TMP0:%.*]] = or i8 [[V0:%.*]], 1
-; SSE-NEXT:    [[TMP1:%.*]] = or i8 [[V1:%.*]], 1
-; SSE-NEXT:    [[TMP2:%.*]] = zext i8 [[TMP0]] to i64
-; SSE-NEXT:    [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i64 [[TMP2]]
-; SSE-NEXT:    [[TMP3:%.*]] = zext i8 [[TMP1]] to i64
-; SSE-NEXT:    [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 [[TMP3]]
+; SSE-NEXT:    [[TMP0:%.*]] = insertelement <2 x i8> poison, i8 [[V0:%.*]], i64 0
+; SSE-NEXT:    [[TMP1:%.*]] = insertelement <2 x i8> [[TMP0]], i8 [[V1:%.*]], i64 1
+; SSE-NEXT:    [[TMP2:%.*]] = or <2 x i8> [[TMP1]], 
+; SSE-NEXT:    [[TMP3:%.*]] = extractelement <2 x i8> [[TMP2]], i64 0
+; SSE-NEXT:    [[TMP4:%.*]] = zext i8 [[TMP3]] to i64
+; SSE-NEXT:    [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i64 [[TMP4]]
+; SSE-NEXT:    [[TMP5:%.*]] = extractelement <2 x i8> [[TMP2]], i64 1
+; SSE-NEXT:    [[TMP6:%.*]] = zext i8 [[TMP5]] to i64
+; SSE-NEXT:    [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 [[TMP6]]
 ; SSE-NEXT:    [[T6:%.*]] = load i8, ptr [[T4]], align 1
 ; SSE-NEXT:    [[T7:%.*]] = load i8, ptr [[T5]], align 1
 ; SSE-NEXT:    [[T8:%.*]] = add i8 [[T6]], [[T7]]
@@ -73,12 +76,15 @@ entry:
 define i8 @PR31243_sext(i8 %v0, i8 %v1, i8 %v2, i8 %v3, ptr %ptr) {
 ; SSE-LABEL: @PR31243_sext(
 ; SSE-NEXT:  entry:
-; SSE-NEXT:    [[TMP0:%.*]] = or i8 [[V0:%.*]], 1
-; SSE-NEXT:    [[TMP1:%.*]] = or i8 [[V1:%.*]], 1
-; SSE-NEXT:    [[TMP2:%.*]] = sext i8 [[TMP0]] to i64
-; SSE-NEXT:    [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i64 [[TMP2]]
-; SSE-NEXT:    [[TMP3:%.*]] = sext i8 [[TMP1]] to i64
-; SSE-NEXT:    [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 [[TMP3]]
+; SSE-NEXT:    [[TMP0:%.*]] = insertelement <2 x i8> poison, i8 [[V0:%.*]], i64 0
+; SSE-NEXT:    [[TMP1:%.*]] = insertelement <2 x i8> [[TMP0]], i8 [[V1:%.*]], i64 1
+; SSE-NEXT:    [[TMP2:%.*]] = or <2 x i8> [[TMP1]], 
+; SSE-NEXT:    [[TMP3:%.*]] = extractelement <2 x i8> [[TMP2]], i64 0
+; SSE-NEXT:    [[TMP4:%.*]] = sext i8 [[TMP3]] to i64
+; SSE-NEXT:    [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i64 [[TMP4]]
+; SSE-NEXT:    [[TMP5:%.*]] = extractelement <2 x i8> [[TMP2]], i64 1
+; SSE-NEXT:    [[TMP6:%.*]] = sext i8 [[TMP5]] to i64
+; SSE-NEXT:    [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 [[TMP6]]
 ; SSE-NEXT:    [[T6:%.*]] = load i8, ptr [[T4]], align 1
 ; SSE-NEXT:    [[T7:%.*]] = load i8, ptr [[T5]], align 1
 ; SSE-NEXT:    [[T8:%.*]] = add i8 [[T6]], [[T7]]
@@ -89,13 +95,12 @@ define i8 @PR31243_sext(i8 %v0, i8 %v1, i8 %v2, i8 %v3, ptr %ptr) {
 ; AVX-NEXT:    [[TMP0:%.*]] = insertelement <2 x i8> poison, i8 [[V0:%.*]], i64 0
 ; AVX-NEXT:    [[TMP1:%.*]] = insertelement <2 x i8> [[TMP0]], i8 [[V1:%.*]], i64 1
 ; AVX-NEXT:    [[TMP2:%.*]] = or <2 x i8> [[TMP1]], 
-; AVX-NEXT:    [[TMP3:%.*]] = sext <2 x i8> [[TMP2]] to <2 x i16>
-; AVX-NEXT:    [[TMP4:%.*]] = extractelement <2 x i16> [[TMP3]], i64 0
-; AVX-NEXT:    [[TMP5:%.*]] = sext i16 [[TMP4]] to i64
-; AVX-NEXT:    [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i64 [[TMP5]]
-; AVX-NEXT:    [[TMP6:%.*]] = extractelement <2 x i16> [[TMP3]], i64 1
-; AVX-NEXT:    [[TMP7:%.*]] = sext i16 [[TMP6]] to i64
-; AVX-NEXT:    [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 [[TMP7]]
+; AVX-NEXT:    [[TMP3:%.*]] = extractelement <2 x i8> [[TMP2]], i64 0
+; AVX-NEXT:    [[TMP4:%.*]] = sext i8 [[TMP3]] to i64
+; AVX-NEXT:    [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i64 [[TMP4]]
+; AVX-NEXT:    [[TMP5:%.*]] = extractelement <2 x i8> [[TMP2]], i64 1
+; AVX-NEXT:    [[TMP6:%.*]] = sext i8 [[TMP5]] to i64
+; AVX-NEXT:    [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 [[TMP6]]
 ; AVX-NEXT:    [[T6:%.*]] = load i8, ptr [[T4]], align 1
 ; AVX-NEXT:    [[T7:%.*]] = load i8, ptr [[T5]], align 1
 ; AVX-NEXT:    [[T8:%.*]] = add i8 [[T6]], [[T7]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/phi-undef-input.ll b/llvm/test/Transforms/SLPVectorizer/X86/phi-undef-input.ll
index 88f75c37846e..3cc32c1fc7b2 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/phi-undef-input.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/phi-undef-input.ll
@@ -15,8 +15,8 @@ define i32 @phi3UndefInput(i1 %cond, i8 %arg0, i8 %arg1, i8 %arg2, i8 %arg3) {
 ; CHECK-NEXT:    br label [[BB3]]
 ; CHECK:       bb3:
 ; CHECK-NEXT:    [[TMP4:%.*]] = phi <4 x i8> [ [[TMP3]], [[BB2]] ], [ , [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[TMP5:%.*]] = zext <4 x i8> [[TMP4]] to <4 x i32>
-; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP5]])
+; CHECK-NEXT:    [[TMP5:%.*]] = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> [[TMP4]])
+; CHECK-NEXT:    [[TMP6:%.*]] = zext i8 [[TMP5]] to i32
 ; CHECK-NEXT:    ret i32 [[TMP6]]
 ;
 entry:
@@ -52,8 +52,8 @@ define i32 @phi2UndefInput(i1 %cond, i8 %arg0, i8 %arg1, i8 %arg2, i8 %arg3) {
 ; CHECK-NEXT:    br label [[BB3]]
 ; CHECK:       bb3:
 ; CHECK-NEXT:    [[TMP4:%.*]] = phi <4 x i8> [ [[TMP3]], [[BB2]] ], [ , [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[TMP5:%.*]] = zext <4 x i8> [[TMP4]] to <4 x i32>
-; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP5]])
+; CHECK-NEXT:    [[TMP5:%.*]] = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> [[TMP4]])
+; CHECK-NEXT:    [[TMP6:%.*]] = zext i8 [[TMP5]] to i32
 ; CHECK-NEXT:    ret i32 [[TMP6]]
 ;
 entry:
@@ -89,8 +89,8 @@ define i32 @phi1UndefInput(i1 %cond, i8 %arg0, i8 %arg1, i8 %arg2, i8 %arg3) {
 ; CHECK-NEXT:    br label [[BB3]]
 ; CHECK:       bb3:
 ; CHECK-NEXT:    [[TMP4:%.*]] = phi <4 x i8> [ [[TMP3]], [[BB2]] ], [ , [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[TMP5:%.*]] = zext <4 x i8> [[TMP4]] to <4 x i32>
-; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP5]])
+; CHECK-NEXT:    [[TMP5:%.*]] = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> [[TMP4]])
+; CHECK-NEXT:    [[TMP6:%.*]] = zext i8 [[TMP5]] to i32
 ; CHECK-NEXT:    ret i32 [[TMP6]]
 ;
 entry:
@@ -127,8 +127,8 @@ define i32 @phi1Undef1PoisonInput(i1 %cond, i8 %arg0, i8 %arg1, i8 %arg2, i8 %ar
 ; CHECK-NEXT:    br label [[BB3]]
 ; CHECK:       bb3:
 ; CHECK-NEXT:    [[TMP4:%.*]] = phi <4 x i8> [ [[TMP3]], [[BB2]] ], [ , [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[TMP5:%.*]] = zext <4 x i8> [[TMP4]] to <4 x i32>
-; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP5]])
+; CHECK-NEXT:    [[TMP5:%.*]] = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> [[TMP4]])
+; CHECK-NEXT:    [[TMP6:%.*]] = zext i8 [[TMP5]] to i32
 ; CHECK-NEXT:    ret i32 [[TMP6]]
 ;
 entry:
@@ -165,8 +165,8 @@ define i32 @phi1Undef2PoisonInputs(i1 %cond, i8 %arg0, i8 %arg1, i8 %arg2, i8 %a
 ; CHECK-NEXT:    br label [[BB3]]
 ; CHECK:       bb3:
 ; CHECK-NEXT:    [[TMP4:%.*]] = phi <4 x i8> [ [[TMP3]], [[BB2]] ], [ , [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[TMP5:%.*]] = zext <4 x i8> [[TMP4]] to <4 x i32>
-; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP5]])
+; CHECK-NEXT:    [[TMP5:%.*]] = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> [[TMP4]])
+; CHECK-NEXT:    [[TMP6:%.*]] = zext i8 [[TMP5]] to i32
 ; CHECK-NEXT:    ret i32 [[TMP6]]
 ;
 entry:
@@ -202,8 +202,8 @@ define i32 @phi1Undef1PoisonGapInput(i1 %cond, i8 %arg0, i8 %arg1, i8 %arg2, i8
 ; CHECK-NEXT:    br label [[BB3]]
 ; CHECK:       bb3:
 ; CHECK-NEXT:    [[TMP4:%.*]] = phi <4 x i8> [ [[TMP3]], [[BB2]] ], [ , [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[TMP5:%.*]] = zext <4 x i8> [[TMP4]] to <4 x i32>
-; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP5]])
+; CHECK-NEXT:    [[TMP5:%.*]] = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> [[TMP4]])
+; CHECK-NEXT:    [[TMP6:%.*]] = zext i8 [[TMP5]] to i32
 ; CHECK-NEXT:    ret i32 [[TMP6]]
 ;
 entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/resched.ll b/llvm/test/Transforms/SLPVectorizer/X86/resched.ll
index 78c6d9516a3d..b7237cbb02bb 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/resched.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/resched.ll
@@ -11,26 +11,26 @@ define fastcc void @_ZN12_GLOBAL__N_127PolynomialMultiplyRecognize9recognizeEv()
 ; CHECK:       if.then22.i:
 ; CHECK-NEXT:    [[SUB_I:%.*]] = add nsw i32 undef, -1
 ; CHECK-NEXT:    [[CONV31_I:%.*]] = and i32 undef, [[SUB_I]]
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i32> poison, i32 [[CONV31_I]], i32 0
-; CHECK-NEXT:    [[SHUFFLE1:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP2:%.*]] = lshr <4 x i32> [[SHUFFLE1]], 
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x i32> poison, i32 [[CONV31_I]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr <4 x i32> [[TMP1]], 
 ; CHECK-NEXT:    [[SHR_4_I_I:%.*]] = lshr i32 [[CONV31_I]], 5
 ; CHECK-NEXT:    [[SHR_5_I_I:%.*]] = lshr i32 [[CONV31_I]], 6
 ; CHECK-NEXT:    [[SHR_6_I_I:%.*]] = lshr i32 [[CONV31_I]], 7
 ; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <8 x i32> poison, i32 [[CONV31_I]], i32 0
-; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <8 x i32> [[TMP3]], <8 x i32> poison, <8 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP4:%.*]] = lshr <8 x i32> [[SHUFFLE]], 
-; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <16 x i32> poison, i32 [[SUB_I]], i32 0
-; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <16 x i32> 
-; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <16 x i32> [[TMP5]], <16 x i32> [[TMP6]], <16 x i32> 
-; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <16 x i32> [[TMP7]], i32 [[SHR_4_I_I]], i32 5
-; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <16 x i32> [[TMP8]], i32 [[SHR_5_I_I]], i32 6
-; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <16 x i32> [[TMP9]], i32 [[SHR_6_I_I]], i32 7
-; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <8 x i32> [[TMP4]], <8 x i32> poison, <16 x i32> 
-; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <16 x i32> [[TMP10]], <16 x i32> [[TMP11]], <16 x i32> 
-; CHECK-NEXT:    [[TMP13:%.*]] = trunc <16 x i32> [[TMP12]] to <16 x i8>
-; CHECK-NEXT:    [[TMP14:%.*]] = and <16 x i8> [[TMP13]], 
-; CHECK-NEXT:    store <16 x i8> [[TMP14]], ptr undef, align 1
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[TMP3]], <8 x i32> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = lshr <8 x i32> [[TMP4]], 
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <16 x i32> poison, i32 [[SUB_I]], i32 0
+; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <16 x i32> 
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <16 x i32> [[TMP6]], <16 x i32> [[TMP7]], <16 x i32> 
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <16 x i32> [[TMP8]], i32 [[SHR_4_I_I]], i32 5
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <16 x i32> [[TMP9]], i32 [[SHR_5_I_I]], i32 6
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <16 x i32> [[TMP10]], i32 [[SHR_6_I_I]], i32 7
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <8 x i32> [[TMP5]], <8 x i32> poison, <16 x i32> 
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <16 x i32> [[TMP11]], <16 x i32> [[TMP12]], <16 x i32> 
+; CHECK-NEXT:    [[TMP14:%.*]] = trunc <16 x i32> [[TMP13]] to <16 x i8>
+; CHECK-NEXT:    [[TMP15:%.*]] = and <16 x i8> [[TMP14]], 
+; CHECK-NEXT:    store <16 x i8> [[TMP15]], ptr undef, align 1
 ; CHECK-NEXT:    unreachable
 ; CHECK:       if.end50.i:
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reused-reductions-with-minbitwidth.ll b/llvm/test/Transforms/SLPVectorizer/X86/reused-reductions-with-minbitwidth.ll
index 5d22b5a4873b..1d1fcec2a7ae 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reused-reductions-with-minbitwidth.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reused-reductions-with-minbitwidth.ll
@@ -7,12 +7,10 @@ define i1 @test(i1 %cmp5.not.31) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x i1> , i1 [[CMP5_NOT_31]], i32 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = select <4 x i1> [[TMP0]], <4 x i32> zeroinitializer, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP2:%.*]] = trunc <4 x i32> [[TMP1]] to <4 x i1>
-; CHECK-NEXT:    [[TMP3:%.*]] = zext <4 x i1> [[TMP2]] to <4 x i32>
-; CHECK-NEXT:    [[TMP4:%.*]] = mul <4 x i32> [[TMP3]], 
-; CHECK-NEXT:    [[TMP5:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP4]])
-; CHECK-NEXT:    [[TMP6:%.*]] = and i32 [[TMP5]], 0
-; CHECK-NEXT:    [[CMP_NOT_I_I:%.*]] = icmp eq i32 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = mul <4 x i32> [[TMP1]], 
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP2]])
+; CHECK-NEXT:    [[TMP4:%.*]] = and i32 [[TMP3]], 0
+; CHECK-NEXT:    [[CMP_NOT_I_I:%.*]] = icmp eq i32 [[TMP4]], 0
 ; CHECK-NEXT:    ret i1 [[CMP_NOT_I_I]]
 ;
 entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/store-insertelement-minbitwidth.ll b/llvm/test/Transforms/SLPVectorizer/X86/store-insertelement-minbitwidth.ll
index c1dd90d0e9a7..2f6868d8dfd6 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/store-insertelement-minbitwidth.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/store-insertelement-minbitwidth.ll
@@ -8,17 +8,18 @@
 ; YAML-NEXT:  Function:        stores
 ; YAML-NEXT:  Args:
 ; YAML-NEXT:    - String:          'Stores SLP vectorized with cost '
-; YAML-NEXT:    - Cost:            '-3'
+; YAML-NEXT:    - Cost:            '-7'
 ; YAML-NEXT:    - String:          ' and with tree size '
 ; YAML-NEXT:    - TreeSize:        '6'
 define void @stores(ptr noalias %in, ptr noalias %inn, ptr noalias %out) {
 ; CHECK-LABEL: @stores(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i8>, ptr [[IN:%.*]], align 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i8>, ptr [[INN:%.*]], align 1
-; CHECK-NEXT:    [[TMP3:%.*]] = zext <4 x i8> [[TMP1]] to <4 x i64>
-; CHECK-NEXT:    [[TMP4:%.*]] = zext <4 x i8> [[TMP2]] to <4 x i64>
-; CHECK-NEXT:    [[TMP5:%.*]] = add <4 x i64> [[TMP3]], [[TMP4]]
-; CHECK-NEXT:    store <4 x i64> [[TMP5]], ptr [[OUT:%.*]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = zext <4 x i8> [[TMP1]] to <4 x i16>
+; CHECK-NEXT:    [[TMP4:%.*]] = zext <4 x i8> [[TMP2]] to <4 x i16>
+; CHECK-NEXT:    [[TMP5:%.*]] = add <4 x i16> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = zext <4 x i16> [[TMP5]] to <4 x i64>
+; CHECK-NEXT:    store <4 x i64> [[TMP6]], ptr [[OUT:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
   %load.1 = load i8, ptr %in, align 1
@@ -63,17 +64,18 @@ define void @stores(ptr noalias %in, ptr noalias %inn, ptr noalias %out) {
 ; YAML-NEXT:  Function:        insertelems
 ; YAML-NEXT:  Args:
 ; YAML-NEXT:    - String:          'SLP vectorized with cost '
-; YAML-NEXT:    - Cost:            '-5'
+; YAML-NEXT:    - Cost:            '-9'
 ; YAML-NEXT:    - String:          ' and with tree size '
 ; YAML-NEXT:    - TreeSize:        '6'
 define <4 x i64> @insertelems(ptr noalias %in, ptr noalias %inn) {
 ; CHECK-LABEL: @insertelems(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i8>, ptr [[IN:%.*]], align 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i8>, ptr [[INN:%.*]], align 1
-; CHECK-NEXT:    [[TMP3:%.*]] = zext <4 x i8> [[TMP1]] to <4 x i64>
-; CHECK-NEXT:    [[TMP4:%.*]] = zext <4 x i8> [[TMP2]] to <4 x i64>
-; CHECK-NEXT:    [[TMP5:%.*]] = add <4 x i64> [[TMP3]], [[TMP4]]
-; CHECK-NEXT:    ret <4 x i64> [[TMP5]]
+; CHECK-NEXT:    [[TMP3:%.*]] = zext <4 x i8> [[TMP1]] to <4 x i16>
+; CHECK-NEXT:    [[TMP4:%.*]] = zext <4 x i8> [[TMP2]] to <4 x i16>
+; CHECK-NEXT:    [[TMP5:%.*]] = add <4 x i16> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = zext <4 x i16> [[TMP5]] to <4 x i64>
+; CHECK-NEXT:    ret <4 x i64> [[TMP6]]
 ;
   %load.1 = load i8, ptr %in, align 1
   %gep.1 = getelementptr inbounds i8, ptr %in, i64 1
diff --git a/llvm/test/Transforms/SLPVectorizer/alt-cmp-vectorize.ll b/llvm/test/Transforms/SLPVectorizer/alt-cmp-vectorize.ll
index 061fbdb45a13..ff6f0bdd3db8 100644
--- a/llvm/test/Transforms/SLPVectorizer/alt-cmp-vectorize.ll
+++ b/llvm/test/Transforms/SLPVectorizer/alt-cmp-vectorize.ll
@@ -10,8 +10,8 @@ define i32 @alt_cmp(i16 %call46) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult <4 x i16> [[TMP0]], [[TMP1]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp ugt <4 x i16> [[TMP0]], [[TMP1]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i1> [[TMP2]], <4 x i1> [[TMP3]], <4 x i32> 
-; CHECK-NEXT:    [[TMP5:%.*]] = zext <4 x i1> [[TMP4]] to <4 x i16>
-; CHECK-NEXT:    [[TMP6:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP5]])
+; CHECK-NEXT:    [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]])
+; CHECK-NEXT:    [[TMP6:%.*]] = zext i1 [[TMP5]] to i16
 ; CHECK-NEXT:    [[OP_RDX:%.*]] = or i16 [[TMP6]], 0
 ; CHECK-NEXT:    [[EXT:%.*]] = zext i16 [[OP_RDX]] to i32
 ; CHECK-NEXT:    ret i32 [[EXT]]
-- 
GitLab


From 9405d5af65853ac548cce2656497195010db1d86 Mon Sep 17 00:00:00 2001
From: Andrew Browne 
Date: Fri, 8 Mar 2024 11:04:04 -0800
Subject: [PATCH 0068/1674] [DFSan] Add missing documentation for
 -dfsan-reaches-function-callbacks. (#84218)

---
 clang/docs/DataFlowSanitizer.rst | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/clang/docs/DataFlowSanitizer.rst b/clang/docs/DataFlowSanitizer.rst
index a18b8ed1948f..5ff50b85dcdc 100644
--- a/clang/docs/DataFlowSanitizer.rst
+++ b/clang/docs/DataFlowSanitizer.rst
@@ -233,6 +233,28 @@ labels of just ``v1`` and ``v2``.
   or, and can be accessed using
   ``dfsan_label dfsan_get_labels_in_signal_conditional();``.
 
+* ``-dfsan-reaches-function-callbacks`` -- An experimental feature that inserts
+  callbacks for data entering a function.
+
+  In addition to this compilation flag, a callback handler must be registered
+  using ``dfsan_set_reaches_function_callback(my_callback);``, where my_callback is
+  a function with a signature matching
+  ``void my_callback(dfsan_label label, dfsan_origin origin, const char *file, unsigned int line, const char *function);``
+  This signature is the same when origin tracking is disabled - in this case
+  the dfsan_origin passed in it will always be 0.
+
+  The callback will be called when a tained value reach stack/registers
+  in the context of a function. Tainted values can reach a function:
+  * via the arguments of the function
+  * via the return value of a call that occurs in the function
+  * via the loaded value of a load that occurs in the function
+
+  The callback will be skipped for conditional expressions inside signal
+  handlers, as this is prone to deadlock. Tainted values reaching functions
+  inside signal handlers will instead be aggregated via bitwise or, and can
+  be accessed using
+  ``dfsan_label dfsan_get_labels_in_signal_reaches_function()``.
+
 * ``-dfsan-track-origins`` -- Controls how to track origins. When its value is
   0, the runtime does not track origins. When its value is 1, the runtime tracks
   origins at memory store operations. When its value is 2, the runtime tracks
-- 
GitLab


From 51207756b0692f325cf75560185cf0336239b3e0 Mon Sep 17 00:00:00 2001
From: Tom Stellard 
Date: Fri, 8 Mar 2024 11:13:34 -0800
Subject: [PATCH 0069/1674] workflows: Fixes for building the release binaries
 (#83694)

Since aa02002491333c42060373bc84f1ff5d2c76b4ce we weren't installing the
correct dependencies, and since 2836d8edbfbcd461b25101ed58f93c862d65903a
we must pass a custom token to github-upload-release.py for verifying
permissions.
---
 .github/workflows/release-binaries.yml | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml
index e487b0f1d4b5..1dba91746dae 100644
--- a/.github/workflows/release-binaries.yml
+++ b/.github/workflows/release-binaries.yml
@@ -48,11 +48,16 @@ jobs:
     - name: Checkout LLVM
       uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
+    - name: Install Dependencies
+      run: |
+        pip install -r ./llvm/utils/git/requirements.txt
+
     - name: Check Permissions
       env:
         GITHUB_TOKEN: ${{ github.token }}
+        USER_TOKEN: ${{ secrets.RELEASE_TASKS_USER_TOKEN }}
       run: |
-        ./llvm/utils/release/./github-upload-release.py --token "$GITHUB_TOKEN" --user ${{ github.actor }} check-permissions
+        ./llvm/utils/release/./github-upload-release.py --token "$GITHUB_TOKEN" --user ${{ github.actor }} --user-token "$USER_TOKEN" check-permissions
 
     - name: Collect Variables
       id: vars
-- 
GitLab


From e460da14ec0d85af657a018605decabdd5221ded Mon Sep 17 00:00:00 2001
From: Jay Foad 
Date: Fri, 8 Mar 2024 19:21:27 +0000
Subject: [PATCH 0070/1674] [AMDGPU] Use get_BUF_ps to default real_name of BUF
 instructions. NFC. (#84524)

---
 llvm/lib/Target/AMDGPU/BUFInstructions.td | 31 +++++++++++++----------
 1 file changed, 18 insertions(+), 13 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/BUFInstructions.td b/llvm/lib/Target/AMDGPU/BUFInstructions.td
index dedf381afca8..d1cbe2840a02 100644
--- a/llvm/lib/Target/AMDGPU/BUFInstructions.td
+++ b/llvm/lib/Target/AMDGPU/BUFInstructions.td
@@ -2484,15 +2484,15 @@ multiclass MUBUF_Real_AllAddr_gfx11_gfx12_Impl2 op, string real_name> :
   MUBUF_Real_AllAddr_gfx11_Impl2,
   MUBUF_Real_AllAddr_gfx12_Impl2;
 
-multiclass MUBUF_Real_AllAddr_gfx11_Impl op, string real_name,
-                                                 bit hasTFE = 1> {
+multiclass MUBUF_Real_AllAddr_gfx11_Impl op, bit hasTFE,
+                                 string real_name = get_BUF_ps.Mnemonic> {
   defm NAME : MUBUF_Real_AllAddr_gfx11_Impl2;
   if hasTFE then
     defm _TFE : MUBUF_Real_AllAddr_gfx11_Impl2;
 }
 
-multiclass MUBUF_Real_AllAddr_gfx11_gfx12_Impl op, string real_name,
-                                                 bit hasTFE = 1> {
+multiclass MUBUF_Real_AllAddr_gfx11_gfx12_Impl op, bit hasTFE,
+                                               string real_name> {
   defm NAME : MUBUF_Real_AllAddr_gfx11_gfx12_Impl2;
   if hasTFE then
     defm _TFE : MUBUF_Real_AllAddr_gfx11_gfx12_Impl2;
@@ -2500,10 +2500,11 @@ multiclass MUBUF_Real_AllAddr_gfx11_gfx12_Impl op, string real_name,
 
 // Non-renamed, non-atomic gfx11/gfx12 mubuf instructions.
 multiclass MUBUF_Real_AllAddr_gfx11 op, bit hasTFE = 1> :
-  MUBUF_Real_AllAddr_gfx11_Impl.Mnemonic, hasTFE>;
+  MUBUF_Real_AllAddr_gfx11_Impl;
 
-multiclass MUBUF_Real_AllAddr_gfx11_gfx12 op, string real_name = !tolower(NAME)> :
-  MUBUF_Real_AllAddr_gfx11_gfx12_Impl {
+multiclass MUBUF_Real_AllAddr_gfx11_gfx12 op,
+                                 string real_name = get_BUF_ps.Mnemonic> :
+  MUBUF_Real_AllAddr_gfx11_gfx12_Impl {
   defvar ps = get_BUF_ps;
   if !ne(ps.Mnemonic, real_name) then
     def : Mnem_gfx11_gfx12;
@@ -2519,7 +2520,7 @@ multiclass MUBUF_Real_Atomic_gfx11_impl op, bit is_return,
 }
 
 multiclass MUBUF_Real_Atomic_gfx12_impl op, bit is_return,
-                                                string real_name> {
+                                 string real_name = get_BUF_ps.Mnemonic> {
   defvar Rtn = !if(is_return, "_RTN", "");
   defm _VBUFFER_BOTHEN#Rtn : VBUFFER_MUBUF_Real_gfx12;
   defm _VBUFFER_IDXEN#Rtn  : VBUFFER_MUBUF_Real_gfx12;
@@ -2533,16 +2534,19 @@ multiclass MUBUF_Real_Atomic_gfx11_gfx12_impl op, bit is_return,
   MUBUF_Real_Atomic_gfx12_impl;
 
 multiclass MUBUF_Real_Atomic_gfx12 op> :
-  MUBUF_Real_Atomic_gfx12_impl.Mnemonic>,
-  MUBUF_Real_Atomic_gfx12_impl.Mnemonic>;
+  MUBUF_Real_Atomic_gfx12_impl,
+  MUBUF_Real_Atomic_gfx12_impl;
 
 multiclass MUBUF_Real_Atomic_gfx11 op, string real_name> :
   MUBUF_Real_Atomic_gfx11_impl,
   MUBUF_Real_Atomic_gfx11_impl {
-  def : Mnem_gfx11_gfx12.Mnemonic, real_name>;
+  defvar ps = get_BUF_ps;
+  def : Mnem_gfx11_gfx12;
 }
 
-multiclass MUBUF_Real_Atomic_gfx11_gfx12 op, string gfx12_name = !tolower(NAME), string gfx11_name = gfx12_name> :
+multiclass MUBUF_Real_Atomic_gfx11_gfx12 op,
+                                  string gfx12_name = get_BUF_ps.Mnemonic,
+                                  string gfx11_name = gfx12_name> :
   MUBUF_Real_Atomic_gfx11_impl,
   MUBUF_Real_Atomic_gfx11_impl,
   MUBUF_Real_Atomic_gfx12_impl,
@@ -2887,7 +2891,8 @@ multiclass MTBUF_Real_AllAddr_gfx11_gfx12_Impl op, string real_name> {
   defm _VBUFFER_OFFSET : VBUFFER_MTBUF_Real_gfx12;
 }
 
-multiclass MTBUF_Real_AllAddr_gfx11_gfx12 op, string real_name = !tolower(NAME)>
+multiclass MTBUF_Real_AllAddr_gfx11_gfx12 op,
+                                   string real_name = get_BUF_ps.Mnemonic>
   : MTBUF_Real_AllAddr_gfx11_gfx12_Impl {
   defvar ps = get_BUF_ps;
   if !ne(ps.Mnemonic, real_name) then
-- 
GitLab


From 074fe3bac634ae9054e2d35be2e1bcf8a19bc256 Mon Sep 17 00:00:00 2001
From: Jay Foad 
Date: Fri, 8 Mar 2024 19:22:08 +0000
Subject: [PATCH 0071/1674] [AMDGPU] Simplify and refactor VBUFFER_Real class
 definitions. NFC. (#84521)

Abstracting out a new base class VBUFFER_Real_gfx12 just highlights that
the only difference between the MUBUF and MTBUF forms is in the handling
of the "format" field.
---
 llvm/lib/Target/AMDGPU/BUFInstructions.td | 31 ++++++++++-------------
 1 file changed, 14 insertions(+), 17 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/BUFInstructions.td b/llvm/lib/Target/AMDGPU/BUFInstructions.td
index d1cbe2840a02..a1bbe170ee29 100644
--- a/llvm/lib/Target/AMDGPU/BUFInstructions.td
+++ b/llvm/lib/Target/AMDGPU/BUFInstructions.td
@@ -2365,7 +2365,7 @@ multiclass MUBUF_Real_gfx7 op> {
 // Base ENC_VBUFFER for GFX12.
 //===----------------------------------------------------------------------===//
 
-class VBUFFER_Real  :
+class VBUFFER_Real  op, BUF_Pseudo ps, string real_name> :
   InstSI , Enc96 {
 
   let isPseudo = 0;
@@ -2389,6 +2389,8 @@ class VBUFFER_Real  :
   let IsAtomicNoRet      = ps.IsAtomicNoRet;
   let VALU               = ps.VALU;
   let LGKM_CNT           = ps.LGKM_CNT;
+  let MUBUF              = ps.MUBUF;
+  let MTBUF              = ps.MTBUF;
 
   bits<24> offset;
   bits<8>  vaddr;
@@ -2405,6 +2407,7 @@ class VBUFFER_Real  :
   let Inst{47-41} = !if(ps.has_srsrc, srsrc, ?);
   let Inst{49-48} = 0b00;
   let Inst{6-0}   = !if(ps.has_soffset, soffset, ?);
+  let Inst{21-14} = op;
   let Inst{22}    = ps.tfe;
   let Inst{62}    = ps.offen;
   let Inst{63}    = ps.idxen;
@@ -2416,33 +2419,27 @@ class VBUFFER_Real  :
   let Inst{31-26} = 0b110001;
 }
 
+class VBUFFER_Real_gfx12 op, BUF_Pseudo ps, string real_name> :
+    VBUFFER_Real,
+    SIMCInstr {
+  let AssemblerPredicate = isGFX12Only;
+  let DecoderNamespace = "GFX12";
+}
+
 multiclass VBUFFER_MUBUF_Real_gfx12 op, string real_name> {
   defvar ps = !cast(NAME);
-  def _gfx12 : VBUFFER_Real,
-               SIMCInstr {
-    let MUBUF = 1;
-
+  def _gfx12 : VBUFFER_Real_gfx12 {
     // Set the last bit of format to 1 to avoid round-trip issues, as some tools
     // print BUF_FMT_INVALID for format 0.
     let Inst{55} = 0b1;
-    let Inst{21-14} = op;
-    let AssemblerPredicate = isGFX12Only;
-    let DecoderNamespace = "GFX12";
   }
 }
 
 multiclass VBUFFER_MTBUF_Real_gfx12 op, string real_name> {
   defvar ps = !cast(NAME);
-  def _gfx12 : VBUFFER_Real, SIMCInstr {
-    let MTBUF = 1;
-
-    bits<7>  format;
-
-    let Inst{17-14} = op;
-    let Inst{21-18} = 0b1000;
+  def _gfx12 : VBUFFER_Real_gfx12<{0b1000, op}, ps, real_name> {
+    bits<7> format;
     let Inst{61-55} = format;
-    let AssemblerPredicate = isGFX12Only;
-    let DecoderNamespace = "GFX12";
   }
 }
 
-- 
GitLab


From 70da9155efb3aeb2c6b37678168cf29f91e80734 Mon Sep 17 00:00:00 2001
From: Mads Marquart 
Date: Fri, 8 Mar 2024 20:34:25 +0100
Subject: [PATCH 0072/1674] Update host CPU detection for Apple M2 and Apple M3
 (#82100)

`CPUFAMILY_ARM_BLIZZARD_AVALANCHE` and `CPUFAMILY_ARM_EVEREST_SAWTOOTH`
are taken from `` in `Kernel.framework`.
---
 llvm/lib/TargetParser/Host.cpp | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/TargetParser/Host.cpp b/llvm/lib/TargetParser/Host.cpp
index ae58ddf03f86..ee4fd0425ca5 100644
--- a/llvm/lib/TargetParser/Host.cpp
+++ b/llvm/lib/TargetParser/Host.cpp
@@ -1466,6 +1466,8 @@ StringRef sys::getHostCPUName() {
 #define CPUFAMILY_ARM_VORTEX_TEMPEST 0x07d34b9f
 #define CPUFAMILY_ARM_LIGHTNING_THUNDER 0x462504d2
 #define CPUFAMILY_ARM_FIRESTORM_ICESTORM 0x1b588bb3
+#define CPUFAMILY_ARM_BLIZZARD_AVALANCHE 0xda33d83d
+#define CPUFAMILY_ARM_EVEREST_SAWTOOTH 0x8765edea
 
 StringRef sys::getHostCPUName() {
   uint32_t Family;
@@ -1491,9 +1493,13 @@ StringRef sys::getHostCPUName() {
     return "apple-a13";
   case CPUFAMILY_ARM_FIRESTORM_ICESTORM:
     return "apple-m1";
+  case CPUFAMILY_ARM_BLIZZARD_AVALANCHE:
+    return "apple-m2";
+  case CPUFAMILY_ARM_EVEREST_SAWTOOTH:
+    return "apple-m3";
   default:
     // Default to the newest CPU we know about.
-    return "apple-m1";
+    return "apple-m3";
   }
 }
 #elif defined(_AIX)
-- 
GitLab


From 087666811452e1b6490d59572337aaa907f93e7c Mon Sep 17 00:00:00 2001
From: Nikolas Klauser 
Date: Fri, 8 Mar 2024 20:43:10 +0100
Subject: [PATCH 0073/1674] [libc++][NFC] Move __format/format_fwd.h to
 __fwd/format.h (#84336)

---
 libcxx/include/CMakeLists.txt                            | 2 +-
 libcxx/include/__format/concepts.h                       | 2 +-
 libcxx/include/__format/format_arg.h                     | 2 +-
 libcxx/include/__format/format_args.h                    | 2 +-
 libcxx/include/__format/format_context.h                 | 2 +-
 libcxx/include/__format/formatter.h                      | 2 +-
 libcxx/include/{__format/format_fwd.h => __fwd/format.h} | 6 +++---
 libcxx/include/format                                    | 2 +-
 libcxx/include/libcxx.imp                                | 2 +-
 libcxx/include/module.modulemap                          | 2 +-
 10 files changed, 12 insertions(+), 12 deletions(-)
 rename libcxx/include/{__format/format_fwd.h => __fwd/format.h} (89%)

diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt
index 3ea3360186dc..e37c4ac4fddd 100644
--- a/libcxx/include/CMakeLists.txt
+++ b/libcxx/include/CMakeLists.txt
@@ -380,7 +380,6 @@ set(files
   __format/format_context.h
   __format/format_error.h
   __format/format_functions.h
-  __format/format_fwd.h
   __format/format_parse_context.h
   __format/format_string.h
   __format/format_to_n_result.h
@@ -430,6 +429,7 @@ set(files
   __fwd/array.h
   __fwd/bit_reference.h
   __fwd/complex.h
+  __fwd/format.h
   __fwd/fstream.h
   __fwd/functional.h
   __fwd/ios.h
diff --git a/libcxx/include/__format/concepts.h b/libcxx/include/__format/concepts.h
index 299c5f40ee35..d7b5a9d16df7 100644
--- a/libcxx/include/__format/concepts.h
+++ b/libcxx/include/__format/concepts.h
@@ -13,8 +13,8 @@
 #include <__concepts/same_as.h>
 #include <__concepts/semiregular.h>
 #include <__config>
-#include <__format/format_fwd.h>
 #include <__format/format_parse_context.h>
+#include <__fwd/format.h>
 #include <__type_traits/is_specialization.h>
 #include <__type_traits/remove_const.h>
 #include <__utility/pair.h>
diff --git a/libcxx/include/__format/format_arg.h b/libcxx/include/__format/format_arg.h
index b786ac3b3620..4924e5fb3253 100644
--- a/libcxx/include/__format/format_arg.h
+++ b/libcxx/include/__format/format_arg.h
@@ -14,9 +14,9 @@
 #include <__concepts/arithmetic.h>
 #include <__config>
 #include <__format/concepts.h>
-#include <__format/format_fwd.h>
 #include <__format/format_parse_context.h>
 #include <__functional/invoke.h>
+#include <__fwd/format.h>
 #include <__memory/addressof.h>
 #include <__type_traits/conditional.h>
 #include <__utility/forward.h>
diff --git a/libcxx/include/__format/format_args.h b/libcxx/include/__format/format_args.h
index 9e0afecc0ae9..79fe51f96c6a 100644
--- a/libcxx/include/__format/format_args.h
+++ b/libcxx/include/__format/format_args.h
@@ -14,7 +14,7 @@
 #include <__config>
 #include <__format/format_arg.h>
 #include <__format/format_arg_store.h>
-#include <__format/format_fwd.h>
+#include <__fwd/format.h>
 #include 
 #include 
 
diff --git a/libcxx/include/__format/format_context.h b/libcxx/include/__format/format_context.h
index 68dcdb49d3aa..d131e942aca6 100644
--- a/libcxx/include/__format/format_context.h
+++ b/libcxx/include/__format/format_context.h
@@ -18,7 +18,7 @@
 #include <__format/format_arg_store.h>
 #include <__format/format_args.h>
 #include <__format/format_error.h>
-#include <__format/format_fwd.h>
+#include <__fwd/format.h>
 #include <__iterator/back_insert_iterator.h>
 #include <__iterator/concepts.h>
 #include <__memory/addressof.h>
diff --git a/libcxx/include/__format/formatter.h b/libcxx/include/__format/formatter.h
index 079befc5bd9c..47e35789b817 100644
--- a/libcxx/include/__format/formatter.h
+++ b/libcxx/include/__format/formatter.h
@@ -12,7 +12,7 @@
 
 #include <__availability>
 #include <__config>
-#include <__format/format_fwd.h>
+#include <__fwd/format.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/__format/format_fwd.h b/libcxx/include/__fwd/format.h
similarity index 89%
rename from libcxx/include/__format/format_fwd.h
rename to libcxx/include/__fwd/format.h
index 120b2fc8d47d..6f5c71243711 100644
--- a/libcxx/include/__format/format_fwd.h
+++ b/libcxx/include/__fwd/format.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___FORMAT_FORMAT_FWD_H
-#define _LIBCPP___FORMAT_FORMAT_FWD_H
+#ifndef _LIBCPP___FWD_FORMAT_H
+#define _LIBCPP___FWD_FORMAT_H
 
 #include <__availability>
 #include <__config>
@@ -36,4 +36,4 @@ struct _LIBCPP_TEMPLATE_VIS formatter;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___FORMAT_FORMAT_FWD_H
+#endif // _LIBCPP___FWD_FORMAT_H
diff --git a/libcxx/include/format b/libcxx/include/format
index b2fe0053b974..c0485c5a1035 100644
--- a/libcxx/include/format
+++ b/libcxx/include/format
@@ -199,7 +199,6 @@ namespace std {
 #include <__format/format_context.h>
 #include <__format/format_error.h>
 #include <__format/format_functions.h>
-#include <__format/format_fwd.h>
 #include <__format/format_parse_context.h>
 #include <__format/format_string.h>
 #include <__format/format_to_n_result.h>
@@ -215,6 +214,7 @@ namespace std {
 #include <__format/range_default_formatter.h>
 #include <__format/range_formatter.h>
 #include <__format/unicode.h>
+#include <__fwd/format.h>
 #include 
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/libcxx.imp b/libcxx/include/libcxx.imp
index cdbb0a63fc0e..e02dc8da6ba1 100644
--- a/libcxx/include/libcxx.imp
+++ b/libcxx/include/libcxx.imp
@@ -374,7 +374,6 @@
   { include: [ "<__format/format_context.h>", "private", "", "public" ] },
   { include: [ "<__format/format_error.h>", "private", "", "public" ] },
   { include: [ "<__format/format_functions.h>", "private", "", "public" ] },
-  { include: [ "<__format/format_fwd.h>", "private", "", "public" ] },
   { include: [ "<__format/format_parse_context.h>", "private", "", "public" ] },
   { include: [ "<__format/format_string.h>", "private", "", "public" ] },
   { include: [ "<__format/format_to_n_result.h>", "private", "", "public" ] },
@@ -425,6 +424,7 @@
   { include: [ "<__fwd/bit_reference.h>", "private", "", "public" ] },
   { include: [ "<__fwd/bit_reference.h>", "private", "", "public" ] },
   { include: [ "<__fwd/complex.h>", "private", "", "public" ] },
+  { include: [ "<__fwd/format.h>", "private", "", "public" ] },
   { include: [ "<__fwd/fstream.h>", "private", "", "public" ] },
   { include: [ "<__fwd/functional.h>", "private", "", "public" ] },
   { include: [ "<__fwd/ios.h>", "private", "", "public" ] },
diff --git a/libcxx/include/module.modulemap b/libcxx/include/module.modulemap
index b247f97c1804..98890e890cdb 100644
--- a/libcxx/include/module.modulemap
+++ b/libcxx/include/module.modulemap
@@ -1315,7 +1315,7 @@ module std_private_format_format_functions                [system] {
   header "__format/format_functions.h"
   export std_string
 }
-module std_private_format_format_fwd                      [system] { header "__format/format_fwd.h" }
+module std_private_format_fwd                             [system] { header "__fwd/format.h" }
 module std_private_format_format_parse_context            [system] { header "__format/format_parse_context.h" }
 module std_private_format_format_string                   [system] { header "__format/format_string.h" }
 module std_private_format_format_to_n_result              [system] {
-- 
GitLab


From 52b3e89ff4517521e40f5e416a40e574321eaee9 Mon Sep 17 00:00:00 2001
From: Fangrui Song 
Date: Fri, 8 Mar 2024 11:45:43 -0800
Subject: [PATCH 0074/1674] [ELF] Simplify GdbIndexSection. NFC

---
 lld/ELF/SyntheticSections.cpp | 27 ++++++++++-----------------
 1 file changed, 10 insertions(+), 17 deletions(-)

diff --git a/lld/ELF/SyntheticSections.cpp b/lld/ELF/SyntheticSections.cpp
index b6bdc350bc0d..66b3e835cabc 100644
--- a/lld/ELF/SyntheticSections.cpp
+++ b/lld/ELF/SyntheticSections.cpp
@@ -2805,21 +2805,16 @@ createSymbols(
     cuIdx += chunks[i].compilationUnits.size();
   }
 
-  // The number of symbols we will handle in this function is of the order
-  // of millions for very large executables, so we use multi-threading to
-  // speed it up.
+  // Collect the compilation unitss for each unique name. Speed it up using
+  // multi-threading as the number of symbols can be in the order of millions.
+  // Shard GdbSymbols by hash's high bits.
   constexpr size_t numShards = 32;
   const size_t concurrency =
       llvm::bit_floor(std::min(config->threadCount, numShards));
-
-  // A sharded map to uniquify symbols by name.
+  const size_t shift = 32 - llvm::countr_zero(numShards);
   auto map =
       std::make_unique[]>(numShards);
-  size_t shift = 32 - llvm::countr_zero(numShards);
-
-  // Instantiate GdbSymbols while uniqufying them by name.
   auto symbols = std::make_unique[]>(numShards);
-
   parallelFor(0, concurrency, [&](size_t threadId) {
     uint32_t i = 0;
     for (ArrayRef entries : nameAttrs) {
@@ -2829,14 +2824,12 @@ createSymbols(
           continue;
 
         uint32_t v = ent.cuIndexAndAttrs + cuIdxs[i];
-        size_t &idx = map[shardId][ent.name];
-        if (idx) {
-          symbols[shardId][idx - 1].cuVector.push_back(v);
-          continue;
-        }
-
-        idx = symbols[shardId].size() + 1;
-        symbols[shardId].push_back({ent.name, {v}, 0, 0});
+        auto [it, inserted] =
+            map[shardId].try_emplace(ent.name, symbols[shardId].size());
+        if (inserted)
+          symbols[shardId].push_back({ent.name, {v}, 0, 0});
+        else
+          symbols[shardId][it->second].cuVector.push_back(v);
       }
       ++i;
     }
-- 
GitLab


From e963d0740e64fc70b4018f39325469d204f6217a Mon Sep 17 00:00:00 2001
From: Shilei Tian 
Date: Fri, 8 Mar 2024 14:49:52 -0500
Subject: [PATCH 0075/1674] [AMDGPU] Replace `isInlinableLiteral16` with
 specific version (#84402)

The current implementation of `isInlinableLiteral16` assumes, a 16-bit
inlinable
literal is either an `i16` or a `fp16`. This is not always true because
of
`bf16`. However, we can't tell `fp16` and `bf16` apart by just looking
at the
value. This patch splits `isInlinableLiteral16` into three versions,
`i16`,
`fp16`, `bf16` respectively, and call the corresponding version.
---
 llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp |  52 ++---
 .../AMDGPU/AsmParser/AMDGPUAsmParser.cpp      |  97 +++++++--
 .../AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp |  29 +--
 .../AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h   |   4 +-
 .../MCTargetDesc/AMDGPUMCCodeEmitter.cpp      |  11 +-
 llvm/lib/Target/AMDGPU/SIISelLowering.cpp     |  28 ++-
 llvm/lib/Target/AMDGPU/SIInstrInfo.cpp        |  25 ++-
 llvm/lib/Target/AMDGPU/SIInstrInfo.h          |   4 +-
 .../Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp    |   8 +-
 llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h |   8 +-
 llvm/lib/Target/AMDGPU/VOP3PInstructions.td   |   1 +
 llvm/test/CodeGen/AMDGPU/immv216.ll           |  42 ++--
 .../test/CodeGen/AMDGPU/inline-constraints.ll |  12 --
 .../CodeGen/AMDGPU/shrink-add-sub-constant.ll | 144 ++++++++-----
 .../test/CodeGen/AMDGPU/wmma-gfx12-w32-imm.ll |   6 +-
 .../test/CodeGen/AMDGPU/wmma-gfx12-w64-imm.ll |   5 +-
 llvm/test/MC/AMDGPU/gfx10_asm_vop1.s          |  16 +-
 llvm/test/MC/AMDGPU/gfx10_asm_vop3.s          | 192 +++++++++---------
 llvm/test/MC/AMDGPU/gfx10_asm_vopc.s          |  96 ++++-----
 llvm/test/MC/AMDGPU/gfx10_asm_vopc_e64.s      | 192 +++++++++---------
 llvm/test/MC/AMDGPU/gfx10_asm_vopcx.s         | 144 ++++++-------
 llvm/test/MC/AMDGPU/gfx11_asm_vop1.s          |  10 +-
 llvm/test/MC/AMDGPU/gfx11_asm_vop3.s          | 108 +++++-----
 .../test/MC/AMDGPU/gfx11_asm_vop3_from_vop1.s |  10 +-
 .../test/MC/AMDGPU/gfx11_asm_vop3_from_vopc.s |  96 ++++-----
 .../MC/AMDGPU/gfx11_asm_vop3_from_vopcx.s     |  48 ++---
 llvm/test/MC/AMDGPU/gfx11_asm_vopc.s          |  48 ++---
 llvm/test/MC/AMDGPU/gfx11_asm_vopcx.s         |  24 +--
 llvm/test/MC/AMDGPU/gfx12_asm_vop1.s          |  10 +-
 llvm/test/MC/AMDGPU/gfx12_asm_vop3.s          | 108 +++++-----
 .../test/MC/AMDGPU/gfx12_asm_vop3_from_vop1.s |  10 +-
 llvm/test/MC/AMDGPU/gfx12_asm_vop3c.s         |  96 ++++-----
 llvm/test/MC/AMDGPU/gfx12_asm_vop3cx.s        |  48 ++---
 llvm/test/MC/AMDGPU/gfx12_asm_vopc.s          |  48 ++---
 llvm/test/MC/AMDGPU/gfx12_asm_vopcx.s         |  24 +--
 llvm/test/MC/AMDGPU/gfx8_asm_vop1.s           |   8 +-
 llvm/test/MC/AMDGPU/gfx8_asm_vop2.s           |  44 ++--
 llvm/test/MC/AMDGPU/gfx8_asm_vopc.s           | 128 ++++++------
 llvm/test/MC/AMDGPU/gfx9-asm-err.s            |  20 +-
 llvm/test/MC/AMDGPU/gfx9_asm_vop1.s           |   8 +-
 llvm/test/MC/AMDGPU/gfx9_asm_vop2.s           |  46 +++--
 llvm/test/MC/AMDGPU/gfx9_asm_vopc.s           | 128 ++++++------
 llvm/test/MC/AMDGPU/gfx9_err_pos.s            |  10 -
 llvm/test/MC/AMDGPU/vop3-gfx10.s              |   4 +-
 llvm/test/MC/AMDGPU/vop3-gfx9.s               |  16 +-
 llvm/test/MC/AMDGPU/vop3.s                    |   4 +-
 llvm/test/MC/AMDGPU/vop_sdwa.s                |   4 +-
 47 files changed, 1180 insertions(+), 1044 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index d1af74e6e0d3..c99e49001466 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -3327,35 +3327,41 @@ bool AMDGPUDAGToDAGISel::SelectWMMAVISrc(SDValue In, SDValue &Src) const {
 
   // 16 bit splat
   SDValue SplatSrc32 = stripBitcast(In);
-  if (auto *SplatSrc32BV = dyn_cast(SplatSrc32)) {
+  if (auto *SplatSrc32BV = dyn_cast(SplatSrc32))
     if (SDValue Splat32 = SplatSrc32BV->getSplatValue()) {
       SDValue SplatSrc16 = stripBitcast(Splat32);
-      if (auto *SplatSrc16BV = dyn_cast(SplatSrc16)) {
+      if (auto *SplatSrc16BV = dyn_cast(SplatSrc16))
         if (SDValue Splat = SplatSrc16BV->getSplatValue()) {
-
-          // f16
-          if (isInlineImmediate(Splat.getNode())) {
-            const ConstantFPSDNode *C = dyn_cast(Splat);
-            int64_t Imm = C->getValueAPF().bitcastToAPInt().getSExtValue();
-            Src = CurDAG->getTargetConstant(Imm, SDLoc(In), MVT::i16);
-            return true;
-          }
-
-          // bf16
-          if (const ConstantSDNode *C = dyn_cast(Splat)) {
-            const SIInstrInfo *TII = Subtarget->getInstrInfo();
-            APInt BF16Value = C->getAPIntValue();
-            APInt F32Value = BF16Value.zext(32).shl(16);
-            if (TII->isInlineConstant(F32Value)) {
-              int64_t Imm = F32Value.getSExtValue();
-              Src = CurDAG->getTargetConstant(Imm, SDLoc(In), MVT::i32);
-              return true;
-            }
+          const SIInstrInfo *TII = Subtarget->getInstrInfo();
+          std::optional RawValue;
+          if (const ConstantFPSDNode *C = dyn_cast(Splat))
+            RawValue = C->getValueAPF().bitcastToAPInt();
+          else if (const ConstantSDNode *C = dyn_cast(Splat))
+            RawValue = C->getAPIntValue();
+
+          if (RawValue.has_value()) {
+            EVT VT = In.getValueType().getScalarType();
+            if (VT.getSimpleVT() == MVT::f16 || VT.getSimpleVT() == MVT::bf16) {
+              APFloat FloatVal(VT.getSimpleVT() == MVT::f16
+                                   ? APFloatBase::IEEEhalf()
+                                   : APFloatBase::BFloat(),
+                               RawValue.value());
+              if (TII->isInlineConstant(FloatVal)) {
+                Src = CurDAG->getTargetConstant(RawValue.value(), SDLoc(In),
+                                                MVT::i16);
+                return true;
+              }
+            } else if (VT.getSimpleVT() == MVT::i16) {
+              if (TII->isInlineConstant(RawValue.value())) {
+                Src = CurDAG->getTargetConstant(RawValue.value(), SDLoc(In),
+                                                MVT::i16);
+                return true;
+              }
+            } else
+              llvm_unreachable("unknown 16-bit type");
           }
         }
-      }
     }
-  }
 
   return false;
 }
diff --git a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
index d5efd4415562..027dd0f2c224 100644
--- a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
+++ b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
@@ -1926,6 +1926,11 @@ static const fltSemantics *getFltSemantics(MVT VT) {
 
 static const fltSemantics *getOpFltSemantics(uint8_t OperandType) {
   switch (OperandType) {
+  // When floating-point immediate is used as operand of type i16, the 32-bit
+   // representation of the constant truncated to the 16 LSBs should be used.
+  case AMDGPU::OPERAND_REG_IMM_INT16:
+  case AMDGPU::OPERAND_REG_INLINE_C_INT16:
+  case AMDGPU::OPERAND_REG_INLINE_AC_INT16:
   case AMDGPU::OPERAND_REG_IMM_INT32:
   case AMDGPU::OPERAND_REG_IMM_FP32:
   case AMDGPU::OPERAND_REG_IMM_FP32_DEFERRED:
@@ -1949,13 +1954,10 @@ static const fltSemantics *getOpFltSemantics(uint8_t OperandType) {
   case AMDGPU::OPERAND_REG_INLINE_C_FP64:
   case AMDGPU::OPERAND_REG_INLINE_AC_FP64:
     return &APFloat::IEEEdouble();
-  case AMDGPU::OPERAND_REG_IMM_INT16:
   case AMDGPU::OPERAND_REG_IMM_FP16:
   case AMDGPU::OPERAND_REG_IMM_FP16_DEFERRED:
-  case AMDGPU::OPERAND_REG_INLINE_C_INT16:
   case AMDGPU::OPERAND_REG_INLINE_C_FP16:
   case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
-  case AMDGPU::OPERAND_REG_INLINE_AC_INT16:
   case AMDGPU::OPERAND_REG_INLINE_AC_FP16:
   case AMDGPU::OPERAND_REG_INLINE_AC_V2FP16:
   case AMDGPU::OPERAND_REG_IMM_V2FP16:
@@ -2001,13 +2003,15 @@ static bool isSafeTruncation(int64_t Val, unsigned Size) {
 }
 
 static bool isInlineableLiteralOp16(int64_t Val, MVT VT, bool HasInv2Pi) {
-  if (VT.getScalarType() == MVT::i16) {
-    // FP immediate values are broken.
-    return isInlinableIntLiteral(Val);
-  }
+  if (VT.getScalarType() == MVT::i16)
+    return isInlinableLiteral32(Val, HasInv2Pi);
+
+  if (VT.getScalarType() == MVT::f16)
+    return AMDGPU::isInlinableLiteralFP16(Val, HasInv2Pi);
 
-  // f16/v2f16 operands work correctly for all values.
-  return AMDGPU::isInlinableLiteral16(Val, HasInv2Pi);
+  assert(VT.getScalarType() == MVT::bf16);
+
+  return AMDGPU::isInlinableLiteralBF16(Val, HasInv2Pi);
 }
 
 bool AMDGPUOperand::isInlinableImm(MVT type) const {
@@ -2041,9 +2045,30 @@ bool AMDGPUOperand::isInlinableImm(MVT type) const {
       return false;
 
     if (type.getScalarSizeInBits() == 16) {
-      return isInlineableLiteralOp16(
-        static_cast(FPLiteral.bitcastToAPInt().getZExtValue()),
-        type, AsmParser->hasInv2PiInlineImm());
+      bool Lost = false;
+      switch (type.getScalarType().SimpleTy) {
+      default:
+        llvm_unreachable("unknown 16-bit type");
+      case MVT::bf16:
+        FPLiteral.convert(APFloatBase::BFloat(), APFloat::rmNearestTiesToEven,
+                          &Lost);
+        break;
+      case MVT::f16:
+        FPLiteral.convert(APFloatBase::IEEEhalf(), APFloat::rmNearestTiesToEven,
+                          &Lost);
+        break;
+      case MVT::i16:
+        FPLiteral.convert(APFloatBase::IEEEsingle(),
+                          APFloat::rmNearestTiesToEven, &Lost);
+        break;
+      }
+      // We need to use 32-bit representation here because when a floating-point
+      // inline constant is used as an i16 operand, its 32-bit representation
+      // representation will be used. We will need the 32-bit value to check if
+      // it is FP inline constant.
+      uint32_t ImmVal = FPLiteral.bitcastToAPInt().getZExtValue();
+      return isInlineableLiteralOp16(ImmVal, type,
+                                     AsmParser->hasInv2PiInlineImm());
     }
 
     // Check if single precision literal is inlinable
@@ -2375,15 +2400,26 @@ void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val, bool ApplyMo
     return;
 
   case AMDGPU::OPERAND_REG_IMM_INT16:
-  case AMDGPU::OPERAND_REG_IMM_FP16:
-  case AMDGPU::OPERAND_REG_IMM_FP16_DEFERRED:
   case AMDGPU::OPERAND_REG_INLINE_C_INT16:
-  case AMDGPU::OPERAND_REG_INLINE_C_FP16:
   case AMDGPU::OPERAND_REG_INLINE_AC_INT16:
+    if (isSafeTruncation(Val, 16) &&
+        AMDGPU::isInlinableIntLiteral(static_cast(Val))) {
+      Inst.addOperand(MCOperand::createImm(Val & 0xffffffff));
+      setImmKindConst();
+      return;
+    }
+
+    Inst.addOperand(MCOperand::createImm(Val & 0xffff));
+    setImmKindLiteral();
+    return;
+
+  case AMDGPU::OPERAND_REG_INLINE_C_FP16:
+  case AMDGPU::OPERAND_REG_IMM_FP16:
+  case AMDGPU::OPERAND_REG_IMM_FP16_DEFERRED:
   case AMDGPU::OPERAND_REG_INLINE_AC_FP16:
     if (isSafeTruncation(Val, 16) &&
-        AMDGPU::isInlinableLiteral16(static_cast(Val),
-                                     AsmParser->hasInv2PiInlineImm())) {
+        AMDGPU::isInlinableLiteralFP16(static_cast(Val),
+                                       AsmParser->hasInv2PiInlineImm())) {
       Inst.addOperand(MCOperand::createImm(Val));
       setImmKindConst();
       return;
@@ -2410,12 +2446,17 @@ void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val, bool ApplyMo
     return;
 
   case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
+  case AMDGPU::OPERAND_REG_INLINE_AC_V2INT16: {
+    assert(isSafeTruncation(Val, 16));
+    assert(AMDGPU::isInlinableIntLiteral(static_cast(Val)));
+    Inst.addOperand(MCOperand::createImm(Val));
+    return;
+  }
   case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
-  case AMDGPU::OPERAND_REG_INLINE_AC_V2INT16:
   case AMDGPU::OPERAND_REG_INLINE_AC_V2FP16: {
     assert(isSafeTruncation(Val, 16));
-    assert(AMDGPU::isInlinableLiteral16(static_cast(Val),
-                                        AsmParser->hasInv2PiInlineImm()));
+    assert(AMDGPU::isInlinableLiteralFP16(static_cast(Val),
+                                          AsmParser->hasInv2PiInlineImm()));
 
     Inst.addOperand(MCOperand::createImm(Val));
     return;
@@ -3542,7 +3583,7 @@ bool AMDGPUAsmParser::isInlineConstant(const MCInst &Inst,
     if (OperandType == AMDGPU::OPERAND_REG_IMM_INT16 ||
         OperandType == AMDGPU::OPERAND_REG_INLINE_C_INT16 ||
         OperandType == AMDGPU::OPERAND_REG_INLINE_AC_INT16)
-      return AMDGPU::isInlinableIntLiteral(Val);
+      return AMDGPU::isInlinableLiteralI16(Val, hasInv2PiInlineImm());
 
     if (OperandType == AMDGPU::OPERAND_REG_INLINE_C_V2INT16 ||
         OperandType == AMDGPU::OPERAND_REG_INLINE_AC_V2INT16 ||
@@ -3559,7 +3600,19 @@ bool AMDGPUAsmParser::isInlineConstant(const MCInst &Inst,
         OperandType == AMDGPU::OPERAND_REG_IMM_V2BF16)
       return AMDGPU::isInlinableLiteralV2BF16(Val);
 
-    return AMDGPU::isInlinableLiteral16(Val, hasInv2PiInlineImm());
+    if (OperandType == AMDGPU::OPERAND_REG_IMM_FP16 ||
+        OperandType == AMDGPU::OPERAND_REG_INLINE_C_FP16 ||
+        OperandType == AMDGPU::OPERAND_REG_INLINE_AC_FP16 ||
+        OperandType == AMDGPU::OPERAND_REG_IMM_FP16_DEFERRED)
+      return AMDGPU::isInlinableLiteralFP16(Val, hasInv2PiInlineImm());
+
+    if (OperandType == AMDGPU::OPERAND_REG_IMM_BF16 ||
+        OperandType == AMDGPU::OPERAND_REG_INLINE_C_BF16 ||
+        OperandType == AMDGPU::OPERAND_REG_INLINE_AC_BF16 ||
+        OperandType == AMDGPU::OPERAND_REG_IMM_BF16_DEFERRED)
+      return AMDGPU::isInlinableLiteralBF16(Val, hasInv2PiInlineImm());
+
+    llvm_unreachable("invalid operand type");
   }
   default:
     llvm_unreachable("invalid operand size");
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp
index a32be1e50a60..b6a95906bc45 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp
@@ -451,19 +451,20 @@ void AMDGPUInstPrinter::printVINTRPDst(const MCInst *MI, unsigned OpNo,
 void AMDGPUInstPrinter::printImmediateInt16(uint32_t Imm,
                                             const MCSubtargetInfo &STI,
                                             raw_ostream &O) {
-  int16_t SImm = static_cast(Imm);
+  int32_t SImm = static_cast(Imm);
   if (isInlinableIntLiteral(SImm)) {
     O << SImm;
-  } else {
-    uint64_t Imm16 = static_cast(Imm);
-    O << formatHex(Imm16);
+    return;
   }
+
+  if (printImmediateFloat32(Imm, STI, O))
+    return;
+
+  O << formatHex(static_cast(Imm & 0xffff));
 }
 
-// This must accept a 32-bit immediate value to correctly handle packed 16-bit
-// operations.
-static bool printImmediateFloat16(uint32_t Imm, const MCSubtargetInfo &STI,
-                                  raw_ostream &O) {
+static bool printImmediateFP16(uint32_t Imm, const MCSubtargetInfo &STI,
+                               raw_ostream &O) {
   if (Imm == 0x3C00)
     O << "1.0";
   else if (Imm == 0xBC00)
@@ -529,9 +530,9 @@ void AMDGPUInstPrinter::printImmediateBF16(uint32_t Imm,
   O << formatHex(static_cast(Imm));
 }
 
-void AMDGPUInstPrinter::printImmediate16(uint32_t Imm,
-                                         const MCSubtargetInfo &STI,
-                                         raw_ostream &O) {
+void AMDGPUInstPrinter::printImmediateF16(uint32_t Imm,
+                                          const MCSubtargetInfo &STI,
+                                          raw_ostream &O) {
   int16_t SImm = static_cast(Imm);
   if (isInlinableIntLiteral(SImm)) {
     O << SImm;
@@ -539,7 +540,7 @@ void AMDGPUInstPrinter::printImmediate16(uint32_t Imm,
   }
 
   uint16_t HImm = static_cast(Imm);
-  if (printImmediateFloat16(HImm, STI, O))
+  if (printImmediateFP16(HImm, STI, O))
     return;
 
   uint64_t Imm16 = static_cast(Imm);
@@ -566,7 +567,7 @@ void AMDGPUInstPrinter::printImmediateV216(uint32_t Imm, uint8_t OpType,
   case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
   case AMDGPU::OPERAND_REG_INLINE_AC_V2FP16:
     if (isUInt<16>(Imm) &&
-        printImmediateFloat16(static_cast(Imm), STI, O))
+        printImmediateFP16(static_cast(Imm), STI, O))
       return;
     break;
   case AMDGPU::OPERAND_REG_IMM_V2BF16:
@@ -845,7 +846,7 @@ void AMDGPUInstPrinter::printRegularOperand(const MCInst *MI, unsigned OpNo,
     case AMDGPU::OPERAND_REG_INLINE_AC_FP16:
     case AMDGPU::OPERAND_REG_IMM_FP16:
     case AMDGPU::OPERAND_REG_IMM_FP16_DEFERRED:
-      printImmediate16(Op.getImm(), STI, O);
+      printImmediateF16(Op.getImm(), STI, O);
       break;
     case AMDGPU::OPERAND_REG_INLINE_C_BF16:
     case AMDGPU::OPERAND_REG_INLINE_AC_BF16:
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h
index 15ecbf2e5e59..c801eaf1111e 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h
@@ -86,10 +86,10 @@ private:
                       raw_ostream &O);
   void printImmediateInt16(uint32_t Imm, const MCSubtargetInfo &STI,
                            raw_ostream &O);
-  void printImmediate16(uint32_t Imm, const MCSubtargetInfo &STI,
-                        raw_ostream &O);
   void printImmediateBF16(uint32_t Imm, const MCSubtargetInfo &STI,
                           raw_ostream &O);
+  void printImmediateF16(uint32_t Imm, const MCSubtargetInfo &STI,
+                         raw_ostream &O);
   void printImmediateV216(uint32_t Imm, uint8_t OpType,
                           const MCSubtargetInfo &STI, raw_ostream &O);
   bool printImmediateFloat32(uint32_t Imm, const MCSubtargetInfo &STI,
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCCodeEmitter.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCCodeEmitter.cpp
index e51bb40132f9..fb93f45e3e87 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCCodeEmitter.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCCodeEmitter.cpp
@@ -116,11 +116,6 @@ static uint32_t getIntInlineImmEncoding(IntTy Imm) {
   return 0;
 }
 
-static uint32_t getLit16IntEncoding(uint16_t Val, const MCSubtargetInfo &STI) {
-  uint16_t IntImm = getIntInlineImmEncoding(static_cast(Val));
-  return IntImm == 0 ? 255 : IntImm;
-}
-
 static uint32_t getLit16Encoding(uint16_t Val, const MCSubtargetInfo &STI) {
   uint16_t IntImm = getIntInlineImmEncoding(static_cast(Val));
   if (IntImm != 0)
@@ -214,6 +209,10 @@ static uint32_t getLit32Encoding(uint32_t Val, const MCSubtargetInfo &STI) {
   return 255;
 }
 
+static uint32_t getLit16IntEncoding(uint32_t Val, const MCSubtargetInfo &STI) {
+  return getLit32Encoding(Val, STI);
+}
+
 static uint32_t getLit64Encoding(uint64_t Val, const MCSubtargetInfo &STI) {
   uint32_t IntImm = getIntInlineImmEncoding(static_cast(Val));
   if (IntImm != 0)
@@ -296,7 +295,7 @@ AMDGPUMCCodeEmitter::getLitEncoding(const MCOperand &MO,
   case AMDGPU::OPERAND_REG_IMM_INT16:
   case AMDGPU::OPERAND_REG_INLINE_C_INT16:
   case AMDGPU::OPERAND_REG_INLINE_AC_INT16:
-    return getLit16IntEncoding(static_cast(Imm), STI);
+    return getLit16IntEncoding(static_cast(Imm), STI);
 
   case AMDGPU::OPERAND_REG_IMM_FP16:
   case AMDGPU::OPERAND_REG_IMM_FP16_DEFERRED:
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index e105db313548..1889ab007288 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -15495,16 +15495,32 @@ bool SITargetLowering::checkAsmConstraintVal(SDValue Op, StringRef Constraint,
   llvm_unreachable("Invalid asm constraint");
 }
 
-bool SITargetLowering::checkAsmConstraintValA(SDValue Op,
-                                              uint64_t Val,
+bool SITargetLowering::checkAsmConstraintValA(SDValue Op, uint64_t Val,
                                               unsigned MaxSize) const {
   unsigned Size = std::min(Op.getScalarValueSizeInBits(), MaxSize);
   bool HasInv2Pi = Subtarget->hasInv2PiInlineImm();
-  if ((Size == 16 && AMDGPU::isInlinableLiteral16(Val, HasInv2Pi)) ||
-      (Size == 32 && AMDGPU::isInlinableLiteral32(Val, HasInv2Pi)) ||
-      (Size == 64 && AMDGPU::isInlinableLiteral64(Val, HasInv2Pi))) {
-    return true;
+  if (Size == 16) {
+    MVT VT = Op.getSimpleValueType();
+    switch (VT.SimpleTy) {
+    default:
+      return false;
+    case MVT::i16:
+      return AMDGPU::isInlinableLiteralI16(Val, HasInv2Pi);
+    case MVT::f16:
+      return AMDGPU::isInlinableLiteralFP16(Val, HasInv2Pi);
+    case MVT::bf16:
+      return AMDGPU::isInlinableLiteralBF16(Val, HasInv2Pi);
+    case MVT::v2i16:
+      return AMDGPU::getInlineEncodingV2I16(Val).has_value();
+    case MVT::v2f16:
+      return AMDGPU::getInlineEncodingV2F16(Val).has_value();
+    case MVT::v2bf16:
+      return AMDGPU::getInlineEncodingV2BF16(Val).has_value();
+    }
   }
+  if ((Size == 32 && AMDGPU::isInlinableLiteral32(Val, HasInv2Pi)) ||
+      (Size == 64 && AMDGPU::isInlinableLiteral64(Val, HasInv2Pi)))
+    return true;
   return false;
 }
 
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index ca2c2d87009f..e8022c8b0afa 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -4121,13 +4121,32 @@ bool SIInstrInfo::isInlineConstant(const APInt &Imm) const {
                                         ST.hasInv2PiInlineImm());
   case 16:
     return ST.has16BitInsts() &&
-           AMDGPU::isInlinableLiteral16(Imm.getSExtValue(),
-                                        ST.hasInv2PiInlineImm());
+           AMDGPU::isInlinableLiteralI16(Imm.getSExtValue(),
+                                         ST.hasInv2PiInlineImm());
   default:
     llvm_unreachable("invalid bitwidth");
   }
 }
 
+bool SIInstrInfo::isInlineConstant(const APFloat &Imm) const {
+  APInt IntImm = Imm.bitcastToAPInt();
+  int64_t IntImmVal = IntImm.getSExtValue();
+  bool HasInv2Pi = ST.hasInv2PiInlineImm();
+  switch (APFloat::SemanticsToEnum(Imm.getSemantics())) {
+  default:
+    llvm_unreachable("invalid fltSemantics");
+  case APFloatBase::S_IEEEsingle:
+  case APFloatBase::S_IEEEdouble:
+    return isInlineConstant(IntImm);
+  case APFloatBase::S_BFloat:
+    return ST.has16BitInsts() &&
+           AMDGPU::isInlinableLiteralBF16(IntImmVal, HasInv2Pi);
+  case APFloatBase::S_IEEEhalf:
+    return ST.has16BitInsts() &&
+           AMDGPU::isInlinableLiteralFP16(IntImmVal, HasInv2Pi);
+  }
+}
+
 bool SIInstrInfo::isInlineConstant(const MachineOperand &MO,
                                    uint8_t OperandType) const {
   assert(!MO.isReg() && "isInlineConstant called on register operand!");
@@ -4200,7 +4219,7 @@ bool SIInstrInfo::isInlineConstant(const MachineOperand &MO,
       // constants in these cases
       int16_t Trunc = static_cast(Imm);
       return ST.has16BitInsts() &&
-             AMDGPU::isInlinableLiteral16(Trunc, ST.hasInv2PiInlineImm());
+             AMDGPU::isInlinableLiteralFP16(Trunc, ST.hasInv2PiInlineImm());
     }
 
     return false;
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index 4200e0c8a29e..a62bf779fe2e 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -984,9 +984,7 @@ public:
 
   bool isInlineConstant(const APInt &Imm) const;
 
-  bool isInlineConstant(const APFloat &Imm) const {
-    return isInlineConstant(Imm.bitcastToAPInt());
-  }
+  bool isInlineConstant(const APFloat &Imm) const;
 
   // Returns true if this non-register operand definitely does not need to be
   // encoded as a 32-bit literal. Note that this function handles all kinds of
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
index 62903a244dc8..edb0e50da289 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
@@ -2657,13 +2657,15 @@ bool isInlinableLiteralBF16(int16_t Literal, bool HasInv2Pi) {
          Val == 0x3E22;   // 1.0 / (2.0 * pi)
 }
 
-bool isInlinableLiteral16(int16_t Literal, bool HasInv2Pi) {
+bool isInlinableLiteralI16(int32_t Literal, bool HasInv2Pi) {
+  return isInlinableLiteral32(Literal, HasInv2Pi);
+}
+
+bool isInlinableLiteralFP16(int16_t Literal, bool HasInv2Pi) {
   if (!HasInv2Pi)
     return false;
-
   if (isInlinableIntLiteral(Literal))
     return true;
-
   uint16_t Val = static_cast(Literal);
   return Val == 0x3C00 || // 1.0
          Val == 0xBC00 || // -1.0
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
index bb307cb67c9b..d7ea2a3eff4b 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
@@ -1404,7 +1404,13 @@ LLVM_READNONE
 bool isInlinableLiteralBF16(int16_t Literal, bool HasInv2Pi);
 
 LLVM_READNONE
-bool isInlinableLiteral16(int16_t Literal, bool HasInv2Pi);
+bool isInlinableLiteralFP16(int16_t Literal, bool HasInv2Pi);
+
+LLVM_READNONE
+bool isInlinableLiteralBF16(int16_t Literal, bool HasInv2Pi);
+
+LLVM_READNONE
+bool isInlinableLiteralI16(int32_t Literal, bool HasInv2Pi);
 
 LLVM_READNONE
 std::optional getInlineEncodingV2I16(uint32_t Literal);
diff --git a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
index 4ca2835eea58..a7d63fdb2e04 100644
--- a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
@@ -1007,6 +1007,7 @@ class VOP3PWMMA_Profile ArgTy, bit _IsSWMMAC, int _IndexType,
                                !cast("VISrc_"#ArgTy[3].Size#
                                                       !cond(IsC_F32: "_f32",
                                                             IsC_F16: "_f16",
+                                                            IsC_BF16: "_bf16",
                                                             1: "_b32")));
 
   // For f16 and bf16 matrices A and B, each element can be modified by
diff --git a/llvm/test/CodeGen/AMDGPU/immv216.ll b/llvm/test/CodeGen/AMDGPU/immv216.ll
index b66ca71a3274..ae51c3edf1c7 100644
--- a/llvm/test/CodeGen/AMDGPU/immv216.ll
+++ b/llvm/test/CodeGen/AMDGPU/immv216.ll
@@ -577,40 +577,40 @@ define amdgpu_kernel void @add_inline_imm_64_v2f16(ptr addrspace(1) %out, <2 x h
 }
 
 ; GCN-LABEL: {{^}}mul_inline_imm_0.5_v2i16:
-; GFX9: s_mov_b32 [[K:s[0-9]+]], 0x38003800
-; GFX9: v_pk_mul_lo_u16 v0, v0, [[K]]
+; GFX9: s_movk_i32 [[K:s[0-9]+]], 0x3800
+; GFX9: v_pk_mul_lo_u16 v0, v0, [[K]] op_sel_hi:[1,0]
 
-; GFX10: v_pk_mul_lo_u16 v0, 0x38003800, v0 ; encoding: [0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0xff,0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x00,0x38,0x00,0x38]
+; GFX10: v_pk_mul_lo_u16 v0, 0x3800, v0 op_sel_hi:[0,1] ; encoding: [0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0xff,0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x00,0x38,0x00,0x00]
 define <2 x i16> @mul_inline_imm_0.5_v2i16(<2 x i16> %x) {
   %y = mul <2 x i16> %x, bitcast (<2 x half>  to <2 x i16>)
   ret <2 x i16> %y
 }
 
 ; GCN-LABEL: {{^}}mul_inline_imm_neg_0.5_v2i16:
-; GFX9: s_mov_b32 [[K:s[0-9]+]], 0xb800b800
-; GFX9: v_pk_mul_lo_u16 v0, v0, [[K]]
+; GFX9: s_movk_i32 [[K:s[0-9]+]], 0xb800
+; GFX9: v_pk_mul_lo_u16 v0, v0, [[K]] op_sel_hi:[1,0]
 
-; GFX10: v_pk_mul_lo_u16 v0, 0xb800b800, v0 ; encoding: [0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0xff,0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x00,0xb8,0x00,0xb8]
+; GFX10: v_pk_mul_lo_u16 v0, 0xffffb800, v0 op_sel_hi:[0,1] ; encoding: [0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0xff,0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x00,0xb8,0xff,0xff]
 define <2 x i16> @mul_inline_imm_neg_0.5_v2i16(<2 x i16> %x) {
   %y = mul <2 x i16> %x, bitcast (<2 x half>  to <2 x i16>)
   ret <2 x i16> %y
 }
 
 ; GCN-LABEL: {{^}}mul_inline_imm_1.0_v2i16:
-; GFX9: s_mov_b32 [[K:s[0-9]+]], 0x3c003c00
-; GFX9: v_pk_mul_lo_u16 v0, v0, [[K]]
+; GFX9: s_movk_i32 [[K:s[0-9]+]], 0x3c00
+; GFX9: v_pk_mul_lo_u16 v0, v0, [[K]] op_sel_hi:[1,0]
 
-; GFX10: v_pk_mul_lo_u16 v0, 0x3c003c00, v0 ; encoding: [0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0xff,0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x00,0x3c,0x00,0x3c]
+; GFX10: v_pk_mul_lo_u16 v0, 0x3c00, v0 op_sel_hi:[0,1] ; encoding: [0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0xff,0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x00,0x3c,0x00,0x00]
 define <2 x i16> @mul_inline_imm_1.0_v2i16(<2 x i16> %x) {
   %y = mul <2 x i16> %x, bitcast (<2 x half>  to <2 x i16>)
   ret <2 x i16> %y
 }
 
 ; GCN-LABEL: {{^}}mul_inline_imm_neg_1.0_v2i16:
-; GFX9: s_mov_b32 [[K:s[0-9]+]], 0xbc00bc00
-; GFX9: v_pk_mul_lo_u16 v0, v0, [[K]]
+; GFX9: s_movk_i32 [[K:s[0-9]+]], 0xbc00
+; GFX9: v_pk_mul_lo_u16 v0, v0, [[K]] op_sel_hi:[1,0]
 
-; GFX10: v_pk_mul_lo_u16 v0, 0xbc00bc00, v0 ; encoding: [0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0xff,0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x00,0xbc,0x00,0xbc]
+; GFX10: v_pk_mul_lo_u16 v0, 0xffffbc00, v0 op_sel_hi:[0,1] ; encoding: [0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0xff,0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x00,0xbc,0xff,0xff]
 define <2 x i16> @mul_inline_imm_neg_1.0_v2i16(<2 x i16> %x) {
   %y = mul <2 x i16> %x, bitcast (<2 x half>  to <2 x i16>)
   ret <2 x i16> %y
@@ -635,10 +635,10 @@ define <2 x i16> @shl_inline_imm_neg_2.0_v2i16(<2 x i16> %x) {
 }
 
 ; GCN-LABEL: {{^}}mul_inline_imm_4.0_v2i16:
-; GFX9: s_mov_b32 [[K:s[0-9]+]], 0x44004400
-; GFX9: v_pk_mul_lo_u16 v0, v0, [[K]]
+; GFX9: s_movk_i32 [[K:s[0-9]+]], 0x4400
+; GFX9: v_pk_mul_lo_u16 v0, v0, [[K]] op_sel_hi:[1,0]
 
-; GFX10: v_pk_mul_lo_u16 v0, 0x44004400, v0 ; encoding: [0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0xff,0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x00,0x44,0x00,0x44]
+; GFX10: v_pk_mul_lo_u16 v0, 0x4400, v0 op_sel_hi:[0,1] ; encoding: [0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0xff,0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x00,0x44,0x00,0x00]
 define <2 x i16> @mul_inline_imm_4.0_v2i16(<2 x i16> %x) {
   %y = mul <2 x i16> %x, bitcast (<2 x half>  to <2 x i16>)
   ret <2 x i16> %y
@@ -646,20 +646,20 @@ define <2 x i16> @mul_inline_imm_4.0_v2i16(<2 x i16> %x) {
 }
 
 ; GCN-LABEL: {{^}}mul_inline_imm_neg_4.0_v2i16:
-; GFX9: s_mov_b32 [[K:s[0-9]+]], 0xc400c400
-; GFX9: v_pk_mul_lo_u16 v0, v0, [[K]]
+; GFX9: s_movk_i32 [[K:s[0-9]+]], 0xc400
+; GFX9: v_pk_mul_lo_u16 v0, v0, [[K]] op_sel_hi:[1,0]
 
-; GFX10: v_pk_mul_lo_u16 v0, 0xc400c400, v0 ; encoding: [0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0xff,0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x00,0xc4,0x00,0xc4]
+; GFX10: v_pk_mul_lo_u16 v0, 0xffffc400, v0 op_sel_hi:[0,1] ; encoding: [0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0xff,0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x00,0xc4,0xff,0xff]
 define <2 x i16> @mul_inline_imm_neg_4.0_v2i16(<2 x i16> %x) {
   %y = mul <2 x i16> %x, bitcast (<2 x half>  to <2 x i16>)
   ret <2 x i16> %y
 }
 
 ; GCN-LABEL: {{^}}mul_inline_imm_inv2pi_v2i16:
-; GFX9: s_mov_b32 [[K:s[0-9]+]], 0x31183118
-; GFX9: v_pk_mul_lo_u16 v0, v0, [[K]]
+; GFX9: s_movk_i32 [[K:s[0-9]+]], 0x3118
+; GFX9: v_pk_mul_lo_u16 v0, v0, [[K]] op_sel_hi:[1,0]
 
-; GFX10: v_pk_mul_lo_u16 v0, 0x31183118, v0 ; encoding: [0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0xff,0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x18,0x31,0x18,0x31]
+; GFX10: v_pk_mul_lo_u16 v0, 0x3118, v0 op_sel_hi:[0,1] ; encoding: [0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0xff,0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x{{[0-9a-f]+}},0x18,0x31,0x00,0x00]
 define <2 x i16> @mul_inline_imm_inv2pi_v2i16(<2 x i16> %x) {
   %y = mul <2 x i16> %x, bitcast (<2 x half>  to <2 x i16>)
   ret <2 x i16> %y
diff --git a/llvm/test/CodeGen/AMDGPU/inline-constraints.ll b/llvm/test/CodeGen/AMDGPU/inline-constraints.ll
index 9ef246fe2e10..7bd6b037386b 100644
--- a/llvm/test/CodeGen/AMDGPU/inline-constraints.ll
+++ b/llvm/test/CodeGen/AMDGPU/inline-constraints.ll
@@ -97,7 +97,6 @@ define i32 @inline_A_constraint_H1() {
 
 ; NOSI: error: invalid operand for inline asm constraint 'A'
 ; VI-LABEL: {{^}}inline_A_constraint_H2:
-; VI: v_mov_b32 {{v[0-9]+}}, 0x3c00
 define i32 @inline_A_constraint_H2() {
   %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,A"(i16 bitcast (half 1.0 to i16))
   ret i32 %v0
@@ -105,7 +104,6 @@ define i32 @inline_A_constraint_H2() {
 
 ; NOSI: error: invalid operand for inline asm constraint 'A'
 ; VI-LABEL: {{^}}inline_A_constraint_H3:
-; VI: v_mov_b32 {{v[0-9]+}}, 0xbc00
 define i32 @inline_A_constraint_H3() {
   %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,A"(i16 bitcast (half -1.0 to i16))
   ret i32 %v0
@@ -113,7 +111,6 @@ define i32 @inline_A_constraint_H3() {
 
 ; NOSI: error: invalid operand for inline asm constraint 'A'
 ; VI-LABEL: {{^}}inline_A_constraint_H4:
-; VI: v_mov_b32 {{v[0-9]+}}, 0x3118
 define i32 @inline_A_constraint_H4() {
   %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,A"(half 0xH3118)
   ret i32 %v0
@@ -121,7 +118,6 @@ define i32 @inline_A_constraint_H4() {
 
 ; NOSI: error: invalid operand for inline asm constraint 'A'
 ; VI-LABEL: {{^}}inline_A_constraint_H5:
-; VI: v_mov_b32 {{v[0-9]+}}, 0x3118
 define i32 @inline_A_constraint_H5() {
   %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,A"(i16 bitcast (half 0xH3118 to i16))
   ret i32 %v0
@@ -129,7 +125,6 @@ define i32 @inline_A_constraint_H5() {
 
 ; NOSI: error: invalid operand for inline asm constraint 'A'
 ; VI-LABEL: {{^}}inline_A_constraint_H6:
-; VI: v_mov_b32 {{v[0-9]+}}, 0xb800
 define i32 @inline_A_constraint_H6() {
   %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,A"(half -0.5)
   ret i32 %v0
@@ -293,7 +288,6 @@ define i32 @inline_A_constraint_V0() {
 
 ; NOSI: error: invalid operand for inline asm constraint 'A'
 ; VI-LABEL: {{^}}inline_A_constraint_V1:
-; VI: v_mov_b32 {{v[0-9]+}}, 0xb800
 define i32 @inline_A_constraint_V1() {
   %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,A"(<2 x half> )
   ret i32 %v0
@@ -970,7 +964,6 @@ define i32 @inline_DA_constraint_H1() {
 
 ; NOSI: error: invalid operand for inline asm constraint 'DA'
 ; VI-LABEL: {{^}}inline_DA_constraint_H2:
-; VI: v_mov_b32 {{v[0-9]+}}, 0x3c00
 define i32 @inline_DA_constraint_H2() {
   %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,^DA"(i16 bitcast (half 1.0 to i16))
   ret i32 %v0
@@ -978,7 +971,6 @@ define i32 @inline_DA_constraint_H2() {
 
 ; NOSI: error: invalid operand for inline asm constraint 'DA'
 ; VI-LABEL: {{^}}inline_DA_constraint_H3:
-; VI: v_mov_b32 {{v[0-9]+}}, 0xbc00
 define i32 @inline_DA_constraint_H3() {
   %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,^DA"(i16 bitcast (half -1.0 to i16))
   ret i32 %v0
@@ -986,7 +978,6 @@ define i32 @inline_DA_constraint_H3() {
 
 ; NOSI: error: invalid operand for inline asm constraint 'DA'
 ; VI-LABEL: {{^}}inline_DA_constraint_H4:
-; VI: v_mov_b32 {{v[0-9]+}}, 0x3118
 define i32 @inline_DA_constraint_H4() {
   %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,^DA"(half 0xH3118)
   ret i32 %v0
@@ -994,7 +985,6 @@ define i32 @inline_DA_constraint_H4() {
 
 ; NOSI: error: invalid operand for inline asm constraint 'DA'
 ; VI-LABEL: {{^}}inline_DA_constraint_H5:
-; VI: v_mov_b32 {{v[0-9]+}}, 0x3118
 define i32 @inline_DA_constraint_H5() {
   %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,^DA"(i16 bitcast (half 0xH3118 to i16))
   ret i32 %v0
@@ -1002,7 +992,6 @@ define i32 @inline_DA_constraint_H5() {
 
 ; NOSI: error: invalid operand for inline asm constraint 'DA'
 ; VI-LABEL: {{^}}inline_DA_constraint_H6:
-; VI: v_mov_b32 {{v[0-9]+}}, 0xb800
 define i32 @inline_DA_constraint_H6() {
   %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,^DA"(half -0.5)
   ret i32 %v0
@@ -1164,7 +1153,6 @@ define i32 @inline_DA_constraint_V0() {
 
 ; NOSI: error: invalid operand for inline asm constraint 'DA'
 ; VI-LABEL: {{^}}inline_DA_constraint_V1:
-; VI: v_mov_b32 {{v[0-9]+}}, 0xb800
 define i32 @inline_DA_constraint_V1() {
   %v0 = tail call i32 asm "v_mov_b32 $0, $1", "=v,^DA"(<2 x half> )
   ret i32 %v0
diff --git a/llvm/test/CodeGen/AMDGPU/shrink-add-sub-constant.ll b/llvm/test/CodeGen/AMDGPU/shrink-add-sub-constant.ll
index 5de9b0b92c9a..1a55bf608ebf 100644
--- a/llvm/test/CodeGen/AMDGPU/shrink-add-sub-constant.ll
+++ b/llvm/test/CodeGen/AMDGPU/shrink-add-sub-constant.ll
@@ -3400,9 +3400,9 @@ define amdgpu_kernel void @v_test_v2i16_x_add_neg_fpone(ptr addrspace(1) %out, p
 ; GFX9-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX9-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-SDAG-NEXT:    global_load_dword v1, v0, s[2:3]
-; GFX9-SDAG-NEXT:    s_mov_b32 s2, 0xc400c400
+; GFX9-SDAG-NEXT:    s_movk_i32 s2, 0xc400
 ; GFX9-SDAG-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-SDAG-NEXT:    v_pk_add_u16 v1, v1, s2
+; GFX9-SDAG-NEXT:    v_pk_add_u16 v1, v1, s2 op_sel_hi:[1,0]
 ; GFX9-SDAG-NEXT:    global_store_dword v0, v1, s[0:1]
 ; GFX9-SDAG-NEXT:    s_endpgm
 ;
@@ -3418,29 +3418,53 @@ define amdgpu_kernel void @v_test_v2i16_x_add_neg_fpone(ptr addrspace(1) %out, p
 ; GFX9-GISEL-NEXT:    global_store_dword v0, v1, s[0:1]
 ; GFX9-GISEL-NEXT:    s_endpgm
 ;
-; GFX10-LABEL: v_test_v2i16_x_add_neg_fpone:
-; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    global_load_dword v1, v0, s[2:3]
-; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_pk_add_u16 v1, 0xc400c400, v1
-; GFX10-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX10-NEXT:    s_endpgm
+; GFX10-SDAG-LABEL: v_test_v2i16_x_add_neg_fpone:
+; GFX10-SDAG:       ; %bb.0:
+; GFX10-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-SDAG-NEXT:    global_load_dword v1, v0, s[2:3]
+; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-SDAG-NEXT:    v_pk_add_u16 v1, 0xffffc400, v1 op_sel_hi:[0,1]
+; GFX10-SDAG-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-SDAG-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: v_test_v2i16_x_add_neg_fpone:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
-; GFX11-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    global_load_b32 v1, v0, s[2:3]
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    v_pk_add_u16 v1, 0xc400c400, v1
-; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
-; GFX11-NEXT:    s_nop 0
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
-; GFX11-NEXT:    s_endpgm
+; GFX10-GISEL-LABEL: v_test_v2i16_x_add_neg_fpone:
+; GFX10-GISEL:       ; %bb.0:
+; GFX10-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-GISEL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT:    global_load_dword v1, v0, s[2:3]
+; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-GISEL-NEXT:    v_pk_add_u16 v1, 0xc400c400, v1
+; GFX10-GISEL-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-GISEL-NEXT:    s_endpgm
+;
+; GFX11-SDAG-LABEL: v_test_v2i16_x_add_neg_fpone:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX11-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT:    v_pk_add_u16 v1, 0xffffc400, v1 op_sel_hi:[0,1]
+; GFX11-SDAG-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-SDAG-NEXT:    s_nop 0
+; GFX11-SDAG-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-SDAG-NEXT:    s_endpgm
+;
+; GFX11-GISEL-LABEL: v_test_v2i16_x_add_neg_fpone:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX11-GISEL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT:    v_pk_add_u16 v1, 0xc400c400, v1
+; GFX11-GISEL-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-GISEL-NEXT:    s_nop 0
+; GFX11-GISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-GISEL-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %gep = getelementptr inbounds <2 x i16>, ptr addrspace(1) %in, i64 %tid.ext
@@ -3541,9 +3565,9 @@ define amdgpu_kernel void @v_test_v2i16_x_add_neg_negfpone(ptr addrspace(1) %out
 ; GFX9-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX9-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-SDAG-NEXT:    global_load_dword v1, v0, s[2:3]
-; GFX9-SDAG-NEXT:    s_mov_b32 s2, 0x44004400
+; GFX9-SDAG-NEXT:    s_movk_i32 s2, 0x4400
 ; GFX9-SDAG-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-SDAG-NEXT:    v_pk_add_u16 v1, v1, s2
+; GFX9-SDAG-NEXT:    v_pk_add_u16 v1, v1, s2 op_sel_hi:[1,0]
 ; GFX9-SDAG-NEXT:    global_store_dword v0, v1, s[0:1]
 ; GFX9-SDAG-NEXT:    s_endpgm
 ;
@@ -3559,29 +3583,53 @@ define amdgpu_kernel void @v_test_v2i16_x_add_neg_negfpone(ptr addrspace(1) %out
 ; GFX9-GISEL-NEXT:    global_store_dword v0, v1, s[0:1]
 ; GFX9-GISEL-NEXT:    s_endpgm
 ;
-; GFX10-LABEL: v_test_v2i16_x_add_neg_negfpone:
-; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    global_load_dword v1, v0, s[2:3]
-; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_pk_add_u16 v1, 0x44004400, v1
-; GFX10-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX10-NEXT:    s_endpgm
+; GFX10-SDAG-LABEL: v_test_v2i16_x_add_neg_negfpone:
+; GFX10-SDAG:       ; %bb.0:
+; GFX10-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-SDAG-NEXT:    global_load_dword v1, v0, s[2:3]
+; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-SDAG-NEXT:    v_pk_add_u16 v1, 0x4400, v1 op_sel_hi:[0,1]
+; GFX10-SDAG-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-SDAG-NEXT:    s_endpgm
 ;
-; GFX11-LABEL: v_test_v2i16_x_add_neg_negfpone:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
-; GFX11-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    global_load_b32 v1, v0, s[2:3]
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    v_pk_add_u16 v1, 0x44004400, v1
-; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
-; GFX11-NEXT:    s_nop 0
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
-; GFX11-NEXT:    s_endpgm
+; GFX10-GISEL-LABEL: v_test_v2i16_x_add_neg_negfpone:
+; GFX10-GISEL:       ; %bb.0:
+; GFX10-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-GISEL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT:    global_load_dword v1, v0, s[2:3]
+; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-GISEL-NEXT:    v_pk_add_u16 v1, 0x44004400, v1
+; GFX10-GISEL-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-GISEL-NEXT:    s_endpgm
+;
+; GFX11-SDAG-LABEL: v_test_v2i16_x_add_neg_negfpone:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX11-SDAG-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT:    v_pk_add_u16 v1, 0x4400, v1 op_sel_hi:[0,1]
+; GFX11-SDAG-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-SDAG-NEXT:    s_nop 0
+; GFX11-SDAG-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-SDAG-NEXT:    s_endpgm
+;
+; GFX11-GISEL-LABEL: v_test_v2i16_x_add_neg_negfpone:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX11-GISEL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT:    v_pk_add_u16 v1, 0x44004400, v1
+; GFX11-GISEL-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-GISEL-NEXT:    s_nop 0
+; GFX11-GISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-GISEL-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
   %gep = getelementptr inbounds <2 x i16>, ptr addrspace(1) %in, i64 %tid.ext
diff --git a/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w32-imm.ll b/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w32-imm.ll
index c4adc8c32128..a3973970d50d 100644
--- a/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w32-imm.ll
+++ b/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w32-imm.ll
@@ -112,7 +112,11 @@ bb:
 define amdgpu_ps void @test_wmma_bf16_16x16x16_bf16_imm(<8 x i16> %A, <8 x i16> %B, ptr addrspace(1) %out) {
 ; GFX12-LABEL: test_wmma_bf16_16x16x16_bf16_imm:
 ; GFX12:       ; %bb.0: ; %bb
-; GFX12-NEXT:    v_wmma_bf16_16x16x16_bf16 v[10:13], v[0:3], v[4:7], 1.0
+; GFX12-NEXT:    v_mov_b32_e32 v10, 0x3f803f80
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    v_dual_mov_b32 v11, v10 :: v_dual_mov_b32 v12, v10
+; GFX12-NEXT:    v_mov_b32_e32 v13, v10
+; GFX12-NEXT:    v_wmma_bf16_16x16x16_bf16 v[10:13], v[0:3], v[4:7], v[10:13]
 ; GFX12-NEXT:    global_store_b128 v[8:9], v[10:13], off
 ; GFX12-NEXT:    s_nop 0
 ; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
diff --git a/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w64-imm.ll b/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w64-imm.ll
index 462fc01e8e79..829d77018ff9 100644
--- a/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w64-imm.ll
+++ b/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w64-imm.ll
@@ -101,7 +101,10 @@ bb:
 define amdgpu_ps void @test_wmma_bf16_16x16x16_bf16_imm(<4 x i16> %A, <4 x i16> %B, ptr addrspace(1) %out) {
 ; GFX12-LABEL: test_wmma_bf16_16x16x16_bf16_imm:
 ; GFX12:       ; %bb.0: ; %bb
-; GFX12-NEXT:    v_wmma_bf16_16x16x16_bf16 v[6:7], v[0:1], v[2:3], 1.0
+; GFX12-NEXT:    v_mov_b32_e32 v6, 0x3f803f80
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    v_mov_b32_e32 v7, v6
+; GFX12-NEXT:    v_wmma_bf16_16x16x16_bf16 v[6:7], v[0:1], v[2:3], v[6:7]
 ; GFX12-NEXT:    global_store_b64 v[4:5], v[6:7], off
 ; GFX12-NEXT:    s_nop 0
 ; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
diff --git a/llvm/test/MC/AMDGPU/gfx10_asm_vop1.s b/llvm/test/MC/AMDGPU/gfx10_asm_vop1.s
index 1cfafebe2c3c..5a1673ef1cce 100644
--- a/llvm/test/MC/AMDGPU/gfx10_asm_vop1.s
+++ b/llvm/test/MC/AMDGPU/gfx10_asm_vop1.s
@@ -11189,10 +11189,10 @@ v_cvt_f16_u16_e32 v5, -1
 // GFX10: encoding: [0xc1,0xa0,0x0a,0x7e]
 
 v_cvt_f16_u16_e32 v5, 0.5
-// GFX10: encoding: [0xff,0xa0,0x0a,0x7e,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0xf0,0xa0,0x0a,0x7e]
 
 v_cvt_f16_u16_e32 v5, -4.0
-// GFX10: encoding: [0xff,0xa0,0x0a,0x7e,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0xf7,0xa0,0x0a,0x7e]
 
 v_cvt_f16_u16_e32 v5, 0xfe0b
 // GFX10: encoding: [0xff,0xa0,0x0a,0x7e,0x0b,0xfe,0x00,0x00]
@@ -11237,10 +11237,10 @@ v_cvt_f16_u16_e64 v5, -1
 // GFX10: encoding: [0x05,0x00,0xd0,0xd5,0xc1,0x00,0x00,0x00]
 
 v_cvt_f16_u16_e64 v5, 0.5
-// GFX10: encoding: [0x05,0x00,0xd0,0xd5,0xff,0x00,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0xd0,0xd5,0xf0,0x00,0x00,0x00]
 
 v_cvt_f16_u16_e64 v5, -4.0
-// GFX10: encoding: [0x05,0x00,0xd0,0xd5,0xff,0x00,0x00,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0xd0,0xd5,0xf7,0x00,0x00,0x00]
 
 v_cvt_f16_u16_e64 v5, v1 clamp
 // GFX10: encoding: [0x05,0x80,0xd0,0xd5,0x01,0x01,0x00,0x00]
@@ -11435,10 +11435,10 @@ v_cvt_f16_i16_e32 v5, -1
 // GFX10: encoding: [0xc1,0xa2,0x0a,0x7e]
 
 v_cvt_f16_i16_e32 v5, 0.5
-// GFX10: encoding: [0xff,0xa2,0x0a,0x7e,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0xf0,0xa2,0x0a,0x7e]
 
 v_cvt_f16_i16_e32 v5, -4.0
-// GFX10: encoding: [0xff,0xa2,0x0a,0x7e,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0xf7,0xa2,0x0a,0x7e]
 
 v_cvt_f16_i16_e32 v5, 0xfe0b
 // GFX10: encoding: [0xff,0xa2,0x0a,0x7e,0x0b,0xfe,0x00,0x00]
@@ -11483,10 +11483,10 @@ v_cvt_f16_i16_e64 v5, -1
 // GFX10: encoding: [0x05,0x00,0xd1,0xd5,0xc1,0x00,0x00,0x00]
 
 v_cvt_f16_i16_e64 v5, 0.5
-// GFX10: encoding: [0x05,0x00,0xd1,0xd5,0xff,0x00,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0xd1,0xd5,0xf0,0x00,0x00,0x00]
 
 v_cvt_f16_i16_e64 v5, -4.0
-// GFX10: encoding: [0x05,0x00,0xd1,0xd5,0xff,0x00,0x00,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0xd1,0xd5,0xf7,0x00,0x00,0x00]
 
 v_cvt_f16_i16_e64 v5, v1 clamp
 // GFX10: encoding: [0x05,0x80,0xd1,0xd5,0x01,0x01,0x00,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx10_asm_vop3.s b/llvm/test/MC/AMDGPU/gfx10_asm_vop3.s
index 09d833d0ce4b..b4256f1a628d 100644
--- a/llvm/test/MC/AMDGPU/gfx10_asm_vop3.s
+++ b/llvm/test/MC/AMDGPU/gfx10_asm_vop3.s
@@ -8741,10 +8741,10 @@ v_add_nc_u16 v5, -1, v2
 // GFX10: encoding: [0x05,0x00,0x03,0xd7,0xc1,0x04,0x02,0x00]
 
 v_add_nc_u16 v5, 0.5, v2
-// GFX10: encoding: [0x05,0x00,0x03,0xd7,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x03,0xd7,0xf0,0x04,0x02,0x00]
 
 v_add_nc_u16 v5, -4.0, v2
-// GFX10: encoding: [0x05,0x00,0x03,0xd7,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x03,0xd7,0xf7,0x04,0x02,0x00]
 
 v_add_nc_u16 v5, v1, v255
 // GFX10: encoding: [0x05,0x00,0x03,0xd7,0x01,0xff,0x03,0x00]
@@ -8777,13 +8777,13 @@ v_add_nc_u16 v5, v1, -1
 // GFX10: encoding: [0x05,0x00,0x03,0xd7,0x01,0x83,0x01,0x00]
 
 v_add_nc_u16 v5, v1, 0.5
-// GFX10: encoding: [0x05,0x00,0x03,0xd7,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x03,0xd7,0x01,0xe1,0x01,0x00]
 
 v_add_nc_u16 v5, v1, -4.0
-// GFX10: encoding: [0x05,0x00,0x03,0xd7,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x03,0xd7,0x01,0xef,0x01,0x00]
 
 v_add_nc_u16 v5, v1, -4.0 clamp
-// GFX10: encoding: [0x05,0x80,0x03,0xd7,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x80,0x03,0xd7,0x01,0xef,0x01,0x00]
 
 v_add_nc_u16 v5, v1, v2 op_sel:[1,1,1]
 // GFX10: [0x05,0x58,0x03,0xd7,0x01,0x05,0x02,0x00]
@@ -8837,10 +8837,10 @@ v_sub_nc_u16 v5, -1, v2
 // GFX10: encoding: [0x05,0x00,0x04,0xd7,0xc1,0x04,0x02,0x00]
 
 v_sub_nc_u16 v5, 0.5, v2
-// GFX10: encoding: [0x05,0x00,0x04,0xd7,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x04,0xd7,0xf0,0x04,0x02,0x00]
 
 v_sub_nc_u16 v5, -4.0, v2
-// GFX10: encoding: [0x05,0x00,0x04,0xd7,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x04,0xd7,0xf7,0x04,0x02,0x00]
 
 v_sub_nc_u16 v5, v1, v255
 // GFX10: encoding: [0x05,0x00,0x04,0xd7,0x01,0xff,0x03,0x00]
@@ -8873,13 +8873,13 @@ v_sub_nc_u16 v5, v1, -1
 // GFX10: encoding: [0x05,0x00,0x04,0xd7,0x01,0x83,0x01,0x00]
 
 v_sub_nc_u16 v5, v1, 0.5
-// GFX10: encoding: [0x05,0x00,0x04,0xd7,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x04,0xd7,0x01,0xe1,0x01,0x00]
 
 v_sub_nc_u16 v5, v1, -4.0
-// GFX10: encoding: [0x05,0x00,0x04,0xd7,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x04,0xd7,0x01,0xef,0x01,0x00]
 
 v_sub_nc_u16 v5, v1, -4.0 clamp
-// GFX10: encoding: [0x05,0x80,0x04,0xd7,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x80,0x04,0xd7,0x01,0xef,0x01,0x00]
 
 v_sub_nc_u16 v5, v1, v2 op_sel:[1,1,1]
 // GFX10: [0x05,0x58,0x04,0xd7,0x01,0x05,0x02,0x00]
@@ -8933,10 +8933,10 @@ v_mul_lo_u16 v5, -1, v2
 // GFX10: encoding: [0x05,0x00,0x05,0xd7,0xc1,0x04,0x02,0x00]
 
 v_mul_lo_u16 v5, 0.5, v2
-// GFX10: encoding: [0x05,0x00,0x05,0xd7,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x05,0xd7,0xf0,0x04,0x02,0x00]
 
 v_mul_lo_u16 v5, -4.0, v2
-// GFX10: encoding: [0x05,0x00,0x05,0xd7,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x05,0xd7,0xf7,0x04,0x02,0x00]
 
 v_mul_lo_u16 v5, v1, v255
 // GFX10: encoding: [0x05,0x00,0x05,0xd7,0x01,0xff,0x03,0x00]
@@ -8969,10 +8969,10 @@ v_mul_lo_u16 v5, v1, -1
 // GFX10: encoding: [0x05,0x00,0x05,0xd7,0x01,0x83,0x01,0x00]
 
 v_mul_lo_u16 v5, v1, 0.5
-// GFX10: encoding: [0x05,0x00,0x05,0xd7,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x05,0xd7,0x01,0xe1,0x01,0x00]
 
 v_mul_lo_u16 v5, v1, -4.0
-// GFX10: encoding: [0x05,0x00,0x05,0xd7,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x05,0xd7,0x01,0xef,0x01,0x00]
 
 v_lshrrev_b16 v5, v1, v2
 // GFX10: encoding: [0x05,0x00,0x07,0xd7,0x01,0x05,0x02,0x00]
@@ -9011,10 +9011,10 @@ v_lshrrev_b16 v5, -1, v2
 // GFX10: encoding: [0x05,0x00,0x07,0xd7,0xc1,0x04,0x02,0x00]
 
 v_lshrrev_b16 v5, 0.5, v2
-// GFX10: encoding: [0x05,0x00,0x07,0xd7,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x07,0xd7,0xf0,0x04,0x02,0x00]
 
 v_lshrrev_b16 v5, -4.0, v2
-// GFX10: encoding: [0x05,0x00,0x07,0xd7,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x07,0xd7,0xf7,0x04,0x02,0x00]
 
 v_lshrrev_b16 v5, v1, v255
 // GFX10: encoding: [0x05,0x00,0x07,0xd7,0x01,0xff,0x03,0x00]
@@ -9047,10 +9047,10 @@ v_lshrrev_b16 v5, v1, -1
 // GFX10: encoding: [0x05,0x00,0x07,0xd7,0x01,0x83,0x01,0x00]
 
 v_lshrrev_b16 v5, v1, 0.5
-// GFX10: encoding: [0x05,0x00,0x07,0xd7,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x07,0xd7,0x01,0xe1,0x01,0x00]
 
 v_lshrrev_b16 v5, v1, -4.0
-// GFX10: encoding: [0x05,0x00,0x07,0xd7,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x07,0xd7,0x01,0xef,0x01,0x00]
 
 v_ashrrev_i16 v5, v1, v2
 // GFX10: encoding: [0x05,0x00,0x08,0xd7,0x01,0x05,0x02,0x00]
@@ -9089,10 +9089,10 @@ v_ashrrev_i16 v5, -1, v2
 // GFX10: encoding: [0x05,0x00,0x08,0xd7,0xc1,0x04,0x02,0x00]
 
 v_ashrrev_i16 v5, 0.5, v2
-// GFX10: encoding: [0x05,0x00,0x08,0xd7,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x08,0xd7,0xf0,0x04,0x02,0x00]
 
 v_ashrrev_i16 v5, -4.0, v2
-// GFX10: encoding: [0x05,0x00,0x08,0xd7,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x08,0xd7,0xf7,0x04,0x02,0x00]
 
 v_ashrrev_i16 v5, v1, v255
 // GFX10: encoding: [0x05,0x00,0x08,0xd7,0x01,0xff,0x03,0x00]
@@ -9125,10 +9125,10 @@ v_ashrrev_i16 v5, v1, -1
 // GFX10: encoding: [0x05,0x00,0x08,0xd7,0x01,0x83,0x01,0x00]
 
 v_ashrrev_i16 v5, v1, 0.5
-// GFX10: encoding: [0x05,0x00,0x08,0xd7,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x08,0xd7,0x01,0xe1,0x01,0x00]
 
 v_ashrrev_i16 v5, v1, -4.0
-// GFX10: encoding: [0x05,0x00,0x08,0xd7,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x08,0xd7,0x01,0xef,0x01,0x00]
 
 v_max_u16 v5, v1, v2
 // GFX10: encoding: [0x05,0x00,0x09,0xd7,0x01,0x05,0x02,0x00]
@@ -9167,10 +9167,10 @@ v_max_u16 v5, -1, v2
 // GFX10: encoding: [0x05,0x00,0x09,0xd7,0xc1,0x04,0x02,0x00]
 
 v_max_u16 v5, 0.5, v2
-// GFX10: encoding: [0x05,0x00,0x09,0xd7,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x09,0xd7,0xf0,0x04,0x02,0x00]
 
 v_max_u16 v5, -4.0, v2
-// GFX10: encoding: [0x05,0x00,0x09,0xd7,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x09,0xd7,0xf7,0x04,0x02,0x00]
 
 v_max_u16 v5, v1, v255
 // GFX10: encoding: [0x05,0x00,0x09,0xd7,0x01,0xff,0x03,0x00]
@@ -9203,10 +9203,10 @@ v_max_u16 v5, v1, -1
 // GFX10: encoding: [0x05,0x00,0x09,0xd7,0x01,0x83,0x01,0x00]
 
 v_max_u16 v5, v1, 0.5
-// GFX10: encoding: [0x05,0x00,0x09,0xd7,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x09,0xd7,0x01,0xe1,0x01,0x00]
 
 v_max_u16 v5, v1, -4.0
-// GFX10: encoding: [0x05,0x00,0x09,0xd7,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x09,0xd7,0x01,0xef,0x01,0x00]
 
 v_max_i16 v5, v1, v2
 // GFX10: encoding: [0x05,0x00,0x0a,0xd7,0x01,0x05,0x02,0x00]
@@ -9245,10 +9245,10 @@ v_max_i16 v5, -1, v2
 // GFX10: encoding: [0x05,0x00,0x0a,0xd7,0xc1,0x04,0x02,0x00]
 
 v_max_i16 v5, 0.5, v2
-// GFX10: encoding: [0x05,0x00,0x0a,0xd7,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x0a,0xd7,0xf0,0x04,0x02,0x00]
 
 v_max_i16 v5, -4.0, v2
-// GFX10: encoding: [0x05,0x00,0x0a,0xd7,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x0a,0xd7,0xf7,0x04,0x02,0x00]
 
 v_max_i16 v5, v1, v255
 // GFX10: encoding: [0x05,0x00,0x0a,0xd7,0x01,0xff,0x03,0x00]
@@ -9281,10 +9281,10 @@ v_max_i16 v5, v1, -1
 // GFX10: encoding: [0x05,0x00,0x0a,0xd7,0x01,0x83,0x01,0x00]
 
 v_max_i16 v5, v1, 0.5
-// GFX10: encoding: [0x05,0x00,0x0a,0xd7,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x0a,0xd7,0x01,0xe1,0x01,0x00]
 
 v_max_i16 v5, v1, -4.0
-// GFX10: encoding: [0x05,0x00,0x0a,0xd7,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x0a,0xd7,0x01,0xef,0x01,0x00]
 
 v_min_u16 v5, v1, v2
 // GFX10: encoding: [0x05,0x00,0x0b,0xd7,0x01,0x05,0x02,0x00]
@@ -9323,10 +9323,10 @@ v_min_u16 v5, -1, v2
 // GFX10: encoding: [0x05,0x00,0x0b,0xd7,0xc1,0x04,0x02,0x00]
 
 v_min_u16 v5, 0.5, v2
-// GFX10: encoding: [0x05,0x00,0x0b,0xd7,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x0b,0xd7,0xf0,0x04,0x02,0x00]
 
 v_min_u16 v5, -4.0, v2
-// GFX10: encoding: [0x05,0x00,0x0b,0xd7,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x0b,0xd7,0xf7,0x04,0x02,0x00]
 
 v_min_u16 v5, v1, v255
 // GFX10: encoding: [0x05,0x00,0x0b,0xd7,0x01,0xff,0x03,0x00]
@@ -9359,10 +9359,10 @@ v_min_u16 v5, v1, -1
 // GFX10: encoding: [0x05,0x00,0x0b,0xd7,0x01,0x83,0x01,0x00]
 
 v_min_u16 v5, v1, 0.5
-// GFX10: encoding: [0x05,0x00,0x0b,0xd7,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x0b,0xd7,0x01,0xe1,0x01,0x00]
 
 v_min_u16 v5, v1, -4.0
-// GFX10: encoding: [0x05,0x00,0x0b,0xd7,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x0b,0xd7,0x01,0xef,0x01,0x00]
 
 v_min_i16 v5, v1, v2
 // GFX10: encoding: [0x05,0x00,0x0c,0xd7,0x01,0x05,0x02,0x00]
@@ -9401,10 +9401,10 @@ v_min_i16 v5, -1, v2
 // GFX10: encoding: [0x05,0x00,0x0c,0xd7,0xc1,0x04,0x02,0x00]
 
 v_min_i16 v5, 0.5, v2
-// GFX10: encoding: [0x05,0x00,0x0c,0xd7,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x0c,0xd7,0xf0,0x04,0x02,0x00]
 
 v_min_i16 v5, -4.0, v2
-// GFX10: encoding: [0x05,0x00,0x0c,0xd7,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x0c,0xd7,0xf7,0x04,0x02,0x00]
 
 v_min_i16 v5, v1, v255
 // GFX10: encoding: [0x05,0x00,0x0c,0xd7,0x01,0xff,0x03,0x00]
@@ -9437,10 +9437,10 @@ v_min_i16 v5, v1, -1
 // GFX10: encoding: [0x05,0x00,0x0c,0xd7,0x01,0x83,0x01,0x00]
 
 v_min_i16 v5, v1, 0.5
-// GFX10: encoding: [0x05,0x00,0x0c,0xd7,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x0c,0xd7,0x01,0xe1,0x01,0x00]
 
 v_min_i16 v5, v1, -4.0
-// GFX10: encoding: [0x05,0x00,0x0c,0xd7,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x0c,0xd7,0x01,0xef,0x01,0x00]
 
 v_add_nc_i16 v5, v1, v2
 // GFX10: encoding: [0x05,0x00,0x0d,0xd7,0x01,0x05,0x02,0x00]
@@ -9479,10 +9479,10 @@ v_add_nc_i16 v5, -1, v2
 // GFX10: encoding: [0x05,0x00,0x0d,0xd7,0xc1,0x04,0x02,0x00]
 
 v_add_nc_i16 v5, 0.5, v2
-// GFX10: encoding: [0x05,0x00,0x0d,0xd7,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x0d,0xd7,0xf0,0x04,0x02,0x00]
 
 v_add_nc_i16 v5, -4.0, v2
-// GFX10: encoding: [0x05,0x00,0x0d,0xd7,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x0d,0xd7,0xf7,0x04,0x02,0x00]
 
 v_add_nc_i16 v5, v1, v255
 // GFX10: encoding: [0x05,0x00,0x0d,0xd7,0x01,0xff,0x03,0x00]
@@ -9515,10 +9515,10 @@ v_add_nc_i16 v5, v1, -1
 // GFX10: encoding: [0x05,0x00,0x0d,0xd7,0x01,0x83,0x01,0x00]
 
 v_add_nc_i16 v5, v1, 0.5
-// GFX10: encoding: [0x05,0x00,0x0d,0xd7,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x0d,0xd7,0x01,0xe1,0x01,0x00]
 
 v_add_nc_i16 v5, v1, -4.0
-// GFX10: encoding: [0x05,0x00,0x0d,0xd7,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x0d,0xd7,0x01,0xef,0x01,0x00]
 
 v_sub_nc_i16 v5, v1, v2
 // GFX10: encoding: [0x05,0x00,0x0e,0xd7,0x01,0x05,0x02,0x00]
@@ -9557,10 +9557,10 @@ v_sub_nc_i16 v5, -1, v2
 // GFX10: encoding: [0x05,0x00,0x0e,0xd7,0xc1,0x04,0x02,0x00]
 
 v_sub_nc_i16 v5, 0.5, v2
-// GFX10: encoding: [0x05,0x00,0x0e,0xd7,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x0e,0xd7,0xf0,0x04,0x02,0x00]
 
 v_sub_nc_i16 v5, -4.0, v2
-// GFX10: encoding: [0x05,0x00,0x0e,0xd7,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x0e,0xd7,0xf7,0x04,0x02,0x00]
 
 v_sub_nc_i16 v5, v1, v255
 // GFX10: encoding: [0x05,0x00,0x0e,0xd7,0x01,0xff,0x03,0x00]
@@ -9593,10 +9593,10 @@ v_sub_nc_i16 v5, v1, -1
 // GFX10: encoding: [0x05,0x00,0x0e,0xd7,0x01,0x83,0x01,0x00]
 
 v_sub_nc_i16 v5, v1, 0.5
-// GFX10: encoding: [0x05,0x00,0x0e,0xd7,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x0e,0xd7,0x01,0xe1,0x01,0x00]
 
 v_sub_nc_i16 v5, v1, -4.0
-// GFX10: encoding: [0x05,0x00,0x0e,0xd7,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x0e,0xd7,0x01,0xef,0x01,0x00]
 
 v_pack_b32_f16 v5, v1, v2
 // GFX10: encoding: [0x05,0x00,0x11,0xd7,0x01,0x05,0x02,0x00]
@@ -9968,10 +9968,10 @@ v_lshlrev_b16 v5, -1, v2
 // GFX10: encoding: [0x05,0x00,0x14,0xd7,0xc1,0x04,0x02,0x00]
 
 v_lshlrev_b16 v5, 0.5, v2
-// GFX10: encoding: [0x05,0x00,0x14,0xd7,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x14,0xd7,0xf0,0x04,0x02,0x00]
 
 v_lshlrev_b16 v5, -4.0, v2
-// GFX10: encoding: [0x05,0x00,0x14,0xd7,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x14,0xd7,0xf7,0x04,0x02,0x00]
 
 v_lshlrev_b16 v5, v1, v255
 // GFX10: encoding: [0x05,0x00,0x14,0xd7,0x01,0xff,0x03,0x00]
@@ -10004,10 +10004,10 @@ v_lshlrev_b16 v5, v1, -1
 // GFX10: encoding: [0x05,0x00,0x14,0xd7,0x01,0x83,0x01,0x00]
 
 v_lshlrev_b16 v5, v1, 0.5
-// GFX10: encoding: [0x05,0x00,0x14,0xd7,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x14,0xd7,0x01,0xe1,0x01,0x00]
 
 v_lshlrev_b16 v5, v1, -4.0
-// GFX10: encoding: [0x05,0x00,0x14,0xd7,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x14,0xd7,0x01,0xef,0x01,0x00]
 
 v_mad_u16 v5, 0, v2, v3
 // GFX10: encoding: [0x05,0x00,0x40,0xd7,0x80,0x04,0x0e,0x04]
@@ -10016,7 +10016,7 @@ v_mad_u16 v5, v1, -1, v3
 // GFX10: encoding: [0x05,0x00,0x40,0xd7,0x01,0x83,0x0d,0x04]
 
 v_mad_u16 v5, v1, v2, -4.0
-// GFX10: encoding: [0x05,0x00,0x40,0xd7,0x01,0x05,0xfe,0x03,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x40,0xd7,0x01,0x05,0xde,0x03]
 
 v_mad_u16 v5, v1, v2, v3 clamp
 // GFX10: encoding: [0x05,0x80,0x40,0xd7,0x01,0x05,0x0e,0x04]
@@ -10755,10 +10755,10 @@ v_min3_i16 v5, -1, v2, v3
 // GFX10: encoding: [0x05,0x00,0x52,0xd7,0xc1,0x04,0x0e,0x04]
 
 v_min3_i16 v5, 0.5, v2, v3
-// GFX10: encoding: [0x05,0x00,0x52,0xd7,0xff,0x04,0x0e,0x04,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x52,0xd7,0xf0,0x04,0x0e,0x04]
 
 v_min3_i16 v5, -4.0, v2, v3
-// GFX10: encoding: [0x05,0x00,0x52,0xd7,0xff,0x04,0x0e,0x04,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x52,0xd7,0xf7,0x04,0x0e,0x04]
 
 v_min3_i16 v5, v1, v255, v3
 // GFX10: encoding: [0x05,0x00,0x52,0xd7,0x01,0xff,0x0f,0x04]
@@ -10791,10 +10791,10 @@ v_min3_i16 v5, v1, -1, v3
 // GFX10: encoding: [0x05,0x00,0x52,0xd7,0x01,0x83,0x0d,0x04]
 
 v_min3_i16 v5, v1, 0.5, v3
-// GFX10: encoding: [0x05,0x00,0x52,0xd7,0x01,0xff,0x0d,0x04,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x52,0xd7,0x01,0xe1,0x0d,0x04]
 
 v_min3_i16 v5, v1, -4.0, v3
-// GFX10: encoding: [0x05,0x00,0x52,0xd7,0x01,0xff,0x0d,0x04,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x52,0xd7,0x01,0xef,0x0d,0x04]
 
 v_min3_i16 v5, v1, v2, v255
 // GFX10: encoding: [0x05,0x00,0x52,0xd7,0x01,0x05,0xfe,0x07]
@@ -10827,10 +10827,10 @@ v_min3_i16 v5, v1, v2, -1
 // GFX10: encoding: [0x05,0x00,0x52,0xd7,0x01,0x05,0x06,0x03]
 
 v_min3_i16 v5, v1, v2, 0.5
-// GFX10: encoding: [0x05,0x00,0x52,0xd7,0x01,0x05,0xfe,0x03,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x52,0xd7,0x01,0x05,0xc2,0x03]
 
 v_min3_i16 v5, v1, v2, -4.0
-// GFX10: encoding: [0x05,0x00,0x52,0xd7,0x01,0x05,0xfe,0x03,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x52,0xd7,0x01,0x05,0xde,0x03]
 
 v_min3_i16 v5, v1, v2, v3 op_sel:[0,0,0,0]
 // GFX10: encoding: [0x05,0x00,0x52,0xd7,0x01,0x05,0x0e,0x04]
@@ -10890,10 +10890,10 @@ v_min3_u16 v5, -1, v2, v3
 // GFX10: encoding: [0x05,0x00,0x53,0xd7,0xc1,0x04,0x0e,0x04]
 
 v_min3_u16 v5, 0.5, v2, v3
-// GFX10: encoding: [0x05,0x00,0x53,0xd7,0xff,0x04,0x0e,0x04,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x53,0xd7,0xf0,0x04,0x0e,0x04]
 
 v_min3_u16 v5, -4.0, v2, v3
-// GFX10: encoding: [0x05,0x00,0x53,0xd7,0xff,0x04,0x0e,0x04,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x53,0xd7,0xf7,0x04,0x0e,0x04]
 
 v_min3_u16 v5, v1, v255, v3
 // GFX10: encoding: [0x05,0x00,0x53,0xd7,0x01,0xff,0x0f,0x04]
@@ -10926,10 +10926,10 @@ v_min3_u16 v5, v1, -1, v3
 // GFX10: encoding: [0x05,0x00,0x53,0xd7,0x01,0x83,0x0d,0x04]
 
 v_min3_u16 v5, v1, 0.5, v3
-// GFX10: encoding: [0x05,0x00,0x53,0xd7,0x01,0xff,0x0d,0x04,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x53,0xd7,0x01,0xe1,0x0d,0x04]
 
 v_min3_u16 v5, v1, -4.0, v3
-// GFX10: encoding: [0x05,0x00,0x53,0xd7,0x01,0xff,0x0d,0x04,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x53,0xd7,0x01,0xef,0x0d,0x04]
 
 v_min3_u16 v5, v1, v2, v255
 // GFX10: encoding: [0x05,0x00,0x53,0xd7,0x01,0x05,0xfe,0x07]
@@ -10962,10 +10962,10 @@ v_min3_u16 v5, v1, v2, -1
 // GFX10: encoding: [0x05,0x00,0x53,0xd7,0x01,0x05,0x06,0x03]
 
 v_min3_u16 v5, v1, v2, 0.5
-// GFX10: encoding: [0x05,0x00,0x53,0xd7,0x01,0x05,0xfe,0x03,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x53,0xd7,0x01,0x05,0xc2,0x03]
 
 v_min3_u16 v5, v1, v2, -4.0
-// GFX10: encoding: [0x05,0x00,0x53,0xd7,0x01,0x05,0xfe,0x03,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x53,0xd7,0x01,0x05,0xde,0x03]
 
 v_min3_u16 v5, v1, v2, v3 op_sel:[0,0,0,0]
 // GFX10: encoding: [0x05,0x00,0x53,0xd7,0x01,0x05,0x0e,0x04]
@@ -11184,10 +11184,10 @@ v_max3_i16 v5, -1, v2, v3
 // GFX10: encoding: [0x05,0x00,0x55,0xd7,0xc1,0x04,0x0e,0x04]
 
 v_max3_i16 v5, 0.5, v2, v3
-// GFX10: encoding: [0x05,0x00,0x55,0xd7,0xff,0x04,0x0e,0x04,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x55,0xd7,0xf0,0x04,0x0e,0x04]
 
 v_max3_i16 v5, -4.0, v2, v3
-// GFX10: encoding: [0x05,0x00,0x55,0xd7,0xff,0x04,0x0e,0x04,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x55,0xd7,0xf7,0x04,0x0e,0x04]
 
 v_max3_i16 v5, v1, v255, v3
 // GFX10: encoding: [0x05,0x00,0x55,0xd7,0x01,0xff,0x0f,0x04]
@@ -11220,10 +11220,10 @@ v_max3_i16 v5, v1, -1, v3
 // GFX10: encoding: [0x05,0x00,0x55,0xd7,0x01,0x83,0x0d,0x04]
 
 v_max3_i16 v5, v1, 0.5, v3
-// GFX10: encoding: [0x05,0x00,0x55,0xd7,0x01,0xff,0x0d,0x04,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x55,0xd7,0x01,0xe1,0x0d,0x04]
 
 v_max3_i16 v5, v1, -4.0, v3
-// GFX10: encoding: [0x05,0x00,0x55,0xd7,0x01,0xff,0x0d,0x04,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x55,0xd7,0x01,0xef,0x0d,0x04]
 
 v_max3_i16 v5, v1, v2, v255
 // GFX10: encoding: [0x05,0x00,0x55,0xd7,0x01,0x05,0xfe,0x07]
@@ -11256,10 +11256,10 @@ v_max3_i16 v5, v1, v2, -1
 // GFX10: encoding: [0x05,0x00,0x55,0xd7,0x01,0x05,0x06,0x03]
 
 v_max3_i16 v5, v1, v2, 0.5
-// GFX10: encoding: [0x05,0x00,0x55,0xd7,0x01,0x05,0xfe,0x03,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x55,0xd7,0x01,0x05,0xc2,0x03]
 
 v_max3_i16 v5, v1, v2, -4.0
-// GFX10: encoding: [0x05,0x00,0x55,0xd7,0x01,0x05,0xfe,0x03,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x55,0xd7,0x01,0x05,0xde,0x03]
 
 v_max3_i16 v5, v1, v2, v3 op_sel:[0,0,0,0]
 // GFX10: encoding: [0x05,0x00,0x55,0xd7,0x01,0x05,0x0e,0x04]
@@ -11319,10 +11319,10 @@ v_max3_u16 v5, -1, v2, v3
 // GFX10: encoding: [0x05,0x00,0x56,0xd7,0xc1,0x04,0x0e,0x04]
 
 v_max3_u16 v5, 0.5, v2, v3
-// GFX10: encoding: [0x05,0x00,0x56,0xd7,0xff,0x04,0x0e,0x04,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x56,0xd7,0xf0,0x04,0x0e,0x04]
 
 v_max3_u16 v5, -4.0, v2, v3
-// GFX10: encoding: [0x05,0x00,0x56,0xd7,0xff,0x04,0x0e,0x04,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x56,0xd7,0xf7,0x04,0x0e,0x04]
 
 v_max3_u16 v5, v1, v255, v3
 // GFX10: encoding: [0x05,0x00,0x56,0xd7,0x01,0xff,0x0f,0x04]
@@ -11355,10 +11355,10 @@ v_max3_u16 v5, v1, -1, v3
 // GFX10: encoding: [0x05,0x00,0x56,0xd7,0x01,0x83,0x0d,0x04]
 
 v_max3_u16 v5, v1, 0.5, v3
-// GFX10: encoding: [0x05,0x00,0x56,0xd7,0x01,0xff,0x0d,0x04,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x56,0xd7,0x01,0xe1,0x0d,0x04]
 
 v_max3_u16 v5, v1, -4.0, v3
-// GFX10: encoding: [0x05,0x00,0x56,0xd7,0x01,0xff,0x0d,0x04,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x56,0xd7,0x01,0xef,0x0d,0x04]
 
 v_max3_u16 v5, v1, v2, v255
 // GFX10: encoding: [0x05,0x00,0x56,0xd7,0x01,0x05,0xfe,0x07]
@@ -11391,10 +11391,10 @@ v_max3_u16 v5, v1, v2, -1
 // GFX10: encoding: [0x05,0x00,0x56,0xd7,0x01,0x05,0x06,0x03]
 
 v_max3_u16 v5, v1, v2, 0.5
-// GFX10: encoding: [0x05,0x00,0x56,0xd7,0x01,0x05,0xfe,0x03,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x56,0xd7,0x01,0x05,0xc2,0x03]
 
 v_max3_u16 v5, v1, v2, -4.0
-// GFX10: encoding: [0x05,0x00,0x56,0xd7,0x01,0x05,0xfe,0x03,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x56,0xd7,0x01,0x05,0xde,0x03]
 
 v_max3_u16 v5, v1, v2, v3 op_sel:[0,0,0,0]
 // GFX10: encoding: [0x05,0x00,0x56,0xd7,0x01,0x05,0x0e,0x04]
@@ -11613,10 +11613,10 @@ v_med3_i16 v5, -1, v2, v3
 // GFX10: encoding: [0x05,0x00,0x58,0xd7,0xc1,0x04,0x0e,0x04]
 
 v_med3_i16 v5, 0.5, v2, v3
-// GFX10: encoding: [0x05,0x00,0x58,0xd7,0xff,0x04,0x0e,0x04,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x58,0xd7,0xf0,0x04,0x0e,0x04]
 
 v_med3_i16 v5, -4.0, v2, v3
-// GFX10: encoding: [0x05,0x00,0x58,0xd7,0xff,0x04,0x0e,0x04,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x58,0xd7,0xf7,0x04,0x0e,0x04]
 
 v_med3_i16 v5, v1, v255, v3
 // GFX10: encoding: [0x05,0x00,0x58,0xd7,0x01,0xff,0x0f,0x04]
@@ -11649,10 +11649,10 @@ v_med3_i16 v5, v1, -1, v3
 // GFX10: encoding: [0x05,0x00,0x58,0xd7,0x01,0x83,0x0d,0x04]
 
 v_med3_i16 v5, v1, 0.5, v3
-// GFX10: encoding: [0x05,0x00,0x58,0xd7,0x01,0xff,0x0d,0x04,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x58,0xd7,0x01,0xe1,0x0d,0x04]
 
 v_med3_i16 v5, v1, -4.0, v3
-// GFX10: encoding: [0x05,0x00,0x58,0xd7,0x01,0xff,0x0d,0x04,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x58,0xd7,0x01,0xef,0x0d,0x04]
 
 v_med3_i16 v5, v1, v2, v255
 // GFX10: encoding: [0x05,0x00,0x58,0xd7,0x01,0x05,0xfe,0x07]
@@ -11685,10 +11685,10 @@ v_med3_i16 v5, v1, v2, -1
 // GFX10: encoding: [0x05,0x00,0x58,0xd7,0x01,0x05,0x06,0x03]
 
 v_med3_i16 v5, v1, v2, 0.5
-// GFX10: encoding: [0x05,0x00,0x58,0xd7,0x01,0x05,0xfe,0x03,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x58,0xd7,0x01,0x05,0xc2,0x03]
 
 v_med3_i16 v5, v1, v2, -4.0
-// GFX10: encoding: [0x05,0x00,0x58,0xd7,0x01,0x05,0xfe,0x03,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x58,0xd7,0x01,0x05,0xde,0x03]
 
 v_med3_i16 v5, v1, v2, v3 op_sel:[0,0,0,0]
 // GFX10: encoding: [0x05,0x00,0x58,0xd7,0x01,0x05,0x0e,0x04]
@@ -11748,10 +11748,10 @@ v_med3_u16 v5, -1, v2, v3
 // GFX10: encoding: [0x05,0x00,0x59,0xd7,0xc1,0x04,0x0e,0x04]
 
 v_med3_u16 v5, 0.5, v2, v3
-// GFX10: encoding: [0x05,0x00,0x59,0xd7,0xff,0x04,0x0e,0x04,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x59,0xd7,0xf0,0x04,0x0e,0x04]
 
 v_med3_u16 v5, -4.0, v2, v3
-// GFX10: encoding: [0x05,0x00,0x59,0xd7,0xff,0x04,0x0e,0x04,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x59,0xd7,0xf7,0x04,0x0e,0x04]
 
 v_med3_u16 v5, v1, v255, v3
 // GFX10: encoding: [0x05,0x00,0x59,0xd7,0x01,0xff,0x0f,0x04]
@@ -11784,10 +11784,10 @@ v_med3_u16 v5, v1, -1, v3
 // GFX10: encoding: [0x05,0x00,0x59,0xd7,0x01,0x83,0x0d,0x04]
 
 v_med3_u16 v5, v1, 0.5, v3
-// GFX10: encoding: [0x05,0x00,0x59,0xd7,0x01,0xff,0x0d,0x04,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x59,0xd7,0x01,0xe1,0x0d,0x04]
 
 v_med3_u16 v5, v1, -4.0, v3
-// GFX10: encoding: [0x05,0x00,0x59,0xd7,0x01,0xff,0x0d,0x04,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x59,0xd7,0x01,0xef,0x0d,0x04]
 
 v_med3_u16 v5, v1, v2, v255
 // GFX10: encoding: [0x05,0x00,0x59,0xd7,0x01,0x05,0xfe,0x07]
@@ -11820,10 +11820,10 @@ v_med3_u16 v5, v1, v2, -1
 // GFX10: encoding: [0x05,0x00,0x59,0xd7,0x01,0x05,0x06,0x03]
 
 v_med3_u16 v5, v1, v2, 0.5
-// GFX10: encoding: [0x05,0x00,0x59,0xd7,0x01,0x05,0xfe,0x03,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x59,0xd7,0x01,0x05,0xc2,0x03]
 
 v_med3_u16 v5, v1, v2, -4.0
-// GFX10: encoding: [0x05,0x00,0x59,0xd7,0x01,0x05,0xfe,0x03,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x59,0xd7,0x01,0x05,0xde,0x03]
 
 v_med3_u16 v5, v1, v2, v3 op_sel:[0,0,0,0]
 // GFX10: encoding: [0x05,0x00,0x59,0xd7,0x01,0x05,0x0e,0x04]
@@ -11868,7 +11868,7 @@ v_mad_i16 v5, v1, -1, v3
 // GFX10: encoding: [0x05,0x00,0x5e,0xd7,0x01,0x83,0x0d,0x04]
 
 v_mad_i16 v5, v1, v2, -4.0
-// GFX10: encoding: [0x05,0x00,0x5e,0xd7,0x01,0x05,0xfe,0x03,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x5e,0xd7,0x01,0x05,0xde,0x03]
 
 v_mad_i16 v5, v1, v2, v3 clamp
 // GFX10: encoding: [0x05,0x80,0x5e,0xd7,0x01,0x05,0x0e,0x04]
@@ -12399,10 +12399,10 @@ v_mad_u32_u16 v5, -1, v2, v3
 // GFX10: encoding: [0x05,0x00,0x73,0xd7,0xc1,0x04,0x0e,0x04]
 
 v_mad_u32_u16 v5, 0.5, v2, v3
-// GFX10: encoding: [0x05,0x00,0x73,0xd7,0xff,0x04,0x0e,0x04,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x73,0xd7,0xf0,0x04,0x0e,0x04]
 
 v_mad_u32_u16 v5, -4.0, v2, v3
-// GFX10: encoding: [0x05,0x00,0x73,0xd7,0xff,0x04,0x0e,0x04,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x73,0xd7,0xf7,0x04,0x0e,0x04]
 
 v_mad_u32_u16 v5, v1, v255, v3
 // GFX10: encoding: [0x05,0x00,0x73,0xd7,0x01,0xff,0x0f,0x04]
@@ -12435,10 +12435,10 @@ v_mad_u32_u16 v5, v1, -1, v3
 // GFX10: encoding: [0x05,0x00,0x73,0xd7,0x01,0x83,0x0d,0x04]
 
 v_mad_u32_u16 v5, v1, 0.5, v3
-// GFX10: encoding: [0x05,0x00,0x73,0xd7,0x01,0xff,0x0d,0x04,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x73,0xd7,0x01,0xe1,0x0d,0x04]
 
 v_mad_u32_u16 v5, v1, -4.0, v3
-// GFX10: encoding: [0x05,0x00,0x73,0xd7,0x01,0xff,0x0d,0x04,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x73,0xd7,0x01,0xef,0x0d,0x04]
 
 v_mad_u32_u16 v5, v1, v2, v255
 // GFX10: encoding: [0x05,0x00,0x73,0xd7,0x01,0x05,0xfe,0x07]
@@ -12534,10 +12534,10 @@ v_mad_i32_i16 v5, -1, v2, v3
 // GFX10: encoding: [0x05,0x00,0x75,0xd7,0xc1,0x04,0x0e,0x04]
 
 v_mad_i32_i16 v5, 0.5, v2, v3
-// GFX10: encoding: [0x05,0x00,0x75,0xd7,0xff,0x04,0x0e,0x04,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x75,0xd7,0xf0,0x04,0x0e,0x04]
 
 v_mad_i32_i16 v5, -4.0, v2, v3
-// GFX10: encoding: [0x05,0x00,0x75,0xd7,0xff,0x04,0x0e,0x04,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x75,0xd7,0xf7,0x04,0x0e,0x04]
 
 v_mad_i32_i16 v5, v1, v255, v3
 // GFX10: encoding: [0x05,0x00,0x75,0xd7,0x01,0xff,0x0f,0x04]
@@ -12570,10 +12570,10 @@ v_mad_i32_i16 v5, v1, -1, v3
 // GFX10: encoding: [0x05,0x00,0x75,0xd7,0x01,0x83,0x0d,0x04]
 
 v_mad_i32_i16 v5, v1, 0.5, v3
-// GFX10: encoding: [0x05,0x00,0x75,0xd7,0x01,0xff,0x0d,0x04,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x75,0xd7,0x01,0xe1,0x0d,0x04]
 
 v_mad_i32_i16 v5, v1, -4.0, v3
-// GFX10: encoding: [0x05,0x00,0x75,0xd7,0x01,0xff,0x0d,0x04,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x75,0xd7,0x01,0xef,0x0d,0x04]
 
 v_mad_i32_i16 v5, v1, v2, v255
 // GFX10: encoding: [0x05,0x00,0x75,0xd7,0x01,0x05,0xfe,0x07]
diff --git a/llvm/test/MC/AMDGPU/gfx10_asm_vopc.s b/llvm/test/MC/AMDGPU/gfx10_asm_vopc.s
index 521394829069..40618d1ea523 100644
--- a/llvm/test/MC/AMDGPU/gfx10_asm_vopc.s
+++ b/llvm/test/MC/AMDGPU/gfx10_asm_vopc.s
@@ -5044,11 +5044,11 @@ v_cmp_lt_i16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_i16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x12,0x7d,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x12,0x7d]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_i16 vcc, -4.0, v2
-// W64: encoding: [0xff,0x04,0x12,0x7d,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0xf7,0x04,0x12,0x7d]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_i16 vcc, 0xfe0b, v2
@@ -5108,11 +5108,11 @@ v_cmp_eq_i16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_i16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x14,0x7d,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x14,0x7d]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_i16 vcc, -4.0, v2
-// W64: encoding: [0xff,0x04,0x14,0x7d,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0xf7,0x04,0x14,0x7d]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_i16 vcc, 0xfe0b, v2
@@ -5172,11 +5172,11 @@ v_cmp_le_i16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_le_i16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x16,0x7d,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x16,0x7d]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_le_i16 vcc, -4.0, v2
-// W64: encoding: [0xff,0x04,0x16,0x7d,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0xf7,0x04,0x16,0x7d]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_le_i16 vcc, 0xfe0b, v2
@@ -5236,11 +5236,11 @@ v_cmp_gt_i16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_i16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x18,0x7d,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x18,0x7d]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_i16 vcc, -4.0, v2
-// W64: encoding: [0xff,0x04,0x18,0x7d,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0xf7,0x04,0x18,0x7d]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_i16 vcc, 0xfe0b, v2
@@ -5300,11 +5300,11 @@ v_cmp_ne_i16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_i16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x1a,0x7d,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x1a,0x7d]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_i16 vcc, -4.0, v2
-// W64: encoding: [0xff,0x04,0x1a,0x7d,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0xf7,0x04,0x1a,0x7d]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_i16 vcc, 0xfe0b, v2
@@ -5364,11 +5364,11 @@ v_cmp_ge_i16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_i16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x1c,0x7d,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x1c,0x7d]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_i16 vcc, -4.0, v2
-// W64: encoding: [0xff,0x04,0x1c,0x7d,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0xf7,0x04,0x1c,0x7d]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_i16 vcc, 0xfe0b, v2
@@ -5428,11 +5428,11 @@ v_cmp_lt_i16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_i16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x12,0x7d,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x12,0x7d]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_i16 vcc_lo, -4.0, v2
-// W32: encoding: [0xff,0x04,0x12,0x7d,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0xf7,0x04,0x12,0x7d]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_i16 vcc_lo, 0xfe0b, v2
@@ -5492,11 +5492,11 @@ v_cmp_eq_i16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_i16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x14,0x7d,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x14,0x7d]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_i16 vcc_lo, -4.0, v2
-// W32: encoding: [0xff,0x04,0x14,0x7d,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0xf7,0x04,0x14,0x7d]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_i16 vcc_lo, 0xfe0b, v2
@@ -5556,11 +5556,11 @@ v_cmp_le_i16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_le_i16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x16,0x7d,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x16,0x7d]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_le_i16 vcc_lo, -4.0, v2
-// W32: encoding: [0xff,0x04,0x16,0x7d,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0xf7,0x04,0x16,0x7d]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_le_i16 vcc_lo, 0xfe0b, v2
@@ -5620,11 +5620,11 @@ v_cmp_gt_i16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_i16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x18,0x7d,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x18,0x7d]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_i16 vcc_lo, -4.0, v2
-// W32: encoding: [0xff,0x04,0x18,0x7d,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0xf7,0x04,0x18,0x7d]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_i16 vcc_lo, 0xfe0b, v2
@@ -5684,11 +5684,11 @@ v_cmp_ne_i16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_i16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x1a,0x7d,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x1a,0x7d]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_i16 vcc_lo, -4.0, v2
-// W32: encoding: [0xff,0x04,0x1a,0x7d,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0xf7,0x04,0x1a,0x7d]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_i16 vcc_lo, 0xfe0b, v2
@@ -5748,11 +5748,11 @@ v_cmp_ge_i16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_i16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x1c,0x7d,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x1c,0x7d]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_i16 vcc_lo, -4.0, v2
-// W32: encoding: [0xff,0x04,0x1c,0x7d,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0xf7,0x04,0x1c,0x7d]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_i16 vcc_lo, 0xfe0b, v2
@@ -7780,11 +7780,11 @@ v_cmp_lt_u16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_u16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x52,0x7d,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x52,0x7d]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_u16 vcc, -4.0, v2
-// W64: encoding: [0xff,0x04,0x52,0x7d,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0xf7,0x04,0x52,0x7d]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_u16 vcc, 0xfe0b, v2
@@ -7844,11 +7844,11 @@ v_cmp_eq_u16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_u16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x54,0x7d,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x54,0x7d]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_u16 vcc, -4.0, v2
-// W64: encoding: [0xff,0x04,0x54,0x7d,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0xf7,0x04,0x54,0x7d]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_u16 vcc, 0xfe0b, v2
@@ -7908,11 +7908,11 @@ v_cmp_le_u16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_le_u16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x56,0x7d,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x56,0x7d]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_le_u16 vcc, -4.0, v2
-// W64: encoding: [0xff,0x04,0x56,0x7d,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0xf7,0x04,0x56,0x7d]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_le_u16 vcc, 0xfe0b, v2
@@ -7972,11 +7972,11 @@ v_cmp_gt_u16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_u16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x58,0x7d,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x58,0x7d]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_u16 vcc, -4.0, v2
-// W64: encoding: [0xff,0x04,0x58,0x7d,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0xf7,0x04,0x58,0x7d]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_u16 vcc, 0xfe0b, v2
@@ -8036,11 +8036,11 @@ v_cmp_ne_u16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_u16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x5a,0x7d,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x5a,0x7d]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_u16 vcc, -4.0, v2
-// W64: encoding: [0xff,0x04,0x5a,0x7d,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0xf7,0x04,0x5a,0x7d]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_u16 vcc, 0xfe0b, v2
@@ -8100,11 +8100,11 @@ v_cmp_ge_u16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_u16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x5c,0x7d,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x5c,0x7d]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_u16 vcc, -4.0, v2
-// W64: encoding: [0xff,0x04,0x5c,0x7d,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0xf7,0x04,0x5c,0x7d]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_u16 vcc, 0xfe0b, v2
@@ -9700,11 +9700,11 @@ v_cmp_lt_u16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_u16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x52,0x7d,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x52,0x7d]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_u16 vcc_lo, -4.0, v2
-// W32: encoding: [0xff,0x04,0x52,0x7d,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0xf7,0x04,0x52,0x7d]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_u16 vcc_lo, 0xfe0b, v2
@@ -9764,11 +9764,11 @@ v_cmp_eq_u16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_u16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x54,0x7d,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x54,0x7d]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_u16 vcc_lo, -4.0, v2
-// W32: encoding: [0xff,0x04,0x54,0x7d,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0xf7,0x04,0x54,0x7d]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_u16 vcc_lo, 0xfe0b, v2
@@ -9828,11 +9828,11 @@ v_cmp_le_u16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_le_u16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x56,0x7d,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x56,0x7d]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_le_u16 vcc_lo, -4.0, v2
-// W32: encoding: [0xff,0x04,0x56,0x7d,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0xf7,0x04,0x56,0x7d]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_le_u16 vcc_lo, 0xfe0b, v2
@@ -9892,11 +9892,11 @@ v_cmp_gt_u16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_u16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x58,0x7d,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x58,0x7d]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_u16 vcc_lo, -4.0, v2
-// W32: encoding: [0xff,0x04,0x58,0x7d,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0xf7,0x04,0x58,0x7d]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_u16 vcc_lo, 0xfe0b, v2
@@ -9956,11 +9956,11 @@ v_cmp_ne_u16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_u16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x5a,0x7d,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x5a,0x7d]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_u16 vcc_lo, -4.0, v2
-// W32: encoding: [0xff,0x04,0x5a,0x7d,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0xf7,0x04,0x5a,0x7d]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_u16 vcc_lo, 0xfe0b, v2
@@ -10020,11 +10020,11 @@ v_cmp_ge_u16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_u16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x5c,0x7d,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x5c,0x7d]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_u16 vcc_lo, -4.0, v2
-// W32: encoding: [0xff,0x04,0x5c,0x7d,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0xf7,0x04,0x5c,0x7d]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_u16 vcc_lo, 0xfe0b, v2
diff --git a/llvm/test/MC/AMDGPU/gfx10_asm_vopc_e64.s b/llvm/test/MC/AMDGPU/gfx10_asm_vopc_e64.s
index fc8fe95a9cf2..669e79cdab10 100644
--- a/llvm/test/MC/AMDGPU/gfx10_asm_vopc_e64.s
+++ b/llvm/test/MC/AMDGPU/gfx10_asm_vopc_e64.s
@@ -9768,11 +9768,11 @@ v_cmp_lt_i16_e64 s[10:11], -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_i16_e64 s[10:11], 0.5, v2
-// W64: encoding: [0x0a,0x00,0x89,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x89,0xd4,0xf0,0x04,0x02,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_i16_e64 s[10:11], -4.0, v2
-// W64: encoding: [0x0a,0x00,0x89,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x89,0xd4,0xf7,0x04,0x02,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_i16_e64 s[10:11], v1, v255
@@ -9816,11 +9816,11 @@ v_cmp_lt_i16_e64 s[10:11], v1, -1
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_i16_e64 s[10:11], v1, 0.5
-// W64: encoding: [0x0a,0x00,0x89,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x89,0xd4,0x01,0xe1,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_i16_e64 s[10:11], v1, -4.0
-// W64: encoding: [0x0a,0x00,0x89,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x89,0xd4,0x01,0xef,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_i16_e64 s[10:11], v1, v2
@@ -9880,11 +9880,11 @@ v_cmp_eq_i16_e64 s[10:11], -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_i16_e64 s[10:11], 0.5, v2
-// W64: encoding: [0x0a,0x00,0x8a,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x8a,0xd4,0xf0,0x04,0x02,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_i16_e64 s[10:11], -4.0, v2
-// W64: encoding: [0x0a,0x00,0x8a,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x8a,0xd4,0xf7,0x04,0x02,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_i16_e64 s[10:11], v1, v255
@@ -9928,11 +9928,11 @@ v_cmp_eq_i16_e64 s[10:11], v1, -1
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_i16_e64 s[10:11], v1, 0.5
-// W64: encoding: [0x0a,0x00,0x8a,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x8a,0xd4,0x01,0xe1,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_i16_e64 s[10:11], v1, -4.0
-// W64: encoding: [0x0a,0x00,0x8a,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x8a,0xd4,0x01,0xef,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_i16_e64 s[10:11], v1, v2
@@ -9992,11 +9992,11 @@ v_cmp_le_i16_e64 s[10:11], -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_i16_e64 s[10:11], 0.5, v2
-// W64: encoding: [0x0a,0x00,0x8b,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x8b,0xd4,0xf0,0x04,0x02,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_i16_e64 s[10:11], -4.0, v2
-// W64: encoding: [0x0a,0x00,0x8b,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x8b,0xd4,0xf7,0x04,0x02,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_i16_e64 s[10:11], v1, v255
@@ -10040,11 +10040,11 @@ v_cmp_le_i16_e64 s[10:11], v1, -1
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_i16_e64 s[10:11], v1, 0.5
-// W64: encoding: [0x0a,0x00,0x8b,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x8b,0xd4,0x01,0xe1,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_i16_e64 s[10:11], v1, -4.0
-// W64: encoding: [0x0a,0x00,0x8b,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x8b,0xd4,0x01,0xef,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_i16_e64 s[10:11], v1, v2
@@ -10104,11 +10104,11 @@ v_cmp_gt_i16_e64 s[10:11], -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_i16_e64 s[10:11], 0.5, v2
-// W64: encoding: [0x0a,0x00,0x8c,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x8c,0xd4,0xf0,0x04,0x02,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_i16_e64 s[10:11], -4.0, v2
-// W64: encoding: [0x0a,0x00,0x8c,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x8c,0xd4,0xf7,0x04,0x02,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_i16_e64 s[10:11], v1, v255
@@ -10152,11 +10152,11 @@ v_cmp_gt_i16_e64 s[10:11], v1, -1
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_i16_e64 s[10:11], v1, 0.5
-// W64: encoding: [0x0a,0x00,0x8c,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x8c,0xd4,0x01,0xe1,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_i16_e64 s[10:11], v1, -4.0
-// W64: encoding: [0x0a,0x00,0x8c,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x8c,0xd4,0x01,0xef,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_i16_e64 s[10:11], v1, v2
@@ -10216,11 +10216,11 @@ v_cmp_ne_i16_e64 s[10:11], -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_i16_e64 s[10:11], 0.5, v2
-// W64: encoding: [0x0a,0x00,0x8d,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x8d,0xd4,0xf0,0x04,0x02,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_i16_e64 s[10:11], -4.0, v2
-// W64: encoding: [0x0a,0x00,0x8d,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x8d,0xd4,0xf7,0x04,0x02,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_i16_e64 s[10:11], v1, v255
@@ -10264,11 +10264,11 @@ v_cmp_ne_i16_e64 s[10:11], v1, -1
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_i16_e64 s[10:11], v1, 0.5
-// W64: encoding: [0x0a,0x00,0x8d,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x8d,0xd4,0x01,0xe1,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_i16_e64 s[10:11], v1, -4.0
-// W64: encoding: [0x0a,0x00,0x8d,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x8d,0xd4,0x01,0xef,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_i16_e64 s[10:11], v1, v2
@@ -10328,11 +10328,11 @@ v_cmp_ge_i16_e64 s[10:11], -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_i16_e64 s[10:11], 0.5, v2
-// W64: encoding: [0x0a,0x00,0x8e,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x8e,0xd4,0xf0,0x04,0x02,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_i16_e64 s[10:11], -4.0, v2
-// W64: encoding: [0x0a,0x00,0x8e,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x8e,0xd4,0xf7,0x04,0x02,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_i16_e64 s[10:11], v1, v255
@@ -10376,11 +10376,11 @@ v_cmp_ge_i16_e64 s[10:11], v1, -1
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_i16_e64 s[10:11], v1, 0.5
-// W64: encoding: [0x0a,0x00,0x8e,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x8e,0xd4,0x01,0xe1,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_i16_e64 s[10:11], v1, -4.0
-// W64: encoding: [0x0a,0x00,0x8e,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x8e,0xd4,0x01,0xef,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_i16_e64 s10, v1, v2
@@ -10440,11 +10440,11 @@ v_cmp_lt_i16_e64 s10, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_i16_e64 s10, 0.5, v2
-// W32: encoding: [0x0a,0x00,0x89,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0x89,0xd4,0xf0,0x04,0x02,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_i16_e64 s10, -4.0, v2
-// W32: encoding: [0x0a,0x00,0x89,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0x89,0xd4,0xf7,0x04,0x02,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_i16_e64 s10, v1, v255
@@ -10488,11 +10488,11 @@ v_cmp_lt_i16_e64 s10, v1, -1
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_i16_e64 s10, v1, 0.5
-// W32: encoding: [0x0a,0x00,0x89,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0x89,0xd4,0x01,0xe1,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_i16_e64 s10, v1, -4.0
-// W32: encoding: [0x0a,0x00,0x89,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0x89,0xd4,0x01,0xef,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_i16_e64 s10, v1, v2
@@ -10552,11 +10552,11 @@ v_cmp_eq_i16_e64 s10, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_i16_e64 s10, 0.5, v2
-// W32: encoding: [0x0a,0x00,0x8a,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0x8a,0xd4,0xf0,0x04,0x02,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_i16_e64 s10, -4.0, v2
-// W32: encoding: [0x0a,0x00,0x8a,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0x8a,0xd4,0xf7,0x04,0x02,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_i16_e64 s10, v1, v255
@@ -10600,11 +10600,11 @@ v_cmp_eq_i16_e64 s10, v1, -1
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_i16_e64 s10, v1, 0.5
-// W32: encoding: [0x0a,0x00,0x8a,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0x8a,0xd4,0x01,0xe1,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_i16_e64 s10, v1, -4.0
-// W32: encoding: [0x0a,0x00,0x8a,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0x8a,0xd4,0x01,0xef,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_i16_e64 s10, v1, v2
@@ -10664,11 +10664,11 @@ v_cmp_le_i16_e64 s10, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_i16_e64 s10, 0.5, v2
-// W32: encoding: [0x0a,0x00,0x8b,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0x8b,0xd4,0xf0,0x04,0x02,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_i16_e64 s10, -4.0, v2
-// W32: encoding: [0x0a,0x00,0x8b,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0x8b,0xd4,0xf7,0x04,0x02,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_i16_e64 s10, v1, v255
@@ -10712,11 +10712,11 @@ v_cmp_le_i16_e64 s10, v1, -1
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_i16_e64 s10, v1, 0.5
-// W32: encoding: [0x0a,0x00,0x8b,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0x8b,0xd4,0x01,0xe1,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_i16_e64 s10, v1, -4.0
-// W32: encoding: [0x0a,0x00,0x8b,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0x8b,0xd4,0x01,0xef,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_i16_e64 s10, v1, v2
@@ -10776,11 +10776,11 @@ v_cmp_gt_i16_e64 s10, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_i16_e64 s10, 0.5, v2
-// W32: encoding: [0x0a,0x00,0x8c,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0x8c,0xd4,0xf0,0x04,0x02,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_i16_e64 s10, -4.0, v2
-// W32: encoding: [0x0a,0x00,0x8c,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0x8c,0xd4,0xf7,0x04,0x02,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_i16_e64 s10, v1, v255
@@ -10824,11 +10824,11 @@ v_cmp_gt_i16_e64 s10, v1, -1
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_i16_e64 s10, v1, 0.5
-// W32: encoding: [0x0a,0x00,0x8c,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0x8c,0xd4,0x01,0xe1,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_i16_e64 s10, v1, -4.0
-// W32: encoding: [0x0a,0x00,0x8c,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0x8c,0xd4,0x01,0xef,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_i16_e64 s10, v1, v2
@@ -10888,11 +10888,11 @@ v_cmp_ne_i16_e64 s10, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_i16_e64 s10, 0.5, v2
-// W32: encoding: [0x0a,0x00,0x8d,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0x8d,0xd4,0xf0,0x04,0x02,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_i16_e64 s10, -4.0, v2
-// W32: encoding: [0x0a,0x00,0x8d,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0x8d,0xd4,0xf7,0x04,0x02,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_i16_e64 s10, v1, v255
@@ -10936,11 +10936,11 @@ v_cmp_ne_i16_e64 s10, v1, -1
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_i16_e64 s10, v1, 0.5
-// W32: encoding: [0x0a,0x00,0x8d,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0x8d,0xd4,0x01,0xe1,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_i16_e64 s10, v1, -4.0
-// W32: encoding: [0x0a,0x00,0x8d,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0x8d,0xd4,0x01,0xef,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_i16_e64 s10, v1, v2
@@ -11000,11 +11000,11 @@ v_cmp_ge_i16_e64 s10, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_i16_e64 s10, 0.5, v2
-// W32: encoding: [0x0a,0x00,0x8e,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0x8e,0xd4,0xf0,0x04,0x02,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_i16_e64 s10, -4.0, v2
-// W32: encoding: [0x0a,0x00,0x8e,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0x8e,0xd4,0xf7,0x04,0x02,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_i16_e64 s10, v1, v255
@@ -11048,11 +11048,11 @@ v_cmp_ge_i16_e64 s10, v1, -1
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_i16_e64 s10, v1, 0.5
-// W32: encoding: [0x0a,0x00,0x8e,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0x8e,0xd4,0x01,0xe1,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_i16_e64 s10, v1, -4.0
-// W32: encoding: [0x0a,0x00,0x8e,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0x8e,0xd4,0x01,0xef,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_class_f16_e64 s[10:11], v1, v2
@@ -14516,11 +14516,11 @@ v_cmp_lt_u16_e64 s[10:11], -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_u16_e64 s[10:11], 0.5, v2
-// W64: encoding: [0x0a,0x00,0xa9,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0xa9,0xd4,0xf0,0x04,0x02,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_u16_e64 s[10:11], -4.0, v2
-// W64: encoding: [0x0a,0x00,0xa9,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0xa9,0xd4,0xf7,0x04,0x02,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_u16_e64 s[10:11], v1, v255
@@ -14564,11 +14564,11 @@ v_cmp_lt_u16_e64 s[10:11], v1, -1
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_u16_e64 s[10:11], v1, 0.5
-// W64: encoding: [0x0a,0x00,0xa9,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0xa9,0xd4,0x01,0xe1,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_u16_e64 s[10:11], v1, -4.0
-// W64: encoding: [0x0a,0x00,0xa9,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0xa9,0xd4,0x01,0xef,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_u16_e64 s[10:11], v1, v2
@@ -14628,11 +14628,11 @@ v_cmp_eq_u16_e64 s[10:11], -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_u16_e64 s[10:11], 0.5, v2
-// W64: encoding: [0x0a,0x00,0xaa,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0xaa,0xd4,0xf0,0x04,0x02,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_u16_e64 s[10:11], -4.0, v2
-// W64: encoding: [0x0a,0x00,0xaa,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0xaa,0xd4,0xf7,0x04,0x02,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_u16_e64 s[10:11], v1, v255
@@ -14676,11 +14676,11 @@ v_cmp_eq_u16_e64 s[10:11], v1, -1
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_u16_e64 s[10:11], v1, 0.5
-// W64: encoding: [0x0a,0x00,0xaa,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0xaa,0xd4,0x01,0xe1,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_u16_e64 s[10:11], v1, -4.0
-// W64: encoding: [0x0a,0x00,0xaa,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0xaa,0xd4,0x01,0xef,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_u16_e64 s[10:11], v1, v2
@@ -14740,11 +14740,11 @@ v_cmp_le_u16_e64 s[10:11], -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_u16_e64 s[10:11], 0.5, v2
-// W64: encoding: [0x0a,0x00,0xab,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0xab,0xd4,0xf0,0x04,0x02,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_u16_e64 s[10:11], -4.0, v2
-// W64: encoding: [0x0a,0x00,0xab,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0xab,0xd4,0xf7,0x04,0x02,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_u16_e64 s[10:11], v1, v255
@@ -14788,11 +14788,11 @@ v_cmp_le_u16_e64 s[10:11], v1, -1
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_u16_e64 s[10:11], v1, 0.5
-// W64: encoding: [0x0a,0x00,0xab,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0xab,0xd4,0x01,0xe1,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_u16_e64 s[10:11], v1, -4.0
-// W64: encoding: [0x0a,0x00,0xab,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0xab,0xd4,0x01,0xef,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_u16_e64 s[10:11], v1, v2
@@ -14852,11 +14852,11 @@ v_cmp_gt_u16_e64 s[10:11], -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_u16_e64 s[10:11], 0.5, v2
-// W64: encoding: [0x0a,0x00,0xac,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0xac,0xd4,0xf0,0x04,0x02,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_u16_e64 s[10:11], -4.0, v2
-// W64: encoding: [0x0a,0x00,0xac,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0xac,0xd4,0xf7,0x04,0x02,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_u16_e64 s[10:11], v1, v255
@@ -14900,11 +14900,11 @@ v_cmp_gt_u16_e64 s[10:11], v1, -1
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_u16_e64 s[10:11], v1, 0.5
-// W64: encoding: [0x0a,0x00,0xac,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0xac,0xd4,0x01,0xe1,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_u16_e64 s[10:11], v1, -4.0
-// W64: encoding: [0x0a,0x00,0xac,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0xac,0xd4,0x01,0xef,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_u16_e64 s[10:11], v1, v2
@@ -14964,11 +14964,11 @@ v_cmp_ne_u16_e64 s[10:11], -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_u16_e64 s[10:11], 0.5, v2
-// W64: encoding: [0x0a,0x00,0xad,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0xad,0xd4,0xf0,0x04,0x02,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_u16_e64 s[10:11], -4.0, v2
-// W64: encoding: [0x0a,0x00,0xad,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0xad,0xd4,0xf7,0x04,0x02,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_u16_e64 s[10:11], v1, v255
@@ -15012,11 +15012,11 @@ v_cmp_ne_u16_e64 s[10:11], v1, -1
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_u16_e64 s[10:11], v1, 0.5
-// W64: encoding: [0x0a,0x00,0xad,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0xad,0xd4,0x01,0xe1,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_u16_e64 s[10:11], v1, -4.0
-// W64: encoding: [0x0a,0x00,0xad,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0xad,0xd4,0x01,0xef,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_u16_e64 s[10:11], v1, v2
@@ -15076,11 +15076,11 @@ v_cmp_ge_u16_e64 s[10:11], -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_u16_e64 s[10:11], 0.5, v2
-// W64: encoding: [0x0a,0x00,0xae,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0xae,0xd4,0xf0,0x04,0x02,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_u16_e64 s[10:11], -4.0, v2
-// W64: encoding: [0x0a,0x00,0xae,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0xae,0xd4,0xf7,0x04,0x02,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_u16_e64 s[10:11], v1, v255
@@ -15124,11 +15124,11 @@ v_cmp_ge_u16_e64 s[10:11], v1, -1
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_u16_e64 s[10:11], v1, 0.5
-// W64: encoding: [0x0a,0x00,0xae,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0xae,0xd4,0x01,0xe1,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_u16_e64 s[10:11], v1, -4.0
-// W64: encoding: [0x0a,0x00,0xae,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0xae,0xd4,0x01,0xef,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_f_u32_e64 s[10:11], v1, v2
@@ -18132,11 +18132,11 @@ v_cmp_lt_u16_e64 s10, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_u16_e64 s10, 0.5, v2
-// W32: encoding: [0x0a,0x00,0xa9,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0xa9,0xd4,0xf0,0x04,0x02,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_u16_e64 s10, -4.0, v2
-// W32: encoding: [0x0a,0x00,0xa9,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0xa9,0xd4,0xf7,0x04,0x02,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_u16_e64 s10, v1, v255
@@ -18180,11 +18180,11 @@ v_cmp_lt_u16_e64 s10, v1, -1
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_u16_e64 s10, v1, 0.5
-// W32: encoding: [0x0a,0x00,0xa9,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0xa9,0xd4,0x01,0xe1,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_u16_e64 s10, v1, -4.0
-// W32: encoding: [0x0a,0x00,0xa9,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0xa9,0xd4,0x01,0xef,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_u16_e64 s10, v1, v2
@@ -18244,11 +18244,11 @@ v_cmp_eq_u16_e64 s10, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_u16_e64 s10, 0.5, v2
-// W32: encoding: [0x0a,0x00,0xaa,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0xaa,0xd4,0xf0,0x04,0x02,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_u16_e64 s10, -4.0, v2
-// W32: encoding: [0x0a,0x00,0xaa,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0xaa,0xd4,0xf7,0x04,0x02,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_u16_e64 s10, v1, v255
@@ -18292,11 +18292,11 @@ v_cmp_eq_u16_e64 s10, v1, -1
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_u16_e64 s10, v1, 0.5
-// W32: encoding: [0x0a,0x00,0xaa,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0xaa,0xd4,0x01,0xe1,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_u16_e64 s10, v1, -4.0
-// W32: encoding: [0x0a,0x00,0xaa,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0xaa,0xd4,0x01,0xef,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_u16_e64 s10, v1, v2
@@ -18356,11 +18356,11 @@ v_cmp_le_u16_e64 s10, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_u16_e64 s10, 0.5, v2
-// W32: encoding: [0x0a,0x00,0xab,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0xab,0xd4,0xf0,0x04,0x02,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_u16_e64 s10, -4.0, v2
-// W32: encoding: [0x0a,0x00,0xab,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0xab,0xd4,0xf7,0x04,0x02,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_u16_e64 s10, v1, v255
@@ -18404,11 +18404,11 @@ v_cmp_le_u16_e64 s10, v1, -1
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_u16_e64 s10, v1, 0.5
-// W32: encoding: [0x0a,0x00,0xab,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0xab,0xd4,0x01,0xe1,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_u16_e64 s10, v1, -4.0
-// W32: encoding: [0x0a,0x00,0xab,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0xab,0xd4,0x01,0xef,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_u16_e64 s10, v1, v2
@@ -18468,11 +18468,11 @@ v_cmp_gt_u16_e64 s10, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_u16_e64 s10, 0.5, v2
-// W32: encoding: [0x0a,0x00,0xac,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0xac,0xd4,0xf0,0x04,0x02,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_u16_e64 s10, -4.0, v2
-// W32: encoding: [0x0a,0x00,0xac,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0xac,0xd4,0xf7,0x04,0x02,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_u16_e64 s10, v1, v255
@@ -18516,11 +18516,11 @@ v_cmp_gt_u16_e64 s10, v1, -1
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_u16_e64 s10, v1, 0.5
-// W32: encoding: [0x0a,0x00,0xac,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0xac,0xd4,0x01,0xe1,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_u16_e64 s10, v1, -4.0
-// W32: encoding: [0x0a,0x00,0xac,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0xac,0xd4,0x01,0xef,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_u16_e64 s10, v1, v2
@@ -18580,11 +18580,11 @@ v_cmp_ne_u16_e64 s10, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_u16_e64 s10, 0.5, v2
-// W32: encoding: [0x0a,0x00,0xad,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0xad,0xd4,0xf0,0x04,0x02,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_u16_e64 s10, -4.0, v2
-// W32: encoding: [0x0a,0x00,0xad,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0xad,0xd4,0xf7,0x04,0x02,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_u16_e64 s10, v1, v255
@@ -18628,11 +18628,11 @@ v_cmp_ne_u16_e64 s10, v1, -1
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_u16_e64 s10, v1, 0.5
-// W32: encoding: [0x0a,0x00,0xad,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0xad,0xd4,0x01,0xe1,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_u16_e64 s10, v1, -4.0
-// W32: encoding: [0x0a,0x00,0xad,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0xad,0xd4,0x01,0xef,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_u16_e64 s10, v1, v2
@@ -18692,11 +18692,11 @@ v_cmp_ge_u16_e64 s10, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_u16_e64 s10, 0.5, v2
-// W32: encoding: [0x0a,0x00,0xae,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0xae,0xd4,0xf0,0x04,0x02,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_u16_e64 s10, -4.0, v2
-// W32: encoding: [0x0a,0x00,0xae,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0xae,0xd4,0xf7,0x04,0x02,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_u16_e64 s10, v1, v255
@@ -18740,11 +18740,11 @@ v_cmp_ge_u16_e64 s10, v1, -1
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_u16_e64 s10, v1, 0.5
-// W32: encoding: [0x0a,0x00,0xae,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0xae,0xd4,0x01,0xe1,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_u16_e64 s10, v1, -4.0
-// W32: encoding: [0x0a,0x00,0xae,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// W32: encoding: [0x0a,0x00,0xae,0xd4,0x01,0xef,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_f_u32_e64 s10, v1, v2
diff --git a/llvm/test/MC/AMDGPU/gfx10_asm_vopcx.s b/llvm/test/MC/AMDGPU/gfx10_asm_vopcx.s
index f441feb88db4..eed284e56bbe 100644
--- a/llvm/test/MC/AMDGPU/gfx10_asm_vopcx.s
+++ b/llvm/test/MC/AMDGPU/gfx10_asm_vopcx.s
@@ -6330,10 +6330,10 @@ v_cmpx_lt_i16 -1, v2
 // GFX10: encoding: [0xc1,0x04,0x32,0x7d]
 
 v_cmpx_lt_i16 0.5, v2
-// GFX10: encoding: [0xff,0x04,0x32,0x7d,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0xf0,0x04,0x32,0x7d]
 
 v_cmpx_lt_i16 -4.0, v2
-// GFX10: encoding: [0xff,0x04,0x32,0x7d,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0xf7,0x04,0x32,0x7d]
 
 v_cmpx_lt_i16 0xfe0b, v2
 // GFX10: encoding: [0xff,0x04,0x32,0x7d,0x0b,0xfe,0x00,0x00]
@@ -6378,10 +6378,10 @@ v_cmpx_lt_i16_e64 -1, v2
 // GFX10: encoding: [0x7e,0x00,0x99,0xd4,0xc1,0x04,0x02,0x00]
 
 v_cmpx_lt_i16_e64 0.5, v2
-// GFX10: encoding: [0x7e,0x00,0x99,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0x99,0xd4,0xf0,0x04,0x02,0x00]
 
 v_cmpx_lt_i16_e64 -4.0, v2
-// GFX10: encoding: [0x7e,0x00,0x99,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0x99,0xd4,0xf7,0x04,0x02,0x00]
 
 v_cmpx_lt_i16_e64 v1, v255
 // GFX10: encoding: [0x7e,0x00,0x99,0xd4,0x01,0xff,0x03,0x00]
@@ -6414,10 +6414,10 @@ v_cmpx_lt_i16_e64 v1, -1
 // GFX10: encoding: [0x7e,0x00,0x99,0xd4,0x01,0x83,0x01,0x00]
 
 v_cmpx_lt_i16_e64 v1, 0.5
-// GFX10: encoding: [0x7e,0x00,0x99,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0x99,0xd4,0x01,0xe1,0x01,0x00]
 
 v_cmpx_lt_i16_e64 v1, -4.0
-// GFX10: encoding: [0x7e,0x00,0x99,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0x99,0xd4,0x01,0xef,0x01,0x00]
 
 v_cmpx_eq_i16 v1, v2
 // GFX10: encoding: [0x01,0x05,0x34,0x7d]
@@ -6453,10 +6453,10 @@ v_cmpx_eq_i16 -1, v2
 // GFX10: encoding: [0xc1,0x04,0x34,0x7d]
 
 v_cmpx_eq_i16 0.5, v2
-// GFX10: encoding: [0xff,0x04,0x34,0x7d,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0xf0,0x04,0x34,0x7d]
 
 v_cmpx_eq_i16 -4.0, v2
-// GFX10: encoding: [0xff,0x04,0x34,0x7d,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0xf7,0x04,0x34,0x7d]
 
 v_cmpx_eq_i16 0xfe0b, v2
 // GFX10: encoding: [0xff,0x04,0x34,0x7d,0x0b,0xfe,0x00,0x00]
@@ -6501,10 +6501,10 @@ v_cmpx_eq_i16_e64 -1, v2
 // GFX10: encoding: [0x7e,0x00,0x9a,0xd4,0xc1,0x04,0x02,0x00]
 
 v_cmpx_eq_i16_e64 0.5, v2
-// GFX10: encoding: [0x7e,0x00,0x9a,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0x9a,0xd4,0xf0,0x04,0x02,0x00]
 
 v_cmpx_eq_i16_e64 -4.0, v2
-// GFX10: encoding: [0x7e,0x00,0x9a,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0x9a,0xd4,0xf7,0x04,0x02,0x00]
 
 v_cmpx_eq_i16_e64 v1, v255
 // GFX10: encoding: [0x7e,0x00,0x9a,0xd4,0x01,0xff,0x03,0x00]
@@ -6537,10 +6537,10 @@ v_cmpx_eq_i16_e64 v1, -1
 // GFX10: encoding: [0x7e,0x00,0x9a,0xd4,0x01,0x83,0x01,0x00]
 
 v_cmpx_eq_i16_e64 v1, 0.5
-// GFX10: encoding: [0x7e,0x00,0x9a,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0x9a,0xd4,0x01,0xe1,0x01,0x00]
 
 v_cmpx_eq_i16_e64 v1, -4.0
-// GFX10: encoding: [0x7e,0x00,0x9a,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0x9a,0xd4,0x01,0xef,0x01,0x00]
 
 v_cmpx_le_i16 v1, v2
 // GFX10: encoding: [0x01,0x05,0x36,0x7d]
@@ -6576,10 +6576,10 @@ v_cmpx_le_i16 -1, v2
 // GFX10: encoding: [0xc1,0x04,0x36,0x7d]
 
 v_cmpx_le_i16 0.5, v2
-// GFX10: encoding: [0xff,0x04,0x36,0x7d,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0xf0,0x04,0x36,0x7d]
 
 v_cmpx_le_i16 -4.0, v2
-// GFX10: encoding: [0xff,0x04,0x36,0x7d,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0xf7,0x04,0x36,0x7d]
 
 v_cmpx_le_i16 0xfe0b, v2
 // GFX10: encoding: [0xff,0x04,0x36,0x7d,0x0b,0xfe,0x00,0x00]
@@ -6624,10 +6624,10 @@ v_cmpx_le_i16_e64 -1, v2
 // GFX10: encoding: [0x7e,0x00,0x9b,0xd4,0xc1,0x04,0x02,0x00]
 
 v_cmpx_le_i16_e64 0.5, v2
-// GFX10: encoding: [0x7e,0x00,0x9b,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0x9b,0xd4,0xf0,0x04,0x02,0x00]
 
 v_cmpx_le_i16_e64 -4.0, v2
-// GFX10: encoding: [0x7e,0x00,0x9b,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0x9b,0xd4,0xf7,0x04,0x02,0x00]
 
 v_cmpx_le_i16_e64 v1, v255
 // GFX10: encoding: [0x7e,0x00,0x9b,0xd4,0x01,0xff,0x03,0x00]
@@ -6660,10 +6660,10 @@ v_cmpx_le_i16_e64 v1, -1
 // GFX10: encoding: [0x7e,0x00,0x9b,0xd4,0x01,0x83,0x01,0x00]
 
 v_cmpx_le_i16_e64 v1, 0.5
-// GFX10: encoding: [0x7e,0x00,0x9b,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0x9b,0xd4,0x01,0xe1,0x01,0x00]
 
 v_cmpx_le_i16_e64 v1, -4.0
-// GFX10: encoding: [0x7e,0x00,0x9b,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0x9b,0xd4,0x01,0xef,0x01,0x00]
 
 v_cmpx_gt_i16 v1, v2
 // GFX10: encoding: [0x01,0x05,0x38,0x7d]
@@ -6699,10 +6699,10 @@ v_cmpx_gt_i16 -1, v2
 // GFX10: encoding: [0xc1,0x04,0x38,0x7d]
 
 v_cmpx_gt_i16 0.5, v2
-// GFX10: encoding: [0xff,0x04,0x38,0x7d,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0xf0,0x04,0x38,0x7d]
 
 v_cmpx_gt_i16 -4.0, v2
-// GFX10: encoding: [0xff,0x04,0x38,0x7d,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0xf7,0x04,0x38,0x7d]
 
 v_cmpx_gt_i16 0xfe0b, v2
 // GFX10: encoding: [0xff,0x04,0x38,0x7d,0x0b,0xfe,0x00,0x00]
@@ -6747,10 +6747,10 @@ v_cmpx_gt_i16_e64 -1, v2
 // GFX10: encoding: [0x7e,0x00,0x9c,0xd4,0xc1,0x04,0x02,0x00]
 
 v_cmpx_gt_i16_e64 0.5, v2
-// GFX10: encoding: [0x7e,0x00,0x9c,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0x9c,0xd4,0xf0,0x04,0x02,0x00]
 
 v_cmpx_gt_i16_e64 -4.0, v2
-// GFX10: encoding: [0x7e,0x00,0x9c,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0x9c,0xd4,0xf7,0x04,0x02,0x00]
 
 v_cmpx_gt_i16_e64 v1, v255
 // GFX10: encoding: [0x7e,0x00,0x9c,0xd4,0x01,0xff,0x03,0x00]
@@ -6783,10 +6783,10 @@ v_cmpx_gt_i16_e64 v1, -1
 // GFX10: encoding: [0x7e,0x00,0x9c,0xd4,0x01,0x83,0x01,0x00]
 
 v_cmpx_gt_i16_e64 v1, 0.5
-// GFX10: encoding: [0x7e,0x00,0x9c,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0x9c,0xd4,0x01,0xe1,0x01,0x00]
 
 v_cmpx_gt_i16_e64 v1, -4.0
-// GFX10: encoding: [0x7e,0x00,0x9c,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0x9c,0xd4,0x01,0xef,0x01,0x00]
 
 v_cmpx_ne_i16 v1, v2
 // GFX10: encoding: [0x01,0x05,0x3a,0x7d]
@@ -6822,10 +6822,10 @@ v_cmpx_ne_i16 -1, v2
 // GFX10: encoding: [0xc1,0x04,0x3a,0x7d]
 
 v_cmpx_ne_i16 0.5, v2
-// GFX10: encoding: [0xff,0x04,0x3a,0x7d,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0xf0,0x04,0x3a,0x7d]
 
 v_cmpx_ne_i16 -4.0, v2
-// GFX10: encoding: [0xff,0x04,0x3a,0x7d,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0xf7,0x04,0x3a,0x7d]
 
 v_cmpx_ne_i16 0xfe0b, v2
 // GFX10: encoding: [0xff,0x04,0x3a,0x7d,0x0b,0xfe,0x00,0x00]
@@ -6870,10 +6870,10 @@ v_cmpx_ne_i16_e64 -1, v2
 // GFX10: encoding: [0x7e,0x00,0x9d,0xd4,0xc1,0x04,0x02,0x00]
 
 v_cmpx_ne_i16_e64 0.5, v2
-// GFX10: encoding: [0x7e,0x00,0x9d,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0x9d,0xd4,0xf0,0x04,0x02,0x00]
 
 v_cmpx_ne_i16_e64 -4.0, v2
-// GFX10: encoding: [0x7e,0x00,0x9d,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0x9d,0xd4,0xf7,0x04,0x02,0x00]
 
 v_cmpx_ne_i16_e64 v1, v255
 // GFX10: encoding: [0x7e,0x00,0x9d,0xd4,0x01,0xff,0x03,0x00]
@@ -6906,10 +6906,10 @@ v_cmpx_ne_i16_e64 v1, -1
 // GFX10: encoding: [0x7e,0x00,0x9d,0xd4,0x01,0x83,0x01,0x00]
 
 v_cmpx_ne_i16_e64 v1, 0.5
-// GFX10: encoding: [0x7e,0x00,0x9d,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0x9d,0xd4,0x01,0xe1,0x01,0x00]
 
 v_cmpx_ne_i16_e64 v1, -4.0
-// GFX10: encoding: [0x7e,0x00,0x9d,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0x9d,0xd4,0x01,0xef,0x01,0x00]
 
 v_cmpx_ge_i16 v1, v2
 // GFX10: encoding: [0x01,0x05,0x3c,0x7d]
@@ -6945,10 +6945,10 @@ v_cmpx_ge_i16 -1, v2
 // GFX10: encoding: [0xc1,0x04,0x3c,0x7d]
 
 v_cmpx_ge_i16 0.5, v2
-// GFX10: encoding: [0xff,0x04,0x3c,0x7d,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0xf0,0x04,0x3c,0x7d]
 
 v_cmpx_ge_i16 -4.0, v2
-// GFX10: encoding: [0xff,0x04,0x3c,0x7d,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0xf7,0x04,0x3c,0x7d]
 
 v_cmpx_ge_i16 0xfe0b, v2
 // GFX10: encoding: [0xff,0x04,0x3c,0x7d,0x0b,0xfe,0x00,0x00]
@@ -6993,10 +6993,10 @@ v_cmpx_ge_i16_e64 -1, v2
 // GFX10: encoding: [0x7e,0x00,0x9e,0xd4,0xc1,0x04,0x02,0x00]
 
 v_cmpx_ge_i16_e64 0.5, v2
-// GFX10: encoding: [0x7e,0x00,0x9e,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0x9e,0xd4,0xf0,0x04,0x02,0x00]
 
 v_cmpx_ge_i16_e64 -4.0, v2
-// GFX10: encoding: [0x7e,0x00,0x9e,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0x9e,0xd4,0xf7,0x04,0x02,0x00]
 
 v_cmpx_ge_i16_e64 v1, v255
 // GFX10: encoding: [0x7e,0x00,0x9e,0xd4,0x01,0xff,0x03,0x00]
@@ -7029,10 +7029,10 @@ v_cmpx_ge_i16_e64 v1, -1
 // GFX10: encoding: [0x7e,0x00,0x9e,0xd4,0x01,0x83,0x01,0x00]
 
 v_cmpx_ge_i16_e64 v1, 0.5
-// GFX10: encoding: [0x7e,0x00,0x9e,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0x9e,0xd4,0x01,0xe1,0x01,0x00]
 
 v_cmpx_ge_i16_e64 v1, -4.0
-// GFX10: encoding: [0x7e,0x00,0x9e,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0x9e,0xd4,0x01,0xef,0x01,0x00]
 
 v_cmpx_class_f16 v1, v2
 // GFX10: encoding: [0x01,0x05,0x3e,0x7d]
@@ -8094,10 +8094,10 @@ v_cmpx_lt_u16 -1, v2
 // GFX10: encoding: [0xc1,0x04,0x72,0x7d]
 
 v_cmpx_lt_u16 0.5, v2
-// GFX10: encoding: [0xff,0x04,0x72,0x7d,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0xf0,0x04,0x72,0x7d]
 
 v_cmpx_lt_u16 -4.0, v2
-// GFX10: encoding: [0xff,0x04,0x72,0x7d,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0xf7,0x04,0x72,0x7d]
 
 v_cmpx_lt_u16 0xfe0b, v2
 // GFX10: encoding: [0xff,0x04,0x72,0x7d,0x0b,0xfe,0x00,0x00]
@@ -8142,10 +8142,10 @@ v_cmpx_lt_u16_e64 -1, v2
 // GFX10: encoding: [0x7e,0x00,0xb9,0xd4,0xc1,0x04,0x02,0x00]
 
 v_cmpx_lt_u16_e64 0.5, v2
-// GFX10: encoding: [0x7e,0x00,0xb9,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0xb9,0xd4,0xf0,0x04,0x02,0x00]
 
 v_cmpx_lt_u16_e64 -4.0, v2
-// GFX10: encoding: [0x7e,0x00,0xb9,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0xb9,0xd4,0xf7,0x04,0x02,0x00]
 
 v_cmpx_lt_u16_e64 v1, v255
 // GFX10: encoding: [0x7e,0x00,0xb9,0xd4,0x01,0xff,0x03,0x00]
@@ -8178,10 +8178,10 @@ v_cmpx_lt_u16_e64 v1, -1
 // GFX10: encoding: [0x7e,0x00,0xb9,0xd4,0x01,0x83,0x01,0x00]
 
 v_cmpx_lt_u16_e64 v1, 0.5
-// GFX10: encoding: [0x7e,0x00,0xb9,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0xb9,0xd4,0x01,0xe1,0x01,0x00]
 
 v_cmpx_lt_u16_e64 v1, -4.0
-// GFX10: encoding: [0x7e,0x00,0xb9,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0xb9,0xd4,0x01,0xef,0x01,0x00]
 
 v_cmpx_eq_u16 v1, v2
 // GFX10: encoding: [0x01,0x05,0x74,0x7d]
@@ -8217,10 +8217,10 @@ v_cmpx_eq_u16 -1, v2
 // GFX10: encoding: [0xc1,0x04,0x74,0x7d]
 
 v_cmpx_eq_u16 0.5, v2
-// GFX10: encoding: [0xff,0x04,0x74,0x7d,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0xf0,0x04,0x74,0x7d]
 
 v_cmpx_eq_u16 -4.0, v2
-// GFX10: encoding: [0xff,0x04,0x74,0x7d,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0xf7,0x04,0x74,0x7d]
 
 v_cmpx_eq_u16 0xfe0b, v2
 // GFX10: encoding: [0xff,0x04,0x74,0x7d,0x0b,0xfe,0x00,0x00]
@@ -8265,10 +8265,10 @@ v_cmpx_eq_u16_e64 -1, v2
 // GFX10: encoding: [0x7e,0x00,0xba,0xd4,0xc1,0x04,0x02,0x00]
 
 v_cmpx_eq_u16_e64 0.5, v2
-// GFX10: encoding: [0x7e,0x00,0xba,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0xba,0xd4,0xf0,0x04,0x02,0x00]
 
 v_cmpx_eq_u16_e64 -4.0, v2
-// GFX10: encoding: [0x7e,0x00,0xba,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0xba,0xd4,0xf7,0x04,0x02,0x00]
 
 v_cmpx_eq_u16_e64 v1, v255
 // GFX10: encoding: [0x7e,0x00,0xba,0xd4,0x01,0xff,0x03,0x00]
@@ -8301,10 +8301,10 @@ v_cmpx_eq_u16_e64 v1, -1
 // GFX10: encoding: [0x7e,0x00,0xba,0xd4,0x01,0x83,0x01,0x00]
 
 v_cmpx_eq_u16_e64 v1, 0.5
-// GFX10: encoding: [0x7e,0x00,0xba,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0xba,0xd4,0x01,0xe1,0x01,0x00]
 
 v_cmpx_eq_u16_e64 v1, -4.0
-// GFX10: encoding: [0x7e,0x00,0xba,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0xba,0xd4,0x01,0xef,0x01,0x00]
 
 v_cmpx_le_u16 v1, v2
 // GFX10: encoding: [0x01,0x05,0x76,0x7d]
@@ -8340,10 +8340,10 @@ v_cmpx_le_u16 -1, v2
 // GFX10: encoding: [0xc1,0x04,0x76,0x7d]
 
 v_cmpx_le_u16 0.5, v2
-// GFX10: encoding: [0xff,0x04,0x76,0x7d,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0xf0,0x04,0x76,0x7d]
 
 v_cmpx_le_u16 -4.0, v2
-// GFX10: encoding: [0xff,0x04,0x76,0x7d,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0xf7,0x04,0x76,0x7d]
 
 v_cmpx_le_u16 0xfe0b, v2
 // GFX10: encoding: [0xff,0x04,0x76,0x7d,0x0b,0xfe,0x00,0x00]
@@ -8388,10 +8388,10 @@ v_cmpx_le_u16_e64 -1, v2
 // GFX10: encoding: [0x7e,0x00,0xbb,0xd4,0xc1,0x04,0x02,0x00]
 
 v_cmpx_le_u16_e64 0.5, v2
-// GFX10: encoding: [0x7e,0x00,0xbb,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0xbb,0xd4,0xf0,0x04,0x02,0x00]
 
 v_cmpx_le_u16_e64 -4.0, v2
-// GFX10: encoding: [0x7e,0x00,0xbb,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0xbb,0xd4,0xf7,0x04,0x02,0x00]
 
 v_cmpx_le_u16_e64 v1, v255
 // GFX10: encoding: [0x7e,0x00,0xbb,0xd4,0x01,0xff,0x03,0x00]
@@ -8424,10 +8424,10 @@ v_cmpx_le_u16_e64 v1, -1
 // GFX10: encoding: [0x7e,0x00,0xbb,0xd4,0x01,0x83,0x01,0x00]
 
 v_cmpx_le_u16_e64 v1, 0.5
-// GFX10: encoding: [0x7e,0x00,0xbb,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0xbb,0xd4,0x01,0xe1,0x01,0x00]
 
 v_cmpx_le_u16_e64 v1, -4.0
-// GFX10: encoding: [0x7e,0x00,0xbb,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0xbb,0xd4,0x01,0xef,0x01,0x00]
 
 v_cmpx_gt_u16 v1, v2
 // GFX10: encoding: [0x01,0x05,0x78,0x7d]
@@ -8463,10 +8463,10 @@ v_cmpx_gt_u16 -1, v2
 // GFX10: encoding: [0xc1,0x04,0x78,0x7d]
 
 v_cmpx_gt_u16 0.5, v2
-// GFX10: encoding: [0xff,0x04,0x78,0x7d,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0xf0,0x04,0x78,0x7d]
 
 v_cmpx_gt_u16 -4.0, v2
-// GFX10: encoding: [0xff,0x04,0x78,0x7d,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0xf7,0x04,0x78,0x7d]
 
 v_cmpx_gt_u16 0xfe0b, v2
 // GFX10: encoding: [0xff,0x04,0x78,0x7d,0x0b,0xfe,0x00,0x00]
@@ -8511,10 +8511,10 @@ v_cmpx_gt_u16_e64 -1, v2
 // GFX10: encoding: [0x7e,0x00,0xbc,0xd4,0xc1,0x04,0x02,0x00]
 
 v_cmpx_gt_u16_e64 0.5, v2
-// GFX10: encoding: [0x7e,0x00,0xbc,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0xbc,0xd4,0xf0,0x04,0x02,0x00]
 
 v_cmpx_gt_u16_e64 -4.0, v2
-// GFX10: encoding: [0x7e,0x00,0xbc,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0xbc,0xd4,0xf7,0x04,0x02,0x00]
 
 v_cmpx_gt_u16_e64 v1, v255
 // GFX10: encoding: [0x7e,0x00,0xbc,0xd4,0x01,0xff,0x03,0x00]
@@ -8547,10 +8547,10 @@ v_cmpx_gt_u16_e64 v1, -1
 // GFX10: encoding: [0x7e,0x00,0xbc,0xd4,0x01,0x83,0x01,0x00]
 
 v_cmpx_gt_u16_e64 v1, 0.5
-// GFX10: encoding: [0x7e,0x00,0xbc,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0xbc,0xd4,0x01,0xe1,0x01,0x00]
 
 v_cmpx_gt_u16_e64 v1, -4.0
-// GFX10: encoding: [0x7e,0x00,0xbc,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0xbc,0xd4,0x01,0xef,0x01,0x00]
 
 v_cmpx_ne_u16 v1, v2
 // GFX10: encoding: [0x01,0x05,0x7a,0x7d]
@@ -8586,10 +8586,10 @@ v_cmpx_ne_u16 -1, v2
 // GFX10: encoding: [0xc1,0x04,0x7a,0x7d]
 
 v_cmpx_ne_u16 0.5, v2
-// GFX10: encoding: [0xff,0x04,0x7a,0x7d,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0xf0,0x04,0x7a,0x7d]
 
 v_cmpx_ne_u16 -4.0, v2
-// GFX10: encoding: [0xff,0x04,0x7a,0x7d,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0xf7,0x04,0x7a,0x7d]
 
 v_cmpx_ne_u16 0xfe0b, v2
 // GFX10: encoding: [0xff,0x04,0x7a,0x7d,0x0b,0xfe,0x00,0x00]
@@ -8634,10 +8634,10 @@ v_cmpx_ne_u16_e64 -1, v2
 // GFX10: encoding: [0x7e,0x00,0xbd,0xd4,0xc1,0x04,0x02,0x00]
 
 v_cmpx_ne_u16_e64 0.5, v2
-// GFX10: encoding: [0x7e,0x00,0xbd,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0xbd,0xd4,0xf0,0x04,0x02,0x00]
 
 v_cmpx_ne_u16_e64 -4.0, v2
-// GFX10: encoding: [0x7e,0x00,0xbd,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0xbd,0xd4,0xf7,0x04,0x02,0x00]
 
 v_cmpx_ne_u16_e64 v1, v255
 // GFX10: encoding: [0x7e,0x00,0xbd,0xd4,0x01,0xff,0x03,0x00]
@@ -8670,10 +8670,10 @@ v_cmpx_ne_u16_e64 v1, -1
 // GFX10: encoding: [0x7e,0x00,0xbd,0xd4,0x01,0x83,0x01,0x00]
 
 v_cmpx_ne_u16_e64 v1, 0.5
-// GFX10: encoding: [0x7e,0x00,0xbd,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0xbd,0xd4,0x01,0xe1,0x01,0x00]
 
 v_cmpx_ne_u16_e64 v1, -4.0
-// GFX10: encoding: [0x7e,0x00,0xbd,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0xbd,0xd4,0x01,0xef,0x01,0x00]
 
 v_cmpx_ge_u16 v1, v2
 // GFX10: encoding: [0x01,0x05,0x7c,0x7d]
@@ -8709,10 +8709,10 @@ v_cmpx_ge_u16 -1, v2
 // GFX10: encoding: [0xc1,0x04,0x7c,0x7d]
 
 v_cmpx_ge_u16 0.5, v2
-// GFX10: encoding: [0xff,0x04,0x7c,0x7d,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0xf0,0x04,0x7c,0x7d]
 
 v_cmpx_ge_u16 -4.0, v2
-// GFX10: encoding: [0xff,0x04,0x7c,0x7d,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0xf7,0x04,0x7c,0x7d]
 
 v_cmpx_ge_u16 0xfe0b, v2
 // GFX10: encoding: [0xff,0x04,0x7c,0x7d,0x0b,0xfe,0x00,0x00]
@@ -8757,10 +8757,10 @@ v_cmpx_ge_u16_e64 -1, v2
 // GFX10: encoding: [0x7e,0x00,0xbe,0xd4,0xc1,0x04,0x02,0x00]
 
 v_cmpx_ge_u16_e64 0.5, v2
-// GFX10: encoding: [0x7e,0x00,0xbe,0xd4,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0xbe,0xd4,0xf0,0x04,0x02,0x00]
 
 v_cmpx_ge_u16_e64 -4.0, v2
-// GFX10: encoding: [0x7e,0x00,0xbe,0xd4,0xff,0x04,0x02,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0xbe,0xd4,0xf7,0x04,0x02,0x00]
 
 v_cmpx_ge_u16_e64 v1, v255
 // GFX10: encoding: [0x7e,0x00,0xbe,0xd4,0x01,0xff,0x03,0x00]
@@ -8793,10 +8793,10 @@ v_cmpx_ge_u16_e64 v1, -1
 // GFX10: encoding: [0x7e,0x00,0xbe,0xd4,0x01,0x83,0x01,0x00]
 
 v_cmpx_ge_u16_e64 v1, 0.5
-// GFX10: encoding: [0x7e,0x00,0xbe,0xd4,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0xbe,0xd4,0x01,0xe1,0x01,0x00]
 
 v_cmpx_ge_u16_e64 v1, -4.0
-// GFX10: encoding: [0x7e,0x00,0xbe,0xd4,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x7e,0x00,0xbe,0xd4,0x01,0xef,0x01,0x00]
 
 v_cmpx_f_u32 v1, v2
 // GFX10: encoding: [0x01,0x05,0xa0,0x7d]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop1.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop1.s
index 5990f7cb4570..993bfedb1b64 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vop1.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop1.s
@@ -485,7 +485,7 @@ v_cvt_f16_i16 v5, -1
 // GFX11: encoding: [0xc1,0xa2,0x0a,0x7e]
 
 v_cvt_f16_i16 v5, 0.5
-// GFX11: encoding: [0xff,0xa2,0x0a,0x7e,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0xf0,0xa2,0x0a,0x7e]
 
 v_cvt_f16_i16 v5, src_scc
 // GFX11: encoding: [0xfd,0xa2,0x0a,0x7e]
@@ -530,7 +530,7 @@ v_cvt_f16_u16 v5, -1
 // GFX11: encoding: [0xc1,0xa0,0x0a,0x7e]
 
 v_cvt_f16_u16 v5, 0.5
-// GFX11: encoding: [0xff,0xa0,0x0a,0x7e,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0xf0,0xa0,0x0a,0x7e]
 
 v_cvt_f16_u16 v5, src_scc
 // GFX11: encoding: [0xfd,0xa0,0x0a,0x7e]
@@ -1277,7 +1277,7 @@ v_cvt_i32_i16 v5, -1
 // GFX11: encoding: [0xc1,0xd4,0x0a,0x7e]
 
 v_cvt_i32_i16 v5, 0.5
-// GFX11: encoding: [0xff,0xd4,0x0a,0x7e,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0xf0,0xd4,0x0a,0x7e]
 
 v_cvt_i32_i16 v5, src_scc
 // GFX11: encoding: [0xfd,0xd4,0x0a,0x7e]
@@ -1673,7 +1673,7 @@ v_cvt_u32_u16 v5, -1
 // GFX11: encoding: [0xc1,0xd6,0x0a,0x7e]
 
 v_cvt_u32_u16 v5, 0.5
-// GFX11: encoding: [0xff,0xd6,0x0a,0x7e,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0xf0,0xd6,0x0a,0x7e]
 
 v_cvt_u32_u16 v5, src_scc
 // GFX11: encoding: [0xfd,0xd6,0x0a,0x7e]
@@ -2666,7 +2666,7 @@ v_not_b16 v5, -1
 // GFX11: encoding: [0xc1,0xd2,0x0a,0x7e]
 
 v_not_b16 v5, 0.5
-// GFX11: encoding: [0xff,0xd2,0x0a,0x7e,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0xf0,0xd2,0x0a,0x7e]
 
 v_not_b16 v5, src_scc
 // GFX11: encoding: [0xfd,0xd2,0x0a,0x7e]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop3.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop3.s
index d288c02a22c9..c0ae7ecbdbdd 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vop3.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop3.s
@@ -266,7 +266,7 @@ v_add_nc_i16 v5, ttmp15, src_scc
 // GFX11: encoding: [0x05,0x00,0x0d,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_add_nc_i16 v5, m0, 0.5
-// GFX11: encoding: [0x05,0x00,0x0d,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x0d,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_add_nc_i16 v5, exec_lo, -1
 // GFX11: encoding: [0x05,0x00,0x0d,0xd7,0x7e,0x82,0x01,0x00]
@@ -281,7 +281,7 @@ v_add_nc_i16 v5, -1, exec_hi op_sel:[0,0,0]
 // GFX11: encoding: [0x05,0x00,0x0d,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_add_nc_i16 v5, 0.5, m0 op_sel:[1,0,0]
-// GFX11: encoding: [0x05,0x08,0x0d,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x08,0x0d,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_add_nc_i16 v5, src_scc, vcc_lo op_sel:[0,1,0]
 // GFX11: encoding: [0x05,0x10,0x0d,0xd7,0xfd,0xd4,0x00,0x00]
@@ -356,7 +356,7 @@ v_add_nc_u16 v5, ttmp15, src_scc
 // GFX11: encoding: [0x05,0x00,0x03,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_add_nc_u16 v5, m0, 0.5
-// GFX11: encoding: [0x05,0x00,0x03,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x03,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_add_nc_u16 v5, exec_lo, -1
 // GFX11: encoding: [0x05,0x00,0x03,0xd7,0x7e,0x82,0x01,0x00]
@@ -371,7 +371,7 @@ v_add_nc_u16 v5, -1, exec_hi op_sel:[0,0,0]
 // GFX11: encoding: [0x05,0x00,0x03,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_add_nc_u16 v5, 0.5, m0 op_sel:[1,0,0]
-// GFX11: encoding: [0x05,0x08,0x03,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x08,0x03,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_add_nc_u16 v5, src_scc, vcc_lo op_sel:[0,1,0]
 // GFX11: encoding: [0x05,0x10,0x03,0xd7,0xfd,0xd4,0x00,0x00]
@@ -491,7 +491,7 @@ v_and_b16 v5, ttmp15, src_scc
 // GFX11: encoding: [0x05,0x00,0x62,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_and_b16 v5, m0, 0.5
-// GFX11: encoding: [0x05,0x00,0x62,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x62,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_and_b16 v5, exec_lo, -1
 // GFX11: encoding: [0x05,0x00,0x62,0xd7,0x7e,0x82,0x01,0x00]
@@ -506,7 +506,7 @@ v_and_b16 v5, -1, exec_hi
 // GFX11: encoding: [0x05,0x00,0x62,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_and_b16 v5, 0.5, m0
-// GFX11: encoding: [0x05,0x00,0x62,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x62,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_and_b16 v5, src_scc, vcc_lo
 // GFX11: encoding: [0x05,0x00,0x62,0xd7,0xfd,0xd4,0x00,0x00]
@@ -581,7 +581,7 @@ v_ashrrev_i16 v5, ttmp15, src_scc
 // GFX11: encoding: [0x05,0x00,0x3a,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_ashrrev_i16 v5, m0, 0.5
-// GFX11: encoding: [0x05,0x00,0x3a,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x3a,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_ashrrev_i16 v5, exec_lo, -1
 // GFX11: encoding: [0x05,0x00,0x3a,0xd7,0x7e,0x82,0x01,0x00]
@@ -596,7 +596,7 @@ v_ashrrev_i16 v5, -1, exec_hi
 // GFX11: encoding: [0x05,0x00,0x3a,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_ashrrev_i16 v5, 0.5, m0
-// GFX11: encoding: [0x05,0x00,0x3a,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x3a,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_ashrrev_i16 v5, src_scc, vcc_lo
 // GFX11: encoding: [0x05,0x00,0x3a,0xd7,0xfd,0xd4,0x00,0x00]
@@ -861,7 +861,7 @@ v_cndmask_b16 v5, v1, src_scc, s3
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cndmask_b16 v5, v255, 0.5, s3
-// W32: encoding: [0x05,0x00,0x5d,0xd6,0xff,0xff,0x0d,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x05,0x00,0x5d,0xd6,0xff,0xe1,0x0d,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cndmask_b16 v5, s105, s105, s3
@@ -897,7 +897,7 @@ v_cndmask_b16 v5, -1, -|vcc_lo|, vcc_lo
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cndmask_b16 v5, 0.5, -1, vcc_hi
-// W32: encoding: [0x05,0x00,0x5d,0xd6,0xff,0x82,0xad,0x01,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x05,0x00,0x5d,0xd6,0xf0,0x82,0xad,0x01]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cndmask_b16 v5, -|src_scc|, null, ttmp15
@@ -909,7 +909,7 @@ v_cndmask_b16 v5, v1, src_scc, s[6:7]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cndmask_b16 v5, v255, 0.5, s[6:7]
-// W64: encoding: [0x05,0x00,0x5d,0xd6,0xff,0xff,0x19,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x05,0x00,0x5d,0xd6,0xff,0xe1,0x19,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cndmask_b16 v5, s105, s105, s[6:7]
@@ -945,7 +945,7 @@ v_cndmask_b16 v5, -1, -|vcc_lo|, s[104:105]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cndmask_b16 v5, 0.5, -1, vcc
-// W64: encoding: [0x05,0x00,0x5d,0xd6,0xff,0x82,0xa9,0x01,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x05,0x00,0x5d,0xd6,0xf0,0x82,0xa9,0x01]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cndmask_b16 v5, -|src_scc|, null, ttmp[14:15]
@@ -2639,7 +2639,7 @@ v_lshlrev_b16 v5, ttmp15, src_scc
 // GFX11: encoding: [0x05,0x00,0x38,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_lshlrev_b16 v5, m0, 0.5
-// GFX11: encoding: [0x05,0x00,0x38,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x38,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_lshlrev_b16 v5, exec_lo, -1
 // GFX11: encoding: [0x05,0x00,0x38,0xd7,0x7e,0x82,0x01,0x00]
@@ -2654,7 +2654,7 @@ v_lshlrev_b16 v5, -1, exec_hi
 // GFX11: encoding: [0x05,0x00,0x38,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_lshlrev_b16 v5, 0.5, m0
-// GFX11: encoding: [0x05,0x00,0x38,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x38,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_lshlrev_b16 v5, src_scc, vcc_lo
 // GFX11: encoding: [0x05,0x00,0x38,0xd7,0xfd,0xd4,0x00,0x00]
@@ -2711,7 +2711,7 @@ v_lshrrev_b16 v5, ttmp15, src_scc
 // GFX11: encoding: [0x05,0x00,0x39,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_lshrrev_b16 v5, m0, 0.5
-// GFX11: encoding: [0x05,0x00,0x39,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x39,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_lshrrev_b16 v5, exec_lo, -1
 // GFX11: encoding: [0x05,0x00,0x39,0xd7,0x7e,0x82,0x01,0x00]
@@ -2726,7 +2726,7 @@ v_lshrrev_b16 v5, -1, exec_hi
 // GFX11: encoding: [0x05,0x00,0x39,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_lshrrev_b16 v5, 0.5, m0
-// GFX11: encoding: [0x05,0x00,0x39,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x39,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_lshrrev_b16 v5, src_scc, vcc_lo
 // GFX11: encoding: [0x05,0x00,0x39,0xd7,0xfd,0xd4,0x00,0x00]
@@ -2783,7 +2783,7 @@ v_mad_i16 v5, ttmp15, src_scc, ttmp15
 // GFX11: encoding: [0x05,0x00,0x53,0xd6,0x7b,0xfa,0xed,0x01]
 
 v_mad_i16 v5, m0, 0.5, m0
-// GFX11: encoding: [0x05,0x00,0x53,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x53,0xd6,0x7d,0xe0,0xf5,0x01]
 
 v_mad_i16 v5, exec_lo, -1, vcc_hi
 // GFX11: encoding: [0x05,0x00,0x53,0xd6,0x7e,0x82,0xad,0x01]
@@ -2798,7 +2798,7 @@ v_mad_i16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0]
 // GFX11: encoding: [0x05,0x08,0x53,0xd6,0xc1,0xfe,0xf4,0x03]
 
 v_mad_i16 v5, 0.5, m0, 0.5 op_sel:[0,1,0,0]
-// GFX11: encoding: [0x05,0x10,0x53,0xd6,0xff,0xfa,0xfc,0x03,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x10,0x53,0xd6,0xf0,0xfa,0xc0,0x03]
 
 v_mad_i16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0]
 // GFX11: encoding: [0x05,0x20,0x53,0xd6,0xfd,0xd4,0x04,0x03]
@@ -2828,7 +2828,7 @@ v_mad_i32_i16 v5, ttmp15, src_scc, ttmp15
 // GFX11: encoding: [0x05,0x00,0x5a,0xd6,0x7b,0xfa,0xed,0x01]
 
 v_mad_i32_i16 v5, m0, 0.5, m0
-// GFX11: encoding: [0x05,0x00,0x5a,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x5a,0xd6,0x7d,0xe0,0xf5,0x01]
 
 v_mad_i32_i16 v5, exec_lo, -1, exec_hi
 // GFX11: encoding: [0x05,0x00,0x5a,0xd6,0x7e,0x82,0xfd,0x01]
@@ -2843,7 +2843,7 @@ v_mad_i32_i16 v5, -1, exec_hi, 0xaf123456
 // GFX11: encoding: [0x05,0x00,0x5a,0xd6,0xc1,0xfe,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
 v_mad_i32_i16 v5, 0.5, m0, -1 op_sel:[0,0,0,0]
-// GFX11: encoding: [0x05,0x00,0x5a,0xd6,0xff,0xfa,0x04,0x03,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x5a,0xd6,0xf0,0xfa,0x04,0x03]
 
 v_mad_i32_i16 v5, src_scc, vcc_lo, src_scc op_sel:[1,0,0,0]
 // GFX11: encoding: [0x05,0x08,0x5a,0xd6,0xfd,0xd4,0xf4,0x03]
@@ -2993,7 +2993,7 @@ v_mad_u16 v5, ttmp15, src_scc, ttmp15
 // GFX11: encoding: [0x05,0x00,0x41,0xd6,0x7b,0xfa,0xed,0x01]
 
 v_mad_u16 v5, m0, 0.5, m0
-// GFX11: encoding: [0x05,0x00,0x41,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x41,0xd6,0x7d,0xe0,0xf5,0x01]
 
 v_mad_u16 v5, exec_lo, -1, vcc_hi
 // GFX11: encoding: [0x05,0x00,0x41,0xd6,0x7e,0x82,0xad,0x01]
@@ -3008,7 +3008,7 @@ v_mad_u16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0]
 // GFX11: encoding: [0x05,0x08,0x41,0xd6,0xc1,0xfe,0xf4,0x03]
 
 v_mad_u16 v5, 0.5, m0, 0.5 op_sel:[0,1,0,0]
-// GFX11: encoding: [0x05,0x10,0x41,0xd6,0xff,0xfa,0xfc,0x03,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x10,0x41,0xd6,0xf0,0xfa,0xc0,0x03]
 
 v_mad_u16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0]
 // GFX11: encoding: [0x05,0x20,0x41,0xd6,0xfd,0xd4,0x04,0x03]
@@ -3038,7 +3038,7 @@ v_mad_u32_u16 v5, ttmp15, src_scc, ttmp15
 // GFX11: encoding: [0x05,0x00,0x59,0xd6,0x7b,0xfa,0xed,0x01]
 
 v_mad_u32_u16 v5, m0, 0.5, m0
-// GFX11: encoding: [0x05,0x00,0x59,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x59,0xd6,0x7d,0xe0,0xf5,0x01]
 
 v_mad_u32_u16 v5, exec_lo, -1, exec_hi
 // GFX11: encoding: [0x05,0x00,0x59,0xd6,0x7e,0x82,0xfd,0x01]
@@ -3053,7 +3053,7 @@ v_mad_u32_u16 v5, -1, exec_hi, 0xaf123456
 // GFX11: encoding: [0x05,0x00,0x59,0xd6,0xc1,0xfe,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
 v_mad_u32_u16 v5, 0.5, m0, -1 op_sel:[0,0,0,0]
-// GFX11: encoding: [0x05,0x00,0x59,0xd6,0xff,0xfa,0x04,0x03,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x59,0xd6,0xf0,0xfa,0x04,0x03]
 
 v_mad_u32_u16 v5, src_scc, vcc_lo, src_scc op_sel:[1,0,0,0]
 // GFX11: encoding: [0x05,0x08,0x59,0xd6,0xfd,0xd4,0xf4,0x03]
@@ -3296,7 +3296,7 @@ v_max3_i16 v5, ttmp15, src_scc, ttmp15
 // GFX11: encoding: [0x05,0x00,0x4d,0xd6,0x7b,0xfa,0xed,0x01]
 
 v_max3_i16 v5, m0, 0.5, m0
-// GFX11: encoding: [0x05,0x00,0x4d,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x4d,0xd6,0x7d,0xe0,0xf5,0x01]
 
 v_max3_i16 v5, exec_lo, -1, vcc_hi
 // GFX11: encoding: [0x05,0x00,0x4d,0xd6,0x7e,0x82,0xad,0x01]
@@ -3311,7 +3311,7 @@ v_max3_i16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0]
 // GFX11: encoding: [0x05,0x08,0x4d,0xd6,0xc1,0xfe,0xf4,0x03]
 
 v_max3_i16 v5, 0.5, m0, 0.5 op_sel:[0,1,0,0]
-// GFX11: encoding: [0x05,0x10,0x4d,0xd6,0xff,0xfa,0xfc,0x03,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x10,0x4d,0xd6,0xf0,0xfa,0xc0,0x03]
 
 v_max3_i16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0]
 // GFX11: encoding: [0x05,0x20,0x4d,0xd6,0xfd,0xd4,0x04,0x03]
@@ -3386,7 +3386,7 @@ v_max3_u16 v5, ttmp15, src_scc, ttmp15
 // GFX11: encoding: [0x05,0x00,0x4e,0xd6,0x7b,0xfa,0xed,0x01]
 
 v_max3_u16 v5, m0, 0.5, m0
-// GFX11: encoding: [0x05,0x00,0x4e,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x4e,0xd6,0x7d,0xe0,0xf5,0x01]
 
 v_max3_u16 v5, exec_lo, -1, vcc_hi
 // GFX11: encoding: [0x05,0x00,0x4e,0xd6,0x7e,0x82,0xad,0x01]
@@ -3401,7 +3401,7 @@ v_max3_u16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0]
 // GFX11: encoding: [0x05,0x08,0x4e,0xd6,0xc1,0xfe,0xf4,0x03]
 
 v_max3_u16 v5, 0.5, m0, 0.5 op_sel:[0,1,0,0]
-// GFX11: encoding: [0x05,0x10,0x4e,0xd6,0xff,0xfa,0xfc,0x03,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x10,0x4e,0xd6,0xf0,0xfa,0xc0,0x03]
 
 v_max3_u16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0]
 // GFX11: encoding: [0x05,0x20,0x4e,0xd6,0xfd,0xd4,0x04,0x03]
@@ -3512,7 +3512,7 @@ v_max_i16 v5, ttmp15, src_scc
 // GFX11: encoding: [0x05,0x00,0x0a,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_max_i16 v5, m0, 0.5
-// GFX11: encoding: [0x05,0x00,0x0a,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x0a,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_max_i16 v5, exec_lo, -1
 // GFX11: encoding: [0x05,0x00,0x0a,0xd7,0x7e,0x82,0x01,0x00]
@@ -3527,7 +3527,7 @@ v_max_i16 v5, -1, exec_hi
 // GFX11: encoding: [0x05,0x00,0x0a,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_max_i16 v5, 0.5, m0
-// GFX11: encoding: [0x05,0x00,0x0a,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x0a,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_max_i16 v5, src_scc, vcc_lo
 // GFX11: encoding: [0x05,0x00,0x0a,0xd7,0xfd,0xd4,0x00,0x00]
@@ -3557,7 +3557,7 @@ v_max_u16 v5, ttmp15, src_scc
 // GFX11: encoding: [0x05,0x00,0x09,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_max_u16 v5, m0, 0.5
-// GFX11: encoding: [0x05,0x00,0x09,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x09,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_max_u16 v5, exec_lo, -1
 // GFX11: encoding: [0x05,0x00,0x09,0xd7,0x7e,0x82,0x01,0x00]
@@ -3572,7 +3572,7 @@ v_max_u16 v5, -1, exec_hi
 // GFX11: encoding: [0x05,0x00,0x09,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_max_u16 v5, 0.5, m0
-// GFX11: encoding: [0x05,0x00,0x09,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x09,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_max_u16 v5, src_scc, vcc_lo
 // GFX11: encoding: [0x05,0x00,0x09,0xd7,0xfd,0xd4,0x00,0x00]
@@ -3965,7 +3965,7 @@ v_med3_i16 v5, ttmp15, src_scc, ttmp15
 // GFX11: encoding: [0x05,0x00,0x50,0xd6,0x7b,0xfa,0xed,0x01]
 
 v_med3_i16 v5, m0, 0.5, m0
-// GFX11: encoding: [0x05,0x00,0x50,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x50,0xd6,0x7d,0xe0,0xf5,0x01]
 
 v_med3_i16 v5, exec_lo, -1, vcc_hi
 // GFX11: encoding: [0x05,0x00,0x50,0xd6,0x7e,0x82,0xad,0x01]
@@ -3980,7 +3980,7 @@ v_med3_i16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0]
 // GFX11: encoding: [0x05,0x08,0x50,0xd6,0xc1,0xfe,0xf4,0x03]
 
 v_med3_i16 v5, 0.5, m0, 0.5 op_sel:[0,1,0,0]
-// GFX11: encoding: [0x05,0x10,0x50,0xd6,0xff,0xfa,0xfc,0x03,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x10,0x50,0xd6,0xf0,0xfa,0xc0,0x03]
 
 v_med3_i16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0]
 // GFX11: encoding: [0x05,0x20,0x50,0xd6,0xfd,0xd4,0x04,0x03]
@@ -4055,7 +4055,7 @@ v_med3_u16 v5, ttmp15, src_scc, ttmp15
 // GFX11: encoding: [0x05,0x00,0x51,0xd6,0x7b,0xfa,0xed,0x01]
 
 v_med3_u16 v5, m0, 0.5, m0
-// GFX11: encoding: [0x05,0x00,0x51,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x51,0xd6,0x7d,0xe0,0xf5,0x01]
 
 v_med3_u16 v5, exec_lo, -1, vcc_hi
 // GFX11: encoding: [0x05,0x00,0x51,0xd6,0x7e,0x82,0xad,0x01]
@@ -4070,7 +4070,7 @@ v_med3_u16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0]
 // GFX11: encoding: [0x05,0x08,0x51,0xd6,0xc1,0xfe,0xf4,0x03]
 
 v_med3_u16 v5, 0.5, m0, 0.5 op_sel:[0,1,0,0]
-// GFX11: encoding: [0x05,0x10,0x51,0xd6,0xff,0xfa,0xfc,0x03,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x10,0x51,0xd6,0xf0,0xfa,0xc0,0x03]
 
 v_med3_u16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0]
 // GFX11: encoding: [0x05,0x20,0x51,0xd6,0xfd,0xd4,0x04,0x03]
@@ -4238,7 +4238,7 @@ v_min3_i16 v5, ttmp15, src_scc, ttmp15
 // GFX11: encoding: [0x05,0x00,0x4a,0xd6,0x7b,0xfa,0xed,0x01]
 
 v_min3_i16 v5, m0, 0.5, m0
-// GFX11: encoding: [0x05,0x00,0x4a,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x4a,0xd6,0x7d,0xe0,0xf5,0x01]
 
 v_min3_i16 v5, exec_lo, -1, vcc_hi
 // GFX11: encoding: [0x05,0x00,0x4a,0xd6,0x7e,0x82,0xad,0x01]
@@ -4253,7 +4253,7 @@ v_min3_i16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0]
 // GFX11: encoding: [0x05,0x08,0x4a,0xd6,0xc1,0xfe,0xf4,0x03]
 
 v_min3_i16 v5, 0.5, m0, 0.5 op_sel:[0,1,0,0]
-// GFX11: encoding: [0x05,0x10,0x4a,0xd6,0xff,0xfa,0xfc,0x03,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x10,0x4a,0xd6,0xf0,0xfa,0xc0,0x03]
 
 v_min3_i16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0]
 // GFX11: encoding: [0x05,0x20,0x4a,0xd6,0xfd,0xd4,0x04,0x03]
@@ -4328,7 +4328,7 @@ v_min3_u16 v5, ttmp15, src_scc, ttmp15
 // GFX11: encoding: [0x05,0x00,0x4b,0xd6,0x7b,0xfa,0xed,0x01]
 
 v_min3_u16 v5, m0, 0.5, m0
-// GFX11: encoding: [0x05,0x00,0x4b,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x4b,0xd6,0x7d,0xe0,0xf5,0x01]
 
 v_min3_u16 v5, exec_lo, -1, vcc_hi
 // GFX11: encoding: [0x05,0x00,0x4b,0xd6,0x7e,0x82,0xad,0x01]
@@ -4343,7 +4343,7 @@ v_min3_u16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0]
 // GFX11: encoding: [0x05,0x08,0x4b,0xd6,0xc1,0xfe,0xf4,0x03]
 
 v_min3_u16 v5, 0.5, m0, 0.5 op_sel:[0,1,0,0]
-// GFX11: encoding: [0x05,0x10,0x4b,0xd6,0xff,0xfa,0xfc,0x03,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x10,0x4b,0xd6,0xf0,0xfa,0xc0,0x03]
 
 v_min3_u16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0]
 // GFX11: encoding: [0x05,0x20,0x4b,0xd6,0xfd,0xd4,0x04,0x03]
@@ -4454,7 +4454,7 @@ v_min_i16 v5, ttmp15, src_scc
 // GFX11: encoding: [0x05,0x00,0x0c,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_min_i16 v5, m0, 0.5
-// GFX11: encoding: [0x05,0x00,0x0c,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x0c,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_min_i16 v5, exec_lo, -1
 // GFX11: encoding: [0x05,0x00,0x0c,0xd7,0x7e,0x82,0x01,0x00]
@@ -4469,7 +4469,7 @@ v_min_i16 v5, -1, exec_hi
 // GFX11: encoding: [0x05,0x00,0x0c,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_min_i16 v5, 0.5, m0
-// GFX11: encoding: [0x05,0x00,0x0c,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x0c,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_min_i16 v5, src_scc, vcc_lo
 // GFX11: encoding: [0x05,0x00,0x0c,0xd7,0xfd,0xd4,0x00,0x00]
@@ -4499,7 +4499,7 @@ v_min_u16 v5, ttmp15, src_scc
 // GFX11: encoding: [0x05,0x00,0x0b,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_min_u16 v5, m0, 0.5
-// GFX11: encoding: [0x05,0x00,0x0b,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x0b,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_min_u16 v5, exec_lo, -1
 // GFX11: encoding: [0x05,0x00,0x0b,0xd7,0x7e,0x82,0x01,0x00]
@@ -4514,7 +4514,7 @@ v_min_u16 v5, -1, exec_hi
 // GFX11: encoding: [0x05,0x00,0x0b,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_min_u16 v5, 0.5, m0
-// GFX11: encoding: [0x05,0x00,0x0b,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x0b,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_min_u16 v5, src_scc, vcc_lo
 // GFX11: encoding: [0x05,0x00,0x0b,0xd7,0xfd,0xd4,0x00,0x00]
@@ -4985,7 +4985,7 @@ v_mul_lo_u16 v5, ttmp15, src_scc
 // GFX11: encoding: [0x05,0x00,0x05,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_mul_lo_u16 v5, m0, 0.5
-// GFX11: encoding: [0x05,0x00,0x05,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x05,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_mul_lo_u16 v5, exec_lo, -1
 // GFX11: encoding: [0x05,0x00,0x05,0xd7,0x7e,0x82,0x01,0x00]
@@ -5000,7 +5000,7 @@ v_mul_lo_u16 v5, -1, exec_hi
 // GFX11: encoding: [0x05,0x00,0x05,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_mul_lo_u16 v5, 0.5, m0
-// GFX11: encoding: [0x05,0x00,0x05,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x05,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_mul_lo_u16 v5, src_scc, vcc_lo
 // GFX11: encoding: [0x05,0x00,0x05,0xd7,0xfd,0xd4,0x00,0x00]
@@ -5165,7 +5165,7 @@ v_or_b16 v5, ttmp15, src_scc
 // GFX11: encoding: [0x05,0x00,0x63,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_or_b16 v5, m0, 0.5
-// GFX11: encoding: [0x05,0x00,0x63,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x63,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_or_b16 v5, exec_lo, -1
 // GFX11: encoding: [0x05,0x00,0x63,0xd7,0x7e,0x82,0x01,0x00]
@@ -5180,7 +5180,7 @@ v_or_b16 v5, -1, exec_hi
 // GFX11: encoding: [0x05,0x00,0x63,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_or_b16 v5, 0.5, m0
-// GFX11: encoding: [0x05,0x00,0x63,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x63,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_or_b16 v5, src_scc, vcc_lo
 // GFX11: encoding: [0x05,0x00,0x63,0xd7,0xfd,0xd4,0x00,0x00]
@@ -5751,7 +5751,7 @@ v_sub_nc_i16 v5, ttmp15, src_scc
 // GFX11: encoding: [0x05,0x00,0x0e,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_sub_nc_i16 v5, m0, 0.5
-// GFX11: encoding: [0x05,0x00,0x0e,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x0e,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_sub_nc_i16 v5, exec_lo, -1
 // GFX11: encoding: [0x05,0x00,0x0e,0xd7,0x7e,0x82,0x01,0x00]
@@ -5766,7 +5766,7 @@ v_sub_nc_i16 v5, -1, exec_hi op_sel:[0,0,0]
 // GFX11: encoding: [0x05,0x00,0x0e,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_sub_nc_i16 v5, 0.5, m0 op_sel:[1,0,0]
-// GFX11: encoding: [0x05,0x08,0x0e,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x08,0x0e,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_sub_nc_i16 v5, src_scc, vcc_lo op_sel:[0,1,0]
 // GFX11: encoding: [0x05,0x10,0x0e,0xd7,0xfd,0xd4,0x00,0x00]
@@ -5841,7 +5841,7 @@ v_sub_nc_u16 v5, ttmp15, src_scc
 // GFX11: encoding: [0x05,0x00,0x04,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_sub_nc_u16 v5, m0, 0.5
-// GFX11: encoding: [0x05,0x00,0x04,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x04,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_sub_nc_u16 v5, exec_lo, -1
 // GFX11: encoding: [0x05,0x00,0x04,0xd7,0x7e,0x82,0x01,0x00]
@@ -5856,7 +5856,7 @@ v_sub_nc_u16 v5, -1, exec_hi op_sel:[0,0,0]
 // GFX11: encoding: [0x05,0x00,0x04,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_sub_nc_u16 v5, 0.5, m0 op_sel:[1,0,0]
-// GFX11: encoding: [0x05,0x08,0x04,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x08,0x04,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_sub_nc_u16 v5, src_scc, vcc_lo op_sel:[0,1,0]
 // GFX11: encoding: [0x05,0x10,0x04,0xd7,0xfd,0xd4,0x00,0x00]
@@ -6175,7 +6175,7 @@ v_xor_b16 v5, ttmp15, src_scc
 // GFX11: encoding: [0x05,0x00,0x64,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_xor_b16 v5, m0, 0.5
-// GFX11: encoding: [0x05,0x00,0x64,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x64,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_xor_b16 v5, exec_lo, -1
 // GFX11: encoding: [0x05,0x00,0x64,0xd7,0x7e,0x82,0x01,0x00]
@@ -6190,7 +6190,7 @@ v_xor_b16 v5, -1, exec_hi
 // GFX11: encoding: [0x05,0x00,0x64,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_xor_b16 v5, 0.5, m0
-// GFX11: encoding: [0x05,0x00,0x64,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x64,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_xor_b16 v5, src_scc, vcc_lo
 // GFX11: encoding: [0x05,0x00,0x64,0xd7,0xfd,0xd4,0x00,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vop1.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vop1.s
index fb4e9108fe1d..71e0f8250415 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vop1.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vop1.s
@@ -478,7 +478,7 @@ v_cvt_f16_i16_e64 v5, -1
 // GFX11: encoding: [0x05,0x00,0xd1,0xd5,0xc1,0x00,0x00,0x00]
 
 v_cvt_f16_i16_e64 v5, 0.5 mul:2
-// GFX11: encoding: [0x05,0x00,0xd1,0xd5,0xff,0x00,0x00,0x08,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0xd1,0xd5,0xf0,0x00,0x00,0x08]
 
 v_cvt_f16_i16_e64 v5, src_scc mul:4
 // GFX11: encoding: [0x05,0x00,0xd1,0xd5,0xfd,0x00,0x00,0x10]
@@ -523,7 +523,7 @@ v_cvt_f16_u16_e64 v5, -1
 // GFX11: encoding: [0x05,0x00,0xd0,0xd5,0xc1,0x00,0x00,0x00]
 
 v_cvt_f16_u16_e64 v5, 0.5 mul:2
-// GFX11: encoding: [0x05,0x00,0xd0,0xd5,0xff,0x00,0x00,0x08,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0xd0,0xd5,0xf0,0x00,0x00,0x08]
 
 v_cvt_f16_u16_e64 v5, src_scc mul:4
 // GFX11: encoding: [0x05,0x00,0xd0,0xd5,0xfd,0x00,0x00,0x10]
@@ -1270,7 +1270,7 @@ v_cvt_i32_i16_e64 v5, -1
 // GFX11: encoding: [0x05,0x00,0xea,0xd5,0xc1,0x00,0x00,0x00]
 
 v_cvt_i32_i16_e64 v5, 0.5
-// GFX11: encoding: [0x05,0x00,0xea,0xd5,0xff,0x00,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0xea,0xd5,0xf0,0x00,0x00,0x00]
 
 v_cvt_i32_i16_e64 v5, src_scc
 // GFX11: encoding: [0x05,0x00,0xea,0xd5,0xfd,0x00,0x00,0x00]
@@ -1666,7 +1666,7 @@ v_cvt_u32_u16_e64 v5, -1
 // GFX11: encoding: [0x05,0x00,0xeb,0xd5,0xc1,0x00,0x00,0x00]
 
 v_cvt_u32_u16_e64 v5, 0.5
-// GFX11: encoding: [0x05,0x00,0xeb,0xd5,0xff,0x00,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0xeb,0xd5,0xf0,0x00,0x00,0x00]
 
 v_cvt_u32_u16_e64 v5, src_scc
 // GFX11: encoding: [0x05,0x00,0xeb,0xd5,0xfd,0x00,0x00,0x00]
@@ -2641,7 +2641,7 @@ v_not_b16_e64 v5, -1
 // GFX11: encoding: [0x05,0x00,0xe9,0xd5,0xc1,0x00,0x00,0x00]
 
 v_not_b16_e64 v5, 0.5
-// GFX11: encoding: [0x05,0x00,0xe9,0xd5,0xff,0x00,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0xe9,0xd5,0xf0,0x00,0x00,0x00]
 
 v_not_b16_e64 v5, src_scc
 // GFX11: encoding: [0x05,0x00,0xe9,0xd5,0xfd,0x00,0x00,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vopc.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vopc.s
index 3a6a61891d29..72894715408a 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vopc.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vopc.s
@@ -702,7 +702,7 @@ v_cmp_eq_i16_e64 s5, ttmp15, src_scc
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_i16_e64 s5, m0, 0.5
-// W32: encoding: [0x05,0x00,0x32,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x05,0x00,0x32,0xd4,0x7d,0xe0,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_i16_e64 s5, exec_lo, -1
@@ -722,7 +722,7 @@ v_cmp_eq_i16_e64 vcc_lo, -1, exec_hi
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_i16_e64 vcc_hi, 0.5, m0
-// W32: encoding: [0x6b,0x00,0x32,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x6b,0x00,0x32,0xd4,0xf0,0xfa,0x00,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_i16_e64 ttmp15, src_scc, vcc_lo
@@ -758,7 +758,7 @@ v_cmp_eq_i16_e64 s[10:11], ttmp15, src_scc
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_i16_e64 s[10:11], m0, 0.5
-// W64: encoding: [0x0a,0x00,0x32,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x32,0xd4,0x7d,0xe0,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_i16_e64 s[10:11], exec_lo, -1
@@ -778,7 +778,7 @@ v_cmp_eq_i16_e64 s[104:105], -1, exec_hi
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_i16_e64 vcc, 0.5, m0
-// W64: encoding: [0x6a,0x00,0x32,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x6a,0x00,0x32,0xd4,0xf0,0xfa,0x00,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_i16_e64 ttmp[14:15], src_scc, vcc_lo
@@ -1023,7 +1023,7 @@ v_cmp_eq_u16_e64 s5, ttmp15, src_scc
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_u16_e64 s5, m0, 0.5
-// W32: encoding: [0x05,0x00,0x3a,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x05,0x00,0x3a,0xd4,0x7d,0xe0,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_u16_e64 s5, exec_lo, -1
@@ -1043,7 +1043,7 @@ v_cmp_eq_u16_e64 vcc_lo, -1, exec_hi
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_u16_e64 vcc_hi, 0.5, m0
-// W32: encoding: [0x6b,0x00,0x3a,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x6b,0x00,0x3a,0xd4,0xf0,0xfa,0x00,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_u16_e64 ttmp15, src_scc, vcc_lo
@@ -1079,7 +1079,7 @@ v_cmp_eq_u16_e64 s[10:11], ttmp15, src_scc
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_u16_e64 s[10:11], m0, 0.5
-// W64: encoding: [0x0a,0x00,0x3a,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x3a,0xd4,0x7d,0xe0,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_u16_e64 s[10:11], exec_lo, -1
@@ -1099,7 +1099,7 @@ v_cmp_eq_u16_e64 s[104:105], -1, exec_hi
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_u16_e64 vcc, 0.5, m0
-// W64: encoding: [0x6a,0x00,0x3a,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x6a,0x00,0x3a,0xd4,0xf0,0xfa,0x00,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_u16_e64 ttmp[14:15], src_scc, vcc_lo
@@ -2398,7 +2398,7 @@ v_cmp_ge_i16_e64 s5, ttmp15, src_scc
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_i16_e64 s5, m0, 0.5
-// W32: encoding: [0x05,0x00,0x36,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x05,0x00,0x36,0xd4,0x7d,0xe0,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_i16_e64 s5, exec_lo, -1
@@ -2418,7 +2418,7 @@ v_cmp_ge_i16_e64 vcc_lo, -1, exec_hi
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_i16_e64 vcc_hi, 0.5, m0
-// W32: encoding: [0x6b,0x00,0x36,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x6b,0x00,0x36,0xd4,0xf0,0xfa,0x00,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_i16_e64 ttmp15, src_scc, vcc_lo
@@ -2454,7 +2454,7 @@ v_cmp_ge_i16_e64 s[10:11], ttmp15, src_scc
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_i16_e64 s[10:11], m0, 0.5
-// W64: encoding: [0x0a,0x00,0x36,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x36,0xd4,0x7d,0xe0,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_i16_e64 s[10:11], exec_lo, -1
@@ -2474,7 +2474,7 @@ v_cmp_ge_i16_e64 s[104:105], -1, exec_hi
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_i16_e64 vcc, 0.5, m0
-// W64: encoding: [0x6a,0x00,0x36,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x6a,0x00,0x36,0xd4,0xf0,0xfa,0x00,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_i16_e64 ttmp[14:15], src_scc, vcc_lo
@@ -2719,7 +2719,7 @@ v_cmp_ge_u16_e64 s5, ttmp15, src_scc
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_u16_e64 s5, m0, 0.5
-// W32: encoding: [0x05,0x00,0x3e,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x05,0x00,0x3e,0xd4,0x7d,0xe0,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_u16_e64 s5, exec_lo, -1
@@ -2739,7 +2739,7 @@ v_cmp_ge_u16_e64 vcc_lo, -1, exec_hi
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_u16_e64 vcc_hi, 0.5, m0
-// W32: encoding: [0x6b,0x00,0x3e,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x6b,0x00,0x3e,0xd4,0xf0,0xfa,0x00,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_u16_e64 ttmp15, src_scc, vcc_lo
@@ -2775,7 +2775,7 @@ v_cmp_ge_u16_e64 s[10:11], ttmp15, src_scc
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_u16_e64 s[10:11], m0, 0.5
-// W64: encoding: [0x0a,0x00,0x3e,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x3e,0xd4,0x7d,0xe0,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_u16_e64 s[10:11], exec_lo, -1
@@ -2795,7 +2795,7 @@ v_cmp_ge_u16_e64 s[104:105], -1, exec_hi
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_u16_e64 vcc, 0.5, m0
-// W64: encoding: [0x6a,0x00,0x3e,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x6a,0x00,0x3e,0xd4,0xf0,0xfa,0x00,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_u16_e64 ttmp[14:15], src_scc, vcc_lo
@@ -3361,7 +3361,7 @@ v_cmp_gt_i16_e64 s5, ttmp15, src_scc
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_i16_e64 s5, m0, 0.5
-// W32: encoding: [0x05,0x00,0x34,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x05,0x00,0x34,0xd4,0x7d,0xe0,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_i16_e64 s5, exec_lo, -1
@@ -3381,7 +3381,7 @@ v_cmp_gt_i16_e64 vcc_lo, -1, exec_hi
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_i16_e64 vcc_hi, 0.5, m0
-// W32: encoding: [0x6b,0x00,0x34,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x6b,0x00,0x34,0xd4,0xf0,0xfa,0x00,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_i16_e64 ttmp15, src_scc, vcc_lo
@@ -3417,7 +3417,7 @@ v_cmp_gt_i16_e64 s[10:11], ttmp15, src_scc
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_i16_e64 s[10:11], m0, 0.5
-// W64: encoding: [0x0a,0x00,0x34,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x34,0xd4,0x7d,0xe0,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_i16_e64 s[10:11], exec_lo, -1
@@ -3437,7 +3437,7 @@ v_cmp_gt_i16_e64 s[104:105], -1, exec_hi
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_i16_e64 vcc, 0.5, m0
-// W64: encoding: [0x6a,0x00,0x34,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x6a,0x00,0x34,0xd4,0xf0,0xfa,0x00,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_i16_e64 ttmp[14:15], src_scc, vcc_lo
@@ -3682,7 +3682,7 @@ v_cmp_gt_u16_e64 s5, ttmp15, src_scc
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_u16_e64 s5, m0, 0.5
-// W32: encoding: [0x05,0x00,0x3c,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x05,0x00,0x3c,0xd4,0x7d,0xe0,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_u16_e64 s5, exec_lo, -1
@@ -3702,7 +3702,7 @@ v_cmp_gt_u16_e64 vcc_lo, -1, exec_hi
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_u16_e64 vcc_hi, 0.5, m0
-// W32: encoding: [0x6b,0x00,0x3c,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x6b,0x00,0x3c,0xd4,0xf0,0xfa,0x00,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_u16_e64 ttmp15, src_scc, vcc_lo
@@ -3738,7 +3738,7 @@ v_cmp_gt_u16_e64 s[10:11], ttmp15, src_scc
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_u16_e64 s[10:11], m0, 0.5
-// W64: encoding: [0x0a,0x00,0x3c,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x3c,0xd4,0x7d,0xe0,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_u16_e64 s[10:11], exec_lo, -1
@@ -3758,7 +3758,7 @@ v_cmp_gt_u16_e64 s[104:105], -1, exec_hi
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_u16_e64 vcc, 0.5, m0
-// W64: encoding: [0x6a,0x00,0x3c,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x6a,0x00,0x3c,0xd4,0xf0,0xfa,0x00,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_u16_e64 ttmp[14:15], src_scc, vcc_lo
@@ -4324,7 +4324,7 @@ v_cmp_le_i16_e64 s5, ttmp15, src_scc
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_i16_e64 s5, m0, 0.5
-// W32: encoding: [0x05,0x00,0x33,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x05,0x00,0x33,0xd4,0x7d,0xe0,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_i16_e64 s5, exec_lo, -1
@@ -4344,7 +4344,7 @@ v_cmp_le_i16_e64 vcc_lo, -1, exec_hi
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_i16_e64 vcc_hi, 0.5, m0
-// W32: encoding: [0x6b,0x00,0x33,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x6b,0x00,0x33,0xd4,0xf0,0xfa,0x00,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_i16_e64 ttmp15, src_scc, vcc_lo
@@ -4380,7 +4380,7 @@ v_cmp_le_i16_e64 s[10:11], ttmp15, src_scc
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_i16_e64 s[10:11], m0, 0.5
-// W64: encoding: [0x0a,0x00,0x33,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x33,0xd4,0x7d,0xe0,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_i16_e64 s[10:11], exec_lo, -1
@@ -4400,7 +4400,7 @@ v_cmp_le_i16_e64 s[104:105], -1, exec_hi
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_i16_e64 vcc, 0.5, m0
-// W64: encoding: [0x6a,0x00,0x33,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x6a,0x00,0x33,0xd4,0xf0,0xfa,0x00,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_i16_e64 ttmp[14:15], src_scc, vcc_lo
@@ -4645,7 +4645,7 @@ v_cmp_le_u16_e64 s5, ttmp15, src_scc
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_u16_e64 s5, m0, 0.5
-// W32: encoding: [0x05,0x00,0x3b,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x05,0x00,0x3b,0xd4,0x7d,0xe0,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_u16_e64 s5, exec_lo, -1
@@ -4665,7 +4665,7 @@ v_cmp_le_u16_e64 vcc_lo, -1, exec_hi
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_u16_e64 vcc_hi, 0.5, m0
-// W32: encoding: [0x6b,0x00,0x3b,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x6b,0x00,0x3b,0xd4,0xf0,0xfa,0x00,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_u16_e64 ttmp15, src_scc, vcc_lo
@@ -4701,7 +4701,7 @@ v_cmp_le_u16_e64 s[10:11], ttmp15, src_scc
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_u16_e64 s[10:11], m0, 0.5
-// W64: encoding: [0x0a,0x00,0x3b,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x3b,0xd4,0x7d,0xe0,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_u16_e64 s[10:11], exec_lo, -1
@@ -4721,7 +4721,7 @@ v_cmp_le_u16_e64 s[104:105], -1, exec_hi
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_u16_e64 vcc, 0.5, m0
-// W64: encoding: [0x6a,0x00,0x3b,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x6a,0x00,0x3b,0xd4,0xf0,0xfa,0x00,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_u16_e64 ttmp[14:15], src_scc, vcc_lo
@@ -5608,7 +5608,7 @@ v_cmp_lt_i16_e64 s5, ttmp15, src_scc
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_i16_e64 s5, m0, 0.5
-// W32: encoding: [0x05,0x00,0x31,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x05,0x00,0x31,0xd4,0x7d,0xe0,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_i16_e64 s5, exec_lo, -1
@@ -5628,7 +5628,7 @@ v_cmp_lt_i16_e64 vcc_lo, -1, exec_hi
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_i16_e64 vcc_hi, 0.5, m0
-// W32: encoding: [0x6b,0x00,0x31,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x6b,0x00,0x31,0xd4,0xf0,0xfa,0x00,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_i16_e64 ttmp15, src_scc, vcc_lo
@@ -5664,7 +5664,7 @@ v_cmp_lt_i16_e64 s[10:11], ttmp15, src_scc
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_i16_e64 s[10:11], m0, 0.5
-// W64: encoding: [0x0a,0x00,0x31,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x31,0xd4,0x7d,0xe0,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_i16_e64 s[10:11], exec_lo, -1
@@ -5684,7 +5684,7 @@ v_cmp_lt_i16_e64 s[104:105], -1, exec_hi
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_i16_e64 vcc, 0.5, m0
-// W64: encoding: [0x6a,0x00,0x31,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x6a,0x00,0x31,0xd4,0xf0,0xfa,0x00,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_i16_e64 ttmp[14:15], src_scc, vcc_lo
@@ -5929,7 +5929,7 @@ v_cmp_lt_u16_e64 s5, ttmp15, src_scc
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_u16_e64 s5, m0, 0.5
-// W32: encoding: [0x05,0x00,0x39,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x05,0x00,0x39,0xd4,0x7d,0xe0,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_u16_e64 s5, exec_lo, -1
@@ -5949,7 +5949,7 @@ v_cmp_lt_u16_e64 vcc_lo, -1, exec_hi
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_u16_e64 vcc_hi, 0.5, m0
-// W32: encoding: [0x6b,0x00,0x39,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x6b,0x00,0x39,0xd4,0xf0,0xfa,0x00,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_u16_e64 ttmp15, src_scc, vcc_lo
@@ -5985,7 +5985,7 @@ v_cmp_lt_u16_e64 s[10:11], ttmp15, src_scc
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_u16_e64 s[10:11], m0, 0.5
-// W64: encoding: [0x0a,0x00,0x39,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x39,0xd4,0x7d,0xe0,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_u16_e64 s[10:11], exec_lo, -1
@@ -6005,7 +6005,7 @@ v_cmp_lt_u16_e64 s[104:105], -1, exec_hi
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_u16_e64 vcc, 0.5, m0
-// W64: encoding: [0x6a,0x00,0x39,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x6a,0x00,0x39,0xd4,0xf0,0xfa,0x00,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_u16_e64 ttmp[14:15], src_scc, vcc_lo
@@ -6250,7 +6250,7 @@ v_cmp_ne_i16_e64 s5, ttmp15, src_scc
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_i16_e64 s5, m0, 0.5
-// W32: encoding: [0x05,0x00,0x35,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x05,0x00,0x35,0xd4,0x7d,0xe0,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_i16_e64 s5, exec_lo, -1
@@ -6270,7 +6270,7 @@ v_cmp_ne_i16_e64 vcc_lo, -1, exec_hi
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_i16_e64 vcc_hi, 0.5, m0
-// W32: encoding: [0x6b,0x00,0x35,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x6b,0x00,0x35,0xd4,0xf0,0xfa,0x00,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_i16_e64 ttmp15, src_scc, vcc_lo
@@ -6306,7 +6306,7 @@ v_cmp_ne_i16_e64 s[10:11], ttmp15, src_scc
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_i16_e64 s[10:11], m0, 0.5
-// W64: encoding: [0x0a,0x00,0x35,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x35,0xd4,0x7d,0xe0,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_i16_e64 s[10:11], exec_lo, -1
@@ -6326,7 +6326,7 @@ v_cmp_ne_i16_e64 s[104:105], -1, exec_hi
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_i16_e64 vcc, 0.5, m0
-// W64: encoding: [0x6a,0x00,0x35,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x6a,0x00,0x35,0xd4,0xf0,0xfa,0x00,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_i16_e64 ttmp[14:15], src_scc, vcc_lo
@@ -6571,7 +6571,7 @@ v_cmp_ne_u16_e64 s5, ttmp15, src_scc
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_u16_e64 s5, m0, 0.5
-// W32: encoding: [0x05,0x00,0x3d,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x05,0x00,0x3d,0xd4,0x7d,0xe0,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_u16_e64 s5, exec_lo, -1
@@ -6591,7 +6591,7 @@ v_cmp_ne_u16_e64 vcc_lo, -1, exec_hi
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_u16_e64 vcc_hi, 0.5, m0
-// W32: encoding: [0x6b,0x00,0x3d,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x6b,0x00,0x3d,0xd4,0xf0,0xfa,0x00,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_u16_e64 ttmp15, src_scc, vcc_lo
@@ -6627,7 +6627,7 @@ v_cmp_ne_u16_e64 s[10:11], ttmp15, src_scc
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_u16_e64 s[10:11], m0, 0.5
-// W64: encoding: [0x0a,0x00,0x3d,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x3d,0xd4,0x7d,0xe0,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_u16_e64 s[10:11], exec_lo, -1
@@ -6647,7 +6647,7 @@ v_cmp_ne_u16_e64 s[104:105], -1, exec_hi
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_u16_e64 vcc, 0.5, m0
-// W64: encoding: [0x6a,0x00,0x3d,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x6a,0x00,0x3d,0xd4,0xf0,0xfa,0x00,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_u16_e64 ttmp[14:15], src_scc, vcc_lo
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vopcx.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vopcx.s
index f50a47777f64..79409a568475 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vopcx.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vopcx.s
@@ -287,7 +287,7 @@ v_cmpx_eq_i16_e64 ttmp15, src_scc
 // GFX11: encoding: [0x7e,0x00,0xb2,0xd4,0x7b,0xfa,0x01,0x00]
 
 v_cmpx_eq_i16_e64 m0, 0.5
-// GFX11: encoding: [0x7e,0x00,0xb2,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x7e,0x00,0xb2,0xd4,0x7d,0xe0,0x01,0x00]
 
 v_cmpx_eq_i16_e64 exec_lo, -1
 // GFX11: encoding: [0x7e,0x00,0xb2,0xd4,0x7e,0x82,0x01,0x00]
@@ -302,7 +302,7 @@ v_cmpx_eq_i16_e64 -1, exec_hi
 // GFX11: encoding: [0x7e,0x00,0xb2,0xd4,0xc1,0xfe,0x00,0x00]
 
 v_cmpx_eq_i16_e64 0.5, m0
-// GFX11: encoding: [0x7e,0x00,0xb2,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x7e,0x00,0xb2,0xd4,0xf0,0xfa,0x00,0x00]
 
 v_cmpx_eq_i16_e64 src_scc, vcc_lo
 // GFX11: encoding: [0x7e,0x00,0xb2,0xd4,0xfd,0xd4,0x00,0x00]
@@ -413,7 +413,7 @@ v_cmpx_eq_u16_e64 ttmp15, src_scc
 // GFX11: encoding: [0x7e,0x00,0xba,0xd4,0x7b,0xfa,0x01,0x00]
 
 v_cmpx_eq_u16_e64 m0, 0.5
-// GFX11: encoding: [0x7e,0x00,0xba,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x7e,0x00,0xba,0xd4,0x7d,0xe0,0x01,0x00]
 
 v_cmpx_eq_u16_e64 exec_lo, -1
 // GFX11: encoding: [0x7e,0x00,0xba,0xd4,0x7e,0x82,0x01,0x00]
@@ -428,7 +428,7 @@ v_cmpx_eq_u16_e64 -1, exec_hi
 // GFX11: encoding: [0x7e,0x00,0xba,0xd4,0xc1,0xfe,0x00,0x00]
 
 v_cmpx_eq_u16_e64 0.5, m0
-// GFX11: encoding: [0x7e,0x00,0xba,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x7e,0x00,0xba,0xd4,0xf0,0xfa,0x00,0x00]
 
 v_cmpx_eq_u16_e64 src_scc, vcc_lo
 // GFX11: encoding: [0x7e,0x00,0xba,0xd4,0xfd,0xd4,0x00,0x00]
@@ -953,7 +953,7 @@ v_cmpx_ge_i16_e64 ttmp15, src_scc
 // GFX11: encoding: [0x7e,0x00,0xb6,0xd4,0x7b,0xfa,0x01,0x00]
 
 v_cmpx_ge_i16_e64 m0, 0.5
-// GFX11: encoding: [0x7e,0x00,0xb6,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x7e,0x00,0xb6,0xd4,0x7d,0xe0,0x01,0x00]
 
 v_cmpx_ge_i16_e64 exec_lo, -1
 // GFX11: encoding: [0x7e,0x00,0xb6,0xd4,0x7e,0x82,0x01,0x00]
@@ -968,7 +968,7 @@ v_cmpx_ge_i16_e64 -1, exec_hi
 // GFX11: encoding: [0x7e,0x00,0xb6,0xd4,0xc1,0xfe,0x00,0x00]
 
 v_cmpx_ge_i16_e64 0.5, m0
-// GFX11: encoding: [0x7e,0x00,0xb6,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x7e,0x00,0xb6,0xd4,0xf0,0xfa,0x00,0x00]
 
 v_cmpx_ge_i16_e64 src_scc, vcc_lo
 // GFX11: encoding: [0x7e,0x00,0xb6,0xd4,0xfd,0xd4,0x00,0x00]
@@ -1079,7 +1079,7 @@ v_cmpx_ge_u16_e64 ttmp15, src_scc
 // GFX11: encoding: [0x7e,0x00,0xbe,0xd4,0x7b,0xfa,0x01,0x00]
 
 v_cmpx_ge_u16_e64 m0, 0.5
-// GFX11: encoding: [0x7e,0x00,0xbe,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x7e,0x00,0xbe,0xd4,0x7d,0xe0,0x01,0x00]
 
 v_cmpx_ge_u16_e64 exec_lo, -1
 // GFX11: encoding: [0x7e,0x00,0xbe,0xd4,0x7e,0x82,0x01,0x00]
@@ -1094,7 +1094,7 @@ v_cmpx_ge_u16_e64 -1, exec_hi
 // GFX11: encoding: [0x7e,0x00,0xbe,0xd4,0xc1,0xfe,0x00,0x00]
 
 v_cmpx_ge_u16_e64 0.5, m0
-// GFX11: encoding: [0x7e,0x00,0xbe,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x7e,0x00,0xbe,0xd4,0xf0,0xfa,0x00,0x00]
 
 v_cmpx_ge_u16_e64 src_scc, vcc_lo
 // GFX11: encoding: [0x7e,0x00,0xbe,0xd4,0xfd,0xd4,0x00,0x00]
@@ -1331,7 +1331,7 @@ v_cmpx_gt_i16_e64 ttmp15, src_scc
 // GFX11: encoding: [0x7e,0x00,0xb4,0xd4,0x7b,0xfa,0x01,0x00]
 
 v_cmpx_gt_i16_e64 m0, 0.5
-// GFX11: encoding: [0x7e,0x00,0xb4,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x7e,0x00,0xb4,0xd4,0x7d,0xe0,0x01,0x00]
 
 v_cmpx_gt_i16_e64 exec_lo, -1
 // GFX11: encoding: [0x7e,0x00,0xb4,0xd4,0x7e,0x82,0x01,0x00]
@@ -1346,7 +1346,7 @@ v_cmpx_gt_i16_e64 -1, exec_hi
 // GFX11: encoding: [0x7e,0x00,0xb4,0xd4,0xc1,0xfe,0x00,0x00]
 
 v_cmpx_gt_i16_e64 0.5, m0
-// GFX11: encoding: [0x7e,0x00,0xb4,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x7e,0x00,0xb4,0xd4,0xf0,0xfa,0x00,0x00]
 
 v_cmpx_gt_i16_e64 src_scc, vcc_lo
 // GFX11: encoding: [0x7e,0x00,0xb4,0xd4,0xfd,0xd4,0x00,0x00]
@@ -1457,7 +1457,7 @@ v_cmpx_gt_u16_e64 ttmp15, src_scc
 // GFX11: encoding: [0x7e,0x00,0xbc,0xd4,0x7b,0xfa,0x01,0x00]
 
 v_cmpx_gt_u16_e64 m0, 0.5
-// GFX11: encoding: [0x7e,0x00,0xbc,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x7e,0x00,0xbc,0xd4,0x7d,0xe0,0x01,0x00]
 
 v_cmpx_gt_u16_e64 exec_lo, -1
 // GFX11: encoding: [0x7e,0x00,0xbc,0xd4,0x7e,0x82,0x01,0x00]
@@ -1472,7 +1472,7 @@ v_cmpx_gt_u16_e64 -1, exec_hi
 // GFX11: encoding: [0x7e,0x00,0xbc,0xd4,0xc1,0xfe,0x00,0x00]
 
 v_cmpx_gt_u16_e64 0.5, m0
-// GFX11: encoding: [0x7e,0x00,0xbc,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x7e,0x00,0xbc,0xd4,0xf0,0xfa,0x00,0x00]
 
 v_cmpx_gt_u16_e64 src_scc, vcc_lo
 // GFX11: encoding: [0x7e,0x00,0xbc,0xd4,0xfd,0xd4,0x00,0x00]
@@ -1709,7 +1709,7 @@ v_cmpx_le_i16_e64 ttmp15, src_scc
 // GFX11: encoding: [0x7e,0x00,0xb3,0xd4,0x7b,0xfa,0x01,0x00]
 
 v_cmpx_le_i16_e64 m0, 0.5
-// GFX11: encoding: [0x7e,0x00,0xb3,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x7e,0x00,0xb3,0xd4,0x7d,0xe0,0x01,0x00]
 
 v_cmpx_le_i16_e64 exec_lo, -1
 // GFX11: encoding: [0x7e,0x00,0xb3,0xd4,0x7e,0x82,0x01,0x00]
@@ -1724,7 +1724,7 @@ v_cmpx_le_i16_e64 -1, exec_hi
 // GFX11: encoding: [0x7e,0x00,0xb3,0xd4,0xc1,0xfe,0x00,0x00]
 
 v_cmpx_le_i16_e64 0.5, m0
-// GFX11: encoding: [0x7e,0x00,0xb3,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x7e,0x00,0xb3,0xd4,0xf0,0xfa,0x00,0x00]
 
 v_cmpx_le_i16_e64 src_scc, vcc_lo
 // GFX11: encoding: [0x7e,0x00,0xb3,0xd4,0xfd,0xd4,0x00,0x00]
@@ -1835,7 +1835,7 @@ v_cmpx_le_u16_e64 ttmp15, src_scc
 // GFX11: encoding: [0x7e,0x00,0xbb,0xd4,0x7b,0xfa,0x01,0x00]
 
 v_cmpx_le_u16_e64 m0, 0.5
-// GFX11: encoding: [0x7e,0x00,0xbb,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x7e,0x00,0xbb,0xd4,0x7d,0xe0,0x01,0x00]
 
 v_cmpx_le_u16_e64 exec_lo, -1
 // GFX11: encoding: [0x7e,0x00,0xbb,0xd4,0x7e,0x82,0x01,0x00]
@@ -1850,7 +1850,7 @@ v_cmpx_le_u16_e64 -1, exec_hi
 // GFX11: encoding: [0x7e,0x00,0xbb,0xd4,0xc1,0xfe,0x00,0x00]
 
 v_cmpx_le_u16_e64 0.5, m0
-// GFX11: encoding: [0x7e,0x00,0xbb,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x7e,0x00,0xbb,0xd4,0xf0,0xfa,0x00,0x00]
 
 v_cmpx_le_u16_e64 src_scc, vcc_lo
 // GFX11: encoding: [0x7e,0x00,0xbb,0xd4,0xfd,0xd4,0x00,0x00]
@@ -2213,7 +2213,7 @@ v_cmpx_lt_i16_e64 ttmp15, src_scc
 // GFX11: encoding: [0x7e,0x00,0xb1,0xd4,0x7b,0xfa,0x01,0x00]
 
 v_cmpx_lt_i16_e64 m0, 0.5
-// GFX11: encoding: [0x7e,0x00,0xb1,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x7e,0x00,0xb1,0xd4,0x7d,0xe0,0x01,0x00]
 
 v_cmpx_lt_i16_e64 exec_lo, -1
 // GFX11: encoding: [0x7e,0x00,0xb1,0xd4,0x7e,0x82,0x01,0x00]
@@ -2228,7 +2228,7 @@ v_cmpx_lt_i16_e64 -1, exec_hi
 // GFX11: encoding: [0x7e,0x00,0xb1,0xd4,0xc1,0xfe,0x00,0x00]
 
 v_cmpx_lt_i16_e64 0.5, m0
-// GFX11: encoding: [0x7e,0x00,0xb1,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x7e,0x00,0xb1,0xd4,0xf0,0xfa,0x00,0x00]
 
 v_cmpx_lt_i16_e64 src_scc, vcc_lo
 // GFX11: encoding: [0x7e,0x00,0xb1,0xd4,0xfd,0xd4,0x00,0x00]
@@ -2339,7 +2339,7 @@ v_cmpx_lt_u16_e64 ttmp15, src_scc
 // GFX11: encoding: [0x7e,0x00,0xb9,0xd4,0x7b,0xfa,0x01,0x00]
 
 v_cmpx_lt_u16_e64 m0, 0.5
-// GFX11: encoding: [0x7e,0x00,0xb9,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x7e,0x00,0xb9,0xd4,0x7d,0xe0,0x01,0x00]
 
 v_cmpx_lt_u16_e64 exec_lo, -1
 // GFX11: encoding: [0x7e,0x00,0xb9,0xd4,0x7e,0x82,0x01,0x00]
@@ -2354,7 +2354,7 @@ v_cmpx_lt_u16_e64 -1, exec_hi
 // GFX11: encoding: [0x7e,0x00,0xb9,0xd4,0xc1,0xfe,0x00,0x00]
 
 v_cmpx_lt_u16_e64 0.5, m0
-// GFX11: encoding: [0x7e,0x00,0xb9,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x7e,0x00,0xb9,0xd4,0xf0,0xfa,0x00,0x00]
 
 v_cmpx_lt_u16_e64 src_scc, vcc_lo
 // GFX11: encoding: [0x7e,0x00,0xb9,0xd4,0xfd,0xd4,0x00,0x00]
@@ -2465,7 +2465,7 @@ v_cmpx_ne_i16_e64 ttmp15, src_scc
 // GFX11: encoding: [0x7e,0x00,0xb5,0xd4,0x7b,0xfa,0x01,0x00]
 
 v_cmpx_ne_i16_e64 m0, 0.5
-// GFX11: encoding: [0x7e,0x00,0xb5,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x7e,0x00,0xb5,0xd4,0x7d,0xe0,0x01,0x00]
 
 v_cmpx_ne_i16_e64 exec_lo, -1
 // GFX11: encoding: [0x7e,0x00,0xb5,0xd4,0x7e,0x82,0x01,0x00]
@@ -2480,7 +2480,7 @@ v_cmpx_ne_i16_e64 -1, exec_hi
 // GFX11: encoding: [0x7e,0x00,0xb5,0xd4,0xc1,0xfe,0x00,0x00]
 
 v_cmpx_ne_i16_e64 0.5, m0
-// GFX11: encoding: [0x7e,0x00,0xb5,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x7e,0x00,0xb5,0xd4,0xf0,0xfa,0x00,0x00]
 
 v_cmpx_ne_i16_e64 src_scc, vcc_lo
 // GFX11: encoding: [0x7e,0x00,0xb5,0xd4,0xfd,0xd4,0x00,0x00]
@@ -2591,7 +2591,7 @@ v_cmpx_ne_u16_e64 ttmp15, src_scc
 // GFX11: encoding: [0x7e,0x00,0xbd,0xd4,0x7b,0xfa,0x01,0x00]
 
 v_cmpx_ne_u16_e64 m0, 0.5
-// GFX11: encoding: [0x7e,0x00,0xbd,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x7e,0x00,0xbd,0xd4,0x7d,0xe0,0x01,0x00]
 
 v_cmpx_ne_u16_e64 exec_lo, -1
 // GFX11: encoding: [0x7e,0x00,0xbd,0xd4,0x7e,0x82,0x01,0x00]
@@ -2606,7 +2606,7 @@ v_cmpx_ne_u16_e64 -1, exec_hi
 // GFX11: encoding: [0x7e,0x00,0xbd,0xd4,0xc1,0xfe,0x00,0x00]
 
 v_cmpx_ne_u16_e64 0.5, m0
-// GFX11: encoding: [0x7e,0x00,0xbd,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x7e,0x00,0xbd,0xd4,0xf0,0xfa,0x00,0x00]
 
 v_cmpx_ne_u16_e64 src_scc, vcc_lo
 // GFX11: encoding: [0x7e,0x00,0xbd,0xd4,0xfd,0xd4,0x00,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vopc.s b/llvm/test/MC/AMDGPU/gfx11_asm_vopc.s
index c9272f24e25a..f2025e4bd6d6 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vopc.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vopc.s
@@ -724,7 +724,7 @@ v_cmp_eq_i16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_i16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x64,0x7c,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x64,0x7c]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_i16 vcc_lo, src_scc, v2
@@ -784,7 +784,7 @@ v_cmp_eq_i16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_i16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x64,0x7c,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x64,0x7c]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_i16 vcc, src_scc, v2
@@ -1060,7 +1060,7 @@ v_cmp_eq_u16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_u16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x74,0x7c,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x74,0x7c]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_u16 vcc_lo, src_scc, v2
@@ -1120,7 +1120,7 @@ v_cmp_eq_u16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_u16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x74,0x7c,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x74,0x7c]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_u16 vcc, src_scc, v2
@@ -2500,7 +2500,7 @@ v_cmp_ge_i16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_i16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x6c,0x7c,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x6c,0x7c]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_i16 vcc_lo, src_scc, v2
@@ -2560,7 +2560,7 @@ v_cmp_ge_i16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_i16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x6c,0x7c,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x6c,0x7c]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_i16 vcc, src_scc, v2
@@ -2836,7 +2836,7 @@ v_cmp_ge_u16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_u16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x7c,0x7c,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x7c,0x7c]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_u16 vcc_lo, src_scc, v2
@@ -2896,7 +2896,7 @@ v_cmp_ge_u16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_u16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x7c,0x7c,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x7c,0x7c]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_u16 vcc, src_scc, v2
@@ -3508,7 +3508,7 @@ v_cmp_gt_i16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_i16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x68,0x7c,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x68,0x7c]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_i16 vcc_lo, src_scc, v2
@@ -3568,7 +3568,7 @@ v_cmp_gt_i16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_i16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x68,0x7c,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x68,0x7c]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_i16 vcc, src_scc, v2
@@ -3844,7 +3844,7 @@ v_cmp_gt_u16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_u16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x78,0x7c,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x78,0x7c]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_u16 vcc_lo, src_scc, v2
@@ -3904,7 +3904,7 @@ v_cmp_gt_u16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_u16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x78,0x7c,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x78,0x7c]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_u16 vcc, src_scc, v2
@@ -4516,7 +4516,7 @@ v_cmp_le_i16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_le_i16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x66,0x7c,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x66,0x7c]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_le_i16 vcc_lo, src_scc, v2
@@ -4576,7 +4576,7 @@ v_cmp_le_i16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_le_i16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x66,0x7c,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x66,0x7c]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_le_i16 vcc, src_scc, v2
@@ -4852,7 +4852,7 @@ v_cmp_le_u16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_le_u16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x76,0x7c,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x76,0x7c]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_le_u16 vcc_lo, src_scc, v2
@@ -4912,7 +4912,7 @@ v_cmp_le_u16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_le_u16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x76,0x7c,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x76,0x7c]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_le_u16 vcc, src_scc, v2
@@ -5860,7 +5860,7 @@ v_cmp_lt_i16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_i16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x62,0x7c,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x62,0x7c]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_i16 vcc_lo, src_scc, v2
@@ -5920,7 +5920,7 @@ v_cmp_lt_i16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_i16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x62,0x7c,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x62,0x7c]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_i16 vcc, src_scc, v2
@@ -6196,7 +6196,7 @@ v_cmp_lt_u16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_u16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x72,0x7c,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x72,0x7c]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_u16 vcc_lo, src_scc, v2
@@ -6256,7 +6256,7 @@ v_cmp_lt_u16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_u16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x72,0x7c,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x72,0x7c]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_u16 vcc, src_scc, v2
@@ -6532,7 +6532,7 @@ v_cmp_ne_i16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_i16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x6a,0x7c,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x6a,0x7c]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_i16 vcc_lo, src_scc, v2
@@ -6592,7 +6592,7 @@ v_cmp_ne_i16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_i16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x6a,0x7c,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x6a,0x7c]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_i16 vcc, src_scc, v2
@@ -6868,7 +6868,7 @@ v_cmp_ne_u16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_u16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x7a,0x7c,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x7a,0x7c]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_u16 vcc_lo, src_scc, v2
@@ -6928,7 +6928,7 @@ v_cmp_ne_u16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_u16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x7a,0x7c,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x7a,0x7c]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_u16 vcc, src_scc, v2
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vopcx.s b/llvm/test/MC/AMDGPU/gfx11_asm_vopcx.s
index 2512f1a4b94b..d52034fedc35 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vopcx.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vopcx.s
@@ -290,7 +290,7 @@ v_cmpx_eq_i16 -1, v2
 // GFX11: encoding: [0xc1,0x04,0x64,0x7d]
 
 v_cmpx_eq_i16 0.5, v2
-// GFX11: encoding: [0xff,0x04,0x64,0x7d,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0xf0,0x04,0x64,0x7d]
 
 v_cmpx_eq_i16 src_scc, v2
 // GFX11: encoding: [0xfd,0x04,0x64,0x7d]
@@ -416,7 +416,7 @@ v_cmpx_eq_u16 -1, v2
 // GFX11: encoding: [0xc1,0x04,0x74,0x7d]
 
 v_cmpx_eq_u16 0.5, v2
-// GFX11: encoding: [0xff,0x04,0x74,0x7d,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0xf0,0x04,0x74,0x7d]
 
 v_cmpx_eq_u16 src_scc, v2
 // GFX11: encoding: [0xfd,0x04,0x74,0x7d]
@@ -956,7 +956,7 @@ v_cmpx_ge_i16 -1, v2
 // GFX11: encoding: [0xc1,0x04,0x6c,0x7d]
 
 v_cmpx_ge_i16 0.5, v2
-// GFX11: encoding: [0xff,0x04,0x6c,0x7d,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0xf0,0x04,0x6c,0x7d]
 
 v_cmpx_ge_i16 src_scc, v2
 // GFX11: encoding: [0xfd,0x04,0x6c,0x7d]
@@ -1082,7 +1082,7 @@ v_cmpx_ge_u16 -1, v2
 // GFX11: encoding: [0xc1,0x04,0x7c,0x7d]
 
 v_cmpx_ge_u16 0.5, v2
-// GFX11: encoding: [0xff,0x04,0x7c,0x7d,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0xf0,0x04,0x7c,0x7d]
 
 v_cmpx_ge_u16 src_scc, v2
 // GFX11: encoding: [0xfd,0x04,0x7c,0x7d]
@@ -1334,7 +1334,7 @@ v_cmpx_gt_i16 -1, v2
 // GFX11: encoding: [0xc1,0x04,0x68,0x7d]
 
 v_cmpx_gt_i16 0.5, v2
-// GFX11: encoding: [0xff,0x04,0x68,0x7d,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0xf0,0x04,0x68,0x7d]
 
 v_cmpx_gt_i16 src_scc, v2
 // GFX11: encoding: [0xfd,0x04,0x68,0x7d]
@@ -1460,7 +1460,7 @@ v_cmpx_gt_u16 -1, v2
 // GFX11: encoding: [0xc1,0x04,0x78,0x7d]
 
 v_cmpx_gt_u16 0.5, v2
-// GFX11: encoding: [0xff,0x04,0x78,0x7d,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0xf0,0x04,0x78,0x7d]
 
 v_cmpx_gt_u16 src_scc, v2
 // GFX11: encoding: [0xfd,0x04,0x78,0x7d]
@@ -1712,7 +1712,7 @@ v_cmpx_le_i16 -1, v2
 // GFX11: encoding: [0xc1,0x04,0x66,0x7d]
 
 v_cmpx_le_i16 0.5, v2
-// GFX11: encoding: [0xff,0x04,0x66,0x7d,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0xf0,0x04,0x66,0x7d]
 
 v_cmpx_le_i16 src_scc, v2
 // GFX11: encoding: [0xfd,0x04,0x66,0x7d]
@@ -1838,7 +1838,7 @@ v_cmpx_le_u16 -1, v2
 // GFX11: encoding: [0xc1,0x04,0x76,0x7d]
 
 v_cmpx_le_u16 0.5, v2
-// GFX11: encoding: [0xff,0x04,0x76,0x7d,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0xf0,0x04,0x76,0x7d]
 
 v_cmpx_le_u16 src_scc, v2
 // GFX11: encoding: [0xfd,0x04,0x76,0x7d]
@@ -2216,7 +2216,7 @@ v_cmpx_lt_i16 -1, v2
 // GFX11: encoding: [0xc1,0x04,0x62,0x7d]
 
 v_cmpx_lt_i16 0.5, v2
-// GFX11: encoding: [0xff,0x04,0x62,0x7d,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0xf0,0x04,0x62,0x7d]
 
 v_cmpx_lt_i16 src_scc, v2
 // GFX11: encoding: [0xfd,0x04,0x62,0x7d]
@@ -2342,7 +2342,7 @@ v_cmpx_lt_u16 -1, v2
 // GFX11: encoding: [0xc1,0x04,0x72,0x7d]
 
 v_cmpx_lt_u16 0.5, v2
-// GFX11: encoding: [0xff,0x04,0x72,0x7d,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0xf0,0x04,0x72,0x7d]
 
 v_cmpx_lt_u16 src_scc, v2
 // GFX11: encoding: [0xfd,0x04,0x72,0x7d]
@@ -2468,7 +2468,7 @@ v_cmpx_ne_i16 -1, v2
 // GFX11: encoding: [0xc1,0x04,0x6a,0x7d]
 
 v_cmpx_ne_i16 0.5, v2
-// GFX11: encoding: [0xff,0x04,0x6a,0x7d,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0xf0,0x04,0x6a,0x7d]
 
 v_cmpx_ne_i16 src_scc, v2
 // GFX11: encoding: [0xfd,0x04,0x6a,0x7d]
@@ -2594,7 +2594,7 @@ v_cmpx_ne_u16 -1, v2
 // GFX11: encoding: [0xc1,0x04,0x7a,0x7d]
 
 v_cmpx_ne_u16 0.5, v2
-// GFX11: encoding: [0xff,0x04,0x7a,0x7d,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0xf0,0x04,0x7a,0x7d]
 
 v_cmpx_ne_u16 src_scc, v2
 // GFX11: encoding: [0xfd,0x04,0x7a,0x7d]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop1.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop1.s
index a21ef11bfb87..5279588f0506 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop1.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop1.s
@@ -524,7 +524,7 @@ v_cvt_f16_i16 v5, -1
 // GFX12: encoding: [0xc1,0xa2,0x0a,0x7e]
 
 v_cvt_f16_i16 v5, 0.5
-// GFX12: encoding: [0xff,0xa2,0x0a,0x7e,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0xf0,0xa2,0x0a,0x7e]
 
 v_cvt_f16_i16 v5, src_scc
 // GFX12: encoding: [0xfd,0xa2,0x0a,0x7e]
@@ -569,7 +569,7 @@ v_cvt_f16_u16 v5, -1
 // GFX12: encoding: [0xc1,0xa0,0x0a,0x7e]
 
 v_cvt_f16_u16 v5, 0.5
-// GFX12: encoding: [0xff,0xa0,0x0a,0x7e,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0xf0,0xa0,0x0a,0x7e]
 
 v_cvt_f16_u16 v5, src_scc
 // GFX12: encoding: [0xfd,0xa0,0x0a,0x7e]
@@ -1316,7 +1316,7 @@ v_cvt_i32_i16 v5, -1
 // GFX12: encoding: [0xc1,0xd4,0x0a,0x7e]
 
 v_cvt_i32_i16 v5, 0.5
-// GFX12: encoding: [0xff,0xd4,0x0a,0x7e,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0xf0,0xd4,0x0a,0x7e]
 
 v_cvt_i32_i16 v5, src_scc
 // GFX12: encoding: [0xfd,0xd4,0x0a,0x7e]
@@ -1712,7 +1712,7 @@ v_cvt_u32_u16 v5, -1
 // GFX12: encoding: [0xc1,0xd6,0x0a,0x7e]
 
 v_cvt_u32_u16 v5, 0.5
-// GFX12: encoding: [0xff,0xd6,0x0a,0x7e,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0xf0,0xd6,0x0a,0x7e]
 
 v_cvt_u32_u16 v5, src_scc
 // GFX12: encoding: [0xfd,0xd6,0x0a,0x7e]
@@ -2687,7 +2687,7 @@ v_not_b16 v5, -1
 // GFX12: encoding: [0xc1,0xd2,0x0a,0x7e]
 
 v_not_b16 v5, 0.5
-// GFX12: encoding: [0xff,0xd2,0x0a,0x7e,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0xf0,0xd2,0x0a,0x7e]
 
 v_not_b16 v5, src_scc
 // GFX12: encoding: [0xfd,0xd2,0x0a,0x7e]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3.s
index 9d0488fd2f1c..be9edc3e019e 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3.s
@@ -230,7 +230,7 @@ v_add_nc_i16 v5, ttmp15, src_scc
 // GFX12: encoding: [0x05,0x00,0x0d,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_add_nc_i16 v5, m0, 0.5
-// GFX12: encoding: [0x05,0x00,0x0d,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x0d,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_add_nc_i16 v5, exec_lo, -1
 // GFX12: encoding: [0x05,0x00,0x0d,0xd7,0x7e,0x82,0x01,0x00]
@@ -245,7 +245,7 @@ v_add_nc_i16 v5, -1, exec_hi op_sel:[0,0,0]
 // GFX12: encoding: [0x05,0x00,0x0d,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_add_nc_i16 v5, 0.5, m0 op_sel:[1,0,0]
-// GFX12: encoding: [0x05,0x08,0x0d,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x08,0x0d,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_add_nc_i16 v5, src_scc, vcc_lo op_sel:[0,1,0]
 // GFX12: encoding: [0x05,0x10,0x0d,0xd7,0xfd,0xd4,0x00,0x00]
@@ -320,7 +320,7 @@ v_add_nc_u16 v5, ttmp15, src_scc
 // GFX12: encoding: [0x05,0x00,0x03,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_add_nc_u16 v5, m0, 0.5
-// GFX12: encoding: [0x05,0x00,0x03,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x03,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_add_nc_u16 v5, exec_lo, -1
 // GFX12: encoding: [0x05,0x00,0x03,0xd7,0x7e,0x82,0x01,0x00]
@@ -335,7 +335,7 @@ v_add_nc_u16 v5, -1, exec_hi op_sel:[0,0,0]
 // GFX12: encoding: [0x05,0x00,0x03,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_add_nc_u16 v5, 0.5, m0 op_sel:[1,0,0]
-// GFX12: encoding: [0x05,0x08,0x03,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x08,0x03,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_add_nc_u16 v5, src_scc, vcc_lo op_sel:[0,1,0]
 // GFX12: encoding: [0x05,0x10,0x03,0xd7,0xfd,0xd4,0x00,0x00]
@@ -455,7 +455,7 @@ v_and_b16 v5, ttmp15, src_scc
 // GFX12: encoding: [0x05,0x00,0x62,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_and_b16 v5, m0, 0.5
-// GFX12: encoding: [0x05,0x00,0x62,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x62,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_and_b16 v5, exec_lo, -1
 // GFX12: encoding: [0x05,0x00,0x62,0xd7,0x7e,0x82,0x01,0x00]
@@ -470,7 +470,7 @@ v_and_b16 v5, -1, exec_hi
 // GFX12: encoding: [0x05,0x00,0x62,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_and_b16 v5, 0.5, m0
-// GFX12: encoding: [0x05,0x00,0x62,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x62,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_and_b16 v5, src_scc, vcc_lo
 // GFX12: encoding: [0x05,0x00,0x62,0xd7,0xfd,0xd4,0x00,0x00]
@@ -545,7 +545,7 @@ v_ashrrev_i16 v5, ttmp15, src_scc
 // GFX12: encoding: [0x05,0x00,0x3a,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_ashrrev_i16 v5, m0, 0.5
-// GFX12: encoding: [0x05,0x00,0x3a,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x3a,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_ashrrev_i16 v5, exec_lo, -1
 // GFX12: encoding: [0x05,0x00,0x3a,0xd7,0x7e,0x82,0x01,0x00]
@@ -560,7 +560,7 @@ v_ashrrev_i16 v5, -1, exec_hi
 // GFX12: encoding: [0x05,0x00,0x3a,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_ashrrev_i16 v5, 0.5, m0
-// GFX12: encoding: [0x05,0x00,0x3a,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x3a,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_ashrrev_i16 v5, src_scc, vcc_lo
 // GFX12: encoding: [0x05,0x00,0x3a,0xd7,0xfd,0xd4,0x00,0x00]
@@ -825,7 +825,7 @@ v_cndmask_b16 v5, v1, src_scc, s3
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cndmask_b16 v5, v255, 0.5, s3
-// W32: encoding: [0x05,0x00,0x5d,0xd6,0xff,0xff,0x0d,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x05,0x00,0x5d,0xd6,0xff,0xe1,0x0d,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cndmask_b16 v5, s105, s105, s3
@@ -861,7 +861,7 @@ v_cndmask_b16 v5, -1, -|vcc_lo|, vcc_lo
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cndmask_b16 v5, 0.5, -1, vcc_hi
-// W32: encoding: [0x05,0x00,0x5d,0xd6,0xff,0x82,0xad,0x01,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x05,0x00,0x5d,0xd6,0xf0,0x82,0xad,0x01]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cndmask_b16 v5, -|src_scc|, null, ttmp15
@@ -873,7 +873,7 @@ v_cndmask_b16 v5, v1, src_scc, s[6:7]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cndmask_b16 v5, v255, 0.5, s[6:7]
-// W64: encoding: [0x05,0x00,0x5d,0xd6,0xff,0xff,0x19,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x05,0x00,0x5d,0xd6,0xff,0xe1,0x19,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cndmask_b16 v5, s105, s105, s[6:7]
@@ -909,7 +909,7 @@ v_cndmask_b16 v5, -1, -|vcc_lo|, s[104:105]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cndmask_b16 v5, 0.5, -1, vcc
-// W64: encoding: [0x05,0x00,0x5d,0xd6,0xff,0x82,0xa9,0x01,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x05,0x00,0x5d,0xd6,0xf0,0x82,0xa9,0x01]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cndmask_b16 v5, -|src_scc|, null, ttmp[14:15]
@@ -2621,7 +2621,7 @@ v_lshlrev_b16 v5, ttmp15, src_scc
 // GFX12: encoding: [0x05,0x00,0x38,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_lshlrev_b16 v5, m0, 0.5
-// GFX12: encoding: [0x05,0x00,0x38,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x38,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_lshlrev_b16 v5, exec_lo, -1
 // GFX12: encoding: [0x05,0x00,0x38,0xd7,0x7e,0x82,0x01,0x00]
@@ -2636,7 +2636,7 @@ v_lshlrev_b16 v5, -1, exec_hi
 // GFX12: encoding: [0x05,0x00,0x38,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_lshlrev_b16 v5, 0.5, m0
-// GFX12: encoding: [0x05,0x00,0x38,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x38,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_lshlrev_b16 v5, src_scc, vcc_lo
 // GFX12: encoding: [0x05,0x00,0x38,0xd7,0xfd,0xd4,0x00,0x00]
@@ -2666,7 +2666,7 @@ v_lshrrev_b16 v5, ttmp15, src_scc
 // GFX12: encoding: [0x05,0x00,0x39,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_lshrrev_b16 v5, m0, 0.5
-// GFX12: encoding: [0x05,0x00,0x39,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x39,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_lshrrev_b16 v5, exec_lo, -1
 // GFX12: encoding: [0x05,0x00,0x39,0xd7,0x7e,0x82,0x01,0x00]
@@ -2681,7 +2681,7 @@ v_lshrrev_b16 v5, -1, exec_hi
 // GFX12: encoding: [0x05,0x00,0x39,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_lshrrev_b16 v5, 0.5, m0
-// GFX12: encoding: [0x05,0x00,0x39,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x39,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_lshrrev_b16 v5, src_scc, vcc_lo
 // GFX12: encoding: [0x05,0x00,0x39,0xd7,0xfd,0xd4,0x00,0x00]
@@ -2738,7 +2738,7 @@ v_mad_i16 v5, ttmp15, src_scc, ttmp15
 // GFX12: encoding: [0x05,0x00,0x53,0xd6,0x7b,0xfa,0xed,0x01]
 
 v_mad_i16 v5, m0, 0.5, m0
-// GFX12: encoding: [0x05,0x00,0x53,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x53,0xd6,0x7d,0xe0,0xf5,0x01]
 
 v_mad_i16 v5, exec_lo, -1, vcc_hi
 // GFX12: encoding: [0x05,0x00,0x53,0xd6,0x7e,0x82,0xad,0x01]
@@ -2753,7 +2753,7 @@ v_mad_i16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0]
 // GFX12: encoding: [0x05,0x08,0x53,0xd6,0xc1,0xfe,0xf4,0x03]
 
 v_mad_i16 v5, 0.5, m0, 0.5 op_sel:[0,1,0,0]
-// GFX12: encoding: [0x05,0x10,0x53,0xd6,0xff,0xfa,0xfc,0x03,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x10,0x53,0xd6,0xf0,0xfa,0xc0,0x03]
 
 v_mad_i16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0]
 // GFX12: encoding: [0x05,0x20,0x53,0xd6,0xfd,0xd4,0x04,0x03]
@@ -2783,7 +2783,7 @@ v_mad_i32_i16 v5, ttmp15, src_scc, ttmp15
 // GFX12: encoding: [0x05,0x00,0x5a,0xd6,0x7b,0xfa,0xed,0x01]
 
 v_mad_i32_i16 v5, m0, 0.5, m0
-// GFX12: encoding: [0x05,0x00,0x5a,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x5a,0xd6,0x7d,0xe0,0xf5,0x01]
 
 v_mad_i32_i16 v5, exec_lo, -1, exec_hi
 // GFX12: encoding: [0x05,0x00,0x5a,0xd6,0x7e,0x82,0xfd,0x01]
@@ -2798,7 +2798,7 @@ v_mad_i32_i16 v5, -1, exec_hi, 0xaf123456
 // GFX12: encoding: [0x05,0x00,0x5a,0xd6,0xc1,0xfe,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
 v_mad_i32_i16 v5, 0.5, m0, -1 op_sel:[0,0,0,0]
-// GFX12: encoding: [0x05,0x00,0x5a,0xd6,0xff,0xfa,0x04,0x03,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x5a,0xd6,0xf0,0xfa,0x04,0x03]
 
 v_mad_i32_i16 v5, src_scc, vcc_lo, src_scc op_sel:[1,0,0,0]
 // GFX12: encoding: [0x05,0x08,0x5a,0xd6,0xfd,0xd4,0xf4,0x03]
@@ -2948,7 +2948,7 @@ v_mad_u16 v5, ttmp15, src_scc, ttmp15
 // GFX12: encoding: [0x05,0x00,0x41,0xd6,0x7b,0xfa,0xed,0x01]
 
 v_mad_u16 v5, m0, 0.5, m0
-// GFX12: encoding: [0x05,0x00,0x41,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x41,0xd6,0x7d,0xe0,0xf5,0x01]
 
 v_mad_u16 v5, exec_lo, -1, vcc_hi
 // GFX12: encoding: [0x05,0x00,0x41,0xd6,0x7e,0x82,0xad,0x01]
@@ -2963,7 +2963,7 @@ v_mad_u16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0]
 // GFX12: encoding: [0x05,0x08,0x41,0xd6,0xc1,0xfe,0xf4,0x03]
 
 v_mad_u16 v5, 0.5, m0, 0.5 op_sel:[0,1,0,0]
-// GFX12: encoding: [0x05,0x10,0x41,0xd6,0xff,0xfa,0xfc,0x03,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x10,0x41,0xd6,0xf0,0xfa,0xc0,0x03]
 
 v_mad_u16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0]
 // GFX12: encoding: [0x05,0x20,0x41,0xd6,0xfd,0xd4,0x04,0x03]
@@ -2993,7 +2993,7 @@ v_mad_u32_u16 v5, ttmp15, src_scc, ttmp15
 // GFX12: encoding: [0x05,0x00,0x59,0xd6,0x7b,0xfa,0xed,0x01]
 
 v_mad_u32_u16 v5, m0, 0.5, m0
-// GFX12: encoding: [0x05,0x00,0x59,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x59,0xd6,0x7d,0xe0,0xf5,0x01]
 
 v_mad_u32_u16 v5, exec_lo, -1, exec_hi
 // GFX12: encoding: [0x05,0x00,0x59,0xd6,0x7e,0x82,0xfd,0x01]
@@ -3008,7 +3008,7 @@ v_mad_u32_u16 v5, -1, exec_hi, 0xaf123456
 // GFX12: encoding: [0x05,0x00,0x59,0xd6,0xc1,0xfe,0xfc,0x03,0x56,0x34,0x12,0xaf]
 
 v_mad_u32_u16 v5, 0.5, m0, -1 op_sel:[0,0,0,0]
-// GFX12: encoding: [0x05,0x00,0x59,0xd6,0xff,0xfa,0x04,0x03,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x59,0xd6,0xf0,0xfa,0x04,0x03]
 
 v_mad_u32_u16 v5, src_scc, vcc_lo, src_scc op_sel:[1,0,0,0]
 // GFX12: encoding: [0x05,0x08,0x59,0xd6,0xfd,0xd4,0xf4,0x03]
@@ -3248,7 +3248,7 @@ v_max3_i16 v5, ttmp15, src_scc, ttmp15
 // GFX12: encoding: [0x05,0x00,0x4d,0xd6,0x7b,0xfa,0xed,0x01]
 
 v_max3_i16 v5, m0, 0.5, m0
-// GFX12: encoding: [0x05,0x00,0x4d,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x4d,0xd6,0x7d,0xe0,0xf5,0x01]
 
 v_max3_i16 v5, exec_lo, -1, vcc_hi
 // GFX12: encoding: [0x05,0x00,0x4d,0xd6,0x7e,0x82,0xad,0x01]
@@ -3263,7 +3263,7 @@ v_max3_i16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0]
 // GFX12: encoding: [0x05,0x08,0x4d,0xd6,0xc1,0xfe,0xf4,0x03]
 
 v_max3_i16 v5, 0.5, m0, 0.5 op_sel:[0,1,0,0]
-// GFX12: encoding: [0x05,0x10,0x4d,0xd6,0xff,0xfa,0xfc,0x03,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x10,0x4d,0xd6,0xf0,0xfa,0xc0,0x03]
 
 v_max3_i16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0]
 // GFX12: encoding: [0x05,0x20,0x4d,0xd6,0xfd,0xd4,0x04,0x03]
@@ -3338,7 +3338,7 @@ v_max3_u16 v5, ttmp15, src_scc, ttmp15
 // GFX12: encoding: [0x05,0x00,0x4e,0xd6,0x7b,0xfa,0xed,0x01]
 
 v_max3_u16 v5, m0, 0.5, m0
-// GFX12: encoding: [0x05,0x00,0x4e,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x4e,0xd6,0x7d,0xe0,0xf5,0x01]
 
 v_max3_u16 v5, exec_lo, -1, vcc_hi
 // GFX12: encoding: [0x05,0x00,0x4e,0xd6,0x7e,0x82,0xad,0x01]
@@ -3353,7 +3353,7 @@ v_max3_u16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0]
 // GFX12: encoding: [0x05,0x08,0x4e,0xd6,0xc1,0xfe,0xf4,0x03]
 
 v_max3_u16 v5, 0.5, m0, 0.5 op_sel:[0,1,0,0]
-// GFX12: encoding: [0x05,0x10,0x4e,0xd6,0xff,0xfa,0xfc,0x03,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x10,0x4e,0xd6,0xf0,0xfa,0xc0,0x03]
 
 v_max3_u16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0]
 // GFX12: encoding: [0x05,0x20,0x4e,0xd6,0xfd,0xd4,0x04,0x03]
@@ -3428,7 +3428,7 @@ v_max_i16 v5, ttmp15, src_scc
 // GFX12: encoding: [0x05,0x00,0x0a,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_max_i16 v5, m0, 0.5
-// GFX12: encoding: [0x05,0x00,0x0a,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x0a,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_max_i16 v5, exec_lo, -1
 // GFX12: encoding: [0x05,0x00,0x0a,0xd7,0x7e,0x82,0x01,0x00]
@@ -3443,7 +3443,7 @@ v_max_i16 v5, -1, exec_hi
 // GFX12: encoding: [0x05,0x00,0x0a,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_max_i16 v5, 0.5, m0
-// GFX12: encoding: [0x05,0x00,0x0a,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x0a,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_max_i16 v5, src_scc, vcc_lo
 // GFX12: encoding: [0x05,0x00,0x0a,0xd7,0xfd,0xd4,0x00,0x00]
@@ -3473,7 +3473,7 @@ v_max_u16 v5, ttmp15, src_scc
 // GFX12: encoding: [0x05,0x00,0x09,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_max_u16 v5, m0, 0.5
-// GFX12: encoding: [0x05,0x00,0x09,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x09,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_max_u16 v5, exec_lo, -1
 // GFX12: encoding: [0x05,0x00,0x09,0xd7,0x7e,0x82,0x01,0x00]
@@ -3488,7 +3488,7 @@ v_max_u16 v5, -1, exec_hi
 // GFX12: encoding: [0x05,0x00,0x09,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_max_u16 v5, 0.5, m0
-// GFX12: encoding: [0x05,0x00,0x09,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x09,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_max_u16 v5, src_scc, vcc_lo
 // GFX12: encoding: [0x05,0x00,0x09,0xd7,0xfd,0xd4,0x00,0x00]
@@ -3878,7 +3878,7 @@ v_med3_i16 v5, ttmp15, src_scc, ttmp15
 // GFX12: encoding: [0x05,0x00,0x50,0xd6,0x7b,0xfa,0xed,0x01]
 
 v_med3_i16 v5, m0, 0.5, m0
-// GFX12: encoding: [0x05,0x00,0x50,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x50,0xd6,0x7d,0xe0,0xf5,0x01]
 
 v_med3_i16 v5, exec_lo, -1, vcc_hi
 // GFX12: encoding: [0x05,0x00,0x50,0xd6,0x7e,0x82,0xad,0x01]
@@ -3893,7 +3893,7 @@ v_med3_i16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0]
 // GFX12: encoding: [0x05,0x08,0x50,0xd6,0xc1,0xfe,0xf4,0x03]
 
 v_med3_i16 v5, 0.5, m0, 0.5 op_sel:[0,1,0,0]
-// GFX12: encoding: [0x05,0x10,0x50,0xd6,0xff,0xfa,0xfc,0x03,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x10,0x50,0xd6,0xf0,0xfa,0xc0,0x03]
 
 v_med3_i16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0]
 // GFX12: encoding: [0x05,0x20,0x50,0xd6,0xfd,0xd4,0x04,0x03]
@@ -3968,7 +3968,7 @@ v_med3_u16 v5, ttmp15, src_scc, ttmp15
 // GFX12: encoding: [0x05,0x00,0x51,0xd6,0x7b,0xfa,0xed,0x01]
 
 v_med3_u16 v5, m0, 0.5, m0
-// GFX12: encoding: [0x05,0x00,0x51,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x51,0xd6,0x7d,0xe0,0xf5,0x01]
 
 v_med3_u16 v5, exec_lo, -1, vcc_hi
 // GFX12: encoding: [0x05,0x00,0x51,0xd6,0x7e,0x82,0xad,0x01]
@@ -3983,7 +3983,7 @@ v_med3_u16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0]
 // GFX12: encoding: [0x05,0x08,0x51,0xd6,0xc1,0xfe,0xf4,0x03]
 
 v_med3_u16 v5, 0.5, m0, 0.5 op_sel:[0,1,0,0]
-// GFX12: encoding: [0x05,0x10,0x51,0xd6,0xff,0xfa,0xfc,0x03,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x10,0x51,0xd6,0xf0,0xfa,0xc0,0x03]
 
 v_med3_u16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0]
 // GFX12: encoding: [0x05,0x20,0x51,0xd6,0xfd,0xd4,0x04,0x03]
@@ -4148,7 +4148,7 @@ v_min3_i16 v5, ttmp15, src_scc, ttmp15
 // GFX12: encoding: [0x05,0x00,0x4a,0xd6,0x7b,0xfa,0xed,0x01]
 
 v_min3_i16 v5, m0, 0.5, m0
-// GFX12: encoding: [0x05,0x00,0x4a,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x4a,0xd6,0x7d,0xe0,0xf5,0x01]
 
 v_min3_i16 v5, exec_lo, -1, vcc_hi
 // GFX12: encoding: [0x05,0x00,0x4a,0xd6,0x7e,0x82,0xad,0x01]
@@ -4163,7 +4163,7 @@ v_min3_i16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0]
 // GFX12: encoding: [0x05,0x08,0x4a,0xd6,0xc1,0xfe,0xf4,0x03]
 
 v_min3_i16 v5, 0.5, m0, 0.5 op_sel:[0,1,0,0]
-// GFX12: encoding: [0x05,0x10,0x4a,0xd6,0xff,0xfa,0xfc,0x03,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x10,0x4a,0xd6,0xf0,0xfa,0xc0,0x03]
 
 v_min3_i16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0]
 // GFX12: encoding: [0x05,0x20,0x4a,0xd6,0xfd,0xd4,0x04,0x03]
@@ -4238,7 +4238,7 @@ v_min3_u16 v5, ttmp15, src_scc, ttmp15
 // GFX12: encoding: [0x05,0x00,0x4b,0xd6,0x7b,0xfa,0xed,0x01]
 
 v_min3_u16 v5, m0, 0.5, m0
-// GFX12: encoding: [0x05,0x00,0x4b,0xd6,0x7d,0xfe,0xf5,0x01,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x4b,0xd6,0x7d,0xe0,0xf5,0x01]
 
 v_min3_u16 v5, exec_lo, -1, vcc_hi
 // GFX12: encoding: [0x05,0x00,0x4b,0xd6,0x7e,0x82,0xad,0x01]
@@ -4253,7 +4253,7 @@ v_min3_u16 v5, -1, exec_hi, src_scc op_sel:[1,0,0,0]
 // GFX12: encoding: [0x05,0x08,0x4b,0xd6,0xc1,0xfe,0xf4,0x03]
 
 v_min3_u16 v5, 0.5, m0, 0.5 op_sel:[0,1,0,0]
-// GFX12: encoding: [0x05,0x10,0x4b,0xd6,0xff,0xfa,0xfc,0x03,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x10,0x4b,0xd6,0xf0,0xfa,0xc0,0x03]
 
 v_min3_u16 v5, src_scc, vcc_lo, -1 op_sel:[0,0,1,0]
 // GFX12: encoding: [0x05,0x20,0x4b,0xd6,0xfd,0xd4,0x04,0x03]
@@ -4328,7 +4328,7 @@ v_min_i16 v5, ttmp15, src_scc
 // GFX12: encoding: [0x05,0x00,0x0c,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_min_i16 v5, m0, 0.5
-// GFX12: encoding: [0x05,0x00,0x0c,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x0c,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_min_i16 v5, exec_lo, -1
 // GFX12: encoding: [0x05,0x00,0x0c,0xd7,0x7e,0x82,0x01,0x00]
@@ -4343,7 +4343,7 @@ v_min_i16 v5, -1, exec_hi
 // GFX12: encoding: [0x05,0x00,0x0c,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_min_i16 v5, 0.5, m0
-// GFX12: encoding: [0x05,0x00,0x0c,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x0c,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_min_i16 v5, src_scc, vcc_lo
 // GFX12: encoding: [0x05,0x00,0x0c,0xd7,0xfd,0xd4,0x00,0x00]
@@ -4373,7 +4373,7 @@ v_min_u16 v5, ttmp15, src_scc
 // GFX12: encoding: [0x05,0x00,0x0b,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_min_u16 v5, m0, 0.5
-// GFX12: encoding: [0x05,0x00,0x0b,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x0b,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_min_u16 v5, exec_lo, -1
 // GFX12: encoding: [0x05,0x00,0x0b,0xd7,0x7e,0x82,0x01,0x00]
@@ -4388,7 +4388,7 @@ v_min_u16 v5, -1, exec_hi
 // GFX12: encoding: [0x05,0x00,0x0b,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_min_u16 v5, 0.5, m0
-// GFX12: encoding: [0x05,0x00,0x0b,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x0b,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_min_u16 v5, src_scc, vcc_lo
 // GFX12: encoding: [0x05,0x00,0x0b,0xd7,0xfd,0xd4,0x00,0x00]
@@ -4823,7 +4823,7 @@ v_mul_lo_u16 v5, ttmp15, src_scc
 // GFX12: encoding: [0x05,0x00,0x05,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_mul_lo_u16 v5, m0, 0.5
-// GFX12: encoding: [0x05,0x00,0x05,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x05,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_mul_lo_u16 v5, exec_lo, -1
 // GFX12: encoding: [0x05,0x00,0x05,0xd7,0x7e,0x82,0x01,0x00]
@@ -4838,7 +4838,7 @@ v_mul_lo_u16 v5, -1, exec_hi
 // GFX12: encoding: [0x05,0x00,0x05,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_mul_lo_u16 v5, 0.5, m0
-// GFX12: encoding: [0x05,0x00,0x05,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x05,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_mul_lo_u16 v5, src_scc, vcc_lo
 // GFX12: encoding: [0x05,0x00,0x05,0xd7,0xfd,0xd4,0x00,0x00]
@@ -5003,7 +5003,7 @@ v_or_b16 v5, ttmp15, src_scc
 // GFX12: encoding: [0x05,0x00,0x63,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_or_b16 v5, m0, 0.5
-// GFX12: encoding: [0x05,0x00,0x63,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x63,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_or_b16 v5, exec_lo, -1
 // GFX12: encoding: [0x05,0x00,0x63,0xd7,0x7e,0x82,0x01,0x00]
@@ -5018,7 +5018,7 @@ v_or_b16 v5, -1, exec_hi
 // GFX12: encoding: [0x05,0x00,0x63,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_or_b16 v5, 0.5, m0
-// GFX12: encoding: [0x05,0x00,0x63,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x63,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_or_b16 v5, src_scc, vcc_lo
 // GFX12: encoding: [0x05,0x00,0x63,0xd7,0xfd,0xd4,0x00,0x00]
@@ -5622,7 +5622,7 @@ v_sub_nc_i16 v5, ttmp15, src_scc
 // GFX12: encoding: [0x05,0x00,0x0e,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_sub_nc_i16 v5, m0, 0.5
-// GFX12: encoding: [0x05,0x00,0x0e,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x0e,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_sub_nc_i16 v5, exec_lo, -1
 // GFX12: encoding: [0x05,0x00,0x0e,0xd7,0x7e,0x82,0x01,0x00]
@@ -5637,7 +5637,7 @@ v_sub_nc_i16 v5, -1, exec_hi op_sel:[0,0,0]
 // GFX12: encoding: [0x05,0x00,0x0e,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_sub_nc_i16 v5, 0.5, m0 op_sel:[1,0,0]
-// GFX12: encoding: [0x05,0x08,0x0e,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x08,0x0e,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_sub_nc_i16 v5, src_scc, vcc_lo op_sel:[0,1,0]
 // GFX12: encoding: [0x05,0x10,0x0e,0xd7,0xfd,0xd4,0x00,0x00]
@@ -5712,7 +5712,7 @@ v_sub_nc_u16 v5, ttmp15, src_scc
 // GFX12: encoding: [0x05,0x00,0x04,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_sub_nc_u16 v5, m0, 0.5
-// GFX12: encoding: [0x05,0x00,0x04,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x04,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_sub_nc_u16 v5, exec_lo, -1
 // GFX12: encoding: [0x05,0x00,0x04,0xd7,0x7e,0x82,0x01,0x00]
@@ -5727,7 +5727,7 @@ v_sub_nc_u16 v5, -1, exec_hi op_sel:[0,0,0]
 // GFX12: encoding: [0x05,0x00,0x04,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_sub_nc_u16 v5, 0.5, m0 op_sel:[1,0,0]
-// GFX12: encoding: [0x05,0x08,0x04,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x08,0x04,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_sub_nc_u16 v5, src_scc, vcc_lo op_sel:[0,1,0]
 // GFX12: encoding: [0x05,0x10,0x04,0xd7,0xfd,0xd4,0x00,0x00]
@@ -6046,7 +6046,7 @@ v_xor_b16 v5, ttmp15, src_scc
 // GFX12: encoding: [0x05,0x00,0x64,0xd7,0x7b,0xfa,0x01,0x00]
 
 v_xor_b16 v5, m0, 0.5
-// GFX12: encoding: [0x05,0x00,0x64,0xd7,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x64,0xd7,0x7d,0xe0,0x01,0x00]
 
 v_xor_b16 v5, exec_lo, -1
 // GFX12: encoding: [0x05,0x00,0x64,0xd7,0x7e,0x82,0x01,0x00]
@@ -6061,7 +6061,7 @@ v_xor_b16 v5, -1, exec_hi
 // GFX12: encoding: [0x05,0x00,0x64,0xd7,0xc1,0xfe,0x00,0x00]
 
 v_xor_b16 v5, 0.5, m0
-// GFX12: encoding: [0x05,0x00,0x64,0xd7,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0x64,0xd7,0xf0,0xfa,0x00,0x00]
 
 v_xor_b16 v5, src_scc, vcc_lo
 // GFX12: encoding: [0x05,0x00,0x64,0xd7,0xfd,0xd4,0x00,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop1.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop1.s
index 9c6a3b8e68cd..662905809ad9 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop1.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop1.s
@@ -616,7 +616,7 @@ v_cvt_f16_i16_e64 v5, -1
 // GFX12: encoding: [0x05,0x00,0xd1,0xd5,0xc1,0x00,0x00,0x00]
 
 v_cvt_f16_i16_e64 v5, 0.5 mul:2
-// GFX12: encoding: [0x05,0x00,0xd1,0xd5,0xff,0x00,0x00,0x08,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0xd1,0xd5,0xf0,0x00,0x00,0x08]
 
 v_cvt_f16_i16_e64 v5, src_scc mul:4
 // GFX12: encoding: [0x05,0x00,0xd1,0xd5,0xfd,0x00,0x00,0x10]
@@ -661,7 +661,7 @@ v_cvt_f16_u16_e64 v5, -1
 // GFX12: encoding: [0x05,0x00,0xd0,0xd5,0xc1,0x00,0x00,0x00]
 
 v_cvt_f16_u16_e64 v5, 0.5 mul:2
-// GFX12: encoding: [0x05,0x00,0xd0,0xd5,0xff,0x00,0x00,0x08,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0xd0,0xd5,0xf0,0x00,0x00,0x08]
 
 v_cvt_f16_u16_e64 v5, src_scc mul:4
 // GFX12: encoding: [0x05,0x00,0xd0,0xd5,0xfd,0x00,0x00,0x10]
@@ -1408,7 +1408,7 @@ v_cvt_i32_i16_e64 v5, -1
 // GFX12: encoding: [0x05,0x00,0xea,0xd5,0xc1,0x00,0x00,0x00]
 
 v_cvt_i32_i16_e64 v5, 0.5
-// GFX12: encoding: [0x05,0x00,0xea,0xd5,0xff,0x00,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0xea,0xd5,0xf0,0x00,0x00,0x00]
 
 v_cvt_i32_i16_e64 v5, src_scc
 // GFX12: encoding: [0x05,0x00,0xea,0xd5,0xfd,0x00,0x00,0x00]
@@ -1804,7 +1804,7 @@ v_cvt_u32_u16_e64 v5, -1
 // GFX12: encoding: [0x05,0x00,0xeb,0xd5,0xc1,0x00,0x00,0x00]
 
 v_cvt_u32_u16_e64 v5, 0.5
-// GFX12: encoding: [0x05,0x00,0xeb,0xd5,0xff,0x00,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0xeb,0xd5,0xf0,0x00,0x00,0x00]
 
 v_cvt_u32_u16_e64 v5, src_scc
 // GFX12: encoding: [0x05,0x00,0xeb,0xd5,0xfd,0x00,0x00,0x00]
@@ -2779,7 +2779,7 @@ v_not_b16_e64 v5, -1
 // GFX12: encoding: [0x05,0x00,0xe9,0xd5,0xc1,0x00,0x00,0x00]
 
 v_not_b16_e64 v5, 0.5
-// GFX12: encoding: [0x05,0x00,0xe9,0xd5,0xff,0x00,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x05,0x00,0xe9,0xd5,0xf0,0x00,0x00,0x00]
 
 v_not_b16_e64 v5, src_scc
 // GFX12: encoding: [0x05,0x00,0xe9,0xd5,0xfd,0x00,0x00,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3c.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3c.s
index 8bc776498c06..e03bb88e0771 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3c.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3c.s
@@ -698,7 +698,7 @@ v_cmp_eq_i16_e64 s5, ttmp15, src_scc
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_i16_e64 s5, m0, 0.5
-// W32: encoding: [0x05,0x00,0x32,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x05,0x00,0x32,0xd4,0x7d,0xe0,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_i16_e64 s5, exec_lo, -1
@@ -718,7 +718,7 @@ v_cmp_eq_i16_e64 vcc_lo, -1, exec_hi
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_i16_e64 vcc_hi, 0.5, m0
-// W32: encoding: [0x6b,0x00,0x32,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x6b,0x00,0x32,0xd4,0xf0,0xfa,0x00,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_i16_e64 ttmp15, src_scc, vcc_lo
@@ -754,7 +754,7 @@ v_cmp_eq_i16_e64 s[10:11], ttmp15, src_scc
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_i16_e64 s[10:11], m0, 0.5
-// W64: encoding: [0x0a,0x00,0x32,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x32,0xd4,0x7d,0xe0,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_i16_e64 s[10:11], exec_lo, -1
@@ -774,7 +774,7 @@ v_cmp_eq_i16_e64 s[104:105], -1, exec_hi
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_i16_e64 vcc, 0.5, m0
-// W64: encoding: [0x6a,0x00,0x32,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x6a,0x00,0x32,0xd4,0xf0,0xfa,0x00,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_i16_e64 ttmp[14:15], src_scc, vcc_lo
@@ -1019,7 +1019,7 @@ v_cmp_eq_u16_e64 s5, ttmp15, src_scc
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_u16_e64 s5, m0, 0.5
-// W32: encoding: [0x05,0x00,0x3a,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x05,0x00,0x3a,0xd4,0x7d,0xe0,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_u16_e64 s5, exec_lo, -1
@@ -1039,7 +1039,7 @@ v_cmp_eq_u16_e64 vcc_lo, -1, exec_hi
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_u16_e64 vcc_hi, 0.5, m0
-// W32: encoding: [0x6b,0x00,0x3a,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x6b,0x00,0x3a,0xd4,0xf0,0xfa,0x00,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_u16_e64 ttmp15, src_scc, vcc_lo
@@ -1075,7 +1075,7 @@ v_cmp_eq_u16_e64 s[10:11], ttmp15, src_scc
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_u16_e64 s[10:11], m0, 0.5
-// W64: encoding: [0x0a,0x00,0x3a,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x3a,0xd4,0x7d,0xe0,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_u16_e64 s[10:11], exec_lo, -1
@@ -1095,7 +1095,7 @@ v_cmp_eq_u16_e64 s[104:105], -1, exec_hi
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_u16_e64 vcc, 0.5, m0
-// W64: encoding: [0x6a,0x00,0x3a,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x6a,0x00,0x3a,0xd4,0xf0,0xfa,0x00,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_eq_u16_e64 ttmp[14:15], src_scc, vcc_lo
@@ -1661,7 +1661,7 @@ v_cmp_ge_i16_e64 s5, ttmp15, src_scc
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_i16_e64 s5, m0, 0.5
-// W32: encoding: [0x05,0x00,0x36,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x05,0x00,0x36,0xd4,0x7d,0xe0,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_i16_e64 s5, exec_lo, -1
@@ -1681,7 +1681,7 @@ v_cmp_ge_i16_e64 vcc_lo, -1, exec_hi
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_i16_e64 vcc_hi, 0.5, m0
-// W32: encoding: [0x6b,0x00,0x36,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x6b,0x00,0x36,0xd4,0xf0,0xfa,0x00,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_i16_e64 ttmp15, src_scc, vcc_lo
@@ -1717,7 +1717,7 @@ v_cmp_ge_i16_e64 s[10:11], ttmp15, src_scc
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_i16_e64 s[10:11], m0, 0.5
-// W64: encoding: [0x0a,0x00,0x36,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x36,0xd4,0x7d,0xe0,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_i16_e64 s[10:11], exec_lo, -1
@@ -1737,7 +1737,7 @@ v_cmp_ge_i16_e64 s[104:105], -1, exec_hi
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_i16_e64 vcc, 0.5, m0
-// W64: encoding: [0x6a,0x00,0x36,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x6a,0x00,0x36,0xd4,0xf0,0xfa,0x00,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_i16_e64 ttmp[14:15], src_scc, vcc_lo
@@ -1982,7 +1982,7 @@ v_cmp_ge_u16_e64 s5, ttmp15, src_scc
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_u16_e64 s5, m0, 0.5
-// W32: encoding: [0x05,0x00,0x3e,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x05,0x00,0x3e,0xd4,0x7d,0xe0,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_u16_e64 s5, exec_lo, -1
@@ -2002,7 +2002,7 @@ v_cmp_ge_u16_e64 vcc_lo, -1, exec_hi
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_u16_e64 vcc_hi, 0.5, m0
-// W32: encoding: [0x6b,0x00,0x3e,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x6b,0x00,0x3e,0xd4,0xf0,0xfa,0x00,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_u16_e64 ttmp15, src_scc, vcc_lo
@@ -2038,7 +2038,7 @@ v_cmp_ge_u16_e64 s[10:11], ttmp15, src_scc
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_u16_e64 s[10:11], m0, 0.5
-// W64: encoding: [0x0a,0x00,0x3e,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x3e,0xd4,0x7d,0xe0,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_u16_e64 s[10:11], exec_lo, -1
@@ -2058,7 +2058,7 @@ v_cmp_ge_u16_e64 s[104:105], -1, exec_hi
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_u16_e64 vcc, 0.5, m0
-// W64: encoding: [0x6a,0x00,0x3e,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x6a,0x00,0x3e,0xd4,0xf0,0xfa,0x00,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ge_u16_e64 ttmp[14:15], src_scc, vcc_lo
@@ -2624,7 +2624,7 @@ v_cmp_gt_i16_e64 s5, ttmp15, src_scc
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_i16_e64 s5, m0, 0.5
-// W32: encoding: [0x05,0x00,0x34,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x05,0x00,0x34,0xd4,0x7d,0xe0,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_i16_e64 s5, exec_lo, -1
@@ -2644,7 +2644,7 @@ v_cmp_gt_i16_e64 vcc_lo, -1, exec_hi
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_i16_e64 vcc_hi, 0.5, m0
-// W32: encoding: [0x6b,0x00,0x34,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x6b,0x00,0x34,0xd4,0xf0,0xfa,0x00,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_i16_e64 ttmp15, src_scc, vcc_lo
@@ -2680,7 +2680,7 @@ v_cmp_gt_i16_e64 s[10:11], ttmp15, src_scc
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_i16_e64 s[10:11], m0, 0.5
-// W64: encoding: [0x0a,0x00,0x34,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x34,0xd4,0x7d,0xe0,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_i16_e64 s[10:11], exec_lo, -1
@@ -2700,7 +2700,7 @@ v_cmp_gt_i16_e64 s[104:105], -1, exec_hi
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_i16_e64 vcc, 0.5, m0
-// W64: encoding: [0x6a,0x00,0x34,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x6a,0x00,0x34,0xd4,0xf0,0xfa,0x00,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_i16_e64 ttmp[14:15], src_scc, vcc_lo
@@ -2945,7 +2945,7 @@ v_cmp_gt_u16_e64 s5, ttmp15, src_scc
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_u16_e64 s5, m0, 0.5
-// W32: encoding: [0x05,0x00,0x3c,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x05,0x00,0x3c,0xd4,0x7d,0xe0,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_u16_e64 s5, exec_lo, -1
@@ -2965,7 +2965,7 @@ v_cmp_gt_u16_e64 vcc_lo, -1, exec_hi
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_u16_e64 vcc_hi, 0.5, m0
-// W32: encoding: [0x6b,0x00,0x3c,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x6b,0x00,0x3c,0xd4,0xf0,0xfa,0x00,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_u16_e64 ttmp15, src_scc, vcc_lo
@@ -3001,7 +3001,7 @@ v_cmp_gt_u16_e64 s[10:11], ttmp15, src_scc
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_u16_e64 s[10:11], m0, 0.5
-// W64: encoding: [0x0a,0x00,0x3c,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x3c,0xd4,0x7d,0xe0,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_u16_e64 s[10:11], exec_lo, -1
@@ -3021,7 +3021,7 @@ v_cmp_gt_u16_e64 s[104:105], -1, exec_hi
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_u16_e64 vcc, 0.5, m0
-// W64: encoding: [0x6a,0x00,0x3c,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x6a,0x00,0x3c,0xd4,0xf0,0xfa,0x00,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_gt_u16_e64 ttmp[14:15], src_scc, vcc_lo
@@ -3587,7 +3587,7 @@ v_cmp_le_i16_e64 s5, ttmp15, src_scc
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_i16_e64 s5, m0, 0.5
-// W32: encoding: [0x05,0x00,0x33,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x05,0x00,0x33,0xd4,0x7d,0xe0,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_i16_e64 s5, exec_lo, -1
@@ -3607,7 +3607,7 @@ v_cmp_le_i16_e64 vcc_lo, -1, exec_hi
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_i16_e64 vcc_hi, 0.5, m0
-// W32: encoding: [0x6b,0x00,0x33,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x6b,0x00,0x33,0xd4,0xf0,0xfa,0x00,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_i16_e64 ttmp15, src_scc, vcc_lo
@@ -3643,7 +3643,7 @@ v_cmp_le_i16_e64 s[10:11], ttmp15, src_scc
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_i16_e64 s[10:11], m0, 0.5
-// W64: encoding: [0x0a,0x00,0x33,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x33,0xd4,0x7d,0xe0,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_i16_e64 s[10:11], exec_lo, -1
@@ -3663,7 +3663,7 @@ v_cmp_le_i16_e64 s[104:105], -1, exec_hi
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_i16_e64 vcc, 0.5, m0
-// W64: encoding: [0x6a,0x00,0x33,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x6a,0x00,0x33,0xd4,0xf0,0xfa,0x00,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_i16_e64 ttmp[14:15], src_scc, vcc_lo
@@ -3908,7 +3908,7 @@ v_cmp_le_u16_e64 s5, ttmp15, src_scc
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_u16_e64 s5, m0, 0.5
-// W32: encoding: [0x05,0x00,0x3b,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x05,0x00,0x3b,0xd4,0x7d,0xe0,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_u16_e64 s5, exec_lo, -1
@@ -3928,7 +3928,7 @@ v_cmp_le_u16_e64 vcc_lo, -1, exec_hi
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_u16_e64 vcc_hi, 0.5, m0
-// W32: encoding: [0x6b,0x00,0x3b,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x6b,0x00,0x3b,0xd4,0xf0,0xfa,0x00,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_u16_e64 ttmp15, src_scc, vcc_lo
@@ -3964,7 +3964,7 @@ v_cmp_le_u16_e64 s[10:11], ttmp15, src_scc
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_u16_e64 s[10:11], m0, 0.5
-// W64: encoding: [0x0a,0x00,0x3b,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x3b,0xd4,0x7d,0xe0,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_u16_e64 s[10:11], exec_lo, -1
@@ -3984,7 +3984,7 @@ v_cmp_le_u16_e64 s[104:105], -1, exec_hi
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_u16_e64 vcc, 0.5, m0
-// W64: encoding: [0x6a,0x00,0x3b,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x6a,0x00,0x3b,0xd4,0xf0,0xfa,0x00,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_le_u16_e64 ttmp[14:15], src_scc, vcc_lo
@@ -4871,7 +4871,7 @@ v_cmp_lt_i16_e64 s5, ttmp15, src_scc
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_i16_e64 s5, m0, 0.5
-// W32: encoding: [0x05,0x00,0x31,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x05,0x00,0x31,0xd4,0x7d,0xe0,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_i16_e64 s5, exec_lo, -1
@@ -4891,7 +4891,7 @@ v_cmp_lt_i16_e64 vcc_lo, -1, exec_hi
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_i16_e64 vcc_hi, 0.5, m0
-// W32: encoding: [0x6b,0x00,0x31,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x6b,0x00,0x31,0xd4,0xf0,0xfa,0x00,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_i16_e64 ttmp15, src_scc, vcc_lo
@@ -4927,7 +4927,7 @@ v_cmp_lt_i16_e64 s[10:11], ttmp15, src_scc
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_i16_e64 s[10:11], m0, 0.5
-// W64: encoding: [0x0a,0x00,0x31,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x31,0xd4,0x7d,0xe0,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_i16_e64 s[10:11], exec_lo, -1
@@ -4947,7 +4947,7 @@ v_cmp_lt_i16_e64 s[104:105], -1, exec_hi
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_i16_e64 vcc, 0.5, m0
-// W64: encoding: [0x6a,0x00,0x31,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x6a,0x00,0x31,0xd4,0xf0,0xfa,0x00,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_i16_e64 ttmp[14:15], src_scc, vcc_lo
@@ -5192,7 +5192,7 @@ v_cmp_lt_u16_e64 s5, ttmp15, src_scc
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_u16_e64 s5, m0, 0.5
-// W32: encoding: [0x05,0x00,0x39,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x05,0x00,0x39,0xd4,0x7d,0xe0,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_u16_e64 s5, exec_lo, -1
@@ -5212,7 +5212,7 @@ v_cmp_lt_u16_e64 vcc_lo, -1, exec_hi
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_u16_e64 vcc_hi, 0.5, m0
-// W32: encoding: [0x6b,0x00,0x39,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x6b,0x00,0x39,0xd4,0xf0,0xfa,0x00,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_u16_e64 ttmp15, src_scc, vcc_lo
@@ -5248,7 +5248,7 @@ v_cmp_lt_u16_e64 s[10:11], ttmp15, src_scc
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_u16_e64 s[10:11], m0, 0.5
-// W64: encoding: [0x0a,0x00,0x39,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x39,0xd4,0x7d,0xe0,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_u16_e64 s[10:11], exec_lo, -1
@@ -5268,7 +5268,7 @@ v_cmp_lt_u16_e64 s[104:105], -1, exec_hi
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_u16_e64 vcc, 0.5, m0
-// W64: encoding: [0x6a,0x00,0x39,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x6a,0x00,0x39,0xd4,0xf0,0xfa,0x00,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_lt_u16_e64 ttmp[14:15], src_scc, vcc_lo
@@ -5513,7 +5513,7 @@ v_cmp_ne_i16_e64 s5, ttmp15, src_scc
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_i16_e64 s5, m0, 0.5
-// W32: encoding: [0x05,0x00,0x35,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x05,0x00,0x35,0xd4,0x7d,0xe0,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_i16_e64 s5, exec_lo, -1
@@ -5533,7 +5533,7 @@ v_cmp_ne_i16_e64 vcc_lo, -1, exec_hi
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_i16_e64 vcc_hi, 0.5, m0
-// W32: encoding: [0x6b,0x00,0x35,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x6b,0x00,0x35,0xd4,0xf0,0xfa,0x00,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_i16_e64 ttmp15, src_scc, vcc_lo
@@ -5569,7 +5569,7 @@ v_cmp_ne_i16_e64 s[10:11], ttmp15, src_scc
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_i16_e64 s[10:11], m0, 0.5
-// W64: encoding: [0x0a,0x00,0x35,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x35,0xd4,0x7d,0xe0,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_i16_e64 s[10:11], exec_lo, -1
@@ -5589,7 +5589,7 @@ v_cmp_ne_i16_e64 s[104:105], -1, exec_hi
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_i16_e64 vcc, 0.5, m0
-// W64: encoding: [0x6a,0x00,0x35,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x6a,0x00,0x35,0xd4,0xf0,0xfa,0x00,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_i16_e64 ttmp[14:15], src_scc, vcc_lo
@@ -5834,7 +5834,7 @@ v_cmp_ne_u16_e64 s5, ttmp15, src_scc
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_u16_e64 s5, m0, 0.5
-// W32: encoding: [0x05,0x00,0x3d,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x05,0x00,0x3d,0xd4,0x7d,0xe0,0x01,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_u16_e64 s5, exec_lo, -1
@@ -5854,7 +5854,7 @@ v_cmp_ne_u16_e64 vcc_lo, -1, exec_hi
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_u16_e64 vcc_hi, 0.5, m0
-// W32: encoding: [0x6b,0x00,0x3d,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W32: encoding: [0x6b,0x00,0x3d,0xd4,0xf0,0xfa,0x00,0x00]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_u16_e64 ttmp15, src_scc, vcc_lo
@@ -5890,7 +5890,7 @@ v_cmp_ne_u16_e64 s[10:11], ttmp15, src_scc
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_u16_e64 s[10:11], m0, 0.5
-// W64: encoding: [0x0a,0x00,0x3d,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x0a,0x00,0x3d,0xd4,0x7d,0xe0,0x01,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_u16_e64 s[10:11], exec_lo, -1
@@ -5910,7 +5910,7 @@ v_cmp_ne_u16_e64 s[104:105], -1, exec_hi
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_u16_e64 vcc, 0.5, m0
-// W64: encoding: [0x6a,0x00,0x3d,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// W64: encoding: [0x6a,0x00,0x3d,0xd4,0xf0,0xfa,0x00,0x00]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction
 
 v_cmp_ne_u16_e64 ttmp[14:15], src_scc, vcc_lo
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3cx.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3cx.s
index 46ecb9f4d2ca..d0fe55ae0a7e 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3cx.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3cx.s
@@ -284,7 +284,7 @@ v_cmpx_eq_i16_e64 ttmp15, src_scc
 // GFX12: encoding: [0x7e,0x00,0xb2,0xd4,0x7b,0xfa,0x01,0x00]
 
 v_cmpx_eq_i16_e64 m0, 0.5
-// GFX12: encoding: [0x7e,0x00,0xb2,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x7e,0x00,0xb2,0xd4,0x7d,0xe0,0x01,0x00]
 
 v_cmpx_eq_i16_e64 exec_lo, -1
 // GFX12: encoding: [0x7e,0x00,0xb2,0xd4,0x7e,0x82,0x01,0x00]
@@ -299,7 +299,7 @@ v_cmpx_eq_i16_e64 -1, exec_hi
 // GFX12: encoding: [0x7e,0x00,0xb2,0xd4,0xc1,0xfe,0x00,0x00]
 
 v_cmpx_eq_i16_e64 0.5, m0
-// GFX12: encoding: [0x7e,0x00,0xb2,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x7e,0x00,0xb2,0xd4,0xf0,0xfa,0x00,0x00]
 
 v_cmpx_eq_i16_e64 src_scc, vcc_lo
 // GFX12: encoding: [0x7e,0x00,0xb2,0xd4,0xfd,0xd4,0x00,0x00]
@@ -410,7 +410,7 @@ v_cmpx_eq_u16_e64 ttmp15, src_scc
 // GFX12: encoding: [0x7e,0x00,0xba,0xd4,0x7b,0xfa,0x01,0x00]
 
 v_cmpx_eq_u16_e64 m0, 0.5
-// GFX12: encoding: [0x7e,0x00,0xba,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x7e,0x00,0xba,0xd4,0x7d,0xe0,0x01,0x00]
 
 v_cmpx_eq_u16_e64 exec_lo, -1
 // GFX12: encoding: [0x7e,0x00,0xba,0xd4,0x7e,0x82,0x01,0x00]
@@ -425,7 +425,7 @@ v_cmpx_eq_u16_e64 -1, exec_hi
 // GFX12: encoding: [0x7e,0x00,0xba,0xd4,0xc1,0xfe,0x00,0x00]
 
 v_cmpx_eq_u16_e64 0.5, m0
-// GFX12: encoding: [0x7e,0x00,0xba,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x7e,0x00,0xba,0xd4,0xf0,0xfa,0x00,0x00]
 
 v_cmpx_eq_u16_e64 src_scc, vcc_lo
 // GFX12: encoding: [0x7e,0x00,0xba,0xd4,0xfd,0xd4,0x00,0x00]
@@ -662,7 +662,7 @@ v_cmpx_ge_i16_e64 ttmp15, src_scc
 // GFX12: encoding: [0x7e,0x00,0xb6,0xd4,0x7b,0xfa,0x01,0x00]
 
 v_cmpx_ge_i16_e64 m0, 0.5
-// GFX12: encoding: [0x7e,0x00,0xb6,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x7e,0x00,0xb6,0xd4,0x7d,0xe0,0x01,0x00]
 
 v_cmpx_ge_i16_e64 exec_lo, -1
 // GFX12: encoding: [0x7e,0x00,0xb6,0xd4,0x7e,0x82,0x01,0x00]
@@ -677,7 +677,7 @@ v_cmpx_ge_i16_e64 -1, exec_hi
 // GFX12: encoding: [0x7e,0x00,0xb6,0xd4,0xc1,0xfe,0x00,0x00]
 
 v_cmpx_ge_i16_e64 0.5, m0
-// GFX12: encoding: [0x7e,0x00,0xb6,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x7e,0x00,0xb6,0xd4,0xf0,0xfa,0x00,0x00]
 
 v_cmpx_ge_i16_e64 src_scc, vcc_lo
 // GFX12: encoding: [0x7e,0x00,0xb6,0xd4,0xfd,0xd4,0x00,0x00]
@@ -788,7 +788,7 @@ v_cmpx_ge_u16_e64 ttmp15, src_scc
 // GFX12: encoding: [0x7e,0x00,0xbe,0xd4,0x7b,0xfa,0x01,0x00]
 
 v_cmpx_ge_u16_e64 m0, 0.5
-// GFX12: encoding: [0x7e,0x00,0xbe,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x7e,0x00,0xbe,0xd4,0x7d,0xe0,0x01,0x00]
 
 v_cmpx_ge_u16_e64 exec_lo, -1
 // GFX12: encoding: [0x7e,0x00,0xbe,0xd4,0x7e,0x82,0x01,0x00]
@@ -803,7 +803,7 @@ v_cmpx_ge_u16_e64 -1, exec_hi
 // GFX12: encoding: [0x7e,0x00,0xbe,0xd4,0xc1,0xfe,0x00,0x00]
 
 v_cmpx_ge_u16_e64 0.5, m0
-// GFX12: encoding: [0x7e,0x00,0xbe,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x7e,0x00,0xbe,0xd4,0xf0,0xfa,0x00,0x00]
 
 v_cmpx_ge_u16_e64 src_scc, vcc_lo
 // GFX12: encoding: [0x7e,0x00,0xbe,0xd4,0xfd,0xd4,0x00,0x00]
@@ -1040,7 +1040,7 @@ v_cmpx_gt_i16_e64 ttmp15, src_scc
 // GFX12: encoding: [0x7e,0x00,0xb4,0xd4,0x7b,0xfa,0x01,0x00]
 
 v_cmpx_gt_i16_e64 m0, 0.5
-// GFX12: encoding: [0x7e,0x00,0xb4,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x7e,0x00,0xb4,0xd4,0x7d,0xe0,0x01,0x00]
 
 v_cmpx_gt_i16_e64 exec_lo, -1
 // GFX12: encoding: [0x7e,0x00,0xb4,0xd4,0x7e,0x82,0x01,0x00]
@@ -1055,7 +1055,7 @@ v_cmpx_gt_i16_e64 -1, exec_hi
 // GFX12: encoding: [0x7e,0x00,0xb4,0xd4,0xc1,0xfe,0x00,0x00]
 
 v_cmpx_gt_i16_e64 0.5, m0
-// GFX12: encoding: [0x7e,0x00,0xb4,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x7e,0x00,0xb4,0xd4,0xf0,0xfa,0x00,0x00]
 
 v_cmpx_gt_i16_e64 src_scc, vcc_lo
 // GFX12: encoding: [0x7e,0x00,0xb4,0xd4,0xfd,0xd4,0x00,0x00]
@@ -1166,7 +1166,7 @@ v_cmpx_gt_u16_e64 ttmp15, src_scc
 // GFX12: encoding: [0x7e,0x00,0xbc,0xd4,0x7b,0xfa,0x01,0x00]
 
 v_cmpx_gt_u16_e64 m0, 0.5
-// GFX12: encoding: [0x7e,0x00,0xbc,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x7e,0x00,0xbc,0xd4,0x7d,0xe0,0x01,0x00]
 
 v_cmpx_gt_u16_e64 exec_lo, -1
 // GFX12: encoding: [0x7e,0x00,0xbc,0xd4,0x7e,0x82,0x01,0x00]
@@ -1181,7 +1181,7 @@ v_cmpx_gt_u16_e64 -1, exec_hi
 // GFX12: encoding: [0x7e,0x00,0xbc,0xd4,0xc1,0xfe,0x00,0x00]
 
 v_cmpx_gt_u16_e64 0.5, m0
-// GFX12: encoding: [0x7e,0x00,0xbc,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x7e,0x00,0xbc,0xd4,0xf0,0xfa,0x00,0x00]
 
 v_cmpx_gt_u16_e64 src_scc, vcc_lo
 // GFX12: encoding: [0x7e,0x00,0xbc,0xd4,0xfd,0xd4,0x00,0x00]
@@ -1418,7 +1418,7 @@ v_cmpx_le_i16_e64 ttmp15, src_scc
 // GFX12: encoding: [0x7e,0x00,0xb3,0xd4,0x7b,0xfa,0x01,0x00]
 
 v_cmpx_le_i16_e64 m0, 0.5
-// GFX12: encoding: [0x7e,0x00,0xb3,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x7e,0x00,0xb3,0xd4,0x7d,0xe0,0x01,0x00]
 
 v_cmpx_le_i16_e64 exec_lo, -1
 // GFX12: encoding: [0x7e,0x00,0xb3,0xd4,0x7e,0x82,0x01,0x00]
@@ -1433,7 +1433,7 @@ v_cmpx_le_i16_e64 -1, exec_hi
 // GFX12: encoding: [0x7e,0x00,0xb3,0xd4,0xc1,0xfe,0x00,0x00]
 
 v_cmpx_le_i16_e64 0.5, m0
-// GFX12: encoding: [0x7e,0x00,0xb3,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x7e,0x00,0xb3,0xd4,0xf0,0xfa,0x00,0x00]
 
 v_cmpx_le_i16_e64 src_scc, vcc_lo
 // GFX12: encoding: [0x7e,0x00,0xb3,0xd4,0xfd,0xd4,0x00,0x00]
@@ -1544,7 +1544,7 @@ v_cmpx_le_u16_e64 ttmp15, src_scc
 // GFX12: encoding: [0x7e,0x00,0xbb,0xd4,0x7b,0xfa,0x01,0x00]
 
 v_cmpx_le_u16_e64 m0, 0.5
-// GFX12: encoding: [0x7e,0x00,0xbb,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x7e,0x00,0xbb,0xd4,0x7d,0xe0,0x01,0x00]
 
 v_cmpx_le_u16_e64 exec_lo, -1
 // GFX12: encoding: [0x7e,0x00,0xbb,0xd4,0x7e,0x82,0x01,0x00]
@@ -1559,7 +1559,7 @@ v_cmpx_le_u16_e64 -1, exec_hi
 // GFX12: encoding: [0x7e,0x00,0xbb,0xd4,0xc1,0xfe,0x00,0x00]
 
 v_cmpx_le_u16_e64 0.5, m0
-// GFX12: encoding: [0x7e,0x00,0xbb,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x7e,0x00,0xbb,0xd4,0xf0,0xfa,0x00,0x00]
 
 v_cmpx_le_u16_e64 src_scc, vcc_lo
 // GFX12: encoding: [0x7e,0x00,0xbb,0xd4,0xfd,0xd4,0x00,0x00]
@@ -1922,7 +1922,7 @@ v_cmpx_lt_i16_e64 ttmp15, src_scc
 // GFX12: encoding: [0x7e,0x00,0xb1,0xd4,0x7b,0xfa,0x01,0x00]
 
 v_cmpx_lt_i16_e64 m0, 0.5
-// GFX12: encoding: [0x7e,0x00,0xb1,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x7e,0x00,0xb1,0xd4,0x7d,0xe0,0x01,0x00]
 
 v_cmpx_lt_i16_e64 exec_lo, -1
 // GFX12: encoding: [0x7e,0x00,0xb1,0xd4,0x7e,0x82,0x01,0x00]
@@ -1937,7 +1937,7 @@ v_cmpx_lt_i16_e64 -1, exec_hi
 // GFX12: encoding: [0x7e,0x00,0xb1,0xd4,0xc1,0xfe,0x00,0x00]
 
 v_cmpx_lt_i16_e64 0.5, m0
-// GFX12: encoding: [0x7e,0x00,0xb1,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x7e,0x00,0xb1,0xd4,0xf0,0xfa,0x00,0x00]
 
 v_cmpx_lt_i16_e64 src_scc, vcc_lo
 // GFX12: encoding: [0x7e,0x00,0xb1,0xd4,0xfd,0xd4,0x00,0x00]
@@ -2048,7 +2048,7 @@ v_cmpx_lt_u16_e64 ttmp15, src_scc
 // GFX12: encoding: [0x7e,0x00,0xb9,0xd4,0x7b,0xfa,0x01,0x00]
 
 v_cmpx_lt_u16_e64 m0, 0.5
-// GFX12: encoding: [0x7e,0x00,0xb9,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x7e,0x00,0xb9,0xd4,0x7d,0xe0,0x01,0x00]
 
 v_cmpx_lt_u16_e64 exec_lo, -1
 // GFX12: encoding: [0x7e,0x00,0xb9,0xd4,0x7e,0x82,0x01,0x00]
@@ -2063,7 +2063,7 @@ v_cmpx_lt_u16_e64 -1, exec_hi
 // GFX12: encoding: [0x7e,0x00,0xb9,0xd4,0xc1,0xfe,0x00,0x00]
 
 v_cmpx_lt_u16_e64 0.5, m0
-// GFX12: encoding: [0x7e,0x00,0xb9,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x7e,0x00,0xb9,0xd4,0xf0,0xfa,0x00,0x00]
 
 v_cmpx_lt_u16_e64 src_scc, vcc_lo
 // GFX12: encoding: [0x7e,0x00,0xb9,0xd4,0xfd,0xd4,0x00,0x00]
@@ -2174,7 +2174,7 @@ v_cmpx_ne_i16_e64 ttmp15, src_scc
 // GFX12: encoding: [0x7e,0x00,0xb5,0xd4,0x7b,0xfa,0x01,0x00]
 
 v_cmpx_ne_i16_e64 m0, 0.5
-// GFX12: encoding: [0x7e,0x00,0xb5,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x7e,0x00,0xb5,0xd4,0x7d,0xe0,0x01,0x00]
 
 v_cmpx_ne_i16_e64 exec_lo, -1
 // GFX12: encoding: [0x7e,0x00,0xb5,0xd4,0x7e,0x82,0x01,0x00]
@@ -2189,7 +2189,7 @@ v_cmpx_ne_i16_e64 -1, exec_hi
 // GFX12: encoding: [0x7e,0x00,0xb5,0xd4,0xc1,0xfe,0x00,0x00]
 
 v_cmpx_ne_i16_e64 0.5, m0
-// GFX12: encoding: [0x7e,0x00,0xb5,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x7e,0x00,0xb5,0xd4,0xf0,0xfa,0x00,0x00]
 
 v_cmpx_ne_i16_e64 src_scc, vcc_lo
 // GFX12: encoding: [0x7e,0x00,0xb5,0xd4,0xfd,0xd4,0x00,0x00]
@@ -2300,7 +2300,7 @@ v_cmpx_ne_u16_e64 ttmp15, src_scc
 // GFX12: encoding: [0x7e,0x00,0xbd,0xd4,0x7b,0xfa,0x01,0x00]
 
 v_cmpx_ne_u16_e64 m0, 0.5
-// GFX12: encoding: [0x7e,0x00,0xbd,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x7e,0x00,0xbd,0xd4,0x7d,0xe0,0x01,0x00]
 
 v_cmpx_ne_u16_e64 exec_lo, -1
 // GFX12: encoding: [0x7e,0x00,0xbd,0xd4,0x7e,0x82,0x01,0x00]
@@ -2315,7 +2315,7 @@ v_cmpx_ne_u16_e64 -1, exec_hi
 // GFX12: encoding: [0x7e,0x00,0xbd,0xd4,0xc1,0xfe,0x00,0x00]
 
 v_cmpx_ne_u16_e64 0.5, m0
-// GFX12: encoding: [0x7e,0x00,0xbd,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0x7e,0x00,0xbd,0xd4,0xf0,0xfa,0x00,0x00]
 
 v_cmpx_ne_u16_e64 src_scc, vcc_lo
 // GFX12: encoding: [0x7e,0x00,0xbd,0xd4,0xfd,0xd4,0x00,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vopc.s b/llvm/test/MC/AMDGPU/gfx12_asm_vopc.s
index e0d48972575e..3ae0c0f314d9 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vopc.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vopc.s
@@ -724,7 +724,7 @@ v_cmp_eq_i16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_i16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x64,0x7c,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x64,0x7c]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_i16 vcc_lo, src_scc, v2
@@ -784,7 +784,7 @@ v_cmp_eq_i16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_i16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x64,0x7c,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x64,0x7c]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_i16 vcc, src_scc, v2
@@ -1060,7 +1060,7 @@ v_cmp_eq_u16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_u16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x74,0x7c,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x74,0x7c]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_u16 vcc_lo, src_scc, v2
@@ -1120,7 +1120,7 @@ v_cmp_eq_u16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_u16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x74,0x7c,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x74,0x7c]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_eq_u16 vcc, src_scc, v2
@@ -1732,7 +1732,7 @@ v_cmp_ge_i16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_i16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x6c,0x7c,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x6c,0x7c]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_i16 vcc_lo, src_scc, v2
@@ -1792,7 +1792,7 @@ v_cmp_ge_i16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_i16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x6c,0x7c,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x6c,0x7c]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_i16 vcc, src_scc, v2
@@ -2068,7 +2068,7 @@ v_cmp_ge_u16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_u16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x7c,0x7c,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x7c,0x7c]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_u16 vcc_lo, src_scc, v2
@@ -2128,7 +2128,7 @@ v_cmp_ge_u16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_u16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x7c,0x7c,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x7c,0x7c]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ge_u16 vcc, src_scc, v2
@@ -2740,7 +2740,7 @@ v_cmp_gt_i16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_i16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x68,0x7c,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x68,0x7c]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_i16 vcc_lo, src_scc, v2
@@ -2800,7 +2800,7 @@ v_cmp_gt_i16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_i16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x68,0x7c,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x68,0x7c]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_i16 vcc, src_scc, v2
@@ -3076,7 +3076,7 @@ v_cmp_gt_u16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_u16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x78,0x7c,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x78,0x7c]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_u16 vcc_lo, src_scc, v2
@@ -3136,7 +3136,7 @@ v_cmp_gt_u16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_u16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x78,0x7c,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x78,0x7c]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_gt_u16 vcc, src_scc, v2
@@ -3748,7 +3748,7 @@ v_cmp_le_i16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_le_i16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x66,0x7c,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x66,0x7c]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_le_i16 vcc_lo, src_scc, v2
@@ -3808,7 +3808,7 @@ v_cmp_le_i16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_le_i16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x66,0x7c,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x66,0x7c]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_le_i16 vcc, src_scc, v2
@@ -4084,7 +4084,7 @@ v_cmp_le_u16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_le_u16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x76,0x7c,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x76,0x7c]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_le_u16 vcc_lo, src_scc, v2
@@ -4144,7 +4144,7 @@ v_cmp_le_u16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_le_u16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x76,0x7c,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x76,0x7c]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_le_u16 vcc, src_scc, v2
@@ -5092,7 +5092,7 @@ v_cmp_lt_i16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_i16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x62,0x7c,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x62,0x7c]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_i16 vcc_lo, src_scc, v2
@@ -5152,7 +5152,7 @@ v_cmp_lt_i16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_i16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x62,0x7c,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x62,0x7c]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_i16 vcc, src_scc, v2
@@ -5428,7 +5428,7 @@ v_cmp_lt_u16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_u16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x72,0x7c,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x72,0x7c]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_u16 vcc_lo, src_scc, v2
@@ -5488,7 +5488,7 @@ v_cmp_lt_u16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_u16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x72,0x7c,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x72,0x7c]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_lt_u16 vcc, src_scc, v2
@@ -5764,7 +5764,7 @@ v_cmp_ne_i16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_i16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x6a,0x7c,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x6a,0x7c]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_i16 vcc_lo, src_scc, v2
@@ -5824,7 +5824,7 @@ v_cmp_ne_i16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_i16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x6a,0x7c,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x6a,0x7c]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_i16 vcc, src_scc, v2
@@ -6100,7 +6100,7 @@ v_cmp_ne_u16 vcc_lo, -1, v2
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_u16 vcc_lo, 0.5, v2
-// W32: encoding: [0xff,0x04,0x7a,0x7c,0x00,0x38,0x00,0x00]
+// W32: encoding: [0xf0,0x04,0x7a,0x7c]
 // W64-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_u16 vcc_lo, src_scc, v2
@@ -6160,7 +6160,7 @@ v_cmp_ne_u16 vcc, -1, v2
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_u16 vcc, 0.5, v2
-// W64: encoding: [0xff,0x04,0x7a,0x7c,0x00,0x38,0x00,0x00]
+// W64: encoding: [0xf0,0x04,0x7a,0x7c]
 // W32-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
 
 v_cmp_ne_u16 vcc, src_scc, v2
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vopcx.s b/llvm/test/MC/AMDGPU/gfx12_asm_vopcx.s
index ffbf50cce76e..68c41abaf05f 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vopcx.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vopcx.s
@@ -290,7 +290,7 @@ v_cmpx_eq_i16 -1, v2
 // GFX12: encoding: [0xc1,0x04,0x64,0x7d]
 
 v_cmpx_eq_i16 0.5, v2
-// GFX12: encoding: [0xff,0x04,0x64,0x7d,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0xf0,0x04,0x64,0x7d]
 
 v_cmpx_eq_i16 src_scc, v2
 // GFX12: encoding: [0xfd,0x04,0x64,0x7d]
@@ -416,7 +416,7 @@ v_cmpx_eq_u16 -1, v2
 // GFX12: encoding: [0xc1,0x04,0x74,0x7d]
 
 v_cmpx_eq_u16 0.5, v2
-// GFX12: encoding: [0xff,0x04,0x74,0x7d,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0xf0,0x04,0x74,0x7d]
 
 v_cmpx_eq_u16 src_scc, v2
 // GFX12: encoding: [0xfd,0x04,0x74,0x7d]
@@ -668,7 +668,7 @@ v_cmpx_ge_i16 -1, v2
 // GFX12: encoding: [0xc1,0x04,0x6c,0x7d]
 
 v_cmpx_ge_i16 0.5, v2
-// GFX12: encoding: [0xff,0x04,0x6c,0x7d,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0xf0,0x04,0x6c,0x7d]
 
 v_cmpx_ge_i16 src_scc, v2
 // GFX12: encoding: [0xfd,0x04,0x6c,0x7d]
@@ -794,7 +794,7 @@ v_cmpx_ge_u16 -1, v2
 // GFX12: encoding: [0xc1,0x04,0x7c,0x7d]
 
 v_cmpx_ge_u16 0.5, v2
-// GFX12: encoding: [0xff,0x04,0x7c,0x7d,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0xf0,0x04,0x7c,0x7d]
 
 v_cmpx_ge_u16 src_scc, v2
 // GFX12: encoding: [0xfd,0x04,0x7c,0x7d]
@@ -1046,7 +1046,7 @@ v_cmpx_gt_i16 -1, v2
 // GFX12: encoding: [0xc1,0x04,0x68,0x7d]
 
 v_cmpx_gt_i16 0.5, v2
-// GFX12: encoding: [0xff,0x04,0x68,0x7d,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0xf0,0x04,0x68,0x7d]
 
 v_cmpx_gt_i16 src_scc, v2
 // GFX12: encoding: [0xfd,0x04,0x68,0x7d]
@@ -1172,7 +1172,7 @@ v_cmpx_gt_u16 -1, v2
 // GFX12: encoding: [0xc1,0x04,0x78,0x7d]
 
 v_cmpx_gt_u16 0.5, v2
-// GFX12: encoding: [0xff,0x04,0x78,0x7d,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0xf0,0x04,0x78,0x7d]
 
 v_cmpx_gt_u16 src_scc, v2
 // GFX12: encoding: [0xfd,0x04,0x78,0x7d]
@@ -1424,7 +1424,7 @@ v_cmpx_le_i16 -1, v2
 // GFX12: encoding: [0xc1,0x04,0x66,0x7d]
 
 v_cmpx_le_i16 0.5, v2
-// GFX12: encoding: [0xff,0x04,0x66,0x7d,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0xf0,0x04,0x66,0x7d]
 
 v_cmpx_le_i16 src_scc, v2
 // GFX12: encoding: [0xfd,0x04,0x66,0x7d]
@@ -1550,7 +1550,7 @@ v_cmpx_le_u16 -1, v2
 // GFX12: encoding: [0xc1,0x04,0x76,0x7d]
 
 v_cmpx_le_u16 0.5, v2
-// GFX12: encoding: [0xff,0x04,0x76,0x7d,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0xf0,0x04,0x76,0x7d]
 
 v_cmpx_le_u16 src_scc, v2
 // GFX12: encoding: [0xfd,0x04,0x76,0x7d]
@@ -1928,7 +1928,7 @@ v_cmpx_lt_i16 -1, v2
 // GFX12: encoding: [0xc1,0x04,0x62,0x7d]
 
 v_cmpx_lt_i16 0.5, v2
-// GFX12: encoding: [0xff,0x04,0x62,0x7d,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0xf0,0x04,0x62,0x7d]
 
 v_cmpx_lt_i16 src_scc, v2
 // GFX12: encoding: [0xfd,0x04,0x62,0x7d]
@@ -2054,7 +2054,7 @@ v_cmpx_lt_u16 -1, v2
 // GFX12: encoding: [0xc1,0x04,0x72,0x7d]
 
 v_cmpx_lt_u16 0.5, v2
-// GFX12: encoding: [0xff,0x04,0x72,0x7d,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0xf0,0x04,0x72,0x7d]
 
 v_cmpx_lt_u16 src_scc, v2
 // GFX12: encoding: [0xfd,0x04,0x72,0x7d]
@@ -2180,7 +2180,7 @@ v_cmpx_ne_i16 -1, v2
 // GFX12: encoding: [0xc1,0x04,0x6a,0x7d]
 
 v_cmpx_ne_i16 0.5, v2
-// GFX12: encoding: [0xff,0x04,0x6a,0x7d,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0xf0,0x04,0x6a,0x7d]
 
 v_cmpx_ne_i16 src_scc, v2
 // GFX12: encoding: [0xfd,0x04,0x6a,0x7d]
@@ -2306,7 +2306,7 @@ v_cmpx_ne_u16 -1, v2
 // GFX12: encoding: [0xc1,0x04,0x7a,0x7d]
 
 v_cmpx_ne_u16 0.5, v2
-// GFX12: encoding: [0xff,0x04,0x7a,0x7d,0x00,0x38,0x00,0x00]
+// GFX12: encoding: [0xf0,0x04,0x7a,0x7d]
 
 v_cmpx_ne_u16 src_scc, v2
 // GFX12: encoding: [0xfd,0x04,0x7a,0x7d]
diff --git a/llvm/test/MC/AMDGPU/gfx8_asm_vop1.s b/llvm/test/MC/AMDGPU/gfx8_asm_vop1.s
index 32ebbd9e1028..52a7762f7fde 100644
--- a/llvm/test/MC/AMDGPU/gfx8_asm_vop1.s
+++ b/llvm/test/MC/AMDGPU/gfx8_asm_vop1.s
@@ -3958,10 +3958,10 @@ v_cvt_f16_u16 v5, -1
 // CHECK: [0xc1,0x72,0x0a,0x7e]
 
 v_cvt_f16_u16 v5, 0.5
-// CHECK: [0xff,0x72,0x0a,0x7e,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x72,0x0a,0x7e]
 
 v_cvt_f16_u16 v5, -4.0
-// CHECK: [0xff,0x72,0x0a,0x7e,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x72,0x0a,0x7e]
 
 v_cvt_f16_u16 v5, src_vccz
 // CHECK: [0xfb,0x72,0x0a,0x7e]
@@ -4039,10 +4039,10 @@ v_cvt_f16_i16 v5, -1
 // CHECK: [0xc1,0x74,0x0a,0x7e]
 
 v_cvt_f16_i16 v5, 0.5
-// CHECK: [0xff,0x74,0x0a,0x7e,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x74,0x0a,0x7e]
 
 v_cvt_f16_i16 v5, -4.0
-// CHECK: [0xff,0x74,0x0a,0x7e,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x74,0x0a,0x7e]
 
 v_cvt_f16_i16 v5, src_vccz
 // CHECK: [0xfb,0x74,0x0a,0x7e]
diff --git a/llvm/test/MC/AMDGPU/gfx8_asm_vop2.s b/llvm/test/MC/AMDGPU/gfx8_asm_vop2.s
index 1c355af95028..5fc7f406a716 100644
--- a/llvm/test/MC/AMDGPU/gfx8_asm_vop2.s
+++ b/llvm/test/MC/AMDGPU/gfx8_asm_vop2.s
@@ -2785,10 +2785,10 @@ v_add_u16 v5, -1, v2
 // CHECK: [0xc1,0x04,0x0a,0x4c]
 
 v_add_u16 v5, 0.5, v2
-// CHECK: [0xff,0x04,0x0a,0x4c,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x0a,0x4c]
 
 v_add_u16 v5, -4.0, v2
-// CHECK: [0xff,0x04,0x0a,0x4c,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x0a,0x4c]
 
 v_add_u16 v5, src_vccz, v2
 // CHECK: [0xfb,0x04,0x0a,0x4c]
@@ -2869,10 +2869,10 @@ v_sub_u16 v5, -1, v2
 // CHECK: [0xc1,0x04,0x0a,0x4e]
 
 v_sub_u16 v5, 0.5, v2
-// CHECK: [0xff,0x04,0x0a,0x4e,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x0a,0x4e]
 
 v_sub_u16 v5, -4.0, v2
-// CHECK: [0xff,0x04,0x0a,0x4e,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x0a,0x4e]
 
 v_sub_u16 v5, src_vccz, v2
 // CHECK: [0xfb,0x04,0x0a,0x4e]
@@ -2953,10 +2953,10 @@ v_subrev_u16 v5, -1, v2
 // CHECK: [0xc1,0x04,0x0a,0x50]
 
 v_subrev_u16 v5, 0.5, v2
-// CHECK: [0xff,0x04,0x0a,0x50,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x0a,0x50]
 
 v_subrev_u16 v5, -4.0, v2
-// CHECK: [0xff,0x04,0x0a,0x50,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x0a,0x50]
 
 v_subrev_u16 v5, src_vccz, v2
 // CHECK: [0xfb,0x04,0x0a,0x50]
@@ -3034,10 +3034,10 @@ v_mul_lo_u16 v5, -1, v2
 // CHECK: [0xc1,0x04,0x0a,0x52]
 
 v_mul_lo_u16 v5, 0.5, v2
-// CHECK: [0xff,0x04,0x0a,0x52,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x0a,0x52]
 
 v_mul_lo_u16 v5, -4.0, v2
-// CHECK: [0xff,0x04,0x0a,0x52,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x0a,0x52]
 
 v_mul_lo_u16 v5, src_vccz, v2
 // CHECK: [0xfb,0x04,0x0a,0x52]
@@ -3118,10 +3118,10 @@ v_lshlrev_b16 v5, -1, v2
 // CHECK: [0xc1,0x04,0x0a,0x54]
 
 v_lshlrev_b16 v5, 0.5, v2
-// CHECK: [0xff,0x04,0x0a,0x54,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x0a,0x54]
 
 v_lshlrev_b16 v5, -4.0, v2
-// CHECK: [0xff,0x04,0x0a,0x54,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x0a,0x54]
 
 v_lshlrev_b16 v5, src_vccz, v2
 // CHECK: [0xfb,0x04,0x0a,0x54]
@@ -3199,10 +3199,10 @@ v_lshrrev_b16 v5, -1, v2
 // CHECK: [0xc1,0x04,0x0a,0x56]
 
 v_lshrrev_b16 v5, 0.5, v2
-// CHECK: [0xff,0x04,0x0a,0x56,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x0a,0x56]
 
 v_lshrrev_b16 v5, -4.0, v2
-// CHECK: [0xff,0x04,0x0a,0x56,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x0a,0x56]
 
 v_lshrrev_b16 v5, src_vccz, v2
 // CHECK: [0xfb,0x04,0x0a,0x56]
@@ -3280,10 +3280,10 @@ v_ashrrev_i16 v5, -1, v2
 // CHECK: [0xc1,0x04,0x0a,0x58]
 
 v_ashrrev_i16 v5, 0.5, v2
-// CHECK: [0xff,0x04,0x0a,0x58,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x0a,0x58]
 
 v_ashrrev_i16 v5, -4.0, v2
-// CHECK: [0xff,0x04,0x0a,0x58,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x0a,0x58]
 
 v_ashrrev_i16 v5, src_vccz, v2
 // CHECK: [0xfb,0x04,0x0a,0x58]
@@ -3529,10 +3529,10 @@ v_max_u16 v5, -1, v2
 // CHECK: [0xc1,0x04,0x0a,0x5e]
 
 v_max_u16 v5, 0.5, v2
-// CHECK: [0xff,0x04,0x0a,0x5e,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x0a,0x5e]
 
 v_max_u16 v5, -4.0, v2
-// CHECK: [0xff,0x04,0x0a,0x5e,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x0a,0x5e]
 
 v_max_u16 v5, src_vccz, v2
 // CHECK: [0xfb,0x04,0x0a,0x5e]
@@ -3613,10 +3613,10 @@ v_max_i16 v5, -1, v2
 // CHECK: [0xc1,0x04,0x0a,0x60]
 
 v_max_i16 v5, 0.5, v2
-// CHECK: [0xff,0x04,0x0a,0x60,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x0a,0x60]
 
 v_max_i16 v5, -4.0, v2
-// CHECK: [0xff,0x04,0x0a,0x60,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x0a,0x60]
 
 v_max_i16 v5, src_vccz, v2
 // CHECK: [0xfb,0x04,0x0a,0x60]
@@ -3697,10 +3697,10 @@ v_min_u16 v5, -1, v2
 // CHECK: [0xc1,0x04,0x0a,0x62]
 
 v_min_u16 v5, 0.5, v2
-// CHECK: [0xff,0x04,0x0a,0x62,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x0a,0x62]
 
 v_min_u16 v5, -4.0, v2
-// CHECK: [0xff,0x04,0x0a,0x62,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x0a,0x62]
 
 v_min_u16 v5, src_vccz, v2
 // CHECK: [0xfb,0x04,0x0a,0x62]
@@ -3781,10 +3781,10 @@ v_min_i16 v5, -1, v2
 // CHECK: [0xc1,0x04,0x0a,0x64]
 
 v_min_i16 v5, 0.5, v2
-// CHECK: [0xff,0x04,0x0a,0x64,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x0a,0x64]
 
 v_min_i16 v5, -4.0, v2
-// CHECK: [0xff,0x04,0x0a,0x64,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x0a,0x64]
 
 v_min_i16 v5, src_vccz, v2
 // CHECK: [0xfb,0x04,0x0a,0x64]
diff --git a/llvm/test/MC/AMDGPU/gfx8_asm_vopc.s b/llvm/test/MC/AMDGPU/gfx8_asm_vopc.s
index adbe3f6a2683..8a431727604b 100644
--- a/llvm/test/MC/AMDGPU/gfx8_asm_vopc.s
+++ b/llvm/test/MC/AMDGPU/gfx8_asm_vopc.s
@@ -7705,10 +7705,10 @@ v_cmp_f_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x40,0x7d]
 
 v_cmp_f_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x40,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x40,0x7d]
 
 v_cmp_f_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x40,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x40,0x7d]
 
 v_cmp_f_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x40,0x7d]
@@ -7786,10 +7786,10 @@ v_cmp_lt_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x42,0x7d]
 
 v_cmp_lt_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x42,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x42,0x7d]
 
 v_cmp_lt_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x42,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x42,0x7d]
 
 v_cmp_lt_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x42,0x7d]
@@ -7867,10 +7867,10 @@ v_cmp_eq_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x44,0x7d]
 
 v_cmp_eq_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x44,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x44,0x7d]
 
 v_cmp_eq_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x44,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x44,0x7d]
 
 v_cmp_eq_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x44,0x7d]
@@ -7948,10 +7948,10 @@ v_cmp_le_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x46,0x7d]
 
 v_cmp_le_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x46,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x46,0x7d]
 
 v_cmp_le_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x46,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x46,0x7d]
 
 v_cmp_le_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x46,0x7d]
@@ -8029,10 +8029,10 @@ v_cmp_gt_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x48,0x7d]
 
 v_cmp_gt_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x48,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x48,0x7d]
 
 v_cmp_gt_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x48,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x48,0x7d]
 
 v_cmp_gt_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x48,0x7d]
@@ -8110,10 +8110,10 @@ v_cmp_ne_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x4a,0x7d]
 
 v_cmp_ne_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x4a,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x4a,0x7d]
 
 v_cmp_ne_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x4a,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x4a,0x7d]
 
 v_cmp_ne_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x4a,0x7d]
@@ -8191,10 +8191,10 @@ v_cmp_ge_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x4c,0x7d]
 
 v_cmp_ge_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x4c,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x4c,0x7d]
 
 v_cmp_ge_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x4c,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x4c,0x7d]
 
 v_cmp_ge_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x4c,0x7d]
@@ -8272,10 +8272,10 @@ v_cmp_t_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x4e,0x7d]
 
 v_cmp_t_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x4e,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x4e,0x7d]
 
 v_cmp_t_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x4e,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x4e,0x7d]
 
 v_cmp_t_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x4e,0x7d]
@@ -8353,10 +8353,10 @@ v_cmp_f_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x50,0x7d]
 
 v_cmp_f_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x50,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x50,0x7d]
 
 v_cmp_f_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x50,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x50,0x7d]
 
 v_cmp_f_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x50,0x7d]
@@ -8434,10 +8434,10 @@ v_cmp_lt_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x52,0x7d]
 
 v_cmp_lt_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x52,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x52,0x7d]
 
 v_cmp_lt_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x52,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x52,0x7d]
 
 v_cmp_lt_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x52,0x7d]
@@ -8515,10 +8515,10 @@ v_cmp_eq_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x54,0x7d]
 
 v_cmp_eq_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x54,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x54,0x7d]
 
 v_cmp_eq_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x54,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x54,0x7d]
 
 v_cmp_eq_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x54,0x7d]
@@ -8596,10 +8596,10 @@ v_cmp_le_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x56,0x7d]
 
 v_cmp_le_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x56,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x56,0x7d]
 
 v_cmp_le_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x56,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x56,0x7d]
 
 v_cmp_le_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x56,0x7d]
@@ -8677,10 +8677,10 @@ v_cmp_gt_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x58,0x7d]
 
 v_cmp_gt_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x58,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x58,0x7d]
 
 v_cmp_gt_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x58,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x58,0x7d]
 
 v_cmp_gt_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x58,0x7d]
@@ -8758,10 +8758,10 @@ v_cmp_ne_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x5a,0x7d]
 
 v_cmp_ne_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x5a,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x5a,0x7d]
 
 v_cmp_ne_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x5a,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x5a,0x7d]
 
 v_cmp_ne_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x5a,0x7d]
@@ -8839,10 +8839,10 @@ v_cmp_ge_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x5c,0x7d]
 
 v_cmp_ge_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x5c,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x5c,0x7d]
 
 v_cmp_ge_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x5c,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x5c,0x7d]
 
 v_cmp_ge_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x5c,0x7d]
@@ -8920,10 +8920,10 @@ v_cmp_t_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x5e,0x7d]
 
 v_cmp_t_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x5e,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x5e,0x7d]
 
 v_cmp_t_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x5e,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x5e,0x7d]
 
 v_cmp_t_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x5e,0x7d]
@@ -9001,10 +9001,10 @@ v_cmpx_f_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x60,0x7d]
 
 v_cmpx_f_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x60,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x60,0x7d]
 
 v_cmpx_f_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x60,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x60,0x7d]
 
 v_cmpx_f_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x60,0x7d]
@@ -9082,10 +9082,10 @@ v_cmpx_lt_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x62,0x7d]
 
 v_cmpx_lt_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x62,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x62,0x7d]
 
 v_cmpx_lt_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x62,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x62,0x7d]
 
 v_cmpx_lt_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x62,0x7d]
@@ -9163,10 +9163,10 @@ v_cmpx_eq_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x64,0x7d]
 
 v_cmpx_eq_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x64,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x64,0x7d]
 
 v_cmpx_eq_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x64,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x64,0x7d]
 
 v_cmpx_eq_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x64,0x7d]
@@ -9244,10 +9244,10 @@ v_cmpx_le_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x66,0x7d]
 
 v_cmpx_le_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x66,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x66,0x7d]
 
 v_cmpx_le_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x66,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x66,0x7d]
 
 v_cmpx_le_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x66,0x7d]
@@ -9325,10 +9325,10 @@ v_cmpx_gt_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x68,0x7d]
 
 v_cmpx_gt_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x68,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x68,0x7d]
 
 v_cmpx_gt_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x68,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x68,0x7d]
 
 v_cmpx_gt_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x68,0x7d]
@@ -9406,10 +9406,10 @@ v_cmpx_ne_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x6a,0x7d]
 
 v_cmpx_ne_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x6a,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x6a,0x7d]
 
 v_cmpx_ne_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x6a,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x6a,0x7d]
 
 v_cmpx_ne_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x6a,0x7d]
@@ -9487,10 +9487,10 @@ v_cmpx_ge_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x6c,0x7d]
 
 v_cmpx_ge_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x6c,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x6c,0x7d]
 
 v_cmpx_ge_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x6c,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x6c,0x7d]
 
 v_cmpx_ge_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x6c,0x7d]
@@ -9568,10 +9568,10 @@ v_cmpx_t_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x6e,0x7d]
 
 v_cmpx_t_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x6e,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x6e,0x7d]
 
 v_cmpx_t_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x6e,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x6e,0x7d]
 
 v_cmpx_t_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x6e,0x7d]
@@ -9649,10 +9649,10 @@ v_cmpx_f_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x70,0x7d]
 
 v_cmpx_f_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x70,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x70,0x7d]
 
 v_cmpx_f_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x70,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x70,0x7d]
 
 v_cmpx_f_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x70,0x7d]
@@ -9730,10 +9730,10 @@ v_cmpx_lt_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x72,0x7d]
 
 v_cmpx_lt_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x72,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x72,0x7d]
 
 v_cmpx_lt_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x72,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x72,0x7d]
 
 v_cmpx_lt_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x72,0x7d]
@@ -9811,10 +9811,10 @@ v_cmpx_eq_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x74,0x7d]
 
 v_cmpx_eq_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x74,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x74,0x7d]
 
 v_cmpx_eq_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x74,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x74,0x7d]
 
 v_cmpx_eq_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x74,0x7d]
@@ -9892,10 +9892,10 @@ v_cmpx_le_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x76,0x7d]
 
 v_cmpx_le_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x76,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x76,0x7d]
 
 v_cmpx_le_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x76,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x76,0x7d]
 
 v_cmpx_le_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x76,0x7d]
@@ -9973,10 +9973,10 @@ v_cmpx_gt_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x78,0x7d]
 
 v_cmpx_gt_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x78,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x78,0x7d]
 
 v_cmpx_gt_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x78,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x78,0x7d]
 
 v_cmpx_gt_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x78,0x7d]
@@ -10054,10 +10054,10 @@ v_cmpx_ne_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x7a,0x7d]
 
 v_cmpx_ne_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x7a,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x7a,0x7d]
 
 v_cmpx_ne_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x7a,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x7a,0x7d]
 
 v_cmpx_ne_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x7a,0x7d]
@@ -10135,10 +10135,10 @@ v_cmpx_ge_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x7c,0x7d]
 
 v_cmpx_ge_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x7c,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x7c,0x7d]
 
 v_cmpx_ge_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x7c,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x7c,0x7d]
 
 v_cmpx_ge_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x7c,0x7d]
@@ -10216,10 +10216,10 @@ v_cmpx_t_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x7e,0x7d]
 
 v_cmpx_t_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x7e,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x7e,0x7d]
 
 v_cmpx_t_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x7e,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x7e,0x7d]
 
 v_cmpx_t_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x7e,0x7d]
diff --git a/llvm/test/MC/AMDGPU/gfx9-asm-err.s b/llvm/test/MC/AMDGPU/gfx9-asm-err.s
index 451fab99e12a..93138a829185 100644
--- a/llvm/test/MC/AMDGPU/gfx9-asm-err.s
+++ b/llvm/test/MC/AMDGPU/gfx9-asm-err.s
@@ -1,33 +1,33 @@
 // RUN: not llvm-mc -triple=amdgcn -mcpu=gfx900 %s 2>&1 | FileCheck -check-prefix=GFX9ERR --implicit-check-not=error: %s
 
-v_cvt_f16_u16_e64 v5, 0.5
+v_cvt_f16_u16_e64 v5, 0.6
 // GFX9ERR: :[[@LINE-1]]:{{[0-9]+}}: error: literal operands are not supported
 
-v_cvt_f16_u16_e64 v5, -4.0
+v_cvt_f16_u16_e64 v5, -4.1
 // GFX9ERR: :[[@LINE-1]]:{{[0-9]+}}: error: literal operands are not supported
 
-v_add_u16_e64 v5, v1, 0.5
+v_add_u16_e64 v5, v1, 0.4
 // GFX9ERR: :[[@LINE-1]]:{{[0-9]+}}: error: literal operands are not supported
 
-v_add_u16_e64 v5, v1, -4.0
+v_add_u16_e64 v5, v1, -4.2
 // GFX9ERR: :[[@LINE-1]]:{{[0-9]+}}: error: literal operands are not supported
 
-v_cvt_f16_i16_e64 v5, 0.5
+v_cvt_f16_i16_e64 v5, 0.3
 // GFX9ERR: :[[@LINE-1]]:{{[0-9]+}}: error: literal operands are not supported
 
-v_cvt_f16_i16_e64 v5, -4.0
+v_cvt_f16_i16_e64 v5, -4.5
 // GFX9ERR: :[[@LINE-1]]:{{[0-9]+}}: error: literal operands are not supported
 
-v_add_u16_e64 v5, 0.5, v2
+v_add_u16_e64 v5, 0.1, v2
 // GFX9ERR: :[[@LINE-1]]:{{[0-9]+}}: error: literal operands are not supported
 
-v_add_u16_e64 v5, -4.0, v2
+v_add_u16_e64 v5, -3.8, v2
 // GFX9ERR: :[[@LINE-1]]:{{[0-9]+}}: error: literal operands are not supported
 
-v_subrev_u16_e64 v5, v1, 0.5
+v_subrev_u16_e64 v5, v1, 0.4
 // GFX9ERR: :[[@LINE-1]]:{{[0-9]+}}: error: literal operands are not supported
 
-v_subrev_u16_e64 v5, v1, -4.0
+v_subrev_u16_e64 v5, v1, -4.2
 // GFX9ERR: :[[@LINE-1]]:{{[0-9]+}}: error: literal operands are not supported
 
 v_cvt_u32_f64 v5, v[0:1] quad_perm:[0,2,1,1] row_mask:0xf bank_mask:0xf
diff --git a/llvm/test/MC/AMDGPU/gfx9_asm_vop1.s b/llvm/test/MC/AMDGPU/gfx9_asm_vop1.s
index 038b4d2185a4..5227fcd9d22a 100644
--- a/llvm/test/MC/AMDGPU/gfx9_asm_vop1.s
+++ b/llvm/test/MC/AMDGPU/gfx9_asm_vop1.s
@@ -3436,10 +3436,10 @@ v_cvt_f16_u16 v5, -1
 // CHECK: [0xc1,0x72,0x0a,0x7e]
 
 v_cvt_f16_u16 v5, 0.5
-// CHECK: [0xff,0x72,0x0a,0x7e,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x72,0x0a,0x7e]
 
 v_cvt_f16_u16 v5, -4.0
-// CHECK: [0xff,0x72,0x0a,0x7e,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x72,0x0a,0x7e]
 
 v_cvt_f16_u16 v5, src_vccz
 // CHECK: [0xfb,0x72,0x0a,0x7e]
@@ -3505,10 +3505,10 @@ v_cvt_f16_i16 v5, -1
 // CHECK: [0xc1,0x74,0x0a,0x7e]
 
 v_cvt_f16_i16 v5, 0.5
-// CHECK: [0xff,0x74,0x0a,0x7e,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x74,0x0a,0x7e]
 
 v_cvt_f16_i16 v5, -4.0
-// CHECK: [0xff,0x74,0x0a,0x7e,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x74,0x0a,0x7e]
 
 v_cvt_f16_i16 v5, src_vccz
 // CHECK: [0xfb,0x74,0x0a,0x7e]
diff --git a/llvm/test/MC/AMDGPU/gfx9_asm_vop2.s b/llvm/test/MC/AMDGPU/gfx9_asm_vop2.s
index 7b53da641b76..fe7fc66d95a2 100644
--- a/llvm/test/MC/AMDGPU/gfx9_asm_vop2.s
+++ b/llvm/test/MC/AMDGPU/gfx9_asm_vop2.s
@@ -1,5 +1,7 @@
 // RUN: llvm-mc -triple=amdgcn -mcpu=gfx900 -show-encoding %s | FileCheck %s
 
+.text
+
 v_cndmask_b32 v5, v1, v2, vcc
 // CHECK: [0x01,0x05,0x0a,0x00]
 
@@ -2419,10 +2421,10 @@ v_add_u16 v5, -1, v2
 // CHECK: [0xc1,0x04,0x0a,0x4c]
 
 v_add_u16 v5, 0.5, v2
-// CHECK: [0xff,0x04,0x0a,0x4c,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x0a,0x4c]
 
 v_add_u16 v5, -4.0, v2
-// CHECK: [0xff,0x04,0x0a,0x4c,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x0a,0x4c]
 
 v_add_u16 v5, src_vccz, v2
 // CHECK: [0xfb,0x04,0x0a,0x4c]
@@ -2491,10 +2493,10 @@ v_sub_u16 v5, -1, v2
 // CHECK: [0xc1,0x04,0x0a,0x4e]
 
 v_sub_u16 v5, 0.5, v2
-// CHECK: [0xff,0x04,0x0a,0x4e,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x0a,0x4e]
 
 v_sub_u16 v5, -4.0, v2
-// CHECK: [0xff,0x04,0x0a,0x4e,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x0a,0x4e]
 
 v_sub_u16 v5, src_vccz, v2
 // CHECK: [0xfb,0x04,0x0a,0x4e]
@@ -2563,10 +2565,10 @@ v_subrev_u16 v5, -1, v2
 // CHECK: [0xc1,0x04,0x0a,0x50]
 
 v_subrev_u16 v5, 0.5, v2
-// CHECK: [0xff,0x04,0x0a,0x50,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x0a,0x50]
 
 v_subrev_u16 v5, -4.0, v2
-// CHECK: [0xff,0x04,0x0a,0x50,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x0a,0x50]
 
 v_subrev_u16 v5, src_vccz, v2
 // CHECK: [0xfb,0x04,0x0a,0x50]
@@ -2632,10 +2634,10 @@ v_mul_lo_u16 v5, -1, v2
 // CHECK: [0xc1,0x04,0x0a,0x52]
 
 v_mul_lo_u16 v5, 0.5, v2
-// CHECK: [0xff,0x04,0x0a,0x52,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x0a,0x52]
 
 v_mul_lo_u16 v5, -4.0, v2
-// CHECK: [0xff,0x04,0x0a,0x52,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x0a,0x52]
 
 v_mul_lo_u16 v5, src_vccz, v2
 // CHECK: [0xfb,0x04,0x0a,0x52]
@@ -2704,10 +2706,10 @@ v_lshlrev_b16 v5, -1, v2
 // CHECK: [0xc1,0x04,0x0a,0x54]
 
 v_lshlrev_b16 v5, 0.5, v2
-// CHECK: [0xff,0x04,0x0a,0x54,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x0a,0x54]
 
 v_lshlrev_b16 v5, -4.0, v2
-// CHECK: [0xff,0x04,0x0a,0x54,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x0a,0x54]
 
 v_lshlrev_b16 v5, src_vccz, v2
 // CHECK: [0xfb,0x04,0x0a,0x54]
@@ -2773,10 +2775,10 @@ v_lshrrev_b16 v5, -1, v2
 // CHECK: [0xc1,0x04,0x0a,0x56]
 
 v_lshrrev_b16 v5, 0.5, v2
-// CHECK: [0xff,0x04,0x0a,0x56,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x0a,0x56]
 
 v_lshrrev_b16 v5, -4.0, v2
-// CHECK: [0xff,0x04,0x0a,0x56,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x0a,0x56]
 
 v_lshrrev_b16 v5, src_vccz, v2
 // CHECK: [0xfb,0x04,0x0a,0x56]
@@ -2842,10 +2844,10 @@ v_ashrrev_i16 v5, -1, v2
 // CHECK: [0xc1,0x04,0x0a,0x58]
 
 v_ashrrev_i16 v5, 0.5, v2
-// CHECK: [0xff,0x04,0x0a,0x58,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x0a,0x58]
 
 v_ashrrev_i16 v5, -4.0, v2
-// CHECK: [0xff,0x04,0x0a,0x58,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x0a,0x58]
 
 v_ashrrev_i16 v5, src_vccz, v2
 // CHECK: [0xfb,0x04,0x0a,0x58]
@@ -3055,10 +3057,10 @@ v_max_u16 v5, -1, v2
 // CHECK: [0xc1,0x04,0x0a,0x5e]
 
 v_max_u16 v5, 0.5, v2
-// CHECK: [0xff,0x04,0x0a,0x5e,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x0a,0x5e]
 
 v_max_u16 v5, -4.0, v2
-// CHECK: [0xff,0x04,0x0a,0x5e,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x0a,0x5e]
 
 v_max_u16 v5, src_vccz, v2
 // CHECK: [0xfb,0x04,0x0a,0x5e]
@@ -3127,10 +3129,10 @@ v_max_i16 v5, -1, v2
 // CHECK: [0xc1,0x04,0x0a,0x60]
 
 v_max_i16 v5, 0.5, v2
-// CHECK: [0xff,0x04,0x0a,0x60,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x0a,0x60]
 
 v_max_i16 v5, -4.0, v2
-// CHECK: [0xff,0x04,0x0a,0x60,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x0a,0x60]
 
 v_max_i16 v5, src_vccz, v2
 // CHECK: [0xfb,0x04,0x0a,0x60]
@@ -3199,10 +3201,10 @@ v_min_u16 v5, -1, v2
 // CHECK: [0xc1,0x04,0x0a,0x62]
 
 v_min_u16 v5, 0.5, v2
-// CHECK: [0xff,0x04,0x0a,0x62,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x0a,0x62]
 
 v_min_u16 v5, -4.0, v2
-// CHECK: [0xff,0x04,0x0a,0x62,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x0a,0x62]
 
 v_min_u16 v5, src_vccz, v2
 // CHECK: [0xfb,0x04,0x0a,0x62]
@@ -3271,10 +3273,10 @@ v_min_i16 v5, -1, v2
 // CHECK: [0xc1,0x04,0x0a,0x64]
 
 v_min_i16 v5, 0.5, v2
-// CHECK: [0xff,0x04,0x0a,0x64,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x0a,0x64]
 
 v_min_i16 v5, -4.0, v2
-// CHECK: [0xff,0x04,0x0a,0x64,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x0a,0x64]
 
 v_min_i16 v5, src_vccz, v2
 // CHECK: [0xfb,0x04,0x0a,0x64]
diff --git a/llvm/test/MC/AMDGPU/gfx9_asm_vopc.s b/llvm/test/MC/AMDGPU/gfx9_asm_vopc.s
index 345afa2149dc..bcb717096607 100644
--- a/llvm/test/MC/AMDGPU/gfx9_asm_vopc.s
+++ b/llvm/test/MC/AMDGPU/gfx9_asm_vopc.s
@@ -6673,10 +6673,10 @@ v_cmp_f_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x40,0x7d]
 
 v_cmp_f_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x40,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x40,0x7d]
 
 v_cmp_f_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x40,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x40,0x7d]
 
 v_cmp_f_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x40,0x7d]
@@ -6742,10 +6742,10 @@ v_cmp_lt_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x42,0x7d]
 
 v_cmp_lt_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x42,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x42,0x7d]
 
 v_cmp_lt_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x42,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x42,0x7d]
 
 v_cmp_lt_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x42,0x7d]
@@ -6811,10 +6811,10 @@ v_cmp_eq_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x44,0x7d]
 
 v_cmp_eq_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x44,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x44,0x7d]
 
 v_cmp_eq_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x44,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x44,0x7d]
 
 v_cmp_eq_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x44,0x7d]
@@ -6880,10 +6880,10 @@ v_cmp_le_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x46,0x7d]
 
 v_cmp_le_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x46,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x46,0x7d]
 
 v_cmp_le_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x46,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x46,0x7d]
 
 v_cmp_le_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x46,0x7d]
@@ -6949,10 +6949,10 @@ v_cmp_gt_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x48,0x7d]
 
 v_cmp_gt_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x48,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x48,0x7d]
 
 v_cmp_gt_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x48,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x48,0x7d]
 
 v_cmp_gt_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x48,0x7d]
@@ -7018,10 +7018,10 @@ v_cmp_ne_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x4a,0x7d]
 
 v_cmp_ne_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x4a,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x4a,0x7d]
 
 v_cmp_ne_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x4a,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x4a,0x7d]
 
 v_cmp_ne_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x4a,0x7d]
@@ -7087,10 +7087,10 @@ v_cmp_ge_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x4c,0x7d]
 
 v_cmp_ge_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x4c,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x4c,0x7d]
 
 v_cmp_ge_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x4c,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x4c,0x7d]
 
 v_cmp_ge_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x4c,0x7d]
@@ -7156,10 +7156,10 @@ v_cmp_t_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x4e,0x7d]
 
 v_cmp_t_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x4e,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x4e,0x7d]
 
 v_cmp_t_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x4e,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x4e,0x7d]
 
 v_cmp_t_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x4e,0x7d]
@@ -7225,10 +7225,10 @@ v_cmp_f_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x50,0x7d]
 
 v_cmp_f_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x50,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x50,0x7d]
 
 v_cmp_f_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x50,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x50,0x7d]
 
 v_cmp_f_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x50,0x7d]
@@ -7294,10 +7294,10 @@ v_cmp_lt_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x52,0x7d]
 
 v_cmp_lt_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x52,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x52,0x7d]
 
 v_cmp_lt_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x52,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x52,0x7d]
 
 v_cmp_lt_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x52,0x7d]
@@ -7363,10 +7363,10 @@ v_cmp_eq_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x54,0x7d]
 
 v_cmp_eq_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x54,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x54,0x7d]
 
 v_cmp_eq_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x54,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x54,0x7d]
 
 v_cmp_eq_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x54,0x7d]
@@ -7432,10 +7432,10 @@ v_cmp_le_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x56,0x7d]
 
 v_cmp_le_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x56,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x56,0x7d]
 
 v_cmp_le_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x56,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x56,0x7d]
 
 v_cmp_le_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x56,0x7d]
@@ -7501,10 +7501,10 @@ v_cmp_gt_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x58,0x7d]
 
 v_cmp_gt_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x58,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x58,0x7d]
 
 v_cmp_gt_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x58,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x58,0x7d]
 
 v_cmp_gt_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x58,0x7d]
@@ -7570,10 +7570,10 @@ v_cmp_ne_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x5a,0x7d]
 
 v_cmp_ne_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x5a,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x5a,0x7d]
 
 v_cmp_ne_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x5a,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x5a,0x7d]
 
 v_cmp_ne_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x5a,0x7d]
@@ -7639,10 +7639,10 @@ v_cmp_ge_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x5c,0x7d]
 
 v_cmp_ge_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x5c,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x5c,0x7d]
 
 v_cmp_ge_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x5c,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x5c,0x7d]
 
 v_cmp_ge_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x5c,0x7d]
@@ -7708,10 +7708,10 @@ v_cmp_t_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x5e,0x7d]
 
 v_cmp_t_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x5e,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x5e,0x7d]
 
 v_cmp_t_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x5e,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x5e,0x7d]
 
 v_cmp_t_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x5e,0x7d]
@@ -7777,10 +7777,10 @@ v_cmpx_f_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x60,0x7d]
 
 v_cmpx_f_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x60,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x60,0x7d]
 
 v_cmpx_f_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x60,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x60,0x7d]
 
 v_cmpx_f_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x60,0x7d]
@@ -7846,10 +7846,10 @@ v_cmpx_lt_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x62,0x7d]
 
 v_cmpx_lt_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x62,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x62,0x7d]
 
 v_cmpx_lt_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x62,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x62,0x7d]
 
 v_cmpx_lt_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x62,0x7d]
@@ -7915,10 +7915,10 @@ v_cmpx_eq_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x64,0x7d]
 
 v_cmpx_eq_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x64,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x64,0x7d]
 
 v_cmpx_eq_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x64,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x64,0x7d]
 
 v_cmpx_eq_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x64,0x7d]
@@ -7984,10 +7984,10 @@ v_cmpx_le_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x66,0x7d]
 
 v_cmpx_le_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x66,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x66,0x7d]
 
 v_cmpx_le_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x66,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x66,0x7d]
 
 v_cmpx_le_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x66,0x7d]
@@ -8053,10 +8053,10 @@ v_cmpx_gt_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x68,0x7d]
 
 v_cmpx_gt_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x68,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x68,0x7d]
 
 v_cmpx_gt_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x68,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x68,0x7d]
 
 v_cmpx_gt_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x68,0x7d]
@@ -8122,10 +8122,10 @@ v_cmpx_ne_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x6a,0x7d]
 
 v_cmpx_ne_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x6a,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x6a,0x7d]
 
 v_cmpx_ne_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x6a,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x6a,0x7d]
 
 v_cmpx_ne_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x6a,0x7d]
@@ -8191,10 +8191,10 @@ v_cmpx_ge_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x6c,0x7d]
 
 v_cmpx_ge_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x6c,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x6c,0x7d]
 
 v_cmpx_ge_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x6c,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x6c,0x7d]
 
 v_cmpx_ge_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x6c,0x7d]
@@ -8260,10 +8260,10 @@ v_cmpx_t_i16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x6e,0x7d]
 
 v_cmpx_t_i16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x6e,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x6e,0x7d]
 
 v_cmpx_t_i16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x6e,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x6e,0x7d]
 
 v_cmpx_t_i16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x6e,0x7d]
@@ -8329,10 +8329,10 @@ v_cmpx_f_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x70,0x7d]
 
 v_cmpx_f_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x70,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x70,0x7d]
 
 v_cmpx_f_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x70,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x70,0x7d]
 
 v_cmpx_f_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x70,0x7d]
@@ -8398,10 +8398,10 @@ v_cmpx_lt_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x72,0x7d]
 
 v_cmpx_lt_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x72,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x72,0x7d]
 
 v_cmpx_lt_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x72,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x72,0x7d]
 
 v_cmpx_lt_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x72,0x7d]
@@ -8467,10 +8467,10 @@ v_cmpx_eq_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x74,0x7d]
 
 v_cmpx_eq_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x74,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x74,0x7d]
 
 v_cmpx_eq_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x74,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x74,0x7d]
 
 v_cmpx_eq_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x74,0x7d]
@@ -8536,10 +8536,10 @@ v_cmpx_le_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x76,0x7d]
 
 v_cmpx_le_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x76,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x76,0x7d]
 
 v_cmpx_le_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x76,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x76,0x7d]
 
 v_cmpx_le_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x76,0x7d]
@@ -8605,10 +8605,10 @@ v_cmpx_gt_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x78,0x7d]
 
 v_cmpx_gt_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x78,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x78,0x7d]
 
 v_cmpx_gt_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x78,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x78,0x7d]
 
 v_cmpx_gt_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x78,0x7d]
@@ -8674,10 +8674,10 @@ v_cmpx_ne_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x7a,0x7d]
 
 v_cmpx_ne_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x7a,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x7a,0x7d]
 
 v_cmpx_ne_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x7a,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x7a,0x7d]
 
 v_cmpx_ne_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x7a,0x7d]
@@ -8743,10 +8743,10 @@ v_cmpx_ge_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x7c,0x7d]
 
 v_cmpx_ge_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x7c,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x7c,0x7d]
 
 v_cmpx_ge_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x7c,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x7c,0x7d]
 
 v_cmpx_ge_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x7c,0x7d]
@@ -8812,10 +8812,10 @@ v_cmpx_t_u16 vcc, -1, v2
 // CHECK: [0xc1,0x04,0x7e,0x7d]
 
 v_cmpx_t_u16 vcc, 0.5, v2
-// CHECK: [0xff,0x04,0x7e,0x7d,0x00,0x38,0x00,0x00]
+// CHECK: [0xf0,0x04,0x7e,0x7d]
 
 v_cmpx_t_u16 vcc, -4.0, v2
-// CHECK: [0xff,0x04,0x7e,0x7d,0x00,0xc4,0x00,0x00]
+// CHECK: [0xf7,0x04,0x7e,0x7d]
 
 v_cmpx_t_u16 vcc, src_vccz, v2
 // CHECK: [0xfb,0x04,0x7e,0x7d]
diff --git a/llvm/test/MC/AMDGPU/gfx9_err_pos.s b/llvm/test/MC/AMDGPU/gfx9_err_pos.s
index 350a699fe0d7..578a063ebb70 100644
--- a/llvm/test/MC/AMDGPU/gfx9_err_pos.s
+++ b/llvm/test/MC/AMDGPU/gfx9_err_pos.s
@@ -168,16 +168,6 @@ v_bfe_u32 v0, v2, undef, v3
 // CHECK-NEXT:{{^}}v_bfe_u32 v0, v2, undef, v3
 // CHECK-NEXT:{{^}}                  ^
 
-v_add_i16 v5, v1, 0.5
-// CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: literal operands are not supported
-// CHECK-NEXT:{{^}}v_add_i16 v5, v1, 0.5
-// CHECK-NEXT:{{^}}                  ^
-
-v_add_i16 v5, 0.5, v2
-// CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: literal operands are not supported
-// CHECK-NEXT:{{^}}v_add_i16 v5, 0.5, v2
-// CHECK-NEXT:{{^}}              ^
-
 //==============================================================================
 // too few operands for instruction
 
diff --git a/llvm/test/MC/AMDGPU/vop3-gfx10.s b/llvm/test/MC/AMDGPU/vop3-gfx10.s
index 50810e82647b..2bd78ea80435 100644
--- a/llvm/test/MC/AMDGPU/vop3-gfx10.s
+++ b/llvm/test/MC/AMDGPU/vop3-gfx10.s
@@ -1,13 +1,13 @@
 // RUN: llvm-mc -triple=amdgcn -mcpu=gfx1010 -show-encoding %s | FileCheck -check-prefix=GFX10 %s
 
 v_mad_i16 v5, v1, 4.0, v3
-// GFX10: v_mad_i16 v5, v1, 0x4400, v3    ; encoding: [0x05,0x00,0x5e,0xd7,0x01,0xff,0x0d,0x04,0x00,0x44,0x00,0x00]
+// GFX10: v_mad_i16 v5, v1, 4.0, v3 ; encoding: [0x05,0x00,0x5e,0xd7,0x01,0xed,0x0d,0x04]
 
 v_mad_i16 v5, v1, 0x4400, v3
 // GFX10: v_mad_i16 v5, v1, 0x4400, v3    ; encoding: [0x05,0x00,0x5e,0xd7,0x01,0xff,0x0d,0x04,0x00,0x44,0x00,0x00]
 
 v_mad_i16 v5, v1, -4.0, v3
-// GFX10: v_mad_i16 v5, v1, 0xc400, v3    ; encoding: [0x05,0x00,0x5e,0xd7,0x01,0xff,0x0d,0x04,0x00,0xc4,0x00,0x00]
+// GFX10: v_mad_i16 v5, v1, -4.0, v3 ; encoding: [0x05,0x00,0x5e,0xd7,0x01,0xef,0x0d,0x04]
 
 v_mad_i16 v5, v1, 0xc400, v3
 // GFX10: v_mad_i16 v5, v1, 0xc400, v3    ; encoding: [0x05,0x00,0x5e,0xd7,0x01,0xff,0x0d,0x04,0x00,0xc4,0x00,0x00]
diff --git a/llvm/test/MC/AMDGPU/vop3-gfx9.s b/llvm/test/MC/AMDGPU/vop3-gfx9.s
index b61f690d2023..a61b0c87e199 100644
--- a/llvm/test/MC/AMDGPU/vop3-gfx9.s
+++ b/llvm/test/MC/AMDGPU/vop3-gfx9.s
@@ -426,9 +426,9 @@ v_mad_i16 v5, v1, -1, v3
 // VI: v_mad_i16 v5, v1, -1, v3 ; encoding: [0x05,0x00,0xec,0xd1,0x01,0x83,0x0d,0x04]
 
 v_mad_i16 v5, v1, v2, -4.0
-// NOGFX9: :[[@LINE-1]]:{{[0-9]+}}: error: literal operands are not supported
+// GFX9: v_mad_i16 v5, v1, v2, -4.0 ; encoding: [0x05,0x00,0x05,0xd2,0x01,0x05,0xde,0x03]
 // NOSICI: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
-// NOVI: :[[@LINE-3]]:{{[0-9]+}}: error: literal operands are not supported
+// VI: v_mad_i16 v5, v1, v2, -4.0 ; encoding: [0x05,0x00,0xec,0xd1,0x01,0x05,0xde,0x03]
 
 v_mad_i16 v5, v1, v2, v3 clamp
 // GFX9: v_mad_i16 v5, v1, v2, v3 clamp ; encoding: [0x05,0x80,0x05,0xd2,0x01,0x05,0x0e,0x04]
@@ -478,11 +478,11 @@ v_mad_legacy_i16 v5, v1, -1, v3
 // NOGCN: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
 
 v_mad_legacy_i16 v5, v1, v2, -4.0
-// NOGFX9: :[[@LINE-1]]:{{[0-9]+}}: error: literal operands are not supported
+// GFX9: v_mad_legacy_i16 v5, v1, v2, -4.0 ; encoding: [0x05,0x00,0xec,0xd1,0x01,0x05,0xde,0x03]
 // NOGCN: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
 
 v_mad_legacy_i16 v5, v1, v2, -4.0 clamp
-// NOGFX9: :[[@LINE-1]]:{{[0-9]+}}: error: literal operands are not supported
+// GFX9: v_mad_legacy_i16 v5, v1, v2, -4.0 clamp ; encoding: [0x05,0x80,0xec,0xd1,0x01,0x05,0xde,0x03]
 // NOGCN: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
 
 v_mad_legacy_u16_e64 v5, 0, v2, v3
@@ -494,11 +494,11 @@ v_mad_legacy_u16 v5, v1, -1, v3
 // NOGCN: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
 
 v_mad_legacy_u16 v5, v1, v2, -4.0
-// NOGFX9: :[[@LINE-1]]:{{[0-9]+}}: error: literal operands are not supported
+// GFX9: v_mad_legacy_u16 v5, v1, v2, -4.0 ; encoding: [0x05,0x00,0xeb,0xd1,0x01,0x05,0xde,0x03]
 // NOGCN: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
 
 v_mad_legacy_u16 v5, v1, v2, -4.0 clamp
-// NOGFX9: :[[@LINE-1]]:{{[0-9]+}}: error: literal operands are not supported
+// GFX9: v_mad_legacy_u16 v5, v1, v2, -4.0 clamp ; encoding: [0x05,0x80,0xeb,0xd1,0x01,0x05,0xde,0x03]
 // NOGCN: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
 
 v_mad_u16_e64 v5, 0, v2, v3
@@ -512,9 +512,9 @@ v_mad_u16 v5, v1, -1, v3
 // VI: v_mad_u16 v5, v1, -1, v3 ; encoding: [0x05,0x00,0xeb,0xd1,0x01,0x83,0x0d,0x04]
 
 v_mad_u16 v5, v1, v2, -4.0
-// NOGFX9: :[[@LINE-1]]:{{[0-9]+}}: error: literal operands are not supported
+// GFX9: v_mad_u16 v5, v1, v2, -4.0 ; encoding: [0x05,0x00,0x04,0xd2,0x01,0x05,0xde,0x03]
 // NOSICI: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
-// NOVI: :[[@LINE-3]]:{{[0-9]+}}: error: literal operands are not supported
+// VI: v_mad_u16 v5, v1, v2, -4.0 ; encoding: [0x05,0x00,0xeb,0xd1,0x01,0x05,0xde,0x03]
 
 v_mad_u16 v5, v1, v2, v3 clamp
 // GFX9: v_mad_u16 v5, v1, v2, v3 clamp ; encoding: [0x05,0x80,0x04,0xd2,0x01,0x05,0x0e,0x04]
diff --git a/llvm/test/MC/AMDGPU/vop3.s b/llvm/test/MC/AMDGPU/vop3.s
index 27fcf5e1ba33..0d2544002a9f 100644
--- a/llvm/test/MC/AMDGPU/vop3.s
+++ b/llvm/test/MC/AMDGPU/vop3.s
@@ -599,7 +599,7 @@ v_mad_i16_e64 v5, -1, v2, v3
 // NOSICI: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
 
 v_mad_i16 v5, v1, -4.0, v3
-// NOVI: :[[@LINE-1]]:{{[0-9]+}}: error: literal operands are not supported
+// VI: v_mad_i16 v5, v1, -4.0, v3 ; encoding: [0x05,0x00,0xec,0xd1,0x01,0xef,0x0d,0x04]
 // NOSICI: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
 
 v_mad_i16 v5, v1, v2, 0
@@ -615,7 +615,7 @@ v_mad_u16 v5, v1, 0, v3
 // NOSICI: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
 
 v_mad_u16 v5, v1, v2, -4.0
-// NOVI: :[[@LINE-1]]:{{[0-9]+}}: error: literal operands are not supported
+// VI: v_mad_u16 v5, v1, v2, -4.0 ; encoding: [0x05,0x00,0xeb,0xd1,0x01,0x05,0xde,0x03]
 // NOSICI: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
 
 ///===---------------------------------------------------------------------===//
diff --git a/llvm/test/MC/AMDGPU/vop_sdwa.s b/llvm/test/MC/AMDGPU/vop_sdwa.s
index 6c8482aaddd9..974bb49174bd 100644
--- a/llvm/test/MC/AMDGPU/vop_sdwa.s
+++ b/llvm/test/MC/AMDGPU/vop_sdwa.s
@@ -963,12 +963,12 @@ v_exp_f16_sdwa v5, -|0.5|
 
 // NOSICI: :[[@LINE+3]]:{{[0-9]+}}: error: instruction not supported on this GPU
 // NOVI: :[[@LINE+2]]:{{[0-9]+}}: error: invalid operand for instruction
-// NOGFX9: :[[@LINE+1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX9: v_max_i16_sdwa v5, -4.0, v2 dst_sel:DWORD dst_unused:UNUSED_PRESERVE src0_sel:DWORD src1_sel:DWORD ; encoding: [0xf9,0x04,0x0a,0x60,0xf7,0x16,0x86,0x06]
 v_max_i16_sdwa v5, -4.0, v2 dst_sel:DWORD dst_unused:UNUSED_PRESERVE src0_sel:DWORD src1_sel:DWORD
 
 // NOSICI: :[[@LINE+3]]:{{[0-9]+}}: error: instruction not supported on this GPU
 // NOVI: :[[@LINE+2]]:{{[0-9]+}}: error: invalid operand for instruction
-// NOGFX9: :[[@LINE+1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX9: v_max_i16_sdwa v5, sext(-4.0), v2 dst_sel:DWORD dst_unused:UNUSED_PRESERVE src0_sel:DWORD src1_sel:DWORD ; encoding: [0xf9,0x04,0x0a,0x60,0xf7,0x16,0x8e,0x06]
 v_max_i16_sdwa v5, sext(-4.0), v2 dst_sel:DWORD dst_unused:UNUSED_PRESERVE src0_sel:DWORD src1_sel:DWORD
 
 // NOSICI: :[[@LINE+3]]:{{[0-9]+}}: error: instruction not supported on this GPU
-- 
GitLab


From 5f935e91810eb28854611faf13bb7d07a8dbf470 Mon Sep 17 00:00:00 2001
From: David Majnemer 
Date: Thu, 7 Mar 2024 19:35:38 +0000
Subject: [PATCH 0076/1674] [AArch64] Optimize fp64 <-> fp16 SIMD conversions

Legalization would result in needless scalarization. Add some
DAGCombines to fix this up.
---
 .../Target/AArch64/AArch64ISelLowering.cpp    | 95 ++++++++++++++++++-
 .../lib/Target/AArch64/AArch64InstrFormats.td |  4 +-
 llvm/lib/Target/AArch64/AArch64InstrInfo.td   | 11 ++-
 llvm/test/CodeGen/AArch64/arm64-vcvt_f.ll     | 56 +++++++++++
 .../CodeGen/AArch64/fp16-v8-instructions.ll   | 50 +++-------
 llvm/test/CodeGen/AArch64/fpext.ll            | 64 +++++++------
 llvm/test/CodeGen/AArch64/fptrunc.ll          | 20 +---
 ...e-streaming-mode-fixed-length-fcopysign.ll | 60 +++++-------
 llvm/test/CodeGen/AArch64/vector-fcopysign.ll | 62 ++++--------
 9 files changed, 252 insertions(+), 170 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 2290223a06f8..89b697b2d515 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -4507,13 +4507,16 @@ SDValue AArch64TargetLowering::LowerINT_TO_FP(SDValue Op,
   };
 
   if (Op.getValueType() == MVT::bf16) {
+    unsigned MaxWidth = IsSigned
+                            ? DAG.ComputeMaxSignificantBits(SrcVal)
+                            : DAG.computeKnownBits(SrcVal).countMaxActiveBits();
     // bf16 conversions are promoted to f32 when converting from i16.
-    if (DAG.ComputeMaxSignificantBits(SrcVal) <= 24) {
+    if (MaxWidth <= 24) {
       return IntToFpViaPromotion(MVT::f32);
     }
 
     // bf16 conversions are promoted to f64 when converting from i32.
-    if (DAG.ComputeMaxSignificantBits(SrcVal) <= 53) {
+    if (MaxWidth <= 53) {
       return IntToFpViaPromotion(MVT::f64);
     }
 
@@ -19376,6 +19379,94 @@ static SDValue performBuildVectorCombine(SDNode *N,
   SDLoc DL(N);
   EVT VT = N->getValueType(0);
 
+  if (VT == MVT::v4f16 || VT == MVT::v4bf16) {
+    SDValue Elt0 = N->getOperand(0), Elt1 = N->getOperand(1),
+            Elt2 = N->getOperand(2), Elt3 = N->getOperand(3);
+    if (Elt0->getOpcode() == ISD::FP_ROUND &&
+        Elt1->getOpcode() == ISD::FP_ROUND &&
+        isa(Elt0->getOperand(1)) &&
+        isa(Elt1->getOperand(1)) &&
+        Elt0->getConstantOperandVal(1) == Elt1->getConstantOperandVal(1) &&
+        Elt0->getOperand(0)->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
+        Elt1->getOperand(0)->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
+        // Constant index.
+        isa(Elt0->getOperand(0)->getOperand(1)) &&
+        isa(Elt1->getOperand(0)->getOperand(1)) &&
+        Elt0->getOperand(0)->getOperand(0) ==
+            Elt1->getOperand(0)->getOperand(0) &&
+        Elt0->getOperand(0)->getConstantOperandVal(1) == 0 &&
+        Elt1->getOperand(0)->getConstantOperandVal(1) == 1) {
+      SDValue LowLanesSrcVec = Elt0->getOperand(0)->getOperand(0);
+      if (LowLanesSrcVec.getValueType() == MVT::v2f64) {
+        SDValue HighLanes;
+        if (Elt2->getOpcode() == ISD::UNDEF &&
+            Elt3->getOpcode() == ISD::UNDEF) {
+          HighLanes = DAG.getUNDEF(MVT::v2f32);
+        } else if (Elt2->getOpcode() == ISD::FP_ROUND &&
+                   Elt3->getOpcode() == ISD::FP_ROUND &&
+                   isa(Elt2->getOperand(1)) &&
+                   isa(Elt3->getOperand(1)) &&
+                   Elt2->getConstantOperandVal(1) ==
+                       Elt3->getConstantOperandVal(1) &&
+                   Elt2->getOperand(0)->getOpcode() ==
+                       ISD::EXTRACT_VECTOR_ELT &&
+                   Elt3->getOperand(0)->getOpcode() ==
+                       ISD::EXTRACT_VECTOR_ELT &&
+                   // Constant index.
+                   isa(Elt2->getOperand(0)->getOperand(1)) &&
+                   isa(Elt3->getOperand(0)->getOperand(1)) &&
+                   Elt2->getOperand(0)->getOperand(0) ==
+                       Elt3->getOperand(0)->getOperand(0) &&
+                   Elt2->getOperand(0)->getConstantOperandVal(1) == 0 &&
+                   Elt3->getOperand(0)->getConstantOperandVal(1) == 1) {
+          SDValue HighLanesSrcVec = Elt2->getOperand(0)->getOperand(0);
+          HighLanes =
+              DAG.getNode(AArch64ISD::FCVTXN, DL, MVT::v2f32, HighLanesSrcVec);
+        }
+        if (HighLanes) {
+          SDValue DoubleToSingleSticky =
+              DAG.getNode(AArch64ISD::FCVTXN, DL, MVT::v2f32, LowLanesSrcVec);
+          SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4f32,
+                                       DoubleToSingleSticky, HighLanes);
+          return DAG.getNode(ISD::FP_ROUND, DL, VT, Concat,
+                             Elt0->getOperand(1));
+        }
+      }
+    }
+  }
+
+  if (VT == MVT::v2f64) {
+    SDValue Elt0 = N->getOperand(0), Elt1 = N->getOperand(1);
+    if (Elt0->getOpcode() == ISD::FP_EXTEND &&
+        Elt1->getOpcode() == ISD::FP_EXTEND &&
+        Elt0->getOperand(0)->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
+        Elt1->getOperand(0)->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
+        Elt0->getOperand(0)->getOperand(0) ==
+            Elt1->getOperand(0)->getOperand(0) &&
+        // Constant index.
+        isa(Elt0->getOperand(0)->getOperand(1)) &&
+        isa(Elt1->getOperand(0)->getOperand(1)) &&
+        Elt0->getOperand(0)->getConstantOperandVal(1) + 1 ==
+            Elt1->getOperand(0)->getConstantOperandVal(1) &&
+        // EXTRACT_SUBVECTOR requires that Idx be a constant multiple of
+        // ResultType's known minimum vector length.
+        Elt0->getOperand(0)->getConstantOperandVal(1) %
+                VT.getVectorMinNumElements() ==
+            0) {
+      SDValue SrcVec = Elt0->getOperand(0)->getOperand(0);
+      if (SrcVec.getValueType() == MVT::v4f16 ||
+          SrcVec.getValueType() == MVT::v4bf16) {
+        SDValue HalfToSingle =
+            DAG.getNode(ISD::FP_EXTEND, DL, MVT::v4f32, SrcVec);
+        SDValue SubvectorIdx = Elt0->getOperand(0)->getOperand(1);
+        SDValue Extract = DAG.getNode(
+            ISD::EXTRACT_SUBVECTOR, DL, VT.changeVectorElementType(MVT::f32),
+            HalfToSingle, SubvectorIdx);
+        return DAG.getNode(ISD::FP_EXTEND, DL, VT, Extract);
+      }
+    }
+  }
+
   // A build vector of two extracted elements is equivalent to an
   // extract subvector where the inner vector is any-extended to the
   // extract_vector_elt VT.
diff --git a/llvm/lib/Target/AArch64/AArch64InstrFormats.td b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
index 091db559a337..8360bef8e2f8 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrFormats.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
@@ -6832,7 +6832,7 @@ multiclass SIMDFPNarrowTwoVector opc, string asm> {
 }
 
 multiclass SIMDFPInexactCvtTwoVector opc, string asm,
-                                     Intrinsic OpNode> {
+                                     SDPatternOperator OpNode> {
   def v2f32 : BaseSIMDFPCvtTwoVector<0, U, {S,1}, opc, V64, V128,
                                      asm, ".2s", ".2d",
                           [(set (v2f32 V64:$Rd), (OpNode (v2f64 V128:$Rn)))]>;
@@ -7547,7 +7547,7 @@ class BaseSIMDCmpTwoScalar size, bits<2> size2, bits<5> opcode,
 let mayRaiseFPException = 1, Uses = [FPCR] in
 class SIMDInexactCvtTwoScalar opcode, string asm>
   : I<(outs FPR32:$Rd), (ins FPR64:$Rn), asm, "\t$Rd, $Rn", "",
-     [(set (f32 FPR32:$Rd), (AArch64fcvtxn (f64 FPR64:$Rn)))]>,
+     [(set (f32 FPR32:$Rd), (AArch64fcvtxnsdr (f64 FPR64:$Rn)))]>,
     Sched<[WriteVd]> {
   bits<5> Rd;
   bits<5> Rn;
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index f8c6d9019ef6..3c67f616c1b9 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -757,9 +757,12 @@ def AArch64fcmlez: SDNode<"AArch64ISD::FCMLEz", SDT_AArch64fcmpz>;
 def AArch64fcmltz: SDNode<"AArch64ISD::FCMLTz", SDT_AArch64fcmpz>;
 
 def AArch64fcvtxn_n: SDNode<"AArch64ISD::FCVTXN", SDTFPRoundOp>;
-def AArch64fcvtxn: PatFrags<(ops node:$Rn),
-                            [(f32 (int_aarch64_sisd_fcvtxn (f64 node:$Rn))),
-                             (f32 (AArch64fcvtxn_n (f64 node:$Rn)))]>;
+def AArch64fcvtxnsdr: PatFrags<(ops node:$Rn),
+                               [(f32 (int_aarch64_sisd_fcvtxn (f64 node:$Rn))),
+                                (f32 (AArch64fcvtxn_n (f64 node:$Rn)))]>;
+def AArch64fcvtxnv: PatFrags<(ops node:$Rn),
+                                 [(int_aarch64_neon_fcvtxn node:$Rn),
+                                  (AArch64fcvtxn_n node:$Rn)]>;
 
 def AArch64bici: SDNode<"AArch64ISD::BICi", SDT_AArch64vecimm>;
 def AArch64orri: SDNode<"AArch64ISD::ORRi", SDT_AArch64vecimm>;
@@ -5042,7 +5045,7 @@ def : Pat<(concat_vectors V64:$Rd, (v4f16 (any_fpround (v4f32 V128:$Rn)))),
 defm FCVTPS : SIMDTwoVectorFPToInt<0,1,0b11010, "fcvtps",int_aarch64_neon_fcvtps>;
 defm FCVTPU : SIMDTwoVectorFPToInt<1,1,0b11010, "fcvtpu",int_aarch64_neon_fcvtpu>;
 defm FCVTXN : SIMDFPInexactCvtTwoVector<1, 0, 0b10110, "fcvtxn",
-                                        int_aarch64_neon_fcvtxn>;
+                                        AArch64fcvtxnv>;
 defm FCVTZS : SIMDTwoVectorFPToInt<0, 1, 0b11011, "fcvtzs", any_fp_to_sint>;
 defm FCVTZU : SIMDTwoVectorFPToInt<1, 1, 0b11011, "fcvtzu", any_fp_to_uint>;
 
diff --git a/llvm/test/CodeGen/AArch64/arm64-vcvt_f.ll b/llvm/test/CodeGen/AArch64/arm64-vcvt_f.ll
index aa6b7cb495f1..cafee32ada68 100644
--- a/llvm/test/CodeGen/AArch64/arm64-vcvt_f.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-vcvt_f.ll
@@ -199,6 +199,60 @@ define <2 x float> @test_vcvt_f32_f64(<2 x double> %v) nounwind readnone ssp {
   ret <2 x float> %vcvt1.i
 }
 
+; FALLBACK-NOT: remark{{.*}}G_FPEXT{{.*}}(in function: test_vcvt_bf16_f64)
+; FALLBACK-NOT: remark{{.*}}fpext{{.*}}(in function: test_vcvt_bf16_f64)
+define <2 x bfloat> @test_vcvt_bf16_f64(<2 x double> %v) nounwind readnone ssp {
+; GENERIC-LABEL: test_vcvt_bf16_f64:
+; GENERIC:       // %bb.0:
+; GENERIC-NEXT:    fcvtxn v0.2s, v0.2d
+; GENERIC-NEXT:    movi.4s v1, #127, msl #8
+; GENERIC-NEXT:    movi.4s v2, #1
+; GENERIC-NEXT:    ushr.4s v3, v0, #16
+; GENERIC-NEXT:    add.4s v1, v0, v1
+; GENERIC-NEXT:    and.16b v2, v3, v2
+; GENERIC-NEXT:    add.4s v1, v2, v1
+; GENERIC-NEXT:    fcmeq.4s v2, v0, v0
+; GENERIC-NEXT:    orr.4s v0, #64, lsl #16
+; GENERIC-NEXT:    bit.16b v0, v1, v2
+; GENERIC-NEXT:    shrn.4h v0, v0, #16
+; GENERIC-NEXT:    ret
+;
+; FAST-LABEL: test_vcvt_bf16_f64:
+; FAST:       // %bb.0:
+; FAST-NEXT:    fcvtxn v1.2s, v0.2d
+; FAST-NEXT:    // implicit-def: $q0
+; FAST-NEXT:    fmov d0, d1
+; FAST-NEXT:    ushr.4s v1, v0, #16
+; FAST-NEXT:    movi.4s v2, #1
+; FAST-NEXT:    and.16b v1, v1, v2
+; FAST-NEXT:    add.4s v1, v1, v0
+; FAST-NEXT:    movi.4s v2, #127, msl #8
+; FAST-NEXT:    add.4s v1, v1, v2
+; FAST-NEXT:    mov.16b v2, v0
+; FAST-NEXT:    orr.4s v2, #64, lsl #16
+; FAST-NEXT:    fcmeq.4s v0, v0, v0
+; FAST-NEXT:    bsl.16b v0, v1, v2
+; FAST-NEXT:    shrn.4h v0, v0, #16
+; FAST-NEXT:    ret
+;
+; GISEL-LABEL: test_vcvt_bf16_f64:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcvtxn v0.2s, v0.2d
+; GISEL-NEXT:    movi.4s v1, #127, msl #8
+; GISEL-NEXT:    movi.4s v2, #1
+; GISEL-NEXT:    ushr.4s v3, v0, #16
+; GISEL-NEXT:    add.4s v1, v0, v1
+; GISEL-NEXT:    and.16b v2, v3, v2
+; GISEL-NEXT:    add.4s v1, v2, v1
+; GISEL-NEXT:    fcmeq.4s v2, v0, v0
+; GISEL-NEXT:    orr.4s v0, #64, lsl #16
+; GISEL-NEXT:    bit.16b v0, v1, v2
+; GISEL-NEXT:    shrn.4h v0, v0, #16
+; GISEL-NEXT:    ret
+  %vcvt1.i = fptrunc <2 x double> %v to <2 x bfloat>
+  ret <2 x bfloat> %vcvt1.i
+}
+
 define half @test_vcvt_f16_f32(<1 x float> %x) {
 ; GENERIC-LABEL: test_vcvt_f16_f32:
 ; GENERIC:       // %bb.0:
@@ -350,3 +404,5 @@ define float @from_half(i16 %in) {
 
 declare float @llvm.convert.from.fp16.f32(i16) #1
 declare i16 @llvm.convert.to.fp16.f32(float) #1
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; FALLBACK: {{.*}}
diff --git a/llvm/test/CodeGen/AArch64/fp16-v8-instructions.ll b/llvm/test/CodeGen/AArch64/fp16-v8-instructions.ll
index 7ff61d9bcb0c..ded343b990ac 100644
--- a/llvm/test/CodeGen/AArch64/fp16-v8-instructions.ll
+++ b/llvm/test/CodeGen/AArch64/fp16-v8-instructions.ll
@@ -312,25 +312,12 @@ define <8 x half> @s_to_h(<8 x float> %a) {
 define <8 x half> @d_to_h(<8 x double> %a) {
 ; CHECK-LABEL: d_to_h:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov d5, v0.d[1]
-; CHECK-NEXT:    fcvt h0, d0
-; CHECK-NEXT:    fcvt h4, d1
-; CHECK-NEXT:    mov d1, v1.d[1]
-; CHECK-NEXT:    fcvt h5, d5
-; CHECK-NEXT:    fcvt h1, d1
-; CHECK-NEXT:    mov v0.h[1], v5.h[0]
-; CHECK-NEXT:    mov v0.h[2], v4.h[0]
-; CHECK-NEXT:    mov v0.h[3], v1.h[0]
-; CHECK-NEXT:    fcvt h1, d2
-; CHECK-NEXT:    mov d2, v2.d[1]
-; CHECK-NEXT:    mov v0.h[4], v1.h[0]
-; CHECK-NEXT:    fcvt h1, d2
-; CHECK-NEXT:    mov d2, v3.d[1]
-; CHECK-NEXT:    mov v0.h[5], v1.h[0]
-; CHECK-NEXT:    fcvt h1, d3
-; CHECK-NEXT:    mov v0.h[6], v1.h[0]
-; CHECK-NEXT:    fcvt h1, d2
-; CHECK-NEXT:    mov v0.h[7], v1.h[0]
+; CHECK-NEXT:    fcvtxn v0.2s, v0.2d
+; CHECK-NEXT:    fcvtxn v2.2s, v2.2d
+; CHECK-NEXT:    fcvtxn2 v0.4s, v1.2d
+; CHECK-NEXT:    fcvtxn2 v2.4s, v3.2d
+; CHECK-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-NEXT:    fcvtn2 v0.8h, v2.4s
 ; CHECK-NEXT:    ret
   %1 = fptrunc <8 x double> %a to <8 x half>
   ret <8 x half> %1
@@ -349,25 +336,12 @@ define <8 x float> @h_to_s(<8 x half> %a) {
 define <8 x double> @h_to_d(<8 x half> %a) {
 ; CHECK-LABEL: h_to_d:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT:    mov h1, v0.h[1]
-; CHECK-NEXT:    mov h3, v0.h[3]
-; CHECK-NEXT:    mov h4, v0.h[2]
-; CHECK-NEXT:    fcvt d0, h0
-; CHECK-NEXT:    mov h5, v2.h[1]
-; CHECK-NEXT:    mov h6, v2.h[3]
-; CHECK-NEXT:    mov h7, v2.h[2]
-; CHECK-NEXT:    fcvt d16, h1
-; CHECK-NEXT:    fcvt d17, h3
-; CHECK-NEXT:    fcvt d1, h4
-; CHECK-NEXT:    fcvt d2, h2
-; CHECK-NEXT:    fcvt d4, h5
-; CHECK-NEXT:    fcvt d5, h6
-; CHECK-NEXT:    fcvt d3, h7
-; CHECK-NEXT:    mov v0.d[1], v16.d[0]
-; CHECK-NEXT:    mov v1.d[1], v17.d[0]
-; CHECK-NEXT:    mov v2.d[1], v4.d[0]
-; CHECK-NEXT:    mov v3.d[1], v5.d[0]
+; CHECK-NEXT:    fcvtl v1.4s, v0.4h
+; CHECK-NEXT:    fcvtl2 v2.4s, v0.8h
+; CHECK-NEXT:    fcvtl v0.2d, v1.2s
+; CHECK-NEXT:    fcvtl2 v3.2d, v2.4s
+; CHECK-NEXT:    fcvtl2 v1.2d, v1.4s
+; CHECK-NEXT:    fcvtl v2.2d, v2.2s
 ; CHECK-NEXT:    ret
   %1 = fpext <8 x half> %a to <8 x double>
   ret <8 x double> %1
diff --git a/llvm/test/CodeGen/AArch64/fpext.ll b/llvm/test/CodeGen/AArch64/fpext.ll
index eca3389bcd88..86f7322f7c4e 100644
--- a/llvm/test/CodeGen/AArch64/fpext.ll
+++ b/llvm/test/CodeGen/AArch64/fpext.ll
@@ -85,29 +85,46 @@ entry:
 }
 
 define <2 x double> @fpext_v2f16_v2f64(<2 x half> %a) {
-; CHECK-LABEL: fpext_v2f16_v2f64:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    mov h1, v0.h[1]
-; CHECK-NEXT:    fcvt d0, h0
-; CHECK-NEXT:    fcvt d1, h1
-; CHECK-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: fpext_v2f16_v2f64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-SD-NEXT:    fcvtl v0.2d, v0.2s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: fpext_v2f16_v2f64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-GI-NEXT:    fcvt d0, h0
+; CHECK-GI-NEXT:    fcvt d1, h1
+; CHECK-GI-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-GI-NEXT:    ret
 entry:
   %c = fpext <2 x half> %a to <2 x double>
   ret <2 x double> %c
 }
 
 define <3 x double> @fpext_v3f16_v3f64(<3 x half> %a) {
-; CHECK-LABEL: fpext_v3f16_v3f64:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    mov h1, v0.h[1]
-; CHECK-NEXT:    mov h2, v0.h[2]
-; CHECK-NEXT:    fcvt d0, h0
-; CHECK-NEXT:    fcvt d1, h1
-; CHECK-NEXT:    fcvt d2, h2
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: fpext_v3f16_v3f64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fcvtl v1.4s, v0.4h
+; CHECK-SD-NEXT:    fcvtl v0.2d, v1.2s
+; CHECK-SD-NEXT:    fcvtl2 v2.2d, v1.4s
+; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: fpext_v3f16_v3f64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-GI-NEXT:    mov h2, v0.h[2]
+; CHECK-GI-NEXT:    fcvt d0, h0
+; CHECK-GI-NEXT:    fcvt d1, h1
+; CHECK-GI-NEXT:    fcvt d2, h2
+; CHECK-GI-NEXT:    ret
 entry:
   %c = fpext <3 x half> %a to <3 x double>
   ret <3 x double> %c
@@ -116,16 +133,9 @@ entry:
 define <4 x double> @fpext_v4f16_v4f64(<4 x half> %a) {
 ; CHECK-SD-LABEL: fpext_v4f16_v4f64:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-NEXT:    mov h2, v0.h[3]
-; CHECK-SD-NEXT:    mov h3, v0.h[2]
-; CHECK-SD-NEXT:    fcvt d0, h0
-; CHECK-SD-NEXT:    fcvt d4, h1
-; CHECK-SD-NEXT:    fcvt d2, h2
-; CHECK-SD-NEXT:    fcvt d1, h3
-; CHECK-SD-NEXT:    mov v0.d[1], v4.d[0]
-; CHECK-SD-NEXT:    mov v1.d[1], v2.d[0]
+; CHECK-SD-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-SD-NEXT:    fcvtl2 v1.2d, v0.4s
+; CHECK-SD-NEXT:    fcvtl v0.2d, v0.2s
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: fpext_v4f16_v4f64:
diff --git a/llvm/test/CodeGen/AArch64/fptrunc.ll b/llvm/test/CodeGen/AArch64/fptrunc.ll
index 9425988af834..3efc98ab5fd5 100644
--- a/llvm/test/CodeGen/AArch64/fptrunc.ll
+++ b/llvm/test/CodeGen/AArch64/fptrunc.ll
@@ -84,11 +84,8 @@ entry:
 define <2 x half> @fptrunc_v2f64_v2f16(<2 x double> %a) {
 ; CHECK-SD-LABEL: fptrunc_v2f64_v2f16:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mov d1, v0.d[1]
-; CHECK-SD-NEXT:    fcvt h0, d0
-; CHECK-SD-NEXT:    fcvt h1, d1
-; CHECK-SD-NEXT:    mov v0.h[1], v1.h[0]
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    fcvtxn v0.2s, v0.2d
+; CHECK-SD-NEXT:    fcvtn v0.4h, v0.4s
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: fptrunc_v2f64_v2f16:
@@ -135,16 +132,9 @@ entry:
 define <4 x half> @fptrunc_v4f64_v4f16(<4 x double> %a) {
 ; CHECK-SD-LABEL: fptrunc_v4f64_v4f16:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mov d2, v0.d[1]
-; CHECK-SD-NEXT:    fcvt h0, d0
-; CHECK-SD-NEXT:    fcvt h2, d2
-; CHECK-SD-NEXT:    mov v0.h[1], v2.h[0]
-; CHECK-SD-NEXT:    fcvt h2, d1
-; CHECK-SD-NEXT:    mov d1, v1.d[1]
-; CHECK-SD-NEXT:    mov v0.h[2], v2.h[0]
-; CHECK-SD-NEXT:    fcvt h1, d1
-; CHECK-SD-NEXT:    mov v0.h[3], v1.h[0]
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    fcvtxn v0.2s, v0.2d
+; CHECK-SD-NEXT:    fcvtxn2 v0.4s, v1.2d
+; CHECK-SD-NEXT:    fcvtn v0.4h, v0.4s
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: fptrunc_v4f64_v4f16:
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fcopysign.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fcopysign.ll
index da9b79a56a95..2ace0bca274a 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fcopysign.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fcopysign.ll
@@ -427,49 +427,35 @@ define void @test_copysign_v4f16_v4f32(ptr %ap, ptr %bp) {
 define void @test_copysign_v4f16_v4f64(ptr %ap, ptr %bp) {
 ; SVE-LABEL: test_copysign_v4f16_v4f64:
 ; SVE:       // %bb.0:
-; SVE-NEXT:    sub sp, sp, #16
-; SVE-NEXT:    .cfi_def_cfa_offset 16
-; SVE-NEXT:    ldp q1, q0, [x1]
-; SVE-NEXT:    ldr d4, [x0]
-; SVE-NEXT:    and z4.h, z4.h, #0x7fff
-; SVE-NEXT:    mov z2.d, z0.d[1]
-; SVE-NEXT:    mov z3.d, z1.d[1]
-; SVE-NEXT:    fcvt h0, d0
-; SVE-NEXT:    fcvt h1, d1
-; SVE-NEXT:    fcvt h2, d2
-; SVE-NEXT:    fcvt h3, d3
-; SVE-NEXT:    str h0, [sp, #12]
-; SVE-NEXT:    str h1, [sp, #8]
-; SVE-NEXT:    str h2, [sp, #14]
-; SVE-NEXT:    str h3, [sp, #10]
-; SVE-NEXT:    ldr d0, [sp, #8]
+; SVE-NEXT:    ldp q0, q1, [x1]
+; SVE-NEXT:    ptrue p0.s, vl2
+; SVE-NEXT:    ptrue p1.s
+; SVE-NEXT:    fcvtxn v1.2s, v1.2d
+; SVE-NEXT:    fcvtxn v0.2s, v0.2d
+; SVE-NEXT:    splice z0.s, p0, z0.s, z1.s
+; SVE-NEXT:    ldr d1, [x0]
+; SVE-NEXT:    and z1.h, z1.h, #0x7fff
+; SVE-NEXT:    fcvt z0.h, p1/m, z0.s
+; SVE-NEXT:    uzp1 z0.h, z0.h, z0.h
 ; SVE-NEXT:    and z0.h, z0.h, #0x8000
-; SVE-NEXT:    orr z0.d, z4.d, z0.d
+; SVE-NEXT:    orr z0.d, z1.d, z0.d
 ; SVE-NEXT:    str d0, [x0]
-; SVE-NEXT:    add sp, sp, #16
 ; SVE-NEXT:    ret
 ;
 ; SVE2-LABEL: test_copysign_v4f16_v4f64:
 ; SVE2:       // %bb.0:
-; SVE2-NEXT:    sub sp, sp, #16
-; SVE2-NEXT:    .cfi_def_cfa_offset 16
-; SVE2-NEXT:    ldp q2, q1, [x1]
-; SVE2-NEXT:    mov z0.h, #32767 // =0x7fff
-; SVE2-NEXT:    ldr d5, [x0]
-; SVE2-NEXT:    mov z3.d, z1.d[1]
-; SVE2-NEXT:    mov z4.d, z2.d[1]
-; SVE2-NEXT:    fcvt h1, d1
-; SVE2-NEXT:    fcvt h2, d2
-; SVE2-NEXT:    fcvt h3, d3
-; SVE2-NEXT:    fcvt h4, d4
-; SVE2-NEXT:    str h1, [sp, #12]
-; SVE2-NEXT:    str h2, [sp, #8]
-; SVE2-NEXT:    str h3, [sp, #14]
-; SVE2-NEXT:    str h4, [sp, #10]
-; SVE2-NEXT:    ldr d1, [sp, #8]
-; SVE2-NEXT:    bsl z5.d, z5.d, z1.d, z0.d
-; SVE2-NEXT:    str d5, [x0]
-; SVE2-NEXT:    add sp, sp, #16
+; SVE2-NEXT:    ldp q0, q1, [x1]
+; SVE2-NEXT:    ptrue p0.s, vl2
+; SVE2-NEXT:    ptrue p1.s
+; SVE2-NEXT:    ldr d2, [x0]
+; SVE2-NEXT:    fcvtxn v1.2s, v1.2d
+; SVE2-NEXT:    fcvtxn v0.2s, v0.2d
+; SVE2-NEXT:    splice z0.s, p0, z0.s, z1.s
+; SVE2-NEXT:    mov z1.h, #32767 // =0x7fff
+; SVE2-NEXT:    fcvt z0.h, p1/m, z0.s
+; SVE2-NEXT:    uzp1 z0.h, z0.h, z0.h
+; SVE2-NEXT:    bsl z2.d, z2.d, z0.d, z1.d
+; SVE2-NEXT:    str d2, [x0]
 ; SVE2-NEXT:    ret
   %a = load <4 x half>, ptr %ap
   %b = load <4 x double>, ptr %bp
diff --git a/llvm/test/CodeGen/AArch64/vector-fcopysign.ll b/llvm/test/CodeGen/AArch64/vector-fcopysign.ll
index c33759331bbc..de26676b5c73 100644
--- a/llvm/test/CodeGen/AArch64/vector-fcopysign.ll
+++ b/llvm/test/CodeGen/AArch64/vector-fcopysign.ll
@@ -209,16 +209,10 @@ define <4 x half> @test_copysign_v4f16_v4f32(<4 x half> %a, <4 x float> %b) #0 {
 define <4 x half> @test_copysign_v4f16_v4f64(<4 x half> %a, <4 x double> %b) #0 {
 ; CHECK-LABEL: test_copysign_v4f16_v4f64:
 ; CHECK:       ; %bb.0:
-; CHECK-NEXT:    mov d3, v1[1]
-; CHECK-NEXT:    fcvt h1, d1
-; CHECK-NEXT:    fcvt h3, d3
-; CHECK-NEXT:    mov.h v1[1], v3[0]
-; CHECK-NEXT:    fcvt h3, d2
-; CHECK-NEXT:    mov d2, v2[1]
-; CHECK-NEXT:    mov.h v1[2], v3[0]
-; CHECK-NEXT:    fcvt h2, d2
-; CHECK-NEXT:    mov.h v1[3], v2[0]
+; CHECK-NEXT:    fcvtxn v1.2s, v1.2d
+; CHECK-NEXT:    fcvtxn2 v1.4s, v2.2d
 ; CHECK-NEXT:    mvni.4h v2, #128, lsl #8
+; CHECK-NEXT:    fcvtn v1.4h, v1.4s
 ; CHECK-NEXT:    bif.8b v0, v1, v2
 ; CHECK-NEXT:    ret
   %tmp0 = fptrunc <4 x double> %b to <4 x half>
@@ -291,42 +285,20 @@ define <4 x bfloat> @test_copysign_v4bf16_v4f32(<4 x bfloat> %a, <4 x float> %b)
 define <4 x bfloat> @test_copysign_v4bf16_v4f64(<4 x bfloat> %a, <4 x double> %b) #0 {
 ; CHECK-LABEL: test_copysign_v4bf16_v4f64:
 ; CHECK:       ; %bb.0:
-; CHECK-NEXT:    mov d3, v1[1]
-; CHECK-NEXT:    fcvtxn s1, d1
-; CHECK-NEXT:    mov w8, #32767 ; =0x7fff
-; CHECK-NEXT:    fcvtxn s3, d3
-; CHECK-NEXT:    fmov w10, s1
-; CHECK-NEXT:    ubfx w12, w10, #16, #1
-; CHECK-NEXT:    add w10, w10, w8
-; CHECK-NEXT:    fmov w9, s3
-; CHECK-NEXT:    fcvtxn s3, d2
-; CHECK-NEXT:    mov d2, v2[1]
-; CHECK-NEXT:    add w10, w12, w10
-; CHECK-NEXT:    lsr w10, w10, #16
-; CHECK-NEXT:    ubfx w11, w9, #16, #1
-; CHECK-NEXT:    add w9, w9, w8
-; CHECK-NEXT:    fcvtxn s1, d2
-; CHECK-NEXT:    add w9, w11, w9
-; CHECK-NEXT:    fmov w11, s3
-; CHECK-NEXT:    fmov s3, w10
-; CHECK-NEXT:    lsr w9, w9, #16
-; CHECK-NEXT:    ubfx w12, w11, #16, #1
-; CHECK-NEXT:    fmov s2, w9
-; CHECK-NEXT:    add w9, w11, w8
-; CHECK-NEXT:    fmov w10, s1
-; CHECK-NEXT:    add w9, w12, w9
-; CHECK-NEXT:    lsr w9, w9, #16
-; CHECK-NEXT:    mov.h v3[1], v2[0]
-; CHECK-NEXT:    ubfx w11, w10, #16, #1
-; CHECK-NEXT:    add w8, w10, w8
-; CHECK-NEXT:    fmov s1, w9
-; CHECK-NEXT:    add w8, w11, w8
-; CHECK-NEXT:    lsr w8, w8, #16
-; CHECK-NEXT:    mov.h v3[2], v1[0]
-; CHECK-NEXT:    fmov s1, w8
-; CHECK-NEXT:    mov.h v3[3], v1[0]
-; CHECK-NEXT:    mvni.4h v1, #128, lsl #8
-; CHECK-NEXT:    bif.8b v0, v3, v1
+; CHECK-NEXT:    fcvtxn v1.2s, v1.2d
+; CHECK-NEXT:    movi.4s v3, #1
+; CHECK-NEXT:    fcvtxn2 v1.4s, v2.2d
+; CHECK-NEXT:    movi.4s v2, #127, msl #8
+; CHECK-NEXT:    ushr.4s v4, v1, #16
+; CHECK-NEXT:    add.4s v2, v1, v2
+; CHECK-NEXT:    and.16b v3, v4, v3
+; CHECK-NEXT:    add.4s v2, v3, v2
+; CHECK-NEXT:    fcmeq.4s v3, v1, v1
+; CHECK-NEXT:    orr.4s v1, #64, lsl #16
+; CHECK-NEXT:    bit.16b v1, v2, v3
+; CHECK-NEXT:    mvni.4h v2, #128, lsl #8
+; CHECK-NEXT:    shrn.4h v1, v1, #16
+; CHECK-NEXT:    bif.8b v0, v1, v2
 ; CHECK-NEXT:    ret
   %tmp0 = fptrunc <4 x double> %b to <4 x bfloat>
   %r = call <4 x bfloat> @llvm.copysign.v4bf16(<4 x bfloat> %a, <4 x bfloat> %tmp0)
-- 
GitLab


From edc1c3d24e6f8ed548340ce0369138fb40427a24 Mon Sep 17 00:00:00 2001
From: David Majnemer 
Date: Fri, 8 Mar 2024 18:22:40 +0000
Subject: [PATCH 0077/1674] [AArch64] Make more vector f16 operations legal

v8f16 is a legal type but promoting to v16f16 would result in an illegal
type.

Let's legalize these by a combination of splitting+promoting resulting
in a pair of v4f16.

Also, we were being overly cautious with different v4f16 nodes. Mark
more of them safe to promote to v4f32.
---
 .../Target/AArch64/AArch64ISelLowering.cpp    |  144 +-
 llvm/lib/Target/AArch64/AArch64InstrInfo.td   |   91 ++
 llvm/test/CodeGen/AArch64/faddp-half.ll       |  218 +--
 llvm/test/CodeGen/AArch64/faddsub.ll          |  496 +-----
 llvm/test/CodeGen/AArch64/fcvt.ll             | 1421 +++--------------
 llvm/test/CodeGen/AArch64/fcvt_combine.ll     |  140 +-
 llvm/test/CodeGen/AArch64/fdiv.ll             |  247 +--
 llvm/test/CodeGen/AArch64/fmla.ll             |  879 ++--------
 llvm/test/CodeGen/AArch64/fmul.ll             |  248 +--
 .../CodeGen/AArch64/fp16-v8-instructions.ll   |  248 +--
 .../test/CodeGen/AArch64/sve-fp-reciprocal.ll |    5 -
 llvm/test/CodeGen/AArch64/vecreduce-fadd.ll   |  196 +--
 llvm/test/CodeGen/AArch64/vecreduce-fmul.ll   |  196 +--
 13 files changed, 864 insertions(+), 3665 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 89b697b2d515..054311d39e7b 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -701,43 +701,45 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
   }
 
   auto LegalizeNarrowFP = [this](MVT ScalarVT) {
-    for (auto Op : {ISD::SETCC,
-                    ISD::SELECT_CC,
-                    ISD::BR_CC,
-                    ISD::FADD,
-                    ISD::FSUB,
-                    ISD::FMUL,
-                    ISD::FDIV,
-                    ISD::FMA,
-                    ISD::FCEIL,
-                    ISD::FSQRT,
-                    ISD::FFLOOR,
-                    ISD::FNEARBYINT,
-                    ISD::FRINT,
-                    ISD::FROUND,
-                    ISD::FROUNDEVEN,
-                    ISD::FTRUNC,
-                    ISD::FMINNUM,
-                    ISD::FMAXNUM,
-                    ISD::FMINIMUM,
-                    ISD::FMAXIMUM,
-                    ISD::STRICT_FADD,
-                    ISD::STRICT_FSUB,
-                    ISD::STRICT_FMUL,
-                    ISD::STRICT_FDIV,
-                    ISD::STRICT_FMA,
-                    ISD::STRICT_FCEIL,
-                    ISD::STRICT_FFLOOR,
-                    ISD::STRICT_FSQRT,
-                    ISD::STRICT_FRINT,
-                    ISD::STRICT_FNEARBYINT,
-                    ISD::STRICT_FROUND,
-                    ISD::STRICT_FTRUNC,
-                    ISD::STRICT_FROUNDEVEN,
-                    ISD::STRICT_FMINNUM,
-                    ISD::STRICT_FMAXNUM,
-                    ISD::STRICT_FMINIMUM,
-                    ISD::STRICT_FMAXIMUM})
+    for (auto Op : {
+             ISD::SETCC,
+             ISD::SELECT_CC,
+             ISD::BR_CC,
+             ISD::FADD,
+             ISD::FSUB,
+             ISD::FMUL,
+             ISD::FDIV,
+             ISD::FMA,
+             ISD::FCEIL,
+             ISD::FSQRT,
+             ISD::FFLOOR,
+             ISD::FNEARBYINT,
+             ISD::FRINT,
+             ISD::FROUND,
+             ISD::FROUNDEVEN,
+             ISD::FTRUNC,
+             ISD::FMINNUM,
+             ISD::FMAXNUM,
+             ISD::FMINIMUM,
+             ISD::FMAXIMUM,
+             ISD::STRICT_FADD,
+             ISD::STRICT_FSUB,
+             ISD::STRICT_FMUL,
+             ISD::STRICT_FDIV,
+             ISD::STRICT_FMA,
+             ISD::STRICT_FCEIL,
+             ISD::STRICT_FFLOOR,
+             ISD::STRICT_FSQRT,
+             ISD::STRICT_FRINT,
+             ISD::STRICT_FNEARBYINT,
+             ISD::STRICT_FROUND,
+             ISD::STRICT_FTRUNC,
+             ISD::STRICT_FROUNDEVEN,
+             ISD::STRICT_FMINNUM,
+             ISD::STRICT_FMAXNUM,
+             ISD::STRICT_FMINIMUM,
+             ISD::STRICT_FMAXIMUM,
+         })
       setOperationAction(Op, ScalarVT, Promote);
 
     for (auto Op : {ISD::FNEG, ISD::FABS})
@@ -752,45 +754,45 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
 
     // promote v4f16 to v4f32 when that is known to be safe.
     auto V4Narrow = MVT::getVectorVT(ScalarVT, 4);
-    setOperationPromotedToType(ISD::FADD, V4Narrow, MVT::v4f32);
-    setOperationPromotedToType(ISD::FSUB, V4Narrow, MVT::v4f32);
-    setOperationPromotedToType(ISD::FMUL, V4Narrow, MVT::v4f32);
-    setOperationPromotedToType(ISD::FDIV, V4Narrow, MVT::v4f32);
-
-    setOperationAction(ISD::FABS, V4Narrow, Legal);
-    setOperationAction(ISD::FNEG, V4Narrow, Legal);
-    setOperationAction(ISD::FROUND,      V4Narrow, Expand);
-    setOperationAction(ISD::FROUNDEVEN,  V4Narrow, Expand);
+    setOperationPromotedToType(ISD::FADD,       V4Narrow, MVT::v4f32);
+    setOperationPromotedToType(ISD::FSUB,       V4Narrow, MVT::v4f32);
+    setOperationPromotedToType(ISD::FMUL,       V4Narrow, MVT::v4f32);
+    setOperationPromotedToType(ISD::FDIV,       V4Narrow, MVT::v4f32);
+    setOperationPromotedToType(ISD::FCEIL,      V4Narrow, MVT::v4f32);
+    setOperationPromotedToType(ISD::FFLOOR,     V4Narrow, MVT::v4f32);
+    setOperationPromotedToType(ISD::FROUND,     V4Narrow, MVT::v4f32);
+    setOperationPromotedToType(ISD::FTRUNC,     V4Narrow, MVT::v4f32);
+    setOperationPromotedToType(ISD::FROUNDEVEN, V4Narrow, MVT::v4f32);
+    setOperationPromotedToType(ISD::FRINT,      V4Narrow, MVT::v4f32);
+    setOperationPromotedToType(ISD::FNEARBYINT, V4Narrow, MVT::v4f32);
+
+    setOperationAction(ISD::FABS,        V4Narrow, Legal);
+    setOperationAction(ISD::FNEG, 	 V4Narrow, Legal);
     setOperationAction(ISD::FMA,         V4Narrow, Expand);
     setOperationAction(ISD::SETCC,       V4Narrow, Custom);
     setOperationAction(ISD::BR_CC,       V4Narrow, Expand);
     setOperationAction(ISD::SELECT,      V4Narrow, Expand);
     setOperationAction(ISD::SELECT_CC,   V4Narrow, Expand);
-    setOperationAction(ISD::FTRUNC,      V4Narrow, Expand);
-    setOperationAction(ISD::FCOPYSIGN, V4Narrow, Custom);
-    setOperationAction(ISD::FFLOOR,      V4Narrow, Expand);
-    setOperationAction(ISD::FCEIL,       V4Narrow, Expand);
-    setOperationAction(ISD::FRINT,       V4Narrow, Expand);
-    setOperationAction(ISD::FNEARBYINT,  V4Narrow, Expand);
+    setOperationAction(ISD::FCOPYSIGN,   V4Narrow, Custom);
     setOperationAction(ISD::FSQRT,       V4Narrow, Expand);
 
     auto V8Narrow = MVT::getVectorVT(ScalarVT, 8);
-    setOperationAction(ISD::FABS, V8Narrow, Legal);
-    setOperationAction(ISD::FADD,        V8Narrow, Expand);
-    setOperationAction(ISD::FCEIL,       V8Narrow, Expand);
-    setOperationAction(ISD::FCOPYSIGN, V8Narrow, Custom);
-    setOperationAction(ISD::FDIV,        V8Narrow, Expand);
-    setOperationAction(ISD::FFLOOR,      V8Narrow, Expand);
+    setOperationAction(ISD::FABS,        V8Narrow, Legal);
+    setOperationAction(ISD::FADD,        V8Narrow, Legal);
+    setOperationAction(ISD::FCEIL,       V8Narrow, Legal);
+    setOperationAction(ISD::FCOPYSIGN,   V8Narrow, Custom);
+    setOperationAction(ISD::FDIV,        V8Narrow, Legal);
+    setOperationAction(ISD::FFLOOR,      V8Narrow, Legal);
     setOperationAction(ISD::FMA,         V8Narrow, Expand);
-    setOperationAction(ISD::FMUL,        V8Narrow, Expand);
-    setOperationAction(ISD::FNEARBYINT,  V8Narrow, Expand);
-    setOperationAction(ISD::FNEG, V8Narrow, Legal);
-    setOperationAction(ISD::FROUND,      V8Narrow, Expand);
-    setOperationAction(ISD::FROUNDEVEN,  V8Narrow, Expand);
-    setOperationAction(ISD::FRINT,       V8Narrow, Expand);
+    setOperationAction(ISD::FMUL,        V8Narrow, Legal);
+    setOperationAction(ISD::FNEARBYINT,  V8Narrow, Legal);
+    setOperationAction(ISD::FNEG, 	 V8Narrow, Legal);
+    setOperationAction(ISD::FROUND,      V8Narrow, Legal);
+    setOperationAction(ISD::FROUNDEVEN,  V8Narrow, Legal);
+    setOperationAction(ISD::FRINT,       V8Narrow, Legal);
     setOperationAction(ISD::FSQRT,       V8Narrow, Expand);
-    setOperationAction(ISD::FSUB,        V8Narrow, Expand);
-    setOperationAction(ISD::FTRUNC,      V8Narrow, Expand);
+    setOperationAction(ISD::FSUB,        V8Narrow, Legal);
+    setOperationAction(ISD::FTRUNC,      V8Narrow, Legal);
     setOperationAction(ISD::SETCC,       V8Narrow, Expand);
     setOperationAction(ISD::BR_CC,       V8Narrow, Expand);
     setOperationAction(ISD::SELECT,      V8Narrow, Expand);
@@ -10593,13 +10595,19 @@ static SDValue getEstimate(const AArch64Subtarget *ST, unsigned Opcode,
         VT == MVT::v4f32)) ||
       (ST->hasSVE() &&
        (VT == MVT::nxv8f16 || VT == MVT::nxv4f32 || VT == MVT::nxv2f64))) {
-    if (ExtraSteps == TargetLoweringBase::ReciprocalEstimate::Unspecified)
+    if (ExtraSteps == TargetLoweringBase::ReciprocalEstimate::Unspecified) {
       // For the reciprocal estimates, convergence is quadratic, so the number
       // of digits is doubled after each iteration.  In ARMv8, the accuracy of
       // the initial estimate is 2^-8.  Thus the number of extra steps to refine
       // the result for float (23 mantissa bits) is 2 and for double (52
       // mantissa bits) is 3.
-      ExtraSteps = VT.getScalarType() == MVT::f64 ? 3 : 2;
+      constexpr unsigned AccurateBits = 8;
+      unsigned DesiredBits =
+          APFloat::semanticsPrecision(DAG.EVTToAPFloatSemantics(VT));
+      ExtraSteps = DesiredBits <= AccurateBits
+                       ? 0
+                       : Log2_64_Ceil(DesiredBits) - Log2_64_Ceil(AccurateBits);
+    }
 
     return DAG.getNode(Opcode, SDLoc(Operand), VT, Operand);
   }
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index 3c67f616c1b9..6254e68326f7 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -128,6 +128,7 @@ def HasRDM           : Predicate<"Subtarget->hasRDM()">,
                                  AssemblerPredicateWithAll<(all_of FeatureRDM), "rdm">;
 def HasFullFP16      : Predicate<"Subtarget->hasFullFP16()">,
                                  AssemblerPredicateWithAll<(all_of FeatureFullFP16), "fullfp16">;
+def HasNoFullFP16    : Predicate<"!Subtarget->hasFullFP16()">;
 def HasFP16FML       : Predicate<"Subtarget->hasFP16FML()">,
                                  AssemblerPredicateWithAll<(all_of FeatureFP16FML), "fp16fml">;
 def HasSPE           : Predicate<"Subtarget->hasSPE()">,
@@ -254,6 +255,7 @@ def HasTRBE          : Predicate<"Subtarget->hasTRBE()">,
                        AssemblerPredicateWithAll<(all_of FeatureTRBE), "trbe">;
 def HasBF16          : Predicate<"Subtarget->hasBF16()">,
                        AssemblerPredicateWithAll<(all_of FeatureBF16), "bf16">;
+def HasNoBF16        : Predicate<"!Subtarget->hasBF16()">;
 def HasMatMulInt8    : Predicate<"Subtarget->hasMatMulInt8()">,
                        AssemblerPredicateWithAll<(all_of FeatureMatMulInt8), "i8mm">;
 def HasMatMulFP32    : Predicate<"Subtarget->hasMatMulFP32()">,
@@ -764,6 +766,8 @@ def AArch64fcvtxnv: PatFrags<(ops node:$Rn),
                                  [(int_aarch64_neon_fcvtxn node:$Rn),
                                   (AArch64fcvtxn_n node:$Rn)]>;
 
+//def Aarch64softf32tobf16v8: SDNode<"AArch64ISD::", SDTFPRoundOp>;
+
 def AArch64bici: SDNode<"AArch64ISD::BICi", SDT_AArch64vecimm>;
 def AArch64orri: SDNode<"AArch64ISD::ORRi", SDT_AArch64vecimm>;
 
@@ -9739,6 +9743,93 @@ let Predicates = [HasCPA] in {
   def MSUBPT : MulAccumCPA<1, "msubpt">;
 }
 
+def round_v4fp32_to_v4bf16 :
+  OutPatFrag<(ops node:$Rn),
+             // NaN? Round : Quiet(NaN)
+             (BSPv16i8 (FCMEQv4f32 $Rn, $Rn),
+                       (ADDv4i32
+                         (ADDv4i32 $Rn,
+                           // Extract the LSB of the fp32 *truncated* to bf16.
+                           (ANDv16i8 (USHRv4i32_shift V128:$Rn, (i32 16)),
+                                     (MOVIv4i32 (i32 1), (i32 0)))),
+                         // Bias which will help us break ties correctly.
+                         (MOVIv4s_msl (i32 127), (i32 264))),
+                       // Set the quiet bit in the NaN.
+                       (ORRv4i32 $Rn, (i32 64), (i32 16)))>;
+
+multiclass PromoteUnaryv8f16Tov4f32 {
+  let Predicates = [HasNoFullFP16] in
+  def : Pat<(InOp (v8f16 V128:$Rn)),
+            (v8f16 (FCVTNv8i16
+              (INSERT_SUBREG (IMPLICIT_DEF),
+                             (v4f16 (FCVTNv4i16
+                               (v4f32 (OutInst
+                                 (v4f32 (FCVTLv4i16 (v4i16 (EXTRACT_SUBREG V128:$Rn, dsub)))))))),
+               dsub),
+              (v4f32 (OutInst (v4f32 (FCVTLv8i16 V128:$Rn))))))>;
+
+  let Predicates = [HasBF16] in
+  def : Pat<(InOp (v8bf16 V128:$Rn)),
+            (v8bf16 (BFCVTN2
+              (v8bf16 (BFCVTN
+                (v4f32 (OutInst
+                  (v4f32 (SHLLv4i16 (v4i16 (EXTRACT_SUBREG V128:$Rn, dsub)))))))),
+              (v4f32 (OutInst (v4f32 (SHLLv8i16 V128:$Rn))))))>;
+
+  let Predicates = [HasNoBF16] in
+  def : Pat<(InOp (v8bf16 V128:$Rn)),
+            (UZP2v8i16
+              (round_v4fp32_to_v4bf16 (v4f32 (OutInst
+                  (v4f32 (SHLLv4i16 (v4i16 (EXTRACT_SUBREG V128:$Rn, dsub))))))),
+              (round_v4fp32_to_v4bf16 (v4f32 (OutInst
+                  (v4f32 (SHLLv8i16 V128:$Rn))))))>;
+}
+defm : PromoteUnaryv8f16Tov4f32;
+defm : PromoteUnaryv8f16Tov4f32;
+defm : PromoteUnaryv8f16Tov4f32;
+defm : PromoteUnaryv8f16Tov4f32;
+defm : PromoteUnaryv8f16Tov4f32;
+defm : PromoteUnaryv8f16Tov4f32;
+defm : PromoteUnaryv8f16Tov4f32;
+
+multiclass PromoteBinaryv8f16Tov4f32 {
+  let Predicates = [HasNoFullFP16] in
+  def : Pat<(InOp (v8f16 V128:$Rn), (v8f16 V128:$Rm)),
+            (v8f16 (FCVTNv8i16
+              (INSERT_SUBREG (IMPLICIT_DEF),
+                             (v4f16 (FCVTNv4i16
+                               (v4f32 (OutInst
+                                 (v4f32 (FCVTLv4i16 (v4i16 (EXTRACT_SUBREG V128:$Rn, dsub)))),
+                                 (v4f32 (FCVTLv4i16 (v4i16 (EXTRACT_SUBREG V128:$Rm, dsub)))))))),
+               dsub),
+              (v4f32 (OutInst (v4f32 (FCVTLv8i16 V128:$Rn)),
+                              (v4f32 (FCVTLv8i16 V128:$Rm))))))>;
+
+  let Predicates = [HasBF16] in
+  def : Pat<(InOp (v8bf16 V128:$Rn), (v8bf16 V128:$Rm)),
+            (v8bf16 (BFCVTN2
+              (v8bf16 (BFCVTN
+                (v4f32 (OutInst
+                  (v4f32 (SHLLv4i16 (v4i16 (EXTRACT_SUBREG V128:$Rn, dsub)))),
+                  (v4f32 (SHLLv4i16 (v4i16 (EXTRACT_SUBREG V128:$Rm, dsub)))))))),
+              (v4f32 (OutInst (v4f32 (SHLLv8i16 V128:$Rn)),
+                              (v4f32 (SHLLv8i16 V128:$Rm))))))>;
+
+  let Predicates = [HasNoBF16] in
+  def : Pat<(InOp (v8bf16 V128:$Rn), (v8bf16 V128:$Rm)),
+            (UZP2v8i16
+              (round_v4fp32_to_v4bf16 (v4f32 (OutInst
+                  (v4f32 (SHLLv4i16 (v4i16 (EXTRACT_SUBREG V128:$Rn, dsub)))),
+                  (v4f32 (SHLLv4i16 (v4i16 (EXTRACT_SUBREG V128:$Rm, dsub))))))),
+              (round_v4fp32_to_v4bf16 (v4f32 (OutInst
+                  (v4f32 (SHLLv8i16 V128:$Rn)),
+                  (v4f32 (SHLLv8i16 V128:$Rm))))))>;
+}
+defm : PromoteBinaryv8f16Tov4f32;
+defm : PromoteBinaryv8f16Tov4f32;
+defm : PromoteBinaryv8f16Tov4f32;
+defm : PromoteBinaryv8f16Tov4f32;
+
 include "AArch64InstrAtomics.td"
 include "AArch64SVEInstrInfo.td"
 include "AArch64SMEInstrInfo.td"
diff --git a/llvm/test/CodeGen/AArch64/faddp-half.ll b/llvm/test/CodeGen/AArch64/faddp-half.ll
index 6068a4742eea..447476e76ec6 100644
--- a/llvm/test/CodeGen/AArch64/faddp-half.ll
+++ b/llvm/test/CodeGen/AArch64/faddp-half.ll
@@ -107,10 +107,15 @@ define half @faddp_8xhalf(<8 x half> %a) {
 ; CHECKNOFP16-LABEL: faddp_8xhalf:
 ; CHECKNOFP16:       // %bb.0: // %entry
 ; CHECKNOFP16-NEXT:    dup v1.8h, v0.h[1]
-; CHECKNOFP16-NEXT:    fcvt s0, h0
-; CHECKNOFP16-NEXT:    fcvt s1, h1
-; CHECKNOFP16-NEXT:    fadd s0, s0, s1
-; CHECKNOFP16-NEXT:    fcvt h0, s0
+; CHECKNOFP16-NEXT:    fcvtl v2.4s, v0.4h
+; CHECKNOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECKNOFP16-NEXT:    fcvtl v3.4s, v1.4h
+; CHECKNOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECKNOFP16-NEXT:    fadd v2.4s, v2.4s, v3.4s
+; CHECKNOFP16-NEXT:    fadd v1.4s, v0.4s, v1.4s
+; CHECKNOFP16-NEXT:    fcvtn v0.4h, v2.4s
+; CHECKNOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
+; CHECKNOFP16-NEXT:    // kill: def $h0 killed $h0 killed $q0
 ; CHECKNOFP16-NEXT:    ret
 entry:
   %shift = shufflevector <8 x half> %a, <8 x half> undef, <8 x i32> 
@@ -128,10 +133,15 @@ define half @faddp_8xhalf_commute(<8 x half> %a) {
 ; CHECKNOFP16-LABEL: faddp_8xhalf_commute:
 ; CHECKNOFP16:       // %bb.0: // %entry
 ; CHECKNOFP16-NEXT:    dup v1.8h, v0.h[1]
-; CHECKNOFP16-NEXT:    fcvt s0, h0
-; CHECKNOFP16-NEXT:    fcvt s1, h1
-; CHECKNOFP16-NEXT:    fadd s0, s1, s0
-; CHECKNOFP16-NEXT:    fcvt h0, s0
+; CHECKNOFP16-NEXT:    fcvtl v2.4s, v0.4h
+; CHECKNOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECKNOFP16-NEXT:    fcvtl v3.4s, v1.4h
+; CHECKNOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECKNOFP16-NEXT:    fadd v2.4s, v3.4s, v2.4s
+; CHECKNOFP16-NEXT:    fadd v1.4s, v1.4s, v0.4s
+; CHECKNOFP16-NEXT:    fcvtn v0.4h, v2.4s
+; CHECKNOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
+; CHECKNOFP16-NEXT:    // kill: def $h0 killed $h0 killed $q0
 ; CHECKNOFP16-NEXT:    ret
 entry:
   %shift = shufflevector <8 x half> %a, <8 x half> undef, <8 x i32> 
@@ -149,61 +159,15 @@ define <8 x half> @addp_v8f16(<8 x half> %a) {
 ;
 ; CHECKNOFP16-LABEL: addp_v8f16:
 ; CHECKNOFP16:       // %bb.0: // %entry
-; CHECKNOFP16-NEXT:    rev32 v2.8h, v0.8h
-; CHECKNOFP16-NEXT:    mov h1, v0.h[1]
-; CHECKNOFP16-NEXT:    fcvt s4, h0
-; CHECKNOFP16-NEXT:    mov h5, v0.h[2]
-; CHECKNOFP16-NEXT:    mov h16, v0.h[3]
-; CHECKNOFP16-NEXT:    mov h3, v2.h[1]
-; CHECKNOFP16-NEXT:    fcvt s6, h2
-; CHECKNOFP16-NEXT:    fcvt s1, h1
-; CHECKNOFP16-NEXT:    mov h7, v2.h[2]
-; CHECKNOFP16-NEXT:    fcvt s5, h5
-; CHECKNOFP16-NEXT:    fcvt s16, h16
-; CHECKNOFP16-NEXT:    fcvt s3, h3
-; CHECKNOFP16-NEXT:    fadd s4, s6, s4
-; CHECKNOFP16-NEXT:    mov h6, v2.h[3]
-; CHECKNOFP16-NEXT:    fcvt s7, h7
-; CHECKNOFP16-NEXT:    fadd s3, s3, s1
-; CHECKNOFP16-NEXT:    fcvt s6, h6
-; CHECKNOFP16-NEXT:    fcvt h1, s4
-; CHECKNOFP16-NEXT:    fadd s4, s7, s5
-; CHECKNOFP16-NEXT:    mov h5, v0.h[4]
-; CHECKNOFP16-NEXT:    mov h7, v2.h[4]
-; CHECKNOFP16-NEXT:    fcvt h3, s3
-; CHECKNOFP16-NEXT:    fadd s6, s6, s16
-; CHECKNOFP16-NEXT:    mov h16, v2.h[5]
-; CHECKNOFP16-NEXT:    fcvt h4, s4
-; CHECKNOFP16-NEXT:    mov v1.h[1], v3.h[0]
-; CHECKNOFP16-NEXT:    fcvt s3, h5
-; CHECKNOFP16-NEXT:    fcvt s5, h7
-; CHECKNOFP16-NEXT:    mov h7, v0.h[5]
-; CHECKNOFP16-NEXT:    fcvt h6, s6
-; CHECKNOFP16-NEXT:    fcvt s16, h16
-; CHECKNOFP16-NEXT:    mov v1.h[2], v4.h[0]
-; CHECKNOFP16-NEXT:    mov h4, v0.h[6]
-; CHECKNOFP16-NEXT:    fadd s3, s5, s3
-; CHECKNOFP16-NEXT:    mov h5, v2.h[6]
-; CHECKNOFP16-NEXT:    fcvt s7, h7
-; CHECKNOFP16-NEXT:    mov h0, v0.h[7]
-; CHECKNOFP16-NEXT:    mov h2, v2.h[7]
-; CHECKNOFP16-NEXT:    mov v1.h[3], v6.h[0]
-; CHECKNOFP16-NEXT:    fcvt s4, h4
-; CHECKNOFP16-NEXT:    fcvt h3, s3
-; CHECKNOFP16-NEXT:    fcvt s5, h5
-; CHECKNOFP16-NEXT:    fadd s6, s16, s7
-; CHECKNOFP16-NEXT:    fcvt s0, h0
-; CHECKNOFP16-NEXT:    fcvt s2, h2
-; CHECKNOFP16-NEXT:    mov v1.h[4], v3.h[0]
-; CHECKNOFP16-NEXT:    fadd s4, s5, s4
-; CHECKNOFP16-NEXT:    fcvt h3, s6
-; CHECKNOFP16-NEXT:    fadd s0, s2, s0
-; CHECKNOFP16-NEXT:    mov v1.h[5], v3.h[0]
-; CHECKNOFP16-NEXT:    fcvt h3, s4
-; CHECKNOFP16-NEXT:    fcvt h0, s0
-; CHECKNOFP16-NEXT:    mov v1.h[6], v3.h[0]
-; CHECKNOFP16-NEXT:    mov v1.h[7], v0.h[0]
-; CHECKNOFP16-NEXT:    mov v0.16b, v1.16b
+; CHECKNOFP16-NEXT:    rev32 v1.8h, v0.8h
+; CHECKNOFP16-NEXT:    fcvtl v2.4s, v0.4h
+; CHECKNOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECKNOFP16-NEXT:    fcvtl v3.4s, v1.4h
+; CHECKNOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECKNOFP16-NEXT:    fadd v2.4s, v3.4s, v2.4s
+; CHECKNOFP16-NEXT:    fadd v1.4s, v1.4s, v0.4s
+; CHECKNOFP16-NEXT:    fcvtn v0.4h, v2.4s
+; CHECKNOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECKNOFP16-NEXT:    ret
 entry:
   %s = shufflevector <8 x half> %a, <8 x half> poison, <8 x i32> 
@@ -221,116 +185,24 @@ define <16 x half> @addp_v16f16(<16 x half> %a) {
 ;
 ; CHECKNOFP16-LABEL: addp_v16f16:
 ; CHECKNOFP16:       // %bb.0: // %entry
-; CHECKNOFP16-NEXT:    rev32 v5.8h, v0.8h
-; CHECKNOFP16-NEXT:    rev32 v4.8h, v1.8h
-; CHECKNOFP16-NEXT:    mov h3, v0.h[1]
-; CHECKNOFP16-NEXT:    mov h6, v1.h[1]
-; CHECKNOFP16-NEXT:    fcvt s16, h0
-; CHECKNOFP16-NEXT:    mov h17, v0.h[2]
-; CHECKNOFP16-NEXT:    fcvt s20, h1
-; CHECKNOFP16-NEXT:    mov h21, v1.h[2]
-; CHECKNOFP16-NEXT:    mov h2, v5.h[1]
-; CHECKNOFP16-NEXT:    mov h7, v4.h[1]
-; CHECKNOFP16-NEXT:    fcvt s3, h3
-; CHECKNOFP16-NEXT:    fcvt s18, h5
-; CHECKNOFP16-NEXT:    mov h19, v5.h[2]
-; CHECKNOFP16-NEXT:    fcvt s6, h6
-; CHECKNOFP16-NEXT:    fcvt s22, h4
-; CHECKNOFP16-NEXT:    mov h23, v4.h[2]
-; CHECKNOFP16-NEXT:    fcvt s17, h17
-; CHECKNOFP16-NEXT:    mov h24, v5.h[3]
-; CHECKNOFP16-NEXT:    fcvt s21, h21
-; CHECKNOFP16-NEXT:    mov h25, v4.h[6]
-; CHECKNOFP16-NEXT:    fcvt s2, h2
-; CHECKNOFP16-NEXT:    fcvt s7, h7
-; CHECKNOFP16-NEXT:    fadd s16, s18, s16
-; CHECKNOFP16-NEXT:    fcvt s18, h19
-; CHECKNOFP16-NEXT:    mov h19, v0.h[3]
-; CHECKNOFP16-NEXT:    fadd s20, s22, s20
-; CHECKNOFP16-NEXT:    fcvt s22, h23
-; CHECKNOFP16-NEXT:    mov h23, v4.h[3]
-; CHECKNOFP16-NEXT:    fadd s3, s2, s3
-; CHECKNOFP16-NEXT:    fadd s6, s7, s6
-; CHECKNOFP16-NEXT:    mov h7, v1.h[3]
-; CHECKNOFP16-NEXT:    fcvt h2, s16
-; CHECKNOFP16-NEXT:    fadd s16, s18, s17
-; CHECKNOFP16-NEXT:    fcvt s18, h19
-; CHECKNOFP16-NEXT:    fcvt s19, h24
-; CHECKNOFP16-NEXT:    mov h24, v5.h[6]
-; CHECKNOFP16-NEXT:    fcvt h17, s3
-; CHECKNOFP16-NEXT:    fcvt h3, s20
-; CHECKNOFP16-NEXT:    fadd s20, s22, s21
-; CHECKNOFP16-NEXT:    fcvt h6, s6
-; CHECKNOFP16-NEXT:    fcvt s7, h7
-; CHECKNOFP16-NEXT:    fcvt s22, h23
-; CHECKNOFP16-NEXT:    mov h21, v0.h[4]
-; CHECKNOFP16-NEXT:    mov h23, v5.h[4]
-; CHECKNOFP16-NEXT:    fcvt h16, s16
-; CHECKNOFP16-NEXT:    fadd s18, s19, s18
-; CHECKNOFP16-NEXT:    mov h19, v4.h[4]
-; CHECKNOFP16-NEXT:    mov v2.h[1], v17.h[0]
-; CHECKNOFP16-NEXT:    mov h17, v1.h[4]
-; CHECKNOFP16-NEXT:    mov v3.h[1], v6.h[0]
-; CHECKNOFP16-NEXT:    fcvt h6, s20
-; CHECKNOFP16-NEXT:    fadd s7, s22, s7
-; CHECKNOFP16-NEXT:    fcvt s20, h21
-; CHECKNOFP16-NEXT:    mov h21, v0.h[5]
-; CHECKNOFP16-NEXT:    mov h22, v5.h[5]
-; CHECKNOFP16-NEXT:    fcvt h18, s18
-; CHECKNOFP16-NEXT:    fcvt s19, h19
-; CHECKNOFP16-NEXT:    mov h5, v5.h[7]
-; CHECKNOFP16-NEXT:    mov v2.h[2], v16.h[0]
-; CHECKNOFP16-NEXT:    fcvt s16, h23
-; CHECKNOFP16-NEXT:    fcvt s17, h17
-; CHECKNOFP16-NEXT:    mov v3.h[2], v6.h[0]
-; CHECKNOFP16-NEXT:    fcvt h6, s7
-; CHECKNOFP16-NEXT:    mov h7, v1.h[5]
-; CHECKNOFP16-NEXT:    mov h23, v4.h[5]
-; CHECKNOFP16-NEXT:    mov h4, v4.h[7]
-; CHECKNOFP16-NEXT:    fcvt s5, h5
-; CHECKNOFP16-NEXT:    fadd s16, s16, s20
-; CHECKNOFP16-NEXT:    mov h20, v0.h[6]
-; CHECKNOFP16-NEXT:    fadd s17, s19, s17
-; CHECKNOFP16-NEXT:    mov h19, v1.h[6]
-; CHECKNOFP16-NEXT:    mov v2.h[3], v18.h[0]
-; CHECKNOFP16-NEXT:    fcvt s18, h21
-; CHECKNOFP16-NEXT:    fcvt s21, h22
-; CHECKNOFP16-NEXT:    mov v3.h[3], v6.h[0]
-; CHECKNOFP16-NEXT:    fcvt s6, h7
-; CHECKNOFP16-NEXT:    fcvt s7, h23
-; CHECKNOFP16-NEXT:    fcvt s22, h24
-; CHECKNOFP16-NEXT:    fcvt s23, h25
-; CHECKNOFP16-NEXT:    fcvt h16, s16
-; CHECKNOFP16-NEXT:    fcvt s20, h20
-; CHECKNOFP16-NEXT:    fcvt h17, s17
-; CHECKNOFP16-NEXT:    fcvt s19, h19
-; CHECKNOFP16-NEXT:    mov h0, v0.h[7]
-; CHECKNOFP16-NEXT:    mov h1, v1.h[7]
-; CHECKNOFP16-NEXT:    fadd s18, s21, s18
-; CHECKNOFP16-NEXT:    fcvt s4, h4
-; CHECKNOFP16-NEXT:    fadd s6, s7, s6
-; CHECKNOFP16-NEXT:    mov v2.h[4], v16.h[0]
-; CHECKNOFP16-NEXT:    fadd s7, s22, s20
-; CHECKNOFP16-NEXT:    mov v3.h[4], v17.h[0]
-; CHECKNOFP16-NEXT:    fadd s16, s23, s19
-; CHECKNOFP16-NEXT:    fcvt s0, h0
-; CHECKNOFP16-NEXT:    fcvt s1, h1
-; CHECKNOFP16-NEXT:    fcvt h17, s18
-; CHECKNOFP16-NEXT:    fcvt h6, s6
-; CHECKNOFP16-NEXT:    fadd s0, s5, s0
-; CHECKNOFP16-NEXT:    fcvt h5, s7
-; CHECKNOFP16-NEXT:    fadd s1, s4, s1
-; CHECKNOFP16-NEXT:    mov v2.h[5], v17.h[0]
-; CHECKNOFP16-NEXT:    mov v3.h[5], v6.h[0]
-; CHECKNOFP16-NEXT:    fcvt h6, s16
-; CHECKNOFP16-NEXT:    fcvt h0, s0
-; CHECKNOFP16-NEXT:    fcvt h1, s1
-; CHECKNOFP16-NEXT:    mov v2.h[6], v5.h[0]
-; CHECKNOFP16-NEXT:    mov v3.h[6], v6.h[0]
-; CHECKNOFP16-NEXT:    mov v2.h[7], v0.h[0]
-; CHECKNOFP16-NEXT:    mov v3.h[7], v1.h[0]
-; CHECKNOFP16-NEXT:    mov v0.16b, v2.16b
-; CHECKNOFP16-NEXT:    mov v1.16b, v3.16b
+; CHECKNOFP16-NEXT:    rev32 v2.8h, v0.8h
+; CHECKNOFP16-NEXT:    rev32 v3.8h, v1.8h
+; CHECKNOFP16-NEXT:    fcvtl v4.4s, v0.4h
+; CHECKNOFP16-NEXT:    fcvtl v6.4s, v1.4h
+; CHECKNOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECKNOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECKNOFP16-NEXT:    fcvtl v5.4s, v2.4h
+; CHECKNOFP16-NEXT:    fcvtl v7.4s, v3.4h
+; CHECKNOFP16-NEXT:    fcvtl2 v2.4s, v2.8h
+; CHECKNOFP16-NEXT:    fcvtl2 v3.4s, v3.8h
+; CHECKNOFP16-NEXT:    fadd v4.4s, v5.4s, v4.4s
+; CHECKNOFP16-NEXT:    fadd v5.4s, v7.4s, v6.4s
+; CHECKNOFP16-NEXT:    fadd v2.4s, v2.4s, v0.4s
+; CHECKNOFP16-NEXT:    fadd v3.4s, v3.4s, v1.4s
+; CHECKNOFP16-NEXT:    fcvtn v0.4h, v4.4s
+; CHECKNOFP16-NEXT:    fcvtn v1.4h, v5.4s
+; CHECKNOFP16-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECKNOFP16-NEXT:    fcvtn2 v1.8h, v3.4s
 ; CHECKNOFP16-NEXT:    ret
 entry:
   %s = shufflevector <16 x half> %a, <16 x half> poison, <16 x i32> 
diff --git a/llvm/test/CodeGen/AArch64/faddsub.ll b/llvm/test/CodeGen/AArch64/faddsub.ll
index 31389f5a77d6..f8970dc9e8d5 100644
--- a/llvm/test/CodeGen/AArch64/faddsub.ll
+++ b/llvm/test/CodeGen/AArch64/faddsub.ll
@@ -169,60 +169,14 @@ entry:
 define <7 x half> @fadd_v7f16(<7 x half> %a, <7 x half> %b) {
 ; CHECK-SD-NOFP16-LABEL: fadd_v7f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h0
-; CHECK-SD-NOFP16-NEXT:    mov h6, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fadd s4, s5, s4
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fadd s3, s3, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s4
-; CHECK-SD-NOFP16-NEXT:    fadd s4, s7, s6
-; CHECK-SD-NOFP16-NEXT:    mov h6, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fadd s5, s5, s16
-; CHECK-SD-NOFP16-NEXT:    mov h16, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[1], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h7
-; CHECK-SD-NOFP16-NEXT:    mov h7, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[2], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    fadd s3, s6, s3
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[3], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h6
-; CHECK-SD-NOFP16-NEXT:    fadd s6, s16, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[4], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fadd s4, s5, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s6
-; CHECK-SD-NOFP16-NEXT:    fadd s0, s0, s1
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[5], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[6], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v2.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v2.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fadd v2.4s, v3.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fadd v1.4s, v0.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: fadd_v7f16:
@@ -309,60 +263,14 @@ entry:
 define <8 x half> @fadd_v8f16(<8 x half> %a, <8 x half> %b) {
 ; CHECK-SD-NOFP16-LABEL: fadd_v8f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h0
-; CHECK-SD-NOFP16-NEXT:    mov h6, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fadd s4, s5, s4
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fadd s3, s3, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s4
-; CHECK-SD-NOFP16-NEXT:    fadd s4, s7, s6
-; CHECK-SD-NOFP16-NEXT:    mov h6, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fadd s5, s5, s16
-; CHECK-SD-NOFP16-NEXT:    mov h16, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[1], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h7
-; CHECK-SD-NOFP16-NEXT:    mov h7, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[2], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    fadd s3, s6, s3
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[3], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h6
-; CHECK-SD-NOFP16-NEXT:    fadd s6, s16, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[4], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fadd s4, s5, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s6
-; CHECK-SD-NOFP16-NEXT:    fadd s0, s0, s1
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[5], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[6], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v2.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v2.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fadd v2.4s, v3.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fadd v1.4s, v0.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: fadd_v8f16:
@@ -394,114 +302,22 @@ entry:
 define <16 x half> @fadd_v16f16(<16 x half> %a, <16 x half> %b) {
 ; CHECK-SD-NOFP16-LABEL: fadd_v16f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h6, v2.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h0
-; CHECK-SD-NOFP16-NEXT:    mov h16, v3.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h18, v2.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h19, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h1
-; CHECK-SD-NOFP16-NEXT:    mov h22, v3.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h23, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    mov h24, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fadd s4, s5, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h17
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h18
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h19
-; CHECK-SD-NOFP16-NEXT:    mov h19, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    fadd s20, s21, s20
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h22
-; CHECK-SD-NOFP16-NEXT:    mov h22, v3.h[3]
-; CHECK-SD-NOFP16-NEXT:    fadd s6, s7, s6
-; CHECK-SD-NOFP16-NEXT:    mov h7, v2.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h25, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    fadd s5, s16, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h23
-; CHECK-SD-NOFP16-NEXT:    mov h23, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    fadd s17, s18, s17
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h19
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt h19, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s20
-; CHECK-SD-NOFP16-NEXT:    fadd s16, s16, s21
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h23
-; CHECK-SD-NOFP16-NEXT:    fcvt h17, s17
-; CHECK-SD-NOFP16-NEXT:    mov h21, v2.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h23, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[1], v6.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h22
-; CHECK-SD-NOFP16-NEXT:    mov h22, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fadd s7, s18, s7
-; CHECK-SD-NOFP16-NEXT:    mov h18, v3.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov v5.h[1], v19.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h16, s16
-; CHECK-SD-NOFP16-NEXT:    fadd s6, s20, s6
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[2], v17.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h21
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h22
-; CHECK-SD-NOFP16-NEXT:    fcvt h7, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h23
-; CHECK-SD-NOFP16-NEXT:    mov h21, v2.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov h22, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov v5.h[2], v16.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v3.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov h23, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s6
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    fadd s17, s19, s17
-; CHECK-SD-NOFP16-NEXT:    mov h19, v2.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[3], v7.h[0]
-; CHECK-SD-NOFP16-NEXT:    fadd s18, s20, s18
-; CHECK-SD-NOFP16-NEXT:    mov h20, v3.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h21
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h22
-; CHECK-SD-NOFP16-NEXT:    fcvt s22, h24
-; CHECK-SD-NOFP16-NEXT:    mov h2, v2.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v5.h[3], v6.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h23
-; CHECK-SD-NOFP16-NEXT:    fcvt h17, s17
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h19
-; CHECK-SD-NOFP16-NEXT:    fcvt s23, h25
-; CHECK-SD-NOFP16-NEXT:    fcvt h18, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h20
-; CHECK-SD-NOFP16-NEXT:    mov h3, v3.h[7]
-; CHECK-SD-NOFP16-NEXT:    fadd s7, s21, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fadd s6, s16, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[4], v17.h[0]
-; CHECK-SD-NOFP16-NEXT:    fadd s16, s22, s19
-; CHECK-SD-NOFP16-NEXT:    mov v5.h[4], v18.h[0]
-; CHECK-SD-NOFP16-NEXT:    fadd s17, s23, s20
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt h7, s7
-; CHECK-SD-NOFP16-NEXT:    fadd s0, s0, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s16
-; CHECK-SD-NOFP16-NEXT:    fadd s1, s1, s3
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[5], v7.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v5.h[5], v6.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s17
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[6], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v5.h[6], v6.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v5.h[7], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v4.16b
-; CHECK-SD-NOFP16-NEXT:    mov v1.16b, v5.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v4.4s, v2.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v5.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v6.4s, v3.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v7.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v2.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v3.4s, v3.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fadd v4.4s, v5.4s, v4.4s
+; CHECK-SD-NOFP16-NEXT:    fadd v5.4s, v7.4s, v6.4s
+; CHECK-SD-NOFP16-NEXT:    fadd v2.4s, v0.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fadd v3.4s, v1.4s, v3.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v4.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v1.4h, v5.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v1.8h, v3.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: fadd_v16f16:
@@ -705,60 +521,14 @@ entry:
 define <7 x half> @fsub_v7f16(<7 x half> %a, <7 x half> %b) {
 ; CHECK-SD-NOFP16-LABEL: fsub_v7f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h0
-; CHECK-SD-NOFP16-NEXT:    mov h6, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fsub s4, s5, s4
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fsub s3, s3, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s4
-; CHECK-SD-NOFP16-NEXT:    fsub s4, s7, s6
-; CHECK-SD-NOFP16-NEXT:    mov h6, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fsub s5, s5, s16
-; CHECK-SD-NOFP16-NEXT:    mov h16, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[1], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h7
-; CHECK-SD-NOFP16-NEXT:    mov h7, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[2], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    fsub s3, s6, s3
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[3], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h6
-; CHECK-SD-NOFP16-NEXT:    fsub s6, s16, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[4], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fsub s4, s5, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s6
-; CHECK-SD-NOFP16-NEXT:    fsub s0, s0, s1
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[5], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[6], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v2.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v2.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fsub v2.4s, v3.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fsub v1.4s, v0.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: fsub_v7f16:
@@ -845,60 +615,14 @@ entry:
 define <8 x half> @fsub_v8f16(<8 x half> %a, <8 x half> %b) {
 ; CHECK-SD-NOFP16-LABEL: fsub_v8f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h0
-; CHECK-SD-NOFP16-NEXT:    mov h6, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fsub s4, s5, s4
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fsub s3, s3, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s4
-; CHECK-SD-NOFP16-NEXT:    fsub s4, s7, s6
-; CHECK-SD-NOFP16-NEXT:    mov h6, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fsub s5, s5, s16
-; CHECK-SD-NOFP16-NEXT:    mov h16, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[1], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h7
-; CHECK-SD-NOFP16-NEXT:    mov h7, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[2], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    fsub s3, s6, s3
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[3], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h6
-; CHECK-SD-NOFP16-NEXT:    fsub s6, s16, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[4], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fsub s4, s5, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s6
-; CHECK-SD-NOFP16-NEXT:    fsub s0, s0, s1
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[5], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[6], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v2.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v2.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fsub v2.4s, v3.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fsub v1.4s, v0.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: fsub_v8f16:
@@ -930,114 +654,22 @@ entry:
 define <16 x half> @fsub_v16f16(<16 x half> %a, <16 x half> %b) {
 ; CHECK-SD-NOFP16-LABEL: fsub_v16f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h6, v2.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h0
-; CHECK-SD-NOFP16-NEXT:    mov h16, v3.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h18, v2.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h19, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h1
-; CHECK-SD-NOFP16-NEXT:    mov h22, v3.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h23, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    mov h24, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fsub s4, s5, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h17
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h18
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h19
-; CHECK-SD-NOFP16-NEXT:    mov h19, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    fsub s20, s21, s20
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h22
-; CHECK-SD-NOFP16-NEXT:    mov h22, v3.h[3]
-; CHECK-SD-NOFP16-NEXT:    fsub s6, s7, s6
-; CHECK-SD-NOFP16-NEXT:    mov h7, v2.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h25, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    fsub s5, s16, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h23
-; CHECK-SD-NOFP16-NEXT:    mov h23, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    fsub s17, s18, s17
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h19
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt h19, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s20
-; CHECK-SD-NOFP16-NEXT:    fsub s16, s16, s21
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h23
-; CHECK-SD-NOFP16-NEXT:    fcvt h17, s17
-; CHECK-SD-NOFP16-NEXT:    mov h21, v2.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h23, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[1], v6.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h22
-; CHECK-SD-NOFP16-NEXT:    mov h22, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fsub s7, s18, s7
-; CHECK-SD-NOFP16-NEXT:    mov h18, v3.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov v5.h[1], v19.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h16, s16
-; CHECK-SD-NOFP16-NEXT:    fsub s6, s20, s6
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[2], v17.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h21
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h22
-; CHECK-SD-NOFP16-NEXT:    fcvt h7, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h23
-; CHECK-SD-NOFP16-NEXT:    mov h21, v2.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov h22, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov v5.h[2], v16.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v3.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov h23, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s6
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    fsub s17, s19, s17
-; CHECK-SD-NOFP16-NEXT:    mov h19, v2.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[3], v7.h[0]
-; CHECK-SD-NOFP16-NEXT:    fsub s18, s20, s18
-; CHECK-SD-NOFP16-NEXT:    mov h20, v3.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h21
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h22
-; CHECK-SD-NOFP16-NEXT:    fcvt s22, h24
-; CHECK-SD-NOFP16-NEXT:    mov h2, v2.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v5.h[3], v6.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h23
-; CHECK-SD-NOFP16-NEXT:    fcvt h17, s17
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h19
-; CHECK-SD-NOFP16-NEXT:    fcvt s23, h25
-; CHECK-SD-NOFP16-NEXT:    fcvt h18, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h20
-; CHECK-SD-NOFP16-NEXT:    mov h3, v3.h[7]
-; CHECK-SD-NOFP16-NEXT:    fsub s7, s21, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fsub s6, s16, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[4], v17.h[0]
-; CHECK-SD-NOFP16-NEXT:    fsub s16, s22, s19
-; CHECK-SD-NOFP16-NEXT:    mov v5.h[4], v18.h[0]
-; CHECK-SD-NOFP16-NEXT:    fsub s17, s23, s20
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt h7, s7
-; CHECK-SD-NOFP16-NEXT:    fsub s0, s0, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s16
-; CHECK-SD-NOFP16-NEXT:    fsub s1, s1, s3
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[5], v7.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v5.h[5], v6.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s17
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[6], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v5.h[6], v6.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v5.h[7], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v4.16b
-; CHECK-SD-NOFP16-NEXT:    mov v1.16b, v5.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v4.4s, v2.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v5.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v6.4s, v3.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v7.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v2.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v3.4s, v3.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fsub v4.4s, v5.4s, v4.4s
+; CHECK-SD-NOFP16-NEXT:    fsub v5.4s, v7.4s, v6.4s
+; CHECK-SD-NOFP16-NEXT:    fsub v2.4s, v0.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fsub v3.4s, v1.4s, v3.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v4.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v1.4h, v5.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v1.8h, v3.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: fsub_v16f16:
diff --git a/llvm/test/CodeGen/AArch64/fcvt.ll b/llvm/test/CodeGen/AArch64/fcvt.ll
index 584174d6d064..3b8a22a052b8 100644
--- a/llvm/test/CodeGen/AArch64/fcvt.ll
+++ b/llvm/test/CodeGen/AArch64/fcvt.ll
@@ -148,45 +148,12 @@ entry:
 define <7 x half> @ceil_v7f16(<7 x half> %a) {
 ; CHECK-SD-NOFP16-LABEL: ceil_v7f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h0
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    frintp s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    frintp s5, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
-; CHECK-SD-NOFP16-NEXT:    frintp s2, s3
-; CHECK-SD-NOFP16-NEXT:    frintp s4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[1], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    frintp s5, s5
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[2], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[3], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    frintp s3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[4], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    frintp s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    frintp s0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[5], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[6], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v1.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    frintp v1.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-SD-NOFP16-NEXT:    frintp v1.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: ceil_v7f16:
@@ -234,26 +201,9 @@ entry:
 define <4 x half> @ceil_v4f16(<4 x half> %a) {
 ; CHECK-SD-NOFP16-LABEL: ceil_v4f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NOFP16-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h0
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    frintp s0, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h4
-; CHECK-SD-NOFP16-NEXT:    frintp s1, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    frintp s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[1], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
-; CHECK-SD-NOFP16-NEXT:    frintp s2, s3
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[2], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[3], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    frintp v0.4s, v0.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: ceil_v4f16:
@@ -280,45 +230,12 @@ entry:
 define <8 x half> @ceil_v8f16(<8 x half> %a) {
 ; CHECK-SD-NOFP16-LABEL: ceil_v8f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h0
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    frintp s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    frintp s5, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
-; CHECK-SD-NOFP16-NEXT:    frintp s2, s3
-; CHECK-SD-NOFP16-NEXT:    frintp s4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[1], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    frintp s5, s5
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[2], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[3], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    frintp s3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[4], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    frintp s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    frintp s0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[5], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[6], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v1.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    frintp v1.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-SD-NOFP16-NEXT:    frintp v1.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: ceil_v8f16:
@@ -348,84 +265,18 @@ entry:
 define <16 x half> @ceil_v16f16(<16 x half> %a) {
 ; CHECK-SD-NOFP16-LABEL: ceil_v16f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h0
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h1
-; CHECK-SD-NOFP16-NEXT:    mov h7, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h20, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h21, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    frintp s4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    frintp s6, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    frintp s18, s2
-; CHECK-SD-NOFP16-NEXT:    frintp s19, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s4
-; CHECK-SD-NOFP16-NEXT:    frintp s4, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s6
-; CHECK-SD-NOFP16-NEXT:    frintp s6, s7
-; CHECK-SD-NOFP16-NEXT:    frintp s16, s16
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt h7, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h20
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h21
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt h16, s16
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[1], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    frintp s5, s17
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[1], v7.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    frintp s18, s18
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[2], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    frintp s4, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s5
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[2], v6.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    mov h19, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[3], v16.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[3], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    frintp s7, s7
-; CHECK-SD-NOFP16-NEXT:    frintp s16, s17
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h19
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[4], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[4], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    frintp s4, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s16
-; CHECK-SD-NOFP16-NEXT:    frintp s7, s17
-; CHECK-SD-NOFP16-NEXT:    frintp s0, s0
-; CHECK-SD-NOFP16-NEXT:    frintp s1, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[5], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[5], v6.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[6], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[6], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[7], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v2.16b
-; CHECK-SD-NOFP16-NEXT:    mov v1.16b, v3.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v2.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v4.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v5.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    frintp v2.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    frintp v1.4s, v3.4s
+; CHECK-SD-NOFP16-NEXT:    frintp v3.4s, v5.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    frintp v2.4s, v4.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v1.4h, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v1.8h, v3.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: ceil_v16f16:
@@ -604,45 +455,12 @@ entry:
 define <7 x half> @floor_v7f16(<7 x half> %a) {
 ; CHECK-SD-NOFP16-LABEL: floor_v7f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h0
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    frintm s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    frintm s5, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
-; CHECK-SD-NOFP16-NEXT:    frintm s2, s3
-; CHECK-SD-NOFP16-NEXT:    frintm s4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[1], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    frintm s5, s5
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[2], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[3], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    frintm s3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[4], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    frintm s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    frintm s0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[5], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[6], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v1.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    frintm v1.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-SD-NOFP16-NEXT:    frintm v1.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: floor_v7f16:
@@ -690,26 +508,9 @@ entry:
 define <4 x half> @floor_v4f16(<4 x half> %a) {
 ; CHECK-SD-NOFP16-LABEL: floor_v4f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NOFP16-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h0
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    frintm s0, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h4
-; CHECK-SD-NOFP16-NEXT:    frintm s1, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    frintm s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[1], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
-; CHECK-SD-NOFP16-NEXT:    frintm s2, s3
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[2], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[3], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    frintm v0.4s, v0.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: floor_v4f16:
@@ -736,45 +537,12 @@ entry:
 define <8 x half> @floor_v8f16(<8 x half> %a) {
 ; CHECK-SD-NOFP16-LABEL: floor_v8f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h0
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    frintm s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    frintm s5, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
-; CHECK-SD-NOFP16-NEXT:    frintm s2, s3
-; CHECK-SD-NOFP16-NEXT:    frintm s4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[1], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    frintm s5, s5
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[2], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[3], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    frintm s3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[4], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    frintm s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    frintm s0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[5], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[6], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v1.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    frintm v1.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-SD-NOFP16-NEXT:    frintm v1.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: floor_v8f16:
@@ -804,84 +572,18 @@ entry:
 define <16 x half> @floor_v16f16(<16 x half> %a) {
 ; CHECK-SD-NOFP16-LABEL: floor_v16f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h0
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h1
-; CHECK-SD-NOFP16-NEXT:    mov h7, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h20, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h21, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    frintm s4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    frintm s6, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    frintm s18, s2
-; CHECK-SD-NOFP16-NEXT:    frintm s19, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s4
-; CHECK-SD-NOFP16-NEXT:    frintm s4, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s6
-; CHECK-SD-NOFP16-NEXT:    frintm s6, s7
-; CHECK-SD-NOFP16-NEXT:    frintm s16, s16
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt h7, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h20
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h21
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt h16, s16
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[1], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    frintm s5, s17
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[1], v7.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    frintm s18, s18
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[2], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    frintm s4, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s5
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[2], v6.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    mov h19, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[3], v16.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[3], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    frintm s7, s7
-; CHECK-SD-NOFP16-NEXT:    frintm s16, s17
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h19
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[4], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[4], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    frintm s4, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s16
-; CHECK-SD-NOFP16-NEXT:    frintm s7, s17
-; CHECK-SD-NOFP16-NEXT:    frintm s0, s0
-; CHECK-SD-NOFP16-NEXT:    frintm s1, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[5], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[5], v6.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[6], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[6], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[7], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v2.16b
-; CHECK-SD-NOFP16-NEXT:    mov v1.16b, v3.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v2.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v4.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v5.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    frintm v2.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    frintm v1.4s, v3.4s
+; CHECK-SD-NOFP16-NEXT:    frintm v3.4s, v5.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    frintm v2.4s, v4.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v1.4h, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v1.8h, v3.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: floor_v16f16:
@@ -1060,45 +762,12 @@ entry:
 define <7 x half> @nearbyint_v7f16(<7 x half> %a) {
 ; CHECK-SD-NOFP16-LABEL: nearbyint_v7f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h0
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    frinti s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    frinti s5, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
-; CHECK-SD-NOFP16-NEXT:    frinti s2, s3
-; CHECK-SD-NOFP16-NEXT:    frinti s4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[1], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    frinti s5, s5
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[2], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[3], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    frinti s3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[4], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    frinti s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    frinti s0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[5], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[6], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v1.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    frinti v1.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-SD-NOFP16-NEXT:    frinti v1.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: nearbyint_v7f16:
@@ -1146,26 +815,9 @@ entry:
 define <4 x half> @nearbyint_v4f16(<4 x half> %a) {
 ; CHECK-SD-NOFP16-LABEL: nearbyint_v4f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NOFP16-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h0
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    frinti s0, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h4
-; CHECK-SD-NOFP16-NEXT:    frinti s1, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    frinti s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[1], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
-; CHECK-SD-NOFP16-NEXT:    frinti s2, s3
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[2], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[3], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    frinti v0.4s, v0.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: nearbyint_v4f16:
@@ -1192,45 +844,12 @@ entry:
 define <8 x half> @nearbyint_v8f16(<8 x half> %a) {
 ; CHECK-SD-NOFP16-LABEL: nearbyint_v8f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h0
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    frinti s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    frinti s5, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
-; CHECK-SD-NOFP16-NEXT:    frinti s2, s3
-; CHECK-SD-NOFP16-NEXT:    frinti s4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[1], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    frinti s5, s5
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[2], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[3], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    frinti s3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[4], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    frinti s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    frinti s0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[5], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[6], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v1.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    frinti v1.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-SD-NOFP16-NEXT:    frinti v1.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: nearbyint_v8f16:
@@ -1260,84 +879,18 @@ entry:
 define <16 x half> @nearbyint_v16f16(<16 x half> %a) {
 ; CHECK-SD-NOFP16-LABEL: nearbyint_v16f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h0
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h1
-; CHECK-SD-NOFP16-NEXT:    mov h7, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h20, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h21, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    frinti s4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    frinti s6, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    frinti s18, s2
-; CHECK-SD-NOFP16-NEXT:    frinti s19, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s4
-; CHECK-SD-NOFP16-NEXT:    frinti s4, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s6
-; CHECK-SD-NOFP16-NEXT:    frinti s6, s7
-; CHECK-SD-NOFP16-NEXT:    frinti s16, s16
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt h7, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h20
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h21
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt h16, s16
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[1], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    frinti s5, s17
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[1], v7.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    frinti s18, s18
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[2], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    frinti s4, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s5
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[2], v6.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    mov h19, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[3], v16.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[3], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    frinti s7, s7
-; CHECK-SD-NOFP16-NEXT:    frinti s16, s17
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h19
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[4], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[4], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    frinti s4, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s16
-; CHECK-SD-NOFP16-NEXT:    frinti s7, s17
-; CHECK-SD-NOFP16-NEXT:    frinti s0, s0
-; CHECK-SD-NOFP16-NEXT:    frinti s1, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[5], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[5], v6.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[6], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[6], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[7], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v2.16b
-; CHECK-SD-NOFP16-NEXT:    mov v1.16b, v3.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v2.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v4.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v5.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    frinti v2.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    frinti v1.4s, v3.4s
+; CHECK-SD-NOFP16-NEXT:    frinti v3.4s, v5.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    frinti v2.4s, v4.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v1.4h, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v1.8h, v3.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: nearbyint_v16f16:
@@ -1516,45 +1069,12 @@ entry:
 define <7 x half> @roundeven_v7f16(<7 x half> %a) {
 ; CHECK-SD-NOFP16-LABEL: roundeven_v7f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h0
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    frintn s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    frintn s5, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
-; CHECK-SD-NOFP16-NEXT:    frintn s2, s3
-; CHECK-SD-NOFP16-NEXT:    frintn s4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[1], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    frintn s5, s5
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[2], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[3], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    frintn s3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[4], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    frintn s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    frintn s0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[5], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[6], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v1.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    frintn v1.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-SD-NOFP16-NEXT:    frintn v1.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: roundeven_v7f16:
@@ -1602,26 +1122,9 @@ entry:
 define <4 x half> @roundeven_v4f16(<4 x half> %a) {
 ; CHECK-SD-NOFP16-LABEL: roundeven_v4f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NOFP16-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h0
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    frintn s0, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h4
-; CHECK-SD-NOFP16-NEXT:    frintn s1, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    frintn s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[1], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
-; CHECK-SD-NOFP16-NEXT:    frintn s2, s3
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[2], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[3], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    frintn v0.4s, v0.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: roundeven_v4f16:
@@ -1648,45 +1151,12 @@ entry:
 define <8 x half> @roundeven_v8f16(<8 x half> %a) {
 ; CHECK-SD-NOFP16-LABEL: roundeven_v8f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h0
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    frintn s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    frintn s5, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
-; CHECK-SD-NOFP16-NEXT:    frintn s2, s3
-; CHECK-SD-NOFP16-NEXT:    frintn s4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[1], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    frintn s5, s5
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[2], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[3], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    frintn s3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[4], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    frintn s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    frintn s0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[5], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[6], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v1.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    frintn v1.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-SD-NOFP16-NEXT:    frintn v1.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: roundeven_v8f16:
@@ -1716,84 +1186,18 @@ entry:
 define <16 x half> @roundeven_v16f16(<16 x half> %a) {
 ; CHECK-SD-NOFP16-LABEL: roundeven_v16f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h0
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h1
-; CHECK-SD-NOFP16-NEXT:    mov h7, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h20, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h21, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    frintn s4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    frintn s6, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    frintn s18, s2
-; CHECK-SD-NOFP16-NEXT:    frintn s19, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s4
-; CHECK-SD-NOFP16-NEXT:    frintn s4, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s6
-; CHECK-SD-NOFP16-NEXT:    frintn s6, s7
-; CHECK-SD-NOFP16-NEXT:    frintn s16, s16
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt h7, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h20
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h21
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt h16, s16
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[1], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    frintn s5, s17
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[1], v7.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    frintn s18, s18
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[2], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    frintn s4, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s5
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[2], v6.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    mov h19, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[3], v16.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[3], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    frintn s7, s7
-; CHECK-SD-NOFP16-NEXT:    frintn s16, s17
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h19
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[4], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[4], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    frintn s4, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s16
-; CHECK-SD-NOFP16-NEXT:    frintn s7, s17
-; CHECK-SD-NOFP16-NEXT:    frintn s0, s0
-; CHECK-SD-NOFP16-NEXT:    frintn s1, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[5], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[5], v6.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[6], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[6], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[7], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v2.16b
-; CHECK-SD-NOFP16-NEXT:    mov v1.16b, v3.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v2.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v4.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v5.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    frintn v2.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    frintn v1.4s, v3.4s
+; CHECK-SD-NOFP16-NEXT:    frintn v3.4s, v5.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    frintn v2.4s, v4.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v1.4h, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v1.8h, v3.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: roundeven_v16f16:
@@ -1972,45 +1376,12 @@ entry:
 define <7 x half> @rint_v7f16(<7 x half> %a) {
 ; CHECK-SD-NOFP16-LABEL: rint_v7f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h0
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    frintx s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    frintx s5, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
-; CHECK-SD-NOFP16-NEXT:    frintx s2, s3
-; CHECK-SD-NOFP16-NEXT:    frintx s4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[1], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    frintx s5, s5
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[2], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[3], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    frintx s3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[4], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    frintx s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    frintx s0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[5], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[6], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v1.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    frintx v1.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-SD-NOFP16-NEXT:    frintx v1.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: rint_v7f16:
@@ -2058,26 +1429,9 @@ entry:
 define <4 x half> @rint_v4f16(<4 x half> %a) {
 ; CHECK-SD-NOFP16-LABEL: rint_v4f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NOFP16-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h0
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    frintx s0, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h4
-; CHECK-SD-NOFP16-NEXT:    frintx s1, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    frintx s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[1], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
-; CHECK-SD-NOFP16-NEXT:    frintx s2, s3
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[2], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[3], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    frintx v0.4s, v0.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: rint_v4f16:
@@ -2104,45 +1458,12 @@ entry:
 define <8 x half> @rint_v8f16(<8 x half> %a) {
 ; CHECK-SD-NOFP16-LABEL: rint_v8f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h0
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    frintx s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    frintx s5, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
-; CHECK-SD-NOFP16-NEXT:    frintx s2, s3
-; CHECK-SD-NOFP16-NEXT:    frintx s4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[1], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    frintx s5, s5
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[2], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[3], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    frintx s3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[4], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    frintx s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    frintx s0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[5], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[6], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v1.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    frintx v1.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-SD-NOFP16-NEXT:    frintx v1.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: rint_v8f16:
@@ -2172,84 +1493,18 @@ entry:
 define <16 x half> @rint_v16f16(<16 x half> %a) {
 ; CHECK-SD-NOFP16-LABEL: rint_v16f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h0
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h1
-; CHECK-SD-NOFP16-NEXT:    mov h7, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h20, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h21, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    frintx s4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    frintx s6, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    frintx s18, s2
-; CHECK-SD-NOFP16-NEXT:    frintx s19, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s4
-; CHECK-SD-NOFP16-NEXT:    frintx s4, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s6
-; CHECK-SD-NOFP16-NEXT:    frintx s6, s7
-; CHECK-SD-NOFP16-NEXT:    frintx s16, s16
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt h7, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h20
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h21
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt h16, s16
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[1], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    frintx s5, s17
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[1], v7.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    frintx s18, s18
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[2], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    frintx s4, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s5
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[2], v6.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    mov h19, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[3], v16.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[3], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    frintx s7, s7
-; CHECK-SD-NOFP16-NEXT:    frintx s16, s17
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h19
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[4], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[4], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    frintx s4, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s16
-; CHECK-SD-NOFP16-NEXT:    frintx s7, s17
-; CHECK-SD-NOFP16-NEXT:    frintx s0, s0
-; CHECK-SD-NOFP16-NEXT:    frintx s1, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[5], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[5], v6.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[6], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[6], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[7], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v2.16b
-; CHECK-SD-NOFP16-NEXT:    mov v1.16b, v3.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v2.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v4.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v5.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    frintx v2.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    frintx v1.4s, v3.4s
+; CHECK-SD-NOFP16-NEXT:    frintx v3.4s, v5.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    frintx v2.4s, v4.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v1.4h, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v1.8h, v3.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: rint_v16f16:
@@ -2428,45 +1683,12 @@ entry:
 define <7 x half> @round_v7f16(<7 x half> %a) {
 ; CHECK-SD-NOFP16-LABEL: round_v7f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h0
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    frinta s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    frinta s5, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
-; CHECK-SD-NOFP16-NEXT:    frinta s2, s3
-; CHECK-SD-NOFP16-NEXT:    frinta s4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[1], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    frinta s5, s5
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[2], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[3], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    frinta s3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[4], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    frinta s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    frinta s0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[5], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[6], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v1.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    frinta v1.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-SD-NOFP16-NEXT:    frinta v1.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: round_v7f16:
@@ -2514,26 +1736,9 @@ entry:
 define <4 x half> @round_v4f16(<4 x half> %a) {
 ; CHECK-SD-NOFP16-LABEL: round_v4f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NOFP16-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h0
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    frinta s0, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h4
-; CHECK-SD-NOFP16-NEXT:    frinta s1, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    frinta s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[1], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
-; CHECK-SD-NOFP16-NEXT:    frinta s2, s3
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[2], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[3], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    frinta v0.4s, v0.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: round_v4f16:
@@ -2560,45 +1765,12 @@ entry:
 define <8 x half> @round_v8f16(<8 x half> %a) {
 ; CHECK-SD-NOFP16-LABEL: round_v8f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h0
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    frinta s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    frinta s5, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
-; CHECK-SD-NOFP16-NEXT:    frinta s2, s3
-; CHECK-SD-NOFP16-NEXT:    frinta s4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[1], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    frinta s5, s5
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[2], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[3], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    frinta s3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[4], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    frinta s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    frinta s0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[5], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[6], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v1.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    frinta v1.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-SD-NOFP16-NEXT:    frinta v1.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: round_v8f16:
@@ -2628,84 +1800,18 @@ entry:
 define <16 x half> @round_v16f16(<16 x half> %a) {
 ; CHECK-SD-NOFP16-LABEL: round_v16f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h0
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h1
-; CHECK-SD-NOFP16-NEXT:    mov h7, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h20, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h21, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    frinta s4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    frinta s6, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    frinta s18, s2
-; CHECK-SD-NOFP16-NEXT:    frinta s19, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s4
-; CHECK-SD-NOFP16-NEXT:    frinta s4, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s6
-; CHECK-SD-NOFP16-NEXT:    frinta s6, s7
-; CHECK-SD-NOFP16-NEXT:    frinta s16, s16
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt h7, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h20
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h21
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt h16, s16
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[1], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    frinta s5, s17
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[1], v7.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    frinta s18, s18
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[2], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    frinta s4, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s5
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[2], v6.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    mov h19, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[3], v16.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[3], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    frinta s7, s7
-; CHECK-SD-NOFP16-NEXT:    frinta s16, s17
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h19
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[4], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[4], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    frinta s4, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s16
-; CHECK-SD-NOFP16-NEXT:    frinta s7, s17
-; CHECK-SD-NOFP16-NEXT:    frinta s0, s0
-; CHECK-SD-NOFP16-NEXT:    frinta s1, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[5], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[5], v6.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[6], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[6], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[7], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v2.16b
-; CHECK-SD-NOFP16-NEXT:    mov v1.16b, v3.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v2.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v4.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v5.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    frinta v2.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    frinta v1.4s, v3.4s
+; CHECK-SD-NOFP16-NEXT:    frinta v3.4s, v5.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    frinta v2.4s, v4.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v1.4h, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v1.8h, v3.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: round_v16f16:
@@ -2884,45 +1990,12 @@ entry:
 define <7 x half> @trunc_v7f16(<7 x half> %a) {
 ; CHECK-SD-NOFP16-LABEL: trunc_v7f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h0
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    frintz s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    frintz s5, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
-; CHECK-SD-NOFP16-NEXT:    frintz s2, s3
-; CHECK-SD-NOFP16-NEXT:    frintz s4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[1], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    frintz s5, s5
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[2], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[3], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    frintz s3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[4], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    frintz s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    frintz s0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[5], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[6], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v1.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    frintz v1.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-SD-NOFP16-NEXT:    frintz v1.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: trunc_v7f16:
@@ -2970,26 +2043,9 @@ entry:
 define <4 x half> @trunc_v4f16(<4 x half> %a) {
 ; CHECK-SD-NOFP16-LABEL: trunc_v4f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NOFP16-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h0
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    frintz s0, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h4
-; CHECK-SD-NOFP16-NEXT:    frintz s1, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    frintz s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[1], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
-; CHECK-SD-NOFP16-NEXT:    frintz s2, s3
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[2], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[3], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    frintz v0.4s, v0.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: trunc_v4f16:
@@ -3016,45 +2072,12 @@ entry:
 define <8 x half> @trunc_v8f16(<8 x half> %a) {
 ; CHECK-SD-NOFP16-LABEL: trunc_v8f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h0
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    frintz s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    frintz s5, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
-; CHECK-SD-NOFP16-NEXT:    frintz s2, s3
-; CHECK-SD-NOFP16-NEXT:    frintz s4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[1], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    frintz s5, s5
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[2], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[3], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    frintz s3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[4], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    frintz s2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    frintz s0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[5], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[6], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v1.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    frintz v1.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-SD-NOFP16-NEXT:    frintz v1.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: trunc_v8f16:
@@ -3084,84 +2107,18 @@ entry:
 define <16 x half> @trunc_v16f16(<16 x half> %a) {
 ; CHECK-SD-NOFP16-LABEL: trunc_v16f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h0
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h1
-; CHECK-SD-NOFP16-NEXT:    mov h7, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h20, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h21, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    frintz s4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    frintz s6, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    frintz s18, s2
-; CHECK-SD-NOFP16-NEXT:    frintz s19, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s4
-; CHECK-SD-NOFP16-NEXT:    frintz s4, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s6
-; CHECK-SD-NOFP16-NEXT:    frintz s6, s7
-; CHECK-SD-NOFP16-NEXT:    frintz s16, s16
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt h7, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h20
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h21
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt h16, s16
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[1], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    frintz s5, s17
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[1], v7.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    frintz s18, s18
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[2], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    frintz s4, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s5
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[2], v6.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    mov h19, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[3], v16.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[3], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    frintz s7, s7
-; CHECK-SD-NOFP16-NEXT:    frintz s16, s17
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h19
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[4], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[4], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    frintz s4, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s16
-; CHECK-SD-NOFP16-NEXT:    frintz s7, s17
-; CHECK-SD-NOFP16-NEXT:    frintz s0, s0
-; CHECK-SD-NOFP16-NEXT:    frintz s1, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[5], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[5], v6.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[6], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[6], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[7], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v2.16b
-; CHECK-SD-NOFP16-NEXT:    mov v1.16b, v3.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v2.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v4.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v5.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    frintz v2.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    frintz v1.4s, v3.4s
+; CHECK-SD-NOFP16-NEXT:    frintz v3.4s, v5.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    frintz v2.4s, v4.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v1.4h, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v1.8h, v3.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: trunc_v16f16:
diff --git a/llvm/test/CodeGen/AArch64/fcvt_combine.ll b/llvm/test/CodeGen/AArch64/fcvt_combine.ll
index 37133cf0aa1d..29170aab9656 100644
--- a/llvm/test/CodeGen/AArch64/fcvt_combine.ll
+++ b/llvm/test/CodeGen/AArch64/fcvt_combine.ll
@@ -180,48 +180,19 @@ define <3 x i32> @test_illegal_fp_to_int(<3 x float> %in) {
 define <8 x i16> @test_v8f16(<8 x half> %in) {
 ; CHECK-NO16-LABEL: test_v8f16:
 ; CHECK-NO16:       // %bb.0:
-; CHECK-NO16-NEXT:    mov h2, v0.h[1]
-; CHECK-NO16-NEXT:    mov h3, v0.h[4]
-; CHECK-NO16-NEXT:    mov h4, v0.h[5]
-; CHECK-NO16-NEXT:    mov h5, v0.h[2]
-; CHECK-NO16-NEXT:    fcvt s6, h0
-; CHECK-NO16-NEXT:    mov h7, v0.h[6]
-; CHECK-NO16-NEXT:    fmov s1, #4.00000000
-; CHECK-NO16-NEXT:    mov h16, v0.h[3]
-; CHECK-NO16-NEXT:    mov h0, v0.h[7]
-; CHECK-NO16-NEXT:    fcvt s2, h2
-; CHECK-NO16-NEXT:    fcvt s3, h3
-; CHECK-NO16-NEXT:    fcvt s4, h4
-; CHECK-NO16-NEXT:    fmul s6, s6, s1
-; CHECK-NO16-NEXT:    fcvt s5, h5
-; CHECK-NO16-NEXT:    fcvt s7, h7
-; CHECK-NO16-NEXT:    fcvt s16, h16
-; CHECK-NO16-NEXT:    fcvt s0, h0
-; CHECK-NO16-NEXT:    fmul s2, s2, s1
-; CHECK-NO16-NEXT:    fmul s3, s3, s1
-; CHECK-NO16-NEXT:    fmul s4, s4, s1
-; CHECK-NO16-NEXT:    fmul s5, s5, s1
-; CHECK-NO16-NEXT:    fcvt h6, s6
-; CHECK-NO16-NEXT:    fmul s7, s7, s1
-; CHECK-NO16-NEXT:    fmul s16, s16, s1
-; CHECK-NO16-NEXT:    fmul s0, s0, s1
-; CHECK-NO16-NEXT:    fcvt h2, s2
-; CHECK-NO16-NEXT:    fcvt h3, s3
-; CHECK-NO16-NEXT:    fcvt h4, s4
-; CHECK-NO16-NEXT:    fcvt h5, s5
-; CHECK-NO16-NEXT:    fcvt h1, s7
-; CHECK-NO16-NEXT:    fcvt h0, s0
-; CHECK-NO16-NEXT:    mov v6.h[1], v2.h[0]
-; CHECK-NO16-NEXT:    fcvt h2, s16
-; CHECK-NO16-NEXT:    mov v3.h[1], v4.h[0]
-; CHECK-NO16-NEXT:    mov v6.h[2], v5.h[0]
-; CHECK-NO16-NEXT:    mov v3.h[2], v1.h[0]
-; CHECK-NO16-NEXT:    mov v6.h[3], v2.h[0]
-; CHECK-NO16-NEXT:    mov v3.h[3], v0.h[0]
-; CHECK-NO16-NEXT:    fcvtl v1.4s, v6.4h
-; CHECK-NO16-NEXT:    fcvtl v0.4s, v3.4h
-; CHECK-NO16-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-NO16-NEXT:    movi v1.8h, #68, lsl #8
+; CHECK-NO16-NEXT:    fcvtl v2.4s, v0.4h
+; CHECK-NO16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-NO16-NEXT:    fcvtl v3.4s, v1.4h
+; CHECK-NO16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-NO16-NEXT:    fmul v2.4s, v2.4s, v3.4s
+; CHECK-NO16-NEXT:    fmul v0.4s, v0.4s, v1.4s
+; CHECK-NO16-NEXT:    fcvtn v1.4h, v2.4s
+; CHECK-NO16-NEXT:    fcvtn2 v1.8h, v0.4s
+; CHECK-NO16-NEXT:    fcvtl2 v0.4s, v1.8h
+; CHECK-NO16-NEXT:    fcvtl v1.4s, v1.4h
 ; CHECK-NO16-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NO16-NEXT:    fcvtzs v1.4s, v1.4s
 ; CHECK-NO16-NEXT:    uzp1 v0.8h, v1.8h, v0.8h
 ; CHECK-NO16-NEXT:    ret
 ;
@@ -496,96 +467,67 @@ define <3 x i32> @test_illegal_fp_to_int_sat_sat(<3 x float> %in) {
 define <8 x i16> @test_v8f16_sat(<8 x half> %in) {
 ; CHECK-NO16-LABEL: test_v8f16_sat:
 ; CHECK-NO16:       // %bb.0:
-; CHECK-NO16-NEXT:    mov h2, v0.h[4]
-; CHECK-NO16-NEXT:    mov h3, v0.h[5]
+; CHECK-NO16-NEXT:    movi v1.8h, #68, lsl #8
+; CHECK-NO16-NEXT:    fcvtl v2.4s, v0.4h
 ; CHECK-NO16-NEXT:    mov w8, #32767 // =0x7fff
-; CHECK-NO16-NEXT:    mov h4, v0.h[6]
-; CHECK-NO16-NEXT:    fmov s1, #4.00000000
+; CHECK-NO16-NEXT:    fcvtl2 v0.4s, v0.8h
 ; CHECK-NO16-NEXT:    mov w11, #-32768 // =0xffff8000
-; CHECK-NO16-NEXT:    mov h5, v0.h[7]
-; CHECK-NO16-NEXT:    mov h6, v0.h[1]
-; CHECK-NO16-NEXT:    mov h7, v0.h[2]
-; CHECK-NO16-NEXT:    fcvt s16, h0
-; CHECK-NO16-NEXT:    mov h0, v0.h[3]
-; CHECK-NO16-NEXT:    fcvt s2, h2
-; CHECK-NO16-NEXT:    fcvt s3, h3
-; CHECK-NO16-NEXT:    fcvt s4, h4
-; CHECK-NO16-NEXT:    fcvt s5, h5
-; CHECK-NO16-NEXT:    fcvt s6, h6
-; CHECK-NO16-NEXT:    fmul s2, s2, s1
-; CHECK-NO16-NEXT:    fmul s3, s3, s1
-; CHECK-NO16-NEXT:    fmul s4, s4, s1
-; CHECK-NO16-NEXT:    fmul s5, s5, s1
-; CHECK-NO16-NEXT:    fmul s6, s6, s1
-; CHECK-NO16-NEXT:    fcvt h2, s2
-; CHECK-NO16-NEXT:    fcvt h3, s3
-; CHECK-NO16-NEXT:    fcvt h4, s4
-; CHECK-NO16-NEXT:    fcvt h5, s5
-; CHECK-NO16-NEXT:    fcvt h6, s6
-; CHECK-NO16-NEXT:    mov v2.h[1], v3.h[0]
-; CHECK-NO16-NEXT:    fcvt s3, h7
-; CHECK-NO16-NEXT:    fmul s7, s16, s1
-; CHECK-NO16-NEXT:    mov v2.h[2], v4.h[0]
-; CHECK-NO16-NEXT:    fcvt s4, h0
-; CHECK-NO16-NEXT:    fmul s3, s3, s1
-; CHECK-NO16-NEXT:    fcvt h0, s7
-; CHECK-NO16-NEXT:    mov v2.h[3], v5.h[0]
-; CHECK-NO16-NEXT:    fmul s1, s4, s1
-; CHECK-NO16-NEXT:    fcvt h3, s3
-; CHECK-NO16-NEXT:    mov v0.h[1], v6.h[0]
-; CHECK-NO16-NEXT:    fcvtl v2.4s, v2.4h
-; CHECK-NO16-NEXT:    fcvt h1, s1
-; CHECK-NO16-NEXT:    mov v0.h[2], v3.h[0]
-; CHECK-NO16-NEXT:    mov s4, v2.s[1]
-; CHECK-NO16-NEXT:    fcvtzs w10, s2
-; CHECK-NO16-NEXT:    mov v0.h[3], v1.h[0]
-; CHECK-NO16-NEXT:    mov s1, v2.s[2]
-; CHECK-NO16-NEXT:    mov s2, v2.s[3]
-; CHECK-NO16-NEXT:    fcvtzs w9, s4
-; CHECK-NO16-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-NO16-NEXT:    fcvtzs w12, s1
-; CHECK-NO16-NEXT:    fcvtzs w13, s2
+; CHECK-NO16-NEXT:    fcvtl v3.4s, v1.4h
+; CHECK-NO16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-NO16-NEXT:    fmul v2.4s, v2.4s, v3.4s
+; CHECK-NO16-NEXT:    fmul v0.4s, v0.4s, v1.4s
+; CHECK-NO16-NEXT:    fcvtn v1.4h, v2.4s
+; CHECK-NO16-NEXT:    fcvtn2 v1.8h, v0.4s
+; CHECK-NO16-NEXT:    fcvtl2 v0.4s, v1.8h
+; CHECK-NO16-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-NO16-NEXT:    mov s2, v0.s[1]
+; CHECK-NO16-NEXT:    fcvtzs w10, s0
+; CHECK-NO16-NEXT:    fcvtzs w15, s1
+; CHECK-NO16-NEXT:    fcvtzs w9, s2
+; CHECK-NO16-NEXT:    mov s2, v0.s[2]
+; CHECK-NO16-NEXT:    mov s0, v0.s[3]
 ; CHECK-NO16-NEXT:    cmp w9, w8
+; CHECK-NO16-NEXT:    fcvtzs w12, s2
+; CHECK-NO16-NEXT:    mov s2, v1.s[1]
 ; CHECK-NO16-NEXT:    csel w9, w9, w8, lt
-; CHECK-NO16-NEXT:    mov s1, v0.s[1]
-; CHECK-NO16-NEXT:    fcvtzs w15, s0
+; CHECK-NO16-NEXT:    fcvtzs w13, s0
+; CHECK-NO16-NEXT:    mov s0, v1.s[2]
 ; CHECK-NO16-NEXT:    cmn w9, #8, lsl #12 // =32768
 ; CHECK-NO16-NEXT:    csel w9, w9, w11, gt
 ; CHECK-NO16-NEXT:    cmp w10, w8
 ; CHECK-NO16-NEXT:    csel w10, w10, w8, lt
+; CHECK-NO16-NEXT:    fcvtzs w14, s2
 ; CHECK-NO16-NEXT:    cmn w10, #8, lsl #12 // =32768
-; CHECK-NO16-NEXT:    fcvtzs w14, s1
-; CHECK-NO16-NEXT:    mov s1, v0.s[2]
+; CHECK-NO16-NEXT:    fcvtzs w16, s0
+; CHECK-NO16-NEXT:    mov s0, v1.s[3]
 ; CHECK-NO16-NEXT:    csel w10, w10, w11, gt
 ; CHECK-NO16-NEXT:    cmp w12, w8
-; CHECK-NO16-NEXT:    mov s0, v0.s[3]
 ; CHECK-NO16-NEXT:    csel w12, w12, w8, lt
+; CHECK-NO16-NEXT:    fmov s1, w10
 ; CHECK-NO16-NEXT:    cmn w12, #8, lsl #12 // =32768
 ; CHECK-NO16-NEXT:    csel w12, w12, w11, gt
 ; CHECK-NO16-NEXT:    cmp w13, w8
-; CHECK-NO16-NEXT:    fcvtzs w16, s1
 ; CHECK-NO16-NEXT:    csel w13, w13, w8, lt
-; CHECK-NO16-NEXT:    fmov s1, w10
+; CHECK-NO16-NEXT:    mov v1.s[1], w9
+; CHECK-NO16-NEXT:    fcvtzs w9, s0
 ; CHECK-NO16-NEXT:    cmn w13, #8, lsl #12 // =32768
 ; CHECK-NO16-NEXT:    csel w13, w13, w11, gt
 ; CHECK-NO16-NEXT:    cmp w14, w8
 ; CHECK-NO16-NEXT:    csel w14, w14, w8, lt
-; CHECK-NO16-NEXT:    mov v1.s[1], w9
-; CHECK-NO16-NEXT:    fcvtzs w9, s0
 ; CHECK-NO16-NEXT:    cmn w14, #8, lsl #12 // =32768
+; CHECK-NO16-NEXT:    mov v1.s[2], w12
 ; CHECK-NO16-NEXT:    csel w14, w14, w11, gt
 ; CHECK-NO16-NEXT:    cmp w15, w8
 ; CHECK-NO16-NEXT:    csel w15, w15, w8, lt
 ; CHECK-NO16-NEXT:    cmn w15, #8, lsl #12 // =32768
-; CHECK-NO16-NEXT:    mov v1.s[2], w12
 ; CHECK-NO16-NEXT:    csel w10, w15, w11, gt
 ; CHECK-NO16-NEXT:    cmp w16, w8
+; CHECK-NO16-NEXT:    mov v1.s[3], w13
 ; CHECK-NO16-NEXT:    fmov s2, w10
 ; CHECK-NO16-NEXT:    csel w10, w16, w8, lt
 ; CHECK-NO16-NEXT:    cmn w10, #8, lsl #12 // =32768
 ; CHECK-NO16-NEXT:    csel w10, w10, w11, gt
 ; CHECK-NO16-NEXT:    cmp w9, w8
-; CHECK-NO16-NEXT:    mov v1.s[3], w13
 ; CHECK-NO16-NEXT:    mov v2.s[1], w14
 ; CHECK-NO16-NEXT:    csel w8, w9, w8, lt
 ; CHECK-NO16-NEXT:    cmn w8, #8, lsl #12 // =32768
diff --git a/llvm/test/CodeGen/AArch64/fdiv.ll b/llvm/test/CodeGen/AArch64/fdiv.ll
index fa87c4fa2d16..e73124fbb595 100644
--- a/llvm/test/CodeGen/AArch64/fdiv.ll
+++ b/llvm/test/CodeGen/AArch64/fdiv.ll
@@ -169,60 +169,14 @@ entry:
 define <7 x half> @fdiv_v7f16(<7 x half> %a, <7 x half> %b) {
 ; CHECK-SD-NOFP16-LABEL: fdiv_v7f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h0
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fdiv s2, s3, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h1
-; CHECK-SD-NOFP16-NEXT:    fdiv s3, s4, s3
-; CHECK-SD-NOFP16-NEXT:    mov h4, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt h18, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fdiv s4, s5, s4
-; CHECK-SD-NOFP16-NEXT:    mov h5, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[1], v18.h[0]
-; CHECK-SD-NOFP16-NEXT:    fdiv s5, s6, s5
-; CHECK-SD-NOFP16-NEXT:    mov h6, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[2], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fdiv s6, s7, s6
-; CHECK-SD-NOFP16-NEXT:    mov h7, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[3], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fdiv s7, s16, s7
-; CHECK-SD-NOFP16-NEXT:    mov h16, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    fdiv s3, s17, s16
-; CHECK-SD-NOFP16-NEXT:    fdiv s0, s0, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s6
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[4], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s7
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[5], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s3
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[6], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v2.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v2.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fdiv v2.4s, v3.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fdiv v1.4s, v0.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: fdiv_v7f16:
@@ -309,60 +263,14 @@ entry:
 define <8 x half> @fdiv_v8f16(<8 x half> %a, <8 x half> %b) {
 ; CHECK-SD-NOFP16-LABEL: fdiv_v8f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h0
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fdiv s2, s3, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h1
-; CHECK-SD-NOFP16-NEXT:    fdiv s3, s4, s3
-; CHECK-SD-NOFP16-NEXT:    mov h4, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt h18, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fdiv s4, s5, s4
-; CHECK-SD-NOFP16-NEXT:    mov h5, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[1], v18.h[0]
-; CHECK-SD-NOFP16-NEXT:    fdiv s5, s6, s5
-; CHECK-SD-NOFP16-NEXT:    mov h6, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[2], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fdiv s6, s7, s6
-; CHECK-SD-NOFP16-NEXT:    mov h7, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[3], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fdiv s7, s16, s7
-; CHECK-SD-NOFP16-NEXT:    mov h16, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    fdiv s3, s17, s16
-; CHECK-SD-NOFP16-NEXT:    fdiv s0, s0, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s6
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[4], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s7
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[5], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s3
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[6], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v2.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v2.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fdiv v2.4s, v3.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fdiv v1.4s, v0.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: fdiv_v8f16:
@@ -394,113 +302,22 @@ entry:
 define <16 x half> @fdiv_v16f16(<16 x half> %a, <16 x half> %b) {
 ; CHECK-SD-NOFP16-LABEL: fdiv_v16f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h4, v2.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h0
-; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h18, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov h19, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h1
-; CHECK-SD-NOFP16-NEXT:    mov h21, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h22, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    mov h23, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h19
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h21
-; CHECK-SD-NOFP16-NEXT:    fcvt s22, h22
-; CHECK-SD-NOFP16-NEXT:    mov h24, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    fdiv s4, s5, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s23, h23
-; CHECK-SD-NOFP16-NEXT:    mov h25, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s24, h24
-; CHECK-SD-NOFP16-NEXT:    fcvt s25, h25
-; CHECK-SD-NOFP16-NEXT:    fdiv s5, s6, s5
-; CHECK-SD-NOFP16-NEXT:    mov h6, v2.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    fdiv s7, s7, s6
-; CHECK-SD-NOFP16-NEXT:    mov h6, v2.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    fdiv s6, s16, s6
-; CHECK-SD-NOFP16-NEXT:    mov h16, v2.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fdiv s16, s17, s16
-; CHECK-SD-NOFP16-NEXT:    mov h17, v2.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    fdiv s17, s18, s17
-; CHECK-SD-NOFP16-NEXT:    mov h18, v2.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h2, v2.h[7]
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fdiv s18, s19, s18
-; CHECK-SD-NOFP16-NEXT:    fdiv s19, s0, s2
-; CHECK-SD-NOFP16-NEXT:    mov h0, v3.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    fdiv s2, s2, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h3
-; CHECK-SD-NOFP16-NEXT:    fdiv s20, s20, s0
-; CHECK-SD-NOFP16-NEXT:    mov h0, v3.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fdiv s21, s21, s0
-; CHECK-SD-NOFP16-NEXT:    mov h0, v3.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fdiv s22, s22, s0
-; CHECK-SD-NOFP16-NEXT:    mov h0, v3.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fdiv s23, s23, s0
-; CHECK-SD-NOFP16-NEXT:    mov h0, v3.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fdiv s24, s24, s0
-; CHECK-SD-NOFP16-NEXT:    mov h0, v3.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v3.h[7]
-; CHECK-SD-NOFP16-NEXT:    fcvt s26, h0
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s20
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[1], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s7
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[1], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s21
-; CHECK-SD-NOFP16-NEXT:    fdiv s20, s25, s26
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[2], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s6
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[2], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s22
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[3], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s23
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[3], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    fdiv s1, s1, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s16
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[4], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s24
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[4], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s17
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[5], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s20
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[5], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s18
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[6], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[6], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[7], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[7], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v1.16b, v2.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v4.4s, v2.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v5.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v2.4s, v2.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fdiv v4.4s, v5.4s, v4.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtl v5.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fdiv v0.4s, v0.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtl v2.4s, v3.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v3.4s, v3.8h
+; CHECK-SD-NOFP16-NEXT:    fdiv v2.4s, v5.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v4.4s
+; CHECK-SD-NOFP16-NEXT:    fdiv v3.4s, v1.4s, v3.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v1.4h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v1.8h, v3.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: fdiv_v16f16:
diff --git a/llvm/test/CodeGen/AArch64/fmla.ll b/llvm/test/CodeGen/AArch64/fmla.ll
index 339ade5fc707..336c9705f399 100644
--- a/llvm/test/CodeGen/AArch64/fmla.ll
+++ b/llvm/test/CodeGen/AArch64/fmla.ll
@@ -840,99 +840,22 @@ entry:
 define <7 x half> @fmuladd_v7f16(<7 x half> %a, <7 x half> %b, <7 x half> %c) {
 ; CHECK-SD-NOFP16-LABEL: fmuladd_v7f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h3, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h0
-; CHECK-SD-NOFP16-NEXT:    mov h7, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h18, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h2
-; CHECK-SD-NOFP16-NEXT:    mov h20, v2.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h21, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov h22, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fmul s5, s6, s5
-; CHECK-SD-NOFP16-NEXT:    mov h6, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h21
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h20
-; CHECK-SD-NOFP16-NEXT:    fmul s3, s4, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h17
-; CHECK-SD-NOFP16-NEXT:    fmul s7, s16, s7
-; CHECK-SD-NOFP16-NEXT:    mov h17, v2.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fmul s5, s6, s5
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    fcvt h7, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fadd s4, s4, s19
-; CHECK-SD-NOFP16-NEXT:    mov h19, v2.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fmul s16, s18, s16
-; CHECK-SD-NOFP16-NEXT:    mov h18, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fadd s17, s3, s17
-; CHECK-SD-NOFP16-NEXT:    fmul s6, s6, s21
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h19
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fadd s4, s7, s20
-; CHECK-SD-NOFP16-NEXT:    fcvt h16, s16
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h22
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt h7, s17
-; CHECK-SD-NOFP16-NEXT:    mov h17, v2.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s6
-; CHECK-SD-NOFP16-NEXT:    fadd s5, s5, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    fmul s18, s20, s18
-; CHECK-SD-NOFP16-NEXT:    fmul s0, s0, s1
-; CHECK-SD-NOFP16-NEXT:    mov h1, v2.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[1], v7.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v2.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    mov h2, v2.h[7]
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fadd s16, s16, s17
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[2], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fadd s6, s6, s7
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[3], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s16
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fadd s0, s0, s2
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[4], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s6
-; CHECK-SD-NOFP16-NEXT:    fadd s1, s5, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[5], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[6], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v3.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v4.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fmul v3.4s, v4.4s, v3.4s
+; CHECK-SD-NOFP16-NEXT:    fmul v0.4s, v0.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v1.4h, v3.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v2.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v2.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v1.8h, v0.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fadd v0.4s, v0.4s, v3.4s
+; CHECK-SD-NOFP16-NEXT:    fadd v1.4s, v1.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: fmuladd_v7f16:
@@ -1045,99 +968,22 @@ entry:
 define <8 x half> @fmuladd_v8f16(<8 x half> %a, <8 x half> %b, <8 x half> %c) {
 ; CHECK-SD-NOFP16-LABEL: fmuladd_v8f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h3, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h0
-; CHECK-SD-NOFP16-NEXT:    mov h7, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h18, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h2
-; CHECK-SD-NOFP16-NEXT:    mov h20, v2.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h21, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov h22, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fmul s5, s6, s5
-; CHECK-SD-NOFP16-NEXT:    mov h6, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h21
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h20
-; CHECK-SD-NOFP16-NEXT:    fmul s3, s4, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h17
-; CHECK-SD-NOFP16-NEXT:    fmul s7, s16, s7
-; CHECK-SD-NOFP16-NEXT:    mov h17, v2.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fmul s5, s6, s5
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    fcvt h7, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fadd s4, s4, s19
-; CHECK-SD-NOFP16-NEXT:    mov h19, v2.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fmul s16, s18, s16
-; CHECK-SD-NOFP16-NEXT:    mov h18, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fadd s17, s3, s17
-; CHECK-SD-NOFP16-NEXT:    fmul s6, s6, s21
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h19
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fadd s4, s7, s20
-; CHECK-SD-NOFP16-NEXT:    fcvt h16, s16
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h22
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt h7, s17
-; CHECK-SD-NOFP16-NEXT:    mov h17, v2.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s6
-; CHECK-SD-NOFP16-NEXT:    fadd s5, s5, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    fmul s18, s20, s18
-; CHECK-SD-NOFP16-NEXT:    fmul s0, s0, s1
-; CHECK-SD-NOFP16-NEXT:    mov h1, v2.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[1], v7.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v2.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    mov h2, v2.h[7]
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fadd s16, s16, s17
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[2], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fadd s6, s6, s7
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[3], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s16
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fadd s0, s0, s2
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[4], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s6
-; CHECK-SD-NOFP16-NEXT:    fadd s1, s5, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[5], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[6], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v3.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v3.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v4.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fmul v3.4s, v4.4s, v3.4s
+; CHECK-SD-NOFP16-NEXT:    fmul v0.4s, v0.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v1.4h, v3.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v2.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v2.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v1.8h, v0.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fadd v0.4s, v0.4s, v3.4s
+; CHECK-SD-NOFP16-NEXT:    fadd v1.4s, v1.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: fmuladd_v8f16:
@@ -1179,192 +1025,38 @@ entry:
 define <16 x half> @fmuladd_v16f16(<16 x half> %a, <16 x half> %b, <16 x half> %c) {
 ; CHECK-SD-NOFP16-LABEL: fmuladd_v16f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h6, v2.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h0
-; CHECK-SD-NOFP16-NEXT:    mov h18, v2.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h19, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h20, v2.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h21, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h24, v3.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h25, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h26, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt s27, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    mov h29, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    fmul s16, s17, s16
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h19
-; CHECK-SD-NOFP16-NEXT:    mov h17, v4.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h20
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h21
-; CHECK-SD-NOFP16-NEXT:    fcvt s24, h24
-; CHECK-SD-NOFP16-NEXT:    fcvt s25, h25
-; CHECK-SD-NOFP16-NEXT:    mov h30, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    fmul s6, s7, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h4
-; CHECK-SD-NOFP16-NEXT:    fmul s23, s19, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt h22, s16
-; CHECK-SD-NOFP16-NEXT:    mov h18, v2.h[4]
-; CHECK-SD-NOFP16-NEXT:    fmul s20, s21, s20
-; CHECK-SD-NOFP16-NEXT:    mov h19, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v4.h[2]
-; CHECK-SD-NOFP16-NEXT:    fmul s24, s25, s24
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    mov h25, v3.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h22
-; CHECK-SD-NOFP16-NEXT:    fcvt h22, s23
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h19
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt h20, s20
-; CHECK-SD-NOFP16-NEXT:    fcvt s23, h6
-; CHECK-SD-NOFP16-NEXT:    mov h6, v4.h[3]
-; CHECK-SD-NOFP16-NEXT:    fadd s7, s21, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h22
-; CHECK-SD-NOFP16-NEXT:    fcvt s22, h3
-; CHECK-SD-NOFP16-NEXT:    fmul s18, s19, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h20
-; CHECK-SD-NOFP16-NEXT:    fadd s17, s23, s17
-; CHECK-SD-NOFP16-NEXT:    mov h23, v3.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt s28, h6
-; CHECK-SD-NOFP16-NEXT:    fmul s22, s27, s22
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s7
-; CHECK-SD-NOFP16-NEXT:    fadd s7, s21, s16
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h26
-; CHECK-SD-NOFP16-NEXT:    mov h26, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt h18, s18
-; CHECK-SD-NOFP16-NEXT:    mov h27, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt h16, s17
-; CHECK-SD-NOFP16-NEXT:    mov h17, v4.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h23
-; CHECK-SD-NOFP16-NEXT:    fadd s19, s19, s28
-; CHECK-SD-NOFP16-NEXT:    fcvt h7, s7
-; CHECK-SD-NOFP16-NEXT:    mov h23, v2.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
-; CHECK-SD-NOFP16-NEXT:    fmul s20, s21, s20
-; CHECK-SD-NOFP16-NEXT:    mov v6.h[1], v16.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v5.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    fcvt h21, s22
-; CHECK-SD-NOFP16-NEXT:    fcvt h22, s24
-; CHECK-SD-NOFP16-NEXT:    fcvt s24, h25
-; CHECK-SD-NOFP16-NEXT:    fcvt s25, h26
-; CHECK-SD-NOFP16-NEXT:    mov h26, v5.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt h19, s19
-; CHECK-SD-NOFP16-NEXT:    mov v6.h[2], v7.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v3.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s28, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt h20, s20
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h21
-; CHECK-SD-NOFP16-NEXT:    fcvt s22, h22
-; CHECK-SD-NOFP16-NEXT:    fadd s16, s18, s17
-; CHECK-SD-NOFP16-NEXT:    fmul s18, s25, s24
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h5
-; CHECK-SD-NOFP16-NEXT:    mov h24, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s25, h26
-; CHECK-SD-NOFP16-NEXT:    fcvt s26, h27
-; CHECK-SD-NOFP16-NEXT:    mov v6.h[3], v19.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h23
-; CHECK-SD-NOFP16-NEXT:    mov h23, v2.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h20
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s27, h29
-; CHECK-SD-NOFP16-NEXT:    fadd s22, s22, s28
-; CHECK-SD-NOFP16-NEXT:    fadd s17, s21, s17
-; CHECK-SD-NOFP16-NEXT:    fcvt h18, s18
-; CHECK-SD-NOFP16-NEXT:    mov h21, v5.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h28, v3.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov h29, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    fmul s19, s26, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt s23, h23
-; CHECK-SD-NOFP16-NEXT:    fcvt s24, h24
-; CHECK-SD-NOFP16-NEXT:    fadd s20, s20, s25
-; CHECK-SD-NOFP16-NEXT:    fmul s25, s27, s7
-; CHECK-SD-NOFP16-NEXT:    mov h27, v3.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt h22, s22
-; CHECK-SD-NOFP16-NEXT:    fcvt h7, s17
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h18
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h21
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h28
-; CHECK-SD-NOFP16-NEXT:    fcvt s28, h29
-; CHECK-SD-NOFP16-NEXT:    fmul s23, s24, s23
-; CHECK-SD-NOFP16-NEXT:    mov h2, v2.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    fcvt s24, h27
-; CHECK-SD-NOFP16-NEXT:    fcvt s26, h30
-; CHECK-SD-NOFP16-NEXT:    fcvt h20, s20
-; CHECK-SD-NOFP16-NEXT:    mov v7.h[1], v22.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h22, s25
-; CHECK-SD-NOFP16-NEXT:    mov h25, v5.h[4]
-; CHECK-SD-NOFP16-NEXT:    fadd s17, s17, s18
-; CHECK-SD-NOFP16-NEXT:    fmul s18, s28, s21
-; CHECK-SD-NOFP16-NEXT:    mov h3, v3.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    fcvt h19, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fmul s21, s26, s24
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    mov h24, v4.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt s22, h22
-; CHECK-SD-NOFP16-NEXT:    fcvt s25, h25
-; CHECK-SD-NOFP16-NEXT:    mov v7.h[2], v20.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h17, s17
-; CHECK-SD-NOFP16-NEXT:    fcvt h18, s18
-; CHECK-SD-NOFP16-NEXT:    mov h20, v5.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt h23, s23
-; CHECK-SD-NOFP16-NEXT:    fmul s0, s0, s2
-; CHECK-SD-NOFP16-NEXT:    mov h2, v4.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt h21, s21
-; CHECK-SD-NOFP16-NEXT:    fadd s22, s22, s25
-; CHECK-SD-NOFP16-NEXT:    mov h25, v5.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt h16, s16
-; CHECK-SD-NOFP16-NEXT:    mov v7.h[3], v17.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h19
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h24
-; CHECK-SD-NOFP16-NEXT:    fmul s1, s1, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h18
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h20
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h23
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h21
-; CHECK-SD-NOFP16-NEXT:    fcvt h22, s22
-; CHECK-SD-NOFP16-NEXT:    fcvt s23, h25
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    fadd s17, s17, s19
-; CHECK-SD-NOFP16-NEXT:    mov h4, v4.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov h5, v5.h[7]
-; CHECK-SD-NOFP16-NEXT:    fadd s3, s3, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
-; CHECK-SD-NOFP16-NEXT:    mov v6.h[4], v16.h[0]
-; CHECK-SD-NOFP16-NEXT:    fadd s2, s20, s2
-; CHECK-SD-NOFP16-NEXT:    mov v7.h[4], v22.h[0]
-; CHECK-SD-NOFP16-NEXT:    fadd s16, s21, s23
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fcvt h17, s17
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    mov v6.h[5], v17.h[0]
-; CHECK-SD-NOFP16-NEXT:    fadd s0, s0, s4
-; CHECK-SD-NOFP16-NEXT:    mov v7.h[5], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s16
-; CHECK-SD-NOFP16-NEXT:    fadd s1, s1, s5
-; CHECK-SD-NOFP16-NEXT:    mov v6.h[6], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v7.h[6], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
-; CHECK-SD-NOFP16-NEXT:    mov v6.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v7.h[7], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v6.16b
-; CHECK-SD-NOFP16-NEXT:    mov v1.16b, v7.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v6.4s, v2.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v7.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v16.4s, v3.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v17.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v2.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v3.4s, v3.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fmul v6.4s, v7.4s, v6.4s
+; CHECK-SD-NOFP16-NEXT:    fmul v7.4s, v17.4s, v16.4s
+; CHECK-SD-NOFP16-NEXT:    fmul v0.4s, v0.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fmul v1.4s, v1.4s, v3.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v2.4h, v6.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtl v6.4s, v5.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtn v3.4h, v7.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v2.8h, v0.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v4.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v4.4s, v4.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v3.8h, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v2.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v2.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v7.4s, v3.4h
+; CHECK-SD-NOFP16-NEXT:    fadd v0.4s, v1.4s, v0.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v3.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v3.4s, v5.8h
+; CHECK-SD-NOFP16-NEXT:    fadd v5.4s, v7.4s, v6.4s
+; CHECK-SD-NOFP16-NEXT:    fadd v2.4s, v2.4s, v4.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-SD-NOFP16-NEXT:    fadd v3.4s, v1.4s, v3.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v1.4h, v5.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v1.8h, v3.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: fmuladd_v16f16:
@@ -1650,99 +1342,22 @@ entry:
 define <7 x half> @fmul_v7f16(<7 x half> %a, <7 x half> %b, <7 x half> %c) {
 ; CHECK-SD-NOFP16-LABEL: fmul_v7f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h3, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h0
-; CHECK-SD-NOFP16-NEXT:    mov h7, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h18, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h19, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h20, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h2
-; CHECK-SD-NOFP16-NEXT:    mov h22, v2.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fmul s5, s6, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h18
-; CHECK-SD-NOFP16-NEXT:    mov h18, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h20
-; CHECK-SD-NOFP16-NEXT:    fmul s3, s4, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h17
-; CHECK-SD-NOFP16-NEXT:    mov h17, v2.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fmul s4, s6, s4
-; CHECK-SD-NOFP16-NEXT:    mov h6, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    fmul s7, s16, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h17
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h18
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h19
-; CHECK-SD-NOFP16-NEXT:    mov h19, v2.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt h7, s7
-; CHECK-SD-NOFP16-NEXT:    fadd s5, s5, s21
-; CHECK-SD-NOFP16-NEXT:    mov h21, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fmul s17, s18, s17
-; CHECK-SD-NOFP16-NEXT:    mov h18, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h19
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    fadd s3, s3, s16
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h22
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fmul s6, s20, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    mov h20, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s5
-; CHECK-SD-NOFP16-NEXT:    mov h5, v2.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fadd s4, s4, s16
-; CHECK-SD-NOFP16-NEXT:    fcvt h16, s17
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h18
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h21
-; CHECK-SD-NOFP16-NEXT:    fadd s7, s7, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[1], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v2.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    fmul s17, s18, s17
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h20
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt h7, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fadd s5, s16, s5
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[2], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v2.h[6]
-; CHECK-SD-NOFP16-NEXT:    fmul s0, s0, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt h16, s17
-; CHECK-SD-NOFP16-NEXT:    mov h2, v2.h[7]
-; CHECK-SD-NOFP16-NEXT:    fadd s3, s6, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s5
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[3], v7.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[4], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fadd s4, s6, s4
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[5], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fadd s0, s0, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[6], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v1.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v4.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fmul v3.4s, v4.4s, v3.4s
+; CHECK-SD-NOFP16-NEXT:    fmul v0.4s, v0.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v1.4h, v3.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v2.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v2.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v1.8h, v0.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fadd v0.4s, v0.4s, v3.4s
+; CHECK-SD-NOFP16-NEXT:    fadd v1.4s, v1.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: fmul_v7f16:
@@ -1857,99 +1472,22 @@ entry:
 define <8 x half> @fmul_v8f16(<8 x half> %a, <8 x half> %b, <8 x half> %c) {
 ; CHECK-SD-NOFP16-LABEL: fmul_v8f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h3, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h0
-; CHECK-SD-NOFP16-NEXT:    mov h7, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h18, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h19, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h20, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h2
-; CHECK-SD-NOFP16-NEXT:    mov h22, v2.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fmul s5, s6, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h18
-; CHECK-SD-NOFP16-NEXT:    mov h18, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h20
-; CHECK-SD-NOFP16-NEXT:    fmul s3, s4, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h17
-; CHECK-SD-NOFP16-NEXT:    mov h17, v2.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fmul s4, s6, s4
-; CHECK-SD-NOFP16-NEXT:    mov h6, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    fmul s7, s16, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h17
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h18
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h19
-; CHECK-SD-NOFP16-NEXT:    mov h19, v2.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt h7, s7
-; CHECK-SD-NOFP16-NEXT:    fadd s5, s5, s21
-; CHECK-SD-NOFP16-NEXT:    mov h21, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fmul s17, s18, s17
-; CHECK-SD-NOFP16-NEXT:    mov h18, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h19
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    fadd s3, s3, s16
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h22
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fmul s6, s20, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    mov h20, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s5
-; CHECK-SD-NOFP16-NEXT:    mov h5, v2.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fadd s4, s4, s16
-; CHECK-SD-NOFP16-NEXT:    fcvt h16, s17
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h18
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h21
-; CHECK-SD-NOFP16-NEXT:    fadd s7, s7, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[1], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v2.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    fmul s17, s18, s17
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h20
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt h7, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fadd s5, s16, s5
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[2], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v2.h[6]
-; CHECK-SD-NOFP16-NEXT:    fmul s0, s0, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt h16, s17
-; CHECK-SD-NOFP16-NEXT:    mov h2, v2.h[7]
-; CHECK-SD-NOFP16-NEXT:    fadd s3, s6, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s5
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[3], v7.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[4], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fadd s4, s6, s4
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[5], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fadd s0, s0, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[6], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v1.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v4.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fmul v3.4s, v4.4s, v3.4s
+; CHECK-SD-NOFP16-NEXT:    fmul v0.4s, v0.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v1.4h, v3.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v2.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v2.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v1.8h, v0.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fadd v0.4s, v0.4s, v3.4s
+; CHECK-SD-NOFP16-NEXT:    fadd v1.4s, v1.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: fmul_v8f16:
@@ -1992,199 +1530,38 @@ entry:
 define <16 x half> @fmul_v16f16(<16 x half> %a, <16 x half> %b, <16 x half> %c) {
 ; CHECK-SD-NOFP16-LABEL: fmul_v16f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    stp d11, d10, [sp, #-32]! // 16-byte Folded Spill
-; CHECK-SD-NOFP16-NEXT:    stp d9, d8, [sp, #16] // 16-byte Folded Spill
-; CHECK-SD-NOFP16-NEXT:    .cfi_def_cfa_offset 32
-; CHECK-SD-NOFP16-NEXT:    .cfi_offset b8, -8
-; CHECK-SD-NOFP16-NEXT:    .cfi_offset b9, -16
-; CHECK-SD-NOFP16-NEXT:    .cfi_offset b10, -24
-; CHECK-SD-NOFP16-NEXT:    .cfi_offset b11, -32
-; CHECK-SD-NOFP16-NEXT:    mov h6, v3.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov h20, v3.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h22, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h23, v3.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov h24, v3.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h26, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v3.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h18, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h27, v3.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h28, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt s25, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h20
-; CHECK-SD-NOFP16-NEXT:    fcvt s22, h22
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h23
-; CHECK-SD-NOFP16-NEXT:    fcvt s23, h24
-; CHECK-SD-NOFP16-NEXT:    fcvt s24, h26
-; CHECK-SD-NOFP16-NEXT:    mov h26, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h19, v3.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s29, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    fmul s6, s25, s6
-; CHECK-SD-NOFP16-NEXT:    mov h25, v2.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s27, h27
-; CHECK-SD-NOFP16-NEXT:    fmul s3, s22, s21
-; CHECK-SD-NOFP16-NEXT:    fcvt s28, h28
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    fmul s23, s24, s23
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s22, h0
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h19
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s24, h25
-; CHECK-SD-NOFP16-NEXT:    fcvt s25, h26
-; CHECK-SD-NOFP16-NEXT:    fmul s26, s1, s29
-; CHECK-SD-NOFP16-NEXT:    fmul s27, s28, s27
-; CHECK-SD-NOFP16-NEXT:    mov h28, v2.h[7]
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fmul s1, s18, s17
-; CHECK-SD-NOFP16-NEXT:    fcvt h17, s23
-; CHECK-SD-NOFP16-NEXT:    mov h29, v4.h[1]
-; CHECK-SD-NOFP16-NEXT:    fmul s21, s22, s21
-; CHECK-SD-NOFP16-NEXT:    fmul s16, s16, s19
-; CHECK-SD-NOFP16-NEXT:    mov h8, v2.h[6]
-; CHECK-SD-NOFP16-NEXT:    fmul s23, s25, s24
-; CHECK-SD-NOFP16-NEXT:    mov h24, v2.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h25, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    fmul s7, s7, s20
-; CHECK-SD-NOFP16-NEXT:    fcvt h18, s26
-; CHECK-SD-NOFP16-NEXT:    fcvt h19, s27
-; CHECK-SD-NOFP16-NEXT:    fcvt s22, h28
-; CHECK-SD-NOFP16-NEXT:    mov h26, v2.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h27, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt h21, s21
-; CHECK-SD-NOFP16-NEXT:    mov h20, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov h11, v2.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt h28, s23
-; CHECK-SD-NOFP16-NEXT:    fcvt s30, h24
-; CHECK-SD-NOFP16-NEXT:    fcvt s31, h25
-; CHECK-SD-NOFP16-NEXT:    mov h24, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt s29, h29
-; CHECK-SD-NOFP16-NEXT:    mov h9, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s25, h26
-; CHECK-SD-NOFP16-NEXT:    fcvt s26, h27
-; CHECK-SD-NOFP16-NEXT:    mov h10, v2.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h20
-; CHECK-SD-NOFP16-NEXT:    mov h23, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h11
-; CHECK-SD-NOFP16-NEXT:    fmul s27, s31, s30
-; CHECK-SD-NOFP16-NEXT:    fcvt s28, h28
-; CHECK-SD-NOFP16-NEXT:    fcvt s30, h21
-; CHECK-SD-NOFP16-NEXT:    fcvt s31, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h8
-; CHECK-SD-NOFP16-NEXT:    mov h8, v5.h[1]
-; CHECK-SD-NOFP16-NEXT:    fmul s25, s26, s25
-; CHECK-SD-NOFP16-NEXT:    fcvt s24, h24
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h19
-; CHECK-SD-NOFP16-NEXT:    fmul s2, s20, s22
-; CHECK-SD-NOFP16-NEXT:    fcvt s22, h9
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h10
-; CHECK-SD-NOFP16-NEXT:    fadd s26, s28, s29
-; CHECK-SD-NOFP16-NEXT:    fcvt s23, h23
-; CHECK-SD-NOFP16-NEXT:    fcvt h27, s27
-; CHECK-SD-NOFP16-NEXT:    fadd s28, s30, s31
-; CHECK-SD-NOFP16-NEXT:    mov h29, v4.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h30, v5.h[2]
-; CHECK-SD-NOFP16-NEXT:    fmul s24, s24, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt s31, h8
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
-; CHECK-SD-NOFP16-NEXT:    fmul s21, s22, s21
-; CHECK-SD-NOFP16-NEXT:    fcvt s8, h5
-; CHECK-SD-NOFP16-NEXT:    fcvt h25, s25
-; CHECK-SD-NOFP16-NEXT:    fmul s20, s23, s20
-; CHECK-SD-NOFP16-NEXT:    fcvt h26, s26
-; CHECK-SD-NOFP16-NEXT:    fcvt s27, h27
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s28
-; CHECK-SD-NOFP16-NEXT:    mov h28, v4.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s29, h29
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    fcvt s30, h30
-; CHECK-SD-NOFP16-NEXT:    fadd s19, s19, s31
-; CHECK-SD-NOFP16-NEXT:    fadd s18, s18, s8
-; CHECK-SD-NOFP16-NEXT:    ldp d9, d8, [sp, #16] // 16-byte Folded Reload
-; CHECK-SD-NOFP16-NEXT:    fcvt h22, s1
-; CHECK-SD-NOFP16-NEXT:    mov h23, v5.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s25, h25
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[1], v26.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt s26, h28
-; CHECK-SD-NOFP16-NEXT:    fadd s27, s27, s29
-; CHECK-SD-NOFP16-NEXT:    fcvt h24, s24
-; CHECK-SD-NOFP16-NEXT:    fadd s17, s17, s30
-; CHECK-SD-NOFP16-NEXT:    mov h28, v4.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt h19, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt h16, s16
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h22
-; CHECK-SD-NOFP16-NEXT:    fcvt s22, h23
-; CHECK-SD-NOFP16-NEXT:    fcvt h20, s20
-; CHECK-SD-NOFP16-NEXT:    fadd s23, s25, s26
-; CHECK-SD-NOFP16-NEXT:    mov h25, v5.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt h26, s27
-; CHECK-SD-NOFP16-NEXT:    fcvt s24, h24
-; CHECK-SD-NOFP16-NEXT:    fcvt s27, h28
-; CHECK-SD-NOFP16-NEXT:    fcvt h7, s7
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[1], v19.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h19, v4.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt h17, s17
-; CHECK-SD-NOFP16-NEXT:    fadd s18, s18, s22
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt h21, s21
-; CHECK-SD-NOFP16-NEXT:    fcvt s22, h25
-; CHECK-SD-NOFP16-NEXT:    mov h25, v5.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[2], v26.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h23, s23
-; CHECK-SD-NOFP16-NEXT:    fadd s24, s24, s27
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h20
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h19
-; CHECK-SD-NOFP16-NEXT:    mov h26, v4.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[2], v17.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h17, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fadd s16, s16, s22
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h25
-; CHECK-SD-NOFP16-NEXT:    mov h22, v5.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[3], v23.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h23, s24
-; CHECK-SD-NOFP16-NEXT:    fadd s19, s20, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h21
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h26
-; CHECK-SD-NOFP16-NEXT:    mov h4, v4.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[3], v17.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h16, s16
-; CHECK-SD-NOFP16-NEXT:    fadd s7, s7, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h22
-; CHECK-SD-NOFP16-NEXT:    mov h5, v5.h[7]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[4], v23.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h18, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    fadd s19, s20, s21
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[4], v16.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h7, s7
-; CHECK-SD-NOFP16-NEXT:    fadd s3, s3, s17
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[5], v18.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h16, s19
-; CHECK-SD-NOFP16-NEXT:    fadd s2, s2, s4
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[5], v7.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fadd s4, s6, s5
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[6], v16.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[6], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s4
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[7], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[7], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    ldp d11, d10, [sp], #32 // 16-byte Folded Reload
+; CHECK-SD-NOFP16-NEXT:    fcvtl v6.4s, v2.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v7.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v16.4s, v3.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v17.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v2.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v3.4s, v3.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fmul v6.4s, v7.4s, v6.4s
+; CHECK-SD-NOFP16-NEXT:    fmul v7.4s, v17.4s, v16.4s
+; CHECK-SD-NOFP16-NEXT:    fmul v0.4s, v0.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fmul v1.4s, v1.4s, v3.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v2.4h, v6.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtl v6.4s, v5.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtn v3.4h, v7.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v2.8h, v0.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v4.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v4.4s, v4.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v3.8h, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v2.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v2.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v7.4s, v3.4h
+; CHECK-SD-NOFP16-NEXT:    fadd v0.4s, v1.4s, v0.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v3.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v3.4s, v5.8h
+; CHECK-SD-NOFP16-NEXT:    fadd v5.4s, v7.4s, v6.4s
+; CHECK-SD-NOFP16-NEXT:    fadd v2.4s, v2.4s, v4.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-SD-NOFP16-NEXT:    fadd v3.4s, v1.4s, v3.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v1.4h, v5.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v1.8h, v3.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: fmul_v16f16:
diff --git a/llvm/test/CodeGen/AArch64/fmul.ll b/llvm/test/CodeGen/AArch64/fmul.ll
index fe84fe1f317a..1f49601a1827 100644
--- a/llvm/test/CodeGen/AArch64/fmul.ll
+++ b/llvm/test/CodeGen/AArch64/fmul.ll
@@ -169,60 +169,14 @@ entry:
 define <7 x half> @fmul_v7f16(<7 x half> %a, <7 x half> %b) {
 ; CHECK-SD-NOFP16-LABEL: fmul_v7f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h0
-; CHECK-SD-NOFP16-NEXT:    mov h6, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fmul s4, s5, s4
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fmul s3, s3, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s4
-; CHECK-SD-NOFP16-NEXT:    fmul s4, s7, s6
-; CHECK-SD-NOFP16-NEXT:    mov h6, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fmul s5, s5, s16
-; CHECK-SD-NOFP16-NEXT:    mov h16, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[1], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h7
-; CHECK-SD-NOFP16-NEXT:    mov h7, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[2], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    fmul s3, s6, s3
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[3], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h6
-; CHECK-SD-NOFP16-NEXT:    fmul s6, s16, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[4], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fmul s4, s5, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s6
-; CHECK-SD-NOFP16-NEXT:    fmul s0, s0, s1
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[5], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[6], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v2.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v2.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fmul v2.4s, v3.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fmul v1.4s, v0.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: fmul_v7f16:
@@ -309,60 +263,14 @@ entry:
 define <8 x half> @fmul_v8f16(<8 x half> %a, <8 x half> %b) {
 ; CHECK-SD-NOFP16-LABEL: fmul_v8f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h0
-; CHECK-SD-NOFP16-NEXT:    mov h6, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fmul s4, s5, s4
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fmul s3, s3, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s4
-; CHECK-SD-NOFP16-NEXT:    fmul s4, s7, s6
-; CHECK-SD-NOFP16-NEXT:    mov h6, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fmul s5, s5, s16
-; CHECK-SD-NOFP16-NEXT:    mov h16, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[1], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h6
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h7
-; CHECK-SD-NOFP16-NEXT:    mov h7, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[2], v4.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h4, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    fmul s3, s6, s3
-; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[3], v5.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h6
-; CHECK-SD-NOFP16-NEXT:    fmul s6, s16, s7
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[4], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fmul s4, s5, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s6
-; CHECK-SD-NOFP16-NEXT:    fmul s0, s0, s1
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[5], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[6], v3.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v2.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v2.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v2.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fmul v2.4s, v3.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fmul v1.4s, v0.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: fmul_v8f16:
@@ -394,114 +302,22 @@ entry:
 define <16 x half> @fmul_v16f16(<16 x half> %a, <16 x half> %b) {
 ; CHECK-SD-NOFP16-LABEL: fmul_v16f16:
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov h4, v2.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h6, v3.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h0
-; CHECK-SD-NOFP16-NEXT:    mov h18, v2.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h19, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h1
-; CHECK-SD-NOFP16-NEXT:    mov h22, v3.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h23, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    fmul s16, s17, s16
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    mov h24, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h18
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h19
-; CHECK-SD-NOFP16-NEXT:    mov h19, v2.h[3]
-; CHECK-SD-NOFP16-NEXT:    fmul s20, s21, s20
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h22
-; CHECK-SD-NOFP16-NEXT:    fcvt s22, h23
-; CHECK-SD-NOFP16-NEXT:    fmul s5, s5, s4
-; CHECK-SD-NOFP16-NEXT:    mov h23, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h25, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    fmul s6, s7, s6
-; CHECK-SD-NOFP16-NEXT:    mov h7, v3.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s24, h24
-; CHECK-SD-NOFP16-NEXT:    fmul s17, s18, s17
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h19
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s16
-; CHECK-SD-NOFP16-NEXT:    fmul s18, s22, s21
-; CHECK-SD-NOFP16-NEXT:    mov h22, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt h16, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt h5, s20
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h23
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    mov h20, v2.h[4]
-; CHECK-SD-NOFP16-NEXT:    fmul s19, s24, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt h17, s17
-; CHECK-SD-NOFP16-NEXT:    mov h23, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h24, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[1], v16.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v3.h[4]
-; CHECK-SD-NOFP16-NEXT:    fmul s7, s21, s7
-; CHECK-SD-NOFP16-NEXT:    mov v5.h[1], v6.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h20
-; CHECK-SD-NOFP16-NEXT:    fcvt h19, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h22
-; CHECK-SD-NOFP16-NEXT:    mov h21, v2.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov h22, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[2], v17.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h23
-; CHECK-SD-NOFP16-NEXT:    mov v5.h[2], v6.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s7
-; CHECK-SD-NOFP16-NEXT:    mov h7, v3.h[5]
-; CHECK-SD-NOFP16-NEXT:    fmul s18, s20, s18
-; CHECK-SD-NOFP16-NEXT:    mov h23, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov h20, v2.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h2, v2.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fmul s16, s17, s16
-; CHECK-SD-NOFP16-NEXT:    mov h17, v3.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[3], v19.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h21
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h22
-; CHECK-SD-NOFP16-NEXT:    mov v5.h[3], v6.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h23
-; CHECK-SD-NOFP16-NEXT:    fcvt h18, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h20
-; CHECK-SD-NOFP16-NEXT:    fcvt s22, h24
-; CHECK-SD-NOFP16-NEXT:    fcvt s23, h25
-; CHECK-SD-NOFP16-NEXT:    fcvt h16, s16
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    mov h3, v3.h[7]
-; CHECK-SD-NOFP16-NEXT:    fmul s19, s21, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    fmul s6, s7, s6
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[4], v18.h[0]
-; CHECK-SD-NOFP16-NEXT:    fmul s7, s22, s20
-; CHECK-SD-NOFP16-NEXT:    mov v5.h[4], v16.h[0]
-; CHECK-SD-NOFP16-NEXT:    fmul s16, s23, s17
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fmul s0, s0, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h17, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s7
-; CHECK-SD-NOFP16-NEXT:    fmul s1, s1, s3
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[5], v17.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov v5.h[5], v6.h[0]
-; CHECK-SD-NOFP16-NEXT:    fcvt h6, s16
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[6], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v5.h[6], v6.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v4.h[7], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v5.h[7], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v4.16b
-; CHECK-SD-NOFP16-NEXT:    mov v1.16b, v5.16b
+; CHECK-SD-NOFP16-NEXT:    fcvtl v4.4s, v2.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v5.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v6.4s, v3.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v7.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v2.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v3.4s, v3.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fmul v4.4s, v5.4s, v4.4s
+; CHECK-SD-NOFP16-NEXT:    fmul v5.4s, v7.4s, v6.4s
+; CHECK-SD-NOFP16-NEXT:    fmul v2.4s, v0.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fmul v3.4s, v1.4s, v3.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v4.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v1.4h, v5.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v1.8h, v3.4s
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: fmul_v16f16:
diff --git a/llvm/test/CodeGen/AArch64/fp16-v8-instructions.ll b/llvm/test/CodeGen/AArch64/fp16-v8-instructions.ll
index ded343b990ac..d4130e7a848b 100644
--- a/llvm/test/CodeGen/AArch64/fp16-v8-instructions.ll
+++ b/llvm/test/CodeGen/AArch64/fp16-v8-instructions.ll
@@ -5,60 +5,14 @@
 define <8 x half> @add_h(<8 x half> %a, <8 x half> %b) {
 ; CHECK-CVT-LABEL: add_h:
 ; CHECK-CVT:       // %bb.0: // %entry
-; CHECK-CVT-NEXT:    mov h2, v1.h[1]
-; CHECK-CVT-NEXT:    mov h3, v0.h[1]
-; CHECK-CVT-NEXT:    fcvt s4, h1
-; CHECK-CVT-NEXT:    fcvt s5, h0
-; CHECK-CVT-NEXT:    mov h6, v1.h[2]
-; CHECK-CVT-NEXT:    mov h7, v0.h[2]
-; CHECK-CVT-NEXT:    mov h16, v1.h[3]
-; CHECK-CVT-NEXT:    fcvt s2, h2
-; CHECK-CVT-NEXT:    fcvt s3, h3
-; CHECK-CVT-NEXT:    fadd s4, s5, s4
-; CHECK-CVT-NEXT:    mov h5, v0.h[3]
-; CHECK-CVT-NEXT:    fcvt s6, h6
-; CHECK-CVT-NEXT:    fcvt s7, h7
-; CHECK-CVT-NEXT:    fcvt s16, h16
-; CHECK-CVT-NEXT:    fadd s3, s3, s2
-; CHECK-CVT-NEXT:    fcvt s5, h5
-; CHECK-CVT-NEXT:    fcvt h2, s4
-; CHECK-CVT-NEXT:    fadd s4, s7, s6
-; CHECK-CVT-NEXT:    mov h6, v1.h[4]
-; CHECK-CVT-NEXT:    mov h7, v0.h[4]
-; CHECK-CVT-NEXT:    fcvt h3, s3
-; CHECK-CVT-NEXT:    fadd s5, s5, s16
-; CHECK-CVT-NEXT:    mov h16, v0.h[5]
-; CHECK-CVT-NEXT:    fcvt h4, s4
-; CHECK-CVT-NEXT:    mov v2.h[1], v3.h[0]
-; CHECK-CVT-NEXT:    fcvt s3, h6
-; CHECK-CVT-NEXT:    fcvt s6, h7
-; CHECK-CVT-NEXT:    mov h7, v1.h[5]
-; CHECK-CVT-NEXT:    fcvt h5, s5
-; CHECK-CVT-NEXT:    fcvt s16, h16
-; CHECK-CVT-NEXT:    mov v2.h[2], v4.h[0]
-; CHECK-CVT-NEXT:    mov h4, v1.h[6]
-; CHECK-CVT-NEXT:    fadd s3, s6, s3
-; CHECK-CVT-NEXT:    mov h6, v0.h[6]
-; CHECK-CVT-NEXT:    fcvt s7, h7
-; CHECK-CVT-NEXT:    mov h1, v1.h[7]
-; CHECK-CVT-NEXT:    mov h0, v0.h[7]
-; CHECK-CVT-NEXT:    mov v2.h[3], v5.h[0]
-; CHECK-CVT-NEXT:    fcvt s4, h4
-; CHECK-CVT-NEXT:    fcvt h3, s3
-; CHECK-CVT-NEXT:    fcvt s5, h6
-; CHECK-CVT-NEXT:    fadd s6, s16, s7
-; CHECK-CVT-NEXT:    fcvt s1, h1
-; CHECK-CVT-NEXT:    fcvt s0, h0
-; CHECK-CVT-NEXT:    mov v2.h[4], v3.h[0]
-; CHECK-CVT-NEXT:    fadd s4, s5, s4
-; CHECK-CVT-NEXT:    fcvt h3, s6
-; CHECK-CVT-NEXT:    fadd s0, s0, s1
-; CHECK-CVT-NEXT:    mov v2.h[5], v3.h[0]
-; CHECK-CVT-NEXT:    fcvt h3, s4
-; CHECK-CVT-NEXT:    fcvt h0, s0
-; CHECK-CVT-NEXT:    mov v2.h[6], v3.h[0]
-; CHECK-CVT-NEXT:    mov v2.h[7], v0.h[0]
-; CHECK-CVT-NEXT:    mov v0.16b, v2.16b
+; CHECK-CVT-NEXT:    fcvtl v2.4s, v1.4h
+; CHECK-CVT-NEXT:    fcvtl v3.4s, v0.4h
+; CHECK-CVT-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-CVT-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-CVT-NEXT:    fadd v2.4s, v3.4s, v2.4s
+; CHECK-CVT-NEXT:    fadd v1.4s, v0.4s, v1.4s
+; CHECK-CVT-NEXT:    fcvtn v0.4h, v2.4s
+; CHECK-CVT-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-CVT-NEXT:    ret
 ;
 ; CHECK-FP16-LABEL: add_h:
@@ -74,60 +28,14 @@ entry:
 define <8 x half> @sub_h(<8 x half> %a, <8 x half> %b) {
 ; CHECK-CVT-LABEL: sub_h:
 ; CHECK-CVT:       // %bb.0: // %entry
-; CHECK-CVT-NEXT:    mov h2, v1.h[1]
-; CHECK-CVT-NEXT:    mov h3, v0.h[1]
-; CHECK-CVT-NEXT:    fcvt s4, h1
-; CHECK-CVT-NEXT:    fcvt s5, h0
-; CHECK-CVT-NEXT:    mov h6, v1.h[2]
-; CHECK-CVT-NEXT:    mov h7, v0.h[2]
-; CHECK-CVT-NEXT:    mov h16, v1.h[3]
-; CHECK-CVT-NEXT:    fcvt s2, h2
-; CHECK-CVT-NEXT:    fcvt s3, h3
-; CHECK-CVT-NEXT:    fsub s4, s5, s4
-; CHECK-CVT-NEXT:    mov h5, v0.h[3]
-; CHECK-CVT-NEXT:    fcvt s6, h6
-; CHECK-CVT-NEXT:    fcvt s7, h7
-; CHECK-CVT-NEXT:    fcvt s16, h16
-; CHECK-CVT-NEXT:    fsub s3, s3, s2
-; CHECK-CVT-NEXT:    fcvt s5, h5
-; CHECK-CVT-NEXT:    fcvt h2, s4
-; CHECK-CVT-NEXT:    fsub s4, s7, s6
-; CHECK-CVT-NEXT:    mov h6, v1.h[4]
-; CHECK-CVT-NEXT:    mov h7, v0.h[4]
-; CHECK-CVT-NEXT:    fcvt h3, s3
-; CHECK-CVT-NEXT:    fsub s5, s5, s16
-; CHECK-CVT-NEXT:    mov h16, v0.h[5]
-; CHECK-CVT-NEXT:    fcvt h4, s4
-; CHECK-CVT-NEXT:    mov v2.h[1], v3.h[0]
-; CHECK-CVT-NEXT:    fcvt s3, h6
-; CHECK-CVT-NEXT:    fcvt s6, h7
-; CHECK-CVT-NEXT:    mov h7, v1.h[5]
-; CHECK-CVT-NEXT:    fcvt h5, s5
-; CHECK-CVT-NEXT:    fcvt s16, h16
-; CHECK-CVT-NEXT:    mov v2.h[2], v4.h[0]
-; CHECK-CVT-NEXT:    mov h4, v1.h[6]
-; CHECK-CVT-NEXT:    fsub s3, s6, s3
-; CHECK-CVT-NEXT:    mov h6, v0.h[6]
-; CHECK-CVT-NEXT:    fcvt s7, h7
-; CHECK-CVT-NEXT:    mov h1, v1.h[7]
-; CHECK-CVT-NEXT:    mov h0, v0.h[7]
-; CHECK-CVT-NEXT:    mov v2.h[3], v5.h[0]
-; CHECK-CVT-NEXT:    fcvt s4, h4
-; CHECK-CVT-NEXT:    fcvt h3, s3
-; CHECK-CVT-NEXT:    fcvt s5, h6
-; CHECK-CVT-NEXT:    fsub s6, s16, s7
-; CHECK-CVT-NEXT:    fcvt s1, h1
-; CHECK-CVT-NEXT:    fcvt s0, h0
-; CHECK-CVT-NEXT:    mov v2.h[4], v3.h[0]
-; CHECK-CVT-NEXT:    fsub s4, s5, s4
-; CHECK-CVT-NEXT:    fcvt h3, s6
-; CHECK-CVT-NEXT:    fsub s0, s0, s1
-; CHECK-CVT-NEXT:    mov v2.h[5], v3.h[0]
-; CHECK-CVT-NEXT:    fcvt h3, s4
-; CHECK-CVT-NEXT:    fcvt h0, s0
-; CHECK-CVT-NEXT:    mov v2.h[6], v3.h[0]
-; CHECK-CVT-NEXT:    mov v2.h[7], v0.h[0]
-; CHECK-CVT-NEXT:    mov v0.16b, v2.16b
+; CHECK-CVT-NEXT:    fcvtl v2.4s, v1.4h
+; CHECK-CVT-NEXT:    fcvtl v3.4s, v0.4h
+; CHECK-CVT-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-CVT-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-CVT-NEXT:    fsub v2.4s, v3.4s, v2.4s
+; CHECK-CVT-NEXT:    fsub v1.4s, v0.4s, v1.4s
+; CHECK-CVT-NEXT:    fcvtn v0.4h, v2.4s
+; CHECK-CVT-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-CVT-NEXT:    ret
 ;
 ; CHECK-FP16-LABEL: sub_h:
@@ -143,60 +51,14 @@ entry:
 define <8 x half> @mul_h(<8 x half> %a, <8 x half> %b) {
 ; CHECK-CVT-LABEL: mul_h:
 ; CHECK-CVT:       // %bb.0: // %entry
-; CHECK-CVT-NEXT:    mov h2, v1.h[1]
-; CHECK-CVT-NEXT:    mov h3, v0.h[1]
-; CHECK-CVT-NEXT:    fcvt s4, h1
-; CHECK-CVT-NEXT:    fcvt s5, h0
-; CHECK-CVT-NEXT:    mov h6, v1.h[2]
-; CHECK-CVT-NEXT:    mov h7, v0.h[2]
-; CHECK-CVT-NEXT:    mov h16, v1.h[3]
-; CHECK-CVT-NEXT:    fcvt s2, h2
-; CHECK-CVT-NEXT:    fcvt s3, h3
-; CHECK-CVT-NEXT:    fmul s4, s5, s4
-; CHECK-CVT-NEXT:    mov h5, v0.h[3]
-; CHECK-CVT-NEXT:    fcvt s6, h6
-; CHECK-CVT-NEXT:    fcvt s7, h7
-; CHECK-CVT-NEXT:    fcvt s16, h16
-; CHECK-CVT-NEXT:    fmul s3, s3, s2
-; CHECK-CVT-NEXT:    fcvt s5, h5
-; CHECK-CVT-NEXT:    fcvt h2, s4
-; CHECK-CVT-NEXT:    fmul s4, s7, s6
-; CHECK-CVT-NEXT:    mov h6, v1.h[4]
-; CHECK-CVT-NEXT:    mov h7, v0.h[4]
-; CHECK-CVT-NEXT:    fcvt h3, s3
-; CHECK-CVT-NEXT:    fmul s5, s5, s16
-; CHECK-CVT-NEXT:    mov h16, v0.h[5]
-; CHECK-CVT-NEXT:    fcvt h4, s4
-; CHECK-CVT-NEXT:    mov v2.h[1], v3.h[0]
-; CHECK-CVT-NEXT:    fcvt s3, h6
-; CHECK-CVT-NEXT:    fcvt s6, h7
-; CHECK-CVT-NEXT:    mov h7, v1.h[5]
-; CHECK-CVT-NEXT:    fcvt h5, s5
-; CHECK-CVT-NEXT:    fcvt s16, h16
-; CHECK-CVT-NEXT:    mov v2.h[2], v4.h[0]
-; CHECK-CVT-NEXT:    mov h4, v1.h[6]
-; CHECK-CVT-NEXT:    fmul s3, s6, s3
-; CHECK-CVT-NEXT:    mov h6, v0.h[6]
-; CHECK-CVT-NEXT:    fcvt s7, h7
-; CHECK-CVT-NEXT:    mov h1, v1.h[7]
-; CHECK-CVT-NEXT:    mov h0, v0.h[7]
-; CHECK-CVT-NEXT:    mov v2.h[3], v5.h[0]
-; CHECK-CVT-NEXT:    fcvt s4, h4
-; CHECK-CVT-NEXT:    fcvt h3, s3
-; CHECK-CVT-NEXT:    fcvt s5, h6
-; CHECK-CVT-NEXT:    fmul s6, s16, s7
-; CHECK-CVT-NEXT:    fcvt s1, h1
-; CHECK-CVT-NEXT:    fcvt s0, h0
-; CHECK-CVT-NEXT:    mov v2.h[4], v3.h[0]
-; CHECK-CVT-NEXT:    fmul s4, s5, s4
-; CHECK-CVT-NEXT:    fcvt h3, s6
-; CHECK-CVT-NEXT:    fmul s0, s0, s1
-; CHECK-CVT-NEXT:    mov v2.h[5], v3.h[0]
-; CHECK-CVT-NEXT:    fcvt h3, s4
-; CHECK-CVT-NEXT:    fcvt h0, s0
-; CHECK-CVT-NEXT:    mov v2.h[6], v3.h[0]
-; CHECK-CVT-NEXT:    mov v2.h[7], v0.h[0]
-; CHECK-CVT-NEXT:    mov v0.16b, v2.16b
+; CHECK-CVT-NEXT:    fcvtl v2.4s, v1.4h
+; CHECK-CVT-NEXT:    fcvtl v3.4s, v0.4h
+; CHECK-CVT-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-CVT-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-CVT-NEXT:    fmul v2.4s, v3.4s, v2.4s
+; CHECK-CVT-NEXT:    fmul v1.4s, v0.4s, v1.4s
+; CHECK-CVT-NEXT:    fcvtn v0.4h, v2.4s
+; CHECK-CVT-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-CVT-NEXT:    ret
 ;
 ; CHECK-FP16-LABEL: mul_h:
@@ -212,60 +74,14 @@ entry:
 define <8 x half> @div_h(<8 x half> %a, <8 x half> %b) {
 ; CHECK-CVT-LABEL: div_h:
 ; CHECK-CVT:       // %bb.0: // %entry
-; CHECK-CVT-NEXT:    mov h2, v1.h[1]
-; CHECK-CVT-NEXT:    mov h3, v0.h[1]
-; CHECK-CVT-NEXT:    fcvt s4, h0
-; CHECK-CVT-NEXT:    mov h5, v0.h[2]
-; CHECK-CVT-NEXT:    mov h6, v0.h[3]
-; CHECK-CVT-NEXT:    mov h7, v0.h[4]
-; CHECK-CVT-NEXT:    mov h16, v0.h[5]
-; CHECK-CVT-NEXT:    mov h17, v0.h[6]
-; CHECK-CVT-NEXT:    mov h0, v0.h[7]
-; CHECK-CVT-NEXT:    fcvt s2, h2
-; CHECK-CVT-NEXT:    fcvt s3, h3
-; CHECK-CVT-NEXT:    fcvt s5, h5
-; CHECK-CVT-NEXT:    fcvt s6, h6
-; CHECK-CVT-NEXT:    fcvt s7, h7
-; CHECK-CVT-NEXT:    fcvt s16, h16
-; CHECK-CVT-NEXT:    fcvt s17, h17
-; CHECK-CVT-NEXT:    fcvt s0, h0
-; CHECK-CVT-NEXT:    fdiv s2, s3, s2
-; CHECK-CVT-NEXT:    fcvt s3, h1
-; CHECK-CVT-NEXT:    fdiv s3, s4, s3
-; CHECK-CVT-NEXT:    mov h4, v1.h[2]
-; CHECK-CVT-NEXT:    fcvt h18, s2
-; CHECK-CVT-NEXT:    fcvt s4, h4
-; CHECK-CVT-NEXT:    fdiv s4, s5, s4
-; CHECK-CVT-NEXT:    mov h5, v1.h[3]
-; CHECK-CVT-NEXT:    fcvt h2, s3
-; CHECK-CVT-NEXT:    fcvt s5, h5
-; CHECK-CVT-NEXT:    mov v2.h[1], v18.h[0]
-; CHECK-CVT-NEXT:    fdiv s5, s6, s5
-; CHECK-CVT-NEXT:    mov h6, v1.h[4]
-; CHECK-CVT-NEXT:    fcvt h4, s4
-; CHECK-CVT-NEXT:    fcvt s6, h6
-; CHECK-CVT-NEXT:    mov v2.h[2], v4.h[0]
-; CHECK-CVT-NEXT:    fdiv s6, s7, s6
-; CHECK-CVT-NEXT:    mov h7, v1.h[5]
-; CHECK-CVT-NEXT:    fcvt h4, s5
-; CHECK-CVT-NEXT:    fcvt s7, h7
-; CHECK-CVT-NEXT:    mov v2.h[3], v4.h[0]
-; CHECK-CVT-NEXT:    fdiv s7, s16, s7
-; CHECK-CVT-NEXT:    mov h16, v1.h[6]
-; CHECK-CVT-NEXT:    mov h1, v1.h[7]
-; CHECK-CVT-NEXT:    fcvt s16, h16
-; CHECK-CVT-NEXT:    fcvt s1, h1
-; CHECK-CVT-NEXT:    fdiv s3, s17, s16
-; CHECK-CVT-NEXT:    fdiv s0, s0, s1
-; CHECK-CVT-NEXT:    fcvt h1, s6
-; CHECK-CVT-NEXT:    mov v2.h[4], v1.h[0]
-; CHECK-CVT-NEXT:    fcvt h1, s7
-; CHECK-CVT-NEXT:    mov v2.h[5], v1.h[0]
-; CHECK-CVT-NEXT:    fcvt h1, s3
-; CHECK-CVT-NEXT:    mov v2.h[6], v1.h[0]
-; CHECK-CVT-NEXT:    fcvt h0, s0
-; CHECK-CVT-NEXT:    mov v2.h[7], v0.h[0]
-; CHECK-CVT-NEXT:    mov v0.16b, v2.16b
+; CHECK-CVT-NEXT:    fcvtl v2.4s, v1.4h
+; CHECK-CVT-NEXT:    fcvtl v3.4s, v0.4h
+; CHECK-CVT-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-CVT-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-CVT-NEXT:    fdiv v2.4s, v3.4s, v2.4s
+; CHECK-CVT-NEXT:    fdiv v1.4s, v0.4s, v1.4s
+; CHECK-CVT-NEXT:    fcvtn v0.4h, v2.4s
+; CHECK-CVT-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-CVT-NEXT:    ret
 ;
 ; CHECK-FP16-LABEL: div_h:
diff --git a/llvm/test/CodeGen/AArch64/sve-fp-reciprocal.ll b/llvm/test/CodeGen/AArch64/sve-fp-reciprocal.ll
index 1b2e0be6111c..aefc8de43143 100644
--- a/llvm/test/CodeGen/AArch64/sve-fp-reciprocal.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fp-reciprocal.ll
@@ -17,8 +17,6 @@ define  @fdiv_recip_8f16( %a,  @fsqrt_recip_8f16( %a) #0 {
 ; CHECK-NEXT:    fcmne p0.h, p0/z, z0.h, #0.0
 ; CHECK-NEXT:    frsqrts z2.h, z0.h, z2.h
 ; CHECK-NEXT:    fmul z1.h, z1.h, z2.h
-; CHECK-NEXT:    fmul z2.h, z1.h, z1.h
-; CHECK-NEXT:    frsqrts z2.h, z0.h, z2.h
-; CHECK-NEXT:    fmul z1.h, z1.h, z2.h
 ; CHECK-NEXT:    fmul z0.h, p0/m, z0.h, z1.h
 ; CHECK-NEXT:    ret
   %fsqrt = call fast  @llvm.sqrt.nxv8f16( %a)
diff --git a/llvm/test/CodeGen/AArch64/vecreduce-fadd.ll b/llvm/test/CodeGen/AArch64/vecreduce-fadd.ll
index 99c6808724b5..03db1d0d433d 100644
--- a/llvm/test/CodeGen/AArch64/vecreduce-fadd.ll
+++ b/llvm/test/CodeGen/AArch64/vecreduce-fadd.ll
@@ -147,79 +147,48 @@ define double @add_D(<2 x double> %bin.rdx)  {
 define half @add_2H(<16 x half> %bin.rdx)  {
 ; CHECK-SD-NOFP16-LABEL: add_2H:
 ; CHECK-SD-NOFP16:       // %bb.0:
-; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h0
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fadd s4, s5, s4
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    fadd s2, s3, s2
-; CHECK-SD-NOFP16-NEXT:    mov h3, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fadd s3, s5, s3
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    fadd s2, s4, s2
-; CHECK-SD-NOFP16-NEXT:    mov h4, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fadd s4, s5, s4
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fadd s2, s2, s3
-; CHECK-SD-NOFP16-NEXT:    mov h3, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
+; CHECK-SD-NOFP16-NEXT:    fcvtl v2.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fadd v2.4s, v3.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fadd v0.4s, v0.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v1.4h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v1.8h, v0.4s
+; CHECK-SD-NOFP16-NEXT:    mov h0, v1.h[1]
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h1
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fadd s0, s2, s0
+; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[2]
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
 ; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fadd s3, s5, s3
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    fadd s2, s2, s4
-; CHECK-SD-NOFP16-NEXT:    mov h4, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fadd s0, s0, s2
+; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[3]
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
 ; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fadd s4, s5, s4
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    fadd s2, s2, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s4
-; CHECK-SD-NOFP16-NEXT:    mov h4, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
 ; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-SD-NOFP16-NEXT:    fadd s0, s0, s2
+; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[4]
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
 ; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fadd s0, s0, s1
-; CHECK-SD-NOFP16-NEXT:    fadd s2, s2, s3
-; CHECK-SD-NOFP16-NEXT:    fadd s3, s5, s4
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fadd s0, s0, s2
+; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[5]
 ; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
 ; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fadd s0, s0, s2
+; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[6]
+; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
 ; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fadd s2, s2, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
 ; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    fadd s0, s1, s0
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fadd s0, s0, s2
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fadd s0, s0, s1
 ; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
@@ -574,78 +543,47 @@ exit:
 define half @fadd_reduct_reassoc_v8f16(<8 x half> %a, <8 x half> %b) {
 ; CHECK-SD-NOFP16-LABEL: fadd_reduct_reassoc_v8f16:
 ; CHECK-SD-NOFP16:       // %bb.0:
-; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h0
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fadd s2, s4, s2
-; CHECK-SD-NOFP16-NEXT:    fadd s3, s5, s3
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h5, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fadd s2, s2, s4
-; CHECK-SD-NOFP16-NEXT:    fadd s3, s3, s5
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h5, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fadd s2, s2, s4
-; CHECK-SD-NOFP16-NEXT:    fadd s3, s3, s5
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h5, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
+; CHECK-SD-NOFP16-NEXT:    fcvtl v2.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fadd v2.4s, v3.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fadd v0.4s, v0.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v1.4h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v1.8h, v0.4s
+; CHECK-SD-NOFP16-NEXT:    mov h0, v1.h[1]
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h1
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fadd s0, s2, s0
+; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[2]
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
 ; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fadd s2, s2, s4
-; CHECK-SD-NOFP16-NEXT:    fadd s3, s3, s5
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov h5, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fadd s0, s0, s2
+; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[3]
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
 ; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fadd s2, s2, s4
-; CHECK-SD-NOFP16-NEXT:    fadd s3, s3, s5
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h5, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
 ; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-SD-NOFP16-NEXT:    fadd s0, s0, s2
+; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[4]
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
 ; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fadd s2, s2, s4
-; CHECK-SD-NOFP16-NEXT:    fadd s3, s3, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fadd s0, s0, s2
+; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[5]
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
 ; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fadd s0, s2, s0
-; CHECK-SD-NOFP16-NEXT:    fadd s1, s3, s1
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fadd s0, s0, s2
+; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[6]
+; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
 ; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
 ; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
 ; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fadd s0, s0, s2
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
 ; CHECK-SD-NOFP16-NEXT:    fadd s0, s0, s1
 ; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
 ; CHECK-SD-NOFP16-NEXT:    ret
diff --git a/llvm/test/CodeGen/AArch64/vecreduce-fmul.ll b/llvm/test/CodeGen/AArch64/vecreduce-fmul.ll
index e85384e46222..da75a8059521 100644
--- a/llvm/test/CodeGen/AArch64/vecreduce-fmul.ll
+++ b/llvm/test/CodeGen/AArch64/vecreduce-fmul.ll
@@ -182,79 +182,48 @@ define double @mul_D(<2 x double> %bin.rdx)  {
 define half @mul_2H(<16 x half> %bin.rdx)  {
 ; CHECK-SD-NOFP16-LABEL: mul_2H:
 ; CHECK-SD-NOFP16:       // %bb.0:
-; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h0
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fmul s4, s5, s4
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    fmul s2, s3, s2
-; CHECK-SD-NOFP16-NEXT:    mov h3, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fmul s3, s5, s3
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    fmul s2, s4, s2
-; CHECK-SD-NOFP16-NEXT:    mov h4, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fmul s4, s5, s4
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    fmul s2, s2, s3
-; CHECK-SD-NOFP16-NEXT:    mov h3, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
+; CHECK-SD-NOFP16-NEXT:    fcvtl v2.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fmul v2.4s, v3.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fmul v0.4s, v0.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v1.4h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v1.8h, v0.4s
+; CHECK-SD-NOFP16-NEXT:    mov h0, v1.h[1]
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h1
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fmul s0, s2, s0
+; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[2]
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
 ; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fmul s3, s5, s3
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    fmul s2, s2, s4
-; CHECK-SD-NOFP16-NEXT:    mov h4, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fmul s0, s0, s2
+; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[3]
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
 ; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fmul s4, s5, s4
-; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    fmul s2, s2, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s4
-; CHECK-SD-NOFP16-NEXT:    mov h4, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
 ; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-SD-NOFP16-NEXT:    fmul s0, s0, s2
+; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[4]
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
 ; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fmul s0, s0, s1
-; CHECK-SD-NOFP16-NEXT:    fmul s2, s2, s3
-; CHECK-SD-NOFP16-NEXT:    fmul s3, s5, s4
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fmul s0, s0, s2
+; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[5]
 ; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
 ; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fmul s0, s0, s2
+; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[6]
+; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
 ; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fmul s2, s2, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
 ; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    fmul s0, s1, s0
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fmul s0, s0, s2
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fmul s0, s0, s1
 ; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
@@ -361,78 +330,47 @@ define float @mul_S_init_42(<4 x float> %bin.rdx)  {
 define half @fmul_reduct_reassoc_v8f16(<8 x half> %a, <8 x half> %b) {
 ; CHECK-SD-NOFP16-LABEL: fmul_reduct_reassoc_v8f16:
 ; CHECK-SD-NOFP16:       // %bb.0:
-; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h0
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fmul s2, s4, s2
-; CHECK-SD-NOFP16-NEXT:    fmul s3, s5, s3
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h5, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fmul s2, s2, s4
-; CHECK-SD-NOFP16-NEXT:    fmul s3, s3, s5
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h5, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fmul s2, s2, s4
-; CHECK-SD-NOFP16-NEXT:    fmul s3, s3, s5
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[4]
-; CHECK-SD-NOFP16-NEXT:    mov h5, v1.h[4]
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
+; CHECK-SD-NOFP16-NEXT:    fcvtl v2.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fmul v2.4s, v3.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fmul v0.4s, v0.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v1.4h, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn2 v1.8h, v0.4s
+; CHECK-SD-NOFP16-NEXT:    mov h0, v1.h[1]
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h1
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fmul s0, s2, s0
+; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[2]
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
 ; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fmul s2, s2, s4
-; CHECK-SD-NOFP16-NEXT:    fmul s3, s3, s5
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[5]
-; CHECK-SD-NOFP16-NEXT:    mov h5, v1.h[5]
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fmul s0, s0, s2
+; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[3]
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
 ; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fmul s2, s2, s4
-; CHECK-SD-NOFP16-NEXT:    fmul s3, s3, s5
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h5, v1.h[6]
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
 ; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-SD-NOFP16-NEXT:    fmul s0, s0, s2
+; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[4]
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
 ; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fmul s2, s2, s4
-; CHECK-SD-NOFP16-NEXT:    fmul s3, s3, s5
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
-; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fmul s0, s0, s2
+; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[5]
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
 ; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fmul s0, s2, s0
-; CHECK-SD-NOFP16-NEXT:    fmul s1, s3, s1
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fmul s0, s0, s2
+; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[6]
+; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
 ; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
 ; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
 ; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fmul s0, s0, s2
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
 ; CHECK-SD-NOFP16-NEXT:    fmul s0, s0, s1
 ; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
 ; CHECK-SD-NOFP16-NEXT:    ret
-- 
GitLab


From 0d7dba6300b452c751cde4eeaedcbfeb86f6185c Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot 
Date: Fri, 8 Mar 2024 20:02:42 +0000
Subject: [PATCH 0078/1674] [gn build] Port 087666811452

---
 llvm/utils/gn/secondary/libcxx/include/BUILD.gn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
index 72aa8b4e8e38..cb52d99e5b2a 100644
--- a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
+++ b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
@@ -450,7 +450,6 @@ if (current_toolchain == default_toolchain) {
       "__format/format_context.h",
       "__format/format_error.h",
       "__format/format_functions.h",
-      "__format/format_fwd.h",
       "__format/format_parse_context.h",
       "__format/format_string.h",
       "__format/format_to_n_result.h",
@@ -500,6 +499,7 @@ if (current_toolchain == default_toolchain) {
       "__fwd/array.h",
       "__fwd/bit_reference.h",
       "__fwd/complex.h",
+      "__fwd/format.h",
       "__fwd/fstream.h",
       "__fwd/functional.h",
       "__fwd/ios.h",
-- 
GitLab


From 0861755e597e671424ee369f36dff5591b7178ee Mon Sep 17 00:00:00 2001
From: Florian Mayer 
Date: Fri, 8 Mar 2024 12:12:35 -0800
Subject: [PATCH 0079/1674] Revert "[IR] Add new Range attribute using new
 ConstantRange Attribute type" (#84549)

Reverts llvm/llvm-project#83171

broke sanitizer buildbot
https://lab.llvm.org/buildbot/#/builders/168/builds/19110/steps/10/logs/stdio
---
 llvm/docs/LangRef.rst                         | 16 ----
 llvm/include/llvm/ADT/FoldingSet.h            |  7 --
 llvm/include/llvm/AsmParser/LLParser.h        |  1 -
 llvm/include/llvm/Bitcode/LLVMBitCodes.h      |  1 -
 llvm/include/llvm/IR/Attributes.h             | 23 ------
 llvm/include/llvm/IR/Attributes.td            |  6 --
 llvm/lib/AsmParser/LLParser.cpp               | 43 ----------
 llvm/lib/Bitcode/Reader/BitcodeReader.cpp     | 41 ----------
 llvm/lib/Bitcode/Writer/BitcodeWriter.cpp     | 61 +++++---------
 llvm/lib/IR/AttributeImpl.h                   | 28 +------
 llvm/lib/IR/Attributes.cpp                    | 80 +------------------
 llvm/lib/IR/Verifier.cpp                      |  5 --
 llvm/lib/Transforms/Utils/CodeExtractor.cpp   |  1 -
 .../range-attribute-invalid-range.ll          |  6 --
 .../Assembler/range-attribute-invalid-type.ll |  6 --
 llvm/test/Bitcode/attributes.ll               | 10 ---
 llvm/test/Verifier/range-attr.ll              | 19 -----
 llvm/utils/TableGen/Attributes.cpp            |  9 +--
 18 files changed, 26 insertions(+), 337 deletions(-)
 delete mode 100644 llvm/test/Assembler/range-attribute-invalid-range.ll
 delete mode 100644 llvm/test/Assembler/range-attribute-invalid-type.ll
 delete mode 100644 llvm/test/Verifier/range-attr.ll

diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index b70220dec926..a7b77d6f776a 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -1635,22 +1635,6 @@ Currently, only the following parameter attributes are defined:
 
     This attribute cannot be applied to return values.
 
-``range( , )``
-    This attribute expresses the possible range of the parameter or return value.
-    If the value is not in the specified range, it is converted to poison.
-    The arguments passed to ``range`` have the following properties:
-
-    -  The type must match the scalar type of the parameter or return value.
-    -  The pair ``a,b`` represents the range ``[a,b)``.
-    -  Both ``a`` and ``b`` are constants.
-    -  The range is allowed to wrap.
-    -  The range should not represent the full or empty set. That is, ``a!=b``.
-    
-    This attribute may only be applied to parameters or return values with integer 
-    or vector of integer types.
-    
-    For vector-typed parameters, the range is applied element-wise.
-
 .. _gc:
 
 Garbage Collector Strategy Names
diff --git a/llvm/include/llvm/ADT/FoldingSet.h b/llvm/include/llvm/ADT/FoldingSet.h
index ddc3e52255d6..f82eabd5044b 100644
--- a/llvm/include/llvm/ADT/FoldingSet.h
+++ b/llvm/include/llvm/ADT/FoldingSet.h
@@ -16,7 +16,6 @@
 #ifndef LLVM_ADT_FOLDINGSET_H
 #define LLVM_ADT_FOLDINGSET_H
 
-#include "llvm/ADT/APInt.h"
 #include "llvm/ADT/Hashing.h"
 #include "llvm/ADT/STLForwardCompat.h"
 #include "llvm/ADT/SmallVector.h"
@@ -355,12 +354,6 @@ public:
     AddInteger(unsigned(I));
     AddInteger(unsigned(I >> 32));
   }
-  void AddInteger(const APInt &Int) {
-    const auto *Parts = Int.getRawData();
-    for (int i = 0, N = Int.getNumWords(); i < N; ++i) {
-      AddInteger(Parts[i]);
-    }
-  }
 
   void AddBoolean(bool B) { AddInteger(B ? 1U : 0U); }
   void AddString(StringRef String);
diff --git a/llvm/include/llvm/AsmParser/LLParser.h b/llvm/include/llvm/AsmParser/LLParser.h
index e85728aa3c0d..e5e1ade8b38b 100644
--- a/llvm/include/llvm/AsmParser/LLParser.h
+++ b/llvm/include/llvm/AsmParser/LLParser.h
@@ -369,7 +369,6 @@ namespace llvm {
     bool parseFnAttributeValuePairs(AttrBuilder &B,
                                     std::vector &FwdRefAttrGrps,
                                     bool inAttrGrp, LocTy &BuiltinLoc);
-    bool parseRangeAttr(AttrBuilder &B);
     bool parseRequiredTypeAttr(AttrBuilder &B, lltok::Kind AttrToken,
                                Attribute::AttrKind AttrKind);
 
diff --git a/llvm/include/llvm/Bitcode/LLVMBitCodes.h b/llvm/include/llvm/Bitcode/LLVMBitCodes.h
index c0a52d64a101..c6f0ddf29a6d 100644
--- a/llvm/include/llvm/Bitcode/LLVMBitCodes.h
+++ b/llvm/include/llvm/Bitcode/LLVMBitCodes.h
@@ -724,7 +724,6 @@ enum AttributeKindCodes {
   ATTR_KIND_WRITABLE = 89,
   ATTR_KIND_CORO_ONLY_DESTROY_WHEN_COMPLETE = 90,
   ATTR_KIND_DEAD_ON_UNWIND = 91,
-  ATTR_KIND_RANGE = 92,
 };
 
 enum ComdatSelectionKindCodes {
diff --git a/llvm/include/llvm/IR/Attributes.h b/llvm/include/llvm/IR/Attributes.h
index 0c2a02514ba0..a4ebe5d732f5 100644
--- a/llvm/include/llvm/IR/Attributes.h
+++ b/llvm/include/llvm/IR/Attributes.h
@@ -37,7 +37,6 @@ class AttributeMask;
 class AttributeImpl;
 class AttributeListImpl;
 class AttributeSetNode;
-class ConstantRange;
 class FoldingSetNodeID;
 class Function;
 class LLVMContext;
@@ -104,9 +103,6 @@ public:
   static bool isTypeAttrKind(AttrKind Kind) {
     return Kind >= FirstTypeAttr && Kind <= LastTypeAttr;
   }
-  static bool isConstantRangeAttrKind(AttrKind Kind) {
-    return Kind >= FirstConstantRangeAttr && Kind <= LastConstantRangeAttr;
-  }
 
   static bool canUseAsFnAttr(AttrKind Kind);
   static bool canUseAsParamAttr(AttrKind Kind);
@@ -129,8 +125,6 @@ public:
   static Attribute get(LLVMContext &Context, StringRef Kind,
                        StringRef Val = StringRef());
   static Attribute get(LLVMContext &Context, AttrKind Kind, Type *Ty);
-  static Attribute get(LLVMContext &Context, AttrKind Kind,
-                       const ConstantRange &CR);
 
   /// Return a uniquified Attribute object that has the specific
   /// alignment set.
@@ -186,9 +180,6 @@ public:
   /// Return true if the attribute is a type attribute.
   bool isTypeAttribute() const;
 
-  /// Return true if the attribute is a ConstantRange attribute.
-  bool isConstantRangeAttribute() const;
-
   /// Return true if the attribute is any kind of attribute.
   bool isValid() const { return pImpl; }
 
@@ -222,10 +213,6 @@ public:
   /// a type attribute.
   Type *getValueAsType() const;
 
-  /// Return the attribute's value as a ConstantRange. This requires the
-  /// attribute to be a ConstantRange attribute.
-  ConstantRange getValueAsConstantRange() const;
-
   /// Returns the alignment field of an attribute as a byte alignment
   /// value.
   MaybeAlign getAlignment() const;
@@ -264,9 +251,6 @@ public:
   /// Return the FPClassTest for nofpclass
   FPClassTest getNoFPClass() const;
 
-  /// Returns the value of the range attribute.
-  ConstantRange getRange() const;
-
   /// The Attribute is converted to a string of equivalent mnemonic. This
   /// is, presumably, for writing out the mnemonics for the assembly writer.
   std::string getAsString(bool InAttrGrp = false) const;
@@ -1205,13 +1189,6 @@ public:
   // Add nofpclass attribute
   AttrBuilder &addNoFPClassAttr(FPClassTest NoFPClassMask);
 
-  /// Add a ConstantRange attribute with the given range.
-  AttrBuilder &addConstantRangeAttr(Attribute::AttrKind Kind,
-                                    const ConstantRange &CR);
-
-  /// Add range attribute.
-  AttrBuilder &addRangeAttr(const ConstantRange &CR);
-
   ArrayRef attrs() const { return Attrs; }
 
   bool operator==(const AttrBuilder &B) const;
diff --git a/llvm/include/llvm/IR/Attributes.td b/llvm/include/llvm/IR/Attributes.td
index cef8b17769f0..08afecf32015 100644
--- a/llvm/include/llvm/IR/Attributes.td
+++ b/llvm/include/llvm/IR/Attributes.td
@@ -44,9 +44,6 @@ class StrBoolAttr : Attr;
 /// Arbitrary string attribute.
 class ComplexStrAttr P> : Attr;
 
-/// ConstantRange attribute.
-class ConstantRangeAttr P> : Attr;
-
 /// Target-independent enum attributes.
 
 /// Alignment of parameter (5 bits) stored as log2 of alignment with +1 bias.
@@ -221,9 +218,6 @@ def OptimizeNone : EnumAttr<"optnone", [FnAttr]>;
 /// Similar to byval but without a copy.
 def Preallocated : TypeAttr<"preallocated", [FnAttr, ParamAttr]>;
 
-/// Parameter or return value is within the specified range.
-def Range : ConstantRangeAttr<"range", [ParamAttr, RetAttr]>;
-
 /// Function does not access memory.
 def ReadNone : EnumAttr<"readnone", [ParamAttr]>;
 
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index 78bcd94e23fa..e140c9419520 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -1596,8 +1596,6 @@ bool LLParser::parseEnumAttribute(Attribute::AttrKind Attr, AttrBuilder &B,
 
     return true;
   }
-  case Attribute::Range:
-    return parseRangeAttr(B);
   default:
     B.addAttribute(Attr);
     Lex.Lex();
@@ -3010,47 +3008,6 @@ bool LLParser::parseRequiredTypeAttr(AttrBuilder &B, lltok::Kind AttrToken,
   return false;
 }
 
-/// parseRangeAttr
-///   ::= range( ,)
-bool LLParser::parseRangeAttr(AttrBuilder &B) {
-  Lex.Lex();
-
-  APInt Lower;
-  APInt Upper;
-  Type *Ty = nullptr;
-  LocTy TyLoc;
-
-  auto ParseAPSInt = [&](unsigned BitWidth, APInt &Val) {
-    if (Lex.getKind() != lltok::APSInt)
-      return tokError("expected integer");
-    if (Lex.getAPSIntVal().getBitWidth() > BitWidth)
-      return tokError(
-          "integer is too large for the bit width of specified type");
-    Val = Lex.getAPSIntVal().extend(BitWidth);
-    Lex.Lex();
-    return false;
-  };
-
-  if (parseToken(lltok::lparen, "expected '('") || parseType(Ty, TyLoc))
-    return true;
-  if (!Ty->isIntegerTy())
-    return error(TyLoc, "the range must have integer type!");
-
-  unsigned BitWidth = Ty->getPrimitiveSizeInBits();
-
-  if (ParseAPSInt(BitWidth, Lower) ||
-      parseToken(lltok::comma, "expected ','") || ParseAPSInt(BitWidth, Upper))
-    return true;
-  if (Lower == Upper)
-    return tokError("the range should not represent the full or empty set!");
-
-  if (parseToken(lltok::rparen, "expected ')'"))
-    return true;
-
-  B.addRangeAttr(ConstantRange(Lower, Upper));
-  return false;
-}
-
 /// parseOptionalOperandBundles
 ///    ::= /*empty*/
 ///    ::= '[' OperandBundle [, OperandBundle ]* ']'
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index 9c63116114f3..832907a3f53f 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -815,30 +815,6 @@ private:
     return getFnValueByID(ValNo, Ty, TyID, ConstExprInsertBB);
   }
 
-  Expected readConstantRange(ArrayRef Record,
-                                            unsigned &OpNum) {
-    if (Record.size() - OpNum < 3)
-      return error("Too few records for range");
-    unsigned BitWidth = Record[OpNum++];
-    if (BitWidth > 64) {
-      unsigned LowerActiveWords = Record[OpNum];
-      unsigned UpperActiveWords = Record[OpNum++] >> 32;
-      if (Record.size() - OpNum < LowerActiveWords + UpperActiveWords)
-        return error("Too few records for range");
-      APInt Lower =
-          readWideAPInt(ArrayRef(&Record[OpNum], LowerActiveWords), BitWidth);
-      OpNum += LowerActiveWords;
-      APInt Upper =
-          readWideAPInt(ArrayRef(&Record[OpNum], UpperActiveWords), BitWidth);
-      OpNum += UpperActiveWords;
-      return ConstantRange(Lower, Upper);
-    } else {
-      int64_t Start = BitcodeReader::decodeSignRotatedValue(Record[OpNum++]);
-      int64_t End = BitcodeReader::decodeSignRotatedValue(Record[OpNum++]);
-      return ConstantRange(APInt(BitWidth, Start), APInt(BitWidth, End));
-    }
-  }
-
   /// Upgrades old-style typeless byval/sret/inalloca attributes by adding the
   /// corresponding argument's pointee type. Also upgrades intrinsics that now
   /// require an elementtype attribute.
@@ -2127,8 +2103,6 @@ static Attribute::AttrKind getAttrFromCode(uint64_t Code) {
     return Attribute::CoroDestroyOnlyWhenComplete;
   case bitc::ATTR_KIND_DEAD_ON_UNWIND:
     return Attribute::DeadOnUnwind;
-  case bitc::ATTR_KIND_RANGE:
-    return Attribute::Range;
   }
 }
 
@@ -2298,21 +2272,6 @@ Error BitcodeReader::parseAttributeGroupBlock() {
             return error("Not a type attribute");
 
           B.addTypeAttr(Kind, HasType ? getTypeByID(Record[++i]) : nullptr);
-        } else if (Record[i] == 7) {
-          Attribute::AttrKind Kind;
-
-          i++;
-          if (Error Err = parseAttrKind(Record[i++], &Kind))
-            return Err;
-          if (!Attribute::isConstantRangeAttrKind(Kind))
-            return error("Not a ConstantRange attribute");
-
-          Expected MaybeCR = readConstantRange(Record, i);
-          if (!MaybeCR)
-            return MaybeCR.takeError();
-          i--;
-
-          B.addConstantRangeAttr(Kind, MaybeCR.get());
         } else {
           return error("Invalid attribute group entry");
         }
diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
index 597f49332fad..656f2a6ce870 100644
--- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -844,8 +844,6 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) {
     return bitc::ATTR_KIND_CORO_ONLY_DESTROY_WHEN_COMPLETE;
   case Attribute::DeadOnUnwind:
     return bitc::ATTR_KIND_DEAD_ON_UNWIND;
-  case Attribute::Range:
-    return bitc::ATTR_KIND_RANGE;
   case Attribute::EndAttrKinds:
     llvm_unreachable("Can not encode end-attribute kinds marker.");
   case Attribute::None:
@@ -858,39 +856,6 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) {
   llvm_unreachable("Trying to encode unknown attribute");
 }
 
-static void emitSignedInt64(SmallVectorImpl &Vals, uint64_t V) {
-  if ((int64_t)V >= 0)
-    Vals.push_back(V << 1);
-  else
-    Vals.push_back((-V << 1) | 1);
-}
-
-static void emitWideAPInt(SmallVectorImpl &Vals, const APInt &A) {
-  // We have an arbitrary precision integer value to write whose
-  // bit width is > 64. However, in canonical unsigned integer
-  // format it is likely that the high bits are going to be zero.
-  // So, we only write the number of active words.
-  unsigned NumWords = A.getActiveWords();
-  const uint64_t *RawData = A.getRawData();
-  for (unsigned i = 0; i < NumWords; i++)
-    emitSignedInt64(Vals, RawData[i]);
-}
-
-static void emitConstantRange(SmallVectorImpl &Record,
-                              const ConstantRange &CR) {
-  unsigned BitWidth = CR.getBitWidth();
-  Record.push_back(BitWidth);
-  if (BitWidth > 64) {
-    Record.push_back(CR.getLower().getActiveWords() |
-                     (uint64_t(CR.getUpper().getActiveWords()) << 32));
-    emitWideAPInt(Record, CR.getLower());
-    emitWideAPInt(Record, CR.getUpper());
-  } else {
-    emitSignedInt64(Record, CR.getLower().getSExtValue());
-    emitSignedInt64(Record, CR.getUpper().getSExtValue());
-  }
-}
-
 void ModuleBitcodeWriter::writeAttributeGroupTable() {
   const std::vector &AttrGrps =
       VE.getAttributeGroups();
@@ -924,17 +889,13 @@ void ModuleBitcodeWriter::writeAttributeGroupTable() {
           Record.append(Val.begin(), Val.end());
           Record.push_back(0);
         }
-      } else if (Attr.isTypeAttribute()) {
+      } else {
+        assert(Attr.isTypeAttribute());
         Type *Ty = Attr.getValueAsType();
         Record.push_back(Ty ? 6 : 5);
         Record.push_back(getAttrKindEncoding(Attr.getKindAsEnum()));
         if (Ty)
           Record.push_back(VE.getTypeID(Attr.getValueAsType()));
-      } else {
-        assert(Attr.isConstantRangeAttribute());
-        Record.push_back(7);
-        Record.push_back(getAttrKindEncoding(Attr.getKindAsEnum()));
-        emitConstantRange(Record, Attr.getValueAsConstantRange());
       }
     }
 
@@ -1755,6 +1716,24 @@ void ModuleBitcodeWriter::writeDIGenericSubrange(
   Record.clear();
 }
 
+static void emitSignedInt64(SmallVectorImpl &Vals, uint64_t V) {
+  if ((int64_t)V >= 0)
+    Vals.push_back(V << 1);
+  else
+    Vals.push_back((-V << 1) | 1);
+}
+
+static void emitWideAPInt(SmallVectorImpl &Vals, const APInt &A) {
+  // We have an arbitrary precision integer value to write whose
+  // bit width is > 64. However, in canonical unsigned integer
+  // format it is likely that the high bits are going to be zero.
+  // So, we only write the number of active words.
+  unsigned NumWords = A.getActiveWords();
+  const uint64_t *RawData = A.getRawData();
+  for (unsigned i = 0; i < NumWords; i++)
+    emitSignedInt64(Vals, RawData[i]);
+}
+
 void ModuleBitcodeWriter::writeDIEnumerator(const DIEnumerator *N,
                                             SmallVectorImpl &Record,
                                             unsigned Abbrev) {
diff --git a/llvm/lib/IR/AttributeImpl.h b/llvm/lib/IR/AttributeImpl.h
index 9a6427bbc3d5..78496786b0ae 100644
--- a/llvm/lib/IR/AttributeImpl.h
+++ b/llvm/lib/IR/AttributeImpl.h
@@ -20,7 +20,6 @@
 #include "llvm/ADT/FoldingSet.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/IR/Attributes.h"
-#include "llvm/IR/ConstantRange.h"
 #include "llvm/Support/TrailingObjects.h"
 #include 
 #include 
@@ -47,7 +46,6 @@ protected:
     IntAttrEntry,
     StringAttrEntry,
     TypeAttrEntry,
-    ConstantRangeAttrEntry,
   };
 
   AttributeImpl(AttrEntryKind KindID) : KindID(KindID) {}
@@ -61,9 +59,6 @@ public:
   bool isIntAttribute() const { return KindID == IntAttrEntry; }
   bool isStringAttribute() const { return KindID == StringAttrEntry; }
   bool isTypeAttribute() const { return KindID == TypeAttrEntry; }
-  bool isConstantRangeAttribute() const {
-    return KindID == ConstantRangeAttrEntry;
-  }
 
   bool hasAttribute(Attribute::AttrKind A) const;
   bool hasAttribute(StringRef Kind) const;
@@ -77,8 +72,6 @@ public:
 
   Type *getValueAsType() const;
 
-  ConstantRange getValueAsConstantRange() const;
-
   /// Used when sorting the attributes.
   bool operator<(const AttributeImpl &AI) const;
 
@@ -89,10 +82,8 @@ public:
       Profile(ID, getKindAsEnum(), getValueAsInt());
     else if (isStringAttribute())
       Profile(ID, getKindAsString(), getValueAsString());
-    else if (isTypeAttribute())
-      Profile(ID, getKindAsEnum(), getValueAsType());
     else
-      Profile(ID, getKindAsEnum(), getValueAsConstantRange());
+      Profile(ID, getKindAsEnum(), getValueAsType());
   }
 
   static void Profile(FoldingSetNodeID &ID, Attribute::AttrKind Kind) {
@@ -117,13 +108,6 @@ public:
     ID.AddInteger(Kind);
     ID.AddPointer(Ty);
   }
-
-  static void Profile(FoldingSetNodeID &ID, Attribute::AttrKind Kind,
-                      const ConstantRange &CR) {
-    ID.AddInteger(Kind);
-    ID.AddInteger(CR.getLower());
-    ID.AddInteger(CR.getUpper());
-  }
 };
 
 static_assert(std::is_trivially_destructible::value,
@@ -212,16 +196,6 @@ public:
   Type *getTypeValue() const { return Ty; }
 };
 
-class ConstantRangeAttributeImpl : public EnumAttributeImpl {
-  ConstantRange CR;
-
-public:
-  ConstantRangeAttributeImpl(Attribute::AttrKind Kind, const ConstantRange &CR)
-      : EnumAttributeImpl(ConstantRangeAttrEntry, Kind), CR(CR) {}
-
-  ConstantRange getConstantRangeValue() const { return CR; }
-};
-
 class AttributeBitSet {
   /// Bitset with a bit for each available attribute Attribute::AttrKind.
   uint8_t AvailableAttrs[12] = {};
diff --git a/llvm/lib/IR/Attributes.cpp b/llvm/lib/IR/Attributes.cpp
index fec34d0a4a81..00acbbe7989d 100644
--- a/llvm/lib/IR/Attributes.cpp
+++ b/llvm/lib/IR/Attributes.cpp
@@ -24,7 +24,6 @@
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/Config/llvm-config.h"
 #include "llvm/IR/AttributeMask.h"
-#include "llvm/IR/ConstantRange.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Type.h"
@@ -166,30 +165,6 @@ Attribute Attribute::get(LLVMContext &Context, Attribute::AttrKind Kind,
   return Attribute(PA);
 }
 
-Attribute Attribute::get(LLVMContext &Context, Attribute::AttrKind Kind,
-                         const ConstantRange &CR) {
-  assert(Attribute::isConstantRangeAttrKind(Kind) &&
-         "Not a ConstantRange attribute");
-  LLVMContextImpl *pImpl = Context.pImpl;
-  FoldingSetNodeID ID;
-  ID.AddInteger(Kind);
-  ID.AddInteger(CR.getLower());
-  ID.AddInteger(CR.getUpper());
-
-  void *InsertPoint;
-  AttributeImpl *PA = pImpl->AttrsSet.FindNodeOrInsertPos(ID, InsertPoint);
-
-  if (!PA) {
-    // If we didn't find any existing attributes of the same shape then create a
-    // new one and insert it.
-    PA = new (pImpl->Alloc) ConstantRangeAttributeImpl(Kind, CR);
-    pImpl->AttrsSet.InsertNode(PA, InsertPoint);
-  }
-
-  // Return the Attribute that we found or created.
-  return Attribute(PA);
-}
-
 Attribute Attribute::getWithAlignment(LLVMContext &Context, Align A) {
   assert(A <= llvm::Value::MaximumAlignment && "Alignment too large.");
   return get(Context, Alignment, A.value());
@@ -312,14 +287,9 @@ bool Attribute::isTypeAttribute() const {
   return pImpl && pImpl->isTypeAttribute();
 }
 
-bool Attribute::isConstantRangeAttribute() const {
-  return pImpl && pImpl->isConstantRangeAttribute();
-}
-
 Attribute::AttrKind Attribute::getKindAsEnum() const {
   if (!pImpl) return None;
-  assert((isEnumAttribute() || isIntAttribute() || isTypeAttribute() ||
-          isConstantRangeAttribute()) &&
+  assert((isEnumAttribute() || isIntAttribute() || isTypeAttribute()) &&
          "Invalid attribute type to get the kind as an enum!");
   return pImpl->getKindAsEnum();
 }
@@ -359,11 +329,6 @@ Type *Attribute::getValueAsType() const {
   return pImpl->getValueAsType();
 }
 
-ConstantRange Attribute::getValueAsConstantRange() const {
-  assert(isConstantRangeAttribute() &&
-         "Invalid attribute type to get the value as a ConstantRange!");
-  return pImpl->getValueAsConstantRange();
-}
 
 bool Attribute::hasAttribute(AttrKind Kind) const {
   return (pImpl && pImpl->hasAttribute(Kind)) || (!pImpl && Kind == None);
@@ -443,12 +408,6 @@ FPClassTest Attribute::getNoFPClass() const {
   return static_cast(pImpl->getValueAsInt());
 }
 
-ConstantRange Attribute::getRange() const {
-  assert(hasAttribute(Attribute::Range) &&
-         "Trying to get range args from non-range attribute");
-  return pImpl->getValueAsConstantRange();
-}
-
 static const char *getModRefStr(ModRefInfo MR) {
   switch (MR) {
   case ModRefInfo::NoModRef:
@@ -603,18 +562,6 @@ std::string Attribute::getAsString(bool InAttrGrp) const {
     return Result;
   }
 
-  if (hasAttribute(Attribute::Range)) {
-    std::string Result;
-    raw_string_ostream OS(Result);
-    ConstantRange CR = getValueAsConstantRange();
-    OS << "range(";
-    OS << "i" << CR.getBitWidth() << " ";
-    OS << CR.getLower() << ", " << CR.getUpper();
-    OS << ")";
-    OS.flush();
-    return Result;
-  }
-
   // Convert target-dependent attributes to strings of the form:
   //
   //   "kind"
@@ -704,8 +651,7 @@ bool AttributeImpl::hasAttribute(StringRef Kind) const {
 }
 
 Attribute::AttrKind AttributeImpl::getKindAsEnum() const {
-  assert(isEnumAttribute() || isIntAttribute() || isTypeAttribute() ||
-         isConstantRangeAttribute());
+  assert(isEnumAttribute() || isIntAttribute() || isTypeAttribute());
   return static_cast(this)->getEnumKind();
 }
 
@@ -734,12 +680,6 @@ Type *AttributeImpl::getValueAsType() const {
   return static_cast(this)->getTypeValue();
 }
 
-ConstantRange AttributeImpl::getValueAsConstantRange() const {
-  assert(isConstantRangeAttribute());
-  return static_cast(this)
-      ->getConstantRangeValue();
-}
-
 bool AttributeImpl::operator<(const AttributeImpl &AI) const {
   if (this == &AI)
     return false;
@@ -753,7 +693,6 @@ bool AttributeImpl::operator<(const AttributeImpl &AI) const {
       return getKindAsEnum() < AI.getKindAsEnum();
     assert(!AI.isEnumAttribute() && "Non-unique attribute");
     assert(!AI.isTypeAttribute() && "Comparison of types would be unstable");
-    assert(!AI.isConstantRangeAttribute() && "Unclear how to compare ranges");
     // TODO: Is this actually needed?
     assert(AI.isIntAttribute() && "Only possibility left");
     return getValueAsInt() < AI.getValueAsInt();
@@ -1942,15 +1881,6 @@ AttrBuilder &AttrBuilder::addInAllocaAttr(Type *Ty) {
   return addTypeAttr(Attribute::InAlloca, Ty);
 }
 
-AttrBuilder &AttrBuilder::addConstantRangeAttr(Attribute::AttrKind Kind,
-                                               const ConstantRange &CR) {
-  return addAttribute(Attribute::get(Ctx, Kind, CR));
-}
-
-AttrBuilder &AttrBuilder::addRangeAttr(const ConstantRange &CR) {
-  return addConstantRangeAttr(Attribute::Range, CR);
-}
-
 AttrBuilder &AttrBuilder::merge(const AttrBuilder &B) {
   // TODO: Could make this O(n) as we're merging two sorted lists.
   for (const auto &I : B.attrs())
@@ -2022,12 +1952,6 @@ AttributeMask AttributeFuncs::typeIncompatible(Type *Ty,
       Incompatible.addAttribute(Attribute::SExt).addAttribute(Attribute::ZExt);
   }
 
-  if (!Ty->isIntOrIntVectorTy()) {
-    // Attributes that only apply to integers or vector of integers.
-    if (ASK & ASK_SAFE_TO_DROP)
-      Incompatible.addAttribute(Attribute::Range);
-  }
-
   if (!Ty->isPointerTy()) {
     // Attributes that only apply to pointers.
     if (ASK & ASK_SAFE_TO_DROP)
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 3cf5e81efb3b..fd5f7d57c258 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -2039,11 +2039,6 @@ void Verifier::verifyParameterAttrs(AttributeSet Attrs, Type *Ty,
     Check((Val & ~static_cast(fcAllFlags)) == 0,
           "Invalid value for 'nofpclass' test mask", V);
   }
-  if (Attrs.hasAttribute(Attribute::Range)) {
-    auto CR = Attrs.getAttribute(Attribute::Range).getValueAsConstantRange();
-    Check(Ty->isIntOrIntVectorTy(CR.getBitWidth()),
-          "Range bit width must match type bit width!", V);
-  }
 }
 
 void Verifier::checkUnsignedBaseTenFuncAttr(AttributeList Attrs, StringRef Attr,
diff --git a/llvm/lib/Transforms/Utils/CodeExtractor.cpp b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
index ab2d25c3f17c..3071ec0c9113 100644
--- a/llvm/lib/Transforms/Utils/CodeExtractor.cpp
+++ b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
@@ -999,7 +999,6 @@ Function *CodeExtractor::constructFunction(const ValueSet &inputs,
       case Attribute::WriteOnly:
       case Attribute::Writable:
       case Attribute::DeadOnUnwind:
-      case Attribute::Range:
       //  These are not really attributes.
       case Attribute::None:
       case Attribute::EndAttrKinds:
diff --git a/llvm/test/Assembler/range-attribute-invalid-range.ll b/llvm/test/Assembler/range-attribute-invalid-range.ll
deleted file mode 100644
index cf6d3f080183..000000000000
--- a/llvm/test/Assembler/range-attribute-invalid-range.ll
+++ /dev/null
@@ -1,6 +0,0 @@
-; RUN: not llvm-as < %s -o /dev/null 2>&1 | FileCheck %s
-
-; CHECK: the range should not represent the full or empty set!
-define void @range_empty(i8 range(i8 0, 0) %a) {
-  ret void
-}
diff --git a/llvm/test/Assembler/range-attribute-invalid-type.ll b/llvm/test/Assembler/range-attribute-invalid-type.ll
deleted file mode 100644
index cc09149a94dc..000000000000
--- a/llvm/test/Assembler/range-attribute-invalid-type.ll
+++ /dev/null
@@ -1,6 +0,0 @@
-; RUN: not llvm-as < %s -o /dev/null 2>&1 | FileCheck %s
-
-; CHECK: the range must have integer type!
-define void @range_vector_type(i8 range(<4 x i32> 0, 0) %a) {
-  ret void
-}
diff --git a/llvm/test/Bitcode/attributes.ll b/llvm/test/Bitcode/attributes.ll
index 26163b4d38c8..6921f11a352d 100644
--- a/llvm/test/Bitcode/attributes.ll
+++ b/llvm/test/Bitcode/attributes.ll
@@ -526,16 +526,6 @@ define void @f91(ptr dead_on_unwind %p) {
   ret void
 }
 
-; CHECK: define range(i32 -1, 42) i32 @range_attribute(<4 x i32> range(i32 -1, 42) %a)
-define range(i32 -1, 42) i32 @range_attribute(<4 x i32> range(i32 -1, 42) %a) {
-  ret i32 0
-}
-
-; CHECK: define void @wide_range_attribute(i128 range(i128 618970019642690137449562111, 618970019642690137449562114) %a)
-define void @wide_range_attribute(i128 range(i128 618970019642690137449562111, 618970019642690137449562114) %a) {
-  ret void
-}
-
 ; CHECK: attributes #0 = { noreturn }
 ; CHECK: attributes #1 = { nounwind }
 ; CHECK: attributes #2 = { memory(none) }
diff --git a/llvm/test/Verifier/range-attr.ll b/llvm/test/Verifier/range-attr.ll
deleted file mode 100644
index f985ab696eac..000000000000
--- a/llvm/test/Verifier/range-attr.ll
+++ /dev/null
@@ -1,19 +0,0 @@
-; RUN: not llvm-as %s -o /dev/null 2>&1 | FileCheck %s
-
-; CHECK: Range bit width must match type bit width!
-; CHECK-NEXT: ptr @bit_widths_do_not_match
-define void @bit_widths_do_not_match(i32 range(i8 1, 0) %a) {
-  ret void
-}
-
-; CHECK: Range bit width must match type bit width!
-; CHECK-NEXT: ptr @bit_widths_do_not_match_vector
-define void @bit_widths_do_not_match_vector(<4 x i32> range(i8 1, 0) %a) {
-  ret void
-}
-
-; CHECK: Attribute 'range(i8 1, 0)' applied to incompatible type!
-; CHECK-NEXT: ptr @not-integer-type
-define void @not-integer-type(ptr range(i8 1, 0) %a) {
-  ret void
-}
diff --git a/llvm/utils/TableGen/Attributes.cpp b/llvm/utils/TableGen/Attributes.cpp
index d9fc7834416c..db3c4decccb4 100644
--- a/llvm/utils/TableGen/Attributes.cpp
+++ b/llvm/utils/TableGen/Attributes.cpp
@@ -53,8 +53,7 @@ void Attributes::emitTargetIndependentNames(raw_ostream &OS) {
   };
 
   // Emit attribute enums in the same order llvm::Attribute::operator< expects.
-  Emit({"EnumAttr", "TypeAttr", "IntAttr", "ConstantRangeAttr"},
-       "ATTRIBUTE_ENUM");
+  Emit({"EnumAttr", "TypeAttr", "IntAttr"}, "ATTRIBUTE_ENUM");
   Emit({"StrBoolAttr"}, "ATTRIBUTE_STRBOOL");
   Emit({"ComplexStrAttr"}, "ATTRIBUTE_COMPLEXSTR");
 
@@ -64,8 +63,7 @@ void Attributes::emitTargetIndependentNames(raw_ostream &OS) {
   OS << "#ifdef GET_ATTR_ENUM\n";
   OS << "#undef GET_ATTR_ENUM\n";
   unsigned Value = 1; // Leave zero for AttrKind::None.
-  for (StringRef KindName :
-       {"EnumAttr", "TypeAttr", "IntAttr", "ConstantRangeAttr"}) {
+  for (StringRef KindName : {"EnumAttr", "TypeAttr", "IntAttr"}) {
     OS << "First" << KindName << " = " << Value << ",\n";
     for (auto *A : Records.getAllDerivedDefinitions(KindName)) {
       OS << A->getName() << " = " << Value << ",\n";
@@ -119,8 +117,7 @@ void Attributes::emitAttributeProperties(raw_ostream &OS) {
   OS << "#ifdef GET_ATTR_PROP_TABLE\n";
   OS << "#undef GET_ATTR_PROP_TABLE\n";
   OS << "static const uint8_t AttrPropTable[] = {\n";
-  for (StringRef KindName :
-       {"EnumAttr", "TypeAttr", "IntAttr", "ConstantRangeAttr"}) {
+  for (StringRef KindName : {"EnumAttr", "TypeAttr", "IntAttr"}) {
     for (auto *A : Records.getAllDerivedDefinitions(KindName)) {
       OS << "0";
       for (Init *P : *A->getValueAsListInit("Properties"))
-- 
GitLab


From 300a39bdad4593fdc2618eb28f6e838df735619a Mon Sep 17 00:00:00 2001
From: Florian Mayer 
Date: Fri, 8 Mar 2024 12:14:22 -0800
Subject: [PATCH 0080/1674] Revert "Change
 GetNumChildren()/CalculateNumChildren() methods return llvm::Expected
 (#84219)"

This reverts commit 99118c809367d518ffe4de60c16da953744b68b9.
---
 lldb/include/lldb/Core/ValueObject.h          | 11 +---
 lldb/include/lldb/Core/ValueObjectCast.h      |  2 +-
 lldb/include/lldb/Core/ValueObjectChild.h     |  2 +-
 .../lldb/Core/ValueObjectConstResult.h        |  2 +-
 .../lldb/Core/ValueObjectDynamicValue.h       |  2 +-
 lldb/include/lldb/Core/ValueObjectMemory.h    |  2 +-
 lldb/include/lldb/Core/ValueObjectRegister.h  |  4 +-
 .../lldb/Core/ValueObjectSyntheticFilter.h    |  2 +-
 lldb/include/lldb/Core/ValueObjectVTable.h    |  2 +-
 lldb/include/lldb/Core/ValueObjectVariable.h  |  2 +-
 .../lldb/DataFormatters/TypeSynthetic.h       | 20 ++----
 .../lldb/DataFormatters/VectorIterator.h      |  2 +-
 lldb/include/lldb/Symbol/CompilerType.h       |  5 +-
 lldb/include/lldb/Symbol/Type.h               |  2 +-
 lldb/include/lldb/Symbol/TypeSystem.h         |  7 +-
 .../lldb/Target/StackFrameRecognizer.h        |  3 +-
 lldb/include/lldb/Utility/Log.h               | 14 ----
 lldb/source/API/SBValue.cpp                   |  2 +-
 lldb/source/Core/FormatEntity.cpp             |  2 +-
 lldb/source/Core/IOHandlerCursesGUI.cpp       |  2 +-
 lldb/source/Core/ValueObject.cpp              | 31 +++------
 lldb/source/Core/ValueObjectCast.cpp          |  6 +-
 lldb/source/Core/ValueObjectChild.cpp         |  6 +-
 lldb/source/Core/ValueObjectConstResult.cpp   |  7 +-
 lldb/source/Core/ValueObjectDynamicValue.cpp  |  7 +-
 lldb/source/Core/ValueObjectMemory.cpp        | 10 +--
 lldb/source/Core/ValueObjectRegister.cpp      | 12 ++--
 .../Core/ValueObjectSyntheticFilter.cpp       | 24 +++----
 lldb/source/Core/ValueObjectVTable.cpp        |  6 +-
 lldb/source/Core/ValueObjectVariable.cpp      |  7 +-
 lldb/source/DataFormatters/FormatManager.cpp  | 11 +---
 lldb/source/DataFormatters/TypeSynthetic.cpp  | 17 +----
 .../DataFormatters/ValueObjectPrinter.cpp     |  2 +-
 lldb/source/DataFormatters/VectorType.cpp     | 16 ++---
 .../Clang/ClangExpressionSourceCode.cpp       |  2 +-
 .../TSan/InstrumentationRuntimeTSan.cpp       |  4 +-
 .../Language/CPlusPlus/BlockPointer.cpp       |  4 +-
 .../Plugins/Language/CPlusPlus/Coroutines.cpp |  8 +--
 .../Plugins/Language/CPlusPlus/Coroutines.h   |  2 +-
 .../Language/CPlusPlus/GenericBitset.cpp      |  4 +-
 .../Language/CPlusPlus/GenericOptional.cpp    |  6 +-
 .../Plugins/Language/CPlusPlus/LibCxx.cpp     | 18 +++---
 .../Plugins/Language/CPlusPlus/LibCxx.h       |  8 +--
 .../Language/CPlusPlus/LibCxxAtomic.cpp       |  6 +-
 .../CPlusPlus/LibCxxInitializerList.cpp       |  6 +-
 .../Plugins/Language/CPlusPlus/LibCxxList.cpp | 12 ++--
 .../Plugins/Language/CPlusPlus/LibCxxMap.cpp  | 14 ++--
 .../Language/CPlusPlus/LibCxxQueue.cpp        |  2 +-
 .../CPlusPlus/LibCxxRangesRefView.cpp         |  2 +-
 .../Plugins/Language/CPlusPlus/LibCxxSpan.cpp |  6 +-
 .../Language/CPlusPlus/LibCxxTuple.cpp        |  4 +-
 .../Language/CPlusPlus/LibCxxUnorderedMap.cpp |  8 +--
 .../Language/CPlusPlus/LibCxxValarray.cpp     |  6 +-
 .../Language/CPlusPlus/LibCxxVariant.cpp      |  2 +-
 .../Language/CPlusPlus/LibCxxVector.cpp       | 14 ++--
 .../Plugins/Language/CPlusPlus/LibStdcpp.cpp  | 17 ++---
 .../Language/CPlusPlus/LibStdcppTuple.cpp     |  7 +-
 .../CPlusPlus/LibStdcppUniquePointer.cpp      |  5 +-
 lldb/source/Plugins/Language/ObjC/Cocoa.cpp   |  2 +-
 lldb/source/Plugins/Language/ObjC/NSArray.cpp | 30 ++++-----
 .../Plugins/Language/ObjC/NSDictionary.cpp    | 64 +++++++++----------
 lldb/source/Plugins/Language/ObjC/NSError.cpp |  2 +-
 .../Plugins/Language/ObjC/NSException.cpp     |  4 +-
 .../Plugins/Language/ObjC/NSIndexPath.cpp     |  6 +-
 lldb/source/Plugins/Language/ObjC/NSSet.cpp   | 32 +++++-----
 .../AppleObjCRuntime/AppleObjCRuntime.cpp     |  3 +-
 .../TypeSystem/Clang/TypeSystemClang.cpp      | 40 ++++--------
 .../TypeSystem/Clang/TypeSystemClang.h        |  7 +-
 lldb/source/Symbol/CompilerType.cpp           |  5 +-
 lldb/source/Symbol/Type.cpp                   |  2 +-
 lldb/source/Symbol/Variable.cpp               |  6 +-
 lldb/source/Target/StackFrame.cpp             | 19 +++---
 72 files changed, 247 insertions(+), 394 deletions(-)

diff --git a/lldb/include/lldb/Core/ValueObject.h b/lldb/include/lldb/Core/ValueObject.h
index e7e35e2b2bff..b4d2c8098edc 100644
--- a/lldb/include/lldb/Core/ValueObject.h
+++ b/lldb/include/lldb/Core/ValueObject.h
@@ -476,13 +476,7 @@ public:
 
   virtual size_t GetIndexOfChildWithName(llvm::StringRef name);
 
-  llvm::Expected GetNumChildren(uint32_t max = UINT32_MAX);
-  /// Like \c GetNumChildren but returns 0 on error.  You probably
-  /// shouldn't be using this function. It exists primarily to ease the
-  /// transition to more pervasive error handling while not all APIs
-  /// have been updated.
-  uint32_t GetNumChildrenIgnoringErrors(uint32_t max = UINT32_MAX);
-  bool HasChildren() { return GetNumChildrenIgnoringErrors() > 0; }
+  uint32_t GetNumChildren(uint32_t max = UINT32_MAX);
 
   const Value &GetValue() const { return m_value; }
 
@@ -964,8 +958,7 @@ protected:
                                           int32_t synthetic_index);
 
   /// Should only be called by ValueObject::GetNumChildren().
-  virtual llvm::Expected
-  CalculateNumChildren(uint32_t max = UINT32_MAX) = 0;
+  virtual uint32_t CalculateNumChildren(uint32_t max = UINT32_MAX) = 0;
 
   void SetNumChildren(uint32_t num_children);
 
diff --git a/lldb/include/lldb/Core/ValueObjectCast.h b/lldb/include/lldb/Core/ValueObjectCast.h
index ba25e166f326..51c647680d52 100644
--- a/lldb/include/lldb/Core/ValueObjectCast.h
+++ b/lldb/include/lldb/Core/ValueObjectCast.h
@@ -33,7 +33,7 @@ public:
 
   std::optional GetByteSize() override;
 
-  llvm::Expected CalculateNumChildren(uint32_t max) override;
+  uint32_t CalculateNumChildren(uint32_t max) override;
 
   lldb::ValueType GetValueType() const override;
 
diff --git a/lldb/include/lldb/Core/ValueObjectChild.h b/lldb/include/lldb/Core/ValueObjectChild.h
index 1f88e607cb57..47a13be08bb8 100644
--- a/lldb/include/lldb/Core/ValueObjectChild.h
+++ b/lldb/include/lldb/Core/ValueObjectChild.h
@@ -39,7 +39,7 @@ public:
 
   lldb::ValueType GetValueType() const override;
 
-  llvm::Expected CalculateNumChildren(uint32_t max) override;
+  uint32_t CalculateNumChildren(uint32_t max) override;
 
   ConstString GetTypeName() override;
 
diff --git a/lldb/include/lldb/Core/ValueObjectConstResult.h b/lldb/include/lldb/Core/ValueObjectConstResult.h
index 37dc0867f26c..9f1246cf2a78 100644
--- a/lldb/include/lldb/Core/ValueObjectConstResult.h
+++ b/lldb/include/lldb/Core/ValueObjectConstResult.h
@@ -67,7 +67,7 @@ public:
 
   lldb::ValueType GetValueType() const override;
 
-  llvm::Expected CalculateNumChildren(uint32_t max) override;
+  uint32_t CalculateNumChildren(uint32_t max) override;
 
   ConstString GetTypeName() override;
 
diff --git a/lldb/include/lldb/Core/ValueObjectDynamicValue.h b/lldb/include/lldb/Core/ValueObjectDynamicValue.h
index 82c20eee0cd4..21a9b409fd5b 100644
--- a/lldb/include/lldb/Core/ValueObjectDynamicValue.h
+++ b/lldb/include/lldb/Core/ValueObjectDynamicValue.h
@@ -43,7 +43,7 @@ public:
 
   ConstString GetDisplayTypeName() override;
 
-  llvm::Expected CalculateNumChildren(uint32_t max) override;
+  uint32_t CalculateNumChildren(uint32_t max) override;
 
   lldb::ValueType GetValueType() const override;
 
diff --git a/lldb/include/lldb/Core/ValueObjectMemory.h b/lldb/include/lldb/Core/ValueObjectMemory.h
index a8fb0353d601..a74b325546b0 100644
--- a/lldb/include/lldb/Core/ValueObjectMemory.h
+++ b/lldb/include/lldb/Core/ValueObjectMemory.h
@@ -47,7 +47,7 @@ public:
 
   ConstString GetDisplayTypeName() override;
 
-  llvm::Expected CalculateNumChildren(uint32_t max) override;
+  uint32_t CalculateNumChildren(uint32_t max) override;
 
   lldb::ValueType GetValueType() const override;
 
diff --git a/lldb/include/lldb/Core/ValueObjectRegister.h b/lldb/include/lldb/Core/ValueObjectRegister.h
index fec8566ba33d..6c470c1a6865 100644
--- a/lldb/include/lldb/Core/ValueObjectRegister.h
+++ b/lldb/include/lldb/Core/ValueObjectRegister.h
@@ -47,7 +47,7 @@ public:
 
   ConstString GetQualifiedTypeName() override;
 
-  llvm::Expected CalculateNumChildren(uint32_t max) override;
+  uint32_t CalculateNumChildren(uint32_t max) override;
 
   ValueObject *CreateChildAtIndex(size_t idx, bool synthetic_array_member,
                                   int32_t synthetic_index) override;
@@ -95,7 +95,7 @@ public:
 
   ConstString GetTypeName() override;
 
-  llvm::Expected CalculateNumChildren(uint32_t max) override;
+  uint32_t CalculateNumChildren(uint32_t max) override;
 
   bool SetValueFromCString(const char *value_str, Status &error) override;
 
diff --git a/lldb/include/lldb/Core/ValueObjectSyntheticFilter.h b/lldb/include/lldb/Core/ValueObjectSyntheticFilter.h
index ca6d6c728005..1e54babc94f3 100644
--- a/lldb/include/lldb/Core/ValueObjectSyntheticFilter.h
+++ b/lldb/include/lldb/Core/ValueObjectSyntheticFilter.h
@@ -47,7 +47,7 @@ public:
 
   bool MightHaveChildren() override;
 
-  llvm::Expected CalculateNumChildren(uint32_t max) override;
+  uint32_t CalculateNumChildren(uint32_t max) override;
 
   lldb::ValueType GetValueType() const override;
 
diff --git a/lldb/include/lldb/Core/ValueObjectVTable.h b/lldb/include/lldb/Core/ValueObjectVTable.h
index 4662f395a4dd..e7e14fc83d78 100644
--- a/lldb/include/lldb/Core/ValueObjectVTable.h
+++ b/lldb/include/lldb/Core/ValueObjectVTable.h
@@ -64,7 +64,7 @@ public:
 
   std::optional GetByteSize() override;
 
-  llvm::Expected CalculateNumChildren(uint32_t max) override;
+  uint32_t CalculateNumChildren(uint32_t max) override;
 
   ValueObject *CreateChildAtIndex(size_t idx, bool synthetic_array_member,
                                   int32_t synthetic_index) override;
diff --git a/lldb/include/lldb/Core/ValueObjectVariable.h b/lldb/include/lldb/Core/ValueObjectVariable.h
index db3847f14a0b..da270300df0b 100644
--- a/lldb/include/lldb/Core/ValueObjectVariable.h
+++ b/lldb/include/lldb/Core/ValueObjectVariable.h
@@ -46,7 +46,7 @@ public:
 
   ConstString GetDisplayTypeName() override;
 
-  llvm::Expected CalculateNumChildren(uint32_t max) override;
+  uint32_t CalculateNumChildren(uint32_t max) override;
 
   lldb::ValueType GetValueType() const override;
 
diff --git a/lldb/include/lldb/DataFormatters/TypeSynthetic.h b/lldb/include/lldb/DataFormatters/TypeSynthetic.h
index ede7442a02bf..38f3ce0fa5f0 100644
--- a/lldb/include/lldb/DataFormatters/TypeSynthetic.h
+++ b/lldb/include/lldb/DataFormatters/TypeSynthetic.h
@@ -38,17 +38,13 @@ public:
 
   virtual ~SyntheticChildrenFrontEnd() = default;
 
-  virtual llvm::Expected CalculateNumChildren() = 0;
+  virtual uint32_t CalculateNumChildren() = 0;
 
-  virtual llvm::Expected CalculateNumChildren(uint32_t max) {
+  virtual uint32_t CalculateNumChildren(uint32_t max) {
     auto count = CalculateNumChildren();
-    if (!count)
-      return count;
-    return *count <= max ? *count : max;
+    return count <= max ? count : max;
   }
 
-  uint32_t CalculateNumChildrenIgnoringErrors(uint32_t max = UINT32_MAX);
-
   virtual lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) = 0;
 
   virtual size_t GetIndexOfChildWithName(ConstString name) = 0;
@@ -113,7 +109,7 @@ public:
 
   ~SyntheticValueProviderFrontEnd() override = default;
 
-  llvm::Expected CalculateNumChildren() override { return 0; }
+  uint32_t CalculateNumChildren() override { return 0; }
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override { return nullptr; }
 
@@ -326,9 +322,7 @@ public:
 
     ~FrontEnd() override = default;
 
-    llvm::Expected CalculateNumChildren() override {
-      return filter->GetCount();
-    }
+    uint32_t CalculateNumChildren() override { return filter->GetCount(); }
 
     lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override {
       if (idx >= filter->GetCount())
@@ -432,9 +426,9 @@ public:
 
     bool IsValid();
 
-    llvm::Expected CalculateNumChildren() override;
+    uint32_t CalculateNumChildren() override;
 
-    llvm::Expected CalculateNumChildren(uint32_t max) override;
+    uint32_t CalculateNumChildren(uint32_t max) override;
 
     lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
diff --git a/lldb/include/lldb/DataFormatters/VectorIterator.h b/lldb/include/lldb/DataFormatters/VectorIterator.h
index 70bcf50ca1b1..7711b9de95db 100644
--- a/lldb/include/lldb/DataFormatters/VectorIterator.h
+++ b/lldb/include/lldb/DataFormatters/VectorIterator.h
@@ -24,7 +24,7 @@ public:
   VectorIteratorSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp,
                                   llvm::ArrayRef item_names);
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
diff --git a/lldb/include/lldb/Symbol/CompilerType.h b/lldb/include/lldb/Symbol/CompilerType.h
index c1dce4ccbf79..414c44275aaa 100644
--- a/lldb/include/lldb/Symbol/CompilerType.h
+++ b/lldb/include/lldb/Symbol/CompilerType.h
@@ -386,9 +386,8 @@ public:
 
   std::optional GetTypeBitAlign(ExecutionContextScope *exe_scope) const;
 
-  llvm::Expected
-  GetNumChildren(bool omit_empty_base_classes,
-                 const ExecutionContext *exe_ctx) const;
+  uint32_t GetNumChildren(bool omit_empty_base_classes,
+                          const ExecutionContext *exe_ctx) const;
 
   lldb::BasicType GetBasicTypeEnumeration() const;
 
diff --git a/lldb/include/lldb/Symbol/Type.h b/lldb/include/lldb/Symbol/Type.h
index b5eac5fa732d..acd1a769f13c 100644
--- a/lldb/include/lldb/Symbol/Type.h
+++ b/lldb/include/lldb/Symbol/Type.h
@@ -440,7 +440,7 @@ public:
 
   std::optional GetByteSize(ExecutionContextScope *exe_scope);
 
-  llvm::Expected GetNumChildren(bool omit_empty_base_classes);
+  uint32_t GetNumChildren(bool omit_empty_base_classes);
 
   bool IsAggregateType();
 
diff --git a/lldb/include/lldb/Symbol/TypeSystem.h b/lldb/include/lldb/Symbol/TypeSystem.h
index f647fcbf1636..63829131556e 100644
--- a/lldb/include/lldb/Symbol/TypeSystem.h
+++ b/lldb/include/lldb/Symbol/TypeSystem.h
@@ -300,10 +300,9 @@ public:
 
   virtual lldb::Format GetFormat(lldb::opaque_compiler_type_t type) = 0;
 
-  virtual llvm::Expected
-  GetNumChildren(lldb::opaque_compiler_type_t type,
-                 bool omit_empty_base_classes,
-                 const ExecutionContext *exe_ctx) = 0;
+  virtual uint32_t GetNumChildren(lldb::opaque_compiler_type_t type,
+                                  bool omit_empty_base_classes,
+                                  const ExecutionContext *exe_ctx) = 0;
 
   virtual CompilerType GetBuiltinTypeByName(ConstString name);
 
diff --git a/lldb/include/lldb/Target/StackFrameRecognizer.h b/lldb/include/lldb/Target/StackFrameRecognizer.h
index 5e8e12b2a4e9..e111f4a4dc70 100644
--- a/lldb/include/lldb/Target/StackFrameRecognizer.h
+++ b/lldb/include/lldb/Target/StackFrameRecognizer.h
@@ -164,8 +164,7 @@ class ValueObjectRecognizerSynthesizedValue : public ValueObject {
     m_value = m_parent->GetValue();
     return true;
   }
-  llvm::Expected
-  CalculateNumChildren(uint32_t max = UINT32_MAX) override {
+  uint32_t CalculateNumChildren(uint32_t max = UINT32_MAX) override {
     return m_parent->GetNumChildren(max);
   }
   CompilerType GetCompilerTypeImpl() override {
diff --git a/lldb/include/lldb/Utility/Log.h b/lldb/include/lldb/Utility/Log.h
index 01876ad732d4..1fe28d61b9da 100644
--- a/lldb/include/lldb/Utility/Log.h
+++ b/lldb/include/lldb/Utility/Log.h
@@ -373,18 +373,4 @@ template  Log *GetLog(Cat mask) {
       ::llvm::consumeError(::std::move(error_private));                        \
   } while (0)
 
-// Write message to the verbose log, if error is set. In the log
-// message refer to the error with {0}. Error is cleared regardless of
-// whether logging is enabled.
-#define LLDB_LOG_ERRORV(log, error, ...)                                       \
-  do {                                                                         \
-    ::lldb_private::Log *log_private = (log);                                  \
-    ::llvm::Error error_private = (error);                                     \
-    if (log_private && log_private->GetVerbose() && error_private) {           \
-      log_private->FormatError(::std::move(error_private), __FILE__, __func__, \
-                               __VA_ARGS__);                                   \
-    } else                                                                     \
-      ::llvm::consumeError(::std::move(error_private));                        \
-  } while (0)
-
 #endif // LLDB_UTILITY_LOG_H
diff --git a/lldb/source/API/SBValue.cpp b/lldb/source/API/SBValue.cpp
index 94a8f3ea319e..89d26a1fbe28 100644
--- a/lldb/source/API/SBValue.cpp
+++ b/lldb/source/API/SBValue.cpp
@@ -947,7 +947,7 @@ uint32_t SBValue::GetNumChildren(uint32_t max) {
   ValueLocker locker;
   lldb::ValueObjectSP value_sp(GetSP(locker));
   if (value_sp)
-    num_children = value_sp->GetNumChildrenIgnoringErrors(max);
+    num_children = value_sp->GetNumChildren(max);
 
   return num_children;
 }
diff --git a/lldb/source/Core/FormatEntity.cpp b/lldb/source/Core/FormatEntity.cpp
index cf82676bedda..fa5eadc6ff4e 100644
--- a/lldb/source/Core/FormatEntity.cpp
+++ b/lldb/source/Core/FormatEntity.cpp
@@ -926,7 +926,7 @@ static bool DumpValue(Stream &s, const SymbolContext *sc,
     s.PutChar('[');
 
     if (index_higher < 0)
-      index_higher = valobj->GetNumChildrenIgnoringErrors() - 1;
+      index_higher = valobj->GetNumChildren() - 1;
 
     uint32_t max_num_children =
         target->GetTargetSP()->GetMaximumNumberOfChildrenToDisplay();
diff --git a/lldb/source/Core/IOHandlerCursesGUI.cpp b/lldb/source/Core/IOHandlerCursesGUI.cpp
index f86dce247135..620e68a28510 100644
--- a/lldb/source/Core/IOHandlerCursesGUI.cpp
+++ b/lldb/source/Core/IOHandlerCursesGUI.cpp
@@ -4519,7 +4519,7 @@ struct Row {
       calculated_children = true;
       ValueObjectSP valobj = value.GetSP();
       if (valobj) {
-        const uint32_t num_children = valobj->GetNumChildrenIgnoringErrors();
+        const size_t num_children = valobj->GetNumChildren();
         for (size_t i = 0; i < num_children; ++i) {
           children.push_back(Row(valobj->GetChildAtIndex(i), this));
         }
diff --git a/lldb/source/Core/ValueObject.cpp b/lldb/source/Core/ValueObject.cpp
index d813044d02ff..0ed7f03be25c 100644
--- a/lldb/source/Core/ValueObject.cpp
+++ b/lldb/source/Core/ValueObject.cpp
@@ -377,7 +377,7 @@ ValueObjectSP ValueObject::GetChildAtIndex(uint32_t idx, bool can_create) {
   // We may need to update our value if we are dynamic
   if (IsPossibleDynamicType())
     UpdateValueIfNeeded(false);
-  if (idx < GetNumChildrenIgnoringErrors()) {
+  if (idx < GetNumChildren()) {
     // Check if we have already made the child value object?
     if (can_create && !m_children.HasChildAtIndex(idx)) {
       // No we haven't created the child at this index, so lets have our
@@ -440,7 +440,7 @@ ValueObjectSP ValueObject::GetChildMemberWithName(llvm::StringRef name,
   return child_sp;
 }
 
-llvm::Expected ValueObject::GetNumChildren(uint32_t max) {
+uint32_t ValueObject::GetNumChildren(uint32_t max) {
   UpdateValueIfNeeded();
 
   if (max < UINT32_MAX) {
@@ -452,24 +452,11 @@ llvm::Expected ValueObject::GetNumChildren(uint32_t max) {
   }
 
   if (!m_flags.m_children_count_valid) {
-    auto num_children_or_err = CalculateNumChildren();
-    if (num_children_or_err)
-      SetNumChildren(*num_children_or_err);
-    else
-      return num_children_or_err;
+    SetNumChildren(CalculateNumChildren());
   }
   return m_children.GetChildrenCount();
 }
 
-uint32_t ValueObject::GetNumChildrenIgnoringErrors(uint32_t max) {
-  auto value_or_err = GetNumChildren(max);
-  if (value_or_err)
-    return *value_or_err;
-  LLDB_LOG_ERRORV(GetLog(LLDBLog::DataFormatters), value_or_err.takeError(),
-                  "{0}");
-  return 0;
-}
-
 bool ValueObject::MightHaveChildren() {
   bool has_children = false;
   const uint32_t type_info = GetTypeInfo();
@@ -477,7 +464,7 @@ bool ValueObject::MightHaveChildren() {
     if (type_info & (eTypeHasChildren | eTypeIsPointer | eTypeIsReference))
       has_children = true;
   } else {
-    has_children = GetNumChildrenIgnoringErrors() > 0;
+    has_children = GetNumChildren() > 0;
   }
   return has_children;
 }
@@ -1189,7 +1176,7 @@ bool ValueObject::DumpPrintableRepresentation(
       if (flags.Test(eTypeIsArray)) {
         if ((custom_format == eFormatBytes) ||
             (custom_format == eFormatBytesWithASCII)) {
-          const size_t count = GetNumChildrenIgnoringErrors();
+          const size_t count = GetNumChildren();
 
           s << '[';
           for (size_t low = 0; low < count; low++) {
@@ -1228,7 +1215,7 @@ bool ValueObject::DumpPrintableRepresentation(
                                                      // format should be printed
                                                      // directly
         {
-          const size_t count = GetNumChildrenIgnoringErrors();
+          const size_t count = GetNumChildren();
 
           Format format = FormatManager::GetSingleItemFormat(custom_format);
 
@@ -1307,7 +1294,7 @@ bool ValueObject::DumpPrintableRepresentation(
       break;
 
     case eValueObjectRepresentationStyleChildrenCount:
-      strm.Printf("%" PRIu64 "", (uint64_t)GetNumChildrenIgnoringErrors());
+      strm.Printf("%" PRIu64 "", (uint64_t)GetNumChildren());
       str = strm.GetString();
       break;
 
@@ -2333,9 +2320,7 @@ ValueObjectSP ValueObject::GetValueForExpressionPath_Impl(
             child_valobj_sp = root->GetSyntheticArrayMember(index, true);
           if (!child_valobj_sp)
             if (root->HasSyntheticValue() &&
-                llvm::expectedToStdOptional(
-                    root->GetSyntheticValue()->GetNumChildren())
-                        .value_or(0) > index)
+                root->GetSyntheticValue()->GetNumChildren() > index)
               child_valobj_sp =
                   root->GetSyntheticValue()->GetChildAtIndex(index);
           if (child_valobj_sp) {
diff --git a/lldb/source/Core/ValueObjectCast.cpp b/lldb/source/Core/ValueObjectCast.cpp
index c8e316415141..a5c555f86b13 100644
--- a/lldb/source/Core/ValueObjectCast.cpp
+++ b/lldb/source/Core/ValueObjectCast.cpp
@@ -41,13 +41,11 @@ ValueObjectCast::~ValueObjectCast() = default;
 
 CompilerType ValueObjectCast::GetCompilerTypeImpl() { return m_cast_type; }
 
-llvm::Expected ValueObjectCast::CalculateNumChildren(uint32_t max) {
+uint32_t ValueObjectCast::CalculateNumChildren(uint32_t max) {
   ExecutionContext exe_ctx(GetExecutionContextRef());
   auto children_count = GetCompilerType().GetNumChildren(
       true, &exe_ctx);
-  if (!children_count)
-    return children_count;
-  return *children_count <= max ? *children_count : max;
+  return children_count <= max ? children_count : max;
 }
 
 std::optional ValueObjectCast::GetByteSize() {
diff --git a/lldb/source/Core/ValueObjectChild.cpp b/lldb/source/Core/ValueObjectChild.cpp
index c6a97dd1a5cd..2e55dd7726bd 100644
--- a/lldb/source/Core/ValueObjectChild.cpp
+++ b/lldb/source/Core/ValueObjectChild.cpp
@@ -49,12 +49,10 @@ lldb::ValueType ValueObjectChild::GetValueType() const {
   return m_parent->GetValueType();
 }
 
-llvm::Expected ValueObjectChild::CalculateNumChildren(uint32_t max) {
+uint32_t ValueObjectChild::CalculateNumChildren(uint32_t max) {
   ExecutionContext exe_ctx(GetExecutionContextRef());
   auto children_count = GetCompilerType().GetNumChildren(true, &exe_ctx);
-  if (!children_count)
-    return children_count;
-  return *children_count <= max ? *children_count : max;
+  return children_count <= max ? children_count : max;
 }
 
 static void AdjustForBitfieldness(ConstString &name,
diff --git a/lldb/source/Core/ValueObjectConstResult.cpp b/lldb/source/Core/ValueObjectConstResult.cpp
index 8ac2c1cac2f6..5c7aa4452b70 100644
--- a/lldb/source/Core/ValueObjectConstResult.cpp
+++ b/lldb/source/Core/ValueObjectConstResult.cpp
@@ -216,13 +216,10 @@ std::optional ValueObjectConstResult::GetByteSize() {
 
 void ValueObjectConstResult::SetByteSize(size_t size) { m_byte_size = size; }
 
-llvm::Expected
-ValueObjectConstResult::CalculateNumChildren(uint32_t max) {
+uint32_t ValueObjectConstResult::CalculateNumChildren(uint32_t max) {
   ExecutionContext exe_ctx(GetExecutionContextRef());
   auto children_count = GetCompilerType().GetNumChildren(true, &exe_ctx);
-  if (!children_count)
-    return children_count;
-  return *children_count <= max ? *children_count : max;
+  return children_count <= max ? children_count : max;
 }
 
 ConstString ValueObjectConstResult::GetTypeName() {
diff --git a/lldb/source/Core/ValueObjectDynamicValue.cpp b/lldb/source/Core/ValueObjectDynamicValue.cpp
index 4695febdf8ca..4e64760371ae 100644
--- a/lldb/source/Core/ValueObjectDynamicValue.cpp
+++ b/lldb/source/Core/ValueObjectDynamicValue.cpp
@@ -85,15 +85,12 @@ ConstString ValueObjectDynamicValue::GetDisplayTypeName() {
   return m_parent->GetDisplayTypeName();
 }
 
-llvm::Expected
-ValueObjectDynamicValue::CalculateNumChildren(uint32_t max) {
+uint32_t ValueObjectDynamicValue::CalculateNumChildren(uint32_t max) {
   const bool success = UpdateValueIfNeeded(false);
   if (success && m_dynamic_type_info.HasType()) {
     ExecutionContext exe_ctx(GetExecutionContextRef());
     auto children_count = GetCompilerType().GetNumChildren(true, &exe_ctx);
-    if (!children_count)
-      return children_count;
-    return *children_count <= max ? *children_count : max;
+    return children_count <= max ? children_count : max;
   } else
     return m_parent->GetNumChildren(max);
 }
diff --git a/lldb/source/Core/ValueObjectMemory.cpp b/lldb/source/Core/ValueObjectMemory.cpp
index f555ab82f441..7f68236c7884 100644
--- a/lldb/source/Core/ValueObjectMemory.cpp
+++ b/lldb/source/Core/ValueObjectMemory.cpp
@@ -126,21 +126,17 @@ ConstString ValueObjectMemory::GetDisplayTypeName() {
   return m_compiler_type.GetDisplayTypeName();
 }
 
-llvm::Expected ValueObjectMemory::CalculateNumChildren(uint32_t max) {
+uint32_t ValueObjectMemory::CalculateNumChildren(uint32_t max) {
   if (m_type_sp) {
     auto child_count = m_type_sp->GetNumChildren(true);
-    if (!child_count)
-      return child_count;
-    return *child_count <= max ? *child_count : max;
+    return child_count <= max ? child_count : max;
   }
 
   ExecutionContext exe_ctx(GetExecutionContextRef());
   const bool omit_empty_base_classes = true;
   auto child_count =
       m_compiler_type.GetNumChildren(omit_empty_base_classes, &exe_ctx);
-  if (!child_count)
-    return child_count;
-  return *child_count <= max ? *child_count : max;
+  return child_count <= max ? child_count : max;
 }
 
 std::optional ValueObjectMemory::GetByteSize() {
diff --git a/lldb/source/Core/ValueObjectRegister.cpp b/lldb/source/Core/ValueObjectRegister.cpp
index 9d9da8bb119d..d4c144cc7edb 100644
--- a/lldb/source/Core/ValueObjectRegister.cpp
+++ b/lldb/source/Core/ValueObjectRegister.cpp
@@ -74,8 +74,7 @@ ConstString ValueObjectRegisterSet::GetQualifiedTypeName() {
   return ConstString();
 }
 
-llvm::Expected
-ValueObjectRegisterSet::CalculateNumChildren(uint32_t max) {
+uint32_t ValueObjectRegisterSet::CalculateNumChildren(uint32_t max) {
   const RegisterSet *reg_set = m_reg_ctx_sp->GetRegisterSet(m_reg_set_idx);
   if (reg_set) {
     auto reg_count = reg_set->num_registers;
@@ -119,7 +118,7 @@ ValueObject *ValueObjectRegisterSet::CreateChildAtIndex(
     size_t idx, bool synthetic_array_member, int32_t synthetic_index) {
   ValueObject *valobj = nullptr;
   if (m_reg_ctx_sp && m_reg_set) {
-    uint32_t num_children = GetNumChildrenIgnoringErrors();
+    const size_t num_children = GetNumChildren();
     if (idx < num_children)
       valobj = new ValueObjectRegister(
           *this, m_reg_ctx_sp,
@@ -221,13 +220,10 @@ ConstString ValueObjectRegister::GetTypeName() {
   return m_type_name;
 }
 
-llvm::Expected
-ValueObjectRegister::CalculateNumChildren(uint32_t max) {
+uint32_t ValueObjectRegister::CalculateNumChildren(uint32_t max) {
   ExecutionContext exe_ctx(GetExecutionContextRef());
   auto children_count = GetCompilerType().GetNumChildren(true, &exe_ctx);
-  if (!children_count)
-    return children_count;
-  return *children_count <= max ? *children_count : max;
+  return children_count <= max ? children_count : max;
 }
 
 std::optional ValueObjectRegister::GetByteSize() {
diff --git a/lldb/source/Core/ValueObjectSyntheticFilter.cpp b/lldb/source/Core/ValueObjectSyntheticFilter.cpp
index adac1b400705..7f8a9a34cb35 100644
--- a/lldb/source/Core/ValueObjectSyntheticFilter.cpp
+++ b/lldb/source/Core/ValueObjectSyntheticFilter.cpp
@@ -31,9 +31,7 @@ public:
   DummySyntheticFrontEnd(ValueObject &backend)
       : SyntheticChildrenFrontEnd(backend) {}
 
-  llvm::Expected CalculateNumChildren() override {
-    return m_backend.GetNumChildren();
-  }
+  uint32_t CalculateNumChildren() override { return m_backend.GetNumChildren(); }
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override {
     return m_backend.GetChildAtIndex(idx);
@@ -86,8 +84,7 @@ ConstString ValueObjectSynthetic::GetDisplayTypeName() {
   return m_parent->GetDisplayTypeName();
 }
 
-llvm::Expected
-ValueObjectSynthetic::CalculateNumChildren(uint32_t max) {
+uint32_t ValueObjectSynthetic::CalculateNumChildren(uint32_t max) {
   Log *log = GetLog(LLDBLog::DataFormatters);
 
   UpdateValueIfNeeded();
@@ -95,23 +92,18 @@ ValueObjectSynthetic::CalculateNumChildren(uint32_t max) {
     return m_synthetic_children_count <= max ? m_synthetic_children_count : max;
 
   if (max < UINT32_MAX) {
-    auto num_children = m_synth_filter_up->CalculateNumChildren(max);
+    size_t num_children = m_synth_filter_up->CalculateNumChildren(max);
     LLDB_LOGF(log,
               "[ValueObjectSynthetic::CalculateNumChildren] for VO of name "
-              "%s and type %s, the filter returned %u child values",
-              GetName().AsCString(), GetTypeName().AsCString(),
-              num_children ? *num_children : 0);
+              "%s and type %s, the filter returned %zu child values",
+              GetName().AsCString(), GetTypeName().AsCString(), num_children);
     return num_children;
   } else {
-    auto num_children_or_err = m_synth_filter_up->CalculateNumChildren(max);
-    if (!num_children_or_err) {
-      m_synthetic_children_count = 0;
-      return num_children_or_err;
-    }
-    auto num_children = (m_synthetic_children_count = *num_children_or_err);
+    size_t num_children = (m_synthetic_children_count =
+                               m_synth_filter_up->CalculateNumChildren(max));
     LLDB_LOGF(log,
               "[ValueObjectSynthetic::CalculateNumChildren] for VO of name "
-              "%s and type %s, the filter returned %u child values",
+              "%s and type %s, the filter returned %zu child values",
               GetName().AsCString(), GetTypeName().AsCString(), num_children);
     return num_children;
   }
diff --git a/lldb/source/Core/ValueObjectVTable.cpp b/lldb/source/Core/ValueObjectVTable.cpp
index 9bfe8dc9700d..4d1cbb8d2f6f 100644
--- a/lldb/source/Core/ValueObjectVTable.cpp
+++ b/lldb/source/Core/ValueObjectVTable.cpp
@@ -33,9 +33,7 @@ public:
 
   std::optional GetByteSize() override { return m_addr_size; };
 
-  llvm::Expected CalculateNumChildren(uint32_t max) override {
-    return 0;
-  };
+  uint32_t CalculateNumChildren(uint32_t max) override { return 0; };
 
   ValueType GetValueType() const override { return eValueTypeVTableEntry; };
 
@@ -161,7 +159,7 @@ std::optional ValueObjectVTable::GetByteSize() {
   return std::nullopt;
 }
 
-llvm::Expected ValueObjectVTable::CalculateNumChildren(uint32_t max) {
+uint32_t ValueObjectVTable::CalculateNumChildren(uint32_t max) {
   if (UpdateValueIfNeeded(false))
     return m_num_vtable_entries <= max ? m_num_vtable_entries : max;
   return 0;
diff --git a/lldb/source/Core/ValueObjectVariable.cpp b/lldb/source/Core/ValueObjectVariable.cpp
index fb29c22c0ab5..dc62bb6358dc 100644
--- a/lldb/source/Core/ValueObjectVariable.cpp
+++ b/lldb/source/Core/ValueObjectVariable.cpp
@@ -94,8 +94,7 @@ ConstString ValueObjectVariable::GetQualifiedTypeName() {
   return ConstString();
 }
 
-llvm::Expected
-ValueObjectVariable::CalculateNumChildren(uint32_t max) {
+uint32_t ValueObjectVariable::CalculateNumChildren(uint32_t max) {
   CompilerType type(GetCompilerType());
 
   if (!type.IsValid())
@@ -104,9 +103,7 @@ ValueObjectVariable::CalculateNumChildren(uint32_t max) {
   ExecutionContext exe_ctx(GetExecutionContextRef());
   const bool omit_empty_base_classes = true;
   auto child_count = type.GetNumChildren(omit_empty_base_classes, &exe_ctx);
-  if (!child_count)
-    return child_count;
-  return *child_count <= max ? *child_count : max;
+  return child_count <= max ? child_count : max;
 }
 
 std::optional ValueObjectVariable::GetByteSize() {
diff --git a/lldb/source/DataFormatters/FormatManager.cpp b/lldb/source/DataFormatters/FormatManager.cpp
index d7ba5b4b70c9..092fa3c8ce49 100644
--- a/lldb/source/DataFormatters/FormatManager.cpp
+++ b/lldb/source/DataFormatters/FormatManager.cpp
@@ -451,13 +451,8 @@ bool FormatManager::ShouldPrintAsOneLiner(ValueObject &valobj) {
   if (valobj.GetSummaryFormat().get() != nullptr)
     return valobj.GetSummaryFormat()->IsOneLiner();
 
-  auto num_children = valobj.GetNumChildren();
-  if (!num_children) {
-    llvm::consumeError(num_children.takeError());
-    return true;
-  }
   // no children, no party
-  if (*num_children == 0)
+  if (valobj.GetNumChildren() == 0)
     return false;
 
   // ask the type if it has any opinion about this eLazyBoolCalculate == no
@@ -476,7 +471,7 @@ bool FormatManager::ShouldPrintAsOneLiner(ValueObject &valobj) {
 
   size_t total_children_name_len = 0;
 
-  for (size_t idx = 0; idx < *num_children; idx++) {
+  for (size_t idx = 0; idx < valobj.GetNumChildren(); idx++) {
     bool is_synth_val = false;
     ValueObjectSP child_sp(valobj.GetChildAtIndex(idx));
     // something is wrong here - bail out
@@ -528,7 +523,7 @@ bool FormatManager::ShouldPrintAsOneLiner(ValueObject &valobj) {
     }
 
     // if this child has children..
-    if (child_sp->HasChildren()) {
+    if (child_sp->GetNumChildren()) {
       // ...and no summary...
       // (if it had a summary and the summary wanted children, we would have
       // bailed out anyway
diff --git a/lldb/source/DataFormatters/TypeSynthetic.cpp b/lldb/source/DataFormatters/TypeSynthetic.cpp
index e3f613606aa8..0ae38c4d31f2 100644
--- a/lldb/source/DataFormatters/TypeSynthetic.cpp
+++ b/lldb/source/DataFormatters/TypeSynthetic.cpp
@@ -115,17 +115,6 @@ std::string CXXSyntheticChildren::GetDescription() {
   return std::string(sstr.GetString());
 }
 
-uint32_t
-SyntheticChildrenFrontEnd::CalculateNumChildrenIgnoringErrors(uint32_t max) {
-  auto value_or_err = CalculateNumChildren(max);
-  if (value_or_err)
-    return *value_or_err;
-  Log *log = GetLog(LLDBLog::DataFormatters);
-  if (log && log->GetVerbose())
-    LLDB_LOG_ERROR(log, value_or_err.takeError(), "{0}");
-  return 0;
-}
-
 lldb::ValueObjectSP SyntheticChildrenFrontEnd::CreateValueObjectFromExpression(
     llvm::StringRef name, llvm::StringRef expression,
     const ExecutionContext &exe_ctx) {
@@ -189,15 +178,13 @@ bool ScriptedSyntheticChildren::FrontEnd::IsValid() {
   return (m_wrapper_sp && m_wrapper_sp->IsValid() && m_interpreter);
 }
 
-llvm::Expected
-ScriptedSyntheticChildren::FrontEnd::CalculateNumChildren() {
+uint32_t ScriptedSyntheticChildren::FrontEnd::CalculateNumChildren() {
   if (!m_wrapper_sp || m_interpreter == nullptr)
     return 0;
   return m_interpreter->CalculateNumChildren(m_wrapper_sp, UINT32_MAX);
 }
 
-llvm::Expected
-ScriptedSyntheticChildren::FrontEnd::CalculateNumChildren(uint32_t max) {
+uint32_t ScriptedSyntheticChildren::FrontEnd::CalculateNumChildren(uint32_t max) {
   if (!m_wrapper_sp || m_interpreter == nullptr)
     return 0;
   return m_interpreter->CalculateNumChildren(m_wrapper_sp, max);
diff --git a/lldb/source/DataFormatters/ValueObjectPrinter.cpp b/lldb/source/DataFormatters/ValueObjectPrinter.cpp
index b853199e878c..46e50a8d421a 100644
--- a/lldb/source/DataFormatters/ValueObjectPrinter.cpp
+++ b/lldb/source/DataFormatters/ValueObjectPrinter.cpp
@@ -627,7 +627,7 @@ uint32_t ValueObjectPrinter::GetMaxNumChildrenToPrint(bool &print_dotdotdot) {
   if (m_options.m_pointer_as_array)
     return m_options.m_pointer_as_array.m_element_count;
 
-  uint32_t num_children = synth_valobj.GetNumChildrenIgnoringErrors();
+  size_t num_children = synth_valobj.GetNumChildren();
   print_dotdotdot = false;
   if (num_children) {
     const size_t max_num_children = GetMostSpecializedValue()
diff --git a/lldb/source/DataFormatters/VectorType.cpp b/lldb/source/DataFormatters/VectorType.cpp
index 19de204c2435..a0626a8cba77 100644
--- a/lldb/source/DataFormatters/VectorType.cpp
+++ b/lldb/source/DataFormatters/VectorType.cpp
@@ -9,7 +9,6 @@
 #include "lldb/DataFormatters/VectorType.h"
 
 #include "lldb/Core/ValueObject.h"
-#include "lldb/Core/ValueObjectConstResult.h"
 #include "lldb/DataFormatters/FormattersHelpers.h"
 #include "lldb/Symbol/CompilerType.h"
 #include "lldb/Symbol/TypeSystem.h"
@@ -225,16 +224,10 @@ public:
 
   ~VectorTypeSyntheticFrontEnd() override = default;
 
-  llvm::Expected CalculateNumChildren() override {
-    return m_num_children;
-  }
+  uint32_t CalculateNumChildren() override { return m_num_children; }
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override {
-    auto num_children_or_err = CalculateNumChildren();
-    if (!num_children_or_err)
-      return ValueObjectConstResult::Create(
-          nullptr, Status(num_children_or_err.takeError()));
-    if (idx >= *num_children_or_err)
+    if (idx >= CalculateNumChildren())
       return {};
     std::optional size = m_child_type.GetByteSize(nullptr);
     if (!size)
@@ -273,7 +266,7 @@ public:
   size_t GetIndexOfChildWithName(ConstString name) override {
     const char *item_name = name.GetCString();
     uint32_t idx = ExtractIndexFromString(item_name);
-    if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
+    if (idx < UINT32_MAX && idx >= CalculateNumChildren())
       return UINT32_MAX;
     return idx;
   }
@@ -300,8 +293,7 @@ bool lldb_private::formatters::VectorTypeSummaryProvider(
   s.PutChar('(');
   bool first = true;
 
-  size_t idx = 0,
-         len = synthetic_children->CalculateNumChildrenIgnoringErrors();
+  size_t idx = 0, len = synthetic_children->CalculateNumChildren();
 
   for (; idx < len; idx++) {
     auto child_sp = synthetic_children->GetChildAtIndex(idx);
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionSourceCode.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionSourceCode.cpp
index 3d43ed3f99ff..b48bbbecc0cd 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionSourceCode.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionSourceCode.cpp
@@ -229,7 +229,7 @@ void AddLambdaCaptureDecls(StreamString &stream, StackFrame *frame,
   assert(frame);
 
   if (auto thisValSP = ClangExpressionUtil::GetLambdaValueObject(frame)) {
-    uint32_t numChildren = thisValSP->GetNumChildrenIgnoringErrors();
+    uint32_t numChildren = thisValSP->GetNumChildren();
     for (uint32_t i = 0; i < numChildren; ++i) {
       auto childVal = thisValSP->GetChildAtIndex(i);
       ConstString childName(childVal ? childVal->GetName() : ConstString(""));
diff --git a/lldb/source/Plugins/InstrumentationRuntime/TSan/InstrumentationRuntimeTSan.cpp b/lldb/source/Plugins/InstrumentationRuntime/TSan/InstrumentationRuntimeTSan.cpp
index b2781aa5e7db..72293c5331f4 100644
--- a/lldb/source/Plugins/InstrumentationRuntime/TSan/InstrumentationRuntimeTSan.cpp
+++ b/lldb/source/Plugins/InstrumentationRuntime/TSan/InstrumentationRuntimeTSan.cpp
@@ -27,8 +27,6 @@
 #include "lldb/Target/StopInfo.h"
 #include "lldb/Target/Target.h"
 #include "lldb/Target/Thread.h"
-#include "lldb/Utility/LLDBLog.h"
-#include "lldb/Utility/Log.h"
 #include "lldb/Utility/RegularExpression.h"
 #include "lldb/Utility/Stream.h"
 
@@ -213,7 +211,7 @@ CreateStackTrace(ValueObjectSP o,
   auto trace_sp = std::make_shared();
   ValueObjectSP trace_value_object =
       o->GetValueForExpressionPath(trace_item_name.c_str());
-  size_t count = trace_value_object->GetNumChildrenIgnoringErrors();
+  size_t count = trace_value_object->GetNumChildren();
   for (size_t j = 0; j < count; j++) {
     addr_t trace_addr =
         trace_value_object->GetChildAtIndex(j)->GetValueAsUnsigned(0);
diff --git a/lldb/source/Plugins/Language/CPlusPlus/BlockPointer.cpp b/lldb/source/Plugins/Language/CPlusPlus/BlockPointer.cpp
index 9a6e135e0083..ef0f67d1e9f9 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/BlockPointer.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/BlockPointer.cpp
@@ -74,7 +74,7 @@ public:
 
   ~BlockPointerSyntheticFrontEnd() override = default;
 
-  llvm::Expected CalculateNumChildren() override {
+  uint32_t CalculateNumChildren() override {
     const bool omit_empty_base_classes = false;
     return m_block_struct_type.GetNumChildren(omit_empty_base_classes, nullptr);
   }
@@ -84,7 +84,7 @@ public:
       return lldb::ValueObjectSP();
     }
 
-    if (idx >= CalculateNumChildrenIgnoringErrors()) {
+    if (idx >= CalculateNumChildren()) {
       return lldb::ValueObjectSP();
     }
 
diff --git a/lldb/source/Plugins/Language/CPlusPlus/Coroutines.cpp b/lldb/source/Plugins/Language/CPlusPlus/Coroutines.cpp
index 5e63d1d7b214..3827f9c21eff 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/Coroutines.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/Coroutines.cpp
@@ -11,8 +11,6 @@
 #include "Plugins/TypeSystem/Clang/TypeSystemClang.h"
 #include "lldb/Symbol/Function.h"
 #include "lldb/Symbol/VariableList.h"
-#include "lldb/Utility/LLDBLog.h"
-#include "lldb/Utility/Log.h"
 
 using namespace lldb;
 using namespace lldb_private;
@@ -24,7 +22,7 @@ static lldb::addr_t GetCoroFramePtrFromHandle(ValueObjectSP valobj_sp) {
 
   // We expect a single pointer in the `coroutine_handle` class.
   // We don't care about its name.
-  if (valobj_sp->GetNumChildrenIgnoringErrors() != 1)
+  if (valobj_sp->GetNumChildren() != 1)
     return LLDB_INVALID_ADDRESS;
   ValueObjectSP ptr_sp(valobj_sp->GetChildAtIndex(0));
   if (!ptr_sp)
@@ -106,8 +104,8 @@ lldb_private::formatters::StdlibCoroutineHandleSyntheticFrontEnd::
 lldb_private::formatters::StdlibCoroutineHandleSyntheticFrontEnd::
     ~StdlibCoroutineHandleSyntheticFrontEnd() = default;
 
-llvm::Expected lldb_private::formatters::
-    StdlibCoroutineHandleSyntheticFrontEnd::CalculateNumChildren() {
+uint32_t lldb_private::formatters::StdlibCoroutineHandleSyntheticFrontEnd::
+    CalculateNumChildren() {
   if (!m_resume_ptr_sp || !m_destroy_ptr_sp)
     return 0;
 
diff --git a/lldb/source/Plugins/Language/CPlusPlus/Coroutines.h b/lldb/source/Plugins/Language/CPlusPlus/Coroutines.h
index 1d4bc65e2637..5c6a80b57ff4 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/Coroutines.h
+++ b/lldb/source/Plugins/Language/CPlusPlus/Coroutines.h
@@ -34,7 +34,7 @@ public:
 
   ~StdlibCoroutineHandleSyntheticFrontEnd() override;
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
diff --git a/lldb/source/Plugins/Language/CPlusPlus/GenericBitset.cpp b/lldb/source/Plugins/Language/CPlusPlus/GenericBitset.cpp
index 33955dccb6cc..6a9da1d17c76 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/GenericBitset.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/GenericBitset.cpp
@@ -34,9 +34,7 @@ public:
 
   bool MightHaveChildren() override { return true; }
   lldb::ChildCacheState Update() override;
-  llvm::Expected CalculateNumChildren() override {
-    return m_elements.size();
-  }
+  uint32_t CalculateNumChildren() override { return m_elements.size(); }
   ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
 private:
diff --git a/lldb/source/Plugins/Language/CPlusPlus/GenericOptional.cpp b/lldb/source/Plugins/Language/CPlusPlus/GenericOptional.cpp
index 23756de7f1e6..c06afb53eb8a 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/GenericOptional.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/GenericOptional.cpp
@@ -19,7 +19,7 @@ using namespace lldb_private;
 bool lldb_private::formatters::GenericOptionalSummaryProvider(
     ValueObject &valobj, Stream &stream, const TypeSummaryOptions &options) {
   stream.Printf(" Has Value=%s ",
-                valobj.GetNumChildrenIgnoringErrors() == 0 ? "false" : "true");
+                valobj.GetNumChildren() == 0 ? "false" : "true");
 
   return true;
 }
@@ -41,9 +41,7 @@ public:
   }
 
   bool MightHaveChildren() override { return true; }
-  llvm::Expected CalculateNumChildren() override {
-    return m_has_value ? 1U : 0U;
-  }
+  uint32_t CalculateNumChildren() override { return m_has_value ? 1U : 0U; }
 
   ValueObjectSP GetChildAtIndex(uint32_t idx) override;
   lldb::ChildCacheState Update() override;
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp
index 10a1fe039189..bba887fec3ac 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp
@@ -63,7 +63,7 @@ lldb::ValueObjectSP
 lldb_private::formatters::GetSecondValueOfLibCXXCompressedPair(
     ValueObject &pair) {
   ValueObjectSP value;
-  if (pair.GetNumChildrenIgnoringErrors() > 1) {
+  if (pair.GetNumChildren() > 1) {
     ValueObjectSP second_child = pair.GetChildAtIndex(1);
     if (second_child) {
       value = second_child->GetChildMemberWithName("__value_");
@@ -351,8 +351,8 @@ lldb_private::formatters::LibCxxMapIteratorSyntheticFrontEnd::Update() {
   return lldb::ChildCacheState::eRefetch;
 }
 
-llvm::Expected lldb_private::formatters::
-    LibCxxMapIteratorSyntheticFrontEnd::CalculateNumChildren() {
+uint32_t lldb_private::formatters::LibCxxMapIteratorSyntheticFrontEnd::
+    CalculateNumChildren() {
   return 2;
 }
 
@@ -509,8 +509,8 @@ lldb::ChildCacheState lldb_private::formatters::
   return lldb::ChildCacheState::eRefetch;
 }
 
-llvm::Expected lldb_private::formatters::
-    LibCxxUnorderedMapIteratorSyntheticFrontEnd::CalculateNumChildren() {
+uint32_t lldb_private::formatters::LibCxxUnorderedMapIteratorSyntheticFrontEnd::
+    CalculateNumChildren() {
   return 2;
 }
 
@@ -566,8 +566,8 @@ lldb_private::formatters::LibcxxSharedPtrSyntheticFrontEnd::
     Update();
 }
 
-llvm::Expected lldb_private::formatters::
-    LibcxxSharedPtrSyntheticFrontEnd::CalculateNumChildren() {
+uint32_t lldb_private::formatters::LibcxxSharedPtrSyntheticFrontEnd::
+    CalculateNumChildren() {
   return (m_cntrl ? 1 : 0);
 }
 
@@ -661,8 +661,8 @@ lldb_private::formatters::LibcxxUniquePtrSyntheticFrontEndCreator(
                     : nullptr);
 }
 
-llvm::Expected lldb_private::formatters::
-    LibcxxUniquePtrSyntheticFrontEnd::CalculateNumChildren() {
+uint32_t lldb_private::formatters::LibcxxUniquePtrSyntheticFrontEnd::
+    CalculateNumChildren() {
   if (m_value_ptr_sp)
     return m_deleter_sp ? 2 : 1;
   return 0;
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h
index a59f21841ec8..ad2f58508ab7 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h
@@ -87,7 +87,7 @@ class LibCxxMapIteratorSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   LibCxxMapIteratorSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -135,7 +135,7 @@ public:
 
   ~LibCxxUnorderedMapIteratorSyntheticFrontEnd() override = default;
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -166,7 +166,7 @@ class LibcxxSharedPtrSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   LibcxxSharedPtrSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -186,7 +186,7 @@ class LibcxxUniquePtrSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   LibcxxUniquePtrSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxAtomic.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxAtomic.cpp
index 7f30dc186291..8e4c36103a74 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxAtomic.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxAtomic.cpp
@@ -90,7 +90,7 @@ public:
 
   ~LibcxxStdAtomicSyntheticFrontEnd() override = default;
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -124,8 +124,8 @@ bool lldb_private::formatters::LibcxxStdAtomicSyntheticFrontEnd::
   return true;
 }
 
-llvm::Expected lldb_private::formatters::
-    LibcxxStdAtomicSyntheticFrontEnd::CalculateNumChildren() {
+uint32_t lldb_private::formatters::LibcxxStdAtomicSyntheticFrontEnd::
+    CalculateNumChildren() {
   return m_real_child ? 1 : 0;
 }
 
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxInitializerList.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxInitializerList.cpp
index bd9c72497664..00012dfc056e 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxInitializerList.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxInitializerList.cpp
@@ -26,7 +26,7 @@ public:
 
   ~LibcxxInitializerListSyntheticFrontEnd() override;
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -59,8 +59,8 @@ lldb_private::formatters::LibcxxInitializerListSyntheticFrontEnd::
   // delete m_start;
 }
 
-llvm::Expected lldb_private::formatters::
-    LibcxxInitializerListSyntheticFrontEnd::CalculateNumChildren() {
+uint32_t lldb_private::formatters::LibcxxInitializerListSyntheticFrontEnd::
+    CalculateNumChildren() {
   m_num_elements = 0;
   ValueObjectSP size_sp(m_backend.GetChildMemberWithName("__size_"));
   if (size_sp)
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxList.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxList.cpp
index d7cfeb30557c..17f6b737d9f6 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxList.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxList.cpp
@@ -136,7 +136,7 @@ class ForwardListFrontEnd : public AbstractListFrontEnd {
 public:
   ForwardListFrontEnd(ValueObject &valobj);
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
   ValueObjectSP GetChildAtIndex(uint32_t idx) override;
   lldb::ChildCacheState Update() override;
 };
@@ -147,7 +147,7 @@ public:
 
   ~ListFrontEnd() override = default;
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -240,7 +240,7 @@ ForwardListFrontEnd::ForwardListFrontEnd(ValueObject &valobj)
   Update();
 }
 
-llvm::Expected ForwardListFrontEnd::CalculateNumChildren() {
+uint32_t ForwardListFrontEnd::CalculateNumChildren() {
   if (m_count != UINT32_MAX)
     return m_count;
 
@@ -254,7 +254,7 @@ llvm::Expected ForwardListFrontEnd::CalculateNumChildren() {
 }
 
 ValueObjectSP ForwardListFrontEnd::GetChildAtIndex(uint32_t idx) {
-  if (idx >= CalculateNumChildrenIgnoringErrors())
+  if (idx >= CalculateNumChildren())
     return nullptr;
 
   if (!m_head)
@@ -308,7 +308,7 @@ ListFrontEnd::ListFrontEnd(lldb::ValueObjectSP valobj_sp)
     Update();
 }
 
-llvm::Expected ListFrontEnd::CalculateNumChildren() {
+uint32_t ListFrontEnd::CalculateNumChildren() {
   if (m_count != UINT32_MAX)
     return m_count;
   if (!m_head || !m_tail || m_node_address == 0)
@@ -347,7 +347,7 @@ lldb::ValueObjectSP ListFrontEnd::GetChildAtIndex(uint32_t idx) {
   static ConstString g_value("__value_");
   static ConstString g_next("__next_");
 
-  if (idx >= CalculateNumChildrenIgnoringErrors())
+  if (idx >= CalculateNumChildren())
     return lldb::ValueObjectSP();
 
   if (!m_head || !m_tail || m_node_address == 0)
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxMap.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxMap.cpp
index ec5b320e2218..6d24eb03779c 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxMap.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxMap.cpp
@@ -177,7 +177,7 @@ public:
 
   ~LibcxxStdMapSyntheticFrontEnd() override = default;
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -209,8 +209,8 @@ lldb_private::formatters::LibcxxStdMapSyntheticFrontEnd::
     Update();
 }
 
-llvm::Expected lldb_private::formatters::
-    LibcxxStdMapSyntheticFrontEnd::CalculateNumChildren() {
+uint32_t lldb_private::formatters::LibcxxStdMapSyntheticFrontEnd::
+    CalculateNumChildren() {
   if (m_count != UINT32_MAX)
     return m_count;
 
@@ -311,13 +311,13 @@ lldb_private::formatters::LibcxxStdMapSyntheticFrontEnd::GetChildAtIndex(
     uint32_t idx) {
   static ConstString g_cc_("__cc_"), g_cc("__cc");
   static ConstString g_nc("__nc");
-  uint32_t num_children = CalculateNumChildrenIgnoringErrors();
-  if (idx >= num_children)
+
+  if (idx >= CalculateNumChildren())
     return lldb::ValueObjectSP();
   if (m_tree == nullptr || m_root_node == nullptr)
     return lldb::ValueObjectSP();
 
-  MapIterator iterator(m_root_node, num_children);
+  MapIterator iterator(m_root_node, CalculateNumChildren());
 
   const bool need_to_skip = (idx > 0);
   size_t actual_advancde = idx;
@@ -382,7 +382,7 @@ lldb_private::formatters::LibcxxStdMapSyntheticFrontEnd::GetChildAtIndex(
   name.Printf("[%" PRIu64 "]", (uint64_t)idx);
   auto potential_child_sp = iterated_sp->Clone(ConstString(name.GetString()));
   if (potential_child_sp) {
-    switch (potential_child_sp->GetNumChildrenIgnoringErrors()) {
+    switch (potential_child_sp->GetNumChildren()) {
     case 1: {
       auto child0_sp = potential_child_sp->GetChildAtIndex(0);
       if (child0_sp &&
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxQueue.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxQueue.cpp
index 5b459a17fe29..fbadee89b7b7 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxQueue.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxQueue.cpp
@@ -28,7 +28,7 @@ public:
   bool MightHaveChildren() override { return true; }
   lldb::ChildCacheState Update() override;
 
-  llvm::Expected CalculateNumChildren() override {
+  uint32_t CalculateNumChildren() override {
     return m_container_sp ? m_container_sp->GetNumChildren() : 0;
   }
 
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxRangesRefView.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxRangesRefView.cpp
index 01a7b8f142ec..74f54f767356 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxRangesRefView.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxRangesRefView.cpp
@@ -27,7 +27,7 @@ public:
 
   ~LibcxxStdRangesRefViewSyntheticFrontEnd() override = default;
 
-  llvm::Expected CalculateNumChildren() override {
+  uint32_t CalculateNumChildren() override {
     // __range_ will be the sole child of this type
     return 1;
   }
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxSpan.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxSpan.cpp
index 9895f336bfd0..af2b51d2b540 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxSpan.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxSpan.cpp
@@ -27,7 +27,7 @@ public:
 
   ~LibcxxStdSpanSyntheticFrontEnd() override = default;
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -73,8 +73,8 @@ lldb_private::formatters::LibcxxStdSpanSyntheticFrontEnd::
     Update();
 }
 
-llvm::Expected lldb_private::formatters::
-    LibcxxStdSpanSyntheticFrontEnd::CalculateNumChildren() {
+uint32_t lldb_private::formatters::LibcxxStdSpanSyntheticFrontEnd::
+    CalculateNumChildren() {
   return m_num_elements;
 }
 
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxTuple.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxTuple.cpp
index 3e3259ab428d..62bb7d619267 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxTuple.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxTuple.cpp
@@ -26,9 +26,7 @@ public:
 
   bool MightHaveChildren() override { return true; }
   lldb::ChildCacheState Update() override;
-  llvm::Expected CalculateNumChildren() override {
-    return m_elements.size();
-  }
+  uint32_t CalculateNumChildren() override { return m_elements.size(); }
   ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
 private:
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxUnorderedMap.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxUnorderedMap.cpp
index af29fdb6d001..b3c364294335 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxUnorderedMap.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxUnorderedMap.cpp
@@ -33,7 +33,7 @@ public:
 
   ~LibcxxStdUnorderedMapSyntheticFrontEnd() override = default;
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -62,8 +62,8 @@ lldb_private::formatters::LibcxxStdUnorderedMapSyntheticFrontEnd::
     Update();
 }
 
-llvm::Expected lldb_private::formatters::
-    LibcxxStdUnorderedMapSyntheticFrontEnd::CalculateNumChildren() {
+uint32_t lldb_private::formatters::LibcxxStdUnorderedMapSyntheticFrontEnd::
+    CalculateNumChildren() {
   return m_num_elements;
 }
 
@@ -94,7 +94,7 @@ static bool isUnorderedMap(ConstString type_name) {
 
 lldb::ValueObjectSP lldb_private::formatters::
     LibcxxStdUnorderedMapSyntheticFrontEnd::GetChildAtIndex(uint32_t idx) {
-  if (idx >= CalculateNumChildrenIgnoringErrors())
+  if (idx >= CalculateNumChildren())
     return lldb::ValueObjectSP();
   if (m_tree == nullptr)
     return lldb::ValueObjectSP();
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxValarray.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxValarray.cpp
index 99f94406e99a..463c7b8d7ce3 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxValarray.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxValarray.cpp
@@ -24,7 +24,7 @@ public:
 
   ~LibcxxStdValarraySyntheticFrontEnd() override;
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -63,8 +63,8 @@ lldb_private::formatters::LibcxxStdValarraySyntheticFrontEnd::
   // delete m_finish;
 }
 
-llvm::Expected lldb_private::formatters::
-    LibcxxStdValarraySyntheticFrontEnd::CalculateNumChildren() {
+uint32_t lldb_private::formatters::LibcxxStdValarraySyntheticFrontEnd::
+    CalculateNumChildren() {
   if (!m_start || !m_finish)
     return 0;
   uint64_t start_val = m_start->GetValueAsUnsigned(0);
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxVariant.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxVariant.cpp
index 62794318e077..1f62062f09be 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxVariant.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxVariant.cpp
@@ -205,7 +205,7 @@ public:
 
   bool MightHaveChildren() override { return true; }
   lldb::ChildCacheState Update() override;
-  llvm::Expected CalculateNumChildren() override { return m_size; }
+  uint32_t CalculateNumChildren() override { return m_size; }
   ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
 private:
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxVector.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxVector.cpp
index 461fed35164b..fcf727ad2ea0 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxVector.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxVector.cpp
@@ -25,7 +25,7 @@ public:
 
   ~LibcxxStdVectorSyntheticFrontEnd() override;
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -46,7 +46,7 @@ class LibcxxVectorBoolSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   LibcxxVectorBoolSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -82,8 +82,8 @@ lldb_private::formatters::LibcxxStdVectorSyntheticFrontEnd::
   // delete m_finish;
 }
 
-llvm::Expected lldb_private::formatters::
-    LibcxxStdVectorSyntheticFrontEnd::CalculateNumChildren() {
+uint32_t lldb_private::formatters::LibcxxStdVectorSyntheticFrontEnd::
+    CalculateNumChildren() {
   if (!m_start || !m_finish)
     return 0;
   uint64_t start_val = m_start->GetValueAsUnsigned(0);
@@ -165,8 +165,8 @@ lldb_private::formatters::LibcxxVectorBoolSyntheticFrontEnd::
   }
 }
 
-llvm::Expected lldb_private::formatters::
-    LibcxxVectorBoolSyntheticFrontEnd::CalculateNumChildren() {
+uint32_t lldb_private::formatters::LibcxxVectorBoolSyntheticFrontEnd::
+    CalculateNumChildren() {
   return m_count;
 }
 
@@ -259,7 +259,7 @@ size_t lldb_private::formatters::LibcxxVectorBoolSyntheticFrontEnd::
     return UINT32_MAX;
   const char *item_name = name.GetCString();
   uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
     return UINT32_MAX;
   return idx;
 }
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.cpp
index 86bb575af5ca..5abb3d50674b 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.cpp
@@ -43,7 +43,7 @@ class LibstdcppMapIteratorSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   explicit LibstdcppMapIteratorSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -64,7 +64,7 @@ class LibStdcppSharedPtrSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   explicit LibStdcppSharedPtrSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -132,8 +132,7 @@ lldb::ChildCacheState LibstdcppMapIteratorSyntheticFrontEnd::Update() {
   return lldb::ChildCacheState::eReuse;
 }
 
-llvm::Expected
-LibstdcppMapIteratorSyntheticFrontEnd::CalculateNumChildren() {
+uint32_t LibstdcppMapIteratorSyntheticFrontEnd::CalculateNumChildren() {
   return 2;
 }
 
@@ -220,10 +219,7 @@ lldb::ChildCacheState VectorIteratorSyntheticFrontEnd::Update() {
   return lldb::ChildCacheState::eRefetch;
 }
 
-llvm::Expected
-VectorIteratorSyntheticFrontEnd::CalculateNumChildren() {
-  return 1;
-}
+uint32_t VectorIteratorSyntheticFrontEnd::CalculateNumChildren() { return 1; }
 
 lldb::ValueObjectSP
 VectorIteratorSyntheticFrontEnd::GetChildAtIndex(uint32_t idx) {
@@ -375,10 +371,7 @@ LibStdcppSharedPtrSyntheticFrontEnd::LibStdcppSharedPtrSyntheticFrontEnd(
     Update();
 }
 
-llvm::Expected
-LibStdcppSharedPtrSyntheticFrontEnd::CalculateNumChildren() {
-  return 1;
-}
+uint32_t LibStdcppSharedPtrSyntheticFrontEnd::CalculateNumChildren() { return 1; }
 
 lldb::ValueObjectSP
 LibStdcppSharedPtrSyntheticFrontEnd::GetChildAtIndex(uint32_t idx) {
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibStdcppTuple.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibStdcppTuple.cpp
index 05199ba35b9a..64d2ec9d943a 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibStdcppTuple.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibStdcppTuple.cpp
@@ -26,7 +26,7 @@ class LibStdcppTupleSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   explicit LibStdcppTupleSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -65,7 +65,7 @@ lldb::ChildCacheState LibStdcppTupleSyntheticFrontEnd::Update() {
     ValueObjectSP current_child = next_child_sp;
     next_child_sp = nullptr;
 
-    size_t child_count = current_child->GetNumChildrenIgnoringErrors();
+    size_t child_count = current_child->GetNumChildren();
     for (size_t i = 0; i < child_count; ++i) {
       ValueObjectSP child_sp = current_child->GetChildAtIndex(i);
       llvm::StringRef name_str = child_sp->GetName().GetStringRef();
@@ -95,8 +95,7 @@ LibStdcppTupleSyntheticFrontEnd::GetChildAtIndex(uint32_t idx) {
   return lldb::ValueObjectSP();
 }
 
-llvm::Expected
-LibStdcppTupleSyntheticFrontEnd::CalculateNumChildren() {
+uint32_t LibStdcppTupleSyntheticFrontEnd::CalculateNumChildren() {
   return m_members.size();
 }
 
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibStdcppUniquePointer.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibStdcppUniquePointer.cpp
index 92f540d9ca52..3a48fe412e07 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibStdcppUniquePointer.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibStdcppUniquePointer.cpp
@@ -26,7 +26,7 @@ class LibStdcppUniquePtrSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   explicit LibStdcppUniquePtrSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -135,8 +135,7 @@ LibStdcppUniquePtrSyntheticFrontEnd::GetChildAtIndex(uint32_t idx) {
   return lldb::ValueObjectSP();
 }
 
-llvm::Expected
-LibStdcppUniquePtrSyntheticFrontEnd::CalculateNumChildren() {
+uint32_t LibStdcppUniquePtrSyntheticFrontEnd::CalculateNumChildren() {
   if (m_del_obj)
     return 2;
   return 1;
diff --git a/lldb/source/Plugins/Language/ObjC/Cocoa.cpp b/lldb/source/Plugins/Language/ObjC/Cocoa.cpp
index 96166657ceeb..cb740f8e71e1 100644
--- a/lldb/source/Plugins/Language/ObjC/Cocoa.cpp
+++ b/lldb/source/Plugins/Language/ObjC/Cocoa.cpp
@@ -1038,7 +1038,7 @@ public:
 
   ~ObjCClassSyntheticChildrenFrontEnd() override = default;
 
-  llvm::Expected CalculateNumChildren() override { return 0; }
+  uint32_t CalculateNumChildren() override { return 0; }
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override {
     return lldb::ValueObjectSP();
diff --git a/lldb/source/Plugins/Language/ObjC/NSArray.cpp b/lldb/source/Plugins/Language/ObjC/NSArray.cpp
index 67d0cd08f51a..7f060b2613d6 100644
--- a/lldb/source/Plugins/Language/ObjC/NSArray.cpp
+++ b/lldb/source/Plugins/Language/ObjC/NSArray.cpp
@@ -50,7 +50,7 @@ public:
 
   ~NSArrayMSyntheticFrontEndBase() override = default;
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -214,7 +214,7 @@ public:
 
   ~GenericNSArrayISyntheticFrontEnd() override;
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -302,7 +302,7 @@ public:
 
   ~NSArray0SyntheticFrontEnd() override = default;
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -319,7 +319,7 @@ public:
 
   ~NSArray1SyntheticFrontEnd() override = default;
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -477,15 +477,15 @@ lldb_private::formatters::
     : NSArrayMSyntheticFrontEndBase(valobj_sp), m_data_32(nullptr),
       m_data_64(nullptr) {}
 
-llvm::Expected lldb_private::formatters::
-    NSArrayMSyntheticFrontEndBase::CalculateNumChildren() {
+uint32_t lldb_private::formatters::NSArrayMSyntheticFrontEndBase::
+    CalculateNumChildren() {
   return GetUsedCount();
 }
 
 lldb::ValueObjectSP
 lldb_private::formatters::NSArrayMSyntheticFrontEndBase::GetChildAtIndex(
     uint32_t idx) {
-  if (idx >= CalculateNumChildrenIgnoringErrors())
+  if (idx >= CalculateNumChildren())
     return lldb::ValueObjectSP();
   lldb::addr_t object_at_idx = GetDataAddress();
   size_t pyhs_idx = idx;
@@ -542,7 +542,7 @@ lldb_private::formatters::NSArrayMSyntheticFrontEndBase::GetIndexOfChildWithName
     ConstString name) {
   const char *item_name = name.GetCString();
   uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
     return UINT32_MAX;
   return idx;
 }
@@ -628,15 +628,15 @@ lldb_private::formatters::GenericNSArrayISyntheticFrontEnd::
   GetIndexOfChildWithName(ConstString name) {
   const char *item_name = name.GetCString();
   uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
     return UINT32_MAX;
   return idx;
 }
 
 template 
-llvm::Expected
-lldb_private::formatters::GenericNSArrayISyntheticFrontEnd<
-    D32, D64, Inline>::CalculateNumChildren() {
+uint32_t
+lldb_private::formatters::GenericNSArrayISyntheticFrontEnd::
+  CalculateNumChildren() {
   return m_data_32 ? m_data_32->used : m_data_64->used;
 }
 
@@ -685,7 +685,7 @@ template 
 lldb::ValueObjectSP
 lldb_private::formatters::GenericNSArrayISyntheticFrontEnd::
   GetChildAtIndex(uint32_t idx) {
-  if (idx >= CalculateNumChildrenIgnoringErrors())
+  if (idx >= CalculateNumChildren())
     return lldb::ValueObjectSP();
   lldb::addr_t object_at_idx;
   if (Inline) {
@@ -719,7 +719,7 @@ lldb_private::formatters::NSArray0SyntheticFrontEnd::GetIndexOfChildWithName(
   return UINT32_MAX;
 }
 
-llvm::Expected
+uint32_t
 lldb_private::formatters::NSArray0SyntheticFrontEnd::CalculateNumChildren() {
   return 0;
 }
@@ -754,7 +754,7 @@ lldb_private::formatters::NSArray1SyntheticFrontEnd::GetIndexOfChildWithName(
   return UINT32_MAX;
 }
 
-llvm::Expected
+uint32_t
 lldb_private::formatters::NSArray1SyntheticFrontEnd::CalculateNumChildren() {
   return 1;
 }
diff --git a/lldb/source/Plugins/Language/ObjC/NSDictionary.cpp b/lldb/source/Plugins/Language/ObjC/NSDictionary.cpp
index ec6fd756394a..da94eda1529c 100644
--- a/lldb/source/Plugins/Language/ObjC/NSDictionary.cpp
+++ b/lldb/source/Plugins/Language/ObjC/NSDictionary.cpp
@@ -103,7 +103,7 @@ public:
 
   ~NSDictionaryISyntheticFrontEnd() override;
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -144,7 +144,7 @@ class NSConstantDictionarySyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   NSConstantDictionarySyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -176,7 +176,7 @@ class NSCFDictionarySyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   NSCFDictionarySyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -209,7 +209,7 @@ public:
 
   ~NSDictionary1SyntheticFrontEnd() override = default;
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -230,7 +230,7 @@ public:
 
   ~GenericNSDictionaryMSyntheticFrontEnd() override;
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -262,9 +262,9 @@ namespace Foundation1100 {
     NSDictionaryMSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
     
     ~NSDictionaryMSyntheticFrontEnd() override;
-
-    llvm::Expected CalculateNumChildren() override;
-
+    
+    uint32_t CalculateNumChildren() override;
+    
     lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
     lldb::ChildCacheState Update() override;
@@ -601,13 +601,13 @@ size_t lldb_private::formatters::NSDictionaryISyntheticFrontEnd::
     GetIndexOfChildWithName(ConstString name) {
   const char *item_name = name.GetCString();
   uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
     return UINT32_MAX;
   return idx;
 }
 
-llvm::Expected lldb_private::formatters::
-    NSDictionaryISyntheticFrontEnd::CalculateNumChildren() {
+uint32_t lldb_private::formatters::NSDictionaryISyntheticFrontEnd::
+    CalculateNumChildren() {
   if (!m_data_32 && !m_data_64)
     return 0;
   return (m_data_32 ? m_data_32->_used : m_data_64->_used);
@@ -656,7 +656,7 @@ bool lldb_private::formatters::NSDictionaryISyntheticFrontEnd::
 lldb::ValueObjectSP
 lldb_private::formatters::NSDictionaryISyntheticFrontEnd::GetChildAtIndex(
     uint32_t idx) {
-  uint32_t num_children = CalculateNumChildrenIgnoringErrors();
+  uint32_t num_children = CalculateNumChildren();
 
   if (idx >= num_children)
     return lldb::ValueObjectSP();
@@ -739,13 +739,13 @@ size_t lldb_private::formatters::NSCFDictionarySyntheticFrontEnd::
     GetIndexOfChildWithName(ConstString name) {
   const char *item_name = name.GetCString();
   const uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
     return UINT32_MAX;
   return idx;
 }
 
-llvm::Expected lldb_private::formatters::
-    NSCFDictionarySyntheticFrontEnd::CalculateNumChildren() {
+uint32_t lldb_private::formatters::NSCFDictionarySyntheticFrontEnd::
+    CalculateNumChildren() {
   if (!m_hashtable.IsValid())
     return 0;
   return m_hashtable.GetCount();
@@ -781,7 +781,7 @@ lldb_private::formatters::NSCFDictionarySyntheticFrontEnd::GetChildAtIndex(
   lldb::addr_t m_keys_ptr = m_hashtable.GetKeyPointer();
   lldb::addr_t m_values_ptr = m_hashtable.GetValuePointer();
 
-  const uint32_t num_children = CalculateNumChildrenIgnoringErrors();
+  const uint32_t num_children = CalculateNumChildren();
 
   if (idx >= num_children)
     return lldb::ValueObjectSP();
@@ -875,13 +875,13 @@ size_t lldb_private::formatters::NSConstantDictionarySyntheticFrontEnd::
     GetIndexOfChildWithName(ConstString name) {
   const char *item_name = name.GetCString();
   uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
     return UINT32_MAX;
   return idx;
 }
 
-llvm::Expected lldb_private::formatters::
-    NSConstantDictionarySyntheticFrontEnd::CalculateNumChildren() {
+uint32_t lldb_private::formatters::NSConstantDictionarySyntheticFrontEnd::
+    CalculateNumChildren() {
   return m_size;
 }
 
@@ -921,7 +921,7 @@ bool lldb_private::formatters::NSConstantDictionarySyntheticFrontEnd::
 
 lldb::ValueObjectSP lldb_private::formatters::
     NSConstantDictionarySyntheticFrontEnd::GetChildAtIndex(uint32_t idx) {
-  uint32_t num_children = CalculateNumChildrenIgnoringErrors();
+  uint32_t num_children = CalculateNumChildren();
 
   if (idx >= num_children)
     return lldb::ValueObjectSP();
@@ -994,8 +994,8 @@ size_t lldb_private::formatters::NSDictionary1SyntheticFrontEnd::
   return name == g_zero ? 0 : UINT32_MAX;
 }
 
-llvm::Expected lldb_private::formatters::
-    NSDictionary1SyntheticFrontEnd::CalculateNumChildren() {
+uint32_t lldb_private::formatters::NSDictionary1SyntheticFrontEnd::
+    CalculateNumChildren() {
   return 1;
 }
 
@@ -1081,18 +1081,17 @@ size_t lldb_private::formatters::GenericNSDictionaryMSyntheticFrontEnd<
     D32, D64>::GetIndexOfChildWithName(ConstString name) {
   const char *item_name = name.GetCString();
   uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
     return UINT32_MAX;
   return idx;
 }
 
 template 
-llvm::Expected
-lldb_private::formatters::GenericNSDictionaryMSyntheticFrontEnd<
-    D32, D64>::CalculateNumChildren() {
+uint32_t
+lldb_private::formatters::GenericNSDictionaryMSyntheticFrontEnd::CalculateNumChildren() {
   if (!m_data_32 && !m_data_64)
     return 0;
-  return (m_data_32 ? (uint32_t)m_data_32->_used : (uint32_t)m_data_64->_used);
+  return (m_data_32 ? m_data_32->_used : m_data_64->_used);
 }
 
 template 
@@ -1154,7 +1153,7 @@ lldb_private::formatters::GenericNSDictionaryMSyntheticFrontEnd<
     m_values_ptr = m_data_64->_buffer + (m_ptr_size * size);
   }
 
-  uint32_t num_children = CalculateNumChildrenIgnoringErrors();
+  uint32_t num_children = CalculateNumChildren();
 
   if (idx >= num_children)
     return lldb::ValueObjectSP();
@@ -1246,13 +1245,14 @@ lldb_private::formatters::Foundation1100::
   NSDictionaryMSyntheticFrontEnd::GetIndexOfChildWithName(ConstString name) {
   const char *item_name = name.GetCString();
   uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
     return UINT32_MAX;
   return idx;
 }
 
-llvm::Expected lldb_private::formatters::Foundation1100::
-    NSDictionaryMSyntheticFrontEnd::CalculateNumChildren() {
+uint32_t
+lldb_private::formatters::Foundation1100::
+  NSDictionaryMSyntheticFrontEnd::CalculateNumChildren() {
   if (!m_data_32 && !m_data_64)
     return 0;
   return (m_data_32 ? m_data_32->_used : m_data_64->_used);
@@ -1306,7 +1306,7 @@ lldb_private::formatters::Foundation1100::
   lldb::addr_t m_values_ptr =
       (m_data_32 ? m_data_32->_objs_addr : m_data_64->_objs_addr);
 
-  uint32_t num_children = CalculateNumChildrenIgnoringErrors();
+  uint32_t num_children = CalculateNumChildren();
 
   if (idx >= num_children)
     return lldb::ValueObjectSP();
diff --git a/lldb/source/Plugins/Language/ObjC/NSError.cpp b/lldb/source/Plugins/Language/ObjC/NSError.cpp
index 5ef7edc7e80c..b034e799b716 100644
--- a/lldb/source/Plugins/Language/ObjC/NSError.cpp
+++ b/lldb/source/Plugins/Language/ObjC/NSError.cpp
@@ -116,7 +116,7 @@ public:
   // no need to delete m_child_ptr - it's kept alive by the cluster manager on
   // our behalf
 
-  llvm::Expected CalculateNumChildren() override {
+  uint32_t CalculateNumChildren() override {
     if (m_child_ptr)
       return 1;
     if (m_child_sp)
diff --git a/lldb/source/Plugins/Language/ObjC/NSException.cpp b/lldb/source/Plugins/Language/ObjC/NSException.cpp
index e7ce26ea4c6f..09d3a1b42b74 100644
--- a/lldb/source/Plugins/Language/ObjC/NSException.cpp
+++ b/lldb/source/Plugins/Language/ObjC/NSException.cpp
@@ -123,7 +123,9 @@ public:
 
   ~NSExceptionSyntheticFrontEnd() override = default;
 
-  llvm::Expected CalculateNumChildren() override { return 4; }
+  uint32_t CalculateNumChildren() override {
+    return 4;
+  }
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override {
     switch (idx) {
diff --git a/lldb/source/Plugins/Language/ObjC/NSIndexPath.cpp b/lldb/source/Plugins/Language/ObjC/NSIndexPath.cpp
index a434cee09d38..10bb907c58ed 100644
--- a/lldb/source/Plugins/Language/ObjC/NSIndexPath.cpp
+++ b/lldb/source/Plugins/Language/ObjC/NSIndexPath.cpp
@@ -40,9 +40,7 @@ public:
 
   ~NSIndexPathSyntheticFrontEnd() override = default;
 
-  llvm::Expected CalculateNumChildren() override {
-    return m_impl.GetNumIndexes();
-  }
+  uint32_t CalculateNumChildren() override { return m_impl.GetNumIndexes(); }
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override {
     return m_impl.GetIndexAtIndex(idx, m_uint_star_type);
@@ -129,7 +127,7 @@ public:
   size_t GetIndexOfChildWithName(ConstString name) override {
     const char *item_name = name.GetCString();
     uint32_t idx = ExtractIndexFromString(item_name);
-    if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
+    if (idx < UINT32_MAX && idx >= CalculateNumChildren())
       return UINT32_MAX;
     return idx;
   }
diff --git a/lldb/source/Plugins/Language/ObjC/NSSet.cpp b/lldb/source/Plugins/Language/ObjC/NSSet.cpp
index 7d0a6a507211..c965a2a13400 100644
--- a/lldb/source/Plugins/Language/ObjC/NSSet.cpp
+++ b/lldb/source/Plugins/Language/ObjC/NSSet.cpp
@@ -46,7 +46,7 @@ public:
 
   ~NSSetISyntheticFrontEnd() override;
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -84,7 +84,7 @@ class NSCFSetSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   NSCFSetSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -117,7 +117,7 @@ public:
 
   ~GenericNSSetMSyntheticFrontEnd() override;
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -233,7 +233,7 @@ public:
 
   ~NSSetCodeRunningSyntheticFrontEnd() override;
 
-  llvm::Expected CalculateNumChildren() override;
+  uint32_t CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -414,12 +414,12 @@ lldb_private::formatters::NSSetISyntheticFrontEnd::GetIndexOfChildWithName(
     ConstString name) {
   const char *item_name = name.GetCString();
   uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
     return UINT32_MAX;
   return idx;
 }
 
-llvm::Expected
+uint32_t
 lldb_private::formatters::NSSetISyntheticFrontEnd::CalculateNumChildren() {
   if (!m_data_32 && !m_data_64)
     return 0;
@@ -468,7 +468,7 @@ bool lldb_private::formatters::NSSetISyntheticFrontEnd::MightHaveChildren() {
 lldb::ValueObjectSP
 lldb_private::formatters::NSSetISyntheticFrontEnd::GetChildAtIndex(
     uint32_t idx) {
-  uint32_t num_children = CalculateNumChildrenIgnoringErrors();
+  uint32_t num_children = CalculateNumChildren();
 
   if (idx >= num_children)
     return lldb::ValueObjectSP();
@@ -551,12 +551,12 @@ lldb_private::formatters::NSCFSetSyntheticFrontEnd::GetIndexOfChildWithName(
     ConstString name) {
   const char *item_name = name.GetCString();
   const uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
     return UINT32_MAX;
   return idx;
 }
 
-llvm::Expected
+uint32_t
 lldb_private::formatters::NSCFSetSyntheticFrontEnd::CalculateNumChildren() {
   if (!m_hashtable.IsValid())
     return 0;
@@ -591,7 +591,7 @@ lldb_private::formatters::NSCFSetSyntheticFrontEnd::GetChildAtIndex(
     uint32_t idx) {
   lldb::addr_t m_values_ptr = m_hashtable.GetValuePointer();
 
-  const uint32_t num_children = CalculateNumChildrenIgnoringErrors();
+  const uint32_t num_children = CalculateNumChildren();
 
   if (idx >= num_children)
     return lldb::ValueObjectSP();
@@ -691,18 +691,18 @@ lldb_private::formatters::
     ConstString name) {
   const char *item_name = name.GetCString();
   uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
     return UINT32_MAX;
   return idx;
 }
 
 template 
-llvm::Expected
-lldb_private::formatters::GenericNSSetMSyntheticFrontEnd<
-    D32, D64>::CalculateNumChildren() {
+uint32_t
+lldb_private::formatters::
+  GenericNSSetMSyntheticFrontEnd::CalculateNumChildren() {
   if (!m_data_32 && !m_data_64)
     return 0;
-  return (m_data_32 ? (uint32_t)m_data_32->_used : (uint32_t)m_data_64->_used);
+  return (m_data_32 ? m_data_32->_used : m_data_64->_used);
 }
 
 template 
@@ -753,7 +753,7 @@ lldb_private::formatters::
   lldb::addr_t m_objs_addr =
       (m_data_32 ? m_data_32->_objs_addr : m_data_64->_objs_addr);
 
-  uint32_t num_children = CalculateNumChildrenIgnoringErrors();
+  uint32_t num_children = CalculateNumChildren();
 
   if (idx >= num_children)
     return lldb::ValueObjectSP();
diff --git a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntime.cpp b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntime.cpp
index 9434376f7d9e..f08f9f0f815d 100644
--- a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntime.cpp
+++ b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntime.cpp
@@ -539,8 +539,7 @@ ThreadSP AppleObjCRuntime::GetBacktraceThreadFromException(
     return object;
   };
 
-  for (size_t idx = 0; idx < reserved_dict->GetNumChildrenIgnoringErrors();
-       idx++) {
+  for (size_t idx = 0; idx < reserved_dict->GetNumChildren(); idx++) {
     ValueObjectSP dict_entry = reserved_dict->GetChildAtIndex(idx);
 
     DataExtractor data;
diff --git a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
index c02b08cb4782..51ab13108feb 100644
--- a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
+++ b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
@@ -5263,10 +5263,9 @@ GetDynamicArrayInfo(TypeSystemClang &ast, SymbolFile *sym_file,
   return std::nullopt;
 }
 
-llvm::Expected
-TypeSystemClang::GetNumChildren(lldb::opaque_compiler_type_t type,
-                                bool omit_empty_base_classes,
-                                const ExecutionContext *exe_ctx) {
+uint32_t TypeSystemClang::GetNumChildren(lldb::opaque_compiler_type_t type,
+                                         bool omit_empty_base_classes,
+                                         const ExecutionContext *exe_ctx) {
   if (!type)
     return 0;
 
@@ -5362,13 +5361,9 @@ TypeSystemClang::GetNumChildren(lldb::opaque_compiler_type_t type,
     CompilerType pointee_clang_type(GetPointeeType(type));
 
     uint32_t num_pointee_children = 0;
-    if (pointee_clang_type.IsAggregateType()) {
-      auto num_children_or_err =
+    if (pointee_clang_type.IsAggregateType())
+      num_pointee_children =
           pointee_clang_type.GetNumChildren(omit_empty_base_classes, exe_ctx);
-      if (!num_children_or_err)
-        return num_children_or_err;
-      num_pointee_children = *num_children_or_err;
-    }
     // If this type points to a simple type, then it has 1 child
     if (num_pointee_children == 0)
       num_children = 1;
@@ -5402,13 +5397,9 @@ TypeSystemClang::GetNumChildren(lldb::opaque_compiler_type_t type,
     clang::QualType pointee_type(pointer_type->getPointeeType());
     CompilerType pointee_clang_type(GetType(pointee_type));
     uint32_t num_pointee_children = 0;
-    if (pointee_clang_type.IsAggregateType()) {
-      auto num_children_or_err =
+    if (pointee_clang_type.IsAggregateType())
+      num_pointee_children =
           pointee_clang_type.GetNumChildren(omit_empty_base_classes, exe_ctx);
-      if (!num_children_or_err)
-        return num_children_or_err;
-      num_pointee_children = *num_children_or_err;
-    }
     if (num_pointee_children == 0) {
       // We have a pointer to a pointee type that claims it has no children. We
       // will want to look at
@@ -6117,15 +6108,8 @@ CompilerType TypeSystemClang::GetChildCompilerTypeAtIndex(
   child_is_base_class = false;
   language_flags = 0;
 
-  auto num_children_or_err =
-      GetNumChildren(type, omit_empty_base_classes, exe_ctx);
-  if (!num_children_or_err) {
-    LLDB_LOG_ERRORV(GetLog(LLDBLog::Types), num_children_or_err.takeError(),
-                    "{0}");
-    return {};
-  }
-
-  const bool idx_is_valid = idx < *num_children_or_err;
+  const bool idx_is_valid =
+      idx < GetNumChildren(type, omit_empty_base_classes, exe_ctx);
   int32_t bit_offset;
   switch (parent_type_class) {
   case clang::Type::Builtin:
@@ -6281,10 +6265,8 @@ CompilerType TypeSystemClang::GetChildCompilerTypeAtIndex(
               CompilerType base_class_clang_type =
                   GetType(getASTContext().getObjCInterfaceType(
                       superclass_interface_decl));
-              if (llvm::expectedToStdOptional(
-                      base_class_clang_type.GetNumChildren(
-                          omit_empty_base_classes, exe_ctx))
-                      .value_or(0) > 0) {
+              if (base_class_clang_type.GetNumChildren(omit_empty_base_classes,
+                                                       exe_ctx) > 0) {
                 if (idx == 0) {
                   clang::QualType ivar_qual_type(
                       getASTContext().getObjCInterfaceType(
diff --git a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
index 05c303baa416..a73164895baa 100644
--- a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
+++ b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
@@ -833,10 +833,9 @@ public:
   GetTypeBitAlign(lldb::opaque_compiler_type_t type,
                   ExecutionContextScope *exe_scope) override;
 
-  llvm::Expected
-  GetNumChildren(lldb::opaque_compiler_type_t type,
-                 bool omit_empty_base_classes,
-                 const ExecutionContext *exe_ctx) override;
+  uint32_t GetNumChildren(lldb::opaque_compiler_type_t type,
+                          bool omit_empty_base_classes,
+                          const ExecutionContext *exe_ctx) override;
 
   CompilerType GetBuiltinTypeByName(ConstString name) override;
 
diff --git a/lldb/source/Symbol/CompilerType.cpp b/lldb/source/Symbol/CompilerType.cpp
index 85dd2d841a5a..76b79daa6ac1 100644
--- a/lldb/source/Symbol/CompilerType.cpp
+++ b/lldb/source/Symbol/CompilerType.cpp
@@ -770,9 +770,8 @@ lldb::Format CompilerType::GetFormat() const {
   return lldb::eFormatDefault;
 }
 
-llvm::Expected
-CompilerType::GetNumChildren(bool omit_empty_base_classes,
-                             const ExecutionContext *exe_ctx) const {
+uint32_t CompilerType::GetNumChildren(bool omit_empty_base_classes,
+                                      const ExecutionContext *exe_ctx) const {
   if (IsValid())
     if (auto type_system_sp = GetTypeSystem())
       return type_system_sp->GetNumChildren(m_type, omit_empty_base_classes,
diff --git a/lldb/source/Symbol/Type.cpp b/lldb/source/Symbol/Type.cpp
index 44a24d7178f5..6069d066eaf6 100644
--- a/lldb/source/Symbol/Type.cpp
+++ b/lldb/source/Symbol/Type.cpp
@@ -488,7 +488,7 @@ std::optional Type::GetByteSize(ExecutionContextScope *exe_scope) {
   return {};
 }
 
-llvm::Expected Type::GetNumChildren(bool omit_empty_base_classes) {
+uint32_t Type::GetNumChildren(bool omit_empty_base_classes) {
   return GetForwardCompilerType().GetNumChildren(omit_empty_base_classes, nullptr);
 }
 
diff --git a/lldb/source/Symbol/Variable.cpp b/lldb/source/Symbol/Variable.cpp
index 90edede0f065..a33c3433d9e2 100644
--- a/lldb/source/Symbol/Variable.cpp
+++ b/lldb/source/Symbol/Variable.cpp
@@ -27,8 +27,6 @@
 #include "lldb/Target/StackFrame.h"
 #include "lldb/Target/Target.h"
 #include "lldb/Target/Thread.h"
-#include "lldb/Utility/LLDBLog.h"
-#include "lldb/Utility/Log.h"
 #include "lldb/Utility/RegularExpression.h"
 #include "lldb/Utility/Stream.h"
 
@@ -570,9 +568,7 @@ static void PrivateAutoComplete(
       case eTypeClassObjCObjectPointer:
       case eTypeClassPointer: {
         bool omit_empty_base_classes = true;
-        if (llvm::expectedToStdOptional(
-                compiler_type.GetNumChildren(omit_empty_base_classes, nullptr))
-                .value_or(0))
+        if (compiler_type.GetNumChildren(omit_empty_base_classes, nullptr) > 0)
           request.AddCompletion((prefix_path + "->").str());
         else {
           request.AddCompletion(prefix_path.str());
diff --git a/lldb/source/Target/StackFrame.cpp b/lldb/source/Target/StackFrame.cpp
index c29a71d92572..50cf01e63cd4 100644
--- a/lldb/source/Target/StackFrame.cpp
+++ b/lldb/source/Target/StackFrame.cpp
@@ -857,11 +857,10 @@ ValueObjectSP StackFrame::GetValueForVariableExpressionPath(
                   "\"(%s) %s\" is not an array type",
                   valobj_sp->GetTypeName().AsCString(""),
                   var_expr_path_strm.GetData());
-            } else if (static_cast(child_index) >=
-                       synthetic
-                           ->GetNumChildrenIgnoringErrors() /* synthetic does
-                                                                not have that
-                                                                many values */) {
+            } else if (
+                static_cast(child_index) >=
+                synthetic
+                    ->GetNumChildren() /* synthetic does not have that many values */) {
               valobj_sp->GetExpressionPath(var_expr_path_strm);
               error.SetErrorStringWithFormat(
                   "array index %ld is not valid for \"(%s) %s\"", child_index,
@@ -930,9 +929,10 @@ ValueObjectSP StackFrame::GetValueForVariableExpressionPath(
                 "\"(%s) %s\" is not an array type",
                 valobj_sp->GetTypeName().AsCString(""),
                 var_expr_path_strm.GetData());
-          } else if (static_cast(child_index) >=
-                     synthetic->GetNumChildrenIgnoringErrors() /* synthetic
-                                     does not have that many values */) {
+          } else if (
+              static_cast(child_index) >=
+              synthetic
+                  ->GetNumChildren() /* synthetic does not have that many values */) {
             valobj_sp->GetExpressionPath(var_expr_path_strm);
             error.SetErrorStringWithFormat(
                 "array index %ld is not valid for \"(%s) %s\"", child_index,
@@ -1397,8 +1397,7 @@ ValueObjectSP GetValueForOffset(StackFrame &frame, ValueObjectSP &parent,
     return parent;
   }
 
-  for (int ci = 0, ce = parent->GetNumChildrenIgnoringErrors(); ci != ce;
-       ++ci) {
+  for (int ci = 0, ce = parent->GetNumChildren(); ci != ce; ++ci) {
     ValueObjectSP child_sp = parent->GetChildAtIndex(ci);
 
     if (!child_sp) {
-- 
GitLab


From 08ddd2ce4048b0adbddc3ab9b770ccf43f81d65d Mon Sep 17 00:00:00 2001
From: Teresa Johnson 
Date: Fri, 8 Mar 2024 12:27:46 -0800
Subject: [PATCH 0081/1674] [PGO] Add support for writing previous indexed
 format (#84505)

Enable temporary support to ease use of new llvm-profdata with slightly
older indexed profiles after 16e74fd48988ac95551d0f64e1b36f78a82a89a2,
which bumped the indexed format for type profiling.
---
 .../llvm/ProfileData/InstrProfWriter.h        |  10 +-
 llvm/lib/ProfileData/InstrProfWriter.cpp      | 113 +++++++++++-------
 .../tools/llvm-profdata/profile-version.test  |   9 ++
 llvm/tools/llvm-profdata/llvm-profdata.cpp    |  13 +-
 4 files changed, 102 insertions(+), 43 deletions(-)
 create mode 100644 llvm/test/tools/llvm-profdata/profile-version.test

diff --git a/llvm/include/llvm/ProfileData/InstrProfWriter.h b/llvm/include/llvm/ProfileData/InstrProfWriter.h
index 7a806fd7fcf3..f70574d1f756 100644
--- a/llvm/include/llvm/ProfileData/InstrProfWriter.h
+++ b/llvm/include/llvm/ProfileData/InstrProfWriter.h
@@ -68,10 +68,18 @@ private:
   // Use raw pointer here for the incomplete type object.
   InstrProfRecordWriterTrait *InfoObj;
 
+  // Temporary support for writing the previous version of the format, to enable
+  // some forward compatibility. Currently this suppresses the writing of the
+  // new vtable names section and header fields.
+  // TODO: Consider enabling this with future version changes as well, to ease
+  // deployment of newer versions of llvm-profdata.
+  bool WritePrevVersion = false;
+
 public:
   InstrProfWriter(bool Sparse = false,
                   uint64_t TemporalProfTraceReservoirSize = 0,
-                  uint64_t MaxTemporalProfTraceLength = 0);
+                  uint64_t MaxTemporalProfTraceLength = 0,
+                  bool WritePrevVersion = false);
   ~InstrProfWriter();
 
   StringMap &getProfileData() { return FunctionData; }
diff --git a/llvm/lib/ProfileData/InstrProfWriter.cpp b/llvm/lib/ProfileData/InstrProfWriter.cpp
index 3e0a0e0d7011..e757c735719b 100644
--- a/llvm/lib/ProfileData/InstrProfWriter.cpp
+++ b/llvm/lib/ProfileData/InstrProfWriter.cpp
@@ -181,10 +181,12 @@ public:
 
 InstrProfWriter::InstrProfWriter(bool Sparse,
                                  uint64_t TemporalProfTraceReservoirSize,
-                                 uint64_t MaxTemporalProfTraceLength)
+                                 uint64_t MaxTemporalProfTraceLength,
+                                 bool WritePrevVersion)
     : Sparse(Sparse), MaxTemporalProfTraceLength(MaxTemporalProfTraceLength),
       TemporalProfTraceReservoirSize(TemporalProfTraceReservoirSize),
-      InfoObj(new InstrProfRecordWriterTrait()) {}
+      InfoObj(new InstrProfRecordWriterTrait()),
+      WritePrevVersion(WritePrevVersion) {}
 
 InstrProfWriter::~InstrProfWriter() { delete InfoObj; }
 
@@ -432,7 +434,13 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) {
   // Write the header.
   IndexedInstrProf::Header Header;
   Header.Magic = IndexedInstrProf::Magic;
-  Header.Version = IndexedInstrProf::ProfVersion::CurrentVersion;
+  Header.Version = WritePrevVersion
+                       ? IndexedInstrProf::ProfVersion::Version11
+                       : IndexedInstrProf::ProfVersion::CurrentVersion;
+  // The WritePrevVersion handling will either need to be removed or updated
+  // if the version is advanced beyond 12.
+  assert(IndexedInstrProf::ProfVersion::CurrentVersion ==
+         IndexedInstrProf::ProfVersion::Version12);
   if (static_cast(ProfileKind & InstrProfKind::IRInstrumentation))
     Header.Version |= VARIANT_MASK_IR_PROF;
   if (static_cast(ProfileKind & InstrProfKind::ContextSensitive))
@@ -484,7 +492,8 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) {
   OS.write(0);
 
   uint64_t VTableNamesOffset = OS.tell();
-  OS.write(0);
+  if (!WritePrevVersion)
+    OS.write(0);
 
   // Reserve space to write profile summary data.
   uint32_t NumEntries = ProfileSummaryBuilder::DefaultCutoffs.size();
@@ -608,27 +617,29 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) {
 
   uint64_t VTableNamesSectionStart = OS.tell();
 
-  // Use a dummy (and uncompressed) string as compressed vtable names and get
-  // the necessary profile format change in place for version 12.
-  // TODO: Store the list of vtable names in InstrProfWriter and use the
-  // real compressed name.
-  std::string CompressedVTableNames = "VTableNames";
+  if (!WritePrevVersion) {
+    // Use a dummy (and uncompressed) string as compressed vtable names and get
+    // the necessary profile format change in place for version 12.
+    // TODO: Store the list of vtable names in InstrProfWriter and use the
+    // real compressed name.
+    std::string CompressedVTableNames = "VTableNames";
 
-  uint64_t CompressedStringLen = CompressedVTableNames.length();
+    uint64_t CompressedStringLen = CompressedVTableNames.length();
 
-  // Record the length of compressed string.
-  OS.write(CompressedStringLen);
+    // Record the length of compressed string.
+    OS.write(CompressedStringLen);
 
-  // Write the chars in compressed strings.
-  for (auto &c : CompressedVTableNames)
-    OS.writeByte(static_cast(c));
+    // Write the chars in compressed strings.
+    for (auto &c : CompressedVTableNames)
+      OS.writeByte(static_cast(c));
 
-  // Pad up to a multiple of 8.
-  // InstrProfReader would read bytes according to 'CompressedStringLen'.
-  uint64_t PaddedLength = alignTo(CompressedStringLen, 8);
+    // Pad up to a multiple of 8.
+    // InstrProfReader would read bytes according to 'CompressedStringLen'.
+    uint64_t PaddedLength = alignTo(CompressedStringLen, 8);
 
-  for (uint64_t K = CompressedStringLen; K < PaddedLength; K++) {
-    OS.writeByte(0);
+    for (uint64_t K = CompressedStringLen; K < PaddedLength; K++) {
+      OS.writeByte(0);
+    }
   }
 
   uint64_t TemporalProfTracesSectionStart = 0;
@@ -662,26 +673,48 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) {
   }
   InfoObj->CSSummaryBuilder = nullptr;
 
-  // Now do the final patch:
-  PatchItem PatchItems[] = {
-      // Patch the Header.HashOffset field.
-      {HashTableStartFieldOffset, &HashTableStart, 1},
-      // Patch the Header.MemProfOffset (=0 for profiles without MemProf
-      // data).
-      {MemProfSectionOffset, &MemProfSectionStart, 1},
-      // Patch the Header.BinaryIdSectionOffset.
-      {BinaryIdSectionOffset, &BinaryIdSectionStart, 1},
-      // Patch the Header.TemporalProfTracesOffset (=0 for profiles without
-      // traces).
-      {TemporalProfTracesOffset, &TemporalProfTracesSectionStart, 1},
-      {VTableNamesOffset, &VTableNamesSectionStart, 1},
-      // Patch the summary data.
-      {SummaryOffset, reinterpret_cast(TheSummary.get()),
-       (int)(SummarySize / sizeof(uint64_t))},
-      {CSSummaryOffset, reinterpret_cast(TheCSSummary.get()),
-       (int)CSSummarySize}};
-
-  OS.patch(PatchItems, std::size(PatchItems));
+  if (!WritePrevVersion) {
+    // Now do the final patch:
+    PatchItem PatchItems[] = {
+        // Patch the Header.HashOffset field.
+        {HashTableStartFieldOffset, &HashTableStart, 1},
+        // Patch the Header.MemProfOffset (=0 for profiles without MemProf
+        // data).
+        {MemProfSectionOffset, &MemProfSectionStart, 1},
+        // Patch the Header.BinaryIdSectionOffset.
+        {BinaryIdSectionOffset, &BinaryIdSectionStart, 1},
+        // Patch the Header.TemporalProfTracesOffset (=0 for profiles without
+        // traces).
+        {TemporalProfTracesOffset, &TemporalProfTracesSectionStart, 1},
+        {VTableNamesOffset, &VTableNamesSectionStart, 1},
+        // Patch the summary data.
+        {SummaryOffset, reinterpret_cast(TheSummary.get()),
+         (int)(SummarySize / sizeof(uint64_t))},
+        {CSSummaryOffset, reinterpret_cast(TheCSSummary.get()),
+         (int)CSSummarySize}};
+
+    OS.patch(PatchItems, std::size(PatchItems));
+  } else {
+    // Now do the final patch:
+    PatchItem PatchItems[] = {
+        // Patch the Header.HashOffset field.
+        {HashTableStartFieldOffset, &HashTableStart, 1},
+        // Patch the Header.MemProfOffset (=0 for profiles without MemProf
+        // data).
+        {MemProfSectionOffset, &MemProfSectionStart, 1},
+        // Patch the Header.BinaryIdSectionOffset.
+        {BinaryIdSectionOffset, &BinaryIdSectionStart, 1},
+        // Patch the Header.TemporalProfTracesOffset (=0 for profiles without
+        // traces).
+        {TemporalProfTracesOffset, &TemporalProfTracesSectionStart, 1},
+        // Patch the summary data.
+        {SummaryOffset, reinterpret_cast(TheSummary.get()),
+         (int)(SummarySize / sizeof(uint64_t))},
+        {CSSummaryOffset, reinterpret_cast(TheCSSummary.get()),
+         (int)CSSummarySize}};
+
+    OS.patch(PatchItems, std::size(PatchItems));
+  }
 
   for (const auto &I : FunctionData)
     for (const auto &F : I.getValue())
diff --git a/llvm/test/tools/llvm-profdata/profile-version.test b/llvm/test/tools/llvm-profdata/profile-version.test
new file mode 100644
index 000000000000..cb68a648d5e5
--- /dev/null
+++ b/llvm/test/tools/llvm-profdata/profile-version.test
@@ -0,0 +1,9 @@
+Test the profile version.
+
+RUN: llvm-profdata merge -o %t.profdata %p/Inputs/basic.proftext
+RUN: llvm-profdata show --profile-version %t.profdata | FileCheck %s
+CHECK: Profile version: 12
+
+RUN: llvm-profdata merge -o %t.prev.profdata %p/Inputs/basic.proftext --write-prev-version
+RUN: llvm-profdata show --profile-version %t.prev.profdata | FileCheck %s --check-prefix=PREV
+PREV: Profile version: 11
diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp
index 8400b0769944..3a7bd061d3d2 100644
--- a/llvm/tools/llvm-profdata/llvm-profdata.cpp
+++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp
@@ -291,6 +291,15 @@ cl::opt DropProfileSymbolList(
     cl::desc("Drop the profile symbol list when merging AutoFDO profiles "
              "(only meaningful for -sample)"));
 
+// Temporary support for writing the previous version of the format, to enable
+// some forward compatibility.
+// TODO: Consider enabling this with future version changes as well, to ease
+// deployment of newer versions of llvm-profdata.
+cl::opt DoWritePrevVersion(
+    "write-prev-version", cl::init(false), cl::Hidden,
+    cl::desc("Write the previous version of indexed format, to enable "
+             "some forward compatibility."));
+
 // Options specific to overlap subcommand.
 cl::opt BaseFilename(cl::Positional, cl::Required,
                                   cl::desc(""),
@@ -579,8 +588,8 @@ struct WriterContext {
   WriterContext(bool IsSparse, std::mutex &ErrLock,
                 SmallSet &WriterErrorCodes,
                 uint64_t ReservoirSize = 0, uint64_t MaxTraceLength = 0)
-      : Writer(IsSparse, ReservoirSize, MaxTraceLength), ErrLock(ErrLock),
-        WriterErrorCodes(WriterErrorCodes) {}
+      : Writer(IsSparse, ReservoirSize, MaxTraceLength, DoWritePrevVersion),
+        ErrLock(ErrLock), WriterErrorCodes(WriterErrorCodes) {}
 };
 
 /// Computer the overlap b/w profile BaseFilename and TestFileName,
-- 
GitLab


From 05280b582aa73ce0f41bb47aea15818658f32929 Mon Sep 17 00:00:00 2001
From: David CARLIER 
Date: Fri, 8 Mar 2024 20:34:43 +0000
Subject: [PATCH 0082/1674] [OpenMP] Implements __kmp_is_address_mapped for
 Solaris/Illumos. (#82930)

Also fixing OpenMP build itself for this platform.
---
 openmp/runtime/cmake/LibompHandleFlags.cmake |  5 +-
 openmp/runtime/src/z_Linux_util.cpp          | 72 ++++++++++++++++++--
 2 files changed, 72 insertions(+), 5 deletions(-)

diff --git a/openmp/runtime/cmake/LibompHandleFlags.cmake b/openmp/runtime/cmake/LibompHandleFlags.cmake
index eed98036dd51..1aba1fbea829 100644
--- a/openmp/runtime/cmake/LibompHandleFlags.cmake
+++ b/openmp/runtime/cmake/LibompHandleFlags.cmake
@@ -147,8 +147,11 @@ function(libomp_get_libflags libflags)
     if (${CMAKE_SYSTEM_NAME} STREQUAL "DragonFly")
       libomp_append(libflags_local "-lkvm")
     endif()
-  elseif(${CMAKE_SYSTEM_NAME} MATCHES "Linux|NetBSD|Solaris")
+  elseif(${CMAKE_SYSTEM_NAME} MATCHES "Linux|NetBSD|SunOS")
     libomp_append(libflags_local -lm)
+    if (${CMAKE_SYSTEM_NAME} STREQUAL "SunOS")
+      libomp_append(libflags_local "-lproc")
+    endif()
   endif()
   set(libflags_local ${libflags_local} ${LIBOMP_LIBFLAGS})
   libomp_setup_flags(libflags_local)
diff --git a/openmp/runtime/src/z_Linux_util.cpp b/openmp/runtime/src/z_Linux_util.cpp
index 3636266677d9..a8e5a9e6bbb0 100644
--- a/openmp/runtime/src/z_Linux_util.cpp
+++ b/openmp/runtime/src/z_Linux_util.cpp
@@ -66,6 +66,9 @@
 #include 
 #include 
 #elif KMP_OS_SOLARIS
+#include 
+#include 
+#include 
 #include 
 #endif
 
@@ -418,7 +421,7 @@ void __kmp_terminate_thread(int gtid) {
   KMP_YIELD(TRUE);
 } //
 
-/* Set thread stack info according to values returned by pthread_getattr_np().
+/* Set thread stack info.
    If values are unreasonable, assume call failed and use incremental stack
    refinement method instead. Returns TRUE if the stack parameters could be
    determined exactly, FALSE if incremental refinement is necessary. */
@@ -426,7 +429,6 @@ static kmp_int32 __kmp_set_stack_info(int gtid, kmp_info_t *th) {
   int stack_data;
 #if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD ||     \
     KMP_OS_HURD || KMP_OS_SOLARIS || KMP_OS_AIX
-  pthread_attr_t attr;
   int status;
   size_t size = 0;
   void *addr = 0;
@@ -436,6 +438,19 @@ static kmp_int32 __kmp_set_stack_info(int gtid, kmp_info_t *th) {
      pthread_attr_getstack may cause thread gtid aliasing */
   if (!KMP_UBER_GTID(gtid)) {
 
+#if KMP_OS_SOLARIS
+    stack_t s;
+    if ((status = thr_stksegment(&s)) < 0) {
+      KMP_CHECK_SYSFAIL("thr_stksegment", status);
+    }
+
+    addr = s.ss_sp;
+    size = s.ss_size;
+    KA_TRACE(60, ("__kmp_set_stack_info: T#%d thr_stksegment returned size:"
+                  " %lu, low addr: %p\n",
+                  gtid, size, addr));
+#else
+    pthread_attr_t attr;
     /* Fetch the real thread attributes */
     status = pthread_attr_init(&attr);
     KMP_CHECK_SYSFAIL("pthread_attr_init", status);
@@ -454,6 +469,7 @@ static kmp_int32 __kmp_set_stack_info(int gtid, kmp_info_t *th) {
               gtid, size, addr));
     status = pthread_attr_destroy(&attr);
     KMP_CHECK_SYSFAIL("pthread_attr_destroy", status);
+#endif
   }
 
   if (size != 0 && addr != 0) { // was stack parameter determination successful?
@@ -2175,6 +2191,54 @@ int __kmp_is_address_mapped(void *addr) {
   }
 
   kvm_close(fd);
+#elif KMP_OS_SOLARIS
+  prmap_t *cur, *map;
+  void *buf;
+  uintptr_t uaddr;
+  ssize_t rd;
+  int err;
+  int file;
+
+  pid_t pid = getpid();
+  struct ps_prochandle *fd = Pgrab(pid, PGRAB_RDONLY, &err);
+  ;
+
+  if (!fd) {
+    return 0;
+  }
+
+  char *name = __kmp_str_format("/proc/%d/map", pid);
+  size_t sz = (1 << 20);
+  file = open(name, O_RDONLY);
+  if (file == -1) {
+    KMP_INTERNAL_FREE(name);
+    return 0;
+  }
+
+  buf = kmpc_malloc(sz);
+
+  while (sz > 0 && (rd = pread(file, buf, sz, 0)) == sz) {
+    void *newbuf;
+    sz <<= 1;
+    newbuf = kmpc_realloc(buf, sz);
+    buf = newbuf;
+  }
+
+  map = reinterpret_cast(buf);
+  uaddr = reinterpret_cast(addr);
+
+  for (cur = map; rd > 0; cur++, rd = -sizeof(*map)) {
+    if ((uaddr >= cur->pr_vaddr) && (uaddr < cur->pr_vaddr)) {
+      if ((cur->pr_mflags & MA_READ) != 0 && (cur->pr_mflags & MA_WRITE) != 0) {
+        found = 1;
+        break;
+      }
+    }
+  }
+
+  kmpc_free(map);
+  close(file);
+  KMP_INTERNAL_FREE(name);
 #elif KMP_OS_DARWIN
 
   /* On OS X*, /proc pseudo filesystem is not available. Try to read memory
@@ -2253,9 +2317,9 @@ int __kmp_is_address_mapped(void *addr) {
   }
 #elif KMP_OS_WASI
   found = (int)addr < (__builtin_wasm_memory_size(0) * PAGESIZE);
-#elif KMP_OS_SOLARIS || KMP_OS_AIX
+#elif KMP_OS_AIX
 
-  // FIXME(Solaris, AIX): Implement this
+  // FIXME(AIX): Implement this
   found = 1;
 
 #else
-- 
GitLab


From 755b439694432d4f68e20e979b479cbc30602bb1 Mon Sep 17 00:00:00 2001
From: yingopq <115543042+yingopq@users.noreply.github.com>
Date: Sat, 9 Mar 2024 04:41:31 +0800
Subject: [PATCH 0083/1674] [Mips] Fix missing sign extension in expansion of
 sub-word atomic max (#77072)

Add sign extension "SEB/SEH" before compare.

Fix #61881
---
 llvm/lib/Target/Mips/MipsExpandPseudo.cpp |  51 +-
 llvm/test/CodeGen/Mips/atomic-min-max.ll  | 655 ++++++++++++++++++----
 2 files changed, 577 insertions(+), 129 deletions(-)

diff --git a/llvm/lib/Target/Mips/MipsExpandPseudo.cpp b/llvm/lib/Target/Mips/MipsExpandPseudo.cpp
index 2c2554b5b4bc..bded59439a73 100644
--- a/llvm/lib/Target/Mips/MipsExpandPseudo.cpp
+++ b/llvm/lib/Target/Mips/MipsExpandPseudo.cpp
@@ -388,18 +388,32 @@ bool MipsExpandPseudo::expandAtomicBinOpSubword(
     Opcode = Mips::XOR;
     break;
   case Mips::ATOMIC_LOAD_UMIN_I8_POSTRA:
+    IsUnsigned = true;
+    IsMin = true;
+    break;
   case Mips::ATOMIC_LOAD_UMIN_I16_POSTRA:
     IsUnsigned = true;
-    [[fallthrough]];
+    IsMin = true;
+    break;
   case Mips::ATOMIC_LOAD_MIN_I8_POSTRA:
+    SEOp = Mips::SEB;
+    IsMin = true;
+    break;
   case Mips::ATOMIC_LOAD_MIN_I16_POSTRA:
     IsMin = true;
     break;
   case Mips::ATOMIC_LOAD_UMAX_I8_POSTRA:
+    IsUnsigned = true;
+    IsMax = true;
+    break;
   case Mips::ATOMIC_LOAD_UMAX_I16_POSTRA:
     IsUnsigned = true;
-    [[fallthrough]];
+    IsMax = true;
+    break;
   case Mips::ATOMIC_LOAD_MAX_I8_POSTRA:
+    SEOp = Mips::SEB;
+    IsMax = true;
+    break;
   case Mips::ATOMIC_LOAD_MAX_I16_POSTRA:
     IsMax = true;
     break;
@@ -461,14 +475,33 @@ bool MipsExpandPseudo::expandAtomicBinOpSubword(
 
     // For little endian we need to clear uninterested bits.
     if (STI->isLittle()) {
-      // and OldVal, OldVal, Mask
-      // and Incr, Incr, Mask
-      BuildMI(loopMBB, DL, TII->get(Mips::AND), OldVal)
-          .addReg(OldVal)
-          .addReg(Mask);
-      BuildMI(loopMBB, DL, TII->get(Mips::AND), Incr).addReg(Incr).addReg(Mask);
+      if (!IsUnsigned) {
+        BuildMI(loopMBB, DL, TII->get(Mips::SRAV), OldVal)
+            .addReg(OldVal)
+            .addReg(ShiftAmnt);
+        BuildMI(loopMBB, DL, TII->get(Mips::SRAV), Incr)
+            .addReg(Incr)
+            .addReg(ShiftAmnt);
+        if (STI->hasMips32r2()) {
+          BuildMI(loopMBB, DL, TII->get(SEOp), OldVal).addReg(OldVal);
+          BuildMI(loopMBB, DL, TII->get(SEOp), Incr).addReg(Incr);
+        } else {
+          const unsigned ShiftImm = SEOp == Mips::SEH ? 16 : 24;
+          BuildMI(loopMBB, DL, TII->get(Mips::SLL), OldVal)
+              .addReg(OldVal, RegState::Kill)
+              .addImm(ShiftImm);
+          BuildMI(loopMBB, DL, TII->get(Mips::SRA), OldVal)
+              .addReg(OldVal, RegState::Kill)
+              .addImm(ShiftImm);
+          BuildMI(loopMBB, DL, TII->get(Mips::SLL), Incr)
+              .addReg(Incr, RegState::Kill)
+              .addImm(ShiftImm);
+          BuildMI(loopMBB, DL, TII->get(Mips::SRA), Incr)
+              .addReg(Incr, RegState::Kill)
+              .addImm(ShiftImm);
+        }
+      }
     }
-
     // unsigned: sltu Scratch4, oldVal, Incr
     // signed:   slt Scratch4, oldVal, Incr
     BuildMI(loopMBB, DL, TII->get(SLTScratch4), Scratch4)
diff --git a/llvm/test/CodeGen/Mips/atomic-min-max.ll b/llvm/test/CodeGen/Mips/atomic-min-max.ll
index f953c885ea73..bc3643f3947a 100644
--- a/llvm/test/CodeGen/Mips/atomic-min-max.ll
+++ b/llvm/test/CodeGen/Mips/atomic-min-max.ll
@@ -3,6 +3,7 @@
 ; RUN: llc -march=mips -O0 -mcpu=mips32r6 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MIPSR6
 ; RUN: llc -march=mips -O0 -mcpu=mips32r2 -mattr=+micromips -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MM
 ; RUN: llc -march=mips -O0 -mcpu=mips32r6 -mattr=+micromips -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MMR6
+; RUN: llc -march=mipsel -O0 -mcpu=mips32 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MIPS32
 ; RUN: llc -march=mipsel -O0 -mcpu=mips32r2 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MIPSEL
 ; RUN: llc -march=mipsel -O0 -mcpu=mips32r6 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MIPSELR6
 ; RUN: llc -march=mipsel -O0 -mcpu=mips32r2 -mattr=+micromips -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MMEL
@@ -77,6 +78,23 @@ define i32 @test_max_32(ptr nocapture %ptr, i32 signext %val) {
 ; MMR6-NEXT:    sync
 ; MMR6-NEXT:    jrc $ra
 ;
+; MIPS32-LABEL: test_max_32:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    sync
+; MIPS32-NEXT:  $BB0_1: # %entry
+; MIPS32-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS32-NEXT:    ll $2, 0($4)
+; MIPS32-NEXT:    slt $3, $2, $5
+; MIPS32-NEXT:    move $1, $2
+; MIPS32-NEXT:    movn $1, $5, $3
+; MIPS32-NEXT:    sc $1, 0($4)
+; MIPS32-NEXT:    beqz $1, $BB0_1
+; MIPS32-NEXT:    nop
+; MIPS32-NEXT:  # %bb.2: # %entry
+; MIPS32-NEXT:    sync
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+;
 ; MIPSEL-LABEL: test_max_32:
 ; MIPSEL:       # %bb.0: # %entry
 ; MIPSEL-NEXT:    sync
@@ -280,6 +298,23 @@ define i32 @test_min_32(ptr nocapture %ptr, i32 signext %val) {
 ; MMR6-NEXT:    sync
 ; MMR6-NEXT:    jrc $ra
 ;
+; MIPS32-LABEL: test_min_32:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    sync
+; MIPS32-NEXT:  $BB1_1: # %entry
+; MIPS32-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS32-NEXT:    ll $2, 0($4)
+; MIPS32-NEXT:    slt $3, $2, $5
+; MIPS32-NEXT:    move $1, $2
+; MIPS32-NEXT:    movz $1, $5, $3
+; MIPS32-NEXT:    sc $1, 0($4)
+; MIPS32-NEXT:    beqz $1, $BB1_1
+; MIPS32-NEXT:    nop
+; MIPS32-NEXT:  # %bb.2: # %entry
+; MIPS32-NEXT:    sync
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+;
 ; MIPSEL-LABEL: test_min_32:
 ; MIPSEL:       # %bb.0: # %entry
 ; MIPSEL-NEXT:    sync
@@ -483,6 +518,23 @@ define i32 @test_umax_32(ptr nocapture %ptr, i32 signext %val) {
 ; MMR6-NEXT:    sync
 ; MMR6-NEXT:    jrc $ra
 ;
+; MIPS32-LABEL: test_umax_32:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    sync
+; MIPS32-NEXT:  $BB2_1: # %entry
+; MIPS32-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS32-NEXT:    ll $2, 0($4)
+; MIPS32-NEXT:    sltu $3, $2, $5
+; MIPS32-NEXT:    move $1, $2
+; MIPS32-NEXT:    movn $1, $5, $3
+; MIPS32-NEXT:    sc $1, 0($4)
+; MIPS32-NEXT:    beqz $1, $BB2_1
+; MIPS32-NEXT:    nop
+; MIPS32-NEXT:  # %bb.2: # %entry
+; MIPS32-NEXT:    sync
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+;
 ; MIPSEL-LABEL: test_umax_32:
 ; MIPSEL:       # %bb.0: # %entry
 ; MIPSEL-NEXT:    sync
@@ -686,6 +738,23 @@ define i32 @test_umin_32(ptr nocapture %ptr, i32 signext %val) {
 ; MMR6-NEXT:    sync
 ; MMR6-NEXT:    jrc $ra
 ;
+; MIPS32-LABEL: test_umin_32:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    sync
+; MIPS32-NEXT:  $BB3_1: # %entry
+; MIPS32-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS32-NEXT:    ll $2, 0($4)
+; MIPS32-NEXT:    sltu $3, $2, $5
+; MIPS32-NEXT:    move $1, $2
+; MIPS32-NEXT:    movz $1, $5, $3
+; MIPS32-NEXT:    sc $1, 0($4)
+; MIPS32-NEXT:    beqz $1, $BB3_1
+; MIPS32-NEXT:    nop
+; MIPS32-NEXT:  # %bb.2: # %entry
+; MIPS32-NEXT:    sync
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+;
 ; MIPSEL-LABEL: test_umin_32:
 ; MIPSEL:       # %bb.0: # %entry
 ; MIPSEL-NEXT:    sync
@@ -980,6 +1049,53 @@ define i16 @test_max_16(ptr nocapture %ptr, i16 signext %val) {
 ; MMR6-NEXT:    sync
 ; MMR6-NEXT:    addiu $sp, $sp, 8
 ; MMR6-NEXT:    jrc $ra
+;
+; MIPS32-LABEL: test_max_16:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    addiu $sp, $sp, -8
+; MIPS32-NEXT:    .cfi_def_cfa_offset 8
+; MIPS32-NEXT:    # kill: def $at killed $a1
+; MIPS32-NEXT:    sync
+; MIPS32-NEXT:    addiu $1, $zero, -4
+; MIPS32-NEXT:    and $6, $4, $1
+; MIPS32-NEXT:    andi $1, $4, 3
+; MIPS32-NEXT:    sll $10, $1, 3
+; MIPS32-NEXT:    ori $1, $zero, 65535
+; MIPS32-NEXT:    sllv $8, $1, $10
+; MIPS32-NEXT:    nor $9, $zero, $8
+; MIPS32-NEXT:    sllv $7, $5, $10
+; MIPS32-NEXT:  $BB4_1: # %entry
+; MIPS32-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS32-NEXT:    ll $2, 0($6)
+; MIPS32-NEXT:    srav $2, $2, $10
+; MIPS32-NEXT:    srav $7, $7, $10
+; MIPS32-NEXT:    sll $2, $2, 16
+; MIPS32-NEXT:    sra $2, $2, 16
+; MIPS32-NEXT:    sll $7, $7, 16
+; MIPS32-NEXT:    sra $7, $7, 16
+; MIPS32-NEXT:    slt $5, $2, $7
+; MIPS32-NEXT:    move $3, $2
+; MIPS32-NEXT:    movn $3, $7, $5
+; MIPS32-NEXT:    and $3, $3, $8
+; MIPS32-NEXT:    and $4, $2, $9
+; MIPS32-NEXT:    or $4, $4, $3
+; MIPS32-NEXT:    sc $4, 0($6)
+; MIPS32-NEXT:    beqz $4, $BB4_1
+; MIPS32-NEXT:    nop
+; MIPS32-NEXT:  # %bb.2: # %entry
+; MIPS32-NEXT:    and $1, $2, $8
+; MIPS32-NEXT:    srlv $1, $1, $10
+; MIPS32-NEXT:    sll $1, $1, 16
+; MIPS32-NEXT:    sra $1, $1, 16
+; MIPS32-NEXT:  # %bb.3: # %entry
+; MIPS32-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:  # %bb.4: # %entry
+; MIPS32-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    sync
+; MIPS32-NEXT:    addiu $sp, $sp, 8
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+
 ;
 ; MIPSEL-LABEL: test_max_16:
 ; MIPSEL:       # %bb.0: # %entry
@@ -998,8 +1114,10 @@ define i16 @test_max_16(ptr nocapture %ptr, i16 signext %val) {
 ; MIPSEL-NEXT:  $BB4_1: # %entry
 ; MIPSEL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPSEL-NEXT:    ll $2, 0($6)
-; MIPSEL-NEXT:    and $2, $2, $8
-; MIPSEL-NEXT:    and $7, $7, $8
+; MIPSEL-NEXT:    srav $2, $2, $10
+; MIPSEL-NEXT:    srav $7, $7, $10
+; MIPSEL-NEXT:    seh $2, $2
+; MIPSEL-NEXT:    seh $7, $7
 ; MIPSEL-NEXT:    slt $5, $2, $7
 ; MIPSEL-NEXT:    move $3, $2
 ; MIPSEL-NEXT:    movn $3, $7, $5
@@ -1039,8 +1157,10 @@ define i16 @test_max_16(ptr nocapture %ptr, i16 signext %val) {
 ; MIPSELR6-NEXT:  $BB4_1: # %entry
 ; MIPSELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPSELR6-NEXT:    ll $2, 0($6)
-; MIPSELR6-NEXT:    and $2, $2, $8
-; MIPSELR6-NEXT:    and $7, $7, $8
+; MIPSELR6-NEXT:    srav $2, $2, $10
+; MIPSELR6-NEXT:    srav $7, $7, $10
+; MIPSELR6-NEXT:    seh $2, $2
+; MIPSELR6-NEXT:    seh $7, $7
 ; MIPSELR6-NEXT:    slt $5, $2, $7
 ; MIPSELR6-NEXT:    seleqz $3, $2, $5
 ; MIPSELR6-NEXT:    selnez $5, $7, $5
@@ -1079,8 +1199,10 @@ define i16 @test_max_16(ptr nocapture %ptr, i16 signext %val) {
 ; MMEL-NEXT:  $BB4_1: # %entry
 ; MMEL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MMEL-NEXT:    ll $2, 0($6)
-; MMEL-NEXT:    and $2, $2, $8
-; MMEL-NEXT:    and $7, $7, $8
+; MMEL-NEXT:    srav $2, $2, $10
+; MMEL-NEXT:    srav $7, $7, $10
+; MMEL-NEXT:    seh $2, $2
+; MMEL-NEXT:    seh $7, $7
 ; MMEL-NEXT:    slt $5, $2, $7
 ; MMEL-NEXT:    or $3, $2, $zero
 ; MMEL-NEXT:    movn $3, $7, $5
@@ -1118,8 +1240,10 @@ define i16 @test_max_16(ptr nocapture %ptr, i16 signext %val) {
 ; MMELR6-NEXT:  $BB4_1: # %entry
 ; MMELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MMELR6-NEXT:    ll $2, 0($6)
-; MMELR6-NEXT:    and $2, $2, $8
-; MMELR6-NEXT:    and $7, $7, $8
+; MMELR6-NEXT:    srav $2, $2, $10
+; MMELR6-NEXT:    srav $7, $7, $10
+; MMELR6-NEXT:    seh $2, $2
+; MMELR6-NEXT:    seh $7, $7
 ; MMELR6-NEXT:    slt $5, $2, $7
 ; MMELR6-NEXT:    seleqz $3, $2, $5
 ; MMELR6-NEXT:    selnez $5, $7, $5
@@ -1237,8 +1361,10 @@ define i16 @test_max_16(ptr nocapture %ptr, i16 signext %val) {
 ; MIPS64EL-NEXT:  .LBB4_1: # %entry
 ; MIPS64EL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPS64EL-NEXT:    ll $2, 0($6)
-; MIPS64EL-NEXT:    and $2, $2, $8
-; MIPS64EL-NEXT:    and $7, $7, $8
+; MIPS64EL-NEXT:    srav $2, $2, $10
+; MIPS64EL-NEXT:    srav $7, $7, $10
+; MIPS64EL-NEXT:    seh $2, $2
+; MIPS64EL-NEXT:    seh $7, $7
 ; MIPS64EL-NEXT:    slt $5, $2, $7
 ; MIPS64EL-NEXT:    move $3, $2
 ; MIPS64EL-NEXT:    movn $3, $7, $5
@@ -1278,8 +1404,10 @@ define i16 @test_max_16(ptr nocapture %ptr, i16 signext %val) {
 ; MIPS64ELR6-NEXT:  .LBB4_1: # %entry
 ; MIPS64ELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPS64ELR6-NEXT:    ll $2, 0($6)
-; MIPS64ELR6-NEXT:    and $2, $2, $8
-; MIPS64ELR6-NEXT:    and $7, $7, $8
+; MIPS64ELR6-NEXT:    srav $2, $2, $10
+; MIPS64ELR6-NEXT:    srav $7, $7, $10
+; MIPS64ELR6-NEXT:    seh $2, $2
+; MIPS64ELR6-NEXT:    seh $7, $7
 ; MIPS64ELR6-NEXT:    slt $5, $2, $7
 ; MIPS64ELR6-NEXT:    seleqz $3, $2, $5
 ; MIPS64ELR6-NEXT:    selnez $5, $7, $5
@@ -1462,6 +1590,52 @@ define i16 @test_min_16(ptr nocapture %ptr, i16 signext %val) {
 ; MMR6-NEXT:    addiu $sp, $sp, 8
 ; MMR6-NEXT:    jrc $ra
 ;
+; MIPS32-LABEL: test_min_16:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    addiu $sp, $sp, -8
+; MIPS32-NEXT:    .cfi_def_cfa_offset 8
+; MIPS32-NEXT:    # kill: def $at killed $a1
+; MIPS32-NEXT:    sync
+; MIPS32-NEXT:    addiu $1, $zero, -4
+; MIPS32-NEXT:    and $6, $4, $1
+; MIPS32-NEXT:    andi $1, $4, 3
+; MIPS32-NEXT:    sll $10, $1, 3
+; MIPS32-NEXT:    ori $1, $zero, 65535
+; MIPS32-NEXT:    sllv $8, $1, $10
+; MIPS32-NEXT:    nor $9, $zero, $8
+; MIPS32-NEXT:    sllv $7, $5, $10
+; MIPS32-NEXT:  $BB5_1: # %entry
+; MIPS32-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS32-NEXT:    ll $2, 0($6)
+; MIPS32-NEXT:    srav $2, $2, $10
+; MIPS32-NEXT:    srav $7, $7, $10
+; MIPS32-NEXT:    sll $2, $2, 16
+; MIPS32-NEXT:    sra $2, $2, 16
+; MIPS32-NEXT:    sll $7, $7, 16
+; MIPS32-NEXT:    sra $7, $7, 16
+; MIPS32-NEXT:    slt $5, $2, $7
+; MIPS32-NEXT:    move $3, $2
+; MIPS32-NEXT:    movz $3, $7, $5
+; MIPS32-NEXT:    and $3, $3, $8
+; MIPS32-NEXT:    and $4, $2, $9
+; MIPS32-NEXT:    or $4, $4, $3
+; MIPS32-NEXT:    sc $4, 0($6)
+; MIPS32-NEXT:    beqz $4, $BB5_1
+; MIPS32-NEXT:    nop
+; MIPS32-NEXT:  # %bb.2: # %entry
+; MIPS32-NEXT:    and $1, $2, $8
+; MIPS32-NEXT:    srlv $1, $1, $10
+; MIPS32-NEXT:    sll $1, $1, 16
+; MIPS32-NEXT:    sra $1, $1, 16
+; MIPS32-NEXT:  # %bb.3: # %entry
+; MIPS32-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:  # %bb.4: # %entry
+; MIPS32-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    sync
+; MIPS32-NEXT:    addiu $sp, $sp, 8
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+;
 ; MIPSEL-LABEL: test_min_16:
 ; MIPSEL:       # %bb.0: # %entry
 ; MIPSEL-NEXT:    addiu $sp, $sp, -8
@@ -1479,8 +1653,10 @@ define i16 @test_min_16(ptr nocapture %ptr, i16 signext %val) {
 ; MIPSEL-NEXT:  $BB5_1: # %entry
 ; MIPSEL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPSEL-NEXT:    ll $2, 0($6)
-; MIPSEL-NEXT:    and $2, $2, $8
-; MIPSEL-NEXT:    and $7, $7, $8
+; MIPSEL-NEXT:    srav $2, $2, $10
+; MIPSEL-NEXT:    srav $7, $7, $10
+; MIPSEL-NEXT:    seh $2, $2
+; MIPSEL-NEXT:    seh $7, $7
 ; MIPSEL-NEXT:    slt $5, $2, $7
 ; MIPSEL-NEXT:    move $3, $2
 ; MIPSEL-NEXT:    movz $3, $7, $5
@@ -1520,8 +1696,10 @@ define i16 @test_min_16(ptr nocapture %ptr, i16 signext %val) {
 ; MIPSELR6-NEXT:  $BB5_1: # %entry
 ; MIPSELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPSELR6-NEXT:    ll $2, 0($6)
-; MIPSELR6-NEXT:    and $2, $2, $8
-; MIPSELR6-NEXT:    and $7, $7, $8
+; MIPSELR6-NEXT:    srav $2, $2, $10
+; MIPSELR6-NEXT:    srav $7, $7, $10
+; MIPSELR6-NEXT:    seh $2, $2
+; MIPSELR6-NEXT:    seh $7, $7
 ; MIPSELR6-NEXT:    slt $5, $2, $7
 ; MIPSELR6-NEXT:    selnez $3, $2, $5
 ; MIPSELR6-NEXT:    seleqz $5, $7, $5
@@ -1560,8 +1738,10 @@ define i16 @test_min_16(ptr nocapture %ptr, i16 signext %val) {
 ; MMEL-NEXT:  $BB5_1: # %entry
 ; MMEL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MMEL-NEXT:    ll $2, 0($6)
-; MMEL-NEXT:    and $2, $2, $8
-; MMEL-NEXT:    and $7, $7, $8
+; MMEL-NEXT:    srav $2, $2, $10
+; MMEL-NEXT:    srav $7, $7, $10
+; MMEL-NEXT:    seh $2, $2
+; MMEL-NEXT:    seh $7, $7
 ; MMEL-NEXT:    slt $5, $2, $7
 ; MMEL-NEXT:    or $3, $2, $zero
 ; MMEL-NEXT:    movz $3, $7, $5
@@ -1599,8 +1779,10 @@ define i16 @test_min_16(ptr nocapture %ptr, i16 signext %val) {
 ; MMELR6-NEXT:  $BB5_1: # %entry
 ; MMELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MMELR6-NEXT:    ll $2, 0($6)
-; MMELR6-NEXT:    and $2, $2, $8
-; MMELR6-NEXT:    and $7, $7, $8
+; MMELR6-NEXT:    srav $2, $2, $10
+; MMELR6-NEXT:    srav $7, $7, $10
+; MMELR6-NEXT:    seh $2, $2
+; MMELR6-NEXT:    seh $7, $7
 ; MMELR6-NEXT:    slt $5, $2, $7
 ; MMELR6-NEXT:    selnez $3, $2, $5
 ; MMELR6-NEXT:    seleqz $5, $7, $5
@@ -1718,8 +1900,10 @@ define i16 @test_min_16(ptr nocapture %ptr, i16 signext %val) {
 ; MIPS64EL-NEXT:  .LBB5_1: # %entry
 ; MIPS64EL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPS64EL-NEXT:    ll $2, 0($6)
-; MIPS64EL-NEXT:    and $2, $2, $8
-; MIPS64EL-NEXT:    and $7, $7, $8
+; MIPS64EL-NEXT:    srav $2, $2, $10
+; MIPS64EL-NEXT:    srav $7, $7, $10
+; MIPS64EL-NEXT:    seh $2, $2
+; MIPS64EL-NEXT:    seh $7, $7
 ; MIPS64EL-NEXT:    slt $5, $2, $7
 ; MIPS64EL-NEXT:    move $3, $2
 ; MIPS64EL-NEXT:    movz $3, $7, $5
@@ -1759,8 +1943,10 @@ define i16 @test_min_16(ptr nocapture %ptr, i16 signext %val) {
 ; MIPS64ELR6-NEXT:  .LBB5_1: # %entry
 ; MIPS64ELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPS64ELR6-NEXT:    ll $2, 0($6)
-; MIPS64ELR6-NEXT:    and $2, $2, $8
-; MIPS64ELR6-NEXT:    and $7, $7, $8
+; MIPS64ELR6-NEXT:    srav $2, $2, $10
+; MIPS64ELR6-NEXT:    srav $7, $7, $10
+; MIPS64ELR6-NEXT:    seh $2, $2
+; MIPS64ELR6-NEXT:    seh $7, $7
 ; MIPS64ELR6-NEXT:    slt $5, $2, $7
 ; MIPS64ELR6-NEXT:    selnez $3, $2, $5
 ; MIPS64ELR6-NEXT:    seleqz $5, $7, $5
@@ -1943,6 +2129,46 @@ define i16 @test_umax_16(ptr nocapture %ptr, i16 signext %val) {
 ; MMR6-NEXT:    addiu $sp, $sp, 8
 ; MMR6-NEXT:    jrc $ra
 ;
+; MIPS32-LABEL: test_umax_16:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    addiu $sp, $sp, -8
+; MIPS32-NEXT:    .cfi_def_cfa_offset 8
+; MIPS32-NEXT:    # kill: def $at killed $a1
+; MIPS32-NEXT:    sync
+; MIPS32-NEXT:    addiu $1, $zero, -4
+; MIPS32-NEXT:    and $6, $4, $1
+; MIPS32-NEXT:    andi $1, $4, 3
+; MIPS32-NEXT:    sll $10, $1, 3
+; MIPS32-NEXT:    ori $1, $zero, 65535
+; MIPS32-NEXT:    sllv $8, $1, $10
+; MIPS32-NEXT:    nor $9, $zero, $8
+; MIPS32-NEXT:    sllv $7, $5, $10
+; MIPS32-NEXT:  $BB6_1: # %entry
+; MIPS32-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS32-NEXT:    ll $2, 0($6)
+; MIPS32-NEXT:    sltu $5, $2, $7
+; MIPS32-NEXT:    move $3, $2
+; MIPS32-NEXT:    movn $3, $7, $5
+; MIPS32-NEXT:    and $3, $3, $8
+; MIPS32-NEXT:    and $4, $2, $9
+; MIPS32-NEXT:    or $4, $4, $3
+; MIPS32-NEXT:    sc $4, 0($6)
+; MIPS32-NEXT:    beqz $4, $BB6_1
+; MIPS32-NEXT:    nop
+; MIPS32-NEXT:  # %bb.2: # %entry
+; MIPS32-NEXT:    and $1, $2, $8
+; MIPS32-NEXT:    srlv $1, $1, $10
+; MIPS32-NEXT:    sll $1, $1, 16
+; MIPS32-NEXT:    sra $1, $1, 16
+; MIPS32-NEXT:  # %bb.3: # %entry
+; MIPS32-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:  # %bb.4: # %entry
+; MIPS32-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    sync
+; MIPS32-NEXT:    addiu $sp, $sp, 8
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+;
 ; MIPSEL-LABEL: test_umax_16:
 ; MIPSEL:       # %bb.0: # %entry
 ; MIPSEL-NEXT:    addiu $sp, $sp, -8
@@ -1960,8 +2186,6 @@ define i16 @test_umax_16(ptr nocapture %ptr, i16 signext %val) {
 ; MIPSEL-NEXT:  $BB6_1: # %entry
 ; MIPSEL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPSEL-NEXT:    ll $2, 0($6)
-; MIPSEL-NEXT:    and $2, $2, $8
-; MIPSEL-NEXT:    and $7, $7, $8
 ; MIPSEL-NEXT:    sltu $5, $2, $7
 ; MIPSEL-NEXT:    move $3, $2
 ; MIPSEL-NEXT:    movn $3, $7, $5
@@ -2001,8 +2225,6 @@ define i16 @test_umax_16(ptr nocapture %ptr, i16 signext %val) {
 ; MIPSELR6-NEXT:  $BB6_1: # %entry
 ; MIPSELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPSELR6-NEXT:    ll $2, 0($6)
-; MIPSELR6-NEXT:    and $2, $2, $8
-; MIPSELR6-NEXT:    and $7, $7, $8
 ; MIPSELR6-NEXT:    sltu $5, $2, $7
 ; MIPSELR6-NEXT:    seleqz $3, $2, $5
 ; MIPSELR6-NEXT:    selnez $5, $7, $5
@@ -2041,8 +2263,6 @@ define i16 @test_umax_16(ptr nocapture %ptr, i16 signext %val) {
 ; MMEL-NEXT:  $BB6_1: # %entry
 ; MMEL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MMEL-NEXT:    ll $2, 0($6)
-; MMEL-NEXT:    and $2, $2, $8
-; MMEL-NEXT:    and $7, $7, $8
 ; MMEL-NEXT:    sltu $5, $2, $7
 ; MMEL-NEXT:    or $3, $2, $zero
 ; MMEL-NEXT:    movn $3, $7, $5
@@ -2080,8 +2300,6 @@ define i16 @test_umax_16(ptr nocapture %ptr, i16 signext %val) {
 ; MMELR6-NEXT:  $BB6_1: # %entry
 ; MMELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MMELR6-NEXT:    ll $2, 0($6)
-; MMELR6-NEXT:    and $2, $2, $8
-; MMELR6-NEXT:    and $7, $7, $8
 ; MMELR6-NEXT:    sltu $5, $2, $7
 ; MMELR6-NEXT:    seleqz $3, $2, $5
 ; MMELR6-NEXT:    selnez $5, $7, $5
@@ -2199,8 +2417,6 @@ define i16 @test_umax_16(ptr nocapture %ptr, i16 signext %val) {
 ; MIPS64EL-NEXT:  .LBB6_1: # %entry
 ; MIPS64EL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPS64EL-NEXT:    ll $2, 0($6)
-; MIPS64EL-NEXT:    and $2, $2, $8
-; MIPS64EL-NEXT:    and $7, $7, $8
 ; MIPS64EL-NEXT:    sltu $5, $2, $7
 ; MIPS64EL-NEXT:    move $3, $2
 ; MIPS64EL-NEXT:    movn $3, $7, $5
@@ -2240,8 +2456,6 @@ define i16 @test_umax_16(ptr nocapture %ptr, i16 signext %val) {
 ; MIPS64ELR6-NEXT:  .LBB6_1: # %entry
 ; MIPS64ELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPS64ELR6-NEXT:    ll $2, 0($6)
-; MIPS64ELR6-NEXT:    and $2, $2, $8
-; MIPS64ELR6-NEXT:    and $7, $7, $8
 ; MIPS64ELR6-NEXT:    sltu $5, $2, $7
 ; MIPS64ELR6-NEXT:    seleqz $3, $2, $5
 ; MIPS64ELR6-NEXT:    selnez $5, $7, $5
@@ -2424,6 +2638,47 @@ define i16 @test_umin_16(ptr nocapture %ptr, i16 signext %val) {
 ; MMR6-NEXT:    addiu $sp, $sp, 8
 ; MMR6-NEXT:    jrc $ra
 ;
+; MIPS32-LABEL: test_umin_16:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    addiu $sp, $sp, -8
+; MIPS32-NEXT:    .cfi_def_cfa_offset 8
+; MIPS32-NEXT:    # kill: def $at killed $a1
+; MIPS32-NEXT:    sync
+; MIPS32-NEXT:    addiu $1, $zero, -4
+; MIPS32-NEXT:    and $6, $4, $1
+; MIPS32-NEXT:    andi $1, $4, 3
+; MIPS32-NEXT:    sll $10, $1, 3
+; MIPS32-NEXT:    ori $1, $zero, 65535
+; MIPS32-NEXT:    sllv $8, $1, $10
+; MIPS32-NEXT:    nor $9, $zero, $8
+; MIPS32-NEXT:    sllv $7, $5, $10
+; MIPS32-NEXT:  $BB7_1: # %entry
+; MIPS32-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS32-NEXT:    ll $2, 0($6)
+; MIPS32-NEXT:    sltu $5, $2, $7
+; MIPS32-NEXT:    move $3, $2
+; MIPS32-NEXT:    movz $3, $7, $5
+; MIPS32-NEXT:    and $3, $3, $8
+; MIPS32-NEXT:    and $4, $2, $9
+; MIPS32-NEXT:    or $4, $4, $3
+; MIPS32-NEXT:    sc $4, 0($6)
+; MIPS32-NEXT:    beqz $4, $BB7_1
+; MIPS32-NEXT:    nop
+; MIPS32-NEXT:  # %bb.2: # %entry
+; MIPS32-NEXT:    and $1, $2, $8
+; MIPS32-NEXT:    srlv $1, $1, $10
+; MIPS32-NEXT:    sll $1, $1, 16
+; MIPS32-NEXT:    sra $1, $1, 16
+; MIPS32-NEXT:  # %bb.3: # %entry
+; MIPS32-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:  # %bb.4: # %entry
+; MIPS32-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    sync
+; MIPS32-NEXT:    addiu $sp, $sp, 8
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+;
+;
 ; MIPSEL-LABEL: test_umin_16:
 ; MIPSEL:       # %bb.0: # %entry
 ; MIPSEL-NEXT:    addiu $sp, $sp, -8
@@ -2441,8 +2696,6 @@ define i16 @test_umin_16(ptr nocapture %ptr, i16 signext %val) {
 ; MIPSEL-NEXT:  $BB7_1: # %entry
 ; MIPSEL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPSEL-NEXT:    ll $2, 0($6)
-; MIPSEL-NEXT:    and $2, $2, $8
-; MIPSEL-NEXT:    and $7, $7, $8
 ; MIPSEL-NEXT:    sltu $5, $2, $7
 ; MIPSEL-NEXT:    move $3, $2
 ; MIPSEL-NEXT:    movz $3, $7, $5
@@ -2482,8 +2735,6 @@ define i16 @test_umin_16(ptr nocapture %ptr, i16 signext %val) {
 ; MIPSELR6-NEXT:  $BB7_1: # %entry
 ; MIPSELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPSELR6-NEXT:    ll $2, 0($6)
-; MIPSELR6-NEXT:    and $2, $2, $8
-; MIPSELR6-NEXT:    and $7, $7, $8
 ; MIPSELR6-NEXT:    sltu $5, $2, $7
 ; MIPSELR6-NEXT:    selnez $3, $2, $5
 ; MIPSELR6-NEXT:    seleqz $5, $7, $5
@@ -2522,8 +2773,6 @@ define i16 @test_umin_16(ptr nocapture %ptr, i16 signext %val) {
 ; MMEL-NEXT:  $BB7_1: # %entry
 ; MMEL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MMEL-NEXT:    ll $2, 0($6)
-; MMEL-NEXT:    and $2, $2, $8
-; MMEL-NEXT:    and $7, $7, $8
 ; MMEL-NEXT:    sltu $5, $2, $7
 ; MMEL-NEXT:    or $3, $2, $zero
 ; MMEL-NEXT:    movz $3, $7, $5
@@ -2561,8 +2810,6 @@ define i16 @test_umin_16(ptr nocapture %ptr, i16 signext %val) {
 ; MMELR6-NEXT:  $BB7_1: # %entry
 ; MMELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MMELR6-NEXT:    ll $2, 0($6)
-; MMELR6-NEXT:    and $2, $2, $8
-; MMELR6-NEXT:    and $7, $7, $8
 ; MMELR6-NEXT:    sltu $5, $2, $7
 ; MMELR6-NEXT:    selnez $3, $2, $5
 ; MMELR6-NEXT:    seleqz $5, $7, $5
@@ -2680,8 +2927,6 @@ define i16 @test_umin_16(ptr nocapture %ptr, i16 signext %val) {
 ; MIPS64EL-NEXT:  .LBB7_1: # %entry
 ; MIPS64EL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPS64EL-NEXT:    ll $2, 0($6)
-; MIPS64EL-NEXT:    and $2, $2, $8
-; MIPS64EL-NEXT:    and $7, $7, $8
 ; MIPS64EL-NEXT:    sltu $5, $2, $7
 ; MIPS64EL-NEXT:    move $3, $2
 ; MIPS64EL-NEXT:    movz $3, $7, $5
@@ -2721,8 +2966,6 @@ define i16 @test_umin_16(ptr nocapture %ptr, i16 signext %val) {
 ; MIPS64ELR6-NEXT:  .LBB7_1: # %entry
 ; MIPS64ELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPS64ELR6-NEXT:    ll $2, 0($6)
-; MIPS64ELR6-NEXT:    and $2, $2, $8
-; MIPS64ELR6-NEXT:    and $7, $7, $8
 ; MIPS64ELR6-NEXT:    sltu $5, $2, $7
 ; MIPS64ELR6-NEXT:    selnez $3, $2, $5
 ; MIPS64ELR6-NEXT:    seleqz $5, $7, $5
@@ -2780,7 +3023,7 @@ define i8 @test_max_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPS-NEXT:  # %bb.2: # %entry
 ; MIPS-NEXT:    and $1, $2, $8
 ; MIPS-NEXT:    srlv $1, $1, $10
-; MIPS-NEXT:    seh $1, $1
+; MIPS-NEXT:    seb $1, $1
 ; MIPS-NEXT:  # %bb.3: # %entry
 ; MIPS-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
 ; MIPS-NEXT:  # %bb.4: # %entry
@@ -2820,7 +3063,7 @@ define i8 @test_max_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPSR6-NEXT:  # %bb.2: # %entry
 ; MIPSR6-NEXT:    and $1, $2, $8
 ; MIPSR6-NEXT:    srlv $1, $1, $10
-; MIPSR6-NEXT:    seh $1, $1
+; MIPSR6-NEXT:    seb $1, $1
 ; MIPSR6-NEXT:  # %bb.3: # %entry
 ; MIPSR6-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
 ; MIPSR6-NEXT:  # %bb.4: # %entry
@@ -2858,7 +3101,7 @@ define i8 @test_max_8(ptr nocapture %ptr, i8 signext %val) {
 ; MM-NEXT:  # %bb.2: # %entry
 ; MM-NEXT:    and $1, $2, $8
 ; MM-NEXT:    srlv $1, $1, $10
-; MM-NEXT:    seh $1, $1
+; MM-NEXT:    seb $1, $1
 ; MM-NEXT:  # %bb.3: # %entry
 ; MM-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
 ; MM-NEXT:  # %bb.4: # %entry
@@ -2897,7 +3140,7 @@ define i8 @test_max_8(ptr nocapture %ptr, i8 signext %val) {
 ; MMR6-NEXT:  # %bb.2: # %entry
 ; MMR6-NEXT:    and $1, $2, $8
 ; MMR6-NEXT:    srlv $1, $1, $10
-; MMR6-NEXT:    seh $1, $1
+; MMR6-NEXT:    seb $1, $1
 ; MMR6-NEXT:  # %bb.3: # %entry
 ; MMR6-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
 ; MMR6-NEXT:  # %bb.4: # %entry
@@ -2906,6 +3149,52 @@ define i8 @test_max_8(ptr nocapture %ptr, i8 signext %val) {
 ; MMR6-NEXT:    addiu $sp, $sp, 8
 ; MMR6-NEXT:    jrc $ra
 ;
+; MIPS32-LABEL: test_max_8:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    addiu $sp, $sp, -8
+; MIPS32-NEXT:    .cfi_def_cfa_offset 8
+; MIPS32-NEXT:    # kill: def $at killed $a1
+; MIPS32-NEXT:    sync
+; MIPS32-NEXT:    addiu $1, $zero, -4
+; MIPS32-NEXT:    and $6, $4, $1
+; MIPS32-NEXT:    andi $1, $4, 3
+; MIPS32-NEXT:    sll $10, $1, 3
+; MIPS32-NEXT:    ori $1, $zero, 255
+; MIPS32-NEXT:    sllv $8, $1, $10
+; MIPS32-NEXT:    nor $9, $zero, $8
+; MIPS32-NEXT:    sllv $7, $5, $10
+; MIPS32-NEXT:  $BB8_1: # %entry
+; MIPS32-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS32-NEXT:    ll $2, 0($6)
+; MIPS32-NEXT:    srav $2, $2, $10
+; MIPS32-NEXT:    srav $7, $7, $10
+; MIPS32-NEXT:    sll $2, $2, 24
+; MIPS32-NEXT:    sra $2, $2, 24
+; MIPS32-NEXT:    sll $7, $7, 24
+; MIPS32-NEXT:    sra $7, $7, 24
+; MIPS32-NEXT:    slt $5, $2, $7
+; MIPS32-NEXT:    move $3, $2
+; MIPS32-NEXT:    movn $3, $7, $5
+; MIPS32-NEXT:    and $3, $3, $8
+; MIPS32-NEXT:    and $4, $2, $9
+; MIPS32-NEXT:    or $4, $4, $3
+; MIPS32-NEXT:    sc $4, 0($6)
+; MIPS32-NEXT:    beqz $4, $BB8_1
+; MIPS32-NEXT:    nop
+; MIPS32-NEXT:  # %bb.2: # %entry
+; MIPS32-NEXT:    and $1, $2, $8
+; MIPS32-NEXT:    srlv $1, $1, $10
+; MIPS32-NEXT:    sll $1, $1, 24
+; MIPS32-NEXT:    sra $1, $1, 24
+; MIPS32-NEXT:  # %bb.3: # %entry
+; MIPS32-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:  # %bb.4: # %entry
+; MIPS32-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    sync
+; MIPS32-NEXT:    addiu $sp, $sp, 8
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+;
 ; MIPSEL-LABEL: test_max_8:
 ; MIPSEL:       # %bb.0: # %entry
 ; MIPSEL-NEXT:    addiu $sp, $sp, -8
@@ -2923,8 +3212,10 @@ define i8 @test_max_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPSEL-NEXT:  $BB8_1: # %entry
 ; MIPSEL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPSEL-NEXT:    ll $2, 0($6)
-; MIPSEL-NEXT:    and $2, $2, $8
-; MIPSEL-NEXT:    and $7, $7, $8
+; MIPSEL-NEXT:    srav $2, $2, $10
+; MIPSEL-NEXT:    srav $7, $7, $10
+; MIPSEL-NEXT:    seb $2, $2
+; MIPSEL-NEXT:    seb $7, $7
 ; MIPSEL-NEXT:    slt $5, $2, $7
 ; MIPSEL-NEXT:    move $3, $2
 ; MIPSEL-NEXT:    movn $3, $7, $5
@@ -2937,7 +3228,7 @@ define i8 @test_max_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPSEL-NEXT:  # %bb.2: # %entry
 ; MIPSEL-NEXT:    and $1, $2, $8
 ; MIPSEL-NEXT:    srlv $1, $1, $10
-; MIPSEL-NEXT:    seh $1, $1
+; MIPSEL-NEXT:    seb $1, $1
 ; MIPSEL-NEXT:  # %bb.3: # %entry
 ; MIPSEL-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
 ; MIPSEL-NEXT:  # %bb.4: # %entry
@@ -2964,8 +3255,10 @@ define i8 @test_max_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPSELR6-NEXT:  $BB8_1: # %entry
 ; MIPSELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPSELR6-NEXT:    ll $2, 0($6)
-; MIPSELR6-NEXT:    and $2, $2, $8
-; MIPSELR6-NEXT:    and $7, $7, $8
+; MIPSELR6-NEXT:    srav $2, $2, $10
+; MIPSELR6-NEXT:    srav $7, $7, $10
+; MIPSELR6-NEXT:    seb $2, $2
+; MIPSELR6-NEXT:    seb $7, $7
 ; MIPSELR6-NEXT:    slt $5, $2, $7
 ; MIPSELR6-NEXT:    seleqz $3, $2, $5
 ; MIPSELR6-NEXT:    selnez $5, $7, $5
@@ -2978,7 +3271,7 @@ define i8 @test_max_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPSELR6-NEXT:  # %bb.2: # %entry
 ; MIPSELR6-NEXT:    and $1, $2, $8
 ; MIPSELR6-NEXT:    srlv $1, $1, $10
-; MIPSELR6-NEXT:    seh $1, $1
+; MIPSELR6-NEXT:    seb $1, $1
 ; MIPSELR6-NEXT:  # %bb.3: # %entry
 ; MIPSELR6-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
 ; MIPSELR6-NEXT:  # %bb.4: # %entry
@@ -3004,8 +3297,10 @@ define i8 @test_max_8(ptr nocapture %ptr, i8 signext %val) {
 ; MMEL-NEXT:  $BB8_1: # %entry
 ; MMEL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MMEL-NEXT:    ll $2, 0($6)
-; MMEL-NEXT:    and $2, $2, $8
-; MMEL-NEXT:    and $7, $7, $8
+; MMEL-NEXT:    srav $2, $2, $10
+; MMEL-NEXT:    srav $7, $7, $10
+; MMEL-NEXT:    seb $2, $2
+; MMEL-NEXT:    seb $7, $7
 ; MMEL-NEXT:    slt $5, $2, $7
 ; MMEL-NEXT:    or $3, $2, $zero
 ; MMEL-NEXT:    movn $3, $7, $5
@@ -3017,7 +3312,7 @@ define i8 @test_max_8(ptr nocapture %ptr, i8 signext %val) {
 ; MMEL-NEXT:  # %bb.2: # %entry
 ; MMEL-NEXT:    and $1, $2, $8
 ; MMEL-NEXT:    srlv $1, $1, $10
-; MMEL-NEXT:    seh $1, $1
+; MMEL-NEXT:    seb $1, $1
 ; MMEL-NEXT:  # %bb.3: # %entry
 ; MMEL-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
 ; MMEL-NEXT:  # %bb.4: # %entry
@@ -3043,8 +3338,10 @@ define i8 @test_max_8(ptr nocapture %ptr, i8 signext %val) {
 ; MMELR6-NEXT:  $BB8_1: # %entry
 ; MMELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MMELR6-NEXT:    ll $2, 0($6)
-; MMELR6-NEXT:    and $2, $2, $8
-; MMELR6-NEXT:    and $7, $7, $8
+; MMELR6-NEXT:    srav $2, $2, $10
+; MMELR6-NEXT:    srav $7, $7, $10
+; MMELR6-NEXT:    seb $2, $2
+; MMELR6-NEXT:    seb $7, $7
 ; MMELR6-NEXT:    slt $5, $2, $7
 ; MMELR6-NEXT:    seleqz $3, $2, $5
 ; MMELR6-NEXT:    selnez $5, $7, $5
@@ -3057,7 +3354,7 @@ define i8 @test_max_8(ptr nocapture %ptr, i8 signext %val) {
 ; MMELR6-NEXT:  # %bb.2: # %entry
 ; MMELR6-NEXT:    and $1, $2, $8
 ; MMELR6-NEXT:    srlv $1, $1, $10
-; MMELR6-NEXT:    seh $1, $1
+; MMELR6-NEXT:    seb $1, $1
 ; MMELR6-NEXT:  # %bb.3: # %entry
 ; MMELR6-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
 ; MMELR6-NEXT:  # %bb.4: # %entry
@@ -3096,7 +3393,7 @@ define i8 @test_max_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPS64-NEXT:  # %bb.2: # %entry
 ; MIPS64-NEXT:    and $1, $2, $8
 ; MIPS64-NEXT:    srlv $1, $1, $10
-; MIPS64-NEXT:    seh $1, $1
+; MIPS64-NEXT:    seb $1, $1
 ; MIPS64-NEXT:  # %bb.3: # %entry
 ; MIPS64-NEXT:    sw $1, 12($sp) # 4-byte Folded Spill
 ; MIPS64-NEXT:  # %bb.4: # %entry
@@ -3136,7 +3433,7 @@ define i8 @test_max_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPS64R6-NEXT:  # %bb.2: # %entry
 ; MIPS64R6-NEXT:    and $1, $2, $8
 ; MIPS64R6-NEXT:    srlv $1, $1, $10
-; MIPS64R6-NEXT:    seh $1, $1
+; MIPS64R6-NEXT:    seb $1, $1
 ; MIPS64R6-NEXT:  # %bb.3: # %entry
 ; MIPS64R6-NEXT:    sw $1, 12($sp) # 4-byte Folded Spill
 ; MIPS64R6-NEXT:  # %bb.4: # %entry
@@ -3162,8 +3459,10 @@ define i8 @test_max_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPS64EL-NEXT:  .LBB8_1: # %entry
 ; MIPS64EL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPS64EL-NEXT:    ll $2, 0($6)
-; MIPS64EL-NEXT:    and $2, $2, $8
-; MIPS64EL-NEXT:    and $7, $7, $8
+; MIPS64EL-NEXT:    srav $2, $2, $10
+; MIPS64EL-NEXT:    srav $7, $7, $10
+; MIPS64EL-NEXT:    seb $2, $2
+; MIPS64EL-NEXT:    seb $7, $7
 ; MIPS64EL-NEXT:    slt $5, $2, $7
 ; MIPS64EL-NEXT:    move $3, $2
 ; MIPS64EL-NEXT:    movn $3, $7, $5
@@ -3176,7 +3475,7 @@ define i8 @test_max_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPS64EL-NEXT:  # %bb.2: # %entry
 ; MIPS64EL-NEXT:    and $1, $2, $8
 ; MIPS64EL-NEXT:    srlv $1, $1, $10
-; MIPS64EL-NEXT:    seh $1, $1
+; MIPS64EL-NEXT:    seb $1, $1
 ; MIPS64EL-NEXT:  # %bb.3: # %entry
 ; MIPS64EL-NEXT:    sw $1, 12($sp) # 4-byte Folded Spill
 ; MIPS64EL-NEXT:  # %bb.4: # %entry
@@ -3203,8 +3502,10 @@ define i8 @test_max_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPS64ELR6-NEXT:  .LBB8_1: # %entry
 ; MIPS64ELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPS64ELR6-NEXT:    ll $2, 0($6)
-; MIPS64ELR6-NEXT:    and $2, $2, $8
-; MIPS64ELR6-NEXT:    and $7, $7, $8
+; MIPS64ELR6-NEXT:    srav $2, $2, $10
+; MIPS64ELR6-NEXT:    srav $7, $7, $10
+; MIPS64ELR6-NEXT:    seb $2, $2
+; MIPS64ELR6-NEXT:    seb $7, $7
 ; MIPS64ELR6-NEXT:    slt $5, $2, $7
 ; MIPS64ELR6-NEXT:    seleqz $3, $2, $5
 ; MIPS64ELR6-NEXT:    selnez $5, $7, $5
@@ -3217,7 +3518,7 @@ define i8 @test_max_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPS64ELR6-NEXT:  # %bb.2: # %entry
 ; MIPS64ELR6-NEXT:    and $1, $2, $8
 ; MIPS64ELR6-NEXT:    srlv $1, $1, $10
-; MIPS64ELR6-NEXT:    seh $1, $1
+; MIPS64ELR6-NEXT:    seb $1, $1
 ; MIPS64ELR6-NEXT:  # %bb.3: # %entry
 ; MIPS64ELR6-NEXT:    sw $1, 12($sp) # 4-byte Folded Spill
 ; MIPS64ELR6-NEXT:  # %bb.4: # %entry
@@ -3261,7 +3562,7 @@ define i8 @test_min_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPS-NEXT:  # %bb.2: # %entry
 ; MIPS-NEXT:    and $1, $2, $8
 ; MIPS-NEXT:    srlv $1, $1, $10
-; MIPS-NEXT:    seh $1, $1
+; MIPS-NEXT:    seb $1, $1
 ; MIPS-NEXT:  # %bb.3: # %entry
 ; MIPS-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
 ; MIPS-NEXT:  # %bb.4: # %entry
@@ -3301,7 +3602,7 @@ define i8 @test_min_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPSR6-NEXT:  # %bb.2: # %entry
 ; MIPSR6-NEXT:    and $1, $2, $8
 ; MIPSR6-NEXT:    srlv $1, $1, $10
-; MIPSR6-NEXT:    seh $1, $1
+; MIPSR6-NEXT:    seb $1, $1
 ; MIPSR6-NEXT:  # %bb.3: # %entry
 ; MIPSR6-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
 ; MIPSR6-NEXT:  # %bb.4: # %entry
@@ -3339,7 +3640,7 @@ define i8 @test_min_8(ptr nocapture %ptr, i8 signext %val) {
 ; MM-NEXT:  # %bb.2: # %entry
 ; MM-NEXT:    and $1, $2, $8
 ; MM-NEXT:    srlv $1, $1, $10
-; MM-NEXT:    seh $1, $1
+; MM-NEXT:    seb $1, $1
 ; MM-NEXT:  # %bb.3: # %entry
 ; MM-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
 ; MM-NEXT:  # %bb.4: # %entry
@@ -3378,7 +3679,7 @@ define i8 @test_min_8(ptr nocapture %ptr, i8 signext %val) {
 ; MMR6-NEXT:  # %bb.2: # %entry
 ; MMR6-NEXT:    and $1, $2, $8
 ; MMR6-NEXT:    srlv $1, $1, $10
-; MMR6-NEXT:    seh $1, $1
+; MMR6-NEXT:    seb $1, $1
 ; MMR6-NEXT:  # %bb.3: # %entry
 ; MMR6-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
 ; MMR6-NEXT:  # %bb.4: # %entry
@@ -3387,6 +3688,52 @@ define i8 @test_min_8(ptr nocapture %ptr, i8 signext %val) {
 ; MMR6-NEXT:    addiu $sp, $sp, 8
 ; MMR6-NEXT:    jrc $ra
 ;
+; MIPS32-LABEL: test_min_8:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    addiu $sp, $sp, -8
+; MIPS32-NEXT:    .cfi_def_cfa_offset 8
+; MIPS32-NEXT:    # kill: def $at killed $a1
+; MIPS32-NEXT:    sync
+; MIPS32-NEXT:    addiu $1, $zero, -4
+; MIPS32-NEXT:    and $6, $4, $1
+; MIPS32-NEXT:    andi $1, $4, 3
+; MIPS32-NEXT:    sll $10, $1, 3
+; MIPS32-NEXT:    ori $1, $zero, 255
+; MIPS32-NEXT:    sllv $8, $1, $10
+; MIPS32-NEXT:    nor $9, $zero, $8
+; MIPS32-NEXT:    sllv $7, $5, $10
+; MIPS32-NEXT:  $BB9_1: # %entry
+; MIPS32-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS32-NEXT:    ll $2, 0($6)
+; MIPS32-NEXT:    srav $2, $2, $10
+; MIPS32-NEXT:    srav $7, $7, $10
+; MIPS32-NEXT:    sll $2, $2, 24
+; MIPS32-NEXT:    sra $2, $2, 24
+; MIPS32-NEXT:    sll $7, $7, 24
+; MIPS32-NEXT:    sra $7, $7, 24
+; MIPS32-NEXT:    slt $5, $2, $7
+; MIPS32-NEXT:    move $3, $2
+; MIPS32-NEXT:    movz $3, $7, $5
+; MIPS32-NEXT:    and $3, $3, $8
+; MIPS32-NEXT:    and $4, $2, $9
+; MIPS32-NEXT:    or $4, $4, $3
+; MIPS32-NEXT:    sc $4, 0($6)
+; MIPS32-NEXT:    beqz $4, $BB9_1
+; MIPS32-NEXT:    nop
+; MIPS32-NEXT:  # %bb.2: # %entry
+; MIPS32-NEXT:    and $1, $2, $8
+; MIPS32-NEXT:    srlv $1, $1, $10
+; MIPS32-NEXT:    sll $1, $1, 24
+; MIPS32-NEXT:    sra $1, $1, 24
+; MIPS32-NEXT:  # %bb.3: # %entry
+; MIPS32-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:  # %bb.4: # %entry
+; MIPS32-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    sync
+; MIPS32-NEXT:    addiu $sp, $sp, 8
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+;
 ; MIPSEL-LABEL: test_min_8:
 ; MIPSEL:       # %bb.0: # %entry
 ; MIPSEL-NEXT:    addiu $sp, $sp, -8
@@ -3404,8 +3751,10 @@ define i8 @test_min_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPSEL-NEXT:  $BB9_1: # %entry
 ; MIPSEL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPSEL-NEXT:    ll $2, 0($6)
-; MIPSEL-NEXT:    and $2, $2, $8
-; MIPSEL-NEXT:    and $7, $7, $8
+; MIPSEL-NEXT:    srav $2, $2, $10
+; MIPSEL-NEXT:    srav $7, $7, $10
+; MIPSEL-NEXT:    seb $2, $2
+; MIPSEL-NEXT:    seb $7, $7
 ; MIPSEL-NEXT:    slt $5, $2, $7
 ; MIPSEL-NEXT:    move $3, $2
 ; MIPSEL-NEXT:    movz $3, $7, $5
@@ -3418,7 +3767,7 @@ define i8 @test_min_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPSEL-NEXT:  # %bb.2: # %entry
 ; MIPSEL-NEXT:    and $1, $2, $8
 ; MIPSEL-NEXT:    srlv $1, $1, $10
-; MIPSEL-NEXT:    seh $1, $1
+; MIPSEL-NEXT:    seb $1, $1
 ; MIPSEL-NEXT:  # %bb.3: # %entry
 ; MIPSEL-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
 ; MIPSEL-NEXT:  # %bb.4: # %entry
@@ -3445,8 +3794,10 @@ define i8 @test_min_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPSELR6-NEXT:  $BB9_1: # %entry
 ; MIPSELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPSELR6-NEXT:    ll $2, 0($6)
-; MIPSELR6-NEXT:    and $2, $2, $8
-; MIPSELR6-NEXT:    and $7, $7, $8
+; MIPSELR6-NEXT:    srav $2, $2, $10
+; MIPSELR6-NEXT:    srav $7, $7, $10
+; MIPSELR6-NEXT:    seb $2, $2
+; MIPSELR6-NEXT:    seb $7, $7
 ; MIPSELR6-NEXT:    slt $5, $2, $7
 ; MIPSELR6-NEXT:    selnez $3, $2, $5
 ; MIPSELR6-NEXT:    seleqz $5, $7, $5
@@ -3459,7 +3810,7 @@ define i8 @test_min_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPSELR6-NEXT:  # %bb.2: # %entry
 ; MIPSELR6-NEXT:    and $1, $2, $8
 ; MIPSELR6-NEXT:    srlv $1, $1, $10
-; MIPSELR6-NEXT:    seh $1, $1
+; MIPSELR6-NEXT:    seb $1, $1
 ; MIPSELR6-NEXT:  # %bb.3: # %entry
 ; MIPSELR6-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
 ; MIPSELR6-NEXT:  # %bb.4: # %entry
@@ -3485,8 +3836,10 @@ define i8 @test_min_8(ptr nocapture %ptr, i8 signext %val) {
 ; MMEL-NEXT:  $BB9_1: # %entry
 ; MMEL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MMEL-NEXT:    ll $2, 0($6)
-; MMEL-NEXT:    and $2, $2, $8
-; MMEL-NEXT:    and $7, $7, $8
+; MMEL-NEXT:    srav $2, $2, $10
+; MMEL-NEXT:    srav $7, $7, $10
+; MMEL-NEXT:    seb $2, $2
+; MMEL-NEXT:    seb $7, $7
 ; MMEL-NEXT:    slt $5, $2, $7
 ; MMEL-NEXT:    or $3, $2, $zero
 ; MMEL-NEXT:    movz $3, $7, $5
@@ -3498,7 +3851,7 @@ define i8 @test_min_8(ptr nocapture %ptr, i8 signext %val) {
 ; MMEL-NEXT:  # %bb.2: # %entry
 ; MMEL-NEXT:    and $1, $2, $8
 ; MMEL-NEXT:    srlv $1, $1, $10
-; MMEL-NEXT:    seh $1, $1
+; MMEL-NEXT:    seb $1, $1
 ; MMEL-NEXT:  # %bb.3: # %entry
 ; MMEL-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
 ; MMEL-NEXT:  # %bb.4: # %entry
@@ -3524,8 +3877,10 @@ define i8 @test_min_8(ptr nocapture %ptr, i8 signext %val) {
 ; MMELR6-NEXT:  $BB9_1: # %entry
 ; MMELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MMELR6-NEXT:    ll $2, 0($6)
-; MMELR6-NEXT:    and $2, $2, $8
-; MMELR6-NEXT:    and $7, $7, $8
+; MMELR6-NEXT:    srav $2, $2, $10
+; MMELR6-NEXT:    srav $7, $7, $10
+; MMELR6-NEXT:    seb $2, $2
+; MMELR6-NEXT:    seb $7, $7
 ; MMELR6-NEXT:    slt $5, $2, $7
 ; MMELR6-NEXT:    selnez $3, $2, $5
 ; MMELR6-NEXT:    seleqz $5, $7, $5
@@ -3538,7 +3893,7 @@ define i8 @test_min_8(ptr nocapture %ptr, i8 signext %val) {
 ; MMELR6-NEXT:  # %bb.2: # %entry
 ; MMELR6-NEXT:    and $1, $2, $8
 ; MMELR6-NEXT:    srlv $1, $1, $10
-; MMELR6-NEXT:    seh $1, $1
+; MMELR6-NEXT:    seb $1, $1
 ; MMELR6-NEXT:  # %bb.3: # %entry
 ; MMELR6-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
 ; MMELR6-NEXT:  # %bb.4: # %entry
@@ -3577,7 +3932,7 @@ define i8 @test_min_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPS64-NEXT:  # %bb.2: # %entry
 ; MIPS64-NEXT:    and $1, $2, $8
 ; MIPS64-NEXT:    srlv $1, $1, $10
-; MIPS64-NEXT:    seh $1, $1
+; MIPS64-NEXT:    seb $1, $1
 ; MIPS64-NEXT:  # %bb.3: # %entry
 ; MIPS64-NEXT:    sw $1, 12($sp) # 4-byte Folded Spill
 ; MIPS64-NEXT:  # %bb.4: # %entry
@@ -3617,7 +3972,7 @@ define i8 @test_min_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPS64R6-NEXT:  # %bb.2: # %entry
 ; MIPS64R6-NEXT:    and $1, $2, $8
 ; MIPS64R6-NEXT:    srlv $1, $1, $10
-; MIPS64R6-NEXT:    seh $1, $1
+; MIPS64R6-NEXT:    seb $1, $1
 ; MIPS64R6-NEXT:  # %bb.3: # %entry
 ; MIPS64R6-NEXT:    sw $1, 12($sp) # 4-byte Folded Spill
 ; MIPS64R6-NEXT:  # %bb.4: # %entry
@@ -3643,8 +3998,10 @@ define i8 @test_min_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPS64EL-NEXT:  .LBB9_1: # %entry
 ; MIPS64EL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPS64EL-NEXT:    ll $2, 0($6)
-; MIPS64EL-NEXT:    and $2, $2, $8
-; MIPS64EL-NEXT:    and $7, $7, $8
+; MIPS64EL-NEXT:    srav $2, $2, $10
+; MIPS64EL-NEXT:    srav $7, $7, $10
+; MIPS64EL-NEXT:    seb $2, $2
+; MIPS64EL-NEXT:    seb $7, $7
 ; MIPS64EL-NEXT:    slt $5, $2, $7
 ; MIPS64EL-NEXT:    move $3, $2
 ; MIPS64EL-NEXT:    movz $3, $7, $5
@@ -3657,7 +4014,7 @@ define i8 @test_min_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPS64EL-NEXT:  # %bb.2: # %entry
 ; MIPS64EL-NEXT:    and $1, $2, $8
 ; MIPS64EL-NEXT:    srlv $1, $1, $10
-; MIPS64EL-NEXT:    seh $1, $1
+; MIPS64EL-NEXT:    seb $1, $1
 ; MIPS64EL-NEXT:  # %bb.3: # %entry
 ; MIPS64EL-NEXT:    sw $1, 12($sp) # 4-byte Folded Spill
 ; MIPS64EL-NEXT:  # %bb.4: # %entry
@@ -3684,8 +4041,10 @@ define i8 @test_min_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPS64ELR6-NEXT:  .LBB9_1: # %entry
 ; MIPS64ELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPS64ELR6-NEXT:    ll $2, 0($6)
-; MIPS64ELR6-NEXT:    and $2, $2, $8
-; MIPS64ELR6-NEXT:    and $7, $7, $8
+; MIPS64ELR6-NEXT:    srav $2, $2, $10
+; MIPS64ELR6-NEXT:    srav $7, $7, $10
+; MIPS64ELR6-NEXT:    seb $2, $2
+; MIPS64ELR6-NEXT:    seb $7, $7
 ; MIPS64ELR6-NEXT:    slt $5, $2, $7
 ; MIPS64ELR6-NEXT:    selnez $3, $2, $5
 ; MIPS64ELR6-NEXT:    seleqz $5, $7, $5
@@ -3698,7 +4057,7 @@ define i8 @test_min_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPS64ELR6-NEXT:  # %bb.2: # %entry
 ; MIPS64ELR6-NEXT:    and $1, $2, $8
 ; MIPS64ELR6-NEXT:    srlv $1, $1, $10
-; MIPS64ELR6-NEXT:    seh $1, $1
+; MIPS64ELR6-NEXT:    seb $1, $1
 ; MIPS64ELR6-NEXT:  # %bb.3: # %entry
 ; MIPS64ELR6-NEXT:    sw $1, 12($sp) # 4-byte Folded Spill
 ; MIPS64ELR6-NEXT:  # %bb.4: # %entry
@@ -3868,6 +4227,46 @@ define i8 @test_umax_8(ptr nocapture %ptr, i8 signext %val) {
 ; MMR6-NEXT:    addiu $sp, $sp, 8
 ; MMR6-NEXT:    jrc $ra
 ;
+; MIPS32-LABEL: test_umax_8:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    addiu $sp, $sp, -8
+; MIPS32-NEXT:    .cfi_def_cfa_offset 8
+; MIPS32-NEXT:    # kill: def $at killed $a1
+; MIPS32-NEXT:    sync
+; MIPS32-NEXT:    addiu $1, $zero, -4
+; MIPS32-NEXT:    and $6, $4, $1
+; MIPS32-NEXT:    andi $1, $4, 3
+; MIPS32-NEXT:    sll $10, $1, 3
+; MIPS32-NEXT:    ori $1, $zero, 255
+; MIPS32-NEXT:    sllv $8, $1, $10
+; MIPS32-NEXT:    nor $9, $zero, $8
+; MIPS32-NEXT:    sllv $7, $5, $10
+; MIPS32-NEXT:  $BB10_1: # %entry
+; MIPS32-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS32-NEXT:    ll $2, 0($6)
+; MIPS32-NEXT:    sltu $5, $2, $7
+; MIPS32-NEXT:    move $3, $2
+; MIPS32-NEXT:    movn $3, $7, $5
+; MIPS32-NEXT:    and $3, $3, $8
+; MIPS32-NEXT:    and $4, $2, $9
+; MIPS32-NEXT:    or $4, $4, $3
+; MIPS32-NEXT:    sc $4, 0($6)
+; MIPS32-NEXT:    beqz $4, $BB10_1
+; MIPS32-NEXT:    nop
+; MIPS32-NEXT:  # %bb.2: # %entry
+; MIPS32-NEXT:    and $1, $2, $8
+; MIPS32-NEXT:    srlv $1, $1, $10
+; MIPS32-NEXT:    sll $1, $1, 16
+; MIPS32-NEXT:    sra $1, $1, 16
+; MIPS32-NEXT:  # %bb.3: # %entry
+; MIPS32-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:  # %bb.4: # %entry
+; MIPS32-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    sync
+; MIPS32-NEXT:    addiu $sp, $sp, 8
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+;
 ; MIPSEL-LABEL: test_umax_8:
 ; MIPSEL:       # %bb.0: # %entry
 ; MIPSEL-NEXT:    addiu $sp, $sp, -8
@@ -3885,8 +4284,6 @@ define i8 @test_umax_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPSEL-NEXT:  $BB10_1: # %entry
 ; MIPSEL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPSEL-NEXT:    ll $2, 0($6)
-; MIPSEL-NEXT:    and $2, $2, $8
-; MIPSEL-NEXT:    and $7, $7, $8
 ; MIPSEL-NEXT:    sltu $5, $2, $7
 ; MIPSEL-NEXT:    move $3, $2
 ; MIPSEL-NEXT:    movn $3, $7, $5
@@ -3926,8 +4323,6 @@ define i8 @test_umax_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPSELR6-NEXT:  $BB10_1: # %entry
 ; MIPSELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPSELR6-NEXT:    ll $2, 0($6)
-; MIPSELR6-NEXT:    and $2, $2, $8
-; MIPSELR6-NEXT:    and $7, $7, $8
 ; MIPSELR6-NEXT:    sltu $5, $2, $7
 ; MIPSELR6-NEXT:    seleqz $3, $2, $5
 ; MIPSELR6-NEXT:    selnez $5, $7, $5
@@ -3966,8 +4361,6 @@ define i8 @test_umax_8(ptr nocapture %ptr, i8 signext %val) {
 ; MMEL-NEXT:  $BB10_1: # %entry
 ; MMEL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MMEL-NEXT:    ll $2, 0($6)
-; MMEL-NEXT:    and $2, $2, $8
-; MMEL-NEXT:    and $7, $7, $8
 ; MMEL-NEXT:    sltu $5, $2, $7
 ; MMEL-NEXT:    or $3, $2, $zero
 ; MMEL-NEXT:    movn $3, $7, $5
@@ -4005,8 +4398,6 @@ define i8 @test_umax_8(ptr nocapture %ptr, i8 signext %val) {
 ; MMELR6-NEXT:  $BB10_1: # %entry
 ; MMELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MMELR6-NEXT:    ll $2, 0($6)
-; MMELR6-NEXT:    and $2, $2, $8
-; MMELR6-NEXT:    and $7, $7, $8
 ; MMELR6-NEXT:    sltu $5, $2, $7
 ; MMELR6-NEXT:    seleqz $3, $2, $5
 ; MMELR6-NEXT:    selnez $5, $7, $5
@@ -4124,8 +4515,6 @@ define i8 @test_umax_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPS64EL-NEXT:  .LBB10_1: # %entry
 ; MIPS64EL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPS64EL-NEXT:    ll $2, 0($6)
-; MIPS64EL-NEXT:    and $2, $2, $8
-; MIPS64EL-NEXT:    and $7, $7, $8
 ; MIPS64EL-NEXT:    sltu $5, $2, $7
 ; MIPS64EL-NEXT:    move $3, $2
 ; MIPS64EL-NEXT:    movn $3, $7, $5
@@ -4165,8 +4554,6 @@ define i8 @test_umax_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPS64ELR6-NEXT:  .LBB10_1: # %entry
 ; MIPS64ELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPS64ELR6-NEXT:    ll $2, 0($6)
-; MIPS64ELR6-NEXT:    and $2, $2, $8
-; MIPS64ELR6-NEXT:    and $7, $7, $8
 ; MIPS64ELR6-NEXT:    sltu $5, $2, $7
 ; MIPS64ELR6-NEXT:    seleqz $3, $2, $5
 ; MIPS64ELR6-NEXT:    selnez $5, $7, $5
@@ -4349,6 +4736,46 @@ define i8 @test_umin_8(ptr nocapture %ptr, i8 signext %val) {
 ; MMR6-NEXT:    addiu $sp, $sp, 8
 ; MMR6-NEXT:    jrc $ra
 ;
+; MIPS32-LABEL: test_umin_8:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    addiu $sp, $sp, -8
+; MIPS32-NEXT:    .cfi_def_cfa_offset 8
+; MIPS32-NEXT:    # kill: def $at killed $a1
+; MIPS32-NEXT:    sync
+; MIPS32-NEXT:    addiu $1, $zero, -4
+; MIPS32-NEXT:    and $6, $4, $1
+; MIPS32-NEXT:    andi $1, $4, 3
+; MIPS32-NEXT:    sll $10, $1, 3
+; MIPS32-NEXT:    ori $1, $zero, 255
+; MIPS32-NEXT:    sllv $8, $1, $10
+; MIPS32-NEXT:    nor $9, $zero, $8
+; MIPS32-NEXT:    sllv $7, $5, $10
+; MIPS32-NEXT:  $BB11_1: # %entry
+; MIPS32-NEXT:    # =>This Inner Loop Header: Depth=1
+; MIPS32-NEXT:    ll $2, 0($6)
+; MIPS32-NEXT:    sltu $5, $2, $7
+; MIPS32-NEXT:    move $3, $2
+; MIPS32-NEXT:    movz $3, $7, $5
+; MIPS32-NEXT:    and $3, $3, $8
+; MIPS32-NEXT:    and $4, $2, $9
+; MIPS32-NEXT:    or $4, $4, $3
+; MIPS32-NEXT:    sc $4, 0($6)
+; MIPS32-NEXT:    beqz $4, $BB11_1
+; MIPS32-NEXT:    nop
+; MIPS32-NEXT:  # %bb.2: # %entry
+; MIPS32-NEXT:    and $1, $2, $8
+; MIPS32-NEXT:    srlv $1, $1, $10
+; MIPS32-NEXT:    sll $1, $1, 16
+; MIPS32-NEXT:    sra $1, $1, 16
+; MIPS32-NEXT:  # %bb.3: # %entry
+; MIPS32-NEXT:    sw $1, 4($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:  # %bb.4: # %entry
+; MIPS32-NEXT:    lw $2, 4($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    sync
+; MIPS32-NEXT:    addiu $sp, $sp, 8
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+;
 ; MIPSEL-LABEL: test_umin_8:
 ; MIPSEL:       # %bb.0: # %entry
 ; MIPSEL-NEXT:    addiu $sp, $sp, -8
@@ -4366,8 +4793,6 @@ define i8 @test_umin_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPSEL-NEXT:  $BB11_1: # %entry
 ; MIPSEL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPSEL-NEXT:    ll $2, 0($6)
-; MIPSEL-NEXT:    and $2, $2, $8
-; MIPSEL-NEXT:    and $7, $7, $8
 ; MIPSEL-NEXT:    sltu $5, $2, $7
 ; MIPSEL-NEXT:    move $3, $2
 ; MIPSEL-NEXT:    movz $3, $7, $5
@@ -4407,8 +4832,6 @@ define i8 @test_umin_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPSELR6-NEXT:  $BB11_1: # %entry
 ; MIPSELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPSELR6-NEXT:    ll $2, 0($6)
-; MIPSELR6-NEXT:    and $2, $2, $8
-; MIPSELR6-NEXT:    and $7, $7, $8
 ; MIPSELR6-NEXT:    sltu $5, $2, $7
 ; MIPSELR6-NEXT:    selnez $3, $2, $5
 ; MIPSELR6-NEXT:    seleqz $5, $7, $5
@@ -4447,8 +4870,6 @@ define i8 @test_umin_8(ptr nocapture %ptr, i8 signext %val) {
 ; MMEL-NEXT:  $BB11_1: # %entry
 ; MMEL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MMEL-NEXT:    ll $2, 0($6)
-; MMEL-NEXT:    and $2, $2, $8
-; MMEL-NEXT:    and $7, $7, $8
 ; MMEL-NEXT:    sltu $5, $2, $7
 ; MMEL-NEXT:    or $3, $2, $zero
 ; MMEL-NEXT:    movz $3, $7, $5
@@ -4486,8 +4907,6 @@ define i8 @test_umin_8(ptr nocapture %ptr, i8 signext %val) {
 ; MMELR6-NEXT:  $BB11_1: # %entry
 ; MMELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MMELR6-NEXT:    ll $2, 0($6)
-; MMELR6-NEXT:    and $2, $2, $8
-; MMELR6-NEXT:    and $7, $7, $8
 ; MMELR6-NEXT:    sltu $5, $2, $7
 ; MMELR6-NEXT:    selnez $3, $2, $5
 ; MMELR6-NEXT:    seleqz $5, $7, $5
@@ -4605,8 +5024,6 @@ define i8 @test_umin_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPS64EL-NEXT:  .LBB11_1: # %entry
 ; MIPS64EL-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPS64EL-NEXT:    ll $2, 0($6)
-; MIPS64EL-NEXT:    and $2, $2, $8
-; MIPS64EL-NEXT:    and $7, $7, $8
 ; MIPS64EL-NEXT:    sltu $5, $2, $7
 ; MIPS64EL-NEXT:    move $3, $2
 ; MIPS64EL-NEXT:    movz $3, $7, $5
@@ -4646,8 +5063,6 @@ define i8 @test_umin_8(ptr nocapture %ptr, i8 signext %val) {
 ; MIPS64ELR6-NEXT:  .LBB11_1: # %entry
 ; MIPS64ELR6-NEXT:    # =>This Inner Loop Header: Depth=1
 ; MIPS64ELR6-NEXT:    ll $2, 0($6)
-; MIPS64ELR6-NEXT:    and $2, $2, $8
-; MIPS64ELR6-NEXT:    and $7, $7, $8
 ; MIPS64ELR6-NEXT:    sltu $5, $2, $7
 ; MIPS64ELR6-NEXT:    selnez $3, $2, $5
 ; MIPS64ELR6-NEXT:    seleqz $5, $7, $5
-- 
GitLab


From 83789ffbd857bb78b69d6e54560310ff2d347f04 Mon Sep 17 00:00:00 2001
From: Florian Hahn 
Date: Fri, 8 Mar 2024 20:43:35 +0000
Subject: [PATCH 0084/1674] [TBAA] Add bail-out to skip tbaa generation to
 getTBAAStructInfo. (#84386)

Without this bail out, we may generate fields with null nodes as tags
are generated by using getTypeInfo which has the same bail out.

PR: https://github.com/llvm/llvm-project/pull/84386
---
 clang/lib/CodeGen/CodeGenTBAA.cpp                             | 3 +++
 clang/test/CodeGen/tbaa-struct-relaxed-aliasing-with-tsan.cpp | 2 --
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/clang/lib/CodeGen/CodeGenTBAA.cpp b/clang/lib/CodeGen/CodeGenTBAA.cpp
index 8a0816121939..1f07205a5af2 100644
--- a/clang/lib/CodeGen/CodeGenTBAA.cpp
+++ b/clang/lib/CodeGen/CodeGenTBAA.cpp
@@ -343,6 +343,9 @@ CodeGenTBAA::CollectFields(uint64_t BaseOffset,
 
 llvm::MDNode *
 CodeGenTBAA::getTBAAStructInfo(QualType QTy) {
+  if (CodeGenOpts.OptimizationLevel == 0 || CodeGenOpts.RelaxedAliasing)
+    return nullptr;
+
   const Type *Ty = Context.getCanonicalType(QTy).getTypePtr();
 
   if (llvm::MDNode *N = StructMetadataCache[Ty])
diff --git a/clang/test/CodeGen/tbaa-struct-relaxed-aliasing-with-tsan.cpp b/clang/test/CodeGen/tbaa-struct-relaxed-aliasing-with-tsan.cpp
index 931ff2476cd1..ce613b9d6b23 100644
--- a/clang/test/CodeGen/tbaa-struct-relaxed-aliasing-with-tsan.cpp
+++ b/clang/test/CodeGen/tbaa-struct-relaxed-aliasing-with-tsan.cpp
@@ -5,9 +5,7 @@
 // RUN:     FileCheck %s
 //
 // Check that we do not create tbaa for instructions generated for copies.
-// FIXME: !tbaa.struct is generated with null node as tag.
 
-// CHECK: !tbaa.struct
 // CHECK-NOT: !tbaa
 
 struct A {
-- 
GitLab


From 8d85cd3a59f65315bbadaf11c4b3f107238456db Mon Sep 17 00:00:00 2001
From: Louis Dionne 
Date: Fri, 8 Mar 2024 15:47:43 -0500
Subject: [PATCH 0085/1674] [libc++] Don't build against libcxxrt by default on
 FreeBSD (#84484)

The libc++ CMake build aims to provide a general-purpose configuration
that matches the canonical LLVM setup (libc++ / libc++abi / libunwind),
not the configuration used for compatibility on any given system. For
these "compatibility" configurations, we use CMake caches like
Apple.cmake and FreeBSD.cmake.

Defaulting to libcxxrt on FreeBSD makes it look as though we're trying
to build the system compatible configuration on FreeBSD, which isn't
really correct and causes confusion more than anything else.

Fixes #84476
---
 libcxx/CMakeLists.txt | 2 --
 1 file changed, 2 deletions(-)

diff --git a/libcxx/CMakeLists.txt b/libcxx/CMakeLists.txt
index d392e95077ac..e565c47c7668 100644
--- a/libcxx/CMakeLists.txt
+++ b/libcxx/CMakeLists.txt
@@ -229,8 +229,6 @@ option(LIBCXX_USE_COMPILER_RT "Use compiler-rt instead of libgcc" OFF)
 # ABI Library options ---------------------------------------------------------
 if (MSVC)
   set(LIBCXX_DEFAULT_ABI_LIBRARY "vcruntime")
-elseif (${CMAKE_SYSTEM_NAME} MATCHES "FreeBSD")
-  set(LIBCXX_DEFAULT_ABI_LIBRARY "libcxxrt")
 else()
   set(LIBCXX_DEFAULT_ABI_LIBRARY "libcxxabi")
 endif()
-- 
GitLab


From 29762e372294e5b591684f57979f33c1c48bbfba Mon Sep 17 00:00:00 2001
From: Joseph Huber 
Date: Fri, 8 Mar 2024 15:02:19 -0600
Subject: [PATCH 0086/1674]  [libc][NFCI] Remove lane size template argument on
 RPC server  (#84557)

Summary:
We previously changed the data layout for the RPC buffer to make it lane
size agnostic. I put off changing the size for the server case to make
the patch smaller. This patch simply reorganizes code by making the lane
size an argument to the port rather than a templtae size. Heavily
simplifies a lot of code, no more `std::variant`.
---
 libc/src/__support/RPC/rpc.h         |  23 +-
 libc/utils/gpu/server/rpc_server.cpp | 484 ++++++++++++---------------
 2 files changed, 234 insertions(+), 273 deletions(-)

diff --git a/libc/src/__support/RPC/rpc.h b/libc/src/__support/RPC/rpc.h
index 5ed39ae0d7f7..5dcae518bb6f 100644
--- a/libc/src/__support/RPC/rpc.h
+++ b/libc/src/__support/RPC/rpc.h
@@ -310,7 +310,7 @@ private:
   LIBC_INLINE Port &operator=(Port &&) = default;
 
   friend struct Client;
-  template  friend struct Server;
+  friend struct Server;
   friend class cpp::optional>;
 
 public:
@@ -369,7 +369,7 @@ static_assert(cpp::is_trivially_copyable::value &&
               "The client is not trivially copyable from the server");
 
 /// The RPC server used to respond to the client.
-template  struct Server {
+struct Server {
   LIBC_INLINE Server() = default;
   LIBC_INLINE Server(const Server &) = delete;
   LIBC_INLINE Server &operator=(const Server &) = delete;
@@ -379,10 +379,12 @@ template  struct Server {
       : process(port_count, buffer) {}
 
   using Port = rpc::Port;
-  LIBC_INLINE cpp::optional try_open(uint32_t start = 0);
-  LIBC_INLINE Port open();
+  LIBC_INLINE cpp::optional try_open(uint32_t lane_size,
+                                           uint32_t start = 0);
+  LIBC_INLINE Port open(uint32_t lane_size);
 
-  LIBC_INLINE static uint64_t allocation_size(uint32_t port_count) {
+  LIBC_INLINE static uint64_t allocation_size(uint32_t lane_size,
+                                              uint32_t port_count) {
     return Process::allocation_size(port_count, lane_size);
   }
 
@@ -556,10 +558,8 @@ template 
 
 /// Attempts to open a port to use as the server. The server can only open a
 /// port if it has a pending receive operation
-template 
-[[clang::convergent]] LIBC_INLINE
-    cpp::optional::Port>
-    Server::try_open(uint32_t start) {
+[[clang::convergent]] LIBC_INLINE cpp::optional
+Server::try_open(uint32_t lane_size, uint32_t start) {
   // Perform a naive linear scan for a port that has a pending request.
   for (uint32_t index = start; index < process.port_count; ++index) {
     uint64_t lane_mask = gpu::get_lane_mask();
@@ -588,10 +588,9 @@ template 
   return cpp::nullopt;
 }
 
-template 
-LIBC_INLINE typename Server::Port Server::open() {
+LIBC_INLINE Server::Port Server::open(uint32_t lane_size) {
   for (;;) {
-    if (cpp::optional p = try_open())
+    if (cpp::optional p = try_open(lane_size))
       return cpp::move(p.value());
     sleep_briefly();
   }
diff --git a/libc/utils/gpu/server/rpc_server.cpp b/libc/utils/gpu/server/rpc_server.cpp
index 707807a5cbaf..2c8186c13ffa 100644
--- a/libc/utils/gpu/server/rpc_server.cpp
+++ b/libc/utils/gpu/server/rpc_server.cpp
@@ -27,228 +27,217 @@ static_assert(sizeof(rpc_buffer_t) == sizeof(rpc::Buffer),
 static_assert(RPC_MAXIMUM_PORT_COUNT == rpc::MAX_PORT_COUNT,
               "Incorrect maximum port count");
 
-// The client needs to support different lane sizes for the SIMT model. Because
-// of this we need to select between the possible sizes that the client can use.
-struct Server {
-  template 
-  Server(std::unique_ptr> &&server)
-      : server(std::move(server)) {}
-
-  rpc_status_t handle_server(
-      const std::unordered_map &callbacks,
-      const std::unordered_map &callback_data,
-      uint32_t &index) {
-    rpc_status_t ret = RPC_STATUS_SUCCESS;
-    std::visit(
-        [&](auto &server) {
-          ret = handle_server(*server, callbacks, callback_data, index);
-        },
-        server);
-    return ret;
-  }
-
-private:
-  template 
-  rpc_status_t handle_server(
-      rpc::Server &server,
-      const std::unordered_map &callbacks,
-      const std::unordered_map &callback_data,
-      uint32_t &index) {
-    auto port = server.try_open(index);
-    if (!port)
-      return RPC_STATUS_SUCCESS;
-
-    switch (port->get_opcode()) {
-    case RPC_WRITE_TO_STREAM:
-    case RPC_WRITE_TO_STDERR:
-    case RPC_WRITE_TO_STDOUT:
-    case RPC_WRITE_TO_STDOUT_NEWLINE: {
-      uint64_t sizes[lane_size] = {0};
-      void *strs[lane_size] = {nullptr};
-      FILE *files[lane_size] = {nullptr};
-      if (port->get_opcode() == RPC_WRITE_TO_STREAM) {
-        port->recv([&](rpc::Buffer *buffer, uint32_t id) {
-          files[id] = reinterpret_cast(buffer->data[0]);
-        });
-      } else if (port->get_opcode() == RPC_WRITE_TO_STDERR) {
-        std::fill(files, files + lane_size, stderr);
-      } else {
-        std::fill(files, files + lane_size, stdout);
-      }
-
-      port->recv_n(strs, sizes, [&](uint64_t size) { return new char[size]; });
-      port->send([&](rpc::Buffer *buffer, uint32_t id) {
-        flockfile(files[id]);
-        buffer->data[0] = fwrite_unlocked(strs[id], 1, sizes[id], files[id]);
-        if (port->get_opcode() == RPC_WRITE_TO_STDOUT_NEWLINE &&
-            buffer->data[0] == sizes[id])
-          buffer->data[0] += fwrite_unlocked("\n", 1, 1, files[id]);
-        funlockfile(files[id]);
-        delete[] reinterpret_cast(strs[id]);
-      });
-      break;
-    }
-    case RPC_READ_FROM_STREAM: {
-      uint64_t sizes[lane_size] = {0};
-      void *data[lane_size] = {nullptr};
-      port->recv([&](rpc::Buffer *buffer, uint32_t id) {
-        data[id] = new char[buffer->data[0]];
-        sizes[id] = fread(data[id], 1, buffer->data[0],
-                          file::to_stream(buffer->data[1]));
-      });
-      port->send_n(data, sizes);
-      port->send([&](rpc::Buffer *buffer, uint32_t id) {
-        delete[] reinterpret_cast(data[id]);
-        std::memcpy(buffer->data, &sizes[id], sizeof(uint64_t));
-      });
-      break;
-    }
-    case RPC_READ_FGETS: {
-      uint64_t sizes[lane_size] = {0};
-      void *data[lane_size] = {nullptr};
-      port->recv([&](rpc::Buffer *buffer, uint32_t id) {
-        data[id] = new char[buffer->data[0]];
-        const char *str =
-            fgets(reinterpret_cast(data[id]), buffer->data[0],
-                  file::to_stream(buffer->data[1]));
-        sizes[id] = !str ? 0 : std::strlen(str) + 1;
-      });
-      port->send_n(data, sizes);
-      for (uint32_t id = 0; id < lane_size; ++id)
-        if (data[id])
-          delete[] reinterpret_cast(data[id]);
-      break;
-    }
-    case RPC_OPEN_FILE: {
-      uint64_t sizes[lane_size] = {0};
-      void *paths[lane_size] = {nullptr};
-      port->recv_n(paths, sizes, [&](uint64_t size) { return new char[size]; });
-      port->recv_and_send([&](rpc::Buffer *buffer, uint32_t id) {
-        FILE *file = fopen(reinterpret_cast(paths[id]),
-                           reinterpret_cast(buffer->data));
-        buffer->data[0] = reinterpret_cast(file);
-      });
-      break;
-    }
-    case RPC_CLOSE_FILE: {
-      port->recv_and_send([&](rpc::Buffer *buffer, uint32_t id) {
-        FILE *file = reinterpret_cast(buffer->data[0]);
-        buffer->data[0] = fclose(file);
-      });
-      break;
-    }
-    case RPC_EXIT: {
-      // Send a response to the client to signal that we are ready to exit.
-      port->recv_and_send([](rpc::Buffer *) {});
-      port->recv([](rpc::Buffer *buffer) {
-        int status = 0;
-        std::memcpy(&status, buffer->data, sizeof(int));
-        exit(status);
-      });
-      break;
-    }
-    case RPC_ABORT: {
-      // Send a response to the client to signal that we are ready to abort.
-      port->recv_and_send([](rpc::Buffer *) {});
-      port->recv([](rpc::Buffer *) {});
-      abort();
-      break;
-    }
-    case RPC_HOST_CALL: {
-      uint64_t sizes[lane_size] = {0};
-      void *args[lane_size] = {nullptr};
-      port->recv_n(args, sizes, [&](uint64_t size) { return new char[size]; });
+template 
+rpc_status_t handle_server_impl(
+    rpc::Server &server,
+    const std::unordered_map &callbacks,
+    const std::unordered_map &callback_data,
+    uint32_t &index) {
+  auto port = server.try_open(lane_size, index);
+  if (!port)
+    return RPC_STATUS_SUCCESS;
+
+  switch (port->get_opcode()) {
+  case RPC_WRITE_TO_STREAM:
+  case RPC_WRITE_TO_STDERR:
+  case RPC_WRITE_TO_STDOUT:
+  case RPC_WRITE_TO_STDOUT_NEWLINE: {
+    uint64_t sizes[lane_size] = {0};
+    void *strs[lane_size] = {nullptr};
+    FILE *files[lane_size] = {nullptr};
+    if (port->get_opcode() == RPC_WRITE_TO_STREAM) {
       port->recv([&](rpc::Buffer *buffer, uint32_t id) {
-        reinterpret_cast(buffer->data[0])(args[id]);
-      });
-      port->send([&](rpc::Buffer *, uint32_t id) {
-        delete[] reinterpret_cast(args[id]);
-      });
-      break;
-    }
-    case RPC_FEOF: {
-      port->recv_and_send([](rpc::Buffer *buffer) {
-        buffer->data[0] = feof(file::to_stream(buffer->data[0]));
-      });
-      break;
-    }
-    case RPC_FERROR: {
-      port->recv_and_send([](rpc::Buffer *buffer) {
-        buffer->data[0] = ferror(file::to_stream(buffer->data[0]));
-      });
-      break;
-    }
-    case RPC_CLEARERR: {
-      port->recv_and_send([](rpc::Buffer *buffer) {
-        clearerr(file::to_stream(buffer->data[0]));
-      });
-      break;
-    }
-    case RPC_FSEEK: {
-      port->recv_and_send([](rpc::Buffer *buffer) {
-        buffer->data[0] = fseek(file::to_stream(buffer->data[0]),
-                                static_cast(buffer->data[1]),
-                                static_cast(buffer->data[2]));
-      });
-      break;
-    }
-    case RPC_FTELL: {
-      port->recv_and_send([](rpc::Buffer *buffer) {
-        buffer->data[0] = ftell(file::to_stream(buffer->data[0]));
-      });
-      break;
-    }
-    case RPC_FFLUSH: {
-      port->recv_and_send([](rpc::Buffer *buffer) {
-        buffer->data[0] = fflush(file::to_stream(buffer->data[0]));
+        files[id] = reinterpret_cast(buffer->data[0]);
       });
-      break;
-    }
-    case RPC_UNGETC: {
-      port->recv_and_send([](rpc::Buffer *buffer) {
-        buffer->data[0] = ungetc(static_cast(buffer->data[0]),
-                                 file::to_stream(buffer->data[1]));
-      });
-      break;
-    }
-    case RPC_NOOP: {
-      port->recv([](rpc::Buffer *) {});
-      break;
-    }
-    default: {
-      auto handler =
-          callbacks.find(static_cast(port->get_opcode()));
-
-      // We error out on an unhandled opcode.
-      if (handler == callbacks.end())
-        return RPC_STATUS_UNHANDLED_OPCODE;
-
-      // Invoke the registered callback with a reference to the port.
-      void *data =
-          callback_data.at(static_cast(port->get_opcode()));
-      rpc_port_t port_ref{reinterpret_cast(&*port), lane_size};
-      (handler->second)(port_ref, data);
-    }
+    } else if (port->get_opcode() == RPC_WRITE_TO_STDERR) {
+      std::fill(files, files + lane_size, stderr);
+    } else {
+      std::fill(files, files + lane_size, stdout);
     }
 
-    // Increment the index so we start the scan after this port.
-    index = port->get_index() + 1;
-    port->close();
-    return RPC_STATUS_CONTINUE;
+    port->recv_n(strs, sizes, [&](uint64_t size) { return new char[size]; });
+    port->send([&](rpc::Buffer *buffer, uint32_t id) {
+      flockfile(files[id]);
+      buffer->data[0] = fwrite_unlocked(strs[id], 1, sizes[id], files[id]);
+      if (port->get_opcode() == RPC_WRITE_TO_STDOUT_NEWLINE &&
+          buffer->data[0] == sizes[id])
+        buffer->data[0] += fwrite_unlocked("\n", 1, 1, files[id]);
+      funlockfile(files[id]);
+      delete[] reinterpret_cast(strs[id]);
+    });
+    break;
+  }
+  case RPC_READ_FROM_STREAM: {
+    uint64_t sizes[lane_size] = {0};
+    void *data[lane_size] = {nullptr};
+    port->recv([&](rpc::Buffer *buffer, uint32_t id) {
+      data[id] = new char[buffer->data[0]];
+      sizes[id] =
+          fread(data[id], 1, buffer->data[0], file::to_stream(buffer->data[1]));
+    });
+    port->send_n(data, sizes);
+    port->send([&](rpc::Buffer *buffer, uint32_t id) {
+      delete[] reinterpret_cast(data[id]);
+      std::memcpy(buffer->data, &sizes[id], sizeof(uint64_t));
+    });
+    break;
+  }
+  case RPC_READ_FGETS: {
+    uint64_t sizes[lane_size] = {0};
+    void *data[lane_size] = {nullptr};
+    port->recv([&](rpc::Buffer *buffer, uint32_t id) {
+      data[id] = new char[buffer->data[0]];
+      const char *str =
+          fgets(reinterpret_cast(data[id]), buffer->data[0],
+                file::to_stream(buffer->data[1]));
+      sizes[id] = !str ? 0 : std::strlen(str) + 1;
+    });
+    port->send_n(data, sizes);
+    for (uint32_t id = 0; id < lane_size; ++id)
+      if (data[id])
+        delete[] reinterpret_cast(data[id]);
+    break;
+  }
+  case RPC_OPEN_FILE: {
+    uint64_t sizes[lane_size] = {0};
+    void *paths[lane_size] = {nullptr};
+    port->recv_n(paths, sizes, [&](uint64_t size) { return new char[size]; });
+    port->recv_and_send([&](rpc::Buffer *buffer, uint32_t id) {
+      FILE *file = fopen(reinterpret_cast(paths[id]),
+                         reinterpret_cast(buffer->data));
+      buffer->data[0] = reinterpret_cast(file);
+    });
+    break;
+  }
+  case RPC_CLOSE_FILE: {
+    port->recv_and_send([&](rpc::Buffer *buffer, uint32_t id) {
+      FILE *file = reinterpret_cast(buffer->data[0]);
+      buffer->data[0] = fclose(file);
+    });
+    break;
+  }
+  case RPC_EXIT: {
+    // Send a response to the client to signal that we are ready to exit.
+    port->recv_and_send([](rpc::Buffer *) {});
+    port->recv([](rpc::Buffer *buffer) {
+      int status = 0;
+      std::memcpy(&status, buffer->data, sizeof(int));
+      exit(status);
+    });
+    break;
+  }
+  case RPC_ABORT: {
+    // Send a response to the client to signal that we are ready to abort.
+    port->recv_and_send([](rpc::Buffer *) {});
+    port->recv([](rpc::Buffer *) {});
+    abort();
+    break;
+  }
+  case RPC_HOST_CALL: {
+    uint64_t sizes[lane_size] = {0};
+    void *args[lane_size] = {nullptr};
+    port->recv_n(args, sizes, [&](uint64_t size) { return new char[size]; });
+    port->recv([&](rpc::Buffer *buffer, uint32_t id) {
+      reinterpret_cast(buffer->data[0])(args[id]);
+    });
+    port->send([&](rpc::Buffer *, uint32_t id) {
+      delete[] reinterpret_cast(args[id]);
+    });
+    break;
+  }
+  case RPC_FEOF: {
+    port->recv_and_send([](rpc::Buffer *buffer) {
+      buffer->data[0] = feof(file::to_stream(buffer->data[0]));
+    });
+    break;
+  }
+  case RPC_FERROR: {
+    port->recv_and_send([](rpc::Buffer *buffer) {
+      buffer->data[0] = ferror(file::to_stream(buffer->data[0]));
+    });
+    break;
+  }
+  case RPC_CLEARERR: {
+    port->recv_and_send([](rpc::Buffer *buffer) {
+      clearerr(file::to_stream(buffer->data[0]));
+    });
+    break;
+  }
+  case RPC_FSEEK: {
+    port->recv_and_send([](rpc::Buffer *buffer) {
+      buffer->data[0] = fseek(file::to_stream(buffer->data[0]),
+                              static_cast(buffer->data[1]),
+                              static_cast(buffer->data[2]));
+    });
+    break;
+  }
+  case RPC_FTELL: {
+    port->recv_and_send([](rpc::Buffer *buffer) {
+      buffer->data[0] = ftell(file::to_stream(buffer->data[0]));
+    });
+    break;
+  }
+  case RPC_FFLUSH: {
+    port->recv_and_send([](rpc::Buffer *buffer) {
+      buffer->data[0] = fflush(file::to_stream(buffer->data[0]));
+    });
+    break;
+  }
+  case RPC_UNGETC: {
+    port->recv_and_send([](rpc::Buffer *buffer) {
+      buffer->data[0] = ungetc(static_cast(buffer->data[0]),
+                               file::to_stream(buffer->data[1]));
+    });
+    break;
+  }
+  case RPC_NOOP: {
+    port->recv([](rpc::Buffer *) {});
+    break;
+  }
+  default: {
+    auto handler =
+        callbacks.find(static_cast(port->get_opcode()));
+
+    // We error out on an unhandled opcode.
+    if (handler == callbacks.end())
+      return RPC_STATUS_UNHANDLED_OPCODE;
+
+    // Invoke the registered callback with a reference to the port.
+    void *data =
+        callback_data.at(static_cast(port->get_opcode()));
+    rpc_port_t port_ref{reinterpret_cast(&*port), lane_size};
+    (handler->second)(port_ref, data);
+  }
   }
 
-  std::variant>,
-               std::unique_ptr>,
-               std::unique_ptr>>
-      server;
-};
+  // Increment the index so we start the scan after this port.
+  index = port->get_index() + 1;
+  port->close();
+
+  return RPC_STATUS_CONTINUE;
+}
 
 struct Device {
-  template 
-  Device(uint32_t num_ports, void *buffer, std::unique_ptr &&server)
-      : buffer(buffer), server(std::move(server)), client(num_ports, buffer) {}
+  Device(uint32_t lane_size, uint32_t num_ports, void *buffer)
+      : lane_size(lane_size), buffer(buffer), server(num_ports, buffer),
+        client(num_ports, buffer) {}
+
+  rpc_status_t handle_server(uint32_t &index) {
+    switch (lane_size) {
+    case 1:
+      return handle_server_impl<1>(server, callbacks, callback_data, index);
+    case 32:
+      return handle_server_impl<32>(server, callbacks, callback_data, index);
+    case 64:
+      return handle_server_impl<64>(server, callbacks, callback_data, index);
+    default:
+      return RPC_STATUS_INVALID_LANE_SIZE;
+    }
+  }
+
+  uint32_t lane_size;
   void *buffer;
-  Server server;
+  rpc::Server server;
   rpc::Client client;
   std::unordered_map callbacks;
   std::unordered_map callback_data;
@@ -287,24 +276,6 @@ rpc_status_t rpc_shutdown(void) {
   return RPC_STATUS_SUCCESS;
 }
 
-template 
-rpc_status_t server_init_impl(uint32_t device_id, uint64_t num_ports,
-                              rpc_alloc_ty alloc, void *data) {
-  uint64_t size = rpc::Server::allocation_size(num_ports);
-  void *buffer = alloc(size, data);
-
-  if (!buffer)
-    return RPC_STATUS_ERROR;
-
-  state->devices[device_id] = std::make_unique(
-      num_ports, buffer,
-      std::make_unique>(num_ports, buffer));
-  if (!state->devices[device_id])
-    return RPC_STATUS_ERROR;
-
-  return RPC_STATUS_SUCCESS;
-}
-
 rpc_status_t rpc_server_init(uint32_t device_id, uint64_t num_ports,
                              uint32_t lane_size, rpc_alloc_ty alloc,
                              void *data) {
@@ -312,28 +283,20 @@ rpc_status_t rpc_server_init(uint32_t device_id, uint64_t num_ports,
     return RPC_STATUS_NOT_INITIALIZED;
   if (device_id >= state->num_devices)
     return RPC_STATUS_OUT_OF_RANGE;
+  if (lane_size != 1 && lane_size != 32 && lane_size != 64)
+    return RPC_STATUS_INVALID_LANE_SIZE;
 
   if (!state->devices[device_id]) {
-    switch (lane_size) {
-    case 1:
-      if (rpc_status_t err =
-              server_init_impl<1>(device_id, num_ports, alloc, data))
-        return err;
-      break;
-    case 32: {
-      if (rpc_status_t err =
-              server_init_impl<32>(device_id, num_ports, alloc, data))
-        return err;
-      break;
-    }
-    case 64:
-      if (rpc_status_t err =
-              server_init_impl<64>(device_id, num_ports, alloc, data))
-        return err;
-      break;
-    default:
-      return RPC_STATUS_INVALID_LANE_SIZE;
-    }
+    uint64_t size = rpc::Server::allocation_size(lane_size, num_ports);
+    void *buffer = alloc(size, data);
+
+    if (!buffer)
+      return RPC_STATUS_ERROR;
+
+    state->devices[device_id] =
+        std::make_unique(lane_size, num_ports, buffer);
+    if (!state->devices[device_id])
+      return RPC_STATUS_ERROR;
   }
 
   return RPC_STATUS_SUCCESS;
@@ -365,9 +328,8 @@ rpc_status_t rpc_handle_server(uint32_t device_id) {
 
   uint32_t index = 0;
   for (;;) {
-    auto &device = *state->devices[device_id];
-    rpc_status_t status = device.server.handle_server(
-        device.callbacks, device.callback_data, index);
+    Device &device = *state->devices[device_id];
+    rpc_status_t status = device.handle_server(index);
     if (status != RPC_STATUS_CONTINUE)
       return status;
   }
@@ -396,26 +358,26 @@ const void *rpc_get_client_buffer(uint32_t device_id) {
 
 uint64_t rpc_get_client_size() { return sizeof(rpc::Client); }
 
-using ServerPort = std::variant::Port *>;
+using ServerPort = std::variant;
 
 ServerPort get_port(rpc_port_t ref) {
-  return reinterpret_cast::Port *>(ref.handle);
+  return reinterpret_cast(ref.handle);
 }
 
 void rpc_send(rpc_port_t ref, rpc_port_callback_ty callback, void *data) {
-  auto port = reinterpret_cast::Port *>(ref.handle);
+  auto port = reinterpret_cast(ref.handle);
   port->send([=](rpc::Buffer *buffer) {
     callback(reinterpret_cast(buffer), data);
   });
 }
 
 void rpc_send_n(rpc_port_t ref, const void *const *src, uint64_t *size) {
-  auto port = reinterpret_cast::Port *>(ref.handle);
+  auto port = reinterpret_cast(ref.handle);
   port->send_n(src, size);
 }
 
 void rpc_recv(rpc_port_t ref, rpc_port_callback_ty callback, void *data) {
-  auto port = reinterpret_cast::Port *>(ref.handle);
+  auto port = reinterpret_cast(ref.handle);
   port->recv([=](rpc::Buffer *buffer) {
     callback(reinterpret_cast(buffer), data);
   });
@@ -423,14 +385,14 @@ void rpc_recv(rpc_port_t ref, rpc_port_callback_ty callback, void *data) {
 
 void rpc_recv_n(rpc_port_t ref, void **dst, uint64_t *size, rpc_alloc_ty alloc,
                 void *data) {
-  auto port = reinterpret_cast::Port *>(ref.handle);
+  auto port = reinterpret_cast(ref.handle);
   auto alloc_fn = [=](uint64_t size) { return alloc(size, data); };
   port->recv_n(dst, size, alloc_fn);
 }
 
 void rpc_recv_and_send(rpc_port_t ref, rpc_port_callback_ty callback,
                        void *data) {
-  auto port = reinterpret_cast::Port *>(ref.handle);
+  auto port = reinterpret_cast(ref.handle);
   port->recv_and_send([=](rpc::Buffer *buffer) {
     callback(reinterpret_cast(buffer), data);
   });
-- 
GitLab


From 03c6c73b022415378f2efbc13398bdb2c22d3180 Mon Sep 17 00:00:00 2001
From: Florian Mayer 
Date: Fri, 8 Mar 2024 13:28:18 -0800
Subject: [PATCH 0087/1674] [NFC] [hwasan] remove unused method

---
 llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
index 6bae679e11be..422406e46bdb 100644
--- a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
@@ -316,7 +316,6 @@ private:
     Value *PtrTag = nullptr;
     Value *MemTag = nullptr;
   };
-  void setSSI(const StackSafetyGlobalInfo *S) { SSI = S; }
 
   void initializeModule();
   void createHwasanCtorComdat();
-- 
GitLab


From 3ceebcb48135612b8d90500f65693398f7e526cd Mon Sep 17 00:00:00 2001
From: Peiming Liu 
Date: Fri, 8 Mar 2024 13:30:34 -0800
Subject: [PATCH 0088/1674] =?UTF-8?q?[mlir]=20tentative=20fix=20for=20"'GT?=
 =?UTF-8?q?EST=5FNO=5FLLVM=5FSUPPORT'=20is=20not=20defined"=20war=E2=80=A6?=
 =?UTF-8?q?=20(#84539)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…ning when unittest is enabled
---
 mlir/unittests/CMakeLists.txt | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/mlir/unittests/CMakeLists.txt b/mlir/unittests/CMakeLists.txt
index d0e222091c9f..6fad249a0b2f 100644
--- a/mlir/unittests/CMakeLists.txt
+++ b/mlir/unittests/CMakeLists.txt
@@ -1,5 +1,8 @@
 set_target_properties(MLIRUnitTests PROPERTIES FOLDER "MLIR Tests")
 
+# To silence warning caused by Wundef.
+add_definitions(-DGTEST_NO_LLVM_SUPPORT=0)
+
 function(add_mlir_unittest test_dirname)
   add_unittest(MLIRUnitTests ${test_dirname} ${ARGN})
 endfunction()
-- 
GitLab


From 1cf428a05a62711200d37a1b12722ca16c33a6ea Mon Sep 17 00:00:00 2001
From: Florian Mayer 
Date: Fri, 8 Mar 2024 13:46:08 -0800
Subject: [PATCH 0089/1674] Skip MemtagBasicDeathTest#Unsupported when running
 with HWASan (#84243)

Tested in AOSP.
---
 compiler-rt/lib/scudo/standalone/tests/memtag_test.cpp | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/compiler-rt/lib/scudo/standalone/tests/memtag_test.cpp b/compiler-rt/lib/scudo/standalone/tests/memtag_test.cpp
index fd277f962a9a..37a18858e67c 100644
--- a/compiler-rt/lib/scudo/standalone/tests/memtag_test.cpp
+++ b/compiler-rt/lib/scudo/standalone/tests/memtag_test.cpp
@@ -12,12 +12,17 @@
 #include "platform.h"
 #include "tests/scudo_unit_test.h"
 
+extern "C" void __hwasan_init() __attribute__((weak));
+
 #if SCUDO_LINUX
 namespace scudo {
 
 TEST(MemtagBasicDeathTest, Unsupported) {
   if (archSupportsMemoryTagging())
     GTEST_SKIP();
+  // Skip when running with HWASan.
+  if (&__hwasan_init != 0)
+    GTEST_SKIP();
 
   EXPECT_DEATH(archMemoryTagGranuleSize(), "not supported");
   EXPECT_DEATH(untagPointer((uptr)0), "not supported");
-- 
GitLab


From e93489c434da5fd93ed54c3c41b0c06f718bb8ff Mon Sep 17 00:00:00 2001
From: Jakub Kuderski 
Date: Fri, 8 Mar 2024 17:43:56 -0500
Subject: [PATCH 0090/1674] [mlir] Add missing build deps for Mesh transforms
 (#84581)

---
 mlir/lib/Dialect/Mesh/Transforms/CMakeLists.txt   | 2 ++
 utils/bazel/llvm-project-overlay/mlir/BUILD.bazel | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/mlir/lib/Dialect/Mesh/Transforms/CMakeLists.txt b/mlir/lib/Dialect/Mesh/Transforms/CMakeLists.txt
index 28af82044007..212ea6d6948b 100644
--- a/mlir/lib/Dialect/Mesh/Transforms/CMakeLists.txt
+++ b/mlir/lib/Dialect/Mesh/Transforms/CMakeLists.txt
@@ -17,7 +17,9 @@ add_mlir_dialect_library(MLIRMeshTransforms
   MLIRArithDialect
   MLIRArithUtils
   MLIRControlFlowDialect
+  MLIRControlFlowInterfaces
   MLIRFuncDialect
+  MLIRFunctionInterfaces
   MLIRIR
   MLIRMeshDialect
   MLIRPass
diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
index cadcbcc5a092..3be8f34ae0f4 100644
--- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
@@ -3410,8 +3410,10 @@ cc_library(
         ":ArithDialect",
         ":ArithUtils",
         ":ControlFlowDialect",
+        ":ControlFlowInterfaces",
         ":DialectUtils",
         ":FuncDialect",
+        ":FunctionInterfaces",
         ":IR",
         ":MeshDialect",
         ":MeshShardingInterface",
-- 
GitLab


From cb6ff746e0c7b9218b6f5c11db44162cacd623a4 Mon Sep 17 00:00:00 2001
From: Kojo Acquah 
Date: Fri, 8 Mar 2024 22:50:13 +0000
Subject: [PATCH 0091/1674] [mlir][ArmNeon] Implements LowerVectorToArmNeon
 Pattern for SMMLA (#81895)

This patch adds a the `LowerVectorToArmNeonPattern` patterns to the
ArmNeon.

This pattern inspects `vector.contract` ops that can be 1-1 mapped to an
`arm.neon.smmla` intrinsic. The contract ops must be separated into
tiles who's inputs must fit that of a single smmla op (`2x8xi32` inputs
and `2x2xi32` output). The `vector.contract` inputs must be sign
extended from narrow types (<=i8) to be converted. If all conditions are
met, an smmla op is inserted with additional `vector.shape_casts` to
handle linearizing the input and output dimension.
---
 .../include/mlir/Dialect/ArmNeon/Transforms.h |  21 +++
 .../Conversion/VectorToLLVM/CMakeLists.txt    |   1 +
 mlir/lib/Dialect/ArmNeon/CMakeLists.txt       |  15 +-
 mlir/lib/Dialect/ArmNeon/IR/CMakeLists.txt    |  13 ++
 .../Dialect/ArmNeon/Transforms/CMakeLists.txt |  14 ++
 .../LowerContractionToSMMLAPattern.cpp        | 146 ++++++++++++++++++
 .../Dialect/ArmNeon/lower-to-arm-neon.mlir    |  42 +++++
 mlir/test/lib/Dialect/ArmNeon/CMakeLists.txt  |  13 ++
 .../Dialect/ArmNeon/TestLowerToArmNeon.cpp    |  61 ++++++++
 mlir/test/lib/Dialect/CMakeLists.txt          |   1 +
 mlir/tools/mlir-opt/CMakeLists.txt            |   1 +
 mlir/tools/mlir-opt/mlir-opt.cpp              |   2 +
 .../llvm-project-overlay/mlir/BUILD.bazel     |  21 +++
 13 files changed, 338 insertions(+), 13 deletions(-)
 create mode 100644 mlir/include/mlir/Dialect/ArmNeon/Transforms.h
 create mode 100644 mlir/lib/Dialect/ArmNeon/IR/CMakeLists.txt
 create mode 100644 mlir/lib/Dialect/ArmNeon/Transforms/CMakeLists.txt
 create mode 100644 mlir/lib/Dialect/ArmNeon/Transforms/LowerContractionToSMMLAPattern.cpp
 create mode 100644 mlir/test/Dialect/ArmNeon/lower-to-arm-neon.mlir
 create mode 100644 mlir/test/lib/Dialect/ArmNeon/CMakeLists.txt
 create mode 100644 mlir/test/lib/Dialect/ArmNeon/TestLowerToArmNeon.cpp

diff --git a/mlir/include/mlir/Dialect/ArmNeon/Transforms.h b/mlir/include/mlir/Dialect/ArmNeon/Transforms.h
new file mode 100644
index 000000000000..49cad22defec
--- /dev/null
+++ b/mlir/include/mlir/Dialect/ArmNeon/Transforms.h
@@ -0,0 +1,21 @@
+//===- Transforms.h - ArmNeon Transformation Entrypoints --------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_ARMNEON_TRANSFORMS_H
+#define MLIR_DIALECT_ARMNEON_TRANSFORMS_H
+
+namespace mlir {
+
+namespace arm_neon {
+void populateLowerContractionToSMMLAPatternPatterns(
+    RewritePatternSet &patterns);
+} // namespace arm_neon
+
+} // namespace mlir
+
+#endif // MLIR_DIALECT_ARMNEON_TRANSFORMS_H
diff --git a/mlir/lib/Conversion/VectorToLLVM/CMakeLists.txt b/mlir/lib/Conversion/VectorToLLVM/CMakeLists.txt
index 5fbb50f62395..a0fce139f274 100644
--- a/mlir/lib/Conversion/VectorToLLVM/CMakeLists.txt
+++ b/mlir/lib/Conversion/VectorToLLVM/CMakeLists.txt
@@ -34,6 +34,7 @@ add_mlir_conversion_library(MLIRVectorToLLVMPass
   MLIRVectorToLLVM
 
   MLIRArmNeonDialect
+  MLIRArmNeonTransforms
   MLIRArmSMEDialect
   MLIRArmSMETransforms
   MLIRArmSVEDialect
diff --git a/mlir/lib/Dialect/ArmNeon/CMakeLists.txt b/mlir/lib/Dialect/ArmNeon/CMakeLists.txt
index 060b6df1b334..9f57627c321f 100644
--- a/mlir/lib/Dialect/ArmNeon/CMakeLists.txt
+++ b/mlir/lib/Dialect/ArmNeon/CMakeLists.txt
@@ -1,13 +1,2 @@
-add_mlir_dialect_library(MLIRArmNeonDialect
-  IR/ArmNeonDialect.cpp
-
-  ADDITIONAL_HEADER_DIRS
-  ${MLIR_MAIN_INCLUDE_DIR}/mlir/Dialect/ArmNeon
-
-  DEPENDS
-  MLIRArmNeonIncGen
-
-  LINK_LIBS PUBLIC
-  MLIRIR
-  MLIRSideEffectInterfaces
-  )
+add_subdirectory(IR)
+add_subdirectory(Transforms)
diff --git a/mlir/lib/Dialect/ArmNeon/IR/CMakeLists.txt b/mlir/lib/Dialect/ArmNeon/IR/CMakeLists.txt
new file mode 100644
index 000000000000..b04919a3a318
--- /dev/null
+++ b/mlir/lib/Dialect/ArmNeon/IR/CMakeLists.txt
@@ -0,0 +1,13 @@
+add_mlir_dialect_library(MLIRArmNeonDialect
+  ArmNeonDialect.cpp
+
+  ADDITIONAL_HEADER_DIRS
+  ${MLIR_MAIN_INCLUDE_DIR}/mlir/Dialect/ArmNeon
+
+  DEPENDS
+  MLIRArmNeonIncGen
+
+  LINK_LIBS PUBLIC
+  MLIRIR
+  MLIRSideEffectInterfaces
+  )
diff --git a/mlir/lib/Dialect/ArmNeon/Transforms/CMakeLists.txt b/mlir/lib/Dialect/ArmNeon/Transforms/CMakeLists.txt
new file mode 100644
index 000000000000..84fb1b0116d2
--- /dev/null
+++ b/mlir/lib/Dialect/ArmNeon/Transforms/CMakeLists.txt
@@ -0,0 +1,14 @@
+add_mlir_dialect_library(MLIRArmNeonTransforms
+  LowerContractionToSMMLAPattern.cpp
+
+  DEPENDS
+  MLIRArmNeonIncGen
+
+  LINK_LIBS PUBLIC
+  MLIRArmNeonDialect
+  MLIRFuncDialect
+  MLIRVectorDialect
+  MLIRIR
+  MLIRLLVMCommonConversion
+  MLIRLLVMDialect
+  )
diff --git a/mlir/lib/Dialect/ArmNeon/Transforms/LowerContractionToSMMLAPattern.cpp b/mlir/lib/Dialect/ArmNeon/Transforms/LowerContractionToSMMLAPattern.cpp
new file mode 100644
index 000000000000..47c84708f3c3
--- /dev/null
+++ b/mlir/lib/Dialect/ArmNeon/Transforms/LowerContractionToSMMLAPattern.cpp
@@ -0,0 +1,146 @@
+//===- LowerContractionToSMMLAPattern.cpp - Contract to SMMLA ---*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements lowering patterns from vector.contract to
+// arm_neon.intr.smmla
+//
+//===---
+
+#include "mlir/Dialect/Arith/IR/Arith.h"
+#include "mlir/Dialect/ArmNeon/ArmNeonDialect.h"
+#include "mlir/Dialect/ArmNeon/Transforms.h"
+#include "mlir/Dialect/Func/IR/FuncOps.h"
+#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
+#include "mlir/Dialect/Vector/IR/VectorOps.h"
+#include "mlir/IR/PatternMatch.h"
+#include "mlir/Support/LogicalResult.h"
+#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
+
+#define DEBUG_TYPE "lower-contract-to-arm-neon"
+
+using namespace mlir;
+using namespace mlir::arm_neon;
+
+namespace {
+
+/// Return the shaped type with new element type.
+static Type matchContainerType(Type element, Type container) {
+  if (auto shapedTy = dyn_cast(container)) {
+    return shapedTy.clone(element);
+  }
+  return element;
+}
+
+/// Lowering from a single vector::contractOp directly to the arm neon smmla
+/// intrinsic. The shapes of the contract and intrinsic must match.
+class LowerContractionToSMMLAPattern
+    : public OpRewritePattern {
+public:
+  using OpRewritePattern::OpRewritePattern;
+  LogicalResult matchAndRewrite(vector::ContractionOp op,
+                                PatternRewriter &rewriter) const override {
+    Location loc = op.getLoc();
+    Value lhs = op.getLhs();
+    Value rhs = op.getRhs();
+    Value res = op.getAcc();
+
+    // Check index maps that represent M N K in contract.
+    auto indexingMaps = op.getIndexingMapsArray();
+    if (llvm::any_of(indexingMaps, [](mlir::AffineMap affineMap) {
+          return affineMap.isPermutation() || affineMap.getNumDims() != 3 ||
+                 affineMap.getNumResults() != 2;
+        })) {
+      return failure();
+    }
+
+    // Check iterator types for contract.
+    auto iteratorTypes = op.getIteratorTypesArray();
+    if (iteratorTypes.size() != 3 ||
+        iteratorTypes[0] != vector::IteratorType::parallel ||
+        iteratorTypes[1] != vector::IteratorType::parallel ||
+        iteratorTypes[2] != vector::IteratorType::reduction) {
+      return failure();
+    }
+
+    // Check the tile size by mapping the dimensions of the contract.
+    mlir::VectorType lhsType = op.getLhsType();
+    mlir::VectorType rhsType = op.getRhsType();
+    auto dimM = lhsType.getDimSize(0);
+    auto dimN = rhsType.getDimSize(0);
+    auto dimK = lhsType.getDimSize(1);
+    if (rhsType.getDimSize(1) != dimK || dimM != 2 || dimN != 2 || dimK != 8) {
+      return failure();
+    }
+
+    // Check two extsi inputs Rhs Lhs for contract.
+    arith::ExtSIOp origLhsExtOp =
+        dyn_cast_or_null(lhs.getDefiningOp());
+    arith::ExtSIOp origRhsExtOp =
+        dyn_cast_or_null(rhs.getDefiningOp());
+    if (!origLhsExtOp || !origRhsExtOp) {
+      return failure();
+    }
+
+    // Match any iX to i32 for X<8 then turn into an i8 output. Feed into
+    // following neon instruction. Check inputs for extsi are <=i8
+    Value extsiLhs;
+    Value extsiRhs;
+    if (auto lhsExtInType =
+            origLhsExtOp.getIn().getType().dyn_cast()) {
+      if (lhsExtInType.getElementTypeBitWidth() <= 8) {
+        Type targetLhsExtTy =
+            matchContainerType(rewriter.getI8Type(), lhsExtInType);
+        extsiLhs = rewriter.createOrFold(loc, targetLhsExtTy,
+                                                         origLhsExtOp.getIn());
+      }
+    }
+    if (auto rhsExtInType =
+            origRhsExtOp.getIn().getType().dyn_cast()) {
+      if (rhsExtInType.getElementTypeBitWidth() <= 8) {
+        Type targetRhsExtTy =
+            matchContainerType(rewriter.getI8Type(), rhsExtInType);
+        extsiRhs = rewriter.createOrFold(loc, targetRhsExtTy,
+                                                         origRhsExtOp.getIn());
+      }
+    }
+
+    if (!extsiLhs || !extsiRhs) {
+      return failure();
+    }
+
+    // Collapse to 1D vectors required by smmla intrinsic
+    auto collapsedInputType = VectorType::get(
+        {16}, extsiLhs.getType().cast().getElementType());
+    auto collapsedOutputType =
+        VectorType::get({4}, res.getType().cast().getElementType());
+    auto collapsedLhs = rewriter.createOrFold(
+        extsiLhs.getLoc(), collapsedInputType, extsiLhs);
+    auto collapsedRhs = rewriter.createOrFold(
+        extsiRhs.getLoc(), collapsedInputType, extsiRhs);
+    auto collapsedRes = rewriter.createOrFold(
+        res.getLoc(), collapsedOutputType, res);
+
+    // Replace the contract with a neon op
+    auto smmlaOp = rewriter.createOrFold(
+        op.getLoc(), collapsedRes.getType(), collapsedRes, collapsedLhs,
+        collapsedRhs);
+
+    // Reshape output back to 2D
+    rewriter.replaceOpWithNewOp(op, op.getResultType(),
+                                                     smmlaOp);
+    return success();
+  }
+};
+
+} // namespace
+
+void mlir::arm_neon::populateLowerContractionToSMMLAPatternPatterns(
+    RewritePatternSet &patterns) {
+  MLIRContext *context = patterns.getContext();
+  patterns.add(context, /*benefit=*/1);
+}
diff --git a/mlir/test/Dialect/ArmNeon/lower-to-arm-neon.mlir b/mlir/test/Dialect/ArmNeon/lower-to-arm-neon.mlir
new file mode 100644
index 000000000000..cba7b00ba77a
--- /dev/null
+++ b/mlir/test/Dialect/ArmNeon/lower-to-arm-neon.mlir
@@ -0,0 +1,42 @@
+// RUN: mlir-opt -test-lower-to-arm-neon -verify-diagnostics -split-input-file %s | FileCheck %s
+
+// CHECK-LABEL: test_lower_vector_arm_neon_mixed_types
+// CHECK-SAME:    %[[A0:.*]]: vector<2x8xi8>, %[[A1:.*]]: vector<2x8xi4>, %[[A2:.*]]: vector<2x2xi32>
+// CHECK-DAG: %[[D0:.*]] = arith.extsi %[[A1]] : vector<2x8xi4> to vector<2x8xi8>
+// CHECK-DAG: %[[D1:.*]] = vector.shape_cast %[[A0]] : vector<2x8xi8> to vector<16xi8>
+// CHECK-DAG: %[[D2:.*]] = vector.shape_cast %[[D0]] : vector<2x8xi8> to vector<16xi8>
+// CHECK-DAG: %[[D3:.*]] = vector.shape_cast %[[A2]] : vector<2x2xi32> to vector<4xi32>
+// CHECK-DAG: %[[D4:.*]] = arm_neon.intr.smmla %[[D3]], %[[D1]], %[[D2]] : vector<16xi8> to vector<4xi32>
+// CHECK-DAG: %[[D5:.*]] = vector.shape_cast %[[D4]] : vector<4xi32> to vector<2x2xi32>
+func.func @test_lower_vector_arm_neon_mixed_types(%lhs: vector<2x8xi8>, %rhs: vector<2x8xi4>, %acc : vector<2x2xi32>) -> vector<2x2xi32> {
+  %lhs_extsi = arith.extsi %lhs : vector<2x8xi8> to vector<2x8xi32>
+  %rhs_extsi = arith.extsi %rhs : vector<2x8xi4> to vector<2x8xi32>
+  %res = vector.contract {indexing_maps = [affine_map<(d0, d1, d2) -> (d0, d2)>, affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1)>], iterator_types = ["parallel", "parallel", "reduction"], kind = #vector.kind} %lhs_extsi, %rhs_extsi, %acc : vector<2x8xi32>, vector<2x8xi32> into vector<2x2xi32>
+  return %res : vector<2x2xi32>
+}
+
+// -----
+
+// CHECK-LABEL: test_lower_vector_arm_neon_same_types
+// CHECK-SAME:    %[[A0:.*]]: vector<2x8xi8>, %[[A1:.*]]: vector<2x8xi8>, %[[A2:.*]]: vector<2x2xi32>
+// CHECK-DAG: %[[D0:.*]] = vector.shape_cast %[[A0]] : vector<2x8xi8> to vector<16xi8>
+// CHECK-DAG: %[[D1:.*]] = vector.shape_cast %[[A1]] : vector<2x8xi8> to vector<16xi8>
+// CHECK-DAG: %[[D2:.*]] = vector.shape_cast %[[A2]] : vector<2x2xi32> to vector<4xi32>
+// CHECK-DAG: %[[D3:.*]] = arm_neon.intr.smmla %[[D2]], %[[D0]], %[[D1]] : vector<16xi8> to vector<4xi32>
+// CHECK-DAG: %[[D4:.*]] = vector.shape_cast %[[D3]] : vector<4xi32> to vector<2x2xi32>
+func.func @test_lower_vector_arm_neon_same_types(%lhs: vector<2x8xi8>, %rhs: vector<2x8xi8>, %acc : vector<2x2xi32>) -> vector<2x2xi32> {
+  %lhs_extsi = arith.extsi %lhs : vector<2x8xi8> to vector<2x8xi32>
+  %rhs_extsi = arith.extsi %rhs : vector<2x8xi8> to vector<2x8xi32>
+  %res = vector.contract {indexing_maps = [affine_map<(d0, d1, d2) -> (d0, d2)>, affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1)>], iterator_types = ["parallel", "parallel", "reduction"], kind = #vector.kind} %lhs_extsi, %rhs_extsi, %acc : vector<2x8xi32>, vector<2x8xi32> into vector<2x2xi32>
+  return %res : vector<2x2xi32>
+}
+
+// -----
+
+// CHECK-LABEL: test_lower_vector_arm_neon_without_extsi
+// CHECK-SAME:    %[[A0:.*]]: vector<2x8xi32>, %[[A1:.*]]: vector<2x8xi32>, %[[A2:.*]]: vector<2x2xi32>
+// CHECK-DAG: %[[D0:.*]] = vector.contract
+func.func @test_lower_vector_arm_neon_without_extsi(%lhs: vector<2x8xi32>, %rhs: vector<2x8xi32>, %acc : vector<2x2xi32>) -> vector<2x2xi32> {
+  %res = vector.contract {indexing_maps = [affine_map<(d0, d1, d2) -> (d0, d2)>, affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1)>], iterator_types = ["parallel", "parallel", "reduction"], kind = #vector.kind} %lhs, %rhs, %acc : vector<2x8xi32>, vector<2x8xi32> into vector<2x2xi32>
+  return %res : vector<2x2xi32>
+}
diff --git a/mlir/test/lib/Dialect/ArmNeon/CMakeLists.txt b/mlir/test/lib/Dialect/ArmNeon/CMakeLists.txt
new file mode 100644
index 000000000000..21548ca57701
--- /dev/null
+++ b/mlir/test/lib/Dialect/ArmNeon/CMakeLists.txt
@@ -0,0 +1,13 @@
+# Exclude tests from libMLIR.so
+add_mlir_library(MLIRArmNeonTestPasses
+  TestLowerToArmNeon.cpp
+
+  EXCLUDE_FROM_LIBMLIR
+
+  LINK_LIBS PUBLIC
+  MLIRArmNeonDialect
+  MLIRArmNeonTransforms
+  MLIRIR
+  MLIRPass
+  MLIRTransforms
+  )
diff --git a/mlir/test/lib/Dialect/ArmNeon/TestLowerToArmNeon.cpp b/mlir/test/lib/Dialect/ArmNeon/TestLowerToArmNeon.cpp
new file mode 100644
index 000000000000..2e5a4a988882
--- /dev/null
+++ b/mlir/test/lib/Dialect/ArmNeon/TestLowerToArmNeon.cpp
@@ -0,0 +1,61 @@
+//===- TestLowerToArmNeon.cpp - Test lowering to ArmNeon as a sink pass -===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements a pass for testing the lowering to ArmNeon as a
+// generally usable sink pass.
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/ArmNeon/ArmNeonDialect.h"
+#include "mlir/Dialect/ArmNeon/Transforms.h"
+#include "mlir/Dialect/Func/IR/FuncOps.h"
+#include "mlir/IR/PatternMatch.h"
+#include "mlir/Pass/Pass.h"
+#include "mlir/Pass/PassManager.h"
+#include "mlir/Support/LogicalResult.h"
+#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
+
+#define PASS_NAME "test-lower-to-arm-neon"
+
+using namespace mlir;
+using namespace mlir::arm_neon;
+
+namespace {
+struct TestLowerToArmNeon
+    : public PassWrapper> {
+  MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestLowerToArmNeon)
+
+  StringRef getArgument() const final { return PASS_NAME; }
+  StringRef getDescription() const final { return "Tests lower to arm Neon."; }
+  TestLowerToArmNeon() = default;
+  TestLowerToArmNeon(const TestLowerToArmNeon &pass) = default;
+
+  void getDependentDialects(DialectRegistry ®istry) const override {
+    registry.insert();
+  }
+
+  void runOnOperation() override;
+};
+
+} // namespace
+
+void TestLowerToArmNeon::runOnOperation() {
+  MLIRContext *context = &getContext();
+  RewritePatternSet patterns(context);
+  populateLowerContractionToSMMLAPatternPatterns(patterns);
+  if (failed(applyPatternsAndFoldGreedily(getOperation(), std::move(patterns))))
+    return signalPassFailure();
+}
+
+namespace mlir {
+namespace test {
+
+void registerTestLowerToArmNeon() { PassRegistration(); }
+
+} // namespace test
+} // namespace mlir
diff --git a/mlir/test/lib/Dialect/CMakeLists.txt b/mlir/test/lib/Dialect/CMakeLists.txt
index e20cd4473a35..29fb4441a24f 100644
--- a/mlir/test/lib/Dialect/CMakeLists.txt
+++ b/mlir/test/lib/Dialect/CMakeLists.txt
@@ -1,5 +1,6 @@
 add_subdirectory(Affine)
 add_subdirectory(Arith)
+add_subdirectory(ArmNeon)
 add_subdirectory(ArmSME)
 add_subdirectory(Bufferization)
 add_subdirectory(ControlFlow)
diff --git a/mlir/tools/mlir-opt/CMakeLists.txt b/mlir/tools/mlir-opt/CMakeLists.txt
index 4389840af10e..e8091bca3326 100644
--- a/mlir/tools/mlir-opt/CMakeLists.txt
+++ b/mlir/tools/mlir-opt/CMakeLists.txt
@@ -17,6 +17,7 @@ if(MLIR_INCLUDE_TESTS)
     MLIRTestFuncToLLVM
     MLIRAffineTransformsTestPasses
     MLIRArithTestPasses
+    MLIRArmNeonTestPasses
     MLIRArmSMETestPasses
     MLIRBufferizationTestPasses
     MLIRControlFlowTestPasses
diff --git a/mlir/tools/mlir-opt/mlir-opt.cpp b/mlir/tools/mlir-opt/mlir-opt.cpp
index e4d05631c35f..82b3881792bf 100644
--- a/mlir/tools/mlir-opt/mlir-opt.cpp
+++ b/mlir/tools/mlir-opt/mlir-opt.cpp
@@ -111,6 +111,7 @@ void registerTestLoopFusion();
 void registerTestCFGLoopInfoPass();
 void registerTestLoopMappingPass();
 void registerTestLoopUnrollingPass();
+void registerTestLowerToArmNeon();
 void registerTestLowerToArmSME();
 void registerTestLowerToLLVM();
 void registerTestMakeIsolatedFromAbovePass();
@@ -237,6 +238,7 @@ void registerTestPasses() {
   mlir::test::registerTestCFGLoopInfoPass();
   mlir::test::registerTestLoopMappingPass();
   mlir::test::registerTestLoopUnrollingPass();
+  mlir::test::registerTestLowerToArmNeon();
   mlir::test::registerTestLowerToArmSME();
   mlir::test::registerTestLowerToLLVM();
   mlir::test::registerTestMakeIsolatedFromAbovePass();
diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
index 3be8f34ae0f4..8da2b51ffc99 100644
--- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
@@ -1931,6 +1931,27 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "ArmNeonTransforms",
+    srcs = ["lib/Dialect/ArmNeon/Transforms/LowerVectorToArmNeon.cpp"],
+    hdrs = ["include/mlir/Dialect/ArmNeon/Transforms.h"],
+    includes = ["include"],
+    deps = [
+        ":ArithDialect",
+        ":ArmNeonIncGen",
+        ":ArmNeonDialect",
+        ":FuncDialect",
+        ":IR",
+        ":LLVMDialect",
+        ":SideEffectInterfaces",
+        ":Support",
+        ":VectorDialect",
+        ":Transforms",
+        "//llvm:Core",
+        "//llvm:Support",
+    ],
+)
+
 gentbl_cc_library(
     name = "ArmNeonConversionIncGen",
     tbl_outs = [
-- 
GitLab


From c22828991e7ca7b99048761c078252e94403ba6e Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh 
Date: Fri, 8 Mar 2024 15:13:20 -0800
Subject: [PATCH 0092/1674] =?UTF-8?q?Revert=20"[builtins]=20Disable=20COMP?=
 =?UTF-8?q?ILER=5FRT=5FCRT=5FUSE=5FEH=5FFRAME=5FREGISTRY=20by=20d=E2=80=A6?=
 =?UTF-8?q?=20(#84580)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…efault (#83201)"

This reverts commit 062cfada643c1aa48a1bb81894e2920d390fe8cf. See issue
#84574.
---
 compiler-rt/lib/builtins/CMakeLists.txt    |  2 +-
 compiler-rt/test/builtins/Unit/ctor_dtor.c | 10 ++++++++++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/compiler-rt/lib/builtins/CMakeLists.txt b/compiler-rt/lib/builtins/CMakeLists.txt
index 3f2c0f043639..f9611574a562 100644
--- a/compiler-rt/lib/builtins/CMakeLists.txt
+++ b/compiler-rt/lib/builtins/CMakeLists.txt
@@ -917,7 +917,7 @@ cmake_dependent_option(COMPILER_RT_BUILD_CRT "Build crtbegin.o/crtend.o" ON "COM
 if (COMPILER_RT_BUILD_CRT)
   add_compiler_rt_component(crt)
 
-  option(COMPILER_RT_CRT_USE_EH_FRAME_REGISTRY "Use eh_frame in crtbegin.o/crtend.o" OFF)
+  option(COMPILER_RT_CRT_USE_EH_FRAME_REGISTRY "Use eh_frame in crtbegin.o/crtend.o" ON)
 
   include(CheckSectionExists)
   check_section_exists(".init_array" COMPILER_RT_HAS_INITFINI_ARRAY
diff --git a/compiler-rt/test/builtins/Unit/ctor_dtor.c b/compiler-rt/test/builtins/Unit/ctor_dtor.c
index 3d5f895a0a1c..47560722a9f7 100644
--- a/compiler-rt/test/builtins/Unit/ctor_dtor.c
+++ b/compiler-rt/test/builtins/Unit/ctor_dtor.c
@@ -9,13 +9,23 @@
 
 // Ensure the various startup functions are called in the proper order.
 
+// CHECK: __register_frame_info()
 /// ctor() is here if ld.so/libc supports DT_INIT/DT_FINI
 // CHECK:      main()
 /// dtor() is here if ld.so/libc supports DT_INIT/DT_FINI
+// CHECK:      __deregister_frame_info()
 
 struct object;
 static int counter;
 
+void __register_frame_info(const void *fi, struct object *obj) {
+  printf("__register_frame_info()\n");
+}
+
+void __deregister_frame_info(const void *fi) {
+  printf("__deregister_frame_info()\n");
+}
+
 void __attribute__((constructor)) ctor() {
   printf("ctor()\n");
   ++counter;
-- 
GitLab


From 624ea68cbc3ce422b3ee110c0c0af839eec2e278 Mon Sep 17 00:00:00 2001
From: Adrian Prantl 
Date: Fri, 8 Mar 2024 10:39:34 -0800
Subject: [PATCH 0093/1674] Change GetNumChildren()/CalculateNumChildren()
 methods return llvm::Expected (#84219)

Change GetNumChildren()/CalculateNumChildren() methods return
llvm::Expected

This is an NFC change that does not yet add any error handling or change
any code to return any errors.

This is the second big change in the patch series started with
https://github.com/llvm/llvm-project/pull/83501

A follow-up PR will wire up error handling.
---
 lldb/include/lldb/Core/ValueObject.h          | 11 +++-
 lldb/include/lldb/Core/ValueObjectCast.h      |  2 +-
 lldb/include/lldb/Core/ValueObjectChild.h     |  2 +-
 .../lldb/Core/ValueObjectConstResult.h        |  2 +-
 .../lldb/Core/ValueObjectDynamicValue.h       |  2 +-
 lldb/include/lldb/Core/ValueObjectMemory.h    |  2 +-
 lldb/include/lldb/Core/ValueObjectRegister.h  |  4 +-
 .../lldb/Core/ValueObjectSyntheticFilter.h    |  2 +-
 lldb/include/lldb/Core/ValueObjectVTable.h    |  2 +-
 lldb/include/lldb/Core/ValueObjectVariable.h  |  2 +-
 .../lldb/DataFormatters/TypeSynthetic.h       | 20 ++++--
 .../lldb/DataFormatters/VectorIterator.h      |  2 +-
 lldb/include/lldb/Symbol/CompilerType.h       |  5 +-
 lldb/include/lldb/Symbol/Type.h               |  2 +-
 lldb/include/lldb/Symbol/TypeSystem.h         |  7 +-
 .../lldb/Target/StackFrameRecognizer.h        |  3 +-
 lldb/include/lldb/Utility/Log.h               | 14 ++++
 lldb/source/API/SBValue.cpp                   |  2 +-
 lldb/source/Core/FormatEntity.cpp             |  2 +-
 lldb/source/Core/IOHandlerCursesGUI.cpp       |  2 +-
 lldb/source/Core/ValueObject.cpp              | 31 ++++++---
 lldb/source/Core/ValueObjectCast.cpp          |  6 +-
 lldb/source/Core/ValueObjectChild.cpp         |  6 +-
 lldb/source/Core/ValueObjectConstResult.cpp   |  7 +-
 lldb/source/Core/ValueObjectDynamicValue.cpp  |  7 +-
 lldb/source/Core/ValueObjectMemory.cpp        | 10 ++-
 lldb/source/Core/ValueObjectRegister.cpp      | 12 ++--
 .../Core/ValueObjectSyntheticFilter.cpp       | 24 ++++---
 lldb/source/Core/ValueObjectVTable.cpp        |  6 +-
 lldb/source/Core/ValueObjectVariable.cpp      |  7 +-
 lldb/source/DataFormatters/FormatManager.cpp  | 11 +++-
 lldb/source/DataFormatters/TypeSynthetic.cpp  | 16 ++++-
 .../DataFormatters/ValueObjectPrinter.cpp     |  2 +-
 lldb/source/DataFormatters/VectorType.cpp     | 16 +++--
 .../Plugins/ABI/PowerPC/ABISysV_ppc64.cpp     |  7 +-
 .../Clang/ClangExpressionSourceCode.cpp       |  2 +-
 .../TSan/InstrumentationRuntimeTSan.cpp       |  4 +-
 .../Language/CPlusPlus/BlockPointer.cpp       |  4 +-
 .../Plugins/Language/CPlusPlus/Coroutines.cpp |  8 ++-
 .../Plugins/Language/CPlusPlus/Coroutines.h   |  2 +-
 .../Language/CPlusPlus/GenericBitset.cpp      |  4 +-
 .../Language/CPlusPlus/GenericOptional.cpp    |  6 +-
 .../Plugins/Language/CPlusPlus/LibCxx.cpp     | 18 +++---
 .../Plugins/Language/CPlusPlus/LibCxx.h       |  8 +--
 .../Language/CPlusPlus/LibCxxAtomic.cpp       |  6 +-
 .../CPlusPlus/LibCxxInitializerList.cpp       |  6 +-
 .../Plugins/Language/CPlusPlus/LibCxxList.cpp | 12 ++--
 .../Plugins/Language/CPlusPlus/LibCxxMap.cpp  | 14 ++--
 .../Language/CPlusPlus/LibCxxQueue.cpp        |  2 +-
 .../CPlusPlus/LibCxxRangesRefView.cpp         |  2 +-
 .../Plugins/Language/CPlusPlus/LibCxxSpan.cpp |  6 +-
 .../Language/CPlusPlus/LibCxxTuple.cpp        |  4 +-
 .../Language/CPlusPlus/LibCxxUnorderedMap.cpp |  8 +--
 .../Language/CPlusPlus/LibCxxValarray.cpp     |  6 +-
 .../Language/CPlusPlus/LibCxxVariant.cpp      |  2 +-
 .../Language/CPlusPlus/LibCxxVector.cpp       | 14 ++--
 .../Plugins/Language/CPlusPlus/LibStdcpp.cpp  | 17 +++--
 .../Language/CPlusPlus/LibStdcppTuple.cpp     |  7 +-
 .../CPlusPlus/LibStdcppUniquePointer.cpp      |  5 +-
 lldb/source/Plugins/Language/ObjC/Cocoa.cpp   |  2 +-
 lldb/source/Plugins/Language/ObjC/NSArray.cpp | 30 ++++-----
 .../Plugins/Language/ObjC/NSDictionary.cpp    | 64 +++++++++----------
 lldb/source/Plugins/Language/ObjC/NSError.cpp |  2 +-
 .../Plugins/Language/ObjC/NSException.cpp     |  4 +-
 .../Plugins/Language/ObjC/NSIndexPath.cpp     |  6 +-
 lldb/source/Plugins/Language/ObjC/NSSet.cpp   | 32 +++++-----
 .../AppleObjCRuntime/AppleObjCRuntime.cpp     |  3 +-
 .../TypeSystem/Clang/TypeSystemClang.cpp      | 40 ++++++++----
 .../TypeSystem/Clang/TypeSystemClang.h        |  7 +-
 lldb/source/Symbol/CompilerType.cpp           |  5 +-
 lldb/source/Symbol/Type.cpp                   |  2 +-
 lldb/source/Symbol/Variable.cpp               |  6 +-
 lldb/source/Target/StackFrame.cpp             | 19 +++---
 73 files changed, 399 insertions(+), 248 deletions(-)

diff --git a/lldb/include/lldb/Core/ValueObject.h b/lldb/include/lldb/Core/ValueObject.h
index b4d2c8098edc..e7e35e2b2bff 100644
--- a/lldb/include/lldb/Core/ValueObject.h
+++ b/lldb/include/lldb/Core/ValueObject.h
@@ -476,7 +476,13 @@ public:
 
   virtual size_t GetIndexOfChildWithName(llvm::StringRef name);
 
-  uint32_t GetNumChildren(uint32_t max = UINT32_MAX);
+  llvm::Expected GetNumChildren(uint32_t max = UINT32_MAX);
+  /// Like \c GetNumChildren but returns 0 on error.  You probably
+  /// shouldn't be using this function. It exists primarily to ease the
+  /// transition to more pervasive error handling while not all APIs
+  /// have been updated.
+  uint32_t GetNumChildrenIgnoringErrors(uint32_t max = UINT32_MAX);
+  bool HasChildren() { return GetNumChildrenIgnoringErrors() > 0; }
 
   const Value &GetValue() const { return m_value; }
 
@@ -958,7 +964,8 @@ protected:
                                           int32_t synthetic_index);
 
   /// Should only be called by ValueObject::GetNumChildren().
-  virtual uint32_t CalculateNumChildren(uint32_t max = UINT32_MAX) = 0;
+  virtual llvm::Expected
+  CalculateNumChildren(uint32_t max = UINT32_MAX) = 0;
 
   void SetNumChildren(uint32_t num_children);
 
diff --git a/lldb/include/lldb/Core/ValueObjectCast.h b/lldb/include/lldb/Core/ValueObjectCast.h
index 51c647680d52..ba25e166f326 100644
--- a/lldb/include/lldb/Core/ValueObjectCast.h
+++ b/lldb/include/lldb/Core/ValueObjectCast.h
@@ -33,7 +33,7 @@ public:
 
   std::optional GetByteSize() override;
 
-  uint32_t CalculateNumChildren(uint32_t max) override;
+  llvm::Expected CalculateNumChildren(uint32_t max) override;
 
   lldb::ValueType GetValueType() const override;
 
diff --git a/lldb/include/lldb/Core/ValueObjectChild.h b/lldb/include/lldb/Core/ValueObjectChild.h
index 47a13be08bb8..1f88e607cb57 100644
--- a/lldb/include/lldb/Core/ValueObjectChild.h
+++ b/lldb/include/lldb/Core/ValueObjectChild.h
@@ -39,7 +39,7 @@ public:
 
   lldb::ValueType GetValueType() const override;
 
-  uint32_t CalculateNumChildren(uint32_t max) override;
+  llvm::Expected CalculateNumChildren(uint32_t max) override;
 
   ConstString GetTypeName() override;
 
diff --git a/lldb/include/lldb/Core/ValueObjectConstResult.h b/lldb/include/lldb/Core/ValueObjectConstResult.h
index 9f1246cf2a78..37dc0867f26c 100644
--- a/lldb/include/lldb/Core/ValueObjectConstResult.h
+++ b/lldb/include/lldb/Core/ValueObjectConstResult.h
@@ -67,7 +67,7 @@ public:
 
   lldb::ValueType GetValueType() const override;
 
-  uint32_t CalculateNumChildren(uint32_t max) override;
+  llvm::Expected CalculateNumChildren(uint32_t max) override;
 
   ConstString GetTypeName() override;
 
diff --git a/lldb/include/lldb/Core/ValueObjectDynamicValue.h b/lldb/include/lldb/Core/ValueObjectDynamicValue.h
index 21a9b409fd5b..82c20eee0cd4 100644
--- a/lldb/include/lldb/Core/ValueObjectDynamicValue.h
+++ b/lldb/include/lldb/Core/ValueObjectDynamicValue.h
@@ -43,7 +43,7 @@ public:
 
   ConstString GetDisplayTypeName() override;
 
-  uint32_t CalculateNumChildren(uint32_t max) override;
+  llvm::Expected CalculateNumChildren(uint32_t max) override;
 
   lldb::ValueType GetValueType() const override;
 
diff --git a/lldb/include/lldb/Core/ValueObjectMemory.h b/lldb/include/lldb/Core/ValueObjectMemory.h
index a74b325546b0..a8fb0353d601 100644
--- a/lldb/include/lldb/Core/ValueObjectMemory.h
+++ b/lldb/include/lldb/Core/ValueObjectMemory.h
@@ -47,7 +47,7 @@ public:
 
   ConstString GetDisplayTypeName() override;
 
-  uint32_t CalculateNumChildren(uint32_t max) override;
+  llvm::Expected CalculateNumChildren(uint32_t max) override;
 
   lldb::ValueType GetValueType() const override;
 
diff --git a/lldb/include/lldb/Core/ValueObjectRegister.h b/lldb/include/lldb/Core/ValueObjectRegister.h
index 6c470c1a6865..fec8566ba33d 100644
--- a/lldb/include/lldb/Core/ValueObjectRegister.h
+++ b/lldb/include/lldb/Core/ValueObjectRegister.h
@@ -47,7 +47,7 @@ public:
 
   ConstString GetQualifiedTypeName() override;
 
-  uint32_t CalculateNumChildren(uint32_t max) override;
+  llvm::Expected CalculateNumChildren(uint32_t max) override;
 
   ValueObject *CreateChildAtIndex(size_t idx, bool synthetic_array_member,
                                   int32_t synthetic_index) override;
@@ -95,7 +95,7 @@ public:
 
   ConstString GetTypeName() override;
 
-  uint32_t CalculateNumChildren(uint32_t max) override;
+  llvm::Expected CalculateNumChildren(uint32_t max) override;
 
   bool SetValueFromCString(const char *value_str, Status &error) override;
 
diff --git a/lldb/include/lldb/Core/ValueObjectSyntheticFilter.h b/lldb/include/lldb/Core/ValueObjectSyntheticFilter.h
index 1e54babc94f3..ca6d6c728005 100644
--- a/lldb/include/lldb/Core/ValueObjectSyntheticFilter.h
+++ b/lldb/include/lldb/Core/ValueObjectSyntheticFilter.h
@@ -47,7 +47,7 @@ public:
 
   bool MightHaveChildren() override;
 
-  uint32_t CalculateNumChildren(uint32_t max) override;
+  llvm::Expected CalculateNumChildren(uint32_t max) override;
 
   lldb::ValueType GetValueType() const override;
 
diff --git a/lldb/include/lldb/Core/ValueObjectVTable.h b/lldb/include/lldb/Core/ValueObjectVTable.h
index e7e14fc83d78..4662f395a4dd 100644
--- a/lldb/include/lldb/Core/ValueObjectVTable.h
+++ b/lldb/include/lldb/Core/ValueObjectVTable.h
@@ -64,7 +64,7 @@ public:
 
   std::optional GetByteSize() override;
 
-  uint32_t CalculateNumChildren(uint32_t max) override;
+  llvm::Expected CalculateNumChildren(uint32_t max) override;
 
   ValueObject *CreateChildAtIndex(size_t idx, bool synthetic_array_member,
                                   int32_t synthetic_index) override;
diff --git a/lldb/include/lldb/Core/ValueObjectVariable.h b/lldb/include/lldb/Core/ValueObjectVariable.h
index da270300df0b..db3847f14a0b 100644
--- a/lldb/include/lldb/Core/ValueObjectVariable.h
+++ b/lldb/include/lldb/Core/ValueObjectVariable.h
@@ -46,7 +46,7 @@ public:
 
   ConstString GetDisplayTypeName() override;
 
-  uint32_t CalculateNumChildren(uint32_t max) override;
+  llvm::Expected CalculateNumChildren(uint32_t max) override;
 
   lldb::ValueType GetValueType() const override;
 
diff --git a/lldb/include/lldb/DataFormatters/TypeSynthetic.h b/lldb/include/lldb/DataFormatters/TypeSynthetic.h
index 38f3ce0fa5f0..ede7442a02bf 100644
--- a/lldb/include/lldb/DataFormatters/TypeSynthetic.h
+++ b/lldb/include/lldb/DataFormatters/TypeSynthetic.h
@@ -38,13 +38,17 @@ public:
 
   virtual ~SyntheticChildrenFrontEnd() = default;
 
-  virtual uint32_t CalculateNumChildren() = 0;
+  virtual llvm::Expected CalculateNumChildren() = 0;
 
-  virtual uint32_t CalculateNumChildren(uint32_t max) {
+  virtual llvm::Expected CalculateNumChildren(uint32_t max) {
     auto count = CalculateNumChildren();
-    return count <= max ? count : max;
+    if (!count)
+      return count;
+    return *count <= max ? *count : max;
   }
 
+  uint32_t CalculateNumChildrenIgnoringErrors(uint32_t max = UINT32_MAX);
+
   virtual lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) = 0;
 
   virtual size_t GetIndexOfChildWithName(ConstString name) = 0;
@@ -109,7 +113,7 @@ public:
 
   ~SyntheticValueProviderFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override { return 0; }
+  llvm::Expected CalculateNumChildren() override { return 0; }
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override { return nullptr; }
 
@@ -322,7 +326,9 @@ public:
 
     ~FrontEnd() override = default;
 
-    uint32_t CalculateNumChildren() override { return filter->GetCount(); }
+    llvm::Expected CalculateNumChildren() override {
+      return filter->GetCount();
+    }
 
     lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override {
       if (idx >= filter->GetCount())
@@ -426,9 +432,9 @@ public:
 
     bool IsValid();
 
-    uint32_t CalculateNumChildren() override;
+    llvm::Expected CalculateNumChildren() override;
 
-    uint32_t CalculateNumChildren(uint32_t max) override;
+    llvm::Expected CalculateNumChildren(uint32_t max) override;
 
     lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
diff --git a/lldb/include/lldb/DataFormatters/VectorIterator.h b/lldb/include/lldb/DataFormatters/VectorIterator.h
index 7711b9de95db..70bcf50ca1b1 100644
--- a/lldb/include/lldb/DataFormatters/VectorIterator.h
+++ b/lldb/include/lldb/DataFormatters/VectorIterator.h
@@ -24,7 +24,7 @@ public:
   VectorIteratorSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp,
                                   llvm::ArrayRef item_names);
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
diff --git a/lldb/include/lldb/Symbol/CompilerType.h b/lldb/include/lldb/Symbol/CompilerType.h
index 414c44275aaa..c1dce4ccbf79 100644
--- a/lldb/include/lldb/Symbol/CompilerType.h
+++ b/lldb/include/lldb/Symbol/CompilerType.h
@@ -386,8 +386,9 @@ public:
 
   std::optional GetTypeBitAlign(ExecutionContextScope *exe_scope) const;
 
-  uint32_t GetNumChildren(bool omit_empty_base_classes,
-                          const ExecutionContext *exe_ctx) const;
+  llvm::Expected
+  GetNumChildren(bool omit_empty_base_classes,
+                 const ExecutionContext *exe_ctx) const;
 
   lldb::BasicType GetBasicTypeEnumeration() const;
 
diff --git a/lldb/include/lldb/Symbol/Type.h b/lldb/include/lldb/Symbol/Type.h
index acd1a769f13c..b5eac5fa732d 100644
--- a/lldb/include/lldb/Symbol/Type.h
+++ b/lldb/include/lldb/Symbol/Type.h
@@ -440,7 +440,7 @@ public:
 
   std::optional GetByteSize(ExecutionContextScope *exe_scope);
 
-  uint32_t GetNumChildren(bool omit_empty_base_classes);
+  llvm::Expected GetNumChildren(bool omit_empty_base_classes);
 
   bool IsAggregateType();
 
diff --git a/lldb/include/lldb/Symbol/TypeSystem.h b/lldb/include/lldb/Symbol/TypeSystem.h
index 63829131556e..f647fcbf1636 100644
--- a/lldb/include/lldb/Symbol/TypeSystem.h
+++ b/lldb/include/lldb/Symbol/TypeSystem.h
@@ -300,9 +300,10 @@ public:
 
   virtual lldb::Format GetFormat(lldb::opaque_compiler_type_t type) = 0;
 
-  virtual uint32_t GetNumChildren(lldb::opaque_compiler_type_t type,
-                                  bool omit_empty_base_classes,
-                                  const ExecutionContext *exe_ctx) = 0;
+  virtual llvm::Expected
+  GetNumChildren(lldb::opaque_compiler_type_t type,
+                 bool omit_empty_base_classes,
+                 const ExecutionContext *exe_ctx) = 0;
 
   virtual CompilerType GetBuiltinTypeByName(ConstString name);
 
diff --git a/lldb/include/lldb/Target/StackFrameRecognizer.h b/lldb/include/lldb/Target/StackFrameRecognizer.h
index e111f4a4dc70..5e8e12b2a4e9 100644
--- a/lldb/include/lldb/Target/StackFrameRecognizer.h
+++ b/lldb/include/lldb/Target/StackFrameRecognizer.h
@@ -164,7 +164,8 @@ class ValueObjectRecognizerSynthesizedValue : public ValueObject {
     m_value = m_parent->GetValue();
     return true;
   }
-  uint32_t CalculateNumChildren(uint32_t max = UINT32_MAX) override {
+  llvm::Expected
+  CalculateNumChildren(uint32_t max = UINT32_MAX) override {
     return m_parent->GetNumChildren(max);
   }
   CompilerType GetCompilerTypeImpl() override {
diff --git a/lldb/include/lldb/Utility/Log.h b/lldb/include/lldb/Utility/Log.h
index 1fe28d61b9da..01876ad732d4 100644
--- a/lldb/include/lldb/Utility/Log.h
+++ b/lldb/include/lldb/Utility/Log.h
@@ -373,4 +373,18 @@ template  Log *GetLog(Cat mask) {
       ::llvm::consumeError(::std::move(error_private));                        \
   } while (0)
 
+// Write message to the verbose log, if error is set. In the log
+// message refer to the error with {0}. Error is cleared regardless of
+// whether logging is enabled.
+#define LLDB_LOG_ERRORV(log, error, ...)                                       \
+  do {                                                                         \
+    ::lldb_private::Log *log_private = (log);                                  \
+    ::llvm::Error error_private = (error);                                     \
+    if (log_private && log_private->GetVerbose() && error_private) {           \
+      log_private->FormatError(::std::move(error_private), __FILE__, __func__, \
+                               __VA_ARGS__);                                   \
+    } else                                                                     \
+      ::llvm::consumeError(::std::move(error_private));                        \
+  } while (0)
+
 #endif // LLDB_UTILITY_LOG_H
diff --git a/lldb/source/API/SBValue.cpp b/lldb/source/API/SBValue.cpp
index 89d26a1fbe28..94a8f3ea319e 100644
--- a/lldb/source/API/SBValue.cpp
+++ b/lldb/source/API/SBValue.cpp
@@ -947,7 +947,7 @@ uint32_t SBValue::GetNumChildren(uint32_t max) {
   ValueLocker locker;
   lldb::ValueObjectSP value_sp(GetSP(locker));
   if (value_sp)
-    num_children = value_sp->GetNumChildren(max);
+    num_children = value_sp->GetNumChildrenIgnoringErrors(max);
 
   return num_children;
 }
diff --git a/lldb/source/Core/FormatEntity.cpp b/lldb/source/Core/FormatEntity.cpp
index fa5eadc6ff4e..cf82676bedda 100644
--- a/lldb/source/Core/FormatEntity.cpp
+++ b/lldb/source/Core/FormatEntity.cpp
@@ -926,7 +926,7 @@ static bool DumpValue(Stream &s, const SymbolContext *sc,
     s.PutChar('[');
 
     if (index_higher < 0)
-      index_higher = valobj->GetNumChildren() - 1;
+      index_higher = valobj->GetNumChildrenIgnoringErrors() - 1;
 
     uint32_t max_num_children =
         target->GetTargetSP()->GetMaximumNumberOfChildrenToDisplay();
diff --git a/lldb/source/Core/IOHandlerCursesGUI.cpp b/lldb/source/Core/IOHandlerCursesGUI.cpp
index 620e68a28510..f86dce247135 100644
--- a/lldb/source/Core/IOHandlerCursesGUI.cpp
+++ b/lldb/source/Core/IOHandlerCursesGUI.cpp
@@ -4519,7 +4519,7 @@ struct Row {
       calculated_children = true;
       ValueObjectSP valobj = value.GetSP();
       if (valobj) {
-        const size_t num_children = valobj->GetNumChildren();
+        const uint32_t num_children = valobj->GetNumChildrenIgnoringErrors();
         for (size_t i = 0; i < num_children; ++i) {
           children.push_back(Row(valobj->GetChildAtIndex(i), this));
         }
diff --git a/lldb/source/Core/ValueObject.cpp b/lldb/source/Core/ValueObject.cpp
index 0ed7f03be25c..d813044d02ff 100644
--- a/lldb/source/Core/ValueObject.cpp
+++ b/lldb/source/Core/ValueObject.cpp
@@ -377,7 +377,7 @@ ValueObjectSP ValueObject::GetChildAtIndex(uint32_t idx, bool can_create) {
   // We may need to update our value if we are dynamic
   if (IsPossibleDynamicType())
     UpdateValueIfNeeded(false);
-  if (idx < GetNumChildren()) {
+  if (idx < GetNumChildrenIgnoringErrors()) {
     // Check if we have already made the child value object?
     if (can_create && !m_children.HasChildAtIndex(idx)) {
       // No we haven't created the child at this index, so lets have our
@@ -440,7 +440,7 @@ ValueObjectSP ValueObject::GetChildMemberWithName(llvm::StringRef name,
   return child_sp;
 }
 
-uint32_t ValueObject::GetNumChildren(uint32_t max) {
+llvm::Expected ValueObject::GetNumChildren(uint32_t max) {
   UpdateValueIfNeeded();
 
   if (max < UINT32_MAX) {
@@ -452,11 +452,24 @@ uint32_t ValueObject::GetNumChildren(uint32_t max) {
   }
 
   if (!m_flags.m_children_count_valid) {
-    SetNumChildren(CalculateNumChildren());
+    auto num_children_or_err = CalculateNumChildren();
+    if (num_children_or_err)
+      SetNumChildren(*num_children_or_err);
+    else
+      return num_children_or_err;
   }
   return m_children.GetChildrenCount();
 }
 
+uint32_t ValueObject::GetNumChildrenIgnoringErrors(uint32_t max) {
+  auto value_or_err = GetNumChildren(max);
+  if (value_or_err)
+    return *value_or_err;
+  LLDB_LOG_ERRORV(GetLog(LLDBLog::DataFormatters), value_or_err.takeError(),
+                  "{0}");
+  return 0;
+}
+
 bool ValueObject::MightHaveChildren() {
   bool has_children = false;
   const uint32_t type_info = GetTypeInfo();
@@ -464,7 +477,7 @@ bool ValueObject::MightHaveChildren() {
     if (type_info & (eTypeHasChildren | eTypeIsPointer | eTypeIsReference))
       has_children = true;
   } else {
-    has_children = GetNumChildren() > 0;
+    has_children = GetNumChildrenIgnoringErrors() > 0;
   }
   return has_children;
 }
@@ -1176,7 +1189,7 @@ bool ValueObject::DumpPrintableRepresentation(
       if (flags.Test(eTypeIsArray)) {
         if ((custom_format == eFormatBytes) ||
             (custom_format == eFormatBytesWithASCII)) {
-          const size_t count = GetNumChildren();
+          const size_t count = GetNumChildrenIgnoringErrors();
 
           s << '[';
           for (size_t low = 0; low < count; low++) {
@@ -1215,7 +1228,7 @@ bool ValueObject::DumpPrintableRepresentation(
                                                      // format should be printed
                                                      // directly
         {
-          const size_t count = GetNumChildren();
+          const size_t count = GetNumChildrenIgnoringErrors();
 
           Format format = FormatManager::GetSingleItemFormat(custom_format);
 
@@ -1294,7 +1307,7 @@ bool ValueObject::DumpPrintableRepresentation(
       break;
 
     case eValueObjectRepresentationStyleChildrenCount:
-      strm.Printf("%" PRIu64 "", (uint64_t)GetNumChildren());
+      strm.Printf("%" PRIu64 "", (uint64_t)GetNumChildrenIgnoringErrors());
       str = strm.GetString();
       break;
 
@@ -2320,7 +2333,9 @@ ValueObjectSP ValueObject::GetValueForExpressionPath_Impl(
             child_valobj_sp = root->GetSyntheticArrayMember(index, true);
           if (!child_valobj_sp)
             if (root->HasSyntheticValue() &&
-                root->GetSyntheticValue()->GetNumChildren() > index)
+                llvm::expectedToStdOptional(
+                    root->GetSyntheticValue()->GetNumChildren())
+                        .value_or(0) > index)
               child_valobj_sp =
                   root->GetSyntheticValue()->GetChildAtIndex(index);
           if (child_valobj_sp) {
diff --git a/lldb/source/Core/ValueObjectCast.cpp b/lldb/source/Core/ValueObjectCast.cpp
index a5c555f86b13..c8e316415141 100644
--- a/lldb/source/Core/ValueObjectCast.cpp
+++ b/lldb/source/Core/ValueObjectCast.cpp
@@ -41,11 +41,13 @@ ValueObjectCast::~ValueObjectCast() = default;
 
 CompilerType ValueObjectCast::GetCompilerTypeImpl() { return m_cast_type; }
 
-uint32_t ValueObjectCast::CalculateNumChildren(uint32_t max) {
+llvm::Expected ValueObjectCast::CalculateNumChildren(uint32_t max) {
   ExecutionContext exe_ctx(GetExecutionContextRef());
   auto children_count = GetCompilerType().GetNumChildren(
       true, &exe_ctx);
-  return children_count <= max ? children_count : max;
+  if (!children_count)
+    return children_count;
+  return *children_count <= max ? *children_count : max;
 }
 
 std::optional ValueObjectCast::GetByteSize() {
diff --git a/lldb/source/Core/ValueObjectChild.cpp b/lldb/source/Core/ValueObjectChild.cpp
index 2e55dd7726bd..c6a97dd1a5cd 100644
--- a/lldb/source/Core/ValueObjectChild.cpp
+++ b/lldb/source/Core/ValueObjectChild.cpp
@@ -49,10 +49,12 @@ lldb::ValueType ValueObjectChild::GetValueType() const {
   return m_parent->GetValueType();
 }
 
-uint32_t ValueObjectChild::CalculateNumChildren(uint32_t max) {
+llvm::Expected ValueObjectChild::CalculateNumChildren(uint32_t max) {
   ExecutionContext exe_ctx(GetExecutionContextRef());
   auto children_count = GetCompilerType().GetNumChildren(true, &exe_ctx);
-  return children_count <= max ? children_count : max;
+  if (!children_count)
+    return children_count;
+  return *children_count <= max ? *children_count : max;
 }
 
 static void AdjustForBitfieldness(ConstString &name,
diff --git a/lldb/source/Core/ValueObjectConstResult.cpp b/lldb/source/Core/ValueObjectConstResult.cpp
index 5c7aa4452b70..8ac2c1cac2f6 100644
--- a/lldb/source/Core/ValueObjectConstResult.cpp
+++ b/lldb/source/Core/ValueObjectConstResult.cpp
@@ -216,10 +216,13 @@ std::optional ValueObjectConstResult::GetByteSize() {
 
 void ValueObjectConstResult::SetByteSize(size_t size) { m_byte_size = size; }
 
-uint32_t ValueObjectConstResult::CalculateNumChildren(uint32_t max) {
+llvm::Expected
+ValueObjectConstResult::CalculateNumChildren(uint32_t max) {
   ExecutionContext exe_ctx(GetExecutionContextRef());
   auto children_count = GetCompilerType().GetNumChildren(true, &exe_ctx);
-  return children_count <= max ? children_count : max;
+  if (!children_count)
+    return children_count;
+  return *children_count <= max ? *children_count : max;
 }
 
 ConstString ValueObjectConstResult::GetTypeName() {
diff --git a/lldb/source/Core/ValueObjectDynamicValue.cpp b/lldb/source/Core/ValueObjectDynamicValue.cpp
index 4e64760371ae..4695febdf8ca 100644
--- a/lldb/source/Core/ValueObjectDynamicValue.cpp
+++ b/lldb/source/Core/ValueObjectDynamicValue.cpp
@@ -85,12 +85,15 @@ ConstString ValueObjectDynamicValue::GetDisplayTypeName() {
   return m_parent->GetDisplayTypeName();
 }
 
-uint32_t ValueObjectDynamicValue::CalculateNumChildren(uint32_t max) {
+llvm::Expected
+ValueObjectDynamicValue::CalculateNumChildren(uint32_t max) {
   const bool success = UpdateValueIfNeeded(false);
   if (success && m_dynamic_type_info.HasType()) {
     ExecutionContext exe_ctx(GetExecutionContextRef());
     auto children_count = GetCompilerType().GetNumChildren(true, &exe_ctx);
-    return children_count <= max ? children_count : max;
+    if (!children_count)
+      return children_count;
+    return *children_count <= max ? *children_count : max;
   } else
     return m_parent->GetNumChildren(max);
 }
diff --git a/lldb/source/Core/ValueObjectMemory.cpp b/lldb/source/Core/ValueObjectMemory.cpp
index 7f68236c7884..f555ab82f441 100644
--- a/lldb/source/Core/ValueObjectMemory.cpp
+++ b/lldb/source/Core/ValueObjectMemory.cpp
@@ -126,17 +126,21 @@ ConstString ValueObjectMemory::GetDisplayTypeName() {
   return m_compiler_type.GetDisplayTypeName();
 }
 
-uint32_t ValueObjectMemory::CalculateNumChildren(uint32_t max) {
+llvm::Expected ValueObjectMemory::CalculateNumChildren(uint32_t max) {
   if (m_type_sp) {
     auto child_count = m_type_sp->GetNumChildren(true);
-    return child_count <= max ? child_count : max;
+    if (!child_count)
+      return child_count;
+    return *child_count <= max ? *child_count : max;
   }
 
   ExecutionContext exe_ctx(GetExecutionContextRef());
   const bool omit_empty_base_classes = true;
   auto child_count =
       m_compiler_type.GetNumChildren(omit_empty_base_classes, &exe_ctx);
-  return child_count <= max ? child_count : max;
+  if (!child_count)
+    return child_count;
+  return *child_count <= max ? *child_count : max;
 }
 
 std::optional ValueObjectMemory::GetByteSize() {
diff --git a/lldb/source/Core/ValueObjectRegister.cpp b/lldb/source/Core/ValueObjectRegister.cpp
index d4c144cc7edb..9d9da8bb119d 100644
--- a/lldb/source/Core/ValueObjectRegister.cpp
+++ b/lldb/source/Core/ValueObjectRegister.cpp
@@ -74,7 +74,8 @@ ConstString ValueObjectRegisterSet::GetQualifiedTypeName() {
   return ConstString();
 }
 
-uint32_t ValueObjectRegisterSet::CalculateNumChildren(uint32_t max) {
+llvm::Expected
+ValueObjectRegisterSet::CalculateNumChildren(uint32_t max) {
   const RegisterSet *reg_set = m_reg_ctx_sp->GetRegisterSet(m_reg_set_idx);
   if (reg_set) {
     auto reg_count = reg_set->num_registers;
@@ -118,7 +119,7 @@ ValueObject *ValueObjectRegisterSet::CreateChildAtIndex(
     size_t idx, bool synthetic_array_member, int32_t synthetic_index) {
   ValueObject *valobj = nullptr;
   if (m_reg_ctx_sp && m_reg_set) {
-    const size_t num_children = GetNumChildren();
+    uint32_t num_children = GetNumChildrenIgnoringErrors();
     if (idx < num_children)
       valobj = new ValueObjectRegister(
           *this, m_reg_ctx_sp,
@@ -220,10 +221,13 @@ ConstString ValueObjectRegister::GetTypeName() {
   return m_type_name;
 }
 
-uint32_t ValueObjectRegister::CalculateNumChildren(uint32_t max) {
+llvm::Expected
+ValueObjectRegister::CalculateNumChildren(uint32_t max) {
   ExecutionContext exe_ctx(GetExecutionContextRef());
   auto children_count = GetCompilerType().GetNumChildren(true, &exe_ctx);
-  return children_count <= max ? children_count : max;
+  if (!children_count)
+    return children_count;
+  return *children_count <= max ? *children_count : max;
 }
 
 std::optional ValueObjectRegister::GetByteSize() {
diff --git a/lldb/source/Core/ValueObjectSyntheticFilter.cpp b/lldb/source/Core/ValueObjectSyntheticFilter.cpp
index 7f8a9a34cb35..adac1b400705 100644
--- a/lldb/source/Core/ValueObjectSyntheticFilter.cpp
+++ b/lldb/source/Core/ValueObjectSyntheticFilter.cpp
@@ -31,7 +31,9 @@ public:
   DummySyntheticFrontEnd(ValueObject &backend)
       : SyntheticChildrenFrontEnd(backend) {}
 
-  uint32_t CalculateNumChildren() override { return m_backend.GetNumChildren(); }
+  llvm::Expected CalculateNumChildren() override {
+    return m_backend.GetNumChildren();
+  }
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override {
     return m_backend.GetChildAtIndex(idx);
@@ -84,7 +86,8 @@ ConstString ValueObjectSynthetic::GetDisplayTypeName() {
   return m_parent->GetDisplayTypeName();
 }
 
-uint32_t ValueObjectSynthetic::CalculateNumChildren(uint32_t max) {
+llvm::Expected
+ValueObjectSynthetic::CalculateNumChildren(uint32_t max) {
   Log *log = GetLog(LLDBLog::DataFormatters);
 
   UpdateValueIfNeeded();
@@ -92,18 +95,23 @@ uint32_t ValueObjectSynthetic::CalculateNumChildren(uint32_t max) {
     return m_synthetic_children_count <= max ? m_synthetic_children_count : max;
 
   if (max < UINT32_MAX) {
-    size_t num_children = m_synth_filter_up->CalculateNumChildren(max);
+    auto num_children = m_synth_filter_up->CalculateNumChildren(max);
     LLDB_LOGF(log,
               "[ValueObjectSynthetic::CalculateNumChildren] for VO of name "
-              "%s and type %s, the filter returned %zu child values",
-              GetName().AsCString(), GetTypeName().AsCString(), num_children);
+              "%s and type %s, the filter returned %u child values",
+              GetName().AsCString(), GetTypeName().AsCString(),
+              num_children ? *num_children : 0);
     return num_children;
   } else {
-    size_t num_children = (m_synthetic_children_count =
-                               m_synth_filter_up->CalculateNumChildren(max));
+    auto num_children_or_err = m_synth_filter_up->CalculateNumChildren(max);
+    if (!num_children_or_err) {
+      m_synthetic_children_count = 0;
+      return num_children_or_err;
+    }
+    auto num_children = (m_synthetic_children_count = *num_children_or_err);
     LLDB_LOGF(log,
               "[ValueObjectSynthetic::CalculateNumChildren] for VO of name "
-              "%s and type %s, the filter returned %zu child values",
+              "%s and type %s, the filter returned %u child values",
               GetName().AsCString(), GetTypeName().AsCString(), num_children);
     return num_children;
   }
diff --git a/lldb/source/Core/ValueObjectVTable.cpp b/lldb/source/Core/ValueObjectVTable.cpp
index 4d1cbb8d2f6f..9bfe8dc9700d 100644
--- a/lldb/source/Core/ValueObjectVTable.cpp
+++ b/lldb/source/Core/ValueObjectVTable.cpp
@@ -33,7 +33,9 @@ public:
 
   std::optional GetByteSize() override { return m_addr_size; };
 
-  uint32_t CalculateNumChildren(uint32_t max) override { return 0; };
+  llvm::Expected CalculateNumChildren(uint32_t max) override {
+    return 0;
+  };
 
   ValueType GetValueType() const override { return eValueTypeVTableEntry; };
 
@@ -159,7 +161,7 @@ std::optional ValueObjectVTable::GetByteSize() {
   return std::nullopt;
 }
 
-uint32_t ValueObjectVTable::CalculateNumChildren(uint32_t max) {
+llvm::Expected ValueObjectVTable::CalculateNumChildren(uint32_t max) {
   if (UpdateValueIfNeeded(false))
     return m_num_vtable_entries <= max ? m_num_vtable_entries : max;
   return 0;
diff --git a/lldb/source/Core/ValueObjectVariable.cpp b/lldb/source/Core/ValueObjectVariable.cpp
index dc62bb6358dc..fb29c22c0ab5 100644
--- a/lldb/source/Core/ValueObjectVariable.cpp
+++ b/lldb/source/Core/ValueObjectVariable.cpp
@@ -94,7 +94,8 @@ ConstString ValueObjectVariable::GetQualifiedTypeName() {
   return ConstString();
 }
 
-uint32_t ValueObjectVariable::CalculateNumChildren(uint32_t max) {
+llvm::Expected
+ValueObjectVariable::CalculateNumChildren(uint32_t max) {
   CompilerType type(GetCompilerType());
 
   if (!type.IsValid())
@@ -103,7 +104,9 @@ uint32_t ValueObjectVariable::CalculateNumChildren(uint32_t max) {
   ExecutionContext exe_ctx(GetExecutionContextRef());
   const bool omit_empty_base_classes = true;
   auto child_count = type.GetNumChildren(omit_empty_base_classes, &exe_ctx);
-  return child_count <= max ? child_count : max;
+  if (!child_count)
+    return child_count;
+  return *child_count <= max ? *child_count : max;
 }
 
 std::optional ValueObjectVariable::GetByteSize() {
diff --git a/lldb/source/DataFormatters/FormatManager.cpp b/lldb/source/DataFormatters/FormatManager.cpp
index 092fa3c8ce49..d7ba5b4b70c9 100644
--- a/lldb/source/DataFormatters/FormatManager.cpp
+++ b/lldb/source/DataFormatters/FormatManager.cpp
@@ -451,8 +451,13 @@ bool FormatManager::ShouldPrintAsOneLiner(ValueObject &valobj) {
   if (valobj.GetSummaryFormat().get() != nullptr)
     return valobj.GetSummaryFormat()->IsOneLiner();
 
+  auto num_children = valobj.GetNumChildren();
+  if (!num_children) {
+    llvm::consumeError(num_children.takeError());
+    return true;
+  }
   // no children, no party
-  if (valobj.GetNumChildren() == 0)
+  if (*num_children == 0)
     return false;
 
   // ask the type if it has any opinion about this eLazyBoolCalculate == no
@@ -471,7 +476,7 @@ bool FormatManager::ShouldPrintAsOneLiner(ValueObject &valobj) {
 
   size_t total_children_name_len = 0;
 
-  for (size_t idx = 0; idx < valobj.GetNumChildren(); idx++) {
+  for (size_t idx = 0; idx < *num_children; idx++) {
     bool is_synth_val = false;
     ValueObjectSP child_sp(valobj.GetChildAtIndex(idx));
     // something is wrong here - bail out
@@ -523,7 +528,7 @@ bool FormatManager::ShouldPrintAsOneLiner(ValueObject &valobj) {
     }
 
     // if this child has children..
-    if (child_sp->GetNumChildren()) {
+    if (child_sp->HasChildren()) {
       // ...and no summary...
       // (if it had a summary and the summary wanted children, we would have
       // bailed out anyway
diff --git a/lldb/source/DataFormatters/TypeSynthetic.cpp b/lldb/source/DataFormatters/TypeSynthetic.cpp
index 0ae38c4d31f2..7aa0670190b2 100644
--- a/lldb/source/DataFormatters/TypeSynthetic.cpp
+++ b/lldb/source/DataFormatters/TypeSynthetic.cpp
@@ -115,6 +115,16 @@ std::string CXXSyntheticChildren::GetDescription() {
   return std::string(sstr.GetString());
 }
 
+uint32_t
+SyntheticChildrenFrontEnd::CalculateNumChildrenIgnoringErrors(uint32_t max) {
+  auto value_or_err = CalculateNumChildren(max);
+  if (value_or_err)
+    return *value_or_err;
+  LLDB_LOG_ERRORV(GetLog(LLDBLog::DataFormatters), value_or_err.takeError(),
+                  "{0}");
+  return 0;
+}
+
 lldb::ValueObjectSP SyntheticChildrenFrontEnd::CreateValueObjectFromExpression(
     llvm::StringRef name, llvm::StringRef expression,
     const ExecutionContext &exe_ctx) {
@@ -178,13 +188,15 @@ bool ScriptedSyntheticChildren::FrontEnd::IsValid() {
   return (m_wrapper_sp && m_wrapper_sp->IsValid() && m_interpreter);
 }
 
-uint32_t ScriptedSyntheticChildren::FrontEnd::CalculateNumChildren() {
+llvm::Expected
+ScriptedSyntheticChildren::FrontEnd::CalculateNumChildren() {
   if (!m_wrapper_sp || m_interpreter == nullptr)
     return 0;
   return m_interpreter->CalculateNumChildren(m_wrapper_sp, UINT32_MAX);
 }
 
-uint32_t ScriptedSyntheticChildren::FrontEnd::CalculateNumChildren(uint32_t max) {
+llvm::Expected
+ScriptedSyntheticChildren::FrontEnd::CalculateNumChildren(uint32_t max) {
   if (!m_wrapper_sp || m_interpreter == nullptr)
     return 0;
   return m_interpreter->CalculateNumChildren(m_wrapper_sp, max);
diff --git a/lldb/source/DataFormatters/ValueObjectPrinter.cpp b/lldb/source/DataFormatters/ValueObjectPrinter.cpp
index 46e50a8d421a..b853199e878c 100644
--- a/lldb/source/DataFormatters/ValueObjectPrinter.cpp
+++ b/lldb/source/DataFormatters/ValueObjectPrinter.cpp
@@ -627,7 +627,7 @@ uint32_t ValueObjectPrinter::GetMaxNumChildrenToPrint(bool &print_dotdotdot) {
   if (m_options.m_pointer_as_array)
     return m_options.m_pointer_as_array.m_element_count;
 
-  size_t num_children = synth_valobj.GetNumChildren();
+  uint32_t num_children = synth_valobj.GetNumChildrenIgnoringErrors();
   print_dotdotdot = false;
   if (num_children) {
     const size_t max_num_children = GetMostSpecializedValue()
diff --git a/lldb/source/DataFormatters/VectorType.cpp b/lldb/source/DataFormatters/VectorType.cpp
index a0626a8cba77..19de204c2435 100644
--- a/lldb/source/DataFormatters/VectorType.cpp
+++ b/lldb/source/DataFormatters/VectorType.cpp
@@ -9,6 +9,7 @@
 #include "lldb/DataFormatters/VectorType.h"
 
 #include "lldb/Core/ValueObject.h"
+#include "lldb/Core/ValueObjectConstResult.h"
 #include "lldb/DataFormatters/FormattersHelpers.h"
 #include "lldb/Symbol/CompilerType.h"
 #include "lldb/Symbol/TypeSystem.h"
@@ -224,10 +225,16 @@ public:
 
   ~VectorTypeSyntheticFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override { return m_num_children; }
+  llvm::Expected CalculateNumChildren() override {
+    return m_num_children;
+  }
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override {
-    if (idx >= CalculateNumChildren())
+    auto num_children_or_err = CalculateNumChildren();
+    if (!num_children_or_err)
+      return ValueObjectConstResult::Create(
+          nullptr, Status(num_children_or_err.takeError()));
+    if (idx >= *num_children_or_err)
       return {};
     std::optional size = m_child_type.GetByteSize(nullptr);
     if (!size)
@@ -266,7 +273,7 @@ public:
   size_t GetIndexOfChildWithName(ConstString name) override {
     const char *item_name = name.GetCString();
     uint32_t idx = ExtractIndexFromString(item_name);
-    if (idx < UINT32_MAX && idx >= CalculateNumChildren())
+    if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
       return UINT32_MAX;
     return idx;
   }
@@ -293,7 +300,8 @@ bool lldb_private::formatters::VectorTypeSummaryProvider(
   s.PutChar('(');
   bool first = true;
 
-  size_t idx = 0, len = synthetic_children->CalculateNumChildren();
+  size_t idx = 0,
+         len = synthetic_children->CalculateNumChildrenIgnoringErrors();
 
   for (; idx < len; idx++) {
     auto child_sp = synthetic_children->GetChildAtIndex(idx);
diff --git a/lldb/source/Plugins/ABI/PowerPC/ABISysV_ppc64.cpp b/lldb/source/Plugins/ABI/PowerPC/ABISysV_ppc64.cpp
index 656f07437095..173b5613d1b8 100644
--- a/lldb/source/Plugins/ABI/PowerPC/ABISysV_ppc64.cpp
+++ b/lldb/source/Plugins/ABI/PowerPC/ABISysV_ppc64.cpp
@@ -768,7 +768,12 @@ private:
 
     // get number of children
     const bool omit_empty_base_classes = true;
-    uint32_t n = m_type.GetNumChildren(omit_empty_base_classes, nullptr);
+    auto n_or_err = m_type.GetNumChildren(omit_empty_base_classes, nullptr);
+    if (!n_or_err) {
+      LLDB_LOG_ERROR(m_log, n_or_err.takeError(), LOG_PREFIX "{0}");
+      return {};
+    }
+    uint32_t n = *n_or_err;
     if (!n) {
       LLDB_LOG(m_log, LOG_PREFIX "No children found in struct");
       return {};
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionSourceCode.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionSourceCode.cpp
index b48bbbecc0cd..3d43ed3f99ff 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionSourceCode.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionSourceCode.cpp
@@ -229,7 +229,7 @@ void AddLambdaCaptureDecls(StreamString &stream, StackFrame *frame,
   assert(frame);
 
   if (auto thisValSP = ClangExpressionUtil::GetLambdaValueObject(frame)) {
-    uint32_t numChildren = thisValSP->GetNumChildren();
+    uint32_t numChildren = thisValSP->GetNumChildrenIgnoringErrors();
     for (uint32_t i = 0; i < numChildren; ++i) {
       auto childVal = thisValSP->GetChildAtIndex(i);
       ConstString childName(childVal ? childVal->GetName() : ConstString(""));
diff --git a/lldb/source/Plugins/InstrumentationRuntime/TSan/InstrumentationRuntimeTSan.cpp b/lldb/source/Plugins/InstrumentationRuntime/TSan/InstrumentationRuntimeTSan.cpp
index 72293c5331f4..b2781aa5e7db 100644
--- a/lldb/source/Plugins/InstrumentationRuntime/TSan/InstrumentationRuntimeTSan.cpp
+++ b/lldb/source/Plugins/InstrumentationRuntime/TSan/InstrumentationRuntimeTSan.cpp
@@ -27,6 +27,8 @@
 #include "lldb/Target/StopInfo.h"
 #include "lldb/Target/Target.h"
 #include "lldb/Target/Thread.h"
+#include "lldb/Utility/LLDBLog.h"
+#include "lldb/Utility/Log.h"
 #include "lldb/Utility/RegularExpression.h"
 #include "lldb/Utility/Stream.h"
 
@@ -211,7 +213,7 @@ CreateStackTrace(ValueObjectSP o,
   auto trace_sp = std::make_shared();
   ValueObjectSP trace_value_object =
       o->GetValueForExpressionPath(trace_item_name.c_str());
-  size_t count = trace_value_object->GetNumChildren();
+  size_t count = trace_value_object->GetNumChildrenIgnoringErrors();
   for (size_t j = 0; j < count; j++) {
     addr_t trace_addr =
         trace_value_object->GetChildAtIndex(j)->GetValueAsUnsigned(0);
diff --git a/lldb/source/Plugins/Language/CPlusPlus/BlockPointer.cpp b/lldb/source/Plugins/Language/CPlusPlus/BlockPointer.cpp
index ef0f67d1e9f9..9a6e135e0083 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/BlockPointer.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/BlockPointer.cpp
@@ -74,7 +74,7 @@ public:
 
   ~BlockPointerSyntheticFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override {
+  llvm::Expected CalculateNumChildren() override {
     const bool omit_empty_base_classes = false;
     return m_block_struct_type.GetNumChildren(omit_empty_base_classes, nullptr);
   }
@@ -84,7 +84,7 @@ public:
       return lldb::ValueObjectSP();
     }
 
-    if (idx >= CalculateNumChildren()) {
+    if (idx >= CalculateNumChildrenIgnoringErrors()) {
       return lldb::ValueObjectSP();
     }
 
diff --git a/lldb/source/Plugins/Language/CPlusPlus/Coroutines.cpp b/lldb/source/Plugins/Language/CPlusPlus/Coroutines.cpp
index 3827f9c21eff..5e63d1d7b214 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/Coroutines.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/Coroutines.cpp
@@ -11,6 +11,8 @@
 #include "Plugins/TypeSystem/Clang/TypeSystemClang.h"
 #include "lldb/Symbol/Function.h"
 #include "lldb/Symbol/VariableList.h"
+#include "lldb/Utility/LLDBLog.h"
+#include "lldb/Utility/Log.h"
 
 using namespace lldb;
 using namespace lldb_private;
@@ -22,7 +24,7 @@ static lldb::addr_t GetCoroFramePtrFromHandle(ValueObjectSP valobj_sp) {
 
   // We expect a single pointer in the `coroutine_handle` class.
   // We don't care about its name.
-  if (valobj_sp->GetNumChildren() != 1)
+  if (valobj_sp->GetNumChildrenIgnoringErrors() != 1)
     return LLDB_INVALID_ADDRESS;
   ValueObjectSP ptr_sp(valobj_sp->GetChildAtIndex(0));
   if (!ptr_sp)
@@ -104,8 +106,8 @@ lldb_private::formatters::StdlibCoroutineHandleSyntheticFrontEnd::
 lldb_private::formatters::StdlibCoroutineHandleSyntheticFrontEnd::
     ~StdlibCoroutineHandleSyntheticFrontEnd() = default;
 
-uint32_t lldb_private::formatters::StdlibCoroutineHandleSyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    StdlibCoroutineHandleSyntheticFrontEnd::CalculateNumChildren() {
   if (!m_resume_ptr_sp || !m_destroy_ptr_sp)
     return 0;
 
diff --git a/lldb/source/Plugins/Language/CPlusPlus/Coroutines.h b/lldb/source/Plugins/Language/CPlusPlus/Coroutines.h
index 5c6a80b57ff4..1d4bc65e2637 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/Coroutines.h
+++ b/lldb/source/Plugins/Language/CPlusPlus/Coroutines.h
@@ -34,7 +34,7 @@ public:
 
   ~StdlibCoroutineHandleSyntheticFrontEnd() override;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
diff --git a/lldb/source/Plugins/Language/CPlusPlus/GenericBitset.cpp b/lldb/source/Plugins/Language/CPlusPlus/GenericBitset.cpp
index 6a9da1d17c76..33955dccb6cc 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/GenericBitset.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/GenericBitset.cpp
@@ -34,7 +34,9 @@ public:
 
   bool MightHaveChildren() override { return true; }
   lldb::ChildCacheState Update() override;
-  uint32_t CalculateNumChildren() override { return m_elements.size(); }
+  llvm::Expected CalculateNumChildren() override {
+    return m_elements.size();
+  }
   ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
 private:
diff --git a/lldb/source/Plugins/Language/CPlusPlus/GenericOptional.cpp b/lldb/source/Plugins/Language/CPlusPlus/GenericOptional.cpp
index c06afb53eb8a..23756de7f1e6 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/GenericOptional.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/GenericOptional.cpp
@@ -19,7 +19,7 @@ using namespace lldb_private;
 bool lldb_private::formatters::GenericOptionalSummaryProvider(
     ValueObject &valobj, Stream &stream, const TypeSummaryOptions &options) {
   stream.Printf(" Has Value=%s ",
-                valobj.GetNumChildren() == 0 ? "false" : "true");
+                valobj.GetNumChildrenIgnoringErrors() == 0 ? "false" : "true");
 
   return true;
 }
@@ -41,7 +41,9 @@ public:
   }
 
   bool MightHaveChildren() override { return true; }
-  uint32_t CalculateNumChildren() override { return m_has_value ? 1U : 0U; }
+  llvm::Expected CalculateNumChildren() override {
+    return m_has_value ? 1U : 0U;
+  }
 
   ValueObjectSP GetChildAtIndex(uint32_t idx) override;
   lldb::ChildCacheState Update() override;
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp
index bba887fec3ac..10a1fe039189 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp
@@ -63,7 +63,7 @@ lldb::ValueObjectSP
 lldb_private::formatters::GetSecondValueOfLibCXXCompressedPair(
     ValueObject &pair) {
   ValueObjectSP value;
-  if (pair.GetNumChildren() > 1) {
+  if (pair.GetNumChildrenIgnoringErrors() > 1) {
     ValueObjectSP second_child = pair.GetChildAtIndex(1);
     if (second_child) {
       value = second_child->GetChildMemberWithName("__value_");
@@ -351,8 +351,8 @@ lldb_private::formatters::LibCxxMapIteratorSyntheticFrontEnd::Update() {
   return lldb::ChildCacheState::eRefetch;
 }
 
-uint32_t lldb_private::formatters::LibCxxMapIteratorSyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    LibCxxMapIteratorSyntheticFrontEnd::CalculateNumChildren() {
   return 2;
 }
 
@@ -509,8 +509,8 @@ lldb::ChildCacheState lldb_private::formatters::
   return lldb::ChildCacheState::eRefetch;
 }
 
-uint32_t lldb_private::formatters::LibCxxUnorderedMapIteratorSyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    LibCxxUnorderedMapIteratorSyntheticFrontEnd::CalculateNumChildren() {
   return 2;
 }
 
@@ -566,8 +566,8 @@ lldb_private::formatters::LibcxxSharedPtrSyntheticFrontEnd::
     Update();
 }
 
-uint32_t lldb_private::formatters::LibcxxSharedPtrSyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    LibcxxSharedPtrSyntheticFrontEnd::CalculateNumChildren() {
   return (m_cntrl ? 1 : 0);
 }
 
@@ -661,8 +661,8 @@ lldb_private::formatters::LibcxxUniquePtrSyntheticFrontEndCreator(
                     : nullptr);
 }
 
-uint32_t lldb_private::formatters::LibcxxUniquePtrSyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    LibcxxUniquePtrSyntheticFrontEnd::CalculateNumChildren() {
   if (m_value_ptr_sp)
     return m_deleter_sp ? 2 : 1;
   return 0;
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h
index ad2f58508ab7..a59f21841ec8 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h
@@ -87,7 +87,7 @@ class LibCxxMapIteratorSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   LibCxxMapIteratorSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -135,7 +135,7 @@ public:
 
   ~LibCxxUnorderedMapIteratorSyntheticFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -166,7 +166,7 @@ class LibcxxSharedPtrSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   LibcxxSharedPtrSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -186,7 +186,7 @@ class LibcxxUniquePtrSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   LibcxxUniquePtrSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxAtomic.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxAtomic.cpp
index 8e4c36103a74..7f30dc186291 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxAtomic.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxAtomic.cpp
@@ -90,7 +90,7 @@ public:
 
   ~LibcxxStdAtomicSyntheticFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -124,8 +124,8 @@ bool lldb_private::formatters::LibcxxStdAtomicSyntheticFrontEnd::
   return true;
 }
 
-uint32_t lldb_private::formatters::LibcxxStdAtomicSyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    LibcxxStdAtomicSyntheticFrontEnd::CalculateNumChildren() {
   return m_real_child ? 1 : 0;
 }
 
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxInitializerList.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxInitializerList.cpp
index 00012dfc056e..bd9c72497664 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxInitializerList.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxInitializerList.cpp
@@ -26,7 +26,7 @@ public:
 
   ~LibcxxInitializerListSyntheticFrontEnd() override;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -59,8 +59,8 @@ lldb_private::formatters::LibcxxInitializerListSyntheticFrontEnd::
   // delete m_start;
 }
 
-uint32_t lldb_private::formatters::LibcxxInitializerListSyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    LibcxxInitializerListSyntheticFrontEnd::CalculateNumChildren() {
   m_num_elements = 0;
   ValueObjectSP size_sp(m_backend.GetChildMemberWithName("__size_"));
   if (size_sp)
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxList.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxList.cpp
index 17f6b737d9f6..d7cfeb30557c 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxList.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxList.cpp
@@ -136,7 +136,7 @@ class ForwardListFrontEnd : public AbstractListFrontEnd {
 public:
   ForwardListFrontEnd(ValueObject &valobj);
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
   ValueObjectSP GetChildAtIndex(uint32_t idx) override;
   lldb::ChildCacheState Update() override;
 };
@@ -147,7 +147,7 @@ public:
 
   ~ListFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -240,7 +240,7 @@ ForwardListFrontEnd::ForwardListFrontEnd(ValueObject &valobj)
   Update();
 }
 
-uint32_t ForwardListFrontEnd::CalculateNumChildren() {
+llvm::Expected ForwardListFrontEnd::CalculateNumChildren() {
   if (m_count != UINT32_MAX)
     return m_count;
 
@@ -254,7 +254,7 @@ uint32_t ForwardListFrontEnd::CalculateNumChildren() {
 }
 
 ValueObjectSP ForwardListFrontEnd::GetChildAtIndex(uint32_t idx) {
-  if (idx >= CalculateNumChildren())
+  if (idx >= CalculateNumChildrenIgnoringErrors())
     return nullptr;
 
   if (!m_head)
@@ -308,7 +308,7 @@ ListFrontEnd::ListFrontEnd(lldb::ValueObjectSP valobj_sp)
     Update();
 }
 
-uint32_t ListFrontEnd::CalculateNumChildren() {
+llvm::Expected ListFrontEnd::CalculateNumChildren() {
   if (m_count != UINT32_MAX)
     return m_count;
   if (!m_head || !m_tail || m_node_address == 0)
@@ -347,7 +347,7 @@ lldb::ValueObjectSP ListFrontEnd::GetChildAtIndex(uint32_t idx) {
   static ConstString g_value("__value_");
   static ConstString g_next("__next_");
 
-  if (idx >= CalculateNumChildren())
+  if (idx >= CalculateNumChildrenIgnoringErrors())
     return lldb::ValueObjectSP();
 
   if (!m_head || !m_tail || m_node_address == 0)
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxMap.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxMap.cpp
index 6d24eb03779c..ec5b320e2218 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxMap.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxMap.cpp
@@ -177,7 +177,7 @@ public:
 
   ~LibcxxStdMapSyntheticFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -209,8 +209,8 @@ lldb_private::formatters::LibcxxStdMapSyntheticFrontEnd::
     Update();
 }
 
-uint32_t lldb_private::formatters::LibcxxStdMapSyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    LibcxxStdMapSyntheticFrontEnd::CalculateNumChildren() {
   if (m_count != UINT32_MAX)
     return m_count;
 
@@ -311,13 +311,13 @@ lldb_private::formatters::LibcxxStdMapSyntheticFrontEnd::GetChildAtIndex(
     uint32_t idx) {
   static ConstString g_cc_("__cc_"), g_cc("__cc");
   static ConstString g_nc("__nc");
-
-  if (idx >= CalculateNumChildren())
+  uint32_t num_children = CalculateNumChildrenIgnoringErrors();
+  if (idx >= num_children)
     return lldb::ValueObjectSP();
   if (m_tree == nullptr || m_root_node == nullptr)
     return lldb::ValueObjectSP();
 
-  MapIterator iterator(m_root_node, CalculateNumChildren());
+  MapIterator iterator(m_root_node, num_children);
 
   const bool need_to_skip = (idx > 0);
   size_t actual_advancde = idx;
@@ -382,7 +382,7 @@ lldb_private::formatters::LibcxxStdMapSyntheticFrontEnd::GetChildAtIndex(
   name.Printf("[%" PRIu64 "]", (uint64_t)idx);
   auto potential_child_sp = iterated_sp->Clone(ConstString(name.GetString()));
   if (potential_child_sp) {
-    switch (potential_child_sp->GetNumChildren()) {
+    switch (potential_child_sp->GetNumChildrenIgnoringErrors()) {
     case 1: {
       auto child0_sp = potential_child_sp->GetChildAtIndex(0);
       if (child0_sp &&
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxQueue.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxQueue.cpp
index fbadee89b7b7..5b459a17fe29 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxQueue.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxQueue.cpp
@@ -28,7 +28,7 @@ public:
   bool MightHaveChildren() override { return true; }
   lldb::ChildCacheState Update() override;
 
-  uint32_t CalculateNumChildren() override {
+  llvm::Expected CalculateNumChildren() override {
     return m_container_sp ? m_container_sp->GetNumChildren() : 0;
   }
 
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxRangesRefView.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxRangesRefView.cpp
index 74f54f767356..01a7b8f142ec 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxRangesRefView.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxRangesRefView.cpp
@@ -27,7 +27,7 @@ public:
 
   ~LibcxxStdRangesRefViewSyntheticFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override {
+  llvm::Expected CalculateNumChildren() override {
     // __range_ will be the sole child of this type
     return 1;
   }
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxSpan.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxSpan.cpp
index af2b51d2b540..9895f336bfd0 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxSpan.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxSpan.cpp
@@ -27,7 +27,7 @@ public:
 
   ~LibcxxStdSpanSyntheticFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -73,8 +73,8 @@ lldb_private::formatters::LibcxxStdSpanSyntheticFrontEnd::
     Update();
 }
 
-uint32_t lldb_private::formatters::LibcxxStdSpanSyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    LibcxxStdSpanSyntheticFrontEnd::CalculateNumChildren() {
   return m_num_elements;
 }
 
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxTuple.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxTuple.cpp
index 62bb7d619267..3e3259ab428d 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxTuple.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxTuple.cpp
@@ -26,7 +26,9 @@ public:
 
   bool MightHaveChildren() override { return true; }
   lldb::ChildCacheState Update() override;
-  uint32_t CalculateNumChildren() override { return m_elements.size(); }
+  llvm::Expected CalculateNumChildren() override {
+    return m_elements.size();
+  }
   ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
 private:
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxUnorderedMap.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxUnorderedMap.cpp
index b3c364294335..af29fdb6d001 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxUnorderedMap.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxUnorderedMap.cpp
@@ -33,7 +33,7 @@ public:
 
   ~LibcxxStdUnorderedMapSyntheticFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -62,8 +62,8 @@ lldb_private::formatters::LibcxxStdUnorderedMapSyntheticFrontEnd::
     Update();
 }
 
-uint32_t lldb_private::formatters::LibcxxStdUnorderedMapSyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    LibcxxStdUnorderedMapSyntheticFrontEnd::CalculateNumChildren() {
   return m_num_elements;
 }
 
@@ -94,7 +94,7 @@ static bool isUnorderedMap(ConstString type_name) {
 
 lldb::ValueObjectSP lldb_private::formatters::
     LibcxxStdUnorderedMapSyntheticFrontEnd::GetChildAtIndex(uint32_t idx) {
-  if (idx >= CalculateNumChildren())
+  if (idx >= CalculateNumChildrenIgnoringErrors())
     return lldb::ValueObjectSP();
   if (m_tree == nullptr)
     return lldb::ValueObjectSP();
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxValarray.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxValarray.cpp
index 463c7b8d7ce3..99f94406e99a 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxValarray.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxValarray.cpp
@@ -24,7 +24,7 @@ public:
 
   ~LibcxxStdValarraySyntheticFrontEnd() override;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -63,8 +63,8 @@ lldb_private::formatters::LibcxxStdValarraySyntheticFrontEnd::
   // delete m_finish;
 }
 
-uint32_t lldb_private::formatters::LibcxxStdValarraySyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    LibcxxStdValarraySyntheticFrontEnd::CalculateNumChildren() {
   if (!m_start || !m_finish)
     return 0;
   uint64_t start_val = m_start->GetValueAsUnsigned(0);
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxVariant.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxVariant.cpp
index 1f62062f09be..62794318e077 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxVariant.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxVariant.cpp
@@ -205,7 +205,7 @@ public:
 
   bool MightHaveChildren() override { return true; }
   lldb::ChildCacheState Update() override;
-  uint32_t CalculateNumChildren() override { return m_size; }
+  llvm::Expected CalculateNumChildren() override { return m_size; }
   ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
 private:
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxVector.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxVector.cpp
index fcf727ad2ea0..461fed35164b 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxVector.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxVector.cpp
@@ -25,7 +25,7 @@ public:
 
   ~LibcxxStdVectorSyntheticFrontEnd() override;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -46,7 +46,7 @@ class LibcxxVectorBoolSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   LibcxxVectorBoolSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -82,8 +82,8 @@ lldb_private::formatters::LibcxxStdVectorSyntheticFrontEnd::
   // delete m_finish;
 }
 
-uint32_t lldb_private::formatters::LibcxxStdVectorSyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    LibcxxStdVectorSyntheticFrontEnd::CalculateNumChildren() {
   if (!m_start || !m_finish)
     return 0;
   uint64_t start_val = m_start->GetValueAsUnsigned(0);
@@ -165,8 +165,8 @@ lldb_private::formatters::LibcxxVectorBoolSyntheticFrontEnd::
   }
 }
 
-uint32_t lldb_private::formatters::LibcxxVectorBoolSyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    LibcxxVectorBoolSyntheticFrontEnd::CalculateNumChildren() {
   return m_count;
 }
 
@@ -259,7 +259,7 @@ size_t lldb_private::formatters::LibcxxVectorBoolSyntheticFrontEnd::
     return UINT32_MAX;
   const char *item_name = name.GetCString();
   uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
     return UINT32_MAX;
   return idx;
 }
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.cpp
index 5abb3d50674b..86bb575af5ca 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.cpp
@@ -43,7 +43,7 @@ class LibstdcppMapIteratorSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   explicit LibstdcppMapIteratorSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -64,7 +64,7 @@ class LibStdcppSharedPtrSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   explicit LibStdcppSharedPtrSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -132,7 +132,8 @@ lldb::ChildCacheState LibstdcppMapIteratorSyntheticFrontEnd::Update() {
   return lldb::ChildCacheState::eReuse;
 }
 
-uint32_t LibstdcppMapIteratorSyntheticFrontEnd::CalculateNumChildren() {
+llvm::Expected
+LibstdcppMapIteratorSyntheticFrontEnd::CalculateNumChildren() {
   return 2;
 }
 
@@ -219,7 +220,10 @@ lldb::ChildCacheState VectorIteratorSyntheticFrontEnd::Update() {
   return lldb::ChildCacheState::eRefetch;
 }
 
-uint32_t VectorIteratorSyntheticFrontEnd::CalculateNumChildren() { return 1; }
+llvm::Expected
+VectorIteratorSyntheticFrontEnd::CalculateNumChildren() {
+  return 1;
+}
 
 lldb::ValueObjectSP
 VectorIteratorSyntheticFrontEnd::GetChildAtIndex(uint32_t idx) {
@@ -371,7 +375,10 @@ LibStdcppSharedPtrSyntheticFrontEnd::LibStdcppSharedPtrSyntheticFrontEnd(
     Update();
 }
 
-uint32_t LibStdcppSharedPtrSyntheticFrontEnd::CalculateNumChildren() { return 1; }
+llvm::Expected
+LibStdcppSharedPtrSyntheticFrontEnd::CalculateNumChildren() {
+  return 1;
+}
 
 lldb::ValueObjectSP
 LibStdcppSharedPtrSyntheticFrontEnd::GetChildAtIndex(uint32_t idx) {
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibStdcppTuple.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibStdcppTuple.cpp
index 64d2ec9d943a..05199ba35b9a 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibStdcppTuple.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibStdcppTuple.cpp
@@ -26,7 +26,7 @@ class LibStdcppTupleSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   explicit LibStdcppTupleSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -65,7 +65,7 @@ lldb::ChildCacheState LibStdcppTupleSyntheticFrontEnd::Update() {
     ValueObjectSP current_child = next_child_sp;
     next_child_sp = nullptr;
 
-    size_t child_count = current_child->GetNumChildren();
+    size_t child_count = current_child->GetNumChildrenIgnoringErrors();
     for (size_t i = 0; i < child_count; ++i) {
       ValueObjectSP child_sp = current_child->GetChildAtIndex(i);
       llvm::StringRef name_str = child_sp->GetName().GetStringRef();
@@ -95,7 +95,8 @@ LibStdcppTupleSyntheticFrontEnd::GetChildAtIndex(uint32_t idx) {
   return lldb::ValueObjectSP();
 }
 
-uint32_t LibStdcppTupleSyntheticFrontEnd::CalculateNumChildren() {
+llvm::Expected
+LibStdcppTupleSyntheticFrontEnd::CalculateNumChildren() {
   return m_members.size();
 }
 
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibStdcppUniquePointer.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibStdcppUniquePointer.cpp
index 3a48fe412e07..92f540d9ca52 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibStdcppUniquePointer.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibStdcppUniquePointer.cpp
@@ -26,7 +26,7 @@ class LibStdcppUniquePtrSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   explicit LibStdcppUniquePtrSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -135,7 +135,8 @@ LibStdcppUniquePtrSyntheticFrontEnd::GetChildAtIndex(uint32_t idx) {
   return lldb::ValueObjectSP();
 }
 
-uint32_t LibStdcppUniquePtrSyntheticFrontEnd::CalculateNumChildren() {
+llvm::Expected
+LibStdcppUniquePtrSyntheticFrontEnd::CalculateNumChildren() {
   if (m_del_obj)
     return 2;
   return 1;
diff --git a/lldb/source/Plugins/Language/ObjC/Cocoa.cpp b/lldb/source/Plugins/Language/ObjC/Cocoa.cpp
index cb740f8e71e1..96166657ceeb 100644
--- a/lldb/source/Plugins/Language/ObjC/Cocoa.cpp
+++ b/lldb/source/Plugins/Language/ObjC/Cocoa.cpp
@@ -1038,7 +1038,7 @@ public:
 
   ~ObjCClassSyntheticChildrenFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override { return 0; }
+  llvm::Expected CalculateNumChildren() override { return 0; }
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override {
     return lldb::ValueObjectSP();
diff --git a/lldb/source/Plugins/Language/ObjC/NSArray.cpp b/lldb/source/Plugins/Language/ObjC/NSArray.cpp
index 7f060b2613d6..67d0cd08f51a 100644
--- a/lldb/source/Plugins/Language/ObjC/NSArray.cpp
+++ b/lldb/source/Plugins/Language/ObjC/NSArray.cpp
@@ -50,7 +50,7 @@ public:
 
   ~NSArrayMSyntheticFrontEndBase() override = default;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -214,7 +214,7 @@ public:
 
   ~GenericNSArrayISyntheticFrontEnd() override;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -302,7 +302,7 @@ public:
 
   ~NSArray0SyntheticFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -319,7 +319,7 @@ public:
 
   ~NSArray1SyntheticFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -477,15 +477,15 @@ lldb_private::formatters::
     : NSArrayMSyntheticFrontEndBase(valobj_sp), m_data_32(nullptr),
       m_data_64(nullptr) {}
 
-uint32_t lldb_private::formatters::NSArrayMSyntheticFrontEndBase::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    NSArrayMSyntheticFrontEndBase::CalculateNumChildren() {
   return GetUsedCount();
 }
 
 lldb::ValueObjectSP
 lldb_private::formatters::NSArrayMSyntheticFrontEndBase::GetChildAtIndex(
     uint32_t idx) {
-  if (idx >= CalculateNumChildren())
+  if (idx >= CalculateNumChildrenIgnoringErrors())
     return lldb::ValueObjectSP();
   lldb::addr_t object_at_idx = GetDataAddress();
   size_t pyhs_idx = idx;
@@ -542,7 +542,7 @@ lldb_private::formatters::NSArrayMSyntheticFrontEndBase::GetIndexOfChildWithName
     ConstString name) {
   const char *item_name = name.GetCString();
   uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
     return UINT32_MAX;
   return idx;
 }
@@ -628,15 +628,15 @@ lldb_private::formatters::GenericNSArrayISyntheticFrontEnd::
   GetIndexOfChildWithName(ConstString name) {
   const char *item_name = name.GetCString();
   uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
     return UINT32_MAX;
   return idx;
 }
 
 template 
-uint32_t
-lldb_private::formatters::GenericNSArrayISyntheticFrontEnd::
-  CalculateNumChildren() {
+llvm::Expected
+lldb_private::formatters::GenericNSArrayISyntheticFrontEnd<
+    D32, D64, Inline>::CalculateNumChildren() {
   return m_data_32 ? m_data_32->used : m_data_64->used;
 }
 
@@ -685,7 +685,7 @@ template 
 lldb::ValueObjectSP
 lldb_private::formatters::GenericNSArrayISyntheticFrontEnd::
   GetChildAtIndex(uint32_t idx) {
-  if (idx >= CalculateNumChildren())
+  if (idx >= CalculateNumChildrenIgnoringErrors())
     return lldb::ValueObjectSP();
   lldb::addr_t object_at_idx;
   if (Inline) {
@@ -719,7 +719,7 @@ lldb_private::formatters::NSArray0SyntheticFrontEnd::GetIndexOfChildWithName(
   return UINT32_MAX;
 }
 
-uint32_t
+llvm::Expected
 lldb_private::formatters::NSArray0SyntheticFrontEnd::CalculateNumChildren() {
   return 0;
 }
@@ -754,7 +754,7 @@ lldb_private::formatters::NSArray1SyntheticFrontEnd::GetIndexOfChildWithName(
   return UINT32_MAX;
 }
 
-uint32_t
+llvm::Expected
 lldb_private::formatters::NSArray1SyntheticFrontEnd::CalculateNumChildren() {
   return 1;
 }
diff --git a/lldb/source/Plugins/Language/ObjC/NSDictionary.cpp b/lldb/source/Plugins/Language/ObjC/NSDictionary.cpp
index da94eda1529c..ec6fd756394a 100644
--- a/lldb/source/Plugins/Language/ObjC/NSDictionary.cpp
+++ b/lldb/source/Plugins/Language/ObjC/NSDictionary.cpp
@@ -103,7 +103,7 @@ public:
 
   ~NSDictionaryISyntheticFrontEnd() override;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -144,7 +144,7 @@ class NSConstantDictionarySyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   NSConstantDictionarySyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -176,7 +176,7 @@ class NSCFDictionarySyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   NSCFDictionarySyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -209,7 +209,7 @@ public:
 
   ~NSDictionary1SyntheticFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -230,7 +230,7 @@ public:
 
   ~GenericNSDictionaryMSyntheticFrontEnd() override;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -262,9 +262,9 @@ namespace Foundation1100 {
     NSDictionaryMSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
     
     ~NSDictionaryMSyntheticFrontEnd() override;
-    
-    uint32_t CalculateNumChildren() override;
-    
+
+    llvm::Expected CalculateNumChildren() override;
+
     lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
     lldb::ChildCacheState Update() override;
@@ -601,13 +601,13 @@ size_t lldb_private::formatters::NSDictionaryISyntheticFrontEnd::
     GetIndexOfChildWithName(ConstString name) {
   const char *item_name = name.GetCString();
   uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
     return UINT32_MAX;
   return idx;
 }
 
-uint32_t lldb_private::formatters::NSDictionaryISyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    NSDictionaryISyntheticFrontEnd::CalculateNumChildren() {
   if (!m_data_32 && !m_data_64)
     return 0;
   return (m_data_32 ? m_data_32->_used : m_data_64->_used);
@@ -656,7 +656,7 @@ bool lldb_private::formatters::NSDictionaryISyntheticFrontEnd::
 lldb::ValueObjectSP
 lldb_private::formatters::NSDictionaryISyntheticFrontEnd::GetChildAtIndex(
     uint32_t idx) {
-  uint32_t num_children = CalculateNumChildren();
+  uint32_t num_children = CalculateNumChildrenIgnoringErrors();
 
   if (idx >= num_children)
     return lldb::ValueObjectSP();
@@ -739,13 +739,13 @@ size_t lldb_private::formatters::NSCFDictionarySyntheticFrontEnd::
     GetIndexOfChildWithName(ConstString name) {
   const char *item_name = name.GetCString();
   const uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
     return UINT32_MAX;
   return idx;
 }
 
-uint32_t lldb_private::formatters::NSCFDictionarySyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    NSCFDictionarySyntheticFrontEnd::CalculateNumChildren() {
   if (!m_hashtable.IsValid())
     return 0;
   return m_hashtable.GetCount();
@@ -781,7 +781,7 @@ lldb_private::formatters::NSCFDictionarySyntheticFrontEnd::GetChildAtIndex(
   lldb::addr_t m_keys_ptr = m_hashtable.GetKeyPointer();
   lldb::addr_t m_values_ptr = m_hashtable.GetValuePointer();
 
-  const uint32_t num_children = CalculateNumChildren();
+  const uint32_t num_children = CalculateNumChildrenIgnoringErrors();
 
   if (idx >= num_children)
     return lldb::ValueObjectSP();
@@ -875,13 +875,13 @@ size_t lldb_private::formatters::NSConstantDictionarySyntheticFrontEnd::
     GetIndexOfChildWithName(ConstString name) {
   const char *item_name = name.GetCString();
   uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
     return UINT32_MAX;
   return idx;
 }
 
-uint32_t lldb_private::formatters::NSConstantDictionarySyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    NSConstantDictionarySyntheticFrontEnd::CalculateNumChildren() {
   return m_size;
 }
 
@@ -921,7 +921,7 @@ bool lldb_private::formatters::NSConstantDictionarySyntheticFrontEnd::
 
 lldb::ValueObjectSP lldb_private::formatters::
     NSConstantDictionarySyntheticFrontEnd::GetChildAtIndex(uint32_t idx) {
-  uint32_t num_children = CalculateNumChildren();
+  uint32_t num_children = CalculateNumChildrenIgnoringErrors();
 
   if (idx >= num_children)
     return lldb::ValueObjectSP();
@@ -994,8 +994,8 @@ size_t lldb_private::formatters::NSDictionary1SyntheticFrontEnd::
   return name == g_zero ? 0 : UINT32_MAX;
 }
 
-uint32_t lldb_private::formatters::NSDictionary1SyntheticFrontEnd::
-    CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::
+    NSDictionary1SyntheticFrontEnd::CalculateNumChildren() {
   return 1;
 }
 
@@ -1081,17 +1081,18 @@ size_t lldb_private::formatters::GenericNSDictionaryMSyntheticFrontEnd<
     D32, D64>::GetIndexOfChildWithName(ConstString name) {
   const char *item_name = name.GetCString();
   uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
     return UINT32_MAX;
   return idx;
 }
 
 template 
-uint32_t
-lldb_private::formatters::GenericNSDictionaryMSyntheticFrontEnd::CalculateNumChildren() {
+llvm::Expected
+lldb_private::formatters::GenericNSDictionaryMSyntheticFrontEnd<
+    D32, D64>::CalculateNumChildren() {
   if (!m_data_32 && !m_data_64)
     return 0;
-  return (m_data_32 ? m_data_32->_used : m_data_64->_used);
+  return (m_data_32 ? (uint32_t)m_data_32->_used : (uint32_t)m_data_64->_used);
 }
 
 template 
@@ -1153,7 +1154,7 @@ lldb_private::formatters::GenericNSDictionaryMSyntheticFrontEnd<
     m_values_ptr = m_data_64->_buffer + (m_ptr_size * size);
   }
 
-  uint32_t num_children = CalculateNumChildren();
+  uint32_t num_children = CalculateNumChildrenIgnoringErrors();
 
   if (idx >= num_children)
     return lldb::ValueObjectSP();
@@ -1245,14 +1246,13 @@ lldb_private::formatters::Foundation1100::
   NSDictionaryMSyntheticFrontEnd::GetIndexOfChildWithName(ConstString name) {
   const char *item_name = name.GetCString();
   uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
     return UINT32_MAX;
   return idx;
 }
 
-uint32_t
-lldb_private::formatters::Foundation1100::
-  NSDictionaryMSyntheticFrontEnd::CalculateNumChildren() {
+llvm::Expected lldb_private::formatters::Foundation1100::
+    NSDictionaryMSyntheticFrontEnd::CalculateNumChildren() {
   if (!m_data_32 && !m_data_64)
     return 0;
   return (m_data_32 ? m_data_32->_used : m_data_64->_used);
@@ -1306,7 +1306,7 @@ lldb_private::formatters::Foundation1100::
   lldb::addr_t m_values_ptr =
       (m_data_32 ? m_data_32->_objs_addr : m_data_64->_objs_addr);
 
-  uint32_t num_children = CalculateNumChildren();
+  uint32_t num_children = CalculateNumChildrenIgnoringErrors();
 
   if (idx >= num_children)
     return lldb::ValueObjectSP();
diff --git a/lldb/source/Plugins/Language/ObjC/NSError.cpp b/lldb/source/Plugins/Language/ObjC/NSError.cpp
index b034e799b716..5ef7edc7e80c 100644
--- a/lldb/source/Plugins/Language/ObjC/NSError.cpp
+++ b/lldb/source/Plugins/Language/ObjC/NSError.cpp
@@ -116,7 +116,7 @@ public:
   // no need to delete m_child_ptr - it's kept alive by the cluster manager on
   // our behalf
 
-  uint32_t CalculateNumChildren() override {
+  llvm::Expected CalculateNumChildren() override {
     if (m_child_ptr)
       return 1;
     if (m_child_sp)
diff --git a/lldb/source/Plugins/Language/ObjC/NSException.cpp b/lldb/source/Plugins/Language/ObjC/NSException.cpp
index 09d3a1b42b74..e7ce26ea4c6f 100644
--- a/lldb/source/Plugins/Language/ObjC/NSException.cpp
+++ b/lldb/source/Plugins/Language/ObjC/NSException.cpp
@@ -123,9 +123,7 @@ public:
 
   ~NSExceptionSyntheticFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override {
-    return 4;
-  }
+  llvm::Expected CalculateNumChildren() override { return 4; }
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override {
     switch (idx) {
diff --git a/lldb/source/Plugins/Language/ObjC/NSIndexPath.cpp b/lldb/source/Plugins/Language/ObjC/NSIndexPath.cpp
index 10bb907c58ed..a434cee09d38 100644
--- a/lldb/source/Plugins/Language/ObjC/NSIndexPath.cpp
+++ b/lldb/source/Plugins/Language/ObjC/NSIndexPath.cpp
@@ -40,7 +40,9 @@ public:
 
   ~NSIndexPathSyntheticFrontEnd() override = default;
 
-  uint32_t CalculateNumChildren() override { return m_impl.GetNumIndexes(); }
+  llvm::Expected CalculateNumChildren() override {
+    return m_impl.GetNumIndexes();
+  }
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override {
     return m_impl.GetIndexAtIndex(idx, m_uint_star_type);
@@ -127,7 +129,7 @@ public:
   size_t GetIndexOfChildWithName(ConstString name) override {
     const char *item_name = name.GetCString();
     uint32_t idx = ExtractIndexFromString(item_name);
-    if (idx < UINT32_MAX && idx >= CalculateNumChildren())
+    if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
       return UINT32_MAX;
     return idx;
   }
diff --git a/lldb/source/Plugins/Language/ObjC/NSSet.cpp b/lldb/source/Plugins/Language/ObjC/NSSet.cpp
index c965a2a13400..7d0a6a507211 100644
--- a/lldb/source/Plugins/Language/ObjC/NSSet.cpp
+++ b/lldb/source/Plugins/Language/ObjC/NSSet.cpp
@@ -46,7 +46,7 @@ public:
 
   ~NSSetISyntheticFrontEnd() override;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -84,7 +84,7 @@ class NSCFSetSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 public:
   NSCFSetSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -117,7 +117,7 @@ public:
 
   ~GenericNSSetMSyntheticFrontEnd() override;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -233,7 +233,7 @@ public:
 
   ~NSSetCodeRunningSyntheticFrontEnd() override;
 
-  uint32_t CalculateNumChildren() override;
+  llvm::Expected CalculateNumChildren() override;
 
   lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
 
@@ -414,12 +414,12 @@ lldb_private::formatters::NSSetISyntheticFrontEnd::GetIndexOfChildWithName(
     ConstString name) {
   const char *item_name = name.GetCString();
   uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
     return UINT32_MAX;
   return idx;
 }
 
-uint32_t
+llvm::Expected
 lldb_private::formatters::NSSetISyntheticFrontEnd::CalculateNumChildren() {
   if (!m_data_32 && !m_data_64)
     return 0;
@@ -468,7 +468,7 @@ bool lldb_private::formatters::NSSetISyntheticFrontEnd::MightHaveChildren() {
 lldb::ValueObjectSP
 lldb_private::formatters::NSSetISyntheticFrontEnd::GetChildAtIndex(
     uint32_t idx) {
-  uint32_t num_children = CalculateNumChildren();
+  uint32_t num_children = CalculateNumChildrenIgnoringErrors();
 
   if (idx >= num_children)
     return lldb::ValueObjectSP();
@@ -551,12 +551,12 @@ lldb_private::formatters::NSCFSetSyntheticFrontEnd::GetIndexOfChildWithName(
     ConstString name) {
   const char *item_name = name.GetCString();
   const uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
     return UINT32_MAX;
   return idx;
 }
 
-uint32_t
+llvm::Expected
 lldb_private::formatters::NSCFSetSyntheticFrontEnd::CalculateNumChildren() {
   if (!m_hashtable.IsValid())
     return 0;
@@ -591,7 +591,7 @@ lldb_private::formatters::NSCFSetSyntheticFrontEnd::GetChildAtIndex(
     uint32_t idx) {
   lldb::addr_t m_values_ptr = m_hashtable.GetValuePointer();
 
-  const uint32_t num_children = CalculateNumChildren();
+  const uint32_t num_children = CalculateNumChildrenIgnoringErrors();
 
   if (idx >= num_children)
     return lldb::ValueObjectSP();
@@ -691,18 +691,18 @@ lldb_private::formatters::
     ConstString name) {
   const char *item_name = name.GetCString();
   uint32_t idx = ExtractIndexFromString(item_name);
-  if (idx < UINT32_MAX && idx >= CalculateNumChildren())
+  if (idx < UINT32_MAX && idx >= CalculateNumChildrenIgnoringErrors())
     return UINT32_MAX;
   return idx;
 }
 
 template 
-uint32_t
-lldb_private::formatters::
-  GenericNSSetMSyntheticFrontEnd::CalculateNumChildren() {
+llvm::Expected
+lldb_private::formatters::GenericNSSetMSyntheticFrontEnd<
+    D32, D64>::CalculateNumChildren() {
   if (!m_data_32 && !m_data_64)
     return 0;
-  return (m_data_32 ? m_data_32->_used : m_data_64->_used);
+  return (m_data_32 ? (uint32_t)m_data_32->_used : (uint32_t)m_data_64->_used);
 }
 
 template 
@@ -753,7 +753,7 @@ lldb_private::formatters::
   lldb::addr_t m_objs_addr =
       (m_data_32 ? m_data_32->_objs_addr : m_data_64->_objs_addr);
 
-  uint32_t num_children = CalculateNumChildren();
+  uint32_t num_children = CalculateNumChildrenIgnoringErrors();
 
   if (idx >= num_children)
     return lldb::ValueObjectSP();
diff --git a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntime.cpp b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntime.cpp
index f08f9f0f815d..9434376f7d9e 100644
--- a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntime.cpp
+++ b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntime.cpp
@@ -539,7 +539,8 @@ ThreadSP AppleObjCRuntime::GetBacktraceThreadFromException(
     return object;
   };
 
-  for (size_t idx = 0; idx < reserved_dict->GetNumChildren(); idx++) {
+  for (size_t idx = 0; idx < reserved_dict->GetNumChildrenIgnoringErrors();
+       idx++) {
     ValueObjectSP dict_entry = reserved_dict->GetChildAtIndex(idx);
 
     DataExtractor data;
diff --git a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
index 51ab13108feb..c02b08cb4782 100644
--- a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
+++ b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
@@ -5263,9 +5263,10 @@ GetDynamicArrayInfo(TypeSystemClang &ast, SymbolFile *sym_file,
   return std::nullopt;
 }
 
-uint32_t TypeSystemClang::GetNumChildren(lldb::opaque_compiler_type_t type,
-                                         bool omit_empty_base_classes,
-                                         const ExecutionContext *exe_ctx) {
+llvm::Expected
+TypeSystemClang::GetNumChildren(lldb::opaque_compiler_type_t type,
+                                bool omit_empty_base_classes,
+                                const ExecutionContext *exe_ctx) {
   if (!type)
     return 0;
 
@@ -5361,9 +5362,13 @@ uint32_t TypeSystemClang::GetNumChildren(lldb::opaque_compiler_type_t type,
     CompilerType pointee_clang_type(GetPointeeType(type));
 
     uint32_t num_pointee_children = 0;
-    if (pointee_clang_type.IsAggregateType())
-      num_pointee_children =
+    if (pointee_clang_type.IsAggregateType()) {
+      auto num_children_or_err =
           pointee_clang_type.GetNumChildren(omit_empty_base_classes, exe_ctx);
+      if (!num_children_or_err)
+        return num_children_or_err;
+      num_pointee_children = *num_children_or_err;
+    }
     // If this type points to a simple type, then it has 1 child
     if (num_pointee_children == 0)
       num_children = 1;
@@ -5397,9 +5402,13 @@ uint32_t TypeSystemClang::GetNumChildren(lldb::opaque_compiler_type_t type,
     clang::QualType pointee_type(pointer_type->getPointeeType());
     CompilerType pointee_clang_type(GetType(pointee_type));
     uint32_t num_pointee_children = 0;
-    if (pointee_clang_type.IsAggregateType())
-      num_pointee_children =
+    if (pointee_clang_type.IsAggregateType()) {
+      auto num_children_or_err =
           pointee_clang_type.GetNumChildren(omit_empty_base_classes, exe_ctx);
+      if (!num_children_or_err)
+        return num_children_or_err;
+      num_pointee_children = *num_children_or_err;
+    }
     if (num_pointee_children == 0) {
       // We have a pointer to a pointee type that claims it has no children. We
       // will want to look at
@@ -6108,8 +6117,15 @@ CompilerType TypeSystemClang::GetChildCompilerTypeAtIndex(
   child_is_base_class = false;
   language_flags = 0;
 
-  const bool idx_is_valid =
-      idx < GetNumChildren(type, omit_empty_base_classes, exe_ctx);
+  auto num_children_or_err =
+      GetNumChildren(type, omit_empty_base_classes, exe_ctx);
+  if (!num_children_or_err) {
+    LLDB_LOG_ERRORV(GetLog(LLDBLog::Types), num_children_or_err.takeError(),
+                    "{0}");
+    return {};
+  }
+
+  const bool idx_is_valid = idx < *num_children_or_err;
   int32_t bit_offset;
   switch (parent_type_class) {
   case clang::Type::Builtin:
@@ -6265,8 +6281,10 @@ CompilerType TypeSystemClang::GetChildCompilerTypeAtIndex(
               CompilerType base_class_clang_type =
                   GetType(getASTContext().getObjCInterfaceType(
                       superclass_interface_decl));
-              if (base_class_clang_type.GetNumChildren(omit_empty_base_classes,
-                                                       exe_ctx) > 0) {
+              if (llvm::expectedToStdOptional(
+                      base_class_clang_type.GetNumChildren(
+                          omit_empty_base_classes, exe_ctx))
+                      .value_or(0) > 0) {
                 if (idx == 0) {
                   clang::QualType ivar_qual_type(
                       getASTContext().getObjCInterfaceType(
diff --git a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
index a73164895baa..05c303baa416 100644
--- a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
+++ b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
@@ -833,9 +833,10 @@ public:
   GetTypeBitAlign(lldb::opaque_compiler_type_t type,
                   ExecutionContextScope *exe_scope) override;
 
-  uint32_t GetNumChildren(lldb::opaque_compiler_type_t type,
-                          bool omit_empty_base_classes,
-                          const ExecutionContext *exe_ctx) override;
+  llvm::Expected
+  GetNumChildren(lldb::opaque_compiler_type_t type,
+                 bool omit_empty_base_classes,
+                 const ExecutionContext *exe_ctx) override;
 
   CompilerType GetBuiltinTypeByName(ConstString name) override;
 
diff --git a/lldb/source/Symbol/CompilerType.cpp b/lldb/source/Symbol/CompilerType.cpp
index 76b79daa6ac1..85dd2d841a5a 100644
--- a/lldb/source/Symbol/CompilerType.cpp
+++ b/lldb/source/Symbol/CompilerType.cpp
@@ -770,8 +770,9 @@ lldb::Format CompilerType::GetFormat() const {
   return lldb::eFormatDefault;
 }
 
-uint32_t CompilerType::GetNumChildren(bool omit_empty_base_classes,
-                                      const ExecutionContext *exe_ctx) const {
+llvm::Expected
+CompilerType::GetNumChildren(bool omit_empty_base_classes,
+                             const ExecutionContext *exe_ctx) const {
   if (IsValid())
     if (auto type_system_sp = GetTypeSystem())
       return type_system_sp->GetNumChildren(m_type, omit_empty_base_classes,
diff --git a/lldb/source/Symbol/Type.cpp b/lldb/source/Symbol/Type.cpp
index 6069d066eaf6..44a24d7178f5 100644
--- a/lldb/source/Symbol/Type.cpp
+++ b/lldb/source/Symbol/Type.cpp
@@ -488,7 +488,7 @@ std::optional Type::GetByteSize(ExecutionContextScope *exe_scope) {
   return {};
 }
 
-uint32_t Type::GetNumChildren(bool omit_empty_base_classes) {
+llvm::Expected Type::GetNumChildren(bool omit_empty_base_classes) {
   return GetForwardCompilerType().GetNumChildren(omit_empty_base_classes, nullptr);
 }
 
diff --git a/lldb/source/Symbol/Variable.cpp b/lldb/source/Symbol/Variable.cpp
index a33c3433d9e2..90edede0f065 100644
--- a/lldb/source/Symbol/Variable.cpp
+++ b/lldb/source/Symbol/Variable.cpp
@@ -27,6 +27,8 @@
 #include "lldb/Target/StackFrame.h"
 #include "lldb/Target/Target.h"
 #include "lldb/Target/Thread.h"
+#include "lldb/Utility/LLDBLog.h"
+#include "lldb/Utility/Log.h"
 #include "lldb/Utility/RegularExpression.h"
 #include "lldb/Utility/Stream.h"
 
@@ -568,7 +570,9 @@ static void PrivateAutoComplete(
       case eTypeClassObjCObjectPointer:
       case eTypeClassPointer: {
         bool omit_empty_base_classes = true;
-        if (compiler_type.GetNumChildren(omit_empty_base_classes, nullptr) > 0)
+        if (llvm::expectedToStdOptional(
+                compiler_type.GetNumChildren(omit_empty_base_classes, nullptr))
+                .value_or(0))
           request.AddCompletion((prefix_path + "->").str());
         else {
           request.AddCompletion(prefix_path.str());
diff --git a/lldb/source/Target/StackFrame.cpp b/lldb/source/Target/StackFrame.cpp
index 50cf01e63cd4..c29a71d92572 100644
--- a/lldb/source/Target/StackFrame.cpp
+++ b/lldb/source/Target/StackFrame.cpp
@@ -857,10 +857,11 @@ ValueObjectSP StackFrame::GetValueForVariableExpressionPath(
                   "\"(%s) %s\" is not an array type",
                   valobj_sp->GetTypeName().AsCString(""),
                   var_expr_path_strm.GetData());
-            } else if (
-                static_cast(child_index) >=
-                synthetic
-                    ->GetNumChildren() /* synthetic does not have that many values */) {
+            } else if (static_cast(child_index) >=
+                       synthetic
+                           ->GetNumChildrenIgnoringErrors() /* synthetic does
+                                                                not have that
+                                                                many values */) {
               valobj_sp->GetExpressionPath(var_expr_path_strm);
               error.SetErrorStringWithFormat(
                   "array index %ld is not valid for \"(%s) %s\"", child_index,
@@ -929,10 +930,9 @@ ValueObjectSP StackFrame::GetValueForVariableExpressionPath(
                 "\"(%s) %s\" is not an array type",
                 valobj_sp->GetTypeName().AsCString(""),
                 var_expr_path_strm.GetData());
-          } else if (
-              static_cast(child_index) >=
-              synthetic
-                  ->GetNumChildren() /* synthetic does not have that many values */) {
+          } else if (static_cast(child_index) >=
+                     synthetic->GetNumChildrenIgnoringErrors() /* synthetic
+                                     does not have that many values */) {
             valobj_sp->GetExpressionPath(var_expr_path_strm);
             error.SetErrorStringWithFormat(
                 "array index %ld is not valid for \"(%s) %s\"", child_index,
@@ -1397,7 +1397,8 @@ ValueObjectSP GetValueForOffset(StackFrame &frame, ValueObjectSP &parent,
     return parent;
   }
 
-  for (int ci = 0, ce = parent->GetNumChildren(); ci != ce; ++ci) {
+  for (int ci = 0, ce = parent->GetNumChildrenIgnoringErrors(); ci != ce;
+       ++ci) {
     ValueObjectSP child_sp = parent->GetChildAtIndex(ci);
 
     if (!child_sp) {
-- 
GitLab


From 4d323e404d43526ad8263769f00aace9db2e57c5 Mon Sep 17 00:00:00 2001
From: Nikolas Klauser 
Date: Sat, 9 Mar 2024 01:09:28 +0100
Subject: [PATCH 0094/1674] [libc++] Allow the use of extensions in the
 implementation (#79532)

We've talked about allowing extensions on
[discourse](https://discourse.llvm.org/t/rfc-use-language-extensions-from-future-standards-in-libc/71898/5)
and in a libc++ monthly meeting and agreed to test it out in the LLVM 18
release. We've done that with the `tuple` constructor overload set
(using conditional `explicit`). Since we haven't heard about any
breakages, it seems safe to do. This patch enables the use of extension
from later C++ standards inside the versioned `std` namespaces. This
should be good enough, since almost all of our code is inside that
namespace. This approach also avoids the use of extensions inside the
test `std` suite. That part of the code base should stay clean, since
it's a test suite that is also used by other vendors to test their
implementations.
---
 libcxx/include/__config | 61 +++++++++++++++++++++++++++--------------
 libcxx/include/tuple    |  6 ----
 2 files changed, 41 insertions(+), 26 deletions(-)

diff --git a/libcxx/include/__config b/libcxx/include/__config
index 3a438e85a7b8..11e13e0c2498 100644
--- a/libcxx/include/__config
+++ b/libcxx/include/__config
@@ -406,6 +406,10 @@ _LIBCPP_HARDENING_MODE_DEBUG
 #    define __has_include(...) 0
 #  endif
 
+#  ifndef __has_warning
+#    define __has_warning(...) 0
+#  endif
+
 #  if !defined(_LIBCPP_COMPILER_CLANG_BASED) && __cplusplus < 201103L
 #    error "libc++ only supports C++03 with Clang-based compilers. Please enable C++11"
 #  endif
@@ -723,6 +727,23 @@ typedef __char32_t char32_t;
 #    define _LIBCPP_EXCLUDE_FROM_EXPLICIT_INSTANTIATION _LIBCPP_ALWAYS_INLINE
 #  endif
 
+#  ifdef _LIBCPP_COMPILER_CLANG_BASED
+#    define _LIBCPP_DIAGNOSTIC_PUSH _Pragma("clang diagnostic push")
+#    define _LIBCPP_DIAGNOSTIC_POP _Pragma("clang diagnostic pop")
+#    define _LIBCPP_CLANG_DIAGNOSTIC_IGNORED(str) _Pragma(_LIBCPP_TOSTRING(clang diagnostic ignored str))
+#    define _LIBCPP_GCC_DIAGNOSTIC_IGNORED(str)
+#  elif defined(_LIBCPP_COMPILER_GCC)
+#    define _LIBCPP_DIAGNOSTIC_PUSH _Pragma("GCC diagnostic push")
+#    define _LIBCPP_DIAGNOSTIC_POP _Pragma("GCC diagnostic pop")
+#    define _LIBCPP_CLANG_DIAGNOSTIC_IGNORED(str)
+#    define _LIBCPP_GCC_DIAGNOSTIC_IGNORED(str) _Pragma(_LIBCPP_TOSTRING(GCC diagnostic ignored str))
+#  else
+#    define _LIBCPP_DIAGNOSTIC_PUSH
+#    define _LIBCPP_DIAGNOSTIC_POP
+#    define _LIBCPP_CLANG_DIAGNOSTIC_IGNORED(str)
+#    define _LIBCPP_GCC_DIAGNOSTIC_IGNORED(str)
+#  endif
+
 #  if _LIBCPP_HARDENING_MODE == _LIBCPP_HARDENING_MODE_FAST
 #    define _LIBCPP_HARDENING_SIG f
 #  elif _LIBCPP_HARDENING_MODE == _LIBCPP_HARDENING_MODE_EXTENSIVE
@@ -810,16 +831,33 @@ typedef __char32_t char32_t;
 #    define _LIBCPP_HIDE_FROM_ABI_AFTER_V1 _LIBCPP_HIDE_FROM_ABI
 #  endif
 
+// TODO: Remove this workaround once we drop support for Clang 16
+#if __has_warning("-Wc++23-extensions")
+#  define _LIBCPP_CLANG_DIAGNOSTIC_IGNORED_CXX23_EXTENSION _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++23-extensions")
+#else
+#  define _LIBCPP_CLANG_DIAGNOSTIC_IGNORED_CXX23_EXTENSION _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++2b-extensions")
+#endif
+
 // Inline namespaces are available in Clang/GCC/MSVC regardless of C++ dialect.
 // clang-format off
-#  define _LIBCPP_BEGIN_NAMESPACE_STD namespace _LIBCPP_TYPE_VISIBILITY_DEFAULT std {                                  \
+#  define _LIBCPP_BEGIN_NAMESPACE_STD _LIBCPP_DIAGNOSTIC_PUSH                                                          \
+                                      _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++11-extensions")                           \
+                                      _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++14-extensions")                           \
+                                      _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++17-extensions")                           \
+                                      _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++20-extensions")                           \
+                                      _LIBCPP_CLANG_DIAGNOSTIC_IGNORED_CXX23_EXTENSION                                 \
+                                      _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wc++14-extensions")                             \
+                                      _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wc++17-extensions")                             \
+                                      _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wc++20-extensions")                             \
+                                      _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wc++23-extensions")                             \
+                                      namespace _LIBCPP_TYPE_VISIBILITY_DEFAULT std {                                  \
                                inline namespace _LIBCPP_ABI_NAMESPACE {
-#  define _LIBCPP_END_NAMESPACE_STD }}
+#  define _LIBCPP_END_NAMESPACE_STD }} _LIBCPP_DIAGNOSTIC_POP
 
 #  define _LIBCPP_BEGIN_NAMESPACE_FILESYSTEM _LIBCPP_BEGIN_NAMESPACE_STD                                               \
                                              inline namespace __fs { namespace filesystem {
 
-#  define _LIBCPP_END_NAMESPACE_FILESYSTEM _LIBCPP_END_NAMESPACE_STD }}
+#  define _LIBCPP_END_NAMESPACE_FILESYSTEM }} _LIBCPP_END_NAMESPACE_STD
 // clang-format on
 
 #  if __has_attribute(__enable_if__)
@@ -1256,23 +1294,6 @@ __sanitizer_verify_double_ended_contiguous_container(const void*, const void*, c
 // the ABI inconsistent.
 #  endif
 
-#  ifdef _LIBCPP_COMPILER_CLANG_BASED
-#    define _LIBCPP_DIAGNOSTIC_PUSH _Pragma("clang diagnostic push")
-#    define _LIBCPP_DIAGNOSTIC_POP _Pragma("clang diagnostic pop")
-#    define _LIBCPP_CLANG_DIAGNOSTIC_IGNORED(str) _Pragma(_LIBCPP_TOSTRING(clang diagnostic ignored str))
-#    define _LIBCPP_GCC_DIAGNOSTIC_IGNORED(str)
-#  elif defined(_LIBCPP_COMPILER_GCC)
-#    define _LIBCPP_DIAGNOSTIC_PUSH _Pragma("GCC diagnostic push")
-#    define _LIBCPP_DIAGNOSTIC_POP _Pragma("GCC diagnostic pop")
-#    define _LIBCPP_CLANG_DIAGNOSTIC_IGNORED(str)
-#    define _LIBCPP_GCC_DIAGNOSTIC_IGNORED(str) _Pragma(_LIBCPP_TOSTRING(GCC diagnostic ignored str))
-#  else
-#    define _LIBCPP_DIAGNOSTIC_PUSH
-#    define _LIBCPP_DIAGNOSTIC_POP
-#    define _LIBCPP_CLANG_DIAGNOSTIC_IGNORED(str)
-#    define _LIBCPP_GCC_DIAGNOSTIC_IGNORED(str)
-#  endif
-
 // c8rtomb() and mbrtoc8() were added in C++20 and C23. Support for these
 // functions is gradually being added to existing C libraries. The conditions
 // below check for known C library versions and conditions under which these
diff --git a/libcxx/include/tuple b/libcxx/include/tuple
index 0101d64aea4a..8808db6739fb 100644
--- a/libcxx/include/tuple
+++ b/libcxx/include/tuple
@@ -548,10 +548,6 @@ class _LIBCPP_TEMPLATE_VIS tuple {
 public:
   // [tuple.cnstr]
 
-  _LIBCPP_DIAGNOSTIC_PUSH
-  _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++20-extensions")
-  _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wc++20-extensions")
-
   // tuple() constructors (including allocator_arg_t variants)
   template