; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=CHECK,SSE ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=CHECK,AVX,AVX1 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX2 declare <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16>, <8 x i16>) nounwind readnone declare <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8>, <16 x i8>) nounwind readnone define <4 x i32> @combine_pmaddwd_zero(<8 x i16> %a0, <8 x i16> %a1) { ; SSE-LABEL: combine_pmaddwd_zero: ; SSE: # %bb.0: ; SSE-NEXT: xorps %xmm0, %xmm0 ; SSE-NEXT: retq ; ; AVX-LABEL: combine_pmaddwd_zero: ; AVX: # %bb.0: ; AVX-NEXT: vxorps %xmm0, %xmm0, %xmm0 ; AVX-NEXT: retq %1 = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %a0, <8 x i16> zeroinitializer) ret <4 x i32> %1 } define <4 x i32> @combine_pmaddwd_zero_commute(<8 x i16> %a0, <8 x i16> %a1) { ; SSE-LABEL: combine_pmaddwd_zero_commute: ; SSE: # %bb.0: ; SSE-NEXT: xorps %xmm0, %xmm0 ; SSE-NEXT: retq ; ; AVX-LABEL: combine_pmaddwd_zero_commute: ; AVX: # %bb.0: ; AVX-NEXT: vxorps %xmm0, %xmm0, %xmm0 ; AVX-NEXT: retq %1 = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> zeroinitializer, <8 x i16> %a0) ret <4 x i32> %1 } define <8 x i32> @combine_pmaddwd_concat(<8 x i16> %a0, <8 x i16> %a1, <8 x i16> %a2, <8 x i16> %a3) { ; SSE-LABEL: combine_pmaddwd_concat: ; SSE: # %bb.0: ; SSE-NEXT: pmaddwd %xmm1, %xmm0 ; SSE-NEXT: pmaddwd %xmm3, %xmm2 ; SSE-NEXT: movdqa %xmm2, %xmm1 ; SSE-NEXT: retq ; ; AVX1-LABEL: combine_pmaddwd_concat: ; AVX1: # %bb.0: ; AVX1-NEXT: vpmaddwd %xmm1, %xmm0, %xmm0 ; AVX1-NEXT: vpmaddwd %xmm3, %xmm2, %xmm1 ; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 ; AVX1-NEXT: retq ; ; AVX2-LABEL: combine_pmaddwd_concat: ; AVX2: # %bb.0: ; AVX2-NEXT: vpmaddwd %xmm1, %xmm0, %xmm0 ; AVX2-NEXT: vpmaddwd %xmm3, %xmm2, %xmm1 ; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 ; AVX2-NEXT: retq %1 = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %a0, <8 x i16> %a1) %2 = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %a2, <8 x i16> %a3) %3 = shufflevector <4 x i32> %1, <4 x i32> %2, <8 x i32> ret <8 x i32> %3 } define <8 x i32> @combine_pmaddwd_concat_freeze(<8 x i16> %a0, <8 x i16> %a1) { ; SSE-LABEL: combine_pmaddwd_concat_freeze: ; SSE: # %bb.0: ; SSE-NEXT: pmovsxbw {{.*#+}} xmm2 = [1,1,1,1,1,1,1,1] ; SSE-NEXT: pmaddwd %xmm2, %xmm0 ; SSE-NEXT: pmaddwd %xmm2, %xmm1 ; SSE-NEXT: retq ; ; AVX1-LABEL: combine_pmaddwd_concat_freeze: ; AVX1: # %bb.0: ; AVX1-NEXT: vbroadcastss {{.*#+}} xmm2 = [1,1,1,1,1,1,1,1] ; AVX1-NEXT: vpmaddwd %xmm2, %xmm0, %xmm0 ; AVX1-NEXT: vpmaddwd %xmm2, %xmm1, %xmm1 ; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 ; AVX1-NEXT: retq ; ; AVX2-LABEL: combine_pmaddwd_concat_freeze: ; AVX2: # %bb.0: ; AVX2-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0 ; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 ; AVX2-NEXT: vpmaddwd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 # [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] ; AVX2-NEXT: retq %lo = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %a0, <8 x i16> ) %hi = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %a1, <8 x i16> ) %flo = freeze <4 x i32> %lo %fhi = freeze <4 x i32> %hi %res = shufflevector <4 x i32> %flo, <4 x i32> %fhi, <8 x i32> ret <8 x i32> %res } define <4 x i32> @combine_pmaddwd_demandedelts(<8 x i16> %a0, <8 x i16> %a1) { ; SSE-LABEL: combine_pmaddwd_demandedelts: ; SSE: # %bb.0: ; SSE-NEXT: pmaddwd %xmm1, %xmm0 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0] ; SSE-NEXT: retq ; ; AVX1-LABEL: combine_pmaddwd_demandedelts: ; AVX1: # %bb.0: ; AVX1-NEXT: vpmaddwd %xmm1, %xmm0, %xmm0 ; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0] ; AVX1-NEXT: retq ; ; AVX2-LABEL: combine_pmaddwd_demandedelts: ; AVX2: # %bb.0: ; AVX2-NEXT: vpmaddwd %xmm1, %xmm0, %xmm0 ; AVX2-NEXT: vpbroadcastd %xmm0, %xmm0 ; AVX2-NEXT: retq %1 = shufflevector <8 x i16> %a0, <8 x i16> poison, <8 x i32> %2 = shufflevector <8 x i16> %a1, <8 x i16> poison, <8 x i32> %3 = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %1, <8 x i16> %2) %4 = shufflevector <4 x i32> %3, <4 x i32> poison, <4 x i32> zeroinitializer ret <4 x i32> %4 } ; [2]: (-5*13)+(6*-15) = -155 = 4294967141 define <4 x i32> @combine_pmaddwd_constant() { ; SSE-LABEL: combine_pmaddwd_constant: ; SSE: # %bb.0: ; SSE-NEXT: movaps {{.*#+}} xmm0 = [19,17,4294967141,271] ; SSE-NEXT: retq ; ; AVX-LABEL: combine_pmaddwd_constant: ; AVX: # %bb.0: ; AVX-NEXT: vmovaps {{.*#+}} xmm0 = [19,17,4294967141,271] ; AVX-NEXT: retq %1 = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> , <8 x i16> ) ret <4 x i32> %1 } ; ensure we don't assume pmaddwd performs add nsw ; [0]: (-32768*-32768)+(-32768*-32768) = 0x80000000 = 2147483648 define <4 x i32> @combine_pmaddwd_constant_nsw() { ; SSE-LABEL: combine_pmaddwd_constant_nsw: ; SSE: # %bb.0: ; SSE-NEXT: movaps {{.*#+}} xmm0 = [2147483648,2147483648,2147483648,2147483648] ; SSE-NEXT: retq ; ; AVX-LABEL: combine_pmaddwd_constant_nsw: ; AVX: # %bb.0: ; AVX-NEXT: vbroadcastss {{.*#+}} xmm0 = [2147483648,2147483648,2147483648,2147483648] ; AVX-NEXT: retq %1 = insertelement <8 x i16> undef, i16 32768, i32 0 %2 = shufflevector <8 x i16> %1, <8 x i16> undef, <8 x i32> zeroinitializer %3 = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %2, <8 x i16> %2) ret <4 x i32> %3 } define <8 x i16> @combine_pmaddubsw_zero(<16 x i8> %a0, <16 x i8> %a1) { ; SSE-LABEL: combine_pmaddubsw_zero: ; SSE: # %bb.0: ; SSE-NEXT: xorps %xmm0, %xmm0 ; SSE-NEXT: retq ; ; AVX-LABEL: combine_pmaddubsw_zero: ; AVX: # %bb.0: ; AVX-NEXT: vxorps %xmm0, %xmm0, %xmm0 ; AVX-NEXT: retq %1 = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %a0, <16 x i8> zeroinitializer) ret <8 x i16> %1 } define <8 x i16> @combine_pmaddubsw_zero_commute(<16 x i8> %a0, <16 x i8> %a1) { ; SSE-LABEL: combine_pmaddubsw_zero_commute: ; SSE: # %bb.0: ; SSE-NEXT: xorps %xmm0, %xmm0 ; SSE-NEXT: retq ; ; AVX-LABEL: combine_pmaddubsw_zero_commute: ; AVX: # %bb.0: ; AVX-NEXT: vxorps %xmm0, %xmm0, %xmm0 ; AVX-NEXT: retq %1 = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> zeroinitializer, <16 x i8> %a0) ret <8 x i16> %1 } define <16 x i16> @combine_pmaddubsw_concat(<32 x i8> %a0, <32 x i8> %a1) { ; SSE-LABEL: combine_pmaddubsw_concat: ; SSE: # %bb.0: ; SSE-NEXT: pmaddubsw %xmm2, %xmm0 ; SSE-NEXT: pmaddubsw %xmm3, %xmm1 ; SSE-NEXT: retq ; ; AVX1-LABEL: combine_pmaddubsw_concat: ; AVX1: # %bb.0: ; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 ; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm3 ; AVX1-NEXT: vpmaddubsw %xmm3, %xmm2, %xmm2 ; AVX1-NEXT: vpmaddubsw %xmm1, %xmm0, %xmm0 ; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 ; AVX1-NEXT: retq ; ; AVX2-LABEL: combine_pmaddubsw_concat: ; AVX2: # %bb.0: ; AVX2-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm0 ; AVX2-NEXT: retq %lo0 = shufflevector <32 x i8> %a0, <32 x i8> undef, <16 x i32> %lo1 = shufflevector <32 x i8> %a1, <32 x i8> undef, <16 x i32> %hi0 = shufflevector <32 x i8> %a0, <32 x i8> undef, <16 x i32> %hi1 = shufflevector <32 x i8> %a1, <32 x i8> undef, <16 x i32> %lo = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %lo0, <16 x i8> %lo1) %hi = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %hi0, <16 x i8> %hi1) %res = shufflevector <8 x i16> %lo, <8 x i16> %hi, <16 x i32> ret <16 x i16> %res } ; Not beneficial to concatenate both inputs just to create a 256-bit pmaddubsw define <16 x i16> @combine_pmaddubsw_concat_unecessary(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> %a2, <16 x i8> %a3) { ; SSE-LABEL: combine_pmaddubsw_concat_unecessary: ; SSE: # %bb.0: ; SSE-NEXT: pmaddubsw %xmm1, %xmm0 ; SSE-NEXT: pmaddubsw %xmm3, %xmm2 ; SSE-NEXT: movdqa %xmm2, %xmm1 ; SSE-NEXT: retq ; ; AVX1-LABEL: combine_pmaddubsw_concat_unecessary: ; AVX1: # %bb.0: ; AVX1-NEXT: vpmaddubsw %xmm1, %xmm0, %xmm0 ; AVX1-NEXT: vpmaddubsw %xmm3, %xmm2, %xmm1 ; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 ; AVX1-NEXT: retq ; ; AVX2-LABEL: combine_pmaddubsw_concat_unecessary: ; AVX2: # %bb.0: ; AVX2-NEXT: vpmaddubsw %xmm1, %xmm0, %xmm0 ; AVX2-NEXT: vpmaddubsw %xmm3, %xmm2, %xmm1 ; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 ; AVX2-NEXT: retq %1 = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %a0, <16 x i8> %a1) %2 = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %a2, <16 x i8> %a3) %3 = shufflevector <8 x i16> %1, <8 x i16> %2, <16 x i32> ret <16 x i16> %3 } define <16 x i16> @combine_pmaddubsw_concat_freeze(<16 x i8> %a0, <16 x i8> %a1) { ; SSE-LABEL: combine_pmaddubsw_concat_freeze: ; SSE: # %bb.0: ; SSE-NEXT: movdqa {{.*#+}} xmm2 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] ; SSE-NEXT: pmaddubsw %xmm2, %xmm0 ; SSE-NEXT: pmaddubsw %xmm2, %xmm1 ; SSE-NEXT: retq ; ; AVX1-LABEL: combine_pmaddubsw_concat_freeze: ; AVX1: # %bb.0: ; AVX1-NEXT: vbroadcastss {{.*#+}} xmm2 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] ; AVX1-NEXT: vpmaddubsw %xmm2, %xmm0, %xmm0 ; AVX1-NEXT: vpmaddubsw %xmm2, %xmm1, %xmm1 ; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 ; AVX1-NEXT: retq ; ; AVX2-LABEL: combine_pmaddubsw_concat_freeze: ; AVX2: # %bb.0: ; AVX2-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0 ; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 ; AVX2-NEXT: vpmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 # [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] ; AVX2-NEXT: retq %lo = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %a0, <16 x i8> ) %hi = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %a1, <16 x i8> ) %flo = freeze <8 x i16> %lo %fhi = freeze <8 x i16> %hi %res = shufflevector <8 x i16> %flo, <8 x i16> %fhi, <16 x i32> ret <16 x i16> %res } define <8 x i16> @combine_pmaddubsw_demandedelts(<16 x i8> %a0, <16 x i8> %a1) { ; SSE-LABEL: combine_pmaddubsw_demandedelts: ; SSE: # %bb.0: ; SSE-NEXT: pmaddubsw %xmm1, %xmm0 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0] ; SSE-NEXT: retq ; ; AVX1-LABEL: combine_pmaddubsw_demandedelts: ; AVX1: # %bb.0: ; AVX1-NEXT: vpmaddubsw %xmm1, %xmm0, %xmm0 ; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0] ; AVX1-NEXT: retq ; ; AVX2-LABEL: combine_pmaddubsw_demandedelts: ; AVX2: # %bb.0: ; AVX2-NEXT: vpmaddubsw %xmm1, %xmm0, %xmm0 ; AVX2-NEXT: vpbroadcastd %xmm0, %xmm0 ; AVX2-NEXT: retq %1 = shufflevector <16 x i8> %a0, <16 x i8> poison, <16 x i32> %2 = shufflevector <16 x i8> %a1, <16 x i8> poison, <16 x i32> %3 = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %1, <16 x i8> %2) %4 = shufflevector <8 x i16> %3, <8 x i16> poison, <8 x i32> ret <8 x i16> %4 } ; [3]: ((uint16_t)-6*7)+(7*-8) = (250*7)+(7*-8) = 1694 define i32 @combine_pmaddubsw_constant() { ; CHECK-LABEL: combine_pmaddubsw_constant: ; CHECK: # %bb.0: ; CHECK-NEXT: movl $1694, %eax # imm = 0x69E ; CHECK-NEXT: retq %1 = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> , <16 x i8> ) %2 = extractelement <8 x i16> %1, i32 3 %3 = sext i16 %2 to i32 ret i32 %3 } ; [0]: add_sat_i16(((uint16_t)-1*-128),((uint16_t)-1*-128)_ = add_sat_i16(255*-128),(255*-128)) = sat_i16(-65280) = -32768 define i32 @combine_pmaddubsw_constant_sat() { ; CHECK-LABEL: combine_pmaddubsw_constant_sat: ; CHECK: # %bb.0: ; CHECK-NEXT: movl $-32768, %eax # imm = 0x8000 ; CHECK-NEXT: retq %1 = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> , <16 x i8> ) %2 = extractelement <8 x i16> %1, i32 0 %3 = sext i16 %2 to i32 ret i32 %3 } ; Constant folding PMADDWD was causing an infinite loop in the PCMPGT commuting between 2 constant values. define i1 @pmaddwd_pcmpgt_infinite_loop() { ; CHECK-LABEL: pmaddwd_pcmpgt_infinite_loop: ; CHECK: # %bb.0: ; CHECK-NEXT: movb $1, %al ; CHECK-NEXT: retq %1 = tail call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> , <8 x i16> ) %2 = icmp eq <4 x i32> %1, %3 = select <4 x i1> %2, <4 x i32> , <4 x i32> zeroinitializer %4 = add <4 x i32> %3, %.not = trunc <4 x i32> %3 to <4 x i1> %5 = icmp sgt <4 x i32> %4, %6 = select <4 x i1> %.not, <4 x i1> %5, <4 x i1> zeroinitializer %7 = bitcast <4 x i1> %6 to i4 %8 = icmp eq i4 %7, 0 ret i1 %8 }