Extend vternlog define_insn_and_split to memory_operand to enable more optimziation.

gcc/ChangeLog:

	PR target/101989
	* config/i386/predicates.md (reg_or_notreg_operand): Rename to ..
	(regmem_or_bitnot_regmem_operand): .. and extend to handle
	memory_operand.
	* config/i386/sse.md (*<avx512>_vpternlog<mode>_1): Force_reg
	the operands which are required to be register_operand.
	(*<avx512>_vpternlog<mode>_2): Ditto.
	(*<avx512>_vpternlog<mode>_3): Ditto.
	(*<avx512>_vternlog<mode>_all): Disallow embeded broadcast for
	vector HFmodes since it's not a real AVX512FP16 instruction.

gcc/testsuite/ChangeLog:

	* gcc.target/i386/pr101989-3.c: New test.

3 files changed, 72 insertions, 15 deletions

diff --git a/gcc/config/i386/predicates.md b/gcc/config/i386/predicates.md
index df5acb4..114d8d4 100644
--- a/gcc/config/i386/predicates.md
+++ b/gcc/config/i386/predicates.md
@@ -1046,10 +1046,10 @@
 
 ;; True for registers, or (not: registers).  Used to optimize 3-operand
 ;; bitwise operation.
-(define_predicate "reg_or_notreg_operand"
-  (ior (match_operand 0 "register_operand")
+(define_predicate "regmem_or_bitnot_regmem_operand"
+  (ior (match_operand 0 "nonimmediate_operand")
        (and (match_code "not")
-	    (match_test "register_operand (XEXP (op, 0), mode)"))))
+	    (match_test "nonimmediate_operand (XEXP (op, 0), mode)"))))
 
 ;; True if OP is acceptable as operand of DImode shift expander.
 (define_predicate "shiftdi_operand"
diff --git a/gcc/config/i386/sse.md b/gcc/config/i386/sse.md
index 22435e5..09f00dc 100644
--- a/gcc/config/i386/sse.md
+++ b/gcc/config/i386/sse.md
@@ -11662,7 +11662,11 @@
 	   (match_operand:V 3 "bcst_vector_operand" "vmBr")
 	   (match_operand:SI 4 "const_0_to_255_operand")]
 	  UNSPEC_VTERNLOG))]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F
+/* Disallow embeded broadcast for vector HFmode since
+   it's not real AVX512FP16 instruction.  */
+  && (GET_MODE_SIZE (GET_MODE_INNER (<MODE>mode)) >= 4
+     || GET_CODE (operands[3]) != VEC_DUPLICATE)"
   "vpternlog<ternlogsuffix>\t{%4, %3, %2, %0|%0, %2, %3, %4}"
   [(set_attr "type" "sselog")
    (set_attr "prefix" "evex")
@@ -11690,11 +11694,11 @@
   [(set (match_operand:V 0 "register_operand")
 	(any_logic:V
 	  (any_logic1:V
-	    (match_operand:V 1 "reg_or_notreg_operand")
-	    (match_operand:V 2 "reg_or_notreg_operand"))
+	    (match_operand:V 1 "regmem_or_bitnot_regmem_operand")
+	    (match_operand:V 2 "regmem_or_bitnot_regmem_operand"))
 	  (any_logic2:V
-	    (match_operand:V 3 "reg_or_notreg_operand")
-	    (match_operand:V 4 "reg_or_notreg_operand"))))]
+	    (match_operand:V 3 "regmem_or_bitnot_regmem_operand")
+	    (match_operand:V 4 "regmem_or_bitnot_regmem_operand"))))]
   "(<MODE_SIZE> == 64 || TARGET_AVX512VL)
    && ix86_pre_reload_split ()
    && (rtx_equal_p (STRIP_UNARY (operands[1]),
@@ -11763,6 +11767,10 @@
   operands[1] = STRIP_UNARY (operands[1]);
   operands[2] = STRIP_UNARY (operands[2]);
   operands[6] = STRIP_UNARY (operands[6]);
+  if (!register_operand (operands[2], <MODE>mode))
+     operands[2] = force_reg (<MODE>mode, operands[2]);
+  if (!register_operand (operands[6], <MODE>mode))
+     operands[6] = force_reg (<MODE>mode, operands[6]);
   operands[5] = GEN_INT (reg_mask);
 })
 
@@ -11771,10 +11779,10 @@
 	(any_logic:V
 	  (any_logic1:V
 	    (any_logic2:V
-	      (match_operand:V 1 "reg_or_notreg_operand")
-	      (match_operand:V 2 "reg_or_notreg_operand"))
-	    (match_operand:V 3 "reg_or_notreg_operand"))
-	  (match_operand:V 4 "reg_or_notreg_operand")))]
+	      (match_operand:V 1 "regmem_or_bitnot_regmem_operand")
+	      (match_operand:V 2 "regmem_or_bitnot_regmem_operand"))
+	    (match_operand:V 3 "regmem_or_bitnot_regmem_operand"))
+	  (match_operand:V 4 "regmem_or_bitnot_regmem_operand")))]
   "(<MODE_SIZE> == 64 || TARGET_AVX512VL)
    && ix86_pre_reload_split ()
    && (rtx_equal_p (STRIP_UNARY (operands[1]),
@@ -11844,15 +11852,20 @@
   operands[2] = STRIP_UNARY (operands[2]);
   operands[6] = STRIP_UNARY (operands[6]);
   operands[5] = GEN_INT (reg_mask);
+  if (!register_operand (operands[2], <MODE>mode))
+    operands[2] = force_reg (<MODE>mode, operands[2]);
+  if (!register_operand (operands[6], <MODE>mode))
+    operands[6] = force_reg (<MODE>mode, operands[6]);
+
 })
 
 (define_insn_and_split "*<avx512>_vpternlog<mode>_3"
   [(set (match_operand:V 0 "register_operand")
 	(any_logic:V
 	  (any_logic1:V
-	    (match_operand:V 1 "reg_or_notreg_operand")
-	    (match_operand:V 2 "reg_or_notreg_operand"))
-	  (match_operand:V 3 "reg_or_notreg_operand")))]
+	    (match_operand:V 1 "regmem_or_bitnot_regmem_operand")
+	    (match_operand:V 2 "regmem_or_bitnot_regmem_operand"))
+	  (match_operand:V 3 "regmem_or_bitnot_regmem_operand")))]
   "(<MODE_SIZE> == 64 || TARGET_AVX512VL)
    && ix86_pre_reload_split ()"
   "#"
@@ -11883,6 +11896,10 @@
   operands[2] = STRIP_UNARY (operands[2]);
   operands[3] = STRIP_UNARY (operands[3]);
   operands[4] = GEN_INT (reg_mask);
+  if (!register_operand (operands[2], <MODE>mode))
+    operands[2] = force_reg (<MODE>mode, operands[2]);
+  if (!register_operand (operands[3], <MODE>mode))
+    operands[3] = force_reg (<MODE>mode, operands[3]);
 })
 
 
diff --git a/gcc/testsuite/gcc.target/i386/pr101989-3.c b/gcc/testsuite/gcc.target/i386/pr101989-3.c
new file mode 100644
index 0000000..dfd8991
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr101989-3.c
@@ -0,0 +1,40 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -mavx512fp16 -mavx512vl" } */
+/* { dg-final { scan-assembler-times "vpternlog" 5 } } */
+/* { dg-final { scan-assembler-not "vpxor" } } */
+/* { dg-final { scan-assembler-not "vpor" } } */
+/* { dg-final { scan-assembler-not "vpand" } } */
+
+#include<immintrin.h>
+
+extern __m256i src1, src2, src3;
+
+__m256i
+foo (void)
+{
+  return (src2 & ~src1) | (src3 & src1);
+}
+
+__m256i
+foo1 (void)
+{
+  return (src2 & src1) | (src3 & ~src1);
+}
+
+__m256i
+foo2 (void)
+{
+  return (src2 & src1) | (~src3 & src1);
+}
+
+__m256i
+foo3 (void)
+{
+  return (~src2 & src1) | (src3 & src1);
+}
+
+__m256i
+foo4 (void)
+{
+  return src3 & src2 ^ src1;
+}