x86: correct VFPCLASSP{S,D} operand size handling

With AVX512VL disabled (e.g. when writing code for the Knights family
of processors) these insns aren't ambiguous when used with a memory
source, and hence should be accepted without suffix or operand size
specifier. When AVX512VL is enabled, to be consistent with this as
well as other ambiguous operand size handling it would seem better to
just warn about the ambiguity in AT&T mode, and still default to 512-bit
operands (on the assumption that the code may have been written without
AVX512VL in mind yet), but it was requested to leave AT&T syntax mode
alone here.

6 files changed, 93 insertions, 5 deletions

diff --git a/gas/ChangeLog b/gas/ChangeLog
index e7fac95..cf1266a 100644
--- a/gas/ChangeLog
+++ b/gas/ChangeLog
@@ -1,5 +1,16 @@
 2020-02-12  Jan Beulich  <jbeulich@suse.com>
 
+	* config/tc-i386.c (avx512): New (at file scope), moved from
+	(check_VecOperands): ... here.
+	(process_suffix): Add [XYZ]MMword operand size handling.
+	* testsuite/gas/i386/avx512dq-inval.s: Add VFPCLASS tests.
+	* testsuite/gas/i386/noavx512-2.s: Add Intel syntax VFPCLASS
+	tests.
+	* testsuite/gas/i386/avx512dq-inval.l,
+	testsuite/gas/i386/noavx512-2.l: Adjust expectations.
+
+2020-02-12  Jan Beulich  <jbeulich@suse.com>
+
 	PR gas/24546
 	* config/tc-i386.c (match_template): Apply AMD64 check to 64-bit
 	code only.
diff --git a/gas/config/tc-i386.c b/gas/config/tc-i386.c
index ac141b8..680016a 100644
--- a/gas/config/tc-i386.c
+++ b/gas/config/tc-i386.c
@@ -1840,6 +1840,8 @@ cpu_flags_and_not (i386_cpu_flags x, i386_cpu_flags y)
   return x;
 }
 
+static const i386_cpu_flags avx512 = CPU_ANY_AVX512F_FLAGS;
+
 #define CPU_FLAGS_ARCH_MATCH		0x1
 #define CPU_FLAGS_64BIT_MATCH		0x2
 
@@ -5369,7 +5371,6 @@ check_VecOperands (const insn_template *t)
 {
   unsigned int op;
   i386_cpu_flags cpu;
-  static const i386_cpu_flags avx512 = CPU_ANY_AVX512F_FLAGS;
 
   /* Templates allowing for ZMMword as well as YMMword and/or XMMword for
      any one operand are implicity requiring AVX512VL support if the actual
@@ -6445,7 +6446,7 @@ process_suffix (void)
       /* Accept FLDENV et al without suffix.  */
       && (i.tm.opcode_modifier.no_ssuf || i.tm.opcode_modifier.floatmf))
     {
-      unsigned int suffixes;
+      unsigned int suffixes, evex = 0;
 
       suffixes = !i.tm.opcode_modifier.no_bsuf;
       if (!i.tm.opcode_modifier.no_wsuf)
@@ -6459,7 +6460,61 @@ process_suffix (void)
       if (flag_code == CODE_64BIT && !i.tm.opcode_modifier.no_qsuf)
 	suffixes |= 1 << 5;
 
-      /* Are multiple suffixes allowed?  */
+      /* For [XYZ]MMWORD operands inspect operand sizes.  While generally
+	 also suitable for AT&T syntax mode, it was requested that this be
+	 restricted to just Intel syntax.  */
+      if (intel_syntax)
+	{
+	  i386_cpu_flags cpu = cpu_flags_and (i.tm.cpu_flags, avx512);
+
+	  if (!cpu_flags_all_zero (&cpu) && !i.broadcast)
+	    {
+	      unsigned int op;
+
+	      for (op = 0; op < i.tm.operands; ++op)
+		{
+		  if (!cpu_arch_flags.bitfield.cpuavx512vl)
+		    {
+		      if (i.tm.operand_types[op].bitfield.ymmword)
+			i.tm.operand_types[op].bitfield.xmmword = 0;
+		      if (i.tm.operand_types[op].bitfield.zmmword)
+			i.tm.operand_types[op].bitfield.ymmword = 0;
+		      if (!i.tm.opcode_modifier.evex
+			  || i.tm.opcode_modifier.evex == EVEXDYN)
+			i.tm.opcode_modifier.evex = EVEX512;
+		    }
+
+		  if (i.tm.operand_types[op].bitfield.xmmword
+		      + i.tm.operand_types[op].bitfield.ymmword
+		      + i.tm.operand_types[op].bitfield.zmmword < 2)
+		    continue;
+
+		  /* Any properly sized operand disambiguates the insn.  */
+		  if (i.types[op].bitfield.xmmword
+		      || i.types[op].bitfield.ymmword
+		      || i.types[op].bitfield.zmmword)
+		    {
+		      suffixes &= ~(7 << 6);
+		      evex = 0;
+		      break;
+		    }
+
+		  if ((i.flags[op] & Operand_Mem)
+		      && i.tm.operand_types[op].bitfield.unspecified)
+		    {
+		      if (i.tm.operand_types[op].bitfield.xmmword)
+			suffixes |= 1 << 6;
+		      if (i.tm.operand_types[op].bitfield.ymmword)
+			suffixes |= 1 << 7;
+		      if (i.tm.operand_types[op].bitfield.zmmword)
+			suffixes |= 1 << 8;
+		      evex = EVEX512;
+		    }
+		}
+	    }
+	}
+
+      /* Are multiple suffixes / operand sizes allowed?  */
       if (suffixes & (suffixes - 1))
 	{
 	  if (intel_syntax
@@ -6485,6 +6540,8 @@ process_suffix (void)
 
 	  if (i.tm.opcode_modifier.floatmf)
 	    i.suffix = SHORT_MNEM_SUFFIX;
+	  else if (evex)
+	    i.tm.opcode_modifier.evex = evex;
 	  else if (flag_code == CODE_16BIT)
 	    i.suffix = WORD_MNEM_SUFFIX;
 	  else if (!i.tm.opcode_modifier.no_lsuf)
diff --git a/gas/testsuite/gas/i386/avx512dq-inval.l b/gas/testsuite/gas/i386/avx512dq-inval.l
index 1533fb4..e8a0274 100644
--- a/gas/testsuite/gas/i386/avx512dq-inval.l
+++ b/gas/testsuite/gas/i386/avx512dq-inval.l
@@ -11,3 +11,7 @@
 .*:[0-9]*: Error:.* `vpinsrq' .*
 .*:[0-9]*: Error:.* `vpinsrq' .*
 .*:[0-9]*: Error:.* `vpinsrq' .*
+.*:[0-9]*: Error:.* `vfpclasspd'
+.*:[0-9]*: Error:.* `vfpclassps'
+.*:[0-9]*: Error:.* `vfpclasspd'
+.*:[0-9]*: Error:.* `vfpclassps'
diff --git a/gas/testsuite/gas/i386/avx512dq-inval.s b/gas/testsuite/gas/i386/avx512dq-inval.s
index 7f0f024..facc5f3 100644
--- a/gas/testsuite/gas/i386/avx512dq-inval.s
+++ b/gas/testsuite/gas/i386/avx512dq-inval.s
@@ -1,4 +1,4 @@
-# Check AVX512DQ instructions not to be accepted outside of 64-bit mode
+# Check AVX512DQ instructions not to be accepted (in part only outside of 64-bit mode)
 
 	.text
 _start:
@@ -20,3 +20,10 @@ _start:
 	       vpinsrq	xmm0, xmm0, qword ptr [eax], 0
 	{evex} vpinsrq	xmm0, xmm0, qword ptr [eax], 0
 
+	vfpclasspd	k0, [eax], 0
+	vfpclassps	k0, [eax], 0
+
+	.att_syntax prefix
+
+	vfpclasspd	$0, (%eax), %k0
+	vfpclassps	$0, (%eax), %k0
diff --git a/gas/testsuite/gas/i386/noavx512-2.l b/gas/testsuite/gas/i386/noavx512-2.l
index bdaaab2..56d1aaa 100644
--- a/gas/testsuite/gas/i386/noavx512-2.l
+++ b/gas/testsuite/gas/i386/noavx512-2.l
@@ -101,5 +101,10 @@ GAS LISTING .*
 [ 	]*50[ 	]+F5
 [ 	]*51[ 	]+\?\?\?\? 660F58F4 		addpd %xmm4, %xmm6
 [ 	]*52[ 	]+
-[ 	]*53[ 	]+\?\?\?\? 0F1F00   		\.p2align 4
+[ 	]*[1-9][0-9]*[ 	]+\.intel_syntax noprefix
+[ 	]*[1-9][0-9]*[ 	]+\?\?\?\? 62F3FD48 		vfpclasspd k0, \[eax], 0
+[ 	]*[1-9][0-9]*[ 	]+660000
+[ 	]*[1-9][0-9]*[ 	]+\?\?\?\? 62F37D48 		vfpclassps k0, \[eax], 0
+[ 	]*[1-9][0-9]*[ 	]+660000
+[ 	]*[1-9][0-9]*[ 	]+
 #pass
diff --git a/gas/testsuite/gas/i386/noavx512-2.s b/gas/testsuite/gas/i386/noavx512-2.s
index b9ef95c..437dc04 100644
--- a/gas/testsuite/gas/i386/noavx512-2.s
+++ b/gas/testsuite/gas/i386/noavx512-2.s
@@ -50,4 +50,8 @@
 	pabsb %xmm5, %xmm6
 	addpd %xmm4, %xmm6
 
+	.intel_syntax noprefix
+	vfpclasspd k0, [eax], 0
+	vfpclassps k0, [eax], 0
+
 	.p2align 4