x86/Intel: improve diagnostics for ambiguous VCVT* operands

Conversions which shrink element size and which have a memory source
can't be disambiguated between their 128- and 256-bit variants by
looking at the register operand. "operand size mismatch", however, is a
pretty misleading diagnostic. Generalize the logic introduced for
VFPCLASSP{S,D} such that, with suitable similar adjustments to the
respective templates, it'll cover these cases too.

For VCVTNEPS2BF16 also fold the two previously separate AVX512VL
templates to achieve the intended effect. This is then also accompanied
by a respective addition to the inval-avx512f testcase.

10 files changed, 115 insertions, 45 deletions

diff --git a/gas/ChangeLog b/gas/ChangeLog
index a148526..7f64228 100644
--- a/gas/ChangeLog
+++ b/gas/ChangeLog
@@ -1,3 +1,19 @@
+2020-02-17  Jan Beulich  <jbeulich@suse.com>
+
+	PR gas/6518
+	* config/tc-i386.c (process_suffix): Re-work Intel-syntax
+	[XYZ]MMWord memory operand ambiguity recognition logic (largely
+	re-indentation).
+	* testsuite/gas/i386/avx512dq-inval.s: Add vcvtqq2ps/vcvtuqq2ps
+	cases.
+	* testsuite/gas/i386/inval-avx512f.s: Also test vcvtneps2bf16.
+	* testsuite/gas/i386/avx512dq-inval.l,
+	testsuite/gas/i386/inval-avx.l,
+	testsuite/gas/i386/inval-avx512f.l: Adjust expectations.
+	* testsuite/gas/i386/avx512vl-ambig.s,
+	testsuite/gas/i386/avx512vl-ambig.l: New.
+	* testsuite/gas/i386/i386.exp: Run new test.
+
 2020-02-16  H.J. Lu  <hongjiu.lu@intel.com>
 
 	* config/tc-i386.c (cpu_arch): Add .sse4a and nosse4a.  Restore
diff --git a/gas/config/tc-i386.c b/gas/config/tc-i386.c
index 6cc7696..acade9d 100644
--- a/gas/config/tc-i386.c
+++ b/gas/config/tc-i386.c
@@ -6466,53 +6466,50 @@ process_suffix (void)
       /* For [XYZ]MMWORD operands inspect operand sizes.  While generally
 	 also suitable for AT&T syntax mode, it was requested that this be
 	 restricted to just Intel syntax.  */
-      if (intel_syntax)
+      if (intel_syntax && is_any_vex_encoding (&i.tm) && !i.broadcast)
 	{
-	  i386_cpu_flags cpu = cpu_flags_and (i.tm.cpu_flags, avx512);
+	  unsigned int op;
 
-	  if (!cpu_flags_all_zero (&cpu) && !i.broadcast)
+	  for (op = 0; op < i.tm.operands; ++op)
 	    {
-	      unsigned int op;
-
-	      for (op = 0; op < i.tm.operands; ++op)
+	      if (is_evex_encoding (&i.tm)
+		  && !cpu_arch_flags.bitfield.cpuavx512vl)
 		{
-		  if (!cpu_arch_flags.bitfield.cpuavx512vl)
-		    {
-		      if (i.tm.operand_types[op].bitfield.ymmword)
-			i.tm.operand_types[op].bitfield.xmmword = 0;
-		      if (i.tm.operand_types[op].bitfield.zmmword)
-			i.tm.operand_types[op].bitfield.ymmword = 0;
-		      if (!i.tm.opcode_modifier.evex
-			  || i.tm.opcode_modifier.evex == EVEXDYN)
-			i.tm.opcode_modifier.evex = EVEX512;
-		    }
+		  if (i.tm.operand_types[op].bitfield.ymmword)
+		    i.tm.operand_types[op].bitfield.xmmword = 0;
+		  if (i.tm.operand_types[op].bitfield.zmmword)
+		    i.tm.operand_types[op].bitfield.ymmword = 0;
+		  if (!i.tm.opcode_modifier.evex
+		      || i.tm.opcode_modifier.evex == EVEXDYN)
+		    i.tm.opcode_modifier.evex = EVEX512;
+		}
 
-		  if (i.tm.operand_types[op].bitfield.xmmword
-		      + i.tm.operand_types[op].bitfield.ymmword
-		      + i.tm.operand_types[op].bitfield.zmmword < 2)
-		    continue;
+	      if (i.tm.operand_types[op].bitfield.xmmword
+		  + i.tm.operand_types[op].bitfield.ymmword
+		  + i.tm.operand_types[op].bitfield.zmmword < 2)
+		continue;
 
-		  /* Any properly sized operand disambiguates the insn.  */
-		  if (i.types[op].bitfield.xmmword
-		      || i.types[op].bitfield.ymmword
-		      || i.types[op].bitfield.zmmword)
-		    {
-		      suffixes &= ~(7 << 6);
-		      evex = 0;
-		      break;
-		    }
+	      /* Any properly sized operand disambiguates the insn.  */
+	      if (i.types[op].bitfield.xmmword
+		  || i.types[op].bitfield.ymmword
+		  || i.types[op].bitfield.zmmword)
+		{
+		  suffixes &= ~(7 << 6);
+		  evex = 0;
+		  break;
+		}
 
-		  if ((i.flags[op] & Operand_Mem)
-		      && i.tm.operand_types[op].bitfield.unspecified)
-		    {
-		      if (i.tm.operand_types[op].bitfield.xmmword)
-			suffixes |= 1 << 6;
-		      if (i.tm.operand_types[op].bitfield.ymmword)
-			suffixes |= 1 << 7;
-		      if (i.tm.operand_types[op].bitfield.zmmword)
-			suffixes |= 1 << 8;
-		      evex = EVEX512;
-		    }
+	      if ((i.flags[op] & Operand_Mem)
+		  && i.tm.operand_types[op].bitfield.unspecified)
+		{
+		  if (i.tm.operand_types[op].bitfield.xmmword)
+		    suffixes |= 1 << 6;
+		  if (i.tm.operand_types[op].bitfield.ymmword)
+		    suffixes |= 1 << 7;
+		  if (i.tm.operand_types[op].bitfield.zmmword)
+		    suffixes |= 1 << 8;
+		  if (is_evex_encoding (&i.tm))
+		    evex = EVEX512;
 		}
 	    }
 	}
diff --git a/gas/testsuite/gas/i386/avx512dq-inval.l b/gas/testsuite/gas/i386/avx512dq-inval.l
index e8a0274..8a43aab 100644
--- a/gas/testsuite/gas/i386/avx512dq-inval.l
+++ b/gas/testsuite/gas/i386/avx512dq-inval.l
@@ -11,7 +11,11 @@
 .*:[0-9]*: Error:.* `vpinsrq' .*
 .*:[0-9]*: Error:.* `vpinsrq' .*
 .*:[0-9]*: Error:.* `vpinsrq' .*
-.*:[0-9]*: Error:.* `vfpclasspd'
-.*:[0-9]*: Error:.* `vfpclassps'
+.*:[0-9]*: Error:.* ambiguous .* `vcvtqq2ps'
+.*:[0-9]*: Error:.* ambiguous .* `vcvtuqq2ps'
+.*:[0-9]*: Error:.* ambiguous .* `vfpclasspd'
+.*:[0-9]*: Error:.* ambiguous .* `vfpclassps'
+.*:[0-9]*: Error:.* `vcvtqq2ps'
+.*:[0-9]*: Error:.* `vcvtuqq2ps'
 .*:[0-9]*: Error:.* `vfpclasspd'
 .*:[0-9]*: Error:.* `vfpclassps'
diff --git a/gas/testsuite/gas/i386/avx512dq-inval.s b/gas/testsuite/gas/i386/avx512dq-inval.s
index facc5f3..36ee3b8 100644
--- a/gas/testsuite/gas/i386/avx512dq-inval.s
+++ b/gas/testsuite/gas/i386/avx512dq-inval.s
@@ -20,10 +20,16 @@ _start:
 	       vpinsrq	xmm0, xmm0, qword ptr [eax], 0
 	{evex} vpinsrq	xmm0, xmm0, qword ptr [eax], 0
 
+	vcvtqq2ps	xmm0, [rax]
+	vcvtuqq2ps	xmm0, [rax]
+
 	vfpclasspd	k0, [eax], 0
 	vfpclassps	k0, [eax], 0
 
 	.att_syntax prefix
 
+	vcvtqq2ps	(%eax), %xmm0
+	vcvtuqq2ps	(%eax), %xmm0
+
 	vfpclasspd	$0, (%eax), %k0
 	vfpclassps	$0, (%eax), %k0
diff --git a/gas/testsuite/gas/i386/avx512vl-ambig.l b/gas/testsuite/gas/i386/avx512vl-ambig.l
new file mode 100644
index 0000000..52f1357
--- /dev/null
+++ b/gas/testsuite/gas/i386/avx512vl-ambig.l
@@ -0,0 +1,7 @@
+.*: Assembler messages:
+.*:[0-9]*: Error:.* ambiguous .* `vcvtneps2bf16'
+.*:[0-9]*: Error:.* ambiguous .* `vcvtpd2dq'
+.*:[0-9]*: Error:.* ambiguous .* `vcvtpd2ps'
+.*:[0-9]*: Error:.* ambiguous .* `vcvtpd2udq'
+.*:[0-9]*: Error:.* ambiguous .* `vcvttpd2dq'
+.*:[0-9]*: Error:.* ambiguous .* `vcvttpd2udq'
diff --git a/gas/testsuite/gas/i386/avx512vl-ambig.s b/gas/testsuite/gas/i386/avx512vl-ambig.s
new file mode 100644
index 0000000..cbcb884
--- /dev/null
+++ b/gas/testsuite/gas/i386/avx512vl-ambig.s
@@ -0,0 +1,11 @@
+# Check AVX512VL instructions with ambiguous operands
+
+	.text
+	.intel_syntax noprefix
+_start:
+	vcvtneps2bf16 xmm0, [ecx]
+	vcvtpd2dq xmm0{k1}, [ecx]
+	vcvtpd2ps xmm0{k1}, [ecx]
+	vcvtpd2udq xmm0, [ecx]
+	vcvttpd2dq xmm0{k1}, [ecx]
+	vcvttpd2udq xmm0, [ecx]
diff --git a/gas/testsuite/gas/i386/i386.exp b/gas/testsuite/gas/i386/i386.exp
index a065583..8d22362 100644
--- a/gas/testsuite/gas/i386/i386.exp
+++ b/gas/testsuite/gas/i386/i386.exp
@@ -479,6 +479,7 @@ if [expr ([istarget "i*86-*-*"] ||  [istarget "x86_64-*-*"]) && [gas_32_check]]
     run_list_test "vp2intersect-inval-bcast"
     run_list_test "avx512vl-1" "-al"
     run_list_test "avx512vl-2" "-al"
+    run_list_test "avx512vl-ambig"
     run_list_test "avx512vl-plain" "-al"
     run_dump_test "fpu-bad"
     run_dump_test "cet"
diff --git a/gas/testsuite/gas/i386/inval-avx.l b/gas/testsuite/gas/i386/inval-avx.l
index a9fb8de..866987e 100644
--- a/gas/testsuite/gas/i386/inval-avx.l
+++ b/gas/testsuite/gas/i386/inval-avx.l
@@ -2,9 +2,9 @@
 .*:4: Error: .*
 .*:5: Error: .*
 .*:6: Error: .*
-.*:9: Error: .*
-.*:10: Error: .*
-.*:11: Error: .*
+.*:9: Error:.* ambiguous .* `vcvtpd2dq'
+.*:10: Error:.* ambiguous .* `vcvtpd2ps'
+.*:11: Error:.* ambiguous .* `vcvttpd2dq'
 GAS LISTING .*
 
 
diff --git a/gas/testsuite/gas/i386/inval-avx512f.l b/gas/testsuite/gas/i386/inval-avx512f.l
index e414129..f42437a 100644
--- a/gas/testsuite/gas/i386/inval-avx512f.l
+++ b/gas/testsuite/gas/i386/inval-avx512f.l
@@ -213,6 +213,11 @@
 .*:306: Error: .*masking.*vscatterpf1qps.*
 .*:308: Error: .*unsupported broadcast for `vdpbf16ps'
 .*:309: Error: .*unsupported broadcast for `vcvtne2ps2bf16'
+.*:311: Error: .*unsupported broadcast for `vcvtneps2bf16'
+.*:312: Error: .*unsupported broadcast for `vcvtneps2bf16'
+.*:313: Error: .*unsupported broadcast for `vcvtneps2bf16'
+.*:316: Error: .*unsupported broadcast for `vcvtneps2bf16'
+.*:319: Error: .*unsupported broadcast for `vcvtneps2bf16'
 GAS LISTING .*
 
 
@@ -551,3 +556,16 @@ GAS LISTING .*
 [ 	]*307[ 	]*
 [ 	]*308[ 	]+vdpbf16ps 8\(%eax\)\{1to8\}, %zmm2, %zmm2
 [ 	]*309[ 	]+vcvtne2ps2bf16 8\(%eax\)\{1to8\}, %zmm2, %zmm2
+[ 	]*310[ 	]*
+[ 	]*311[ 	]+vcvtneps2bf16 \(%eax\)\{1to2\}, %ymm1
+[ 	]*312[ 	]+vcvtneps2bf16 \(%eax\)\{1to4\}, %ymm1
+[ 	]*313[ 	]+vcvtneps2bf16 \(%eax\)\{1to8\}, %ymm1
+[ 	]*314 \?\?\?\? 62F27E58[ 	]+vcvtneps2bf16 \(%eax\)\{1to16\}, %ymm1
+[ 	]*314[ 	]+7208
+[ 	]*315[ 	]*
+[ 	]*316[ 	]+vcvtneps2bf16 \(%eax\)\{1to2\}, %xmm1
+[ 	]*317 \?\?\?\? 62F27E18[ 	]+vcvtneps2bf16 \(%eax\)\{1to4\}, %xmm1
+[ 	]*317[ 	]+7208
+[ 	]*318 \?\?\?\? 62F27E38[ 	]+vcvtneps2bf16 \(%eax\)\{1to8\}, %xmm1
+[ 	]*318[ 	]+7208
+[ 	]*319[ 	]+vcvtneps2bf16 \(%eax\)\{1to16\}, %xmm1
diff --git a/gas/testsuite/gas/i386/inval-avx512f.s b/gas/testsuite/gas/i386/inval-avx512f.s
index 2833b1e..6aa6f14 100644
--- a/gas/testsuite/gas/i386/inval-avx512f.s
+++ b/gas/testsuite/gas/i386/inval-avx512f.s
@@ -307,3 +307,13 @@ _start:
 
 	vdpbf16ps 8(%eax){1to8}, %zmm2, %zmm2
 	vcvtne2ps2bf16 8(%eax){1to8}, %zmm2, %zmm2
+
+	vcvtneps2bf16 (%eax){1to2}, %ymm1
+	vcvtneps2bf16 (%eax){1to4}, %ymm1
+	vcvtneps2bf16 (%eax){1to8}, %ymm1
+	vcvtneps2bf16 (%eax){1to16}, %ymm1
+
+	vcvtneps2bf16 (%eax){1to2}, %xmm1
+	vcvtneps2bf16 (%eax){1to4}, %xmm1
+	vcvtneps2bf16 (%eax){1to8}, %xmm1
+	vcvtneps2bf16 (%eax){1to16}, %xmm1