x86: also refuse data size prefix on SIMD insns

The data size prefix alters the meaning of legacy encoded SIMD insns,
and hence shouldn't be accepted there. Use of it also leads to
inconsistencies in SSE2AVX mode. Don't match insns with data size prefix
against SSE2AVX templates.

8 files changed, 37 insertions, 14 deletions

diff --git a/gas/ChangeLog b/gas/ChangeLog
index 83f0e6f..835a552 100644
--- a/gas/ChangeLog
+++ b/gas/ChangeLog
@@ -1,5 +1,18 @@
 2020-06-25  Jan Beulich  <jbeulich@suse.com>
 
+	* config/tc-i386.c (cpu_flags_match): Only match SSE2AVX
+	templates when there's no data size prefix.
+	(md_assemble): Reject data size prefix also for legacy encoded
+	SIMD templates.
+	* testsuite/gas/i386/prefix32.s, testsuite/gas/i386/prefix64.s:
+	Uncomment previously not working line.
+	* testsuite/gas/i386/sse2avx.s: Add ldmxcsr/stmxcsr cases with
+	data16 prefix.
+	* testsuite/gas/i386/prefix32.l, testsuite/gas/i386/prefix64.l,
+	testsuite/gas/i386/sse2avx.d: Adjust expectations.
+
+2020-06-25  Jan Beulich  <jbeulich@suse.com>
+
 	* config/tc-i386.c (build_evex_prefix): Drop early setting of
 	vec_length.
 
diff --git a/gas/config/tc-i386.c b/gas/config/tc-i386.c
index 18f4a24..5a652a9 100644
--- a/gas/config/tc-i386.c
+++ b/gas/config/tc-i386.c
@@ -1944,7 +1944,8 @@ cpu_flags_match (const insn_template *t)
 	    {
 	      /* We need to check a few extra flags with AVX.  */
 	      if (cpu.bitfield.cpuavx
-		  && (!t->opcode_modifier.sse2avx || sse2avx)
+		  && (!t->opcode_modifier.sse2avx
+		      || (sse2avx && !i.prefix[DATA_PREFIX]))
 		  && (!x.bitfield.cpuaes || cpu.bitfield.cpuaes)
 		  && (!x.bitfield.cpugfni || cpu.bitfield.cpugfni)
 		  && (!x.bitfield.cpupclmul || cpu.bitfield.cpupclmul))
@@ -4787,8 +4788,11 @@ md_assemble (char *line)
       return;
     }
 
-  /* Check for data size prefix on VEX/XOP/EVEX encoded insns.  */
-  if (i.prefix[DATA_PREFIX] && is_any_vex_encoding (&i.tm))
+  /* Check for data size prefix on VEX/XOP/EVEX encoded and SIMD insns.  */
+  if (i.prefix[DATA_PREFIX]
+      && (is_any_vex_encoding (&i.tm)
+	  || i.tm.operand_types[i.imm_operands].bitfield.class >= RegMMX
+	  || i.tm.operand_types[i.imm_operands + 1].bitfield.class >= RegMMX))
     {
       as_bad (_("data size prefix invalid with `%s'"), i.tm.name);
       return;
diff --git a/gas/testsuite/gas/i386/prefix32.l b/gas/testsuite/gas/i386/prefix32.l
index 09a00cd..e43abbd 100644
--- a/gas/testsuite/gas/i386/prefix32.l
+++ b/gas/testsuite/gas/i386/prefix32.l
@@ -5,7 +5,8 @@
 .*:9: Error: invalid .* `vaddss' after `repne'
 .*:14: Error: same type of prefix .*
 .*:15: Error: same type of prefix .*
-.*:19: Error: same type of prefix .*
+.*:18: Error: data size .* `addps'
+.*:19: Error: data size .* `addpd'
 .*:20: Error: data size .* `vaddps'
 .*:21: Error: data size .* `vaddpd'
 .*:25: Error: same type of prefix .*
@@ -28,9 +29,8 @@ GAS LISTING .*
 [ 	]*15[ 	]+rep bnd ret
 [ 	]*16[ 	]*
 [ 	]*17[ 	]+\.Ldata16:
-[ 	]*18[ 	]+#bogus	data16 addps	%xmm0, %xmm0
-[ 	]*19[ 	]+\?\?\?\? 660F58C0[ 	]+data16 addpd	%xmm0, %xmm0
-\*\*\*\*  Error: .*
+[ 	]*18[ 	]+data16 addps	%xmm0, %xmm0
+[ 	]*19[ 	]+data16 addpd	%xmm0, %xmm0
 [ 	]*20[ 	]+data16 vaddps	%xmm0, %xmm0, %xmm0
 [ 	]*21[ 	]+data16 vaddpd	%xmm0, %xmm0, %xmm0
 [ 	]*22[ 	]*
diff --git a/gas/testsuite/gas/i386/prefix32.s b/gas/testsuite/gas/i386/prefix32.s
index a4382a2..598b0a7 100644
--- a/gas/testsuite/gas/i386/prefix32.s
+++ b/gas/testsuite/gas/i386/prefix32.s
@@ -15,7 +15,7 @@ prefix:
 	rep bnd ret
 
 .Ldata16:
-#bogus	data16 addps	%xmm0, %xmm0
+	data16 addps	%xmm0, %xmm0
 	data16 addpd	%xmm0, %xmm0
 	data16 vaddps	%xmm0, %xmm0, %xmm0
 	data16 vaddpd	%xmm0, %xmm0, %xmm0
diff --git a/gas/testsuite/gas/i386/prefix64.l b/gas/testsuite/gas/i386/prefix64.l
index 0659d9c..712f4e0 100644
--- a/gas/testsuite/gas/i386/prefix64.l
+++ b/gas/testsuite/gas/i386/prefix64.l
@@ -5,7 +5,8 @@
 .*:9: Error: invalid .* `vaddss' after `repne'
 .*:14: Error: same type of prefix .*
 .*:15: Error: same type of prefix .*
-.*:19: Error: same type of prefix .*
+.*:18: Error: data size .* `addps'
+.*:19: Error: data size .* `addpd'
 .*:20: Error: data size .* `vaddps'
 .*:21: Error: data size .* `vaddpd'
 GAS LISTING .*
@@ -27,10 +28,9 @@ GAS LISTING .*
 [ 	]*15[ 	]+rep bnd ret
 [ 	]*16[ 	]*
 [ 	]*17[ 	]+\.Ldata16:
-[ 	]*18[ 	]+#bogus	data16 addps	%xmm0, %xmm0
-[ 	]*19[ 	]+\?\?\?\? 660F58C0[ 	]+data16 addpd	%xmm0, %xmm0
-\*\*\*\*  Error: .*
+[ 	]*18[ 	]+data16 addps	%xmm0, %xmm0
+[ 	]*19[ 	]+data16 addpd	%xmm0, %xmm0
 [ 	]*20[ 	]+data16 vaddps	%xmm0, %xmm0, %xmm0
 [ 	]*21[ 	]+data16 vaddpd	%xmm0, %xmm0, %xmm0
 [ 	]*22[ 	]*
-[ 	]*23[ 	]*[\?]+ 0+[ \t]+\.p2align	4,0
+#pass
diff --git a/gas/testsuite/gas/i386/prefix64.s b/gas/testsuite/gas/i386/prefix64.s
index 39fb477..32091c7 100644
--- a/gas/testsuite/gas/i386/prefix64.s
+++ b/gas/testsuite/gas/i386/prefix64.s
@@ -15,7 +15,7 @@ prefix:
 	rep bnd ret
 
 .Ldata16:
-#bogus	data16 addps	%xmm0, %xmm0
+	data16 addps	%xmm0, %xmm0
 	data16 addpd	%xmm0, %xmm0
 	data16 vaddps	%xmm0, %xmm0, %xmm0
 	data16 vaddpd	%xmm0, %xmm0, %xmm0
diff --git a/gas/testsuite/gas/i386/sse2avx.d b/gas/testsuite/gas/i386/sse2avx.d
index d035491..1a7df71 100644
--- a/gas/testsuite/gas/i386/sse2avx.d
+++ b/gas/testsuite/gas/i386/sse2avx.d
@@ -9,6 +9,8 @@ Disassembly of section .text:
 0+ <_start>:
 [ 	]*[a-f0-9]+:	c5 f8 ae 11          	vldmxcsr \(%ecx\)
 [ 	]*[a-f0-9]+:	c5 f8 ae 19          	vstmxcsr \(%ecx\)
+[ 	]*[a-f0-9]+:	66 0f ae 11          	data16 ldmxcsr \(%ecx\)
+[ 	]*[a-f0-9]+:	66 0f ae 19          	data16 stmxcsr \(%ecx\)
 [ 	]*[a-f0-9]+:	c5 f8 5b f4          	vcvtdq2ps %xmm4,%xmm6
 [ 	]*[a-f0-9]+:	c5 f8 5b 21          	vcvtdq2ps \(%ecx\),%xmm4
 [ 	]*[a-f0-9]+:	c5 fb e6 f4          	vcvtpd2dq %xmm4,%xmm6
diff --git a/gas/testsuite/gas/i386/sse2avx.s b/gas/testsuite/gas/i386/sse2avx.s
index 00080cf..0ad600c 100644
--- a/gas/testsuite/gas/i386/sse2avx.s
+++ b/gas/testsuite/gas/i386/sse2avx.s
@@ -7,6 +7,10 @@ _start:
 	ldmxcsr (%ecx)
 	stmxcsr (%ecx)
 
+# These should not be converted
+	data16 ldmxcsr (%ecx)
+	data16 stmxcsr (%ecx)
+
 # Tests for op xmm/mem128, xmm
 	cvtdq2ps %xmm4,%xmm6
 	cvtdq2ps (%ecx),%xmm4