x86: don't abort() upon DATA16 prefix on (E)VEX encoded insn

Instead of hitting the abort() in output_insn() (commented by "There
should be no other prefixes for instructions with VEX prefix"), report
a proper diagnostic instead, just like we do e.g. for invalid REP
prefixes.

7 files changed, 142 insertions, 5 deletions

diff --git a/gas/ChangeLog b/gas/ChangeLog
index af24a44..568a47f 100644
--- a/gas/ChangeLog
+++ b/gas/ChangeLog
@@ -1,5 +1,16 @@
 2018-07-31  Jan Beulich  <jbeulich@suse.com>
 
+	* config/tc-i386.c (is_any_vex_encoding): New.
+	(process_immext, process_suffix): Use it.
+	(md_assemble): Likewise. Reject DATA_PREFIX with VEX/XOP/EVEX
+	insn.
+	* testsuite/gas/i386/prefix32.s, testsuite/gas/i386/prefix32.l,
+	testsuite/gas/i386/prefix64.s, testsuite/gas/i386/prefix64.l
+	 New.
+	* testsuite/gas/i386/i386.exp: Run new tests.
+
+2018-07-31  Jan Beulich  <jbeulich@suse.com>
+
 	* config/tc-i386.c (parse_real_register): Use cpuavx512f instead
 	of cpuvrex.
 
diff --git a/gas/config/tc-i386.c b/gas/config/tc-i386.c
index 5a5bf5f..438bae9 100644
--- a/gas/config/tc-i386.c
+++ b/gas/config/tc-i386.c
@@ -3478,6 +3478,13 @@ is_evex_encoding (const insn_template *t)
 	 || t->opcode_modifier.staticrounding || t->opcode_modifier.sae;
 }
 
+static INLINE bfd_boolean
+is_any_vex_encoding (const insn_template *t)
+{
+  return t->opcode_modifier.vex || t->opcode_modifier.vexopcode
+	 || is_evex_encoding (t);
+}
+
 /* Build the EVEX prefix.  */
 
 static void
@@ -3760,9 +3767,7 @@ bad_register_operand:
 
   gas_assert (i.imm_operands <= 1
 	      && (i.operands <= 2
-		  || ((i.tm.opcode_modifier.vex
-		       || i.tm.opcode_modifier.vexopcode
-		       || is_evex_encoding (&i.tm))
+		  || (is_any_vex_encoding (&i.tm)
 		      && i.operands <= 4)));
 
   exp = &im_expressions[i.imm_operands++];
@@ -4125,6 +4130,13 @@ md_assemble (char *line)
       return;
     }
 
+  /* Check for data size prefix on VEX/XOP/EVEX encoded insns.  */
+  if (i.prefix[DATA_PREFIX] && is_any_vex_encoding (&i.tm))
+    {
+      as_bad (_("data size prefix invalid with `%s'"), i.tm.name);
+      return;
+    }
+
   /* Check if HLE prefix is OK.  */
   if (i.hle_prefix && !check_hle ())
     return;
@@ -4211,8 +4223,7 @@ md_assemble (char *line)
       as_warn (_("translating to `%sp'"), i.tm.name);
     }
 
-  if (i.tm.opcode_modifier.vex || i.tm.opcode_modifier.vexopcode
-      || is_evex_encoding (&i.tm))
+  if (is_any_vex_encoding (&i.tm))
     {
       if (flag_code == CODE_16BIT)
 	{
@@ -6137,6 +6148,9 @@ process_suffix (void)
       else if (i.suffix != QWORD_MNEM_SUFFIX
 	       && !i.tm.opcode_modifier.ignoresize
 	       && !i.tm.opcode_modifier.floatmf
+	       && !i.tm.opcode_modifier.vex
+	       && !i.tm.opcode_modifier.vexopcode
+	       && !is_evex_encoding (&i.tm)
 	       && ((i.suffix == LONG_MNEM_SUFFIX) == (flag_code == CODE_16BIT)
 		   || (flag_code == CODE_64BIT
 		       && i.tm.opcode_modifier.jumpbyte)))
diff --git a/gas/testsuite/gas/i386/i386.exp b/gas/testsuite/gas/i386/i386.exp
index 6e22958..babeb94 100644
--- a/gas/testsuite/gas/i386/i386.exp
+++ b/gas/testsuite/gas/i386/i386.exp
@@ -57,6 +57,7 @@ if [expr ([istarget "i*86-*-*"] ||  [istarget "x86_64-*-*"]) && [gas_32_check]]
     run_list_test "intelbad" ""
     run_dump_test "intelok"
     run_dump_test "prefix"
+    run_list_test "prefix32" "-al"
     run_dump_test "amd"
     run_dump_test "katmai"
     run_dump_test "jump"
@@ -682,6 +683,7 @@ if [expr ([istarget "i*86-*-*"] || [istarget "x86_64-*-*"]) && [gas_64_check]] t
     run_list_test "suffix-bad"
     run_list_test "x86-64-suffix-bad"
     run_list_test "unspec64" ""
+    run_list_test "prefix64" "-al"
     run_dump_test "x86-64-fxsave"
     run_dump_test "x86-64-fxsave-intel"
     run_dump_test "x86-64-arch-1"
diff --git a/gas/testsuite/gas/i386/prefix32.l b/gas/testsuite/gas/i386/prefix32.l
new file mode 100644
index 0000000..ea7f304
--- /dev/null
+++ b/gas/testsuite/gas/i386/prefix32.l
@@ -0,0 +1,34 @@
+.*: Assembler messages:
+.*:6: Error: invalid .* `addss' after `repe'
+.*:7: Error: invalid .* `addss' after `repne'
+.*:8: Error: invalid .* `vaddss' after `repe'
+.*:9: Error: invalid .* `vaddss' after `repne'
+.*:14: Error: same type of prefix .*
+.*:15: Error: same type of prefix .*
+.*:19: Error: same type of prefix .*
+.*:20: Error: data size .* `vaddps'
+.*:21: Error: data size .* `vaddpd'
+GAS LISTING .*
+#...
+[ 	]*1[ 	]+\.text
+[ 	]*2[ 	]+prefix:
+[ 	]*3[ 	]+\?\?\?\? 67E203[ 	]+addr16 loop	\.Lrep_ret
+[ 	]*4[ 	]+\?\?\?\? 67E304[ 	]+addr16 jecxz	\.Ldata16
+[ 	]*5[ 	]*
+[ 	]*6[ 	]+repe addss	%xmm0, %xmm0
+[ 	]*7[ 	]+repne addss	%xmm0, %xmm0
+[ 	]*8[ 	]+repe vaddss	%xmm0, %xmm0, %xmm0
+[ 	]*9[ 	]+repne vaddss	%xmm0, %xmm0, %xmm0
+[ 	]*10[ 	]*
+[ 	]*11[ 	]+\.Lrep_ret:
+[ 	]*12[ 	]+\?\?\?\? F2C3[ 	]+bnd ret
+[ 	]*13[ 	]+\?\?\?\? F3C3[ 	]+rep ret
+[ 	]*14[ 	]+bnd rep ret
+[ 	]*15[ 	]+rep bnd ret
+[ 	]*16[ 	]*
+[ 	]*17[ 	]+\.Ldata16:
+[ 	]*18[ 	]+#bogus	data16 addps	%xmm0, %xmm0
+[ 	]*19[ 	]+\?\?\?\? 660F58C0[ 	]+data16 addpd	%xmm0, %xmm0
+\*\*\*\*  Error: .*
+[ 	]*20[ 	]+data16 vaddps	%xmm0, %xmm0, %xmm0
+[ 	]*21[ 	]+data16 vaddpd	%xmm0, %xmm0, %xmm0
diff --git a/gas/testsuite/gas/i386/prefix32.s b/gas/testsuite/gas/i386/prefix32.s
new file mode 100644
index 0000000..d20cb47
--- /dev/null
+++ b/gas/testsuite/gas/i386/prefix32.s
@@ -0,0 +1,21 @@
+	.text
+prefix:
+	addr16 loop	.Lrep_ret
+	addr16 jecxz	.Ldata16
+
+	repe addss	%xmm0, %xmm0
+	repne addss	%xmm0, %xmm0
+	repe vaddss	%xmm0, %xmm0, %xmm0
+	repne vaddss	%xmm0, %xmm0, %xmm0
+
+.Lrep_ret:
+	bnd ret
+	rep ret
+	bnd rep ret
+	rep bnd ret
+
+.Ldata16:
+#bogus	data16 addps	%xmm0, %xmm0
+	data16 addpd	%xmm0, %xmm0
+	data16 vaddps	%xmm0, %xmm0, %xmm0
+	data16 vaddpd	%xmm0, %xmm0, %xmm0
diff --git a/gas/testsuite/gas/i386/prefix64.l b/gas/testsuite/gas/i386/prefix64.l
new file mode 100644
index 0000000..0a5615a
--- /dev/null
+++ b/gas/testsuite/gas/i386/prefix64.l
@@ -0,0 +1,34 @@
+.*: Assembler messages:
+.*:6: Error: invalid .* `addss' after `repe'
+.*:7: Error: invalid .* `addss' after `repne'
+.*:8: Error: invalid .* `vaddss' after `repe'
+.*:9: Error: invalid .* `vaddss' after `repne'
+.*:14: Error: same type of prefix .*
+.*:15: Error: same type of prefix .*
+.*:19: Error: same type of prefix .*
+.*:20: Error: data size .* `vaddps'
+.*:21: Error: data size .* `vaddpd'
+GAS LISTING .*
+#...
+[ 	]*1[ 	]+\.text
+[ 	]*2[ 	]+prefix:
+[ 	]*3[ 	]+\?\?\?\? 67E203[ 	]+addr32 loop	\.Lrep_ret
+[ 	]*4[ 	]+\?\?\?\? 67E304[ 	]+addr32 jrcxz	\.Ldata16
+[ 	]*5[ 	]*
+[ 	]*6[ 	]+repe addss	%xmm0, %xmm0
+[ 	]*7[ 	]+repne addss	%xmm0, %xmm0
+[ 	]*8[ 	]+repe vaddss	%xmm0, %xmm0, %xmm0
+[ 	]*9[ 	]+repne vaddss	%xmm0, %xmm0, %xmm0
+[ 	]*10[ 	]*
+[ 	]*11[ 	]+\.Lrep_ret:
+[ 	]*12[ 	]+\?\?\?\? F2C3[ 	]+bnd ret
+[ 	]*13[ 	]+\?\?\?\? F3C3[ 	]+rep ret
+[ 	]*14[ 	]+bnd rep ret
+[ 	]*15[ 	]+rep bnd ret
+[ 	]*16[ 	]*
+[ 	]*17[ 	]+\.Ldata16:
+[ 	]*18[ 	]+#bogus	data16 addps	%xmm0, %xmm0
+[ 	]*19[ 	]+\?\?\?\? 660F58C0[ 	]+data16 addpd	%xmm0, %xmm0
+\*\*\*\*  Error: .*
+[ 	]*20[ 	]+data16 vaddps	%xmm0, %xmm0, %xmm0
+[ 	]*21[ 	]+data16 vaddpd	%xmm0, %xmm0, %xmm0
diff --git a/gas/testsuite/gas/i386/prefix64.s b/gas/testsuite/gas/i386/prefix64.s
new file mode 100644
index 0000000..ca79f49
--- /dev/null
+++ b/gas/testsuite/gas/i386/prefix64.s
@@ -0,0 +1,21 @@
+	.text
+prefix:
+	addr32 loop	.Lrep_ret
+	addr32 jrcxz	.Ldata16
+
+	repe addss	%xmm0, %xmm0
+	repne addss	%xmm0, %xmm0
+	repe vaddss	%xmm0, %xmm0, %xmm0
+	repne vaddss	%xmm0, %xmm0, %xmm0
+
+.Lrep_ret:
+	bnd ret
+	rep ret
+	bnd rep ret
+	rep bnd ret
+
+.Ldata16:
+#bogus	data16 addps	%xmm0, %xmm0
+	data16 addpd	%xmm0, %xmm0
+	data16 vaddps	%xmm0, %xmm0, %xmm0
+	data16 vaddpd	%xmm0, %xmm0, %xmm0