x86: consistently print prefixes explicitly which are invalid with VEX etc

All of data size, rep, lock, and rex prefixes are invalid with VEX- and
alike encoded insns. Make sure they get printed explicitly in all cases,
to signal the anomaly. With this, do away with "rex_ignored" - if there
is a rex prefix, we want to print it anyway for VEX etc (and there's
nothing "ignored" about it in the first place - such an instruction will
raise #UD).

5 files changed, 26 insertions, 14 deletions

diff --git a/gas/ChangeLog b/gas/ChangeLog
index 78ffd80..8b78964 100644
--- a/gas/ChangeLog
+++ b/gas/ChangeLog
@@ -1,5 +1,11 @@
 2020-06-09  Jan Beulich  <jbeulich@suse.com>
 
+	* testsuite/gas/i386/prefix.s: Add bogus prefix-with-VEX/EVEX
+	encoding tests.
+	* testsuite/gas/i386/prefix.d: Adjust expectations.
+
+2020-06-09  Jan Beulich  <jbeulich@suse.com>
+
 	* testsuite/gas/i386/prefix.s: Add bogus REP / EVEX.W prefix
 	with VEX/EVEX encoding tests.
 	* testsuite/gas/i386/prefix.d: Adjust expectations.
diff --git a/gas/testsuite/gas/i386/prefix.d b/gas/testsuite/gas/i386/prefix.d
index 9e293bc..02e1813 100644
--- a/gas/testsuite/gas/i386/prefix.d
+++ b/gas/testsuite/gas/i386/prefix.d
@@ -86,8 +86,11 @@ Disassembly of section .text:
 [ 	]*[a-f0-9]+:	ff cc                	dec    %esp
 [ 	]*[a-f0-9]+:	62 f1 ff 08 28       	\(bad\) *
 [ 	]*[a-f0-9]+:	ff cc                	dec    %esp
+[ 	]*[a-f0-9]+:	66 c5 f8 28 c0       	data16 vmovaps %xmm0,%xmm0
+[ 	]*[a-f0-9]+:	f3 c4 e1 78 28 c0    	repz vmovaps %xmm0,%xmm0
+[ 	]*[a-f0-9]+:	f2 c5 f8 28 c0       	repnz vmovaps %xmm0,%xmm0
+[ 	]*[a-f0-9]+:	f0 62 f1 7c 08 28 c0 	lock vmovaps %xmm0,%xmm0
 [ 	]*[a-f0-9]+:	c5 fb e6 40 20       	vcvtpd2dqx 0x20\(%eax\),%xmm0
 [ 	]*[a-f0-9]+:	62 f1 ff 18 e6 40 04 	vcvtpd2dq 0x20\(%eax\)\{1to2\},%xmm0
 [ 	]*[a-f0-9]+:	c5 fb e6 40 20       	vcvtpd2dqx 0x20\(%eax\),%xmm0
-	...
 #pass
diff --git a/gas/testsuite/gas/i386/prefix.s b/gas/testsuite/gas/i386/prefix.s
index 78fd478..8d563e1 100644
--- a/gas/testsuite/gas/i386/prefix.s
+++ b/gas/testsuite/gas/i386/prefix.s
@@ -451,6 +451,11 @@
 
 	int $3
 
+	.byte 0x66; vmovaps %xmm0, %xmm0
+	repz; {vex3} vmovaps %xmm0, %xmm0
+	repnz; vmovaps %xmm0, %xmm0
+	lock; {evex} vmovaps %xmm0, %xmm0
+
 	vcvtpd2dqx 0x20(%eax),%xmm0
 	vcvtpd2dq 0x20(%eax){1to2},%xmm0
 	vcvtpd2dqx 0x20(%eax),%xmm0
diff --git a/opcodes/ChangeLog b/opcodes/ChangeLog
index c6f1d26..6b3869f 100644
--- a/opcodes/ChangeLog
+++ b/opcodes/ChangeLog
@@ -1,5 +1,13 @@
 2020-06-09  Jan Beulich  <jbeulich@suse.com>
 
+	* i386-dis.c (rex_ignored): Delete.
+	(ckprefix): Drop rex_ignored initialization.
+	(get_valid_dis386): Drop setting of rex_ignored.
+	(print_insn): Drop checking of rex_ignored. Don't record data
+	size prefix as used with VEX-and-alike encodings.
+
+2020-06-09  Jan Beulich  <jbeulich@suse.com>
+
 	* i386-dis.c (MOD_0F12_PREFIX_2, MOD_0F16_PREFIX_2,
 	MOD_VEX_0F12_PREFIX_2, MOD_VEX_0F16_PREFIX_2): New enumerators.
 	(VEX_LEN_0F12_P_2, VEX_LEN_0F16_P_2): Delete.
diff --git a/opcodes/i386-dis.c b/opcodes/i386-dis.c
index 3861371..be6958a 100644
--- a/opcodes/i386-dis.c
+++ b/opcodes/i386-dis.c
@@ -153,8 +153,6 @@ static int prefixes;
 static int rex;
 /* Bits of REX we've already used.  */
 static int rex_used;
-/* REX bits in original REX prefix ignored.  */
-static int rex_ignored;
 /* Mark parts used in the REX prefix.  When we are testing for
    empty prefix (for 8bit register REX extension), just mask it
    out.  Otherwise test for REX bit is excuse for existence of REX
@@ -11057,7 +11055,6 @@ ckprefix (void)
 {
   int newrex, i, length;
   rex = 0;
-  rex_ignored = 0;
   prefixes = 0;
   used_prefixes = 0;
   rex_used = 0;
@@ -11517,8 +11514,6 @@ get_valid_dis386 (const struct dis386 *dp, disassemble_info *info)
 
     case USE_XOP_8F_TABLE:
       FETCH_DATA (info, codep + 3);
-      /* All bits in the REX prefix are ignored.  */
-      rex_ignored = rex;
       rex = ~(*codep >> 5) & 0x7;
 
       /* VEX_TABLE_INDEX is the mmmmm part of the XOP byte 1 "RCB.mmmmm".  */
@@ -11580,8 +11575,6 @@ get_valid_dis386 (const struct dis386 *dp, disassemble_info *info)
     case USE_VEX_C4_TABLE:
       /* VEX prefix.  */
       FETCH_DATA (info, codep + 3);
-      /* All bits in the REX prefix are ignored.  */
-      rex_ignored = rex;
       rex = ~(*codep >> 5) & 0x7;
       switch ((*codep & 0x1f))
 	{
@@ -11647,8 +11640,6 @@ get_valid_dis386 (const struct dis386 *dp, disassemble_info *info)
     case USE_VEX_C5_TABLE:
       /* VEX prefix.  */
       FETCH_DATA (info, codep + 2);
-      /* All bits in the REX prefix are ignored.  */
-      rex_ignored = rex;
       rex = (*codep & 0x80) ? 0 : REX_R;
 
       /* For the 2-byte VEX prefix in 32-bit mode, the highest bit in
@@ -11697,8 +11688,6 @@ get_valid_dis386 (const struct dis386 *dp, disassemble_info *info)
       /* EVEX prefix.  */
       vex.evex = 1;
       FETCH_DATA (info, codep + 4);
-      /* All bits in the REX prefix are ignored.  */
-      rex_ignored = rex;
       /* The first byte after 0x62.  */
       rex = ~(*codep >> 5) & 0x7;
       vex.r = *codep & 0x10;
@@ -12179,7 +12168,7 @@ print_insn (bfd_vma pc, disassemble_info *info)
     }
 
   /* Check if the REX prefix is used.  */
-  if (rex_ignored == 0 && (rex ^ rex_used) == 0 && last_rex_prefix >= 0)
+  if ((rex ^ rex_used) == 0 && !need_vex && last_rex_prefix >= 0)
     all_prefixes[last_rex_prefix] = 0;
 
   /* Check if the SEG prefix is used.  */
@@ -12195,7 +12184,8 @@ print_insn (bfd_vma pc, disassemble_info *info)
 
   /* Check if the DATA prefix is used.  */
   if ((prefixes & PREFIX_DATA) != 0
-      && (used_prefixes & PREFIX_DATA) != 0)
+      && (used_prefixes & PREFIX_DATA) != 0
+      && !need_vex)
     all_prefixes[last_data_prefix] = 0;
 
   /* Print the extra prefixes.  */