x86: correct decoding of packed-FP-only AVX encodings

Various AVX insns utilizing the X macro fail to reject F3/F2 embedded
prefix encodings. As the PREFIX_OPCODE attribute wasn't used by any
non-legacy-encoded insns so far, re-use it to achieve the intended
effect.

5 files changed, 119 insertions, 31 deletions

diff --git a/gas/ChangeLog b/gas/ChangeLog
index b4476bf..78ffd80 100644
--- a/gas/ChangeLog
+++ b/gas/ChangeLog
@@ -1,5 +1,11 @@
 2020-06-09  Jan Beulich  <jbeulich@suse.com>
 
+	* testsuite/gas/i386/prefix.s: Add bogus REP / EVEX.W prefix
+	with VEX/EVEX encoding tests.
+	* testsuite/gas/i386/prefix.d: Adjust expectations.
+
+2020-06-09  Jan Beulich  <jbeulich@suse.com>
+
 	* config/tc-i386.c (process_suffix): Restrict defaulting to 'q'
 	suffix.
 	* testsuite/gas/i386/noreg64.s: Add lcall/ljmp cases.
diff --git a/gas/testsuite/gas/i386/prefix.d b/gas/testsuite/gas/i386/prefix.d
index e9ad5eb..9e293bc 100644
--- a/gas/testsuite/gas/i386/prefix.d
+++ b/gas/testsuite/gas/i386/prefix.d
@@ -72,6 +72,20 @@ Disassembly of section .text:
 [ 	]*[a-f0-9]+:	90                   	nop
 [ 	]*[a-f0-9]+:	f2 0f c7             	\(bad\)  
 [ 	]*[a-f0-9]+:	f0 90                	lock nop
+[ 	]*[a-f0-9]+:	f3 0f 28             	repz \(bad\) *
+[ 	]*[a-f0-9]+:	ff cc                	dec    %esp
+[ 	]*[a-f0-9]+:	c5 fa 28             	\(bad\) *
+[ 	]*[a-f0-9]+:	ff cc                	dec    %esp
+[ 	]*[a-f0-9]+:	c4 e1 7b 28          	\(bad\) *
+[ 	]*[a-f0-9]+:	ff cc                	dec    %esp
+[ 	]*[a-f0-9]+:	62 f1 fc 08 28       	\(bad\) *
+[ 	]*[a-f0-9]+:	ff cc                	dec    %esp
+[ 	]*[a-f0-9]+:	62 f1 7e 08 28       	\(bad\) *
+[ 	]*[a-f0-9]+:	ff cc                	dec    %esp
+[ 	]*[a-f0-9]+:	62 f1 7d 08 28       	\(bad\) *
+[ 	]*[a-f0-9]+:	ff cc                	dec    %esp
+[ 	]*[a-f0-9]+:	62 f1 ff 08 28       	\(bad\) *
+[ 	]*[a-f0-9]+:	ff cc                	dec    %esp
 [ 	]*[a-f0-9]+:	c5 fb e6 40 20       	vcvtpd2dqx 0x20\(%eax\),%xmm0
 [ 	]*[a-f0-9]+:	62 f1 ff 18 e6 40 04 	vcvtpd2dq 0x20\(%eax\)\{1to2\},%xmm0
 [ 	]*[a-f0-9]+:	c5 fb e6 40 20       	vcvtpd2dqx 0x20\(%eax\),%xmm0
diff --git a/gas/testsuite/gas/i386/prefix.s b/gas/testsuite/gas/i386/prefix.s
index a4c60a7..78fd478 100644
--- a/gas/testsuite/gas/i386/prefix.s
+++ b/gas/testsuite/gas/i386/prefix.s
@@ -391,6 +391,66 @@
 
 	nop
 
+	repz; movaps %xmm7, %xmm7
+	int $3
+
+# "repz" vmovaps %xmm7, %xmm7
+	.byte 0xc5
+	.byte 0xfa
+	.byte 0x28
+	.byte 0xff
+
+	int $3
+
+# "repnz" {vex3} vmovaps %xmm7, %xmm7
+	.byte 0xc4
+	.byte 0xe1
+	.byte 0x7b
+	.byte 0x28
+	.byte 0xff
+
+	int $3
+
+# "EVEX.W1" vmovaps %xmm7, %xmm7
+	.byte 0x62
+	.byte 0xf1
+	.byte 0xfc
+	.byte 0x08
+	.byte 0x28
+	.byte 0xff
+
+	int $3
+
+# "repz" vmovaps %xmm7, %xmm7
+	.byte 0x62
+	.byte 0xf1
+	.byte 0x7e
+	.byte 0x08
+	.byte 0x28
+	.byte 0xff
+
+	int $3
+
+# "EVEX.W0" vmovapd %xmm7, %xmm7
+	.byte 0x62
+	.byte 0xf1
+	.byte 0x7d
+	.byte 0x08
+	.byte 0x28
+	.byte 0xff
+
+	int $3
+
+# "repnz" vmovapd %xmm7, %xmm7
+	.byte 0x62
+	.byte 0xf1
+	.byte 0xff
+	.byte 0x08
+	.byte 0x28
+	.byte 0xff
+
+	int $3
+
 	vcvtpd2dqx 0x20(%eax),%xmm0
 	vcvtpd2dq 0x20(%eax){1to2},%xmm0
 	vcvtpd2dqx 0x20(%eax),%xmm0
diff --git a/opcodes/ChangeLog b/opcodes/ChangeLog
index 0ef06c9..feda626 100644
--- a/opcodes/ChangeLog
+++ b/opcodes/ChangeLog
@@ -1,5 +1,15 @@
 2020-06-09  Jan Beulich  <jbeulich@suse.com>
 
+	* i386-dis.c (vex_table): Use PREFIX_OPCODE for vunpcklpX,
+	vunpckhpX, vmovapX, vandpX, vandnpX, vorpX, vxorpX and vshufpX.
+	(vex_len_table) : Likewise for vmovlpX, vmovhpX, vmovntpX, and
+	vmovmskpX.
+	(print_insn): Drop pointless check against bad_opcode. Split
+	prefix validation into legacy and VEX-and-alike parts.
+	(putop): Re-work 'X' macro handling.
+
+2020-06-09  Jan Beulich  <jbeulich@suse.com>
+
 	* i386-dis.c (MOD_0F51): Rename to ...
 	(MOD_0F50): ... this.
 
diff --git a/opcodes/i386-dis.c b/opcodes/i386-dis.c
index 6721fac..799d9d4 100644
--- a/opcodes/i386-dis.c
+++ b/opcodes/i386-dis.c
@@ -8485,8 +8485,8 @@ static const struct dis386 vex_table[][256] = {
     { PREFIX_TABLE (PREFIX_VEX_0F11) },
     { PREFIX_TABLE (PREFIX_VEX_0F12) },
     { MOD_TABLE (MOD_VEX_0F13) },
-    { "vunpcklpX",	{ XM, Vex, EXx }, 0 },
-    { "vunpckhpX",	{ XM, Vex, EXx }, 0 },
+    { "vunpcklpX",	{ XM, Vex, EXx }, PREFIX_OPCODE },
+    { "vunpckhpX",	{ XM, Vex, EXx }, PREFIX_OPCODE },
     { PREFIX_TABLE (PREFIX_VEX_0F16) },
     { MOD_TABLE (MOD_VEX_0F17) },
     /* 18 */
@@ -8508,8 +8508,8 @@ static const struct dis386 vex_table[][256] = {
     { Bad_Opcode },
     { Bad_Opcode },
     /* 28 */
-    { "vmovapX",	{ XM, EXx }, 0 },
-    { "vmovapX",	{ EXxS, XM }, 0 },
+    { "vmovapX",	{ XM, EXx }, PREFIX_OPCODE },
+    { "vmovapX",	{ EXxS, XM }, PREFIX_OPCODE },
     { PREFIX_TABLE (PREFIX_VEX_0F2A) },
     { MOD_TABLE (MOD_VEX_0F2B) },
     { PREFIX_TABLE (PREFIX_VEX_0F2C) },
@@ -8557,10 +8557,10 @@ static const struct dis386 vex_table[][256] = {
     { PREFIX_TABLE (PREFIX_VEX_0F51) },
     { PREFIX_TABLE (PREFIX_VEX_0F52) },
     { PREFIX_TABLE (PREFIX_VEX_0F53) },
-    { "vandpX",		{ XM, Vex, EXx }, 0 },
-    { "vandnpX",	{ XM, Vex, EXx }, 0 },
-    { "vorpX",		{ XM, Vex, EXx }, 0 },
-    { "vxorpX",		{ XM, Vex, EXx }, 0 },
+    { "vandpX",		{ XM, Vex, EXx }, PREFIX_OPCODE },
+    { "vandnpX",	{ XM, Vex, EXx }, PREFIX_OPCODE },
+    { "vorpX",		{ XM, Vex, EXx }, PREFIX_OPCODE },
+    { "vxorpX",		{ XM, Vex, EXx }, PREFIX_OPCODE },
     /* 58 */
     { PREFIX_TABLE (PREFIX_VEX_0F58) },
     { PREFIX_TABLE (PREFIX_VEX_0F59) },
@@ -8685,7 +8685,7 @@ static const struct dis386 vex_table[][256] = {
     { Bad_Opcode },
     { PREFIX_TABLE (PREFIX_VEX_0FC4) },
     { PREFIX_TABLE (PREFIX_VEX_0FC5) },
-    { "vshufpX",	{ XM, Vex, EXx, Ib }, 0 },
+    { "vshufpX",	{ XM, Vex, EXx, Ib }, PREFIX_OPCODE },
     { Bad_Opcode },
     /* c8 */
     { Bad_Opcode },
@@ -9355,7 +9355,7 @@ static const struct dis386 vex_len_table[][2] = {
 
   /* VEX_LEN_0F13_M_0 */
   {
-    { "vmovlpX",	{ EXq, XM }, 0 },
+    { "vmovlpX",	{ EXq, XM }, PREFIX_OPCODE },
   },
 
   /* VEX_LEN_0F16_P_0_M_0 */
@@ -9375,7 +9375,7 @@ static const struct dis386 vex_len_table[][2] = {
 
   /* VEX_LEN_0F17_M_0 */
   {
-    { "vmovhpX",	{ EXq, XM }, 0 },
+    { "vmovhpX",	{ EXq, XM }, PREFIX_OPCODE },
   },
 
   /* VEX_LEN_0F41_P_0 */
@@ -10592,7 +10592,7 @@ static const struct dis386 mod_table[][2] = {
   },
   {
     /* MOD_VEX_0F2B */
-    { "vmovntpX",	{ Mx, XM }, 0 },
+    { "vmovntpX",	{ Mx, XM }, PREFIX_OPCODE },
   },
   {
     /* MOD_VEX_W_0_0F41_P_0_LEN_1 */
@@ -10752,7 +10752,7 @@ static const struct dis386 mod_table[][2] = {
   {
     /* MOD_VEX_0F50 */
     { Bad_Opcode },
-    { "vmovmskpX",	{ Gdq, XS }, 0 },
+    { "vmovmskpX",	{ Gdq, XS }, PREFIX_OPCODE },
   },
   {
     /* MOD_VEX_0F71_REG_2 */
@@ -12266,14 +12266,18 @@ print_insn (bfd_vma pc, disassemble_info *info)
      PREFIX_REPZ/PREFIX_REPNZ fix, we check the PREFIX_DATA prefix
      separately.  */
   if (dp->prefix_requirement == PREFIX_OPCODE
-      && dp != &bad_opcode
-      && (((prefixes
-	    & (PREFIX_REPZ | PREFIX_REPNZ)) != 0
+      && (((need_vex
+	    ? vex.prefix == REPE_PREFIX_OPCODE
+	      || vex.prefix == REPNE_PREFIX_OPCODE
+	    : (prefixes
+	       & (PREFIX_REPZ | PREFIX_REPNZ)) != 0)
 	   && (used_prefixes
 	       & (PREFIX_REPZ | PREFIX_REPNZ)) == 0)
-	  || ((((prefixes
-		 & (PREFIX_REPZ | PREFIX_REPNZ | PREFIX_DATA))
-		== PREFIX_DATA)
+	  || (((need_vex
+		? vex.prefix == DATA_PREFIX_OPCODE
+		: ((prefixes
+		    & (PREFIX_REPZ | PREFIX_REPNZ | PREFIX_DATA))
+		   == PREFIX_DATA))
 	       && (used_prefixes & PREFIX_DATA) == 0))))
     {
       (*info->fprintf_func) (info->stream, "(bad)");
@@ -13230,21 +13234,15 @@ putop (const char *in_template, int sizeflag)
 	      SAVE_LAST (*p);
 	      break;
 	    }
-	  if (need_vex && vex.prefix)
+	  if (need_vex
+	      ? vex.prefix == DATA_PREFIX_OPCODE
+	      : prefixes & PREFIX_DATA)
 	    {
-	      if (vex.prefix == DATA_PREFIX_OPCODE)
-		*obufp++ = 'd';
-	      else
-		*obufp++ = 's';
+	      *obufp++ = 'd';
+	      used_prefixes |= PREFIX_DATA;
 	    }
 	  else
-	    {
-	      if (prefixes & PREFIX_DATA)
-		*obufp++ = 'd';
-	      else
-		*obufp++ = 's';
-	      used_prefixes |= (prefixes & PREFIX_DATA);
-	    }
+	    *obufp++ = 's';
 	  break;
 	case 'Y':
 	  if (l == 0 && len == 1)