x86: correct decoding of packed-FP-only AVX encodings

Various AVX insns utilizing the X macro fail to reject F3/F2 embedded
prefix encodings. As the PREFIX_OPCODE attribute wasn't used by any
non-legacy-encoded insns so far, re-use it to achieve the intended
effect.

2 files changed, 39 insertions, 31 deletions

diff --git a/opcodes/ChangeLog b/opcodes/ChangeLog
index 0ef06c9..feda626 100644
--- a/opcodes/ChangeLog
+++ b/opcodes/ChangeLog
@@ -1,5 +1,15 @@
 2020-06-09  Jan Beulich  <jbeulich@suse.com>
 
+	* i386-dis.c (vex_table): Use PREFIX_OPCODE for vunpcklpX,
+	vunpckhpX, vmovapX, vandpX, vandnpX, vorpX, vxorpX and vshufpX.
+	(vex_len_table) : Likewise for vmovlpX, vmovhpX, vmovntpX, and
+	vmovmskpX.
+	(print_insn): Drop pointless check against bad_opcode. Split
+	prefix validation into legacy and VEX-and-alike parts.
+	(putop): Re-work 'X' macro handling.
+
+2020-06-09  Jan Beulich  <jbeulich@suse.com>
+
 	* i386-dis.c (MOD_0F51): Rename to ...
 	(MOD_0F50): ... this.
 
diff --git a/opcodes/i386-dis.c b/opcodes/i386-dis.c
index 6721fac..799d9d4 100644
--- a/opcodes/i386-dis.c
+++ b/opcodes/i386-dis.c
@@ -8485,8 +8485,8 @@ static const struct dis386 vex_table[][256] = {
     { PREFIX_TABLE (PREFIX_VEX_0F11) },
     { PREFIX_TABLE (PREFIX_VEX_0F12) },
     { MOD_TABLE (MOD_VEX_0F13) },
-    { "vunpcklpX",	{ XM, Vex, EXx }, 0 },
-    { "vunpckhpX",	{ XM, Vex, EXx }, 0 },
+    { "vunpcklpX",	{ XM, Vex, EXx }, PREFIX_OPCODE },
+    { "vunpckhpX",	{ XM, Vex, EXx }, PREFIX_OPCODE },
     { PREFIX_TABLE (PREFIX_VEX_0F16) },
     { MOD_TABLE (MOD_VEX_0F17) },
     /* 18 */
@@ -8508,8 +8508,8 @@ static const struct dis386 vex_table[][256] = {
     { Bad_Opcode },
     { Bad_Opcode },
     /* 28 */
-    { "vmovapX",	{ XM, EXx }, 0 },
-    { "vmovapX",	{ EXxS, XM }, 0 },
+    { "vmovapX",	{ XM, EXx }, PREFIX_OPCODE },
+    { "vmovapX",	{ EXxS, XM }, PREFIX_OPCODE },
     { PREFIX_TABLE (PREFIX_VEX_0F2A) },
     { MOD_TABLE (MOD_VEX_0F2B) },
     { PREFIX_TABLE (PREFIX_VEX_0F2C) },
@@ -8557,10 +8557,10 @@ static const struct dis386 vex_table[][256] = {
     { PREFIX_TABLE (PREFIX_VEX_0F51) },
     { PREFIX_TABLE (PREFIX_VEX_0F52) },
     { PREFIX_TABLE (PREFIX_VEX_0F53) },
-    { "vandpX",		{ XM, Vex, EXx }, 0 },
-    { "vandnpX",	{ XM, Vex, EXx }, 0 },
-    { "vorpX",		{ XM, Vex, EXx }, 0 },
-    { "vxorpX",		{ XM, Vex, EXx }, 0 },
+    { "vandpX",		{ XM, Vex, EXx }, PREFIX_OPCODE },
+    { "vandnpX",	{ XM, Vex, EXx }, PREFIX_OPCODE },
+    { "vorpX",		{ XM, Vex, EXx }, PREFIX_OPCODE },
+    { "vxorpX",		{ XM, Vex, EXx }, PREFIX_OPCODE },
     /* 58 */
     { PREFIX_TABLE (PREFIX_VEX_0F58) },
     { PREFIX_TABLE (PREFIX_VEX_0F59) },
@@ -8685,7 +8685,7 @@ static const struct dis386 vex_table[][256] = {
     { Bad_Opcode },
     { PREFIX_TABLE (PREFIX_VEX_0FC4) },
     { PREFIX_TABLE (PREFIX_VEX_0FC5) },
-    { "vshufpX",	{ XM, Vex, EXx, Ib }, 0 },
+    { "vshufpX",	{ XM, Vex, EXx, Ib }, PREFIX_OPCODE },
     { Bad_Opcode },
     /* c8 */
     { Bad_Opcode },
@@ -9355,7 +9355,7 @@ static const struct dis386 vex_len_table[][2] = {
 
   /* VEX_LEN_0F13_M_0 */
   {
-    { "vmovlpX",	{ EXq, XM }, 0 },
+    { "vmovlpX",	{ EXq, XM }, PREFIX_OPCODE },
   },
 
   /* VEX_LEN_0F16_P_0_M_0 */
@@ -9375,7 +9375,7 @@ static const struct dis386 vex_len_table[][2] = {
 
   /* VEX_LEN_0F17_M_0 */
   {
-    { "vmovhpX",	{ EXq, XM }, 0 },
+    { "vmovhpX",	{ EXq, XM }, PREFIX_OPCODE },
   },
 
   /* VEX_LEN_0F41_P_0 */
@@ -10592,7 +10592,7 @@ static const struct dis386 mod_table[][2] = {
   },
   {
     /* MOD_VEX_0F2B */
-    { "vmovntpX",	{ Mx, XM }, 0 },
+    { "vmovntpX",	{ Mx, XM }, PREFIX_OPCODE },
   },
   {
     /* MOD_VEX_W_0_0F41_P_0_LEN_1 */
@@ -10752,7 +10752,7 @@ static const struct dis386 mod_table[][2] = {
   {
     /* MOD_VEX_0F50 */
     { Bad_Opcode },
-    { "vmovmskpX",	{ Gdq, XS }, 0 },
+    { "vmovmskpX",	{ Gdq, XS }, PREFIX_OPCODE },
   },
   {
     /* MOD_VEX_0F71_REG_2 */
@@ -12266,14 +12266,18 @@ print_insn (bfd_vma pc, disassemble_info *info)
      PREFIX_REPZ/PREFIX_REPNZ fix, we check the PREFIX_DATA prefix
      separately.  */
   if (dp->prefix_requirement == PREFIX_OPCODE
-      && dp != &bad_opcode
-      && (((prefixes
-	    & (PREFIX_REPZ | PREFIX_REPNZ)) != 0
+      && (((need_vex
+	    ? vex.prefix == REPE_PREFIX_OPCODE
+	      || vex.prefix == REPNE_PREFIX_OPCODE
+	    : (prefixes
+	       & (PREFIX_REPZ | PREFIX_REPNZ)) != 0)
 	   && (used_prefixes
 	       & (PREFIX_REPZ | PREFIX_REPNZ)) == 0)
-	  || ((((prefixes
-		 & (PREFIX_REPZ | PREFIX_REPNZ | PREFIX_DATA))
-		== PREFIX_DATA)
+	  || (((need_vex
+		? vex.prefix == DATA_PREFIX_OPCODE
+		: ((prefixes
+		    & (PREFIX_REPZ | PREFIX_REPNZ | PREFIX_DATA))
+		   == PREFIX_DATA))
 	       && (used_prefixes & PREFIX_DATA) == 0))))
     {
       (*info->fprintf_func) (info->stream, "(bad)");
@@ -13230,21 +13234,15 @@ putop (const char *in_template, int sizeflag)
 	      SAVE_LAST (*p);
 	      break;
 	    }
-	  if (need_vex && vex.prefix)
+	  if (need_vex
+	      ? vex.prefix == DATA_PREFIX_OPCODE
+	      : prefixes & PREFIX_DATA)
 	    {
-	      if (vex.prefix == DATA_PREFIX_OPCODE)
-		*obufp++ = 'd';
-	      else
-		*obufp++ = 's';
+	      *obufp++ = 'd';
+	      used_prefixes |= PREFIX_DATA;
 	    }
 	  else
-	    {
-	      if (prefixes & PREFIX_DATA)
-		*obufp++ = 'd';
-	      else
-		*obufp++ = 's';
-	      used_prefixes |= (prefixes & PREFIX_DATA);
-	    }
+	    *obufp++ = 's';
 	  break;
 	case 'Y':
 	  if (l == 0 && len == 1)