x86: flag EVEX masking when destination is GPR(-like)

Masking is not permitted in this case. See the code comment for how this
is being dealt with.

To avoid excess special casing of modes, have OP_M() call OP_E_memory()
directly.

3 files changed, 24 insertions, 1 deletions

diff --git a/gas/testsuite/gas/i386/avx512f-nondef.d b/gas/testsuite/gas/i386/avx512f-nondef.d
index 9c1d327..aa7d292 100644
--- a/gas/testsuite/gas/i386/avx512f-nondef.d
+++ b/gas/testsuite/gas/i386/avx512f-nondef.d
@@ -22,4 +22,6 @@ Disassembly of section .text:
 [ 	]*[a-f0-9]+:	62 f2 7d 48 92 04 08 	vgatherdps \(%eax,%zmm1(,1)?\),%zmm0/\(bad\)
 [ 	]*[a-f0-9]+:	62 f1 7c cf c2 c0 00 	vcmpeqps %zmm0,%zmm0,%k0\{%k7\}\{z\}/\(bad\)
 [ 	]*[a-f0-9]+:	62 f1 7c cf 29 00    	vmovaps %zmm0,\(%eax\)\{%k7\}\{z\}/\(bad\)
+[ 	]*[a-f0-9]+:	62 f1 7d 0a c5 c8 00 	vpextrw \$(0x)?0,%xmm0,%ecx\{%k2\}/\(bad\)
+[ 	]*[a-f0-9]+:	62 f3 7d 0a 16 01 00 	vpextrd \$(0x)?0,%xmm0,\(%ecx\)\{%k2\}/\(bad\)
 #pass
diff --git a/gas/testsuite/gas/i386/avx512f-nondef.s b/gas/testsuite/gas/i386/avx512f-nondef.s
index 77426b1..39ba23b 100644
--- a/gas/testsuite/gas/i386/avx512f-nondef.s
+++ b/gas/testsuite/gas/i386/avx512f-nondef.s
@@ -32,3 +32,9 @@
 
 	# vmovaps %zmm0, (%eax){%k7} with EVEX.z set
 	.insn EVEX.0f 0x29, %zmm0, (%eax){%k7}{z}
+
+	# vpextrw $0, %xmm0, %ecx with non-zero EVEX.aaa
+	.insn EVEX.66.0f 0xc5, $0, %xmm0, %ecx{%k2}
+
+	# vpextrd $0, %xmm0, (%ecx) with non-zero EVEX.aaa
+	.insn EVEX.66.0f3a 0x16, $0, %xmm0, (%ecx){%k2}
diff --git a/opcodes/i386-dis.c b/opcodes/i386-dis.c
index 7c41302..d5e3547 100644
--- a/opcodes/i386-dis.c
+++ b/opcodes/i386-dis.c
@@ -11641,6 +11641,11 @@ print_register (instr_info *ins, unsigned int reg, unsigned int rexmask,
 {
   const char (*names)[8];
 
+  /* Masking is invalid for insns with GPR destination. Set the flag uniformly,
+     as the consumer will inspect it only for the destination operand.  */
+  if (bytemode != mask_mode && ins->vex.mask_register_specifier)
+    ins->illegal_masking = true;
+
   USED_REX (rexmask);
   if (ins->rex & rexmask)
     reg += 8;
@@ -12375,6 +12380,12 @@ OP_E (instr_info *ins, int bytemode, int sizeflag)
       return true;
     }
 
+  /* Masking is invalid for insns with GPR-like memory destination. Set the
+     flag uniformly, as the consumer will inspect it only for the destination
+     operand.  */
+  if (ins->vex.mask_register_specifier)
+    ins->illegal_masking = true;
+
   return OP_E_memory (ins, bytemode, sizeflag);
 }
 
@@ -13157,10 +13168,14 @@ OP_XS (instr_info *ins, int bytemode, int sizeflag)
 static bool
 OP_M (instr_info *ins, int bytemode, int sizeflag)
 {
+  /* Skip mod/rm byte.  */
+  MODRM_CHECK;
+  ins->codep++;
+
   if (ins->modrm.mod == 3)
     /* bad bound,lea,lds,les,lfs,lgs,lss,cmpxchg8b,vmptrst modrm */
     return BadOp (ins);
-  return OP_E (ins, bytemode, sizeflag);
+  return OP_E_memory (ins, bytemode, sizeflag);
 }
 
 static bool