amd64-tdep: need_modrm = 1 for VEX/EVEX instructions, except vzeroall/vzeroupper

VEX and EVEX-encoded instructions generally require a ModR/M byte, with the
notable exception of vzeroall and vzeroupper (opcode 0x77), which do not
use ModR/M.

This change sets need_modrm = 1 for VEX instructions, and adds an exception
for instructions where *insn == 0x77, following Intel’s SDM.

EVEX has no exceptions and thus always sets need_modrm to 1.

Additionally, the legacy twobyte_has_modrm table cannot be used for VEX and
EVEX instructions, as these encodings have different requirements and
exceptions. The logic is now explicit for VEX/EVEX handling.

Add vpblendw to selftest amd64_insn_decode.

The Intel SDM says the following:

  1. Intel® 64 and IA-32 Architectures Software Developer’s Manual

  Section 2.2.1.2 — Instruction Prefixes

  "The VEX prefix is a multi-byte prefix that replaces several legacy prefixes
  and opcode bytes. The VEX prefix is not an opcode; it is a prefix that
  modifies the instruction that follows."

  Section 2.2.1.3 — Opcode Bytes

  "The opcode byte(s) follow any instruction prefixes (including VEX). The
  opcode specifies the operation to be performed."

  Section 2.2.2 — Instruction Format

  "If a VEX prefix is present, it is processed as a single prefix, and the
  opcode bytes follow immediately after the VEX prefix."

  Source: Intel® SDM Vol. 2A, Section 2.2.1.2 and 2.2.2 (See Vol. 2A,
  PDF pages 2-4, 2-5, and 2-7)

  2. ModRM Byte Requirement

  Intel® SDM Vol. 2A, Table 2-2 — VEX Prefix Encoding

  "Most VEX-encoded instructions require a ModRM byte, except for a few
  instructions such as VZEROALL and VZEROUPPER."

  Source: Intel® SDM Vol. 2A, Table 2-2 (See Vol. 2A, PDF page 2-13)

Approved-By: Tom de Vries <tdevries@suse.de>

1 files changed, 32 insertions, 6 deletions

diff --git a/gdb/amd64-tdep.c b/gdb/amd64-tdep.c
index d5ea4af..fdd4d14 100644
--- a/gdb/amd64-tdep.c
+++ b/gdb/amd64-tdep.c
@@ -1383,13 +1383,12 @@ amd64_get_insn_details (gdb_byte *insn, struct amd64_insn *details)
     }
   else if (prefix != nullptr && vex2_prefix_p (*prefix))
     {
-      need_modrm = twobyte_has_modrm[*insn];
+      /* All VEX2 instructions need ModR/M, except vzeroupper/vzeroall.  */
+      need_modrm = *insn != 0x77 ? 1 : 0;
       details->opcode_len = 2;
     }
   else if (prefix != nullptr && vex3_prefix_p (*prefix))
     {
-      need_modrm = twobyte_has_modrm[*insn];
-
       gdb_byte m = prefix[1] & 0x1f;
       if (m == 0)
 	{
@@ -1398,12 +1397,16 @@ amd64_get_insn_details (gdb_byte *insn, struct amd64_insn *details)
 	}
       else if (m == 1)
 	{
-	  /* Escape 0x0f.  */
+	  /* Escape 0x0f.  All VEX3 instructions in this map need ModR/M,
+	     except vzeroupper/vzeroall.  */
+	  need_modrm = *insn != 0x77 ? 1 : 0;
 	  details->opcode_len = 2;
 	}
       else if (m == 2 || m == 3)
 	{
-	  /* Escape 0x0f 0x38 or 0x0f 0x3a.  */
+	  /* Escape 0x0f 0x38 or 0x0f 0x3a.  All VEX3 instructions in
+	     this map need ModR/M.  */
+	  need_modrm = 1;
 	  details->opcode_len = 3;
 	}
       else if (m == 7)
@@ -1419,7 +1422,8 @@ amd64_get_insn_details (gdb_byte *insn, struct amd64_insn *details)
     }
   else if (prefix != nullptr && evex_prefix_p (*prefix))
     {
-      need_modrm = twobyte_has_modrm[*insn];
+      /* All EVEX instructions need ModR/M.  */
+      need_modrm = 1;
 
       gdb_byte m = prefix[1] & 0x7;
       if (m == 1)
@@ -3755,6 +3759,28 @@ test_amd64_get_insn_details (void)
     = { 0x62, 0xf1, 0x7c, 0x48, 0x28, 0x81, 0x00, 0xfc, 0xff, 0xff };
   fixup_riprel (details, insn.data (), ECX_REG_NUM);
   SELF_CHECK (insn == updated_insn);
+
+  /* INSN: vpblendw $0x7,%xmm4,%xmm6,%xmm2, vex3 prefix.  */
+  insn = { 0xc4, 0xe3, 0x49, 0x0e, 0xd4, 0x07 };
+  amd64_get_insn_details (insn.data (), &details);
+  SELF_CHECK (details.opcode_len == 3);
+  SELF_CHECK (details.enc_prefix_offset == 0);
+  SELF_CHECK (details.opcode_offset == 3);
+  SELF_CHECK (details.modrm_offset == 4);
+
+  /* INSN: vpblendw $0x7,0xff(%rip),%ymm6,%ymm2, vex3 prefix.  */
+  insn = { 0xc4, 0xe3, 0x4d, 0x0e, 0x15, 0xff, 0x00, 0x00, 0x00, 0x07 };
+  amd64_get_insn_details (insn.data (), &details);
+  SELF_CHECK (details.opcode_len == 3);
+  SELF_CHECK (details.enc_prefix_offset == 0);
+  SELF_CHECK (details.opcode_offset == 3);
+  SELF_CHECK (details.modrm_offset == 4);
+
+  /* INSN: vpblendw $0x7,0xff(%ecx),%ymm6,%ymm2, vex3 prefix.  */
+  fixup_riprel (details, insn.data (), ECX_REG_NUM);
+  updated_insn
+    = { 0xc4, 0xe3, 0x4d, 0x0e, 0x91, 0xff, 0x00, 0x00, 0x00, 0x07 };
+  SELF_CHECK (insn == updated_insn);
 }
 
 static void