[gdb/tdep] Add XOP support in amd64_get_insn_details

Implement support for XOP instructions [1] in amd64_get_insn_details.

The encoding scheme is documented here [2].  Essentially it's a variant of the
VEX3 encoding scheme, with:
- 0x8f as the first byte instead of 0xc4, and
- an opcode map >= 8.

The changes are roughly the same as the XOP part of an earlier submission [3],
hence the tag.

The only real difference is that that patch proposed to implement xop_prefix_p
using:
...
  return pfx[0] == 0x8f && (pfx[1] & 0x38);
...
which tries to resolve the conflict between the XOP prefix (starts with 0x8f)
and the POP instruction (opcode 0x8f) by detecting that it's not a POP
instruction.

Instead, use the way AMD has resolved this conflict in the specification, by
checking for opcode map >= 8:
...
  gdb_byte m = pfx[1] & 0x1f;
  return pfx[0] == 0x8f && m >= 8;
...

Tested on x86_64-linux.

Co-Authored-By: Jan Beulich <jbeulich@suse.com>
Reviewed-By: Klaus Gerlicher<klaus.gerlicher.@intel.com>

[1] https://en.wikipedia.org/wiki/XOP_instruction_set
[2] https://www.amd.com/content/dam/amd/en/documents/archived-tech-docs/programmer-references/43479.pdf
[3] https://sourceware.org/pipermail/gdb-patches/2019-February/155347.html

1 files changed, 30 insertions, 2 deletions

diff --git a/gdb/amd64-tdep.c b/gdb/amd64-tdep.c
index fdd4d14..3848a85 100644
--- a/gdb/amd64-tdep.c
+++ b/gdb/amd64-tdep.c
@@ -1181,6 +1181,15 @@ vex3_prefix_p (gdb_byte pfx)
   return pfx == 0xc4;
 }
 
+/* True if PFX is the start of an XOP prefix.  */
+
+static bool
+xop_prefix_p (const gdb_byte *pfx)
+{
+  gdb_byte m = pfx[1] & 0x1f;
+  return pfx[0] == 0x8f && m >= 8;
+}
+
 /* Return true if PFX is the start of the 4-byte EVEX prefix.  */
 
 static bool
@@ -1351,7 +1360,7 @@ amd64_get_insn_details (gdb_byte *insn, struct amd64_insn *details)
       details->enc_prefix_offset = insn - start;
       insn += 2;
     }
-  else if (vex3_prefix_p (*insn))
+  else if (vex3_prefix_p (*insn) || xop_prefix_p (insn))
     {
       details->enc_prefix_offset = insn - start;
       insn += 3;
@@ -1442,6 +1451,11 @@ amd64_get_insn_details (gdb_byte *insn, struct amd64_insn *details)
 	  return;
 	}
     }
+  else if (prefix != nullptr && xop_prefix_p (prefix))
+    {
+      details->opcode_len = 1;
+      need_modrm = 1;
+    }
   else if (*insn == TWO_BYTE_OPCODE_ESCAPE)
     {
       /* Two or three-byte opcode.  */
@@ -1512,7 +1526,7 @@ fixup_riprel (const struct amd64_insn &details, gdb_byte *insn,
 	{
 	  /* VEX.!B is set implicitly.  */
 	}
-      else if (vex3_prefix_p (pfx[0]))
+      else if (vex3_prefix_p (pfx[0]) || xop_prefix_p (pfx))
 	pfx[1] |= VEX3_NOT_B;
       else if (evex_prefix_p (pfx[0]))
 	{
@@ -3781,6 +3795,20 @@ test_amd64_get_insn_details (void)
   updated_insn
     = { 0xc4, 0xe3, 0x4d, 0x0e, 0x91, 0xff, 0x00, 0x00, 0x00, 0x07 };
   SELF_CHECK (insn == updated_insn);
+
+  /* INSN: vpcomtrueuq 0x0(%rip),%xmm7,%xmm0, xop prefix.  */
+  insn = { 0x8f, 0xe8, 0x40, 0xef, 0x05, 0x00, 0x00, 0x00, 0x00, 0x07 };
+  amd64_get_insn_details (insn.data (), &details);
+  SELF_CHECK (details.opcode_len == 1);
+  SELF_CHECK (details.enc_prefix_offset == 0);
+  SELF_CHECK (details.opcode_offset == 3);
+  SELF_CHECK (details.modrm_offset == 4);
+
+  /* INSN: vpcomtrueuq 0x0(%ecx),%xmm7,%xmm0, xop prefix.  */
+  fixup_riprel (details, insn.data (), ECX_REG_NUM);
+  updated_insn
+    = { 0x8f, 0xe8, 0x40, 0xef, 0x81, 0x00, 0x00, 0x00, 0x00, 0x07 };
+  SELF_CHECK (insn == updated_insn);
 }
 
 static void