Fix displaced-stepping RIP-relative VEX-encoded instructions (AVX) (PR gdb/22499)

PR gdb/22499 is about a latent bug exposed by the switch to "maint set
target-non-stop on" by default on x86-64 GNU/Linux, a while ago.  With
that on, GDB is also preferring to use displaced-stepping by default.

The testcase in the bug is failing because GDB ends up incorrectly
displaced-stepping over a RIP-relative VEX-encoded instruction, like
this:

 0x00000000004007f5 <+15>:    c5 fb 10 05 8b 01 00 00 vmovsd 0x18b(%rip),%xmm0        # 0x400988

While RIP-relative instructions need adjustment when relocated to the
scratch pad, GDB ends up just copying VEX-encoded instructions to the
scratch pad unmodified, with the end result that the inferior ends up
executing an instruction that fetches/writes memory from the wrong
address...

This patch teaches GDB about the VEX-encoding prefixes, fixing the
problem, and adds a testcase that fails without the GDB fix.

I think we may need a similar treatment for EVEX-encoded instructions,
but I didn't address that simply because I couldn't find any
EVEX-encoded RIP-relative instruction in the gas testsuite.  In any
case, this commit is forward progress as-is already.

gdb/ChangeLog:
2017-12-04  Pedro Alves  <palves@redhat.com>

	PR gdb/22499
	* amd64-tdep.c (amd64_insn::rex_offset): Rename to...
	(amd64_insn::enc_prefix_offset): ... this, and tweak comment.
	(vex2_prefix_p, vex3_prefix_p): New functions.
	(amd64_get_insn_details): Adjust to rename.  Also skip VEX2 and
	VEX3 prefixes.
	(fixup_riprel): Set VEX3.!B.

gdb/testsuite/ChangeLog:
2017-12-04  Pedro Alves  <palves@redhat.com>

	PR gdb/22499
	* gdb.arch/amd64-disp-step-avx.S: New file.
	* gdb.arch/amd64-disp-step-avx.exp: New file.

1 files changed, 49 insertions, 9 deletions

diff --git a/gdb/amd64-tdep.c b/gdb/amd64-tdep.c
index bcd37ef..0eb3670 100644
--- a/gdb/amd64-tdep.c
+++ b/gdb/amd64-tdep.c
@@ -1037,8 +1037,9 @@ struct amd64_insn
 {
   /* The number of opcode bytes.  */
   int opcode_len;
-  /* The offset of the rex prefix or -1 if not present.  */
-  int rex_offset;
+  /* The offset of the REX/VEX instruction encoding prefix or -1 if
+     not present.  */
+  int enc_prefix_offset;
   /* The offset to the first opcode byte.  */
   int opcode_offset;
   /* The offset to the modrm byte or -1 if not present.  */
@@ -1124,6 +1125,22 @@ rex_prefix_p (gdb_byte pfx)
   return REX_PREFIX_P (pfx);
 }
 
+/* True if PFX is the start of the 2-byte VEX prefix.  */
+
+static bool
+vex2_prefix_p (gdb_byte pfx)
+{
+  return pfx == 0xc5;
+}
+
+/* True if PFX is the start of the 3-byte VEX prefix.  */
+
+static bool
+vex3_prefix_p (gdb_byte pfx)
+{
+  return pfx == 0xc4;
+}
+
 /* Skip the legacy instruction prefixes in INSN.
    We assume INSN is properly sentineled so we don't have to worry
    about falling off the end of the buffer.  */
@@ -1242,19 +1259,30 @@ amd64_get_insn_details (gdb_byte *insn, struct amd64_insn *details)
   details->raw_insn = insn;
 
   details->opcode_len = -1;
-  details->rex_offset = -1;
+  details->enc_prefix_offset = -1;
   details->opcode_offset = -1;
   details->modrm_offset = -1;
 
   /* Skip legacy instruction prefixes.  */
   insn = amd64_skip_prefixes (insn);
 
-  /* Skip REX instruction prefix.  */
+  /* Skip REX/VEX instruction encoding prefixes.  */
   if (rex_prefix_p (*insn))
     {
-      details->rex_offset = insn - start;
+      details->enc_prefix_offset = insn - start;
       ++insn;
     }
+  else if (vex2_prefix_p (*insn))
+    {
+      /* Don't record the offset in this case because this prefix has
+	 no REX.B equivalent.  */
+      insn += 2;
+    }
+  else if (vex3_prefix_p (*insn))
+    {
+      details->enc_prefix_offset = insn - start;
+      insn += 3;
+    }
 
   details->opcode_offset = insn - start;
 
@@ -1329,10 +1357,22 @@ fixup_riprel (struct gdbarch *gdbarch, amd64_displaced_step_closure *dsc,
   arch_tmp_regno = amd64_get_unused_input_int_reg (insn_details);
   tmp_regno = amd64_arch_reg_to_regnum (arch_tmp_regno);
 
-  /* REX.B should be unset as we were using rip-relative addressing,
-     but ensure it's unset anyway, tmp_regno is not r8-r15.  */
-  if (insn_details->rex_offset != -1)
-    dsc->insn_buf[insn_details->rex_offset] &= ~REX_B;
+  /* Position of the not-B bit in the 3-byte VEX prefix (in byte 1).  */
+  static constexpr gdb_byte VEX3_NOT_B = 0x20;
+
+  /* REX.B should be unset (VEX.!B set) as we were using rip-relative
+     addressing, but ensure it's unset (set for VEX) anyway, tmp_regno
+     is not r8-r15.  */
+  if (insn_details->enc_prefix_offset != -1)
+    {
+      gdb_byte *pfx = &dsc->insn_buf[insn_details->enc_prefix_offset];
+      if (rex_prefix_p (pfx[0]))
+	pfx[0] &= ~REX_B;
+      else if (vex3_prefix_p (pfx[0]))
+	pfx[1] |= VEX3_NOT_B;
+      else
+	gdb_assert_not_reached ("unhandled prefix");
+    }
 
   regcache_cooked_read_unsigned (regs, tmp_regno, &orig_value);
   dsc->tmp_regno = tmp_regno;