gdbarch: add instruction predicate methods

Add new methods to gdbarch for analyzing the instruction at a given address.
Implement those methods for i386 and amd64 architectures.

This is needed by "record btrace" to detect function calls in the
execution trace.

2014-01-16  Markus Metzger  <markus.t.metzger@intel.com>

	* amd64-tdep.c (amd64_classify_insn_at, amd64_insn_is_call)
	(amd64_insn_is_ret, amd64_insn_is_jump, amd64_jmp_p): New.
	(amd64_init_abi): Add insn_is_call, insn_is_ret, and insn_is_jump
	to gdbarch.
	* i386-tdep.c (i386_insn_is_call, i386_insn_is_ret)
	(i386_insn_is_jump, i386_jmp_p): New.
	(i386_gdbarch_init): Add insn_is_call, insn_is_ret, and
	insn_is_jump to gdbarch.
	* gdbarch.sh (insn_is_call, insn_is_ret, insn_is_jump): New.
	* gdbarch.h: Regenerated.
	* gdbarch.c: Regenerated.
	* arch-utils.h (default_insn_is_call, default_insn_is_ret)
	(default_insn_is_jump): New.
	* arch-utils.c (default_insn_is_call, default_insn_is_ret)
	(default_insn_is_jump): New.

8 files changed, 262 insertions, 0 deletions

diff --git a/gdb/ChangeLog b/gdb/ChangeLog
index 153c508..d80ab85 100644
--- a/gdb/ChangeLog
+++ b/gdb/ChangeLog
@@ -1,5 +1,23 @@
 2014-01-16  Markus Metzger  <markus.t.metzger@intel.com>
 
+	* amd64-tdep.c (amd64_classify_insn_at, amd64_insn_is_call)
+	(amd64_insn_is_ret, amd64_insn_is_jump, amd64_jmp_p): New.
+	(amd64_init_abi): Add insn_is_call, insn_is_ret, and insn_is_jump
+	to gdbarch.
+	* i386-tdep.c (i386_insn_is_call, i386_insn_is_ret)
+	(i386_insn_is_jump, i386_jmp_p): New.
+	(i386_gdbarch_init): Add insn_is_call, insn_is_ret, and
+	insn_is_jump to gdbarch.
+	* gdbarch.sh (insn_is_call, insn_is_ret, insn_is_jump): New.
+	* gdbarch.h: Regenerated.
+	* gdbarch.c: Regenerated.
+	* arch-utils.h (default_insn_is_call, default_insn_is_ret)
+	(default_insn_is_jump): New.
+	* arch-utils.c (default_insn_is_call, default_insn_is_ret)
+	(default_insn_is_jump): New.
+
+2014-01-16  Markus Metzger  <markus.t.metzger@intel.com>
+
 	* common/btrace-common.h (btrace_read_type) <btrace_read_all>:
 	Change to ...
 	(btrace_read_type) <BTRACE_READ_ALL>: ... this.  Update users.
diff --git a/gdb/amd64-tdep.c b/gdb/amd64-tdep.c
index 2055a3b..72d748e 100644
--- a/gdb/amd64-tdep.c
+++ b/gdb/amd64-tdep.c
@@ -1345,6 +1345,24 @@ amd64_absolute_jmp_p (const struct amd64_insn *details)
   return 0;
 }
 
+/* Return non-zero if the instruction DETAILS is a jump, zero otherwise.  */
+
+static int
+amd64_jmp_p (const struct amd64_insn *details)
+{
+  const gdb_byte *insn = &details->raw_insn[details->opcode_offset];
+
+  /* jump short, relative.  */
+  if (insn[0] == 0xeb)
+    return 1;
+
+  /* jump near, relative.  */
+  if (insn[0] == 0xe9)
+    return 1;
+
+  return amd64_absolute_jmp_p (details);
+}
+
 static int
 amd64_absolute_call_p (const struct amd64_insn *details)
 {
@@ -1416,6 +1434,52 @@ amd64_syscall_p (const struct amd64_insn *details, int *lengthp)
   return 0;
 }
 
+/* Classify the instruction at ADDR using PRED.
+   Throw an error if the memory can't be read.  */
+
+static int
+amd64_classify_insn_at (struct gdbarch *gdbarch, CORE_ADDR addr,
+			int (*pred) (const struct amd64_insn *))
+{
+  struct amd64_insn details;
+  gdb_byte *buf;
+  int len, classification;
+
+  len = gdbarch_max_insn_length (gdbarch);
+  buf = alloca (len);
+
+  read_code (addr, buf, len);
+  amd64_get_insn_details (buf, &details);
+
+  classification = pred (&details);
+
+  return classification;
+}
+
+/* The gdbarch insn_is_call method.  */
+
+static int
+amd64_insn_is_call (struct gdbarch *gdbarch, CORE_ADDR addr)
+{
+  return amd64_classify_insn_at (gdbarch, addr, amd64_call_p);
+}
+
+/* The gdbarch insn_is_ret method.  */
+
+static int
+amd64_insn_is_ret (struct gdbarch *gdbarch, CORE_ADDR addr)
+{
+  return amd64_classify_insn_at (gdbarch, addr, amd64_ret_p);
+}
+
+/* The gdbarch insn_is_jump method.  */
+
+static int
+amd64_insn_is_jump (struct gdbarch *gdbarch, CORE_ADDR addr)
+{
+  return amd64_classify_insn_at (gdbarch, addr, amd64_jmp_p);
+}
+
 /* Fix up the state of registers and memory after having single-stepped
    a displaced instruction.  */
 
@@ -2966,6 +3030,9 @@ amd64_init_abi (struct gdbarch_info info, struct gdbarch *gdbarch)
 				      i386_stap_is_single_operand);
   set_gdbarch_stap_parse_special_token (gdbarch,
 					i386_stap_parse_special_token);
+  set_gdbarch_insn_is_call (gdbarch, amd64_insn_is_call);
+  set_gdbarch_insn_is_ret (gdbarch, amd64_insn_is_ret);
+  set_gdbarch_insn_is_jump (gdbarch, amd64_insn_is_jump);
 }
 
 
diff --git a/gdb/arch-utils.c b/gdb/arch-utils.c
index 66cebd7..bff1a10 100644
--- a/gdb/arch-utils.c
+++ b/gdb/arch-utils.c
@@ -804,6 +804,21 @@ default_return_in_first_hidden_param_p (struct gdbarch *gdbarch,
   return language_pass_by_reference (type);
 }
 
+int default_insn_is_call (struct gdbarch *gdbarch, CORE_ADDR addr)
+{
+  return 0;
+}
+
+int default_insn_is_ret (struct gdbarch *gdbarch, CORE_ADDR addr)
+{
+  return 0;
+}
+
+int default_insn_is_jump (struct gdbarch *gdbarch, CORE_ADDR addr)
+{
+  return 0;
+}
+
 /* */
 
 /* -Wmissing-prototypes */
diff --git a/gdb/arch-utils.h b/gdb/arch-utils.h
index 08ad4f9..46d1573 100644
--- a/gdb/arch-utils.h
+++ b/gdb/arch-utils.h
@@ -170,4 +170,8 @@ extern const char *default_auto_wide_charset (void);
 
 extern int default_return_in_first_hidden_param_p (struct gdbarch *,
 						   struct type *);
+
+extern int default_insn_is_call (struct gdbarch *, CORE_ADDR);
+extern int default_insn_is_ret (struct gdbarch *, CORE_ADDR);
+extern int default_insn_is_jump (struct gdbarch *, CORE_ADDR);
 #endif
diff --git a/gdb/gdbarch.c b/gdb/gdbarch.c
index e1766d3..ee41d50 100644
--- a/gdb/gdbarch.c
+++ b/gdb/gdbarch.c
@@ -317,6 +317,9 @@ struct gdbarch
   gdbarch_core_info_proc_ftype *core_info_proc;
   gdbarch_iterate_over_objfiles_in_search_order_ftype *iterate_over_objfiles_in_search_order;
   struct ravenscar_arch_ops * ravenscar_ops;
+  gdbarch_insn_is_call_ftype *insn_is_call;
+  gdbarch_insn_is_ret_ftype *insn_is_ret;
+  gdbarch_insn_is_jump_ftype *insn_is_jump;
 };
 
 
@@ -490,6 +493,9 @@ struct gdbarch startup_gdbarch =
   0,  /* core_info_proc */
   default_iterate_over_objfiles_in_search_order,  /* iterate_over_objfiles_in_search_order */
   NULL,  /* ravenscar_ops */
+  default_insn_is_call,  /* insn_is_call */
+  default_insn_is_ret,  /* insn_is_ret */
+  default_insn_is_jump,  /* insn_is_jump */
   /* startup_gdbarch() */
 };
 
@@ -581,6 +587,9 @@ gdbarch_alloc (const struct gdbarch_info *info,
   gdbarch->gen_return_address = default_gen_return_address;
   gdbarch->iterate_over_objfiles_in_search_order = default_iterate_over_objfiles_in_search_order;
   gdbarch->ravenscar_ops = NULL;
+  gdbarch->insn_is_call = default_insn_is_call;
+  gdbarch->insn_is_ret = default_insn_is_ret;
+  gdbarch->insn_is_jump = default_insn_is_jump;
   /* gdbarch_alloc() */
 
   return gdbarch;
@@ -795,6 +804,9 @@ verify_gdbarch (struct gdbarch *gdbarch)
   /* Skip verify of core_info_proc, has predicate.  */
   /* Skip verify of iterate_over_objfiles_in_search_order, invalid_p == 0 */
   /* Skip verify of ravenscar_ops, invalid_p == 0 */
+  /* Skip verify of insn_is_call, invalid_p == 0 */
+  /* Skip verify of insn_is_ret, invalid_p == 0 */
+  /* Skip verify of insn_is_jump, invalid_p == 0 */
   buf = ui_file_xstrdup (log, &length);
   make_cleanup (xfree, buf);
   if (length > 0)
@@ -1128,6 +1140,15 @@ gdbarch_dump (struct gdbarch *gdbarch, struct ui_file *file)
                       "gdbarch_dump: inner_than = <%s>\n",
                       host_address_to_string (gdbarch->inner_than));
   fprintf_unfiltered (file,
+                      "gdbarch_dump: insn_is_call = <%s>\n",
+                      host_address_to_string (gdbarch->insn_is_call));
+  fprintf_unfiltered (file,
+                      "gdbarch_dump: insn_is_jump = <%s>\n",
+                      host_address_to_string (gdbarch->insn_is_jump));
+  fprintf_unfiltered (file,
+                      "gdbarch_dump: insn_is_ret = <%s>\n",
+                      host_address_to_string (gdbarch->insn_is_ret));
+  fprintf_unfiltered (file,
                       "gdbarch_dump: int_bit = %s\n",
                       plongest (gdbarch->int_bit));
   fprintf_unfiltered (file,
@@ -4451,6 +4472,57 @@ set_gdbarch_ravenscar_ops (struct gdbarch *gdbarch,
   gdbarch->ravenscar_ops = ravenscar_ops;
 }
 
+int
+gdbarch_insn_is_call (struct gdbarch *gdbarch, CORE_ADDR addr)
+{
+  gdb_assert (gdbarch != NULL);
+  gdb_assert (gdbarch->insn_is_call != NULL);
+  if (gdbarch_debug >= 2)
+    fprintf_unfiltered (gdb_stdlog, "gdbarch_insn_is_call called\n");
+  return gdbarch->insn_is_call (gdbarch, addr);
+}
+
+void
+set_gdbarch_insn_is_call (struct gdbarch *gdbarch,
+                          gdbarch_insn_is_call_ftype insn_is_call)
+{
+  gdbarch->insn_is_call = insn_is_call;
+}
+
+int
+gdbarch_insn_is_ret (struct gdbarch *gdbarch, CORE_ADDR addr)
+{
+  gdb_assert (gdbarch != NULL);
+  gdb_assert (gdbarch->insn_is_ret != NULL);
+  if (gdbarch_debug >= 2)
+    fprintf_unfiltered (gdb_stdlog, "gdbarch_insn_is_ret called\n");
+  return gdbarch->insn_is_ret (gdbarch, addr);
+}
+
+void
+set_gdbarch_insn_is_ret (struct gdbarch *gdbarch,
+                         gdbarch_insn_is_ret_ftype insn_is_ret)
+{
+  gdbarch->insn_is_ret = insn_is_ret;
+}
+
+int
+gdbarch_insn_is_jump (struct gdbarch *gdbarch, CORE_ADDR addr)
+{
+  gdb_assert (gdbarch != NULL);
+  gdb_assert (gdbarch->insn_is_jump != NULL);
+  if (gdbarch_debug >= 2)
+    fprintf_unfiltered (gdb_stdlog, "gdbarch_insn_is_jump called\n");
+  return gdbarch->insn_is_jump (gdbarch, addr);
+}
+
+void
+set_gdbarch_insn_is_jump (struct gdbarch *gdbarch,
+                          gdbarch_insn_is_jump_ftype insn_is_jump)
+{
+  gdbarch->insn_is_jump = insn_is_jump;
+}
+
 
 /* Keep a registry of per-architecture data-pointers required by GDB
    modules.  */
diff --git a/gdb/gdbarch.h b/gdb/gdbarch.h
index 6ff875e..6b59371 100644
--- a/gdb/gdbarch.h
+++ b/gdb/gdbarch.h
@@ -1267,6 +1267,24 @@ extern void set_gdbarch_iterate_over_objfiles_in_search_order (struct gdbarch *g
 extern struct ravenscar_arch_ops * gdbarch_ravenscar_ops (struct gdbarch *gdbarch);
 extern void set_gdbarch_ravenscar_ops (struct gdbarch *gdbarch, struct ravenscar_arch_ops * ravenscar_ops);
 
+/* Return non-zero if the instruction at ADDR is a call; zero otherwise. */
+
+typedef int (gdbarch_insn_is_call_ftype) (struct gdbarch *gdbarch, CORE_ADDR addr);
+extern int gdbarch_insn_is_call (struct gdbarch *gdbarch, CORE_ADDR addr);
+extern void set_gdbarch_insn_is_call (struct gdbarch *gdbarch, gdbarch_insn_is_call_ftype *insn_is_call);
+
+/* Return non-zero if the instruction at ADDR is a return; zero otherwise. */
+
+typedef int (gdbarch_insn_is_ret_ftype) (struct gdbarch *gdbarch, CORE_ADDR addr);
+extern int gdbarch_insn_is_ret (struct gdbarch *gdbarch, CORE_ADDR addr);
+extern void set_gdbarch_insn_is_ret (struct gdbarch *gdbarch, gdbarch_insn_is_ret_ftype *insn_is_ret);
+
+/* Return non-zero if the instruction at ADDR is a jump; zero otherwise. */
+
+typedef int (gdbarch_insn_is_jump_ftype) (struct gdbarch *gdbarch, CORE_ADDR addr);
+extern int gdbarch_insn_is_jump (struct gdbarch *gdbarch, CORE_ADDR addr);
+extern void set_gdbarch_insn_is_jump (struct gdbarch *gdbarch, gdbarch_insn_is_jump_ftype *insn_is_jump);
+
 /* Definition for an unknown syscall, used basically in error-cases.  */
 #define UNKNOWN_SYSCALL (-1)
 
diff --git a/gdb/gdbarch.sh b/gdb/gdbarch.sh
index 7e18d85..9c1229e 100755
--- a/gdb/gdbarch.sh
+++ b/gdb/gdbarch.sh
@@ -994,6 +994,15 @@ m:void:iterate_over_objfiles_in_search_order:iterate_over_objfiles_in_search_ord
 
 # Ravenscar arch-dependent ops.
 v:struct ravenscar_arch_ops *:ravenscar_ops:::NULL:NULL::0:host_address_to_string (gdbarch->ravenscar_ops)
+
+# Return non-zero if the instruction at ADDR is a call; zero otherwise.
+m:int:insn_is_call:CORE_ADDR addr:addr::default_insn_is_call::0
+
+# Return non-zero if the instruction at ADDR is a return; zero otherwise.
+m:int:insn_is_ret:CORE_ADDR addr:addr::default_insn_is_ret::0
+
+# Return non-zero if the instruction at ADDR is a jump; zero otherwise.
+m:int:insn_is_jump:CORE_ADDR addr:addr::default_insn_is_jump::0
 EOF
 }
 
diff --git a/gdb/i386-tdep.c b/gdb/i386-tdep.c
index 4a5f652..57f18cb 100644
--- a/gdb/i386-tdep.c
+++ b/gdb/i386-tdep.c
@@ -530,6 +530,22 @@ i386_absolute_jmp_p (const gdb_byte *insn)
   return 0;
 }
 
+/* Return non-zero if INSN is a jump, zero otherwise.  */
+
+static int
+i386_jmp_p (const gdb_byte *insn)
+{
+  /* jump short, relative.  */
+  if (insn[0] == 0xeb)
+    return 1;
+
+  /* jump near, relative.  */
+  if (insn[0] == 0xe9)
+    return 1;
+
+  return i386_absolute_jmp_p (insn);
+}
+
 static int
 i386_absolute_call_p (const gdb_byte *insn)
 {
@@ -601,6 +617,45 @@ i386_syscall_p (const gdb_byte *insn, int *lengthp)
   return 0;
 }
 
+/* The gdbarch insn_is_call method.  */
+
+static int
+i386_insn_is_call (struct gdbarch *gdbarch, CORE_ADDR addr)
+{
+  gdb_byte buf[I386_MAX_INSN_LEN], *insn;
+
+  read_code (addr, buf, I386_MAX_INSN_LEN);
+  insn = i386_skip_prefixes (buf, I386_MAX_INSN_LEN);
+
+  return i386_call_p (insn);
+}
+
+/* The gdbarch insn_is_ret method.  */
+
+static int
+i386_insn_is_ret (struct gdbarch *gdbarch, CORE_ADDR addr)
+{
+  gdb_byte buf[I386_MAX_INSN_LEN], *insn;
+
+  read_code (addr, buf, I386_MAX_INSN_LEN);
+  insn = i386_skip_prefixes (buf, I386_MAX_INSN_LEN);
+
+  return i386_ret_p (insn);
+}
+
+/* The gdbarch insn_is_jump method.  */
+
+static int
+i386_insn_is_jump (struct gdbarch *gdbarch, CORE_ADDR addr)
+{
+  gdb_byte buf[I386_MAX_INSN_LEN], *insn;
+
+  read_code (addr, buf, I386_MAX_INSN_LEN);
+  insn = i386_skip_prefixes (buf, I386_MAX_INSN_LEN);
+
+  return i386_jmp_p (insn);
+}
+
 /* Some kernels may run one past a syscall insn, so we have to cope.
    Otherwise this is just simple_displaced_step_copy_insn.  */
 
@@ -8017,6 +8072,10 @@ i386_gdbarch_init (struct gdbarch_info info, struct gdbarch_list *arches)
 
   set_gdbarch_gen_return_address (gdbarch, i386_gen_return_address);
 
+  set_gdbarch_insn_is_call (gdbarch, i386_insn_is_call);
+  set_gdbarch_insn_is_ret (gdbarch, i386_insn_is_ret);
+  set_gdbarch_insn_is_jump (gdbarch, i386_insn_is_jump);
+
   /* Hook in ABI-specific overrides, if they have been registered.  */
   info.tdep_info = (void *) tdesc_data;
   gdbarch_init_osabi (info, gdbarch);