gdb: Improve syscall entry/return tracking on Linux

The existing logic was simply to flip syscall entry/return state when a
syscall trap was seen, and even then only with active 'catch syscall'.
That can get out of sync if 'catch syscall' is toggled at odd times.

This patch updates the entry/return state for all syscall traps,
regardless of catching state, and also updates known syscall state for
other kinds of traps.  Almost all PTRACE_EVENT stops are delivered from
the middle of a syscall, so this can act like an entry.  Every other
kind of ptrace stop is only delivered outside of syscall event pairs, so
marking them ignored ensures the next syscall trap looks like an entry.

Three new test scenarios are added to catch-syscall.exp:

- Disable 'catch syscall' from an entry to deliberately miss the return
  event, then re-enable to make sure a new entry is recognized.

- Enable 'catch syscall' for the first time from a vfork event, which is
  a PTRACE_EVENT_VFORK in the middle of the syscall.  Make sure the next
  syscall event is recognized as the return.

- Make sure entry and return are recognized for an ENOSYS syscall.  This
  is to defeat a common x86 hack that uses the pre-filled ENOSYS return
  value as a sign of being on the entry side.

gdb/ChangeLog:

2015-10-19  Josh Stone  <jistone@redhat.com>

	* linux-nat.c (linux_handle_syscall_trap): Always update entry/
	return state, even when not actively catching syscalls at all.
	(linux_handle_extended_wait): Mark syscall_state like an entry.
	(wait_lwp): Set syscall_state ignored for other traps.
	(linux_nat_filter_event): Likewise.

gdb/testsuite/ChangeLog:

2015-10-19  Josh Stone  <jistone@redhat.com>

	* gdb.base/catch-syscall.c: Include <sched.h>.
	(unknown_syscall): New variable.
	(main): Trigger a vfork and an unknown syscall.
	* gdb.base/catch-syscall.exp (vfork_syscalls): New variable.
	(unknown_syscall_number): Likewise.
	(check_call_to_syscall): Accept an optional syscall pattern.
	(check_return_from_syscall): Likewise.
	(check_continue): Likewise.
	(test_catch_syscall_without_args): Check for vfork and ENOSYS.
	(test_catch_syscall_skipping_return): New test toggling off 'catch
	syscall' to step over the syscall return, then toggling back on.
	(test_catch_syscall_mid_vfork): New test turning on 'catch syscall'
	during a PTRACE_EVENT_VFORK stop, in the middle of a vfork syscall.
	(do_syscall_tests): Call test_catch_syscall_without_args and
	test_catch_syscall_mid_vfork.
	(test_catch_syscall_without_args_noxml): Check for vfork and ENOSYS.
	(fill_all_syscalls_numbers): Initialize unknown_syscall_number.

1 files changed, 26 insertions, 9 deletions

diff --git a/gdb/linux-nat.c b/gdb/linux-nat.c
index eb9f5bb..841ec39 100644
--- a/gdb/linux-nat.c
+++ b/gdb/linux-nat.c
@@ -1916,17 +1916,17 @@ linux_handle_syscall_trap (struct lwp_info *lp, int stopping)
       return 1;
     }
 
+  /* Always update the entry/return state, even if this particular
+     syscall isn't interesting to the core now.  In async mode,
+     the user could install a new catchpoint for this syscall
+     between syscall enter/return, and we'll need to know to
+     report a syscall return if that happens.  */
+  lp->syscall_state = (lp->syscall_state == TARGET_WAITKIND_SYSCALL_ENTRY
+		       ? TARGET_WAITKIND_SYSCALL_RETURN
+		       : TARGET_WAITKIND_SYSCALL_ENTRY);
+
   if (catch_syscall_enabled ())
     {
-      /* Always update the entry/return state, even if this particular
-	 syscall isn't interesting to the core now.  In async mode,
-	 the user could install a new catchpoint for this syscall
-	 between syscall enter/return, and we'll need to know to
-	 report a syscall return if that happens.  */
-      lp->syscall_state = (lp->syscall_state == TARGET_WAITKIND_SYSCALL_ENTRY
-			   ? TARGET_WAITKIND_SYSCALL_RETURN
-			   : TARGET_WAITKIND_SYSCALL_ENTRY);
-
       if (catching_syscall_number (syscall_number))
 	{
 	  /* Alright, an event to report.  */
@@ -2006,6 +2006,11 @@ linux_handle_extended_wait (struct lwp_info *lp, int status)
   struct target_waitstatus *ourstatus = &lp->waitstatus;
   int event = linux_ptrace_get_extended_event (status);
 
+  /* All extended events we currently use are mid-syscall.  Only
+     PTRACE_EVENT_STOP is delivered more like a signal-stop, but
+     you have to be using PTRACE_SEIZE to get that.  */
+  lp->syscall_state = TARGET_WAITKIND_SYSCALL_ENTRY;
+
   if (event == PTRACE_EVENT_FORK || event == PTRACE_EVENT_VFORK
       || event == PTRACE_EVENT_CLONE)
     {
@@ -2324,6 +2329,12 @@ wait_lwp (struct lwp_info *lp)
       if (linux_handle_syscall_trap (lp, 1))
 	return wait_lwp (lp);
     }
+  else
+    {
+      /* Almost all other ptrace-stops are known to be outside of system
+	 calls, with further exceptions in linux_handle_extended_wait.  */
+      lp->syscall_state = TARGET_WAITKIND_IGNORE;
+    }
 
   /* Handle GNU/Linux's extended waitstatus for trace events.  */
   if (WIFSTOPPED (status) && WSTOPSIG (status) == SIGTRAP
@@ -3126,6 +3137,12 @@ linux_nat_filter_event (int lwpid, int status)
       if (linux_handle_syscall_trap (lp, 0))
 	return NULL;
     }
+  else
+    {
+      /* Almost all other ptrace-stops are known to be outside of system
+	 calls, with further exceptions in linux_handle_extended_wait.  */
+      lp->syscall_state = TARGET_WAITKIND_IGNORE;
+    }
 
   /* Handle GNU/Linux's extended waitstatus for trace events.  */
   if (WIFSTOPPED (status) && WSTOPSIG (status) == SIGTRAP