Add multi-target tests

This adds a testcase exercising multi-target features.  It spawns 6
inferiors, like this:

 inferior 1 -> native
 inferior 2 -> extended-remote 1
 inferior 3 -> core
 inferior 4 -> native
 inferior 5 -> extended-remote 2
 inferior 6 -> core

and then tests various details, including:

 - running to breakpoints

 - interrupting with Ctrl-C and "interrupt -a"

 - "next" bouncing between two breakpoints in two threads running in
   different targets.

 - since we have cores and live inferiors mixed in the same session,
   this makes sure that gdb doesn't try to remove a core dump's
   threads.

 - all-stop and non-stop modes.

This testcase caught a _lot_ of bugs in development.

gdb/testsuite/ChangeLog:
2020-01-10  Pedro Alves  <palves@redhat.com>

	* gdb.multi/multi-target.c: New file.
	* gdb.multi/multi-target.exp: New file.
	* lib/gdbserver-support.exp (gdb_target_cmd): Handle "Non-stop
	mode requested, but remote does not support non-stop".

4 files changed, 472 insertions, 0 deletions

diff --git a/gdb/testsuite/ChangeLog b/gdb/testsuite/ChangeLog
index 0ff7595..9297fa2 100644
--- a/gdb/testsuite/ChangeLog
+++ b/gdb/testsuite/ChangeLog
@@ -1,5 +1,12 @@
 2020-01-10  Pedro Alves  <palves@redhat.com>
 
+	* gdb.multi/multi-target.c: New file.
+	* gdb.multi/multi-target.exp: New file.
+	* lib/gdbserver-support.exp (gdb_target_cmd): Handle "Non-stop
+	mode requested, but remote does not support non-stop".
+
+2020-01-10  Pedro Alves  <palves@redhat.com>
+
 	* gdb.server/extended-remote-restart.exp (test_reload): Explicitly
 	disconnect before reconnecting.
 
diff --git a/gdb/testsuite/gdb.multi/multi-target.c b/gdb/testsuite/gdb.multi/multi-target.c
new file mode 100644
index 0000000..23ec6bd
--- /dev/null
+++ b/gdb/testsuite/gdb.multi/multi-target.c
@@ -0,0 +1,100 @@
+/* This testcase is part of GDB, the GNU debugger.
+
+   Copyright 2017-2020 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.  */
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <limits.h>
+#include <string.h>
+#include <pthread.h>
+
+#define NUM_THREADS 1
+
+static pthread_barrier_t barrier;
+
+static void *
+thread_start (void *arg)
+{
+  pthread_barrier_wait (&barrier);
+
+  while (1)
+    sleep (1);
+  return NULL;
+}
+
+static void
+all_started (void)
+{
+}
+
+int wait_for_gdb;
+
+static void
+function1 (void)
+{
+  while (wait_for_gdb)
+    sleep (1);
+}
+
+static void
+function2 (void)
+{
+  while (wait_for_gdb)
+    sleep (1);
+}
+
+static void
+function3 (void)
+{
+}
+
+static void
+function4 (void)
+{
+}
+
+static void
+function5 (void)
+{
+}
+
+int
+main (int argc, char ** argv)
+{
+  pthread_t thread;
+  int len;
+
+  alarm (360);
+
+  pthread_barrier_init (&barrier, NULL, NUM_THREADS + 1);
+  pthread_create (&thread, NULL, thread_start, NULL);
+
+  pthread_barrier_wait (&barrier);
+  all_started ();
+
+  while (1)
+    {
+      function1 (); /* set break 1 here */
+      function2 (); /* set break 2 here */
+      function3 ();
+      function4 ();
+      function5 ();
+      sleep (1);
+    }
+
+  return 0;
+}
diff --git a/gdb/testsuite/gdb.multi/multi-target.exp b/gdb/testsuite/gdb.multi/multi-target.exp
new file mode 100644
index 0000000..cd0db12
--- /dev/null
+++ b/gdb/testsuite/gdb.multi/multi-target.exp
@@ -0,0 +1,361 @@
+# Copyright 2017-2020 Free Software Foundation, Inc.
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Test multi-target features.
+
+load_lib gdbserver-support.exp
+
+standard_testfile
+
+# The plain remote target can't do multiple inferiors.
+if {[target_info gdb_protocol] != ""} {
+    return
+}
+
+if { [prepare_for_testing "failed to prepare" ${binfile} "${srcfile}" \
+	  {debug pthreads}] } {
+    return
+}
+
+proc connect_target_extended_remote {binfile} {
+    set res [gdbserver_start "--multi" ""]
+    set gdbserver_gdbport [lindex $res 1]
+    return [gdb_target_cmd "extended-remote" $gdbserver_gdbport]
+}
+
+# Add and start inferior number NUM.  Returns true on success, false
+# otherwise.
+proc add_inferior {num target binfile {gcorefile ""}} {
+    # Start another inferior.
+    gdb_test "add-inferior -no-connection" "Added inferior $num" \
+	"add empty inferior $num"
+    gdb_test "inferior $num" "Switching to inferior $num.*" \
+	"switch to inferior $num"
+    gdb_test "file ${binfile}" ".*" "load file in inferior $num"
+    gdb_test_no_output "set remote exec-file ${binfile}" \
+	"set remote-exec file in inferior $num"
+
+    if {$target == "core"} {
+	gdb_test "core $gcorefile" "Core was generated by.*" \
+	    "core [file tail $gcorefile], inf $num"
+	return 1
+    }
+
+    if {$target == "extended-remote"} {
+	if {[connect_target_extended_remote $binfile]} {
+	    return 0
+	}
+    }
+    if ![runto "all_started"] then {
+	return 0
+    }
+    delete_breakpoints
+
+    return 1
+}
+
+proc prepare_core {} {
+    global gcorefile gcore_created
+    global binfile
+
+    clean_restart ${binfile}
+
+    if ![runto all_started] then {
+	return -1
+    }
+
+    global testfile
+    set gcorefile [standard_output_file $testfile.gcore]
+    set gcore_created [gdb_gcore_cmd $gcorefile "save a core file"]
+}
+
+proc next_live_inferior {inf} {
+    incr inf
+    if {$inf == 3} {
+	# 3 is a core.
+	return 4
+    }
+    if {$inf > 5} {
+	# 6 is a core.
+	return 1
+    }
+
+    return $inf
+}
+
+# Return true on success, false otherwise.
+
+proc setup {non-stop} {
+    global gcorefile gcore_created
+    global binfile
+
+    clean_restart ${binfile}
+
+    # multi-target depends on target running in non-stop mode.  Force
+    # it on for remote targets, until this is the default.
+    gdb_test_no_output "maint set target-non-stop on"
+
+    gdb_test_no_output "set non-stop ${non-stop}"
+
+    if ![runto all_started] then {
+	return 0
+    }
+
+    delete_breakpoints
+
+    # inferior 1 -> native
+    # inferior 2 -> extended-remote
+    # inferior 3 -> core
+    # inferior 4 -> native
+    # inferior 5 -> extended-remote
+    # inferior 6 -> core
+    if {![add_inferior 2 "extended-remote" $binfile]} {
+	return 0
+    }
+    if {![add_inferior 3 "core" $binfile $gcorefile]} {
+	return 0
+    }
+    if {![add_inferior 4 "native" $binfile]} {
+	return 0
+    }
+    if {![add_inferior 5 "extended-remote" $binfile]} {
+	return 0
+    }
+    if {![add_inferior 6 "core" $binfile $gcorefile]} {
+	return 0
+    }
+
+    # For debugging.
+    gdb_test "info inferiors" ".*"
+    gdb_test "info threads" ".*"
+
+    # Make "continue" resume all inferiors.
+    if {${non-stop} == "off"} {
+	gdb_test_no_output "set schedule-multiple on"
+    }
+
+    return 1
+}
+
+# Test "continue" to breakpoints in different targets.  In non-stop
+# mode, also tests "interrupt -a".
+proc test_continue {non-stop} {
+    if {![setup ${non-stop}]} {
+	untested "setup failed"
+	return
+    }
+
+    proc set_break {inf} {
+	gdb_test "break function${inf} thread ${inf}.1" \
+	"Breakpoint .* function${inf}\\..*"
+    }
+
+    # Select inferior INF, and then run to a breakpoint on inferior
+    # INF+1.
+    proc test_continue_inf {inf} {
+	upvar 1 non-stop non-stop
+
+	global gdb_prompt
+	delete_breakpoints
+
+	set next_inf [next_live_inferior $inf]
+
+	gdb_test "inferior $inf" "Switching to inferior $inf.*"
+	set_break $next_inf
+
+	if {${non-stop} == "off"} {
+	    gdb_test "continue" "hit Breakpoint .* function${next_inf}.*"
+	} else {
+	    set msg "continue"
+	    gdb_test_multiple "continue -a&" $msg {
+		-re "Continuing.*$gdb_prompt " {
+		    pass $msg
+		}
+	    }
+
+	    set msg "hit bp"
+	    gdb_test_multiple "" $msg {
+		-re "hit Breakpoint .* function${next_inf}" {
+		    pass $msg
+		}
+	    }
+
+	    set msg "stop all threads"
+	    gdb_test_multiple "interrupt -a" $msg {
+		-re "$gdb_prompt " {
+		    for {set i 0} {$i < 7} {incr i} {
+			set ok 0
+			gdb_test_multiple "" $msg {
+			    -re "Thread\[^\r\n\]*stopped\\." {
+				set ok 1
+			    }
+			}
+			if {!$ok} {
+			    break
+			}
+		    }
+		    gdb_assert $ok $msg
+		}
+	    }
+	}
+    }
+
+    for {set i 1} {$i <= 5} {incr i} {
+	if {$i == 3} {
+	    # This is a core inferior.
+	    continue
+	}
+
+	with_test_prefix "inf$i" {
+	    test_continue_inf $i
+	}
+    }
+}
+
+# Test interrupting multiple targets with Ctrl-C.
+
+proc test_ctrlc {} {
+    if {![setup "off"]} {
+	untested "setup failed"
+	return
+    }
+
+    delete_breakpoints
+
+    # Select inferior INF, continue all inferiors, and then Ctrl-C.
+    proc test_ctrlc_inf {inf} {
+	global gdb_prompt
+
+	gdb_test "inferior $inf" "Switching to inferior $inf.*"
+
+	set msg "continue"
+	gdb_test_multiple "continue" $msg {
+	    -re "Continuing" {
+		pass $msg
+	    }
+	}
+
+	after 200 { send_gdb "\003" }
+
+	set msg "send_gdb control C"
+	gdb_test_multiple "" $msg {
+	    -re "received signal SIGINT.*$gdb_prompt $" {
+		pass $msg
+	    }
+	}
+
+	set msg "all threads stopped"
+	gdb_test_multiple "info threads" "$msg" {
+	    -re "\\\(running\\\).*$gdb_prompt $" {
+		fail $msg
+	    }
+	    -re "$gdb_prompt $" {
+		pass $msg
+	    }
+	}
+    }
+
+    for {set i 1} {$i <= 5} {incr i} {
+	if {$i == 3} {
+	    # This is a core inferior.
+	    continue
+	}
+
+	with_test_prefix "inf$i" {
+	    test_ctrlc_inf $i
+	}
+    }
+}
+
+# Test "next" bouncing between two breakpoints in two threads running
+# in different targets.
+proc test_ping_pong_next {} {
+    global srcfile
+
+    if {![setup "off"]} {
+	untested "setup failed"
+	return
+    }
+
+    # block/unblock inferiors 1 and 2 according to INF1 and INF2.
+    proc block {inf1 inf2} {
+	gdb_test "thread apply 1.1 p wait_for_gdb = $inf1" " = $inf1"
+	gdb_test "thread apply 2.1 p wait_for_gdb = $inf2" " = $inf2"
+    }
+
+    # We're use inferiors 1 and 2.  Make sure they're really connected
+    # to different targets.
+    gdb_test "thread apply 1.1 maint print target-stack" \
+	"- native.*"
+    gdb_test "thread apply 2.1 maint print target-stack" \
+	"- extended-remote.*"
+
+    # Set two breakpoints, one for each of inferior 1 and 2.  Inferior
+    # 1 is running on the native target, and inferior 2 is running on
+    # extended-gdbserver.  Run to breakpoint 1 to gets things started.
+    set line1 [gdb_get_line_number "set break 1 here"]
+    set line2 [gdb_get_line_number "set break 2 here"]
+
+    gdb_test "thread 1.1" "Switching to thread 1.1 .*"
+
+    gdb_test "break $srcfile:$line1 thread 1.1" \
+	"Breakpoint .*$srcfile:$line1\\..*"
+
+    gdb_test "continue" "hit Breakpoint .*"
+
+    gdb_test "break $srcfile:$line2 thread 2.1" \
+	"Breakpoint .*$srcfile:$line2\\..*"
+
+    # Now block inferior 1 and issue "next".  We should stop at the
+    # breakpoint for inferior 2, given schedlock off.
+    with_test_prefix "next inf 1" {
+	block 1 0
+	gdb_test "next" "Thread 2.1 .*hit Breakpoint .*$srcfile:$line2.*"
+    }
+
+    # Now unblock inferior 2 and block inferior 1.  "next" should run
+    # into the breakpoint in inferior 1.
+    with_test_prefix "next inf 2" {
+	block 0 1
+	gdb_test "next" "Thread 1.1 .*hit Breakpoint .*$srcfile:$line1.*"
+    }
+
+    # Try nexting inferior 1 again.
+    with_test_prefix "next inf 1 again" {
+	block 1 0
+	gdb_test "next" "Thread 2.1 .*hit Breakpoint .*$srcfile:$line2.*"
+    }
+}
+
+# Make a core file with two threads upfront.  Several tests load the
+# same core file.
+prepare_core
+
+# Some basic "continue" + breakpoints tests.
+with_test_prefix "continue" {
+    foreach_with_prefix non-stop {"off" "on"} {
+	test_continue ${non-stop}
+    }
+}
+
+# Some basic all-stop Ctrl-C tests.
+with_test_prefix "interrupt" {
+    test_ctrlc
+}
+
+# Test ping-ponging between two targets with "next".
+with_test_prefix "ping-pong" {
+    test_ping_pong_next
+}
diff --git a/gdb/testsuite/lib/gdbserver-support.exp b/gdb/testsuite/lib/gdbserver-support.exp
index 21ec418..12796e8 100644
--- a/gdb/testsuite/lib/gdbserver-support.exp
+++ b/gdb/testsuite/lib/gdbserver-support.exp
@@ -64,6 +64,10 @@ proc gdb_target_cmd_ext { targetname serialport {additional_text ""} } {
 	    -re "Couldn't establish connection to remote.*$gdb_prompt $" {
 		verbose "Connection failed"
 	    }
+	    -re "Non-stop mode requested, but remote does not support non-stop.*$gdb_prompt $" {
+		verbose "remote does not support non-stop"
+		return 1
+	    }
 	    -re "Remote MIPS debugging.*$additional_text.*$gdb_prompt" {
 		verbose "Set target to $targetname"
 		return 0