Test daisy chained homogeneous spike instances. (#334)

* Test debugging multiple spikes in a daisy chain.

* Hugely speed up rbb_daisychain.

Now 2 dual-hart spikes are less than 4x slower than a single dual-hart
spike.

* WIP

* Test daisy chained homogeneous spike instances.

For OpenOCD, this means we're checking that we can talk to multiple
TAPs. Next up is heterogeneous testing.

* Enable Sv48Test.

Didn't mean to disable it with this commit.

* Test authentication again.

Another change I hadn't meant to push...

9 files changed, 303 insertions, 40 deletions

diff --git a/debug/Makefile b/debug/Makefile
index 8f40a14..f6e8efb 100644
--- a/debug/Makefile
+++ b/debug/Makefile
@@ -8,7 +8,7 @@ TESTS = $(shell $(GDBSERVER_PY) --list-tests $(src_dir)/targets/RISC-V/spike32.p
 default: spike$(XLEN) spike$(XLEN)-2
 
 all-tests: spike32 spike32-2 spike32-2-hwthread \
-	spike64 spike64-2 spike64-2-hwthread
+	spike64 spike64-2 spike64-2-hwthread spike-multi
 
 all:	pylint all-tests
 
diff --git a/debug/gdbserver.py b/debug/gdbserver.py
index bcac02c..ea5bb22 100755
--- a/debug/gdbserver.py
+++ b/debug/gdbserver.py
@@ -1012,18 +1012,18 @@ class MulticoreRegTest(GdbTest):
             self.gdb.c()
             assertIn("main_end", self.gdb.where())
 
-        hart_ids = []
+        hart_ids = set()
         for hart in self.target.harts:
             self.gdb.select_hart(hart)
             # Check register values.
             x1 = self.gdb.p("$x1")
             hart_id = self.gdb.p("$mhartid")
             assertEqual(x1, hart_id << 8)
-            assertNotIn(hart_id, hart_ids)
-            hart_ids.append(hart_id)
+            assertNotIn((hart.system, hart_id), hart_ids)
+            hart_ids.add((hart.system, hart_id))
             for n in range(2, 32):
                 value = self.gdb.p("$x%d" % n)
-                assertEqual(value, (hart_ids[-1] << 8) + n - 1)
+                assertEqual(value, (hart_id << 8) + n - 1)
 
         # Confirmed that we read different register values for different harts.
         # Write a new value to x1, and run through the add sequence again.
@@ -1471,7 +1471,8 @@ class DownloadTest(GdbTest):
         self.gdb.load()
         self.parkOtherHarts()
         self.gdb.command("b _exit")
-        self.gdb.c(ops=100)
+        #self.gdb.c(ops=100)
+        self.gdb.c()
         assertEqual(self.gdb.p("status"), self.crc)
         os.unlink(self.download_c.name)
 
diff --git a/debug/rbb_daisychain.py b/debug/rbb_daisychain.py
new file mode 100755
index 0000000..b7c21e6
--- /dev/null
+++ b/debug/rbb_daisychain.py
@@ -0,0 +1,111 @@
+#!/usr/bin/python3
+
+import argparse
+import sys
+import socket
+
+# https://github.com/ntfreak/openocd/blob/master/doc/manual/jtag/drivers/remote_bitbang.txt
+
+class Tap:
+    def __init__(self, port):
+        self.port = port
+        self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        self.socket.connect(("localhost", port))
+
+    def execute(self, commands):
+        sent = self.socket.send(commands)
+        assert len(commands) == sent
+        read_count = 0
+        for command in commands:
+            if command == ord('R'):
+                read_count += 1
+        result = b""
+        while len(result) < read_count:
+            result += self.socket.recv(read_count - len(result))
+        assert len(result) == read_count
+        return result
+
+class Chain:
+    def __init__(self, debug=False):
+        self.debug = debug
+        self.taps = []
+
+    def append(self, tap):
+        self.taps.append(tap)
+
+    def execute(self, commands):
+        values = []
+        for i, tap in enumerate(self.taps):
+            tmp_commands = []
+            for command in commands:
+                if i > 0 and ord('0') <= command <= ord('7'):
+                    # Replace TDI with the value from the previous TAP.
+                    v = values.pop(0)
+                    command &= 0xfe
+                    if v == ord('1'):
+                        command |= 1
+
+                if i < len(self.taps) - 1:
+                    if command != ord('R'):
+                        tmp_commands.append(command)
+                    if ord('0') <= command <= ord('7'):
+                        # Read TDO before every scan.
+                        tmp_commands.append(ord('R'))
+                else:
+                    tmp_commands.append(command)
+            assert len(values) == 0
+            values = list(tap.execute(bytes(tmp_commands)))
+            if self.debug:
+                sys.stdout.write("    %d %r -> %r\n" % (i, bytes(tmp_commands),
+                                                        bytes(values)))
+        return bytes(values)
+
+def main():
+    parser = argparse.ArgumentParser(
+            description='Combine multiple remote_bitbang processes into a '
+            'single scan-chain.')
+    parser.add_argument("listen_port", type=int,
+            help="port to listen on")
+    parser.add_argument("tap_port", nargs="+", type=int,
+            help="port of a remote_bitbang TAP to connect to")
+    parser.add_argument("--debug", action='store_true',
+                        help="Print out debug messages.")
+    args = parser.parse_args()
+
+    server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    server.bind(("localhost", args.listen_port))
+    server.listen(1)
+
+    chain = Chain(args.debug)
+    for port in args.tap_port:
+        chain.append(Tap(port))
+
+    sys.stdout.write("Listening on port %d.\n" % server.getsockname()[1])
+    sys.stdout.flush()
+
+    while True:
+        (client, _) = server.accept()
+
+        while True:
+            try:
+                commands = client.recv(4096)
+            except (ConnectionResetError, OSError):
+                sys.stdout.write("Client disconnected due to exception.\n")
+                break
+
+            if len(commands) == 0:
+                sys.stdout.write("Client disconnected.\n")
+                break
+
+            if args.debug:
+                sys.stdout.write("%r\n" % commands)
+            result = chain.execute(commands)
+            if args.debug:
+                sys.stdout.write("   -> %r\n" % result)
+            client.send(result)
+
+        client.close()
+        sys.stdout.flush()
+
+if __name__ == '__main__':
+    sys.exit(main())
diff --git a/debug/targets.py b/debug/targets.py
index 0890465..be45c62 100644
--- a/debug/targets.py
+++ b/debug/targets.py
@@ -44,6 +44,13 @@ class Hart:
     # jumpers.
     reset_vectors = []
 
+    def __init__(self, system=None):
+        """system is set to an identifier of the system this hart belongs to.
+        Harts within the same system are assumed to share memory, and to have
+        unique hartids within that system.  So for most cases the default
+        value of None is fine."""
+        self.system = system
+
     def extensionSupported(self, letter):
         # target.misa is set by testlib.ExamineTarget
         if self.misa:
diff --git a/debug/targets/RISC-V/spike-multi.cfg b/debug/targets/RISC-V/spike-multi.cfg
new file mode 100644
index 0000000..ef6dfc6
--- /dev/null
+++ b/debug/targets/RISC-V/spike-multi.cfg
@@ -0,0 +1,46 @@
+# Connect to a mult-icore RISC-V target, exposing each hart as a thread.
+adapter_khz     10000
+
+interface remote_bitbang
+remote_bitbang_host $::env(REMOTE_BITBANG_HOST)
+remote_bitbang_port $::env(REMOTE_BITBANG_PORT)
+
+jtag newtap riscv.0 cpu -irlen 5 -expected-id 0x10e31913
+jtag newtap riscv.1 cpu -irlen 5 -expected-id 0x10e31913
+
+target create riscv.0.cpu0 riscv -chain-position riscv.0.cpu -coreid 0
+target create riscv.0.cpu1 riscv -chain-position riscv.0.cpu -coreid 1
+target create riscv.1.cpu0 riscv -chain-position riscv.1.cpu -coreid 0
+target create riscv.1.cpu1 riscv -chain-position riscv.1.cpu -coreid 1
+
+riscv.0.cpu0 configure -work-area-phys $::env(WORK_AREA) -work-area-size 8096 -work-area-backup 1
+riscv.0.cpu1 configure -work-area-phys $::env(WORK_AREA) -work-area-size 8096 -work-area-backup 1
+riscv.1.cpu0 configure -work-area-phys $::env(WORK_AREA) -work-area-size 8096 -work-area-backup 1
+riscv.1.cpu1 configure -work-area-phys $::env(WORK_AREA) -work-area-size 8096 -work-area-backup 1
+
+gdb_report_data_abort enable
+gdb_report_register_access_error enable
+
+# Expose an unimplemented CSR so we can test non-existent register access
+# behavior.
+foreach t [target names] {
+    targets $t
+    riscv expose_csrs 2288
+    riscv expose_custom 1,12345-12348
+}
+
+init
+
+targets riscv.0.cpu0
+set challenge [riscv authdata_read]
+riscv authdata_write [expr $challenge + 1]
+
+targets riscv.1.cpu0
+set challenge [riscv authdata_read]
+riscv authdata_write [expr $challenge + 1]
+
+foreach t [target names] {
+    targets $t
+    halt
+    arm semihosting enable
+}
diff --git a/debug/targets/RISC-V/spike-multi.py b/debug/targets/RISC-V/spike-multi.py
new file mode 100644
index 0000000..9e85a35
--- /dev/null
+++ b/debug/targets/RISC-V/spike-multi.py
@@ -0,0 +1,39 @@
+import targets
+import testlib
+
+import spike32  # pylint: disable=import-error
+#import spike64  # pylint: disable=import-error
+
+class multispike(targets.Target):
+    harts = [
+        spike32.spike32_hart(misa=0x4034112d, system=0),
+        spike32.spike32_hart(misa=0x4034112d, system=0),
+        spike32.spike32_hart(misa=0x4034112d, system=1),
+        spike32.spike32_hart(misa=0x4034112d, system=1)]
+        #spike64.spike64_hart(misa=0x8000000000341129),
+        #spike64.spike64_hart(misa=0x8000000000341129)]
+    openocd_config_path = "spike-multi.cfg"
+    # Increased timeout because we use abstract_rti to artificially slow things
+    # down.
+    timeout_sec = 30
+    implements_custom_test = True
+    support_hasel = False
+    support_memory_sampling = False # Needs SBA
+
+    def create(self):
+        # TODO: It would be nice to test with slen=128, but spike currently
+        # requires vlen==slen.
+        return testlib.MultiSpike(
+            [
+#                testlib.Spike(self, isa="RV64IMAFDV", abstract_rti=30,
+#                    support_hasel=False, support_abstract_csr=False,
+#                    vlen=512, elen=64, slen=512, harts=self.harts[:2]),
+                testlib.Spike(self, isa="RV32IMAFDCV", dmi_rti=4,
+                    support_abstract_csr=True, support_haltgroups=False,
+                    # elen must be at least 64 because D is supported.
+                    elen=64, harts=self.harts[2:]),
+                testlib.Spike(self, isa="RV32IMAFDCV", dmi_rti=4,
+                    support_abstract_csr=True, support_haltgroups=False,
+                    # elen must be at least 64 because D is supported.
+                    elen=64, harts=self.harts[2:])
+                ])
diff --git a/debug/targets/RISC-V/spike32.py b/debug/targets/RISC-V/spike32.py
index 381aea7..6256574 100644
--- a/debug/targets/RISC-V/spike32.py
+++ b/debug/targets/RISC-V/spike32.py
@@ -10,7 +10,8 @@ class spike32_hart(targets.Hart):
     reset_vectors = [0x1000]
     link_script_path = "spike32.lds"
 
-    def __init__(self, misa):
+    def __init__(self, misa, system=0):
+        super().__init__(system=system)
         self.misa = misa
 
 class spike32(targets.Target):
diff --git a/debug/targets/RISC-V/spike64.py b/debug/targets/RISC-V/spike64.py
index c4c66c8..4313968 100644
--- a/debug/targets/RISC-V/spike64.py
+++ b/debug/targets/RISC-V/spike64.py
@@ -10,7 +10,8 @@ class spike64_hart(targets.Hart):
     reset_vectors = [0x1000]
     link_script_path = "spike64.lds"
 
-    def __init__(self, misa=0x8000000000141125):
+    def __init__(self, misa=0x8000000000141125, system=0):
+        super().__init__(system=system)
         self.misa = misa
 
 class spike64(targets.Target):
diff --git a/debug/testlib.py b/debug/testlib.py
index 501e0e6..cde6597 100644
--- a/debug/testlib.py
+++ b/debug/testlib.py
@@ -65,7 +65,7 @@ class Spike:
     def __init__(self, target, halted=False, timeout=None, with_jtag_gdb=True,
             isa=None, progbufsize=None, dmi_rti=None, abstract_rti=None,
             support_hasel=True, support_abstract_csr=True,
-            support_haltgroups=True, vlen=128, elen=64, slen=128):
+            support_haltgroups=True, vlen=128, elen=64, slen=128, harts=None):
         """Launch spike. Return tuple of its process and the port it's running
         on."""
         self.process = None
@@ -80,21 +80,19 @@ class Spike:
         self.elen = elen
         self.slen = slen
 
-        if target.harts:
-            harts = target.harts
-        else:
-            harts = [target]
+        self.harts = harts or target.harts or [target]
 
-        cmd = self.command(target, harts, halted, timeout, with_jtag_gdb)
-        self.infinite_loop = target.compile(harts[0],
+        cmd = self.command(target, halted, timeout, with_jtag_gdb)
+        self.infinite_loop = target.compile(self.harts[0],
                 "programs/checksum.c", "programs/tiny-malloc.c",
                 "programs/infinite_loop.S", "-DDEFINE_MALLOC", "-DDEFINE_FREE")
         cmd.append(self.infinite_loop)
         self.logfile = tempfile.NamedTemporaryFile(prefix="spike-",
                 suffix=".log")
-        self.logname = self.logfile.name
+        logname = self.logfile.name
+        self.lognames = [logname]
         if print_log_names:
-            real_stdout.write("Temporary spike log: %s\n" % self.logname)
+            real_stdout.write("Temporary spike log: %s\n" % logname)
         self.logfile.write(("+ %s\n" % " ".join(cmd)).encode())
         self.logfile.flush()
         self.process = subprocess.Popen(cmd, stdin=subprocess.PIPE,
@@ -104,34 +102,34 @@ class Spike:
             self.port = None
             for _ in range(30):
                 m = re.search(r"Listening for remote bitbang connection on "
-                        r"port (\d+).", open(self.logname).read())
+                        r"port (\d+).", open(logname).read())
                 if m:
                     self.port = int(m.group(1))
                     os.environ['REMOTE_BITBANG_PORT'] = m.group(1)
                     break
                 time.sleep(0.11)
             if not self.port:
-                print_log(self.logname)
+                print_log(logname)
                 raise Exception("Didn't get spike message about bitbang "
                         "connection")
 
     # pylint: disable=too-many-branches
-    def command(self, target, harts, halted, timeout, with_jtag_gdb):
+    def command(self, target, halted, timeout, with_jtag_gdb):
         # pylint: disable=no-self-use
         if target.sim_cmd:
             cmd = shlex.split(target.sim_cmd)
         else:
             cmd = ["spike"]
 
-        cmd += ["-p%d" % len(harts)]
+        cmd += ["-p%d" % len(self.harts)]
 
-        assert len(set(t.xlen for t in harts)) == 1, \
+        assert len(set(t.xlen for t in self.harts)) == 1, \
                 "All spike harts must have the same XLEN"
 
         if self.isa:
             isa = self.isa
         else:
-            isa = "RV%dG" % harts[0].xlen
+            isa = "RV%dG" % self.harts[0].xlen
 
         cmd += ["--isa", isa]
         cmd += ["--dm-auth"]
@@ -159,12 +157,12 @@ class Spike:
             cmd.append("--varch=vlen:%d,elen:%d,slen:%d" % (self.vlen,
                 self.elen, self.slen))
 
-        assert len(set(t.ram for t in harts)) == 1, \
+        assert len(set(t.ram for t in self.harts)) == 1, \
                 "All spike harts must have the same RAM layout"
-        assert len(set(t.ram_size for t in harts)) == 1, \
+        assert len(set(t.ram_size for t in self.harts)) == 1, \
                 "All spike harts must have the same RAM layout"
-        os.environ['WORK_AREA'] = '0x%x' % harts[0].ram
-        cmd += ["-m0x%x:0x%x" % (harts[0].ram, harts[0].ram_size)]
+        os.environ['WORK_AREA'] = '0x%x' % self.harts[0].ram
+        cmd += ["-m0x%x:0x%x" % (self.harts[0].ram, self.harts[0].ram_size)]
 
         if timeout:
             cmd = ["timeout", str(timeout)] + cmd
@@ -188,6 +186,48 @@ class Spike:
     def wait(self, *args, **kwargs):
         return self.process.wait(*args, **kwargs)
 
+class MultiSpike:
+    def __init__(self, spikes):
+        self.process = None
+
+        self.spikes = spikes
+        self.lognames = sum((spike.lognames for spike in spikes), [])
+        self.logfile = tempfile.NamedTemporaryFile(prefix="daisychain-",
+                suffix=".log")
+        self.lognames.append(self.logfile.name)
+
+        # Now create the daisy-chain process.
+        cmd = ["./rbb_daisychain.py", "0"] + \
+            [str(spike.port) for spike in spikes]
+        self.logfile.write(("+ %s\n" % cmd).encode())
+        self.logfile.flush()
+        self.process = subprocess.Popen(cmd, stdin=subprocess.PIPE,
+                stdout=self.logfile, stderr=self.logfile)
+
+        self.port = None
+        for _ in range(30):
+            m = re.search(r"Listening on port (\d+).",
+                          open(self.lognames[-1]).read())
+            if m:
+                self.port = int(m.group(1))
+                break
+            time.sleep(0.11)
+        if not self.port:
+            print_log(self.lognames[-1])
+            raise Exception("Didn't get daisy chain message about which port "
+                            "it's listening on.")
+
+        os.environ['REMOTE_BITBANG_HOST'] = 'localhost'
+        os.environ['REMOTE_BITBANG_PORT'] = str(self.port)
+
+    def __del__(self):
+        if self.process:
+            try:
+                self.process.kill()
+                self.process.wait()
+            except OSError:
+                pass
+
 class VcsSim:
     logfile = tempfile.NamedTemporaryFile(prefix='simv', suffix='.log')
     logname = logfile.name
@@ -519,11 +559,12 @@ class Gdb:
             11, 149, 107, 163, 73, 47, 43, 173, 7, 109, 101, 103, 191, 2, 139,
             97, 193, 157, 3, 29, 79, 113, 5, 89, 19, 37, 71, 179, 59, 137, 53)
 
-    def __init__(self, ports,
+    def __init__(self, target, ports,
             cmd="riscv64-unknown-elf-gdb",
             timeout=60, binary=None):
         assert ports
 
+        self.target = target
         self.ports = ports
         self.cmd = cmd
         self.timeout = timeout
@@ -551,14 +592,16 @@ class Gdb:
         for port, child in zip(self.ports, self.children):
             self.select_child(child)
             self.wait()
-            self.command("set style enabled off")
-            self.command("set confirm off")
-            self.command("set width 0")
-            self.command("set height 0")
+            self.command("set style enabled off", reset_delays=None)
+            self.command("set confirm off", reset_delays=None)
+            self.command("set width 0", reset_delays=None)
+            self.command("set height 0", reset_delays=None)
             # Force consistency.
-            self.command("set print entry-values no")
-            self.command("set remotetimeout %d" % self.timeout)
-            self.command("target extended-remote localhost:%d" % port, ops=10)
+            self.command("set print entry-values no", reset_delays=None)
+            self.command("set remotetimeout %d" % self.timeout,
+                         reset_delays=None)
+            self.command("target extended-remote localhost:%d" % port, ops=10,
+                         reset_delays=None)
             if self.binary:
                 self.command("file %s" % self.binary)
             threads = self.threads()
@@ -635,6 +678,20 @@ class Gdb:
                 self.select_child(child)
                 self.command(command)
 
+    def system_command(self, command, ops=20):
+        """Execute this command on every unique system that we control."""
+        done = set()
+        output = ""
+        with PrivateState(self):
+            for i, child in enumerate(self.children):
+                self.select_child(child)
+                if self.target.harts[i].system in done:
+                    self.command("set $pc=_start")
+                else:
+                    output += self.command(command, ops=ops)
+                    done.add(self.target.harts[i].system)
+        return output
+
     def c(self, wait=True, sync=True, checkOutput=True, ops=20):
         """
         Dumb c command.
@@ -742,10 +799,10 @@ class Gdb:
         return output
 
     def load(self):
-        output = self.command("load", ops=1000)
+        output = self.system_command("load", ops=1000)
         assert "failed" not in  output
         assert "Transfer rate" in output
-        output = self.command("compare-sections", ops=1000)
+        output = self.system_command("compare-sections", ops=1000)
         assert "matched" in output
         assert "MIS" not in output
 
@@ -986,7 +1043,7 @@ class BaseTest:
         self.compile()
         self.target_process = self.target.create()
         if self.target_process:
-            self.logs.append(self.target_process.logname)
+            self.logs += self.target_process.lognames
         try:
             self.server = self.target.server(self)
             self.logs.append(self.server.logname)
@@ -1076,10 +1133,10 @@ class GdbTest(BaseTest):
         BaseTest.classSetup(self)
 
         if gdb_cmd:
-            self.gdb = Gdb(self.server.gdb_ports, gdb_cmd,
+            self.gdb = Gdb(self.target, self.server.gdb_ports, gdb_cmd,
                     timeout=self.target.timeout_sec, binary=self.binary)
         else:
-            self.gdb = Gdb(self.server.gdb_ports,
+            self.gdb = Gdb(self.target, self.server.gdb_ports,
                     timeout=self.target.timeout_sec, binary=self.binary)
 
         self.logs += self.gdb.lognames()