From c9f43c1652c1d8abf85f3466f41ffd5ec4d911d6 Mon Sep 17 00:00:00 2001 From: Tim Newsome Date: Fri, 7 May 2021 11:49:42 -0700 Subject: Test daisy chained homogeneous spike instances. (#334) * Test debugging multiple spikes in a daisy chain. * Hugely speed up rbb_daisychain. Now 2 dual-hart spikes are less than 4x slower than a single dual-hart spike. * WIP * Test daisy chained homogeneous spike instances. For OpenOCD, this means we're checking that we can talk to multiple TAPs. Next up is heterogeneous testing. * Enable Sv48Test. Didn't mean to disable it with this commit. * Test authentication again. Another change I hadn't meant to push... --- debug/Makefile | 2 +- debug/gdbserver.py | 11 ++-- debug/rbb_daisychain.py | 111 ++++++++++++++++++++++++++++++++ debug/targets.py | 7 ++ debug/targets/RISC-V/spike-multi.cfg | 46 +++++++++++++ debug/targets/RISC-V/spike-multi.py | 39 +++++++++++ debug/targets/RISC-V/spike32.py | 3 +- debug/targets/RISC-V/spike64.py | 3 +- debug/testlib.py | 121 ++++++++++++++++++++++++++--------- 9 files changed, 303 insertions(+), 40 deletions(-) create mode 100755 debug/rbb_daisychain.py create mode 100644 debug/targets/RISC-V/spike-multi.cfg create mode 100644 debug/targets/RISC-V/spike-multi.py diff --git a/debug/Makefile b/debug/Makefile index 8f40a14..f6e8efb 100644 --- a/debug/Makefile +++ b/debug/Makefile @@ -8,7 +8,7 @@ TESTS = $(shell $(GDBSERVER_PY) --list-tests $(src_dir)/targets/RISC-V/spike32.p default: spike$(XLEN) spike$(XLEN)-2 all-tests: spike32 spike32-2 spike32-2-hwthread \ - spike64 spike64-2 spike64-2-hwthread + spike64 spike64-2 spike64-2-hwthread spike-multi all: pylint all-tests diff --git a/debug/gdbserver.py b/debug/gdbserver.py index bcac02c..ea5bb22 100755 --- a/debug/gdbserver.py +++ b/debug/gdbserver.py @@ -1012,18 +1012,18 @@ class MulticoreRegTest(GdbTest): self.gdb.c() assertIn("main_end", self.gdb.where()) - hart_ids = [] + hart_ids = set() for hart in self.target.harts: self.gdb.select_hart(hart) # Check register values. x1 = self.gdb.p("$x1") hart_id = self.gdb.p("$mhartid") assertEqual(x1, hart_id << 8) - assertNotIn(hart_id, hart_ids) - hart_ids.append(hart_id) + assertNotIn((hart.system, hart_id), hart_ids) + hart_ids.add((hart.system, hart_id)) for n in range(2, 32): value = self.gdb.p("$x%d" % n) - assertEqual(value, (hart_ids[-1] << 8) + n - 1) + assertEqual(value, (hart_id << 8) + n - 1) # Confirmed that we read different register values for different harts. # Write a new value to x1, and run through the add sequence again. @@ -1471,7 +1471,8 @@ class DownloadTest(GdbTest): self.gdb.load() self.parkOtherHarts() self.gdb.command("b _exit") - self.gdb.c(ops=100) + #self.gdb.c(ops=100) + self.gdb.c() assertEqual(self.gdb.p("status"), self.crc) os.unlink(self.download_c.name) diff --git a/debug/rbb_daisychain.py b/debug/rbb_daisychain.py new file mode 100755 index 0000000..b7c21e6 --- /dev/null +++ b/debug/rbb_daisychain.py @@ -0,0 +1,111 @@ +#!/usr/bin/python3 + +import argparse +import sys +import socket + +# https://github.com/ntfreak/openocd/blob/master/doc/manual/jtag/drivers/remote_bitbang.txt + +class Tap: + def __init__(self, port): + self.port = port + self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.socket.connect(("localhost", port)) + + def execute(self, commands): + sent = self.socket.send(commands) + assert len(commands) == sent + read_count = 0 + for command in commands: + if command == ord('R'): + read_count += 1 + result = b"" + while len(result) < read_count: + result += self.socket.recv(read_count - len(result)) + assert len(result) == read_count + return result + +class Chain: + def __init__(self, debug=False): + self.debug = debug + self.taps = [] + + def append(self, tap): + self.taps.append(tap) + + def execute(self, commands): + values = [] + for i, tap in enumerate(self.taps): + tmp_commands = [] + for command in commands: + if i > 0 and ord('0') <= command <= ord('7'): + # Replace TDI with the value from the previous TAP. + v = values.pop(0) + command &= 0xfe + if v == ord('1'): + command |= 1 + + if i < len(self.taps) - 1: + if command != ord('R'): + tmp_commands.append(command) + if ord('0') <= command <= ord('7'): + # Read TDO before every scan. + tmp_commands.append(ord('R')) + else: + tmp_commands.append(command) + assert len(values) == 0 + values = list(tap.execute(bytes(tmp_commands))) + if self.debug: + sys.stdout.write(" %d %r -> %r\n" % (i, bytes(tmp_commands), + bytes(values))) + return bytes(values) + +def main(): + parser = argparse.ArgumentParser( + description='Combine multiple remote_bitbang processes into a ' + 'single scan-chain.') + parser.add_argument("listen_port", type=int, + help="port to listen on") + parser.add_argument("tap_port", nargs="+", type=int, + help="port of a remote_bitbang TAP to connect to") + parser.add_argument("--debug", action='store_true', + help="Print out debug messages.") + args = parser.parse_args() + + server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + server.bind(("localhost", args.listen_port)) + server.listen(1) + + chain = Chain(args.debug) + for port in args.tap_port: + chain.append(Tap(port)) + + sys.stdout.write("Listening on port %d.\n" % server.getsockname()[1]) + sys.stdout.flush() + + while True: + (client, _) = server.accept() + + while True: + try: + commands = client.recv(4096) + except (ConnectionResetError, OSError): + sys.stdout.write("Client disconnected due to exception.\n") + break + + if len(commands) == 0: + sys.stdout.write("Client disconnected.\n") + break + + if args.debug: + sys.stdout.write("%r\n" % commands) + result = chain.execute(commands) + if args.debug: + sys.stdout.write(" -> %r\n" % result) + client.send(result) + + client.close() + sys.stdout.flush() + +if __name__ == '__main__': + sys.exit(main()) diff --git a/debug/targets.py b/debug/targets.py index 0890465..be45c62 100644 --- a/debug/targets.py +++ b/debug/targets.py @@ -44,6 +44,13 @@ class Hart: # jumpers. reset_vectors = [] + def __init__(self, system=None): + """system is set to an identifier of the system this hart belongs to. + Harts within the same system are assumed to share memory, and to have + unique hartids within that system. So for most cases the default + value of None is fine.""" + self.system = system + def extensionSupported(self, letter): # target.misa is set by testlib.ExamineTarget if self.misa: diff --git a/debug/targets/RISC-V/spike-multi.cfg b/debug/targets/RISC-V/spike-multi.cfg new file mode 100644 index 0000000..ef6dfc6 --- /dev/null +++ b/debug/targets/RISC-V/spike-multi.cfg @@ -0,0 +1,46 @@ +# Connect to a mult-icore RISC-V target, exposing each hart as a thread. +adapter_khz 10000 + +interface remote_bitbang +remote_bitbang_host $::env(REMOTE_BITBANG_HOST) +remote_bitbang_port $::env(REMOTE_BITBANG_PORT) + +jtag newtap riscv.0 cpu -irlen 5 -expected-id 0x10e31913 +jtag newtap riscv.1 cpu -irlen 5 -expected-id 0x10e31913 + +target create riscv.0.cpu0 riscv -chain-position riscv.0.cpu -coreid 0 +target create riscv.0.cpu1 riscv -chain-position riscv.0.cpu -coreid 1 +target create riscv.1.cpu0 riscv -chain-position riscv.1.cpu -coreid 0 +target create riscv.1.cpu1 riscv -chain-position riscv.1.cpu -coreid 1 + +riscv.0.cpu0 configure -work-area-phys $::env(WORK_AREA) -work-area-size 8096 -work-area-backup 1 +riscv.0.cpu1 configure -work-area-phys $::env(WORK_AREA) -work-area-size 8096 -work-area-backup 1 +riscv.1.cpu0 configure -work-area-phys $::env(WORK_AREA) -work-area-size 8096 -work-area-backup 1 +riscv.1.cpu1 configure -work-area-phys $::env(WORK_AREA) -work-area-size 8096 -work-area-backup 1 + +gdb_report_data_abort enable +gdb_report_register_access_error enable + +# Expose an unimplemented CSR so we can test non-existent register access +# behavior. +foreach t [target names] { + targets $t + riscv expose_csrs 2288 + riscv expose_custom 1,12345-12348 +} + +init + +targets riscv.0.cpu0 +set challenge [riscv authdata_read] +riscv authdata_write [expr $challenge + 1] + +targets riscv.1.cpu0 +set challenge [riscv authdata_read] +riscv authdata_write [expr $challenge + 1] + +foreach t [target names] { + targets $t + halt + arm semihosting enable +} diff --git a/debug/targets/RISC-V/spike-multi.py b/debug/targets/RISC-V/spike-multi.py new file mode 100644 index 0000000..9e85a35 --- /dev/null +++ b/debug/targets/RISC-V/spike-multi.py @@ -0,0 +1,39 @@ +import targets +import testlib + +import spike32 # pylint: disable=import-error +#import spike64 # pylint: disable=import-error + +class multispike(targets.Target): + harts = [ + spike32.spike32_hart(misa=0x4034112d, system=0), + spike32.spike32_hart(misa=0x4034112d, system=0), + spike32.spike32_hart(misa=0x4034112d, system=1), + spike32.spike32_hart(misa=0x4034112d, system=1)] + #spike64.spike64_hart(misa=0x8000000000341129), + #spike64.spike64_hart(misa=0x8000000000341129)] + openocd_config_path = "spike-multi.cfg" + # Increased timeout because we use abstract_rti to artificially slow things + # down. + timeout_sec = 30 + implements_custom_test = True + support_hasel = False + support_memory_sampling = False # Needs SBA + + def create(self): + # TODO: It would be nice to test with slen=128, but spike currently + # requires vlen==slen. + return testlib.MultiSpike( + [ +# testlib.Spike(self, isa="RV64IMAFDV", abstract_rti=30, +# support_hasel=False, support_abstract_csr=False, +# vlen=512, elen=64, slen=512, harts=self.harts[:2]), + testlib.Spike(self, isa="RV32IMAFDCV", dmi_rti=4, + support_abstract_csr=True, support_haltgroups=False, + # elen must be at least 64 because D is supported. + elen=64, harts=self.harts[2:]), + testlib.Spike(self, isa="RV32IMAFDCV", dmi_rti=4, + support_abstract_csr=True, support_haltgroups=False, + # elen must be at least 64 because D is supported. + elen=64, harts=self.harts[2:]) + ]) diff --git a/debug/targets/RISC-V/spike32.py b/debug/targets/RISC-V/spike32.py index 381aea7..6256574 100644 --- a/debug/targets/RISC-V/spike32.py +++ b/debug/targets/RISC-V/spike32.py @@ -10,7 +10,8 @@ class spike32_hart(targets.Hart): reset_vectors = [0x1000] link_script_path = "spike32.lds" - def __init__(self, misa): + def __init__(self, misa, system=0): + super().__init__(system=system) self.misa = misa class spike32(targets.Target): diff --git a/debug/targets/RISC-V/spike64.py b/debug/targets/RISC-V/spike64.py index c4c66c8..4313968 100644 --- a/debug/targets/RISC-V/spike64.py +++ b/debug/targets/RISC-V/spike64.py @@ -10,7 +10,8 @@ class spike64_hart(targets.Hart): reset_vectors = [0x1000] link_script_path = "spike64.lds" - def __init__(self, misa=0x8000000000141125): + def __init__(self, misa=0x8000000000141125, system=0): + super().__init__(system=system) self.misa = misa class spike64(targets.Target): diff --git a/debug/testlib.py b/debug/testlib.py index 501e0e6..cde6597 100644 --- a/debug/testlib.py +++ b/debug/testlib.py @@ -65,7 +65,7 @@ class Spike: def __init__(self, target, halted=False, timeout=None, with_jtag_gdb=True, isa=None, progbufsize=None, dmi_rti=None, abstract_rti=None, support_hasel=True, support_abstract_csr=True, - support_haltgroups=True, vlen=128, elen=64, slen=128): + support_haltgroups=True, vlen=128, elen=64, slen=128, harts=None): """Launch spike. Return tuple of its process and the port it's running on.""" self.process = None @@ -80,21 +80,19 @@ class Spike: self.elen = elen self.slen = slen - if target.harts: - harts = target.harts - else: - harts = [target] + self.harts = harts or target.harts or [target] - cmd = self.command(target, harts, halted, timeout, with_jtag_gdb) - self.infinite_loop = target.compile(harts[0], + cmd = self.command(target, halted, timeout, with_jtag_gdb) + self.infinite_loop = target.compile(self.harts[0], "programs/checksum.c", "programs/tiny-malloc.c", "programs/infinite_loop.S", "-DDEFINE_MALLOC", "-DDEFINE_FREE") cmd.append(self.infinite_loop) self.logfile = tempfile.NamedTemporaryFile(prefix="spike-", suffix=".log") - self.logname = self.logfile.name + logname = self.logfile.name + self.lognames = [logname] if print_log_names: - real_stdout.write("Temporary spike log: %s\n" % self.logname) + real_stdout.write("Temporary spike log: %s\n" % logname) self.logfile.write(("+ %s\n" % " ".join(cmd)).encode()) self.logfile.flush() self.process = subprocess.Popen(cmd, stdin=subprocess.PIPE, @@ -104,34 +102,34 @@ class Spike: self.port = None for _ in range(30): m = re.search(r"Listening for remote bitbang connection on " - r"port (\d+).", open(self.logname).read()) + r"port (\d+).", open(logname).read()) if m: self.port = int(m.group(1)) os.environ['REMOTE_BITBANG_PORT'] = m.group(1) break time.sleep(0.11) if not self.port: - print_log(self.logname) + print_log(logname) raise Exception("Didn't get spike message about bitbang " "connection") # pylint: disable=too-many-branches - def command(self, target, harts, halted, timeout, with_jtag_gdb): + def command(self, target, halted, timeout, with_jtag_gdb): # pylint: disable=no-self-use if target.sim_cmd: cmd = shlex.split(target.sim_cmd) else: cmd = ["spike"] - cmd += ["-p%d" % len(harts)] + cmd += ["-p%d" % len(self.harts)] - assert len(set(t.xlen for t in harts)) == 1, \ + assert len(set(t.xlen for t in self.harts)) == 1, \ "All spike harts must have the same XLEN" if self.isa: isa = self.isa else: - isa = "RV%dG" % harts[0].xlen + isa = "RV%dG" % self.harts[0].xlen cmd += ["--isa", isa] cmd += ["--dm-auth"] @@ -159,12 +157,12 @@ class Spike: cmd.append("--varch=vlen:%d,elen:%d,slen:%d" % (self.vlen, self.elen, self.slen)) - assert len(set(t.ram for t in harts)) == 1, \ + assert len(set(t.ram for t in self.harts)) == 1, \ "All spike harts must have the same RAM layout" - assert len(set(t.ram_size for t in harts)) == 1, \ + assert len(set(t.ram_size for t in self.harts)) == 1, \ "All spike harts must have the same RAM layout" - os.environ['WORK_AREA'] = '0x%x' % harts[0].ram - cmd += ["-m0x%x:0x%x" % (harts[0].ram, harts[0].ram_size)] + os.environ['WORK_AREA'] = '0x%x' % self.harts[0].ram + cmd += ["-m0x%x:0x%x" % (self.harts[0].ram, self.harts[0].ram_size)] if timeout: cmd = ["timeout", str(timeout)] + cmd @@ -188,6 +186,48 @@ class Spike: def wait(self, *args, **kwargs): return self.process.wait(*args, **kwargs) +class MultiSpike: + def __init__(self, spikes): + self.process = None + + self.spikes = spikes + self.lognames = sum((spike.lognames for spike in spikes), []) + self.logfile = tempfile.NamedTemporaryFile(prefix="daisychain-", + suffix=".log") + self.lognames.append(self.logfile.name) + + # Now create the daisy-chain process. + cmd = ["./rbb_daisychain.py", "0"] + \ + [str(spike.port) for spike in spikes] + self.logfile.write(("+ %s\n" % cmd).encode()) + self.logfile.flush() + self.process = subprocess.Popen(cmd, stdin=subprocess.PIPE, + stdout=self.logfile, stderr=self.logfile) + + self.port = None + for _ in range(30): + m = re.search(r"Listening on port (\d+).", + open(self.lognames[-1]).read()) + if m: + self.port = int(m.group(1)) + break + time.sleep(0.11) + if not self.port: + print_log(self.lognames[-1]) + raise Exception("Didn't get daisy chain message about which port " + "it's listening on.") + + os.environ['REMOTE_BITBANG_HOST'] = 'localhost' + os.environ['REMOTE_BITBANG_PORT'] = str(self.port) + + def __del__(self): + if self.process: + try: + self.process.kill() + self.process.wait() + except OSError: + pass + class VcsSim: logfile = tempfile.NamedTemporaryFile(prefix='simv', suffix='.log') logname = logfile.name @@ -519,11 +559,12 @@ class Gdb: 11, 149, 107, 163, 73, 47, 43, 173, 7, 109, 101, 103, 191, 2, 139, 97, 193, 157, 3, 29, 79, 113, 5, 89, 19, 37, 71, 179, 59, 137, 53) - def __init__(self, ports, + def __init__(self, target, ports, cmd="riscv64-unknown-elf-gdb", timeout=60, binary=None): assert ports + self.target = target self.ports = ports self.cmd = cmd self.timeout = timeout @@ -551,14 +592,16 @@ class Gdb: for port, child in zip(self.ports, self.children): self.select_child(child) self.wait() - self.command("set style enabled off") - self.command("set confirm off") - self.command("set width 0") - self.command("set height 0") + self.command("set style enabled off", reset_delays=None) + self.command("set confirm off", reset_delays=None) + self.command("set width 0", reset_delays=None) + self.command("set height 0", reset_delays=None) # Force consistency. - self.command("set print entry-values no") - self.command("set remotetimeout %d" % self.timeout) - self.command("target extended-remote localhost:%d" % port, ops=10) + self.command("set print entry-values no", reset_delays=None) + self.command("set remotetimeout %d" % self.timeout, + reset_delays=None) + self.command("target extended-remote localhost:%d" % port, ops=10, + reset_delays=None) if self.binary: self.command("file %s" % self.binary) threads = self.threads() @@ -635,6 +678,20 @@ class Gdb: self.select_child(child) self.command(command) + def system_command(self, command, ops=20): + """Execute this command on every unique system that we control.""" + done = set() + output = "" + with PrivateState(self): + for i, child in enumerate(self.children): + self.select_child(child) + if self.target.harts[i].system in done: + self.command("set $pc=_start") + else: + output += self.command(command, ops=ops) + done.add(self.target.harts[i].system) + return output + def c(self, wait=True, sync=True, checkOutput=True, ops=20): """ Dumb c command. @@ -742,10 +799,10 @@ class Gdb: return output def load(self): - output = self.command("load", ops=1000) + output = self.system_command("load", ops=1000) assert "failed" not in output assert "Transfer rate" in output - output = self.command("compare-sections", ops=1000) + output = self.system_command("compare-sections", ops=1000) assert "matched" in output assert "MIS" not in output @@ -986,7 +1043,7 @@ class BaseTest: self.compile() self.target_process = self.target.create() if self.target_process: - self.logs.append(self.target_process.logname) + self.logs += self.target_process.lognames try: self.server = self.target.server(self) self.logs.append(self.server.logname) @@ -1076,10 +1133,10 @@ class GdbTest(BaseTest): BaseTest.classSetup(self) if gdb_cmd: - self.gdb = Gdb(self.server.gdb_ports, gdb_cmd, + self.gdb = Gdb(self.target, self.server.gdb_ports, gdb_cmd, timeout=self.target.timeout_sec, binary=self.binary) else: - self.gdb = Gdb(self.server.gdb_ports, + self.gdb = Gdb(self.target, self.server.gdb_ports, timeout=self.target.timeout_sec, binary=self.binary) self.logs += self.gdb.lognames() -- cgit v1.1