Merge pull request #79 from riscv/multigdb

Multigdb support

13 files changed, 236 insertions, 96 deletions

diff --git a/debug/Makefile b/debug/Makefile
index 33988dd..9f7cb2e 100644
--- a/debug/Makefile
+++ b/debug/Makefile
@@ -6,7 +6,7 @@ GDBSERVER_PY = $(src_dir)/gdbserver.py
 
 default: spike$(XLEN)-2
 
-all:	pylint spike32 spike64 spike32-2 spike64-2
+all:	pylint spike32 spike32-2 spike32-2-rtos spike64 spike64-2 spike64-2-rtos
 
 pylint:
 	pylint --rcfile=pylint.rc `git ls-files '*.py'`
diff --git a/debug/gdbserver.py b/debug/gdbserver.py
index 135dab8..924f42a 100755
--- a/debug/gdbserver.py
+++ b/debug/gdbserver.py
@@ -363,7 +363,7 @@ class Hwbp2(DebugTest):
         self.exit()
 
 class TooManyHwbp(DebugTest):
-    def run(self):
+    def test(self):
         for i in range(30):
             self.gdb.hbreak("*rot13 + %d" % (i * 4))
 
@@ -476,21 +476,27 @@ class MulticoreRegTest(GdbTest):
 
     def test(self):
         # Run to main
+        # Hart 0 is the first to be resumed, so we have to set the breakpoint
+        # there. gdb won't actually set the breakpoint until we tell it to
+        # resume.
+        self.gdb.select_hart(self.target.harts[0])
         self.gdb.b("main")
-        self.gdb.c()
-        for t in self.gdb.threads():
-            assertIn("main", t.frame)
+        self.gdb.c_all()
+        for hart in self.target.harts:
+            self.gdb.select_hart(hart)
+            assertIn("main", self.gdb.where())
+        self.gdb.select_hart(self.target.harts[0])
         self.gdb.command("delete breakpoints")
 
         # Run through the entire loop.
         self.gdb.b("main_end")
-        self.gdb.c()
+        self.gdb.c_all()
 
         hart_ids = []
-        for t in self.gdb.threads():
-            assertIn("main_end", t.frame)
+        for hart in self.target.harts:
+            self.gdb.select_hart(hart)
+            assertIn("main_end", self.gdb.where())
             # Check register values.
-            self.gdb.thread(t)
             hart_id = self.gdb.p("$x1")
             assertNotIn(hart_id, hart_ids)
             hart_ids.append(hart_id)
@@ -505,12 +511,11 @@ class MulticoreRegTest(GdbTest):
             self.gdb.select_hart(hart)
             self.gdb.p("$x1=0x%x" % (hart.index * 0x800))
             self.gdb.p("$pc=main_post_csrr")
-        self.gdb.c()
-        for t in self.gdb.threads():
-            assertIn("main_end", t.frame)
+        self.gdb.c_all()
         for hart in self.target.harts:
-            # Check register values.
             self.gdb.select_hart(hart)
+            assertIn("main", self.gdb.where())
+            # Check register values.
             for n in range(1, 32):
                 value = self.gdb.p("$x%d" % n)
                 assertEqual(value, hart.index * 0x800 + n - 1)
diff --git a/debug/targets.py b/debug/targets.py
index d09b576..624eb71 100644
--- a/debug/targets.py
+++ b/debug/targets.py
@@ -96,6 +96,8 @@ class Target(object):
                 self.openocd_config_path)
         for i, hart in enumerate(self.harts):
             hart.index = i
+            if not hasattr(hart, 'id'):
+                hart.id = i
             if not hart.name:
                 hart.name = "%s-%d" % (self.name, i)
             # Default link script to <name>.lds
diff --git a/debug/targets/RISC-V/spike.cfg b/debug/targets/RISC-V/spike-1.cfg
index fc20b53..fc20b53 100644
--- a/debug/targets/RISC-V/spike.cfg
+++ b/debug/targets/RISC-V/spike-1.cfg
diff --git a/debug/targets/RISC-V/spike-2.cfg b/debug/targets/RISC-V/spike-2.cfg
new file mode 100644
index 0000000..17526ec
--- /dev/null
+++ b/debug/targets/RISC-V/spike-2.cfg
@@ -0,0 +1,19 @@
+# Connect to a mult-icore RISC-V target, exposing each hart as a thread.
+adapter_khz     10000
+
+interface remote_bitbang
+remote_bitbang_host $::env(REMOTE_BITBANG_HOST)
+remote_bitbang_port $::env(REMOTE_BITBANG_PORT)
+
+set _CHIPNAME riscv
+jtag newtap $_CHIPNAME cpu -irlen 5 -expected-id 0x10e31913
+
+set _TARGETNAME_0 $_CHIPNAME.cpu0
+set _TARGETNAME_1 $_CHIPNAME.cpu1
+target create $_TARGETNAME_0 riscv -chain-position $_CHIPNAME.cpu -coreid 0
+target create $_TARGETNAME_1 riscv -chain-position $_CHIPNAME.cpu -coreid 1
+
+gdb_report_data_abort enable
+
+init
+reset halt
diff --git a/debug/targets/RISC-V/spike-rtos.cfg b/debug/targets/RISC-V/spike-rtos.cfg
index 9b1841c..799e3cb 100644
--- a/debug/targets/RISC-V/spike-rtos.cfg
+++ b/debug/targets/RISC-V/spike-rtos.cfg
@@ -1,3 +1,4 @@
+# Connect to a mult-icore RISC-V target, exposing each hart as a thread.
 adapter_khz     10000
 
 interface remote_bitbang
diff --git a/debug/targets/RISC-V/spike32-2-rtos.py b/debug/targets/RISC-V/spike32-2-rtos.py
new file mode 100644
index 0000000..a7b9a1c
--- /dev/null
+++ b/debug/targets/RISC-V/spike32-2-rtos.py
@@ -0,0 +1,12 @@
+import targets
+import testlib
+
+import spike32  # pylint: disable=import-error
+
+class spike32_2(targets.Target):
+    harts = [spike32.spike32_hart(), spike32.spike32_hart()]
+    openocd_config_path = "spike-rtos.cfg"
+    timeout_sec = 30
+
+    def create(self):
+        return testlib.Spike(self)
diff --git a/debug/targets/RISC-V/spike32-2.py b/debug/targets/RISC-V/spike32-2.py
index a7b9a1c..719009d 100644
--- a/debug/targets/RISC-V/spike32-2.py
+++ b/debug/targets/RISC-V/spike32-2.py
@@ -5,7 +5,7 @@ import spike32  # pylint: disable=import-error
 
 class spike32_2(targets.Target):
     harts = [spike32.spike32_hart(), spike32.spike32_hart()]
-    openocd_config_path = "spike-rtos.cfg"
+    openocd_config_path = "spike-2.cfg"
     timeout_sec = 30
 
     def create(self):
diff --git a/debug/targets/RISC-V/spike32.py b/debug/targets/RISC-V/spike32.py
index bcb5892..809463c 100644
--- a/debug/targets/RISC-V/spike32.py
+++ b/debug/targets/RISC-V/spike32.py
@@ -11,7 +11,7 @@ class spike32_hart(targets.Hart):
 
 class spike32(targets.Target):
     harts = [spike32_hart()]
-    openocd_config_path = "spike.cfg"
+    openocd_config_path = "spike-1.cfg"
     timeout_sec = 30
 
     def create(self):
diff --git a/debug/targets/RISC-V/spike64-2-rtos.py b/debug/targets/RISC-V/spike64-2-rtos.py
new file mode 100644
index 0000000..d65d2ab
--- /dev/null
+++ b/debug/targets/RISC-V/spike64-2-rtos.py
@@ -0,0 +1,12 @@
+import targets
+import testlib
+
+import spike64  # pylint: disable=import-error
+
+class spike64_2_rtos(targets.Target):
+    harts = [spike64.spike64_hart(), spike64.spike64_hart()]
+    openocd_config_path = "spike-rtos.cfg"
+    timeout_sec = 30
+
+    def create(self):
+        return testlib.Spike(self)
diff --git a/debug/targets/RISC-V/spike64-2.py b/debug/targets/RISC-V/spike64-2.py
index 4f6f1ff..709ebbe 100644
--- a/debug/targets/RISC-V/spike64-2.py
+++ b/debug/targets/RISC-V/spike64-2.py
@@ -5,7 +5,7 @@ import spike64  # pylint: disable=import-error
 
 class spike64_2(targets.Target):
     harts = [spike64.spike64_hart(), spike64.spike64_hart()]
-    openocd_config_path = "spike-rtos.cfg"
+    openocd_config_path = "spike-2.cfg"
     timeout_sec = 30
 
     def create(self):
diff --git a/debug/targets/RISC-V/spike64.py b/debug/targets/RISC-V/spike64.py
index 9c37f87..2cd67a5 100644
--- a/debug/targets/RISC-V/spike64.py
+++ b/debug/targets/RISC-V/spike64.py
@@ -11,7 +11,7 @@ class spike64_hart(targets.Hart):
 
 class spike64(targets.Target):
     harts = [spike64_hart()]
-    openocd_config_path = "spike.cfg"
+    openocd_config_path = "spike-1.cfg"
     timeout_sec = 30
 
     def create(self):
diff --git a/debug/testlib.py b/debug/testlib.py
index 94694a0..66b7b38 100644
--- a/debug/testlib.py
+++ b/debug/testlib.py
@@ -1,4 +1,5 @@
 import collections
+import os
 import os.path
 import random
 import re
@@ -50,18 +51,7 @@ def compile(args, xlen=32): # pylint: disable=redefined-builtin
         header("")
         raise Exception("Compile failed!")
 
-def unused_port():
-    # http://stackoverflow.com/questions/2838244/get-open-tcp-port-in-python/2838309#2838309
-    import socket
-    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-    s.bind(("", 0))
-    port = s.getsockname()[1]
-    s.close()
-    return port
-
 class Spike(object):
-    logname = "spike-%d.log" % os.getpid()
-
     def __init__(self, target, halted=False, timeout=None, with_jtag_gdb=True):
         """Launch spike. Return tuple of its process and the port it's running
         on."""
@@ -77,11 +67,13 @@ class Spike(object):
                 "programs/checksum.c", "programs/tiny-malloc.c",
                 "programs/infinite_loop.S", "-DDEFINE_MALLOC", "-DDEFINE_FREE")
         cmd.append(self.infinite_loop)
-        logfile = open(self.logname, "w")
-        logfile.write("+ %s\n" % " ".join(cmd))
-        logfile.flush()
+        self.logfile = tempfile.NamedTemporaryFile(prefix="spike-",
+                suffix=".log")
+        self.logname = self.logfile.name
+        self.logfile.write("+ %s\n" % " ".join(cmd))
+        self.logfile.flush()
         self.process = subprocess.Popen(cmd, stdin=subprocess.PIPE,
-                stdout=logfile, stderr=logfile)
+                stdout=self.logfile, stderr=self.logfile)
 
         if with_jtag_gdb:
             self.port = None
@@ -232,8 +224,7 @@ class Openocd(object):
         logfile.write("+ %s\n" % " ".join(cmd))
         logfile.flush()
 
-        self.ports = []
-        self.port = None
+        self.gdb_ports = []
         self.process = self.start(cmd, logfile)
 
     def start(self, cmd, logfile):
@@ -247,31 +238,32 @@ class Openocd(object):
             # attempt too early.
             start = time.time()
             messaged = False
+            fd = open(Openocd.logname, "r")
             while True:
-                log = open(Openocd.logname).read()
+                line = fd.readline()
+                if not line:
+                    if not process.poll() is None:
+                        raise Exception("OpenOCD exited early.")
+                    time.sleep(0.1)
+                    continue
+
                 m = re.search(r"Listening on port (\d+) for gdb connections",
-                        log)
+                        line)
                 if m:
-                    if not self.ports:
-                        self.port = int(m.group(1))
-                    self.ports.append(int(m.group(1)))
+                    self.gdb_ports.append(int(m.group(1)))
 
-                if "telnet server disabled" in log:
-                    break
+                if "telnet server disabled" in line:
+                    return process
 
-                if not process.poll() is None:
-                    raise Exception(
-                            "OpenOCD exited before completing riscv_examine()")
                 if not messaged and time.time() - start > 1:
                     messaged = True
                     print "Waiting for OpenOCD to start..."
                 if (time.time() - start) > self.timeout:
-                    raise Exception("ERROR: Timed out waiting for OpenOCD to "
+                    raise Exception("Timed out waiting for OpenOCD to "
                             "listen for gdb")
-            return process
+
         except Exception:
-            header("OpenOCD log")
-            sys.stdout.write(log)
+            print_log(Openocd.logname)
             raise
 
     def __del__(self):
@@ -312,42 +304,109 @@ class CannotAccess(Exception):
         Exception.__init__(self)
         self.address = address
 
-Thread = collections.namedtuple('Thread', ('id', 'target_id', 'name',
-    'frame'))
+Thread = collections.namedtuple('Thread', ('id', 'description', 'target_id',
+    'name', 'frame'))
 
 class Gdb(object):
-    logfile = tempfile.NamedTemporaryFile(prefix="gdb", suffix=".log")
-    logname = logfile.name
-    print "GDB Temporary Log File: %s" % logname
-
-    def __init__(self,
-            cmd=os.path.expandvars("$RISCV/bin/riscv64-unknown-elf-gdb")):
-        self.child = pexpect.spawn(cmd)
-        self.child.logfile = open(self.logname, "w")
-        self.child.logfile.write("+ %s\n" % cmd)
-        self.wait()
-        self.command("set confirm off")
-        self.command("set width 0")
-        self.command("set height 0")
-        # Force consistency.
-        self.command("set print entry-values no")
+    """A single gdb class which can interact with one or more gdb instances."""
+
+    # pylint: disable=too-many-public-methods
+
+    def __init__(self, ports,
+            cmd=os.path.expandvars("$RISCV/bin/riscv64-unknown-elf-gdb"),
+            binary=None):
+        assert ports
+
+        self.stack = []
+
+        self.logfiles = []
+        self.children = []
+        for port in ports:
+            logfile = tempfile.NamedTemporaryFile(prefix="gdb@%d-" % port,
+                    suffix=".log")
+            self.logfiles.append(logfile)
+            child = pexpect.spawn(cmd)
+            child.logfile = logfile
+            child.logfile.write("+ %s\n" % cmd)
+            self.children.append(child)
+        self.active_child = self.children[0]
+
+        self.harts = {}
+        for port, child in zip(ports, self.children):
+            self.select_child(child)
+            self.wait()
+            self.command("set confirm off")
+            self.command("set width 0")
+            self.command("set height 0")
+            # Force consistency.
+            self.command("set print entry-values no")
+            self.command("target extended-remote localhost:%d" % port)
+            if binary:
+                self.command("file %s" % binary)
+            threads = self.threads()
+            for t in threads:
+                hartid = None
+                if t.name:
+                    m = re.search(r"Hart (\d+)", t.name)
+                    if m:
+                        hartid = int(m.group(1))
+                if hartid is None:
+                    if self.harts:
+                        hartid = max(self.harts) + 1
+                    else:
+                        hartid = 0
+                self.harts[hartid] = (child, t)
+
+    def __del__(self):
+        for child in self.children:
+            del child
+
+    def lognames(self):
+        return [logfile.name for logfile in self.logfiles]
+
+    def select_child(self, child):
+        self.active_child = child
 
     def select_hart(self, hart):
-        output = self.command("thread %d" % (hart.index + 1))
+        child, thread = self.harts[hart.id]
+        self.select_child(child)
+        output = self.command("thread %s" % thread.id)
         assert "Unknown" not in output
 
+    def push_state(self):
+        self.stack.append({
+            'active_child': self.active_child
+            })
+
+    def pop_state(self):
+        state = self.stack.pop()
+        self.active_child = state['active_child']
+
     def wait(self):
         """Wait for prompt."""
-        self.child.expect(r"\(gdb\)")
+        self.active_child.expect(r"\(gdb\)")
 
     def command(self, command, timeout=6000):
         """timeout is in seconds"""
-        self.child.sendline(command)
-        self.child.expect("\n", timeout=timeout)
-        self.child.expect(r"\(gdb\)", timeout=timeout)
-        return self.child.before.strip()
+        self.active_child.sendline(command)
+        self.active_child.expect("\n", timeout=timeout)
+        self.active_child.expect(r"\(gdb\)", timeout=timeout)
+        return self.active_child.before.strip()
+
+    def global_command(self, command):
+        """Execute this command on every gdb that we control."""
+        with PrivateState(self):
+            for child in self.children:
+                self.select_child(child)
+                self.command(command)
 
     def c(self, wait=True, timeout=-1, async=False):
+        """
+        Dumb c command.
+        In RTOS mode, gdb will resume all harts.
+        In multi-gdb mode, this command will just go to the current gdb, so
+        will only resume one hart.
+        """
         if async:
             async = "&"
         else:
@@ -357,13 +416,24 @@ class Gdb(object):
             assert "Continuing" in output
             return output
         else:
-            self.child.sendline("c%s" % async)
-            self.child.expect("Continuing")
+            self.active_child.sendline("c%s" % async)
+            self.active_child.expect("Continuing")
+
+    def c_all(self):
+        """Resume every hart."""
+        with PrivateState(self):
+            for child in self.children:
+                child.sendline("c")
+                child.expect("Continuing")
+
+            # Now wait for them all to halt
+            for child in self.children:
+                child.expect(r"\(gdb\)")
 
     def interrupt(self):
-        self.child.send("\003")
-        self.child.expect(r"\(gdb\)", timeout=6000)
-        return self.child.before.strip()
+        self.active_child.send("\003")
+        self.active_child.expect(r"\(gdb\)", timeout=6000)
+        return self.active_child.before.strip()
 
     def x(self, address, size='w'):
         output = self.command("x/%s %s" % (size, address))
@@ -426,17 +496,32 @@ class Gdb(object):
         threads = []
         for line in output.splitlines():
             m = re.match(
-                    r"[\s\*]*(\d+)\s*Thread (\d+)\s*\(Name: ([^\)]+)\s*(.*)",
-                    line)
+                    r"[\s\*]*(\d+)\s*"
+                    r"(Remote target|Thread (\d+)\s*\(Name: ([^\)]+))"
+                    r"\s*(.*)", line)
             if m:
                 threads.append(Thread(*m.groups()))
-        if not threads:
-            threads.append(Thread('1', '1', 'Default', '???'))
+        assert threads
+        #>>>if not threads:
+        #>>>    threads.append(Thread('1', '1', 'Default', '???'))
         return threads
 
     def thread(self, thread):
         return self.command("thread %s" % thread.id)
 
+    def where(self):
+        return self.command("where 1")
+
+class PrivateState(object):
+    def __init__(self, gdb):
+        self.gdb = gdb
+
+    def __enter__(self):
+        self.gdb.push_state()
+
+    def __exit__(self, _type, _value, _traceback):
+        self.gdb.pop_state()
+
 def run_all_tests(module, target, parsed):
     if not os.path.exists(parsed.logs):
         os.makedirs(parsed.logs)
@@ -480,7 +565,7 @@ def run_tests(parsed, target, todo):
         log_name = os.path.join(parsed.logs, "%s-%s-%s.log" %
                 (time.strftime("%Y%m%d-%H%M%S"), type(target).__name__, name))
         log_fd = open(log_name, 'w')
-        print "Running %s > %s ..." % (name, log_name),
+        print "[%s] Starting > %s" % (name, log_name)
         instance = definition(target, hart)
         sys.stdout.flush()
         log_fd.write("Test: %s\n" % name)
@@ -489,12 +574,12 @@ def run_tests(parsed, target, todo):
         real_stdout = sys.stdout
         sys.stdout = log_fd
         try:
-            result = instance.run()
+            result = instance.run(real_stdout)
             log_fd.write("Result: %s\n" % result)
         finally:
             sys.stdout = real_stdout
             log_fd.write("Time elapsed: %.2fs\n" % (time.time() - start))
-        print "%s in %.2fs" % (result, time.time() - start)
+        print "[%s] %s in %.2fs" % (name, result, time.time() - start)
         if result not in good_results and parsed.print_failures:
             sys.stdout.write(open(log_name).read())
         sys.stdout.flush()
@@ -600,7 +685,7 @@ class BaseTest(object):
     def postMortem(self):
         pass
 
-    def run(self):
+    def run(self, real_stdout):
         """
         If compile_args is set, compile a program and set self.binary.
 
@@ -619,6 +704,8 @@ class BaseTest(object):
 
         try:
             self.classSetup()
+            real_stdout.write("[%s] Temporary logs: %s\n" % (
+                type(self).__name__, ", ".join(self.logs)))
             self.setup()
             result = self.test()    # pylint: disable=no-member
         except TestNotApplicable:
@@ -633,7 +720,12 @@ class BaseTest(object):
                 print e.message
             header("Traceback")
             traceback.print_exc(file=sys.stdout)
-            self.postMortem()
+            try:
+                self.postMortem()
+            except Exception as e:  # pylint: disable=broad-except
+                header("postMortem Exception")
+                print e
+                traceback.print_exc(file=sys.stdout)
             return result
 
         finally:
@@ -656,25 +748,22 @@ class GdbTest(BaseTest):
         BaseTest.classSetup(self)
 
         if gdb_cmd:
-            self.gdb = Gdb(gdb_cmd)
+            self.gdb = Gdb(self.server.gdb_ports, gdb_cmd, binary=self.binary)
         else:
-            self.gdb = Gdb()
+            self.gdb = Gdb(self.server.gdb_ports, binary=self.binary)
 
-        self.logs.append(self.gdb.logname)
+        self.logs += self.gdb.lognames()
 
-        if self.binary:
-            self.gdb.command("file %s" % self.binary)
         if self.target:
-            self.gdb.command("set arch riscv:rv%d" % self.hart.xlen)
-            self.gdb.command("set remotetimeout %d" % self.target.timeout_sec)
-        if self.server.port:
-            self.gdb.command(
-                    "target extended-remote localhost:%d" % self.server.port)
-            self.gdb.select_hart(self.hart)
+            self.gdb.global_command("set arch riscv:rv%d" % self.hart.xlen)
+            self.gdb.global_command("set remotetimeout %d" %
+                    self.target.timeout_sec)
 
         for cmd in self.target.gdb_setup:
             self.gdb.command(cmd)
 
+        self.gdb.select_hart(self.hart)
+
         # FIXME: OpenOCD doesn't handle PRIV now
         #self.gdb.p("$priv=3")