Merge branch 'master' into interruptsinterrupts

17 files changed, 410 insertions, 204 deletions

diff --git a/benchmarks/Makefile b/benchmarks/Makefile
index d1c2362..fb0d297 100644
--- a/benchmarks/Makefile
+++ b/benchmarks/Makefile
@@ -39,7 +39,7 @@ RISCV_PREFIX ?= riscv$(XLEN)-unknown-elf-
 RISCV_GCC ?= $(RISCV_PREFIX)gcc
 RISCV_GCC_OPTS ?= -DPREALLOCATE=1 -mcmodel=medany -static -std=gnu99 -O2 -ffast-math -fno-common -fno-builtin-printf
 RISCV_LINK ?= $(RISCV_GCC) -T $(src_dir)/common/test.ld $(incs)
-RISCV_LINK_OPTS ?= -static -nostdlib -nostartfiles -lgcc -T $(src_dir)/common/test.ld
+RISCV_LINK_OPTS ?= -static -nostdlib -nostartfiles -lm -lgcc -T $(src_dir)/common/test.ld
 RISCV_OBJDUMP ?= $(RISCV_PREFIX)objdump --disassemble-all --disassemble-zeroes --section=.text --section=.text.startup --section=.data
 RISCV_SIM ?= spike --isa=rv$(XLEN)gc
 
@@ -48,7 +48,7 @@ objs  :=
 
 define compile_template
 $(1).riscv: $(wildcard $(src_dir)/$(1)/*) $(wildcard $(src_dir)/common/*)
-	$$(RISCV_GCC) $$(incs) $$(RISCV_GCC_OPTS) $$(RISCV_LINK_OPTS) -o $$@ $(wildcard $(src_dir)/$(1)/*.c) $(wildcard $(src_dir)/common/*.c) $(wildcard $(src_dir)/common/*.S)
+	$$(RISCV_GCC) $$(incs) $$(RISCV_GCC_OPTS) -o $$@ $(wildcard $(src_dir)/$(1)/*.c) $(wildcard $(src_dir)/common/*.c) $(wildcard $(src_dir)/common/*.S) $$(RISCV_LINK_OPTS)
 endef
 
 $(foreach bmark,$(bmarks),$(eval $(call compile_template,$(bmark))))
diff --git a/debug/Makefile b/debug/Makefile
index 33988dd..9f7cb2e 100644
--- a/debug/Makefile
+++ b/debug/Makefile
@@ -6,7 +6,7 @@ GDBSERVER_PY = $(src_dir)/gdbserver.py
 
 default: spike$(XLEN)-2
 
-all:	pylint spike32 spike64 spike32-2 spike64-2
+all:	pylint spike32 spike32-2 spike32-2-rtos spike64 spike64-2 spike64-2-rtos
 
 pylint:
 	pylint --rcfile=pylint.rc `git ls-files '*.py'`
diff --git a/debug/gdbserver.py b/debug/gdbserver.py
index 9fedbca..d2c4fe9 100755
--- a/debug/gdbserver.py
+++ b/debug/gdbserver.py
@@ -12,7 +12,7 @@ import targets
 import testlib
 from testlib import assertEqual, assertNotEqual, assertIn, assertNotIn
 from testlib import assertGreater, assertRegexpMatches, assertLess
-from testlib import GdbTest, GdbSingleHartTest, TestFailed
+from testlib import GdbTest, GdbSingleHartTest, TestFailed, assertTrue
 
 MSTATUS_UIE = 0x00000001
 MSTATUS_SIE = 0x00000002
@@ -217,7 +217,7 @@ class InstantHaltTest(GdbTest):
             self.gdb.thread(t)
             pcs.append(self.gdb.p("$pc"))
         for pc in pcs:
-            assertEqual(self.hart.reset_vector, pc)
+            assertIn(pc, self.hart.reset_vectors)
         # mcycle and minstret have no defined reset value.
         mstatus = self.gdb.p("$mstatus")
         assertEqual(mstatus & (MSTATUS_MIE | MSTATUS_MPRV |
@@ -363,7 +363,7 @@ class Hwbp2(DebugTest):
         self.exit()
 
 class TooManyHwbp(DebugTest):
-    def run(self):
+    def test(self):
         for i in range(30):
             self.gdb.hbreak("*rot13 + %d" % (i * 4))
 
@@ -476,21 +476,27 @@ class MulticoreRegTest(GdbTest):
 
     def test(self):
         # Run to main
+        # Hart 0 is the first to be resumed, so we have to set the breakpoint
+        # there. gdb won't actually set the breakpoint until we tell it to
+        # resume.
+        self.gdb.select_hart(self.target.harts[0])
         self.gdb.b("main")
-        self.gdb.c()
-        for t in self.gdb.threads():
-            assertIn("main", t.frame)
+        self.gdb.c_all()
+        for hart in self.target.harts:
+            self.gdb.select_hart(hart)
+            assertIn("main", self.gdb.where())
+        self.gdb.select_hart(self.target.harts[0])
         self.gdb.command("delete breakpoints")
 
         # Run through the entire loop.
         self.gdb.b("main_end")
-        self.gdb.c()
+        self.gdb.c_all()
 
         hart_ids = []
-        for t in self.gdb.threads():
-            assertIn("main_end", t.frame)
+        for hart in self.target.harts:
+            self.gdb.select_hart(hart)
+            assertIn("main_end", self.gdb.where())
             # Check register values.
-            self.gdb.thread(t)
             hart_id = self.gdb.p("$x1")
             assertNotIn(hart_id, hart_ids)
             hart_ids.append(hart_id)
@@ -505,12 +511,11 @@ class MulticoreRegTest(GdbTest):
             self.gdb.select_hart(hart)
             self.gdb.p("$x1=0x%x" % (hart.index * 0x800))
             self.gdb.p("$pc=main_post_csrr")
-        self.gdb.c()
-        for t in self.gdb.threads():
-            assertIn("main_end", t.frame)
+        self.gdb.c_all()
         for hart in self.target.harts:
-            # Check register values.
             self.gdb.select_hart(hart)
+            assertIn("main", self.gdb.where())
+            # Check register values.
             for n in range(1, 32):
                 value = self.gdb.p("$x%d" % n)
                 assertEqual(value, hart.index * 0x800 + n - 1)
@@ -529,14 +534,21 @@ class MulticoreRunHaltStepiTest(GdbTest):
 
     def test(self):
         previous_hart_count = [0 for h in self.target.harts]
+        previous_interrupt_count = [0 for h in self.target.harts]
         for _ in range(10):
             self.gdb.c(wait=False)
-            time.sleep(1)
+            time.sleep(2)
             self.gdb.interrupt()
+            self.gdb.p("$mie")
+            self.gdb.p("$mip")
+            self.gdb.p("$mstatus")
+            self.gdb.p("$priv")
             self.gdb.p("buf", fmt="")
             hart_count = self.gdb.p("hart_count")
+            interrupt_count = self.gdb.p("interrupt_count")
             for i, h in enumerate(self.target.harts):
                 assertGreater(hart_count[i], previous_hart_count[i])
+                assertGreater(interrupt_count[i], previous_interrupt_count[i])
                 self.gdb.select_hart(h)
                 pc = self.gdb.p("$pc")
                 self.gdb.stepi()
@@ -762,7 +774,6 @@ class DownloadTest(GdbTest):
         assertEqual(self.gdb.p("status"), self.crc)
         os.unlink(self.download_c.name)
 
-# FIXME: PRIV isn't implemented in the current OpenOCD
 #class MprvTest(GdbTest):
 #    compile_args = ("programs/mprv.S", )
 #    def setup(self):
@@ -775,56 +786,56 @@ class DownloadTest(GdbTest):
 #        self.gdb.interrupt()
 #        output = self.gdb.command("p/x *(int*)(((char*)&data)-0x80000000)")
 #        assertIn("0xbead", output)
-#
-#class PrivTest(GdbTest):
-#    compile_args = ("programs/priv.S", )
-#    def setup(self):
-#        # pylint: disable=attribute-defined-outside-init
-#        self.gdb.load()
-#
-#        misa = self.hart.misa
-#        self.supported = set()
-#        if misa & (1<<20):
-#            self.supported.add(0)
-#        if misa & (1<<18):
-#            self.supported.add(1)
-#        if misa & (1<<7):
-#            self.supported.add(2)
-#        self.supported.add(3)
-#
-#class PrivRw(PrivTest):
-#    def test(self):
-#        """Test reading/writing priv."""
-#        for privilege in range(4):
-#            self.gdb.p("$priv=%d" % privilege)
-#            self.gdb.stepi()
-#            actual = self.gdb.p("$priv")
-#            assertIn(actual, self.supported)
-#            if privilege in self.supported:
-#                assertEqual(actual, privilege)
-#
-#class PrivChange(PrivTest):
-#    def test(self):
-#        """Test that the core's privilege level actually changes."""
-#
-#        if 0 not in self.supported:
-#            return 'not_applicable'
-#
-#        self.gdb.b("main")
-#        self.gdb.c()
-#
-#        # Machine mode
-#        self.gdb.p("$priv=3")
-#        main_address = self.gdb.p("$pc")
-#        self.gdb.stepi()
-#        assertEqual("%x" % self.gdb.p("$pc"), "%x" % (main_address+4))
-#
-#        # User mode
-#        self.gdb.p("$priv=0")
-#        self.gdb.stepi()
-#        # Should have taken an exception, so be nowhere near main.
-#        pc = self.gdb.p("$pc")
-#        assertTrue(pc < main_address or pc > main_address + 0x100)
+
+class PrivTest(GdbTest):
+    compile_args = ("programs/priv.S", )
+    def setup(self):
+        # pylint: disable=attribute-defined-outside-init
+        self.gdb.load()
+
+        misa = self.hart.misa
+        self.supported = set()
+        if misa & (1<<20):
+            self.supported.add(0)
+        if misa & (1<<18):
+            self.supported.add(1)
+        if misa & (1<<7):
+            self.supported.add(2)
+        self.supported.add(3)
+
+class PrivRw(PrivTest):
+    def test(self):
+        """Test reading/writing priv."""
+        for privilege in range(4):
+            self.gdb.p("$priv=%d" % privilege)
+            self.gdb.stepi()
+            actual = self.gdb.p("$priv")
+            assertIn(actual, self.supported)
+            if privilege in self.supported:
+                assertEqual(actual, privilege)
+
+class PrivChange(PrivTest):
+    def test(self):
+        """Test that the core's privilege level actually changes."""
+
+        if 0 not in self.supported:
+            return 'not_applicable'
+
+        self.gdb.b("main")
+        self.gdb.c()
+
+        # Machine mode
+        self.gdb.p("$priv=3")
+        main_address = self.gdb.p("$pc")
+        self.gdb.stepi()
+        assertEqual("%x" % self.gdb.p("$pc"), "%x" % (main_address+4))
+
+        # User mode
+        self.gdb.p("$priv=0")
+        self.gdb.stepi()
+        # Should have taken an exception, so be nowhere near main.
+        pc = self.gdb.p("$pc")
+        assertTrue(pc < main_address or pc > main_address + 0x100)
 
 parsed = None
 def main():
diff --git a/debug/programs/entry.S b/debug/programs/entry.S
index 97b62a3..35c233e 100755
--- a/debug/programs/entry.S
+++ b/debug/programs/entry.S
@@ -1,6 +1,6 @@
 #include "encoding.h"
 
-#define STACK_SIZE (74 * XLEN / 8)
+#define STACK_SIZE (90 * XLEN / 8)
 
 #if XLEN == 64
 # define LREG ld
diff --git a/debug/programs/init.h b/debug/programs/init.h
new file mode 100644
index 0000000..9aaa398
--- /dev/null
+++ b/debug/programs/init.h
@@ -0,0 +1,20 @@
+#ifndef INIT_H
+#define INIT_H
+
+#define MTIME           (*(volatile long long *)(0x02000000 + 0xbff8))
+#define MTIMECMP        ((volatile long long *)(0x02000000 + 0x4000))
+
+#define csr_read(csr)                                   \
+({                                                      \
+    register unsigned long __v;                         \
+    __asm__ __volatile__ ("csrr %0, " #csr              \
+                  : "=r" (__v));                        \
+    __v;                                                \
+})
+
+typedef void* (*trap_handler_t)(unsigned hartid, unsigned mcause, void *mepc,
+        void *sp);
+void set_trap_handler(trap_handler_t handler);
+void enable_timer_interrupts();
+
+#endif
diff --git a/debug/programs/multicore.c b/debug/programs/multicore.c
index d7dd845..272baea 100644
--- a/debug/programs/multicore.c
+++ b/debug/programs/multicore.c
@@ -1,5 +1,7 @@
 #include <stdint.h>
 
+#include "init.h"
+
 typedef struct {
     int counter;
 } atomic_t;
@@ -15,14 +17,6 @@ static inline int atomic_xchg(atomic_t *v, int n)
     return c;
 }
 
-#define csr_read(csr)                                   \
-({                                                      \
-    register unsigned long __v;                         \
-    __asm__ __volatile__ ("csrr %0, " #csr              \
-                  : "=r" (__v));                        \
-    __v;                                                \
-})
-
 static inline void mb(void)
 {
     __asm__ __volatile__ ("fence");
@@ -44,19 +38,31 @@ void put_lock(atomic_t *lock)
 static atomic_t buf_lock = { .counter = 0 };
 static char buf[32];
 static int buf_initialized;
-static unsigned hart_count[2];
+static unsigned hart_count[NHARTS];
+static unsigned interrupt_count[NHARTS];
 
-static const char case_bit = 'a' - 'A';
-volatile int initialized;
+static unsigned delta = 0x100;
+void *increment_count(unsigned hartid, unsigned mcause, void *mepc, void *sp)
+{
+    interrupt_count[hartid]++;
+    MTIMECMP[hartid] = MTIME + delta;
+    return mepc;
+}
 
 int main()
 {
     uint32_t hartid = csr_read(mhartid);
     hart_count[hartid] = 0;
+    interrupt_count[hartid] = 0;
+
+    set_trap_handler(increment_count);
+    // Despite being memory-mapped, there appears to be one mtimecmp
+    // register per hart. The spec does not address this.
+    MTIMECMP[hartid] = MTIME + delta;
+    enable_timer_interrupts();
 
     while (1) {
         get_lock(&buf_lock);
-        hart_count[hartid]++;
 
         if (!buf_initialized) {
             for (unsigned i = 0; i < sizeof(buf); i++) {
@@ -77,5 +83,6 @@ int main()
                 buf[i] = 'a' + ((i + hartid + hart_count[hartid]) % 26);
         }
         put_lock(&buf_lock);
+        hart_count[hartid]++;
     }
 }
diff --git a/debug/targets.py b/debug/targets.py
index d661d14..624eb71 100644
--- a/debug/targets.py
+++ b/debug/targets.py
@@ -34,6 +34,11 @@ class Hart(object):
     # Defaults to target-<index>
     name = None
 
+    # When reset, the PC must be at one of the values listed here.
+    # This is a list because on some boards the reset vector depends on
+    # jumpers.
+    reset_vectors = []
+
     def extensionSupported(self, letter):
         # target.misa is set by testlib.ExamineTarget
         if self.misa:
@@ -91,6 +96,8 @@ class Target(object):
                 self.openocd_config_path)
         for i, hart in enumerate(self.harts):
             hart.index = i
+            if not hasattr(hart, 'id'):
+                hart.id = i
             if not hart.name:
                 hart.name = "%s-%d" % (self.name, i)
             # Default link script to <name>.lds
diff --git a/debug/targets/RISC-V/spike-1.cfg b/debug/targets/RISC-V/spike-1.cfg
new file mode 100644
index 0000000..fc20b53
--- /dev/null
+++ b/debug/targets/RISC-V/spike-1.cfg
@@ -0,0 +1,16 @@
+adapter_khz     10000
+
+interface remote_bitbang
+remote_bitbang_host $::env(REMOTE_BITBANG_HOST)
+remote_bitbang_port $::env(REMOTE_BITBANG_PORT)
+
+set _CHIPNAME riscv
+jtag newtap $_CHIPNAME cpu -irlen 5 -expected-id 0x10e31913
+
+set _TARGETNAME $_CHIPNAME.cpu
+target create $_TARGETNAME riscv -chain-position $_TARGETNAME
+
+gdb_report_data_abort enable
+
+init
+reset halt
diff --git a/debug/targets/RISC-V/spike-2.cfg b/debug/targets/RISC-V/spike-2.cfg
new file mode 100644
index 0000000..17526ec
--- /dev/null
+++ b/debug/targets/RISC-V/spike-2.cfg
@@ -0,0 +1,19 @@
+# Connect to a mult-icore RISC-V target, exposing each hart as a thread.
+adapter_khz     10000
+
+interface remote_bitbang
+remote_bitbang_host $::env(REMOTE_BITBANG_HOST)
+remote_bitbang_port $::env(REMOTE_BITBANG_PORT)
+
+set _CHIPNAME riscv
+jtag newtap $_CHIPNAME cpu -irlen 5 -expected-id 0x10e31913
+
+set _TARGETNAME_0 $_CHIPNAME.cpu0
+set _TARGETNAME_1 $_CHIPNAME.cpu1
+target create $_TARGETNAME_0 riscv -chain-position $_CHIPNAME.cpu -coreid 0
+target create $_TARGETNAME_1 riscv -chain-position $_CHIPNAME.cpu -coreid 1
+
+gdb_report_data_abort enable
+
+init
+reset halt
diff --git a/debug/targets/RISC-V/spike.cfg b/debug/targets/RISC-V/spike-rtos.cfg
index 9b1841c..799e3cb 100644
--- a/debug/targets/RISC-V/spike.cfg
+++ b/debug/targets/RISC-V/spike-rtos.cfg
@@ -1,3 +1,4 @@
+# Connect to a mult-icore RISC-V target, exposing each hart as a thread.
 adapter_khz     10000
 
 interface remote_bitbang
diff --git a/debug/targets/RISC-V/spike32-2-rtos.py b/debug/targets/RISC-V/spike32-2-rtos.py
new file mode 100644
index 0000000..a7b9a1c
--- /dev/null
+++ b/debug/targets/RISC-V/spike32-2-rtos.py
@@ -0,0 +1,12 @@
+import targets
+import testlib
+
+import spike32  # pylint: disable=import-error
+
+class spike32_2(targets.Target):
+    harts = [spike32.spike32_hart(), spike32.spike32_hart()]
+    openocd_config_path = "spike-rtos.cfg"
+    timeout_sec = 30
+
+    def create(self):
+        return testlib.Spike(self)
diff --git a/debug/targets/RISC-V/spike32-2.py b/debug/targets/RISC-V/spike32-2.py
index 6cf558d..719009d 100644
--- a/debug/targets/RISC-V/spike32-2.py
+++ b/debug/targets/RISC-V/spike32-2.py
@@ -5,7 +5,7 @@ import spike32  # pylint: disable=import-error
 
 class spike32_2(targets.Target):
     harts = [spike32.spike32_hart(), spike32.spike32_hart()]
-    openocd_config_path = "spike.cfg"
+    openocd_config_path = "spike-2.cfg"
     timeout_sec = 30
 
     def create(self):
diff --git a/debug/targets/RISC-V/spike32.py b/debug/targets/RISC-V/spike32.py
index 665d7e9..809463c 100644
--- a/debug/targets/RISC-V/spike32.py
+++ b/debug/targets/RISC-V/spike32.py
@@ -6,12 +6,12 @@ class spike32_hart(targets.Hart):
     ram = 0x10000000
     ram_size = 0x10000000
     instruction_hardware_breakpoint_count = 4
-    reset_vector = 0x1000
+    reset_vectors = [0x1000]
     link_script_path = "spike32.lds"
 
 class spike32(targets.Target):
     harts = [spike32_hart()]
-    openocd_config_path = "spike.cfg"
+    openocd_config_path = "spike-1.cfg"
     timeout_sec = 30
 
     def create(self):
diff --git a/debug/targets/RISC-V/spike64-2-rtos.py b/debug/targets/RISC-V/spike64-2-rtos.py
new file mode 100644
index 0000000..d65d2ab
--- /dev/null
+++ b/debug/targets/RISC-V/spike64-2-rtos.py
@@ -0,0 +1,12 @@
+import targets
+import testlib
+
+import spike64  # pylint: disable=import-error
+
+class spike64_2_rtos(targets.Target):
+    harts = [spike64.spike64_hart(), spike64.spike64_hart()]
+    openocd_config_path = "spike-rtos.cfg"
+    timeout_sec = 30
+
+    def create(self):
+        return testlib.Spike(self)
diff --git a/debug/targets/RISC-V/spike64-2.py b/debug/targets/RISC-V/spike64-2.py
index c6321dc..709ebbe 100644
--- a/debug/targets/RISC-V/spike64-2.py
+++ b/debug/targets/RISC-V/spike64-2.py
@@ -5,7 +5,7 @@ import spike64  # pylint: disable=import-error
 
 class spike64_2(targets.Target):
     harts = [spike64.spike64_hart(), spike64.spike64_hart()]
-    openocd_config_path = "spike.cfg"
+    openocd_config_path = "spike-2.cfg"
     timeout_sec = 30
 
     def create(self):
diff --git a/debug/targets/RISC-V/spike64.py b/debug/targets/RISC-V/spike64.py
index 6e3da89..2cd67a5 100644
--- a/debug/targets/RISC-V/spike64.py
+++ b/debug/targets/RISC-V/spike64.py
@@ -6,12 +6,12 @@ class spike64_hart(targets.Hart):
     ram = 0x1212340000
     ram_size = 0x10000000
     instruction_hardware_breakpoint_count = 4
-    reset_vector = 0x1000
+    reset_vectors = [0x1000]
     link_script_path = "spike64.lds"
 
 class spike64(targets.Target):
     harts = [spike64_hart()]
-    openocd_config_path = "spike.cfg"
+    openocd_config_path = "spike-1.cfg"
     timeout_sec = 30
 
     def create(self):
diff --git a/debug/testlib.py b/debug/testlib.py
index bb81cfb..66b7b38 100644
--- a/debug/testlib.py
+++ b/debug/testlib.py
@@ -1,4 +1,5 @@
 import collections
+import os
 import os.path
 import random
 import re
@@ -50,18 +51,7 @@ def compile(args, xlen=32): # pylint: disable=redefined-builtin
         header("")
         raise Exception("Compile failed!")
 
-def unused_port():
-    # http://stackoverflow.com/questions/2838244/get-open-tcp-port-in-python/2838309#2838309
-    import socket
-    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-    s.bind(("", 0))
-    port = s.getsockname()[1]
-    s.close()
-    return port
-
 class Spike(object):
-    logname = "spike-%d.log" % os.getpid()
-
     def __init__(self, target, halted=False, timeout=None, with_jtag_gdb=True):
         """Launch spike. Return tuple of its process and the port it's running
         on."""
@@ -72,6 +62,36 @@ class Spike(object):
         else:
             harts = [target]
 
+        cmd = self.command(target, harts, halted, timeout, with_jtag_gdb)
+        self.infinite_loop = target.compile(harts[0],
+                "programs/checksum.c", "programs/tiny-malloc.c",
+                "programs/infinite_loop.S", "-DDEFINE_MALLOC", "-DDEFINE_FREE")
+        cmd.append(self.infinite_loop)
+        self.logfile = tempfile.NamedTemporaryFile(prefix="spike-",
+                suffix=".log")
+        self.logname = self.logfile.name
+        self.logfile.write("+ %s\n" % " ".join(cmd))
+        self.logfile.flush()
+        self.process = subprocess.Popen(cmd, stdin=subprocess.PIPE,
+                stdout=self.logfile, stderr=self.logfile)
+
+        if with_jtag_gdb:
+            self.port = None
+            for _ in range(30):
+                m = re.search(r"Listening for remote bitbang connection on "
+                        r"port (\d+).", open(self.logname).read())
+                if m:
+                    self.port = int(m.group(1))
+                    os.environ['REMOTE_BITBANG_PORT'] = m.group(1)
+                    break
+                time.sleep(0.11)
+            if not self.port:
+                print_log(self.logname)
+                raise Exception("Didn't get spike message about bitbang "
+                        "connection")
+
+    def command(self, target, harts, halted, timeout, with_jtag_gdb):
+        # pylint: disable=no-self-use
         if target.sim_cmd:
             cmd = shlex.split(target.sim_cmd)
         else:
@@ -102,28 +122,8 @@ class Spike(object):
         if with_jtag_gdb:
             cmd += ['--rbb-port', '0']
             os.environ['REMOTE_BITBANG_HOST'] = 'localhost'
-        self.infinite_loop = target.compile(harts[0],
-                "programs/checksum.c", "programs/tiny-malloc.c",
-                "programs/infinite_loop.S", "-DDEFINE_MALLOC", "-DDEFINE_FREE")
-        cmd.append(self.infinite_loop)
-        logfile = open(self.logname, "w")
-        logfile.write("+ %s\n" % " ".join(cmd))
-        logfile.flush()
-        self.process = subprocess.Popen(cmd, stdin=subprocess.PIPE,
-                stdout=logfile, stderr=logfile)
 
-        if with_jtag_gdb:
-            self.port = None
-            for _ in range(30):
-                m = re.search(r"Listening for remote bitbang connection on "
-                        r"port (\d+).", open(self.logname).read())
-                if m:
-                    self.port = int(m.group(1))
-                    os.environ['REMOTE_BITBANG_PORT'] = m.group(1)
-                    break
-                time.sleep(0.11)
-            assert self.port, "Didn't get spike message about bitbang " \
-                    "connection"
+        return cmd
 
     def __del__(self):
         if self.process:
@@ -224,8 +224,7 @@ class Openocd(object):
         logfile.write("+ %s\n" % " ".join(cmd))
         logfile.flush()
 
-        self.ports = []
-        self.port = None
+        self.gdb_ports = []
         self.process = self.start(cmd, logfile)
 
     def start(self, cmd, logfile):
@@ -239,31 +238,32 @@ class Openocd(object):
             # attempt too early.
             start = time.time()
             messaged = False
+            fd = open(Openocd.logname, "r")
             while True:
-                log = open(Openocd.logname).read()
+                line = fd.readline()
+                if not line:
+                    if not process.poll() is None:
+                        raise Exception("OpenOCD exited early.")
+                    time.sleep(0.1)
+                    continue
+
                 m = re.search(r"Listening on port (\d+) for gdb connections",
-                        log)
+                        line)
                 if m:
-                    if not self.ports:
-                        self.port = int(m.group(1))
-                    self.ports.append(int(m.group(1)))
+                    self.gdb_ports.append(int(m.group(1)))
 
-                if "telnet server disabled" in log:
-                    break
+                if "telnet server disabled" in line:
+                    return process
 
-                if not process.poll() is None:
-                    raise Exception(
-                            "OpenOCD exited before completing riscv_examine()")
                 if not messaged and time.time() - start > 1:
                     messaged = True
                     print "Waiting for OpenOCD to start..."
                 if (time.time() - start) > self.timeout:
-                    raise Exception("ERROR: Timed out waiting for OpenOCD to "
+                    raise Exception("Timed out waiting for OpenOCD to "
                             "listen for gdb")
-            return process
+
         except Exception:
-            header("OpenOCD log")
-            sys.stdout.write(log)
+            print_log(Openocd.logname)
             raise
 
     def __del__(self):
@@ -304,42 +304,109 @@ class CannotAccess(Exception):
         Exception.__init__(self)
         self.address = address
 
-Thread = collections.namedtuple('Thread', ('id', 'target_id', 'name',
-    'frame'))
+Thread = collections.namedtuple('Thread', ('id', 'description', 'target_id',
+    'name', 'frame'))
 
 class Gdb(object):
-    logfile = tempfile.NamedTemporaryFile(prefix="gdb", suffix=".log")
-    logname = logfile.name
-    print "GDB Temporary Log File: %s" % logname
-
-    def __init__(self,
-            cmd=os.path.expandvars("$RISCV/bin/riscv64-unknown-elf-gdb")):
-        self.child = pexpect.spawn(cmd)
-        self.child.logfile = open(self.logname, "w")
-        self.child.logfile.write("+ %s\n" % cmd)
-        self.wait()
-        self.command("set confirm off")
-        self.command("set width 0")
-        self.command("set height 0")
-        # Force consistency.
-        self.command("set print entry-values no")
+    """A single gdb class which can interact with one or more gdb instances."""
+
+    # pylint: disable=too-many-public-methods
+
+    def __init__(self, ports,
+            cmd=os.path.expandvars("$RISCV/bin/riscv64-unknown-elf-gdb"),
+            binary=None):
+        assert ports
+
+        self.stack = []
+
+        self.logfiles = []
+        self.children = []
+        for port in ports:
+            logfile = tempfile.NamedTemporaryFile(prefix="gdb@%d-" % port,
+                    suffix=".log")
+            self.logfiles.append(logfile)
+            child = pexpect.spawn(cmd)
+            child.logfile = logfile
+            child.logfile.write("+ %s\n" % cmd)
+            self.children.append(child)
+        self.active_child = self.children[0]
+
+        self.harts = {}
+        for port, child in zip(ports, self.children):
+            self.select_child(child)
+            self.wait()
+            self.command("set confirm off")
+            self.command("set width 0")
+            self.command("set height 0")
+            # Force consistency.
+            self.command("set print entry-values no")
+            self.command("target extended-remote localhost:%d" % port)
+            if binary:
+                self.command("file %s" % binary)
+            threads = self.threads()
+            for t in threads:
+                hartid = None
+                if t.name:
+                    m = re.search(r"Hart (\d+)", t.name)
+                    if m:
+                        hartid = int(m.group(1))
+                if hartid is None:
+                    if self.harts:
+                        hartid = max(self.harts) + 1
+                    else:
+                        hartid = 0
+                self.harts[hartid] = (child, t)
+
+    def __del__(self):
+        for child in self.children:
+            del child
+
+    def lognames(self):
+        return [logfile.name for logfile in self.logfiles]
+
+    def select_child(self, child):
+        self.active_child = child
 
     def select_hart(self, hart):
-        output = self.command("thread %d" % (hart.index + 1))
+        child, thread = self.harts[hart.id]
+        self.select_child(child)
+        output = self.command("thread %s" % thread.id)
         assert "Unknown" not in output
 
+    def push_state(self):
+        self.stack.append({
+            'active_child': self.active_child
+            })
+
+    def pop_state(self):
+        state = self.stack.pop()
+        self.active_child = state['active_child']
+
     def wait(self):
         """Wait for prompt."""
-        self.child.expect(r"\(gdb\)")
+        self.active_child.expect(r"\(gdb\)")
 
     def command(self, command, timeout=6000):
         """timeout is in seconds"""
-        self.child.sendline(command)
-        self.child.expect("\n", timeout=timeout)
-        self.child.expect(r"\(gdb\)", timeout=timeout)
-        return self.child.before.strip()
+        self.active_child.sendline(command)
+        self.active_child.expect("\n", timeout=timeout)
+        self.active_child.expect(r"\(gdb\)", timeout=timeout)
+        return self.active_child.before.strip()
+
+    def global_command(self, command):
+        """Execute this command on every gdb that we control."""
+        with PrivateState(self):
+            for child in self.children:
+                self.select_child(child)
+                self.command(command)
 
     def c(self, wait=True, timeout=-1, async=False):
+        """
+        Dumb c command.
+        In RTOS mode, gdb will resume all harts.
+        In multi-gdb mode, this command will just go to the current gdb, so
+        will only resume one hart.
+        """
         if async:
             async = "&"
         else:
@@ -349,13 +416,24 @@ class Gdb(object):
             assert "Continuing" in output
             return output
         else:
-            self.child.sendline("c%s" % async)
-            self.child.expect("Continuing")
+            self.active_child.sendline("c%s" % async)
+            self.active_child.expect("Continuing")
+
+    def c_all(self):
+        """Resume every hart."""
+        with PrivateState(self):
+            for child in self.children:
+                child.sendline("c")
+                child.expect("Continuing")
+
+            # Now wait for them all to halt
+            for child in self.children:
+                child.expect(r"\(gdb\)")
 
     def interrupt(self):
-        self.child.send("\003")
-        self.child.expect(r"\(gdb\)", timeout=6000)
-        return self.child.before.strip()
+        self.active_child.send("\003")
+        self.active_child.expect(r"\(gdb\)", timeout=6000)
+        return self.active_child.before.strip()
 
     def x(self, address, size='w'):
         output = self.command("x/%s %s" % (size, address))
@@ -418,17 +496,32 @@ class Gdb(object):
         threads = []
         for line in output.splitlines():
             m = re.match(
-                    r"[\s\*]*(\d+)\s*Thread (\d+)\s*\(Name: ([^\)]+)\s*(.*)",
-                    line)
+                    r"[\s\*]*(\d+)\s*"
+                    r"(Remote target|Thread (\d+)\s*\(Name: ([^\)]+))"
+                    r"\s*(.*)", line)
             if m:
                 threads.append(Thread(*m.groups()))
-        if not threads:
-            threads.append(Thread('1', '1', 'Default', '???'))
+        assert threads
+        #>>>if not threads:
+        #>>>    threads.append(Thread('1', '1', 'Default', '???'))
         return threads
 
     def thread(self, thread):
         return self.command("thread %s" % thread.id)
 
+    def where(self):
+        return self.command("where 1")
+
+class PrivateState(object):
+    def __init__(self, gdb):
+        self.gdb = gdb
+
+    def __enter__(self):
+        self.gdb.push_state()
+
+    def __exit__(self, _type, _value, _traceback):
+        self.gdb.pop_state()
+
 def run_all_tests(module, target, parsed):
     if not os.path.exists(parsed.logs):
         os.makedirs(parsed.logs)
@@ -439,14 +532,16 @@ def run_all_tests(module, target, parsed):
     gdb_cmd = parsed.gdb
 
     todo = []
+    examine_added = False
     for hart in target.harts:
         if parsed.misaval:
             hart.misa = int(parsed.misaval, 16)
             print "Using $misa from command line: 0x%x" % hart.misa
         elif hart.misa:
             print "Using $misa from hart definition: 0x%x" % hart.misa
-        else:
-            todo.append(("ExamineTarget", ExamineTarget, hart))
+        elif not examine_added:
+            todo.append(("ExamineTarget", ExamineTarget, None))
+            examine_added = True
 
     for name in dir(module):
         definition = getattr(module, name)
@@ -470,7 +565,7 @@ def run_tests(parsed, target, todo):
         log_name = os.path.join(parsed.logs, "%s-%s-%s.log" %
                 (time.strftime("%Y%m%d-%H%M%S"), type(target).__name__, name))
         log_fd = open(log_name, 'w')
-        print "Running %s > %s ..." % (name, log_name),
+        print "[%s] Starting > %s" % (name, log_name)
         instance = definition(target, hart)
         sys.stdout.flush()
         log_fd.write("Test: %s\n" % name)
@@ -479,12 +574,12 @@ def run_tests(parsed, target, todo):
         real_stdout = sys.stdout
         sys.stdout = log_fd
         try:
-            result = instance.run()
+            result = instance.run(real_stdout)
             log_fd.write("Result: %s\n" % result)
         finally:
             sys.stdout = real_stdout
             log_fd.write("Time elapsed: %.2fs\n" % (time.time() - start))
-        print "%s in %.2fs" % (result, time.time() - start)
+        print "[%s] %s in %.2fs" % (name, result, time.time() - start)
         if result not in good_results and parsed.print_failures:
             sys.stdout.write(open(log_name).read())
         sys.stdout.flush()
@@ -532,8 +627,7 @@ def header(title, dash='-', length=78):
 
 def print_log(path):
     header(path)
-    lines = open(path, "r").readlines()
-    for l in lines:
+    for l in open(path, "r"):
         sys.stdout.write(l)
     print
 
@@ -591,7 +685,7 @@ class BaseTest(object):
     def postMortem(self):
         pass
 
-    def run(self):
+    def run(self, real_stdout):
         """
         If compile_args is set, compile a program and set self.binary.
 
@@ -610,6 +704,8 @@ class BaseTest(object):
 
         try:
             self.classSetup()
+            real_stdout.write("[%s] Temporary logs: %s\n" % (
+                type(self).__name__, ", ".join(self.logs)))
             self.setup()
             result = self.test()    # pylint: disable=no-member
         except TestNotApplicable:
@@ -624,7 +720,12 @@ class BaseTest(object):
                 print e.message
             header("Traceback")
             traceback.print_exc(file=sys.stdout)
-            self.postMortem()
+            try:
+                self.postMortem()
+            except Exception as e:  # pylint: disable=broad-except
+                header("postMortem Exception")
+                print e
+                traceback.print_exc(file=sys.stdout)
             return result
 
         finally:
@@ -647,25 +748,22 @@ class GdbTest(BaseTest):
         BaseTest.classSetup(self)
 
         if gdb_cmd:
-            self.gdb = Gdb(gdb_cmd)
+            self.gdb = Gdb(self.server.gdb_ports, gdb_cmd, binary=self.binary)
         else:
-            self.gdb = Gdb()
+            self.gdb = Gdb(self.server.gdb_ports, binary=self.binary)
 
-        self.logs.append(self.gdb.logname)
+        self.logs += self.gdb.lognames()
 
-        if self.binary:
-            self.gdb.command("file %s" % self.binary)
         if self.target:
-            self.gdb.command("set arch riscv:rv%d" % self.hart.xlen)
-            self.gdb.command("set remotetimeout %d" % self.target.timeout_sec)
-        if self.server.port:
-            self.gdb.command(
-                    "target extended-remote localhost:%d" % self.server.port)
-            self.gdb.select_hart(self.hart)
+            self.gdb.global_command("set arch riscv:rv%d" % self.hart.xlen)
+            self.gdb.global_command("set remotetimeout %d" %
+                    self.target.timeout_sec)
 
         for cmd in self.target.gdb_setup:
             self.gdb.command(cmd)
 
+        self.gdb.select_hart(self.hart)
+
         # FIXME: OpenOCD doesn't handle PRIV now
         #self.gdb.p("$priv=3")
 
@@ -692,23 +790,26 @@ class GdbSingleHartTest(GdbTest):
 
 class ExamineTarget(GdbTest):
     def test(self):
-        self.target.misa = self.gdb.p("$misa")
-
-        txt = "RV"
-        if (self.target.misa >> 30) == 1:
-            txt += "32"
-        elif (self.target.misa >> 62) == 2:
-            txt += "64"
-        elif (self.target.misa >> 126) == 3:
-            txt += "128"
-        else:
-            raise TestFailed("Couldn't determine XLEN from $misa (0x%x)" %
-                    self.target.misa)
+        for hart in self.target.harts:
+            self.gdb.select_hart(hart)
+
+            hart.misa = self.gdb.p("$misa")
+
+            txt = "RV"
+            if (hart.misa >> 30) == 1:
+                txt += "32"
+            elif (hart.misa >> 62) == 2:
+                txt += "64"
+            elif (hart.misa >> 126) == 3:
+                txt += "128"
+            else:
+                raise TestFailed("Couldn't determine XLEN from $misa (0x%x)" %
+                        self.hart.misa)
 
-        for i in range(26):
-            if self.target.misa & (1<<i):
-                txt += chr(i + ord('A'))
-        print txt,
+            for i in range(26):
+                if hart.misa & (1<<i):
+                    txt += chr(i + ord('A'))
+            print txt,
 
 class TestFailed(Exception):
     def __init__(self, message):