runtime: implement cheaper context switch on Linux/AMD64

    
    Currently, goroutine switches are implemented with libc
    getcontext/setcontext functions, which saves/restores the machine
    register states and also the signal context. This does more than
    what we need, and performs an expensive syscall.
    
    This CL implements a simplified version of getcontext/setcontext,
    in assembly, that only saves/restores the necessary part, i.e.
    the callee-save registers, and the PC, SP. A simplified version
    of makecontext, written in C, is also added. Currently this is
    only implemented on Linux/AMD64.
    
    Reviewed-on: https://go-review.googlesource.com/c/gofrontend/+/178298

From-SVN: r271818

3 files changed, 103 insertions, 19 deletions

diff --git a/libgo/runtime/go-context.S b/libgo/runtime/go-context.S
new file mode 100644
index 0000000..0cd2242
--- /dev/null
+++ b/libgo/runtime/go-context.S
@@ -0,0 +1,69 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This provides a simplified version of getcontext and
+// setcontext. They are like the corresponding functions
+// in libc, but we only save/restore the callee-save
+// registers and PC, SP. Unlike the libc functions, we
+// don't save/restore the signal masks and floating point
+// environment.
+
+#if defined(__x86_64__) && defined(__linux__) && !defined(__CET__)
+
+#define RBP_OFF	(0*8)
+#define RBX_OFF	(1*8)
+#define R12_OFF	(2*8)
+#define R13_OFF	(3*8)
+#define R14_OFF	(4*8)
+#define R15_OFF	(5*8)
+#define SP_OFF	(6*8)
+#define PC_OFF	(7*8)
+
+.globl __go_getcontext
+.text
+__go_getcontext:
+	movq	%rbx, RBX_OFF(%rdi)
+	movq	%rbp, RBP_OFF(%rdi)
+	movq	%r12, R12_OFF(%rdi)
+	movq	%r13, R13_OFF(%rdi)
+	movq	%r14, R14_OFF(%rdi)
+	movq	%r15, R15_OFF(%rdi)
+
+	movq	(%rsp), %rax	// return PC
+	movq	%rax, PC_OFF(%rdi)
+	leaq	8(%rsp), %rax	// the SP before pushing return PC
+	movq	%rax, SP_OFF(%rdi)
+
+	ret
+
+.globl __go_setcontext
+.text
+__go_setcontext:
+	movq	RBX_OFF(%rdi), %rbx
+	movq	RBP_OFF(%rdi), %rbp
+	movq	R12_OFF(%rdi), %r12
+	movq	R13_OFF(%rdi), %r13
+	movq	R14_OFF(%rdi), %r14
+	movq	R15_OFF(%rdi), %r15
+	movq	SP_OFF(%rdi), %rsp
+	movq	PC_OFF(%rdi), %rdx
+
+	jmp	*%rdx
+
+.globl __go_makecontext
+.text
+__go_makecontext:
+	addq	%rcx, %rdx
+
+	// Align the SP, and push a dummy return address.
+	andq	$~0xfULL, %rdx
+	subq	$8, %rdx
+	movq	$0, (%rdx)
+
+	movq	%rdx, SP_OFF(%rdi)
+	movq	%rsi, PC_OFF(%rdi)
+
+	ret
+
+#endif
diff --git a/libgo/runtime/proc.c b/libgo/runtime/proc.c
index 1569b5b..5ef421f 100644
--- a/libgo/runtime/proc.c
+++ b/libgo/runtime/proc.c
@@ -75,7 +75,7 @@ initcontext(void)
 }
 
 static inline void
-fixcontext(ucontext_t *c __attribute__ ((unused)))
+fixcontext(__go_context_t *c __attribute__ ((unused)))
 {
 }
 
@@ -182,18 +182,18 @@ fixcontext(ucontext_t* c)
 // Go, and Go has no simple way to align a field to such a boundary.
 // So we make the field larger in runtime2.go and pick an appropriate
 // offset within the field here.
-static ucontext_t*
+static __go_context_t*
 ucontext_arg(uintptr_t* go_ucontext)
 {
 	uintptr_t p = (uintptr_t)go_ucontext;
-	size_t align = __alignof__(ucontext_t);
+	size_t align = __alignof__(__go_context_t);
 	if(align > 16) {
 		// We only ensured space for up to a 16 byte alignment
 		// in libgo/go/runtime/runtime2.go.
-		runtime_throw("required alignment of ucontext_t too large");
+		runtime_throw("required alignment of __go_context_t too large");
 	}
 	p = (p + align - 1) &~ (uintptr_t)(align - 1);
-	return (ucontext_t*)p;
+	return (__go_context_t*)p;
 }
 
 // We can not always refer to the TLS variables directly.  The
@@ -289,7 +289,7 @@ runtime_gogo(G* newg)
 	g = newg;
 	newg->fromgogo = true;
 	fixcontext(ucontext_arg(&newg->context[0]));
-	setcontext(ucontext_arg(&newg->context[0]));
+	__go_setcontext(ucontext_arg(&newg->context[0]));
 	runtime_throw("gogo setcontext returned");
 }
 
@@ -328,7 +328,7 @@ runtime_mcall(FuncVal *fv)
 		gp->gcnextsp2 = (uintptr)(secondary_stack_pointer());
 #endif
 		gp->fromgogo = false;
-		getcontext(ucontext_arg(&gp->context[0]));
+		__go_getcontext(ucontext_arg(&gp->context[0]));
 
 		// When we return from getcontext, we may be running
 		// in a new thread.  That means that g may have
@@ -358,7 +358,7 @@ runtime_mcall(FuncVal *fv)
 		g = mp->g0;
 
 		fixcontext(ucontext_arg(&mp->g0->context[0]));
-		setcontext(ucontext_arg(&mp->g0->context[0]));
+		__go_setcontext(ucontext_arg(&mp->g0->context[0]));
 		runtime_throw("runtime: mcall function returned");
 	}
 }
@@ -450,7 +450,7 @@ void getTraceback(G* me, G* gp)
 #ifdef USING_SPLIT_STACK
 	__splitstack_getcontext((void*)(&me->stackcontext[0]));
 #endif
-	getcontext(ucontext_arg(&me->context[0]));
+	__go_getcontext(ucontext_arg(&me->context[0]));
 
 	if (gp->traceback != 0) {
 		runtime_gogo(gp);
@@ -493,7 +493,7 @@ doscanstackswitch(G* me, G* gp)
 #ifdef USING_SPLIT_STACK
 	__splitstack_getcontext((void*)(&me->stackcontext[0]));
 #endif
-	getcontext(ucontext_arg(&me->context[0]));
+	__go_getcontext(ucontext_arg(&me->context[0]));
 
 	if(me->entry != nil) {
 		// Got here from mcall.
@@ -574,7 +574,7 @@ runtime_mstart(void *arg)
 
 	// Save the currently active context.  This will return
 	// multiple times via the setcontext call in mcall.
-	getcontext(ucontext_arg(&gp->context[0]));
+	__go_getcontext(ucontext_arg(&gp->context[0]));
 
 	if(gp->traceback != 0) {
 		// Got here from getTraceback.
@@ -652,7 +652,7 @@ setGContext(void)
 	gp->gcinitialsp2 = secondary_stack_pointer();
 	gp->gcnextsp2 = (uintptr)(gp->gcinitialsp2);
 #endif
-	getcontext(ucontext_arg(&gp->context[0]));
+	__go_getcontext(ucontext_arg(&gp->context[0]));
 
 	if(gp->entry != nil) {
 		// Got here from mcall.
@@ -672,13 +672,11 @@ void makeGContext(G*, byte*, uintptr)
 // makeGContext makes a new context for a g.
 void
 makeGContext(G* gp, byte* sp, uintptr spsize) {
-	ucontext_t *uc;
+	__go_context_t *uc;
 
 	uc = ucontext_arg(&gp->context[0]);
-	getcontext(uc);
-	uc->uc_stack.ss_sp = sp;
-	uc->uc_stack.ss_size = (size_t)spsize;
-	makecontext(uc, kickoff, 0);
+	__go_getcontext(uc);
+	__go_makecontext(uc, kickoff, sp, (size_t)spsize);
 }
 
 // The goroutine g is about to enter a system call.
@@ -700,7 +698,7 @@ runtime_entersyscall()
 	// Save the registers in the g structure so that any pointers
 	// held in registers will be seen by the garbage collector.
 	if (!runtime_usestackmaps)
-		getcontext(ucontext_arg(&g->gcregs[0]));
+		__go_getcontext(ucontext_arg(&g->gcregs[0]));
 
 	// Note that if this function does save any registers itself,
 	// we might store the wrong value in the call to getcontext.
@@ -747,7 +745,7 @@ runtime_entersyscallblock()
 	// Save the registers in the g structure so that any pointers
 	// held in registers will be seen by the garbage collector.
 	if (!runtime_usestackmaps)
-		getcontext(ucontext_arg(&g->gcregs[0]));
+		__go_getcontext(ucontext_arg(&g->gcregs[0]));
 
 	// See comment in runtime_entersyscall.
 	doentersyscallblock((uintptr)runtime_getcallerpc(),
diff --git a/libgo/runtime/runtime.h b/libgo/runtime/runtime.h
index 71c1a3e..a421dea 100644
--- a/libgo/runtime/runtime.h
+++ b/libgo/runtime/runtime.h
@@ -510,3 +510,20 @@ bool probestackmaps(void)
 // older versions of glibc when a SIGPROF signal arrives while
 // collecting a backtrace.
 extern uint32 __go_runtime_in_callers;
+
+// Cheaper context switch functions.  Currently only defined on
+// Linux/AMD64.
+#if defined(__x86_64__) && defined(__linux__) && !defined(__CET__)
+typedef struct {
+	uint64 regs[8];
+} __go_context_t;
+int __go_getcontext(__go_context_t*);
+int __go_setcontext(__go_context_t*);
+void __go_makecontext(__go_context_t*, void (*)(), void*, size_t);
+#else
+#define __go_context_t	ucontext_t
+#define __go_getcontext(c)	getcontext(c)
+#define __go_setcontext(c)	setcontext(c)
+#define __go_makecontext(c, fn, sp, size) \
+	((c)->uc_stack.ss_sp = sp, (c)->uc_stack.ss_size = size, makecontext(c, fn, 0))
+#endif