diff options
Diffstat (limited to 'libgo')
| -rw-r--r-- | libgo/Makefile.am | 1 | ||||
| -rw-r--r-- | libgo/Makefile.in | 8 | ||||
| -rw-r--r-- | libgo/go/context/context_test.go | 2 | ||||
| -rw-r--r-- | libgo/go/runtime/chan.go | 724 | ||||
| -rw-r--r-- | libgo/go/runtime/runtime2.go | 8 | ||||
| -rw-r--r-- | libgo/go/runtime/select.go | 697 | ||||
| -rw-r--r-- | libgo/go/runtime/stubs.go | 64 | ||||
| -rw-r--r-- | libgo/go/runtime/trace.go | 1008 | ||||
| -rw-r--r-- | libgo/runtime/chan.goc | 1130 | ||||
| -rw-r--r-- | libgo/runtime/chan.h | 76 | ||||
| -rw-r--r-- | libgo/runtime/go-cgo.c | 5 | ||||
| -rw-r--r-- | libgo/runtime/heapdump.c | 4 | ||||
| -rw-r--r-- | libgo/runtime/mgc0.c | 9 | ||||
| -rw-r--r-- | libgo/runtime/proc.c | 44 | ||||
| -rw-r--r-- | libgo/runtime/runtime.h | 4 | ||||
| -rw-r--r-- | libgo/runtime/sema.goc | 15 |
16 files changed, 2558 insertions, 1241 deletions
diff --git a/libgo/Makefile.am b/libgo/Makefile.am index a0566c1..bc47be6 100644 --- a/libgo/Makefile.am +++ b/libgo/Makefile.am @@ -520,7 +520,6 @@ runtime_files = \ $(runtime_thread_files) \ runtime/yield.c \ $(rtems_task_variable_add_file) \ - chan.c \ cpuprof.c \ go-iface.c \ lfstack.c \ diff --git a/libgo/Makefile.in b/libgo/Makefile.in index 7a133f1..5806d75 100644 --- a/libgo/Makefile.in +++ b/libgo/Makefile.in @@ -263,9 +263,9 @@ am__objects_6 = go-append.lo go-assert.lo go-assert-interface.lo \ $(am__objects_1) mfixalloc.lo mgc0.lo mheap.lo msize.lo \ $(am__objects_2) panic.lo parfor.lo print.lo proc.lo \ runtime.lo signal_unix.lo thread.lo $(am__objects_3) yield.lo \ - $(am__objects_4) chan.lo cpuprof.lo go-iface.lo lfstack.lo \ - malloc.lo mprof.lo netpoll.lo rdebug.lo reflect.lo runtime1.lo \ - sema.lo sigqueue.lo string.lo time.lo $(am__objects_5) + $(am__objects_4) cpuprof.lo go-iface.lo lfstack.lo malloc.lo \ + mprof.lo netpoll.lo rdebug.lo reflect.lo runtime1.lo sema.lo \ + sigqueue.lo string.lo time.lo $(am__objects_5) am_libgo_llgo_la_OBJECTS = $(am__objects_6) libgo_llgo_la_OBJECTS = $(am_libgo_llgo_la_OBJECTS) libgo_llgo_la_LINK = $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) \ @@ -921,7 +921,6 @@ runtime_files = \ $(runtime_thread_files) \ runtime/yield.c \ $(rtems_task_variable_add_file) \ - chan.c \ cpuprof.c \ go-iface.c \ lfstack.c \ @@ -1557,7 +1556,6 @@ mostlyclean-compile: distclean-compile: -rm -f *.tab.c -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/chan.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/cpuprof.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/env_posix.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/getncpu-bsd.Plo@am__quote@ diff --git a/libgo/go/context/context_test.go b/libgo/go/context/context_test.go index cf18211..cdfec07 100644 --- a/libgo/go/context/context_test.go +++ b/libgo/go/context/context_test.go @@ -381,7 +381,7 @@ func TestAllocs(t *testing.T) { <-c.Done() }, limit: 8, - gccgoLimit: 15, + gccgoLimit: 18, }, { desc: "WithCancel(bg)", diff --git a/libgo/go/runtime/chan.go b/libgo/go/runtime/chan.go new file mode 100644 index 0000000..eb2cad6 --- /dev/null +++ b/libgo/go/runtime/chan.go @@ -0,0 +1,724 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +// This file contains the implementation of Go channels. + +// Invariants: +// At least one of c.sendq and c.recvq is empty. +// For buffered channels, also: +// c.qcount > 0 implies that c.recvq is empty. +// c.qcount < c.dataqsiz implies that c.sendq is empty. +import ( + "runtime/internal/atomic" + "unsafe" +) + +// For gccgo, use go:linkname to rename compiler-called functions to +// themselves, so that the compiler will export them. +// +//go:linkname makechan runtime.makechan +//go:linkname chansend1 runtime.chansend1 +//go:linkname chanrecv1 runtime.chanrecv1 +//go:linkname chanrecv2 runtime.chanrecv2 +//go:linkname closechan runtime.closechan + +const ( + maxAlign = 8 + hchanSize = unsafe.Sizeof(hchan{}) + uintptr(-int(unsafe.Sizeof(hchan{}))&(maxAlign-1)) + debugChan = false +) + +type hchan struct { + qcount uint // total data in the queue + dataqsiz uint // size of the circular queue + buf unsafe.Pointer // points to an array of dataqsiz elements + elemsize uint16 + closed uint32 + elemtype *_type // element type + sendx uint // send index + recvx uint // receive index + recvq waitq // list of recv waiters + sendq waitq // list of send waiters + + // lock protects all fields in hchan, as well as several + // fields in sudogs blocked on this channel. + // + // Do not change another G's status while holding this lock + // (in particular, do not ready a G), as this can deadlock + // with stack shrinking. + lock mutex +} + +type waitq struct { + first *sudog + last *sudog +} + +//go:linkname reflect_makechan reflect.makechan +func reflect_makechan(t *chantype, size int64) *hchan { + return makechan(t, size) +} + +func makechan(t *chantype, size int64) *hchan { + elem := t.elem + + // compiler checks this but be safe. + if elem.size >= 1<<16 { + throw("makechan: invalid channel element type") + } + if hchanSize%maxAlign != 0 || elem.align > maxAlign { + throw("makechan: bad alignment") + } + if size < 0 || int64(uintptr(size)) != size || (elem.size > 0 && uintptr(size) > (_MaxMem-hchanSize)/elem.size) { + panic(plainError("makechan: size out of range")) + } + + var c *hchan + if elem.kind&kindNoPointers != 0 || size == 0 { + // Allocate memory in one call. + // Hchan does not contain pointers interesting for GC in this case: + // buf points into the same allocation, elemtype is persistent. + // SudoG's are referenced from their owning thread so they can't be collected. + // TODO(dvyukov,rlh): Rethink when collector can move allocated objects. + c = (*hchan)(mallocgc(hchanSize+uintptr(size)*elem.size, nil, true)) + if size > 0 && elem.size != 0 { + c.buf = add(unsafe.Pointer(c), hchanSize) + } else { + // race detector uses this location for synchronization + // Also prevents us from pointing beyond the allocation (see issue 9401). + c.buf = unsafe.Pointer(c) + } + } else { + c = new(hchan) + c.buf = newarray(elem, int(size)) + } + c.elemsize = uint16(elem.size) + c.elemtype = elem + c.dataqsiz = uint(size) + + if debugChan { + print("makechan: chan=", c, "; elemsize=", elem.size, "; dataqsiz=", size, "\n") + } + return c +} + +// chanbuf(c, i) is pointer to the i'th slot in the buffer. +func chanbuf(c *hchan, i uint) unsafe.Pointer { + return add(c.buf, uintptr(i)*uintptr(c.elemsize)) +} + +// entry point for c <- x from compiled code +//go:nosplit +func chansend1(t *chantype, c *hchan, elem unsafe.Pointer) { + chansend(t, c, elem, true, getcallerpc(unsafe.Pointer(&t))) +} + +/* + * generic single channel send/recv + * If block is not nil, + * then the protocol will not + * sleep but return if it could + * not complete. + * + * sleep can wake up with g.param == nil + * when a channel involved in the sleep has + * been closed. it is easiest to loop and re-run + * the operation; we'll see that it's now closed. + */ +func chansend(t *chantype, c *hchan, ep unsafe.Pointer, block bool, callerpc uintptr) bool { + if raceenabled { + raceReadObjectPC(t.elem, ep, callerpc, funcPC(chansend)) + } + if msanenabled { + msanread(ep, t.elem.size) + } + + if c == nil { + if !block { + return false + } + gopark(nil, nil, "chan send (nil chan)", traceEvGoStop, 2) + throw("unreachable") + } + + if debugChan { + print("chansend: chan=", c, "\n") + } + + if raceenabled { + racereadpc(unsafe.Pointer(c), callerpc, funcPC(chansend)) + } + + // Fast path: check for failed non-blocking operation without acquiring the lock. + // + // After observing that the channel is not closed, we observe that the channel is + // not ready for sending. Each of these observations is a single word-sized read + // (first c.closed and second c.recvq.first or c.qcount depending on kind of channel). + // Because a closed channel cannot transition from 'ready for sending' to + // 'not ready for sending', even if the channel is closed between the two observations, + // they imply a moment between the two when the channel was both not yet closed + // and not ready for sending. We behave as if we observed the channel at that moment, + // and report that the send cannot proceed. + // + // It is okay if the reads are reordered here: if we observe that the channel is not + // ready for sending and then observe that it is not closed, that implies that the + // channel wasn't closed during the first observation. + if !block && c.closed == 0 && ((c.dataqsiz == 0 && c.recvq.first == nil) || + (c.dataqsiz > 0 && c.qcount == c.dataqsiz)) { + return false + } + + var t0 int64 + if blockprofilerate > 0 { + t0 = cputicks() + } + + lock(&c.lock) + + if c.closed != 0 { + unlock(&c.lock) + panic(plainError("send on closed channel")) + } + + if sg := c.recvq.dequeue(); sg != nil { + // Found a waiting receiver. We pass the value we want to send + // directly to the receiver, bypassing the channel buffer (if any). + send(c, sg, ep, func() { unlock(&c.lock) }) + return true + } + + if c.qcount < c.dataqsiz { + // Space is available in the channel buffer. Enqueue the element to send. + qp := chanbuf(c, c.sendx) + if raceenabled { + raceacquire(qp) + racerelease(qp) + } + typedmemmove(c.elemtype, qp, ep) + c.sendx++ + if c.sendx == c.dataqsiz { + c.sendx = 0 + } + c.qcount++ + unlock(&c.lock) + return true + } + + if !block { + unlock(&c.lock) + return false + } + + // Block on the channel. Some receiver will complete our operation for us. + gp := getg() + mysg := acquireSudog() + mysg.releasetime = 0 + if t0 != 0 { + mysg.releasetime = -1 + } + // No stack splits between assigning elem and enqueuing mysg + // on gp.waiting where copystack can find it. + mysg.elem = ep + mysg.waitlink = nil + mysg.g = gp + mysg.selectdone = nil + mysg.c = c + gp.waiting = mysg + gp.param = nil + c.sendq.enqueue(mysg) + goparkunlock(&c.lock, "chan send", traceEvGoBlockSend, 3) + + // someone woke us up. + if mysg != gp.waiting { + throw("G waiting list is corrupted") + } + gp.waiting = nil + if gp.param == nil { + if c.closed == 0 { + throw("chansend: spurious wakeup") + } + panic(plainError("send on closed channel")) + } + gp.param = nil + if mysg.releasetime > 0 { + blockevent(mysg.releasetime-t0, 2) + } + mysg.c = nil + releaseSudog(mysg) + return true +} + +// send processes a send operation on an empty channel c. +// The value ep sent by the sender is copied to the receiver sg. +// The receiver is then woken up to go on its merry way. +// Channel c must be empty and locked. send unlocks c with unlockf. +// sg must already be dequeued from c. +// ep must be non-nil and point to the heap or the caller's stack. +func send(c *hchan, sg *sudog, ep unsafe.Pointer, unlockf func()) { + if raceenabled { + if c.dataqsiz == 0 { + racesync(c, sg) + } else { + // Pretend we go through the buffer, even though + // we copy directly. Note that we need to increment + // the head/tail locations only when raceenabled. + qp := chanbuf(c, c.recvx) + raceacquire(qp) + racerelease(qp) + raceacquireg(sg.g, qp) + racereleaseg(sg.g, qp) + c.recvx++ + if c.recvx == c.dataqsiz { + c.recvx = 0 + } + c.sendx = c.recvx // c.sendx = (c.sendx+1) % c.dataqsiz + } + } + if sg.elem != nil { + sendDirect(c.elemtype, sg, ep) + sg.elem = nil + } + gp := sg.g + unlockf() + gp.param = unsafe.Pointer(sg) + if sg.releasetime != 0 { + sg.releasetime = cputicks() + } + goready(gp, 4) +} + +func sendDirect(t *_type, sg *sudog, src unsafe.Pointer) { + // Send on an unbuffered or empty-buffered channel is the only operation + // in the entire runtime where one goroutine + // writes to the stack of another goroutine. The GC assumes that + // stack writes only happen when the goroutine is running and are + // only done by that goroutine. Using a write barrier is sufficient to + // make up for violating that assumption, but the write barrier has to work. + // typedmemmove will call heapBitsBulkBarrier, but the target bytes + // are not in the heap, so that will not help. We arrange to call + // memmove and typeBitsBulkBarrier instead. + + // Once we read sg.elem out of sg, it will no longer + // be updated if the destination's stack gets copied (shrunk). + // So make sure that no preemption points can happen between read & use. + dst := sg.elem + memmove(dst, src, t.size) + typeBitsBulkBarrier(t, uintptr(dst), t.size) +} + +func closechan(c *hchan) { + if c == nil { + panic(plainError("close of nil channel")) + } + + lock(&c.lock) + if c.closed != 0 { + unlock(&c.lock) + panic(plainError("close of closed channel")) + } + + if raceenabled { + callerpc := getcallerpc(unsafe.Pointer(&c)) + racewritepc(unsafe.Pointer(c), callerpc, funcPC(closechan)) + racerelease(unsafe.Pointer(c)) + } + + c.closed = 1 + + var glist *g + + // release all readers + for { + sg := c.recvq.dequeue() + if sg == nil { + break + } + if sg.elem != nil { + memclr(sg.elem, uintptr(c.elemsize)) + sg.elem = nil + } + if sg.releasetime != 0 { + sg.releasetime = cputicks() + } + gp := sg.g + gp.param = nil + if raceenabled { + raceacquireg(gp, unsafe.Pointer(c)) + } + gp.schedlink.set(glist) + glist = gp + } + + // release all writers (they will panic) + for { + sg := c.sendq.dequeue() + if sg == nil { + break + } + sg.elem = nil + if sg.releasetime != 0 { + sg.releasetime = cputicks() + } + gp := sg.g + gp.param = nil + if raceenabled { + raceacquireg(gp, unsafe.Pointer(c)) + } + gp.schedlink.set(glist) + glist = gp + } + unlock(&c.lock) + + // Ready all Gs now that we've dropped the channel lock. + for glist != nil { + gp := glist + glist = glist.schedlink.ptr() + gp.schedlink = 0 + goready(gp, 3) + } +} + +// entry points for <- c from compiled code +//go:nosplit +func chanrecv1(t *chantype, c *hchan, elem unsafe.Pointer) { + chanrecv(t, c, elem, true) +} + +//go:nosplit +func chanrecv2(t *chantype, c *hchan, elem unsafe.Pointer) (received bool) { + _, received = chanrecv(t, c, elem, true) + return +} + +// chanrecv receives on channel c and writes the received data to ep. +// ep may be nil, in which case received data is ignored. +// If block == false and no elements are available, returns (false, false). +// Otherwise, if c is closed, zeros *ep and returns (true, false). +// Otherwise, fills in *ep with an element and returns (true, true). +// A non-nil ep must point to the heap or the caller's stack. +func chanrecv(t *chantype, c *hchan, ep unsafe.Pointer, block bool) (selected, received bool) { + // raceenabled: don't need to check ep, as it is always on the stack + // or is new memory allocated by reflect. + + if debugChan { + print("chanrecv: chan=", c, "\n") + } + + if c == nil { + if !block { + return + } + gopark(nil, nil, "chan receive (nil chan)", traceEvGoStop, 2) + throw("unreachable") + } + + // Fast path: check for failed non-blocking operation without acquiring the lock. + // + // After observing that the channel is not ready for receiving, we observe that the + // channel is not closed. Each of these observations is a single word-sized read + // (first c.sendq.first or c.qcount, and second c.closed). + // Because a channel cannot be reopened, the later observation of the channel + // being not closed implies that it was also not closed at the moment of the + // first observation. We behave as if we observed the channel at that moment + // and report that the receive cannot proceed. + // + // The order of operations is important here: reversing the operations can lead to + // incorrect behavior when racing with a close. + if !block && (c.dataqsiz == 0 && c.sendq.first == nil || + c.dataqsiz > 0 && atomic.Loaduint(&c.qcount) == 0) && + atomic.Load(&c.closed) == 0 { + return + } + + var t0 int64 + if blockprofilerate > 0 { + t0 = cputicks() + } + + lock(&c.lock) + + if c.closed != 0 && c.qcount == 0 { + if raceenabled { + raceacquire(unsafe.Pointer(c)) + } + unlock(&c.lock) + if ep != nil { + memclr(ep, uintptr(c.elemsize)) + } + return true, false + } + + if sg := c.sendq.dequeue(); sg != nil { + // Found a waiting sender. If buffer is size 0, receive value + // directly from sender. Otherwise, receive from head of queue + // and add sender's value to the tail of the queue (both map to + // the same buffer slot because the queue is full). + recv(c, sg, ep, func() { unlock(&c.lock) }) + return true, true + } + + if c.qcount > 0 { + // Receive directly from queue + qp := chanbuf(c, c.recvx) + if raceenabled { + raceacquire(qp) + racerelease(qp) + } + if ep != nil { + typedmemmove(c.elemtype, ep, qp) + } + memclr(qp, uintptr(c.elemsize)) + c.recvx++ + if c.recvx == c.dataqsiz { + c.recvx = 0 + } + c.qcount-- + unlock(&c.lock) + return true, true + } + + if !block { + unlock(&c.lock) + return false, false + } + + // no sender available: block on this channel. + gp := getg() + mysg := acquireSudog() + mysg.releasetime = 0 + if t0 != 0 { + mysg.releasetime = -1 + } + // No stack splits between assigning elem and enqueuing mysg + // on gp.waiting where copystack can find it. + mysg.elem = ep + mysg.waitlink = nil + gp.waiting = mysg + mysg.g = gp + mysg.selectdone = nil + mysg.c = c + gp.param = nil + c.recvq.enqueue(mysg) + goparkunlock(&c.lock, "chan receive", traceEvGoBlockRecv, 3) + + // someone woke us up + if mysg != gp.waiting { + throw("G waiting list is corrupted") + } + gp.waiting = nil + if mysg.releasetime > 0 { + blockevent(mysg.releasetime-t0, 2) + } + closed := gp.param == nil + gp.param = nil + mysg.c = nil + releaseSudog(mysg) + return true, !closed +} + +// recv processes a receive operation on a full channel c. +// There are 2 parts: +// 1) The value sent by the sender sg is put into the channel +// and the sender is woken up to go on its merry way. +// 2) The value received by the receiver (the current G) is +// written to ep. +// For synchronous channels, both values are the same. +// For asynchronous channels, the receiver gets its data from +// the channel buffer and the sender's data is put in the +// channel buffer. +// Channel c must be full and locked. recv unlocks c with unlockf. +// sg must already be dequeued from c. +// A non-nil ep must point to the heap or the caller's stack. +func recv(c *hchan, sg *sudog, ep unsafe.Pointer, unlockf func()) { + if c.dataqsiz == 0 { + if raceenabled { + racesync(c, sg) + } + if ep != nil { + // copy data from sender + // ep points to our own stack or heap, so nothing + // special (ala sendDirect) needed here. + typedmemmove(c.elemtype, ep, sg.elem) + } + } else { + // Queue is full. Take the item at the + // head of the queue. Make the sender enqueue + // its item at the tail of the queue. Since the + // queue is full, those are both the same slot. + qp := chanbuf(c, c.recvx) + if raceenabled { + raceacquire(qp) + racerelease(qp) + raceacquireg(sg.g, qp) + racereleaseg(sg.g, qp) + } + // copy data from queue to receiver + if ep != nil { + typedmemmove(c.elemtype, ep, qp) + } + // copy data from sender to queue + typedmemmove(c.elemtype, qp, sg.elem) + c.recvx++ + if c.recvx == c.dataqsiz { + c.recvx = 0 + } + c.sendx = c.recvx // c.sendx = (c.sendx+1) % c.dataqsiz + } + sg.elem = nil + gp := sg.g + unlockf() + gp.param = unsafe.Pointer(sg) + if sg.releasetime != 0 { + sg.releasetime = cputicks() + } + goready(gp, 4) +} + +// compiler implements +// +// select { +// case c <- v: +// ... foo +// default: +// ... bar +// } +// +// as +// +// if selectnbsend(c, v) { +// ... foo +// } else { +// ... bar +// } +// +func selectnbsend(t *chantype, c *hchan, elem unsafe.Pointer) (selected bool) { + return chansend(t, c, elem, false, getcallerpc(unsafe.Pointer(&t))) +} + +// compiler implements +// +// select { +// case v = <-c: +// ... foo +// default: +// ... bar +// } +// +// as +// +// if selectnbrecv(&v, c) { +// ... foo +// } else { +// ... bar +// } +// +func selectnbrecv(t *chantype, elem unsafe.Pointer, c *hchan) (selected bool) { + selected, _ = chanrecv(t, c, elem, false) + return +} + +// compiler implements +// +// select { +// case v, ok = <-c: +// ... foo +// default: +// ... bar +// } +// +// as +// +// if c != nil && selectnbrecv2(&v, &ok, c) { +// ... foo +// } else { +// ... bar +// } +// +func selectnbrecv2(t *chantype, elem unsafe.Pointer, received *bool, c *hchan) (selected bool) { + // TODO(khr): just return 2 values from this function, now that it is in Go. + selected, *received = chanrecv(t, c, elem, false) + return +} + +//go:linkname reflect_chansend reflect.chansend +func reflect_chansend(t *chantype, c *hchan, elem unsafe.Pointer, nb bool) (selected bool) { + return chansend(t, c, elem, !nb, getcallerpc(unsafe.Pointer(&t))) +} + +//go:linkname reflect_chanrecv reflect.chanrecv +func reflect_chanrecv(t *chantype, c *hchan, nb bool, elem unsafe.Pointer) (selected bool, received bool) { + return chanrecv(t, c, elem, !nb) +} + +//go:linkname reflect_chanlen reflect.chanlen +func reflect_chanlen(c *hchan) int { + if c == nil { + return 0 + } + return int(c.qcount) +} + +//go:linkname reflect_chancap reflect.chancap +func reflect_chancap(c *hchan) int { + if c == nil { + return 0 + } + return int(c.dataqsiz) +} + +//go:linkname reflect_chanclose reflect.chanclose +func reflect_chanclose(c *hchan) { + closechan(c) +} + +func (q *waitq) enqueue(sgp *sudog) { + sgp.next = nil + x := q.last + if x == nil { + sgp.prev = nil + q.first = sgp + q.last = sgp + return + } + sgp.prev = x + x.next = sgp + q.last = sgp +} + +func (q *waitq) dequeue() *sudog { + for { + sgp := q.first + if sgp == nil { + return nil + } + y := sgp.next + if y == nil { + q.first = nil + q.last = nil + } else { + y.prev = nil + q.first = y + sgp.next = nil // mark as removed (see dequeueSudog) + } + + // if sgp participates in a select and is already signaled, ignore it + if sgp.selectdone != nil { + // claim the right to signal + if *sgp.selectdone != 0 || !atomic.Cas(sgp.selectdone, 0, 1) { + continue + } + } + + return sgp + } +} + +func racesync(c *hchan, sg *sudog) { + racerelease(chanbuf(c, 0)) + raceacquireg(sg.g, chanbuf(c, 0)) + racereleaseg(sg.g, chanbuf(c, 0)) + raceacquire(chanbuf(c, 0)) +} diff --git a/libgo/go/runtime/runtime2.go b/libgo/go/runtime/runtime2.go index 17447eb..688efcd 100644 --- a/libgo/go/runtime/runtime2.go +++ b/libgo/go/runtime/runtime2.go @@ -231,9 +231,6 @@ func (mp *muintptr) set(m *m) { *mp = muintptr(unsafe.Pointer(m)) } // // sudogs are allocated from a special pool. Use acquireSudog and // releaseSudog to allocate and free them. -/* -Commented out for gccgo for now. - type sudog struct { // The following fields are protected by the hchan.lock of the // channel this sudog is blocking on. shrinkstack depends on @@ -253,7 +250,6 @@ type sudog struct { waitlink *sudog // g.waiting list c *hchan // channel } -*/ type gcstats struct { // the struct must consist of only uint64's, @@ -364,7 +360,7 @@ type g struct { gopc uintptr // pc of go statement that created this goroutine startpc uintptr // pc of goroutine function racectx uintptr - // Not for gccgo for now: waiting *sudog // sudog structures this g is waiting on (that have a valid elem ptr); in lock order + waiting *sudog // sudog structures this g is waiting on (that have a valid elem ptr); in lock order // Not for gccgo: cgoCtxt []uintptr // cgo traceback context // Per-G GC state @@ -528,7 +524,7 @@ type p struct { gfree *g gfreecnt int32 - // Not for gccgo for now: sudogcache []*sudog + sudogcache []*sudog // Not for gccgo for now: sudogbuf [128]*sudog // Not for gccgo for now: tracebuf traceBufPtr diff --git a/libgo/go/runtime/select.go b/libgo/go/runtime/select.go new file mode 100644 index 0000000..08446a1 --- /dev/null +++ b/libgo/go/runtime/select.go @@ -0,0 +1,697 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +// This file contains the implementation of Go select statements. + +import ( + "runtime/internal/sys" + "unsafe" +) + +// For gccgo, use go:linkname to rename compiler-called functions to +// themselves, so that the compiler will export them. +// +//go:linkname newselect runtime.newselect +//go:linkname selectdefault runtime.selectdefault +//go:linkname selectsend runtime.selectsend +//go:linkname selectrecv runtime.selectrecv +//go:linkname selectrecv2 runtime.selectrecv2 +//go:linkname selectgo runtime.selectgo + +const ( + debugSelect = false + + // scase.kind + caseRecv = iota + caseSend + caseDefault +) + +// Select statement header. +// Known to compiler. +// Changes here must also be made in src/cmd/internal/gc/select.go's selecttype. +type hselect struct { + tcase uint16 // total count of scase[] + ncase uint16 // currently filled scase[] + pollorder *uint16 // case poll order + lockorder *uint16 // channel lock order + scase [1]scase // one per case (in order of appearance) +} + +// Select case descriptor. +// Known to compiler. +// Changes here must also be made in src/cmd/internal/gc/select.go's selecttype. +type scase struct { + elem unsafe.Pointer // data element + c *hchan // chan + pc uintptr // return pc + kind uint16 + index uint16 // case index + receivedp *bool // pointer to received bool (recv2) + releasetime int64 +} + +var ( + chansendpc = funcPC(chansend) + chanrecvpc = funcPC(chanrecv) +) + +func selectsize(size uintptr) uintptr { + selsize := unsafe.Sizeof(hselect{}) + + (size-1)*unsafe.Sizeof(hselect{}.scase[0]) + + size*unsafe.Sizeof(*hselect{}.lockorder) + + size*unsafe.Sizeof(*hselect{}.pollorder) + return round(selsize, sys.Int64Align) +} + +func newselect(sel *hselect, selsize int64, size int32) { + if selsize != int64(selectsize(uintptr(size))) { + print("runtime: bad select size ", selsize, ", want ", selectsize(uintptr(size)), "\n") + throw("bad select size") + } + if size != int32(uint16(size)) { + throw("select size too large") + } + sel.tcase = uint16(size) + sel.ncase = 0 + sel.lockorder = (*uint16)(add(unsafe.Pointer(&sel.scase), uintptr(size)*unsafe.Sizeof(hselect{}.scase[0]))) + sel.pollorder = (*uint16)(add(unsafe.Pointer(sel.lockorder), uintptr(size)*unsafe.Sizeof(*hselect{}.lockorder))) + + // For gccgo the temporary variable will not have been zeroed. + memclr(unsafe.Pointer(&sel.scase), uintptr(size)*unsafe.Sizeof(hselect{}.scase[0])+uintptr(size)*unsafe.Sizeof(*hselect{}.lockorder)+uintptr(size)*unsafe.Sizeof(*hselect{}.pollorder)) + + if debugSelect { + print("newselect s=", sel, " size=", size, "\n") + } +} + +func selectsend(sel *hselect, c *hchan, elem unsafe.Pointer, index int32) { + // nil cases do not compete + if c != nil { + selectsendImpl(sel, c, getcallerpc(unsafe.Pointer(&sel)), elem, index) + } + return +} + +// cut in half to give stack a chance to split +func selectsendImpl(sel *hselect, c *hchan, pc uintptr, elem unsafe.Pointer, index int32) { + i := sel.ncase + if i >= sel.tcase { + throw("selectsend: too many cases") + } + sel.ncase = i + 1 + cas := (*scase)(add(unsafe.Pointer(&sel.scase), uintptr(i)*unsafe.Sizeof(sel.scase[0]))) + + cas.pc = pc + cas.c = c + cas.index = uint16(index) + cas.kind = caseSend + cas.elem = elem + + if debugSelect { + print("selectsend s=", sel, " pc=", hex(cas.pc), " chan=", cas.c, " index=", cas.index, "\n") + } +} + +func selectrecv(sel *hselect, c *hchan, elem unsafe.Pointer, index int32) { + // nil cases do not compete + if c != nil { + selectrecvImpl(sel, c, getcallerpc(unsafe.Pointer(&sel)), elem, nil, index) + } + return +} + +func selectrecv2(sel *hselect, c *hchan, elem unsafe.Pointer, received *bool, index int32) { + // nil cases do not compete + if c != nil { + selectrecvImpl(sel, c, getcallerpc(unsafe.Pointer(&sel)), elem, received, index) + } + return +} + +func selectrecvImpl(sel *hselect, c *hchan, pc uintptr, elem unsafe.Pointer, received *bool, index int32) { + i := sel.ncase + if i >= sel.tcase { + throw("selectrecv: too many cases") + } + sel.ncase = i + 1 + cas := (*scase)(add(unsafe.Pointer(&sel.scase), uintptr(i)*unsafe.Sizeof(sel.scase[0]))) + cas.pc = pc + cas.c = c + cas.index = uint16(index) + cas.kind = caseRecv + cas.elem = elem + cas.receivedp = received + + if debugSelect { + print("selectrecv s=", sel, " pc=", hex(cas.pc), " chan=", cas.c, " index=", cas.index, "\n") + } +} + +func selectdefault(sel *hselect, index int32) { + selectdefaultImpl(sel, getcallerpc(unsafe.Pointer(&sel)), index) + return +} + +func selectdefaultImpl(sel *hselect, callerpc uintptr, index int32) { + i := sel.ncase + if i >= sel.tcase { + throw("selectdefault: too many cases") + } + sel.ncase = i + 1 + cas := (*scase)(add(unsafe.Pointer(&sel.scase), uintptr(i)*unsafe.Sizeof(sel.scase[0]))) + cas.pc = callerpc + cas.c = nil + cas.index = uint16(index) + cas.kind = caseDefault + + if debugSelect { + print("selectdefault s=", sel, " pc=", hex(cas.pc), " index=", cas.index, "\n") + } +} + +func sellock(scases []scase, lockorder []uint16) { + var c *hchan + for _, o := range lockorder { + c0 := scases[o].c + if c0 != nil && c0 != c { + c = c0 + lock(&c.lock) + } + } +} + +func selunlock(scases []scase, lockorder []uint16) { + // We must be very careful here to not touch sel after we have unlocked + // the last lock, because sel can be freed right after the last unlock. + // Consider the following situation. + // First M calls runtime·park() in runtime·selectgo() passing the sel. + // Once runtime·park() has unlocked the last lock, another M makes + // the G that calls select runnable again and schedules it for execution. + // When the G runs on another M, it locks all the locks and frees sel. + // Now if the first M touches sel, it will access freed memory. + n := len(scases) + r := 0 + // skip the default case + if n > 0 && scases[lockorder[0]].c == nil { + r = 1 + } + for i := n - 1; i >= r; i-- { + c := scases[lockorder[i]].c + if i > 0 && c == scases[lockorder[i-1]].c { + continue // will unlock it on the next iteration + } + unlock(&c.lock) + } +} + +func selparkcommit(gp *g, _ unsafe.Pointer) bool { + // This must not access gp's stack (see gopark). In + // particular, it must not access the *hselect. That's okay, + // because by the time this is called, gp.waiting has all + // channels in lock order. + var lastc *hchan + for sg := gp.waiting; sg != nil; sg = sg.waitlink { + if sg.c != lastc && lastc != nil { + // As soon as we unlock the channel, fields in + // any sudog with that channel may change, + // including c and waitlink. Since multiple + // sudogs may have the same channel, we unlock + // only after we've passed the last instance + // of a channel. + unlock(&lastc.lock) + } + lastc = sg.c + } + if lastc != nil { + unlock(&lastc.lock) + } + return true +} + +func block() { + gopark(nil, nil, "select (no cases)", traceEvGoStop, 1) // forever +} + +// selectgo implements the select statement. +// +// *sel is on the current goroutine's stack (regardless of any +// escaping in selectgo). +// +// selectgo does not return. Instead, it overwrites its return PC and +// returns directly to the triggered select case. Because of this, it +// cannot appear at the top of a split stack. +func selectgo(sel *hselect) int32 { + _, index := selectgoImpl(sel) + return int32(index) +} + +// selectgoImpl returns scase.pc and scase.so for the select +// case which fired. +func selectgoImpl(sel *hselect) (uintptr, uint16) { + if debugSelect { + print("select: sel=", sel, "\n") + } + + scaseslice := slice{unsafe.Pointer(&sel.scase), int(sel.ncase), int(sel.ncase)} + scases := *(*[]scase)(unsafe.Pointer(&scaseslice)) + + var t0 int64 + if blockprofilerate > 0 { + t0 = cputicks() + for i := 0; i < int(sel.ncase); i++ { + scases[i].releasetime = -1 + } + } + + // The compiler rewrites selects that statically have + // only 0 or 1 cases plus default into simpler constructs. + // The only way we can end up with such small sel.ncase + // values here is for a larger select in which most channels + // have been nilled out. The general code handles those + // cases correctly, and they are rare enough not to bother + // optimizing (and needing to test). + + // generate permuted order + pollslice := slice{unsafe.Pointer(sel.pollorder), int(sel.ncase), int(sel.ncase)} + pollorder := *(*[]uint16)(unsafe.Pointer(&pollslice)) + for i := 1; i < int(sel.ncase); i++ { + j := int(fastrand1()) % (i + 1) + pollorder[i] = pollorder[j] + pollorder[j] = uint16(i) + } + + // sort the cases by Hchan address to get the locking order. + // simple heap sort, to guarantee n log n time and constant stack footprint. + lockslice := slice{unsafe.Pointer(sel.lockorder), int(sel.ncase), int(sel.ncase)} + lockorder := *(*[]uint16)(unsafe.Pointer(&lockslice)) + for i := 0; i < int(sel.ncase); i++ { + j := i + // Start with the pollorder to permute cases on the same channel. + c := scases[pollorder[i]].c + for j > 0 && scases[lockorder[(j-1)/2]].c.sortkey() < c.sortkey() { + k := (j - 1) / 2 + lockorder[j] = lockorder[k] + j = k + } + lockorder[j] = pollorder[i] + } + for i := int(sel.ncase) - 1; i >= 0; i-- { + o := lockorder[i] + c := scases[o].c + lockorder[i] = lockorder[0] + j := 0 + for { + k := j*2 + 1 + if k >= i { + break + } + if k+1 < i && scases[lockorder[k]].c.sortkey() < scases[lockorder[k+1]].c.sortkey() { + k++ + } + if c.sortkey() < scases[lockorder[k]].c.sortkey() { + lockorder[j] = lockorder[k] + j = k + continue + } + break + } + lockorder[j] = o + } + /* + for i := 0; i+1 < int(sel.ncase); i++ { + if scases[lockorder[i]].c.sortkey() > scases[lockorder[i+1]].c.sortkey() { + print("i=", i, " x=", lockorder[i], " y=", lockorder[i+1], "\n") + throw("select: broken sort") + } + } + */ + + // lock all the channels involved in the select + sellock(scases, lockorder) + + var ( + gp *g + done uint32 + sg *sudog + c *hchan + k *scase + sglist *sudog + sgnext *sudog + qp unsafe.Pointer + nextp **sudog + ) + +loop: + // pass 1 - look for something already waiting + var dfl *scase + var cas *scase + for i := 0; i < int(sel.ncase); i++ { + cas = &scases[pollorder[i]] + c = cas.c + + switch cas.kind { + case caseRecv: + sg = c.sendq.dequeue() + if sg != nil { + goto recv + } + if c.qcount > 0 { + goto bufrecv + } + if c.closed != 0 { + goto rclose + } + + case caseSend: + if raceenabled { + racereadpc(unsafe.Pointer(c), cas.pc, chansendpc) + } + if c.closed != 0 { + goto sclose + } + sg = c.recvq.dequeue() + if sg != nil { + goto send + } + if c.qcount < c.dataqsiz { + goto bufsend + } + + case caseDefault: + dfl = cas + } + } + + if dfl != nil { + selunlock(scases, lockorder) + cas = dfl + goto retc + } + + // pass 2 - enqueue on all chans + gp = getg() + done = 0 + if gp.waiting != nil { + throw("gp.waiting != nil") + } + nextp = &gp.waiting + for _, casei := range lockorder { + cas = &scases[casei] + c = cas.c + sg := acquireSudog() + sg.g = gp + // Note: selectdone is adjusted for stack copies in stack1.go:adjustsudogs + sg.selectdone = (*uint32)(noescape(unsafe.Pointer(&done))) + // No stack splits between assigning elem and enqueuing + // sg on gp.waiting where copystack can find it. + sg.elem = cas.elem + sg.releasetime = 0 + if t0 != 0 { + sg.releasetime = -1 + } + sg.c = c + // Construct waiting list in lock order. + *nextp = sg + nextp = &sg.waitlink + + switch cas.kind { + case caseRecv: + c.recvq.enqueue(sg) + + case caseSend: + c.sendq.enqueue(sg) + } + } + + // wait for someone to wake us up + gp.param = nil + gopark(selparkcommit, nil, "select", traceEvGoBlockSelect, 2) + + // someone woke us up + sellock(scases, lockorder) + sg = (*sudog)(gp.param) + gp.param = nil + + // pass 3 - dequeue from unsuccessful chans + // otherwise they stack up on quiet channels + // record the successful case, if any. + // We singly-linked up the SudoGs in lock order. + cas = nil + sglist = gp.waiting + // Clear all elem before unlinking from gp.waiting. + for sg1 := gp.waiting; sg1 != nil; sg1 = sg1.waitlink { + sg1.selectdone = nil + sg1.elem = nil + sg1.c = nil + } + gp.waiting = nil + + for _, casei := range lockorder { + k = &scases[casei] + if sglist.releasetime > 0 { + k.releasetime = sglist.releasetime + } + if sg == sglist { + // sg has already been dequeued by the G that woke us up. + cas = k + } else { + c = k.c + if k.kind == caseSend { + c.sendq.dequeueSudoG(sglist) + } else { + c.recvq.dequeueSudoG(sglist) + } + } + sgnext = sglist.waitlink + sglist.waitlink = nil + releaseSudog(sglist) + sglist = sgnext + } + + if cas == nil { + // This can happen if we were woken up by a close(). + // TODO: figure that out explicitly so we don't need this loop. + goto loop + } + + c = cas.c + + if debugSelect { + print("wait-return: sel=", sel, " c=", c, " cas=", cas, " kind=", cas.kind, "\n") + } + + if cas.kind == caseRecv { + if cas.receivedp != nil { + *cas.receivedp = true + } + } + + if raceenabled { + if cas.kind == caseRecv && cas.elem != nil { + raceWriteObjectPC(c.elemtype, cas.elem, cas.pc, chanrecvpc) + } else if cas.kind == caseSend { + raceReadObjectPC(c.elemtype, cas.elem, cas.pc, chansendpc) + } + } + if msanenabled { + if cas.kind == caseRecv && cas.elem != nil { + msanwrite(cas.elem, c.elemtype.size) + } else if cas.kind == caseSend { + msanread(cas.elem, c.elemtype.size) + } + } + + selunlock(scases, lockorder) + goto retc + +bufrecv: + // can receive from buffer + if raceenabled { + if cas.elem != nil { + raceWriteObjectPC(c.elemtype, cas.elem, cas.pc, chanrecvpc) + } + raceacquire(chanbuf(c, c.recvx)) + racerelease(chanbuf(c, c.recvx)) + } + if msanenabled && cas.elem != nil { + msanwrite(cas.elem, c.elemtype.size) + } + if cas.receivedp != nil { + *cas.receivedp = true + } + qp = chanbuf(c, c.recvx) + if cas.elem != nil { + typedmemmove(c.elemtype, cas.elem, qp) + } + memclr(qp, uintptr(c.elemsize)) + c.recvx++ + if c.recvx == c.dataqsiz { + c.recvx = 0 + } + c.qcount-- + selunlock(scases, lockorder) + goto retc + +bufsend: + // can send to buffer + if raceenabled { + raceacquire(chanbuf(c, c.sendx)) + racerelease(chanbuf(c, c.sendx)) + raceReadObjectPC(c.elemtype, cas.elem, cas.pc, chansendpc) + } + if msanenabled { + msanread(cas.elem, c.elemtype.size) + } + typedmemmove(c.elemtype, chanbuf(c, c.sendx), cas.elem) + c.sendx++ + if c.sendx == c.dataqsiz { + c.sendx = 0 + } + c.qcount++ + selunlock(scases, lockorder) + goto retc + +recv: + // can receive from sleeping sender (sg) + recv(c, sg, cas.elem, func() { selunlock(scases, lockorder) }) + if debugSelect { + print("syncrecv: sel=", sel, " c=", c, "\n") + } + if cas.receivedp != nil { + *cas.receivedp = true + } + goto retc + +rclose: + // read at end of closed channel + selunlock(scases, lockorder) + if cas.receivedp != nil { + *cas.receivedp = false + } + if cas.elem != nil { + memclr(cas.elem, uintptr(c.elemsize)) + } + if raceenabled { + raceacquire(unsafe.Pointer(c)) + } + goto retc + +send: + // can send to a sleeping receiver (sg) + if raceenabled { + raceReadObjectPC(c.elemtype, cas.elem, cas.pc, chansendpc) + } + if msanenabled { + msanread(cas.elem, c.elemtype.size) + } + send(c, sg, cas.elem, func() { selunlock(scases, lockorder) }) + if debugSelect { + print("syncsend: sel=", sel, " c=", c, "\n") + } + goto retc + +retc: + if cas.releasetime > 0 { + blockevent(cas.releasetime-t0, 2) + } + return cas.pc, cas.index + +sclose: + // send on closed channel + selunlock(scases, lockorder) + panic(plainError("send on closed channel")) +} + +func (c *hchan) sortkey() uintptr { + // TODO(khr): if we have a moving garbage collector, we'll need to + // change this function. + return uintptr(unsafe.Pointer(c)) +} + +// A runtimeSelect is a single case passed to rselect. +// This must match ../reflect/value.go:/runtimeSelect +type runtimeSelect struct { + dir selectDir + typ unsafe.Pointer // channel type (not used here) + ch *hchan // channel + val unsafe.Pointer // ptr to data (SendDir) or ptr to receive buffer (RecvDir) +} + +// These values must match ../reflect/value.go:/SelectDir. +type selectDir int + +const ( + _ selectDir = iota + selectSend // case Chan <- Send + selectRecv // case <-Chan: + selectDefault // default +) + +//go:linkname reflect_rselect reflect.rselect +func reflect_rselect(cases []runtimeSelect) (chosen int, recvOK bool) { + // flagNoScan is safe here, because all objects are also referenced from cases. + size := selectsize(uintptr(len(cases))) + sel := (*hselect)(mallocgc(size, nil, true)) + newselect(sel, int64(size), int32(len(cases))) + r := new(bool) + for i := range cases { + rc := &cases[i] + switch rc.dir { + case selectDefault: + selectdefaultImpl(sel, uintptr(i), 0) + case selectSend: + if rc.ch == nil { + break + } + selectsendImpl(sel, rc.ch, uintptr(i), rc.val, 0) + case selectRecv: + if rc.ch == nil { + break + } + selectrecvImpl(sel, rc.ch, uintptr(i), rc.val, r, 0) + } + } + + pc, _ := selectgoImpl(sel) + chosen = int(pc) + recvOK = *r + return +} + +func (q *waitq) dequeueSudoG(sgp *sudog) { + x := sgp.prev + y := sgp.next + if x != nil { + if y != nil { + // middle of queue + x.next = y + y.prev = x + sgp.next = nil + sgp.prev = nil + return + } + // end of queue + x.next = nil + q.last = x + sgp.prev = nil + return + } + if y != nil { + // start of queue + y.prev = nil + q.first = y + sgp.next = nil + return + } + + // x==y==nil. Either sgp is the only element in the queue, + // or it has already been removed. Use q.first to disambiguate. + if q.first == sgp { + q.first = nil + q.last = nil + } +} diff --git a/libgo/go/runtime/stubs.go b/libgo/go/runtime/stubs.go index bbeac41..c687cbf 100644 --- a/libgo/go/runtime/stubs.go +++ b/libgo/go/runtime/stubs.go @@ -384,3 +384,67 @@ func errno() int func entersyscall(int32) func entersyscallblock(int32) func exitsyscall(int32) +func gopark(func(*g, unsafe.Pointer) bool, unsafe.Pointer, string, byte, int) +func goparkunlock(*mutex, string, byte, int) +func goready(*g, int) + +// Temporary for gccgo until we port mprof.go. +var blockprofilerate uint64 + +func blockevent(cycles int64, skip int) {} + +// Temporary hack for gccgo until we port proc.go. +//go:nosplit +func acquireSudog() *sudog { + mp := acquirem() + pp := mp.p.ptr() + if len(pp.sudogcache) == 0 { + pp.sudogcache = append(pp.sudogcache, new(sudog)) + } + n := len(pp.sudogcache) + s := pp.sudogcache[n-1] + pp.sudogcache[n-1] = nil + pp.sudogcache = pp.sudogcache[:n-1] + if s.elem != nil { + throw("acquireSudog: found s.elem != nil in cache") + } + releasem(mp) + return s +} + +// Temporary hack for gccgo until we port proc.go. +//go:nosplit +func releaseSudog(s *sudog) { + if s.elem != nil { + throw("runtime: sudog with non-nil elem") + } + if s.selectdone != nil { + throw("runtime: sudog with non-nil selectdone") + } + if s.next != nil { + throw("runtime: sudog with non-nil next") + } + if s.prev != nil { + throw("runtime: sudog with non-nil prev") + } + if s.waitlink != nil { + throw("runtime: sudog with non-nil waitlink") + } + if s.c != nil { + throw("runtime: sudog with non-nil c") + } + gp := getg() + if gp.param != nil { + throw("runtime: releaseSudog with non-nil gp.param") + } + mp := acquirem() // avoid rescheduling to another P + pp := mp.p.ptr() + pp.sudogcache = append(pp.sudogcache, s) + releasem(mp) +} + +// Temporary hack for gccgo until we port the garbage collector. +func typeBitsBulkBarrier(typ *_type, p, size uintptr) {} + +// Temporary for gccgo until we port print.go. +type hex uint64 diff --git a/libgo/go/runtime/trace.go b/libgo/go/runtime/trace.go new file mode 100644 index 0000000..35126f1 --- /dev/null +++ b/libgo/go/runtime/trace.go @@ -0,0 +1,1008 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Go execution tracer. +// The tracer captures a wide range of execution events like goroutine +// creation/blocking/unblocking, syscall enter/exit/block, GC-related events, +// changes of heap size, processor start/stop, etc and writes them to a buffer +// in a compact form. A precise nanosecond-precision timestamp and a stack +// trace is captured for most events. +// See https://golang.org/s/go15trace for more info. + +package runtime + +import ( + "runtime/internal/sys" + "unsafe" +) + +// Event types in the trace, args are given in square brackets. +const ( + traceEvNone = 0 // unused + traceEvBatch = 1 // start of per-P batch of events [pid, timestamp] + traceEvFrequency = 2 // contains tracer timer frequency [frequency (ticks per second)] + traceEvStack = 3 // stack [stack id, number of PCs, array of {PC, func string ID, file string ID, line}] + traceEvGomaxprocs = 4 // current value of GOMAXPROCS [timestamp, GOMAXPROCS, stack id] + traceEvProcStart = 5 // start of P [timestamp, thread id] + traceEvProcStop = 6 // stop of P [timestamp] + traceEvGCStart = 7 // GC start [timestamp, seq, stack id] + traceEvGCDone = 8 // GC done [timestamp] + traceEvGCScanStart = 9 // GC scan start [timestamp] + traceEvGCScanDone = 10 // GC scan done [timestamp] + traceEvGCSweepStart = 11 // GC sweep start [timestamp, stack id] + traceEvGCSweepDone = 12 // GC sweep done [timestamp] + traceEvGoCreate = 13 // goroutine creation [timestamp, new goroutine id, new stack id, stack id] + traceEvGoStart = 14 // goroutine starts running [timestamp, goroutine id, seq] + traceEvGoEnd = 15 // goroutine ends [timestamp] + traceEvGoStop = 16 // goroutine stops (like in select{}) [timestamp, stack] + traceEvGoSched = 17 // goroutine calls Gosched [timestamp, stack] + traceEvGoPreempt = 18 // goroutine is preempted [timestamp, stack] + traceEvGoSleep = 19 // goroutine calls Sleep [timestamp, stack] + traceEvGoBlock = 20 // goroutine blocks [timestamp, stack] + traceEvGoUnblock = 21 // goroutine is unblocked [timestamp, goroutine id, seq, stack] + traceEvGoBlockSend = 22 // goroutine blocks on chan send [timestamp, stack] + traceEvGoBlockRecv = 23 // goroutine blocks on chan recv [timestamp, stack] + traceEvGoBlockSelect = 24 // goroutine blocks on select [timestamp, stack] + traceEvGoBlockSync = 25 // goroutine blocks on Mutex/RWMutex [timestamp, stack] + traceEvGoBlockCond = 26 // goroutine blocks on Cond [timestamp, stack] + traceEvGoBlockNet = 27 // goroutine blocks on network [timestamp, stack] + traceEvGoSysCall = 28 // syscall enter [timestamp, stack] + traceEvGoSysExit = 29 // syscall exit [timestamp, goroutine id, seq, real timestamp] + traceEvGoSysBlock = 30 // syscall blocks [timestamp] + traceEvGoWaiting = 31 // denotes that goroutine is blocked when tracing starts [timestamp, goroutine id] + traceEvGoInSyscall = 32 // denotes that goroutine is in syscall when tracing starts [timestamp, goroutine id] + traceEvHeapAlloc = 33 // memstats.heap_live change [timestamp, heap_alloc] + traceEvNextGC = 34 // memstats.next_gc change [timestamp, next_gc] + traceEvTimerGoroutine = 35 // denotes timer goroutine [timer goroutine id] + traceEvFutileWakeup = 36 // denotes that the previous wakeup of this goroutine was futile [timestamp] + traceEvString = 37 // string dictionary entry [ID, length, string] + traceEvGoStartLocal = 38 // goroutine starts running on the same P as the last event [timestamp, goroutine id] + traceEvGoUnblockLocal = 39 // goroutine is unblocked on the same P as the last event [timestamp, goroutine id, stack] + traceEvGoSysExitLocal = 40 // syscall exit on the same P as the last event [timestamp, goroutine id, real timestamp] + traceEvCount = 41 +) + +const ( + // Timestamps in trace are cputicks/traceTickDiv. + // This makes absolute values of timestamp diffs smaller, + // and so they are encoded in less number of bytes. + // 64 on x86 is somewhat arbitrary (one tick is ~20ns on a 3GHz machine). + // The suggested increment frequency for PowerPC's time base register is + // 512 MHz according to Power ISA v2.07 section 6.2, so we use 16 on ppc64 + // and ppc64le. + // Tracing won't work reliably for architectures where cputicks is emulated + // by nanotime, so the value doesn't matter for those architectures. + traceTickDiv = 16 + 48*(sys.Goarch386|sys.GoarchAmd64|sys.GoarchAmd64p32) + // Maximum number of PCs in a single stack trace. + // Since events contain only stack id rather than whole stack trace, + // we can allow quite large values here. + traceStackSize = 128 + // Identifier of a fake P that is used when we trace without a real P. + traceGlobProc = -1 + // Maximum number of bytes to encode uint64 in base-128. + traceBytesPerNumber = 10 + // Shift of the number of arguments in the first event byte. + traceArgCountShift = 6 + // Flag passed to traceGoPark to denote that the previous wakeup of this + // goroutine was futile. For example, a goroutine was unblocked on a mutex, + // but another goroutine got ahead and acquired the mutex before the first + // goroutine is scheduled, so the first goroutine has to block again. + // Such wakeups happen on buffered channels and sync.Mutex, + // but are generally not interesting for end user. + traceFutileWakeup byte = 128 +) + +// trace is global tracing context. +var trace struct { + lock mutex // protects the following members + lockOwner *g // to avoid deadlocks during recursive lock locks + enabled bool // when set runtime traces events + shutdown bool // set when we are waiting for trace reader to finish after setting enabled to false + headerWritten bool // whether ReadTrace has emitted trace header + footerWritten bool // whether ReadTrace has emitted trace footer + shutdownSema uint32 // used to wait for ReadTrace completion + seqStart uint64 // sequence number when tracing was started + ticksStart int64 // cputicks when tracing was started + ticksEnd int64 // cputicks when tracing was stopped + timeStart int64 // nanotime when tracing was started + timeEnd int64 // nanotime when tracing was stopped + seqGC uint64 // GC start/done sequencer + reading traceBufPtr // buffer currently handed off to user + empty traceBufPtr // stack of empty buffers + fullHead traceBufPtr // queue of full buffers + fullTail traceBufPtr + reader *g // goroutine that called ReadTrace, or nil + stackTab traceStackTable // maps stack traces to unique ids + + // Dictionary for traceEvString. + // Currently this is used only for func/file:line info after tracing session, + // so we assume single-threaded access. + strings map[string]uint64 + stringSeq uint64 + + bufLock mutex // protects buf + buf traceBufPtr // global trace buffer, used when running without a p +} + +// traceBufHeader is per-P tracing buffer. +type traceBufHeader struct { + link traceBufPtr // in trace.empty/full + lastTicks uint64 // when we wrote the last event + pos int // next write offset in arr + stk [traceStackSize]uintptr // scratch buffer for traceback +} + +// traceBuf is per-P tracing buffer. +type traceBuf struct { + traceBufHeader + arr [64<<10 - unsafe.Sizeof(traceBufHeader{})]byte // underlying buffer for traceBufHeader.buf +} + +// traceBufPtr is a *traceBuf that is not traced by the garbage +// collector and doesn't have write barriers. traceBufs are not +// allocated from the GC'd heap, so this is safe, and are often +// manipulated in contexts where write barriers are not allowed, so +// this is necessary. +type traceBufPtr uintptr + +func (tp traceBufPtr) ptr() *traceBuf { return (*traceBuf)(unsafe.Pointer(tp)) } +func (tp *traceBufPtr) set(b *traceBuf) { *tp = traceBufPtr(unsafe.Pointer(b)) } +func traceBufPtrOf(b *traceBuf) traceBufPtr { + return traceBufPtr(unsafe.Pointer(b)) +} + +/* +Commented out for gccgo for now. + +// StartTrace enables tracing for the current process. +// While tracing, the data will be buffered and available via ReadTrace. +// StartTrace returns an error if tracing is already enabled. +// Most clients should use the runtime/trace package or the testing package's +// -test.trace flag instead of calling StartTrace directly. +func StartTrace() error { + // Stop the world, so that we can take a consistent snapshot + // of all goroutines at the beginning of the trace. + stopTheWorld("start tracing") + + // We are in stop-the-world, but syscalls can finish and write to trace concurrently. + // Exitsyscall could check trace.enabled long before and then suddenly wake up + // and decide to write to trace at a random point in time. + // However, such syscall will use the global trace.buf buffer, because we've + // acquired all p's by doing stop-the-world. So this protects us from such races. + lock(&trace.bufLock) + + if trace.enabled || trace.shutdown { + unlock(&trace.bufLock) + startTheWorld() + return errorString("tracing is already enabled") + } + + // Can't set trace.enabled yet. While the world is stopped, exitsyscall could + // already emit a delayed event (see exitTicks in exitsyscall) if we set trace.enabled here. + // That would lead to an inconsistent trace: + // - either GoSysExit appears before EvGoInSyscall, + // - or GoSysExit appears for a goroutine for which we don't emit EvGoInSyscall below. + // To instruct traceEvent that it must not ignore events below, we set startingtrace. + // trace.enabled is set afterwards once we have emitted all preliminary events. + _g_ := getg() + _g_.m.startingtrace = true + for _, gp := range allgs { + status := readgstatus(gp) + if status != _Gdead { + traceGoCreate(gp, gp.startpc) // also resets gp.traceseq/tracelastp + } + if status == _Gwaiting { + // traceEvGoWaiting is implied to have seq=1. + gp.traceseq++ + traceEvent(traceEvGoWaiting, -1, uint64(gp.goid)) + } + if status == _Gsyscall { + gp.traceseq++ + traceEvent(traceEvGoInSyscall, -1, uint64(gp.goid)) + } else { + gp.sysblocktraced = false + } + } + traceProcStart() + traceGoStart() + // Note: ticksStart needs to be set after we emit traceEvGoInSyscall events. + // If we do it the other way around, it is possible that exitsyscall will + // query sysexitticks after ticksStart but before traceEvGoInSyscall timestamp. + // It will lead to a false conclusion that cputicks is broken. + trace.ticksStart = cputicks() + trace.timeStart = nanotime() + trace.headerWritten = false + trace.footerWritten = false + trace.strings = make(map[string]uint64) + trace.stringSeq = 0 + trace.seqGC = 0 + _g_.m.startingtrace = false + trace.enabled = true + + unlock(&trace.bufLock) + + startTheWorld() + return nil +} + +// StopTrace stops tracing, if it was previously enabled. +// StopTrace only returns after all the reads for the trace have completed. +func StopTrace() { + // Stop the world so that we can collect the trace buffers from all p's below, + // and also to avoid races with traceEvent. + stopTheWorld("stop tracing") + + // See the comment in StartTrace. + lock(&trace.bufLock) + + if !trace.enabled { + unlock(&trace.bufLock) + startTheWorld() + return + } + + traceGoSched() + + for _, p := range &allp { + if p == nil { + break + } + buf := p.tracebuf + if buf != 0 { + traceFullQueue(buf) + p.tracebuf = 0 + } + } + if trace.buf != 0 && trace.buf.ptr().pos != 0 { + buf := trace.buf + trace.buf = 0 + traceFullQueue(buf) + } + + for { + trace.ticksEnd = cputicks() + trace.timeEnd = nanotime() + // Windows time can tick only every 15ms, wait for at least one tick. + if trace.timeEnd != trace.timeStart { + break + } + osyield() + } + + trace.enabled = false + trace.shutdown = true + unlock(&trace.bufLock) + + startTheWorld() + + // The world is started but we've set trace.shutdown, so new tracing can't start. + // Wait for the trace reader to flush pending buffers and stop. + semacquire(&trace.shutdownSema, false) + if raceenabled { + raceacquire(unsafe.Pointer(&trace.shutdownSema)) + } + + // The lock protects us from races with StartTrace/StopTrace because they do stop-the-world. + lock(&trace.lock) + for _, p := range &allp { + if p == nil { + break + } + if p.tracebuf != 0 { + throw("trace: non-empty trace buffer in proc") + } + } + if trace.buf != 0 { + throw("trace: non-empty global trace buffer") + } + if trace.fullHead != 0 || trace.fullTail != 0 { + throw("trace: non-empty full trace buffer") + } + if trace.reading != 0 || trace.reader != nil { + throw("trace: reading after shutdown") + } + for trace.empty != 0 { + buf := trace.empty + trace.empty = buf.ptr().link + sysFree(unsafe.Pointer(buf), unsafe.Sizeof(*buf.ptr()), &memstats.other_sys) + } + trace.strings = nil + trace.shutdown = false + unlock(&trace.lock) +} + +// ReadTrace returns the next chunk of binary tracing data, blocking until data +// is available. If tracing is turned off and all the data accumulated while it +// was on has been returned, ReadTrace returns nil. The caller must copy the +// returned data before calling ReadTrace again. +// ReadTrace must be called from one goroutine at a time. +func ReadTrace() []byte { + // This function may need to lock trace.lock recursively + // (goparkunlock -> traceGoPark -> traceEvent -> traceFlush). + // To allow this we use trace.lockOwner. + // Also this function must not allocate while holding trace.lock: + // allocation can call heap allocate, which will try to emit a trace + // event while holding heap lock. + lock(&trace.lock) + trace.lockOwner = getg() + + if trace.reader != nil { + // More than one goroutine reads trace. This is bad. + // But we rather do not crash the program because of tracing, + // because tracing can be enabled at runtime on prod servers. + trace.lockOwner = nil + unlock(&trace.lock) + println("runtime: ReadTrace called from multiple goroutines simultaneously") + return nil + } + // Recycle the old buffer. + if buf := trace.reading; buf != 0 { + buf.ptr().link = trace.empty + trace.empty = buf + trace.reading = 0 + } + // Write trace header. + if !trace.headerWritten { + trace.headerWritten = true + trace.lockOwner = nil + unlock(&trace.lock) + return []byte("go 1.7 trace\x00\x00\x00\x00") + } + // Wait for new data. + if trace.fullHead == 0 && !trace.shutdown { + trace.reader = getg() + goparkunlock(&trace.lock, "trace reader (blocked)", traceEvGoBlock, 2) + lock(&trace.lock) + } + // Write a buffer. + if trace.fullHead != 0 { + buf := traceFullDequeue() + trace.reading = buf + trace.lockOwner = nil + unlock(&trace.lock) + return buf.ptr().arr[:buf.ptr().pos] + } + // Write footer with timer frequency. + if !trace.footerWritten { + trace.footerWritten = true + // Use float64 because (trace.ticksEnd - trace.ticksStart) * 1e9 can overflow int64. + freq := float64(trace.ticksEnd-trace.ticksStart) * 1e9 / float64(trace.timeEnd-trace.timeStart) / traceTickDiv + trace.lockOwner = nil + unlock(&trace.lock) + var data []byte + data = append(data, traceEvFrequency|0<<traceArgCountShift) + data = traceAppend(data, uint64(freq)) + if timers.gp != nil { + data = append(data, traceEvTimerGoroutine|0<<traceArgCountShift) + data = traceAppend(data, uint64(timers.gp.goid)) + } + // This will emit a bunch of full buffers, we will pick them up + // on the next iteration. + trace.stackTab.dump() + return data + } + // Done. + if trace.shutdown { + trace.lockOwner = nil + unlock(&trace.lock) + if raceenabled { + // Model synchronization on trace.shutdownSema, which race + // detector does not see. This is required to avoid false + // race reports on writer passed to trace.Start. + racerelease(unsafe.Pointer(&trace.shutdownSema)) + } + // trace.enabled is already reset, so can call traceable functions. + semrelease(&trace.shutdownSema) + return nil + } + // Also bad, but see the comment above. + trace.lockOwner = nil + unlock(&trace.lock) + println("runtime: spurious wakeup of trace reader") + return nil +} + +// traceReader returns the trace reader that should be woken up, if any. +func traceReader() *g { + if trace.reader == nil || (trace.fullHead == 0 && !trace.shutdown) { + return nil + } + lock(&trace.lock) + if trace.reader == nil || (trace.fullHead == 0 && !trace.shutdown) { + unlock(&trace.lock) + return nil + } + gp := trace.reader + trace.reader = nil + unlock(&trace.lock) + return gp +} + +// traceProcFree frees trace buffer associated with pp. +func traceProcFree(pp *p) { + buf := pp.tracebuf + pp.tracebuf = 0 + if buf == 0 { + return + } + lock(&trace.lock) + traceFullQueue(buf) + unlock(&trace.lock) +} + +// traceFullQueue queues buf into queue of full buffers. +func traceFullQueue(buf traceBufPtr) { + buf.ptr().link = 0 + if trace.fullHead == 0 { + trace.fullHead = buf + } else { + trace.fullTail.ptr().link = buf + } + trace.fullTail = buf +} + +// traceFullDequeue dequeues from queue of full buffers. +func traceFullDequeue() traceBufPtr { + buf := trace.fullHead + if buf == 0 { + return 0 + } + trace.fullHead = buf.ptr().link + if trace.fullHead == 0 { + trace.fullTail = 0 + } + buf.ptr().link = 0 + return buf +} + +// traceEvent writes a single event to trace buffer, flushing the buffer if necessary. +// ev is event type. +// If skip > 0, write current stack id as the last argument (skipping skip top frames). +// If skip = 0, this event type should contain a stack, but we don't want +// to collect and remember it for this particular call. +func traceEvent(ev byte, skip int, args ...uint64) { + mp, pid, bufp := traceAcquireBuffer() + // Double-check trace.enabled now that we've done m.locks++ and acquired bufLock. + // This protects from races between traceEvent and StartTrace/StopTrace. + + // The caller checked that trace.enabled == true, but trace.enabled might have been + // turned off between the check and now. Check again. traceLockBuffer did mp.locks++, + // StopTrace does stopTheWorld, and stopTheWorld waits for mp.locks to go back to zero, + // so if we see trace.enabled == true now, we know it's true for the rest of the function. + // Exitsyscall can run even during stopTheWorld. The race with StartTrace/StopTrace + // during tracing in exitsyscall is resolved by locking trace.bufLock in traceLockBuffer. + if !trace.enabled && !mp.startingtrace { + traceReleaseBuffer(pid) + return + } + buf := (*bufp).ptr() + const maxSize = 2 + 5*traceBytesPerNumber // event type, length, sequence, timestamp, stack id and two add params + if buf == nil || len(buf.arr)-buf.pos < maxSize { + buf = traceFlush(traceBufPtrOf(buf)).ptr() + (*bufp).set(buf) + } + + ticks := uint64(cputicks()) / traceTickDiv + tickDiff := ticks - buf.lastTicks + if buf.pos == 0 { + buf.byte(traceEvBatch | 1<<traceArgCountShift) + buf.varint(uint64(pid)) + buf.varint(ticks) + tickDiff = 0 + } + buf.lastTicks = ticks + narg := byte(len(args)) + if skip >= 0 { + narg++ + } + // We have only 2 bits for number of arguments. + // If number is >= 3, then the event type is followed by event length in bytes. + if narg > 3 { + narg = 3 + } + startPos := buf.pos + buf.byte(ev | narg<<traceArgCountShift) + var lenp *byte + if narg == 3 { + // Reserve the byte for length assuming that length < 128. + buf.varint(0) + lenp = &buf.arr[buf.pos-1] + } + buf.varint(tickDiff) + for _, a := range args { + buf.varint(a) + } + if skip == 0 { + buf.varint(0) + } else if skip > 0 { + _g_ := getg() + gp := mp.curg + var nstk int + if gp == _g_ { + nstk = callers(skip, buf.stk[:]) + } else if gp != nil { + gp = mp.curg + // This may happen when tracing a system call, + // so we must lock the stack. + if gcTryLockStackBarriers(gp) { + nstk = gcallers(gp, skip, buf.stk[:]) + gcUnlockStackBarriers(gp) + } + } + if nstk > 0 { + nstk-- // skip runtime.goexit + } + if nstk > 0 && gp.goid == 1 { + nstk-- // skip runtime.main + } + id := trace.stackTab.put(buf.stk[:nstk]) + buf.varint(uint64(id)) + } + evSize := buf.pos - startPos + if evSize > maxSize { + throw("invalid length of trace event") + } + if lenp != nil { + // Fill in actual length. + *lenp = byte(evSize - 2) + } + traceReleaseBuffer(pid) +} + +// traceAcquireBuffer returns trace buffer to use and, if necessary, locks it. +func traceAcquireBuffer() (mp *m, pid int32, bufp *traceBufPtr) { + mp = acquirem() + if p := mp.p.ptr(); p != nil { + return mp, p.id, &p.tracebuf + } + lock(&trace.bufLock) + return mp, traceGlobProc, &trace.buf +} + +// traceReleaseBuffer releases a buffer previously acquired with traceAcquireBuffer. +func traceReleaseBuffer(pid int32) { + if pid == traceGlobProc { + unlock(&trace.bufLock) + } + releasem(getg().m) +} + +// traceFlush puts buf onto stack of full buffers and returns an empty buffer. +func traceFlush(buf traceBufPtr) traceBufPtr { + owner := trace.lockOwner + dolock := owner == nil || owner != getg().m.curg + if dolock { + lock(&trace.lock) + } + if buf != 0 { + traceFullQueue(buf) + } + if trace.empty != 0 { + buf = trace.empty + trace.empty = buf.ptr().link + } else { + buf = traceBufPtr(sysAlloc(unsafe.Sizeof(traceBuf{}), &memstats.other_sys)) + if buf == 0 { + throw("trace: out of memory") + } + } + bufp := buf.ptr() + bufp.link.set(nil) + bufp.pos = 0 + bufp.lastTicks = 0 + if dolock { + unlock(&trace.lock) + } + return buf +} + +func traceString(buf *traceBuf, s string) (uint64, *traceBuf) { + if s == "" { + return 0, buf + } + if id, ok := trace.strings[s]; ok { + return id, buf + } + + trace.stringSeq++ + id := trace.stringSeq + trace.strings[s] = id + + size := 1 + 2*traceBytesPerNumber + len(s) + if len(buf.arr)-buf.pos < size { + buf = traceFlush(traceBufPtrOf(buf)).ptr() + } + buf.byte(traceEvString) + buf.varint(id) + buf.varint(uint64(len(s))) + buf.pos += copy(buf.arr[buf.pos:], s) + return id, buf +} + +// traceAppend appends v to buf in little-endian-base-128 encoding. +func traceAppend(buf []byte, v uint64) []byte { + for ; v >= 0x80; v >>= 7 { + buf = append(buf, 0x80|byte(v)) + } + buf = append(buf, byte(v)) + return buf +} + +// varint appends v to buf in little-endian-base-128 encoding. +func (buf *traceBuf) varint(v uint64) { + pos := buf.pos + for ; v >= 0x80; v >>= 7 { + buf.arr[pos] = 0x80 | byte(v) + pos++ + } + buf.arr[pos] = byte(v) + pos++ + buf.pos = pos +} + +// byte appends v to buf. +func (buf *traceBuf) byte(v byte) { + buf.arr[buf.pos] = v + buf.pos++ +} + +*/ + +// traceStackTable maps stack traces (arrays of PC's) to unique uint32 ids. +// It is lock-free for reading. +type traceStackTable struct { + lock mutex + seq uint32 + mem traceAlloc + tab [1 << 13]traceStackPtr +} + +// traceStack is a single stack in traceStackTable. +type traceStack struct { + link traceStackPtr + hash uintptr + id uint32 + n int + stk [0]uintptr // real type [n]uintptr +} + +type traceStackPtr uintptr + +/* +Commented out for gccgo for now. + +func (tp traceStackPtr) ptr() *traceStack { return (*traceStack)(unsafe.Pointer(tp)) } + +// stack returns slice of PCs. +func (ts *traceStack) stack() []uintptr { + return (*[traceStackSize]uintptr)(unsafe.Pointer(&ts.stk))[:ts.n] +} + +// put returns a unique id for the stack trace pcs and caches it in the table, +// if it sees the trace for the first time. +func (tab *traceStackTable) put(pcs []uintptr) uint32 { + if len(pcs) == 0 { + return 0 + } + hash := memhash(unsafe.Pointer(&pcs[0]), 0, uintptr(len(pcs))*unsafe.Sizeof(pcs[0])) + // First, search the hashtable w/o the mutex. + if id := tab.find(pcs, hash); id != 0 { + return id + } + // Now, double check under the mutex. + lock(&tab.lock) + if id := tab.find(pcs, hash); id != 0 { + unlock(&tab.lock) + return id + } + // Create new record. + tab.seq++ + stk := tab.newStack(len(pcs)) + stk.hash = hash + stk.id = tab.seq + stk.n = len(pcs) + stkpc := stk.stack() + for i, pc := range pcs { + stkpc[i] = pc + } + part := int(hash % uintptr(len(tab.tab))) + stk.link = tab.tab[part] + atomicstorep(unsafe.Pointer(&tab.tab[part]), unsafe.Pointer(stk)) + unlock(&tab.lock) + return stk.id +} + +// find checks if the stack trace pcs is already present in the table. +func (tab *traceStackTable) find(pcs []uintptr, hash uintptr) uint32 { + part := int(hash % uintptr(len(tab.tab))) +Search: + for stk := tab.tab[part].ptr(); stk != nil; stk = stk.link.ptr() { + if stk.hash == hash && stk.n == len(pcs) { + for i, stkpc := range stk.stack() { + if stkpc != pcs[i] { + continue Search + } + } + return stk.id + } + } + return 0 +} + +// newStack allocates a new stack of size n. +func (tab *traceStackTable) newStack(n int) *traceStack { + return (*traceStack)(tab.mem.alloc(unsafe.Sizeof(traceStack{}) + uintptr(n)*sys.PtrSize)) +} + +// dump writes all previously cached stacks to trace buffers, +// releases all memory and resets state. +func (tab *traceStackTable) dump() { + frames := make(map[uintptr]traceFrame) + var tmp [(2 + 4*traceStackSize) * traceBytesPerNumber]byte + buf := traceFlush(0).ptr() + for _, stk := range tab.tab { + stk := stk.ptr() + for ; stk != nil; stk = stk.link.ptr() { + tmpbuf := tmp[:0] + tmpbuf = traceAppend(tmpbuf, uint64(stk.id)) + tmpbuf = traceAppend(tmpbuf, uint64(stk.n)) + for _, pc := range stk.stack() { + var frame traceFrame + frame, buf = traceFrameForPC(buf, frames, pc) + tmpbuf = traceAppend(tmpbuf, uint64(pc)) + tmpbuf = traceAppend(tmpbuf, uint64(frame.funcID)) + tmpbuf = traceAppend(tmpbuf, uint64(frame.fileID)) + tmpbuf = traceAppend(tmpbuf, uint64(frame.line)) + } + // Now copy to the buffer. + size := 1 + traceBytesPerNumber + len(tmpbuf) + if len(buf.arr)-buf.pos < size { + buf = traceFlush(traceBufPtrOf(buf)).ptr() + } + buf.byte(traceEvStack | 3<<traceArgCountShift) + buf.varint(uint64(len(tmpbuf))) + buf.pos += copy(buf.arr[buf.pos:], tmpbuf) + } + } + + lock(&trace.lock) + traceFullQueue(traceBufPtrOf(buf)) + unlock(&trace.lock) + + tab.mem.drop() + *tab = traceStackTable{} +} + +type traceFrame struct { + funcID uint64 + fileID uint64 + line uint64 +} + +func traceFrameForPC(buf *traceBuf, frames map[uintptr]traceFrame, pc uintptr) (traceFrame, *traceBuf) { + if frame, ok := frames[pc]; ok { + return frame, buf + } + + var frame traceFrame + f := findfunc(pc) + if f == nil { + frames[pc] = frame + return frame, buf + } + + fn := funcname(f) + const maxLen = 1 << 10 + if len(fn) > maxLen { + fn = fn[len(fn)-maxLen:] + } + frame.funcID, buf = traceString(buf, fn) + file, line := funcline(f, pc-sys.PCQuantum) + frame.line = uint64(line) + if len(file) > maxLen { + file = file[len(file)-maxLen:] + } + frame.fileID, buf = traceString(buf, file) + return frame, buf +} + +*/ + +// traceAlloc is a non-thread-safe region allocator. +// It holds a linked list of traceAllocBlock. +type traceAlloc struct { + head traceAllocBlockPtr + off uintptr +} + +// traceAllocBlock is a block in traceAlloc. +// +// traceAllocBlock is allocated from non-GC'd memory, so it must not +// contain heap pointers. Writes to pointers to traceAllocBlocks do +// not need write barriers. +type traceAllocBlock struct { + next traceAllocBlockPtr + data [64<<10 - sys.PtrSize]byte +} + +type traceAllocBlockPtr uintptr + +func (p traceAllocBlockPtr) ptr() *traceAllocBlock { return (*traceAllocBlock)(unsafe.Pointer(p)) } +func (p *traceAllocBlockPtr) set(x *traceAllocBlock) { *p = traceAllocBlockPtr(unsafe.Pointer(x)) } + +/* +Commented out for gccgo for now. + +// alloc allocates n-byte block. +func (a *traceAlloc) alloc(n uintptr) unsafe.Pointer { + n = round(n, sys.PtrSize) + if a.head == 0 || a.off+n > uintptr(len(a.head.ptr().data)) { + if n > uintptr(len(a.head.ptr().data)) { + throw("trace: alloc too large") + } + block := (*traceAllocBlock)(sysAlloc(unsafe.Sizeof(traceAllocBlock{}), &memstats.other_sys)) + if block == nil { + throw("trace: out of memory") + } + block.next.set(a.head.ptr()) + a.head.set(block) + a.off = 0 + } + p := &a.head.ptr().data[a.off] + a.off += n + return unsafe.Pointer(p) +} + +// drop frees all previously allocated memory and resets the allocator. +func (a *traceAlloc) drop() { + for a.head != 0 { + block := a.head.ptr() + a.head.set(block.next.ptr()) + sysFree(unsafe.Pointer(block), unsafe.Sizeof(traceAllocBlock{}), &memstats.other_sys) + } +} + +// The following functions write specific events to trace. + +func traceGomaxprocs(procs int32) { + traceEvent(traceEvGomaxprocs, 1, uint64(procs)) +} + +func traceProcStart() { + traceEvent(traceEvProcStart, -1, uint64(getg().m.id)) +} + +func traceProcStop(pp *p) { + // Sysmon and stopTheWorld can stop Ps blocked in syscalls, + // to handle this we temporary employ the P. + mp := acquirem() + oldp := mp.p + mp.p.set(pp) + traceEvent(traceEvProcStop, -1) + mp.p = oldp + releasem(mp) +} + +func traceGCStart() { + traceEvent(traceEvGCStart, 3, trace.seqGC) + trace.seqGC++ +} + +func traceGCDone() { + traceEvent(traceEvGCDone, -1) +} + +func traceGCScanStart() { + traceEvent(traceEvGCScanStart, -1) +} + +func traceGCScanDone() { + traceEvent(traceEvGCScanDone, -1) +} + +func traceGCSweepStart() { + traceEvent(traceEvGCSweepStart, 1) +} + +func traceGCSweepDone() { + traceEvent(traceEvGCSweepDone, -1) +} + +func traceGoCreate(newg *g, pc uintptr) { + newg.traceseq = 0 + newg.tracelastp = getg().m.p + // +PCQuantum because traceFrameForPC expects return PCs and subtracts PCQuantum. + id := trace.stackTab.put([]uintptr{pc + sys.PCQuantum}) + traceEvent(traceEvGoCreate, 2, uint64(newg.goid), uint64(id)) +} + +func traceGoStart() { + _g_ := getg().m.curg + _p_ := _g_.m.p + _g_.traceseq++ + if _g_.tracelastp == _p_ { + traceEvent(traceEvGoStartLocal, -1, uint64(_g_.goid)) + } else { + _g_.tracelastp = _p_ + traceEvent(traceEvGoStart, -1, uint64(_g_.goid), _g_.traceseq) + } +} + +func traceGoEnd() { + traceEvent(traceEvGoEnd, -1) +} + +func traceGoSched() { + _g_ := getg() + _g_.tracelastp = _g_.m.p + traceEvent(traceEvGoSched, 1) +} + +func traceGoPreempt() { + _g_ := getg() + _g_.tracelastp = _g_.m.p + traceEvent(traceEvGoPreempt, 1) +} + +func traceGoPark(traceEv byte, skip int, gp *g) { + if traceEv&traceFutileWakeup != 0 { + traceEvent(traceEvFutileWakeup, -1) + } + traceEvent(traceEv & ^traceFutileWakeup, skip) +} + +func traceGoUnpark(gp *g, skip int) { + _p_ := getg().m.p + gp.traceseq++ + if gp.tracelastp == _p_ { + traceEvent(traceEvGoUnblockLocal, skip, uint64(gp.goid)) + } else { + gp.tracelastp = _p_ + traceEvent(traceEvGoUnblock, skip, uint64(gp.goid), gp.traceseq) + } +} + +func traceGoSysCall() { + traceEvent(traceEvGoSysCall, 1) +} + +func traceGoSysExit(ts int64) { + if ts != 0 && ts < trace.ticksStart { + // There is a race between the code that initializes sysexitticks + // (in exitsyscall, which runs without a P, and therefore is not + // stopped with the rest of the world) and the code that initializes + // a new trace. The recorded sysexitticks must therefore be treated + // as "best effort". If they are valid for this trace, then great, + // use them for greater accuracy. But if they're not valid for this + // trace, assume that the trace was started after the actual syscall + // exit (but before we actually managed to start the goroutine, + // aka right now), and assign a fresh time stamp to keep the log consistent. + ts = 0 + } + _g_ := getg().m.curg + _g_.traceseq++ + _g_.tracelastp = _g_.m.p + traceEvent(traceEvGoSysExit, -1, uint64(_g_.goid), _g_.traceseq, uint64(ts)/traceTickDiv) +} + +func traceGoSysBlock(pp *p) { + // Sysmon and stopTheWorld can declare syscalls running on remote Ps as blocked, + // to handle this we temporary employ the P. + mp := acquirem() + oldp := mp.p + mp.p.set(pp) + traceEvent(traceEvGoSysBlock, -1) + mp.p = oldp + releasem(mp) +} + +func traceHeapAlloc() { + traceEvent(traceEvHeapAlloc, -1, memstats.heap_live) +} + +func traceNextGC() { + traceEvent(traceEvNextGC, -1, memstats.next_gc) +} + +*/ diff --git a/libgo/runtime/chan.goc b/libgo/runtime/chan.goc deleted file mode 100644 index 6e4c8fd..0000000 --- a/libgo/runtime/chan.goc +++ /dev/null @@ -1,1130 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package runtime -#include "runtime.h" -#include "arch.h" -#include "go-type.h" -#include "malloc.h" -#include "chan.h" - -uint32 runtime_Hchansize = sizeof(Hchan); - -static void dequeueg(WaitQ*); -static SudoG* dequeue(WaitQ*); -static void enqueue(WaitQ*, SudoG*); - -static Hchan* -makechan(ChanType *t, int64 hint) -{ - Hchan *c; - uintptr n; - const Type *elem; - - elem = t->__element_type; - - // compiler checks this but be safe. - if(elem->__size >= (1<<16)) - runtime_throw("makechan: invalid channel element type"); - - if(hint < 0 || (intgo)hint != hint || (elem->__size > 0 && (uintptr)hint > (MaxMem - sizeof(*c)) / elem->__size)) - runtime_panicstring("makechan: size out of range"); - - n = sizeof(*c); - n = ROUND(n, elem->__align); - - // allocate memory in one call - c = (Hchan*)runtime_mallocgc(sizeof(*c) + hint*elem->__size, (uintptr)t | TypeInfo_Chan, 0); - c->elemsize = elem->__size; - c->elemtype = elem; - c->dataqsiz = hint; - - if(debug) - runtime_printf("makechan: chan=%p; elemsize=%D; dataqsiz=%D\n", - c, (int64)elem->__size, (int64)c->dataqsiz); - - return c; -} - -func reflect.makechan(t *ChanType, size uint64) (c *Hchan) { - c = makechan(t, size); -} - -Hchan* -__go_new_channel(ChanType *t, uintptr hint) -{ - return makechan(t, hint); -} - -Hchan* -__go_new_channel_big(ChanType *t, uint64 hint) -{ - return makechan(t, hint); -} - -/* - * generic single channel send/recv - * if the bool pointer is nil, - * then the full exchange will - * occur. if pres is not nil, - * then the protocol will not - * sleep but return if it could - * not complete. - * - * sleep can wake up with g->param == nil - * when a channel involved in the sleep has - * been closed. it is easiest to loop and re-run - * the operation; we'll see that it's now closed. - */ -static bool -chansend(ChanType *t, Hchan *c, byte *ep, bool block, void *pc) -{ - USED(pc); - SudoG *sg; - SudoG mysg; - G* gp; - int64 t0; - G* g; - - g = runtime_g(); - - if(c == nil) { - USED(t); - if(!block) - return false; - runtime_park(nil, nil, "chan send (nil chan)"); - return false; // not reached - } - - if(runtime_gcwaiting()) - runtime_gosched(); - - if(debug) { - runtime_printf("chansend: chan=%p\n", c); - } - - t0 = 0; - mysg.releasetime = 0; - if(runtime_blockprofilerate > 0) { - t0 = runtime_cputicks(); - mysg.releasetime = -1; - } - - runtime_lock(c); - if(c->closed) - goto closed; - - if(c->dataqsiz > 0) - goto asynch; - - sg = dequeue(&c->recvq); - if(sg != nil) { - runtime_unlock(c); - - gp = sg->g; - gp->param = sg; - if(sg->elem != nil) - runtime_memmove(sg->elem, ep, c->elemsize); - if(sg->releasetime) - sg->releasetime = runtime_cputicks(); - runtime_ready(gp); - return true; - } - - if(!block) { - runtime_unlock(c); - return false; - } - - mysg.elem = ep; - mysg.g = g; - mysg.selectdone = nil; - g->param = nil; - enqueue(&c->sendq, &mysg); - runtime_parkunlock(c, "chan send"); - - if(g->param == nil) { - runtime_lock(c); - if(!c->closed) - runtime_throw("chansend: spurious wakeup"); - goto closed; - } - - if(mysg.releasetime > 0) - runtime_blockevent(mysg.releasetime - t0, 2); - - return true; - -asynch: - if(c->closed) - goto closed; - - if(c->qcount >= c->dataqsiz) { - if(!block) { - runtime_unlock(c); - return false; - } - mysg.g = g; - mysg.elem = nil; - mysg.selectdone = nil; - enqueue(&c->sendq, &mysg); - runtime_parkunlock(c, "chan send"); - - runtime_lock(c); - goto asynch; - } - - runtime_memmove(chanbuf(c, c->sendx), ep, c->elemsize); - if(++c->sendx == c->dataqsiz) - c->sendx = 0; - c->qcount++; - - sg = dequeue(&c->recvq); - if(sg != nil) { - gp = sg->g; - runtime_unlock(c); - if(sg->releasetime) - sg->releasetime = runtime_cputicks(); - runtime_ready(gp); - } else - runtime_unlock(c); - if(mysg.releasetime > 0) - runtime_blockevent(mysg.releasetime - t0, 2); - return true; - -closed: - runtime_unlock(c); - runtime_panicstring("send on closed channel"); - return false; // not reached -} - - -static bool -chanrecv(ChanType *t, Hchan* c, byte *ep, bool block, bool *received) -{ - SudoG *sg; - SudoG mysg; - G *gp; - int64 t0; - G *g; - - if(runtime_gcwaiting()) - runtime_gosched(); - - if(debug) - runtime_printf("chanrecv: chan=%p\n", c); - - g = runtime_g(); - - if(c == nil) { - USED(t); - if(!block) - return false; - runtime_park(nil, nil, "chan receive (nil chan)"); - return false; // not reached - } - - t0 = 0; - mysg.releasetime = 0; - if(runtime_blockprofilerate > 0) { - t0 = runtime_cputicks(); - mysg.releasetime = -1; - } - - runtime_lock(c); - if(c->dataqsiz > 0) - goto asynch; - - if(c->closed) - goto closed; - - sg = dequeue(&c->sendq); - if(sg != nil) { - runtime_unlock(c); - - if(ep != nil) - runtime_memmove(ep, sg->elem, c->elemsize); - gp = sg->g; - gp->param = sg; - if(sg->releasetime) - sg->releasetime = runtime_cputicks(); - runtime_ready(gp); - - if(received != nil) - *received = true; - return true; - } - - if(!block) { - runtime_unlock(c); - return false; - } - - mysg.elem = ep; - mysg.g = g; - mysg.selectdone = nil; - g->param = nil; - enqueue(&c->recvq, &mysg); - runtime_parkunlock(c, "chan receive"); - - if(g->param == nil) { - runtime_lock(c); - if(!c->closed) - runtime_throw("chanrecv: spurious wakeup"); - goto closed; - } - - if(received != nil) - *received = true; - if(mysg.releasetime > 0) - runtime_blockevent(mysg.releasetime - t0, 2); - return true; - -asynch: - if(c->qcount <= 0) { - if(c->closed) - goto closed; - - if(!block) { - runtime_unlock(c); - if(received != nil) - *received = false; - return false; - } - mysg.g = g; - mysg.elem = nil; - mysg.selectdone = nil; - enqueue(&c->recvq, &mysg); - runtime_parkunlock(c, "chan receive"); - - runtime_lock(c); - goto asynch; - } - - if(ep != nil) - runtime_memmove(ep, chanbuf(c, c->recvx), c->elemsize); - runtime_memclr(chanbuf(c, c->recvx), c->elemsize); - if(++c->recvx == c->dataqsiz) - c->recvx = 0; - c->qcount--; - - sg = dequeue(&c->sendq); - if(sg != nil) { - gp = sg->g; - runtime_unlock(c); - if(sg->releasetime) - sg->releasetime = runtime_cputicks(); - runtime_ready(gp); - } else - runtime_unlock(c); - - if(received != nil) - *received = true; - if(mysg.releasetime > 0) - runtime_blockevent(mysg.releasetime - t0, 2); - return true; - -closed: - if(ep != nil) - runtime_memclr(ep, c->elemsize); - if(received != nil) - *received = false; - runtime_unlock(c); - if(mysg.releasetime > 0) - runtime_blockevent(mysg.releasetime - t0, 2); - return true; -} - -// The compiler generates a call to __go_send_small to send a value 8 -// bytes or smaller. -void -__go_send_small(ChanType *t, Hchan* c, uint64 val) -{ - union - { - byte b[sizeof(uint64)]; - uint64 v; - } u; - byte *v; - - u.v = val; -#ifndef WORDS_BIGENDIAN - v = u.b; -#else - v = u.b + sizeof(uint64) - t->__element_type->__size; -#endif - chansend(t, c, v, true, runtime_getcallerpc(&t)); -} - -// The compiler generates a call to __go_send_big to send a value -// larger than 8 bytes or smaller. -void -__go_send_big(ChanType *t, Hchan* c, byte* v) -{ - chansend(t, c, v, true, runtime_getcallerpc(&t)); -} - -// The compiler generates a call to __go_receive to receive a -// value from a channel. -void -__go_receive(ChanType *t, Hchan* c, byte* v) -{ - chanrecv(t, c, v, true, nil); -} - -_Bool runtime_chanrecv2(ChanType *t, Hchan* c, byte* v) - __asm__ (GOSYM_PREFIX "runtime.chanrecv2"); - -_Bool -runtime_chanrecv2(ChanType *t, Hchan* c, byte* v) -{ - bool received = false; - - chanrecv(t, c, v, true, &received); - return received; -} - -// compiler implements -// -// select { -// case c <- v: -// ... foo -// default: -// ... bar -// } -// -// as -// -// if selectnbsend(c, v) { -// ... foo -// } else { -// ... bar -// } -// -func selectnbsend(t *ChanType, c *Hchan, elem *byte) (selected bool) { - selected = chansend(t, c, elem, false, runtime_getcallerpc(&t)); -} - -// compiler implements -// -// select { -// case v = <-c: -// ... foo -// default: -// ... bar -// } -// -// as -// -// if selectnbrecv(&v, c) { -// ... foo -// } else { -// ... bar -// } -// -func selectnbrecv(t *ChanType, elem *byte, c *Hchan) (selected bool) { - selected = chanrecv(t, c, elem, false, nil); -} - -// compiler implements -// -// select { -// case v, ok = <-c: -// ... foo -// default: -// ... bar -// } -// -// as -// -// if c != nil && selectnbrecv2(&v, &ok, c) { -// ... foo -// } else { -// ... bar -// } -// -func selectnbrecv2(t *ChanType, elem *byte, received *bool, c *Hchan) (selected bool) { - bool r; - - selected = chanrecv(t, c, elem, false, received == nil ? nil : &r); - if(received != nil) - *received = r; -} - -func reflect.chansend(t *ChanType, c *Hchan, elem *byte, nb bool) (selected bool) { - selected = chansend(t, c, elem, !nb, runtime_getcallerpc(&t)); -} - -func reflect.chanrecv(t *ChanType, c *Hchan, nb bool, elem *byte) (selected bool, received bool) { - received = false; - selected = chanrecv(t, c, elem, !nb, &received); -} - -static Select* newselect(int32); - -func newselect(size int32) (sel *byte) { - sel = (byte*)newselect(size); -} - -static Select* -newselect(int32 size) -{ - int32 n; - Select *sel; - - n = 0; - if(size > 1) - n = size-1; - - // allocate all the memory we need in a single allocation - // start with Select with size cases - // then lockorder with size entries - // then pollorder with size entries - sel = runtime_mal(sizeof(*sel) + - n*sizeof(sel->scase[0]) + - size*sizeof(sel->lockorder[0]) + - size*sizeof(sel->pollorder[0])); - - sel->tcase = size; - sel->ncase = 0; - sel->lockorder = (void*)(sel->scase + size); - sel->pollorder = (void*)(sel->lockorder + size); - - if(debug) - runtime_printf("newselect s=%p size=%d\n", sel, size); - return sel; -} - -// cut in half to give stack a chance to split -static void selectsend(Select *sel, Hchan *c, int index, void *elem); - -func selectsend(sel *Select, c *Hchan, elem *byte, index int32) { - // nil cases do not compete - if(c != nil) - selectsend(sel, c, index, elem); -} - -static void -selectsend(Select *sel, Hchan *c, int index, void *elem) -{ - int32 i; - Scase *cas; - - i = sel->ncase; - if(i >= sel->tcase) - runtime_throw("selectsend: too many cases"); - sel->ncase = i+1; - cas = &sel->scase[i]; - - cas->index = index; - cas->chan = c; - cas->kind = CaseSend; - cas->sg.elem = elem; - - if(debug) - runtime_printf("selectsend s=%p index=%d chan=%p\n", - sel, cas->index, cas->chan); -} - -// cut in half to give stack a chance to split -static void selectrecv(Select *sel, Hchan *c, int index, void *elem, bool*); - -func selectrecv(sel *Select, c *Hchan, elem *byte, index int32) { - // nil cases do not compete - if(c != nil) - selectrecv(sel, c, index, elem, nil); -} - -func selectrecv2(sel *Select, c *Hchan, elem *byte, received *bool, index int32) { - // nil cases do not compete - if(c != nil) - selectrecv(sel, c, index, elem, received); -} - -static void -selectrecv(Select *sel, Hchan *c, int index, void *elem, bool *received) -{ - int32 i; - Scase *cas; - - i = sel->ncase; - if(i >= sel->tcase) - runtime_throw("selectrecv: too many cases"); - sel->ncase = i+1; - cas = &sel->scase[i]; - cas->index = index; - cas->chan = c; - - cas->kind = CaseRecv; - cas->sg.elem = elem; - cas->receivedp = received; - - if(debug) - runtime_printf("selectrecv s=%p index=%d chan=%p\n", - sel, cas->index, cas->chan); -} - -// cut in half to give stack a chance to split -static void selectdefault(Select*, int); - -func selectdefault(sel *Select, index int32) { - selectdefault(sel, index); -} - -static void -selectdefault(Select *sel, int32 index) -{ - int32 i; - Scase *cas; - - i = sel->ncase; - if(i >= sel->tcase) - runtime_throw("selectdefault: too many cases"); - sel->ncase = i+1; - cas = &sel->scase[i]; - cas->index = index; - cas->chan = nil; - - cas->kind = CaseDefault; - - if(debug) - runtime_printf("selectdefault s=%p index=%d\n", - sel, cas->index); -} - -static void -sellock(Select *sel) -{ - uint32 i; - Hchan *c, *c0; - - c = nil; - for(i=0; i<sel->ncase; i++) { - c0 = sel->lockorder[i]; - if(c0 && c0 != c) { - c = sel->lockorder[i]; - runtime_lock(c); - } - } -} - -static void -selunlock(Select *sel) -{ - int32 i, n, r; - Hchan *c; - - // We must be very careful here to not touch sel after we have unlocked - // the last lock, because sel can be freed right after the last unlock. - // Consider the following situation. - // First M calls runtime_park() in runtime_selectgo() passing the sel. - // Once runtime_park() has unlocked the last lock, another M makes - // the G that calls select runnable again and schedules it for execution. - // When the G runs on another M, it locks all the locks and frees sel. - // Now if the first M touches sel, it will access freed memory. - n = (int32)sel->ncase; - r = 0; - // skip the default case - if(n>0 && sel->lockorder[0] == nil) - r = 1; - for(i = n-1; i >= r; i--) { - c = sel->lockorder[i]; - if(i>0 && sel->lockorder[i-1] == c) - continue; // will unlock it on the next iteration - runtime_unlock(c); - } -} - -static bool -selparkcommit(G *gp, void *sel) -{ - USED(gp); - selunlock(sel); - return true; -} - -func block() { - runtime_park(nil, nil, "select (no cases)"); // forever -} - -static int selectgo(Select**); - -// selectgo(sel *byte); - -func selectgo(sel *Select) (ret int32) { - return selectgo(&sel); -} - -static int -selectgo(Select **selp) -{ - Select *sel; - uint32 o, i, j, k, done; - int64 t0; - Scase *cas, *dfl; - Hchan *c; - SudoG *sg; - G *gp; - int index; - G *g; - - sel = *selp; - if(runtime_gcwaiting()) - runtime_gosched(); - - if(debug) - runtime_printf("select: sel=%p\n", sel); - - g = runtime_g(); - - t0 = 0; - if(runtime_blockprofilerate > 0) { - t0 = runtime_cputicks(); - for(i=0; i<sel->ncase; i++) - sel->scase[i].sg.releasetime = -1; - } - - // The compiler rewrites selects that statically have - // only 0 or 1 cases plus default into simpler constructs. - // The only way we can end up with such small sel->ncase - // values here is for a larger select in which most channels - // have been nilled out. The general code handles those - // cases correctly, and they are rare enough not to bother - // optimizing (and needing to test). - - // generate permuted order - for(i=0; i<sel->ncase; i++) - sel->pollorder[i] = i; - for(i=1; i<sel->ncase; i++) { - o = sel->pollorder[i]; - j = runtime_fastrand1()%(i+1); - sel->pollorder[i] = sel->pollorder[j]; - sel->pollorder[j] = o; - } - - // sort the cases by Hchan address to get the locking order. - // simple heap sort, to guarantee n log n time and constant stack footprint. - for(i=0; i<sel->ncase; i++) { - j = i; - c = sel->scase[j].chan; - while(j > 0 && sel->lockorder[k=(j-1)/2] < c) { - sel->lockorder[j] = sel->lockorder[k]; - j = k; - } - sel->lockorder[j] = c; - } - for(i=sel->ncase; i-->0; ) { - c = sel->lockorder[i]; - sel->lockorder[i] = sel->lockorder[0]; - j = 0; - for(;;) { - k = j*2+1; - if(k >= i) - break; - if(k+1 < i && sel->lockorder[k] < sel->lockorder[k+1]) - k++; - if(c < sel->lockorder[k]) { - sel->lockorder[j] = sel->lockorder[k]; - j = k; - continue; - } - break; - } - sel->lockorder[j] = c; - } - /* - for(i=0; i+1<sel->ncase; i++) - if(sel->lockorder[i] > sel->lockorder[i+1]) { - runtime_printf("i=%d %p %p\n", i, sel->lockorder[i], sel->lockorder[i+1]); - runtime_throw("select: broken sort"); - } - */ - sellock(sel); - -loop: - // pass 1 - look for something already waiting - dfl = nil; - for(i=0; i<sel->ncase; i++) { - o = sel->pollorder[i]; - cas = &sel->scase[o]; - c = cas->chan; - - switch(cas->kind) { - case CaseRecv: - if(c->dataqsiz > 0) { - if(c->qcount > 0) - goto asyncrecv; - } else { - sg = dequeue(&c->sendq); - if(sg != nil) - goto syncrecv; - } - if(c->closed) - goto rclose; - break; - - case CaseSend: - if(c->closed) - goto sclose; - if(c->dataqsiz > 0) { - if(c->qcount < c->dataqsiz) - goto asyncsend; - } else { - sg = dequeue(&c->recvq); - if(sg != nil) - goto syncsend; - } - break; - - case CaseDefault: - dfl = cas; - break; - } - } - - if(dfl != nil) { - selunlock(sel); - cas = dfl; - goto retc; - } - - - // pass 2 - enqueue on all chans - done = 0; - for(i=0; i<sel->ncase; i++) { - o = sel->pollorder[i]; - cas = &sel->scase[o]; - c = cas->chan; - sg = &cas->sg; - sg->g = g; - sg->selectdone = &done; - - switch(cas->kind) { - case CaseRecv: - enqueue(&c->recvq, sg); - break; - - case CaseSend: - enqueue(&c->sendq, sg); - break; - } - } - - g->param = nil; - runtime_park(selparkcommit, sel, "select"); - - sellock(sel); - sg = g->param; - - // pass 3 - dequeue from unsuccessful chans - // otherwise they stack up on quiet channels - for(i=0; i<sel->ncase; i++) { - cas = &sel->scase[i]; - if(cas != (Scase*)sg) { - c = cas->chan; - if(cas->kind == CaseSend) - dequeueg(&c->sendq); - else - dequeueg(&c->recvq); - } - } - - if(sg == nil) - goto loop; - - cas = (Scase*)sg; - c = cas->chan; - - if(c->dataqsiz > 0) - runtime_throw("selectgo: shouldn't happen"); - - if(debug) - runtime_printf("wait-return: sel=%p c=%p cas=%p kind=%d\n", - sel, c, cas, cas->kind); - - if(cas->kind == CaseRecv) { - if(cas->receivedp != nil) - *cas->receivedp = true; - } - - selunlock(sel); - goto retc; - -asyncrecv: - // can receive from buffer - if(cas->receivedp != nil) - *cas->receivedp = true; - if(cas->sg.elem != nil) - runtime_memmove(cas->sg.elem, chanbuf(c, c->recvx), c->elemsize); - runtime_memclr(chanbuf(c, c->recvx), c->elemsize); - if(++c->recvx == c->dataqsiz) - c->recvx = 0; - c->qcount--; - sg = dequeue(&c->sendq); - if(sg != nil) { - gp = sg->g; - selunlock(sel); - if(sg->releasetime) - sg->releasetime = runtime_cputicks(); - runtime_ready(gp); - } else { - selunlock(sel); - } - goto retc; - -asyncsend: - // can send to buffer - runtime_memmove(chanbuf(c, c->sendx), cas->sg.elem, c->elemsize); - if(++c->sendx == c->dataqsiz) - c->sendx = 0; - c->qcount++; - sg = dequeue(&c->recvq); - if(sg != nil) { - gp = sg->g; - selunlock(sel); - if(sg->releasetime) - sg->releasetime = runtime_cputicks(); - runtime_ready(gp); - } else { - selunlock(sel); - } - goto retc; - -syncrecv: - // can receive from sleeping sender (sg) - selunlock(sel); - if(debug) - runtime_printf("syncrecv: sel=%p c=%p o=%d\n", sel, c, o); - if(cas->receivedp != nil) - *cas->receivedp = true; - if(cas->sg.elem != nil) - runtime_memmove(cas->sg.elem, sg->elem, c->elemsize); - gp = sg->g; - gp->param = sg; - if(sg->releasetime) - sg->releasetime = runtime_cputicks(); - runtime_ready(gp); - goto retc; - -rclose: - // read at end of closed channel - selunlock(sel); - if(cas->receivedp != nil) - *cas->receivedp = false; - if(cas->sg.elem != nil) - runtime_memclr(cas->sg.elem, c->elemsize); - goto retc; - -syncsend: - // can send to sleeping receiver (sg) - selunlock(sel); - if(debug) - runtime_printf("syncsend: sel=%p c=%p o=%d\n", sel, c, o); - if(sg->elem != nil) - runtime_memmove(sg->elem, cas->sg.elem, c->elemsize); - gp = sg->g; - gp->param = sg; - if(sg->releasetime) - sg->releasetime = runtime_cputicks(); - runtime_ready(gp); - -retc: - // return index corresponding to chosen case - index = cas->index; - if(cas->sg.releasetime > 0) - runtime_blockevent(cas->sg.releasetime - t0, 2); - runtime_free(sel); - return index; - -sclose: - // send on closed channel - selunlock(sel); - runtime_panicstring("send on closed channel"); - return 0; // not reached -} - -// This struct must match ../reflect/value.go:/runtimeSelect. -typedef struct runtimeSelect runtimeSelect; -struct runtimeSelect -{ - intgo dir; - ChanType *typ; - Hchan *ch; - byte *val; -}; - -// This enum must match ../reflect/value.go:/SelectDir. -enum SelectDir { - SelectSend = 1, - SelectRecv, - SelectDefault, -}; - -func reflect.rselect(cases Slice) (chosen int, recvOK bool) { - int32 i; - Select *sel; - runtimeSelect* rcase, *rc; - - chosen = -1; - recvOK = false; - - rcase = (runtimeSelect*)cases.__values; - - sel = newselect(cases.__count); - for(i=0; i<cases.__count; i++) { - rc = &rcase[i]; - switch(rc->dir) { - case SelectDefault: - selectdefault(sel, i); - break; - case SelectSend: - if(rc->ch == nil) - break; - selectsend(sel, rc->ch, i, rc->val); - break; - case SelectRecv: - if(rc->ch == nil) - break; - selectrecv(sel, rc->ch, i, rc->val, &recvOK); - break; - } - } - - chosen = (intgo)(uintptr)selectgo(&sel); -} - -static void closechan(Hchan *c, void *pc); - -func closechan(c *Hchan) { - closechan(c, runtime_getcallerpc(&c)); -} - -func reflect.chanclose(c *Hchan) { - closechan(c, runtime_getcallerpc(&c)); -} - -static void -closechan(Hchan *c, void *pc) -{ - USED(pc); - SudoG *sg; - G* gp; - - if(c == nil) - runtime_panicstring("close of nil channel"); - - if(runtime_gcwaiting()) - runtime_gosched(); - - runtime_lock(c); - if(c->closed) { - runtime_unlock(c); - runtime_panicstring("close of closed channel"); - } - c->closed = true; - - // release all readers - for(;;) { - sg = dequeue(&c->recvq); - if(sg == nil) - break; - gp = sg->g; - gp->param = nil; - if(sg->releasetime) - sg->releasetime = runtime_cputicks(); - runtime_ready(gp); - } - - // release all writers - for(;;) { - sg = dequeue(&c->sendq); - if(sg == nil) - break; - gp = sg->g; - gp->param = nil; - if(sg->releasetime) - sg->releasetime = runtime_cputicks(); - runtime_ready(gp); - } - - runtime_unlock(c); -} - -void -__go_builtin_close(Hchan *c) -{ - runtime_closechan(c); -} - -func reflect.chanlen(c *Hchan) (len int) { - if(c == nil) - len = 0; - else - len = c->qcount; -} - -func reflect.chancap(c *Hchan) (cap int) { - if(c == nil) - cap = 0; - else - cap = c->dataqsiz; -} - -intgo -__go_chan_cap(Hchan *c) -{ - return reflect_chancap(c); -} - -static SudoG* -dequeue(WaitQ *q) -{ - SudoG *sgp; - -loop: - sgp = q->first; - if(sgp == nil) - return nil; - q->first = sgp->link; - - // if sgp participates in a select and is already signaled, ignore it - if(sgp->selectdone != nil) { - // claim the right to signal - if(*sgp->selectdone != 0 || !runtime_cas(sgp->selectdone, 0, 1)) - goto loop; - } - - return sgp; -} - -static void -dequeueg(WaitQ *q) -{ - SudoG **l, *sgp, *prevsgp; - G *g; - - g = runtime_g(); - prevsgp = nil; - for(l=&q->first; (sgp=*l) != nil; l=&sgp->link, prevsgp=sgp) { - if(sgp->g == g) { - *l = sgp->link; - if(q->last == sgp) - q->last = prevsgp; - break; - } - } -} - -static void -enqueue(WaitQ *q, SudoG *sgp) -{ - sgp->link = nil; - if(q->first == nil) { - q->first = sgp; - q->last = sgp; - return; - } - q->last->link = sgp; - q->last = sgp; -} diff --git a/libgo/runtime/chan.h b/libgo/runtime/chan.h deleted file mode 100644 index 473f365..0000000 --- a/libgo/runtime/chan.h +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -typedef struct WaitQ WaitQ; -typedef struct SudoG SudoG; -typedef struct Select Select; -typedef struct Scase Scase; - -typedef struct __go_type_descriptor Type; -typedef struct __go_channel_type ChanType; - -struct SudoG -{ - G* g; - uint32* selectdone; - SudoG* link; - int64 releasetime; - byte* elem; // data element - uint32 ticket; -}; - -struct WaitQ -{ - SudoG* first; - SudoG* last; -}; - -// The garbage collector is assuming that Hchan can only contain pointers into the stack -// and cannot contain pointers into the heap. -struct Hchan -{ - uintgo qcount; // total data in the q - uintgo dataqsiz; // size of the circular q - uint16 elemsize; - uint16 pad; // ensures proper alignment of the buffer that follows Hchan in memory - bool closed; - const Type* elemtype; // element type - uintgo sendx; // send index - uintgo recvx; // receive index - WaitQ recvq; // list of recv waiters - WaitQ sendq; // list of send waiters - Lock; -}; - -// Buffer follows Hchan immediately in memory. -// chanbuf(c, i) is pointer to the i'th slot in the buffer. -#define chanbuf(c, i) ((byte*)((c)+1)+(uintptr)(c)->elemsize*(i)) - -enum -{ - debug = 0, - - // Scase.kind - CaseRecv, - CaseSend, - CaseDefault, -}; - -struct Scase -{ - SudoG sg; // must be first member (cast to Scase) - Hchan* chan; // chan - uint16 kind; - uint16 index; // index to return - bool* receivedp; // pointer to received bool (recv2) -}; - -struct Select -{ - uint16 tcase; // total count of scase[] - uint16 ncase; // currently filled scase[] - uint16* pollorder; // case poll order - Hchan** lockorder; // channel lock order - Scase scase[1]; // one per case (in order of appearance) -}; diff --git a/libgo/runtime/go-cgo.c b/libgo/runtime/go-cgo.c index 7d0494c..a7d87a7 100644 --- a/libgo/runtime/go-cgo.c +++ b/libgo/runtime/go-cgo.c @@ -10,7 +10,8 @@ #include "go-panic.h" #include "go-type.h" -extern void __go_receive (ChanType *, Hchan *, byte *); +extern void chanrecv1 (ChanType *, Hchan *, void *) + __asm__ (GOSYM_PREFIX "runtime.chanrecv1"); /* Prepare to call from code written in Go to code written in C or C++. This takes the current goroutine out of the Go scheduler, as @@ -97,7 +98,7 @@ syscall_cgocallback () Go. In the case of -buildmode=c-archive or c-shared, this call may be coming in before package initialization is complete. Wait until it is. */ - __go_receive (NULL, runtime_main_init_done, NULL); + chanrecv1 (NULL, runtime_main_init_done, NULL); } mp = runtime_m (); diff --git a/libgo/runtime/heapdump.c b/libgo/runtime/heapdump.c index 18fe913..3cc0c1d 100644 --- a/libgo/runtime/heapdump.c +++ b/libgo/runtime/heapdump.c @@ -462,7 +462,7 @@ dumpparams(void) else dumpbool(true); // big-endian ptrs dumpint(PtrSize); - dumpint(runtime_Hchansize); + dumpint(hchanSize); dumpint((uintptr)runtime_mheap.arena_start); dumpint((uintptr)runtime_mheap.arena_used); dumpint(0); @@ -769,7 +769,7 @@ dumpefacetypes(void *obj __attribute__ ((unused)), uintptr size, const Type *typ case TypeInfo_Chan: if(type->__size == 0) // channels may have zero-sized objects in them break; - for(i = runtime_Hchansize; i <= size - type->__size; i += type->__size) { + for(i = hchanSize; i <= size - type->__size; i += type->__size) { //playgcprog(i, (uintptr*)type->gc + 1, dumpeface_callback, obj); } break; diff --git a/libgo/runtime/mgc0.c b/libgo/runtime/mgc0.c index 341544c..ac6e396 100644 --- a/libgo/runtime/mgc0.c +++ b/libgo/runtime/mgc0.c @@ -56,7 +56,6 @@ #include "arch.h" #include "malloc.h" #include "mgc0.h" -#include "chan.h" #include "go-type.h" // Map gccgo field names to gc field names. @@ -1112,15 +1111,13 @@ scanblock(Workbuf *wbuf, bool keepworking) // There are no heap pointers in struct Hchan, // so we can ignore the leading sizeof(Hchan) bytes. if(!(chantype->elem->__code & kindNoPointers)) { - // Channel's buffer follows Hchan immediately in memory. - // Size of buffer (cap(c)) is second int in the chan struct. - chancap = ((uintgo*)chan)[1]; - if(chancap > 0) { + chancap = chan->dataqsiz; + if(chancap > 0 && markonly(chan->buf)) { // TODO(atom): split into two chunks so that only the // in-use part of the circular buffer is scanned. // (Channel routines zero the unused part, so the current // code does not lead to leaks, it's just a little inefficient.) - *sbuf.obj.pos++ = (Obj){(byte*)chan+runtime_Hchansize, chancap*chantype->elem->__size, + *sbuf.obj.pos++ = (Obj){chan->buf, chancap*chantype->elem->__size, (uintptr)chantype->elem->__gc | PRECISE | LOOP}; if(sbuf.obj.pos == sbuf.obj.end) flushobjbuf(&sbuf); diff --git a/libgo/runtime/proc.c b/libgo/runtime/proc.c index dac32eb..02b62be 100644 --- a/libgo/runtime/proc.c +++ b/libgo/runtime/proc.c @@ -564,7 +564,8 @@ static struct __go_channel_type chan_bool_type_descriptor = CHANNEL_BOTH_DIR }; -extern Hchan *__go_new_channel (ChanType *, uintptr); +extern Hchan *makechan (ChanType *, int64) + __asm__ (GOSYM_PREFIX "runtime.makechan"); extern void closechan(Hchan *) __asm__ (GOSYM_PREFIX "runtime.closechan"); static void @@ -613,7 +614,7 @@ runtime_main(void* dummy __attribute__((unused))) runtime_throw("runtime_main not on m0"); __go_go(runtime_MHeap_Scavenger, nil); - runtime_main_init_done = __go_new_channel(&chan_bool_type_descriptor, 0); + runtime_main_init_done = makechan(&chan_bool_type_descriptor, 0); _cgo_notify_runtime_init_done(); @@ -853,6 +854,14 @@ runtime_ready(G *gp) g->m->locks--; } +void goready(G*, int) __asm__ (GOSYM_PREFIX "runtime.goready"); + +void +goready(G* gp, int traceskip __attribute__ ((unused))) +{ + runtime_ready(gp); +} + int32 runtime_gcprocs(void) { @@ -1898,6 +1907,22 @@ runtime_park(bool(*unlockf)(G*, void*), void *lock, const char *reason) runtime_mcall(park0); } +void gopark(FuncVal *, void *, String, byte, int) + __asm__ ("runtime.gopark"); + +void +gopark(FuncVal *unlockf, void *lock, String reason, + byte traceEv __attribute__ ((unused)), + int traceskip __attribute__ ((unused))) +{ + if(g->atomicstatus != _Grunning) + runtime_throw("bad g status"); + g->m->waitlock = lock; + g->m->waitunlockf = unlockf == nil ? nil : (void*)unlockf->fn; + g->waitreason = reason; + runtime_mcall(park0); +} + static bool parkunlock(G *gp, void *lock) { @@ -1914,6 +1939,21 @@ runtime_parkunlock(Lock *lock, const char *reason) runtime_park(parkunlock, lock, reason); } +void goparkunlock(Lock *, String, byte, int) + __asm__ (GOSYM_PREFIX "runtime.goparkunlock"); + +void +goparkunlock(Lock *lock, String reason, byte traceEv __attribute__ ((unused)), + int traceskip __attribute__ ((unused))) +{ + if(g->atomicstatus != _Grunning) + runtime_throw("bad g status"); + g->m->waitlock = lock; + g->m->waitunlockf = parkunlock; + g->waitreason = reason; + runtime_mcall(park0); +} + // runtime_park continuation on g0. static void park0(G *gp) diff --git a/libgo/runtime/runtime.h b/libgo/runtime/runtime.h index d1aad1e..6f96b2b 100644 --- a/libgo/runtime/runtime.h +++ b/libgo/runtime/runtime.h @@ -66,7 +66,7 @@ typedef struct FuncVal FuncVal; typedef struct SigTab SigTab; typedef struct mcache MCache; typedef struct FixAlloc FixAlloc; -typedef struct Hchan Hchan; +typedef struct hchan Hchan; typedef struct Timers Timers; typedef struct Timer Timer; typedef struct gcstats GCStats; @@ -75,6 +75,7 @@ typedef struct ParFor ParFor; typedef struct ParForThread ParForThread; typedef struct cgoMal CgoMal; typedef struct PollDesc PollDesc; +typedef struct sudog SudoG; typedef struct __go_open_array Slice; typedef struct __go_interface Iface; @@ -294,7 +295,6 @@ extern uint32 runtime_panicking; extern int8* runtime_goos; extern int32 runtime_ncpu; extern void (*runtime_sysargs)(int32, uint8**); -extern uint32 runtime_Hchansize; extern struct debugVars runtime_debug; extern uintptr runtime_maxstacksize; diff --git a/libgo/runtime/sema.goc b/libgo/runtime/sema.goc index b49c7b7..b0d198e 100644 --- a/libgo/runtime/sema.goc +++ b/libgo/runtime/sema.goc @@ -19,7 +19,6 @@ package sync #include "runtime.h" -#include "chan.h" #include "arch.h" typedef struct SemaWaiter SemaWaiter; @@ -373,7 +372,7 @@ func runtime_notifyListWait(l *notifyList, t uint32) { if (l->tail == nil) { l->head = &s; } else { - l->tail->link = &s; + l->tail->next = &s; } l->tail = &s; runtime_parkunlock(&l->lock, "semacquire"); @@ -409,8 +408,8 @@ func runtime_notifyListNotifyAll(l *notifyList) { // Go through the local list and ready all waiters. while (s != nil) { - SudoG* next = s->link; - s->link = nil; + SudoG* next = s->next; + s->next = nil; readyWithTime(s, 4); s = next; } @@ -442,11 +441,11 @@ func runtime_notifyListNotifyOne(l *notifyList) { // needs to be notified. If it hasn't made it to the list yet we won't // find it, but it won't park itself once it sees the new notify number. runtime_atomicstore(&l->notify, t+1); - for (p = nil, s = l->head; s != nil; p = s, s = s->link) { + for (p = nil, s = l->head; s != nil; p = s, s = s->next) { if (s->ticket == t) { - SudoG *n = s->link; + SudoG *n = s->next; if (p != nil) { - p->link = n; + p->next = n; } else { l->head = n; } @@ -454,7 +453,7 @@ func runtime_notifyListNotifyOne(l *notifyList) { l->tail = p; } runtime_unlock(&l->lock); - s->link = nil; + s->next = nil; readyWithTime(s, 4); return; } |