diff options
author | Ian Lance Taylor <ian@gcc.gnu.org> | 2016-10-10 16:52:09 +0000 |
---|---|---|
committer | Ian Lance Taylor <ian@gcc.gnu.org> | 2016-10-10 16:52:09 +0000 |
commit | 5d8c099edebfe908256c2bd77a0e2b67182b0f57 (patch) | |
tree | 6033d9fc6d1e3f5c3019c9b13d28e70655697a67 | |
parent | 40962ac03a869cf7e07e0672c0a649371896277b (diff) | |
download | gcc-5d8c099edebfe908256c2bd77a0e2b67182b0f57.zip gcc-5d8c099edebfe908256c2bd77a0e2b67182b0f57.tar.gz gcc-5d8c099edebfe908256c2bd77a0e2b67182b0f57.tar.bz2 |
runtime: copy channel code from Go 1.7 runtime
Change the compiler to use the new routines. Drop the separation of
small and large values when sending on a channel. Allocate the select
struct on the stack. Remove the old C implementation of channels. Adjust
the garbage collector for the new data structure.
Bring in part of the tracing code, enough for the channel code to call.
Bump the permitted number of allocations in one of the tests in
context_test.go. The difference is that now receiving from a channel
allocates a sudog, which the C code used to simply put on the
stack. This will be somewhat better when we port proc.go.
Reviewed-on: https://go-review.googlesource.com/30714
From-SVN: r240941
-rw-r--r-- | gcc/go/gofrontend/MERGE | 2 | ||||
-rw-r--r-- | gcc/go/gofrontend/escape.cc | 4 | ||||
-rw-r--r-- | gcc/go/gofrontend/expressions.cc | 38 | ||||
-rw-r--r-- | gcc/go/gofrontend/runtime.def | 19 | ||||
-rw-r--r-- | gcc/go/gofrontend/statements.cc | 62 | ||||
-rw-r--r-- | gcc/go/gofrontend/types.cc | 47 | ||||
-rw-r--r-- | gcc/go/gofrontend/types.h | 3 | ||||
-rw-r--r-- | libgo/Makefile.am | 1 | ||||
-rw-r--r-- | libgo/Makefile.in | 8 | ||||
-rw-r--r-- | libgo/go/context/context_test.go | 2 | ||||
-rw-r--r-- | libgo/go/runtime/chan.go | 724 | ||||
-rw-r--r-- | libgo/go/runtime/runtime2.go | 8 | ||||
-rw-r--r-- | libgo/go/runtime/select.go | 697 | ||||
-rw-r--r-- | libgo/go/runtime/stubs.go | 64 | ||||
-rw-r--r-- | libgo/go/runtime/trace.go | 1008 | ||||
-rw-r--r-- | libgo/runtime/chan.goc | 1130 | ||||
-rw-r--r-- | libgo/runtime/chan.h | 76 | ||||
-rw-r--r-- | libgo/runtime/go-cgo.c | 5 | ||||
-rw-r--r-- | libgo/runtime/heapdump.c | 4 | ||||
-rw-r--r-- | libgo/runtime/mgc0.c | 9 | ||||
-rw-r--r-- | libgo/runtime/proc.c | 44 | ||||
-rw-r--r-- | libgo/runtime/runtime.h | 4 | ||||
-rw-r--r-- | libgo/runtime/sema.goc | 15 |
23 files changed, 2676 insertions, 1298 deletions
diff --git a/gcc/go/gofrontend/MERGE b/gcc/go/gofrontend/MERGE index dc9e1f4..05752dac 100644 --- a/gcc/go/gofrontend/MERGE +++ b/gcc/go/gofrontend/MERGE @@ -1,4 +1,4 @@ -2431267d513804a3b1aa71adde9aefba9e3c3c59 +9401e714d690e3907a64ac5c8cd5aed9e28f511b The first line of this file holds the git revision number of the last merge done from the gofrontend repository. diff --git a/gcc/go/gofrontend/escape.cc b/gcc/go/gofrontend/escape.cc index ba82d80..e0df77d 100644 --- a/gcc/go/gofrontend/escape.cc +++ b/gcc/go/gofrontend/escape.cc @@ -293,7 +293,6 @@ Node::op_format() const break; case Runtime::MAKECHAN: - case Runtime::MAKECHANBIG: case Runtime::MAKEMAP: case Runtime::MAKESLICE1: case Runtime::MAKESLICE2: @@ -1229,7 +1228,6 @@ Escape_analysis_assign::expression(Expression** pexpr) break; case Runtime::MAKECHAN: - case Runtime::MAKECHANBIG: case Runtime::MAKEMAP: case Runtime::MAKESLICE1: case Runtime::MAKESLICE2: @@ -1838,7 +1836,6 @@ Escape_analysis_assign::assign(Node* dst, Node* src) } case Runtime::MAKECHAN: - case Runtime::MAKECHANBIG: case Runtime::MAKEMAP: case Runtime::MAKESLICE1: case Runtime::MAKESLICE2: @@ -2612,7 +2609,6 @@ Escape_analysis_flood::flood(Level level, Node* dst, Node* src, break; case Runtime::MAKECHAN: - case Runtime::MAKECHANBIG: case Runtime::MAKEMAP: case Runtime::MAKESLICE1: case Runtime::MAKESLICE2: diff --git a/gcc/go/gofrontend/expressions.cc b/gcc/go/gofrontend/expressions.cc index 343d354..36000ea 100644 --- a/gcc/go/gofrontend/expressions.cc +++ b/gcc/go/gofrontend/expressions.cc @@ -3604,6 +3604,7 @@ Unsafe_type_conversion_expression::do_get_backend(Translate_context* context) || et->channel_type() != NULL || et->map_type() != NULL || et->function_type() != NULL + || et->integer_type() != NULL || et->is_nil_type()); else if (et->is_unsafe_pointer_type()) go_assert(t->points_to() != NULL); @@ -7077,6 +7078,7 @@ Builtin_call_expression::do_flatten(Gogo*, Named_object*, break; case BUILTIN_LEN: + case BUILTIN_CAP: Expression_list::iterator pa = this->args()->begin(); if (!(*pa)->is_variable() && ((*pa)->type()->map_type() != NULL @@ -7217,10 +7219,7 @@ Builtin_call_expression::lower_make() Expression::make_nil(loc), Expression::make_nil(loc)); else if (is_chan) - call = Runtime::make_call((have_big_args - ? Runtime::MAKECHANBIG - : Runtime::MAKECHAN), - loc, 2, type_arg, len_arg); + call = Runtime::make_call(Runtime::MAKECHAN, loc, 2, type_arg, len_arg); else go_unreachable(); @@ -8300,7 +8299,31 @@ Builtin_call_expression::do_get_backend(Translate_context* context) this->seen_ = false; } else if (arg_type->channel_type() != NULL) - val = Runtime::make_call(Runtime::CHAN_CAP, location, 1, arg); + { + // The second field is the capacity. If the pointer + // is nil, the capacity is zero. + Type* uintptr_type = Type::lookup_integer_type("uintptr"); + Type* pint_type = Type::make_pointer_type(int_type); + Expression* parg = Expression::make_unsafe_cast(uintptr_type, + arg, + location); + int off = int_type->integer_type()->bits() / 8; + Expression* eoff = Expression::make_integer_ul(off, + uintptr_type, + location); + parg = Expression::make_binary(OPERATOR_PLUS, parg, eoff, + location); + parg = Expression::make_unsafe_cast(pint_type, parg, location); + Expression* nil = Expression::make_nil(location); + nil = Expression::make_cast(pint_type, nil, location); + Expression* cmp = Expression::make_binary(OPERATOR_EQEQ, + arg, nil, location); + Expression* zero = Expression::make_integer_ul(0, int_type, + location); + Expression* indir = Expression::make_unary(OPERATOR_MULT, + parg, location); + val = Expression::make_conditional(cmp, zero, indir, location); + } else go_unreachable(); } @@ -13729,9 +13752,8 @@ Receive_expression::do_get_backend(Translate_context* context) Expression* recv_addr = Expression::make_temporary_reference(this->temp_receiver_, loc); recv_addr = Expression::make_unary(OPERATOR_AND, recv_addr, loc); - Expression* recv = - Runtime::make_call(Runtime::RECEIVE, loc, 3, - td, this->channel_, recv_addr); + Expression* recv = Runtime::make_call(Runtime::CHANRECV1, loc, 3, + td, this->channel_, recv_addr); return Expression::make_compound(recv, recv_ref, loc)->get_backend(context); } diff --git a/gcc/go/gofrontend/runtime.def b/gcc/go/gofrontend/runtime.def index 2be772b..168f473 100644 --- a/gcc/go/gofrontend/runtime.def +++ b/gcc/go/gofrontend/runtime.def @@ -127,20 +127,13 @@ DEF_GO_RUNTIME(MAPITERNEXT, "runtime.mapiternext", P1(POINTER), R0()) // Make a channel. -DEF_GO_RUNTIME(MAKECHAN, "__go_new_channel", P2(TYPE, UINTPTR), R1(CHAN)) -DEF_GO_RUNTIME(MAKECHANBIG, "__go_new_channel_big", P2(TYPE, UINT64), R1(CHAN)) +DEF_GO_RUNTIME(MAKECHAN, "runtime.makechan", P2(TYPE, INT64), R1(CHAN)) -// Get the capacity of a channel (the size of the buffer). -DEF_GO_RUNTIME(CHAN_CAP, "__go_chan_cap", P1(CHAN), R1(INT)) - -// Send a small value on a channel. -DEF_GO_RUNTIME(SEND_SMALL, "__go_send_small", P3(TYPE, CHAN, UINT64), R0()) - -// Send a big value on a channel. -DEF_GO_RUNTIME(SEND_BIG, "__go_send_big", P3(TYPE, CHAN, POINTER), R0()) +// Send a value on a channel. +DEF_GO_RUNTIME(CHANSEND, "runtime.chansend1", P3(TYPE, CHAN, POINTER), R0()) // Receive a value from a channel. -DEF_GO_RUNTIME(RECEIVE, "__go_receive", P3(TYPE, CHAN, POINTER), R0()) +DEF_GO_RUNTIME(CHANRECV1, "runtime.chanrecv1", P3(TYPE, CHAN, POINTER), R0()) // Receive a value from a channel returning whether it is closed. DEF_GO_RUNTIME(CHANRECV2, "runtime.chanrecv2", P3(TYPE, CHAN, POINTER), @@ -148,7 +141,7 @@ DEF_GO_RUNTIME(CHANRECV2, "runtime.chanrecv2", P3(TYPE, CHAN, POINTER), // Start building a select statement. -DEF_GO_RUNTIME(NEWSELECT, "runtime.newselect", P1(INT32), R1(POINTER)) +DEF_GO_RUNTIME(NEWSELECT, "runtime.newselect", P3(POINTER, INT64, INT32), R0()) // Add a default clause to a select statement. DEF_GO_RUNTIME(SELECTDEFAULT, "runtime.selectdefault", @@ -202,7 +195,7 @@ DEF_GO_RUNTIME(RUNTIME_ERROR, "__go_runtime_error", P1(INT32), R0()) // Close. -DEF_GO_RUNTIME(CLOSE, "__go_builtin_close", P1(CHAN), R0()) +DEF_GO_RUNTIME(CLOSE, "runtime.closechan", P1(CHAN), R0()) // Copy. diff --git a/gcc/go/gofrontend/statements.cc b/gcc/go/gofrontend/statements.cc index 2eb363a..41f1ffb 100644 --- a/gcc/go/gofrontend/statements.cc +++ b/gcc/go/gofrontend/statements.cc @@ -4330,7 +4330,6 @@ Send_statement::do_get_backend(Translate_context* context) element_type, this->val_, loc); - bool is_small; bool can_take_address; switch (element_type->base()->classification()) { @@ -4340,25 +4339,18 @@ Send_statement::do_get_backend(Translate_context* context) case Type::TYPE_POINTER: case Type::TYPE_MAP: case Type::TYPE_CHANNEL: - is_small = true; - can_take_address = false; - break; - case Type::TYPE_FLOAT: case Type::TYPE_COMPLEX: case Type::TYPE_STRING: case Type::TYPE_INTERFACE: - is_small = false; can_take_address = false; break; case Type::TYPE_STRUCT: - is_small = false; can_take_address = true; break; case Type::TYPE_ARRAY: - is_small = false; can_take_address = !element_type->is_slice_type(); break; @@ -4384,28 +4376,19 @@ Send_statement::do_get_backend(Translate_context* context) Expression* td = Expression::make_type_descriptor(this->channel_->type(), loc); - Runtime::Function code; Bstatement* btemp = NULL; - if (is_small) - { - // Type is small enough to handle as uint64. - code = Runtime::SEND_SMALL; - val = Expression::make_unsafe_cast(Type::lookup_integer_type("uint64"), - val, loc); - } - else if (can_take_address) - { - // Must pass address of value. The function doesn't change the - // value, so just take its address directly. - code = Runtime::SEND_BIG; + if (can_take_address) + { + // The function doesn't change the value, so just take its + // address directly. val = Expression::make_unary(OPERATOR_AND, val, loc); } else { - // Must pass address of value, but the value is small enough - // that it might be in registers. Copy value into temporary - // variable to take address. - code = Runtime::SEND_BIG; + // The value is not in a variable, or is small enough that it + // might be in a register, and taking the address would push it + // on the stack. Copy it into a temporary variable to take the + // address. Temporary_statement* temp = Statement::make_temporary(element_type, val, loc); Expression* ref = Expression::make_temporary_reference(temp, loc); @@ -4413,7 +4396,8 @@ Send_statement::do_get_backend(Translate_context* context) btemp = temp->get_backend(context); } - Expression* call = Runtime::make_call(code, loc, 3, td, this->channel_, val); + Expression* call = Runtime::make_call(Runtime::CHANSEND, loc, 3, td, + this->channel_, val); context->gogo()->lower_expression(context->function(), NULL, &call); Bexpression* bcall = call->get_backend(context); @@ -4491,6 +4475,7 @@ Select_clauses::Select_clause::lower(Gogo* gogo, Named_object* function, Location loc = this->location_; Expression* selref = Expression::make_temporary_reference(sel, loc); + selref = Expression::make_unary(OPERATOR_AND, selref, loc); Expression* index_expr = Expression::make_integer_ul(this->index_, NULL, loc); @@ -4854,6 +4839,7 @@ Select_clauses::get_backend(Translate_context* context, } Expression* selref = Expression::make_temporary_reference(sel, location); + selref = Expression::make_unary(OPERATOR_AND, selref, location); Expression* call = Runtime::make_call(Runtime::SELECTGO, location, 1, selref); context->gogo()->lower_expression(context->function(), NULL, &call); @@ -4920,13 +4906,27 @@ Select_statement::do_lower(Gogo* gogo, Named_object* function, go_assert(this->sel_ == NULL); - Expression* size_expr = Expression::make_integer_ul(this->clauses_->size(), - NULL, loc); - Expression* call = Runtime::make_call(Runtime::NEWSELECT, loc, 1, size_expr); - - this->sel_ = Statement::make_temporary(NULL, call, loc); + int ncases = this->clauses_->size(); + Type* selstruct_type = Channel_type::select_type(ncases); + this->sel_ = Statement::make_temporary(selstruct_type, NULL, loc); b->add_statement(this->sel_); + int64_t selstruct_size; + if (!selstruct_type->backend_type_size(gogo, &selstruct_size)) + { + go_assert(saw_errors()); + return Statement::make_error_statement(loc); + } + + Expression* ref = Expression::make_temporary_reference(this->sel_, loc); + ref = Expression::make_unary(OPERATOR_AND, ref, loc); + Expression* selstruct_size_expr = + Expression::make_integer_int64(selstruct_size, NULL, loc); + Expression* size_expr = Expression::make_integer_ul(ncases, NULL, loc); + Expression* call = Runtime::make_call(Runtime::NEWSELECT, loc, 3, + ref, selstruct_size_expr, size_expr); + b->add_statement(Statement::make_statement(call, true)); + this->clauses_->lower(gogo, function, b, this->sel_); this->is_lowered_ = true; b->add_statement(this); diff --git a/gcc/go/gofrontend/types.cc b/gcc/go/gofrontend/types.cc index 01af8f3..38613bb 100644 --- a/gcc/go/gofrontend/types.cc +++ b/gcc/go/gofrontend/types.cc @@ -7771,6 +7771,53 @@ Channel_type::do_import(Import* imp) return Type::make_channel_type(may_send, may_receive, element_type); } +// Return the type to manage a select statement with ncases case +// statements. A value of this type is allocated on the stack. This +// must match the type hselect in libgo/go/runtime/select.go. + +Type* +Channel_type::select_type(int ncases) +{ + Type* unsafe_pointer_type = Type::make_pointer_type(Type::make_void_type()); + Type* uint16_type = Type::lookup_integer_type("uint16"); + + static Struct_type* scase_type; + if (scase_type == NULL) + { + Type* uintptr_type = Type::lookup_integer_type("uintptr"); + Type* uint64_type = Type::lookup_integer_type("uint64"); + scase_type = + Type::make_builtin_struct_type(7, + "elem", unsafe_pointer_type, + "chan", unsafe_pointer_type, + "pc", uintptr_type, + "kind", uint16_type, + "index", uint16_type, + "receivedp", unsafe_pointer_type, + "releasetime", uint64_type); + scase_type->set_is_struct_incomparable(); + } + + Expression* ncases_expr = + Expression::make_integer_ul(ncases, NULL, Linemap::predeclared_location()); + Array_type* scases = Type::make_array_type(scase_type, ncases_expr); + scases->set_is_array_incomparable(); + Array_type* order = Type::make_array_type(uint16_type, ncases_expr); + order->set_is_array_incomparable(); + + Struct_type* ret = + Type::make_builtin_struct_type(7, + "tcase", uint16_type, + "ncase", uint16_type, + "pollorder", unsafe_pointer_type, + "lockorder", unsafe_pointer_type, + "scase", scases, + "lockorderarr", order, + "pollorderarr", order); + ret->set_is_struct_incomparable(); + return ret; +} + // Make a new channel type. Channel_type* diff --git a/gcc/go/gofrontend/types.h b/gcc/go/gofrontend/types.h index 3d9a3c4..58d60e5 100644 --- a/gcc/go/gofrontend/types.h +++ b/gcc/go/gofrontend/types.h @@ -2809,6 +2809,9 @@ class Channel_type : public Type static Type* make_chan_type_descriptor_type(); + static Type* + select_type(int ncases); + protected: int do_traverse(Traverse* traverse) diff --git a/libgo/Makefile.am b/libgo/Makefile.am index a0566c1..bc47be6 100644 --- a/libgo/Makefile.am +++ b/libgo/Makefile.am @@ -520,7 +520,6 @@ runtime_files = \ $(runtime_thread_files) \ runtime/yield.c \ $(rtems_task_variable_add_file) \ - chan.c \ cpuprof.c \ go-iface.c \ lfstack.c \ diff --git a/libgo/Makefile.in b/libgo/Makefile.in index 7a133f1..5806d75 100644 --- a/libgo/Makefile.in +++ b/libgo/Makefile.in @@ -263,9 +263,9 @@ am__objects_6 = go-append.lo go-assert.lo go-assert-interface.lo \ $(am__objects_1) mfixalloc.lo mgc0.lo mheap.lo msize.lo \ $(am__objects_2) panic.lo parfor.lo print.lo proc.lo \ runtime.lo signal_unix.lo thread.lo $(am__objects_3) yield.lo \ - $(am__objects_4) chan.lo cpuprof.lo go-iface.lo lfstack.lo \ - malloc.lo mprof.lo netpoll.lo rdebug.lo reflect.lo runtime1.lo \ - sema.lo sigqueue.lo string.lo time.lo $(am__objects_5) + $(am__objects_4) cpuprof.lo go-iface.lo lfstack.lo malloc.lo \ + mprof.lo netpoll.lo rdebug.lo reflect.lo runtime1.lo sema.lo \ + sigqueue.lo string.lo time.lo $(am__objects_5) am_libgo_llgo_la_OBJECTS = $(am__objects_6) libgo_llgo_la_OBJECTS = $(am_libgo_llgo_la_OBJECTS) libgo_llgo_la_LINK = $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) \ @@ -921,7 +921,6 @@ runtime_files = \ $(runtime_thread_files) \ runtime/yield.c \ $(rtems_task_variable_add_file) \ - chan.c \ cpuprof.c \ go-iface.c \ lfstack.c \ @@ -1557,7 +1556,6 @@ mostlyclean-compile: distclean-compile: -rm -f *.tab.c -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/chan.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/cpuprof.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/env_posix.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/getncpu-bsd.Plo@am__quote@ diff --git a/libgo/go/context/context_test.go b/libgo/go/context/context_test.go index cf18211..cdfec07 100644 --- a/libgo/go/context/context_test.go +++ b/libgo/go/context/context_test.go @@ -381,7 +381,7 @@ func TestAllocs(t *testing.T) { <-c.Done() }, limit: 8, - gccgoLimit: 15, + gccgoLimit: 18, }, { desc: "WithCancel(bg)", diff --git a/libgo/go/runtime/chan.go b/libgo/go/runtime/chan.go new file mode 100644 index 0000000..eb2cad6 --- /dev/null +++ b/libgo/go/runtime/chan.go @@ -0,0 +1,724 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +// This file contains the implementation of Go channels. + +// Invariants: +// At least one of c.sendq and c.recvq is empty. +// For buffered channels, also: +// c.qcount > 0 implies that c.recvq is empty. +// c.qcount < c.dataqsiz implies that c.sendq is empty. +import ( + "runtime/internal/atomic" + "unsafe" +) + +// For gccgo, use go:linkname to rename compiler-called functions to +// themselves, so that the compiler will export them. +// +//go:linkname makechan runtime.makechan +//go:linkname chansend1 runtime.chansend1 +//go:linkname chanrecv1 runtime.chanrecv1 +//go:linkname chanrecv2 runtime.chanrecv2 +//go:linkname closechan runtime.closechan + +const ( + maxAlign = 8 + hchanSize = unsafe.Sizeof(hchan{}) + uintptr(-int(unsafe.Sizeof(hchan{}))&(maxAlign-1)) + debugChan = false +) + +type hchan struct { + qcount uint // total data in the queue + dataqsiz uint // size of the circular queue + buf unsafe.Pointer // points to an array of dataqsiz elements + elemsize uint16 + closed uint32 + elemtype *_type // element type + sendx uint // send index + recvx uint // receive index + recvq waitq // list of recv waiters + sendq waitq // list of send waiters + + // lock protects all fields in hchan, as well as several + // fields in sudogs blocked on this channel. + // + // Do not change another G's status while holding this lock + // (in particular, do not ready a G), as this can deadlock + // with stack shrinking. + lock mutex +} + +type waitq struct { + first *sudog + last *sudog +} + +//go:linkname reflect_makechan reflect.makechan +func reflect_makechan(t *chantype, size int64) *hchan { + return makechan(t, size) +} + +func makechan(t *chantype, size int64) *hchan { + elem := t.elem + + // compiler checks this but be safe. + if elem.size >= 1<<16 { + throw("makechan: invalid channel element type") + } + if hchanSize%maxAlign != 0 || elem.align > maxAlign { + throw("makechan: bad alignment") + } + if size < 0 || int64(uintptr(size)) != size || (elem.size > 0 && uintptr(size) > (_MaxMem-hchanSize)/elem.size) { + panic(plainError("makechan: size out of range")) + } + + var c *hchan + if elem.kind&kindNoPointers != 0 || size == 0 { + // Allocate memory in one call. + // Hchan does not contain pointers interesting for GC in this case: + // buf points into the same allocation, elemtype is persistent. + // SudoG's are referenced from their owning thread so they can't be collected. + // TODO(dvyukov,rlh): Rethink when collector can move allocated objects. + c = (*hchan)(mallocgc(hchanSize+uintptr(size)*elem.size, nil, true)) + if size > 0 && elem.size != 0 { + c.buf = add(unsafe.Pointer(c), hchanSize) + } else { + // race detector uses this location for synchronization + // Also prevents us from pointing beyond the allocation (see issue 9401). + c.buf = unsafe.Pointer(c) + } + } else { + c = new(hchan) + c.buf = newarray(elem, int(size)) + } + c.elemsize = uint16(elem.size) + c.elemtype = elem + c.dataqsiz = uint(size) + + if debugChan { + print("makechan: chan=", c, "; elemsize=", elem.size, "; dataqsiz=", size, "\n") + } + return c +} + +// chanbuf(c, i) is pointer to the i'th slot in the buffer. +func chanbuf(c *hchan, i uint) unsafe.Pointer { + return add(c.buf, uintptr(i)*uintptr(c.elemsize)) +} + +// entry point for c <- x from compiled code +//go:nosplit +func chansend1(t *chantype, c *hchan, elem unsafe.Pointer) { + chansend(t, c, elem, true, getcallerpc(unsafe.Pointer(&t))) +} + +/* + * generic single channel send/recv + * If block is not nil, + * then the protocol will not + * sleep but return if it could + * not complete. + * + * sleep can wake up with g.param == nil + * when a channel involved in the sleep has + * been closed. it is easiest to loop and re-run + * the operation; we'll see that it's now closed. + */ +func chansend(t *chantype, c *hchan, ep unsafe.Pointer, block bool, callerpc uintptr) bool { + if raceenabled { + raceReadObjectPC(t.elem, ep, callerpc, funcPC(chansend)) + } + if msanenabled { + msanread(ep, t.elem.size) + } + + if c == nil { + if !block { + return false + } + gopark(nil, nil, "chan send (nil chan)", traceEvGoStop, 2) + throw("unreachable") + } + + if debugChan { + print("chansend: chan=", c, "\n") + } + + if raceenabled { + racereadpc(unsafe.Pointer(c), callerpc, funcPC(chansend)) + } + + // Fast path: check for failed non-blocking operation without acquiring the lock. + // + // After observing that the channel is not closed, we observe that the channel is + // not ready for sending. Each of these observations is a single word-sized read + // (first c.closed and second c.recvq.first or c.qcount depending on kind of channel). + // Because a closed channel cannot transition from 'ready for sending' to + // 'not ready for sending', even if the channel is closed between the two observations, + // they imply a moment between the two when the channel was both not yet closed + // and not ready for sending. We behave as if we observed the channel at that moment, + // and report that the send cannot proceed. + // + // It is okay if the reads are reordered here: if we observe that the channel is not + // ready for sending and then observe that it is not closed, that implies that the + // channel wasn't closed during the first observation. + if !block && c.closed == 0 && ((c.dataqsiz == 0 && c.recvq.first == nil) || + (c.dataqsiz > 0 && c.qcount == c.dataqsiz)) { + return false + } + + var t0 int64 + if blockprofilerate > 0 { + t0 = cputicks() + } + + lock(&c.lock) + + if c.closed != 0 { + unlock(&c.lock) + panic(plainError("send on closed channel")) + } + + if sg := c.recvq.dequeue(); sg != nil { + // Found a waiting receiver. We pass the value we want to send + // directly to the receiver, bypassing the channel buffer (if any). + send(c, sg, ep, func() { unlock(&c.lock) }) + return true + } + + if c.qcount < c.dataqsiz { + // Space is available in the channel buffer. Enqueue the element to send. + qp := chanbuf(c, c.sendx) + if raceenabled { + raceacquire(qp) + racerelease(qp) + } + typedmemmove(c.elemtype, qp, ep) + c.sendx++ + if c.sendx == c.dataqsiz { + c.sendx = 0 + } + c.qcount++ + unlock(&c.lock) + return true + } + + if !block { + unlock(&c.lock) + return false + } + + // Block on the channel. Some receiver will complete our operation for us. + gp := getg() + mysg := acquireSudog() + mysg.releasetime = 0 + if t0 != 0 { + mysg.releasetime = -1 + } + // No stack splits between assigning elem and enqueuing mysg + // on gp.waiting where copystack can find it. + mysg.elem = ep + mysg.waitlink = nil + mysg.g = gp + mysg.selectdone = nil + mysg.c = c + gp.waiting = mysg + gp.param = nil + c.sendq.enqueue(mysg) + goparkunlock(&c.lock, "chan send", traceEvGoBlockSend, 3) + + // someone woke us up. + if mysg != gp.waiting { + throw("G waiting list is corrupted") + } + gp.waiting = nil + if gp.param == nil { + if c.closed == 0 { + throw("chansend: spurious wakeup") + } + panic(plainError("send on closed channel")) + } + gp.param = nil + if mysg.releasetime > 0 { + blockevent(mysg.releasetime-t0, 2) + } + mysg.c = nil + releaseSudog(mysg) + return true +} + +// send processes a send operation on an empty channel c. +// The value ep sent by the sender is copied to the receiver sg. +// The receiver is then woken up to go on its merry way. +// Channel c must be empty and locked. send unlocks c with unlockf. +// sg must already be dequeued from c. +// ep must be non-nil and point to the heap or the caller's stack. +func send(c *hchan, sg *sudog, ep unsafe.Pointer, unlockf func()) { + if raceenabled { + if c.dataqsiz == 0 { + racesync(c, sg) + } else { + // Pretend we go through the buffer, even though + // we copy directly. Note that we need to increment + // the head/tail locations only when raceenabled. + qp := chanbuf(c, c.recvx) + raceacquire(qp) + racerelease(qp) + raceacquireg(sg.g, qp) + racereleaseg(sg.g, qp) + c.recvx++ + if c.recvx == c.dataqsiz { + c.recvx = 0 + } + c.sendx = c.recvx // c.sendx = (c.sendx+1) % c.dataqsiz + } + } + if sg.elem != nil { + sendDirect(c.elemtype, sg, ep) + sg.elem = nil + } + gp := sg.g + unlockf() + gp.param = unsafe.Pointer(sg) + if sg.releasetime != 0 { + sg.releasetime = cputicks() + } + goready(gp, 4) +} + +func sendDirect(t *_type, sg *sudog, src unsafe.Pointer) { + // Send on an unbuffered or empty-buffered channel is the only operation + // in the entire runtime where one goroutine + // writes to the stack of another goroutine. The GC assumes that + // stack writes only happen when the goroutine is running and are + // only done by that goroutine. Using a write barrier is sufficient to + // make up for violating that assumption, but the write barrier has to work. + // typedmemmove will call heapBitsBulkBarrier, but the target bytes + // are not in the heap, so that will not help. We arrange to call + // memmove and typeBitsBulkBarrier instead. + + // Once we read sg.elem out of sg, it will no longer + // be updated if the destination's stack gets copied (shrunk). + // So make sure that no preemption points can happen between read & use. + dst := sg.elem + memmove(dst, src, t.size) + typeBitsBulkBarrier(t, uintptr(dst), t.size) +} + +func closechan(c *hchan) { + if c == nil { + panic(plainError("close of nil channel")) + } + + lock(&c.lock) + if c.closed != 0 { + unlock(&c.lock) + panic(plainError("close of closed channel")) + } + + if raceenabled { + callerpc := getcallerpc(unsafe.Pointer(&c)) + racewritepc(unsafe.Pointer(c), callerpc, funcPC(closechan)) + racerelease(unsafe.Pointer(c)) + } + + c.closed = 1 + + var glist *g + + // release all readers + for { + sg := c.recvq.dequeue() + if sg == nil { + break + } + if sg.elem != nil { + memclr(sg.elem, uintptr(c.elemsize)) + sg.elem = nil + } + if sg.releasetime != 0 { + sg.releasetime = cputicks() + } + gp := sg.g + gp.param = nil + if raceenabled { + raceacquireg(gp, unsafe.Pointer(c)) + } + gp.schedlink.set(glist) + glist = gp + } + + // release all writers (they will panic) + for { + sg := c.sendq.dequeue() + if sg == nil { + break + } + sg.elem = nil + if sg.releasetime != 0 { + sg.releasetime = cputicks() + } + gp := sg.g + gp.param = nil + if raceenabled { + raceacquireg(gp, unsafe.Pointer(c)) + } + gp.schedlink.set(glist) + glist = gp + } + unlock(&c.lock) + + // Ready all Gs now that we've dropped the channel lock. + for glist != nil { + gp := glist + glist = glist.schedlink.ptr() + gp.schedlink = 0 + goready(gp, 3) + } +} + +// entry points for <- c from compiled code +//go:nosplit +func chanrecv1(t *chantype, c *hchan, elem unsafe.Pointer) { + chanrecv(t, c, elem, true) +} + +//go:nosplit +func chanrecv2(t *chantype, c *hchan, elem unsafe.Pointer) (received bool) { + _, received = chanrecv(t, c, elem, true) + return +} + +// chanrecv receives on channel c and writes the received data to ep. +// ep may be nil, in which case received data is ignored. +// If block == false and no elements are available, returns (false, false). +// Otherwise, if c is closed, zeros *ep and returns (true, false). +// Otherwise, fills in *ep with an element and returns (true, true). +// A non-nil ep must point to the heap or the caller's stack. +func chanrecv(t *chantype, c *hchan, ep unsafe.Pointer, block bool) (selected, received bool) { + // raceenabled: don't need to check ep, as it is always on the stack + // or is new memory allocated by reflect. + + if debugChan { + print("chanrecv: chan=", c, "\n") + } + + if c == nil { + if !block { + return + } + gopark(nil, nil, "chan receive (nil chan)", traceEvGoStop, 2) + throw("unreachable") + } + + // Fast path: check for failed non-blocking operation without acquiring the lock. + // + // After observing that the channel is not ready for receiving, we observe that the + // channel is not closed. Each of these observations is a single word-sized read + // (first c.sendq.first or c.qcount, and second c.closed). + // Because a channel cannot be reopened, the later observation of the channel + // being not closed implies that it was also not closed at the moment of the + // first observation. We behave as if we observed the channel at that moment + // and report that the receive cannot proceed. + // + // The order of operations is important here: reversing the operations can lead to + // incorrect behavior when racing with a close. + if !block && (c.dataqsiz == 0 && c.sendq.first == nil || + c.dataqsiz > 0 && atomic.Loaduint(&c.qcount) == 0) && + atomic.Load(&c.closed) == 0 { + return + } + + var t0 int64 + if blockprofilerate > 0 { + t0 = cputicks() + } + + lock(&c.lock) + + if c.closed != 0 && c.qcount == 0 { + if raceenabled { + raceacquire(unsafe.Pointer(c)) + } + unlock(&c.lock) + if ep != nil { + memclr(ep, uintptr(c.elemsize)) + } + return true, false + } + + if sg := c.sendq.dequeue(); sg != nil { + // Found a waiting sender. If buffer is size 0, receive value + // directly from sender. Otherwise, receive from head of queue + // and add sender's value to the tail of the queue (both map to + // the same buffer slot because the queue is full). + recv(c, sg, ep, func() { unlock(&c.lock) }) + return true, true + } + + if c.qcount > 0 { + // Receive directly from queue + qp := chanbuf(c, c.recvx) + if raceenabled { + raceacquire(qp) + racerelease(qp) + } + if ep != nil { + typedmemmove(c.elemtype, ep, qp) + } + memclr(qp, uintptr(c.elemsize)) + c.recvx++ + if c.recvx == c.dataqsiz { + c.recvx = 0 + } + c.qcount-- + unlock(&c.lock) + return true, true + } + + if !block { + unlock(&c.lock) + return false, false + } + + // no sender available: block on this channel. + gp := getg() + mysg := acquireSudog() + mysg.releasetime = 0 + if t0 != 0 { + mysg.releasetime = -1 + } + // No stack splits between assigning elem and enqueuing mysg + // on gp.waiting where copystack can find it. + mysg.elem = ep + mysg.waitlink = nil + gp.waiting = mysg + mysg.g = gp + mysg.selectdone = nil + mysg.c = c + gp.param = nil + c.recvq.enqueue(mysg) + goparkunlock(&c.lock, "chan receive", traceEvGoBlockRecv, 3) + + // someone woke us up + if mysg != gp.waiting { + throw("G waiting list is corrupted") + } + gp.waiting = nil + if mysg.releasetime > 0 { + blockevent(mysg.releasetime-t0, 2) + } + closed := gp.param == nil + gp.param = nil + mysg.c = nil + releaseSudog(mysg) + return true, !closed +} + +// recv processes a receive operation on a full channel c. +// There are 2 parts: +// 1) The value sent by the sender sg is put into the channel +// and the sender is woken up to go on its merry way. +// 2) The value received by the receiver (the current G) is +// written to ep. +// For synchronous channels, both values are the same. +// For asynchronous channels, the receiver gets its data from +// the channel buffer and the sender's data is put in the +// channel buffer. +// Channel c must be full and locked. recv unlocks c with unlockf. +// sg must already be dequeued from c. +// A non-nil ep must point to the heap or the caller's stack. +func recv(c *hchan, sg *sudog, ep unsafe.Pointer, unlockf func()) { + if c.dataqsiz == 0 { + if raceenabled { + racesync(c, sg) + } + if ep != nil { + // copy data from sender + // ep points to our own stack or heap, so nothing + // special (ala sendDirect) needed here. + typedmemmove(c.elemtype, ep, sg.elem) + } + } else { + // Queue is full. Take the item at the + // head of the queue. Make the sender enqueue + // its item at the tail of the queue. Since the + // queue is full, those are both the same slot. + qp := chanbuf(c, c.recvx) + if raceenabled { + raceacquire(qp) + racerelease(qp) + raceacquireg(sg.g, qp) + racereleaseg(sg.g, qp) + } + // copy data from queue to receiver + if ep != nil { + typedmemmove(c.elemtype, ep, qp) + } + // copy data from sender to queue + typedmemmove(c.elemtype, qp, sg.elem) + c.recvx++ + if c.recvx == c.dataqsiz { + c.recvx = 0 + } + c.sendx = c.recvx // c.sendx = (c.sendx+1) % c.dataqsiz + } + sg.elem = nil + gp := sg.g + unlockf() + gp.param = unsafe.Pointer(sg) + if sg.releasetime != 0 { + sg.releasetime = cputicks() + } + goready(gp, 4) +} + +// compiler implements +// +// select { +// case c <- v: +// ... foo +// default: +// ... bar +// } +// +// as +// +// if selectnbsend(c, v) { +// ... foo +// } else { +// ... bar +// } +// +func selectnbsend(t *chantype, c *hchan, elem unsafe.Pointer) (selected bool) { + return chansend(t, c, elem, false, getcallerpc(unsafe.Pointer(&t))) +} + +// compiler implements +// +// select { +// case v = <-c: +// ... foo +// default: +// ... bar +// } +// +// as +// +// if selectnbrecv(&v, c) { +// ... foo +// } else { +// ... bar +// } +// +func selectnbrecv(t *chantype, elem unsafe.Pointer, c *hchan) (selected bool) { + selected, _ = chanrecv(t, c, elem, false) + return +} + +// compiler implements +// +// select { +// case v, ok = <-c: +// ... foo +// default: +// ... bar +// } +// +// as +// +// if c != nil && selectnbrecv2(&v, &ok, c) { +// ... foo +// } else { +// ... bar +// } +// +func selectnbrecv2(t *chantype, elem unsafe.Pointer, received *bool, c *hchan) (selected bool) { + // TODO(khr): just return 2 values from this function, now that it is in Go. + selected, *received = chanrecv(t, c, elem, false) + return +} + +//go:linkname reflect_chansend reflect.chansend +func reflect_chansend(t *chantype, c *hchan, elem unsafe.Pointer, nb bool) (selected bool) { + return chansend(t, c, elem, !nb, getcallerpc(unsafe.Pointer(&t))) +} + +//go:linkname reflect_chanrecv reflect.chanrecv +func reflect_chanrecv(t *chantype, c *hchan, nb bool, elem unsafe.Pointer) (selected bool, received bool) { + return chanrecv(t, c, elem, !nb) +} + +//go:linkname reflect_chanlen reflect.chanlen +func reflect_chanlen(c *hchan) int { + if c == nil { + return 0 + } + return int(c.qcount) +} + +//go:linkname reflect_chancap reflect.chancap +func reflect_chancap(c *hchan) int { + if c == nil { + return 0 + } + return int(c.dataqsiz) +} + +//go:linkname reflect_chanclose reflect.chanclose +func reflect_chanclose(c *hchan) { + closechan(c) +} + +func (q *waitq) enqueue(sgp *sudog) { + sgp.next = nil + x := q.last + if x == nil { + sgp.prev = nil + q.first = sgp + q.last = sgp + return + } + sgp.prev = x + x.next = sgp + q.last = sgp +} + +func (q *waitq) dequeue() *sudog { + for { + sgp := q.first + if sgp == nil { + return nil + } + y := sgp.next + if y == nil { + q.first = nil + q.last = nil + } else { + y.prev = nil + q.first = y + sgp.next = nil // mark as removed (see dequeueSudog) + } + + // if sgp participates in a select and is already signaled, ignore it + if sgp.selectdone != nil { + // claim the right to signal + if *sgp.selectdone != 0 || !atomic.Cas(sgp.selectdone, 0, 1) { + continue + } + } + + return sgp + } +} + +func racesync(c *hchan, sg *sudog) { + racerelease(chanbuf(c, 0)) + raceacquireg(sg.g, chanbuf(c, 0)) + racereleaseg(sg.g, chanbuf(c, 0)) + raceacquire(chanbuf(c, 0)) +} diff --git a/libgo/go/runtime/runtime2.go b/libgo/go/runtime/runtime2.go index 17447eb..688efcd 100644 --- a/libgo/go/runtime/runtime2.go +++ b/libgo/go/runtime/runtime2.go @@ -231,9 +231,6 @@ func (mp *muintptr) set(m *m) { *mp = muintptr(unsafe.Pointer(m)) } // // sudogs are allocated from a special pool. Use acquireSudog and // releaseSudog to allocate and free them. -/* -Commented out for gccgo for now. - type sudog struct { // The following fields are protected by the hchan.lock of the // channel this sudog is blocking on. shrinkstack depends on @@ -253,7 +250,6 @@ type sudog struct { waitlink *sudog // g.waiting list c *hchan // channel } -*/ type gcstats struct { // the struct must consist of only uint64's, @@ -364,7 +360,7 @@ type g struct { gopc uintptr // pc of go statement that created this goroutine startpc uintptr // pc of goroutine function racectx uintptr - // Not for gccgo for now: waiting *sudog // sudog structures this g is waiting on (that have a valid elem ptr); in lock order + waiting *sudog // sudog structures this g is waiting on (that have a valid elem ptr); in lock order // Not for gccgo: cgoCtxt []uintptr // cgo traceback context // Per-G GC state @@ -528,7 +524,7 @@ type p struct { gfree *g gfreecnt int32 - // Not for gccgo for now: sudogcache []*sudog + sudogcache []*sudog // Not for gccgo for now: sudogbuf [128]*sudog // Not for gccgo for now: tracebuf traceBufPtr diff --git a/libgo/go/runtime/select.go b/libgo/go/runtime/select.go new file mode 100644 index 0000000..08446a1 --- /dev/null +++ b/libgo/go/runtime/select.go @@ -0,0 +1,697 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +// This file contains the implementation of Go select statements. + +import ( + "runtime/internal/sys" + "unsafe" +) + +// For gccgo, use go:linkname to rename compiler-called functions to +// themselves, so that the compiler will export them. +// +//go:linkname newselect runtime.newselect +//go:linkname selectdefault runtime.selectdefault +//go:linkname selectsend runtime.selectsend +//go:linkname selectrecv runtime.selectrecv +//go:linkname selectrecv2 runtime.selectrecv2 +//go:linkname selectgo runtime.selectgo + +const ( + debugSelect = false + + // scase.kind + caseRecv = iota + caseSend + caseDefault +) + +// Select statement header. +// Known to compiler. +// Changes here must also be made in src/cmd/internal/gc/select.go's selecttype. +type hselect struct { + tcase uint16 // total count of scase[] + ncase uint16 // currently filled scase[] + pollorder *uint16 // case poll order + lockorder *uint16 // channel lock order + scase [1]scase // one per case (in order of appearance) +} + +// Select case descriptor. +// Known to compiler. +// Changes here must also be made in src/cmd/internal/gc/select.go's selecttype. +type scase struct { + elem unsafe.Pointer // data element + c *hchan // chan + pc uintptr // return pc + kind uint16 + index uint16 // case index + receivedp *bool // pointer to received bool (recv2) + releasetime int64 +} + +var ( + chansendpc = funcPC(chansend) + chanrecvpc = funcPC(chanrecv) +) + +func selectsize(size uintptr) uintptr { + selsize := unsafe.Sizeof(hselect{}) + + (size-1)*unsafe.Sizeof(hselect{}.scase[0]) + + size*unsafe.Sizeof(*hselect{}.lockorder) + + size*unsafe.Sizeof(*hselect{}.pollorder) + return round(selsize, sys.Int64Align) +} + +func newselect(sel *hselect, selsize int64, size int32) { + if selsize != int64(selectsize(uintptr(size))) { + print("runtime: bad select size ", selsize, ", want ", selectsize(uintptr(size)), "\n") + throw("bad select size") + } + if size != int32(uint16(size)) { + throw("select size too large") + } + sel.tcase = uint16(size) + sel.ncase = 0 + sel.lockorder = (*uint16)(add(unsafe.Pointer(&sel.scase), uintptr(size)*unsafe.Sizeof(hselect{}.scase[0]))) + sel.pollorder = (*uint16)(add(unsafe.Pointer(sel.lockorder), uintptr(size)*unsafe.Sizeof(*hselect{}.lockorder))) + + // For gccgo the temporary variable will not have been zeroed. + memclr(unsafe.Pointer(&sel.scase), uintptr(size)*unsafe.Sizeof(hselect{}.scase[0])+uintptr(size)*unsafe.Sizeof(*hselect{}.lockorder)+uintptr(size)*unsafe.Sizeof(*hselect{}.pollorder)) + + if debugSelect { + print("newselect s=", sel, " size=", size, "\n") + } +} + +func selectsend(sel *hselect, c *hchan, elem unsafe.Pointer, index int32) { + // nil cases do not compete + if c != nil { + selectsendImpl(sel, c, getcallerpc(unsafe.Pointer(&sel)), elem, index) + } + return +} + +// cut in half to give stack a chance to split +func selectsendImpl(sel *hselect, c *hchan, pc uintptr, elem unsafe.Pointer, index int32) { + i := sel.ncase + if i >= sel.tcase { + throw("selectsend: too many cases") + } + sel.ncase = i + 1 + cas := (*scase)(add(unsafe.Pointer(&sel.scase), uintptr(i)*unsafe.Sizeof(sel.scase[0]))) + + cas.pc = pc + cas.c = c + cas.index = uint16(index) + cas.kind = caseSend + cas.elem = elem + + if debugSelect { + print("selectsend s=", sel, " pc=", hex(cas.pc), " chan=", cas.c, " index=", cas.index, "\n") + } +} + +func selectrecv(sel *hselect, c *hchan, elem unsafe.Pointer, index int32) { + // nil cases do not compete + if c != nil { + selectrecvImpl(sel, c, getcallerpc(unsafe.Pointer(&sel)), elem, nil, index) + } + return +} + +func selectrecv2(sel *hselect, c *hchan, elem unsafe.Pointer, received *bool, index int32) { + // nil cases do not compete + if c != nil { + selectrecvImpl(sel, c, getcallerpc(unsafe.Pointer(&sel)), elem, received, index) + } + return +} + +func selectrecvImpl(sel *hselect, c *hchan, pc uintptr, elem unsafe.Pointer, received *bool, index int32) { + i := sel.ncase + if i >= sel.tcase { + throw("selectrecv: too many cases") + } + sel.ncase = i + 1 + cas := (*scase)(add(unsafe.Pointer(&sel.scase), uintptr(i)*unsafe.Sizeof(sel.scase[0]))) + cas.pc = pc + cas.c = c + cas.index = uint16(index) + cas.kind = caseRecv + cas.elem = elem + cas.receivedp = received + + if debugSelect { + print("selectrecv s=", sel, " pc=", hex(cas.pc), " chan=", cas.c, " index=", cas.index, "\n") + } +} + +func selectdefault(sel *hselect, index int32) { + selectdefaultImpl(sel, getcallerpc(unsafe.Pointer(&sel)), index) + return +} + +func selectdefaultImpl(sel *hselect, callerpc uintptr, index int32) { + i := sel.ncase + if i >= sel.tcase { + throw("selectdefault: too many cases") + } + sel.ncase = i + 1 + cas := (*scase)(add(unsafe.Pointer(&sel.scase), uintptr(i)*unsafe.Sizeof(sel.scase[0]))) + cas.pc = callerpc + cas.c = nil + cas.index = uint16(index) + cas.kind = caseDefault + + if debugSelect { + print("selectdefault s=", sel, " pc=", hex(cas.pc), " index=", cas.index, "\n") + } +} + +func sellock(scases []scase, lockorder []uint16) { + var c *hchan + for _, o := range lockorder { + c0 := scases[o].c + if c0 != nil && c0 != c { + c = c0 + lock(&c.lock) + } + } +} + +func selunlock(scases []scase, lockorder []uint16) { + // We must be very careful here to not touch sel after we have unlocked + // the last lock, because sel can be freed right after the last unlock. + // Consider the following situation. + // First M calls runtime·park() in runtime·selectgo() passing the sel. + // Once runtime·park() has unlocked the last lock, another M makes + // the G that calls select runnable again and schedules it for execution. + // When the G runs on another M, it locks all the locks and frees sel. + // Now if the first M touches sel, it will access freed memory. + n := len(scases) + r := 0 + // skip the default case + if n > 0 && scases[lockorder[0]].c == nil { + r = 1 + } + for i := n - 1; i >= r; i-- { + c := scases[lockorder[i]].c + if i > 0 && c == scases[lockorder[i-1]].c { + continue // will unlock it on the next iteration + } + unlock(&c.lock) + } +} + +func selparkcommit(gp *g, _ unsafe.Pointer) bool { + // This must not access gp's stack (see gopark). In + // particular, it must not access the *hselect. That's okay, + // because by the time this is called, gp.waiting has all + // channels in lock order. + var lastc *hchan + for sg := gp.waiting; sg != nil; sg = sg.waitlink { + if sg.c != lastc && lastc != nil { + // As soon as we unlock the channel, fields in + // any sudog with that channel may change, + // including c and waitlink. Since multiple + // sudogs may have the same channel, we unlock + // only after we've passed the last instance + // of a channel. + unlock(&lastc.lock) + } + lastc = sg.c + } + if lastc != nil { + unlock(&lastc.lock) + } + return true +} + +func block() { + gopark(nil, nil, "select (no cases)", traceEvGoStop, 1) // forever +} + +// selectgo implements the select statement. +// +// *sel is on the current goroutine's stack (regardless of any +// escaping in selectgo). +// +// selectgo does not return. Instead, it overwrites its return PC and +// returns directly to the triggered select case. Because of this, it +// cannot appear at the top of a split stack. +func selectgo(sel *hselect) int32 { + _, index := selectgoImpl(sel) + return int32(index) +} + +// selectgoImpl returns scase.pc and scase.so for the select +// case which fired. +func selectgoImpl(sel *hselect) (uintptr, uint16) { + if debugSelect { + print("select: sel=", sel, "\n") + } + + scaseslice := slice{unsafe.Pointer(&sel.scase), int(sel.ncase), int(sel.ncase)} + scases := *(*[]scase)(unsafe.Pointer(&scaseslice)) + + var t0 int64 + if blockprofilerate > 0 { + t0 = cputicks() + for i := 0; i < int(sel.ncase); i++ { + scases[i].releasetime = -1 + } + } + + // The compiler rewrites selects that statically have + // only 0 or 1 cases plus default into simpler constructs. + // The only way we can end up with such small sel.ncase + // values here is for a larger select in which most channels + // have been nilled out. The general code handles those + // cases correctly, and they are rare enough not to bother + // optimizing (and needing to test). + + // generate permuted order + pollslice := slice{unsafe.Pointer(sel.pollorder), int(sel.ncase), int(sel.ncase)} + pollorder := *(*[]uint16)(unsafe.Pointer(&pollslice)) + for i := 1; i < int(sel.ncase); i++ { + j := int(fastrand1()) % (i + 1) + pollorder[i] = pollorder[j] + pollorder[j] = uint16(i) + } + + // sort the cases by Hchan address to get the locking order. + // simple heap sort, to guarantee n log n time and constant stack footprint. + lockslice := slice{unsafe.Pointer(sel.lockorder), int(sel.ncase), int(sel.ncase)} + lockorder := *(*[]uint16)(unsafe.Pointer(&lockslice)) + for i := 0; i < int(sel.ncase); i++ { + j := i + // Start with the pollorder to permute cases on the same channel. + c := scases[pollorder[i]].c + for j > 0 && scases[lockorder[(j-1)/2]].c.sortkey() < c.sortkey() { + k := (j - 1) / 2 + lockorder[j] = lockorder[k] + j = k + } + lockorder[j] = pollorder[i] + } + for i := int(sel.ncase) - 1; i >= 0; i-- { + o := lockorder[i] + c := scases[o].c + lockorder[i] = lockorder[0] + j := 0 + for { + k := j*2 + 1 + if k >= i { + break + } + if k+1 < i && scases[lockorder[k]].c.sortkey() < scases[lockorder[k+1]].c.sortkey() { + k++ + } + if c.sortkey() < scases[lockorder[k]].c.sortkey() { + lockorder[j] = lockorder[k] + j = k + continue + } + break + } + lockorder[j] = o + } + /* + for i := 0; i+1 < int(sel.ncase); i++ { + if scases[lockorder[i]].c.sortkey() > scases[lockorder[i+1]].c.sortkey() { + print("i=", i, " x=", lockorder[i], " y=", lockorder[i+1], "\n") + throw("select: broken sort") + } + } + */ + + // lock all the channels involved in the select + sellock(scases, lockorder) + + var ( + gp *g + done uint32 + sg *sudog + c *hchan + k *scase + sglist *sudog + sgnext *sudog + qp unsafe.Pointer + nextp **sudog + ) + +loop: + // pass 1 - look for something already waiting + var dfl *scase + var cas *scase + for i := 0; i < int(sel.ncase); i++ { + cas = &scases[pollorder[i]] + c = cas.c + + switch cas.kind { + case caseRecv: + sg = c.sendq.dequeue() + if sg != nil { + goto recv + } + if c.qcount > 0 { + goto bufrecv + } + if c.closed != 0 { + goto rclose + } + + case caseSend: + if raceenabled { + racereadpc(unsafe.Pointer(c), cas.pc, chansendpc) + } + if c.closed != 0 { + goto sclose + } + sg = c.recvq.dequeue() + if sg != nil { + goto send + } + if c.qcount < c.dataqsiz { + goto bufsend + } + + case caseDefault: + dfl = cas + } + } + + if dfl != nil { + selunlock(scases, lockorder) + cas = dfl + goto retc + } + + // pass 2 - enqueue on all chans + gp = getg() + done = 0 + if gp.waiting != nil { + throw("gp.waiting != nil") + } + nextp = &gp.waiting + for _, casei := range lockorder { + cas = &scases[casei] + c = cas.c + sg := acquireSudog() + sg.g = gp + // Note: selectdone is adjusted for stack copies in stack1.go:adjustsudogs + sg.selectdone = (*uint32)(noescape(unsafe.Pointer(&done))) + // No stack splits between assigning elem and enqueuing + // sg on gp.waiting where copystack can find it. + sg.elem = cas.elem + sg.releasetime = 0 + if t0 != 0 { + sg.releasetime = -1 + } + sg.c = c + // Construct waiting list in lock order. + *nextp = sg + nextp = &sg.waitlink + + switch cas.kind { + case caseRecv: + c.recvq.enqueue(sg) + + case caseSend: + c.sendq.enqueue(sg) + } + } + + // wait for someone to wake us up + gp.param = nil + gopark(selparkcommit, nil, "select", traceEvGoBlockSelect, 2) + + // someone woke us up + sellock(scases, lockorder) + sg = (*sudog)(gp.param) + gp.param = nil + + // pass 3 - dequeue from unsuccessful chans + // otherwise they stack up on quiet channels + // record the successful case, if any. + // We singly-linked up the SudoGs in lock order. + cas = nil + sglist = gp.waiting + // Clear all elem before unlinking from gp.waiting. + for sg1 := gp.waiting; sg1 != nil; sg1 = sg1.waitlink { + sg1.selectdone = nil + sg1.elem = nil + sg1.c = nil + } + gp.waiting = nil + + for _, casei := range lockorder { + k = &scases[casei] + if sglist.releasetime > 0 { + k.releasetime = sglist.releasetime + } + if sg == sglist { + // sg has already been dequeued by the G that woke us up. + cas = k + } else { + c = k.c + if k.kind == caseSend { + c.sendq.dequeueSudoG(sglist) + } else { + c.recvq.dequeueSudoG(sglist) + } + } + sgnext = sglist.waitlink + sglist.waitlink = nil + releaseSudog(sglist) + sglist = sgnext + } + + if cas == nil { + // This can happen if we were woken up by a close(). + // TODO: figure that out explicitly so we don't need this loop. + goto loop + } + + c = cas.c + + if debugSelect { + print("wait-return: sel=", sel, " c=", c, " cas=", cas, " kind=", cas.kind, "\n") + } + + if cas.kind == caseRecv { + if cas.receivedp != nil { + *cas.receivedp = true + } + } + + if raceenabled { + if cas.kind == caseRecv && cas.elem != nil { + raceWriteObjectPC(c.elemtype, cas.elem, cas.pc, chanrecvpc) + } else if cas.kind == caseSend { + raceReadObjectPC(c.elemtype, cas.elem, cas.pc, chansendpc) + } + } + if msanenabled { + if cas.kind == caseRecv && cas.elem != nil { + msanwrite(cas.elem, c.elemtype.size) + } else if cas.kind == caseSend { + msanread(cas.elem, c.elemtype.size) + } + } + + selunlock(scases, lockorder) + goto retc + +bufrecv: + // can receive from buffer + if raceenabled { + if cas.elem != nil { + raceWriteObjectPC(c.elemtype, cas.elem, cas.pc, chanrecvpc) + } + raceacquire(chanbuf(c, c.recvx)) + racerelease(chanbuf(c, c.recvx)) + } + if msanenabled && cas.elem != nil { + msanwrite(cas.elem, c.elemtype.size) + } + if cas.receivedp != nil { + *cas.receivedp = true + } + qp = chanbuf(c, c.recvx) + if cas.elem != nil { + typedmemmove(c.elemtype, cas.elem, qp) + } + memclr(qp, uintptr(c.elemsize)) + c.recvx++ + if c.recvx == c.dataqsiz { + c.recvx = 0 + } + c.qcount-- + selunlock(scases, lockorder) + goto retc + +bufsend: + // can send to buffer + if raceenabled { + raceacquire(chanbuf(c, c.sendx)) + racerelease(chanbuf(c, c.sendx)) + raceReadObjectPC(c.elemtype, cas.elem, cas.pc, chansendpc) + } + if msanenabled { + msanread(cas.elem, c.elemtype.size) + } + typedmemmove(c.elemtype, chanbuf(c, c.sendx), cas.elem) + c.sendx++ + if c.sendx == c.dataqsiz { + c.sendx = 0 + } + c.qcount++ + selunlock(scases, lockorder) + goto retc + +recv: + // can receive from sleeping sender (sg) + recv(c, sg, cas.elem, func() { selunlock(scases, lockorder) }) + if debugSelect { + print("syncrecv: sel=", sel, " c=", c, "\n") + } + if cas.receivedp != nil { + *cas.receivedp = true + } + goto retc + +rclose: + // read at end of closed channel + selunlock(scases, lockorder) + if cas.receivedp != nil { + *cas.receivedp = false + } + if cas.elem != nil { + memclr(cas.elem, uintptr(c.elemsize)) + } + if raceenabled { + raceacquire(unsafe.Pointer(c)) + } + goto retc + +send: + // can send to a sleeping receiver (sg) + if raceenabled { + raceReadObjectPC(c.elemtype, cas.elem, cas.pc, chansendpc) + } + if msanenabled { + msanread(cas.elem, c.elemtype.size) + } + send(c, sg, cas.elem, func() { selunlock(scases, lockorder) }) + if debugSelect { + print("syncsend: sel=", sel, " c=", c, "\n") + } + goto retc + +retc: + if cas.releasetime > 0 { + blockevent(cas.releasetime-t0, 2) + } + return cas.pc, cas.index + +sclose: + // send on closed channel + selunlock(scases, lockorder) + panic(plainError("send on closed channel")) +} + +func (c *hchan) sortkey() uintptr { + // TODO(khr): if we have a moving garbage collector, we'll need to + // change this function. + return uintptr(unsafe.Pointer(c)) +} + +// A runtimeSelect is a single case passed to rselect. +// This must match ../reflect/value.go:/runtimeSelect +type runtimeSelect struct { + dir selectDir + typ unsafe.Pointer // channel type (not used here) + ch *hchan // channel + val unsafe.Pointer // ptr to data (SendDir) or ptr to receive buffer (RecvDir) +} + +// These values must match ../reflect/value.go:/SelectDir. +type selectDir int + +const ( + _ selectDir = iota + selectSend // case Chan <- Send + selectRecv // case <-Chan: + selectDefault // default +) + +//go:linkname reflect_rselect reflect.rselect +func reflect_rselect(cases []runtimeSelect) (chosen int, recvOK bool) { + // flagNoScan is safe here, because all objects are also referenced from cases. + size := selectsize(uintptr(len(cases))) + sel := (*hselect)(mallocgc(size, nil, true)) + newselect(sel, int64(size), int32(len(cases))) + r := new(bool) + for i := range cases { + rc := &cases[i] + switch rc.dir { + case selectDefault: + selectdefaultImpl(sel, uintptr(i), 0) + case selectSend: + if rc.ch == nil { + break + } + selectsendImpl(sel, rc.ch, uintptr(i), rc.val, 0) + case selectRecv: + if rc.ch == nil { + break + } + selectrecvImpl(sel, rc.ch, uintptr(i), rc.val, r, 0) + } + } + + pc, _ := selectgoImpl(sel) + chosen = int(pc) + recvOK = *r + return +} + +func (q *waitq) dequeueSudoG(sgp *sudog) { + x := sgp.prev + y := sgp.next + if x != nil { + if y != nil { + // middle of queue + x.next = y + y.prev = x + sgp.next = nil + sgp.prev = nil + return + } + // end of queue + x.next = nil + q.last = x + sgp.prev = nil + return + } + if y != nil { + // start of queue + y.prev = nil + q.first = y + sgp.next = nil + return + } + + // x==y==nil. Either sgp is the only element in the queue, + // or it has already been removed. Use q.first to disambiguate. + if q.first == sgp { + q.first = nil + q.last = nil + } +} diff --git a/libgo/go/runtime/stubs.go b/libgo/go/runtime/stubs.go index bbeac41..c687cbf 100644 --- a/libgo/go/runtime/stubs.go +++ b/libgo/go/runtime/stubs.go @@ -384,3 +384,67 @@ func errno() int func entersyscall(int32) func entersyscallblock(int32) func exitsyscall(int32) +func gopark(func(*g, unsafe.Pointer) bool, unsafe.Pointer, string, byte, int) +func goparkunlock(*mutex, string, byte, int) +func goready(*g, int) + +// Temporary for gccgo until we port mprof.go. +var blockprofilerate uint64 + +func blockevent(cycles int64, skip int) {} + +// Temporary hack for gccgo until we port proc.go. +//go:nosplit +func acquireSudog() *sudog { + mp := acquirem() + pp := mp.p.ptr() + if len(pp.sudogcache) == 0 { + pp.sudogcache = append(pp.sudogcache, new(sudog)) + } + n := len(pp.sudogcache) + s := pp.sudogcache[n-1] + pp.sudogcache[n-1] = nil + pp.sudogcache = pp.sudogcache[:n-1] + if s.elem != nil { + throw("acquireSudog: found s.elem != nil in cache") + } + releasem(mp) + return s +} + +// Temporary hack for gccgo until we port proc.go. +//go:nosplit +func releaseSudog(s *sudog) { + if s.elem != nil { + throw("runtime: sudog with non-nil elem") + } + if s.selectdone != nil { + throw("runtime: sudog with non-nil selectdone") + } + if s.next != nil { + throw("runtime: sudog with non-nil next") + } + if s.prev != nil { + throw("runtime: sudog with non-nil prev") + } + if s.waitlink != nil { + throw("runtime: sudog with non-nil waitlink") + } + if s.c != nil { + throw("runtime: sudog with non-nil c") + } + gp := getg() + if gp.param != nil { + throw("runtime: releaseSudog with non-nil gp.param") + } + mp := acquirem() // avoid rescheduling to another P + pp := mp.p.ptr() + pp.sudogcache = append(pp.sudogcache, s) + releasem(mp) +} + +// Temporary hack for gccgo until we port the garbage collector. +func typeBitsBulkBarrier(typ *_type, p, size uintptr) {} + +// Temporary for gccgo until we port print.go. +type hex uint64 diff --git a/libgo/go/runtime/trace.go b/libgo/go/runtime/trace.go new file mode 100644 index 0000000..35126f1 --- /dev/null +++ b/libgo/go/runtime/trace.go @@ -0,0 +1,1008 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Go execution tracer. +// The tracer captures a wide range of execution events like goroutine +// creation/blocking/unblocking, syscall enter/exit/block, GC-related events, +// changes of heap size, processor start/stop, etc and writes them to a buffer +// in a compact form. A precise nanosecond-precision timestamp and a stack +// trace is captured for most events. +// See https://golang.org/s/go15trace for more info. + +package runtime + +import ( + "runtime/internal/sys" + "unsafe" +) + +// Event types in the trace, args are given in square brackets. +const ( + traceEvNone = 0 // unused + traceEvBatch = 1 // start of per-P batch of events [pid, timestamp] + traceEvFrequency = 2 // contains tracer timer frequency [frequency (ticks per second)] + traceEvStack = 3 // stack [stack id, number of PCs, array of {PC, func string ID, file string ID, line}] + traceEvGomaxprocs = 4 // current value of GOMAXPROCS [timestamp, GOMAXPROCS, stack id] + traceEvProcStart = 5 // start of P [timestamp, thread id] + traceEvProcStop = 6 // stop of P [timestamp] + traceEvGCStart = 7 // GC start [timestamp, seq, stack id] + traceEvGCDone = 8 // GC done [timestamp] + traceEvGCScanStart = 9 // GC scan start [timestamp] + traceEvGCScanDone = 10 // GC scan done [timestamp] + traceEvGCSweepStart = 11 // GC sweep start [timestamp, stack id] + traceEvGCSweepDone = 12 // GC sweep done [timestamp] + traceEvGoCreate = 13 // goroutine creation [timestamp, new goroutine id, new stack id, stack id] + traceEvGoStart = 14 // goroutine starts running [timestamp, goroutine id, seq] + traceEvGoEnd = 15 // goroutine ends [timestamp] + traceEvGoStop = 16 // goroutine stops (like in select{}) [timestamp, stack] + traceEvGoSched = 17 // goroutine calls Gosched [timestamp, stack] + traceEvGoPreempt = 18 // goroutine is preempted [timestamp, stack] + traceEvGoSleep = 19 // goroutine calls Sleep [timestamp, stack] + traceEvGoBlock = 20 // goroutine blocks [timestamp, stack] + traceEvGoUnblock = 21 // goroutine is unblocked [timestamp, goroutine id, seq, stack] + traceEvGoBlockSend = 22 // goroutine blocks on chan send [timestamp, stack] + traceEvGoBlockRecv = 23 // goroutine blocks on chan recv [timestamp, stack] + traceEvGoBlockSelect = 24 // goroutine blocks on select [timestamp, stack] + traceEvGoBlockSync = 25 // goroutine blocks on Mutex/RWMutex [timestamp, stack] + traceEvGoBlockCond = 26 // goroutine blocks on Cond [timestamp, stack] + traceEvGoBlockNet = 27 // goroutine blocks on network [timestamp, stack] + traceEvGoSysCall = 28 // syscall enter [timestamp, stack] + traceEvGoSysExit = 29 // syscall exit [timestamp, goroutine id, seq, real timestamp] + traceEvGoSysBlock = 30 // syscall blocks [timestamp] + traceEvGoWaiting = 31 // denotes that goroutine is blocked when tracing starts [timestamp, goroutine id] + traceEvGoInSyscall = 32 // denotes that goroutine is in syscall when tracing starts [timestamp, goroutine id] + traceEvHeapAlloc = 33 // memstats.heap_live change [timestamp, heap_alloc] + traceEvNextGC = 34 // memstats.next_gc change [timestamp, next_gc] + traceEvTimerGoroutine = 35 // denotes timer goroutine [timer goroutine id] + traceEvFutileWakeup = 36 // denotes that the previous wakeup of this goroutine was futile [timestamp] + traceEvString = 37 // string dictionary entry [ID, length, string] + traceEvGoStartLocal = 38 // goroutine starts running on the same P as the last event [timestamp, goroutine id] + traceEvGoUnblockLocal = 39 // goroutine is unblocked on the same P as the last event [timestamp, goroutine id, stack] + traceEvGoSysExitLocal = 40 // syscall exit on the same P as the last event [timestamp, goroutine id, real timestamp] + traceEvCount = 41 +) + +const ( + // Timestamps in trace are cputicks/traceTickDiv. + // This makes absolute values of timestamp diffs smaller, + // and so they are encoded in less number of bytes. + // 64 on x86 is somewhat arbitrary (one tick is ~20ns on a 3GHz machine). + // The suggested increment frequency for PowerPC's time base register is + // 512 MHz according to Power ISA v2.07 section 6.2, so we use 16 on ppc64 + // and ppc64le. + // Tracing won't work reliably for architectures where cputicks is emulated + // by nanotime, so the value doesn't matter for those architectures. + traceTickDiv = 16 + 48*(sys.Goarch386|sys.GoarchAmd64|sys.GoarchAmd64p32) + // Maximum number of PCs in a single stack trace. + // Since events contain only stack id rather than whole stack trace, + // we can allow quite large values here. + traceStackSize = 128 + // Identifier of a fake P that is used when we trace without a real P. + traceGlobProc = -1 + // Maximum number of bytes to encode uint64 in base-128. + traceBytesPerNumber = 10 + // Shift of the number of arguments in the first event byte. + traceArgCountShift = 6 + // Flag passed to traceGoPark to denote that the previous wakeup of this + // goroutine was futile. For example, a goroutine was unblocked on a mutex, + // but another goroutine got ahead and acquired the mutex before the first + // goroutine is scheduled, so the first goroutine has to block again. + // Such wakeups happen on buffered channels and sync.Mutex, + // but are generally not interesting for end user. + traceFutileWakeup byte = 128 +) + +// trace is global tracing context. +var trace struct { + lock mutex // protects the following members + lockOwner *g // to avoid deadlocks during recursive lock locks + enabled bool // when set runtime traces events + shutdown bool // set when we are waiting for trace reader to finish after setting enabled to false + headerWritten bool // whether ReadTrace has emitted trace header + footerWritten bool // whether ReadTrace has emitted trace footer + shutdownSema uint32 // used to wait for ReadTrace completion + seqStart uint64 // sequence number when tracing was started + ticksStart int64 // cputicks when tracing was started + ticksEnd int64 // cputicks when tracing was stopped + timeStart int64 // nanotime when tracing was started + timeEnd int64 // nanotime when tracing was stopped + seqGC uint64 // GC start/done sequencer + reading traceBufPtr // buffer currently handed off to user + empty traceBufPtr // stack of empty buffers + fullHead traceBufPtr // queue of full buffers + fullTail traceBufPtr + reader *g // goroutine that called ReadTrace, or nil + stackTab traceStackTable // maps stack traces to unique ids + + // Dictionary for traceEvString. + // Currently this is used only for func/file:line info after tracing session, + // so we assume single-threaded access. + strings map[string]uint64 + stringSeq uint64 + + bufLock mutex // protects buf + buf traceBufPtr // global trace buffer, used when running without a p +} + +// traceBufHeader is per-P tracing buffer. +type traceBufHeader struct { + link traceBufPtr // in trace.empty/full + lastTicks uint64 // when we wrote the last event + pos int // next write offset in arr + stk [traceStackSize]uintptr // scratch buffer for traceback +} + +// traceBuf is per-P tracing buffer. +type traceBuf struct { + traceBufHeader + arr [64<<10 - unsafe.Sizeof(traceBufHeader{})]byte // underlying buffer for traceBufHeader.buf +} + +// traceBufPtr is a *traceBuf that is not traced by the garbage +// collector and doesn't have write barriers. traceBufs are not +// allocated from the GC'd heap, so this is safe, and are often +// manipulated in contexts where write barriers are not allowed, so +// this is necessary. +type traceBufPtr uintptr + +func (tp traceBufPtr) ptr() *traceBuf { return (*traceBuf)(unsafe.Pointer(tp)) } +func (tp *traceBufPtr) set(b *traceBuf) { *tp = traceBufPtr(unsafe.Pointer(b)) } +func traceBufPtrOf(b *traceBuf) traceBufPtr { + return traceBufPtr(unsafe.Pointer(b)) +} + +/* +Commented out for gccgo for now. + +// StartTrace enables tracing for the current process. +// While tracing, the data will be buffered and available via ReadTrace. +// StartTrace returns an error if tracing is already enabled. +// Most clients should use the runtime/trace package or the testing package's +// -test.trace flag instead of calling StartTrace directly. +func StartTrace() error { + // Stop the world, so that we can take a consistent snapshot + // of all goroutines at the beginning of the trace. + stopTheWorld("start tracing") + + // We are in stop-the-world, but syscalls can finish and write to trace concurrently. + // Exitsyscall could check trace.enabled long before and then suddenly wake up + // and decide to write to trace at a random point in time. + // However, such syscall will use the global trace.buf buffer, because we've + // acquired all p's by doing stop-the-world. So this protects us from such races. + lock(&trace.bufLock) + + if trace.enabled || trace.shutdown { + unlock(&trace.bufLock) + startTheWorld() + return errorString("tracing is already enabled") + } + + // Can't set trace.enabled yet. While the world is stopped, exitsyscall could + // already emit a delayed event (see exitTicks in exitsyscall) if we set trace.enabled here. + // That would lead to an inconsistent trace: + // - either GoSysExit appears before EvGoInSyscall, + // - or GoSysExit appears for a goroutine for which we don't emit EvGoInSyscall below. + // To instruct traceEvent that it must not ignore events below, we set startingtrace. + // trace.enabled is set afterwards once we have emitted all preliminary events. + _g_ := getg() + _g_.m.startingtrace = true + for _, gp := range allgs { + status := readgstatus(gp) + if status != _Gdead { + traceGoCreate(gp, gp.startpc) // also resets gp.traceseq/tracelastp + } + if status == _Gwaiting { + // traceEvGoWaiting is implied to have seq=1. + gp.traceseq++ + traceEvent(traceEvGoWaiting, -1, uint64(gp.goid)) + } + if status == _Gsyscall { + gp.traceseq++ + traceEvent(traceEvGoInSyscall, -1, uint64(gp.goid)) + } else { + gp.sysblocktraced = false + } + } + traceProcStart() + traceGoStart() + // Note: ticksStart needs to be set after we emit traceEvGoInSyscall events. + // If we do it the other way around, it is possible that exitsyscall will + // query sysexitticks after ticksStart but before traceEvGoInSyscall timestamp. + // It will lead to a false conclusion that cputicks is broken. + trace.ticksStart = cputicks() + trace.timeStart = nanotime() + trace.headerWritten = false + trace.footerWritten = false + trace.strings = make(map[string]uint64) + trace.stringSeq = 0 + trace.seqGC = 0 + _g_.m.startingtrace = false + trace.enabled = true + + unlock(&trace.bufLock) + + startTheWorld() + return nil +} + +// StopTrace stops tracing, if it was previously enabled. +// StopTrace only returns after all the reads for the trace have completed. +func StopTrace() { + // Stop the world so that we can collect the trace buffers from all p's below, + // and also to avoid races with traceEvent. + stopTheWorld("stop tracing") + + // See the comment in StartTrace. + lock(&trace.bufLock) + + if !trace.enabled { + unlock(&trace.bufLock) + startTheWorld() + return + } + + traceGoSched() + + for _, p := range &allp { + if p == nil { + break + } + buf := p.tracebuf + if buf != 0 { + traceFullQueue(buf) + p.tracebuf = 0 + } + } + if trace.buf != 0 && trace.buf.ptr().pos != 0 { + buf := trace.buf + trace.buf = 0 + traceFullQueue(buf) + } + + for { + trace.ticksEnd = cputicks() + trace.timeEnd = nanotime() + // Windows time can tick only every 15ms, wait for at least one tick. + if trace.timeEnd != trace.timeStart { + break + } + osyield() + } + + trace.enabled = false + trace.shutdown = true + unlock(&trace.bufLock) + + startTheWorld() + + // The world is started but we've set trace.shutdown, so new tracing can't start. + // Wait for the trace reader to flush pending buffers and stop. + semacquire(&trace.shutdownSema, false) + if raceenabled { + raceacquire(unsafe.Pointer(&trace.shutdownSema)) + } + + // The lock protects us from races with StartTrace/StopTrace because they do stop-the-world. + lock(&trace.lock) + for _, p := range &allp { + if p == nil { + break + } + if p.tracebuf != 0 { + throw("trace: non-empty trace buffer in proc") + } + } + if trace.buf != 0 { + throw("trace: non-empty global trace buffer") + } + if trace.fullHead != 0 || trace.fullTail != 0 { + throw("trace: non-empty full trace buffer") + } + if trace.reading != 0 || trace.reader != nil { + throw("trace: reading after shutdown") + } + for trace.empty != 0 { + buf := trace.empty + trace.empty = buf.ptr().link + sysFree(unsafe.Pointer(buf), unsafe.Sizeof(*buf.ptr()), &memstats.other_sys) + } + trace.strings = nil + trace.shutdown = false + unlock(&trace.lock) +} + +// ReadTrace returns the next chunk of binary tracing data, blocking until data +// is available. If tracing is turned off and all the data accumulated while it +// was on has been returned, ReadTrace returns nil. The caller must copy the +// returned data before calling ReadTrace again. +// ReadTrace must be called from one goroutine at a time. +func ReadTrace() []byte { + // This function may need to lock trace.lock recursively + // (goparkunlock -> traceGoPark -> traceEvent -> traceFlush). + // To allow this we use trace.lockOwner. + // Also this function must not allocate while holding trace.lock: + // allocation can call heap allocate, which will try to emit a trace + // event while holding heap lock. + lock(&trace.lock) + trace.lockOwner = getg() + + if trace.reader != nil { + // More than one goroutine reads trace. This is bad. + // But we rather do not crash the program because of tracing, + // because tracing can be enabled at runtime on prod servers. + trace.lockOwner = nil + unlock(&trace.lock) + println("runtime: ReadTrace called from multiple goroutines simultaneously") + return nil + } + // Recycle the old buffer. + if buf := trace.reading; buf != 0 { + buf.ptr().link = trace.empty + trace.empty = buf + trace.reading = 0 + } + // Write trace header. + if !trace.headerWritten { + trace.headerWritten = true + trace.lockOwner = nil + unlock(&trace.lock) + return []byte("go 1.7 trace\x00\x00\x00\x00") + } + // Wait for new data. + if trace.fullHead == 0 && !trace.shutdown { + trace.reader = getg() + goparkunlock(&trace.lock, "trace reader (blocked)", traceEvGoBlock, 2) + lock(&trace.lock) + } + // Write a buffer. + if trace.fullHead != 0 { + buf := traceFullDequeue() + trace.reading = buf + trace.lockOwner = nil + unlock(&trace.lock) + return buf.ptr().arr[:buf.ptr().pos] + } + // Write footer with timer frequency. + if !trace.footerWritten { + trace.footerWritten = true + // Use float64 because (trace.ticksEnd - trace.ticksStart) * 1e9 can overflow int64. + freq := float64(trace.ticksEnd-trace.ticksStart) * 1e9 / float64(trace.timeEnd-trace.timeStart) / traceTickDiv + trace.lockOwner = nil + unlock(&trace.lock) + var data []byte + data = append(data, traceEvFrequency|0<<traceArgCountShift) + data = traceAppend(data, uint64(freq)) + if timers.gp != nil { + data = append(data, traceEvTimerGoroutine|0<<traceArgCountShift) + data = traceAppend(data, uint64(timers.gp.goid)) + } + // This will emit a bunch of full buffers, we will pick them up + // on the next iteration. + trace.stackTab.dump() + return data + } + // Done. + if trace.shutdown { + trace.lockOwner = nil + unlock(&trace.lock) + if raceenabled { + // Model synchronization on trace.shutdownSema, which race + // detector does not see. This is required to avoid false + // race reports on writer passed to trace.Start. + racerelease(unsafe.Pointer(&trace.shutdownSema)) + } + // trace.enabled is already reset, so can call traceable functions. + semrelease(&trace.shutdownSema) + return nil + } + // Also bad, but see the comment above. + trace.lockOwner = nil + unlock(&trace.lock) + println("runtime: spurious wakeup of trace reader") + return nil +} + +// traceReader returns the trace reader that should be woken up, if any. +func traceReader() *g { + if trace.reader == nil || (trace.fullHead == 0 && !trace.shutdown) { + return nil + } + lock(&trace.lock) + if trace.reader == nil || (trace.fullHead == 0 && !trace.shutdown) { + unlock(&trace.lock) + return nil + } + gp := trace.reader + trace.reader = nil + unlock(&trace.lock) + return gp +} + +// traceProcFree frees trace buffer associated with pp. +func traceProcFree(pp *p) { + buf := pp.tracebuf + pp.tracebuf = 0 + if buf == 0 { + return + } + lock(&trace.lock) + traceFullQueue(buf) + unlock(&trace.lock) +} + +// traceFullQueue queues buf into queue of full buffers. +func traceFullQueue(buf traceBufPtr) { + buf.ptr().link = 0 + if trace.fullHead == 0 { + trace.fullHead = buf + } else { + trace.fullTail.ptr().link = buf + } + trace.fullTail = buf +} + +// traceFullDequeue dequeues from queue of full buffers. +func traceFullDequeue() traceBufPtr { + buf := trace.fullHead + if buf == 0 { + return 0 + } + trace.fullHead = buf.ptr().link + if trace.fullHead == 0 { + trace.fullTail = 0 + } + buf.ptr().link = 0 + return buf +} + +// traceEvent writes a single event to trace buffer, flushing the buffer if necessary. +// ev is event type. +// If skip > 0, write current stack id as the last argument (skipping skip top frames). +// If skip = 0, this event type should contain a stack, but we don't want +// to collect and remember it for this particular call. +func traceEvent(ev byte, skip int, args ...uint64) { + mp, pid, bufp := traceAcquireBuffer() + // Double-check trace.enabled now that we've done m.locks++ and acquired bufLock. + // This protects from races between traceEvent and StartTrace/StopTrace. + + // The caller checked that trace.enabled == true, but trace.enabled might have been + // turned off between the check and now. Check again. traceLockBuffer did mp.locks++, + // StopTrace does stopTheWorld, and stopTheWorld waits for mp.locks to go back to zero, + // so if we see trace.enabled == true now, we know it's true for the rest of the function. + // Exitsyscall can run even during stopTheWorld. The race with StartTrace/StopTrace + // during tracing in exitsyscall is resolved by locking trace.bufLock in traceLockBuffer. + if !trace.enabled && !mp.startingtrace { + traceReleaseBuffer(pid) + return + } + buf := (*bufp).ptr() + const maxSize = 2 + 5*traceBytesPerNumber // event type, length, sequence, timestamp, stack id and two add params + if buf == nil || len(buf.arr)-buf.pos < maxSize { + buf = traceFlush(traceBufPtrOf(buf)).ptr() + (*bufp).set(buf) + } + + ticks := uint64(cputicks()) / traceTickDiv + tickDiff := ticks - buf.lastTicks + if buf.pos == 0 { + buf.byte(traceEvBatch | 1<<traceArgCountShift) + buf.varint(uint64(pid)) + buf.varint(ticks) + tickDiff = 0 + } + buf.lastTicks = ticks + narg := byte(len(args)) + if skip >= 0 { + narg++ + } + // We have only 2 bits for number of arguments. + // If number is >= 3, then the event type is followed by event length in bytes. + if narg > 3 { + narg = 3 + } + startPos := buf.pos + buf.byte(ev | narg<<traceArgCountShift) + var lenp *byte + if narg == 3 { + // Reserve the byte for length assuming that length < 128. + buf.varint(0) + lenp = &buf.arr[buf.pos-1] + } + buf.varint(tickDiff) + for _, a := range args { + buf.varint(a) + } + if skip == 0 { + buf.varint(0) + } else if skip > 0 { + _g_ := getg() + gp := mp.curg + var nstk int + if gp == _g_ { + nstk = callers(skip, buf.stk[:]) + } else if gp != nil { + gp = mp.curg + // This may happen when tracing a system call, + // so we must lock the stack. + if gcTryLockStackBarriers(gp) { + nstk = gcallers(gp, skip, buf.stk[:]) + gcUnlockStackBarriers(gp) + } + } + if nstk > 0 { + nstk-- // skip runtime.goexit + } + if nstk > 0 && gp.goid == 1 { + nstk-- // skip runtime.main + } + id := trace.stackTab.put(buf.stk[:nstk]) + buf.varint(uint64(id)) + } + evSize := buf.pos - startPos + if evSize > maxSize { + throw("invalid length of trace event") + } + if lenp != nil { + // Fill in actual length. + *lenp = byte(evSize - 2) + } + traceReleaseBuffer(pid) +} + +// traceAcquireBuffer returns trace buffer to use and, if necessary, locks it. +func traceAcquireBuffer() (mp *m, pid int32, bufp *traceBufPtr) { + mp = acquirem() + if p := mp.p.ptr(); p != nil { + return mp, p.id, &p.tracebuf + } + lock(&trace.bufLock) + return mp, traceGlobProc, &trace.buf +} + +// traceReleaseBuffer releases a buffer previously acquired with traceAcquireBuffer. +func traceReleaseBuffer(pid int32) { + if pid == traceGlobProc { + unlock(&trace.bufLock) + } + releasem(getg().m) +} + +// traceFlush puts buf onto stack of full buffers and returns an empty buffer. +func traceFlush(buf traceBufPtr) traceBufPtr { + owner := trace.lockOwner + dolock := owner == nil || owner != getg().m.curg + if dolock { + lock(&trace.lock) + } + if buf != 0 { + traceFullQueue(buf) + } + if trace.empty != 0 { + buf = trace.empty + trace.empty = buf.ptr().link + } else { + buf = traceBufPtr(sysAlloc(unsafe.Sizeof(traceBuf{}), &memstats.other_sys)) + if buf == 0 { + throw("trace: out of memory") + } + } + bufp := buf.ptr() + bufp.link.set(nil) + bufp.pos = 0 + bufp.lastTicks = 0 + if dolock { + unlock(&trace.lock) + } + return buf +} + +func traceString(buf *traceBuf, s string) (uint64, *traceBuf) { + if s == "" { + return 0, buf + } + if id, ok := trace.strings[s]; ok { + return id, buf + } + + trace.stringSeq++ + id := trace.stringSeq + trace.strings[s] = id + + size := 1 + 2*traceBytesPerNumber + len(s) + if len(buf.arr)-buf.pos < size { + buf = traceFlush(traceBufPtrOf(buf)).ptr() + } + buf.byte(traceEvString) + buf.varint(id) + buf.varint(uint64(len(s))) + buf.pos += copy(buf.arr[buf.pos:], s) + return id, buf +} + +// traceAppend appends v to buf in little-endian-base-128 encoding. +func traceAppend(buf []byte, v uint64) []byte { + for ; v >= 0x80; v >>= 7 { + buf = append(buf, 0x80|byte(v)) + } + buf = append(buf, byte(v)) + return buf +} + +// varint appends v to buf in little-endian-base-128 encoding. +func (buf *traceBuf) varint(v uint64) { + pos := buf.pos + for ; v >= 0x80; v >>= 7 { + buf.arr[pos] = 0x80 | byte(v) + pos++ + } + buf.arr[pos] = byte(v) + pos++ + buf.pos = pos +} + +// byte appends v to buf. +func (buf *traceBuf) byte(v byte) { + buf.arr[buf.pos] = v + buf.pos++ +} + +*/ + +// traceStackTable maps stack traces (arrays of PC's) to unique uint32 ids. +// It is lock-free for reading. +type traceStackTable struct { + lock mutex + seq uint32 + mem traceAlloc + tab [1 << 13]traceStackPtr +} + +// traceStack is a single stack in traceStackTable. +type traceStack struct { + link traceStackPtr + hash uintptr + id uint32 + n int + stk [0]uintptr // real type [n]uintptr +} + +type traceStackPtr uintptr + +/* +Commented out for gccgo for now. + +func (tp traceStackPtr) ptr() *traceStack { return (*traceStack)(unsafe.Pointer(tp)) } + +// stack returns slice of PCs. +func (ts *traceStack) stack() []uintptr { + return (*[traceStackSize]uintptr)(unsafe.Pointer(&ts.stk))[:ts.n] +} + +// put returns a unique id for the stack trace pcs and caches it in the table, +// if it sees the trace for the first time. +func (tab *traceStackTable) put(pcs []uintptr) uint32 { + if len(pcs) == 0 { + return 0 + } + hash := memhash(unsafe.Pointer(&pcs[0]), 0, uintptr(len(pcs))*unsafe.Sizeof(pcs[0])) + // First, search the hashtable w/o the mutex. + if id := tab.find(pcs, hash); id != 0 { + return id + } + // Now, double check under the mutex. + lock(&tab.lock) + if id := tab.find(pcs, hash); id != 0 { + unlock(&tab.lock) + return id + } + // Create new record. + tab.seq++ + stk := tab.newStack(len(pcs)) + stk.hash = hash + stk.id = tab.seq + stk.n = len(pcs) + stkpc := stk.stack() + for i, pc := range pcs { + stkpc[i] = pc + } + part := int(hash % uintptr(len(tab.tab))) + stk.link = tab.tab[part] + atomicstorep(unsafe.Pointer(&tab.tab[part]), unsafe.Pointer(stk)) + unlock(&tab.lock) + return stk.id +} + +// find checks if the stack trace pcs is already present in the table. +func (tab *traceStackTable) find(pcs []uintptr, hash uintptr) uint32 { + part := int(hash % uintptr(len(tab.tab))) +Search: + for stk := tab.tab[part].ptr(); stk != nil; stk = stk.link.ptr() { + if stk.hash == hash && stk.n == len(pcs) { + for i, stkpc := range stk.stack() { + if stkpc != pcs[i] { + continue Search + } + } + return stk.id + } + } + return 0 +} + +// newStack allocates a new stack of size n. +func (tab *traceStackTable) newStack(n int) *traceStack { + return (*traceStack)(tab.mem.alloc(unsafe.Sizeof(traceStack{}) + uintptr(n)*sys.PtrSize)) +} + +// dump writes all previously cached stacks to trace buffers, +// releases all memory and resets state. +func (tab *traceStackTable) dump() { + frames := make(map[uintptr]traceFrame) + var tmp [(2 + 4*traceStackSize) * traceBytesPerNumber]byte + buf := traceFlush(0).ptr() + for _, stk := range tab.tab { + stk := stk.ptr() + for ; stk != nil; stk = stk.link.ptr() { + tmpbuf := tmp[:0] + tmpbuf = traceAppend(tmpbuf, uint64(stk.id)) + tmpbuf = traceAppend(tmpbuf, uint64(stk.n)) + for _, pc := range stk.stack() { + var frame traceFrame + frame, buf = traceFrameForPC(buf, frames, pc) + tmpbuf = traceAppend(tmpbuf, uint64(pc)) + tmpbuf = traceAppend(tmpbuf, uint64(frame.funcID)) + tmpbuf = traceAppend(tmpbuf, uint64(frame.fileID)) + tmpbuf = traceAppend(tmpbuf, uint64(frame.line)) + } + // Now copy to the buffer. + size := 1 + traceBytesPerNumber + len(tmpbuf) + if len(buf.arr)-buf.pos < size { + buf = traceFlush(traceBufPtrOf(buf)).ptr() + } + buf.byte(traceEvStack | 3<<traceArgCountShift) + buf.varint(uint64(len(tmpbuf))) + buf.pos += copy(buf.arr[buf.pos:], tmpbuf) + } + } + + lock(&trace.lock) + traceFullQueue(traceBufPtrOf(buf)) + unlock(&trace.lock) + + tab.mem.drop() + *tab = traceStackTable{} +} + +type traceFrame struct { + funcID uint64 + fileID uint64 + line uint64 +} + +func traceFrameForPC(buf *traceBuf, frames map[uintptr]traceFrame, pc uintptr) (traceFrame, *traceBuf) { + if frame, ok := frames[pc]; ok { + return frame, buf + } + + var frame traceFrame + f := findfunc(pc) + if f == nil { + frames[pc] = frame + return frame, buf + } + + fn := funcname(f) + const maxLen = 1 << 10 + if len(fn) > maxLen { + fn = fn[len(fn)-maxLen:] + } + frame.funcID, buf = traceString(buf, fn) + file, line := funcline(f, pc-sys.PCQuantum) + frame.line = uint64(line) + if len(file) > maxLen { + file = file[len(file)-maxLen:] + } + frame.fileID, buf = traceString(buf, file) + return frame, buf +} + +*/ + +// traceAlloc is a non-thread-safe region allocator. +// It holds a linked list of traceAllocBlock. +type traceAlloc struct { + head traceAllocBlockPtr + off uintptr +} + +// traceAllocBlock is a block in traceAlloc. +// +// traceAllocBlock is allocated from non-GC'd memory, so it must not +// contain heap pointers. Writes to pointers to traceAllocBlocks do +// not need write barriers. +type traceAllocBlock struct { + next traceAllocBlockPtr + data [64<<10 - sys.PtrSize]byte +} + +type traceAllocBlockPtr uintptr + +func (p traceAllocBlockPtr) ptr() *traceAllocBlock { return (*traceAllocBlock)(unsafe.Pointer(p)) } +func (p *traceAllocBlockPtr) set(x *traceAllocBlock) { *p = traceAllocBlockPtr(unsafe.Pointer(x)) } + +/* +Commented out for gccgo for now. + +// alloc allocates n-byte block. +func (a *traceAlloc) alloc(n uintptr) unsafe.Pointer { + n = round(n, sys.PtrSize) + if a.head == 0 || a.off+n > uintptr(len(a.head.ptr().data)) { + if n > uintptr(len(a.head.ptr().data)) { + throw("trace: alloc too large") + } + block := (*traceAllocBlock)(sysAlloc(unsafe.Sizeof(traceAllocBlock{}), &memstats.other_sys)) + if block == nil { + throw("trace: out of memory") + } + block.next.set(a.head.ptr()) + a.head.set(block) + a.off = 0 + } + p := &a.head.ptr().data[a.off] + a.off += n + return unsafe.Pointer(p) +} + +// drop frees all previously allocated memory and resets the allocator. +func (a *traceAlloc) drop() { + for a.head != 0 { + block := a.head.ptr() + a.head.set(block.next.ptr()) + sysFree(unsafe.Pointer(block), unsafe.Sizeof(traceAllocBlock{}), &memstats.other_sys) + } +} + +// The following functions write specific events to trace. + +func traceGomaxprocs(procs int32) { + traceEvent(traceEvGomaxprocs, 1, uint64(procs)) +} + +func traceProcStart() { + traceEvent(traceEvProcStart, -1, uint64(getg().m.id)) +} + +func traceProcStop(pp *p) { + // Sysmon and stopTheWorld can stop Ps blocked in syscalls, + // to handle this we temporary employ the P. + mp := acquirem() + oldp := mp.p + mp.p.set(pp) + traceEvent(traceEvProcStop, -1) + mp.p = oldp + releasem(mp) +} + +func traceGCStart() { + traceEvent(traceEvGCStart, 3, trace.seqGC) + trace.seqGC++ +} + +func traceGCDone() { + traceEvent(traceEvGCDone, -1) +} + +func traceGCScanStart() { + traceEvent(traceEvGCScanStart, -1) +} + +func traceGCScanDone() { + traceEvent(traceEvGCScanDone, -1) +} + +func traceGCSweepStart() { + traceEvent(traceEvGCSweepStart, 1) +} + +func traceGCSweepDone() { + traceEvent(traceEvGCSweepDone, -1) +} + +func traceGoCreate(newg *g, pc uintptr) { + newg.traceseq = 0 + newg.tracelastp = getg().m.p + // +PCQuantum because traceFrameForPC expects return PCs and subtracts PCQuantum. + id := trace.stackTab.put([]uintptr{pc + sys.PCQuantum}) + traceEvent(traceEvGoCreate, 2, uint64(newg.goid), uint64(id)) +} + +func traceGoStart() { + _g_ := getg().m.curg + _p_ := _g_.m.p + _g_.traceseq++ + if _g_.tracelastp == _p_ { + traceEvent(traceEvGoStartLocal, -1, uint64(_g_.goid)) + } else { + _g_.tracelastp = _p_ + traceEvent(traceEvGoStart, -1, uint64(_g_.goid), _g_.traceseq) + } +} + +func traceGoEnd() { + traceEvent(traceEvGoEnd, -1) +} + +func traceGoSched() { + _g_ := getg() + _g_.tracelastp = _g_.m.p + traceEvent(traceEvGoSched, 1) +} + +func traceGoPreempt() { + _g_ := getg() + _g_.tracelastp = _g_.m.p + traceEvent(traceEvGoPreempt, 1) +} + +func traceGoPark(traceEv byte, skip int, gp *g) { + if traceEv&traceFutileWakeup != 0 { + traceEvent(traceEvFutileWakeup, -1) + } + traceEvent(traceEv & ^traceFutileWakeup, skip) +} + +func traceGoUnpark(gp *g, skip int) { + _p_ := getg().m.p + gp.traceseq++ + if gp.tracelastp == _p_ { + traceEvent(traceEvGoUnblockLocal, skip, uint64(gp.goid)) + } else { + gp.tracelastp = _p_ + traceEvent(traceEvGoUnblock, skip, uint64(gp.goid), gp.traceseq) + } +} + +func traceGoSysCall() { + traceEvent(traceEvGoSysCall, 1) +} + +func traceGoSysExit(ts int64) { + if ts != 0 && ts < trace.ticksStart { + // There is a race between the code that initializes sysexitticks + // (in exitsyscall, which runs without a P, and therefore is not + // stopped with the rest of the world) and the code that initializes + // a new trace. The recorded sysexitticks must therefore be treated + // as "best effort". If they are valid for this trace, then great, + // use them for greater accuracy. But if they're not valid for this + // trace, assume that the trace was started after the actual syscall + // exit (but before we actually managed to start the goroutine, + // aka right now), and assign a fresh time stamp to keep the log consistent. + ts = 0 + } + _g_ := getg().m.curg + _g_.traceseq++ + _g_.tracelastp = _g_.m.p + traceEvent(traceEvGoSysExit, -1, uint64(_g_.goid), _g_.traceseq, uint64(ts)/traceTickDiv) +} + +func traceGoSysBlock(pp *p) { + // Sysmon and stopTheWorld can declare syscalls running on remote Ps as blocked, + // to handle this we temporary employ the P. + mp := acquirem() + oldp := mp.p + mp.p.set(pp) + traceEvent(traceEvGoSysBlock, -1) + mp.p = oldp + releasem(mp) +} + +func traceHeapAlloc() { + traceEvent(traceEvHeapAlloc, -1, memstats.heap_live) +} + +func traceNextGC() { + traceEvent(traceEvNextGC, -1, memstats.next_gc) +} + +*/ diff --git a/libgo/runtime/chan.goc b/libgo/runtime/chan.goc deleted file mode 100644 index 6e4c8fd..0000000 --- a/libgo/runtime/chan.goc +++ /dev/null @@ -1,1130 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package runtime -#include "runtime.h" -#include "arch.h" -#include "go-type.h" -#include "malloc.h" -#include "chan.h" - -uint32 runtime_Hchansize = sizeof(Hchan); - -static void dequeueg(WaitQ*); -static SudoG* dequeue(WaitQ*); -static void enqueue(WaitQ*, SudoG*); - -static Hchan* -makechan(ChanType *t, int64 hint) -{ - Hchan *c; - uintptr n; - const Type *elem; - - elem = t->__element_type; - - // compiler checks this but be safe. - if(elem->__size >= (1<<16)) - runtime_throw("makechan: invalid channel element type"); - - if(hint < 0 || (intgo)hint != hint || (elem->__size > 0 && (uintptr)hint > (MaxMem - sizeof(*c)) / elem->__size)) - runtime_panicstring("makechan: size out of range"); - - n = sizeof(*c); - n = ROUND(n, elem->__align); - - // allocate memory in one call - c = (Hchan*)runtime_mallocgc(sizeof(*c) + hint*elem->__size, (uintptr)t | TypeInfo_Chan, 0); - c->elemsize = elem->__size; - c->elemtype = elem; - c->dataqsiz = hint; - - if(debug) - runtime_printf("makechan: chan=%p; elemsize=%D; dataqsiz=%D\n", - c, (int64)elem->__size, (int64)c->dataqsiz); - - return c; -} - -func reflect.makechan(t *ChanType, size uint64) (c *Hchan) { - c = makechan(t, size); -} - -Hchan* -__go_new_channel(ChanType *t, uintptr hint) -{ - return makechan(t, hint); -} - -Hchan* -__go_new_channel_big(ChanType *t, uint64 hint) -{ - return makechan(t, hint); -} - -/* - * generic single channel send/recv - * if the bool pointer is nil, - * then the full exchange will - * occur. if pres is not nil, - * then the protocol will not - * sleep but return if it could - * not complete. - * - * sleep can wake up with g->param == nil - * when a channel involved in the sleep has - * been closed. it is easiest to loop and re-run - * the operation; we'll see that it's now closed. - */ -static bool -chansend(ChanType *t, Hchan *c, byte *ep, bool block, void *pc) -{ - USED(pc); - SudoG *sg; - SudoG mysg; - G* gp; - int64 t0; - G* g; - - g = runtime_g(); - - if(c == nil) { - USED(t); - if(!block) - return false; - runtime_park(nil, nil, "chan send (nil chan)"); - return false; // not reached - } - - if(runtime_gcwaiting()) - runtime_gosched(); - - if(debug) { - runtime_printf("chansend: chan=%p\n", c); - } - - t0 = 0; - mysg.releasetime = 0; - if(runtime_blockprofilerate > 0) { - t0 = runtime_cputicks(); - mysg.releasetime = -1; - } - - runtime_lock(c); - if(c->closed) - goto closed; - - if(c->dataqsiz > 0) - goto asynch; - - sg = dequeue(&c->recvq); - if(sg != nil) { - runtime_unlock(c); - - gp = sg->g; - gp->param = sg; - if(sg->elem != nil) - runtime_memmove(sg->elem, ep, c->elemsize); - if(sg->releasetime) - sg->releasetime = runtime_cputicks(); - runtime_ready(gp); - return true; - } - - if(!block) { - runtime_unlock(c); - return false; - } - - mysg.elem = ep; - mysg.g = g; - mysg.selectdone = nil; - g->param = nil; - enqueue(&c->sendq, &mysg); - runtime_parkunlock(c, "chan send"); - - if(g->param == nil) { - runtime_lock(c); - if(!c->closed) - runtime_throw("chansend: spurious wakeup"); - goto closed; - } - - if(mysg.releasetime > 0) - runtime_blockevent(mysg.releasetime - t0, 2); - - return true; - -asynch: - if(c->closed) - goto closed; - - if(c->qcount >= c->dataqsiz) { - if(!block) { - runtime_unlock(c); - return false; - } - mysg.g = g; - mysg.elem = nil; - mysg.selectdone = nil; - enqueue(&c->sendq, &mysg); - runtime_parkunlock(c, "chan send"); - - runtime_lock(c); - goto asynch; - } - - runtime_memmove(chanbuf(c, c->sendx), ep, c->elemsize); - if(++c->sendx == c->dataqsiz) - c->sendx = 0; - c->qcount++; - - sg = dequeue(&c->recvq); - if(sg != nil) { - gp = sg->g; - runtime_unlock(c); - if(sg->releasetime) - sg->releasetime = runtime_cputicks(); - runtime_ready(gp); - } else - runtime_unlock(c); - if(mysg.releasetime > 0) - runtime_blockevent(mysg.releasetime - t0, 2); - return true; - -closed: - runtime_unlock(c); - runtime_panicstring("send on closed channel"); - return false; // not reached -} - - -static bool -chanrecv(ChanType *t, Hchan* c, byte *ep, bool block, bool *received) -{ - SudoG *sg; - SudoG mysg; - G *gp; - int64 t0; - G *g; - - if(runtime_gcwaiting()) - runtime_gosched(); - - if(debug) - runtime_printf("chanrecv: chan=%p\n", c); - - g = runtime_g(); - - if(c == nil) { - USED(t); - if(!block) - return false; - runtime_park(nil, nil, "chan receive (nil chan)"); - return false; // not reached - } - - t0 = 0; - mysg.releasetime = 0; - if(runtime_blockprofilerate > 0) { - t0 = runtime_cputicks(); - mysg.releasetime = -1; - } - - runtime_lock(c); - if(c->dataqsiz > 0) - goto asynch; - - if(c->closed) - goto closed; - - sg = dequeue(&c->sendq); - if(sg != nil) { - runtime_unlock(c); - - if(ep != nil) - runtime_memmove(ep, sg->elem, c->elemsize); - gp = sg->g; - gp->param = sg; - if(sg->releasetime) - sg->releasetime = runtime_cputicks(); - runtime_ready(gp); - - if(received != nil) - *received = true; - return true; - } - - if(!block) { - runtime_unlock(c); - return false; - } - - mysg.elem = ep; - mysg.g = g; - mysg.selectdone = nil; - g->param = nil; - enqueue(&c->recvq, &mysg); - runtime_parkunlock(c, "chan receive"); - - if(g->param == nil) { - runtime_lock(c); - if(!c->closed) - runtime_throw("chanrecv: spurious wakeup"); - goto closed; - } - - if(received != nil) - *received = true; - if(mysg.releasetime > 0) - runtime_blockevent(mysg.releasetime - t0, 2); - return true; - -asynch: - if(c->qcount <= 0) { - if(c->closed) - goto closed; - - if(!block) { - runtime_unlock(c); - if(received != nil) - *received = false; - return false; - } - mysg.g = g; - mysg.elem = nil; - mysg.selectdone = nil; - enqueue(&c->recvq, &mysg); - runtime_parkunlock(c, "chan receive"); - - runtime_lock(c); - goto asynch; - } - - if(ep != nil) - runtime_memmove(ep, chanbuf(c, c->recvx), c->elemsize); - runtime_memclr(chanbuf(c, c->recvx), c->elemsize); - if(++c->recvx == c->dataqsiz) - c->recvx = 0; - c->qcount--; - - sg = dequeue(&c->sendq); - if(sg != nil) { - gp = sg->g; - runtime_unlock(c); - if(sg->releasetime) - sg->releasetime = runtime_cputicks(); - runtime_ready(gp); - } else - runtime_unlock(c); - - if(received != nil) - *received = true; - if(mysg.releasetime > 0) - runtime_blockevent(mysg.releasetime - t0, 2); - return true; - -closed: - if(ep != nil) - runtime_memclr(ep, c->elemsize); - if(received != nil) - *received = false; - runtime_unlock(c); - if(mysg.releasetime > 0) - runtime_blockevent(mysg.releasetime - t0, 2); - return true; -} - -// The compiler generates a call to __go_send_small to send a value 8 -// bytes or smaller. -void -__go_send_small(ChanType *t, Hchan* c, uint64 val) -{ - union - { - byte b[sizeof(uint64)]; - uint64 v; - } u; - byte *v; - - u.v = val; -#ifndef WORDS_BIGENDIAN - v = u.b; -#else - v = u.b + sizeof(uint64) - t->__element_type->__size; -#endif - chansend(t, c, v, true, runtime_getcallerpc(&t)); -} - -// The compiler generates a call to __go_send_big to send a value -// larger than 8 bytes or smaller. -void -__go_send_big(ChanType *t, Hchan* c, byte* v) -{ - chansend(t, c, v, true, runtime_getcallerpc(&t)); -} - -// The compiler generates a call to __go_receive to receive a -// value from a channel. -void -__go_receive(ChanType *t, Hchan* c, byte* v) -{ - chanrecv(t, c, v, true, nil); -} - -_Bool runtime_chanrecv2(ChanType *t, Hchan* c, byte* v) - __asm__ (GOSYM_PREFIX "runtime.chanrecv2"); - -_Bool -runtime_chanrecv2(ChanType *t, Hchan* c, byte* v) -{ - bool received = false; - - chanrecv(t, c, v, true, &received); - return received; -} - -// compiler implements -// -// select { -// case c <- v: -// ... foo -// default: -// ... bar -// } -// -// as -// -// if selectnbsend(c, v) { -// ... foo -// } else { -// ... bar -// } -// -func selectnbsend(t *ChanType, c *Hchan, elem *byte) (selected bool) { - selected = chansend(t, c, elem, false, runtime_getcallerpc(&t)); -} - -// compiler implements -// -// select { -// case v = <-c: -// ... foo -// default: -// ... bar -// } -// -// as -// -// if selectnbrecv(&v, c) { -// ... foo -// } else { -// ... bar -// } -// -func selectnbrecv(t *ChanType, elem *byte, c *Hchan) (selected bool) { - selected = chanrecv(t, c, elem, false, nil); -} - -// compiler implements -// -// select { -// case v, ok = <-c: -// ... foo -// default: -// ... bar -// } -// -// as -// -// if c != nil && selectnbrecv2(&v, &ok, c) { -// ... foo -// } else { -// ... bar -// } -// -func selectnbrecv2(t *ChanType, elem *byte, received *bool, c *Hchan) (selected bool) { - bool r; - - selected = chanrecv(t, c, elem, false, received == nil ? nil : &r); - if(received != nil) - *received = r; -} - -func reflect.chansend(t *ChanType, c *Hchan, elem *byte, nb bool) (selected bool) { - selected = chansend(t, c, elem, !nb, runtime_getcallerpc(&t)); -} - -func reflect.chanrecv(t *ChanType, c *Hchan, nb bool, elem *byte) (selected bool, received bool) { - received = false; - selected = chanrecv(t, c, elem, !nb, &received); -} - -static Select* newselect(int32); - -func newselect(size int32) (sel *byte) { - sel = (byte*)newselect(size); -} - -static Select* -newselect(int32 size) -{ - int32 n; - Select *sel; - - n = 0; - if(size > 1) - n = size-1; - - // allocate all the memory we need in a single allocation - // start with Select with size cases - // then lockorder with size entries - // then pollorder with size entries - sel = runtime_mal(sizeof(*sel) + - n*sizeof(sel->scase[0]) + - size*sizeof(sel->lockorder[0]) + - size*sizeof(sel->pollorder[0])); - - sel->tcase = size; - sel->ncase = 0; - sel->lockorder = (void*)(sel->scase + size); - sel->pollorder = (void*)(sel->lockorder + size); - - if(debug) - runtime_printf("newselect s=%p size=%d\n", sel, size); - return sel; -} - -// cut in half to give stack a chance to split -static void selectsend(Select *sel, Hchan *c, int index, void *elem); - -func selectsend(sel *Select, c *Hchan, elem *byte, index int32) { - // nil cases do not compete - if(c != nil) - selectsend(sel, c, index, elem); -} - -static void -selectsend(Select *sel, Hchan *c, int index, void *elem) -{ - int32 i; - Scase *cas; - - i = sel->ncase; - if(i >= sel->tcase) - runtime_throw("selectsend: too many cases"); - sel->ncase = i+1; - cas = &sel->scase[i]; - - cas->index = index; - cas->chan = c; - cas->kind = CaseSend; - cas->sg.elem = elem; - - if(debug) - runtime_printf("selectsend s=%p index=%d chan=%p\n", - sel, cas->index, cas->chan); -} - -// cut in half to give stack a chance to split -static void selectrecv(Select *sel, Hchan *c, int index, void *elem, bool*); - -func selectrecv(sel *Select, c *Hchan, elem *byte, index int32) { - // nil cases do not compete - if(c != nil) - selectrecv(sel, c, index, elem, nil); -} - -func selectrecv2(sel *Select, c *Hchan, elem *byte, received *bool, index int32) { - // nil cases do not compete - if(c != nil) - selectrecv(sel, c, index, elem, received); -} - -static void -selectrecv(Select *sel, Hchan *c, int index, void *elem, bool *received) -{ - int32 i; - Scase *cas; - - i = sel->ncase; - if(i >= sel->tcase) - runtime_throw("selectrecv: too many cases"); - sel->ncase = i+1; - cas = &sel->scase[i]; - cas->index = index; - cas->chan = c; - - cas->kind = CaseRecv; - cas->sg.elem = elem; - cas->receivedp = received; - - if(debug) - runtime_printf("selectrecv s=%p index=%d chan=%p\n", - sel, cas->index, cas->chan); -} - -// cut in half to give stack a chance to split -static void selectdefault(Select*, int); - -func selectdefault(sel *Select, index int32) { - selectdefault(sel, index); -} - -static void -selectdefault(Select *sel, int32 index) -{ - int32 i; - Scase *cas; - - i = sel->ncase; - if(i >= sel->tcase) - runtime_throw("selectdefault: too many cases"); - sel->ncase = i+1; - cas = &sel->scase[i]; - cas->index = index; - cas->chan = nil; - - cas->kind = CaseDefault; - - if(debug) - runtime_printf("selectdefault s=%p index=%d\n", - sel, cas->index); -} - -static void -sellock(Select *sel) -{ - uint32 i; - Hchan *c, *c0; - - c = nil; - for(i=0; i<sel->ncase; i++) { - c0 = sel->lockorder[i]; - if(c0 && c0 != c) { - c = sel->lockorder[i]; - runtime_lock(c); - } - } -} - -static void -selunlock(Select *sel) -{ - int32 i, n, r; - Hchan *c; - - // We must be very careful here to not touch sel after we have unlocked - // the last lock, because sel can be freed right after the last unlock. - // Consider the following situation. - // First M calls runtime_park() in runtime_selectgo() passing the sel. - // Once runtime_park() has unlocked the last lock, another M makes - // the G that calls select runnable again and schedules it for execution. - // When the G runs on another M, it locks all the locks and frees sel. - // Now if the first M touches sel, it will access freed memory. - n = (int32)sel->ncase; - r = 0; - // skip the default case - if(n>0 && sel->lockorder[0] == nil) - r = 1; - for(i = n-1; i >= r; i--) { - c = sel->lockorder[i]; - if(i>0 && sel->lockorder[i-1] == c) - continue; // will unlock it on the next iteration - runtime_unlock(c); - } -} - -static bool -selparkcommit(G *gp, void *sel) -{ - USED(gp); - selunlock(sel); - return true; -} - -func block() { - runtime_park(nil, nil, "select (no cases)"); // forever -} - -static int selectgo(Select**); - -// selectgo(sel *byte); - -func selectgo(sel *Select) (ret int32) { - return selectgo(&sel); -} - -static int -selectgo(Select **selp) -{ - Select *sel; - uint32 o, i, j, k, done; - int64 t0; - Scase *cas, *dfl; - Hchan *c; - SudoG *sg; - G *gp; - int index; - G *g; - - sel = *selp; - if(runtime_gcwaiting()) - runtime_gosched(); - - if(debug) - runtime_printf("select: sel=%p\n", sel); - - g = runtime_g(); - - t0 = 0; - if(runtime_blockprofilerate > 0) { - t0 = runtime_cputicks(); - for(i=0; i<sel->ncase; i++) - sel->scase[i].sg.releasetime = -1; - } - - // The compiler rewrites selects that statically have - // only 0 or 1 cases plus default into simpler constructs. - // The only way we can end up with such small sel->ncase - // values here is for a larger select in which most channels - // have been nilled out. The general code handles those - // cases correctly, and they are rare enough not to bother - // optimizing (and needing to test). - - // generate permuted order - for(i=0; i<sel->ncase; i++) - sel->pollorder[i] = i; - for(i=1; i<sel->ncase; i++) { - o = sel->pollorder[i]; - j = runtime_fastrand1()%(i+1); - sel->pollorder[i] = sel->pollorder[j]; - sel->pollorder[j] = o; - } - - // sort the cases by Hchan address to get the locking order. - // simple heap sort, to guarantee n log n time and constant stack footprint. - for(i=0; i<sel->ncase; i++) { - j = i; - c = sel->scase[j].chan; - while(j > 0 && sel->lockorder[k=(j-1)/2] < c) { - sel->lockorder[j] = sel->lockorder[k]; - j = k; - } - sel->lockorder[j] = c; - } - for(i=sel->ncase; i-->0; ) { - c = sel->lockorder[i]; - sel->lockorder[i] = sel->lockorder[0]; - j = 0; - for(;;) { - k = j*2+1; - if(k >= i) - break; - if(k+1 < i && sel->lockorder[k] < sel->lockorder[k+1]) - k++; - if(c < sel->lockorder[k]) { - sel->lockorder[j] = sel->lockorder[k]; - j = k; - continue; - } - break; - } - sel->lockorder[j] = c; - } - /* - for(i=0; i+1<sel->ncase; i++) - if(sel->lockorder[i] > sel->lockorder[i+1]) { - runtime_printf("i=%d %p %p\n", i, sel->lockorder[i], sel->lockorder[i+1]); - runtime_throw("select: broken sort"); - } - */ - sellock(sel); - -loop: - // pass 1 - look for something already waiting - dfl = nil; - for(i=0; i<sel->ncase; i++) { - o = sel->pollorder[i]; - cas = &sel->scase[o]; - c = cas->chan; - - switch(cas->kind) { - case CaseRecv: - if(c->dataqsiz > 0) { - if(c->qcount > 0) - goto asyncrecv; - } else { - sg = dequeue(&c->sendq); - if(sg != nil) - goto syncrecv; - } - if(c->closed) - goto rclose; - break; - - case CaseSend: - if(c->closed) - goto sclose; - if(c->dataqsiz > 0) { - if(c->qcount < c->dataqsiz) - goto asyncsend; - } else { - sg = dequeue(&c->recvq); - if(sg != nil) - goto syncsend; - } - break; - - case CaseDefault: - dfl = cas; - break; - } - } - - if(dfl != nil) { - selunlock(sel); - cas = dfl; - goto retc; - } - - - // pass 2 - enqueue on all chans - done = 0; - for(i=0; i<sel->ncase; i++) { - o = sel->pollorder[i]; - cas = &sel->scase[o]; - c = cas->chan; - sg = &cas->sg; - sg->g = g; - sg->selectdone = &done; - - switch(cas->kind) { - case CaseRecv: - enqueue(&c->recvq, sg); - break; - - case CaseSend: - enqueue(&c->sendq, sg); - break; - } - } - - g->param = nil; - runtime_park(selparkcommit, sel, "select"); - - sellock(sel); - sg = g->param; - - // pass 3 - dequeue from unsuccessful chans - // otherwise they stack up on quiet channels - for(i=0; i<sel->ncase; i++) { - cas = &sel->scase[i]; - if(cas != (Scase*)sg) { - c = cas->chan; - if(cas->kind == CaseSend) - dequeueg(&c->sendq); - else - dequeueg(&c->recvq); - } - } - - if(sg == nil) - goto loop; - - cas = (Scase*)sg; - c = cas->chan; - - if(c->dataqsiz > 0) - runtime_throw("selectgo: shouldn't happen"); - - if(debug) - runtime_printf("wait-return: sel=%p c=%p cas=%p kind=%d\n", - sel, c, cas, cas->kind); - - if(cas->kind == CaseRecv) { - if(cas->receivedp != nil) - *cas->receivedp = true; - } - - selunlock(sel); - goto retc; - -asyncrecv: - // can receive from buffer - if(cas->receivedp != nil) - *cas->receivedp = true; - if(cas->sg.elem != nil) - runtime_memmove(cas->sg.elem, chanbuf(c, c->recvx), c->elemsize); - runtime_memclr(chanbuf(c, c->recvx), c->elemsize); - if(++c->recvx == c->dataqsiz) - c->recvx = 0; - c->qcount--; - sg = dequeue(&c->sendq); - if(sg != nil) { - gp = sg->g; - selunlock(sel); - if(sg->releasetime) - sg->releasetime = runtime_cputicks(); - runtime_ready(gp); - } else { - selunlock(sel); - } - goto retc; - -asyncsend: - // can send to buffer - runtime_memmove(chanbuf(c, c->sendx), cas->sg.elem, c->elemsize); - if(++c->sendx == c->dataqsiz) - c->sendx = 0; - c->qcount++; - sg = dequeue(&c->recvq); - if(sg != nil) { - gp = sg->g; - selunlock(sel); - if(sg->releasetime) - sg->releasetime = runtime_cputicks(); - runtime_ready(gp); - } else { - selunlock(sel); - } - goto retc; - -syncrecv: - // can receive from sleeping sender (sg) - selunlock(sel); - if(debug) - runtime_printf("syncrecv: sel=%p c=%p o=%d\n", sel, c, o); - if(cas->receivedp != nil) - *cas->receivedp = true; - if(cas->sg.elem != nil) - runtime_memmove(cas->sg.elem, sg->elem, c->elemsize); - gp = sg->g; - gp->param = sg; - if(sg->releasetime) - sg->releasetime = runtime_cputicks(); - runtime_ready(gp); - goto retc; - -rclose: - // read at end of closed channel - selunlock(sel); - if(cas->receivedp != nil) - *cas->receivedp = false; - if(cas->sg.elem != nil) - runtime_memclr(cas->sg.elem, c->elemsize); - goto retc; - -syncsend: - // can send to sleeping receiver (sg) - selunlock(sel); - if(debug) - runtime_printf("syncsend: sel=%p c=%p o=%d\n", sel, c, o); - if(sg->elem != nil) - runtime_memmove(sg->elem, cas->sg.elem, c->elemsize); - gp = sg->g; - gp->param = sg; - if(sg->releasetime) - sg->releasetime = runtime_cputicks(); - runtime_ready(gp); - -retc: - // return index corresponding to chosen case - index = cas->index; - if(cas->sg.releasetime > 0) - runtime_blockevent(cas->sg.releasetime - t0, 2); - runtime_free(sel); - return index; - -sclose: - // send on closed channel - selunlock(sel); - runtime_panicstring("send on closed channel"); - return 0; // not reached -} - -// This struct must match ../reflect/value.go:/runtimeSelect. -typedef struct runtimeSelect runtimeSelect; -struct runtimeSelect -{ - intgo dir; - ChanType *typ; - Hchan *ch; - byte *val; -}; - -// This enum must match ../reflect/value.go:/SelectDir. -enum SelectDir { - SelectSend = 1, - SelectRecv, - SelectDefault, -}; - -func reflect.rselect(cases Slice) (chosen int, recvOK bool) { - int32 i; - Select *sel; - runtimeSelect* rcase, *rc; - - chosen = -1; - recvOK = false; - - rcase = (runtimeSelect*)cases.__values; - - sel = newselect(cases.__count); - for(i=0; i<cases.__count; i++) { - rc = &rcase[i]; - switch(rc->dir) { - case SelectDefault: - selectdefault(sel, i); - break; - case SelectSend: - if(rc->ch == nil) - break; - selectsend(sel, rc->ch, i, rc->val); - break; - case SelectRecv: - if(rc->ch == nil) - break; - selectrecv(sel, rc->ch, i, rc->val, &recvOK); - break; - } - } - - chosen = (intgo)(uintptr)selectgo(&sel); -} - -static void closechan(Hchan *c, void *pc); - -func closechan(c *Hchan) { - closechan(c, runtime_getcallerpc(&c)); -} - -func reflect.chanclose(c *Hchan) { - closechan(c, runtime_getcallerpc(&c)); -} - -static void -closechan(Hchan *c, void *pc) -{ - USED(pc); - SudoG *sg; - G* gp; - - if(c == nil) - runtime_panicstring("close of nil channel"); - - if(runtime_gcwaiting()) - runtime_gosched(); - - runtime_lock(c); - if(c->closed) { - runtime_unlock(c); - runtime_panicstring("close of closed channel"); - } - c->closed = true; - - // release all readers - for(;;) { - sg = dequeue(&c->recvq); - if(sg == nil) - break; - gp = sg->g; - gp->param = nil; - if(sg->releasetime) - sg->releasetime = runtime_cputicks(); - runtime_ready(gp); - } - - // release all writers - for(;;) { - sg = dequeue(&c->sendq); - if(sg == nil) - break; - gp = sg->g; - gp->param = nil; - if(sg->releasetime) - sg->releasetime = runtime_cputicks(); - runtime_ready(gp); - } - - runtime_unlock(c); -} - -void -__go_builtin_close(Hchan *c) -{ - runtime_closechan(c); -} - -func reflect.chanlen(c *Hchan) (len int) { - if(c == nil) - len = 0; - else - len = c->qcount; -} - -func reflect.chancap(c *Hchan) (cap int) { - if(c == nil) - cap = 0; - else - cap = c->dataqsiz; -} - -intgo -__go_chan_cap(Hchan *c) -{ - return reflect_chancap(c); -} - -static SudoG* -dequeue(WaitQ *q) -{ - SudoG *sgp; - -loop: - sgp = q->first; - if(sgp == nil) - return nil; - q->first = sgp->link; - - // if sgp participates in a select and is already signaled, ignore it - if(sgp->selectdone != nil) { - // claim the right to signal - if(*sgp->selectdone != 0 || !runtime_cas(sgp->selectdone, 0, 1)) - goto loop; - } - - return sgp; -} - -static void -dequeueg(WaitQ *q) -{ - SudoG **l, *sgp, *prevsgp; - G *g; - - g = runtime_g(); - prevsgp = nil; - for(l=&q->first; (sgp=*l) != nil; l=&sgp->link, prevsgp=sgp) { - if(sgp->g == g) { - *l = sgp->link; - if(q->last == sgp) - q->last = prevsgp; - break; - } - } -} - -static void -enqueue(WaitQ *q, SudoG *sgp) -{ - sgp->link = nil; - if(q->first == nil) { - q->first = sgp; - q->last = sgp; - return; - } - q->last->link = sgp; - q->last = sgp; -} diff --git a/libgo/runtime/chan.h b/libgo/runtime/chan.h deleted file mode 100644 index 473f365..0000000 --- a/libgo/runtime/chan.h +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -typedef struct WaitQ WaitQ; -typedef struct SudoG SudoG; -typedef struct Select Select; -typedef struct Scase Scase; - -typedef struct __go_type_descriptor Type; -typedef struct __go_channel_type ChanType; - -struct SudoG -{ - G* g; - uint32* selectdone; - SudoG* link; - int64 releasetime; - byte* elem; // data element - uint32 ticket; -}; - -struct WaitQ -{ - SudoG* first; - SudoG* last; -}; - -// The garbage collector is assuming that Hchan can only contain pointers into the stack -// and cannot contain pointers into the heap. -struct Hchan -{ - uintgo qcount; // total data in the q - uintgo dataqsiz; // size of the circular q - uint16 elemsize; - uint16 pad; // ensures proper alignment of the buffer that follows Hchan in memory - bool closed; - const Type* elemtype; // element type - uintgo sendx; // send index - uintgo recvx; // receive index - WaitQ recvq; // list of recv waiters - WaitQ sendq; // list of send waiters - Lock; -}; - -// Buffer follows Hchan immediately in memory. -// chanbuf(c, i) is pointer to the i'th slot in the buffer. -#define chanbuf(c, i) ((byte*)((c)+1)+(uintptr)(c)->elemsize*(i)) - -enum -{ - debug = 0, - - // Scase.kind - CaseRecv, - CaseSend, - CaseDefault, -}; - -struct Scase -{ - SudoG sg; // must be first member (cast to Scase) - Hchan* chan; // chan - uint16 kind; - uint16 index; // index to return - bool* receivedp; // pointer to received bool (recv2) -}; - -struct Select -{ - uint16 tcase; // total count of scase[] - uint16 ncase; // currently filled scase[] - uint16* pollorder; // case poll order - Hchan** lockorder; // channel lock order - Scase scase[1]; // one per case (in order of appearance) -}; diff --git a/libgo/runtime/go-cgo.c b/libgo/runtime/go-cgo.c index 7d0494c..a7d87a7 100644 --- a/libgo/runtime/go-cgo.c +++ b/libgo/runtime/go-cgo.c @@ -10,7 +10,8 @@ #include "go-panic.h" #include "go-type.h" -extern void __go_receive (ChanType *, Hchan *, byte *); +extern void chanrecv1 (ChanType *, Hchan *, void *) + __asm__ (GOSYM_PREFIX "runtime.chanrecv1"); /* Prepare to call from code written in Go to code written in C or C++. This takes the current goroutine out of the Go scheduler, as @@ -97,7 +98,7 @@ syscall_cgocallback () Go. In the case of -buildmode=c-archive or c-shared, this call may be coming in before package initialization is complete. Wait until it is. */ - __go_receive (NULL, runtime_main_init_done, NULL); + chanrecv1 (NULL, runtime_main_init_done, NULL); } mp = runtime_m (); diff --git a/libgo/runtime/heapdump.c b/libgo/runtime/heapdump.c index 18fe913..3cc0c1d 100644 --- a/libgo/runtime/heapdump.c +++ b/libgo/runtime/heapdump.c @@ -462,7 +462,7 @@ dumpparams(void) else dumpbool(true); // big-endian ptrs dumpint(PtrSize); - dumpint(runtime_Hchansize); + dumpint(hchanSize); dumpint((uintptr)runtime_mheap.arena_start); dumpint((uintptr)runtime_mheap.arena_used); dumpint(0); @@ -769,7 +769,7 @@ dumpefacetypes(void *obj __attribute__ ((unused)), uintptr size, const Type *typ case TypeInfo_Chan: if(type->__size == 0) // channels may have zero-sized objects in them break; - for(i = runtime_Hchansize; i <= size - type->__size; i += type->__size) { + for(i = hchanSize; i <= size - type->__size; i += type->__size) { //playgcprog(i, (uintptr*)type->gc + 1, dumpeface_callback, obj); } break; diff --git a/libgo/runtime/mgc0.c b/libgo/runtime/mgc0.c index 341544c..ac6e396 100644 --- a/libgo/runtime/mgc0.c +++ b/libgo/runtime/mgc0.c @@ -56,7 +56,6 @@ #include "arch.h" #include "malloc.h" #include "mgc0.h" -#include "chan.h" #include "go-type.h" // Map gccgo field names to gc field names. @@ -1112,15 +1111,13 @@ scanblock(Workbuf *wbuf, bool keepworking) // There are no heap pointers in struct Hchan, // so we can ignore the leading sizeof(Hchan) bytes. if(!(chantype->elem->__code & kindNoPointers)) { - // Channel's buffer follows Hchan immediately in memory. - // Size of buffer (cap(c)) is second int in the chan struct. - chancap = ((uintgo*)chan)[1]; - if(chancap > 0) { + chancap = chan->dataqsiz; + if(chancap > 0 && markonly(chan->buf)) { // TODO(atom): split into two chunks so that only the // in-use part of the circular buffer is scanned. // (Channel routines zero the unused part, so the current // code does not lead to leaks, it's just a little inefficient.) - *sbuf.obj.pos++ = (Obj){(byte*)chan+runtime_Hchansize, chancap*chantype->elem->__size, + *sbuf.obj.pos++ = (Obj){chan->buf, chancap*chantype->elem->__size, (uintptr)chantype->elem->__gc | PRECISE | LOOP}; if(sbuf.obj.pos == sbuf.obj.end) flushobjbuf(&sbuf); diff --git a/libgo/runtime/proc.c b/libgo/runtime/proc.c index dac32eb..02b62be 100644 --- a/libgo/runtime/proc.c +++ b/libgo/runtime/proc.c @@ -564,7 +564,8 @@ static struct __go_channel_type chan_bool_type_descriptor = CHANNEL_BOTH_DIR }; -extern Hchan *__go_new_channel (ChanType *, uintptr); +extern Hchan *makechan (ChanType *, int64) + __asm__ (GOSYM_PREFIX "runtime.makechan"); extern void closechan(Hchan *) __asm__ (GOSYM_PREFIX "runtime.closechan"); static void @@ -613,7 +614,7 @@ runtime_main(void* dummy __attribute__((unused))) runtime_throw("runtime_main not on m0"); __go_go(runtime_MHeap_Scavenger, nil); - runtime_main_init_done = __go_new_channel(&chan_bool_type_descriptor, 0); + runtime_main_init_done = makechan(&chan_bool_type_descriptor, 0); _cgo_notify_runtime_init_done(); @@ -853,6 +854,14 @@ runtime_ready(G *gp) g->m->locks--; } +void goready(G*, int) __asm__ (GOSYM_PREFIX "runtime.goready"); + +void +goready(G* gp, int traceskip __attribute__ ((unused))) +{ + runtime_ready(gp); +} + int32 runtime_gcprocs(void) { @@ -1898,6 +1907,22 @@ runtime_park(bool(*unlockf)(G*, void*), void *lock, const char *reason) runtime_mcall(park0); } +void gopark(FuncVal *, void *, String, byte, int) + __asm__ ("runtime.gopark"); + +void +gopark(FuncVal *unlockf, void *lock, String reason, + byte traceEv __attribute__ ((unused)), + int traceskip __attribute__ ((unused))) +{ + if(g->atomicstatus != _Grunning) + runtime_throw("bad g status"); + g->m->waitlock = lock; + g->m->waitunlockf = unlockf == nil ? nil : (void*)unlockf->fn; + g->waitreason = reason; + runtime_mcall(park0); +} + static bool parkunlock(G *gp, void *lock) { @@ -1914,6 +1939,21 @@ runtime_parkunlock(Lock *lock, const char *reason) runtime_park(parkunlock, lock, reason); } +void goparkunlock(Lock *, String, byte, int) + __asm__ (GOSYM_PREFIX "runtime.goparkunlock"); + +void +goparkunlock(Lock *lock, String reason, byte traceEv __attribute__ ((unused)), + int traceskip __attribute__ ((unused))) +{ + if(g->atomicstatus != _Grunning) + runtime_throw("bad g status"); + g->m->waitlock = lock; + g->m->waitunlockf = parkunlock; + g->waitreason = reason; + runtime_mcall(park0); +} + // runtime_park continuation on g0. static void park0(G *gp) diff --git a/libgo/runtime/runtime.h b/libgo/runtime/runtime.h index d1aad1e..6f96b2b 100644 --- a/libgo/runtime/runtime.h +++ b/libgo/runtime/runtime.h @@ -66,7 +66,7 @@ typedef struct FuncVal FuncVal; typedef struct SigTab SigTab; typedef struct mcache MCache; typedef struct FixAlloc FixAlloc; -typedef struct Hchan Hchan; +typedef struct hchan Hchan; typedef struct Timers Timers; typedef struct Timer Timer; typedef struct gcstats GCStats; @@ -75,6 +75,7 @@ typedef struct ParFor ParFor; typedef struct ParForThread ParForThread; typedef struct cgoMal CgoMal; typedef struct PollDesc PollDesc; +typedef struct sudog SudoG; typedef struct __go_open_array Slice; typedef struct __go_interface Iface; @@ -294,7 +295,6 @@ extern uint32 runtime_panicking; extern int8* runtime_goos; extern int32 runtime_ncpu; extern void (*runtime_sysargs)(int32, uint8**); -extern uint32 runtime_Hchansize; extern struct debugVars runtime_debug; extern uintptr runtime_maxstacksize; diff --git a/libgo/runtime/sema.goc b/libgo/runtime/sema.goc index b49c7b7..b0d198e 100644 --- a/libgo/runtime/sema.goc +++ b/libgo/runtime/sema.goc @@ -19,7 +19,6 @@ package sync #include "runtime.h" -#include "chan.h" #include "arch.h" typedef struct SemaWaiter SemaWaiter; @@ -373,7 +372,7 @@ func runtime_notifyListWait(l *notifyList, t uint32) { if (l->tail == nil) { l->head = &s; } else { - l->tail->link = &s; + l->tail->next = &s; } l->tail = &s; runtime_parkunlock(&l->lock, "semacquire"); @@ -409,8 +408,8 @@ func runtime_notifyListNotifyAll(l *notifyList) { // Go through the local list and ready all waiters. while (s != nil) { - SudoG* next = s->link; - s->link = nil; + SudoG* next = s->next; + s->next = nil; readyWithTime(s, 4); s = next; } @@ -442,11 +441,11 @@ func runtime_notifyListNotifyOne(l *notifyList) { // needs to be notified. If it hasn't made it to the list yet we won't // find it, but it won't park itself once it sees the new notify number. runtime_atomicstore(&l->notify, t+1); - for (p = nil, s = l->head; s != nil; p = s, s = s->link) { + for (p = nil, s = l->head; s != nil; p = s, s = s->next) { if (s->ticket == t) { - SudoG *n = s->link; + SudoG *n = s->next; if (p != nil) { - p->link = n; + p->next = n; } else { l->head = n; } @@ -454,7 +453,7 @@ func runtime_notifyListNotifyOne(l *notifyList) { l->tail = p; } runtime_unlock(&l->lock); - s->link = nil; + s->next = nil; readyWithTime(s, 4); return; } |