diff options
Diffstat (limited to 'libgo/runtime/mgc0.c')
| -rw-r--r-- | libgo/runtime/mgc0.c | 853 |
1 files changed, 693 insertions, 160 deletions
diff --git a/libgo/runtime/mgc0.c b/libgo/runtime/mgc0.c index 88283cc..36afd2b 100644 --- a/libgo/runtime/mgc0.c +++ b/libgo/runtime/mgc0.c @@ -21,8 +21,11 @@ #define tab __methods // Eface aka __go_empty_interface. #define type __type_descriptor +// Hmap aka __go_map +typedef struct __go_map Hmap; // Type aka __go_type_descriptor #define kind __code +#define string __reflection #define KindPtr GO_PTR #define KindNoPointers GO_NO_POINTERS // PtrType aka __go_ptr_type @@ -41,6 +44,9 @@ extern void * __splitstack_find_context (void *context[10], size_t *, void **, enum { Debug = 0, DebugMark = 0, // run second pass to check mark + CollectStats = 0, + ScanStackByFrames = 0, + IgnorePreciseGC = 0, // Four bits per word (see #defines below). wordsPerBitmapWord = sizeof(void*)*8/4, @@ -147,6 +153,7 @@ static Workbuf* getempty(Workbuf*); static Workbuf* getfull(Workbuf*); static void putempty(Workbuf*); static Workbuf* handoff(Workbuf*); +static void gchelperstart(void); static struct { uint64 full; // lock-free list of full blocks @@ -170,11 +177,114 @@ static struct { } work; enum { - // TODO(atom): to be expanded in a next CL GC_DEFAULT_PTR = GC_NUM_INSTR, + GC_MAP_NEXT, + GC_CHAN, + + GC_NUM_INSTR2 }; -// PtrTarget and BitTarget are structures used by intermediate buffers. +static struct { + struct { + uint64 sum; + uint64 cnt; + } ptr; + uint64 nbytes; + struct { + uint64 sum; + uint64 cnt; + uint64 notype; + uint64 typelookup; + } obj; + uint64 rescan; + uint64 rescanbytes; + uint64 instr[GC_NUM_INSTR2]; + uint64 putempty; + uint64 getfull; +} gcstats; + +// markonly marks an object. It returns true if the object +// has been marked by this function, false otherwise. +// This function doesn't append the object to any buffer. +static bool +markonly(void *obj) +{ + byte *p; + uintptr *bitp, bits, shift, x, xbits, off; + MSpan *s; + PageID k; + + // Words outside the arena cannot be pointers. + if((byte*)obj < runtime_mheap->arena_start || (byte*)obj >= runtime_mheap->arena_used) + return false; + + // obj may be a pointer to a live object. + // Try to find the beginning of the object. + + // Round down to word boundary. + obj = (void*)((uintptr)obj & ~((uintptr)PtrSize-1)); + + // Find bits for this word. + off = (uintptr*)obj - (uintptr*)runtime_mheap->arena_start; + bitp = (uintptr*)runtime_mheap->arena_start - off/wordsPerBitmapWord - 1; + shift = off % wordsPerBitmapWord; + xbits = *bitp; + bits = xbits >> shift; + + // Pointing at the beginning of a block? + if((bits & (bitAllocated|bitBlockBoundary)) != 0) + goto found; + + // Otherwise consult span table to find beginning. + // (Manually inlined copy of MHeap_LookupMaybe.) + k = (uintptr)obj>>PageShift; + x = k; + if(sizeof(void*) == 8) + x -= (uintptr)runtime_mheap->arena_start>>PageShift; + s = runtime_mheap->map[x]; + if(s == nil || k < s->start || k - s->start >= s->npages || s->state != MSpanInUse) + return false; + p = (byte*)((uintptr)s->start<<PageShift); + if(s->sizeclass == 0) { + obj = p; + } else { + if((byte*)obj >= (byte*)s->limit) + return false; + uintptr size = s->elemsize; + int32 i = ((byte*)obj - p)/size; + obj = p+i*size; + } + + // Now that we know the object header, reload bits. + off = (uintptr*)obj - (uintptr*)runtime_mheap->arena_start; + bitp = (uintptr*)runtime_mheap->arena_start - off/wordsPerBitmapWord - 1; + shift = off % wordsPerBitmapWord; + xbits = *bitp; + bits = xbits >> shift; + +found: + // Now we have bits, bitp, and shift correct for + // obj pointing at the base of the object. + // Only care about allocated and not marked. + if((bits & (bitAllocated|bitMarked)) != bitAllocated) + return false; + if(work.nproc == 1) + *bitp |= bitMarked<<shift; + else { + for(;;) { + x = *bitp; + if(x & (bitMarked<<shift)) + return false; + if(runtime_casp((void**)bitp, (void*)x, (void*)(x|(bitMarked<<shift)))) + break; + } + } + + // The object is now marked + return true; +} + +// PtrTarget is a structure used by intermediate buffers. // The intermediate buffers hold GC data before it // is moved/flushed to the work buffer (Workbuf). // The size of an intermediate buffer is very small, @@ -186,24 +296,16 @@ struct PtrTarget uintptr ti; }; -typedef struct BitTarget BitTarget; -struct BitTarget -{ - void *p; - uintptr ti; - uintptr *bitp, shift; -}; - typedef struct BufferList BufferList; struct BufferList { PtrTarget ptrtarget[IntermediateBufferCapacity]; - BitTarget bittarget[IntermediateBufferCapacity]; - BufferList *next; + Obj obj[IntermediateBufferCapacity]; + uint32 busy; + byte pad[CacheLineSize]; }; -static BufferList *bufferList; +static BufferList bufferList[MaxGcproc]; -static Lock lock; static Type *itabtype; static void enqueue(Obj obj, Workbuf **_wbuf, Obj **_wp, uintptr *_nobj); @@ -214,7 +316,6 @@ static void enqueue(Obj obj, Workbuf **_wbuf, Obj **_wp, uintptr *_nobj); // and are prepared to be scanned by the garbage collector. // // _wp, _wbuf, _nobj are input/output parameters and are specifying the work buffer. -// bitbuf holds temporary data generated by this function. // // A simplified drawing explaining how the todo-list moves from a structure to another: // @@ -222,14 +323,12 @@ static void enqueue(Obj obj, Workbuf **_wbuf, Obj **_wp, uintptr *_nobj); // (find pointers) // Obj ------> PtrTarget (pointer targets) // ↑ | -// | | flushptrbuf (1st part, -// | | find block start) -// | ↓ -// `--------- BitTarget (pointer targets and the corresponding locations in bitmap) -// flushptrbuf -// (2nd part, mark and enqueue) +// | | +// `----------' +// flushptrbuf +// (find block start, mark and enqueue) static void -flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf, uintptr *_nobj, BitTarget *bitbuf) +flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf, uintptr *_nobj) { byte *p, *arena_start, *obj; uintptr size, *bitp, bits, shift, j, x, xbits, off, nobj, ti, n; @@ -238,9 +337,8 @@ flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf Obj *wp; Workbuf *wbuf; PtrTarget *ptrbuf_end; - BitTarget *bitbufpos, *bt; - arena_start = runtime_mheap.arena_start; + arena_start = runtime_mheap->arena_start; wp = *_wp; wbuf = *_wbuf; @@ -250,6 +348,11 @@ flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf n = ptrbuf_end - ptrbuf; *ptrbufpos = ptrbuf; + if(CollectStats) { + runtime_xadd64(&gcstats.ptr.sum, n); + runtime_xadd64(&gcstats.ptr.cnt, 1); + } + // If buffer is nearly full, get a new one. if(wbuf == nil || nobj+n >= nelem(wbuf->obj)) { if(wbuf != nil) @@ -267,8 +370,6 @@ flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf { // Multi-threaded version. - bitbufpos = bitbuf; - while(ptrbuf < ptrbuf_end) { obj = ptrbuf->p; ti = ptrbuf->ti; @@ -276,7 +377,7 @@ flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf // obj belongs to interval [mheap.arena_start, mheap.arena_used). if(Debug > 1) { - if(obj < runtime_mheap.arena_start || obj >= runtime_mheap.arena_used) + if(obj < runtime_mheap->arena_start || obj >= runtime_mheap->arena_used) runtime_throw("object is outside of mheap"); } @@ -319,7 +420,7 @@ flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf x = k; if(sizeof(void*) == 8) x -= (uintptr)arena_start>>PageShift; - s = runtime_mheap.map[x]; + s = runtime_mheap->map[x]; if(s == nil || k < s->start || k - s->start >= s->npages || s->state != MSpanInUse) continue; p = (byte*)((uintptr)s->start<<PageShift); @@ -346,40 +447,36 @@ flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf // Only care about allocated and not marked. if((bits & (bitAllocated|bitMarked)) != bitAllocated) continue; - - *bitbufpos++ = (BitTarget){obj, ti, bitp, shift}; - } - - runtime_lock(&lock); - for(bt=bitbuf; bt<bitbufpos; bt++){ - xbits = *bt->bitp; - bits = xbits >> bt->shift; - if((bits & bitMarked) != 0) - continue; - - // Mark the block - *bt->bitp = xbits | (bitMarked << bt->shift); + if(work.nproc == 1) + *bitp |= bitMarked<<shift; + else { + for(;;) { + x = *bitp; + if(x & (bitMarked<<shift)) + goto continue_obj; + if(runtime_casp((void**)bitp, (void*)x, (void*)(x|(bitMarked<<shift)))) + break; + } + } // If object has no pointers, don't need to scan further. if((bits & bitNoPointers) != 0) continue; - obj = bt->p; - // Ask span about size class. // (Manually inlined copy of MHeap_Lookup.) x = (uintptr)obj >> PageShift; if(sizeof(void*) == 8) x -= (uintptr)arena_start>>PageShift; - s = runtime_mheap.map[x]; + s = runtime_mheap->map[x]; PREFETCH(obj); - *wp = (Obj){obj, s->elemsize, bt->ti}; + *wp = (Obj){obj, s->elemsize, ti}; wp++; nobj++; + continue_obj:; } - runtime_unlock(&lock); // If another proc wants a pointer, give it some. if(work.nwait > 0 && nobj > handoffThreshold && work.full == 0) { @@ -395,9 +492,73 @@ flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf *_nobj = nobj; } +static void +flushobjbuf(Obj *objbuf, Obj **objbufpos, Obj **_wp, Workbuf **_wbuf, uintptr *_nobj) +{ + uintptr nobj, off; + Obj *wp, obj; + Workbuf *wbuf; + Obj *objbuf_end; + + wp = *_wp; + wbuf = *_wbuf; + nobj = *_nobj; + + objbuf_end = *objbufpos; + *objbufpos = objbuf; + + while(objbuf < objbuf_end) { + obj = *objbuf++; + + // Align obj.b to a word boundary. + off = (uintptr)obj.p & (PtrSize-1); + if(off != 0) { + obj.p += PtrSize - off; + obj.n -= PtrSize - off; + obj.ti = 0; + } + + if(obj.p == nil || obj.n == 0) + continue; + + // If buffer is full, get a new one. + if(wbuf == nil || nobj >= nelem(wbuf->obj)) { + if(wbuf != nil) + wbuf->nobj = nobj; + wbuf = getempty(wbuf); + wp = wbuf->obj; + nobj = 0; + } + + *wp = obj; + wp++; + nobj++; + } + + // If another proc wants a pointer, give it some. + if(work.nwait > 0 && nobj > handoffThreshold && work.full == 0) { + wbuf->nobj = nobj; + wbuf = handoff(wbuf); + nobj = wbuf->nobj; + wp = wbuf->obj + nobj; + } + + *_wp = wp; + *_wbuf = wbuf; + *_nobj = nobj; +} + // Program that scans the whole block and treats every block element as a potential pointer static uintptr defaultProg[2] = {PtrSize, GC_DEFAULT_PTR}; +#if 0 +// Hashmap iterator program +static uintptr mapProg[2] = {0, GC_MAP_NEXT}; + +// Hchan program +static uintptr chanProg[2] = {0, GC_CHAN}; +#endif + // Local variables of a program fragment or loop typedef struct Frame Frame; struct Frame { @@ -405,6 +566,61 @@ struct Frame { uintptr *loop_or_ret; }; +// Sanity check for the derived type info objti. +static void +checkptr(void *obj, uintptr objti) +{ + uintptr type, tisize, i, x; + byte *objstart; + Type *t; + MSpan *s; + + if(!Debug) + runtime_throw("checkptr is debug only"); + + if((byte*)obj < runtime_mheap->arena_start || (byte*)obj >= runtime_mheap->arena_used) + return; + type = runtime_gettype(obj); + t = (Type*)(type & ~(uintptr)(PtrSize-1)); + if(t == nil) + return; + x = (uintptr)obj >> PageShift; + if(sizeof(void*) == 8) + x -= (uintptr)(runtime_mheap->arena_start)>>PageShift; + s = runtime_mheap->map[x]; + objstart = (byte*)((uintptr)s->start<<PageShift); + if(s->sizeclass != 0) { + i = ((byte*)obj - objstart)/s->elemsize; + objstart += i*s->elemsize; + } + tisize = *(uintptr*)objti; + // Sanity check for object size: it should fit into the memory block. + if((byte*)obj + tisize > objstart + s->elemsize) + runtime_throw("invalid gc type info"); + if(obj != objstart) + return; + // If obj points to the beginning of the memory block, + // check type info as well. + if(t->string == nil || + // Gob allocates unsafe pointers for indirection. + (runtime_strcmp((const char *)t->string->str, (const char*)"unsafe.Pointer") && + // Runtime and gc think differently about closures. + runtime_strstr((const char *)t->string->str, (const char*)"struct { F uintptr") != (const char *)t->string->str)) { +#if 0 + pc1 = (uintptr*)objti; + pc2 = (uintptr*)t->gc; + // A simple best-effort check until first GC_END. + for(j = 1; pc1[j] != GC_END && pc2[j] != GC_END; j++) { + if(pc1[j] != pc2[j]) { + runtime_printf("invalid gc type info for '%s' at %p, type info %p, block info %p\n", + t->string ? (const int8*)t->string->str : (const int8*)"?", j, pc1[j], pc2[j]); + runtime_throw("invalid gc type info"); + } + } +#endif + } +} + // scanblock scans a block of n bytes starting at pointer b for references // to other objects, scanning any it finds recursively until there are no // unscanned objects left. Instead of using an explicit recursion, it keeps @@ -419,49 +635,64 @@ static void scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking) { byte *b, *arena_start, *arena_used; - uintptr n, i, end_b, elemsize, ti, objti, count /* , type */; + uintptr n, i, end_b, elemsize, size, ti, objti, count /* , type */; uintptr *pc, precise_type, nominal_size; +#if 0 + uintptr *map_ret, mapkey_size, mapval_size, mapkey_ti, mapval_ti, *chan_ret, chancap; +#endif void *obj; const Type *t; Slice *sliceptr; Frame *stack_ptr, stack_top, stack[GC_STACK_CAPACITY+4]; BufferList *scanbuffers; PtrTarget *ptrbuf, *ptrbuf_end, *ptrbufpos; - BitTarget *bitbuf; + Obj *objbuf, *objbuf_end, *objbufpos; Eface *eface; Iface *iface; +#if 0 + Hmap *hmap; + MapType *maptype; + bool mapkey_kind, mapval_kind; + struct hash_gciter map_iter; + struct hash_gciter_data d; + Hchan *chan; + ChanType *chantype; +#endif if(sizeof(Workbuf) % PageSize != 0) runtime_throw("scanblock: size of Workbuf is suboptimal"); // Memory arena parameters. - arena_start = runtime_mheap.arena_start; - arena_used = runtime_mheap.arena_used; + arena_start = runtime_mheap->arena_start; + arena_used = runtime_mheap->arena_used; stack_ptr = stack+nelem(stack)-1; precise_type = false; nominal_size = 0; - // Allocate ptrbuf, bitbuf + // Allocate ptrbuf { - runtime_lock(&lock); - - if(bufferList == nil) { - bufferList = runtime_SysAlloc(sizeof(*bufferList)); - bufferList->next = nil; - } - scanbuffers = bufferList; - bufferList = bufferList->next; - + scanbuffers = &bufferList[runtime_m()->helpgc]; ptrbuf = &scanbuffers->ptrtarget[0]; ptrbuf_end = &scanbuffers->ptrtarget[0] + nelem(scanbuffers->ptrtarget); - bitbuf = &scanbuffers->bittarget[0]; - - runtime_unlock(&lock); + objbuf = &scanbuffers->obj[0]; + objbuf_end = &scanbuffers->obj[0] + nelem(scanbuffers->obj); } ptrbufpos = ptrbuf; + objbufpos = objbuf; + + // (Silence the compiler) +#if 0 + map_ret = nil; + mapkey_size = mapval_size = 0; + mapkey_kind = mapval_kind = false; + mapkey_ti = mapval_ti = 0; + chan = nil; + chantype = nil; + chan_ret = nil; +#endif goto next_block; @@ -472,7 +703,13 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking) runtime_printf("scanblock %p %D\n", b, (int64)n); } - if(ti != 0 && 0) { + if(CollectStats) { + runtime_xadd64(&gcstats.nbytes, n); + runtime_xadd64(&gcstats.obj.sum, nobj); + runtime_xadd64(&gcstats.obj.cnt, 1); + } + + if(ti != 0 && false) { pc = (uintptr*)(ti & ~(uintptr)PC_BITS); precise_type = (ti & PRECISE); stack_top.elemsize = pc[0]; @@ -484,10 +721,27 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking) } else { stack_top.count = 1; } - } else if(UseSpanType && 0) { + if(Debug) { + // Simple sanity check for provided type info ti: + // The declared size of the object must be not larger than the actual size + // (it can be smaller due to inferior pointers). + // It's difficult to make a comprehensive check due to inferior pointers, + // reflection, gob, etc. + if(pc[0] > n) { + runtime_printf("invalid gc type info: type info size %p, block size %p\n", pc[0], n); + runtime_throw("invalid gc type info"); + } + } + } else if(UseSpanType && false) { + if(CollectStats) + runtime_xadd64(&gcstats.obj.notype, 1); + #if 0 type = runtime_gettype(b); if(type != 0) { + if(CollectStats) + runtime_xadd64(&gcstats.obj.typelookup, 1); + t = (Type*)(type & ~(uintptr)(PtrSize-1)); switch(type & (PtrSize-1)) { case TypeInfo_SingleObject: @@ -506,8 +760,27 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking) stack_top.loop_or_ret = pc+1; break; case TypeInfo_Map: - // TODO(atom): to be expanded in a next CL - pc = defaultProg; + hmap = (Hmap*)b; + maptype = (MapType*)t; + if(hash_gciter_init(hmap, &map_iter)) { + mapkey_size = maptype->key->size; + mapkey_kind = maptype->key->kind; + mapkey_ti = (uintptr)maptype->key->gc | PRECISE; + mapval_size = maptype->elem->size; + mapval_kind = maptype->elem->kind; + mapval_ti = (uintptr)maptype->elem->gc | PRECISE; + + map_ret = nil; + pc = mapProg; + } else { + goto next_block; + } + break; + case TypeInfo_Chan: + chan = (Hchan*)b; + chantype = (ChanType*)t; + chan_ret = nil; + pc = chanProg; break; default: runtime_throw("scanblock: invalid type"); @@ -521,12 +794,18 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking) pc = defaultProg; } + if(IgnorePreciseGC) + pc = defaultProg; + pc++; stack_top.b = (uintptr)b; end_b = (uintptr)b + n - PtrSize; for(;;) { + if(CollectStats) + runtime_xadd64(&gcstats.instr[pc[0]], 1); + obj = nil; objti = 0; switch(pc[0]) { @@ -534,13 +813,19 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking) obj = *(void**)(stack_top.b + pc[1]); objti = pc[2]; pc += 3; + if(Debug) + checkptr(obj, objti); break; case GC_SLICE: sliceptr = (Slice*)(stack_top.b + pc[1]); if(sliceptr->cap != 0) { obj = sliceptr->array; - objti = pc[2] | PRECISE | LOOP; + // Can't use slice element type for scanning, + // because if it points to an array embedded + // in the beginning of a struct, + // we will scan the whole struct as the slice. + // So just obtain type info from heap. } pc += 3; break; @@ -552,17 +837,31 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking) case GC_STRING: obj = *(void**)(stack_top.b + pc[1]); + markonly(obj); pc += 2; - break; + continue; case GC_EFACE: eface = (Eface*)(stack_top.b + pc[1]); pc += 2; - if(eface->type != nil && ((byte*)eface->__object >= arena_start && (byte*)eface->__object < arena_used)) { - t = eface->type; + if(eface->type == nil) + continue; + + // eface->type + t = eface->type; + if((const byte*)t >= arena_start && (const byte*)t < arena_used) { + union { const Type *tc; Type *tr; } u; + u.tc = t; + *ptrbufpos++ = (struct PtrTarget){(void*)u.tr, 0}; + if(ptrbufpos == ptrbuf_end) + flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj); + } + + // eface->__object + if((byte*)eface->__object >= arena_start && (byte*)eface->__object < arena_used) { if(t->__size <= sizeof(void*)) { if((t->kind & KindNoPointers)) - break; + continue; obj = eface->__object; if((t->kind & ~KindNoPointers) == KindPtr) @@ -580,14 +879,14 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking) iface = (Iface*)(stack_top.b + pc[1]); pc += 2; if(iface->tab == nil) - break; + continue; // iface->tab if((byte*)iface->tab >= arena_start && (byte*)iface->tab < arena_used) { // *ptrbufpos++ = (struct PtrTarget){iface->tab, (uintptr)itabtype->gc}; *ptrbufpos++ = (struct PtrTarget){iface->tab, 0}; if(ptrbufpos == ptrbuf_end) - flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj, bitbuf); + flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj); } // iface->data @@ -596,7 +895,7 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking) t = nil; if(t->__size <= sizeof(void*)) { if((t->kind & KindNoPointers)) - break; + continue; obj = iface->__object; if((t->kind & ~KindNoPointers) == KindPtr) @@ -611,13 +910,13 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking) break; case GC_DEFAULT_PTR: - while((i = stack_top.b) <= end_b) { + while(stack_top.b <= end_b) { + obj = *(byte**)stack_top.b; stack_top.b += PtrSize; - obj = *(byte**)i; if((byte*)obj >= arena_start && (byte*)obj < arena_used) { *ptrbufpos++ = (struct PtrTarget){obj, 0}; if(ptrbufpos == ptrbuf_end) - flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj, bitbuf); + flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj); } } goto next_block; @@ -625,9 +924,8 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking) case GC_END: if(--stack_top.count != 0) { // Next iteration of a loop if possible. - elemsize = stack_top.elemsize; - stack_top.b += elemsize; - if(stack_top.b + elemsize <= end_b+PtrSize) { + stack_top.b += stack_top.elemsize; + if(stack_top.b + stack_top.elemsize <= end_b+PtrSize) { pc = stack_top.loop_or_ret; continue; } @@ -648,6 +946,10 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking) // Found a value that may be a pointer. // Do a rescan of the entire block. enqueue((Obj){b, n, 0}, &wbuf, &wp, &nobj); + if(CollectStats) { + runtime_xadd64(&gcstats.rescan, 1); + runtime_xadd64(&gcstats.rescanbytes, n); + } break; } } @@ -680,20 +982,136 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking) // Stack push. *stack_ptr-- = stack_top; stack_top = (Frame){1, 0, stack_top.b + pc[1], pc+3 /*return address*/}; - pc = (uintptr*)pc[2]; // target of the CALL instruction + pc = (uintptr*)((byte*)pc + *(int32*)(pc+2)); // target of the CALL instruction continue; +#if 0 case GC_MAP_PTR: - // TODO(atom): to be expanded in a next CL. Same as GC_APTR for now. - obj = *(void**)(stack_top.b + pc[1]); - pc += 3; - break; + hmap = *(Hmap**)(stack_top.b + pc[1]); + if(hmap == nil) { + pc += 3; + continue; + } + if(markonly(hmap)) { + maptype = (MapType*)pc[2]; + if(hash_gciter_init(hmap, &map_iter)) { + mapkey_size = maptype->key->size; + mapkey_kind = maptype->key->kind; + mapkey_ti = (uintptr)maptype->key->gc | PRECISE; + mapval_size = maptype->elem->size; + mapval_kind = maptype->elem->kind; + mapval_ti = (uintptr)maptype->elem->gc | PRECISE; + + // Start mapProg. + map_ret = pc+3; + pc = mapProg+1; + } else { + pc += 3; + } + } else { + pc += 3; + } + continue; + + case GC_MAP_NEXT: + // Add all keys and values to buffers, mark all subtables. + while(hash_gciter_next(&map_iter, &d)) { + // buffers: reserve space for 2 objects. + if(ptrbufpos+2 >= ptrbuf_end) + flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj); + if(objbufpos+2 >= objbuf_end) + flushobjbuf(objbuf, &objbufpos, &wp, &wbuf, &nobj); + + if(d.st != nil) + markonly(d.st); + + if(d.key_data != nil) { + if(!(mapkey_kind & KindNoPointers) || d.indirectkey) { + if(!d.indirectkey) + *objbufpos++ = (Obj){d.key_data, mapkey_size, mapkey_ti}; + else { + if(Debug) { + obj = *(void**)d.key_data; + if(!(arena_start <= obj && obj < arena_used)) + runtime_throw("scanblock: inconsistent hashmap"); + } + *ptrbufpos++ = (struct PtrTarget){*(void**)d.key_data, mapkey_ti}; + } + } + if(!(mapval_kind & KindNoPointers) || d.indirectval) { + if(!d.indirectval) + *objbufpos++ = (Obj){d.val_data, mapval_size, mapval_ti}; + else { + if(Debug) { + obj = *(void**)d.val_data; + if(!(arena_start <= obj && obj < arena_used)) + runtime_throw("scanblock: inconsistent hashmap"); + } + *ptrbufpos++ = (struct PtrTarget){*(void**)d.val_data, mapval_ti}; + } + } + } + } + if(map_ret == nil) + goto next_block; + pc = map_ret; + continue; +#endif case GC_REGION: - // TODO(atom): to be expanded in a next CL. Same as GC_APTR for now. obj = (void*)(stack_top.b + pc[1]); + size = pc[2]; + objti = pc[3]; pc += 4; - break; + + *objbufpos++ = (Obj){obj, size, objti}; + if(objbufpos == objbuf_end) + flushobjbuf(objbuf, &objbufpos, &wp, &wbuf, &nobj); + continue; + +#if 0 + case GC_CHAN_PTR: + // Similar to GC_MAP_PTR + chan = *(Hchan**)(stack_top.b + pc[1]); + if(chan == nil) { + pc += 3; + continue; + } + if(markonly(chan)) { + chantype = (ChanType*)pc[2]; + if(!(chantype->elem->kind & KindNoPointers)) { + // Start chanProg. + chan_ret = pc+3; + pc = chanProg+1; + continue; + } + } + pc += 3; + continue; + + case GC_CHAN: + // There are no heap pointers in struct Hchan, + // so we can ignore the leading sizeof(Hchan) bytes. + if(!(chantype->elem->kind & KindNoPointers)) { + // Channel's buffer follows Hchan immediately in memory. + // Size of buffer (cap(c)) is second int in the chan struct. + chancap = ((uintgo*)chan)[1]; + if(chancap > 0) { + // TODO(atom): split into two chunks so that only the + // in-use part of the circular buffer is scanned. + // (Channel routines zero the unused part, so the current + // code does not lead to leaks, it's just a little inefficient.) + *objbufpos++ = (Obj){(byte*)chan+runtime_Hchansize, chancap*chantype->elem->size, + (uintptr)chantype->elem->gc | PRECISE | LOOP}; + if(objbufpos == objbuf_end) + flushobjbuf(objbuf, &objbufpos, &wp, &wbuf, &nobj); + } + } + if(chan_ret == nil) + goto next_block; + pc = chan_ret; + continue; +#endif default: runtime_throw("scanblock: invalid GC instruction"); @@ -701,9 +1119,9 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking) } if((byte*)obj >= arena_start && (byte*)obj < arena_used) { - *ptrbufpos++ = (PtrTarget){obj, objti}; + *ptrbufpos++ = (struct PtrTarget){obj, objti}; if(ptrbufpos == ptrbuf_end) - flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj, bitbuf); + flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj); } } @@ -712,7 +1130,8 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking) // the loop by setting b, n, ti to the parameters for the next block. if(nobj == 0) { - flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj, bitbuf); + flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj); + flushobjbuf(objbuf, &objbufpos, &wp, &wbuf, &nobj); if(nobj == 0) { if(!keepworking) { @@ -737,11 +1156,7 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking) nobj--; } -endscan: - runtime_lock(&lock); - scanbuffers->next = bufferList; - bufferList = scanbuffers; - runtime_unlock(&lock); +endscan:; } // debug_scanblock is the debug copy of scanblock. @@ -776,14 +1191,14 @@ debug_scanblock(byte *b, uintptr n) obj = (byte*)vp[i]; // Words outside the arena cannot be pointers. - if((byte*)obj < runtime_mheap.arena_start || (byte*)obj >= runtime_mheap.arena_used) + if((byte*)obj < runtime_mheap->arena_start || (byte*)obj >= runtime_mheap->arena_used) continue; // Round down to word boundary. obj = (void*)((uintptr)obj & ~((uintptr)PtrSize-1)); // Consult span table to find beginning. - s = runtime_MHeap_LookupMaybe(&runtime_mheap, obj); + s = runtime_MHeap_LookupMaybe(runtime_mheap, obj); if(s == nil) continue; @@ -799,8 +1214,8 @@ debug_scanblock(byte *b, uintptr n) } // Now that we know the object header, reload bits. - off = (uintptr*)obj - (uintptr*)runtime_mheap.arena_start; - bitp = (uintptr*)runtime_mheap.arena_start - off/wordsPerBitmapWord - 1; + off = (uintptr*)obj - (uintptr*)runtime_mheap->arena_start; + bitp = (uintptr*)runtime_mheap->arena_start - off/wordsPerBitmapWord - 1; shift = off % wordsPerBitmapWord; xbits = *bitp; bits = xbits >> shift; @@ -906,6 +1321,8 @@ getempty(Workbuf *b) if(work.nchunk < sizeof *b) { work.nchunk = 1<<20; work.chunk = runtime_SysAlloc(work.nchunk); + if(work.chunk == nil) + runtime_throw("runtime: cannot allocate memory"); } b = (Workbuf*)work.chunk; work.chunk += sizeof *b; @@ -919,6 +1336,9 @@ getempty(Workbuf *b) static void putempty(Workbuf *b) { + if(CollectStats) + runtime_xadd64(&gcstats.putempty, 1); + runtime_lfstackpush(&work.empty, &b->node); } @@ -929,6 +1349,9 @@ getfull(Workbuf *b) M *m; int32 i; + if(CollectStats) + runtime_xadd64(&gcstats.getfull, 1); + if(b != nil) runtime_lfstackpush(&work.empty, &b->node); b = (Workbuf*)runtime_lfstackpop(&work.full); @@ -994,6 +1417,8 @@ addroot(Obj obj) if(cap < 2*work.rootcap) cap = 2*work.rootcap; new = (Obj*)runtime_SysAlloc(cap*sizeof(Obj)); + if(new == nil) + runtime_throw("runtime: cannot allocate memory"); if(work.roots != nil) { runtime_memmove(new, work.roots, work.rootcap*sizeof(Obj)); runtime_SysFree(work.roots, work.rootcap*sizeof(Obj)); @@ -1081,13 +1506,14 @@ static void addfinroots(void *v) { uintptr size; + void *base; size = 0; - if(!runtime_mlookup(v, (byte**)&v, &size, nil) || !runtime_blockspecial(v)) + if(!runtime_mlookup(v, (byte**)&base, &size, nil) || !runtime_blockspecial(base)) runtime_throw("mark - finalizer inconsistency"); // do not mark the finalizer block itself. just mark the things it points at. - addroot((Obj){v, size, 0}); + addroot((Obj){base, size, 0}); } static struct root_list* roots; @@ -1128,22 +1554,27 @@ addroots(void) addroot((Obj){(byte*)&runtime_g0, sizeof runtime_g0, 0}); addroot((Obj){(byte*)&runtime_allg, sizeof runtime_allg, 0}); addroot((Obj){(byte*)&runtime_allm, sizeof runtime_allm, 0}); + addroot((Obj){(byte*)&runtime_allp, sizeof runtime_allp, 0}); + runtime_proc_scan(addroot); runtime_MProf_Mark(addroot); runtime_time_scan(addroot); // MSpan.types - allspans = runtime_mheap.allspans; - for(spanidx=0; spanidx<runtime_mheap.nspan; spanidx++) { + allspans = runtime_mheap->allspans; + for(spanidx=0; spanidx<runtime_mheap->nspan; spanidx++) { s = allspans[spanidx]; if(s->state == MSpanInUse) { + // The garbage collector ignores type pointers stored in MSpan.types: + // - Compiler-generated types are stored outside of heap. + // - The reflect package has runtime-generated types cached in its data structures. + // The garbage collector relies on finding the references via that cache. switch(s->types.compression) { case MTypes_Empty: case MTypes_Single: break; case MTypes_Words: case MTypes_Bytes: - // TODO(atom): consider using defaultProg instead of 0 - addroot((Obj){(byte*)&s->types.data, sizeof(void*), 0}); + markonly((byte*)s->types.data); break; } } @@ -1196,6 +1627,8 @@ handlespecial(byte *p, uintptr size) if(finq == nil || finq->cnt == finq->cap) { if(finc == nil) { finc = runtime_SysAlloc(PageSize); + if(finc == nil) + runtime_throw("runtime: cannot allocate memory"); finc->cap = (PageSize - sizeof(FinBlock)) / sizeof(Finalizer) + 1; finc->alllink = allfin; allfin = finc; @@ -1235,10 +1668,10 @@ sweepspan(ParFor *desc, uint32 idx) m = runtime_m(); USED(&desc); - s = runtime_mheap.allspans[idx]; + s = runtime_mheap->allspans[idx]; if(s->state != MSpanInUse) return; - arena_start = runtime_mheap.arena_start; + arena_start = runtime_mheap->arena_start; p = (byte*)(s->start << PageShift); cl = s->sizeclass; size = s->elemsize; @@ -1301,8 +1734,8 @@ sweepspan(ParFor *desc, uint32 idx) if(cl == 0) { // Free large span. runtime_unmarkspan(p, 1<<PageShift); - *(uintptr*)p = 1; // needs zeroing - runtime_MHeap_Free(&runtime_mheap, s, 1); + *(uintptr*)p = (uintptr)0xdeaddeaddeaddeadll; // needs zeroing + runtime_MHeap_Free(runtime_mheap, s, 1); c->local_alloc -= size; c->local_nfree++; } else { @@ -1316,7 +1749,7 @@ sweepspan(ParFor *desc, uint32 idx) break; } if(size > sizeof(uintptr)) - ((uintptr*)p)[1] = 1; // mark as "needs to be zeroed" + ((uintptr*)p)[1] = (uintptr)0xdeaddeaddeaddeadll; // mark as "needs to be zeroed" end->next = (MLink*)p; end = (MLink*)p; @@ -1330,7 +1763,7 @@ sweepspan(ParFor *desc, uint32 idx) c->local_nfree += nfree; c->local_cachealloc -= nfree * size; c->local_objects -= nfree; - runtime_MCentral_FreeSpan(&runtime_mheap.central[cl], s, nfree, head.next, end); + runtime_MCentral_FreeSpan(&runtime_mheap->central[cl], s, nfree, head.next, end); } } @@ -1344,10 +1777,10 @@ dumpspan(uint32 idx) MSpan *s; bool allocated, special; - s = runtime_mheap.allspans[idx]; + s = runtime_mheap->allspans[idx]; if(s->state != MSpanInUse) return; - arena_start = runtime_mheap.arena_start; + arena_start = runtime_mheap->arena_start; p = (byte*)(s->start << PageShift); sizeclass = s->sizeclass; size = s->elemsize; @@ -1405,7 +1838,7 @@ runtime_memorydump(void) { uint32 spanidx; - for(spanidx=0; spanidx<runtime_mheap.nspan; spanidx++) { + for(spanidx=0; spanidx<runtime_mheap->nspan; spanidx++) { dumpspan(spanidx); } } @@ -1413,6 +1846,8 @@ runtime_memorydump(void) void runtime_gchelper(void) { + gchelperstart(); + // parallel mark for over gc roots runtime_parfordo(work.markfor); @@ -1426,10 +1861,13 @@ runtime_gchelper(void) } runtime_parfordo(work.sweepfor); + bufferList[runtime_m()->helpgc].busy = 0; if(runtime_xadd(&work.ndone, +1) == work.nproc-1) runtime_notewakeup(&work.alldone); } +#define GcpercentUnknown (-2) + // Initialized from $GOGC. GOGC=off means no gc. // // Next gc is after we've allocated an extra amount of @@ -1439,22 +1877,14 @@ runtime_gchelper(void) // proportion to the allocation cost. Adjusting gcpercent // just changes the linear constant (and also the amount of // extra memory used). -static int32 gcpercent = -2; - -static void -stealcache(void) -{ - M *mp; - - for(mp=runtime_allm; mp; mp=mp->alllink) - runtime_MCache_ReleaseAll(mp->mcache); -} +static int32 gcpercent = GcpercentUnknown; static void cachestats(GCStats *stats) { M *mp; MCache *c; + P *p, **pp; uint32 i; uint64 stacks_inuse; uint64 *src, *dst; @@ -1463,9 +1893,7 @@ cachestats(GCStats *stats) runtime_memclr((byte*)stats, sizeof(*stats)); stacks_inuse = 0; for(mp=runtime_allm; mp; mp=mp->alllink) { - c = mp->mcache; - runtime_purgecachedstats(c); - // stacks_inuse += mp->stackinuse*FixedStack; + //stacks_inuse += mp->stackinuse*FixedStack; if(stats) { src = (uint64*)&mp->gcstats; dst = (uint64*)stats; @@ -1473,6 +1901,12 @@ cachestats(GCStats *stats) dst[i] += src[i]; runtime_memclr((byte*)&mp->gcstats, sizeof(mp->gcstats)); } + } + for(pp=runtime_allp; (p=*pp) != nil; pp++) { + c = p->mcache; + if(c==nil) + continue; + runtime_purgecachedstats(c); for(i=0; i<nelem(c->local_by_size); i++) { mstats.by_size[i].nmalloc += c->local_by_size[i].nmalloc; c->local_by_size[i].nmalloc = 0; @@ -1492,6 +1926,19 @@ struct gc_args static void gc(struct gc_args *args); +static int32 +readgogc(void) +{ + const byte *p; + + p = runtime_getenv("GOGC"); + if(p == nil || p[0] == '\0') + return 100; + if(runtime_strcmp((const char *)p, "off") == 0) + return -1; + return runtime_atoi(p); +} + void runtime_gc(int32 force) { @@ -1504,6 +1951,8 @@ runtime_gc(int32 force) // a problem in the past. if((((uintptr)&work.empty) & 7) != 0) runtime_throw("runtime: gc work buffer is misaligned"); + if((((uintptr)&work.full) & 7) != 0) + runtime_throw("runtime: gc work buffer is misaligned"); // Make sure all registers are saved on stack so that // scanstack sees them. @@ -1521,14 +1970,8 @@ runtime_gc(int32 force) if(!mstats.enablegc || m->locks > 0 || runtime_panicking) return; - if(gcpercent == -2) { // first time through - p = runtime_getenv("GOGC"); - if(p == nil || p[0] == '\0') - gcpercent = 100; - else if(runtime_strcmp((const char*)p, "off") == 0) - gcpercent = -1; - else - gcpercent = runtime_atoi(p); + if(gcpercent == GcpercentUnknown) { // first time through + gcpercent = readgogc(); p = runtime_getenv("GOGCTRACE"); if(p != nil) @@ -1555,7 +1998,7 @@ gc(struct gc_args *args) { M *m; int64 t0, t1, t2, t3, t4; - uint64 heap0, heap1, obj0, obj1; + uint64 heap0, heap1, obj0, obj1, ninstr; GCStats stats; M *mp; uint32 i; @@ -1574,6 +2017,9 @@ gc(struct gc_args *args) m->gcing = 1; runtime_stoptheworld(); + if(CollectStats) + runtime_memclr((byte*)&gcstats, sizeof(gcstats)); + for(mp=runtime_allm; mp; mp=mp->alllink) runtime_settype_flush(mp, false); @@ -1604,7 +2050,7 @@ gc(struct gc_args *args) work.nproc = runtime_gcprocs(); addroots(); runtime_parforsetup(work.markfor, work.nproc, work.nroot, nil, false, markroot); - runtime_parforsetup(work.sweepfor, work.nproc, runtime_mheap.nspan, nil, true, sweepspan); + runtime_parforsetup(work.sweepfor, work.nproc, runtime_mheap->nspan, nil, true, sweepspan); if(work.nproc > 1) { runtime_noteclear(&work.alldone); runtime_helpgc(work.nproc); @@ -1612,6 +2058,7 @@ gc(struct gc_args *args) t1 = runtime_nanotime(); + gchelperstart(); runtime_parfordo(work.markfor); scanblock(nil, nil, 0, true); @@ -1623,14 +2070,14 @@ gc(struct gc_args *args) t2 = runtime_nanotime(); runtime_parfordo(work.sweepfor); + bufferList[m->helpgc].busy = 0; t3 = runtime_nanotime(); - stealcache(); - cachestats(&stats); - if(work.nproc > 1) runtime_notesleep(&work.alldone); + cachestats(&stats); + stats.nprocyield += work.sweepfor->nprocyield; stats.nosyield += work.sweepfor->nosyield; stats.nsleep += work.sweepfor->nsleep; @@ -1670,6 +2117,27 @@ gc(struct gc_args *args) stats.nhandoff, stats.nhandoffcnt, work.sweepfor->nsteal, work.sweepfor->nstealcnt, stats.nprocyield, stats.nosyield, stats.nsleep); + if(CollectStats) { + runtime_printf("scan: %D bytes, %D objects, %D untyped, %D types from MSpan\n", + gcstats.nbytes, gcstats.obj.cnt, gcstats.obj.notype, gcstats.obj.typelookup); + if(gcstats.ptr.cnt != 0) + runtime_printf("avg ptrbufsize: %D (%D/%D)\n", + gcstats.ptr.sum/gcstats.ptr.cnt, gcstats.ptr.sum, gcstats.ptr.cnt); + if(gcstats.obj.cnt != 0) + runtime_printf("avg nobj: %D (%D/%D)\n", + gcstats.obj.sum/gcstats.obj.cnt, gcstats.obj.sum, gcstats.obj.cnt); + runtime_printf("rescans: %D, %D bytes\n", gcstats.rescan, gcstats.rescanbytes); + + runtime_printf("instruction counts:\n"); + ninstr = 0; + for(i=0; i<nelem(gcstats.instr); i++) { + runtime_printf("\t%d:\t%D\n", i, gcstats.instr[i]); + ninstr += gcstats.instr[i]; + } + runtime_printf("\ttotal:\t%D\n", ninstr); + + runtime_printf("putempty: %D, getfull: %D\n", gcstats.putempty, gcstats.getfull); + } } runtime_MProf_GC(); @@ -1704,6 +2172,71 @@ runtime_ReadMemStats(MStats *stats) runtime_starttheworld(); } +void runtime_debug_readGCStats(Slice*) + __asm__("runtime_debug.readGCStats"); + +void +runtime_debug_readGCStats(Slice *pauses) +{ + uint64 *p; + uint32 i, n; + + // Calling code in runtime/debug should make the slice large enough. + if((size_t)pauses->cap < nelem(mstats.pause_ns)+3) + runtime_throw("runtime: short slice passed to readGCStats"); + + // Pass back: pauses, last gc (absolute time), number of gc, total pause ns. + p = (uint64*)pauses->array; + runtime_lock(runtime_mheap); + n = mstats.numgc; + if(n > nelem(mstats.pause_ns)) + n = nelem(mstats.pause_ns); + + // The pause buffer is circular. The most recent pause is at + // pause_ns[(numgc-1)%nelem(pause_ns)], and then backward + // from there to go back farther in time. We deliver the times + // most recent first (in p[0]). + for(i=0; i<n; i++) + p[i] = mstats.pause_ns[(mstats.numgc-1-i)%nelem(mstats.pause_ns)]; + + p[n] = mstats.last_gc; + p[n+1] = mstats.numgc; + p[n+2] = mstats.pause_total_ns; + runtime_unlock(runtime_mheap); + pauses->__count = n+3; +} + +intgo runtime_debug_setGCPercent(intgo) + __asm__("runtime_debug.setGCPercent"); + +intgo +runtime_debug_setGCPercent(intgo in) +{ + intgo out; + + runtime_lock(runtime_mheap); + if(gcpercent == GcpercentUnknown) + gcpercent = readgogc(); + out = gcpercent; + if(in < 0) + in = -1; + gcpercent = in; + runtime_unlock(runtime_mheap); + return out; +} + +static void +gchelperstart(void) +{ + M *m; + + m = runtime_m(); + if(m->helpgc < 0 || m->helpgc >= MaxGcproc) + runtime_throw("gchelperstart: bad m->helpgc"); + if(runtime_xchg(&bufferList[m->helpgc].busy, 1)) + runtime_throw("gchelperstart: already busy"); +} + static void runfinq(void* dummy __attribute__ ((unused))) { @@ -1757,11 +2290,11 @@ runtime_markallocated(void *v, uintptr n, bool noptr) if(0) runtime_printf("markallocated %p+%p\n", v, n); - if((byte*)v+n > (byte*)runtime_mheap.arena_used || (byte*)v < runtime_mheap.arena_start) + if((byte*)v+n > (byte*)runtime_mheap->arena_used || (byte*)v < runtime_mheap->arena_start) runtime_throw("markallocated: bad pointer"); - off = (uintptr*)v - (uintptr*)runtime_mheap.arena_start; // word offset - b = (uintptr*)runtime_mheap.arena_start - off/wordsPerBitmapWord - 1; + off = (uintptr*)v - (uintptr*)runtime_mheap->arena_start; // word offset + b = (uintptr*)runtime_mheap->arena_start - off/wordsPerBitmapWord - 1; shift = off % wordsPerBitmapWord; for(;;) { @@ -1789,11 +2322,11 @@ runtime_markfreed(void *v, uintptr n) if(0) runtime_printf("markallocated %p+%p\n", v, n); - if((byte*)v+n > (byte*)runtime_mheap.arena_used || (byte*)v < runtime_mheap.arena_start) + if((byte*)v+n > (byte*)runtime_mheap->arena_used || (byte*)v < runtime_mheap->arena_start) runtime_throw("markallocated: bad pointer"); - off = (uintptr*)v - (uintptr*)runtime_mheap.arena_start; // word offset - b = (uintptr*)runtime_mheap.arena_start - off/wordsPerBitmapWord - 1; + off = (uintptr*)v - (uintptr*)runtime_mheap->arena_start; // word offset + b = (uintptr*)runtime_mheap->arena_start - off/wordsPerBitmapWord - 1; shift = off % wordsPerBitmapWord; for(;;) { @@ -1819,11 +2352,11 @@ runtime_checkfreed(void *v, uintptr n) if(!runtime_checking) return; - if((byte*)v+n > (byte*)runtime_mheap.arena_used || (byte*)v < runtime_mheap.arena_start) + if((byte*)v+n > (byte*)runtime_mheap->arena_used || (byte*)v < runtime_mheap->arena_start) return; // not allocated, so okay - off = (uintptr*)v - (uintptr*)runtime_mheap.arena_start; // word offset - b = (uintptr*)runtime_mheap.arena_start - off/wordsPerBitmapWord - 1; + off = (uintptr*)v - (uintptr*)runtime_mheap->arena_start; // word offset + b = (uintptr*)runtime_mheap->arena_start - off/wordsPerBitmapWord - 1; shift = off % wordsPerBitmapWord; bits = *b>>shift; @@ -1842,7 +2375,7 @@ runtime_markspan(void *v, uintptr size, uintptr n, bool leftover) uintptr *b, off, shift; byte *p; - if((byte*)v+size*n > (byte*)runtime_mheap.arena_used || (byte*)v < runtime_mheap.arena_start) + if((byte*)v+size*n > (byte*)runtime_mheap->arena_used || (byte*)v < runtime_mheap->arena_start) runtime_throw("markspan: bad pointer"); p = v; @@ -1853,8 +2386,8 @@ runtime_markspan(void *v, uintptr size, uintptr n, bool leftover) // the entire span, and each bitmap word has bits for only // one span, so no other goroutines are changing these // bitmap words. - off = (uintptr*)p - (uintptr*)runtime_mheap.arena_start; // word offset - b = (uintptr*)runtime_mheap.arena_start - off/wordsPerBitmapWord - 1; + off = (uintptr*)p - (uintptr*)runtime_mheap->arena_start; // word offset + b = (uintptr*)runtime_mheap->arena_start - off/wordsPerBitmapWord - 1; shift = off % wordsPerBitmapWord; *b = (*b & ~(bitMask<<shift)) | (bitBlockBoundary<<shift); } @@ -1866,14 +2399,14 @@ runtime_unmarkspan(void *v, uintptr n) { uintptr *p, *b, off; - if((byte*)v+n > (byte*)runtime_mheap.arena_used || (byte*)v < runtime_mheap.arena_start) + if((byte*)v+n > (byte*)runtime_mheap->arena_used || (byte*)v < runtime_mheap->arena_start) runtime_throw("markspan: bad pointer"); p = v; - off = p - (uintptr*)runtime_mheap.arena_start; // word offset + off = p - (uintptr*)runtime_mheap->arena_start; // word offset if(off % wordsPerBitmapWord != 0) runtime_throw("markspan: unaligned pointer"); - b = (uintptr*)runtime_mheap.arena_start - off/wordsPerBitmapWord - 1; + b = (uintptr*)runtime_mheap->arena_start - off/wordsPerBitmapWord - 1; n /= PtrSize; if(n%wordsPerBitmapWord != 0) runtime_throw("unmarkspan: unaligned length"); @@ -1894,8 +2427,8 @@ runtime_blockspecial(void *v) if(DebugMark) return true; - off = (uintptr*)v - (uintptr*)runtime_mheap.arena_start; - b = (uintptr*)runtime_mheap.arena_start - off/wordsPerBitmapWord - 1; + off = (uintptr*)v - (uintptr*)runtime_mheap->arena_start; + b = (uintptr*)runtime_mheap->arena_start - off/wordsPerBitmapWord - 1; shift = off % wordsPerBitmapWord; return (*b & (bitSpecial<<shift)) != 0; @@ -1909,8 +2442,8 @@ runtime_setblockspecial(void *v, bool s) if(DebugMark) return; - off = (uintptr*)v - (uintptr*)runtime_mheap.arena_start; - b = (uintptr*)runtime_mheap.arena_start - off/wordsPerBitmapWord - 1; + off = (uintptr*)v - (uintptr*)runtime_mheap->arena_start; + b = (uintptr*)runtime_mheap->arena_start - off/wordsPerBitmapWord - 1; shift = off % wordsPerBitmapWord; for(;;) { |