diff options
Diffstat (limited to 'libgo')
-rw-r--r-- | libgo/go/runtime/mem.go | 77 | ||||
-rw-r--r-- | libgo/go/runtime/mstats.go | 418 | ||||
-rw-r--r-- | libgo/go/runtime/stubs.go | 48 | ||||
-rw-r--r-- | libgo/runtime/cpuprof.goc | 2 | ||||
-rw-r--r-- | libgo/runtime/heapdump.c | 60 | ||||
-rw-r--r-- | libgo/runtime/malloc.goc | 34 | ||||
-rw-r--r-- | libgo/runtime/malloc.h | 63 | ||||
-rw-r--r-- | libgo/runtime/mem_posix_memalign.c | 4 | ||||
-rw-r--r-- | libgo/runtime/mgc0.c | 175 | ||||
-rw-r--r-- | libgo/runtime/mheap.c | 53 | ||||
-rw-r--r-- | libgo/runtime/mprof.goc | 26 | ||||
-rw-r--r-- | libgo/runtime/msize.c | 4 | ||||
-rw-r--r-- | libgo/runtime/netpoll.goc | 2 | ||||
-rw-r--r-- | libgo/runtime/netpoll_select.c | 4 | ||||
-rw-r--r-- | libgo/runtime/proc.c | 20 | ||||
-rw-r--r-- | libgo/runtime/runtime.h | 11 |
16 files changed, 652 insertions, 349 deletions
diff --git a/libgo/go/runtime/mem.go b/libgo/go/runtime/mem.go deleted file mode 100644 index b41d741..0000000 --- a/libgo/go/runtime/mem.go +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package runtime - -import "unsafe" - -// Note: the MemStats struct should be kept in sync with -// struct MStats in malloc.h - -// A MemStats records statistics about the memory allocator. -type MemStats struct { - // General statistics. - Alloc uint64 // bytes allocated and still in use - TotalAlloc uint64 // bytes allocated (even if freed) - Sys uint64 // bytes obtained from system (sum of XxxSys below) - Lookups uint64 // number of pointer lookups - Mallocs uint64 // number of mallocs - Frees uint64 // number of frees - - // Main allocation heap statistics. - HeapAlloc uint64 // bytes allocated and still in use - HeapSys uint64 // bytes obtained from system - HeapIdle uint64 // bytes in idle spans - HeapInuse uint64 // bytes in non-idle span - HeapReleased uint64 // bytes released to the OS - HeapObjects uint64 // total number of allocated objects - - // Low-level fixed-size structure allocator statistics. - // Inuse is bytes used now. - // Sys is bytes obtained from system. - StackInuse uint64 // bootstrap stacks - StackSys uint64 - MSpanInuse uint64 // mspan structures - MSpanSys uint64 - MCacheInuse uint64 // mcache structures - MCacheSys uint64 - BuckHashSys uint64 // profiling bucket hash table - GCSys uint64 // GC metadata - OtherSys uint64 // other system allocations - - // Garbage collector statistics. - NextGC uint64 // next run in HeapAlloc time (bytes) - LastGC uint64 // last run in absolute time (ns) - PauseTotalNs uint64 - PauseNs [256]uint64 // circular buffer of recent GC pause times, most recent at [(NumGC+255)%256] - PauseEnd [256]uint64 // circular buffer of recent GC pause end times - NumGC uint32 - GCCPUFraction float64 // fraction of CPU time used by GC - EnableGC bool - DebugGC bool - - // Per-size allocation statistics. - // 61 is NumSizeClasses in the C code. - BySize [61]struct { - Size uint32 - Mallocs uint64 - Frees uint64 - } -} - -var Sizeof_C_MStats uintptr // filled in by malloc.goc - -func init() { - var memStats MemStats - if Sizeof_C_MStats != unsafe.Sizeof(memStats) { - println(Sizeof_C_MStats, unsafe.Sizeof(memStats)) - panic("MStats vs MemStatsType size mismatch") - } -} - -// ReadMemStats populates m with memory allocator statistics. -func ReadMemStats(m *MemStats) - -// GC runs a garbage collection. -func GC() diff --git a/libgo/go/runtime/mstats.go b/libgo/go/runtime/mstats.go new file mode 100644 index 0000000..6ec268d --- /dev/null +++ b/libgo/go/runtime/mstats.go @@ -0,0 +1,418 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Memory statistics + +package runtime + +import ( + "runtime/internal/atomic" + "runtime/internal/sys" + "unsafe" +) + +// Statistics. +// If you edit this structure, also edit type MemStats below. +type mstats struct { + // General statistics. + alloc uint64 // bytes allocated and not yet freed + total_alloc uint64 // bytes allocated (even if freed) + sys uint64 // bytes obtained from system (should be sum of xxx_sys below, no locking, approximate) + nlookup uint64 // number of pointer lookups + nmalloc uint64 // number of mallocs + nfree uint64 // number of frees + + // Statistics about malloc heap. + // protected by mheap.lock + heap_alloc uint64 // bytes allocated and not yet freed (same as alloc above) + heap_sys uint64 // bytes obtained from system + heap_idle uint64 // bytes in idle spans + heap_inuse uint64 // bytes in non-idle spans + heap_released uint64 // bytes released to the os + heap_objects uint64 // total number of allocated objects + + // Statistics about allocation of low-level fixed-size structures. + // Protected by FixAlloc locks. + stacks_inuse uint64 // this number is included in heap_inuse above + stacks_sys uint64 // always 0 in mstats + mspan_inuse uint64 // mspan structures + mspan_sys uint64 + mcache_inuse uint64 // mcache structures + mcache_sys uint64 + buckhash_sys uint64 // profiling bucket hash table + gc_sys uint64 + other_sys uint64 + + // Statistics about garbage collector. + // Protected by mheap or stopping the world during GC. + next_gc uint64 // next gc (in heap_live time) + last_gc uint64 // last gc (in absolute time) + pause_total_ns uint64 + pause_ns [256]uint64 // circular buffer of recent gc pause lengths + pause_end [256]uint64 // circular buffer of recent gc end times (nanoseconds since 1970) + numgc uint32 + gc_cpu_fraction float64 // fraction of CPU time used by GC + enablegc bool + debuggc bool + + // Statistics about allocation size classes. + + by_size [_NumSizeClasses]struct { + size uint32 + nmalloc uint64 + nfree uint64 + } + + // Statistics below here are not exported to Go directly. + + tinyallocs uint64 // number of tiny allocations that didn't cause actual allocation; not exported to go directly + + // heap_live is the number of bytes considered live by the GC. + // That is: retained by the most recent GC plus allocated + // since then. heap_live <= heap_alloc, since heap_alloc + // includes unmarked objects that have not yet been swept (and + // hence goes up as we allocate and down as we sweep) while + // heap_live excludes these objects (and hence only goes up + // between GCs). + // + // This is updated atomically without locking. To reduce + // contention, this is updated only when obtaining a span from + // an mcentral and at this point it counts all of the + // unallocated slots in that span (which will be allocated + // before that mcache obtains another span from that + // mcentral). Hence, it slightly overestimates the "true" live + // heap size. It's better to overestimate than to + // underestimate because 1) this triggers the GC earlier than + // necessary rather than potentially too late and 2) this + // leads to a conservative GC rate rather than a GC rate that + // is potentially too low. + // + // Whenever this is updated, call traceHeapAlloc() and + // gcController.revise(). + heap_live uint64 + + // heap_scan is the number of bytes of "scannable" heap. This + // is the live heap (as counted by heap_live), but omitting + // no-scan objects and no-scan tails of objects. + // + // Whenever this is updated, call gcController.revise(). + heap_scan uint64 + + // heap_marked is the number of bytes marked by the previous + // GC. After mark termination, heap_live == heap_marked, but + // unlike heap_live, heap_marked does not change until the + // next mark termination. + heap_marked uint64 + + // heap_reachable is an estimate of the reachable heap bytes + // at the end of the previous GC. + heap_reachable uint64 +} + +var memstats mstats + +// A MemStats records statistics about the memory allocator. +type MemStats struct { + // General statistics. + Alloc uint64 // bytes allocated and not yet freed + TotalAlloc uint64 // bytes allocated (even if freed) + Sys uint64 // bytes obtained from system (sum of XxxSys below) + Lookups uint64 // number of pointer lookups + Mallocs uint64 // number of mallocs + Frees uint64 // number of frees + + // Main allocation heap statistics. + HeapAlloc uint64 // bytes allocated and not yet freed (same as Alloc above) + HeapSys uint64 // bytes obtained from system + HeapIdle uint64 // bytes in idle spans + HeapInuse uint64 // bytes in non-idle span + HeapReleased uint64 // bytes released to the OS + HeapObjects uint64 // total number of allocated objects + + // Low-level fixed-size structure allocator statistics. + // Inuse is bytes used now. + // Sys is bytes obtained from system. + StackInuse uint64 // bytes used by stack allocator + StackSys uint64 + MSpanInuse uint64 // mspan structures + MSpanSys uint64 + MCacheInuse uint64 // mcache structures + MCacheSys uint64 + BuckHashSys uint64 // profiling bucket hash table + GCSys uint64 // GC metadata + OtherSys uint64 // other system allocations + + // Garbage collector statistics. + NextGC uint64 // next collection will happen when HeapAlloc ≥ this amount + LastGC uint64 // end time of last collection (nanoseconds since 1970) + PauseTotalNs uint64 + PauseNs [256]uint64 // circular buffer of recent GC pause durations, most recent at [(NumGC+255)%256] + PauseEnd [256]uint64 // circular buffer of recent GC pause end times + NumGC uint32 + GCCPUFraction float64 // fraction of CPU time used by GC + EnableGC bool + DebugGC bool + + // Per-size allocation statistics. + // 61 is NumSizeClasses in the C code. + BySize [61]struct { + Size uint32 + Mallocs uint64 + Frees uint64 + } +} + +// Size of the trailing by_size array differs between Go and C, +// and all data after by_size is local to runtime, not exported. +// NumSizeClasses was changed, but we cannot change Go struct because of backward compatibility. +// sizeof_C_MStats is what C thinks about size of Go struct. +var sizeof_C_MStats = unsafe.Offsetof(memstats.by_size) + 61*unsafe.Sizeof(memstats.by_size[0]) + +func init() { + var memStats MemStats + if sizeof_C_MStats != unsafe.Sizeof(memStats) { + println(sizeof_C_MStats, unsafe.Sizeof(memStats)) + throw("MStats vs MemStatsType size mismatch") + } +} + +// ReadMemStats populates m with memory allocator statistics. +func ReadMemStats(m *MemStats) { + stopTheWorld("read mem stats") + + systemstack(func() { + readmemstats_m(m) + }) + + startTheWorld() +} + +func readmemstats_m(stats *MemStats) { + updatememstats(nil) + + // Size of the trailing by_size array differs between Go and C, + // NumSizeClasses was changed, but we cannot change Go struct because of backward compatibility. + memmove(unsafe.Pointer(stats), unsafe.Pointer(&memstats), sizeof_C_MStats) + + // Stack numbers are part of the heap numbers, separate those out for user consumption + stats.StackSys += stats.StackInuse + stats.HeapInuse -= stats.StackInuse + stats.HeapSys -= stats.StackInuse +} + +// For gccgo this is in runtime/mgc0.c. +func updatememstats(stats *gcstats) + +/* +For gccgo these are still in runtime/mgc0.c. + +//go:linkname readGCStats runtime/debug.readGCStats +func readGCStats(pauses *[]uint64) { + systemstack(func() { + readGCStats_m(pauses) + }) +} + +func readGCStats_m(pauses *[]uint64) { + p := *pauses + // Calling code in runtime/debug should make the slice large enough. + if cap(p) < len(memstats.pause_ns)+3 { + throw("short slice passed to readGCStats") + } + + // Pass back: pauses, pause ends, last gc (absolute time), number of gc, total pause ns. + lock(&mheap_.lock) + + n := memstats.numgc + if n > uint32(len(memstats.pause_ns)) { + n = uint32(len(memstats.pause_ns)) + } + + // The pause buffer is circular. The most recent pause is at + // pause_ns[(numgc-1)%len(pause_ns)], and then backward + // from there to go back farther in time. We deliver the times + // most recent first (in p[0]). + p = p[:cap(p)] + for i := uint32(0); i < n; i++ { + j := (memstats.numgc - 1 - i) % uint32(len(memstats.pause_ns)) + p[i] = memstats.pause_ns[j] + p[n+i] = memstats.pause_end[j] + } + + p[n+n] = memstats.last_gc + p[n+n+1] = uint64(memstats.numgc) + p[n+n+2] = memstats.pause_total_ns + unlock(&mheap_.lock) + *pauses = p[:n+n+3] +} + +//go:nowritebarrier +func updatememstats(stats *gcstats) { + if stats != nil { + *stats = gcstats{} + } + for mp := allm; mp != nil; mp = mp.alllink { + if stats != nil { + src := (*[unsafe.Sizeof(gcstats{}) / 8]uint64)(unsafe.Pointer(&mp.gcstats)) + dst := (*[unsafe.Sizeof(gcstats{}) / 8]uint64)(unsafe.Pointer(stats)) + for i, v := range src { + dst[i] += v + } + mp.gcstats = gcstats{} + } + } + + memstats.mcache_inuse = uint64(mheap_.cachealloc.inuse) + memstats.mspan_inuse = uint64(mheap_.spanalloc.inuse) + memstats.sys = memstats.heap_sys + memstats.stacks_sys + memstats.mspan_sys + + memstats.mcache_sys + memstats.buckhash_sys + memstats.gc_sys + memstats.other_sys + + // Calculate memory allocator stats. + // During program execution we only count number of frees and amount of freed memory. + // Current number of alive object in the heap and amount of alive heap memory + // are calculated by scanning all spans. + // Total number of mallocs is calculated as number of frees plus number of alive objects. + // Similarly, total amount of allocated memory is calculated as amount of freed memory + // plus amount of alive heap memory. + memstats.alloc = 0 + memstats.total_alloc = 0 + memstats.nmalloc = 0 + memstats.nfree = 0 + for i := 0; i < len(memstats.by_size); i++ { + memstats.by_size[i].nmalloc = 0 + memstats.by_size[i].nfree = 0 + } + + // Flush MCache's to MCentral. + systemstack(flushallmcaches) + + // Aggregate local stats. + cachestats() + + // Scan all spans and count number of alive objects. + lock(&mheap_.lock) + for i := uint32(0); i < mheap_.nspan; i++ { + s := h_allspans[i] + if s.state != mSpanInUse { + continue + } + if s.sizeclass == 0 { + memstats.nmalloc++ + memstats.alloc += uint64(s.elemsize) + } else { + memstats.nmalloc += uint64(s.allocCount) + memstats.by_size[s.sizeclass].nmalloc += uint64(s.allocCount) + memstats.alloc += uint64(s.allocCount) * uint64(s.elemsize) + } + } + unlock(&mheap_.lock) + + // Aggregate by size class. + smallfree := uint64(0) + memstats.nfree = mheap_.nlargefree + for i := 0; i < len(memstats.by_size); i++ { + memstats.nfree += mheap_.nsmallfree[i] + memstats.by_size[i].nfree = mheap_.nsmallfree[i] + memstats.by_size[i].nmalloc += mheap_.nsmallfree[i] + smallfree += mheap_.nsmallfree[i] * uint64(class_to_size[i]) + } + memstats.nfree += memstats.tinyallocs + memstats.nmalloc += memstats.nfree + + // Calculate derived stats. + memstats.total_alloc = memstats.alloc + mheap_.largefree + smallfree + memstats.heap_alloc = memstats.alloc + memstats.heap_objects = memstats.nmalloc - memstats.nfree +} + +//go:nowritebarrier +func cachestats() { + for i := 0; ; i++ { + p := allp[i] + if p == nil { + break + } + c := p.mcache + if c == nil { + continue + } + purgecachedstats(c) + } +} + +//go:nowritebarrier +func flushallmcaches() { + for i := 0; ; i++ { + p := allp[i] + if p == nil { + break + } + c := p.mcache + if c == nil { + continue + } + c.releaseAll() + stackcache_clear(c) + } +} + +//go:nosplit +func purgecachedstats(c *mcache) { + // Protected by either heap or GC lock. + h := &mheap_ + memstats.heap_scan += uint64(c.local_scan) + c.local_scan = 0 + memstats.tinyallocs += uint64(c.local_tinyallocs) + c.local_tinyallocs = 0 + memstats.nlookup += uint64(c.local_nlookup) + c.local_nlookup = 0 + h.largefree += uint64(c.local_largefree) + c.local_largefree = 0 + h.nlargefree += uint64(c.local_nlargefree) + c.local_nlargefree = 0 + for i := 0; i < len(c.local_nsmallfree); i++ { + h.nsmallfree[i] += uint64(c.local_nsmallfree[i]) + c.local_nsmallfree[i] = 0 + } +} + +*/ + +// Atomically increases a given *system* memory stat. We are counting on this +// stat never overflowing a uintptr, so this function must only be used for +// system memory stats. +// +// The current implementation for little endian architectures is based on +// xadduintptr(), which is less than ideal: xadd64() should really be used. +// Using xadduintptr() is a stop-gap solution until arm supports xadd64() that +// doesn't use locks. (Locks are a problem as they require a valid G, which +// restricts their useability.) +// +// A side-effect of using xadduintptr() is that we need to check for +// overflow errors. +//go:nosplit +func mSysStatInc(sysStat *uint64, n uintptr) { + if sys.BigEndian != 0 { + atomic.Xadd64(sysStat, int64(n)) + return + } + if val := atomic.Xadduintptr((*uintptr)(unsafe.Pointer(sysStat)), n); val < n { + print("runtime: stat overflow: val ", val, ", n ", n, "\n") + exit(2) + } +} + +// Atomically decreases a given *system* memory stat. Same comments as +// mSysStatInc apply. +//go:nosplit +func mSysStatDec(sysStat *uint64, n uintptr) { + if sys.BigEndian != 0 { + atomic.Xadd64(sysStat, -int64(n)) + return + } + if val := atomic.Xadduintptr((*uintptr)(unsafe.Pointer(sysStat)), uintptr(-int64(n))); val+n < n { + print("runtime: stat underflow: val ", val, ", n ", n, "\n") + exit(2) + } +} diff --git a/libgo/go/runtime/stubs.go b/libgo/go/runtime/stubs.go index d598a0a..13fc5e5 100644 --- a/libgo/go/runtime/stubs.go +++ b/libgo/go/runtime/stubs.go @@ -367,3 +367,51 @@ func typeBitsBulkBarrier(typ *_type, p, size uintptr) {} // Here for gccgo until we port msize.go. func roundupsize(uintptr) uintptr + +// Here for gccgo until we port mgc.go. +func GC() + +// Here for gccgo until we port proc.go. +var worldsema uint32 = 1 + +func stopTheWorldWithSema() +func startTheWorldWithSema() + +// For gccgo to call from C code. +//go:linkname acquireWorldsema runtime.acquireWorldsema +func acquireWorldsema() { + semacquire(&worldsema, false) +} + +// For gccgo to call from C code. +//go:linkname releaseWorldsema runtime.releaseWorldsema +func releaseWorldsema() { + semrelease(&worldsema) +} + +// Here for gccgo until we port proc.go. +func stopTheWorld(reason string) { + semacquire(&worldsema, false) + getg().m.preemptoff = reason + getg().m.gcing = 1 + systemstack(stopTheWorldWithSema) +} + +// Here for gccgo until we port proc.go. +func startTheWorld() { + getg().m.gcing = 0 + getg().m.locks++ + systemstack(startTheWorldWithSema) + // worldsema must be held over startTheWorldWithSema to ensure + // gomaxprocs cannot change while worldsema is held. + semrelease(&worldsema) + getg().m.preemptoff = "" + getg().m.locks-- +} + +// For gccgo to call from C code, so that the C code and the Go code +// can share the memstats variable for now. +//go:linkname getMstats runtime.getMstats +func getMstats() *mstats { + return &memstats +} diff --git a/libgo/runtime/cpuprof.goc b/libgo/runtime/cpuprof.goc index 7d27bc6..123e074 100644 --- a/libgo/runtime/cpuprof.goc +++ b/libgo/runtime/cpuprof.goc @@ -146,7 +146,7 @@ runtime_SetCPUProfileRate(intgo hz) runtime_lock(&lk); if(hz > 0) { if(prof == nil) { - prof = runtime_SysAlloc(sizeof *prof, &mstats.other_sys); + prof = runtime_SysAlloc(sizeof *prof, &mstats()->other_sys); if(prof == nil) { runtime_printf("runtime: cpu profiling cannot allocate memory\n"); runtime_unlock(&lk); diff --git a/libgo/runtime/heapdump.c b/libgo/runtime/heapdump.c index 3cc0c1d..158ff5e 100644 --- a/libgo/runtime/heapdump.c +++ b/libgo/runtime/heapdump.c @@ -489,33 +489,33 @@ dumpmemstats(void) int32 i; dumpint(TagMemStats); - dumpint(mstats.alloc); - dumpint(mstats.total_alloc); - dumpint(mstats.sys); - dumpint(mstats.nlookup); - dumpint(mstats.nmalloc); - dumpint(mstats.nfree); - dumpint(mstats.heap_alloc); - dumpint(mstats.heap_sys); - dumpint(mstats.heap_idle); - dumpint(mstats.heap_inuse); - dumpint(mstats.heap_released); - dumpint(mstats.heap_objects); - dumpint(mstats.stacks_inuse); - dumpint(mstats.stacks_sys); - dumpint(mstats.mspan_inuse); - dumpint(mstats.mspan_sys); - dumpint(mstats.mcache_inuse); - dumpint(mstats.mcache_sys); - dumpint(mstats.buckhash_sys); - dumpint(mstats.gc_sys); - dumpint(mstats.other_sys); - dumpint(mstats.next_gc); - dumpint(mstats.last_gc); - dumpint(mstats.pause_total_ns); + dumpint(mstats()->alloc); + dumpint(mstats()->total_alloc); + dumpint(mstats()->sys); + dumpint(mstats()->nlookup); + dumpint(mstats()->nmalloc); + dumpint(mstats()->nfree); + dumpint(mstats()->heap_alloc); + dumpint(mstats()->heap_sys); + dumpint(mstats()->heap_idle); + dumpint(mstats()->heap_inuse); + dumpint(mstats()->heap_released); + dumpint(mstats()->heap_objects); + dumpint(mstats()->stacks_inuse); + dumpint(mstats()->stacks_sys); + dumpint(mstats()->mspan_inuse); + dumpint(mstats()->mspan_sys); + dumpint(mstats()->mcache_inuse); + dumpint(mstats()->mcache_sys); + dumpint(mstats()->buckhash_sys); + dumpint(mstats()->gc_sys); + dumpint(mstats()->other_sys); + dumpint(mstats()->next_gc); + dumpint(mstats()->last_gc); + dumpint(mstats()->pause_total_ns); for(i = 0; i < 256; i++) - dumpint(mstats.pause_ns[i]); - dumpint(mstats.numgc); + dumpint(mstats()->pause_ns[i]); + dumpint(mstats()->numgc); } static void @@ -615,11 +615,11 @@ runtime_debug_WriteHeapDump(uintptr fd) G *g; // Stop the world. - runtime_semacquire(&runtime_worldsema, false); + runtime_acquireWorldsema(); m = runtime_m(); m->gcing = 1; m->locks++; - runtime_stoptheworld(); + runtime_stopTheWorldWithSema(); // Update stats so we can dump them. // As a side effect, flushes all the MCaches so the MSpan.freelist @@ -640,8 +640,8 @@ runtime_debug_WriteHeapDump(uintptr fd) // Start up the world again. m->gcing = 0; - runtime_semrelease(&runtime_worldsema); - runtime_starttheworld(); + runtime_releaseWorldsema(); + runtime_startTheWorldWithSema(); m->locks--; } diff --git a/libgo/runtime/malloc.goc b/libgo/runtime/malloc.goc index a924b8a..4f81d82 100644 --- a/libgo/runtime/malloc.goc +++ b/libgo/runtime/malloc.goc @@ -51,12 +51,9 @@ package runtime // Mark mheap as 'no pointers', it does not contain interesting pointers but occupies ~45K. MHeap runtime_mheap; -MStats mstats; int32 runtime_checking; -extern MStats mstats; // defined in zruntime_def_$GOOS_$GOARCH.go - extern volatile intgo runtime_MemProfileRate __asm__ (GOSYM_PREFIX "runtime.MemProfileRate"); @@ -81,6 +78,7 @@ runtime_mallocgc(uintptr size, uintptr typ, uint32 flag) MLink *v, *next; byte *tiny; bool incallback; + MStats *pmstats; if(size == 0) { // All 0-length allocations use this pointer. @@ -105,7 +103,7 @@ runtime_mallocgc(uintptr size, uintptr typ, uint32 flag) flag |= FlagNoInvokeGC; } - if(runtime_gcwaiting() && g != m->g0 && m->locks == 0 && !(flag & FlagNoInvokeGC)) { + if(runtime_gcwaiting() && g != m->g0 && m->locks == 0 && !(flag & FlagNoInvokeGC) && m->preemptoff.len == 0) { runtime_gosched(); m = runtime_m(); } @@ -252,7 +250,8 @@ runtime_mallocgc(uintptr size, uintptr typ, uint32 flag) m->locks--; - if(!(flag & FlagNoInvokeGC) && mstats.heap_alloc >= mstats.next_gc) + pmstats = mstats(); + if(!(flag & FlagNoInvokeGC) && pmstats->heap_alloc >= pmstats->next_gc) runtime_gc(0); if(incallback) @@ -472,9 +471,9 @@ runtime_purgecachedstats(MCache *c) // Protected by either heap or GC lock. h = &runtime_mheap; - mstats.heap_alloc += (intptr)c->local_cachealloc; + mstats()->heap_alloc += (intptr)c->local_cachealloc; c->local_cachealloc = 0; - mstats.nlookup += c->local_nlookup; + mstats()->nlookup += c->local_nlookup; c->local_nlookup = 0; h->largefree += c->local_largefree; c->local_largefree = 0; @@ -486,13 +485,6 @@ runtime_purgecachedstats(MCache *c) } } -extern uintptr runtime_sizeof_C_MStats - __asm__ (GOSYM_PREFIX "runtime.Sizeof_C_MStats"); - -// Size of the trailing by_size array differs between Go and C, -// _NumSizeClasses was changed, but we can not change Go struct because of backward compatibility. -// sizeof_C_MStats is what C thinks about size of Go struct. - // Initialized in mallocinit because it's defined in go/runtime/mem.go. #define MaxArena32 (2U<<30) @@ -508,8 +500,6 @@ runtime_mallocinit(void) uint64 i; bool reserved; - runtime_sizeof_C_MStats = sizeof(MStats) - (_NumSizeClasses - 61) * sizeof(mstats.by_size[0]); - p = nil; p_size = 0; arena_size = 0; @@ -685,7 +675,7 @@ runtime_MHeap_SysAlloc(MHeap *h, uintptr n) if(n <= (uintptr)(h->arena_end - h->arena_used)) { // Keep taking from our reservation. p = h->arena_used; - runtime_SysMap(p, n, h->arena_reserved, &mstats.heap_sys); + runtime_SysMap(p, n, h->arena_reserved, &mstats()->heap_sys); h->arena_used += n; runtime_MHeap_MapBits(h); runtime_MHeap_MapSpans(h); @@ -703,14 +693,14 @@ runtime_MHeap_SysAlloc(MHeap *h, uintptr n) // try to get memory at a location chosen by the OS // and hope that it is in the range we allocated bitmap for. p_size = ROUND(n, PageSize) + PageSize; - p = runtime_SysAlloc(p_size, &mstats.heap_sys); + p = runtime_SysAlloc(p_size, &mstats()->heap_sys); if(p == nil) return nil; if(p < h->arena_start || (uintptr)(p+p_size - h->arena_start) >= MaxArena32) { runtime_printf("runtime: memory allocated by OS (%p) not in usable range [%p,%p)\n", p, h->arena_start, h->arena_start+MaxArena32); - runtime_SysFree(p, p_size, &mstats.heap_sys); + runtime_SysFree(p, p_size, &mstats()->heap_sys); return nil; } @@ -763,7 +753,7 @@ runtime_persistentalloc(uintptr size, uintptr align, uint64 *stat) runtime_lock(&persistent); persistent.pos = (byte*)ROUND((uintptr)persistent.pos, align); if(persistent.pos + size > persistent.end) { - persistent.pos = runtime_SysAlloc(PersistentAllocChunk, &mstats.other_sys); + persistent.pos = runtime_SysAlloc(PersistentAllocChunk, &mstats()->other_sys); if(persistent.pos == nil) { runtime_unlock(&persistent); runtime_throw("runtime: cannot allocate memory"); @@ -773,10 +763,10 @@ runtime_persistentalloc(uintptr size, uintptr align, uint64 *stat) p = persistent.pos; persistent.pos += size; runtime_unlock(&persistent); - if(stat != &mstats.other_sys) { + if(stat != &mstats()->other_sys) { // reaccount the allocation against provided stat runtime_xadd64(stat, size); - runtime_xadd64(&mstats.other_sys, -(uint64)size); + runtime_xadd64(&mstats()->other_sys, -(uint64)size); } return p; } diff --git a/libgo/runtime/malloc.h b/libgo/runtime/malloc.h index b2dbf90..5e74b8c 100644 --- a/libgo/runtime/malloc.h +++ b/libgo/runtime/malloc.h @@ -83,7 +83,7 @@ typedef struct MCentral MCentral; typedef struct MHeap MHeap; typedef struct mspan MSpan; -typedef struct MStats MStats; +typedef struct mstats MStats; typedef struct mlink MLink; typedef struct mtypes MTypes; typedef struct gcstats GCStats; @@ -216,63 +216,10 @@ void runtime_FixAlloc_Init(FixAlloc *f, uintptr size, void (*first)(void*, byte* void* runtime_FixAlloc_Alloc(FixAlloc *f); void runtime_FixAlloc_Free(FixAlloc *f, void *p); - -// Statistics. -// Shared with Go: if you edit this structure, also edit type MemStats in mem.go. -struct MStats -{ - // General statistics. - uint64 alloc; // bytes allocated and still in use - uint64 total_alloc; // bytes allocated (even if freed) - uint64 sys; // bytes obtained from system (should be sum of xxx_sys below, no locking, approximate) - uint64 nlookup; // number of pointer lookups - uint64 nmalloc; // number of mallocs - uint64 nfree; // number of frees - - // Statistics about malloc heap. - // protected by mheap.Lock - uint64 heap_alloc; // bytes allocated and still in use - uint64 heap_sys; // bytes obtained from system - uint64 heap_idle; // bytes in idle spans - uint64 heap_inuse; // bytes in non-idle spans - uint64 heap_released; // bytes released to the OS - uint64 heap_objects; // total number of allocated objects - - // Statistics about allocation of low-level fixed-size structures. - // Protected by FixAlloc locks. - uint64 stacks_inuse; // bootstrap stacks - uint64 stacks_sys; - uint64 mspan_inuse; // MSpan structures - uint64 mspan_sys; - uint64 mcache_inuse; // MCache structures - uint64 mcache_sys; - uint64 buckhash_sys; // profiling bucket hash table - uint64 gc_sys; - uint64 other_sys; - - // Statistics about garbage collector. - // Protected by mheap or stopping the world during GC. - uint64 next_gc; // next GC (in heap_alloc time) - uint64 last_gc; // last GC (in absolute time) - uint64 pause_total_ns; - uint64 pause_ns[256]; - uint64 pause_end[256]; - uint32 numgc; - float64 gc_cpu_fraction; - bool enablegc; - bool debuggc; - - // Statistics about allocation size classes. - struct { - uint32 size; - uint64 nmalloc; - uint64 nfree; - } by_size[_NumSizeClasses]; -}; - -extern MStats mstats - __asm__ (GOSYM_PREFIX "runtime.memStats"); -void runtime_updatememstats(GCStats *stats); +extern MStats *mstats(void) + __asm__ (GOSYM_PREFIX "runtime.getMstats"); +void runtime_updatememstats(GCStats *stats) + __asm__ (GOSYM_PREFIX "runtime.updatememstats"); // Size classes. Computed and initialized by InitSizes. // diff --git a/libgo/runtime/mem_posix_memalign.c b/libgo/runtime/mem_posix_memalign.c index 8acdf07..853b5c7 100644 --- a/libgo/runtime/mem_posix_memalign.c +++ b/libgo/runtime/mem_posix_memalign.c @@ -9,7 +9,7 @@ runtime_SysAlloc(uintptr n) { void *p; - mstats.sys += n; + mstats()->sys += n; errno = posix_memalign(&p, PageSize, n); if (errno > 0) { perror("posix_memalign"); @@ -29,7 +29,7 @@ runtime_SysUnused(void *v, uintptr n) void runtime_SysFree(void *v, uintptr n) { - mstats.sys -= n; + mstats()->sys -= n; free(v); } diff --git a/libgo/runtime/mgc0.c b/libgo/runtime/mgc0.c index 0044821..4877678 100644 --- a/libgo/runtime/mgc0.c +++ b/libgo/runtime/mgc0.c @@ -145,21 +145,6 @@ clearpools(void) } } -// Holding worldsema grants an M the right to try to stop the world. -// The procedure is: -// -// runtime_semacquire(&runtime_worldsema); -// m->gcing = 1; -// runtime_stoptheworld(); -// -// ... do stuff ... -// -// m->gcing = 0; -// runtime_semrelease(&runtime_worldsema); -// runtime_starttheworld(); -// -uint32 runtime_worldsema = 1; - typedef struct Workbuf Workbuf; struct Workbuf { @@ -1377,7 +1362,7 @@ getempty(Workbuf *b) runtime_lock(&work); if(work.nchunk < sizeof *b) { work.nchunk = 1<<20; - work.chunk = runtime_SysAlloc(work.nchunk, &mstats.gc_sys); + work.chunk = runtime_SysAlloc(work.nchunk, &mstats()->gc_sys); if(work.chunk == nil) runtime_throw("runtime: cannot allocate memory"); } @@ -1558,7 +1543,7 @@ runtime_queuefinalizer(void *p, FuncVal *fn, const FuncType *ft, const PtrType * runtime_lock(&finlock); if(finq == nil || finq->cnt == finq->cap) { if(finc == nil) { - finc = runtime_persistentalloc(FinBlockSize, 0, &mstats.gc_sys); + finc = runtime_persistentalloc(FinBlockSize, 0, &mstats()->gc_sys); finc->cap = (FinBlockSize - sizeof(FinBlock)) / sizeof(Finalizer) + 1; finc->alllink = allfin; allfin = finc; @@ -1755,7 +1740,7 @@ runtime_MSpan_Sweep(MSpan *s) runtime_MHeap_Free(&runtime_mheap, s, 1); c->local_nlargefree++; c->local_largefree += size; - runtime_xadd64(&mstats.next_gc, -(uint64)(size * (gcpercent + 100)/100)); + runtime_xadd64(&mstats()->next_gc, -(uint64)(size * (gcpercent + 100)/100)); res = true; } else { // Free small object. @@ -1797,7 +1782,7 @@ runtime_MSpan_Sweep(MSpan *s) if(nfree > 0) { c->local_nsmallfree[cl] += nfree; c->local_cachealloc -= nfree * size; - runtime_xadd64(&mstats.next_gc, -(uint64)(nfree * size * (gcpercent + 100)/100)); + runtime_xadd64(&mstats()->next_gc, -(uint64)(nfree * size * (gcpercent + 100)/100)); res = runtime_MCentral_FreeSpan(&runtime_mheap.central[cl], s, nfree, head.next, end); //MCentral_FreeSpan updates sweepgen } @@ -2010,6 +1995,7 @@ runtime_updatememstats(GCStats *stats) uint32 i; uint64 stacks_inuse, smallfree; uint64 *src, *dst; + MStats *pmstats; if(stats) runtime_memclr((byte*)stats, sizeof(*stats)); @@ -2024,11 +2010,12 @@ runtime_updatememstats(GCStats *stats) runtime_memclr((byte*)&mp->gcstats, sizeof(mp->gcstats)); } } - mstats.stacks_inuse = stacks_inuse; - mstats.mcache_inuse = runtime_mheap.cachealloc.inuse; - mstats.mspan_inuse = runtime_mheap.spanalloc.inuse; - mstats.sys = mstats.heap_sys + mstats.stacks_sys + mstats.mspan_sys + - mstats.mcache_sys + mstats.buckhash_sys + mstats.gc_sys + mstats.other_sys; + pmstats = mstats(); + pmstats->stacks_inuse = stacks_inuse; + pmstats->mcache_inuse = runtime_mheap.cachealloc.inuse; + pmstats->mspan_inuse = runtime_mheap.spanalloc.inuse; + pmstats->sys = pmstats->heap_sys + pmstats->stacks_sys + pmstats->mspan_sys + + pmstats->mcache_sys + pmstats->buckhash_sys + pmstats->gc_sys + pmstats->other_sys; // Calculate memory allocator stats. // During program execution we only count number of frees and amount of freed memory. @@ -2037,13 +2024,13 @@ runtime_updatememstats(GCStats *stats) // Total number of mallocs is calculated as number of frees plus number of alive objects. // Similarly, total amount of allocated memory is calculated as amount of freed memory // plus amount of alive heap memory. - mstats.alloc = 0; - mstats.total_alloc = 0; - mstats.nmalloc = 0; - mstats.nfree = 0; - for(i = 0; i < nelem(mstats.by_size); i++) { - mstats.by_size[i].nmalloc = 0; - mstats.by_size[i].nfree = 0; + pmstats->alloc = 0; + pmstats->total_alloc = 0; + pmstats->nmalloc = 0; + pmstats->nfree = 0; + for(i = 0; i < nelem(pmstats->by_size); i++) { + pmstats->by_size[i].nmalloc = 0; + pmstats->by_size[i].nfree = 0; } // Flush MCache's to MCentral. @@ -2058,30 +2045,30 @@ runtime_updatememstats(GCStats *stats) if(s->state != MSpanInUse) continue; if(s->sizeclass == 0) { - mstats.nmalloc++; - mstats.alloc += s->elemsize; + pmstats->nmalloc++; + pmstats->alloc += s->elemsize; } else { - mstats.nmalloc += s->ref; - mstats.by_size[s->sizeclass].nmalloc += s->ref; - mstats.alloc += s->ref*s->elemsize; + pmstats->nmalloc += s->ref; + pmstats->by_size[s->sizeclass].nmalloc += s->ref; + pmstats->alloc += s->ref*s->elemsize; } } // Aggregate by size class. smallfree = 0; - mstats.nfree = runtime_mheap.nlargefree; - for(i = 0; i < nelem(mstats.by_size); i++) { - mstats.nfree += runtime_mheap.nsmallfree[i]; - mstats.by_size[i].nfree = runtime_mheap.nsmallfree[i]; - mstats.by_size[i].nmalloc += runtime_mheap.nsmallfree[i]; + pmstats->nfree = runtime_mheap.nlargefree; + for(i = 0; i < nelem(pmstats->by_size); i++) { + pmstats->nfree += runtime_mheap.nsmallfree[i]; + pmstats->by_size[i].nfree = runtime_mheap.nsmallfree[i]; + pmstats->by_size[i].nmalloc += runtime_mheap.nsmallfree[i]; smallfree += runtime_mheap.nsmallfree[i] * runtime_class_to_size[i]; } - mstats.nmalloc += mstats.nfree; + pmstats->nmalloc += pmstats->nfree; // Calculate derived stats. - mstats.total_alloc = mstats.alloc + runtime_mheap.largefree + smallfree; - mstats.heap_alloc = mstats.alloc; - mstats.heap_objects = mstats.nmalloc - mstats.nfree; + pmstats->total_alloc = pmstats->alloc + runtime_mheap.largefree + smallfree; + pmstats->heap_alloc = pmstats->alloc; + pmstats->heap_objects = pmstats->nmalloc - pmstats->nfree; } // Structure of arguments passed to function gc(). @@ -2119,6 +2106,7 @@ runtime_gc(int32 force) G *g; struct gc_args a; int32 i; + MStats *pmstats; // The atomic operations are not atomic if the uint64s // are not aligned on uint64 boundaries. This has been @@ -2141,7 +2129,8 @@ runtime_gc(int32 force) // while holding a lock. The next mallocgc // without a lock will do the gc instead. m = runtime_m(); - if(!mstats.enablegc || runtime_g() == m->g0 || m->locks > 0 || runtime_panicking) + pmstats = mstats(); + if(!pmstats->enablegc || runtime_g() == m->g0 || m->locks > 0 || runtime_panicking || m->preemptoff.len > 0) return; if(gcpercent == GcpercentUnknown) { // first time through @@ -2153,11 +2142,11 @@ runtime_gc(int32 force) if(gcpercent < 0) return; - runtime_semacquire(&runtime_worldsema, false); - if(force==0 && mstats.heap_alloc < mstats.next_gc) { + runtime_acquireWorldsema(); + if(force==0 && pmstats->heap_alloc < pmstats->next_gc) { // typically threads which lost the race to grab // worldsema exit here when gc is done. - runtime_semrelease(&runtime_worldsema); + runtime_releaseWorldsema(); return; } @@ -2165,7 +2154,7 @@ runtime_gc(int32 force) a.start_time = runtime_nanotime(); a.eagersweep = force >= 2; m->gcing = 1; - runtime_stoptheworld(); + runtime_stopTheWorldWithSema(); clearpools(); @@ -2189,8 +2178,8 @@ runtime_gc(int32 force) // all done m->gcing = 0; m->locks++; - runtime_semrelease(&runtime_worldsema); - runtime_starttheworld(); + runtime_releaseWorldsema(); + runtime_startTheWorldWithSema(); m->locks--; // now that gc is done, kick off finalizer thread if needed @@ -2220,6 +2209,7 @@ gc(struct gc_args *args) uint64 heap0, heap1, obj, ninstr; GCStats stats; uint32 i; + MStats *pmstats; // Eface eface; m = runtime_m(); @@ -2275,28 +2265,29 @@ gc(struct gc_args *args) cachestats(); // next_gc calculation is tricky with concurrent sweep since we don't know size of live heap // estimate what was live heap size after previous GC (for tracing only) - heap0 = mstats.next_gc*100/(gcpercent+100); + pmstats = mstats(); + heap0 = pmstats->next_gc*100/(gcpercent+100); // conservatively set next_gc to high value assuming that everything is live // concurrent/lazy sweep will reduce this number while discovering new garbage - mstats.next_gc = mstats.heap_alloc+(mstats.heap_alloc-runtime_stacks_sys)*gcpercent/100; + pmstats->next_gc = pmstats->heap_alloc+(pmstats->heap_alloc-runtime_stacks_sys)*gcpercent/100; tm4 = runtime_nanotime(); - mstats.last_gc = runtime_unixnanotime(); // must be Unix time to make sense to user - mstats.pause_ns[mstats.numgc%nelem(mstats.pause_ns)] = tm4 - tm0; - mstats.pause_end[mstats.numgc%nelem(mstats.pause_end)] = mstats.last_gc; - mstats.pause_total_ns += tm4 - tm0; - mstats.numgc++; - if(mstats.debuggc) + pmstats->last_gc = runtime_unixnanotime(); // must be Unix time to make sense to user + pmstats->pause_ns[pmstats->numgc%nelem(pmstats->pause_ns)] = tm4 - tm0; + pmstats->pause_end[pmstats->numgc%nelem(pmstats->pause_end)] = pmstats->last_gc; + pmstats->pause_total_ns += tm4 - tm0; + pmstats->numgc++; + if(pmstats->debuggc) runtime_printf("pause %D\n", tm4-tm0); if(runtime_debug.gctrace) { - heap1 = mstats.heap_alloc; + heap1 = pmstats->heap_alloc; runtime_updatememstats(&stats); - if(heap1 != mstats.heap_alloc) { - runtime_printf("runtime: mstats skew: heap=%D/%D\n", heap1, mstats.heap_alloc); + if(heap1 != pmstats->heap_alloc) { + runtime_printf("runtime: mstats skew: heap=%D/%D\n", heap1, pmstats->heap_alloc); runtime_throw("mstats skew"); } - obj = mstats.nmalloc - mstats.nfree; + obj = pmstats->nmalloc - pmstats->nfree; stats.nprocyield += work.markfor->nprocyield; stats.nosyield += work.markfor->nosyield; @@ -2305,9 +2296,9 @@ gc(struct gc_args *args) runtime_printf("gc%d(%d): %D+%D+%D+%D us, %D -> %D MB, %D (%D-%D) objects," " %d/%d/%d sweeps," " %D(%D) handoff, %D(%D) steal, %D/%D/%D yields\n", - mstats.numgc, work.nproc, (tm1-tm0)/1000, (tm2-tm1)/1000, (tm3-tm2)/1000, (tm4-tm3)/1000, + pmstats->numgc, work.nproc, (tm1-tm0)/1000, (tm2-tm1)/1000, (tm3-tm2)/1000, (tm4-tm3)/1000, heap0>>20, heap1>>20, obj, - mstats.nmalloc, mstats.nfree, + pmstats->nmalloc, pmstats->nfree, sweep.nspan, gcstats.nbgsweep, gcstats.npausesweep, stats.nhandoff, stats.nhandoffcnt, work.markfor->nsteal, work.markfor->nstealcnt, @@ -2346,7 +2337,7 @@ gc(struct gc_args *args) // Free the old cached array if necessary. if(sweep.spans && sweep.spans != runtime_mheap.allspans) - runtime_SysFree(sweep.spans, sweep.nspan*sizeof(sweep.spans[0]), &mstats.other_sys); + runtime_SysFree(sweep.spans, sweep.nspan*sizeof(sweep.spans[0]), &pmstats->other_sys); // Cache the current array. runtime_mheap.sweepspans = runtime_mheap.allspans; runtime_mheap.sweepgen += 2; @@ -2377,36 +2368,6 @@ gc(struct gc_args *args) m->traceback = 0; } -extern uintptr runtime_sizeof_C_MStats - __asm__ (GOSYM_PREFIX "runtime.Sizeof_C_MStats"); - -void runtime_ReadMemStats(MStats *) - __asm__ (GOSYM_PREFIX "runtime.ReadMemStats"); - -void -runtime_ReadMemStats(MStats *stats) -{ - M *m; - - // Have to acquire worldsema to stop the world, - // because stoptheworld can only be used by - // one goroutine at a time, and there might be - // a pending garbage collection already calling it. - runtime_semacquire(&runtime_worldsema, false); - m = runtime_m(); - m->gcing = 1; - runtime_stoptheworld(); - runtime_updatememstats(nil); - // Size of the trailing by_size array differs between Go and C, - // _NumSizeClasses was changed, but we can not change Go struct because of backward compatibility. - runtime_memmove(stats, &mstats, runtime_sizeof_C_MStats); - m->gcing = 0; - m->locks++; - runtime_semrelease(&runtime_worldsema); - runtime_starttheworld(); - m->locks--; -} - void runtime_debug_readGCStats(Slice*) __asm__("runtime_debug.readGCStats"); @@ -2415,28 +2376,30 @@ runtime_debug_readGCStats(Slice *pauses) { uint64 *p; uint32 i, n; + MStats *pmstats; // Calling code in runtime/debug should make the slice large enough. - if((size_t)pauses->cap < nelem(mstats.pause_ns)+3) + pmstats = mstats(); + if((size_t)pauses->cap < nelem(pmstats->pause_ns)+3) runtime_throw("runtime: short slice passed to readGCStats"); // Pass back: pauses, last gc (absolute time), number of gc, total pause ns. p = (uint64*)pauses->array; runtime_lock(&runtime_mheap); - n = mstats.numgc; - if(n > nelem(mstats.pause_ns)) - n = nelem(mstats.pause_ns); + n = pmstats->numgc; + if(n > nelem(pmstats->pause_ns)) + n = nelem(pmstats->pause_ns); // The pause buffer is circular. The most recent pause is at // pause_ns[(numgc-1)%nelem(pause_ns)], and then backward // from there to go back farther in time. We deliver the times // most recent first (in p[0]). for(i=0; i<n; i++) - p[i] = mstats.pause_ns[(mstats.numgc-1-i)%nelem(mstats.pause_ns)]; + p[i] = pmstats->pause_ns[(pmstats->numgc-1-i)%nelem(pmstats->pause_ns)]; - p[n] = mstats.last_gc; - p[n+1] = mstats.numgc; - p[n+2] = mstats.pause_total_ns; + p[n] = pmstats->last_gc; + p[n+1] = pmstats->numgc; + p[n+2] = pmstats->pause_total_ns; runtime_unlock(&runtime_mheap); pauses->__count = n+3; } @@ -2745,7 +2708,7 @@ runtime_MHeap_MapBits(MHeap *h) if(h->bitmap_mapped >= n) return; - runtime_SysMap(h->arena_start - n, n - h->bitmap_mapped, h->arena_reserved, &mstats.gc_sys); + runtime_SysMap(h->arena_start - n, n - h->bitmap_mapped, h->arena_reserved, &mstats()->gc_sys); h->bitmap_mapped = n; } diff --git a/libgo/runtime/mheap.c b/libgo/runtime/mheap.c index 04a5b98..c167bdc 100644 --- a/libgo/runtime/mheap.c +++ b/libgo/runtime/mheap.c @@ -36,7 +36,7 @@ RecordSpan(void *vh, byte *p) cap = 64*1024/sizeof(all[0]); if(cap < h->nspancap*3/2) cap = h->nspancap*3/2; - all = (MSpan**)runtime_SysAlloc(cap*sizeof(all[0]), &mstats.other_sys); + all = (MSpan**)runtime_SysAlloc(cap*sizeof(all[0]), &mstats()->other_sys); if(all == nil) runtime_throw("runtime: cannot allocate memory"); if(h->allspans) { @@ -44,7 +44,7 @@ RecordSpan(void *vh, byte *p) // Don't free the old array if it's referenced by sweep. // See the comment in mgc0.c. if(h->allspans != runtime_mheap.sweepspans) - runtime_SysFree(h->allspans, h->nspancap*sizeof(all[0]), &mstats.other_sys); + runtime_SysFree(h->allspans, h->nspancap*sizeof(all[0]), &mstats()->other_sys); } h->allspans = all; h->nspancap = cap; @@ -56,12 +56,14 @@ RecordSpan(void *vh, byte *p) void runtime_MHeap_Init(MHeap *h) { + MStats *pmstats; uint32 i; - runtime_FixAlloc_Init(&h->spanalloc, sizeof(MSpan), RecordSpan, h, &mstats.mspan_sys); - runtime_FixAlloc_Init(&h->cachealloc, sizeof(MCache), nil, nil, &mstats.mcache_sys); - runtime_FixAlloc_Init(&h->specialfinalizeralloc, sizeof(SpecialFinalizer), nil, nil, &mstats.other_sys); - runtime_FixAlloc_Init(&h->specialprofilealloc, sizeof(SpecialProfile), nil, nil, &mstats.other_sys); + pmstats = mstats(); + runtime_FixAlloc_Init(&h->spanalloc, sizeof(MSpan), RecordSpan, h, &pmstats->mspan_sys); + runtime_FixAlloc_Init(&h->cachealloc, sizeof(MCache), nil, nil, &pmstats->mcache_sys); + runtime_FixAlloc_Init(&h->specialfinalizeralloc, sizeof(SpecialFinalizer), nil, nil, &pmstats->other_sys); + runtime_FixAlloc_Init(&h->specialprofilealloc, sizeof(SpecialProfile), nil, nil, &pmstats->other_sys); // h->mapcache needs no init for(i=0; i<nelem(h->free); i++) { runtime_MSpanList_Init(&h->free[i]); @@ -88,7 +90,7 @@ runtime_MHeap_MapSpans(MHeap *h) n = ROUND(n, pagesize); if(h->spans_mapped >= n) return; - runtime_SysMap((byte*)h->spans + h->spans_mapped, n - h->spans_mapped, h->arena_reserved, &mstats.other_sys); + runtime_SysMap((byte*)h->spans + h->spans_mapped, n - h->spans_mapped, h->arena_reserved, &mstats()->other_sys); h->spans_mapped = n; } @@ -173,17 +175,19 @@ MHeap_Reclaim(MHeap *h, uintptr npage) MSpan* runtime_MHeap_Alloc(MHeap *h, uintptr npage, int32 sizeclass, bool large, bool needzero) { + MStats *pmstats; MSpan *s; runtime_lock(h); - mstats.heap_alloc += (intptr)runtime_m()->mcache->local_cachealloc; + pmstats = mstats(); + pmstats->heap_alloc += (intptr)runtime_m()->mcache->local_cachealloc; runtime_m()->mcache->local_cachealloc = 0; s = MHeap_AllocLocked(h, npage, sizeclass); if(s != nil) { - mstats.heap_inuse += npage<<PageShift; + pmstats->heap_inuse += npage<<PageShift; if(large) { - mstats.heap_objects++; - mstats.heap_alloc += npage<<PageShift; + pmstats->heap_objects++; + pmstats->heap_alloc += npage<<PageShift; // Swept spans are at the end of lists. if(s->npages < nelem(h->free)) runtime_MSpanList_InsertBack(&h->busy[s->npages], s); @@ -237,8 +241,8 @@ HaveSpan: runtime_MSpanList_Remove(s); runtime_atomicstore(&s->sweepgen, h->sweepgen); s->state = MSpanInUse; - mstats.heap_idle -= s->npages<<PageShift; - mstats.heap_released -= s->npreleased<<PageShift; + mstats()->heap_idle -= s->npages<<PageShift; + mstats()->heap_released -= s->npreleased<<PageShift; if(s->npreleased > 0) runtime_SysUsed((void*)(s->start<<PageShift), s->npages<<PageShift); s->npreleased = 0; @@ -326,7 +330,7 @@ MHeap_Grow(MHeap *h, uintptr npage) v = runtime_MHeap_SysAlloc(h, ask); } if(v == nil) { - runtime_printf("runtime: out of memory: cannot allocate %D-byte block (%D in use)\n", (uint64)ask, mstats.heap_sys); + runtime_printf("runtime: out of memory: cannot allocate %D-byte block (%D in use)\n", (uint64)ask, mstats()->heap_sys); return false; } } @@ -386,13 +390,16 @@ runtime_MHeap_LookupMaybe(MHeap *h, void *v) void runtime_MHeap_Free(MHeap *h, MSpan *s, int32 acct) { + MStats *pmstats; + runtime_lock(h); - mstats.heap_alloc += (intptr)runtime_m()->mcache->local_cachealloc; + pmstats = mstats(); + pmstats->heap_alloc += (intptr)runtime_m()->mcache->local_cachealloc; runtime_m()->mcache->local_cachealloc = 0; - mstats.heap_inuse -= s->npages<<PageShift; + pmstats->heap_inuse -= s->npages<<PageShift; if(acct) { - mstats.heap_alloc -= s->npages<<PageShift; - mstats.heap_objects--; + pmstats->heap_alloc -= s->npages<<PageShift; + pmstats->heap_objects--; } MHeap_FreeLocked(h, s); runtime_unlock(h); @@ -411,7 +418,7 @@ MHeap_FreeLocked(MHeap *h, MSpan *s) s, s->start<<PageShift, s->state, s->ref, s->sweepgen, h->sweepgen); runtime_throw("MHeap_FreeLocked - invalid free"); } - mstats.heap_idle += s->npages<<PageShift; + mstats()->heap_idle += s->npages<<PageShift; s->state = MSpanFree; runtime_MSpanList_Remove(s); // Stamp newly unused spans. The scavenger will use that @@ -472,7 +479,7 @@ scavengelist(MSpan *list, uint64 now, uint64 limit) for(s=list->next; s != list; s=s->next) { if((now - s->unusedsince) > limit && s->npreleased != s->npages) { released = (s->npages - s->npreleased) << PageShift; - mstats.heap_released += released; + mstats()->heap_released += released; sumreleased += released; s->npreleased = s->npages; @@ -508,8 +515,8 @@ scavenge(int32 k, uint64 now, uint64 limit) if(sumreleased > 0) runtime_printf("scvg%d: %D MB released\n", k, (uint64)sumreleased>>20); runtime_printf("scvg%d: inuse: %D, idle: %D, sys: %D, released: %D, consumed: %D (MB)\n", - k, mstats.heap_inuse>>20, mstats.heap_idle>>20, mstats.heap_sys>>20, - mstats.heap_released>>20, (mstats.heap_sys - mstats.heap_released)>>20); + k, mstats()->heap_inuse>>20, mstats()->heap_idle>>20, mstats()->heap_sys>>20, + mstats()->heap_released>>20, (mstats()->heap_sys - mstats()->heap_released)>>20); } } @@ -550,7 +557,7 @@ runtime_MHeap_Scavenger(void* dummy) runtime_lock(h); unixnow = runtime_unixnanotime(); - if(unixnow - mstats.last_gc > forcegc) { + if(unixnow - mstats()->last_gc > forcegc) { runtime_unlock(h); // The scavenger can not block other goroutines, // otherwise deadlock detector can fire spuriously. diff --git a/libgo/runtime/mprof.goc b/libgo/runtime/mprof.goc index be2c17e..c4966a4 100644 --- a/libgo/runtime/mprof.goc +++ b/libgo/runtime/mprof.goc @@ -90,7 +90,7 @@ stkbucket(int32 typ, uintptr size, Location *stk, int32 nstk, bool alloc) Bucket *b; if(buckhash == nil) { - buckhash = runtime_SysAlloc(BuckHashSize*sizeof buckhash[0], &mstats.buckhash_sys); + buckhash = runtime_SysAlloc(BuckHashSize*sizeof buckhash[0], &mstats()->buckhash_sys); if(buckhash == nil) runtime_throw("runtime: cannot allocate memory"); } @@ -127,7 +127,7 @@ stkbucket(int32 typ, uintptr size, Location *stk, int32 nstk, bool alloc) if(!alloc) return nil; - b = runtime_persistentalloc(sizeof *b + nstk*sizeof stk[0], 0, &mstats.buckhash_sys); + b = runtime_persistentalloc(sizeof *b + nstk*sizeof stk[0], 0, &mstats()->buckhash_sys); bucketmem += sizeof *b + nstk*sizeof stk[0]; runtime_memmove(b->stk, stk, nstk*sizeof stk[0]); b->typ = typ; @@ -408,11 +408,11 @@ func Stack(b Slice, all bool) (n int) { pc = (byte*)(uintptr)runtime_getcallerpc(&b); if(all) { - runtime_semacquire(&runtime_worldsema, false); + runtime_acquireWorldsema(); runtime_m()->gcing = 1; - runtime_stoptheworld(); - enablegc = mstats.enablegc; - mstats.enablegc = false; + runtime_stopTheWorldWithSema(); + enablegc = mstats()->enablegc; + mstats()->enablegc = false; } if(b.__count == 0) @@ -436,9 +436,9 @@ func Stack(b Slice, all bool) (n int) { if(all) { runtime_m()->gcing = 0; - mstats.enablegc = enablegc; - runtime_semrelease(&runtime_worldsema); - runtime_starttheworld(); + mstats()->enablegc = enablegc; + runtime_releaseWorldsema(); + runtime_startTheWorldWithSema(); } } @@ -469,9 +469,9 @@ func GoroutineProfile(b Slice) (n int, ok bool) { ok = false; n = runtime_gcount(); if(n <= b.__count) { - runtime_semacquire(&runtime_worldsema, false); + runtime_acquireWorldsema(); runtime_m()->gcing = 1; - runtime_stoptheworld(); + runtime_stopTheWorldWithSema(); n = runtime_gcount(); if(n <= b.__count) { @@ -488,8 +488,8 @@ func GoroutineProfile(b Slice) (n int, ok bool) { } runtime_m()->gcing = 0; - runtime_semrelease(&runtime_worldsema); - runtime_starttheworld(); + runtime_releaseWorldsema(); + runtime_startTheWorldWithSema(); } } diff --git a/libgo/runtime/msize.c b/libgo/runtime/msize.c index 1bafc82..b82c709 100644 --- a/libgo/runtime/msize.c +++ b/libgo/runtime/msize.c @@ -60,6 +60,7 @@ runtime_InitSizes(void) int32 align, sizeclass, size, nextsize, n; uint32 i; uintptr allocsize, npages; + MStats *pmstats; // Initialize the runtime_class_to_size table (and choose class sizes in the process). runtime_class_to_size[0] = 0; @@ -134,8 +135,9 @@ runtime_InitSizes(void) } // Copy out for statistics table. + pmstats = mstats(); for(i=0; i<nelem(runtime_class_to_size); i++) - mstats.by_size[i].size = runtime_class_to_size[i]; + pmstats->by_size[i].size = runtime_class_to_size[i]; return; dump: diff --git a/libgo/runtime/netpoll.goc b/libgo/runtime/netpoll.goc index 2744ec5..ecd426f 100644 --- a/libgo/runtime/netpoll.goc +++ b/libgo/runtime/netpoll.goc @@ -459,7 +459,7 @@ allocPollDesc(void) n = 1; // Must be in non-GC memory because can be referenced // only from epoll/kqueue internals. - pd = runtime_persistentalloc(n*sizeof(*pd), 0, &mstats.other_sys); + pd = runtime_persistentalloc(n*sizeof(*pd), 0, &mstats()->other_sys); for(i = 0; i < n; i++) { pd[i].link = pollcache.first; pollcache.first = &pd[i]; diff --git a/libgo/runtime/netpoll_select.c b/libgo/runtime/netpoll_select.c index 033661d..b32a1d5 100644 --- a/libgo/runtime/netpoll_select.c +++ b/libgo/runtime/netpoll_select.c @@ -149,7 +149,7 @@ runtime_netpoll(bool block) if(inuse) { if(!allocatedfds) { - prfds = runtime_SysAlloc(4 * sizeof fds, &mstats.other_sys); + prfds = runtime_SysAlloc(4 * sizeof fds, &mstats()->other_sys); pwfds = prfds + 1; pefds = pwfds + 1; ptfds = pefds + 1; @@ -239,7 +239,7 @@ runtime_netpoll(bool block) goto retry; if(allocatedfds) { - runtime_SysFree(prfds, 4 * sizeof fds, &mstats.other_sys); + runtime_SysFree(prfds, 4 * sizeof fds, &mstats()->other_sys); } else { runtime_lock(&selectlock); inuse = false; diff --git a/libgo/runtime/proc.c b/libgo/runtime/proc.c index eb9e6c2..d8a26fd 100644 --- a/libgo/runtime/proc.c +++ b/libgo/runtime/proc.c @@ -508,7 +508,7 @@ runtime_schedinit(void) procresize(procs); // Can not enable GC until all roots are registered. - // mstats.enablegc = 1; + // mstats()->enablegc = 1; } extern void main_init(void) __asm__ (GOSYM_PREFIX "__go_init_main"); @@ -633,7 +633,7 @@ runtime_main(void* dummy __attribute__((unused))) // For gccgo we have to wait until after main is initialized // to enable GC, because initializing main registers the GC // roots. - mstats.enablegc = 1; + mstats()->enablegc = 1; if(runtime_isarchive) { // This is not a complete program, but is instead a @@ -951,7 +951,7 @@ runtime_freezetheworld(void) } void -runtime_stoptheworld(void) +runtime_stopTheWorldWithSema(void) { int32 i; uint32 s; @@ -1001,7 +1001,7 @@ mhelpgc(void) } void -runtime_starttheworld(void) +runtime_startTheWorldWithSema(void) { P *p, *p1; M *mp; @@ -1045,7 +1045,7 @@ runtime_starttheworld(void) mp = (M*)p->m; p->m = 0; if(mp->nextp) - runtime_throw("starttheworld: inconsistent mp->nextp"); + runtime_throw("startTheWorldWithSema: inconsistent mp->nextp"); mp->nextp = (uintptr)p; runtime_notewakeup(&mp->park); } else { @@ -2373,7 +2373,7 @@ runtime_malg(int32 stacksize, byte** ret_stack, uintptr* ret_stacksize) // 32-bit mode, the Go allocation space is all of // memory anyhow. if(sizeof(void*) == 8) { - void *p = runtime_SysAlloc(stacksize, &mstats.other_sys); + void *p = runtime_SysAlloc(stacksize, &mstats()->other_sys); if(p == nil) runtime_throw("runtime: cannot allocate memory for goroutine stack"); *ret_stack = (byte*)p; @@ -2583,13 +2583,13 @@ runtime_gomaxprocsfunc(int32 n) } runtime_unlock(&runtime_sched); - runtime_semacquire(&runtime_worldsema, false); + runtime_acquireWorldsema(); g->m->gcing = 1; - runtime_stoptheworld(); + runtime_stopTheWorldWithSema(); newprocs = n; g->m->gcing = 0; - runtime_semrelease(&runtime_worldsema); - runtime_starttheworld(); + runtime_releaseWorldsema(); + runtime_startTheWorldWithSema(); return ret; } diff --git a/libgo/runtime/runtime.h b/libgo/runtime/runtime.h index 8c12265..b7e5902 100644 --- a/libgo/runtime/runtime.h +++ b/libgo/runtime/runtime.h @@ -448,9 +448,14 @@ int32 runtime_setmaxthreads(int32); G* runtime_timejump(void); void runtime_iterate_finq(void (*callback)(FuncVal*, void*, const FuncType*, const PtrType*)); -void runtime_stoptheworld(void); -void runtime_starttheworld(void); -extern uint32 runtime_worldsema; +void runtime_stopTheWorldWithSema(void) + __asm__(GOSYM_PREFIX "runtime.stopTheWorldWithSema"); +void runtime_startTheWorldWithSema(void) + __asm__(GOSYM_PREFIX "runtime.startTheWorldWithSema"); +void runtime_acquireWorldsema(void) + __asm__(GOSYM_PREFIX "runtime.acquireWorldsema"); +void runtime_releaseWorldsema(void) + __asm__(GOSYM_PREFIX "runtime.releaseWorldsema"); /* * mutual exclusion locks. in the uncontended case, |