aboutsummaryrefslogtreecommitdiff
path: root/libgo/runtime
diff options
context:
space:
mode:
Diffstat (limited to 'libgo/runtime')
-rw-r--r--libgo/runtime/chan.c33
-rw-r--r--libgo/runtime/cpuprof.c2
-rw-r--r--libgo/runtime/go-reflect-map.c178
-rw-r--r--libgo/runtime/go-type.h5
-rw-r--r--libgo/runtime/go-unsafe-pointer.c11
-rw-r--r--libgo/runtime/malloc.goc22
-rw-r--r--libgo/runtime/malloc.h5
-rw-r--r--libgo/runtime/mgc0.c533
-rw-r--r--libgo/runtime/mprof.goc48
-rw-r--r--libgo/runtime/print.c5
-rw-r--r--libgo/runtime/race.h2
-rw-r--r--libgo/runtime/runtime.c4
-rw-r--r--libgo/runtime/runtime.h7
-rw-r--r--libgo/runtime/signal_unix.c9
14 files changed, 491 insertions, 373 deletions
diff --git a/libgo/runtime/chan.c b/libgo/runtime/chan.c
index 38a2aaf..6bd12e4 100644
--- a/libgo/runtime/chan.c
+++ b/libgo/runtime/chan.c
@@ -123,19 +123,16 @@ runtime_makechan_c(ChanType *t, int64 hint)
// For reflect
// func makechan(typ *ChanType, size uint64) (chan)
-uintptr reflect_makechan(ChanType *, uint64)
+Hchan *reflect_makechan(ChanType *, uint64)
__asm__ (GOSYM_PREFIX "reflect.makechan");
-uintptr
+Hchan *
reflect_makechan(ChanType *t, uint64 size)
{
- void *ret;
Hchan *c;
c = runtime_makechan_c(t, size);
- ret = runtime_mal(sizeof(void*));
- __builtin_memcpy(ret, &c, sizeof(void*));
- return (uintptr)ret;
+ return c;
}
// makechan(t *ChanType, hint int64) (hchan *chan any);
@@ -1308,12 +1305,12 @@ runtime_closechan(Hchan *c)
// For reflect
// func chanclose(c chan)
-void reflect_chanclose(uintptr) __asm__ (GOSYM_PREFIX "reflect.chanclose");
+void reflect_chanclose(Hchan *) __asm__ (GOSYM_PREFIX "reflect.chanclose");
void
-reflect_chanclose(uintptr c)
+reflect_chanclose(Hchan *c)
{
- closechan((Hchan*)c, runtime_getcallerpc(&c));
+ closechan(c, runtime_getcallerpc(&c));
}
static void
@@ -1377,15 +1374,13 @@ __go_builtin_close(Hchan *c)
// For reflect
// func chanlen(c chan) (len int)
-intgo reflect_chanlen(uintptr) __asm__ (GOSYM_PREFIX "reflect.chanlen");
+intgo reflect_chanlen(Hchan *) __asm__ (GOSYM_PREFIX "reflect.chanlen");
intgo
-reflect_chanlen(uintptr ca)
+reflect_chanlen(Hchan *c)
{
- Hchan *c;
intgo len;
- c = (Hchan*)ca;
if(c == nil)
len = 0;
else
@@ -1396,21 +1391,19 @@ reflect_chanlen(uintptr ca)
intgo
__go_chan_len(Hchan *c)
{
- return reflect_chanlen((uintptr)c);
+ return reflect_chanlen(c);
}
// For reflect
-// func chancap(c chan) (cap intgo)
+// func chancap(c chan) int
-intgo reflect_chancap(uintptr) __asm__ (GOSYM_PREFIX "reflect.chancap");
+intgo reflect_chancap(Hchan *) __asm__ (GOSYM_PREFIX "reflect.chancap");
intgo
-reflect_chancap(uintptr ca)
+reflect_chancap(Hchan *c)
{
- Hchan *c;
intgo cap;
- c = (Hchan*)ca;
if(c == nil)
cap = 0;
else
@@ -1421,7 +1414,7 @@ reflect_chancap(uintptr ca)
intgo
__go_chan_cap(Hchan *c)
{
- return reflect_chancap((uintptr)c);
+ return reflect_chancap(c);
}
static SudoG*
diff --git a/libgo/runtime/cpuprof.c b/libgo/runtime/cpuprof.c
index a2a1a05..a07029c 100644
--- a/libgo/runtime/cpuprof.c
+++ b/libgo/runtime/cpuprof.c
@@ -177,7 +177,7 @@ runtime_SetCPUProfileRate(intgo hz)
runtime_noteclear(&prof->wait);
runtime_setcpuprofilerate(tick, hz);
- } else if(prof->on) {
+ } else if(prof != nil && prof->on) {
runtime_setcpuprofilerate(nil, 0);
prof->on = false;
diff --git a/libgo/runtime/go-reflect-map.c b/libgo/runtime/go-reflect-map.c
index 1ae7c96..ab116e8 100644
--- a/libgo/runtime/go-reflect-map.c
+++ b/libgo/runtime/go-reflect-map.c
@@ -16,112 +16,55 @@
/* This file implements support for reflection on maps. These
functions are called from reflect/value.go. */
-struct mapaccess_ret
-{
- uintptr_t val;
- _Bool pres;
-};
-
-extern struct mapaccess_ret mapaccess (struct __go_map_type *, uintptr_t,
- uintptr_t)
+extern void *mapaccess (struct __go_map_type *, void *, void *)
__asm__ (GOSYM_PREFIX "reflect.mapaccess");
-struct mapaccess_ret
-mapaccess (struct __go_map_type *mt, uintptr_t m, uintptr_t key_i)
+void *
+mapaccess (struct __go_map_type *mt, void *m, void *key)
{
struct __go_map *map = (struct __go_map *) m;
- void *key;
- const struct __go_type_descriptor *key_descriptor;
- void *p;
- const struct __go_type_descriptor *val_descriptor;
- struct mapaccess_ret ret;
- void *val;
- void *pv;
__go_assert (mt->__common.__code == GO_MAP);
-
- key_descriptor = mt->__key_type;
- if (__go_is_pointer_type (key_descriptor))
- key = &key_i;
- else
- key = (void *) key_i;
-
if (map == NULL)
- p = NULL;
+ return NULL;
else
- p = __go_map_index (map, key, 0);
-
- val_descriptor = mt->__val_type;
- if (__go_is_pointer_type (val_descriptor))
- {
- val = NULL;
- pv = &val;
- }
- else
- {
- val = __go_alloc (val_descriptor->__size);
- pv = val;
- }
-
- if (p == NULL)
- ret.pres = 0;
- else
- {
- __builtin_memcpy (pv, p, val_descriptor->__size);
- ret.pres = 1;
- }
-
- ret.val = (uintptr_t) val;
- return ret;
+ return __go_map_index (map, key, 0);
}
-extern void mapassign (struct __go_map_type *, uintptr_t, uintptr_t,
- uintptr_t, _Bool)
+extern void mapassign (struct __go_map_type *, void *, void *, void *)
__asm__ (GOSYM_PREFIX "reflect.mapassign");
void
-mapassign (struct __go_map_type *mt, uintptr_t m, uintptr_t key_i,
- uintptr_t val_i, _Bool pres)
+mapassign (struct __go_map_type *mt, void *m, void *key, void *val)
{
struct __go_map *map = (struct __go_map *) m;
- const struct __go_type_descriptor *key_descriptor;
- void *key;
+ void *p;
__go_assert (mt->__common.__code == GO_MAP);
-
if (map == NULL)
runtime_panicstring ("assignment to entry in nil map");
+ p = __go_map_index (map, key, 1);
+ __builtin_memcpy (p, val, mt->__val_type->__size);
+}
- key_descriptor = mt->__key_type;
- if (__go_is_pointer_type (key_descriptor))
- key = &key_i;
- else
- key = (void *) key_i;
+extern void mapdelete (struct __go_map_type *, void *, void *)
+ __asm__ (GOSYM_PREFIX "reflect.mapdelete");
- if (!pres)
- __go_map_delete (map, key);
- else
- {
- void *p;
- const struct __go_type_descriptor *val_descriptor;
- void *pv;
-
- p = __go_map_index (map, key, 1);
-
- val_descriptor = mt->__val_type;
- if (__go_is_pointer_type (val_descriptor))
- pv = &val_i;
- else
- pv = (void *) val_i;
- __builtin_memcpy (p, pv, val_descriptor->__size);
- }
+void
+mapdelete (struct __go_map_type *mt, void *m, void *key)
+{
+ struct __go_map *map = (struct __go_map *) m;
+
+ __go_assert (mt->__common.__code == GO_MAP);
+ if (map == NULL)
+ return;
+ __go_map_delete (map, key);
}
-extern int32_t maplen (uintptr_t)
- __asm__ (GOSYM_PREFIX "reflect.maplen");
+extern int32_t maplen (void *) __asm__ (GOSYM_PREFIX "reflect.maplen");
int32_t
-maplen (uintptr_t m)
+maplen (void *m)
{
struct __go_map *map = (struct __go_map *) m;
@@ -130,11 +73,11 @@ maplen (uintptr_t m)
return (int32_t) map->__element_count;
}
-extern unsigned char *mapiterinit (struct __go_map_type *, uintptr_t)
+extern unsigned char *mapiterinit (struct __go_map_type *, void *)
__asm__ (GOSYM_PREFIX "reflect.mapiterinit");
unsigned char *
-mapiterinit (struct __go_map_type *mt, uintptr_t m)
+mapiterinit (struct __go_map_type *mt, void *m)
{
struct __go_hash_iter *it;
@@ -144,78 +87,45 @@ mapiterinit (struct __go_map_type *mt, uintptr_t m)
return (unsigned char *) it;
}
-extern void mapiternext (unsigned char *)
- __asm__ (GOSYM_PREFIX "reflect.mapiternext");
+extern void mapiternext (void *) __asm__ (GOSYM_PREFIX "reflect.mapiternext");
void
-mapiternext (unsigned char *it)
+mapiternext (void *it)
{
__go_mapiternext ((struct __go_hash_iter *) it);
}
-struct mapiterkey_ret
-{
- uintptr_t key;
- _Bool ok;
-};
-
-extern struct mapiterkey_ret mapiterkey (unsigned char *)
- __asm__ (GOSYM_PREFIX "reflect.mapiterkey");
+extern void *mapiterkey (void *) __asm__ (GOSYM_PREFIX "reflect.mapiterkey");
-struct mapiterkey_ret
-mapiterkey (unsigned char *ita)
+void *
+mapiterkey (void *ita)
{
struct __go_hash_iter *it = (struct __go_hash_iter *) ita;
- struct mapiterkey_ret ret;
+ const struct __go_type_descriptor *key_descriptor;
+ void *key;
if (it->entry == NULL)
- {
- ret.key = 0;
- ret.ok = 0;
- }
- else
- {
- const struct __go_type_descriptor *key_descriptor;
- void *key;
- void *pk;
-
- key_descriptor = it->map->__descriptor->__map_descriptor->__key_type;
- if (__go_is_pointer_type (key_descriptor))
- {
- key = NULL;
- pk = &key;
- }
- else
- {
- key = __go_alloc (key_descriptor->__size);
- pk = key;
- }
-
- __go_mapiter1 (it, pk);
-
- ret.key = (uintptr_t) key;
- ret.ok = 1;
- }
-
- return ret;
+ return NULL;
+
+ key_descriptor = it->map->__descriptor->__map_descriptor->__key_type;
+ key = __go_alloc (key_descriptor->__size);
+ __go_mapiter1 (it, key);
+ return key;
}
/* Make a new map. We have to build our own map descriptor. */
-extern uintptr_t makemap (const struct __go_map_type *)
+extern struct __go_map *makemap (const struct __go_map_type *)
__asm__ (GOSYM_PREFIX "reflect.makemap");
-uintptr_t
+struct __go_map *
makemap (const struct __go_map_type *t)
{
struct __go_map_descriptor *md;
unsigned int o;
const struct __go_type_descriptor *kt;
const struct __go_type_descriptor *vt;
- struct __go_map* map;
- void *ret;
- /* FIXME: Reference count. */
md = (struct __go_map_descriptor *) __go_alloc (sizeof (*md));
md->__map_descriptor = t;
o = sizeof (void *);
@@ -232,11 +142,7 @@ makemap (const struct __go_map_type *t)
o = (o + vt->__field_align - 1) & ~ (vt->__field_align - 1);
md->__entry_size = o;
- map = __go_new_map (md, 0);
-
- ret = __go_alloc (sizeof (void *));
- __builtin_memcpy (ret, &map, sizeof (void *));
- return (uintptr_t) ret;
+ return __go_new_map (md, 0);
}
extern _Bool ismapkey (const struct __go_type_descriptor *)
diff --git a/libgo/runtime/go-type.h b/libgo/runtime/go-type.h
index 2269ae6..fa355ce 100644
--- a/libgo/runtime/go-type.h
+++ b/libgo/runtime/go-type.h
@@ -103,6 +103,11 @@ struct __go_type_descriptor
/* The descriptor for the type which is a pointer to this type.
This may be NULL. */
const struct __go_type_descriptor *__pointer_to_this;
+
+ /* A pointer to a zero value for this type. All types will point to
+ the same zero value, go$zerovalue, which is a common variable so
+ that it will be large enough. */
+ void *__zero;
};
/* The information we store for each method of a type. */
diff --git a/libgo/runtime/go-unsafe-pointer.c b/libgo/runtime/go-unsafe-pointer.c
index ca1d253..b71804a 100644
--- a/libgo/runtime/go-unsafe-pointer.c
+++ b/libgo/runtime/go-unsafe-pointer.c
@@ -9,6 +9,9 @@
#include "runtime.h"
#include "go-type.h"
+/* A pointer with a zero value. */
+static void *zero_pointer;
+
/* This file provides the type descriptor for the unsafe.Pointer type.
The unsafe package is defined by the compiler itself, which means
that there is no package to compile to define the type
@@ -53,7 +56,9 @@ const struct __go_type_descriptor unsafe_Pointer =
/* __uncommon */
NULL,
/* __pointer_to_this */
- NULL
+ NULL,
+ /* __zero */
+ &zero_pointer
};
/* We also need the type descriptor for the pointer to unsafe.Pointer,
@@ -94,7 +99,9 @@ const struct __go_ptr_type pointer_unsafe_Pointer =
/* __uncommon */
NULL,
/* __pointer_to_this */
- NULL
+ NULL,
+ /* __zero */
+ &zero_pointer
},
/* __element_type */
&unsafe_Pointer
diff --git a/libgo/runtime/malloc.goc b/libgo/runtime/malloc.goc
index 7120457..33d0c39 100644
--- a/libgo/runtime/malloc.goc
+++ b/libgo/runtime/malloc.goc
@@ -118,7 +118,7 @@ runtime_mallocgc(uintptr size, uintptr typ, uint32 flag)
size += sizeof(uintptr);
c = m->mcache;
- if(size <= MaxSmallSize) {
+ if(!runtime_debug.efence && size <= MaxSmallSize) {
// Allocate from mcache free lists.
// Inlined version of SizeToClass().
if(size <= 1024-8)
@@ -157,8 +157,10 @@ runtime_mallocgc(uintptr size, uintptr typ, uint32 flag)
runtime_markspan(v, 0, 0, true);
}
- if(!(flag & FlagNoGC))
- runtime_markallocated(v, size, (flag&FlagNoScan) != 0);
+ if(flag & FlagNoGC)
+ runtime_marknogc(v);
+ else if(!(flag & FlagNoScan))
+ runtime_markscan(v);
if(DebugTypeAtBlockEnd)
*(uintptr*)((uintptr)v+size-sizeof(uintptr)) = typ;
@@ -180,6 +182,9 @@ runtime_mallocgc(uintptr size, uintptr typ, uint32 flag)
runtime_settype_flush(m);
m->locks--;
+ if(runtime_debug.allocfreetrace)
+ goto profile;
+
if(!(flag & FlagNoProfiling) && (rate = runtime_MemProfileRate) > 0) {
if(size >= (uint32) rate)
goto profile;
@@ -193,7 +198,7 @@ runtime_mallocgc(uintptr size, uintptr typ, uint32 flag)
m->mcache->next_sample = runtime_fastrand1() % (2*rate);
profile:
runtime_setblockspecial(v, true);
- runtime_MProf_Malloc(v, size);
+ runtime_MProf_Malloc(v, size, typ);
}
}
@@ -257,7 +262,10 @@ __go_free(void *v)
// they might coalesce v into other spans and change the bitmap further.
runtime_markfreed(v, size);
runtime_unmarkspan(v, 1<<PageShift);
- runtime_MHeap_Free(&runtime_mheap, s, 1);
+ if(runtime_debug.efence)
+ runtime_SysFree((void*)(s->start<<PageShift), size, &mstats.heap_sys);
+ else
+ runtime_MHeap_Free(&runtime_mheap, s, 1);
c->local_nlargefree++;
c->local_largefree += size;
} else {
@@ -819,6 +827,10 @@ func SetFinalizer(obj Eface, finalizer Eface) {
runtime_printf("runtime.SetFinalizer: first argument is %S, not pointer\n", *obj.__type_descriptor->__reflection);
goto throw;
}
+ ot = (const PtrType*)obj.type;
+ if(ot->__element_type != nil && ot->__element_type->__size == 0) {
+ return;
+ }
if(!runtime_mlookup(obj.__object, &base, &size, nil) || obj.__object != base) {
runtime_printf("runtime.SetFinalizer: pointer not at beginning of allocated block\n");
goto throw;
diff --git a/libgo/runtime/malloc.h b/libgo/runtime/malloc.h
index e1a5be9..16f51a5 100644
--- a/libgo/runtime/malloc.h
+++ b/libgo/runtime/malloc.h
@@ -449,7 +449,8 @@ void* runtime_mallocgc(uintptr size, uintptr typ, uint32 flag);
void* runtime_persistentalloc(uintptr size, uintptr align, uint64 *stat);
int32 runtime_mlookup(void *v, byte **base, uintptr *size, MSpan **s);
void runtime_gc(int32 force);
-void runtime_markallocated(void *v, uintptr n, bool noptr);
+void runtime_markscan(void *v);
+void runtime_marknogc(void *v);
void runtime_checkallocated(void *v, uintptr n);
void runtime_markfreed(void *v, uintptr n);
void runtime_checkfreed(void *v, uintptr n);
@@ -484,7 +485,7 @@ struct Obj
uintptr ti; // type info
};
-void runtime_MProf_Malloc(void*, uintptr);
+void runtime_MProf_Malloc(void*, uintptr, uintptr);
void runtime_MProf_Free(void*, uintptr);
void runtime_MProf_GC(void);
void runtime_MProf_Mark(void (*addroot)(Obj));
diff --git a/libgo/runtime/mgc0.c b/libgo/runtime/mgc0.c
index f963686..d665d92 100644
--- a/libgo/runtime/mgc0.c
+++ b/libgo/runtime/mgc0.c
@@ -45,7 +45,7 @@ enum {
Debug = 0,
DebugMark = 0, // run second pass to check mark
CollectStats = 0,
- ScanStackByFrames = 0,
+ ScanStackByFrames = 1,
IgnorePreciseGC = 0,
// Four bits per word (see #defines below).
@@ -68,6 +68,39 @@ enum {
BitsEface = 3,
};
+static struct
+{
+ Lock;
+ void* head;
+} pools;
+
+void sync_runtime_registerPool(void **)
+ __asm__ (GOSYM_PREFIX "sync.runtime_registerPool");
+
+void
+sync_runtime_registerPool(void **p)
+{
+ runtime_lock(&pools);
+ p[0] = pools.head;
+ pools.head = p;
+ runtime_unlock(&pools);
+}
+
+static void
+clearpools(void)
+{
+ void **p, **next;
+
+ for(p = pools.head; p != nil; p = next) {
+ next = p[0];
+ p[0] = nil; // next
+ p[1] = nil; // slice
+ p[2] = nil;
+ p[3] = nil;
+ }
+ pools.head = nil;
+}
+
// Bits in per-word bitmap.
// #defines because enum might not be able to hold the values.
//
@@ -77,7 +110,7 @@ enum {
// The bits in the word are packed together by type first, then by
// heap location, so each 64-bit bitmap word consists of, from top to bottom,
// the 16 bitSpecial bits for the corresponding heap words, then the 16 bitMarked bits,
-// then the 16 bitNoScan/bitBlockBoundary bits, then the 16 bitAllocated bits.
+// then the 16 bitScan/bitBlockBoundary bits, then the 16 bitAllocated bits.
// This layout makes it easier to iterate over the bits of a given type.
//
// The bitmap starts at mheap.arena_start and extends *backward* from
@@ -93,13 +126,13 @@ enum {
// bits = *b >> shift;
// /* then test bits & bitAllocated, bits & bitMarked, etc. */
//
-#define bitAllocated ((uintptr)1<<(bitShift*0))
-#define bitNoScan ((uintptr)1<<(bitShift*1)) /* when bitAllocated is set */
+#define bitAllocated ((uintptr)1<<(bitShift*0)) /* block start; eligible for garbage collection */
+#define bitScan ((uintptr)1<<(bitShift*1)) /* when bitAllocated is set */
#define bitMarked ((uintptr)1<<(bitShift*2)) /* when bitAllocated is set */
#define bitSpecial ((uintptr)1<<(bitShift*3)) /* when bitAllocated is set - has finalizer or being profiled */
-#define bitBlockBoundary ((uintptr)1<<(bitShift*1)) /* when bitAllocated is NOT set */
+#define bitBlockBoundary ((uintptr)1<<(bitShift*1)) /* when bitAllocated is NOT set - mark for FlagNoGC objects */
-#define bitMask (bitBlockBoundary | bitAllocated | bitMarked | bitSpecial)
+#define bitMask (bitAllocated | bitScan | bitMarked | bitSpecial)
// Holding worldsema grants an M the right to try to stop the world.
// The procedure is:
@@ -185,6 +218,7 @@ static struct {
enum {
GC_DEFAULT_PTR = GC_NUM_INSTR,
GC_CHAN,
+ GC_G_PTR,
GC_NUM_INSTR2
};
@@ -325,6 +359,24 @@ struct PtrTarget
uintptr ti;
};
+typedef struct Scanbuf Scanbuf;
+struct Scanbuf
+{
+ struct {
+ PtrTarget *begin;
+ PtrTarget *end;
+ PtrTarget *pos;
+ } ptr;
+ struct {
+ Obj *begin;
+ Obj *end;
+ Obj *pos;
+ } obj;
+ Workbuf *wbuf;
+ Obj *wp;
+ uintptr nobj;
+};
+
typedef struct BufferList BufferList;
struct BufferList
{
@@ -357,7 +409,7 @@ static void enqueue(Obj obj, Workbuf **_wbuf, Obj **_wp, uintptr *_nobj);
// flushptrbuf
// (find block start, mark and enqueue)
static void
-flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf, uintptr *_nobj)
+flushptrbuf(Scanbuf *sbuf)
{
byte *p, *arena_start, *obj;
uintptr size, *bitp, bits, shift, j, x, xbits, off, nobj, ti, n;
@@ -365,17 +417,19 @@ flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf
PageID k;
Obj *wp;
Workbuf *wbuf;
+ PtrTarget *ptrbuf;
PtrTarget *ptrbuf_end;
arena_start = runtime_mheap.arena_start;
- wp = *_wp;
- wbuf = *_wbuf;
- nobj = *_nobj;
+ wp = sbuf->wp;
+ wbuf = sbuf->wbuf;
+ nobj = sbuf->nobj;
- ptrbuf_end = *ptrbufpos;
- n = ptrbuf_end - ptrbuf;
- *ptrbufpos = ptrbuf;
+ ptrbuf = sbuf->ptr.begin;
+ ptrbuf_end = sbuf->ptr.pos;
+ n = ptrbuf_end - sbuf->ptr.begin;
+ sbuf->ptr.pos = sbuf->ptr.begin;
if(CollectStats) {
runtime_xadd64(&gcstats.ptr.sum, n);
@@ -394,150 +448,146 @@ flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf
runtime_throw("ptrbuf has to be smaller than WorkBuf");
}
- // TODO(atom): This block is a branch of an if-then-else statement.
- // The single-threaded branch may be added in a next CL.
- {
- // Multi-threaded version.
+ while(ptrbuf < ptrbuf_end) {
+ obj = ptrbuf->p;
+ ti = ptrbuf->ti;
+ ptrbuf++;
- while(ptrbuf < ptrbuf_end) {
- obj = ptrbuf->p;
- ti = ptrbuf->ti;
- ptrbuf++;
+ // obj belongs to interval [mheap.arena_start, mheap.arena_used).
+ if(Debug > 1) {
+ if(obj < runtime_mheap.arena_start || obj >= runtime_mheap.arena_used)
+ runtime_throw("object is outside of mheap");
+ }
- // obj belongs to interval [mheap.arena_start, mheap.arena_used).
- if(Debug > 1) {
- if(obj < runtime_mheap.arena_start || obj >= runtime_mheap.arena_used)
- runtime_throw("object is outside of mheap");
- }
+ // obj may be a pointer to a live object.
+ // Try to find the beginning of the object.
+
+ // Round down to word boundary.
+ if(((uintptr)obj & ((uintptr)PtrSize-1)) != 0) {
+ obj = (void*)((uintptr)obj & ~((uintptr)PtrSize-1));
+ ti = 0;
+ }
- // obj may be a pointer to a live object.
- // Try to find the beginning of the object.
+ // Find bits for this word.
+ off = (uintptr*)obj - (uintptr*)arena_start;
+ bitp = (uintptr*)arena_start - off/wordsPerBitmapWord - 1;
+ shift = off % wordsPerBitmapWord;
+ xbits = *bitp;
+ bits = xbits >> shift;
- // Round down to word boundary.
- if(((uintptr)obj & ((uintptr)PtrSize-1)) != 0) {
- obj = (void*)((uintptr)obj & ~((uintptr)PtrSize-1));
- ti = 0;
- }
+ // Pointing at the beginning of a block?
+ if((bits & (bitAllocated|bitBlockBoundary)) != 0) {
+ if(CollectStats)
+ runtime_xadd64(&gcstats.flushptrbuf.foundbit, 1);
+ goto found;
+ }
- // Find bits for this word.
- off = (uintptr*)obj - (uintptr*)arena_start;
- bitp = (uintptr*)arena_start - off/wordsPerBitmapWord - 1;
- shift = off % wordsPerBitmapWord;
- xbits = *bitp;
- bits = xbits >> shift;
+ ti = 0;
- // Pointing at the beginning of a block?
- if((bits & (bitAllocated|bitBlockBoundary)) != 0) {
+ // Pointing just past the beginning?
+ // Scan backward a little to find a block boundary.
+ for(j=shift; j-->0; ) {
+ if(((xbits>>j) & (bitAllocated|bitBlockBoundary)) != 0) {
+ obj = (byte*)obj - (shift-j)*PtrSize;
+ shift = j;
+ bits = xbits>>shift;
if(CollectStats)
- runtime_xadd64(&gcstats.flushptrbuf.foundbit, 1);
+ runtime_xadd64(&gcstats.flushptrbuf.foundword, 1);
goto found;
}
+ }
- ti = 0;
-
- // Pointing just past the beginning?
- // Scan backward a little to find a block boundary.
- for(j=shift; j-->0; ) {
- if(((xbits>>j) & (bitAllocated|bitBlockBoundary)) != 0) {
- obj = (byte*)obj - (shift-j)*PtrSize;
- shift = j;
- bits = xbits>>shift;
- if(CollectStats)
- runtime_xadd64(&gcstats.flushptrbuf.foundword, 1);
- goto found;
- }
- }
-
- // Otherwise consult span table to find beginning.
- // (Manually inlined copy of MHeap_LookupMaybe.)
- k = (uintptr)obj>>PageShift;
- x = k;
- x -= (uintptr)arena_start>>PageShift;
- s = runtime_mheap.spans[x];
- if(s == nil || k < s->start || obj >= s->limit || s->state != MSpanInUse)
- continue;
- p = (byte*)((uintptr)s->start<<PageShift);
- if(s->sizeclass == 0) {
- obj = p;
- } else {
- size = s->elemsize;
- int32 i = ((byte*)obj - p)/size;
- obj = p+i*size;
- }
+ // Otherwise consult span table to find beginning.
+ // (Manually inlined copy of MHeap_LookupMaybe.)
+ k = (uintptr)obj>>PageShift;
+ x = k;
+ x -= (uintptr)arena_start>>PageShift;
+ s = runtime_mheap.spans[x];
+ if(s == nil || k < s->start || obj >= s->limit || s->state != MSpanInUse)
+ continue;
+ p = (byte*)((uintptr)s->start<<PageShift);
+ if(s->sizeclass == 0) {
+ obj = p;
+ } else {
+ size = s->elemsize;
+ int32 i = ((byte*)obj - p)/size;
+ obj = p+i*size;
+ }
- // Now that we know the object header, reload bits.
- off = (uintptr*)obj - (uintptr*)arena_start;
- bitp = (uintptr*)arena_start - off/wordsPerBitmapWord - 1;
- shift = off % wordsPerBitmapWord;
- xbits = *bitp;
- bits = xbits >> shift;
- if(CollectStats)
- runtime_xadd64(&gcstats.flushptrbuf.foundspan, 1);
+ // Now that we know the object header, reload bits.
+ off = (uintptr*)obj - (uintptr*)arena_start;
+ bitp = (uintptr*)arena_start - off/wordsPerBitmapWord - 1;
+ shift = off % wordsPerBitmapWord;
+ xbits = *bitp;
+ bits = xbits >> shift;
+ if(CollectStats)
+ runtime_xadd64(&gcstats.flushptrbuf.foundspan, 1);
- found:
- // Now we have bits, bitp, and shift correct for
- // obj pointing at the base of the object.
- // Only care about allocated and not marked.
- if((bits & (bitAllocated|bitMarked)) != bitAllocated)
- continue;
- if(work.nproc == 1)
- *bitp |= bitMarked<<shift;
- else {
- for(;;) {
- x = *bitp;
- if(x & (bitMarked<<shift))
- goto continue_obj;
- if(runtime_casp((void**)bitp, (void*)x, (void*)(x|(bitMarked<<shift))))
- break;
- }
+ found:
+ // Now we have bits, bitp, and shift correct for
+ // obj pointing at the base of the object.
+ // Only care about allocated and not marked.
+ if((bits & (bitAllocated|bitMarked)) != bitAllocated)
+ continue;
+ if(work.nproc == 1)
+ *bitp |= bitMarked<<shift;
+ else {
+ for(;;) {
+ x = *bitp;
+ if(x & (bitMarked<<shift))
+ goto continue_obj;
+ if(runtime_casp((void**)bitp, (void*)x, (void*)(x|(bitMarked<<shift))))
+ break;
}
+ }
- // If object has no pointers, don't need to scan further.
- if((bits & bitNoScan) != 0)
- continue;
+ // If object has no pointers, don't need to scan further.
+ if((bits & bitScan) == 0)
+ continue;
- // Ask span about size class.
- // (Manually inlined copy of MHeap_Lookup.)
- x = (uintptr)obj >> PageShift;
- x -= (uintptr)arena_start>>PageShift;
- s = runtime_mheap.spans[x];
+ // Ask span about size class.
+ // (Manually inlined copy of MHeap_Lookup.)
+ x = (uintptr)obj >> PageShift;
+ x -= (uintptr)arena_start>>PageShift;
+ s = runtime_mheap.spans[x];
- PREFETCH(obj);
+ PREFETCH(obj);
- *wp = (Obj){obj, s->elemsize, ti};
- wp++;
- nobj++;
- continue_obj:;
- }
+ *wp = (Obj){obj, s->elemsize, ti};
+ wp++;
+ nobj++;
+ continue_obj:;
+ }
- // If another proc wants a pointer, give it some.
- if(work.nwait > 0 && nobj > handoffThreshold && work.full == 0) {
- wbuf->nobj = nobj;
- wbuf = handoff(wbuf);
- nobj = wbuf->nobj;
- wp = wbuf->obj + nobj;
- }
+ // If another proc wants a pointer, give it some.
+ if(work.nwait > 0 && nobj > handoffThreshold && work.full == 0) {
+ wbuf->nobj = nobj;
+ wbuf = handoff(wbuf);
+ nobj = wbuf->nobj;
+ wp = wbuf->obj + nobj;
}
- *_wp = wp;
- *_wbuf = wbuf;
- *_nobj = nobj;
+ sbuf->wp = wp;
+ sbuf->wbuf = wbuf;
+ sbuf->nobj = nobj;
}
static void
-flushobjbuf(Obj *objbuf, Obj **objbufpos, Obj **_wp, Workbuf **_wbuf, uintptr *_nobj)
+flushobjbuf(Scanbuf *sbuf)
{
uintptr nobj, off;
Obj *wp, obj;
Workbuf *wbuf;
+ Obj *objbuf;
Obj *objbuf_end;
- wp = *_wp;
- wbuf = *_wbuf;
- nobj = *_nobj;
+ wp = sbuf->wp;
+ wbuf = sbuf->wbuf;
+ nobj = sbuf->nobj;
- objbuf_end = *objbufpos;
- *objbufpos = objbuf;
+ objbuf = sbuf->obj.begin;
+ objbuf_end = sbuf->obj.pos;
+ sbuf->obj.pos = sbuf->obj.begin;
while(objbuf < objbuf_end) {
obj = *objbuf++;
@@ -575,9 +625,9 @@ flushobjbuf(Obj *objbuf, Obj **objbufpos, Obj **_wp, Workbuf **_wbuf, uintptr *_
wp = wbuf->obj + nobj;
}
- *_wp = wp;
- *_wbuf = wbuf;
- *_nobj = nobj;
+ sbuf->wp = wp;
+ sbuf->wbuf = wbuf;
+ sbuf->nobj = nobj;
}
// Program that scans the whole block and treats every block element as a potential pointer
@@ -588,6 +638,11 @@ static uintptr defaultProg[2] = {PtrSize, GC_DEFAULT_PTR};
static uintptr chanProg[2] = {0, GC_CHAN};
#endif
+#if 0
+// G* program
+static uintptr gptrProg[2] = {0, GC_G_PTR};
+#endif
+
// Local variables of a program fragment or loop
typedef struct Frame Frame;
struct Frame {
@@ -676,8 +731,7 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
Slice *sliceptr;
Frame *stack_ptr, stack_top, stack[GC_STACK_CAPACITY+4];
BufferList *scanbuffers;
- PtrTarget *ptrbuf, *ptrbuf_end, *ptrbufpos;
- Obj *objbuf, *objbuf_end, *objbufpos;
+ Scanbuf sbuf;
Eface *eface;
Iface *iface;
#if 0
@@ -693,21 +747,22 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
arena_used = runtime_mheap.arena_used;
stack_ptr = stack+nelem(stack)-1;
-
+
precise_type = false;
nominal_size = 0;
- // Allocate ptrbuf
- {
- scanbuffers = &bufferList[runtime_m()->helpgc];
- ptrbuf = &scanbuffers->ptrtarget[0];
- ptrbuf_end = &scanbuffers->ptrtarget[0] + nelem(scanbuffers->ptrtarget);
- objbuf = &scanbuffers->obj[0];
- objbuf_end = &scanbuffers->obj[0] + nelem(scanbuffers->obj);
- }
+ // Initialize sbuf
+ scanbuffers = &bufferList[runtime_m()->helpgc];
+
+ sbuf.ptr.begin = sbuf.ptr.pos = &scanbuffers->ptrtarget[0];
+ sbuf.ptr.end = sbuf.ptr.begin + nelem(scanbuffers->ptrtarget);
+
+ sbuf.obj.begin = sbuf.obj.pos = &scanbuffers->obj[0];
+ sbuf.obj.end = sbuf.obj.begin + nelem(scanbuffers->obj);
- ptrbufpos = ptrbuf;
- objbufpos = objbuf;
+ sbuf.wbuf = wbuf;
+ sbuf.wp = wp;
+ sbuf.nobj = nobj;
// (Silence the compiler)
#if 0
@@ -727,7 +782,7 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
if(CollectStats) {
runtime_xadd64(&gcstats.nbytes, n);
- runtime_xadd64(&gcstats.obj.sum, nobj);
+ runtime_xadd64(&gcstats.obj.sum, sbuf.nobj);
runtime_xadd64(&gcstats.obj.cnt, 1);
}
@@ -857,9 +912,9 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
if((const byte*)t >= arena_start && (const byte*)t < arena_used) {
union { const Type *tc; Type *tr; } u;
u.tc = t;
- *ptrbufpos++ = (struct PtrTarget){(void*)u.tr, 0};
- if(ptrbufpos == ptrbuf_end)
- flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj);
+ *sbuf.ptr.pos++ = (PtrTarget){u.tr, 0};
+ if(sbuf.ptr.pos == sbuf.ptr.end)
+ flushptrbuf(&sbuf);
}
// eface->__object
@@ -888,10 +943,9 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
// iface->tab
if((byte*)iface->tab >= arena_start && (byte*)iface->tab < arena_used) {
- // *ptrbufpos++ = (struct PtrTarget){iface->tab, (uintptr)itabtype->gc};
- *ptrbufpos++ = (struct PtrTarget){iface->tab, 0};
- if(ptrbufpos == ptrbuf_end)
- flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj);
+ *sbuf.ptr.pos++ = (PtrTarget){iface->tab, /* (uintptr)itabtype->gc */ 0};
+ if(sbuf.ptr.pos == sbuf.ptr.end)
+ flushptrbuf(&sbuf);
}
// iface->data
@@ -919,9 +973,9 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
obj = *(byte**)stack_top.b;
stack_top.b += PtrSize;
if((byte*)obj >= arena_start && (byte*)obj < arena_used) {
- *ptrbufpos++ = (struct PtrTarget){obj, 0};
- if(ptrbufpos == ptrbuf_end)
- flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj);
+ *sbuf.ptr.pos++ = (PtrTarget){obj, 0};
+ if(sbuf.ptr.pos == sbuf.ptr.end)
+ flushptrbuf(&sbuf);
}
}
goto next_block;
@@ -950,7 +1004,7 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
if(*(byte**)i != nil) {
// Found a value that may be a pointer.
// Do a rescan of the entire block.
- enqueue((Obj){b, n, 0}, &wbuf, &wp, &nobj);
+ enqueue((Obj){b, n, 0}, &sbuf.wbuf, &sbuf.wp, &sbuf.nobj);
if(CollectStats) {
runtime_xadd64(&gcstats.rescan, 1);
runtime_xadd64(&gcstats.rescanbytes, n);
@@ -996,9 +1050,9 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
objti = pc[3];
pc += 4;
- *objbufpos++ = (Obj){obj, size, objti};
- if(objbufpos == objbuf_end)
- flushobjbuf(objbuf, &objbufpos, &wp, &wbuf, &nobj);
+ *sbuf.obj.pos++ = (Obj){obj, size, objti};
+ if(sbuf.obj.pos == sbuf.obj.end)
+ flushobjbuf(&sbuf);
continue;
#if 0
@@ -1032,10 +1086,10 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
// in-use part of the circular buffer is scanned.
// (Channel routines zero the unused part, so the current
// code does not lead to leaks, it's just a little inefficient.)
- *objbufpos++ = (Obj){(byte*)chan+runtime_Hchansize, chancap*chantype->elem->size,
+ *sbuf.obj.pos++ = (Obj){(byte*)chan+runtime_Hchansize, chancap*chantype->elem->size,
(uintptr)chantype->elem->gc | PRECISE | LOOP};
- if(objbufpos == objbuf_end)
- flushobjbuf(objbuf, &objbufpos, &wp, &wbuf, &nobj);
+ if(sbuf.obj.pos == sbuf.obj.end)
+ flushobjbuf(&sbuf);
}
}
if(chan_ret == nil)
@@ -1044,15 +1098,22 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
continue;
#endif
+#if 0
+ case GC_G_PTR:
+ obj = (void*)stack_top.b;
+ scanstack(obj, &sbuf);
+ goto next_block;
+#endif
+
default:
runtime_throw("scanblock: invalid GC instruction");
return;
}
if((byte*)obj >= arena_start && (byte*)obj < arena_used) {
- *ptrbufpos++ = (struct PtrTarget){obj, objti};
- if(ptrbufpos == ptrbuf_end)
- flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj);
+ *sbuf.ptr.pos++ = (PtrTarget){obj, objti};
+ if(sbuf.ptr.pos == sbuf.ptr.end)
+ flushptrbuf(&sbuf);
}
}
@@ -1060,34 +1121,32 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
// Done scanning [b, b+n). Prepare for the next iteration of
// the loop by setting b, n, ti to the parameters for the next block.
- if(nobj == 0) {
- flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj);
- flushobjbuf(objbuf, &objbufpos, &wp, &wbuf, &nobj);
+ if(sbuf.nobj == 0) {
+ flushptrbuf(&sbuf);
+ flushobjbuf(&sbuf);
- if(nobj == 0) {
+ if(sbuf.nobj == 0) {
if(!keepworking) {
- if(wbuf)
- putempty(wbuf);
- goto endscan;
+ if(sbuf.wbuf)
+ putempty(sbuf.wbuf);
+ return;
}
// Emptied our buffer: refill.
- wbuf = getfull(wbuf);
- if(wbuf == nil)
- goto endscan;
- nobj = wbuf->nobj;
- wp = wbuf->obj + wbuf->nobj;
+ sbuf.wbuf = getfull(sbuf.wbuf);
+ if(sbuf.wbuf == nil)
+ return;
+ sbuf.nobj = sbuf.wbuf->nobj;
+ sbuf.wp = sbuf.wbuf->obj + sbuf.wbuf->nobj;
}
}
// Fetch b from the work buffer.
- --wp;
- b = wp->p;
- n = wp->n;
- ti = wp->ti;
- nobj--;
+ --sbuf.wp;
+ b = sbuf.wp->p;
+ n = sbuf.wp->n;
+ ti = sbuf.wp->ti;
+ sbuf.nobj--;
}
-
-endscan:;
}
// debug_scanblock is the debug copy of scanblock.
@@ -1159,7 +1218,7 @@ debug_scanblock(byte *b, uintptr n)
runtime_printf("found unmarked block %p in %p\n", obj, vp+i);
// If object has no pointers, don't need to scan further.
- if((bits & bitNoScan) != 0)
+ if((bits & bitScan) == 0)
continue;
debug_scanblock(obj, size);
@@ -1536,6 +1595,28 @@ addroots(void)
addroot((Obj){(byte*)&work, sizeof work, 0});
}
+static void
+addfreelists(void)
+{
+ int32 i;
+ P *p, **pp;
+ MCache *c;
+ MLink *m;
+
+ // Mark objects in the MCache of each P so we don't collect them.
+ for(pp=runtime_allp; (p=*pp); pp++) {
+ c = p->mcache;
+ if(c==nil)
+ continue;
+ for(i = 0; i < NumSizeClasses; i++) {
+ for(m = c->list[i].list; m != nil; m = m->next) {
+ markonly(m);
+ }
+ }
+ }
+ // Note: the sweeper will mark objects in each span's freelist.
+}
+
static bool
handlespecial(byte *p, uintptr size)
{
@@ -1581,7 +1662,7 @@ sweepspan(ParFor *desc, uint32 idx)
{
M *m;
int32 cl, n, npages;
- uintptr size;
+ uintptr size, off, *bitp, shift;
byte *p;
MCache *c;
byte *arena_start;
@@ -1591,6 +1672,7 @@ sweepspan(ParFor *desc, uint32 idx)
byte compression;
uintptr type_data_inc;
MSpan *s;
+ MLink *x;
m = runtime_m();
@@ -1612,6 +1694,17 @@ sweepspan(ParFor *desc, uint32 idx)
nfree = 0;
end = &head;
c = m->mcache;
+
+ // mark any free objects in this span so we don't collect them
+ for(x = s->freelist; x != nil; x = x->next) {
+ // This is markonly(x) but faster because we don't need
+ // atomic access and we're guaranteed to be pointing at
+ // the head of a valid object.
+ off = (uintptr*)x - (uintptr*)runtime_mheap.arena_start;
+ bitp = (uintptr*)runtime_mheap.arena_start - off/wordsPerBitmapWord - 1;
+ shift = off % wordsPerBitmapWord;
+ *bitp |= bitMarked<<shift;
+ }
type_data = (byte*)s->types.data;
type_data_inc = sizeof(uintptr);
@@ -1655,14 +1748,17 @@ sweepspan(ParFor *desc, uint32 idx)
continue;
}
- // Mark freed; restore block boundary bit.
- *bitp = (*bitp & ~(bitMask<<shift)) | (bitBlockBoundary<<shift);
+ // Clear mark, scan, and special bits.
+ *bitp &= ~((bitScan|bitMarked|bitSpecial)<<shift);
if(cl == 0) {
// Free large span.
runtime_unmarkspan(p, 1<<PageShift);
*(uintptr*)p = (uintptr)0xdeaddeaddeaddeadll; // needs zeroing
- runtime_MHeap_Free(&runtime_mheap, s, 1);
+ if(runtime_debug.efence)
+ runtime_SysFree(p, size, &mstats.gc_sys);
+ else
+ runtime_MHeap_Free(&runtime_mheap, s, 1);
c->local_nlargefree++;
c->local_largefree += size;
} else {
@@ -1985,7 +2081,9 @@ runtime_gc(int32 force)
a.start_time = runtime_nanotime();
m->gcing = 1;
runtime_stoptheworld();
-
+
+ clearpools();
+
// Run gc on the g0 stack. We do this so that the g stack
// we're currently running on will no longer change. Cuts
// the root set down a bit (g0 stacks are not scanned, and
@@ -2081,6 +2179,7 @@ gc(struct gc_args *args)
work.debugmarkdone = 0;
work.nproc = runtime_gcprocs();
addroots();
+ addfreelists();
runtime_parforsetup(work.markfor, work.nproc, work.nroot, nil, false, markroot);
runtime_parforsetup(work.sweepfor, work.nproc, runtime_mheap.nspan, nil, true, sweepspan);
if(work.nproc > 1) {
@@ -2317,18 +2416,35 @@ runfinq(void* dummy __attribute__ ((unused)))
}
}
-// mark the block at v of size n as allocated.
-// If noscan is true, mark it as not needing scanning.
void
-runtime_markallocated(void *v, uintptr n, bool noscan)
+runtime_marknogc(void *v)
{
uintptr *b, obits, bits, off, shift;
- if(0)
- runtime_printf("markallocated %p+%p\n", v, n);
+ off = (uintptr*)v - (uintptr*)runtime_mheap.arena_start; // word offset
+ b = (uintptr*)runtime_mheap.arena_start - off/wordsPerBitmapWord - 1;
+ shift = off % wordsPerBitmapWord;
- if((byte*)v+n > (byte*)runtime_mheap.arena_used || (byte*)v < runtime_mheap.arena_start)
- runtime_throw("markallocated: bad pointer");
+ for(;;) {
+ obits = *b;
+ if((obits>>shift & bitMask) != bitAllocated)
+ runtime_throw("bad initial state for marknogc");
+ bits = (obits & ~(bitAllocated<<shift)) | bitBlockBoundary<<shift;
+ if(runtime_gomaxprocs == 1) {
+ *b = bits;
+ break;
+ } else {
+ // more than one goroutine is potentially running: use atomic op
+ if(runtime_casp((void**)b, (void*)obits, (void*)bits))
+ break;
+ }
+ }
+}
+
+void
+runtime_markscan(void *v)
+{
+ uintptr *b, obits, bits, off, shift;
off = (uintptr*)v - (uintptr*)runtime_mheap.arena_start; // word offset
b = (uintptr*)runtime_mheap.arena_start - off/wordsPerBitmapWord - 1;
@@ -2336,9 +2452,9 @@ runtime_markallocated(void *v, uintptr n, bool noscan)
for(;;) {
obits = *b;
- bits = (obits & ~(bitMask<<shift)) | (bitAllocated<<shift);
- if(noscan)
- bits |= bitNoScan<<shift;
+ if((obits>>shift & bitMask) != bitAllocated)
+ runtime_throw("bad initial state for markscan");
+ bits = obits | bitScan<<shift;
if(runtime_gomaxprocs == 1) {
*b = bits;
break;
@@ -2368,7 +2484,10 @@ runtime_markfreed(void *v, uintptr n)
for(;;) {
obits = *b;
- bits = (obits & ~(bitMask<<shift)) | (bitBlockBoundary<<shift);
+ // This could be a free of a gc-eligible object (bitAllocated + others) or
+ // a FlagNoGC object (bitBlockBoundary set). In either case, we revert to
+ // a simple no-scan allocated object because it is going on a free list.
+ bits = (obits & ~(bitMask<<shift)) | (bitAllocated<<shift);
if(runtime_gomaxprocs == 1) {
*b = bits;
break;
@@ -2409,12 +2528,22 @@ runtime_checkfreed(void *v, uintptr n)
void
runtime_markspan(void *v, uintptr size, uintptr n, bool leftover)
{
- uintptr *b, off, shift;
+ uintptr *b, off, shift, i;
byte *p;
if((byte*)v+size*n > (byte*)runtime_mheap.arena_used || (byte*)v < runtime_mheap.arena_start)
runtime_throw("markspan: bad pointer");
+ if(runtime_checking) {
+ // bits should be all zero at the start
+ off = (byte*)v + size - runtime_mheap.arena_start;
+ b = (uintptr*)(runtime_mheap.arena_start - off/wordsPerBitmapWord);
+ for(i = 0; i < size/PtrSize/wordsPerBitmapWord; i++) {
+ if(b[i] != 0)
+ runtime_throw("markspan: span bits not zero");
+ }
+ }
+
p = v;
if(leftover) // mark a boundary just past end of last block too
n++;
@@ -2426,7 +2555,7 @@ runtime_markspan(void *v, uintptr size, uintptr n, bool leftover)
off = (uintptr*)p - (uintptr*)runtime_mheap.arena_start; // word offset
b = (uintptr*)runtime_mheap.arena_start - off/wordsPerBitmapWord - 1;
shift = off % wordsPerBitmapWord;
- *b = (*b & ~(bitMask<<shift)) | (bitBlockBoundary<<shift);
+ *b = (*b & ~(bitMask<<shift)) | (bitAllocated<<shift);
}
}
diff --git a/libgo/runtime/mprof.goc b/libgo/runtime/mprof.goc
index 7507dfc..469ddfe 100644
--- a/libgo/runtime/mprof.goc
+++ b/libgo/runtime/mprof.goc
@@ -256,16 +256,56 @@ found:
return nil;
}
+static const char*
+typeinfoname(int32 typeinfo)
+{
+ if(typeinfo == TypeInfo_SingleObject)
+ return "single object";
+ else if(typeinfo == TypeInfo_Array)
+ return "array";
+ else if(typeinfo == TypeInfo_Chan)
+ return "channel";
+ // runtime_throw("typinfoname: unknown type info");
+ return "unknown";
+}
+
+static void
+printstackframes(Location *stk, int32 nstk)
+{
+ Location *loc;
+ int32 frame;
+
+ for(frame = 0; frame < nstk; frame++) {
+ loc = &stk[frame];
+ if (loc->function.len > 0) {
+ runtime_printf("\t#%d %p %S %S:%d\n", frame, loc->pc, loc->function, loc->filename, (int32)loc->lineno);
+ } else {
+ runtime_printf("\t#%d %p\n", frame, loc->pc);
+ }
+ }
+}
+
// Called by malloc to record a profiled block.
void
-runtime_MProf_Malloc(void *p, uintptr size)
+runtime_MProf_Malloc(void *p, uintptr size, uintptr typ)
{
- int32 nstk;
Location stk[32];
Bucket *b;
+ Type *type;
+ const char *name;
+ int32 nstk;
nstk = runtime_callers(1, stk, 32);
runtime_lock(&proflock);
+ if(runtime_debug.allocfreetrace) {
+ type = (Type*)(typ & ~3);
+ name = typeinfoname(typ & 3);
+ runtime_printf("MProf_Malloc(p=%p, size=%p, type=%p <%s", p, size, type, name);
+ if(type != nil)
+ runtime_printf(" of %S", *type->__reflection);
+ runtime_printf(">)\n");
+ printstackframes(stk, nstk);
+ }
b = stkbucket(MProf, stk, nstk, true);
b->recent_allocs++;
b->recent_alloc_bytes += size;
@@ -284,6 +324,10 @@ runtime_MProf_Free(void *p, uintptr size)
if(b != nil) {
b->recent_frees++;
b->recent_free_bytes += size;
+ if(runtime_debug.allocfreetrace) {
+ runtime_printf("MProf_Free(p=%p, size=%p)\n", p, size);
+ printstackframes(b->stk, b->nstk);
+ }
}
runtime_unlock(&proflock);
}
diff --git a/libgo/runtime/print.c b/libgo/runtime/print.c
index ba0b86a..d00b638 100644
--- a/libgo/runtime/print.c
+++ b/libgo/runtime/print.c
@@ -208,7 +208,10 @@ runtime_printfloat(double v)
n = 7; // digits printed
e = 0; // exp
s = 0; // sign
- if(v != 0) {
+ if(v == 0) {
+ if(isinf(1/v) && 1/v < 0)
+ s = 1;
+ } else {
// sign
if(v < 0) {
v = -v;
diff --git a/libgo/runtime/race.h b/libgo/runtime/race.h
index 884245c..c97e672 100644
--- a/libgo/runtime/race.h
+++ b/libgo/runtime/race.h
@@ -24,6 +24,8 @@ void runtime_racewritepc(void *addr, void *callpc, void *pc);
void runtime_racereadpc(void *addr, void *callpc, void *pc);
void runtime_racewriterangepc(void *addr, uintptr sz, void *callpc, void *pc);
void runtime_racereadrangepc(void *addr, uintptr sz, void *callpc, void *pc);
+void runtime_racereadobjectpc(void *addr, Type *t, void *callpc, void *pc);
+void runtime_racewriteobjectpc(void *addr, Type *t, void *callpc, void *pc);
void runtime_racefingo(void);
void runtime_raceacquire(void *addr);
void runtime_raceacquireg(G *gp, void *addr);
diff --git a/libgo/runtime/runtime.c b/libgo/runtime/runtime.c
index 1a7c3c7..4f9909b 100644
--- a/libgo/runtime/runtime.c
+++ b/libgo/runtime/runtime.c
@@ -282,9 +282,11 @@ static struct {
const char* name;
int32* value;
} dbgvar[] = {
+ {"allocfreetrace", &runtime_debug.allocfreetrace},
+ {"efence", &runtime_debug.efence},
{"gctrace", &runtime_debug.gctrace},
- {"schedtrace", &runtime_debug.schedtrace},
{"scheddetail", &runtime_debug.scheddetail},
+ {"schedtrace", &runtime_debug.schedtrace},
};
void
diff --git a/libgo/runtime/runtime.h b/libgo/runtime/runtime.h
index 7a57850..ef6090f 100644
--- a/libgo/runtime/runtime.h
+++ b/libgo/runtime/runtime.h
@@ -427,9 +427,11 @@ struct CgoMal
// Holds variables parsed from GODEBUG env var.
struct DebugVars
{
+ int32 allocfreetrace;
+ int32 efence;
int32 gctrace;
- int32 schedtrace;
int32 scheddetail;
+ int32 schedtrace;
};
extern bool runtime_precisestack;
@@ -741,6 +743,9 @@ void runtime_lockOSThread(void);
void runtime_unlockOSThread(void);
bool runtime_showframe(String, bool);
+Hchan* runtime_makechan_c(ChanType*, int64);
+void runtime_chansend(ChanType*, Hchan*, byte*, bool*, void*);
+void runtime_chanrecv(ChanType*, Hchan*, byte*, bool*, bool*);
void runtime_printcreatedby(G*);
uintptr runtime_memlimit(void);
diff --git a/libgo/runtime/signal_unix.c b/libgo/runtime/signal_unix.c
index ea0a58f..6c191d0 100644
--- a/libgo/runtime/signal_unix.c
+++ b/libgo/runtime/signal_unix.c
@@ -122,6 +122,14 @@ os_sigpipe(void)
}
void
+runtime_unblocksignals(void)
+{
+ sigset_t sigset_none;
+ sigemptyset(&sigset_none);
+ pthread_sigmask(SIG_SETMASK, &sigset_none, nil);
+}
+
+void
runtime_crash(void)
{
int32 i;
@@ -137,6 +145,7 @@ runtime_crash(void)
return;
#endif
+ runtime_unblocksignals();
for(i = 0; runtime_sigtab[i].sig != -1; i++)
if(runtime_sigtab[i].sig == SIGABRT)
break;
generated by cgit v1.1 at 2024-11-11 17:37:12 +0000