mstats.alloc += size;
mstats.total_alloc += size;
mstats.by_size[sizeclass].nmalloc++;
+
+ if(!mlookup(v, nil, nil, nil, &ref)) {
+ printf("malloc %D; mlookup failed\n", (uint64)size);
+ throw("malloc mlookup");
+ }
+ *ref = RefNone | refflag;
} else {
// TODO(rsc): Report tracebacks for very large allocations.
mstats.alloc += npages<<PageShift;
mstats.total_alloc += npages<<PageShift;
v = (void*)(s->start << PageShift);
- }
- // setup for mark sweep
- if(!mlookup(v, nil, nil, &ref)) {
- printf("malloc %D; mlookup failed\n", (uint64)size);
- throw("malloc mlookup");
+ // setup for mark sweep
+ s->gcref0 = RefNone | refflag;
}
- *ref = RefNone | refflag;
m->mallocing = 0;
free(void *v)
{
int32 sizeclass, size;
- uintptr page, tmp;
MSpan *s;
MCache *c;
uint32 *ref;
throw("malloc/free - deadlock");
m->mallocing = 1;
- if(!mlookup(v, nil, nil, &ref)) {
+ if(!mlookup(v, nil, nil, &s, &ref)) {
printf("free %p: not an allocated block\n", v);
throw("free mlookup");
}
*ref = RefFree;
// Find size class for v.
- page = (uintptr)v >> PageShift;
- sizeclass = MHeapMapCache_GET(&mheap.mapcache, page, tmp);
+ sizeclass = s->sizeclass;
if(sizeclass == 0) {
- // Missed in cache.
- s = MHeap_Lookup(&mheap, page);
- if(s == nil)
- throw("free - invalid pointer");
- sizeclass = s->sizeclass;
- if(sizeclass == 0) {
- // Large object.
- mstats.alloc -= s->npages<<PageShift;
- runtime_memclr(v, s->npages<<PageShift);
- MHeap_Free(&mheap, s);
- goto out;
- }
- MHeapMapCache_SET(&mheap.mapcache, page, sizeclass);
+ // Large object.
+ mstats.alloc -= s->npages<<PageShift;
+ runtime_memclr(v, s->npages<<PageShift);
+ MHeap_Free(&mheap, s);
+ } else {
+ // Small object.
+ c = m->mcache;
+ size = class_to_size[sizeclass];
+ if(size > sizeof(uintptr))
+ ((uintptr*)v)[1] = 1; // mark as "needs to be zeroed"
+ mstats.alloc -= size;
+ mstats.by_size[sizeclass].nfree++;
+ MCache_Free(c, v, sizeclass, size);
}
-
- // Small object.
- c = m->mcache;
- size = class_to_size[sizeclass];
- if(size > sizeof(uintptr))
- ((uintptr*)v)[1] = 1; // mark as "needs to be zeroed"
- mstats.alloc -= size;
- mstats.by_size[sizeclass].nfree++;
- MCache_Free(c, v, sizeclass, size);
-
-out:
m->mallocing = 0;
}
int32
-mlookup(void *v, byte **base, uintptr *size, uint32 **ref)
+mlookup(void *v, byte **base, uintptr *size, MSpan **sp, uint32 **ref)
{
uintptr n, nobj, i;
byte *p;
mstats.nlookup++;
s = MHeap_LookupMaybe(&mheap, (uintptr)v>>PageShift);
+ if(sp)
+ *sp = s;
if(s == nil) {
if(base)
*base = nil;
return v;
}
v = mallocgc(n, 0, 0, 0);
- if(!mlookup(v, nil, nil, &ref))
+ if(!mlookup(v, nil, nil, nil, &ref))
throw("stackalloc mlookup");
*ref = RefStack;
return v;
}
func Lookup(p *byte) (base *byte, size uintptr) {
- mlookup(p, &base, &size, nil);
+ mlookup(p, &base, &size, nil, nil);
}
func GC() {
printf("runtime.SetFinalizer: first argument is %S, not pointer\n", *obj.type->string);
goto throw;
}
- if(!mlookup(obj.data, &base, &size, nil) || obj.data != base) {
+ if(!mlookup(obj.data, &base, &size, nil, nil) || obj.data != base) {
printf("runtime.SetFinalizer: pointer not at beginning of allocated block\n");
goto throw;
}
// MHeap: the malloc heap, managed at page (4096-byte) granularity.
// MSpan: a run of pages managed by the MHeap.
// MHeapMap: a mapping from page IDs to MSpans.
-// MHeapMapCache: a small cache of MHeapMap mapping page IDs
-// to size classes for pages used for small objects.
// MCentral: a shared free list for a given size class.
// MCache: a per-thread (in Go, per-M) cache for small objects.
// MStats: allocation statistics.
typedef struct MCentral MCentral;
typedef struct MHeap MHeap;
typedef struct MHeapMap MHeapMap;
-typedef struct MHeapMapCache MHeapMapCache;
typedef struct MSpan MSpan;
typedef struct MStats MStats;
typedef struct MLink MLink;
// span lookup
MHeapMap map;
- MHeapMapCache mapcache;
// range of addresses we might see in the heap
byte *min;
void MGetSizeClassInfo(int32 sizeclass, int32 *size, int32 *npages, int32 *nobj);
void* mallocgc(uintptr size, uint32 flag, int32 dogc, int32 zeroed);
-int32 mlookup(void *v, byte **base, uintptr *size, uint32 **ref);
+int32 mlookup(void *v, byte **base, uintptr *size, MSpan **s, uint32 **ref);
void gc(int32 force);
void* SysAlloc(uintptr);
uint32 *ref;
byte *base;
- if(!mlookup(p, &base, nil, &ref) || p != base)
+ if(!mlookup(p, &base, nil, nil, &ref) || p != base)
throw("addfinalizer on invalid pointer");
if(f == nil) {
if(*ref & RefHasFinalizer) {
obj = vp[i];
if(obj == nil || (byte*)obj < mheap.min || (byte*)obj >= mheap.max)
continue;
- if(mlookup(obj, &obj, &size, &refp)) {
+ if(mlookup(obj, &obj, &size, nil, &refp)) {
ref = *refp;
switch(ref & ~(RefNoPointers|RefHasFinalizer)) {
case RefFinalize:
MHeap_FreeLocked(h, t);
}
- // If span is being used for small objects, cache size class.
- // No matter what, cache span info, because gc needs to be
+ // Record span info, because gc needs to be
// able to map interior pointer to containing span.
s->sizeclass = sizeclass;
for(n=0; n<npage; n++)
MHeapMap_Set(&h->map, s->start+n, s);
- if(sizeclass == 0) {
- uintptr tmp;
-
- // If there are entries for this span, invalidate them,
- // but don't blow out cache entries about other spans.
- for(n=0; n<npage; n++)
- if(MHeapMapCache_GET(&h->mapcache, s->start+n, tmp) != 0)
- MHeapMapCache_SET(&h->mapcache, s->start+n, 0);
- } else {
- // Save cache entries for this span.
- // If there's a size class, there aren't that many pages.
- for(n=0; n<npage; n++)
- MHeapMapCache_SET(&h->mapcache, s->start+n, sizeclass);
- }
-
return s;
}
void MHeapMap_Set(MHeapMap *m, PageID k, MSpan *v);
-// Much of the time, free(v) needs to know only the size class for v,
-// not which span it came from. The MHeapMap finds the size class
-// by looking up the span.
-//
-// An MHeapMapCache is a simple direct-mapped cache translating
-// page numbers to size classes. It avoids the expensive MHeapMap
-// lookup for hot pages.
-//
-// The cache entries are 32 bits, with the page number in the low part
-// and the value at the top.
-//
-// NOTE(rsc): On a machine with 32-bit addresses (= 20-bit page numbers),
-// we can use a 16-bit cache entry by not storing the redundant 12 bits
-// of the key that are used as the entry index. For now, keep it simple.
-enum
-{
- MHeapMapCache_HashBits = 12
-};
-
-struct MHeapMapCache
-{
- uint32 array[1<<MHeapMapCache_HashBits];
-};
-
-// All macros for speed (sorry).
-#define HMASK ((1<<MHeapMapCache_HashBits)-1)
-#define KBITS MHeapMap_TotalBits
-#define KMASK ((1LL<<KBITS)-1)
-
-#define MHeapMapCache_SET(cache, key, value) \
- ((cache)->array[(key) & HMASK] = (key) | ((uintptr)(value) << KBITS))
-
-#define MHeapMapCache_GET(cache, key, tmp) \
- (tmp = (cache)->array[(key) & HMASK], \
- (tmp & KMASK) == (key) ? (tmp >> KBITS) : 0)
void MHeapMap_Set(MHeapMap *m, PageID k, MSpan *v);
-// Much of the time, free(v) needs to know only the size class for v,
-// not which span it came from. The MHeapMap finds the size class
-// by looking up the span.
-//
-// An MHeapMapCache is a simple direct-mapped cache translating
-// page numbers to size classes. It avoids the expensive MHeapMap
-// lookup for hot pages.
-//
-// The cache entries are 64 bits, with the page number in the low part
-// and the value at the top.
-//
-// NOTE(rsc): On a machine with 32-bit addresses (= 20-bit page numbers),
-// we can use a 16-bit cache entry by not storing the redundant 12 bits
-// of the key that are used as the entry index. Here in 64-bit land,
-// that trick won't work unless the hash table has 2^28 entries.
-enum
-{
- MHeapMapCache_HashBits = 12
-};
-
-struct MHeapMapCache
-{
- uintptr array[1<<MHeapMapCache_HashBits];
-};
-
-// All macros for speed (sorry).
-#define HMASK ((1<<MHeapMapCache_HashBits)-1)
-#define KBITS MHeapMap_TotalBits
-#define KMASK ((1LL<<KBITS)-1)
-
-#define MHeapMapCache_SET(cache, key, value) \
- ((cache)->array[(key) & HMASK] = (key) | ((uintptr)(value) << KBITS))
-
-#define MHeapMapCache_GET(cache, key, tmp) \
- (tmp = (cache)->array[(key) & HMASK], \
- (tmp & KMASK) == (key) ? (tmp >> KBITS) : 0)