runtime: add per-M caches for MemStats

Avoid touching centralized state during
memory manager opreations.

R=rsc
CC=golang-dev
https://golang.org/cl/4766042

diff --git a/src/pkg/runtime/malloc.goc b/src/pkg/runtime/malloc.goc
index 4274e3e162cda6dd12f3b6d9b9989ea97681ce80..b9fe36db6d814de4bf8fbb07f1c9d76221f9f799 100644 (file)
--- a/src/pkg/runtime/malloc.goc
+++ b/src/pkg/runtime/malloc.goc
@@ -38,18 +38,18 @@ runtime·mallocgc(uintptr size, uint32 flag, int32 dogc, int32 zeroed)
        if(size == 0)
                size = 1;
 
-       mstats.nmalloc++;
+       c = m->mcache;
+       c->local_nmalloc++;
        if(size <= MaxSmallSize) {
                // Allocate from mcache free lists.
                sizeclass = runtime·SizeToClass(size);
                size = runtime·class_to_size[sizeclass];
-               c = m->mcache;
                v = runtime·MCache_Alloc(c, sizeclass, size, zeroed);
                if(v == nil)
                        runtime·throw("out of memory");
-               mstats.alloc += size;
-               mstats.total_alloc += size;
-               mstats.by_size[sizeclass].nmalloc++;
+               c->local_alloc += size;
+               c->local_total_alloc += size;
+               c->local_by_size[sizeclass].nmalloc++;
        } else {
                // TODO(rsc): Report tracebacks for very large allocations.
 
@@ -61,8 +61,8 @@ runtime·mallocgc(uintptr size, uint32 flag, int32 dogc, int32 zeroed)
                if(s == nil)
                        runtime·throw("out of memory");
                size = npages<<PageShift;
-               mstats.alloc += size;
-               mstats.total_alloc += size;
+               c->local_alloc += size;
+               c->local_total_alloc += size;
                v = (void*)(s->start << PageShift);
 
                // setup for mark sweep
@@ -128,6 +128,7 @@ runtime·free(void *v)
 
        // Find size class for v.
        sizeclass = s->sizeclass;
+       c = m->mcache;
        if(sizeclass == 0) {
                // Large object.
                size = s->npages<<PageShift;
@@ -139,7 +140,6 @@ runtime·free(void *v)
                runtime·MHeap_Free(&runtime·mheap, s, 1);
        } else {
                // Small object.
-               c = m->mcache;
                size = runtime·class_to_size[sizeclass];
                if(size > sizeof(uintptr))
                        ((uintptr*)v)[1] = 1;   // mark as "needs to be zeroed"
@@ -147,10 +147,10 @@ runtime·free(void *v)
                // it might coalesce v and other blocks into a bigger span
                // and change the bitmap further.
                runtime·markfreed(v, size);
-               mstats.by_size[sizeclass].nfree++;
+               c->local_by_size[sizeclass].nfree++;
                runtime·MCache_Free(c, v, sizeclass, size);
        }
-       mstats.alloc -= size;
+       c->local_alloc -= size;
        if(prof)
                runtime·MProf_Free(v, size);
        m->mallocing = 0;
@@ -163,7 +163,7 @@ runtime·mlookup(void *v, byte **base, uintptr *size, MSpan **sp)
        byte *p;
        MSpan *s;
 
-       mstats.nlookup++;
+       m->mcache->local_nlookup++;
        s = runtime·MHeap_LookupMaybe(&runtime·mheap, v);
        if(sp)
                *sp = s;
@@ -192,9 +192,10 @@ runtime·mlookup(void *v, byte **base, uintptr *size, MSpan **sp)
        }
 
        n = runtime·class_to_size[s->sizeclass];
-       i = ((byte*)v - p)/n;
-       if(base)
+       if(base) {
+               i = ((byte*)v - p)/n;
                *base = p + i*n;
+       }
        if(size)
                *size = n;
 
@@ -214,6 +215,29 @@ runtime·allocmcache(void)
        return c;
 }
 
+void
+runtime·purgecachedstats(M* m)
+{
+       MCache *c;
+
+       // Protected by either heap or GC lock.
+       c = m->mcache;
+       mstats.heap_alloc += c->local_cachealloc;
+       c->local_cachealloc = 0;
+       mstats.heap_objects += c->local_objects;
+       c->local_objects = 0;
+       mstats.nmalloc += c->local_nmalloc;
+       c->local_nmalloc = 0;
+       mstats.nfree += c->local_nfree;
+       c->local_nfree = 0;
+       mstats.nlookup += c->local_nlookup;
+       c->local_nlookup = 0;
+       mstats.alloc += c->local_alloc;
+       c->local_alloc= 0;
+       mstats.total_alloc += c->local_total_alloc;
+       c->local_total_alloc= 0;
+}
+
 uintptr runtime·sizeof_C_MStats = sizeof(MStats);
 
 #define MaxArena32 (2U<<30)
@@ -361,9 +385,6 @@ func new(n uint32) (ret *uint8) {
 void*
 runtime·stackalloc(uint32 n)
 {
-       void *v;
-       uintptr sys0;
-
        // Stackalloc must be called on scheduler stack, so that we
        // never try to grow the stack during the code that stackalloc runs.
        // Doing so would cause a deadlock (issue 1547).
@@ -382,11 +403,7 @@ runtime·stackalloc(uint32 n)
                        runtime·printf("stackalloc: in malloc, size=%d want %d", FixedStack, n);
                        runtime·throw("stackalloc");
                }
-               sys0 = m->stackalloc->sys;
-               v = runtime·FixAlloc_Alloc(m->stackalloc);
-               mstats.stacks_inuse += FixedStack;
-               mstats.stacks_sys += m->stackalloc->sys - sys0;
-               return v;
+               return runtime·FixAlloc_Alloc(m->stackalloc);
        }
        return runtime·mallocgc(n, FlagNoProfiling|FlagNoGC, 0, 0);
 }
@@ -394,13 +411,8 @@ runtime·stackalloc(uint32 n)
 void
 runtime·stackfree(void *v, uintptr n)
 {
-       uintptr sys0;
-
        if(m->mallocing || m->gcing || n == FixedStack) {
-               sys0 = m->stackalloc->sys;
                runtime·FixAlloc_Free(m->stackalloc, v);
-               mstats.stacks_inuse -= FixedStack;
-               mstats.stacks_sys += m->stackalloc->sys - sys0;
                return;
        }
        runtime·free(v);

diff --git a/src/pkg/runtime/malloc.h b/src/pkg/runtime/malloc.h
index d8d2111cf7b762b8adb70a2c4a8daa83bc2c8fd4..2865317ef65cfd9d0c3018a26311cc62cee28630 100644 (file)
--- a/src/pkg/runtime/malloc.h
+++ b/src/pkg/runtime/malloc.h
@@ -185,10 +185,10 @@ void      runtime·FixAlloc_Free(FixAlloc *f, void *p);
 // Shared with Go: if you edit this structure, also edit extern.go.
 struct MStats
 {
-       // General statistics.  No locking; approximate.
+       // General statistics.
        uint64  alloc;          // bytes allocated and still in use
        uint64  total_alloc;    // bytes allocated (even if freed)
-       uint64  sys;            // bytes obtained from system (should be sum of xxx_sys below)
+       uint64  sys;            // bytes obtained from system (should be sum of xxx_sys below, no locking, approximate)
        uint64  nlookup;        // number of pointer lookups
        uint64  nmalloc;        // number of mallocs
        uint64  nfree;  // number of frees
@@ -221,7 +221,6 @@ struct MStats
        bool    debuggc;
        
        // Statistics about allocation size classes.
-       // No locking; approximate.
        struct {
                uint32 size;
                uint64 nmalloc;
@@ -267,9 +266,20 @@ struct MCache
 {
        MCacheList list[NumSizeClasses];
        uint64 size;
-       int64 local_alloc;      // bytes allocated (or freed) since last lock of heap
-       int64 local_objects;    // objects allocated (or freed) since last lock of heap
+       int64 local_cachealloc; // bytes allocated (or freed) from cache since last lock of heap
+       int64 local_objects;    // objects allocated (or freed) from cache since last lock of heap
+       int64 local_alloc;      // bytes allocated and still in use since last lock of heap
+       int64 local_total_alloc;        // bytes allocated (even if freed) since last lock of heap
+       int64 local_nmalloc;    // number of mallocs since last lock of heap
+       int64 local_nfree;      // number of frees since last lock of heap
+       int64 local_nlookup;    // number of pointer lookups since last lock of heap
        int32 next_sample;      // trigger heap sample after allocating this many bytes
+       // Statistics about allocation size classes since last lock of heap
+       struct {
+               int64 nmalloc;
+               int64 nfree;
+       } local_by_size[NumSizeClasses];
+       
 };
 
 void*  runtime·MCache_Alloc(MCache *c, int32 sizeclass, uintptr size, int32 zeroed);
@@ -378,6 +388,7 @@ void        runtime·markspan(void *v, uintptr size, uintptr n, bool leftover);
 void   runtime·unmarkspan(void *v, uintptr size);
 bool   runtime·blockspecial(void*);
 void   runtime·setblockspecial(void*);
+void   runtime·purgecachedstats(M*);
 
 enum
 {

diff --git a/src/pkg/runtime/mcache.c b/src/pkg/runtime/mcache.c
index e406211862e3fe8bb86ca2cda389e05de3390d77..711e938fc56b3994f747a54f35af1857ef800406 100644 (file)
--- a/src/pkg/runtime/mcache.c
+++ b/src/pkg/runtime/mcache.c
@@ -48,7 +48,7 @@ runtime·MCache_Alloc(MCache *c, int32 sizeclass, uintptr size, int32 zeroed)
                        v->next = nil;
                }
        }
-       c->local_alloc += size;
+       c->local_cachealloc += size;
        c->local_objects++;
        return v;
 }
@@ -90,7 +90,7 @@ runtime·MCache_Free(MCache *c, void *v, int32 sizeclass, uintptr size)
        l->list = p;
        l->nlist++;
        c->size += size;
-       c->local_alloc -= size;
+       c->local_cachealloc -= size;
        c->local_objects--;
 
        if(l->nlist >= MaxMCacheListLen) {

diff --git a/src/pkg/runtime/mgc0.c b/src/pkg/runtime/mgc0.c
index ac6a1fa40deb6dc97dd7094dc4236ce6647a94f1..bc373d890904eb45cd15e68573adabefa7c84540 100644 (file)
--- a/src/pkg/runtime/mgc0.c
+++ b/src/pkg/runtime/mgc0.c
@@ -484,6 +484,7 @@ sweep(void)
                        // Mark freed; restore block boundary bit.
                        *bitp = (*bitp & ~(bitMask<<shift)) | (bitBlockBoundary<<shift);
 
+                       c = m->mcache;
                        if(s->sizeclass == 0) {
                                // Free large span.
                                runtime·unmarkspan(p, 1<<PageShift);
@@ -491,14 +492,13 @@ sweep(void)
                                runtime·MHeap_Free(&runtime·mheap, s, 1);
                        } else {
                                // Free small object.
-                               c = m->mcache;
                                if(size > sizeof(uintptr))
                                        ((uintptr*)p)[1] = 1;   // mark as "needs to be zeroed"
-                               mstats.by_size[s->sizeclass].nfree++;
+                               c->local_by_size[s->sizeclass].nfree++;
                                runtime·MCache_Free(c, p, s->sizeclass, size);
                        }
-                       mstats.alloc -= size;
-                       mstats.nfree++;
+                       c->local_alloc -= size;
+                       c->local_nfree++;
                }
        }
 }
@@ -533,14 +533,26 @@ cachestats(void)
 {
        M *m;
        MCache *c;
+       int32 i;
+       uint64 stacks_inuse;
+       uint64 stacks_sys;
 
+       stacks_inuse = 0;
+       stacks_sys = 0;
        for(m=runtime·allm; m; m=m->alllink) {
+               runtime·purgecachedstats(m);
+               stacks_inuse += m->stackalloc->inuse;
+               stacks_sys += m->stackalloc->sys;
                c = m->mcache;
-               mstats.heap_alloc += c->local_alloc;
-               c->local_alloc = 0;
-               mstats.heap_objects += c->local_objects;
-               c->local_objects = 0;
+               for(i=0; i<nelem(c->local_by_size); i++) {
+                       mstats.by_size[i].nmalloc += c->local_by_size[i].nmalloc;
+                       c->local_by_size[i].nmalloc = 0;
+                       mstats.by_size[i].nfree += c->local_by_size[i].nfree;
+                       c->local_by_size[i].nfree = 0;
+               }
        }
+       mstats.stacks_inuse = stacks_inuse;
+       mstats.stacks_sys = stacks_sys;
 }
 
 void
@@ -603,6 +615,7 @@ runtime·gc(int32 force)
        sweep();
        t2 = runtime·nanotime();
        stealcache();
+       cachestats();
 
        mstats.next_gc = mstats.heap_alloc+mstats.heap_alloc*gcpercent/100;
        m->gcing = 0;

diff --git a/src/pkg/runtime/mheap.c b/src/pkg/runtime/mheap.c
index dde31ce3457f2e887fe925fffc842608458dd607..37d50568154053eb2530a6267dee2f01f8930b61 100644 (file)
--- a/src/pkg/runtime/mheap.c
+++ b/src/pkg/runtime/mheap.c
@@ -57,10 +57,7 @@ runtime·MHeap_Alloc(MHeap *h, uintptr npage, int32 sizeclass, int32 acct)
        MSpan *s;
 
        runtime·lock(h);
-       mstats.heap_alloc += m->mcache->local_alloc;
-       m->mcache->local_alloc = 0;
-       mstats.heap_objects += m->mcache->local_objects;
-       m->mcache->local_objects = 0;
+       runtime·purgecachedstats(m);
        s = MHeap_AllocLocked(h, npage, sizeclass);
        if(s != nil) {
                mstats.heap_inuse += npage<<PageShift;
@@ -258,10 +255,7 @@ void
 runtime·MHeap_Free(MHeap *h, MSpan *s, int32 acct)
 {
        runtime·lock(h);
-       mstats.heap_alloc += m->mcache->local_alloc;
-       m->mcache->local_alloc = 0;
-       mstats.heap_objects += m->mcache->local_objects;
-       m->mcache->local_objects = 0;
+       runtime·purgecachedstats(m);
        mstats.heap_inuse -= s->npages<<PageShift;
        if(acct) {
                mstats.heap_alloc -= s->npages<<PageShift;

diff --git a/test/mallocrep.go b/test/mallocrep.go
index 762f3754f5fbec2c07030504435b73f33ff54ee5..43233b7b74b042234704c862bc4d2afbc5bb19aa 100644 (file)
--- a/test/mallocrep.go
+++ b/test/mallocrep.go
@@ -47,6 +47,7 @@ func main() {
                        b := runtime.Alloc(uintptr(j))
                        during := runtime.MemStats.Alloc
                        runtime.Free(b)
+                       runtime.GC()
                        if a := runtime.MemStats.Alloc; a != 0 {
                                println("allocated ", j, ": wrong stats: during=", during, " after=", a, " (want 0)")
                                panic("fail")

diff --git a/test/mallocrep1.go b/test/mallocrep1.go
index eb67bed86babd26ed32a3eafb011cab54353bcf5..079ae94226084935c3216bfb3f7368795005f886 100644 (file)
--- a/test/mallocrep1.go
+++ b/test/mallocrep1.go
@@ -60,6 +60,7 @@ func AllocAndFree(size, count int) {
                fmt.Printf("size=%d count=%d stats=%+v\n", size, count, *stats)
        }
        n3 := stats.Alloc
+       runtime.GC()
        for j := 0; j < count; j++ {
                i := j
                if *reverse {
@@ -72,6 +73,7 @@ func AllocAndFree(size, count int) {
                        panic("fail")
                }
                runtime.Free(b[i])
+               runtime.GC()
                if stats.Alloc != uint64(alloc-n) {
                        println("free alloc got", stats.Alloc, "expected", alloc-n, "after free of", n)
                        panic("fail")