runtime: fix and speedup malloc stats

Currently per-sizeclass stats are lost for destroyed MCache's. This patch fixes this.
Also, only update mstats.heap_alloc on heap operations, because that's the only
stat that needs to be promptly updated. Everything else needs to be up-to-date only in ReadMemStats().

R=golang-dev, remyoudompheng, dave, iant
CC=golang-dev
https://golang.org/cl/9207047

diff --git a/src/pkg/runtime/malloc.goc b/src/pkg/runtime/malloc.goc
index 5326551fee4f99e39d6663bc37632d714ab8771a..7e691fe9c8b9b21164fb0f92ed392ef8bab05cc1 100644 (file)
--- a/src/pkg/runtime/malloc.goc
+++ b/src/pkg/runtime/malloc.goc
@@ -278,6 +278,8 @@ runtime·freemcache(MCache *c)
 void
 runtime·purgecachedstats(MCache *c)
 {
+       int32 i;
+
        // Protected by either heap or GC lock.
        mstats.heap_alloc += c->local_cachealloc;
        c->local_cachealloc = 0;
@@ -293,6 +295,12 @@ runtime·purgecachedstats(MCache *c)
        c->local_alloc= 0;
        mstats.total_alloc += c->local_total_alloc;
        c->local_total_alloc= 0;
+       for(i=0; i<nelem(c->local_by_size); i++) {
+               mstats.by_size[i].nmalloc += c->local_by_size[i].nmalloc;
+               c->local_by_size[i].nmalloc = 0;
+               mstats.by_size[i].nfree += c->local_by_size[i].nfree;
+               c->local_by_size[i].nfree = 0;
+       }
 }
 
 uintptr runtime·sizeof_C_MStats = sizeof(MStats);

diff --git a/src/pkg/runtime/malloc.h b/src/pkg/runtime/malloc.h
index cbcc09cdb73f1c829e75c4837a77c212d76cdb05..0d31326a22b7439d60e8e5c6e1a4895123e33523 100644 (file)
--- a/src/pkg/runtime/malloc.h
+++ b/src/pkg/runtime/malloc.h
@@ -285,15 +285,18 @@ struct MCacheList
 
 struct MCache
 {
-       MCacheList list[NumSizeClasses];
+       // The following members are accessed on every malloc,
+       // so they are grouped here for better caching.
+       int32 next_sample;      // trigger heap sample after allocating this many bytes
        intptr local_cachealloc;        // bytes allocated (or freed) from cache since last lock of heap
+       // The rest is not accessed on every malloc.
+       MCacheList list[NumSizeClasses];
        intptr local_objects;   // objects allocated (or freed) from cache since last lock of heap
        intptr local_alloc;     // bytes allocated (or freed) since last lock of heap
        uintptr local_total_alloc;      // bytes allocated (even if freed) since last lock of heap
        uintptr local_nmalloc;  // number of mallocs since last lock of heap
        uintptr local_nfree;    // number of frees since last lock of heap
        uintptr local_nlookup;  // number of pointer lookups since last lock of heap
-       int32 next_sample;      // trigger heap sample after allocating this many bytes
        // Statistics about allocation size classes since last lock of heap
        struct {
                uintptr nmalloc;

diff --git a/src/pkg/runtime/mgc0.c b/src/pkg/runtime/mgc0.c
index 11248772536e584c3174fce8f1a87e081448f2b6..28cc4353efa3826e6edc5da883fb48581afeba74 100644 (file)
--- a/src/pkg/runtime/mgc0.c
+++ b/src/pkg/runtime/mgc0.c
@@ -1863,12 +1863,6 @@ cachestats(GCStats *stats)
                if(c==nil)
                        continue;
                runtime·purgecachedstats(c);
-               for(i=0; i<nelem(c->local_by_size); i++) {
-                       mstats.by_size[i].nmalloc += c->local_by_size[i].nmalloc;
-                       c->local_by_size[i].nmalloc = 0;
-                       mstats.by_size[i].nfree += c->local_by_size[i].nfree;
-                       c->local_by_size[i].nfree = 0;
-               }
        }
        mstats.stacks_inuse = stacks_inuse;
 }

diff --git a/src/pkg/runtime/mheap.c b/src/pkg/runtime/mheap.c
index f4fbbee7a4f39b9e6d41b7ab34aa2241665b6416..e839e9fc5ac753cf2614b37ed23eddca8aabe5cf 100644 (file)
--- a/src/pkg/runtime/mheap.c
+++ b/src/pkg/runtime/mheap.c
@@ -73,7 +73,8 @@ runtime·MHeap_Alloc(MHeap *h, uintptr npage, int32 sizeclass, int32 acct, int32
        MSpan *s;
 
        runtime·lock(h);
-       runtime·purgecachedstats(m->mcache);
+       mstats.heap_alloc += m->mcache->local_cachealloc;
+       m->mcache->local_cachealloc = 0;
        s = MHeap_AllocLocked(h, npage, sizeclass);
        if(s != nil) {
                mstats.heap_inuse += npage<<PageShift;
@@ -296,7 +297,8 @@ void
 runtime·MHeap_Free(MHeap *h, MSpan *s, int32 acct)
 {
        runtime·lock(h);
-       runtime·purgecachedstats(m->mcache);
+       mstats.heap_alloc += m->mcache->local_cachealloc;
+       m->mcache->local_cachealloc = 0;
        mstats.heap_inuse -= s->npages<<PageShift;
        if(acct) {
                mstats.heap_alloc -= s->npages<<PageShift;