throw("bad sweepgen in refill")
}
mheap_.central[spc].mcentral.uncacheSpan(s)
+
+ // Count up how many slots were used and record it.
+ stats := memstats.heapStats.acquire()
+ slotsUsed := uintptr(s.allocCount) - uintptr(s.allocCountBeforeCache)
+ atomic.Xadduintptr(&stats.smallAllocCount[spc.sizeclass()], slotsUsed)
+
+ // Flush tinyAllocs.
+ if spc == tinySpanClass {
+ atomic.Xadduintptr(&stats.tinyAllocCount, c.tinyAllocs)
+ c.tinyAllocs = 0
+ }
+ memstats.heapStats.release()
+
+ // Update heapLive and flush scanAlloc.
+ gcController.update(int64(slotsUsed*s.elemsize), int64(c.scanAlloc))
+ c.scanAlloc = 0
+
+ // Clear the second allocCount just to be safe.
+ s.allocCountBeforeCache = 0
}
// Get a new cached span from the central lists.
// sweeping in the next sweep phase.
s.sweepgen = mheap_.sweepgen + 3
- // Assume all objects from this span will be allocated in the
- // mcache. If it gets uncached, we'll adjust this.
- stats := memstats.heapStats.acquire()
- atomic.Xadduintptr(&stats.smallAllocCount[spc.sizeclass()], uintptr(s.nelems)-uintptr(s.allocCount))
-
- // Flush tinyAllocs.
- if spc == tinySpanClass {
- atomic.Xadduintptr(&stats.tinyAllocCount, c.tinyAllocs)
- c.tinyAllocs = 0
- }
- memstats.heapStats.release()
-
- // Update heapLive with the same assumption.
- // While we're here, flush scanAlloc, since we have to call
- // revise anyway.
- usedBytes := uintptr(s.allocCount) * s.elemsize
- gcController.update(int64(s.npages*pageSize)-int64(usedBytes), int64(c.scanAlloc))
- c.scanAlloc = 0
+ // Store the current alloc count for accounting later.
+ s.allocCountBeforeCache = s.allocCount
c.alloc[spc] = s
}
scanAlloc := int64(c.scanAlloc)
c.scanAlloc = 0
- sg := mheap_.sweepgen
- dHeapLive := int64(0)
for i := range c.alloc {
s := c.alloc[i]
if s != &emptymspan {
- // Adjust nsmallalloc in case the span wasn't fully allocated.
- n := uintptr(s.nelems) - uintptr(s.allocCount)
+ // Adjust smallAllocCount for whatever was allocated.
stats := memstats.heapStats.acquire()
- atomic.Xadduintptr(&stats.smallAllocCount[spanClass(i).sizeclass()], -n)
+ slotsUsed := uintptr(s.allocCount) - uintptr(s.allocCountBeforeCache)
+ atomic.Xadduintptr(&stats.smallAllocCount[spanClass(i).sizeclass()], slotsUsed)
memstats.heapStats.release()
- if s.sweepgen != sg+1 {
- // refill conservatively counted unallocated slots in gcController.heapLive.
- // Undo this.
- //
- // If this span was cached before sweep, then
- // gcController.heapLive was totally recomputed since
- // caching this span, so we don't do this for
- // stale spans.
- dHeapLive -= int64(n) * int64(s.elemsize)
- }
+ s.allocCountBeforeCache = 0
+
// Release the span to the mcentral.
mheap_.central[i].mcentral.uncacheSpan(s)
c.alloc[i] = &emptymspan
c.tinyAllocs = 0
memstats.heapStats.release()
- // Updated heapScan and heapLive.
- gcController.update(dHeapLive, scanAlloc)
+ // Updated heapScan.
+ gcController.update(0, scanAlloc)
}
// prepareForSweep flushes c if the system has entered a new sweep phase
"runtime/metrics"
"sort"
"strings"
+ "sync"
"testing"
"time"
"unsafe"
b.ReportMetric(float64(latencies[len(latencies)*90/100]), "p90-ns")
b.ReportMetric(float64(latencies[len(latencies)*99/100]), "p99-ns")
}
+
+var readMetricsSink [1024]interface{}
+
+func TestReadMetricsCumulative(t *testing.T) {
+ // Set up the set of metrics marked cumulative.
+ descs := metrics.All()
+ var samples [2][]metrics.Sample
+ samples[0] = make([]metrics.Sample, len(descs))
+ samples[1] = make([]metrics.Sample, len(descs))
+ total := 0
+ for i := range samples[0] {
+ if !descs[i].Cumulative {
+ continue
+ }
+ samples[0][total].Name = descs[i].Name
+ total++
+ }
+ samples[0] = samples[0][:total]
+ samples[1] = samples[1][:total]
+ copy(samples[1], samples[0])
+
+ // Start some noise in the background.
+ var wg sync.WaitGroup
+ wg.Add(1)
+ done := make(chan struct{})
+ go func() {
+ defer wg.Done()
+ for {
+ // Add more things here that could influence metrics.
+ for i := 0; i < len(readMetricsSink); i++ {
+ readMetricsSink[i] = make([]byte, 1024)
+ select {
+ case <-done:
+ return
+ default:
+ }
+ }
+ runtime.GC()
+ }
+ }()
+
+ sum := func(us []uint64) uint64 {
+ total := uint64(0)
+ for _, u := range us {
+ total += u
+ }
+ return total
+ }
+
+ // Populate the first generation.
+ metrics.Read(samples[0])
+
+ // Check to make sure that these metrics only grow monotonically.
+ for gen := 1; gen < 10; gen++ {
+ metrics.Read(samples[gen%2])
+ for i := range samples[gen%2] {
+ name := samples[gen%2][i].Name
+ vNew, vOld := samples[gen%2][i].Value, samples[1-(gen%2)][i].Value
+
+ switch vNew.Kind() {
+ case metrics.KindUint64:
+ new := vNew.Uint64()
+ old := vOld.Uint64()
+ if new < old {
+ t.Errorf("%s decreased: %d < %d", name, new, old)
+ }
+ case metrics.KindFloat64:
+ new := vNew.Float64()
+ old := vOld.Float64()
+ if new < old {
+ t.Errorf("%s decreased: %f < %f", name, new, old)
+ }
+ case metrics.KindFloat64Histogram:
+ new := sum(vNew.Float64Histogram().Counts)
+ old := sum(vOld.Float64Histogram().Counts)
+ if new < old {
+ t.Errorf("%s counts decreased: %d < %d", name, new, old)
+ }
+ }
+ }
+ }
+ close(done)
+
+ wg.Wait()
+}
// if sweepgen == h->sweepgen + 3, the span was swept and then cached and is still cached
// h->sweepgen is incremented by 2 after every GC
- sweepgen uint32
- divMul uint32 // for divide by elemsize
- allocCount uint16 // number of allocated objects
- spanclass spanClass // size class and noscan (uint8)
- state mSpanStateBox // mSpanInUse etc; accessed atomically (get/set methods)
- needzero uint8 // needs to be zeroed before allocation
- elemsize uintptr // computed from sizeclass or from npages
- limit uintptr // end of data in span
- speciallock mutex // guards specials list
- specials *special // linked list of special records sorted by offset.
+ sweepgen uint32
+ divMul uint32 // for divide by elemsize
+ allocCount uint16 // number of allocated objects
+ spanclass spanClass // size class and noscan (uint8)
+ state mSpanStateBox // mSpanInUse etc; accessed atomically (get/set methods)
+ needzero uint8 // needs to be zeroed before allocation
+ allocCountBeforeCache uint16 // a copy of allocCount that is stored just before this span is cached
+ elemsize uintptr // computed from sizeclass or from npages
+ limit uintptr // end of data in span
+ speciallock mutex // guards specials list
+ specials *special // linked list of special records sorted by offset.
}
func (s *mspan) base() uintptr {