From: Dmitriy Vyukov Date: Thu, 16 Oct 2014 18:11:26 +0000 (+0400) Subject: runtime: fix memory profiler X-Git-Tag: go1.4beta1~105 X-Git-Url: http://www.git.cypherpunks.su/?a=commitdiff_plain;h=8f47c837fd4f550c5b3f25f3725c7101c281e363;p=gostls13.git runtime: fix memory profiler There are 3 issues: 1. Skip argument of callers is off by 3, so that all allocations are deep inside of memory profiler. 2. Memory profiling statistics are not updated after runtime.GC. 3. Testing package does not update memory profiling statistics before capturing the profile. Also add an end-to-end test. Fixes #8867. LGTM=rsc R=rsc CC=golang-codereviews https://golang.org/cl/148710043 --- diff --git a/src/runtime/mgc0.c b/src/runtime/mgc0.c index 05cabe7085..2ff64aaa30 100644 --- a/src/runtime/mgc0.c +++ b/src/runtime/mgc0.c @@ -1507,7 +1507,6 @@ gc(struct gc_args *args) runtime·sweep.spanidx = 0; runtime·unlock(&runtime·mheap.lock); - // Temporary disable concurrent sweep, because we see failures on builders. if(ConcurrentSweep && !args->eagersweep) { runtime·lock(&runtime·gclock); if(runtime·sweep.g == nil) @@ -1521,6 +1520,8 @@ gc(struct gc_args *args) // Sweep all spans eagerly. while(runtime·sweepone() != -1) runtime·sweep.npausesweep++; + // Do an additional mProf_GC, because all 'free' events are now real as well. + runtime·mProf_GC(); } runtime·mProf_GC(); diff --git a/src/runtime/mprof.go b/src/runtime/mprof.go index 89e9915236..f4676fad6e 100644 --- a/src/runtime/mprof.go +++ b/src/runtime/mprof.go @@ -234,7 +234,7 @@ func mProf_GC() { // Called by malloc to record a profiled block. func mProf_Malloc(p unsafe.Pointer, size uintptr) { var stk [maxStack]uintptr - nstk := callers(1, &stk[0], len(stk)) + nstk := callers(4, &stk[0], len(stk)) lock(&proflock) b := stkbucket(memProfile, size, stk[:nstk], true) mp := b.mp() diff --git a/src/runtime/pprof/mprof_test.go b/src/runtime/pprof/mprof_test.go new file mode 100644 index 0000000000..1880b9a707 --- /dev/null +++ b/src/runtime/pprof/mprof_test.go @@ -0,0 +1,146 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package pprof_test + +import ( + "bufio" + "bytes" + "fmt" + "io/ioutil" + "os" + "os/exec" + "regexp" + "runtime" + . "runtime/pprof" + "testing" + "unsafe" +) + +var memSink interface{} + +func allocateTransient1M() { + for i := 0; i < 1024; i++ { + memSink = &struct{ x [1024]byte }{} + } +} + +func allocateTransient2M() { + // prevent inlining + if memSink == nil { + panic("bad") + } + memSink = make([]byte, 2<<20) +} + +type Obj32 struct { + link *Obj32 + pad [32 - unsafe.Sizeof(uintptr(0))]byte +} + +var persistentMemSink *Obj32 + +func allocatePersistent1K() { + for i := 0; i < 32; i++ { + // Can't use slice because that will introduce implicit allocations. + obj := &Obj32{link: persistentMemSink} + persistentMemSink = obj + } +} + +var memoryProfilerRun = 0 + +func TestMemoryProfiler(t *testing.T) { + // Create temp file for the profile. + f, err := ioutil.TempFile("", "memprof") + if err != nil { + t.Fatalf("failed to create temp file: %v", err) + } + defer func() { + f.Close() + os.Remove(f.Name()) + }() + + // Disable sampling, otherwise it's difficult to assert anything. + oldRate := runtime.MemProfileRate + runtime.MemProfileRate = 1 + defer func() { + runtime.MemProfileRate = oldRate + }() + // Allocate a meg to ensure that mcache.next_sample is updated to 1. + for i := 0; i < 1024; i++ { + memSink = make([]byte, 1024) + } + + // Do the interesting allocations. + allocateTransient1M() + allocateTransient2M() + allocatePersistent1K() + memSink = nil + + runtime.GC() // materialize stats + if err := WriteHeapProfile(f); err != nil { + t.Fatalf("failed to write heap profile: %v", err) + } + f.Close() + + memoryProfilerRun++ + checkMemProfile(t, f.Name(), []string{"--alloc_space", "--show_bytes", "--lines"}, []string{ + fmt.Sprintf(`%v .* runtime/pprof_test\.allocateTransient1M .*mprof_test.go:25`, 1<<20*memoryProfilerRun), + fmt.Sprintf(`%v .* runtime/pprof_test\.allocateTransient2M .*mprof_test.go:34`, 2<<20*memoryProfilerRun), + fmt.Sprintf(`%v .* runtime/pprof_test\.allocatePersistent1K .*mprof_test.go:47`, 1<<10*memoryProfilerRun), + }, []string{}) + + checkMemProfile(t, f.Name(), []string{"--inuse_space", "--show_bytes", "--lines"}, []string{ + fmt.Sprintf(`%v .* runtime/pprof_test\.allocatePersistent1K .*mprof_test.go:47`, 1<<10*memoryProfilerRun), + }, []string{ + "allocateTransient1M", + "allocateTransient2M", + }) +} + +func checkMemProfile(t *testing.T, file string, addArgs []string, what []string, whatnot []string) { + args := []string{"tool", "pprof", "--text"} + args = append(args, addArgs...) + args = append(args, os.Args[0], file) + out, err := exec.Command("go", args...).CombinedOutput() + if err != nil { + t.Fatalf("failed to execute pprof: %v\n%v\n", err, string(out)) + } + + matched := make(map[*regexp.Regexp]bool) + for _, s := range what { + matched[regexp.MustCompile(s)] = false + } + var not []*regexp.Regexp + for _, s := range whatnot { + not = append(not, regexp.MustCompile(s)) + } + + s := bufio.NewScanner(bytes.NewReader(out)) + for s.Scan() { + ln := s.Text() + for re := range matched { + if re.MatchString(ln) { + if matched[re] { + t.Errorf("entry '%s' is matched twice", re.String()) + } + matched[re] = true + } + } + for _, re := range not { + if re.MatchString(ln) { + t.Errorf("entry '%s' is matched, but must not", re.String()) + } + } + } + for re, ok := range matched { + if !ok { + t.Errorf("entry '%s' is not matched", re.String()) + } + } + if t.Failed() { + t.Logf("profile:\n%v", string(out)) + } +} diff --git a/src/testing/testing.go b/src/testing/testing.go index f91d860a94..e54a3b8ce4 100644 --- a/src/testing/testing.go +++ b/src/testing/testing.go @@ -620,6 +620,7 @@ func after() { fmt.Fprintf(os.Stderr, "testing: %s\n", err) os.Exit(2) } + runtime.GC() // materialize all statistics if err = pprof.WriteHeapProfile(f); err != nil { fmt.Fprintf(os.Stderr, "testing: can't write %s: %s\n", *memProfile, err) os.Exit(2)