cmd/trace: fix goroutine view

Currently, trace processing interleaves state/statistics updates and
emitting trace viewer objects. As a result, if events are being
filtered, either by time or by goroutines, we'll miss those
state/statistics updates. At best, this leads to bad statistics;
however, since we're now strictly checking G state transitions, it
usually leads to a failure to process the trace if there is any
filtering.

Fix this by separating state updates from emitting trace object. State
updates are done before filtering, so we always have correct state
information and statistics. Trace objects are only emitted if we pass
the filter. To determine when we need to emit trace counters, rather
than duplicating the knowledge of which events might modify
statistics, we keep track of the previously emitted counters and emit
a trace counter object whenever these have changed.

Fixes #17719.

Change-Id: Ic66e3ddaef60d1acaaf2ff4c62baa5352799cf99
Reviewed-on: https://go-review.googlesource.com/32810
Reviewed-by: Dmitry Vyukov <dvyukov@google.com>

diff --git a/src/cmd/trace/trace.go b/src/cmd/trace/trace.go
index f92852f2a0e5a403646bd85f518fb42c9d788bb7..f5a2df47ff7f520a310ecf4ddf56af56e5ed543b 100644 (file)
--- a/src/cmd/trace/trace.go
+++ b/src/cmd/trace/trace.go
@@ -254,10 +254,19 @@ type traceContext struct {
        frameTree frameNode
        frameSeq  int
        arrowSeq  uint64
+       gcount    uint64
+
+       heapStats, prevHeapStats     heapStats
+       threadStats, prevThreadStats threadStats
+       gstates, prevGstates         [gStateCount]uint64
+}
+
+type heapStats struct {
        heapAlloc uint64
        nextGC    uint64
-       gcount    uint64
-       gstates   [gStateCount]uint64
+}
+
+type threadStats struct {
        insyscall uint64
        prunning  uint64
 }
@@ -344,12 +353,12 @@ func generateTrace(params *traceParams) (ViewerData, error) {
                ctx.gstates[gstates[g]]--
                ctx.gstates[newState]++
                gstates[g] = newState
-               ctx.emitGoroutineCounters(ev)
        }
        for _, ev := range ctx.events {
-               // Handle trace.EvGoStart separately, because we need the goroutine name
-               // even if ignore the event otherwise.
-               if ev.Type == trace.EvGoStart {
+               // Handle state transitions before we filter out events.
+               switch ev.Type {
+               case trace.EvGoStart, trace.EvGoStartLabel:
+                       setGState(ev, ev.G, gRunnable, gRunning)
                        if _, ok := gnames[ev.G]; !ok {
                                if len(ev.Stk) > 0 {
                                        gnames[ev.G] = fmt.Sprintf("G%v %s", ev.G, ev.Stk[0].Fn)
@@ -357,6 +366,48 @@ func generateTrace(params *traceParams) (ViewerData, error) {
                                        gnames[ev.G] = fmt.Sprintf("G%v", ev.G)
                                }
                        }
+               case trace.EvProcStart:
+                       ctx.threadStats.prunning++
+               case trace.EvProcStop:
+                       ctx.threadStats.prunning--
+               case trace.EvGoCreate:
+                       ctx.gcount++
+                       setGState(ev, ev.Args[0], gDead, gRunnable)
+               case trace.EvGoEnd:
+                       ctx.gcount--
+                       setGState(ev, ev.G, gRunning, gDead)
+               case trace.EvGoUnblock:
+                       setGState(ev, ev.Args[0], gWaiting, gRunnable)
+               case trace.EvGoSysExit:
+                       setGState(ev, ev.G, gWaiting, gRunnable)
+                       ctx.threadStats.insyscall--
+               case trace.EvGoSysBlock:
+                       setGState(ev, ev.G, gRunning, gWaiting)
+                       ctx.threadStats.insyscall++
+               case trace.EvGoSched, trace.EvGoPreempt:
+                       setGState(ev, ev.G, gRunning, gRunnable)
+               case trace.EvGoStop,
+                       trace.EvGoSleep, trace.EvGoBlock, trace.EvGoBlockSend, trace.EvGoBlockRecv,
+                       trace.EvGoBlockSelect, trace.EvGoBlockSync, trace.EvGoBlockCond, trace.EvGoBlockNet:
+                       setGState(ev, ev.G, gRunning, gWaiting)
+               case trace.EvGoBlockGC:
+                       setGState(ev, ev.G, gRunning, gWaitingGC)
+               case trace.EvGoWaiting:
+                       setGState(ev, ev.G, gRunnable, gWaiting)
+               case trace.EvGoInSyscall:
+                       // Cancel out the effect of EvGoCreate at the beginning.
+                       setGState(ev, ev.G, gRunnable, gWaiting)
+                       ctx.threadStats.insyscall++
+               case trace.EvHeapAlloc:
+                       ctx.heapStats.heapAlloc = ev.Args[0]
+               case trace.EvNextGC:
+                       ctx.heapStats.nextGC = ev.Args[0]
+               }
+               if setGStateErr != nil {
+                       return ctx.data, setGStateErr
+               }
+               if ctx.gstates[gRunnable] < 0 || ctx.gstates[gRunning] < 0 || ctx.threadStats.insyscall < 0 {
+                       return ctx.data, fmt.Errorf("invalid state after processing %v: runnable=%d running=%d insyscall=%d", ev, ctx.gstates[gRunnable], ctx.gstates[gRunning], ctx.threadStats.insyscall)
                }
 
                // Ignore events that are from uninteresting goroutines
@@ -372,20 +423,17 @@ func generateTrace(params *traceParams) (ViewerData, error) {
                        maxProc = ev.P
                }
 
+               // Emit trace objects.
                switch ev.Type {
                case trace.EvProcStart:
                        if ctx.gtrace {
                                continue
                        }
-                       ctx.prunning++
-                       ctx.emitThreadCounters(ev)
                        ctx.emitInstant(ev, "proc start")
                case trace.EvProcStop:
                        if ctx.gtrace {
                                continue
                        }
-                       ctx.prunning--
-                       ctx.emitThreadCounters(ev)
                        ctx.emitInstant(ev, "proc stop")
                case trace.EvGCStart:
                        ctx.emitSlice(ev, "GC")
@@ -399,62 +447,23 @@ func generateTrace(params *traceParams) (ViewerData, error) {
                case trace.EvGCSweepStart:
                        ctx.emitSlice(ev, "SWEEP")
                case trace.EvGCSweepDone:
-               case trace.EvGoStart, trace.EvGoStartLabel:
-                       setGState(ev, ev.G, gRunnable, gRunning)
-                       if ev.Type == trace.EvGoStartLabel {
-                               ctx.emitSlice(ev, ev.SArgs[0])
-                       } else {
-                               ctx.emitSlice(ev, gnames[ev.G])
-                       }
+               case trace.EvGoStart:
+                       ctx.emitSlice(ev, gnames[ev.G])
+               case trace.EvGoStartLabel:
+                       ctx.emitSlice(ev, ev.SArgs[0])
                case trace.EvGoCreate:
-                       ctx.gcount++
-                       setGState(ev, ev.Args[0], gDead, gRunnable)
                        ctx.emitArrow(ev, "go")
-               case trace.EvGoEnd:
-                       ctx.gcount--
-                       setGState(ev, ev.G, gRunning, gDead)
                case trace.EvGoUnblock:
-                       setGState(ev, ev.Args[0], gWaiting, gRunnable)
                        ctx.emitArrow(ev, "unblock")
                case trace.EvGoSysCall:
                        ctx.emitInstant(ev, "syscall")
                case trace.EvGoSysExit:
-                       setGState(ev, ev.G, gWaiting, gRunnable)
-                       ctx.insyscall--
-                       ctx.emitThreadCounters(ev)
                        ctx.emitArrow(ev, "sysexit")
-               case trace.EvGoSysBlock:
-                       setGState(ev, ev.G, gRunning, gWaiting)
-                       ctx.insyscall++
-                       ctx.emitThreadCounters(ev)
-               case trace.EvGoSched, trace.EvGoPreempt:
-                       setGState(ev, ev.G, gRunning, gRunnable)
-               case trace.EvGoStop,
-                       trace.EvGoSleep, trace.EvGoBlock, trace.EvGoBlockSend, trace.EvGoBlockRecv,
-                       trace.EvGoBlockSelect, trace.EvGoBlockSync, trace.EvGoBlockCond, trace.EvGoBlockNet:
-                       setGState(ev, ev.G, gRunning, gWaiting)
-               case trace.EvGoBlockGC:
-                       setGState(ev, ev.G, gRunning, gWaitingGC)
-               case trace.EvGoWaiting:
-                       setGState(ev, ev.G, gRunnable, gWaiting)
-               case trace.EvGoInSyscall:
-                       // Cancel out the effect of EvGoCreate at the beginning.
-                       setGState(ev, ev.G, gRunnable, gWaiting)
-                       ctx.insyscall++
-                       ctx.emitThreadCounters(ev)
-               case trace.EvHeapAlloc:
-                       ctx.heapAlloc = ev.Args[0]
-                       ctx.emitHeapCounters(ev)
-               case trace.EvNextGC:
-                       ctx.nextGC = ev.Args[0]
-                       ctx.emitHeapCounters(ev)
-               }
-               if setGStateErr != nil {
-                       return ctx.data, setGStateErr
-               }
-               if ctx.gstates[gRunnable] < 0 || ctx.gstates[gRunning] < 0 || ctx.insyscall < 0 {
-                       return ctx.data, fmt.Errorf("invalid state after processing %v: runnable=%d running=%d insyscall=%d", ev, ctx.gstates[gRunnable], ctx.gstates[gRunning], ctx.insyscall)
                }
+               // Emit any counter updates.
+               ctx.emitThreadCounters(ev)
+               ctx.emitHeapCounters(ev)
+               ctx.emitGoroutineCounters(ev)
        }
 
        ctx.data.footer = len(ctx.data.Events)
@@ -535,11 +544,15 @@ func (ctx *traceContext) emitHeapCounters(ev *trace.Event) {
        if ctx.gtrace {
                return
        }
+       if ctx.prevHeapStats == ctx.heapStats {
+               return
+       }
        diff := uint64(0)
-       if ctx.nextGC > ctx.heapAlloc {
-               diff = ctx.nextGC - ctx.heapAlloc
+       if ctx.heapStats.nextGC > ctx.heapStats.heapAlloc {
+               diff = ctx.heapStats.nextGC - ctx.heapStats.heapAlloc
        }
-       ctx.emit(&ViewerEvent{Name: "Heap", Phase: "C", Time: ctx.time(ev), Pid: 1, Arg: &heapCountersArg{ctx.heapAlloc, diff}})
+       ctx.emit(&ViewerEvent{Name: "Heap", Phase: "C", Time: ctx.time(ev), Pid: 1, Arg: &heapCountersArg{ctx.heapStats.heapAlloc, diff}})
+       ctx.prevHeapStats = ctx.heapStats
 }
 
 type goroutineCountersArg struct {
@@ -552,7 +565,11 @@ func (ctx *traceContext) emitGoroutineCounters(ev *trace.Event) {
        if ctx.gtrace {
                return
        }
+       if ctx.prevGstates == ctx.gstates {
+               return
+       }
        ctx.emit(&ViewerEvent{Name: "Goroutines", Phase: "C", Time: ctx.time(ev), Pid: 1, Arg: &goroutineCountersArg{ctx.gstates[gRunning], ctx.gstates[gRunnable], ctx.gstates[gWaitingGC]}})
+       ctx.prevGstates = ctx.gstates
 }
 
 type threadCountersArg struct {
@@ -564,7 +581,11 @@ func (ctx *traceContext) emitThreadCounters(ev *trace.Event) {
        if ctx.gtrace {
                return
        }
-       ctx.emit(&ViewerEvent{Name: "Threads", Phase: "C", Time: ctx.time(ev), Pid: 1, Arg: &threadCountersArg{ctx.prunning, ctx.insyscall}})
+       if ctx.prevThreadStats == ctx.threadStats {
+               return
+       }
+       ctx.emit(&ViewerEvent{Name: "Threads", Phase: "C", Time: ctx.time(ev), Pid: 1, Arg: &threadCountersArg{ctx.threadStats.prunning, ctx.threadStats.insyscall}})
+       ctx.prevThreadStats = ctx.threadStats
 }
 
 func (ctx *traceContext) emitInstant(ev *trace.Event, name string) {

diff --git a/src/cmd/trace/trace_test.go b/src/cmd/trace/trace_test.go
index 73a2883f1ea2e4ac5749a9d4997cac75bdd396bf..d14239cfe16be560ad108e33256ba963334544b5 100644 (file)
--- a/src/cmd/trace/trace_test.go
+++ b/src/cmd/trace/trace_test.go
@@ -60,3 +60,42 @@ func TestGoroutineCount(t *testing.T) {
                }
        }
 }
+
+func TestGoroutineFilter(t *testing.T) {
+       // Test that we handle state changes to selected goroutines
+       // caused by events on goroutines that are not selected.
+
+       w := trace.NewWriter()
+       w.Emit(trace.EvBatch, 0, 0)  // start of per-P batch event [pid, timestamp]
+       w.Emit(trace.EvFrequency, 1) // [ticks per second]
+
+       // goroutine 10: blocked
+       w.Emit(trace.EvGoCreate, 1, 10, 1, 1) // [timestamp, new goroutine id, new stack id, stack id]
+       w.Emit(trace.EvGoWaiting, 1, 10)      // [timestamp, goroutine id]
+
+       // goroutine 20: runnable->running->unblock 10
+       w.Emit(trace.EvGoCreate, 1, 20, 7, 1)
+       w.Emit(trace.EvGoStartLocal, 1, 20)      // [timestamp, goroutine id]
+       w.Emit(trace.EvGoUnblockLocal, 1, 10, 8) // [timestamp, goroutine id, stack]
+       w.Emit(trace.EvGoEnd, 1)                 // [timestamp]
+
+       // goroutine 10: runnable->running->block
+       w.Emit(trace.EvGoStartLocal, 1, 10) // [timestamp, goroutine id]
+       w.Emit(trace.EvGoBlock, 1, 9)       // [timestamp, stack]
+
+       events, err := trace.Parse(w, "")
+       if err != nil {
+               t.Fatalf("failed to parse test trace: %v", err)
+       }
+
+       params := &traceParams{
+               events:  events,
+               endTime: int64(1<<63 - 1),
+               gs:      map[uint64]bool{10: true},
+       }
+
+       _, err = generateTrace(params)
+       if err != nil {
+               t.Fatalf("generateTrace failed: %v", err)
+       }
+}