From d2b5a6f332b011c75c17bfb99216cc51ac7a0b5f Mon Sep 17 00:00:00 2001 From: Michael Pratt Date: Wed, 26 Oct 2022 18:12:21 -0400 Subject: [PATCH] cmd/trace: only include required frames in splits Though we split traces into 100MB chunks, currently each chunk always includes the entire stack frame map, including frames for all events in the trace file, even if they aren't needed by events in this chunk. This means that if the stack frame JSON alone is >100MB then there is no space at all for events. In that case, we'll generate splits each containing 1 event, which is effectively useless. Handle this more efficiently by only including stack frames referenced by events in the chunk. Each marginal events only adds at most a few dozen stack frames, so it should now longer be possible to only include a tiny number of events. Fixes #56444. Change-Id: I58aa8f271c32678028b72d82df16e6ea762ebb39 Reviewed-on: https://go-review.googlesource.com/c/go/+/445895 Run-TryBot: Michael Pratt Reviewed-by: Michael Knyszek Auto-Submit: Michael Pratt TryBot-Result: Gopher Robot --- src/cmd/trace/trace.go | 177 ++++++++++++++++++++++++++++++++++------- 1 file changed, 147 insertions(+), 30 deletions(-) diff --git a/src/cmd/trace/trace.go b/src/cmd/trace/trace.go index 253b5dafe5..84fca62a04 100644 --- a/src/cmd/trace/trace.go +++ b/src/cmd/trace/trace.go @@ -310,15 +310,76 @@ type splitter struct { Ranges []Range } +// walkStackFrames calls fn for id and all of its parent frames from allFrames. +func walkStackFrames(allFrames map[string]traceviewer.Frame, id int, fn func(id int)) { + for id != 0 { + f, ok := allFrames[strconv.Itoa(id)] + if !ok { + break + } + fn(id) + id = f.Parent + } +} + +func stackFrameEncodedSize(id uint, f traceviewer.Frame) int { + // We want to know the marginal size of traceviewer.Data.Frames for + // each event. Running full JSON encoding of the map for each event is + // far too slow. + // + // Since the format is fixed, we can easily compute the size without + // encoding. + // + // A single entry looks like one of the following: + // + // "1":{"name":"main.main:30"}, + // "10":{"name":"pkg.NewSession:173","parent":9}, + // + // The parent is omitted if 0. The trailing comma is omitted from the + // last entry, but we don't need that much precision. + const ( + baseSize = len(`"`) + len (`":{"name":"`) + len(`"},`) + + // Don't count the trailing quote on the name, as that is + // counted in baseSize. + parentBaseSize = len(`,"parent":`) + ) + + size := baseSize + + size += len(f.Name) + + // Bytes for id (always positive). + for id > 0 { + size += 1 + id /= 10 + } + + if f.Parent > 0 { + size += parentBaseSize + // Bytes for parent (always positive). + for f.Parent > 0 { + size += 1 + f.Parent /= 10 + } + } + + return size +} + func splittingTraceConsumer(max int) (*splitter, traceConsumer) { type eventSz struct { - Time float64 - Sz int + Time float64 + Sz int + Frames []int } var ( + // data.Frames contains only the frames for required events. data = traceviewer.Data{Frames: make(map[string]traceviewer.Frame)} + allFrames = make(map[string]traceviewer.Frame) + sizes []eventSz cw countingWriter ) @@ -331,48 +392,95 @@ func splittingTraceConsumer(max int) (*splitter, traceConsumer) { }, consumeViewerEvent: func(v *traceviewer.Event, required bool) { if required { - // Store required events inside data - // so flush can include them in the required - // part of the trace. + // Store required events inside data so flush + // can include them in the required part of the + // trace. data.Events = append(data.Events, v) + walkStackFrames(allFrames, v.Stack, func(id int) { + s := strconv.Itoa(id) + data.Frames[s] = allFrames[s] + }) + walkStackFrames(allFrames, v.EndStack, func(id int) { + s := strconv.Itoa(id) + data.Frames[s] = allFrames[s] + }) return } enc := json.NewEncoder(&cw) enc.Encode(v) - sizes = append(sizes, eventSz{v.Time, cw.size + 1}) // +1 for ",". + size := eventSz{Time: v.Time, Sz: cw.size + 1} // +1 for ",". + // Add referenced stack frames. Their size is computed + // in flush, where we can dedup across events. + walkStackFrames(allFrames, v.Stack, func(id int) { + size.Frames = append(size.Frames, id) + }) + walkStackFrames(allFrames, v.EndStack, func(id int) { + size.Frames = append(size.Frames, id) // This may add duplicates. We'll dedup later. + }) + sizes = append(sizes, size) cw.size = 0 }, consumeViewerFrame: func(k string, v traceviewer.Frame) { - data.Frames[k] = v + allFrames[k] = v }, flush: func() { // Calculate size of the mandatory part of the trace. - // This includes stack traces and thread names. + // This includes thread names and stack frames for + // required events. cw.size = 0 enc := json.NewEncoder(&cw) enc.Encode(data) - minSize := cw.size + requiredSize := cw.size + + // Then calculate size of each individual event and + // their stack frames, grouping them into ranges. We + // only include stack frames relevant to the events in + // the range to reduce overhead. + + var ( + start = 0 - // Then calculate size of each individual event - // and group them into ranges. - sum := minSize - start := 0 + eventsSize = 0 + + frames = make(map[string]traceviewer.Frame) + framesSize = 0 + ) for i, ev := range sizes { - if sum+ev.Sz > max { - startTime := time.Duration(sizes[start].Time * 1000) - endTime := time.Duration(ev.Time * 1000) - ranges = append(ranges, Range{ - Name: fmt.Sprintf("%v-%v", startTime, endTime), - Start: start, - End: i + 1, - StartTime: int64(startTime), - EndTime: int64(endTime), - }) - start = i + 1 - sum = minSize - } else { - sum += ev.Sz + 1 + eventsSize += ev.Sz + + // Add required stack frames. Note that they + // may already be in the map. + for _, id := range ev.Frames { + s := strconv.Itoa(id) + _, ok := frames[s] + if ok { + continue + } + f := allFrames[s] + frames[s] = f + framesSize += stackFrameEncodedSize(uint(id), f) + } + + total := requiredSize + framesSize + eventsSize + if total < max { + continue } + + // Reached max size, commit this range and + // start a new range. + startTime := time.Duration(sizes[start].Time * 1000) + endTime := time.Duration(ev.Time * 1000) + ranges = append(ranges, Range{ + Name: fmt.Sprintf("%v-%v", startTime, endTime), + Start: start, + End: i + 1, + StartTime: int64(startTime), + EndTime: int64(endTime), + }) + start = i + 1 + frames = make(map[string]traceviewer.Frame) + framesSize = 0 + eventsSize = 0 } if len(ranges) <= 1 { s.Ranges = nil @@ -1151,7 +1259,8 @@ type jsonWriter struct { } func viewerDataTraceConsumer(w io.Writer, start, end int64) traceConsumer { - frames := make(map[string]traceviewer.Frame) + allFrames := make(map[string]traceviewer.Frame) + requiredFrames := make(map[string]traceviewer.Frame) enc := json.NewEncoder(w) written := 0 index := int64(-1) @@ -1169,6 +1278,14 @@ func viewerDataTraceConsumer(w io.Writer, start, end int64) traceConsumer { // not in the range. Skip! return } + walkStackFrames(allFrames, v.Stack, func(id int) { + s := strconv.Itoa(id) + requiredFrames[s] = allFrames[s] + }) + walkStackFrames(allFrames, v.EndStack, func(id int) { + s := strconv.Itoa(id) + requiredFrames[s] = allFrames[s] + }) if written == 0 { io.WriteString(w, `"traceEvents": [`) } @@ -1181,11 +1298,11 @@ func viewerDataTraceConsumer(w io.Writer, start, end int64) traceConsumer { written++ }, consumeViewerFrame: func(k string, v traceviewer.Frame) { - frames[k] = v + allFrames[k] = v }, flush: func() { io.WriteString(w, `], "stackFrames":`) - enc.Encode(frames) + enc.Encode(requiredFrames) io.WriteString(w, `}`) }, } -- 2.50.0