From: Austin Clements Date: Mon, 6 Feb 2023 02:37:07 +0000 (-0500) Subject: runtime: use inlineUnwinder X-Git-Tag: go1.21rc1~1323 X-Git-Url: http://www.git.cypherpunks.su/?a=commitdiff_plain;h=166e5ee4f2b5f170663c25b56d4929919c13debf;p=gostls13.git runtime: use inlineUnwinder This converts all places in the runtime that perform inline expansion to use the new inlineUnwinder abstraction. For #54466. Change-Id: I48d996fb6263ed5225bd21d30914a27ae434528d Reviewed-on: https://go-review.googlesource.com/c/go/+/466099 Run-TryBot: Austin Clements TryBot-Result: Gopher Robot Reviewed-by: Michael Pratt --- diff --git a/src/runtime/preempt.go b/src/runtime/preempt.go index 4f62fc628b..a6623c0ec2 100644 --- a/src/runtime/preempt.go +++ b/src/runtime/preempt.go @@ -413,14 +413,9 @@ func isAsyncSafePoint(gp *g, pc, sp, lr uintptr) (bool, uintptr) { // except the ones that have funcFlag_SPWRITE set in f.flag. return false, 0 } - name := funcname(f) - if inldata := funcdata(f, _FUNCDATA_InlTree); inldata != nil { - inltree := (*[1 << 20]inlinedCall)(inldata) - ix := pcdatavalue(f, _PCDATA_InlTreeIndex, pc, nil) - if ix >= 0 { - name = funcnameFromNameOff(f, inltree[ix].nameOff) - } - } + // Check the inner-most name + u, uf := newInlineUnwinder(f, pc, nil) + name := u.srcFunc(uf).name() if hasPrefix(name, "runtime.") || hasPrefix(name, "runtime/internal/") || hasPrefix(name, "reflect.") { diff --git a/src/runtime/race.go b/src/runtime/race.go index 144043bb66..33360d192f 100644 --- a/src/runtime/race.go +++ b/src/runtime/race.go @@ -171,39 +171,32 @@ func racecallback(cmd uintptr, ctx unsafe.Pointer) { func raceSymbolizeCode(ctx *symbolizeCodeContext) { pc := ctx.pc fi := findfunc(pc) - f := fi._Func() - if f != nil { - file, line := f.FileLine(pc) - if line != 0 { - if inldata := funcdata(fi, _FUNCDATA_InlTree); inldata != nil { - inltree := (*[1 << 20]inlinedCall)(inldata) - for { - ix := pcdatavalue(fi, _PCDATA_InlTreeIndex, pc, nil) - if ix >= 0 { - if inltree[ix].funcID == funcID_wrapper { - // ignore wrappers - // Back up to an instruction in the "caller". - pc = f.Entry() + uintptr(inltree[ix].parentPc) - continue - } - ctx.pc = f.Entry() + uintptr(inltree[ix].parentPc) // "caller" pc - name := funcnameFromNameOff(fi, inltree[ix].nameOff) - ctx.fn = &bytes(name)[0] // assume NUL-terminated - ctx.line = uintptr(line) - ctx.file = &bytes(file)[0] // assume NUL-terminated - ctx.off = pc - f.Entry() - ctx.res = 1 - return - } - break - } + if fi.valid() { + u, uf := newInlineUnwinder(fi, pc, nil) + for ; uf.valid(); uf = u.next(uf) { + sf := u.srcFunc(uf) + if sf.funcID == funcID_wrapper { + // ignore wrappers + continue + } + + name := sf.name() + file, line := u.fileLine(uf) + if line == 0 { + // Failure to symbolize + continue } - name := funcname(fi) ctx.fn = &bytes(name)[0] // assume NUL-terminated ctx.line = uintptr(line) ctx.file = &bytes(file)[0] // assume NUL-terminated - ctx.off = pc - f.Entry() + ctx.off = pc - fi.entry() ctx.res = 1 + if u.isInlined(uf) { + // Set ctx.pc to the "caller" so the race detector calls this again + // to further unwind. + uf = u.next(uf) + ctx.pc = uf.pc + } return } } diff --git a/src/runtime/stubs.go b/src/runtime/stubs.go index 5fe3506d5e..373445d613 100644 --- a/src/runtime/stubs.go +++ b/src/runtime/stubs.go @@ -222,6 +222,15 @@ func noescape(p unsafe.Pointer) unsafe.Pointer { return unsafe.Pointer(x ^ 0) } +// noEscapePtr hides a pointer from escape analysis. See noescape. +// USE CAREFULLY! +// +//go:nosplit +func noEscapePtr[T any](p *T) *T { + x := uintptr(unsafe.Pointer(p)) + return (*T)(unsafe.Pointer(x ^ 0)) +} + // Not all cgocallback frames are actually cgocallback, // so not all have these arguments. Mark them uintptr so that the GC // does not misinterpret memory when the arguments are not present. diff --git a/src/runtime/symtab.go b/src/runtime/symtab.go index c3329568b7..67648a4ebc 100644 --- a/src/runtime/symtab.go +++ b/src/runtime/symtab.go @@ -116,28 +116,21 @@ func (ci *Frames) Next() (frame Frame, more bool) { // work correctly for entries in the result of runtime.Callers. pc-- } - name := funcname(funcInfo) - startLine := f.startLine() - if inldata := funcdata(funcInfo, _FUNCDATA_InlTree); inldata != nil { - inltree := (*[1 << 20]inlinedCall)(inldata) - // Non-strict as cgoTraceback may have added bogus PCs - // with a valid funcInfo but invalid PCDATA. - ix := pcdatavalue1(funcInfo, _PCDATA_InlTreeIndex, pc, nil, false) - if ix >= 0 { - // Note: entry is not modified. It always refers to a real frame, not an inlined one. - f = nil - ic := inltree[ix] - name = funcnameFromNameOff(funcInfo, ic.nameOff) - startLine = ic.startLine - // File/line from funcline1 below are already correct. - } + // It's important that interpret pc non-strictly as cgoTraceback may + // have added bogus PCs with a valid funcInfo but invalid PCDATA. + u, uf := newInlineUnwinder(funcInfo, pc, nil) + sf := u.srcFunc(uf) + if u.isInlined(uf) { + // Note: entry is not modified. It always refers to a real frame, not an inlined one. + // File/line from funcline1 below are already correct. + f = nil } ci.frames = append(ci.frames, Frame{ PC: pc, Func: f, - Function: name, + Function: sf.name(), Entry: entry, - startLine: int(startLine), + startLine: int(sf.startLine), funcInfo: funcInfo, // Note: File,Line set below }) @@ -182,6 +175,8 @@ func runtime_FrameStartLine(f *Frame) int { // //go:linkname runtime_expandFinalInlineFrame runtime/pprof.runtime_expandFinalInlineFrame func runtime_expandFinalInlineFrame(stk []uintptr) []uintptr { + // TODO: It would be more efficient to report only physical PCs to pprof and + // just expand the whole stack. if len(stk) == 0 { return stk } @@ -194,46 +189,29 @@ func runtime_expandFinalInlineFrame(stk []uintptr) []uintptr { return stk } - inldata := funcdata(f, _FUNCDATA_InlTree) - if inldata == nil { - // Nothing inline in f. + var cache pcvalueCache + u, uf := newInlineUnwinder(f, tracepc, &cache) + if !u.isInlined(uf) { + // Nothing inline at tracepc. return stk } // Treat the previous func as normal. We haven't actually checked, but // since this pc was included in the stack, we know it shouldn't be // elided. - lastFuncID := funcID_normal + calleeID := funcID_normal // Remove pc from stk; we'll re-add it below. stk = stk[:len(stk)-1] - // See inline expansion in gentraceback. - var cache pcvalueCache - inltree := (*[1 << 20]inlinedCall)(inldata) - for { - // Non-strict as cgoTraceback may have added bogus PCs - // with a valid funcInfo but invalid PCDATA. - ix := pcdatavalue1(f, _PCDATA_InlTreeIndex, tracepc, &cache, false) - if ix < 0 { - break - } - if inltree[ix].funcID == funcID_wrapper && elideWrapperCalling(lastFuncID) { + for ; uf.valid(); uf = u.next(uf) { + funcID := u.srcFunc(uf).funcID + if funcID == funcID_wrapper && elideWrapperCalling(calleeID) { // ignore wrappers } else { - stk = append(stk, pc) + stk = append(stk, uf.pc+1) } - lastFuncID = inltree[ix].funcID - // Back up to an instruction in the "caller". - tracepc = f.entry() + uintptr(inltree[ix].parentPc) - pc = tracepc + 1 - } - - // N.B. we want to keep the last parentPC which is not inline. - if f.funcID == funcID_wrapper && elideWrapperCalling(lastFuncID) { - // Ignore wrapper functions (except when they trigger panics). - } else { - stk = append(stk, pc) + calleeID = funcID } return stk @@ -752,28 +730,25 @@ func FuncForPC(pc uintptr) *Func { if !f.valid() { return nil } - if inldata := funcdata(f, _FUNCDATA_InlTree); inldata != nil { - // Note: strict=false so bad PCs (those between functions) don't crash the runtime. - // We just report the preceding function in that situation. See issue 29735. - // TODO: Perhaps we should report no function at all in that case. - // The runtime currently doesn't have function end info, alas. - if ix := pcdatavalue1(f, _PCDATA_InlTreeIndex, pc, nil, false); ix >= 0 { - inltree := (*[1 << 20]inlinedCall)(inldata) - ic := inltree[ix] - name := funcnameFromNameOff(f, ic.nameOff) - file, line := funcline(f, pc) - fi := &funcinl{ - ones: ^uint32(0), - entry: f.entry(), // entry of the real (the outermost) function. - name: name, - file: file, - line: line, - startLine: ic.startLine, - } - return (*Func)(unsafe.Pointer(fi)) - } + // This must interpret PC non-strictly so bad PCs (those between functions) don't crash the runtime. + // We just report the preceding function in that situation. See issue 29735. + // TODO: Perhaps we should report no function at all in that case. + // The runtime currently doesn't have function end info, alas. + u, uf := newInlineUnwinder(f, pc, nil) + if !u.isInlined(uf) { + return f._Func() } - return f._Func() + sf := u.srcFunc(uf) + file, line := u.fileLine(uf) + fi := &funcinl{ + ones: ^uint32(0), + entry: f.entry(), // entry of the real (the outermost) function. + name: sf.name(), + file: file, + line: int32(line), + startLine: sf.startLine, + } + return (*Func)(unsafe.Pointer(fi)) } // Name returns the name of the function. @@ -1059,13 +1034,6 @@ func funcpkgpath(f funcInfo) string { return name[:i] } -func funcnameFromNameOff(f funcInfo, nameOff int32) string { - if !f.valid() { - return "" - } - return f.datap.funcName(nameOff) -} - func funcfile(f funcInfo, fileno int32) string { datap := f.datap if !f.valid() { diff --git a/src/runtime/traceback.go b/src/runtime/traceback.go index e873ac74be..b4717ab164 100644 --- a/src/runtime/traceback.go +++ b/src/runtime/traceback.go @@ -306,9 +306,8 @@ func gentraceback(pc0, sp0, lr0 uintptr, gp *g, skip int, pcbuf *uintptr, max in } if pcbuf != nil { - pc := frame.pc // backup to CALL instruction to read inlining info (same logic as below) - tracepc := pc + tracepc := frame.pc // Normally, pc is a return address. In that case, we want to look up // file/line information using pc-1, because that is the pc of the // call instruction (more precisely, the last byte of the call instruction). @@ -320,42 +319,24 @@ func gentraceback(pc0, sp0, lr0 uintptr, gp *g, skip int, pcbuf *uintptr, max in // See issue 34123. // The pc can be at function entry when the frame is initialized without // actually running code, like runtime.mstart. - if (n == 0 && flags&_TraceTrap != 0) || calleeFuncID == funcID_sigpanic || pc == f.entry() { - pc++ - } else { + if !((n == 0 && flags&_TraceTrap != 0) || calleeFuncID == funcID_sigpanic || tracepc == f.entry()) { tracepc-- } - - // If there is inlining info, record the inner frames. - if inldata := funcdata(f, _FUNCDATA_InlTree); inldata != nil { - inltree := (*[1 << 20]inlinedCall)(inldata) - for { - ix := pcdatavalue(f, _PCDATA_InlTreeIndex, tracepc, &cache) - if ix < 0 { - break - } - if inltree[ix].funcID == funcID_wrapper && elideWrapperCalling(calleeFuncID) { - // ignore wrappers - } else if skip > 0 { - skip-- - } else if n < max { - (*[1 << 20]uintptr)(unsafe.Pointer(pcbuf))[n] = pc - n++ - } - calleeFuncID = inltree[ix].funcID - // Back up to an instruction in the "caller". - tracepc = frame.fn.entry() + uintptr(inltree[ix].parentPc) - pc = tracepc + 1 + // TODO: Why does cache escape? (Same below) + for iu, uf := newInlineUnwinder(f, tracepc, noEscapePtr(&cache)); uf.valid(); uf = iu.next(uf) { + sf := iu.srcFunc(uf) + if sf.funcID == funcID_wrapper && elideWrapperCalling(calleeFuncID) { + // ignore wrappers + } else if skip > 0 { + skip-- + } else if n < max { + // Callers expect the pc buffer to contain return addresses + // and do the -1 themselves, so we add 1 to the call PC to + // create a return PC. + (*[1 << 20]uintptr)(unsafe.Pointer(pcbuf))[n] = uf.pc + 1 + n++ } - } - // Record the main frame. - if f.funcID == funcID_wrapper && elideWrapperCalling(calleeFuncID) { - // Ignore wrapper functions (except when they trigger panics). - } else if skip > 0 { - skip-- - } else if n < max { - (*[1 << 20]uintptr)(unsafe.Pointer(pcbuf))[n] = pc - n++ + calleeFuncID = sf.funcID } n-- // offset n++ below } @@ -373,52 +354,38 @@ func gentraceback(pc0, sp0, lr0 uintptr, gp *g, skip int, pcbuf *uintptr, max in if (n > 0 || flags&_TraceTrap == 0) && frame.pc > f.entry() && calleeFuncID != funcID_sigpanic { tracepc-- } - // If there is inlining info, print the inner frames. - if inldata := funcdata(f, _FUNCDATA_InlTree); inldata != nil { - inltree := (*[1 << 20]inlinedCall)(inldata) - for { - ix := pcdatavalue(f, _PCDATA_InlTreeIndex, tracepc, nil) - if ix < 0 { - break + for iu, uf := newInlineUnwinder(f, tracepc, noEscapePtr(&cache)); uf.valid(); uf = iu.next(uf) { + sf := iu.srcFunc(uf) + if (flags&_TraceRuntimeFrames) != 0 || showframe(sf, gp, nprint == 0, calleeFuncID) { + name := sf.name() + file, line := iu.fileLine(uf) + if name == "runtime.gopanic" { + name = "panic" } - - sf := srcFunc{f.datap, inltree[ix].nameOff, inltree[ix].startLine, inltree[ix].funcID} - - if (flags&_TraceRuntimeFrames) != 0 || showframe(sf, gp, nprint == 0, calleeFuncID) { - name := sf.name() - file, line := funcline(f, tracepc) - print(name, "(...)\n") - print("\t", file, ":", line, "\n") - nprint++ + // Print during crash. + // main(0x1, 0x2, 0x3) + // /home/rsc/go/src/runtime/x.go:23 +0xf + // + print(name, "(") + if iu.isInlined(uf) { + print("...") + } else { + argp := unsafe.Pointer(frame.argp) + printArgs(f, argp, tracepc) } - calleeFuncID = inltree[ix].funcID - // Back up to an instruction in the "caller". - tracepc = frame.fn.entry() + uintptr(inltree[ix].parentPc) - } - } - if (flags&_TraceRuntimeFrames) != 0 || showframe(f.srcFunc(), gp, nprint == 0, calleeFuncID) { - // Print during crash. - // main(0x1, 0x2, 0x3) - // /home/rsc/go/src/runtime/x.go:23 +0xf - // - name := funcname(f) - file, line := funcline(f, tracepc) - if name == "runtime.gopanic" { - name = "panic" - } - print(name, "(") - argp := unsafe.Pointer(frame.argp) - printArgs(f, argp, tracepc) - print(")\n") - print("\t", file, ":", line) - if frame.pc > f.entry() { - print(" +", hex(frame.pc-f.entry())) - } - if gp.m != nil && gp.m.throwing >= throwTypeRuntime && gp == gp.m.curg || level >= 2 { - print(" fp=", hex(frame.fp), " sp=", hex(frame.sp), " pc=", hex(frame.pc)) + print(")\n") + print("\t", file, ":", line) + if !iu.isInlined(uf) { + if frame.pc > f.entry() { + print(" +", hex(frame.pc-f.entry())) + } + if gp.m != nil && gp.m.throwing >= throwTypeRuntime && gp == gp.m.curg || level >= 2 { + print(" fp=", hex(frame.fp), " sp=", hex(frame.sp), " pc=", hex(frame.pc)) + } + } + print("\n") + nprint++ } - print("\n") - nprint++ } } n++ @@ -807,15 +774,9 @@ func printAncestorTraceback(ancestor ancestorInfo) { // due to only have access to the pcs at the time of the caller // goroutine being created. func printAncestorTracebackFuncInfo(f funcInfo, pc uintptr) { - name := funcname(f) - if inldata := funcdata(f, _FUNCDATA_InlTree); inldata != nil { - inltree := (*[1 << 20]inlinedCall)(inldata) - ix := pcdatavalue(f, _PCDATA_InlTreeIndex, pc, nil) - if ix >= 0 { - name = funcnameFromNameOff(f, inltree[ix].nameOff) - } - } - file, line := funcline(f, pc) + u, uf := newInlineUnwinder(f, pc, nil) + name := u.srcFunc(uf).name() + file, line := u.fileLine(uf) if name == "runtime.gopanic" { name = "panic" }