From: Michael Pratt Date: Tue, 28 Mar 2023 16:45:17 +0000 (-0400) Subject: cmd/compile: don't inline hot calls into big functions X-Git-Tag: go1.21rc1~972 X-Git-Url: http://www.git.cypherpunks.su/?a=commitdiff_plain;h=c1ac63e9732a54ef8349fe0bdbb3fa47bd0847c2;p=gostls13.git cmd/compile: don't inline hot calls into big functions Standard inlining has a reduced maximum cost of callees (20 instead of 80) when inlining into a "big" function, to limit how much bigger we will make an already big function. When adding PGO hot call budget increases, we inadvertently bypassed this "big" function restriction, allowing hot calls of up to inlineHotMaxBudget, even into big functions. Add the restriction back, even for hot calls. If a function is already very large, we probably shouldn't inline even more. A very important note here is that function "big"-ness is computed prior to any inlining. One potential problem with PGO is that many hot calls inline into an initially-small function and ultimately make it very large. This CL does nothing to address that case, which would require recomputing size after inlining. This CL has no impact on sweet PGO benchmarks. I specifically dug into tile38, which contained 0 hot big functions. Other benchmarks are probably similar. Change-Id: I3b6304eaf7738a219359d4b8bb121d68babfea8b Reviewed-on: https://go-review.googlesource.com/c/go/+/482157 TryBot-Result: Gopher Robot Reviewed-by: Cherry Mui Reviewed-by: Matthew Dempsky Run-TryBot: Michael Pratt --- diff --git a/src/cmd/compile/internal/inline/inl.go b/src/cmd/compile/internal/inline/inl.go index 3e4498eff5..9a2df95718 100644 --- a/src/cmd/compile/internal/inline/inl.go +++ b/src/cmd/compile/internal/inline/inl.go @@ -791,17 +791,14 @@ func inlcopy(n ir.Node) ir.Node { func InlineCalls(fn *ir.Func, profile *pgo.Profile) { savefn := ir.CurFunc ir.CurFunc = fn - maxCost := int32(inlineMaxBudget) - if isBigFunc(fn) { - if base.Flag.LowerM > 1 { - fmt.Printf("%v: function %v considered 'big'; revising maxCost from %d to %d\n", ir.Line(fn), fn, maxCost, inlineBigFunctionMaxCost) - } - maxCost = inlineBigFunctionMaxCost + bigCaller := isBigFunc(fn) + if bigCaller && base.Flag.LowerM > 1 { + fmt.Printf("%v: function %v considered 'big'; reducing max cost of inlinees\n", ir.Line(fn), fn) } var inlCalls []*ir.InlinedCallExpr var edit func(ir.Node) ir.Node edit = func(n ir.Node) ir.Node { - return inlnode(n, maxCost, &inlCalls, edit, profile) + return inlnode(n, bigCaller, &inlCalls, edit, profile) } ir.EditChildren(fn, edit) @@ -832,7 +829,7 @@ func InlineCalls(fn *ir.Func, profile *pgo.Profile) { // The result of inlnode MUST be assigned back to n, e.g. // // n.Left = inlnode(n.Left) -func inlnode(n ir.Node, maxCost int32, inlCalls *[]*ir.InlinedCallExpr, edit func(ir.Node) ir.Node, profile *pgo.Profile) ir.Node { +func inlnode(n ir.Node, bigCaller bool, inlCalls *[]*ir.InlinedCallExpr, edit func(ir.Node) ir.Node, profile *pgo.Profile) ir.Node { if n == nil { return n } @@ -894,7 +891,7 @@ func inlnode(n ir.Node, maxCost int32, inlCalls *[]*ir.InlinedCallExpr, edit fun break } if fn := inlCallee(call.X, profile); fn != nil && typecheck.HaveInlineBody(fn) { - n = mkinlcall(call, fn, maxCost, inlCalls, edit) + n = mkinlcall(call, fn, bigCaller, inlCalls, edit) if fn.IsHiddenClosure() { // Visit function to pick out any contained hidden // closures to mark them as dead, since they will no @@ -969,6 +966,54 @@ var InlineCall = func(call *ir.CallExpr, fn *ir.Func, inlIndex int) *ir.InlinedC panic("unreachable") } +// inlineCostOK returns true if call n from caller to callee is cheap enough to +// inline. bigCaller indicates that caller is a big function. +// +// If inlineCostOK returns false, it also returns the max cost that the callee +// exceeded. +func inlineCostOK(n *ir.CallExpr, caller, callee *ir.Func, bigCaller bool) (bool, int32) { + maxCost := int32(inlineMaxBudget) + if bigCaller { + // We use this to restrict inlining into very big functions. + // See issue 26546 and 17566. + maxCost = inlineBigFunctionMaxCost + } + + if callee.Inl.Cost <= maxCost { + // Simple case. Function is already cheap enough. + return true, 0 + } + + // We'll also allow inlining of hot functions below inlineHotMaxBudget, + // but only in small functions. + + lineOffset := pgo.NodeLineOffset(n, caller) + csi := pgo.CallSiteInfo{LineOffset: lineOffset, Caller: caller} + if _, ok := candHotEdgeMap[csi]; !ok { + // Cold + return false, maxCost + } + + // Hot + + if bigCaller { + if base.Debug.PGOInline > 0 { + fmt.Printf("hot-big check disallows inlining for call %s (cost %d) at %v in big function %s\n", ir.PkgFuncName(callee), callee.Inl.Cost, ir.Line(n), ir.PkgFuncName(caller)) + } + return false, maxCost + } + + if callee.Inl.Cost > inlineHotMaxBudget { + return false, inlineHotMaxBudget + } + + if base.Debug.PGOInline > 0 { + fmt.Printf("hot-budget check allows inlining for call %s (cost %d) at %v in function %s\n", ir.PkgFuncName(callee), callee.Inl.Cost, ir.Line(n), ir.PkgFuncName(caller)) + } + + return true, 0 +} + // If n is a OCALLFUNC node, and fn is an ONAME node for a // function with an inlinable body, return an OINLCALL node that can replace n. // The returned node's Ninit has the parameter assignments, the Nbody is the @@ -977,7 +1022,7 @@ var InlineCall = func(call *ir.CallExpr, fn *ir.Func, inlIndex int) *ir.InlinedC // The result of mkinlcall MUST be assigned back to n, e.g. // // n.Left = mkinlcall(n.Left, fn, isddd) -func mkinlcall(n *ir.CallExpr, fn *ir.Func, maxCost int32, inlCalls *[]*ir.InlinedCallExpr, edit func(ir.Node) ir.Node) ir.Node { +func mkinlcall(n *ir.CallExpr, fn *ir.Func, bigCaller bool, inlCalls *[]*ir.InlinedCallExpr, edit func(ir.Node) ir.Node) ir.Node { if fn.Inl == nil { if logopt.Enabled() { logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(ir.CurFunc), @@ -985,30 +1030,13 @@ func mkinlcall(n *ir.CallExpr, fn *ir.Func, maxCost int32, inlCalls *[]*ir.Inlin } return n } - if fn.Inl.Cost > maxCost { - // If the callsite is hot and it is under the inlineHotMaxBudget budget, then try to inline it, or else bail. - lineOffset := pgo.NodeLineOffset(n, ir.CurFunc) - csi := pgo.CallSiteInfo{LineOffset: lineOffset, Caller: ir.CurFunc} - if _, ok := candHotEdgeMap[csi]; ok { - if fn.Inl.Cost > inlineHotMaxBudget { - if logopt.Enabled() { - logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(ir.CurFunc), - fmt.Sprintf("cost %d of %s exceeds max large caller cost %d", fn.Inl.Cost, ir.PkgFuncName(fn), inlineHotMaxBudget)) - } - return n - } - if base.Debug.PGOInline > 0 { - fmt.Printf("hot-budget check allows inlining for call %s at %v\n", ir.PkgFuncName(fn), ir.Line(n)) - } - } else { - // The inlined function body is too big. Typically we use this check to restrict - // inlining into very big functions. See issue 26546 and 17566. - if logopt.Enabled() { - logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(ir.CurFunc), - fmt.Sprintf("cost %d of %s exceeds max large caller cost %d", fn.Inl.Cost, ir.PkgFuncName(fn), maxCost)) - } - return n + + if ok, maxCost := inlineCostOK(n, ir.CurFunc, fn, bigCaller); !ok { + if logopt.Enabled() { + logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(ir.CurFunc), + fmt.Sprintf("cost %d of %s exceeds max caller cost %d", fn.Inl.Cost, ir.PkgFuncName(fn), maxCost)) } + return n } if fn == ir.CurFunc {