inlineExtraPanicCost = 1 // do not penalize inlining panics.
inlineExtraThrowCost = inlineMaxBudget // with current (2018-05/1.11) code, inlining runtime.throw does not help.
- inlineBigFunctionNodes = 5000 // Functions with this many nodes are considered "big".
- inlineBigFunctionMaxCost = 20 // Max cost of inlinee when inlining into a "big" function.
+ inlineBigFunctionNodes = 5000 // Functions with this many nodes are considered "big".
+ inlineBigFunctionMaxCost = 20 // Max cost of inlinee when inlining into a "big" function.
+ inlineClosureCalledOnceCost = 10 * inlineMaxBudget // if a closure is just called once, inline it.
)
var (
budget += inlheur.BudgetExpansion(inlineMaxBudget)
}
if fn.ClosureParent != nil {
- budget *= 2
+ // be very liberal here, if the closure is only called once, the budget is large
+ budget = max(budget, inlineClosureCalledOnceCost)
}
return budget
}
break
}
- if callee := inlCallee(v.curFunc, n.Fun, v.profile); callee != nil && typecheck.HaveInlineBody(callee) {
+ if callee := inlCallee(v.curFunc, n.Fun, v.profile, false); callee != nil && typecheck.HaveInlineBody(callee) {
// Check whether we'd actually inline this call. Set
// log == false since we aren't actually doing inlining
// yet.
- if ok, _, _ := canInlineCallExpr(v.curFunc, n, callee, v.isBigFunc, false); ok {
+ if ok, _, _ := canInlineCallExpr(v.curFunc, n, callee, v.isBigFunc, false, false); ok {
// mkinlcall would inline this call [1], so use
// the cost of the inline body as the cost of
// the call, as that is what will actually
// by looking at what has already been inlined.
// Since we haven't done any inlining yet we
// will miss those.
+ //
+ // TODO: in the case of a single-call closure, the inlining budget here is potentially much, much larger.
+ //
v.budget -= callee.Inl.Cost
break
}
})
}
-// TryInlineCall returns an inlined call expression for call, or nil
-// if inlining is not possible.
-func TryInlineCall(callerfn *ir.Func, call *ir.CallExpr, bigCaller bool, profile *pgoir.Profile) *ir.InlinedCallExpr {
+// inlineCallCheck returns whether a call will never be inlineable
+// for basic reasons, and whether the call is an intrinisic call.
+// The intrinsic result singles out intrinsic calls for debug logging.
+func inlineCallCheck(callerfn *ir.Func, call *ir.CallExpr) (bool, bool) {
if base.Flag.LowerL == 0 {
- return nil
+ return false, false
}
if call.Op() != ir.OCALLFUNC {
- return nil
+ return false, false
}
if call.GoDefer || call.NoInline {
- return nil
+ return false, false
}
// Prevent inlining some reflect.Value methods when using checkptr,
if method := ir.MethodExprName(call.Fun); method != nil {
switch types.ReflectSymName(method.Sym()) {
case "Value.UnsafeAddr", "Value.Pointer":
- return nil
+ return false, false
}
}
}
+ if ir.IsIntrinsicCall(call) {
+ return false, true
+ }
+ return true, false
+}
+
+// InlineCallTarget returns the resolved-for-inlining target of a call.
+// It does not necessarily guarantee that the target can be inlined, though
+// obvious exclusions are applied.
+func InlineCallTarget(callerfn *ir.Func, call *ir.CallExpr, profile *pgoir.Profile) *ir.Func {
+ if mightInline, _ := inlineCallCheck(callerfn, call); !mightInline {
+ return nil
+ }
+ return inlCallee(callerfn, call.Fun, profile, true)
+}
+
+// TryInlineCall returns an inlined call expression for call, or nil
+// if inlining is not possible.
+func TryInlineCall(callerfn *ir.Func, call *ir.CallExpr, bigCaller bool, profile *pgoir.Profile, closureCalledOnce bool) *ir.InlinedCallExpr {
+ mightInline, isIntrinsic := inlineCallCheck(callerfn, call)
- if base.Flag.LowerM > 3 {
+ // Preserve old logging behavior
+ if (mightInline || isIntrinsic) && base.Flag.LowerM > 3 {
fmt.Printf("%v:call to func %+v\n", ir.Line(call), call.Fun)
}
- if ir.IsIntrinsicCall(call) {
+ if !mightInline {
return nil
}
- if fn := inlCallee(callerfn, call.Fun, profile); fn != nil && typecheck.HaveInlineBody(fn) {
- return mkinlcall(callerfn, call, fn, bigCaller)
+
+ if fn := inlCallee(callerfn, call.Fun, profile, false); fn != nil && typecheck.HaveInlineBody(fn) {
+ return mkinlcall(callerfn, call, fn, bigCaller, closureCalledOnce)
}
return nil
}
// inlCallee takes a function-typed expression and returns the underlying function ONAME
// that it refers to if statically known. Otherwise, it returns nil.
-func inlCallee(caller *ir.Func, fn ir.Node, profile *pgoir.Profile) (res *ir.Func) {
+// resolveOnly skips cost-based inlineability checks for closures; the result may not actually be inlineable.
+func inlCallee(caller *ir.Func, fn ir.Node, profile *pgoir.Profile, resolveOnly bool) (res *ir.Func) {
fn = ir.StaticValue(fn)
switch fn.Op() {
case ir.OMETHEXPR:
if len(c.ClosureVars) != 0 && c.ClosureVars[0].Outer.Curfn != caller {
return nil // inliner doesn't support inlining across closure frames
}
- CanInline(c, profile)
+ if !resolveOnly {
+ CanInline(c, profile)
+ }
return c
}
return nil
// - the "max cost" limit used to make the decision (which may differ depending on func size)
// - the score assigned to this specific callsite
// - whether the inlined function is "hot" according to PGO.
-func inlineCostOK(n *ir.CallExpr, caller, callee *ir.Func, bigCaller bool) (bool, int32, int32, bool) {
+func inlineCostOK(n *ir.CallExpr, caller, callee *ir.Func, bigCaller, closureCalledOnce bool) (bool, int32, int32, bool) {
maxCost := int32(inlineMaxBudget)
- if callee.ClosureParent != nil {
- maxCost *= 2 // favor inlining closures
- }
if bigCaller {
// We use this to restrict inlining into very big functions.
maxCost = inlineBigFunctionMaxCost
}
+ if callee.ClosureParent != nil {
+ maxCost *= 2 // favor inlining closures
+ if closureCalledOnce { // really favor inlining the one call to this closure
+ maxCost = max(maxCost, inlineClosureCalledOnceCost)
+ }
+ }
+
metric := callee.Inl.Cost
if inlheur.Enabled() {
score, ok := inlheur.GetCallSiteScore(caller, n)
// indicates that the 'cannot inline' reason should be logged.
//
// Preconditions: CanInline(callee) has already been called.
-func canInlineCallExpr(callerfn *ir.Func, n *ir.CallExpr, callee *ir.Func, bigCaller bool, log bool) (bool, int32, bool) {
+func canInlineCallExpr(callerfn *ir.Func, n *ir.CallExpr, callee *ir.Func, bigCaller, closureCalledOnce bool, log bool) (bool, int32, bool) {
if callee.Inl == nil {
// callee is never inlinable.
if log && logopt.Enabled() {
return false, 0, false
}
- ok, maxCost, callSiteScore, hot := inlineCostOK(n, callerfn, callee, bigCaller)
+ ok, maxCost, callSiteScore, hot := inlineCostOK(n, callerfn, callee, bigCaller, closureCalledOnce)
if !ok {
// callee cost too high for this call site.
if log && logopt.Enabled() {
// The result of mkinlcall MUST be assigned back to n, e.g.
//
// n.Left = mkinlcall(n.Left, fn, isddd)
-func mkinlcall(callerfn *ir.Func, n *ir.CallExpr, fn *ir.Func, bigCaller bool) *ir.InlinedCallExpr {
- ok, score, hot := canInlineCallExpr(callerfn, n, fn, bigCaller, true)
+func mkinlcall(callerfn *ir.Func, n *ir.CallExpr, fn *ir.Func, bigCaller, closureCalledOnce bool) *ir.InlinedCallExpr {
+ ok, score, hot := canInlineCallExpr(callerfn, n, fn, bigCaller, closureCalledOnce, true)
if !ok {
return nil
}
inline.CanInlineFuncs(pkg.Funcs, inlProfile)
inlState := make(map[*ir.Func]*inlClosureState)
+ calleeUseCounts := make(map[*ir.Func]int)
+ // Pre-process all the functions, adding parentheses around call sites and starting their "inl state".
for _, fn := range typecheck.Target.Funcs {
- // Pre-process all the functions, adding parentheses around call sites.
bigCaller := base.Flag.LowerL != 0 && inline.IsBigFunc(fn)
if bigCaller && base.Flag.LowerM > 1 {
fmt.Printf("%v: function %v considered 'big'; reducing max cost of inlinees\n", ir.Line(fn), fn)
}
- s := &inlClosureState{bigCaller: bigCaller, profile: profile, fn: fn, callSites: make(map[*ir.ParenExpr]bool)}
+ s := &inlClosureState{bigCaller: bigCaller, profile: profile, fn: fn, callSites: make(map[*ir.ParenExpr]bool), useCounts: calleeUseCounts}
s.parenthesize()
inlState[fn] = s
+
+ // Do a first pass at counting call sites.
+ for i := range s.parens {
+ s.resolve(i)
+ }
}
ir.VisitFuncsBottomUp(typecheck.Target.Funcs, func(list []*ir.Func, recursive bool) {
s := inlState[fn]
ir.WithFunc(fn, func() {
- for i := 0; i < len(s.parens); i++ { // can't use "range parens" here
- paren := s.parens[i]
- if new := s.edit(paren.X); new != nil {
- // Update AST and recursively mark nodes.
- paren.X = new
- ir.EditChildren(new, s.mark) // mark may append to parens
- done = false
+ l1 := len(s.parens)
+ l0 := 0
+
+ // Batch iterations so that newly discovered call sites are
+ // resolved in a batch before inlining attempts.
+ // Do this to avoid discovering new closure calls 1 at a time
+ // which might cause first call to be seen as a single (high-budget)
+ // call before the second is observed.
+ for {
+ for i := l0; i < l1; i++ { // can't use "range parens" here
+ paren := s.parens[i]
+ if new := s.edit(i); new != nil {
+ // Update AST and recursively mark nodes.
+ paren.X = new
+ ir.EditChildren(new, s.mark) // mark may append to parens
+ done = false
+ }
+ }
+ l0, l1 = l1, len(s.parens)
+ if l0 == l1 {
+ break
}
+ for i := l0; i < l1; i++ {
+ s.resolve(i)
+ }
+
}
+
}) // WithFunc
}
fmt.Printf("%v: function %v considered 'big'; reducing max cost of inlinees\n", ir.Line(fn), fn)
}
- s := &inlClosureState{bigCaller: bigCaller, profile: profile, fn: fn, callSites: make(map[*ir.ParenExpr]bool)}
+ s := &inlClosureState{bigCaller: bigCaller, profile: profile, fn: fn, callSites: make(map[*ir.ParenExpr]bool), useCounts: make(map[*ir.Func]int)}
s.parenthesize()
s.fixpoint()
s.unparenthesize()
})
}
+type callSite struct {
+ fn *ir.Func
+ whichParen int
+}
+
type inlClosureState struct {
fn *ir.Func
profile *pgoir.Profile
callSites map[*ir.ParenExpr]bool // callSites[p] == "p appears in parens" (do not append again)
+ resolved []*ir.Func // for each call in parens, the resolved target of the call
+ useCounts map[*ir.Func]int // shared among all InlClosureStates
parens []*ir.ParenExpr
bigCaller bool
}
-func (s *inlClosureState) edit(n ir.Node) ir.Node {
+// resolve attempts to resolve a call to a potentially inlineable callee
+// and updates use counts on the callees. Returns the call site count
+// for that callee.
+func (s *inlClosureState) resolve(i int) (*ir.Func, int) {
+ p := s.parens[i]
+ if i < len(s.resolved) {
+ if callee := s.resolved[i]; callee != nil {
+ return callee, s.useCounts[callee]
+ }
+ }
+ n := p.X
call, ok := n.(*ir.CallExpr)
if !ok { // previously inlined
- return nil
+ return nil, -1
}
-
devirtualize.StaticCall(call)
- if inlCall := inline.TryInlineCall(s.fn, call, s.bigCaller, s.profile); inlCall != nil {
+ if callee := inline.InlineCallTarget(s.fn, call, s.profile); callee != nil {
+ for len(s.resolved) <= i {
+ s.resolved = append(s.resolved, nil)
+ }
+ s.resolved[i] = callee
+ c := s.useCounts[callee] + 1
+ s.useCounts[callee] = c
+ return callee, c
+ }
+ return nil, 0
+}
+
+func (s *inlClosureState) edit(i int) ir.Node {
+ n := s.parens[i].X
+ call, ok := n.(*ir.CallExpr)
+ if !ok {
+ return nil
+ }
+ // This is redundant with earlier calls to
+ // resolve, but because things can change it
+ // must be re-checked.
+ callee, count := s.resolve(i)
+ if count <= 0 {
+ return nil
+ }
+ if inlCall := inline.TryInlineCall(s.fn, call, s.bigCaller, s.profile, count == 1 && callee.ClosureParent != nil); inlCall != nil {
return inlCall
}
return nil
done = true
for i := 0; i < len(s.parens); i++ { // can't use "range parens" here
paren := s.parens[i]
- if new := s.edit(paren.X); new != nil {
+ if new := s.edit(i); new != nil {
// Update AST and recursively mark nodes.
paren.X = new
ir.EditChildren(new, s.mark) // mark may append to parens
}
{
- func() { // ERROR "func literal does not escape"
+ func() { // ERROR "can inline main.func7"
y := func(x int) int { // ERROR "can inline main.func7.1" "func literal does not escape"
return x + 2
}
if y(40) != 41 {
ppanic("y(40) != 41")
}
- }()
+ }() // ERROR "func literal does not escape" "inlining call to main.func7"
}
{
}
{
- func() { // ERROR "func literal does not escape"
+ func() { // ERROR "can inline main.func10"
y := func(x int) int { // ERROR "can inline main.func10.1" "func literal does not escape"
return x + 2
}
if y(40) != 41 {
ppanic("y(40) != 41")
}
- }()
+ }() // ERROR "func literal does not escape" "inlining call to main.func10"
}
{
}
{
- func() { // ERROR "func literal does not escape"
+ func() { // ERROR "can inline main.func13"
y := func(x int) int { // ERROR "func literal does not escape" "can inline main.func13.1"
return x + 2
}
- y, sink = func() (func(int) int, int) { // ERROR "can inline main.func13.2"
+ y, sink = func() (func(int) int, int) { // ERROR "can inline main.func13.2" "can inline main.main.func13.func35"
return func(x int) int { // ERROR "can inline main.func13.2" "func literal escapes to heap"
return x + 1
}, 42
if y(40) != 41 {
ppanic("y(40) != 41")
}
- }()
+ }() // ERROR "func literal does not escape" "inlining call to main.func13" "inlining call to main.main.func13.func35"
}
{
}
{
- func() { // ERROR "func literal does not escape"
+ func() { // ERROR "can inline main.func16"
y := func(x int) int { // ERROR "can inline main.func16.1" "func literal does not escape"
return x + 2
}
if y(40) != 41 {
ppanic("y(40) != 41")
}
- }()
+ }() // ERROR "func literal does not escape" "inlining call to main.func16" "map\[int\]func\(int\) int{...} does not escape" "func literal escapes to heap"
}
{
}
{
- func() { // ERROR "func literal does not escape"
+ func() { // ERROR "can inline main.func19"
y := func(x int) int { // ERROR "can inline main.func19.1" "func literal does not escape"
return x + 2
}
if y(40) != 41 {
ppanic("y(40) != 41")
}
- }()
+ }() // ERROR "func literal does not escape" "inlining call to main.func19"
}
{
{
x := 42
if z := func(y int) int { // ERROR "can inline main.func22"
- return func() int { // ERROR "can inline main.func22.1" "can inline main.main.func22.func30"
+ return func() int { // ERROR "can inline main.func22.1" "can inline main.main.func22.func40"
return x + y
}() // ERROR "inlining call to main.func22.1"
- }(1); z != 43 { // ERROR "inlining call to main.func22" "inlining call to main.main.func22.func30"
+ }(1); z != 43 { // ERROR "inlining call to main.func22" "inlining call to main.main.func22.func40"
ppanic("z != 43")
}
if z := func(y int) int { // ERROR "func literal does not escape" "can inline main.func23"
- return func() int { // ERROR "can inline main.func23.1" "can inline main.main.func23.func31"
+ return func() int { // ERROR "can inline main.func23.1" "can inline main.main.func23.func41"
return x + y
}() // ERROR "inlining call to main.func23.1"
- }; z(1) != 43 { // ERROR "inlining call to main.func23" "inlining call to main.main.func23.func31"
+ }; z(1) != 43 { // ERROR "inlining call to main.func23" "inlining call to main.main.func23.func41"
_ = z // prevent simple deadcode elimination after inlining
ppanic("z(1) != 43")
}
{
a := 1
func() { // ERROR "can inline main.func24"
- func() { // ERROR "can inline main.func24" "can inline main.main.func24.func32"
+ func() { // ERROR "can inline main.func24" "can inline main.main.func24.func42"
a = 2
}() // ERROR "inlining call to main.func24"
- }() // ERROR "inlining call to main.func24" "inlining call to main.main.func24.func32"
+ }() // ERROR "inlining call to main.func24" "inlining call to main.main.func24.func42"
if a != 2 {
ppanic("a != 2")
}
{
b := 2
func(b int) { // ERROR "can inline main.func25"
- func() { // ERROR "can inline main.func25.1" "can inline main.main.func25.func33"
+ func() { // ERROR "can inline main.func25.1" "can inline main.main.func25.func43"
b = 3
}() // ERROR "inlining call to main.func25.1"
if b != 3 {
ppanic("b != 3")
}
- }(b) // ERROR "inlining call to main.func25" "inlining call to main.main.func25.func33"
+ }(b) // ERROR "inlining call to main.func25" "inlining call to main.main.func25.func43"
if b != 2 {
ppanic("b != 2")
}
// revisit those. E.g., func34 and func36 are constructed by the inliner.
if r := func(x int) int { // ERROR "can inline main.func27"
b := 3
- return func(y int) int { // ERROR "can inline main.func27.1" "can inline main.main.func27.func35"
+ return func(y int) int { // ERROR "can inline main.func27.1" "can inline main.main.func27.func45"
c := 5
- return func(z int) int { // ERROR "can inline main.func27.1.1" "can inline main.main.func27.func35.1" "can inline main.func27.main.func27.1.2" "can inline main.main.func27.main.main.func27.func35.func37"
+ return func(z int) int { // ERROR "can inline main.func27.1.1" "can inline main.main.func27.func45.1" "can inline main.func27.main.func27.1.2" "can inline main.main.func27.main.main.func27.func45.func48"
return a*x + b*y + c*z
}(10) // ERROR "inlining call to main.func27.1.1"
}(100) // ERROR "inlining call to main.func27.1" "inlining call to main.func27.main.func27.1.2"
- }(1000); r != 2350 { // ERROR "inlining call to main.func27" "inlining call to main.main.func27.func35" "inlining call to main.main.func27.main.main.func27.func35.func37"
+ }(1000); r != 2350 { // ERROR "inlining call to main.func27" "inlining call to main.main.func27.func45" "inlining call to main.main.func27.main.main.func27.func45.func48"
ppanic("r != 2350")
}
}
a := 2
if r := func(x int) int { // ERROR "can inline main.func28"
b := 3
- return func(y int) int { // ERROR "can inline main.func28.1" "can inline main.main.func28.func36"
+ return func(y int) int { // ERROR "can inline main.func28.1" "can inline main.main.func28.func46"
c := 5
- func(z int) { // ERROR "can inline main.func28.1.1" "can inline main.func28.main.func28.1.2" "can inline main.main.func28.func36.1" "can inline main.main.func28.main.main.func28.func36.func38"
+ func(z int) { // ERROR "can inline main.func28.1.1" "can inline main.func28.main.func28.1.2" "can inline main.main.func28.func46.1" "can inline main.main.func28.main.main.func28.func46.func49"
a = a * x
b = b * y
c = c * z
}(10) // ERROR "inlining call to main.func28.1.1"
return a + c
}(100) + b // ERROR "inlining call to main.func28.1" "inlining call to main.func28.main.func28.1.2"
- }(1000); r != 2350 { // ERROR "inlining call to main.func28" "inlining call to main.main.func28.func36" "inlining call to main.main.func28.main.main.func28.func36.func38"
+ }(1000); r != 2350 { // ERROR "inlining call to main.func28" "inlining call to main.main.func28.func46" "inlining call to main.main.func28.main.main.func28.func46.func49"
ppanic("r != 2350")
}
if a != 2000 {