cmd/compile: reorder operations in SCCs to enable more inlining

This patch changes the relative order of "CanInline" and "InlineCalls"
operations within the inliner for clumps of functions corresponding to
strongly connected components in the call graph. This helps increase
the amount of inlining within SCCs, particularly in Go's runtime
package, which has a couple of very large SCCs.

For a given SCC of the form { fn1, fn2, ... fnk }, the inliner would
(prior to this point) walk through the list of functions and for each
function first compute inlinability ("CanInline") and then perform
inlining ("InlineCalls"). This meant that if there was an inlinable
call from fn3 to fn4 (for example), this call would never be inlined,
since at the point fn3 was visited, we would not have computed
inlinability for fn4.

We now do inlinability analysis for all functions in an SCC first,
then do actual inlining for everything. This results in 47 additional
inlines in the Go runtime package (a fairly modest increase
percentage-wise of 0.6%).

Updates #58905.

Change-Id: I48dbb1ca16f0b12f256d9eeba8cf7f3e6dd853cd
Reviewed-on: https://go-review.googlesource.com/c/go/+/474955
Run-TryBot: Than McIntosh <thanm@google.com>
Reviewed-by: Matthew Dempsky <mdempsky@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: Austin Clements <austin@google.com>

diff --git a/src/cmd/compile/internal/base/debug.go b/src/cmd/compile/internal/base/debug.go
index ec20b181349f7d1c0698484164120071be54d0cf..81e8ed645dfa8fe523337f692f0645d25b2758af 100644 (file)
--- a/src/cmd/compile/internal/base/debug.go
+++ b/src/cmd/compile/internal/base/debug.go
@@ -32,6 +32,7 @@ type DebugFlags struct {
        InlFuncsWithClosures  int    `help:"allow functions with closures to be inlined" concurrent:"ok"`
        InlStaticInit         int    `help:"allow static initialization of inlined calls" concurrent:"ok"`
        InterfaceCycles       int    `help:"allow anonymous interface cycles"`
+       InlineSCCOnePass      int    `help:"visit SCC funcs only once during inlining (legacy behavior)"`
        Libfuzzer             int    `help:"enable coverage instrumentation for libfuzzer"`
        LoopVar               int    `help:"shared (0, default), 1 (private loop variables), 2, private + log"`
        LoopVarHash           string `help:"for debugging changes in loop behavior. Overrides experiment and loopvar flag."`

diff --git a/src/cmd/compile/internal/inline/inl.go b/src/cmd/compile/internal/inline/inl.go
index 03f565e9d3c1763deb0cc16bf99e856b7ba3a47d..80be841efa22b1b7393461fc7f86a85dfa4bb11f 100644 (file)
--- a/src/cmd/compile/internal/inline/inl.go
+++ b/src/cmd/compile/internal/inline/inl.go
@@ -186,21 +186,45 @@ func InlineDecls(p *pgo.Profile, decls []ir.Node, doInline bool) {
                pgoInlinePrologue(p, decls)
        }
 
+       doCanInline := func(n *ir.Func, recursive bool, numfns int) {
+               if !recursive || numfns > 1 {
+                       // We allow inlining if there is no
+                       // recursion, or the recursion cycle is
+                       // across more than one function.
+                       CanInline(n, p)
+               } else {
+                       if base.Flag.LowerM > 1 && n.OClosure == nil {
+                               fmt.Printf("%v: cannot inline %v: recursive\n", ir.Line(n), n.Nname)
+                       }
+               }
+       }
+
        ir.VisitFuncsBottomUp(decls, func(list []*ir.Func, recursive bool) {
                numfns := numNonClosures(list)
-               for _, n := range list {
-                       if !recursive || numfns > 1 {
-                               // We allow inlining if there is no
-                               // recursion, or the recursion cycle is
-                               // across more than one function.
-                               CanInline(n, p)
-                       } else {
-                               if base.Flag.LowerM > 1 && n.OClosure == nil {
-                                       fmt.Printf("%v: cannot inline %v: recursive\n", ir.Line(n), n.Nname)
-                               }
+               // We visit functions within an SCC in fairly arbitrary order,
+               // so by computing inlinability for all functions in the SCC
+               // before performing any inlining, the results are less
+               // sensitive to the order within the SCC (see #58905 for an
+               // example).
+               if base.Debug.InlineSCCOnePass == 0 {
+                       // Compute inlinability for all functions in the SCC ...
+                       for _, n := range list {
+                               doCanInline(n, recursive, numfns)
                        }
+                       // ... then make a second pass to do inlining of calls.
                        if doInline {
-                               InlineCalls(n, p)
+                               for _, n := range list {
+                                       InlineCalls(n, p)
+                               }
+                       }
+               } else {
+                       // Legacy ordering to make it easier to triage any bugs
+                       // or compile time issues that might crop up.
+                       for _, n := range list {
+                               doCanInline(n, recursive, numfns)
+                               if doInline {
+                                       InlineCalls(n, p)
+                               }
                        }
                }
        })

diff --git a/test/inline.go b/test/inline.go
index 1aa8fccbbdc62509d2f23789c680137ab4491d0d..3bc102f769c2a63e8cdd911dd94f6af4cb0feb9d 100644 (file)
--- a/test/inline.go
+++ b/test/inline.go
@@ -246,13 +246,13 @@ func ff(x int) { // ERROR "can inline ff"
        if x < 0 {
                return
        }
-       gg(x - 1)
+       gg(x - 1) // ERROR "inlining call to gg" "inlining call to hh"
 }
 func gg(x int) { // ERROR "can inline gg"
-       hh(x - 1)
+       hh(x - 1) // ERROR "inlining call to hh" "inlining call to ff"
 }
 func hh(x int) { // ERROR "can inline hh"
-       ff(x - 1) // ERROR "inlining call to ff"  // ERROR "inlining call to gg"
+       ff(x - 1) // ERROR "inlining call to ff" "inlining call to gg"
 }
 
 // Issue #14768 - make sure we can inline for loops.