[release-branch.go1.24] cmd/compile: avoid infinite recursion when inlining closures

CL 630696 changes budget for once-called closures, making them more
inlinable. However, when recursive inlining involve both the closure and
its parent, the inliner goes into an infinite loop:

	parent (a closure)  -> closure -> parent -> ...

The problem here dues to the closure name mangling, causing the inlined
checking condition failed, since the closure name affects how the
linker symbol generated.

To fix this, just prevent the closure from inlining its parent into
itself, avoid the infinite inlining loop.

Fixes #71829

Change-Id: Ib27626d70f95e5f1c24a3eb1c8e6c3443b7d90c8
Reviewed-on: https://go-review.googlesource.com/c/go/+/649656
Reviewed-by: David Chase <drchase@google.com>
Auto-Submit: Cuong Manh Le <cuong.manhle.vn@gmail.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Michael Knyszek <mknyszek@google.com>
Reviewed-on: https://go-review.googlesource.com/c/go/+/650555

diff --git a/src/cmd/compile/internal/inline/inl.go b/src/cmd/compile/internal/inline/inl.go
index f298f69ec1b6a7a0a3cecd06b9d82da35eef225c..a8809f368283168758d74ca7c523f3c20d106851 100644 (file)
--- a/src/cmd/compile/internal/inline/inl.go
+++ b/src/cmd/compile/internal/inline/inl.go
@@ -1009,6 +1009,38 @@ func canInlineCallExpr(callerfn *ir.Func, n *ir.CallExpr, callee *ir.Func, bigCa
                return false, 0, false
        }
 
+       isClosureParent := func(closure, parent *ir.Func) bool {
+               for p := closure.ClosureParent; p != nil; p = p.ClosureParent {
+                       if p == parent {
+                               return true
+                       }
+               }
+               return false
+       }
+       if isClosureParent(callerfn, callee) {
+               // Can't recursively inline a parent of the closure into itself.
+               if log && logopt.Enabled() {
+                       logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", fmt.Sprintf("recursive call to closure parent: %s, %s", ir.FuncName(callerfn), ir.FuncName(callee)))
+               }
+               return false, 0, false
+       }
+       if isClosureParent(callee, callerfn) {
+               // Can't recursively inline a closure if there's a call to the parent in closure body.
+               if ir.Any(callee, func(node ir.Node) bool {
+                       if call, ok := node.(*ir.CallExpr); ok {
+                               if name, ok := call.Fun.(*ir.Name); ok && isClosureParent(callerfn, name.Func) {
+                                       return true
+                               }
+                       }
+                       return false
+               }) {
+                       if log && logopt.Enabled() {
+                               logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", fmt.Sprintf("recursive call to closure parent: %s, %s", ir.FuncName(callerfn), ir.FuncName(callee)))
+                       }
+                       return false, 0, false
+               }
+       }
+
        if base.Flag.Cfg.Instrumenting && types.IsNoInstrumentPkg(callee.Sym().Pkg) {
                // Runtime package must not be instrumented.
                // Instrument skips runtime package. However, some runtime code can be

diff --git a/test/fixedbugs/issue71680.go b/test/fixedbugs/issue71680.go
new file mode 100644 (file)
index 0000000..1013b8f
--- /dev/null
+++ b/test/fixedbugs/issue71680.go
@@ -0,0 +1,28 @@
+// compile
+
+// Copyright 2025 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package p
+
+type Parser struct{}
+type Node struct{}
+
+type parserState func(p *Parser) parserState
+
+func parserStateData(root *Node) parserState {
+       return func(p *Parser) parserState {
+               return parserStateOpenMap(root)(p)
+       }
+}
+
+func parserStateOpenMap(root *Node) parserState {
+       return func(p *Parser) parserState {
+               switch {
+               case p != nil:
+                       return parserStateData(root)(p)
+               }
+               return parserStateOpenMap(root)(p)
+       }
+}