[release-branch.go1.24] cmd/compile: fix out of memory when inlining closure

CL 629195 strongly favor closure inlining, allowing closures to be
inlined more aggressively.

However, if the closure body contains a call to a function, which itself
is one of the call arguments, it causes the infinite inlining.

Fixing this by prevent this kind of functions from being inlinable.

Fixes #72067

Change-Id: I5fb5723a819b1e2c5aadb57c1023ec84ca9fa53c
Reviewed-on: https://go-review.googlesource.com/c/go/+/654195
Reviewed-by: David Chase <drchase@google.com>
Reviewed-by: Dmitri Shuralyov <dmitshur@google.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-on: https://go-review.googlesource.com/c/go/+/654517
Commit-Queue: Junyang Shao <shaojunyang@google.com>
Auto-Submit: Junyang Shao <shaojunyang@google.com>
Reviewed-by: Junyang Shao <shaojunyang@google.com>
Reviewed-by: Jorropo <jorropo.pgm@gmail.com>

diff --git a/src/cmd/compile/internal/inline/inl.go b/src/cmd/compile/internal/inline/inl.go
index a8809f368283168758d74ca7c523f3c20d106851..f17af5283a5520765765a9b3d933878dbecd890a 100644 (file)
--- a/src/cmd/compile/internal/inline/inl.go
+++ b/src/cmd/compile/internal/inline/inl.go
@@ -1040,6 +1040,28 @@ func canInlineCallExpr(callerfn *ir.Func, n *ir.CallExpr, callee *ir.Func, bigCa
                        return false, 0, false
                }
        }
+       do := func(fn *ir.Func) bool {
+               // Can't recursively inline a function if the function body contains
+               // a call to a function f, which the function f is one of the call arguments.
+               return ir.Any(fn, func(node ir.Node) bool {
+                       if call, ok := node.(*ir.CallExpr); ok {
+                               for _, arg := range call.Args {
+                                       if call.Fun == arg {
+                                               return true
+                                       }
+                               }
+                       }
+                       return false
+               })
+       }
+       for _, fn := range []*ir.Func{callerfn, callee} {
+               if do(fn) {
+                       if log && logopt.Enabled() {
+                               logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", fmt.Sprintf("recursive call to function: %s", ir.FuncName(fn)))
+                       }
+                       return false, 0, false
+               }
+       }
 
        if base.Flag.Cfg.Instrumenting && types.IsNoInstrumentPkg(callee.Sym().Pkg) {
                // Runtime package must not be instrumented.

diff --git a/test/fixedbugs/issue72063.go b/test/fixedbugs/issue72063.go
new file mode 100644 (file)
index 0000000..7b9c13e
--- /dev/null
+++ b/test/fixedbugs/issue72063.go
@@ -0,0 +1,40 @@
+// run
+
+// Copyright 2025 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import "fmt"
+
+// Y is the Y-combinator based on https://dreamsongs.com/Files/WhyOfY.pdf
+func Y[Endo ~func(RecFct) RecFct, RecFct ~func(T) R, T, R any](f Endo) RecFct {
+
+       type internal[RecFct ~func(T) R, T, R any] func(internal[RecFct, T, R]) RecFct
+
+       g := func(h internal[RecFct, T, R]) RecFct {
+               return func(t T) R {
+                       return f(h(h))(t)
+               }
+       }
+       return g(g)
+}
+
+func main() {
+
+       fct := Y(func(r func(int) int) func(int) int {
+               return func(n int) int {
+                       if n <= 0 {
+                               return 1
+                       }
+                       return n * r(n-1)
+               }
+       })
+
+       want := 3628800
+       if got := fct(10); got != want {
+               msg := fmt.Sprintf("unexpected result, got: %d, want: %d", got, want)
+               panic(msg)
+       }
+}