cmd/compile: restore tail call for method wrappers

For certain type of method wrappers we used to generate a tail
call. That was disabled in CL 307234 when register ABI is used,
because with the current IR it was difficult to generate a tail
call with the arguments in the right places. The problem was that
the IR does not contain a CALL-like node with arguments; instead,
it contains an OAS node that adjusts the receiver, than an
OTAILCALL node that just contains the target, but no argument
(with the assumption that the OAS node will put the adjusted
receiver in the right place). With register ABI, putting
arguments in registers are done in SSA. The assignment (OAS)
doesn't put the receiver in register.

This CL changes the IR of a tail call to take an actual OCALL
node. Specifically, a tail call is represented as

OTAILCALL (OCALL target args...)

This way, the call target and args are connected through the OCALL
node. So the call can be analyzed in SSA and the args can be passed
in the right places.

(Alternatively, we could have OTAILCALL node directly take the
target and the args, without the OCALL node. Using an OCALL node is
convenient as there are existing code that processes OCALL nodes
which do not need to be changed. Also, a tail call is similar to
ORETURN (OCALL target args...), except it doesn't preserve the
frame. I did the former but I'm open to change.)

The SSA representation is similar. Previously, the IR lowers to
a Store the receiver then a BlockRetJmp which jumps to the target
(without putting the arg in register). Now we use a TailCall op,
which takes the target and the args. The call expansion pass and
the register allocator handles TailCall pretty much like a
StaticCall, and it will do the right ABI analysis and put the args
in the right places. (Args other than the receiver are already in
the right places. For register args it generates no code for them.
For stack args currently it generates a self copy. I'll work on
optimize that out.) BlockRetJmp is still used, signaling it is a
tail call. The actual call is made in the TailCall op so
BlockRetJmp generates no code (we could use BlockExit if we like).

This slightly reduces binary size:
              old        new
cmd/go     14003088   13953936
cmd/link    6275552    6271456

Change-Id: I2d16d8d419fe1f17554916d317427383e17e27f0
Reviewed-on: https://go-review.googlesource.com/c/go/+/350145
Trust: Cherry Mui <cherryyz@google.com>
Run-TryBot: Cherry Mui <cherryyz@google.com>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Matthew Dempsky <mdempsky@google.com>
Reviewed-by: David Chase <drchase@google.com>

diff --git a/src/cmd/compile/internal/amd64/ssa.go b/src/cmd/compile/internal/amd64/ssa.go
index fc547ebba0177ee651071613ef884d88c919d0a6..30131bd5590ee8292e8de10b7fa12eff1b520995 100644 (file)
--- a/src/cmd/compile/internal/amd64/ssa.go
+++ b/src/cmd/compile/internal/amd64/ssa.go
@@ -1008,7 +1008,7 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
                }
                r := v.Reg()
                getgFromTLS(s, r)
-       case ssa.OpAMD64CALLstatic:
+       case ssa.OpAMD64CALLstatic, ssa.OpAMD64CALLtail:
                if s.ABI == obj.ABI0 && v.Aux.(*ssa.AuxCall).Fn.ABI() == obj.ABIInternal {
                        // zeroing X15 when entering ABIInternal from ABI0
                        if buildcfg.GOOS != "plan9" { // do not use SSE on Plan 9
@@ -1017,6 +1017,10 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
                        // set G register from TLS
                        getgFromTLS(s, x86.REG_R14)
                }
+               if v.Op == ssa.OpAMD64CALLtail {
+                       s.TailCall(v)
+                       break
+               }
                s.Call(v)
                if s.ABI == obj.ABIInternal && v.Aux.(*ssa.AuxCall).Fn.ABI() == obj.ABI0 {
                        // zeroing X15 when entering ABIInternal from ABI0
@@ -1314,22 +1318,9 @@ func ssaGenBlock(s *ssagen.State, b, next *ssa.Block) {
                        p.To.Type = obj.TYPE_BRANCH
                        s.Branches = append(s.Branches, ssagen.Branch{P: p, B: b.Succs[0].Block()})
                }
-       case ssa.BlockExit:
+       case ssa.BlockExit, ssa.BlockRetJmp:
        case ssa.BlockRet:
                s.Prog(obj.ARET)
-       case ssa.BlockRetJmp:
-               if s.ABI == obj.ABI0 && b.Aux.(*obj.LSym).ABI() == obj.ABIInternal {
-                       // zeroing X15 when entering ABIInternal from ABI0
-                       if buildcfg.GOOS != "plan9" { // do not use SSE on Plan 9
-                               opregreg(s, x86.AXORPS, x86.REG_X15, x86.REG_X15)
-                       }
-                       // set G register from TLS
-                       getgFromTLS(s, x86.REG_R14)
-               }
-               p := s.Prog(obj.ARET)
-               p.To.Type = obj.TYPE_MEM
-               p.To.Name = obj.NAME_EXTERN
-               p.To.Sym = b.Aux.(*obj.LSym)
 
        case ssa.BlockAMD64EQF:
                s.CombJump(b, next, &eqfJumps)

diff --git a/src/cmd/compile/internal/arm/ssa.go b/src/cmd/compile/internal/arm/ssa.go
index 4b083cec46b4a5cff59981de626a75f62fc0e1b7..8aac80a22eca0dea2a2f8446c2af242dcfe3cc60 100644 (file)
--- a/src/cmd/compile/internal/arm/ssa.go
+++ b/src/cmd/compile/internal/arm/ssa.go
@@ -696,6 +696,8 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
                p.To.Reg = v.Reg()
        case ssa.OpARMCALLstatic, ssa.OpARMCALLclosure, ssa.OpARMCALLinter:
                s.Call(v)
+       case ssa.OpARMCALLtail:
+               s.TailCall(v)
        case ssa.OpARMCALLudiv:
                p := s.Prog(obj.ACALL)
                p.To.Type = obj.TYPE_MEM
@@ -936,17 +938,11 @@ func ssaGenBlock(s *ssagen.State, b, next *ssa.Block) {
                        s.Branches = append(s.Branches, ssagen.Branch{P: p, B: b.Succs[0].Block()})
                }
 
-       case ssa.BlockExit:
+       case ssa.BlockExit, ssa.BlockRetJmp:
 
        case ssa.BlockRet:
                s.Prog(obj.ARET)
 
-       case ssa.BlockRetJmp:
-               p := s.Prog(obj.ARET)
-               p.To.Type = obj.TYPE_MEM
-               p.To.Name = obj.NAME_EXTERN
-               p.To.Sym = b.Aux.(*obj.LSym)
-
        case ssa.BlockARMEQ, ssa.BlockARMNE,
                ssa.BlockARMLT, ssa.BlockARMGE,
                ssa.BlockARMLE, ssa.BlockARMGT,

diff --git a/src/cmd/compile/internal/arm64/ssa.go b/src/cmd/compile/internal/arm64/ssa.go
index b985246117d95d0eac4fda43b251069c98b3d827..c5e7a0891482942057146569e7c84630dd00a254 100644 (file)
--- a/src/cmd/compile/internal/arm64/ssa.go
+++ b/src/cmd/compile/internal/arm64/ssa.go
@@ -1046,6 +1046,8 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
                p4.To.SetTarget(p)
        case ssa.OpARM64CALLstatic, ssa.OpARM64CALLclosure, ssa.OpARM64CALLinter:
                s.Call(v)
+       case ssa.OpARM64CALLtail:
+               s.TailCall(v)
        case ssa.OpARM64LoweredWB:
                p := s.Prog(obj.ACALL)
                p.To.Type = obj.TYPE_MEM
@@ -1241,17 +1243,11 @@ func ssaGenBlock(s *ssagen.State, b, next *ssa.Block) {
                        s.Branches = append(s.Branches, ssagen.Branch{P: p, B: b.Succs[0].Block()})
                }
 
-       case ssa.BlockExit:
+       case ssa.BlockExit, ssa.BlockRetJmp:
 
        case ssa.BlockRet:
                s.Prog(obj.ARET)
 
-       case ssa.BlockRetJmp:
-               p := s.Prog(obj.ARET)
-               p.To.Type = obj.TYPE_MEM
-               p.To.Name = obj.NAME_EXTERN
-               p.To.Sym = b.Aux.(*obj.LSym)
-
        case ssa.BlockARM64EQ, ssa.BlockARM64NE,
                ssa.BlockARM64LT, ssa.BlockARM64GE,
                ssa.BlockARM64LE, ssa.BlockARM64GT,

diff --git a/src/cmd/compile/internal/escape/stmt.go b/src/cmd/compile/internal/escape/stmt.go
index c71848b8a1ca678c0c78934d1b75c44e686b997f..0afb5d64ef681433aa031e2334da33c71d3e3991 100644 (file)
--- a/src/cmd/compile/internal/escape/stmt.go
+++ b/src/cmd/compile/internal/escape/stmt.go
@@ -180,7 +180,8 @@ func (e *escape) stmt(n ir.Node) {
                e.goDeferStmt(n)
 
        case ir.OTAILCALL:
-               // TODO(mdempsky): Treat like a normal call? esc.go used to just ignore it.
+               n := n.(*ir.TailCallStmt)
+               e.call(nil, n.Call)
        }
 }
 

diff --git a/src/cmd/compile/internal/inline/inl.go b/src/cmd/compile/internal/inline/inl.go
index 073373144d4a6c45b4a30b818114708fb6d3b610..04d751869b4ff874c66a9652882cd061d455c6da 100644 (file)
--- a/src/cmd/compile/internal/inline/inl.go
+++ b/src/cmd/compile/internal/inline/inl.go
@@ -544,6 +544,9 @@ func inlnode(n ir.Node, maxCost int32, inlMap map[*ir.Func]bool, edit func(ir.No
                        call := call.(*ir.CallExpr)
                        call.NoInline = true
                }
+       case ir.OTAILCALL:
+               n := n.(*ir.TailCallStmt)
+               n.Call.NoInline = true // Not inline a tail call for now. Maybe we could inline it just like RETURN fn(arg)?
 
        // TODO do them here (or earlier),
        // so escape analysis can avoid more heapmoves.

diff --git a/src/cmd/compile/internal/ir/fmt.go b/src/cmd/compile/internal/ir/fmt.go
index a99cb5ed9872c31c2cab22dbfa86b0b139b78e4b..29505357ccaa9325bc29c17bbc6541ac0cdc704a 100644 (file)
--- a/src/cmd/compile/internal/ir/fmt.go
+++ b/src/cmd/compile/internal/ir/fmt.go
@@ -386,7 +386,7 @@ func stmtFmt(n Node, s fmt.State) {
 
        case OTAILCALL:
                n := n.(*TailCallStmt)
-               fmt.Fprintf(s, "tailcall %v", n.Target)
+               fmt.Fprintf(s, "tailcall %v", n.Call)
 
        case OINLMARK:
                n := n.(*InlineMarkStmt)

diff --git a/src/cmd/compile/internal/ir/node_gen.go b/src/cmd/compile/internal/ir/node_gen.go
index aa41c03beb97dd9f0d136cdcd6b15409de733850..44988880c8c2a7a52a37517cc57d361e3b3ab144 100644 (file)
--- a/src/cmd/compile/internal/ir/node_gen.go
+++ b/src/cmd/compile/internal/ir/node_gen.go
@@ -1331,15 +1331,15 @@ func (n *TailCallStmt) doChildren(do func(Node) bool) bool {
        if doNodes(n.init, do) {
                return true
        }
-       if n.Target != nil && do(n.Target) {
+       if n.Call != nil && do(n.Call) {
                return true
        }
        return false
 }
 func (n *TailCallStmt) editChildren(edit func(Node) Node) {
        editNodes(n.init, edit)
-       if n.Target != nil {
-               n.Target = edit(n.Target).(*Name)
+       if n.Call != nil {
+               n.Call = edit(n.Call).(*CallExpr)
        }
 }
 

diff --git a/src/cmd/compile/internal/ir/stmt.go b/src/cmd/compile/internal/ir/stmt.go
index 69a74b9fdd6c402a7f6d619a18dde68df07fea8f..3482d7972ec56216f20e5ff48f63227a3d88ddbf 100644 (file)
--- a/src/cmd/compile/internal/ir/stmt.go
+++ b/src/cmd/compile/internal/ir/stmt.go
@@ -385,14 +385,11 @@ func NewSwitchStmt(pos src.XPos, tag Node, cases []*CaseClause) *SwitchStmt {
 // code generation to jump directly to another function entirely.
 type TailCallStmt struct {
        miniStmt
-       Target *Name
+       Call *CallExpr // the underlying call
 }
 
-func NewTailCallStmt(pos src.XPos, target *Name) *TailCallStmt {
-       if target.Op() != ONAME || target.Class != PFUNC {
-               base.FatalfAt(pos, "tail call to non-func %v", target)
-       }
-       n := &TailCallStmt{Target: target}
+func NewTailCallStmt(pos src.XPos, call *CallExpr) *TailCallStmt {
+       n := &TailCallStmt{Call: call}
        n.pos = pos
        n.op = OTAILCALL
        return n

diff --git a/src/cmd/compile/internal/mips/ssa.go b/src/cmd/compile/internal/mips/ssa.go
index e0447f38cbf23347c05a831d304581259eb9b092..6326f966bf2158807d6bdab361a19393c884f831 100644 (file)
--- a/src/cmd/compile/internal/mips/ssa.go
+++ b/src/cmd/compile/internal/mips/ssa.go
@@ -475,6 +475,8 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
                p6.To.SetTarget(p2)
        case ssa.OpMIPSCALLstatic, ssa.OpMIPSCALLclosure, ssa.OpMIPSCALLinter:
                s.Call(v)
+       case ssa.OpMIPSCALLtail:
+               s.TailCall(v)
        case ssa.OpMIPSLoweredWB:
                p := s.Prog(obj.ACALL)
                p.To.Type = obj.TYPE_MEM
@@ -841,14 +843,9 @@ func ssaGenBlock(s *ssagen.State, b, next *ssa.Block) {
                        p.To.Type = obj.TYPE_BRANCH
                        s.Branches = append(s.Branches, ssagen.Branch{P: p, B: b.Succs[0].Block()})
                }
-       case ssa.BlockExit:
+       case ssa.BlockExit, ssa.BlockRetJmp:
        case ssa.BlockRet:
                s.Prog(obj.ARET)
-       case ssa.BlockRetJmp:
-               p := s.Prog(obj.ARET)
-               p.To.Type = obj.TYPE_MEM
-               p.To.Name = obj.NAME_EXTERN
-               p.To.Sym = b.Aux.(*obj.LSym)
        case ssa.BlockMIPSEQ, ssa.BlockMIPSNE,
                ssa.BlockMIPSLTZ, ssa.BlockMIPSGEZ,
                ssa.BlockMIPSLEZ, ssa.BlockMIPSGTZ,

diff --git a/src/cmd/compile/internal/mips64/ssa.go b/src/cmd/compile/internal/mips64/ssa.go
index e821a00876fa799bc6b0a4413ed48acb445e336e..990b9788f765e39974873f07f544b218b171df6f 100644 (file)
--- a/src/cmd/compile/internal/mips64/ssa.go
+++ b/src/cmd/compile/internal/mips64/ssa.go
@@ -491,6 +491,8 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
                p6.To.SetTarget(p2)
        case ssa.OpMIPS64CALLstatic, ssa.OpMIPS64CALLclosure, ssa.OpMIPS64CALLinter:
                s.Call(v)
+       case ssa.OpMIPS64CALLtail:
+               s.TailCall(v)
        case ssa.OpMIPS64LoweredWB:
                p := s.Prog(obj.ACALL)
                p.To.Type = obj.TYPE_MEM
@@ -808,14 +810,9 @@ func ssaGenBlock(s *ssagen.State, b, next *ssa.Block) {
                        p.To.Type = obj.TYPE_BRANCH
                        s.Branches = append(s.Branches, ssagen.Branch{P: p, B: b.Succs[0].Block()})
                }
-       case ssa.BlockExit:
+       case ssa.BlockExit, ssa.BlockRetJmp:
        case ssa.BlockRet:
                s.Prog(obj.ARET)
-       case ssa.BlockRetJmp:
-               p := s.Prog(obj.ARET)
-               p.To.Type = obj.TYPE_MEM
-               p.To.Name = obj.NAME_EXTERN
-               p.To.Sym = b.Aux.(*obj.LSym)
        case ssa.BlockMIPS64EQ, ssa.BlockMIPS64NE,
                ssa.BlockMIPS64LTZ, ssa.BlockMIPS64GEZ,
                ssa.BlockMIPS64LEZ, ssa.BlockMIPS64GTZ,

diff --git a/src/cmd/compile/internal/ppc64/ssa.go b/src/cmd/compile/internal/ppc64/ssa.go
index 11226f65a0f01639359fe2992e2f266b2301be20..e366e06949dca29a40d9c26fc912155fa6f2dc61 100644 (file)
--- a/src/cmd/compile/internal/ppc64/ssa.go
+++ b/src/cmd/compile/internal/ppc64/ssa.go
@@ -1829,6 +1829,9 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
        case ssa.OpPPC64CALLstatic:
                s.Call(v)
 
+       case ssa.OpPPC64CALLtail:
+               s.TailCall(v)
+
        case ssa.OpPPC64CALLclosure, ssa.OpPPC64CALLinter:
                p := s.Prog(ppc64.AMOVD)
                p.From.Type = obj.TYPE_REG
@@ -1980,14 +1983,9 @@ func ssaGenBlock(s *ssagen.State, b, next *ssa.Block) {
                        p.To.Type = obj.TYPE_BRANCH
                        s.Branches = append(s.Branches, ssagen.Branch{P: p, B: b.Succs[0].Block()})
                }
-       case ssa.BlockExit:
+       case ssa.BlockExit, ssa.BlockRetJmp:
        case ssa.BlockRet:
                s.Prog(obj.ARET)
-       case ssa.BlockRetJmp:
-               p := s.Prog(obj.AJMP)
-               p.To.Type = obj.TYPE_MEM
-               p.To.Name = obj.NAME_EXTERN
-               p.To.Sym = b.Aux.(*obj.LSym)
 
        case ssa.BlockPPC64EQ, ssa.BlockPPC64NE,
                ssa.BlockPPC64LT, ssa.BlockPPC64GE,

diff --git a/src/cmd/compile/internal/reflectdata/reflect.go b/src/cmd/compile/internal/reflectdata/reflect.go
index f42bb338d0ee64f03e2a32446a14e10f5f986ce6..6dbe3cb455c7d56360ba5d6695bace0de843ee26 100644 (file)
--- a/src/cmd/compile/internal/reflectdata/reflect.go
+++ b/src/cmd/compile/internal/reflectdata/reflect.go
@@ -7,7 +7,6 @@ package reflectdata
 import (
        "encoding/binary"
        "fmt"
-       "internal/buildcfg"
        "os"
        "sort"
        "strings"
@@ -1869,15 +1868,11 @@ func methodWrapper(rcvr *types.Type, method *types.Field, forItab bool) *obj.LSy
        // Disable tailcall for RegabiArgs for now. The IR does not connect the
        // arguments with the OTAILCALL node, and the arguments are not marshaled
        // correctly.
-       if !base.Flag.Cfg.Instrumenting && rcvr.IsPtr() && methodrcvr.IsPtr() && method.Embedded != 0 && !types.IsInterfaceMethod(method.Type) && !(base.Ctxt.Arch.Name == "ppc64le" && base.Ctxt.Flag_dynlink) && !buildcfg.Experiment.RegabiArgs && !generic {
-               // generate tail call: adjust pointer receiver and jump to embedded method.
-               left := dot.X // skip final .M
-               if !left.Type().IsPtr() {
-                       left = typecheck.NodAddr(left)
-               }
-               as := ir.NewAssignStmt(base.Pos, nthis, typecheck.ConvNop(left, rcvr))
-               fn.Body.Append(as)
-               fn.Body.Append(ir.NewTailCallStmt(base.Pos, method.Nname.(*ir.Name)))
+       if !base.Flag.Cfg.Instrumenting && rcvr.IsPtr() && methodrcvr.IsPtr() && method.Embedded != 0 && !types.IsInterfaceMethod(method.Type) && !(base.Ctxt.Arch.Name == "ppc64le" && base.Ctxt.Flag_dynlink) && !generic {
+               call := ir.NewCallExpr(base.Pos, ir.OCALL, dot, nil)
+               call.Args = ir.ParamNames(tfn.Type())
+               call.IsDDD = tfn.Type().IsVariadic()
+               fn.Body.Append(ir.NewTailCallStmt(base.Pos, call))
        } else {
                fn.SetWrapper(true) // ignore frame for panic+recover matching
                var call *ir.CallExpr

diff --git a/src/cmd/compile/internal/riscv64/ssa.go b/src/cmd/compile/internal/riscv64/ssa.go
index e400ca1ffecedd396e21a4fd9ed3dff3eef8f5fb..1359b6a0c388b416dbcda35ef746d0c88497ef44 100644 (file)
--- a/src/cmd/compile/internal/riscv64/ssa.go
+++ b/src/cmd/compile/internal/riscv64/ssa.go
@@ -413,6 +413,8 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
                p.To.Reg = v.Reg()
        case ssa.OpRISCV64CALLstatic, ssa.OpRISCV64CALLclosure, ssa.OpRISCV64CALLinter:
                s.Call(v)
+       case ssa.OpRISCV64CALLtail:
+               s.TailCall(v)
        case ssa.OpRISCV64LoweredWB:
                p := s.Prog(obj.ACALL)
                p.To.Type = obj.TYPE_MEM
@@ -725,14 +727,9 @@ func ssaGenBlock(s *ssagen.State, b, next *ssa.Block) {
                        p.To.Type = obj.TYPE_BRANCH
                        s.Branches = append(s.Branches, ssagen.Branch{P: p, B: b.Succs[0].Block()})
                }
-       case ssa.BlockExit:
+       case ssa.BlockExit, ssa.BlockRetJmp:
        case ssa.BlockRet:
                s.Prog(obj.ARET)
-       case ssa.BlockRetJmp:
-               p := s.Prog(obj.ARET)
-               p.To.Type = obj.TYPE_MEM
-               p.To.Name = obj.NAME_EXTERN
-               p.To.Sym = b.Aux.(*obj.LSym)
        case ssa.BlockRISCV64BEQ, ssa.BlockRISCV64BEQZ, ssa.BlockRISCV64BNE, ssa.BlockRISCV64BNEZ,
                ssa.BlockRISCV64BLT, ssa.BlockRISCV64BLEZ, ssa.BlockRISCV64BGE, ssa.BlockRISCV64BGEZ,
                ssa.BlockRISCV64BLTZ, ssa.BlockRISCV64BGTZ, ssa.BlockRISCV64BLTU, ssa.BlockRISCV64BGEU:

diff --git a/src/cmd/compile/internal/s390x/ssa.go b/src/cmd/compile/internal/s390x/ssa.go
index ddc05b36add0fa1b9fd22c3697c12be3b124e6eb..deb6c790069ca29870bbe2a96ba9e6c53725a554 100644 (file)
--- a/src/cmd/compile/internal/s390x/ssa.go
+++ b/src/cmd/compile/internal/s390x/ssa.go
@@ -556,6 +556,8 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
                p.To.Reg = v.Reg()
        case ssa.OpS390XCALLstatic, ssa.OpS390XCALLclosure, ssa.OpS390XCALLinter:
                s.Call(v)
+       case ssa.OpS390XCALLtail:
+               s.TailCall(v)
        case ssa.OpS390XLoweredWB:
                p := s.Prog(obj.ACALL)
                p.To.Type = obj.TYPE_MEM
@@ -899,17 +901,11 @@ func ssaGenBlock(s *ssagen.State, b, next *ssa.Block) {
                        s.Br(s390x.ABR, b.Succs[0].Block())
                }
                return
-       case ssa.BlockExit:
+       case ssa.BlockExit, ssa.BlockRetJmp:
                return
        case ssa.BlockRet:
                s.Prog(obj.ARET)
                return
-       case ssa.BlockRetJmp:
-               p := s.Prog(s390x.ABR)
-               p.To.Type = obj.TYPE_MEM
-               p.To.Name = obj.NAME_EXTERN
-               p.To.Sym = b.Aux.(*obj.LSym)
-               return
        }
 
        // Handle s390x-specific blocks. These blocks all have a

diff --git a/src/cmd/compile/internal/ssa/check.go b/src/cmd/compile/internal/ssa/check.go
index 969fd96dbf573cf67483a1f740f5880fb5d8940e..28edfd2237d9253c88731c3c6791e2db5700663b 100644 (file)
--- a/src/cmd/compile/internal/ssa/check.go
+++ b/src/cmd/compile/internal/ssa/check.go
@@ -66,9 +66,6 @@ func checkFunc(f *Func) {
                        if !b.Controls[0].Type.IsMemory() {
                                f.Fatalf("retjmp block %s has non-memory control value %s", b, b.Controls[0].LongString())
                        }
-                       if b.Aux == nil {
-                               f.Fatalf("retjmp block %s has nil Aux field", b)
-                       }
                case BlockPlain:
                        if len(b.Succs) != 1 {
                                f.Fatalf("plain block %s len(Succs)==%d, want 1", b, len(b.Succs))

diff --git a/src/cmd/compile/internal/ssa/expand_calls.go b/src/cmd/compile/internal/ssa/expand_calls.go
index 12c7b16acd3027d3a39750d5d432f9114c3f83c5..79434f33d39246edb76de232f003d81464edefbe 100644 (file)
--- a/src/cmd/compile/internal/ssa/expand_calls.go
+++ b/src/cmd/compile/internal/ssa/expand_calls.go
@@ -1082,6 +1082,12 @@ func (x *expandState) rewriteArgs(v *Value, firstArg int) {
        mem := m0
        newArgs := []*Value{}
        oldArgs := []*Value{}
+       sp := x.sp
+       if v.Op == OpTailLECall {
+               // For tail call, we unwind the frame before the call so we'll use the caller's
+               // SP.
+               sp = x.f.Entry.NewValue0(src.NoXPos, OpGetCallerSP, x.typs.Uintptr)
+       }
        for i, a := range v.Args[firstArg : len(v.Args)-1] { // skip leading non-parameter SSA Args and trailing mem SSA Arg.
                oldArgs = append(oldArgs, a)
                auxI := int64(i)
@@ -1094,7 +1100,7 @@ func (x *expandState) rewriteArgs(v *Value, firstArg int) {
                        }
                        // "Dereference" of addressed (probably not-SSA-eligible) value becomes Move
                        // TODO(register args) this will be more complicated with registers in the picture.
-                       mem = x.rewriteDereference(v.Block, x.sp, a, mem, aOffset, aux.SizeOfArg(auxI), aType, a.Pos)
+                       mem = x.rewriteDereference(v.Block, sp, a, mem, aOffset, aux.SizeOfArg(auxI), aType, a.Pos)
                } else {
                        var rc registerCursor
                        var result *[]*Value
@@ -1107,7 +1113,7 @@ func (x *expandState) rewriteArgs(v *Value, firstArg int) {
                        if x.debug > 1 {
                                x.Printf("...storeArg %s, %v, %d\n", a.LongString(), aType, aOffset)
                        }
-                       rc.init(aRegs, aux.abiInfo, result, x.sp)
+                       rc.init(aRegs, aux.abiInfo, result, sp)
                        mem = x.storeArgOrLoad(a.Pos, v.Block, a, mem, aType, aOffset, 0, rc)
                }
        }
@@ -1207,7 +1213,7 @@ func expandCalls(f *Func) {
                for _, v := range b.Values {
                        firstArg := 0
                        switch v.Op {
-                       case OpStaticLECall:
+                       case OpStaticLECall, OpTailLECall:
                        case OpInterLECall:
                                firstArg = 1
                        case OpClosureLECall:
@@ -1525,6 +1531,10 @@ func expandCalls(f *Func) {
                                v.Op = OpStaticCall
                                rts := abi.RegisterTypes(v.Aux.(*AuxCall).abiInfo.OutParams())
                                v.Type = types.NewResults(append(rts, types.TypeMem))
+                       case OpTailLECall:
+                               v.Op = OpTailCall
+                               rts := abi.RegisterTypes(v.Aux.(*AuxCall).abiInfo.OutParams())
+                               v.Type = types.NewResults(append(rts, types.TypeMem))
                        case OpClosureLECall:
                                v.Op = OpClosureCall
                                rts := abi.RegisterTypes(v.Aux.(*AuxCall).abiInfo.OutParams())

diff --git a/src/cmd/compile/internal/ssa/gen/386.rules b/src/cmd/compile/internal/ssa/gen/386.rules
index 199b73c42f436fcfc88e422d196ffd8904780787..7bdebedafef81f6a1e1f3821e1a5fc1d59322365 100644 (file)
--- a/src/cmd/compile/internal/ssa/gen/386.rules
+++ b/src/cmd/compile/internal/ssa/gen/386.rules
@@ -317,6 +317,7 @@
 (StaticCall ...) => (CALLstatic ...)
 (ClosureCall ...) => (CALLclosure ...)
 (InterCall ...) => (CALLinter ...)
+(TailCall ...) => (CALLtail ...)
 
 // Miscellaneous
 (IsNonNil p) => (SETNE (TESTL p p))

diff --git a/src/cmd/compile/internal/ssa/gen/386Ops.go b/src/cmd/compile/internal/ssa/gen/386Ops.go
index 91f33c837457a3c944029de97383e8ebee9698af..3512d60865f6dc9a28fcb0cd50eaa77e54380414 100644 (file)
--- a/src/cmd/compile/internal/ssa/gen/386Ops.go
+++ b/src/cmd/compile/internal/ssa/gen/386Ops.go
@@ -455,6 +455,7 @@ func init() {
                },
 
                {name: "CALLstatic", argLength: 1, reg: regInfo{clobbers: callerSave}, aux: "CallOff", clobberFlags: true, call: true},                                              // call static function aux.(*obj.LSym).  arg0=mem, auxint=argsize, returns mem
+               {name: "CALLtail", argLength: 1, reg: regInfo{clobbers: callerSave}, aux: "CallOff", clobberFlags: true, call: true},                                                // tail call static function aux.(*obj.LSym).  arg0=mem, auxint=argsize, returns mem
                {name: "CALLclosure", argLength: 3, reg: regInfo{inputs: []regMask{gpsp, buildReg("DX"), 0}, clobbers: callerSave}, aux: "CallOff", clobberFlags: true, call: true}, // call function via closure.  arg0=codeptr, arg1=closure, arg2=mem, auxint=argsize, returns mem
                {name: "CALLinter", argLength: 2, reg: regInfo{inputs: []regMask{gp}, clobbers: callerSave}, aux: "CallOff", clobberFlags: true, call: true},                        // call fn by pointer.  arg0=codeptr, arg1=mem, auxint=argsize, returns mem
 

diff --git a/src/cmd/compile/internal/ssa/gen/AMD64.rules b/src/cmd/compile/internal/ssa/gen/AMD64.rules
index 5b127c98e79473c614522770c20497f7b338c8f8..bfed3bc7fdaa0340958598f7feb3bf00b8398c80 100644 (file)
--- a/src/cmd/compile/internal/ssa/gen/AMD64.rules
+++ b/src/cmd/compile/internal/ssa/gen/AMD64.rules
@@ -408,6 +408,7 @@
 (StaticCall ...) => (CALLstatic ...)
 (ClosureCall ...) => (CALLclosure ...)
 (InterCall ...) => (CALLinter ...)
+(TailCall ...) => (CALLtail ...)
 
 // Lowering conditional moves
 // If the condition is a SETxx, we can just run a CMOV from the comparison that was

diff --git a/src/cmd/compile/internal/ssa/gen/AMD64Ops.go b/src/cmd/compile/internal/ssa/gen/AMD64Ops.go
index 52ea7ac5e04600731a347a1ec71e77a4681eaf1d..51cbf5f78a215ec21318d6a83177cf41c7818aa0 100644 (file)
--- a/src/cmd/compile/internal/ssa/gen/AMD64Ops.go
+++ b/src/cmd/compile/internal/ssa/gen/AMD64Ops.go
@@ -765,6 +765,7 @@ func init() {
 
                // With a register ABI, the actual register info for these instructions (i.e., what is used in regalloc) is augmented with per-call-site bindings of additional arguments to specific in and out registers.
                {name: "CALLstatic", argLength: -1, reg: regInfo{clobbers: callerSave}, aux: "CallOff", clobberFlags: true, call: true},                                              // call static function aux.(*obj.LSym).  last arg=mem, auxint=argsize, returns mem
+               {name: "CALLtail", argLength: -1, reg: regInfo{clobbers: callerSave}, aux: "CallOff", clobberFlags: true, call: true},                                                // tail call static function aux.(*obj.LSym).  last arg=mem, auxint=argsize, returns mem
                {name: "CALLclosure", argLength: -1, reg: regInfo{inputs: []regMask{gpsp, buildReg("DX"), 0}, clobbers: callerSave}, aux: "CallOff", clobberFlags: true, call: true}, // call function via closure.  arg0=codeptr, arg1=closure, last arg=mem, auxint=argsize, returns mem
                {name: "CALLinter", argLength: -1, reg: regInfo{inputs: []regMask{gp}, clobbers: callerSave}, aux: "CallOff", clobberFlags: true, call: true},                        // call fn by pointer.  arg0=codeptr, last arg=mem, auxint=argsize, returns mem
 

diff --git a/src/cmd/compile/internal/ssa/gen/ARM.rules b/src/cmd/compile/internal/ssa/gen/ARM.rules
index bcacbafe3a5f0299dd47e45962a46170bd9896d4..bfb97e5271b41761904d951970eade41414f5bbc 100644 (file)
--- a/src/cmd/compile/internal/ssa/gen/ARM.rules
+++ b/src/cmd/compile/internal/ssa/gen/ARM.rules
@@ -351,6 +351,7 @@
 (StaticCall ...) => (CALLstatic ...)
 (ClosureCall ...) => (CALLclosure ...)
 (InterCall ...) => (CALLinter ...)
+(TailCall ...) => (CALLtail ...)
 
 // checks
 (NilCheck ...) => (LoweredNilCheck ...)

diff --git a/src/cmd/compile/internal/ssa/gen/ARM64.rules b/src/cmd/compile/internal/ssa/gen/ARM64.rules
index d99487c31b52d36703756d3d1274bd2414ed69a7..7f9d5ec2bd43a1b37f17f893f1eea38a0fbbc174 100644 (file)
--- a/src/cmd/compile/internal/ssa/gen/ARM64.rules
+++ b/src/cmd/compile/internal/ssa/gen/ARM64.rules
@@ -503,6 +503,7 @@
 (StaticCall ...) => (CALLstatic ...)
 (ClosureCall ...) => (CALLclosure ...)
 (InterCall ...) => (CALLinter ...)
+(TailCall ...) => (CALLtail ...)
 
 // checks
 (NilCheck ...) => (LoweredNilCheck ...)

diff --git a/src/cmd/compile/internal/ssa/gen/ARM64Ops.go b/src/cmd/compile/internal/ssa/gen/ARM64Ops.go
index acfb2880c2992801a7f205fc042d58276655b728..41231fb809a312012d23ae47c6de4a4001502187 100644 (file)
--- a/src/cmd/compile/internal/ssa/gen/ARM64Ops.go
+++ b/src/cmd/compile/internal/ssa/gen/ARM64Ops.go
@@ -484,6 +484,7 @@ func init() {
 
                // function calls
                {name: "CALLstatic", argLength: -1, reg: regInfo{clobbers: callerSave}, aux: "CallOff", clobberFlags: true, call: true},                                               // call static function aux.(*obj.LSym).  last arg=mem, auxint=argsize, returns mem
+               {name: "CALLtail", argLength: -1, reg: regInfo{clobbers: callerSave}, aux: "CallOff", clobberFlags: true, call: true},                                                 // tail call static function aux.(*obj.LSym).  last arg=mem, auxint=argsize, returns mem
                {name: "CALLclosure", argLength: -1, reg: regInfo{inputs: []regMask{gpsp, buildReg("R26"), 0}, clobbers: callerSave}, aux: "CallOff", clobberFlags: true, call: true}, // call function via closure.  arg0=codeptr, arg1=closure, last arg=mem, auxint=argsize, returns mem
                {name: "CALLinter", argLength: -1, reg: regInfo{inputs: []regMask{gp}, clobbers: callerSave}, aux: "CallOff", clobberFlags: true, call: true},                         // call fn by pointer.  arg0=codeptr, last arg=mem, auxint=argsize, returns mem
 

diff --git a/src/cmd/compile/internal/ssa/gen/ARMOps.go b/src/cmd/compile/internal/ssa/gen/ARMOps.go
index d1f86039a36e5c98ab7d6681928d67ea50e66302..eea0703f2cf87bbdae86766c4d985e062376ea6f 100644 (file)
--- a/src/cmd/compile/internal/ssa/gen/ARMOps.go
+++ b/src/cmd/compile/internal/ssa/gen/ARMOps.go
@@ -431,6 +431,7 @@ func init() {
 
                // function calls
                {name: "CALLstatic", argLength: 1, reg: regInfo{clobbers: callerSave}, aux: "CallOff", clobberFlags: true, call: true},                                              // call static function aux.(*obj.LSym).  arg0=mem, auxint=argsize, returns mem
+               {name: "CALLtail", argLength: 1, reg: regInfo{clobbers: callerSave}, aux: "CallOff", clobberFlags: true, call: true},                                                // tail call static function aux.(*obj.LSym).  arg0=mem, auxint=argsize, returns mem
                {name: "CALLclosure", argLength: 3, reg: regInfo{inputs: []regMask{gpsp, buildReg("R7"), 0}, clobbers: callerSave}, aux: "CallOff", clobberFlags: true, call: true}, // call function via closure.  arg0=codeptr, arg1=closure, arg2=mem, auxint=argsize, returns mem
                {name: "CALLinter", argLength: 2, reg: regInfo{inputs: []regMask{gp}, clobbers: callerSave}, aux: "CallOff", clobberFlags: true, call: true},                        // call fn by pointer.  arg0=codeptr, arg1=mem, auxint=argsize, returns mem
 

diff --git a/src/cmd/compile/internal/ssa/gen/MIPS.rules b/src/cmd/compile/internal/ssa/gen/MIPS.rules
index 4ac9668ea9cb566154a506ba1c3c0ecac7a144e7..639dda4b075492d3d8a499b71fa6616a2cbcb10c 100644 (file)
--- a/src/cmd/compile/internal/ssa/gen/MIPS.rules
+++ b/src/cmd/compile/internal/ssa/gen/MIPS.rules
@@ -334,6 +334,7 @@
 (StaticCall ...)  => (CALLstatic ...)
 (ClosureCall ...) => (CALLclosure ...)
 (InterCall ...)   => (CALLinter ...)
+(TailCall ...) => (CALLtail ...)
 
 // atomic intrinsics
 (AtomicLoad(8|32) ...) => (LoweredAtomicLoad(8|32) ...)

diff --git a/src/cmd/compile/internal/ssa/gen/MIPS64.rules b/src/cmd/compile/internal/ssa/gen/MIPS64.rules
index fd04a6c3a85c7ae768fa36d3a4a9bafb65b53012..292ff2fc7966092fac267f78a39a1d939e221c12 100644 (file)
--- a/src/cmd/compile/internal/ssa/gen/MIPS64.rules
+++ b/src/cmd/compile/internal/ssa/gen/MIPS64.rules
@@ -379,6 +379,7 @@
 (StaticCall ...) => (CALLstatic ...)
 (ClosureCall ...) => (CALLclosure ...)
 (InterCall ...) => (CALLinter ...)
+(TailCall ...) => (CALLtail ...)
 
 // atomic intrinsics
 (AtomicLoad(8|32|64) ...) => (LoweredAtomicLoad(8|32|64) ...)

diff --git a/src/cmd/compile/internal/ssa/gen/MIPS64Ops.go b/src/cmd/compile/internal/ssa/gen/MIPS64Ops.go
index a18cd4289d4fc6b735311158e994ec1b4b8c1155..54c0741efdeeb0cd5b7310469c9debbb0bcda0f0 100644 (file)
--- a/src/cmd/compile/internal/ssa/gen/MIPS64Ops.go
+++ b/src/cmd/compile/internal/ssa/gen/MIPS64Ops.go
@@ -276,6 +276,7 @@ func init() {
 
                // function calls
                {name: "CALLstatic", argLength: 1, reg: regInfo{clobbers: callerSave}, aux: "CallOff", clobberFlags: true, call: true},                                               // call static function aux.(*obj.LSym).  arg0=mem, auxint=argsize, returns mem
+               {name: "CALLtail", argLength: 1, reg: regInfo{clobbers: callerSave}, aux: "CallOff", clobberFlags: true, call: true},                                                 // tail call static function aux.(*obj.LSym).  arg0=mem, auxint=argsize, returns mem
                {name: "CALLclosure", argLength: 3, reg: regInfo{inputs: []regMask{gpsp, buildReg("R22"), 0}, clobbers: callerSave}, aux: "CallOff", clobberFlags: true, call: true}, // call function via closure.  arg0=codeptr, arg1=closure, arg2=mem, auxint=argsize, returns mem
                {name: "CALLinter", argLength: 2, reg: regInfo{inputs: []regMask{gp}, clobbers: callerSave}, aux: "CallOff", clobberFlags: true, call: true},                         // call fn by pointer.  arg0=codeptr, arg1=mem, auxint=argsize, returns mem
 

diff --git a/src/cmd/compile/internal/ssa/gen/MIPSOps.go b/src/cmd/compile/internal/ssa/gen/MIPSOps.go
index 8177c7e2d17cfff541418718755150f7947cf54c..5f73e9f2dcf7daa56a5c2497437574fc185c3280 100644 (file)
--- a/src/cmd/compile/internal/ssa/gen/MIPSOps.go
+++ b/src/cmd/compile/internal/ssa/gen/MIPSOps.go
@@ -258,6 +258,7 @@ func init() {
 
                // function calls
                {name: "CALLstatic", argLength: 1, reg: regInfo{clobbers: callerSave}, aux: "CallOff", clobberFlags: true, call: true},                                               // call static function aux.(*obj.LSym).  arg0=mem, auxint=argsize, returns mem
+               {name: "CALLtail", argLength: 1, reg: regInfo{clobbers: callerSave}, aux: "CallOff", clobberFlags: true, call: true},                                                 //  tail call static function aux.(*obj.LSym).  arg0=mem, auxint=argsize, returns mem
                {name: "CALLclosure", argLength: 3, reg: regInfo{inputs: []regMask{gpsp, buildReg("R22"), 0}, clobbers: callerSave}, aux: "CallOff", clobberFlags: true, call: true}, // call function via closure.  arg0=codeptr, arg1=closure, arg2=mem, auxint=argsize, returns mem
                {name: "CALLinter", argLength: 2, reg: regInfo{inputs: []regMask{gp}, clobbers: callerSave}, aux: "CallOff", clobberFlags: true, call: true},                         // call fn by pointer.  arg0=codeptr, arg1=mem, auxint=argsize, returns mem
 

diff --git a/src/cmd/compile/internal/ssa/gen/PPC64.rules b/src/cmd/compile/internal/ssa/gen/PPC64.rules
index ce4b324b5e10227bef0406b69f33b2bdef3652c9..411bb8d29d2c368b5ad7361510d0adaa448720e0 100644 (file)
--- a/src/cmd/compile/internal/ssa/gen/PPC64.rules
+++ b/src/cmd/compile/internal/ssa/gen/PPC64.rules
@@ -670,6 +670,7 @@
 (StaticCall ...) => (CALLstatic ...)
 (ClosureCall ...) => (CALLclosure ...)
 (InterCall ...) => (CALLinter ...)
+(TailCall ...) => (CALLtail ...)
 
 // Miscellaneous
 (GetClosurePtr ...) => (LoweredGetClosurePtr ...)

diff --git a/src/cmd/compile/internal/ssa/gen/PPC64Ops.go b/src/cmd/compile/internal/ssa/gen/PPC64Ops.go
index d7d8a33a0a10a52f8bd19ddb5ab2f5bbdc14e1cc..a14d9cd4904ac02729112efb3ef064727d53fe65 100644 (file)
--- a/src/cmd/compile/internal/ssa/gen/PPC64Ops.go
+++ b/src/cmd/compile/internal/ssa/gen/PPC64Ops.go
@@ -429,6 +429,7 @@ func init() {
                {name: "LoweredRound64F", argLength: 1, reg: fp11, resultInArg0: true, zeroWidth: true},
 
                {name: "CALLstatic", argLength: 1, reg: regInfo{clobbers: callerSave}, aux: "CallOff", clobberFlags: true, call: true},                                       // call static function aux.(*obj.LSym).  arg0=mem, auxint=argsize, returns mem
+               {name: "CALLtail", argLength: 1, reg: regInfo{clobbers: callerSave}, aux: "CallOff", clobberFlags: true, call: true},                                         // tail call static function aux.(*obj.LSym).  arg0=mem, auxint=argsize, returns mem
                {name: "CALLclosure", argLength: 3, reg: regInfo{inputs: []regMask{callptr, ctxt, 0}, clobbers: callerSave}, aux: "CallOff", clobberFlags: true, call: true}, // call function via closure.  arg0=codeptr, arg1=closure, arg2=mem, auxint=argsize, returns mem
                {name: "CALLinter", argLength: 2, reg: regInfo{inputs: []regMask{callptr}, clobbers: callerSave}, aux: "CallOff", clobberFlags: true, call: true},            // call fn by pointer.  arg0=codeptr, arg1=mem, auxint=argsize, returns mem
 

diff --git a/src/cmd/compile/internal/ssa/gen/RISCV64.rules b/src/cmd/compile/internal/ssa/gen/RISCV64.rules
index aa7c452d05f694ba4651dda8d8fbd7052a55219b..4290d1b85c14459a0cd0293821bf6db0b2d2afa6 100644 (file)
--- a/src/cmd/compile/internal/ssa/gen/RISCV64.rules
+++ b/src/cmd/compile/internal/ssa/gen/RISCV64.rules
@@ -546,6 +546,7 @@
 (StaticCall  ...) => (CALLstatic  ...)
 (ClosureCall ...) => (CALLclosure ...)
 (InterCall   ...) => (CALLinter   ...)
+(TailCall ...) => (CALLtail ...)
 
 // Atomic Intrinsics
 (AtomicLoad8   ...) => (LoweredAtomicLoad8  ...)

diff --git a/src/cmd/compile/internal/ssa/gen/RISCV64Ops.go b/src/cmd/compile/internal/ssa/gen/RISCV64Ops.go
index ac1bcd2a065756ccd9069ccf828f242137fda708..8a3fdf75f7838ef41d3f83b47adde44a70c3f64a 100644 (file)
--- a/src/cmd/compile/internal/ssa/gen/RISCV64Ops.go
+++ b/src/cmd/compile/internal/ssa/gen/RISCV64Ops.go
@@ -241,6 +241,7 @@ func init() {
 
                // Calls
                {name: "CALLstatic", argLength: 1, reg: call, aux: "CallOff", call: true},         // call static function aux.(*gc.Sym). arg0=mem, auxint=argsize, returns mem
+               {name: "CALLtail", argLength: 1, reg: call, aux: "CallOff", call: true},           // tail call static function aux.(*gc.Sym). arg0=mem, auxint=argsize, returns mem
                {name: "CALLclosure", argLength: 3, reg: callClosure, aux: "CallOff", call: true}, // call function via closure. arg0=codeptr, arg1=closure, arg2=mem, auxint=argsize, returns mem
                {name: "CALLinter", argLength: 2, reg: callInter, aux: "CallOff", call: true},     // call fn by pointer. arg0=codeptr, arg1=mem, auxint=argsize, returns mem
 

diff --git a/src/cmd/compile/internal/ssa/gen/S390X.rules b/src/cmd/compile/internal/ssa/gen/S390X.rules
index 88762f704589067c018bb110090397d7d51261de..b3928c6a1e6e4df84c241835c310dde974287b1d 100644 (file)
--- a/src/cmd/compile/internal/ssa/gen/S390X.rules
+++ b/src/cmd/compile/internal/ssa/gen/S390X.rules
@@ -434,6 +434,7 @@
 (StaticCall ...) => (CALLstatic ...)
 (ClosureCall ...) => (CALLclosure ...)
 (InterCall ...) => (CALLinter ...)
+(TailCall ...) => (CALLtail ...)
 
 // Miscellaneous
 (IsNonNil p) => (LOCGR {s390x.NotEqual} (MOVDconst [0]) (MOVDconst [1]) (CMPconst p [0]))

diff --git a/src/cmd/compile/internal/ssa/gen/S390XOps.go b/src/cmd/compile/internal/ssa/gen/S390XOps.go
index 00fce8e0e58b9a12728418aff98de10c0cbb28d0..9b6ac2bfb679bcf390650d7717031144c030369c 100644 (file)
--- a/src/cmd/compile/internal/ssa/gen/S390XOps.go
+++ b/src/cmd/compile/internal/ssa/gen/S390XOps.go
@@ -480,6 +480,7 @@ func init() {
                {name: "CLEAR", argLength: 2, reg: regInfo{inputs: []regMask{ptr, 0}}, asm: "CLEAR", aux: "SymValAndOff", typ: "Mem", clobberFlags: true, faultOnNilArg0: true, symEffect: "Write"},
 
                {name: "CALLstatic", argLength: 1, reg: regInfo{clobbers: callerSave}, aux: "CallOff", clobberFlags: true, call: true},                                                // call static function aux.(*obj.LSym).  arg0=mem, auxint=argsize, returns mem
+               {name: "CALLtail", argLength: 1, reg: regInfo{clobbers: callerSave}, aux: "CallOff", clobberFlags: true, call: true},                                                  // tail call static function aux.(*obj.LSym).  arg0=mem, auxint=argsize, returns mem
                {name: "CALLclosure", argLength: 3, reg: regInfo{inputs: []regMask{ptrsp, buildReg("R12"), 0}, clobbers: callerSave}, aux: "CallOff", clobberFlags: true, call: true}, // call function via closure.  arg0=codeptr, arg1=closure, arg2=mem, auxint=argsize, returns mem
                {name: "CALLinter", argLength: 2, reg: regInfo{inputs: []regMask{ptr}, clobbers: callerSave}, aux: "CallOff", clobberFlags: true, call: true},                         // call fn by pointer.  arg0=codeptr, arg1=mem, auxint=argsize, returns mem
 

diff --git a/src/cmd/compile/internal/ssa/gen/Wasm.rules b/src/cmd/compile/internal/ssa/gen/Wasm.rules
index 7ad3d1c72e1fb04db56f2b937410dd030de4fea7..9e683b116c1613423505e365cc7ffc3bfd071901 100644 (file)
--- a/src/cmd/compile/internal/ssa/gen/Wasm.rules
+++ b/src/cmd/compile/internal/ssa/gen/Wasm.rules
@@ -307,6 +307,7 @@
 (StaticCall ...) => (LoweredStaticCall ...)
 (ClosureCall ...) => (LoweredClosureCall ...)
 (InterCall ...) => (LoweredInterCall ...)
+(TailCall ...) => (LoweredTailCall ...)
 
 // Miscellaneous
 (Convert ...) => (LoweredConvert ...)

diff --git a/src/cmd/compile/internal/ssa/gen/WasmOps.go b/src/cmd/compile/internal/ssa/gen/WasmOps.go
index 7f7ae5e83701f17278102596ac3819ed0fcd4455..0d7327109a3b9bcb27cc27588464cb3ffc12f440 100644 (file)
--- a/src/cmd/compile/internal/ssa/gen/WasmOps.go
+++ b/src/cmd/compile/internal/ssa/gen/WasmOps.go
@@ -124,6 +124,7 @@ func init() {
 
        var WasmOps = []opData{
                {name: "LoweredStaticCall", argLength: 1, reg: regInfo{clobbers: callerSave}, aux: "CallOff", call: true},                                // call static function aux.(*obj.LSym). arg0=mem, auxint=argsize, returns mem
+               {name: "LoweredTailCall", argLength: 1, reg: regInfo{clobbers: callerSave}, aux: "CallOff", call: true},                                  // tail call static function aux.(*obj.LSym). arg0=mem, auxint=argsize, returns mem
                {name: "LoweredClosureCall", argLength: 3, reg: regInfo{inputs: []regMask{gp, gp, 0}, clobbers: callerSave}, aux: "CallOff", call: true}, // call function via closure. arg0=codeptr, arg1=closure, arg2=mem, auxint=argsize, returns mem
                {name: "LoweredInterCall", argLength: 2, reg: regInfo{inputs: []regMask{gp}, clobbers: callerSave}, aux: "CallOff", call: true},          // call fn by pointer. arg0=codeptr, arg1=mem, auxint=argsize, returns mem
 

diff --git a/src/cmd/compile/internal/ssa/gen/genericOps.go b/src/cmd/compile/internal/ssa/gen/genericOps.go
index c183aedf2d33770998548c700f4ac560e317f4a2..a0166f58f4e765f8f957552b064fedd136daa9a1 100644 (file)
--- a/src/cmd/compile/internal/ssa/gen/genericOps.go
+++ b/src/cmd/compile/internal/ssa/gen/genericOps.go
@@ -417,10 +417,12 @@ var genericOps = []opData{
        {name: "ClosureCall", argLength: -1, aux: "CallOff", call: true}, // arg0=code pointer, arg1=context ptr, arg2..argN-1 are register inputs, argN=memory.  auxint=arg size.  Returns Result of register results, plus memory.
        {name: "StaticCall", argLength: -1, aux: "CallOff", call: true},  // call function aux.(*obj.LSym), arg0..argN-1 are register inputs, argN=memory.  auxint=arg size.  Returns Result of register results, plus memory.
        {name: "InterCall", argLength: -1, aux: "CallOff", call: true},   // interface call.  arg0=code pointer, arg1..argN-1 are register inputs, argN=memory, auxint=arg size.  Returns Result of register results, plus memory.
+       {name: "TailCall", argLength: -1, aux: "CallOff", call: true},    // tail call function aux.(*obj.LSym), arg0..argN-1 are register inputs, argN=memory.  auxint=arg size.  Returns Result of register results, plus memory.
 
        {name: "ClosureLECall", argLength: -1, aux: "CallOff", call: true}, // late-expanded closure call. arg0=code pointer, arg1=context ptr,  arg2..argN-1 are inputs, argN is mem. auxint = arg size. Result is tuple of result(s), plus mem.
        {name: "StaticLECall", argLength: -1, aux: "CallOff", call: true},  // late-expanded static call function aux.(*ssa.AuxCall.Fn). arg0..argN-1 are inputs, argN is mem. auxint = arg size. Result is tuple of result(s), plus mem.
        {name: "InterLECall", argLength: -1, aux: "CallOff", call: true},   // late-expanded interface call. arg0=code pointer, arg1..argN-1 are inputs, argN is mem. auxint = arg size. Result is tuple of result(s), plus mem.
+       {name: "TailLECall", argLength: -1, aux: "CallOff", call: true},    // late-expanded static tail call function aux.(*ssa.AuxCall.Fn). arg0..argN-1 are inputs, argN is mem. auxint = arg size. Result is tuple of result(s), plus mem.
 
        // Conversions: signed extensions, zero (unsigned) extensions, truncations
        {name: "SignExt8to16", argLength: 1, typ: "Int16"},
@@ -638,7 +640,7 @@ var genericBlocks = []blockData{
        {name: "If", controls: 1},     // if Controls[0] goto Succs[0] else goto Succs[1]
        {name: "Defer", controls: 1},  // Succs[0]=defer queued, Succs[1]=defer recovered. Controls[0] is call op (of memory type)
        {name: "Ret", controls: 1},    // no successors, Controls[0] value is memory result
-       {name: "RetJmp", controls: 1}, // no successors, Controls[0] value is memory result, jumps to b.Aux.(*gc.Sym)
+       {name: "RetJmp", controls: 1}, // no successors, Controls[0] value is a tail call
        {name: "Exit", controls: 1},   // no successors, Controls[0] value generates a panic
 
        // transient block state used for dead code removal

diff --git a/src/cmd/compile/internal/ssa/opGen.go b/src/cmd/compile/internal/ssa/opGen.go
index 1ca99c1ba96ec0526fd5bc4aff2be6fa86941547..4ec13a6629c21bf823c08165f4ec5200c668e6b4 100644 (file)
--- a/src/cmd/compile/internal/ssa/opGen.go
+++ b/src/cmd/compile/internal/ssa/opGen.go
@@ -515,6 +515,7 @@ const (
        Op386DUFFZERO
        Op386REPSTOSL
        Op386CALLstatic
+       Op386CALLtail
        Op386CALLclosure
        Op386CALLinter
        Op386DUFFCOPY
@@ -993,6 +994,7 @@ const (
        OpAMD64DUFFZERO
        OpAMD64REPSTOSQ
        OpAMD64CALLstatic
+       OpAMD64CALLtail
        OpAMD64CALLclosure
        OpAMD64CALLinter
        OpAMD64DUFFCOPY
@@ -1269,6 +1271,7 @@ const (
        OpARMCMOVWLSconst
        OpARMSRAcond
        OpARMCALLstatic
+       OpARMCALLtail
        OpARMCALLclosure
        OpARMCALLinter
        OpARMLoweredNilCheck
@@ -1552,6 +1555,7 @@ const (
        OpARM64CSNEG
        OpARM64CSETM
        OpARM64CALLstatic
+       OpARM64CALLtail
        OpARM64CALLclosure
        OpARM64CALLinter
        OpARM64LoweredNilCheck
@@ -1697,6 +1701,7 @@ const (
        OpMIPSMOVFD
        OpMIPSMOVDF
        OpMIPSCALLstatic
+       OpMIPSCALLtail
        OpMIPSCALLclosure
        OpMIPSCALLinter
        OpMIPSLoweredAtomicLoad8
@@ -1813,6 +1818,7 @@ const (
        OpMIPS64MOVFD
        OpMIPS64MOVDF
        OpMIPS64CALLstatic
+       OpMIPS64CALLtail
        OpMIPS64CALLclosure
        OpMIPS64CALLinter
        OpMIPS64DUFFZERO
@@ -2025,6 +2031,7 @@ const (
        OpPPC64LoweredRound32F
        OpPPC64LoweredRound64F
        OpPPC64CALLstatic
+       OpPPC64CALLtail
        OpPPC64CALLclosure
        OpPPC64CALLinter
        OpPPC64LoweredZero
@@ -2128,6 +2135,7 @@ const (
        OpRISCV64SLTIU
        OpRISCV64MOVconvert
        OpRISCV64CALLstatic
+       OpRISCV64CALLtail
        OpRISCV64CALLclosure
        OpRISCV64CALLinter
        OpRISCV64DUFFZERO
@@ -2386,6 +2394,7 @@ const (
        OpS390XMOVDstoreconst
        OpS390XCLEAR
        OpS390XCALLstatic
+       OpS390XCALLtail
        OpS390XCALLclosure
        OpS390XCALLinter
        OpS390XInvertFlags
@@ -2439,6 +2448,7 @@ const (
        OpS390XLoweredZero
 
        OpWasmLoweredStaticCall
+       OpWasmLoweredTailCall
        OpWasmLoweredClosureCall
        OpWasmLoweredInterCall
        OpWasmLoweredAddr
@@ -2785,9 +2795,11 @@ const (
        OpClosureCall
        OpStaticCall
        OpInterCall
+       OpTailCall
        OpClosureLECall
        OpStaticLECall
        OpInterLECall
+       OpTailLECall
        OpSignExt8to16
        OpSignExt8to32
        OpSignExt8to64
@@ -5906,6 +5918,16 @@ var opcodeTable = [...]opInfo{
                        clobbers: 65519, // AX CX DX BX BP SI DI X0 X1 X2 X3 X4 X5 X6 X7
                },
        },
+       {
+               name:         "CALLtail",
+               auxType:      auxCallOff,
+               argLen:       1,
+               clobberFlags: true,
+               call:         true,
+               reg: regInfo{
+                       clobbers: 65519, // AX CX DX BX BP SI DI X0 X1 X2 X3 X4 X5 X6 X7
+               },
+       },
        {
                name:         "CALLclosure",
                auxType:      auxCallOff,
@@ -13103,6 +13125,16 @@ var opcodeTable = [...]opInfo{
                        clobbers: 2147483631, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 g R15 X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14
                },
        },
+       {
+               name:         "CALLtail",
+               auxType:      auxCallOff,
+               argLen:       -1,
+               clobberFlags: true,
+               call:         true,
+               reg: regInfo{
+                       clobbers: 2147483631, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 g R15 X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14
+               },
+       },
        {
                name:         "CALLclosure",
                auxType:      auxCallOff,
@@ -16939,6 +16971,16 @@ var opcodeTable = [...]opInfo{
                        clobbers: 4294924287, // R0 R1 R2 R3 R4 R5 R6 R7 R8 R9 g R12 R14 F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15
                },
        },
+       {
+               name:         "CALLtail",
+               auxType:      auxCallOff,
+               argLen:       1,
+               clobberFlags: true,
+               call:         true,
+               reg: regInfo{
+                       clobbers: 4294924287, // R0 R1 R2 R3 R4 R5 R6 R7 R8 R9 g R12 R14 F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15
+               },
+       },
        {
                name:         "CALLclosure",
                auxType:      auxCallOff,
@@ -20706,6 +20748,16 @@ var opcodeTable = [...]opInfo{
                        clobbers: 9223372035512336383, // R0 R1 R2 R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R13 R14 R15 R16 R17 R19 R20 R21 R22 R23 R24 R25 R26 g R30 F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31
                },
        },
+       {
+               name:         "CALLtail",
+               auxType:      auxCallOff,
+               argLen:       -1,
+               clobberFlags: true,
+               call:         true,
+               reg: regInfo{
+                       clobbers: 9223372035512336383, // R0 R1 R2 R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R13 R14 R15 R16 R17 R19 R20 R21 R22 R23 R24 R25 R26 g R30 F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31
+               },
+       },
        {
                name:         "CALLclosure",
                auxType:      auxCallOff,
@@ -22639,6 +22691,16 @@ var opcodeTable = [...]opInfo{
                        clobbers: 140737421246462, // R1 R2 R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R13 R14 R15 R16 R17 R18 R19 R20 R21 R22 R24 R25 R28 g R31 F0 F2 F4 F6 F8 F10 F12 F14 F16 F18 F20 F22 F24 F26 F28 F30 HI LO
                },
        },
+       {
+               name:         "CALLtail",
+               auxType:      auxCallOff,
+               argLen:       1,
+               clobberFlags: true,
+               call:         true,
+               reg: regInfo{
+                       clobbers: 140737421246462, // R1 R2 R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R13 R14 R15 R16 R17 R18 R19 R20 R21 R22 R24 R25 R28 g R31 F0 F2 F4 F6 F8 F10 F12 F14 F16 F18 F20 F22 F24 F26 F28 F30 HI LO
+               },
+       },
        {
                name:         "CALLclosure",
                auxType:      auxCallOff,
@@ -24198,6 +24260,16 @@ var opcodeTable = [...]opInfo{
                        clobbers: 4611686018393833470, // R1 R2 R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R13 R14 R15 R16 R17 R18 R19 R20 R21 R22 R24 R25 g R31 F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 HI LO
                },
        },
+       {
+               name:         "CALLtail",
+               auxType:      auxCallOff,
+               argLen:       1,
+               clobberFlags: true,
+               call:         true,
+               reg: regInfo{
+                       clobbers: 4611686018393833470, // R1 R2 R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R13 R14 R15 R16 R17 R18 R19 R20 R21 R22 R24 R25 g R31 F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 HI LO
+               },
+       },
        {
                name:         "CALLclosure",
                auxType:      auxCallOff,
@@ -27026,6 +27098,16 @@ var opcodeTable = [...]opInfo{
                        clobbers: 576460745860964344, // R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R14 R15 R16 R17 R18 R19 R20 R21 R22 R23 R24 R25 R26 R27 R28 R29 g F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26
                },
        },
+       {
+               name:         "CALLtail",
+               auxType:      auxCallOff,
+               argLen:       1,
+               clobberFlags: true,
+               call:         true,
+               reg: regInfo{
+                       clobbers: 576460745860964344, // R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R14 R15 R16 R17 R18 R19 R20 R21 R22 R23 R24 R25 R26 R27 R28 R29 g F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26
+               },
+       },
        {
                name:         "CALLclosure",
                auxType:      auxCallOff,
@@ -28432,6 +28514,15 @@ var opcodeTable = [...]opInfo{
                        clobbers: 9223372035781033972, // X3 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 g X28 X29 X30 F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31
                },
        },
+       {
+               name:    "CALLtail",
+               auxType: auxCallOff,
+               argLen:  1,
+               call:    true,
+               reg: regInfo{
+                       clobbers: 9223372035781033972, // X3 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 g X28 X29 X30 F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31
+               },
+       },
        {
                name:    "CALLclosure",
                auxType: auxCallOff,
@@ -32187,6 +32278,16 @@ var opcodeTable = [...]opInfo{
                        clobbers: 4294933503, // R0 R1 R2 R3 R4 R5 R6 R7 R8 R9 R11 R12 g R14 F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15
                },
        },
+       {
+               name:         "CALLtail",
+               auxType:      auxCallOff,
+               argLen:       1,
+               clobberFlags: true,
+               call:         true,
+               reg: regInfo{
+                       clobbers: 4294933503, // R0 R1 R2 R3 R4 R5 R6 R7 R8 R9 R11 R12 g R14 F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15
+               },
+       },
        {
                name:         "CALLclosure",
                auxType:      auxCallOff,
@@ -32857,6 +32958,15 @@ var opcodeTable = [...]opInfo{
                        clobbers: 844424930131967, // R0 R1 R2 R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R13 R14 R15 F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 g
                },
        },
+       {
+               name:    "LoweredTailCall",
+               auxType: auxCallOff,
+               argLen:  1,
+               call:    true,
+               reg: regInfo{
+                       clobbers: 844424930131967, // R0 R1 R2 R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R13 R14 R15 F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 g
+               },
+       },
        {
                name:    "LoweredClosureCall",
                auxType: auxCallOff,
@@ -35633,6 +35743,13 @@ var opcodeTable = [...]opInfo{
                call:    true,
                generic: true,
        },
+       {
+               name:    "TailCall",
+               auxType: auxCallOff,
+               argLen:  -1,
+               call:    true,
+               generic: true,
+       },
        {
                name:    "ClosureLECall",
                auxType: auxCallOff,
@@ -35654,6 +35771,13 @@ var opcodeTable = [...]opInfo{
                call:    true,
                generic: true,
        },
+       {
+               name:    "TailLECall",
+               auxType: auxCallOff,
+               argLen:  -1,
+               call:    true,
+               generic: true,
+       },
        {
                name:    "SignExt8to16",
                argLen:  1,

diff --git a/src/cmd/compile/internal/ssa/rewrite386.go b/src/cmd/compile/internal/ssa/rewrite386.go
index 1ec2d26f750f6b7998933fca8cbeb668d9188f1a..34f37867cf6149140eb883b4f67f37e26c1cb179 100644 (file)
--- a/src/cmd/compile/internal/ssa/rewrite386.go
+++ b/src/cmd/compile/internal/ssa/rewrite386.go
@@ -652,6 +652,9 @@ func rewriteValue386(v *Value) bool {
        case OpSubPtr:
                v.Op = Op386SUBL
                return true
+       case OpTailCall:
+               v.Op = Op386CALLtail
+               return true
        case OpTrunc16to8:
                v.Op = OpCopy
                return true

diff --git a/src/cmd/compile/internal/ssa/rewriteAMD64.go b/src/cmd/compile/internal/ssa/rewriteAMD64.go
index aa9293e34778fae8cdf5c19b2d0e92704e047631..e20161c920923724949a4bec144a5d97c551e35f 100644 (file)
--- a/src/cmd/compile/internal/ssa/rewriteAMD64.go
+++ b/src/cmd/compile/internal/ssa/rewriteAMD64.go
@@ -1103,6 +1103,9 @@ func rewriteValueAMD64(v *Value) bool {
        case OpSubPtr:
                v.Op = OpAMD64SUBQ
                return true
+       case OpTailCall:
+               v.Op = OpAMD64CALLtail
+               return true
        case OpTrunc:
                return rewriteValueAMD64_OpTrunc(v)
        case OpTrunc16to8:

diff --git a/src/cmd/compile/internal/ssa/rewriteARM.go b/src/cmd/compile/internal/ssa/rewriteARM.go
index febb5566e338492cb6f42e30d013b454b579fa4a..3d2f862705090b7e394f611de196cc9eb5340b84 100644 (file)
--- a/src/cmd/compile/internal/ssa/rewriteARM.go
+++ b/src/cmd/compile/internal/ssa/rewriteARM.go
@@ -855,6 +855,9 @@ func rewriteValueARM(v *Value) bool {
        case OpSubPtr:
                v.Op = OpARMSUB
                return true
+       case OpTailCall:
+               v.Op = OpARMCALLtail
+               return true
        case OpTrunc16to8:
                v.Op = OpCopy
                return true

diff --git a/src/cmd/compile/internal/ssa/rewriteARM64.go b/src/cmd/compile/internal/ssa/rewriteARM64.go
index 83dd771436c67aae62fae126c2f4127ea30d1479..3df84e161a274c073b0909c995f85711656f8ad9 100644 (file)
--- a/src/cmd/compile/internal/ssa/rewriteARM64.go
+++ b/src/cmd/compile/internal/ssa/rewriteARM64.go
@@ -1042,6 +1042,9 @@ func rewriteValueARM64(v *Value) bool {
        case OpSubPtr:
                v.Op = OpARM64SUB
                return true
+       case OpTailCall:
+               v.Op = OpARM64CALLtail
+               return true
        case OpTrunc:
                v.Op = OpARM64FRINTZD
                return true

diff --git a/src/cmd/compile/internal/ssa/rewriteMIPS.go b/src/cmd/compile/internal/ssa/rewriteMIPS.go
index 429369d631eb37bf910c4b6a845fbac06da68463..811ea9d9d326f239a20e3d2c1d06fd62a7f9c914 100644 (file)
--- a/src/cmd/compile/internal/ssa/rewriteMIPS.go
+++ b/src/cmd/compile/internal/ssa/rewriteMIPS.go
@@ -544,6 +544,9 @@ func rewriteValueMIPS(v *Value) bool {
        case OpSubPtr:
                v.Op = OpMIPSSUB
                return true
+       case OpTailCall:
+               v.Op = OpMIPSCALLtail
+               return true
        case OpTrunc16to8:
                v.Op = OpCopy
                return true

diff --git a/src/cmd/compile/internal/ssa/rewriteMIPS64.go b/src/cmd/compile/internal/ssa/rewriteMIPS64.go
index 772d7b66efeebdf3bd3048c2d5285be80ea69b03..1fbd556b5cbcd7d32cb9ad6424d5c5d2a14580e7 100644 (file)
--- a/src/cmd/compile/internal/ssa/rewriteMIPS64.go
+++ b/src/cmd/compile/internal/ssa/rewriteMIPS64.go
@@ -625,6 +625,9 @@ func rewriteValueMIPS64(v *Value) bool {
        case OpSubPtr:
                v.Op = OpMIPS64SUBV
                return true
+       case OpTailCall:
+               v.Op = OpMIPS64CALLtail
+               return true
        case OpTrunc16to8:
                v.Op = OpCopy
                return true

diff --git a/src/cmd/compile/internal/ssa/rewritePPC64.go b/src/cmd/compile/internal/ssa/rewritePPC64.go
index 96dee0bd21baf6d9f753dcbf615fee2797506fee..b35331a62487600e3e8199f296c3177158081e23 100644 (file)
--- a/src/cmd/compile/internal/ssa/rewritePPC64.go
+++ b/src/cmd/compile/internal/ssa/rewritePPC64.go
@@ -772,6 +772,9 @@ func rewriteValuePPC64(v *Value) bool {
        case OpSubPtr:
                v.Op = OpPPC64SUB
                return true
+       case OpTailCall:
+               v.Op = OpPPC64CALLtail
+               return true
        case OpTrunc:
                v.Op = OpPPC64FTRUNC
                return true

diff --git a/src/cmd/compile/internal/ssa/rewriteRISCV64.go b/src/cmd/compile/internal/ssa/rewriteRISCV64.go
index 3a277ca369e3d700184f68c0f85f80a51714ac12..f856a26d4917c0a5f8860069fd4b06ff5a07ef80 100644 (file)
--- a/src/cmd/compile/internal/ssa/rewriteRISCV64.go
+++ b/src/cmd/compile/internal/ssa/rewriteRISCV64.go
@@ -639,6 +639,9 @@ func rewriteValueRISCV64(v *Value) bool {
        case OpSubPtr:
                v.Op = OpRISCV64SUB
                return true
+       case OpTailCall:
+               v.Op = OpRISCV64CALLtail
+               return true
        case OpTrunc16to8:
                v.Op = OpCopy
                return true

diff --git a/src/cmd/compile/internal/ssa/rewriteS390X.go b/src/cmd/compile/internal/ssa/rewriteS390X.go
index 8b41d62c315bd25de6fa2adbdd553c17e1ca2a3e..0d6358614929c1adc323bd5eaadd7bb12c41a796 100644 (file)
--- a/src/cmd/compile/internal/ssa/rewriteS390X.go
+++ b/src/cmd/compile/internal/ssa/rewriteS390X.go
@@ -819,6 +819,9 @@ func rewriteValueS390X(v *Value) bool {
        case OpSubPtr:
                v.Op = OpS390XSUB
                return true
+       case OpTailCall:
+               v.Op = OpS390XCALLtail
+               return true
        case OpTrunc:
                return rewriteValueS390X_OpTrunc(v)
        case OpTrunc16to8:

diff --git a/src/cmd/compile/internal/ssa/rewriteWasm.go b/src/cmd/compile/internal/ssa/rewriteWasm.go
index 5dab09f85b35780f7113c4cb5e89e70337c7ae49..defd40ddd195e6a233e6205fcb74896ce5b52550 100644 (file)
--- a/src/cmd/compile/internal/ssa/rewriteWasm.go
+++ b/src/cmd/compile/internal/ssa/rewriteWasm.go
@@ -556,6 +556,9 @@ func rewriteValueWasm(v *Value) bool {
        case OpSubPtr:
                v.Op = OpWasmI64Sub
                return true
+       case OpTailCall:
+               v.Op = OpWasmLoweredTailCall
+               return true
        case OpTrunc:
                v.Op = OpWasmF64Trunc
                return true

diff --git a/src/cmd/compile/internal/ssa/writebarrier.go b/src/cmd/compile/internal/ssa/writebarrier.go
index d7510965f6b31a6b9a3b83e76ab9ab1587834449..52f060b601785426794719ea614db436e10412a9 100644 (file)
--- a/src/cmd/compile/internal/ssa/writebarrier.go
+++ b/src/cmd/compile/internal/ssa/writebarrier.go
@@ -544,7 +544,7 @@ func IsStackAddr(v *Value) bool {
                v = v.Args[0]
        }
        switch v.Op {
-       case OpSP, OpLocalAddr, OpSelectNAddr:
+       case OpSP, OpLocalAddr, OpSelectNAddr, OpGetCallerSP:
                return true
        }
        return false

diff --git a/src/cmd/compile/internal/ssagen/abi.go b/src/cmd/compile/internal/ssagen/abi.go
index 6d8c53e7225fd93893852a1348171be89cc1d7f2..c54a734c750fc2a118a471cadd3892f389093250 100644 (file)
--- a/src/cmd/compile/internal/ssagen/abi.go
+++ b/src/cmd/compile/internal/ssagen/abi.go
@@ -382,18 +382,16 @@ func makeABIWrapper(f *ir.Func, wrapperABI obj.ABI) {
        }
 
        var tail ir.Node
+       call := ir.NewCallExpr(base.Pos, ir.OCALL, f.Nname, nil)
+       call.Args = ir.ParamNames(tfn.Type())
+       call.IsDDD = tfn.Type().IsVariadic()
+       tail = call
        if tailcall {
-               tail = ir.NewTailCallStmt(base.Pos, f.Nname)
-       } else {
-               call := ir.NewCallExpr(base.Pos, ir.OCALL, f.Nname, nil)
-               call.Args = ir.ParamNames(tfn.Type())
-               call.IsDDD = tfn.Type().IsVariadic()
-               tail = call
-               if tfn.Type().NumResults() > 0 {
-                       n := ir.NewReturnStmt(base.Pos, nil)
-                       n.Results = []ir.Node{call}
-                       tail = n
-               }
+               tail = ir.NewTailCallStmt(base.Pos, call)
+       } else if tfn.Type().NumResults() > 0 {
+               n := ir.NewReturnStmt(base.Pos, nil)
+               n.Results = []ir.Node{call}
+               tail = n
        }
        fn.Body.Append(tail)
 

diff --git a/src/cmd/compile/internal/ssagen/ssa.go b/src/cmd/compile/internal/ssagen/ssa.go
index 1e7eda94fc1fc8e24b47c8cf946e8253e83f3e28..346a801508596556ee72d634fc04af0b7420b18c 100644 (file)
--- a/src/cmd/compile/internal/ssagen/ssa.go
+++ b/src/cmd/compile/internal/ssagen/ssa.go
@@ -1696,9 +1696,11 @@ func (s *state) stmt(n ir.Node) {
 
        case ir.OTAILCALL:
                n := n.(*ir.TailCallStmt)
-               b := s.exit()
-               b.Kind = ssa.BlockRetJmp // override BlockRet
-               b.Aux = callTargetLSym(n.Target)
+               s.callResult(n.Call, callTail)
+               call := s.mem()
+               b := s.endBlock()
+               b.Kind = ssa.BlockRetJmp // could use BlockExit. BlockRetJmp is mostly for clarity.
+               b.SetControl(call)
 
        case ir.OCONTINUE, ir.OBREAK:
                n := n.(*ir.BranchStmt)
@@ -3645,6 +3647,7 @@ const (
        callDefer
        callDeferStack
        callGo
+       callTail
 )
 
 type sfRtCallDef struct {
@@ -4911,13 +4914,13 @@ func (s *state) call(n *ir.CallExpr, k callKind, returnResultAddr bool) *ssa.Val
                }
        }
 
-       if k != callNormal && (len(n.Args) != 0 || n.Op() == ir.OCALLINTER || n.X.Type().NumResults() != 0) {
+       if k != callNormal && k != callTail && (len(n.Args) != 0 || n.Op() == ir.OCALLINTER || n.X.Type().NumResults() != 0) {
                s.Fatalf("go/defer call with arguments: %v", n)
        }
 
        switch n.Op() {
        case ir.OCALLFUNC:
-               if k == callNormal && fn.Op() == ir.ONAME && fn.(*ir.Name).Class == ir.PFUNC {
+               if (k == callNormal || k == callTail) && fn.Op() == ir.ONAME && fn.(*ir.Name).Class == ir.PFUNC {
                        fn := fn.(*ir.Name)
                        callee = fn
                        if buildcfg.Experiment.RegabiArgs {
@@ -4971,7 +4974,7 @@ func (s *state) call(n *ir.CallExpr, k callKind, returnResultAddr bool) *ssa.Val
        stksize := params.ArgWidth() // includes receiver, args, and results
 
        res := n.X.Type().Results()
-       if k == callNormal {
+       if k == callNormal || k == callTail {
                for _, p := range params.OutParams() {
                        ACResults = append(ACResults, p.Type)
                }
@@ -5018,7 +5021,7 @@ func (s *state) call(n *ir.CallExpr, k callKind, returnResultAddr bool) *ssa.Val
                // These are written in SP-offset order.
                argStart := base.Ctxt.FixedFrameSize()
                // Defer/go args.
-               if k != callNormal {
+               if k != callNormal && k != callTail {
                        // Write closure (arg to newproc/deferproc).
                        ACArgs = append(ACArgs, types.Types[types.TUINTPTR]) // not argExtra
                        callArgs = append(callArgs, closure)
@@ -5068,6 +5071,10 @@ func (s *state) call(n *ir.CallExpr, k callKind, returnResultAddr bool) *ssa.Val
                case callee != nil:
                        aux := ssa.StaticAuxCall(callTargetLSym(callee), params)
                        call = s.newValue0A(ssa.OpStaticLECall, aux.LateExpansionResultType(), aux)
+                       if k == callTail {
+                               call.Op = ssa.OpTailLECall
+                               stksize = 0 // Tail call does not use stack. We reuse caller's frame.
+                       }
                default:
                        s.Fatalf("bad call type %v %v", n.Op(), n)
                }
@@ -7399,6 +7406,14 @@ func (s *State) Call(v *ssa.Value) *obj.Prog {
        return p
 }
 
+// TailCall returns a new tail call instruction for the SSA value v.
+// It is like Call, but for a tail call.
+func (s *State) TailCall(v *ssa.Value) *obj.Prog {
+       p := s.Call(v)
+       p.As = obj.ARET
+       return p
+}
+
 // PrepareCall prepares to emit a CALL instruction for v and does call-related bookkeeping.
 // It must be called immediately before emitting the actual CALL instruction,
 // since it emits PCDATA for the stack map at the call (calls are safe points).

diff --git a/src/cmd/compile/internal/typecheck/typecheck.go b/src/cmd/compile/internal/typecheck/typecheck.go
index 404af5b1b2c3f009189d85bda6cffec83a878310..42970f6a5e279fa4a133c2d56d946d7d3dbfe9b2 100644 (file)
--- a/src/cmd/compile/internal/typecheck/typecheck.go
+++ b/src/cmd/compile/internal/typecheck/typecheck.go
@@ -879,6 +879,7 @@ func typecheck1(n ir.Node, top int) ir.Node {
 
        case ir.OTAILCALL:
                n := n.(*ir.TailCallStmt)
+               n.Call = typecheck(n.Call, ctxStmt|ctxExpr).(*ir.CallExpr)
                return n
 
        case ir.OCHECKNIL:

diff --git a/src/cmd/compile/internal/walk/stmt.go b/src/cmd/compile/internal/walk/stmt.go
index 4581bca3dfe42da847008085e9d62f9944305171..f09e9165464bcf9e31979752111a881c5091c84c 100644 (file)
--- a/src/cmd/compile/internal/walk/stmt.go
+++ b/src/cmd/compile/internal/walk/stmt.go
@@ -136,6 +136,14 @@ func walkStmt(n ir.Node) ir.Node {
 
        case ir.OTAILCALL:
                n := n.(*ir.TailCallStmt)
+
+               var init ir.Nodes
+               n.Call.X = walkExpr(n.Call.X, &init)
+
+               if len(init) > 0 {
+                       init.Append(n)
+                       return ir.NewBlockStmt(n.Pos(), init)
+               }
                return n
 
        case ir.OINLMARK:

diff --git a/src/cmd/compile/internal/wasm/ssa.go b/src/cmd/compile/internal/wasm/ssa.go
index 0b2ca3fdbb83af3573a2af995d795ffd7bd80718..765051c9445b293e84f372fcd890598c048f3ba3 100644 (file)
--- a/src/cmd/compile/internal/wasm/ssa.go
+++ b/src/cmd/compile/internal/wasm/ssa.go
@@ -88,13 +88,7 @@ func ssaGenBlock(s *ssagen.State, b, next *ssa.Block) {
        case ssa.BlockRet:
                s.Prog(obj.ARET)
 
-       case ssa.BlockRetJmp:
-               p := s.Prog(obj.ARET)
-               p.To.Type = obj.TYPE_MEM
-               p.To.Name = obj.NAME_EXTERN
-               p.To.Sym = b.Aux.(*obj.LSym)
-
-       case ssa.BlockExit:
+       case ssa.BlockExit, ssa.BlockRetJmp:
 
        case ssa.BlockDefer:
                p := s.Prog(wasm.AGet)
@@ -122,7 +116,7 @@ func ssaGenBlock(s *ssagen.State, b, next *ssa.Block) {
 
 func ssaGenValue(s *ssagen.State, v *ssa.Value) {
        switch v.Op {
-       case ssa.OpWasmLoweredStaticCall, ssa.OpWasmLoweredClosureCall, ssa.OpWasmLoweredInterCall:
+       case ssa.OpWasmLoweredStaticCall, ssa.OpWasmLoweredClosureCall, ssa.OpWasmLoweredInterCall, ssa.OpWasmLoweredTailCall:
                s.PrepareCall(v)
                if call, ok := v.Aux.(*ssa.AuxCall); ok && call.Fn == ir.Syms.Deferreturn {
                        // The runtime needs to inject jumps to
@@ -141,6 +135,9 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
                        p := s.Prog(obj.ACALL)
                        p.To = obj.Addr{Type: obj.TYPE_MEM, Name: obj.NAME_EXTERN, Sym: sym}
                        p.Pos = v.Pos
+                       if v.Op == ssa.OpWasmLoweredTailCall {
+                               p.As = obj.ARET
+                       }
                } else {
                        getValue64(s, v.Args[0])
                        p := s.Prog(obj.ACALL)

diff --git a/src/cmd/compile/internal/x86/ssa.go b/src/cmd/compile/internal/x86/ssa.go
index a06fdbcb71770ebce782dcbba69fdc673b236ac7..32e29f347b9c7f675414ee46c037aa62ea682b00 100644 (file)
--- a/src/cmd/compile/internal/x86/ssa.go
+++ b/src/cmd/compile/internal/x86/ssa.go
@@ -752,6 +752,8 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
 
        case ssa.Op386CALLstatic, ssa.Op386CALLclosure, ssa.Op386CALLinter:
                s.Call(v)
+       case ssa.Op386CALLtail:
+               s.TailCall(v)
        case ssa.Op386NEGL,
                ssa.Op386BSWAPL,
                ssa.Op386NOTL:
@@ -892,14 +894,9 @@ func ssaGenBlock(s *ssagen.State, b, next *ssa.Block) {
                        p.To.Type = obj.TYPE_BRANCH
                        s.Branches = append(s.Branches, ssagen.Branch{P: p, B: b.Succs[0].Block()})
                }
-       case ssa.BlockExit:
+       case ssa.BlockExit, ssa.BlockRetJmp:
        case ssa.BlockRet:
                s.Prog(obj.ARET)
-       case ssa.BlockRetJmp:
-               p := s.Prog(obj.AJMP)
-               p.To.Type = obj.TYPE_MEM
-               p.To.Name = obj.NAME_EXTERN
-               p.To.Sym = b.Aux.(*obj.LSym)
 
        case ssa.Block386EQF:
                s.CombJump(b, next, &eqfJumps)

diff --git a/test/abi/method_wrapper.go b/test/abi/method_wrapper.go
new file mode 100644 (file)
index 0000000..7aa262f
--- /dev/null
+++ b/test/abi/method_wrapper.go
@@ -0,0 +1,35 @@
+// run
+
+// Copyright 2021 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+type S int
+
+type T struct {
+       a int
+       S
+}
+
+//go:noinline
+func (s *S) M(a int, x [2]int, b float64, y [2]float64) (S, int, [2]int, float64, [2]float64) {
+       return *s, a, x, b, y
+}
+
+var s S = 42
+var t = &T{S: s}
+
+var fn = (*T).M // force a method wrapper
+
+func main() {
+       a := 123
+       x := [2]int{456, 789}
+       b := 1.2
+       y := [2]float64{3.4, 5.6}
+       s1, a1, x1, b1, y1 := fn(t, a, x, b, y)
+       if a1 != a || x1 != x || b1 != b || y1 != y || s1 != s {
+               panic("FAIL")
+       }
+}