cmd/compile,runtime: open code unsafe.Slice

So prevent heavy runtime call overhead, and the compiler will have a
chance to optimize the bound check.

With this optimization, changing runtime/stack.go to use unsafe.Slice
no longer negatively impacts stack copying performance:

name                   old time/op    new time/op    delta
StackCopyWithStkobj-8    16.3ms ± 6%    16.5ms ± 5%   ~     (p=0.382 n=8+8)

name                   old alloc/op   new alloc/op   delta
StackCopyWithStkobj-8     17.0B ± 0%     17.0B ± 0%   ~     (all equal)

name                   old allocs/op  new allocs/op  delta
StackCopyWithStkobj-8      1.00 ± 0%      1.00 ± 0%   ~     (all equal)

Fixes #48798

Change-Id: I731a9a4abd6dd6846f44eece7f86025b7bb1141b
Reviewed-on: https://go-review.googlesource.com/c/go/+/362934
Reviewed-by: Keith Randall <khr@golang.org>
Reviewed-by: Matthew Dempsky <mdempsky@google.com>
Reviewed-by: Keith Randall <khr@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>
Run-TryBot: Cuong Manh Le <cuong.manhle.vn@gmail.com>

diff --git a/src/cmd/compile/internal/ssagen/ssa.go b/src/cmd/compile/internal/ssagen/ssa.go
index 686547f17571ea7a74ebc00c286e42b3fe804b4e..f8ed8765f9ed0d62b864f8cd80546e849797ede7 100644 (file)
--- a/src/cmd/compile/internal/ssagen/ssa.go
+++ b/src/cmd/compile/internal/ssagen/ssa.go
@@ -3921,6 +3921,7 @@ func InitTables() {
                        return s.newValue2(ssa.OpMul64uover, types.NewTuple(types.Types[types.TUINT], types.Types[types.TUINT]), args[0], args[1])
                },
                sys.AMD64, sys.I386, sys.MIPS64, sys.RISCV64)
+       alias("runtime", "mulUintptr", "runtime/internal/math", "MulUintptr", all...)
        add("runtime", "KeepAlive",
                func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
                        data := s.newValue1(ssa.OpIData, s.f.Config.Types.BytePtr, args[0])

diff --git a/src/cmd/compile/internal/typecheck/builtin.go b/src/cmd/compile/internal/typecheck/builtin.go
index 581928c00515e75065a21d56c0c9278c2fe1b7bd..875af37215eb9060901be70b41dc9f6340159856 100644 (file)
--- a/src/cmd/compile/internal/typecheck/builtin.go
+++ b/src/cmd/compile/internal/typecheck/builtin.go
@@ -134,9 +134,10 @@ var runtimeDecls = [...]struct {
        {"makeslice64", funcTag, 114},
        {"makeslicecopy", funcTag, 115},
        {"growslice", funcTag, 117},
-       {"unsafeslice", funcTag, 118},
-       {"unsafeslice64", funcTag, 119},
-       {"unsafeslicecheckptr", funcTag, 119},
+       {"unsafeslicecheckptr", funcTag, 118},
+       {"panicunsafeslicelen", funcTag, 9},
+       {"panicunsafeslicenilptr", funcTag, 9},
+       {"mulUintptr", funcTag, 119},
        {"memmove", funcTag, 120},
        {"memclrNoHeapPointers", funcTag, 121},
        {"memclrHasPointers", funcTag, 121},
@@ -346,8 +347,8 @@ func runtimeTypes() []*types.Type {
        typs[115] = newSig(params(typs[1], typs[15], typs[15], typs[7]), params(typs[7]))
        typs[116] = types.NewSlice(typs[2])
        typs[117] = newSig(params(typs[1], typs[116], typs[15]), params(typs[116]))
-       typs[118] = newSig(params(typs[1], typs[7], typs[15]), nil)
-       typs[119] = newSig(params(typs[1], typs[7], typs[22]), nil)
+       typs[118] = newSig(params(typs[1], typs[7], typs[22]), nil)
+       typs[119] = newSig(params(typs[5], typs[5]), params(typs[5], typs[6]))
        typs[120] = newSig(params(typs[3], typs[3], typs[5]), nil)
        typs[121] = newSig(params(typs[7], typs[5]), nil)
        typs[122] = newSig(params(typs[3], typs[3], typs[5]), params(typs[6]))

diff --git a/src/cmd/compile/internal/typecheck/builtin/runtime.go b/src/cmd/compile/internal/typecheck/builtin/runtime.go
index 04ae4f23a387790c11cc5d9e98360c55beff5019..dd19eefa29817e99dff838e65381ee5d40d99228 100644 (file)
--- a/src/cmd/compile/internal/typecheck/builtin/runtime.go
+++ b/src/cmd/compile/internal/typecheck/builtin/runtime.go
@@ -180,9 +180,11 @@ func makeslice(typ *byte, len int, cap int) unsafe.Pointer
 func makeslice64(typ *byte, len int64, cap int64) unsafe.Pointer
 func makeslicecopy(typ *byte, tolen int, fromlen int, from unsafe.Pointer) unsafe.Pointer
 func growslice(typ *byte, old []any, cap int) (ary []any)
-func unsafeslice(typ *byte, ptr unsafe.Pointer, len int)
-func unsafeslice64(typ *byte, ptr unsafe.Pointer, len int64)
 func unsafeslicecheckptr(typ *byte, ptr unsafe.Pointer, len int64)
+func panicunsafeslicelen()
+func panicunsafeslicenilptr()
+
+func mulUintptr(x, y uintptr) (uintptr, bool)
 
 func memmove(to *any, frm *any, length uintptr)
 func memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr)

diff --git a/src/cmd/compile/internal/walk/builtin.go b/src/cmd/compile/internal/walk/builtin.go
index d7b553ed0c4920905ce231ab3c29c4a923c57216..a11031b3d0bf7b5ecae7168daf30666ab284edcc 100644 (file)
--- a/src/cmd/compile/internal/walk/builtin.go
+++ b/src/cmd/compile/internal/walk/builtin.go
@@ -645,28 +645,67 @@ func walkRecoverFP(nn *ir.CallExpr, init *ir.Nodes) ir.Node {
 func walkUnsafeSlice(n *ir.BinaryExpr, init *ir.Nodes) ir.Node {
        ptr := safeExpr(n.X, init)
        len := safeExpr(n.Y, init)
+       sliceType := n.Type()
 
-       fnname := "unsafeslice64"
        lenType := types.Types[types.TINT64]
+       unsafePtr := typecheck.Conv(ptr, types.Types[types.TUNSAFEPTR])
 
+       // If checkptr enabled, call runtime.unsafeslicecheckptr to check ptr and len.
+       // for simplicity, unsafeslicecheckptr always uses int64.
        // Type checking guarantees that TIDEAL len/cap are positive and fit in an int.
        // The case of len or cap overflow when converting TUINT or TUINTPTR to TINT
        // will be handled by the negative range checks in unsafeslice during runtime.
        if ir.ShouldCheckPtr(ir.CurFunc, 1) {
-               fnname = "unsafeslicecheckptr"
-               // for simplicity, unsafeslicecheckptr always uses int64
-       } else if len.Type().IsKind(types.TIDEAL) || len.Type().Size() <= types.Types[types.TUINT].Size() {
-               fnname = "unsafeslice"
-               lenType = types.Types[types.TINT]
-       }
-
-       t := n.Type()
+               fnname := "unsafeslicecheckptr"
+               fn := typecheck.LookupRuntime(fnname)
+               init.Append(mkcall1(fn, nil, init, reflectdata.TypePtr(sliceType.Elem()), unsafePtr, typecheck.Conv(len, lenType)))
+       } else {
+               // Otherwise, open code unsafe.Slice to prevent runtime call overhead.
+               // Keep this code in sync with runtime.unsafeslice{,64}
+               if len.Type().IsKind(types.TIDEAL) || len.Type().Size() <= types.Types[types.TUINT].Size() {
+                       lenType = types.Types[types.TINT]
+               } else {
+                       // len64 := int64(len)
+                       // if int64(int(len64)) != len64 {
+                       //     panicunsafeslicelen()
+                       // }
+                       len64 := typecheck.Conv(len, lenType)
+                       nif := ir.NewIfStmt(base.Pos, nil, nil, nil)
+                       nif.Cond = ir.NewBinaryExpr(base.Pos, ir.ONE, typecheck.Conv(typecheck.Conv(len64, types.Types[types.TINT]), lenType), len64)
+                       nif.Body.Append(mkcall("panicunsafeslicelen", nil, &nif.Body))
+                       appendWalkStmt(init, nif)
+               }
 
-       // Call runtime.unsafeslice{,64,checkptr} to check ptr and len.
-       fn := typecheck.LookupRuntime(fnname)
-       init.Append(mkcall1(fn, nil, init, reflectdata.TypePtr(t.Elem()), typecheck.Conv(ptr, types.Types[types.TUNSAFEPTR]), typecheck.Conv(len, lenType)))
+               // if len < 0 { panicunsafeslicelen() }
+               nif := ir.NewIfStmt(base.Pos, nil, nil, nil)
+               nif.Cond = ir.NewBinaryExpr(base.Pos, ir.OLT, typecheck.Conv(len, lenType), ir.NewInt(0))
+               nif.Body.Append(mkcall("panicunsafeslicelen", nil, &nif.Body))
+               appendWalkStmt(init, nif)
+
+               // mem, overflow := runtime.mulUintptr(et.size, len)
+               mem := typecheck.Temp(types.Types[types.TUINTPTR])
+               overflow := typecheck.Temp(types.Types[types.TBOOL])
+               fn := typecheck.LookupRuntime("mulUintptr")
+               call := mkcall1(fn, fn.Type().Results(), init, ir.NewInt(sliceType.Elem().Size()), typecheck.Conv(typecheck.Conv(len, lenType), types.Types[types.TUINTPTR]))
+               appendWalkStmt(init, ir.NewAssignListStmt(base.Pos, ir.OAS2, []ir.Node{mem, overflow}, []ir.Node{call}))
+
+               // if overflow || mem > -uintptr(ptr) {
+               //     if ptr == nil {
+               //         panicunsafesliceptrnil()
+               //     }
+               //     panicunsafeslicelen()
+               // }
+               nif = ir.NewIfStmt(base.Pos, nil, nil, nil)
+               memCond := ir.NewBinaryExpr(base.Pos, ir.OGT, mem, ir.NewUnaryExpr(base.Pos, ir.ONEG, typecheck.Conv(unsafePtr, types.Types[types.TUINTPTR])))
+               nif.Cond = ir.NewLogicalExpr(base.Pos, ir.OOROR, overflow, memCond)
+               nifPtr := ir.NewIfStmt(base.Pos, nil, nil, nil)
+               nifPtr.Cond = ir.NewBinaryExpr(base.Pos, ir.OEQ, unsafePtr, typecheck.NodNil())
+               nifPtr.Body.Append(mkcall("panicunsafeslicenilptr", nil, &nifPtr.Body))
+               nif.Body.Append(nifPtr, mkcall("panicunsafeslicelen", nil, &nif.Body))
+               appendWalkStmt(init, nif)
+       }
 
-       h := ir.NewSliceHeaderExpr(n.Pos(), t,
+       h := ir.NewSliceHeaderExpr(n.Pos(), sliceType,
                typecheck.Conv(ptr, types.Types[types.TUNSAFEPTR]),
                typecheck.Conv(len, types.Types[types.TINT]),
                typecheck.Conv(len, types.Types[types.TINT]))

diff --git a/src/runtime/slice.go b/src/runtime/slice.go
index e0aeba604fd8cec503005a4b81ab22ce5ae94809..2413a46d6a28fa49e5c44763db4e02cfc95f7a48 100644 (file)
--- a/src/runtime/slice.go
+++ b/src/runtime/slice.go
@@ -117,6 +117,13 @@ func makeslice64(et *_type, len64, cap64 int64) unsafe.Pointer {
        return makeslice(et, len, cap)
 }
 
+// This is a wrapper over runtime/internal/math.MulUintptr,
+// so the compiler can recognize and treat it as an intrinsic.
+func mulUintptr(a, b uintptr) (uintptr, bool) {
+       return math.MulUintptr(a, b)
+}
+
+// Keep this code in sync with cmd/compile/internal/walk/builtin.go:walkUnsafeSlice
 func unsafeslice(et *_type, ptr unsafe.Pointer, len int) {
        if len < 0 {
                panicunsafeslicelen()
@@ -125,12 +132,13 @@ func unsafeslice(et *_type, ptr unsafe.Pointer, len int) {
        mem, overflow := math.MulUintptr(et.size, uintptr(len))
        if overflow || mem > -uintptr(ptr) {
                if ptr == nil {
-                       panic(errorString("unsafe.Slice: ptr is nil and len is not zero"))
+                       panicunsafeslicenilptr()
                }
                panicunsafeslicelen()
        }
 }
 
+// Keep this code in sync with cmd/compile/internal/walk/builtin.go:walkUnsafeSlice
 func unsafeslice64(et *_type, ptr unsafe.Pointer, len64 int64) {
        len := int(len64)
        if int64(len) != len64 {
@@ -153,6 +161,10 @@ func panicunsafeslicelen() {
        panic(errorString("unsafe.Slice: len out of range"))
 }
 
+func panicunsafeslicenilptr() {
+       panic(errorString("unsafe.Slice: ptr is nil and len is not zero"))
+}
+
 // growslice handles slice growth during append.
 // It is passed the slice element type, the old slice, and the desired new minimum capacity,
 // and it returns a new slice with at least that capacity, with the old data