--- /dev/null
+// Copyright 2024 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package abi
+
+import "unsafe"
+
+// NoEscape hides the pointer p from escape analysis, preventing it
+// from escaping to the heap. It compiles down to nothing.
+//
+// WARNING: This is very subtle to use correctly. The caller must
+// ensure that it's truly safe for p to not escape to the heap by
+// maintaining runtime pointer invariants (for example, that globals
+// and the heap may not generally point into a stack).
+//
+//go:nosplit
+//go:nocheckptr
+func NoEscape(p unsafe.Pointer) unsafe.Pointer {
+ x := uintptr(p)
+ return unsafe.Pointer(x ^ 0)
+}
package abi
+import "unsafe"
+
// The first word of every non-empty interface type contains an *ITab.
// It records the underlying concrete type (Type), the interface type it
// is implementing (Inter), and some ancillary information.
Hash uint32 // copy of Type.Hash. Used for type switches.
Fun [1]uintptr // variable sized. fun[0]==0 means Type does not implement Inter.
}
+
+// EmptyInterface describes the layout of a "interface{}" or a "any."
+// These are represented differently than non-empty interface, as the first
+// word always points to an abi.Type.
+type EmptyInterface struct {
+ Type *Type
+ Data unsafe.Pointer
+}
UnsafePointer: "unsafe.Pointer",
}
+// TypeOf returns the abi.Type of some value.
+func TypeOf(a any) *Type {
+ eface := *(*EmptyInterface)(unsafe.Pointer(&a))
+ // Types are either static (for compiler-created types) or
+ // heap-allocated but always reachable (for reflection-created
+ // types, held in the central map). So there is no need to
+ // escape types. noescape here help avoid unnecessary escape
+ // of v.
+ return (*Type)(NoEscape(unsafe.Pointer(eface.Type)))
+}
+
func (t *Type) Kind() Kind { return t.Kind_ & KindMask }
func (t *Type) HasName() bool {
// TypeOf returns the reflection Type that represents the dynamic type of i.
// If i is a nil interface value, TypeOf returns nil.
func TypeOf(i any) Type {
- eface := *(*emptyInterface)(unsafe.Pointer(&i))
- // Noescape so this doesn't make i to escape. See the comment
- // at Value.typ for why this is safe.
- return toType((*abi.Type)(noescape(unsafe.Pointer(eface.typ))))
+ return toType(abi.TypeOf(i))
}
func (t rtype) Implements(u Type) bool {
// types, held in the central map). So there is no need to
// escape types. noescape here help avoid unnecessary escape
// of v.
- return (*abi.Type)(noescape(unsafe.Pointer(v.typ_)))
+ return (*abi.Type)(abi.NoEscape(unsafe.Pointer(v.typ_)))
}
// pointer returns the underlying pointer represented by v.
func packEface(v Value) any {
t := v.typ()
var i any
- e := (*emptyInterface)(unsafe.Pointer(&i))
+ e := (*abi.EmptyInterface)(unsafe.Pointer(&i))
// First, fill in the data portion of the interface.
switch {
case ifaceIndir(t):
typedmemmove(t, c, ptr)
ptr = c
}
- e.word = ptr
+ e.Data = ptr
case v.flag&flagIndir != 0:
// Value is indirect, but interface is direct. We need
// to load the data at v.ptr into the interface data word.
- e.word = *(*unsafe.Pointer)(v.ptr)
+ e.Data = *(*unsafe.Pointer)(v.ptr)
default:
// Value is direct, and so is the interface.
- e.word = v.ptr
+ e.Data = v.ptr
}
// Now, fill in the type portion. We're very careful here not
// to have any operation between the e.word and e.typ assignments
// that would let the garbage collector observe the partially-built
// interface value.
- e.typ = t
+ e.Type = t
return i
}
// unpackEface converts the empty interface i to a Value.
func unpackEface(i any) Value {
- e := (*emptyInterface)(unsafe.Pointer(&i))
+ e := (*abi.EmptyInterface)(unsafe.Pointer(&i))
// NOTE: don't read e.word until we know whether it is really a pointer or not.
- t := e.typ
+ t := e.Type
if t == nil {
return Value{}
}
if ifaceIndir(t) {
f |= flagIndir
}
- return Value{t, e.word, f}
+ return Value{t, e.Data, f}
}
// A ValueError occurs when a Value method is invoked on
return f.Name()
}
-// emptyInterface is the header for an interface{} value.
-type emptyInterface struct {
- typ *abi.Type
- word unsafe.Pointer
-}
-
// mustBeExported panics if f records that the value was obtained using
// an unexported field.
func (f flag) mustBeExported() {
b bool
x any
}
-
-//go:nosplit
-func noescape(p unsafe.Pointer) unsafe.Pointer {
- x := uintptr(p)
- return unsafe.Pointer(x ^ 0)
-}
// TypeOf returns the reflection [Type] that represents the dynamic type of i.
// If i is a nil interface value, TypeOf returns nil.
func TypeOf(i any) Type {
- eface := *(*emptyInterface)(unsafe.Pointer(&i))
- // Noescape so this doesn't make i to escape. See the comment
- // at Value.typ for why this is safe.
- return toType((*abi.Type)(noescape(unsafe.Pointer(eface.typ))))
+ return toType(abi.TypeOf(i))
}
// rtypeOf directly extracts the *rtype of the provided value.
func rtypeOf(i any) *abi.Type {
- eface := *(*emptyInterface)(unsafe.Pointer(&i))
- return eface.typ
+ return abi.TypeOf(i)
}
// ptrMap is the cache for PointerTo.
// types, held in the central map). So there is no need to
// escape types. noescape here help avoid unnecessary escape
// of v.
- return (*abi.Type)(noescape(unsafe.Pointer(v.typ_)))
+ return (*abi.Type)(abi.NoEscape(unsafe.Pointer(v.typ_)))
}
// pointer returns the underlying pointer represented by v.
func packEface(v Value) any {
t := v.typ()
var i any
- e := (*emptyInterface)(unsafe.Pointer(&i))
+ e := (*abi.EmptyInterface)(unsafe.Pointer(&i))
// First, fill in the data portion of the interface.
switch {
case t.IfaceIndir():
typedmemmove(t, c, ptr)
ptr = c
}
- e.word = ptr
+ e.Data = ptr
case v.flag&flagIndir != 0:
// Value is indirect, but interface is direct. We need
// to load the data at v.ptr into the interface data word.
- e.word = *(*unsafe.Pointer)(v.ptr)
+ e.Data = *(*unsafe.Pointer)(v.ptr)
default:
// Value is direct, and so is the interface.
- e.word = v.ptr
+ e.Data = v.ptr
}
// Now, fill in the type portion. We're very careful here not
// to have any operation between the e.word and e.typ assignments
// that would let the garbage collector observe the partially-built
// interface value.
- e.typ = t
+ e.Type = t
return i
}
// unpackEface converts the empty interface i to a Value.
func unpackEface(i any) Value {
- e := (*emptyInterface)(unsafe.Pointer(&i))
+ e := (*abi.EmptyInterface)(unsafe.Pointer(&i))
// NOTE: don't read e.word until we know whether it is really a pointer or not.
- t := e.typ
+ t := e.Type
if t == nil {
return Value{}
}
if t.IfaceIndir() {
f |= flagIndir
}
- return Value{t, e.word, f}
+ return Value{t, e.Data, f}
}
// A ValueError occurs when a Value method is invoked on
return "unknown method"
}
-// emptyInterface is the header for an interface{} value.
-type emptyInterface struct {
- typ *abi.Type
- word unsafe.Pointer
-}
-
// nonEmptyInterface is the header for an interface value with methods.
type nonEmptyInterface struct {
itab *abi.ITab
// v.ptr doesn't escape, as Equal functions are compiler generated
// and never escape. The escape analysis doesn't know, as it is a
// function pointer call.
- return typ.Equal(noescape(v.ptr), unsafe.Pointer(&zeroVal[0]))
+ return typ.Equal(abi.NoEscape(v.ptr), unsafe.Pointer(&zeroVal[0]))
}
if typ.TFlag&abi.TFlagRegularMemory != 0 {
// For some types where the zero value is a value where all bits of this type are 0
// If the type is comparable, then compare directly with zero.
if typ.Equal != nil && typ.Size() <= abi.ZeroValSize {
// See noescape justification above.
- return typ.Equal(noescape(v.ptr), unsafe.Pointer(&zeroVal[0]))
+ return typ.Equal(abi.NoEscape(v.ptr), unsafe.Pointer(&zeroVal[0]))
}
if typ.TFlag&abi.TFlagRegularMemory != 0 {
// For some types where the zero value is a value where all bits of this type are 0
case Slice:
*(*unsafeheader.Slice)(v.ptr) = unsafeheader.Slice{}
case Interface:
- *(*emptyInterface)(v.ptr) = emptyInterface{}
+ *(*abi.EmptyInterface)(v.ptr) = abi.EmptyInterface{}
case Chan, Func, Map, Pointer, UnsafePointer:
*(*unsafe.Pointer)(v.ptr) = nil
case Array, Struct:
}
}
+// This is just a wrapper around abi.NoEscape. The inlining heuristics are
+// finnicky and for whatever reason treat the local call to noescape as much
+// lower cost with respect to the inliner budget. (That is, replacing calls to
+// noescape with abi.NoEscape will cause inlining tests to fail.)
+//
//go:nosplit
func noescape(p unsafe.Pointer) unsafe.Pointer {
- x := uintptr(p)
- return unsafe.Pointer(x ^ 0)
+ return abi.NoEscape(p)
}
package strings
import (
+ "internal/abi"
"internal/bytealg"
"unicode/utf8"
"unsafe"
buf []byte
}
-// noescape hides a pointer from escape analysis. It is the identity function
-// but escape analysis doesn't think the output depends on the input.
-// noescape is inlined and currently compiles down to zero instructions.
-// USE CAREFULLY!
-// This was copied from the runtime; see issues 23382 and 7921.
-//
-//go:nosplit
-//go:nocheckptr
-func noescape(p unsafe.Pointer) unsafe.Pointer {
- x := uintptr(p)
- return unsafe.Pointer(x ^ 0)
-}
-
func (b *Builder) copyCheck() {
if b.addr == nil {
// This hack works around a failing of Go's escape analysis
// See issue 23382.
// TODO: once issue 7921 is fixed, this should be reverted to
// just "b.addr = b".
- b.addr = (*Builder)(noescape(unsafe.Pointer(b)))
+ b.addr = (*Builder)(abi.NoEscape(unsafe.Pointer(b)))
} else if b.addr != b {
panic("strings: illegal use of non-zero Builder copied by value")
}