// If/when midstack inlining is enabled (-l=4), the compiler gets both larger and slower.
// Not-inlining this method is a help (*Value.reset and *Block.NewValue0 are similar).
+//
//go:noinline
func (v *Value) AddArg(w *Value) {
if v.Args == nil {
// reset is called from most rewrite rules.
// Allowing it to be inlined increases the size
// of cmd/compile by almost 10%, and slows it down.
+//
//go:noinline
func (v *Value) reset(op Op) {
if v.InCache {
// copyOf is called from rewrite rules.
// It modifies v to be (Copy a).
+//
//go:noinline
func (v *Value) copyOf(a *Value) {
if v == a {
}
// make sure to cover int, uint cases (issue #16738)
+//
//go:noinline
func cvt9(a float64) int {
return int(a)
// and y.val() should be equal to which and y.p.val() should
// be equal to z.val(). Also, x(.p)**8 == x; that is, the
// autos are all linked into a ring.
+//
//go:noinline
func (v V) autos_ssa(which, w1, x1, w2, x2 int64) (y, z V) {
fill_ssa(v.w, v.x, &v, v.p) // gratuitous no-op to force addressing
// gets is an address-mentioning way of implementing
// structure assignment.
+//
//go:noinline
func (to *V) gets(from *V) {
*to = *from
// gets is an address-and-interface-mentioning way of
// implementing structure assignment.
+//
//go:noinline
func (to *V) getsI(from interface{}) {
*to = *from.(*V)
}
// fill_ssa initializes r with V{w:w, x:x, p:p}
+//
//go:noinline
func fill_ssa(w, x int64, r, p *V) {
*r = V{w: w, x: x, p: p}
// overflowConstShift_ssa verifes that constant folding for shift
// doesn't wrap (i.e. x << MAX_INT << 1 doesn't get folded to x << 0).
+//
//go:noinline
func overflowConstShift64_ssa(x int64) int64 {
return x << uint64(0xffffffffffffffff) << uint64(1)
// flagOverwrite_ssa is intended to reproduce an issue seen where a XOR
// was scheduled between a compare and branch, clearing flags.
+//
//go:noinline
func flagOverwrite_ssa(s *junk, c int) int {
if '0' <= c && c <= '9' {
// manysub_ssa is designed to tickle bugs that depend on register
// pressure or unfriendly operand ordering in registers (and at
// least once it succeeded in this).
+//
//go:noinline
func manysub_ssa(a, b, c, d float64) (aa, ab, ac, ad, ba, bb, bc, bd, ca, cb, cc, cd, da, db, dc, dd float64) {
aa = a + 11.0 - a
// fpspill_ssa attempts to trigger a bug where phis with floating point values
// were stored in non-fp registers causing an error in doasm.
+//
//go:noinline
func fpspill_ssa(a int) float64 {
// testDeadStorePanic_ssa ensures that we don't optimize away stores
// that could be read by after recover(). Modeled after fixedbugs/issue1304.
+//
//go:noinline
func testDeadStorePanic_ssa(a int) (r int) {
defer func() {
}
// Not inlining this function removes a significant chunk of init code.
+//
//go:noinline
func newSig(params, results []*types.Field) *types.Type {
return types.NewSignature(types.NoPkg, nil, nil, params, results)
fmt.Fprintln(w, `
// Not inlining this function removes a significant chunk of init code.
+//
//go:noinline
func newSig(params, results []*types.Field) *types.Type {
return types.NewSignature(types.NoPkg, nil, nil, params, results)
// license that can be found in the LICENSE file.
// js does not support inter-process file locking.
+//
//go:build !js
package lockedfile_test
// license that can be found in the LICENSE file.
// js does not support inter-process file locking.
+//
//go:build !js
package lockedfile_test
// These are used to index into cmd/link/internal/sym/AbiSymKindToSymKind
//
// TODO(rsc): Give idiomatic Go names.
+//
//go:generate stringer -type=SymKind
const (
// An otherwise invalid zero value for the type
)
// Implemented in the syscall package.
+//
//go:linkname fcntl syscall.fcntl
func fcntl(fd int, cmd int, arg int) (int, error)
// Defined SymKind values.
//
// TODO(rsc): Give idiomatic Go names.
+//
//go:generate stringer -type=SymKind
const (
Sxxx SymKind = iota
// cryptBlocksChain invokes the cipher message with chaining (KMC) instruction
// with the given function code. The length must be a multiple of BlockSize (16).
+//
//go:noescape
func cryptBlocksChain(c code, iv, key, dst, src *byte, length int)
// cryptBlocks invokes the cipher message (KM) instruction with
// the given function code. This is equivalent to AES in ECB
// mode. The length must be a multiple of BlockSize (16).
+//
//go:noescape
func cryptBlocks(c code, key, dst, src *byte, length int)
// dst. If a and b are not the same length then the number of bytes processed
// will be equal to the length of shorter of the two. Returns the number
// of bytes processed.
+//
//go:noescape
func xorBytes(dst, a, b []byte) int
// ghash uses the GHASH algorithm to hash data with the given key. The initial
// hash value is given by hash which will be updated with the new hash value.
// The length of data must be a multiple of 16-bytes.
+//
//go:noescape
func ghash(key *gcmHashKey, hash *[16]byte, data []byte)
// The lengths of both dst and buf must be greater than or equal to the length
// of src. buf may be partially or completely overwritten during the execution
// of the function.
+//
//go:noescape
func cryptBlocksGCM(fn code, key, dst, src, buf []byte, cnt *gcmCount)
// will be calculated and written to tag. cnt should contain the current
// counter state and will be overwritten with the updated counter state.
// TODO(mundaym): could pass in hash subkey
+//
//go:noescape
func kmaGCM(fn code, key, dst, src, aad []byte, tag *[16]byte, cnt *gcmCount)
// The return value corresponds to the condition code set by the
// instruction. Interrupted invocations are handled by the
// function.
+//
//go:noescape
func kdsa(fc uint64, params *[4096]byte) (errn uint64)
package field
// feMul sets out = a * b. It works like feMulGeneric.
+//
//go:noescape
func feMul(out *Element, a *Element, b *Element)
// feSquare sets out = a * a. It works like feSquareGeneric.
+//
//go:noescape
func feSquare(out *Element, a *Element)
// Functions implemented in p256_asm_*64.s
// Montgomery multiplication modulo P256
+//
//go:noescape
func p256Mul(res, in1, in2 []uint64)
// Montgomery square modulo P256, repeated n times (n >= 1)
+//
//go:noescape
func p256Sqr(res, in []uint64, n int)
// Montgomery multiplication by 1
+//
//go:noescape
func p256FromMont(res, in []uint64)
// iff cond == 1 val <- -val
+//
//go:noescape
func p256NegCond(val []uint64, cond int)
// if cond == 0 res <- b; else res <- a
+//
//go:noescape
func p256MovCond(res, a, b []uint64, cond int)
// Endianness swap
+//
//go:noescape
func p256BigToLittle(res []uint64, in []byte)
func p256LittleToBig(res []byte, in []uint64)
// Constant time table access
+//
//go:noescape
func p256Select(point, table []uint64, idx int)
func p256SelectBase(point *[12]uint64, table string, idx int)
// Montgomery multiplication modulo Ord(G)
+//
//go:noescape
func p256OrdMul(res, in1, in2 []uint64)
// Montgomery square modulo Ord(G), repeated n times
+//
//go:noescape
func p256OrdSqr(res, in []uint64, n int)
// If sign == 1 -> in2 = -in2
// If sel == 0 -> res = in1
// if zero == 0 -> res = in2
+//
//go:noescape
func p256PointAddAffineAsm(res, in1, in2 []uint64, sign, sel, zero int)
// Point add. Returns one if the two input points were equal and zero
// otherwise. (Note that, due to the way that the equations work out, some
// representations of ∞ are considered equal to everything by this function.)
+//
//go:noescape
func p256PointAddAsm(res, in1, in2 []uint64) int
// Point double
+//
//go:noescape
func p256PointDoubleAsm(res, in []uint64)
)
// Core Foundation linker flags for the external linker. See Issue 42459.
+//
//go:cgo_ldflag "-framework"
//go:cgo_ldflag "CoreFoundation"
)
// Security.framework linker flags for the external linker. See Issue 42459.
+//
//go:cgo_ldflag "-framework"
//go:cgo_ldflag "Security"
// argument to the latest security_certificates version from
// https://opensource.apple.com/source/security_certificates/
// and run "go generate". See https://golang.org/issue/38843.
+//
//go:generate go run root_ios_gen.go -version 55188.120.1.0.1
import "sync"
// license that can be found in the LICENSE file.
// Delete the next line to include in the gob package.
+//
//go:build ignore
package gob
// castagnoliSSE42 is defined in crc32_amd64.s and uses the SSE 4.2 CRC32
// instruction.
+//
//go:noescape
func castagnoliSSE42(crc uint32, p []byte) uint32
// castagnoliSSE42Triple is defined in crc32_amd64.s and uses the SSE 4.2 CRC32
// instruction.
+//
//go:noescape
func castagnoliSSE42Triple(
crcA, crcB, crcC uint32,
// ieeeCLMUL is defined in crc_amd64.s and uses the PCLMULQDQ
// instruction as well as SSE 4.1.
+//
//go:noescape
func ieeeCLMUL(crc uint32, p []byte) uint32
func ppc64SlicingUpdateBy8(crc uint32, table8 *slicing8Table, p []byte) uint32
// this function requires the buffer to be 16 byte aligned and > 16 bytes long
+//
//go:noescape
func vectorCrc32(crc uint32, poly uint32, p []byte) uint32
// vectorizedCastagnoli implements CRC32 using vector instructions.
// It is defined in crc32_s390x.s.
+//
//go:noescape
func vectorizedCastagnoli(crc uint32, p []byte) uint32
// vectorizedIEEE implements CRC32 using vector instructions.
// It is defined in crc32_s390x.s.
+//
//go:noescape
func vectorizedIEEE(crc uint32, p []byte) uint32
// per-arch information, including constants named $GOARCH for every
// GOARCH. The constant is 1 on the current system, 0 otherwise; multiplying
// by them is useful for defining GOARCH-specific constants.
+//
//go:generate go run gengoarch.go
type ArchFamilyType int
// per-OS information, including constants named Is$GOOS for every
// known GOOS. The constant is 1 on the current system, 0 otherwise;
// multiplying by them is useful for defining GOOS-specific constants.
+//
//go:generate go run gengoos.go
// We play unsafe games that violate Go's rules (and assume a non-moving
// collector). So we quiet Go here.
// See the comment below Get for more implementation details.
+//
//go:nocheckptr
func get(k key) *Value {
mu.Lock()
import _ "unsafe" // for go:linkname
// Implemented in the syscall package.
+//
//go:linkname fcntl syscall.fcntl
func fcntl(fd int, cmd int, arg int) (int, error)
}
// Implemented in syscall/syscall_darwin.go.
+//
//go:linkname fdopendir syscall.fdopendir
func fdopendir(fd int) (dir uintptr, err error)
)
// runtimeNano returns the current value of the runtime clock in nanoseconds.
+//
//go:linkname runtimeNano runtime.nanotime
func runtimeNano() int64
)
// Implemented in syscall/syscall_darwin.go.
+//
//go:linkname writev syscall.writev
func writev(fd int, iovecs []syscall.Iovec) (uintptr, error)
import "syscall"
// Not strictly needed, but very helpful for debugging, see issue #10221.
+//
//go:cgo_import_dynamic _ _ "libsendfile.so"
//go:cgo_import_dynamic _ _ "libsocket.so"
func ifaceE2I(t *rtype, src any, dst unsafe.Pointer)
// typedmemmove copies a value of type t to dst from src.
+//
//go:noescape
func typedmemmove(t *rtype, dst, src unsafe.Pointer)
}
// Implemented in the syscall package.
+//
//go:linkname fcntl syscall.fcntl
func fcntl(fd int, cmd int, arg int) (int, error)
import _ "unsafe" // for go:linkname
// Implemented in the syscall package.
+//
//go:linkname fcntl syscall.fcntl
func fcntl(fd int, cmd int, arg int) (int, error)
// license that can be found in the LICENSE file.
// Test broken pipes on Unix systems.
+//
//go:build !plan9 && !js
package os_test
// license that can be found in the LICENSE file.
// Test use of raw connections.
+//
//go:build !plan9 && !js
package os_test
func lastmoduleinit() (pluginpath string, syms map[string]any, errstr string)
// doInit is defined in package runtime
+//
//go:linkname doInit runtime.doInit
func doInit(t unsafe.Pointer) // t should be a *runtime.initTask
// This test case forces a large argument to the stack followed by more
// in-register arguments.
+//
//go:registerparams
//go:noinline
func passStruct10AndSmall(a Struct10, b byte, c uint) (Struct10, byte, uint) {
// nosplit because pointers are being held in uintptr slots in args, so
// having our stack scanned now could lead to accidentally freeing
// memory.
+//
//go:nosplit
func moveMakeFuncArgPtrs(ctxt *makeFuncCtxt, args *abi.RegArgs) {
for i, arg := range args.Ints {
// If the case was a receive, val is filled in with the received value.
// The conventional OK bool indicates whether the receive corresponds
// to a sent value.
+//
//go:noescape
func rselect([]runtimeSelect) (chosen int, recvOK bool)
// Arguments passed through to call do not escape. The type is used only in a
// very limited callee of call, the stackArgs are copied, and regArgs is only
// used in the call frame.
+//
//go:noescape
//go:linkname call runtime.reflectcall
func call(stackArgsType *rtype, f, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
func ifaceE2I(t *rtype, src any, dst unsafe.Pointer)
// memmove copies size bytes to dst from src. No write barriers are used.
+//
//go:noescape
func memmove(dst, src unsafe.Pointer, size uintptr)
// typedmemmove copies a value of type t to dst from src.
+//
//go:noescape
func typedmemmove(t *rtype, dst, src unsafe.Pointer)
// typedmemmovepartial is like typedmemmove but assumes that
// dst and src point off bytes into the value and only copies size bytes.
+//
//go:noescape
func typedmemmovepartial(t *rtype, dst, src unsafe.Pointer, off, size uintptr)
// typedmemclr zeros the value at ptr of type t.
+//
//go:noescape
func typedmemclr(t *rtype, ptr unsafe.Pointer)
// typedmemclrpartial is like typedmemclr but assumes that
// dst points off bytes into the value and only clears size bytes.
+//
//go:noescape
func typedmemclrpartial(t *rtype, ptr unsafe.Pointer, off, size uintptr)
// typedslicecopy copies a slice of elemType values from src to dst,
// returning the number of elements copied.
+//
//go:noescape
func typedslicecopy(elemType *rtype, dst, src unsafeheader.Slice) int
func asanpoison(addr unsafe.Pointer, sz uintptr)
// These are called from asan_GOARCH.s
+//
//go:cgo_import_static __asan_read_go
//go:cgo_import_static __asan_write_go
//go:cgo_import_static __asan_unpoison_go
// that pattern working. In particular, crosscall2 actually takes four
// arguments, but it works to call it with three arguments when
// calling _cgo_panic.
+//
//go:cgo_export_static crosscall2
//go:cgo_export_dynamic crosscall2
// These functions must be exported in order to perform
// longcall on cgo programs (cf gcc_aix_ppc64.c).
+//
//go:cgo_export_static __cgo_topofstack
//go:cgo_export_static runtime.rt0_go
//go:cgo_export_static _rt0_ppc64_aix_lib
// This is normally marked as hidden and placed in the
// .openbsd.randomdata section.
+//
//go:cgo_export_dynamic __guard_local __guard_local
// _cgo_mmap is filled in by runtime/cgo when it is linked into the
// program, so it is only non-nil when using cgo.
+//
//go:linkname _cgo_mmap _cgo_mmap
var _cgo_mmap unsafe.Pointer
// _cgo_munmap is filled in by runtime/cgo when it is linked into the
// program, so it is only non-nil when using cgo.
+//
//go:linkname _cgo_munmap _cgo_munmap
var _cgo_munmap unsafe.Pointer
// support sanitizer interceptors. Don't allow stack splits, since this function
// (used by sysAlloc) is called in a lot of low-level parts of the runtime and
// callers often assume it won't acquire any locks.
+//
//go:nosplit
func mmap(addr unsafe.Pointer, n uintptr, prot, flags, fd int32, off uint32) (unsafe.Pointer, int) {
if _cgo_mmap != nil {
// crosscall_ppc64 calls into the runtime to set up the registers the
// Go runtime expects and so the symbol it calls needs to be exported
// for external linking to work.
+//
//go:cgo_export_static _cgo_reginit
// _cgo_sigaction is filled in by runtime/cgo when it is linked into the
// program, so it is only non-nil when using cgo.
+//
//go:linkname _cgo_sigaction _cgo_sigaction
var _cgo_sigaction unsafe.Pointer
// callCgoSigaction calls the sigaction function in the runtime/cgo package
// using the GCC calling convention. It is implemented in assembly.
+//
//go:noescape
func callCgoSigaction(sig uintptr, new, old *sigactiont) int32
}
// wrapper for syscall package to call cgocall for libc (cgo) calls.
+//
//go:linkname syscall_cgocaller syscall.cgocaller
//go:nosplit
//go:uintptrescapes
}
// Call from C back to Go. fn must point to an ABIInternal Go entry-point.
+//
//go:nosplit
func cgocallbackg(fn, frame unsafe.Pointer, ctxt uintptr) {
gp := getg()
// cgoIsGoPointer reports whether the pointer is a Go pointer--a
// pointer to Go memory. We only care about Go memory that might
// contain pointers.
+//
//go:nosplit
//go:nowritebarrierrec
func cgoIsGoPointer(p unsafe.Pointer) bool {
}
// cgoInRange reports whether p is between start and end.
+//
//go:nosplit
//go:nowritebarrierrec
func cgoInRange(p unsafe.Pointer, start, end uintptr) bool {
// size is the number of bytes to copy.
// It throws if the program is copying a block that contains a Go pointer
// into non-Go memory.
+//
//go:nosplit
//go:nowritebarrier
func cgoCheckMemmove(typ *_type, dst, src unsafe.Pointer, off, size uintptr) {
// typ is the element type of the slice.
// It throws if the program is copying slice elements that contain Go pointers
// into non-Go memory.
+//
//go:nosplit
//go:nowritebarrier
func cgoCheckSliceCopy(typ *_type, dst, src unsafe.Pointer, n int) {
// cgoCheckTypedBlock checks the block of memory at src, for up to size bytes,
// and throws if it finds a Go pointer. The type of the memory is typ,
// and src is off bytes into that type.
+//
//go:nosplit
//go:nowritebarrier
func cgoCheckTypedBlock(typ *_type, src unsafe.Pointer, off, size uintptr) {
// cgoCheckBits checks the block of memory at src, for up to size
// bytes, and throws if it finds a Go pointer. The gcbits mark each
// pointer value. The src pointer is off bytes into the gcbits.
+//
//go:nosplit
//go:nowritebarrier
func cgoCheckBits(src unsafe.Pointer, gcbits *byte, off, size uintptr) {
// We only use this when looking at a value on the stack when the type
// uses a GC program, because otherwise it's more efficient to use the
// GC bits. This is called on the system stack.
+//
//go:nowritebarrier
//go:systemstack
func cgoCheckUsingType(typ *_type, src unsafe.Pointer, off, size uintptr) {
}
// entry point for c <- x from compiled code
+//
//go:nosplit
func chansend1(c *hchan, elem unsafe.Pointer) {
chansend(c, elem, true, getcallerpc())
}
// entry points for <- c from compiled code
+//
//go:nosplit
func chanrecv1(c *hchan, elem unsafe.Pointer) {
chanrecv(c, elem, true)
// and cannot allocate memory or acquire locks that might be
// held at the time of the signal, nor can it use substantial amounts
// of stack.
+//
//go:nowritebarrierrec
func (p *cpuProfile) add(tagPtr *unsafe.Pointer, stk []uintptr) {
// Simple cas-lock to coordinate with setcpuprofilerate.
// Instead, we copy the stack into cpuprof.extra,
// which will be drained the next time a Go thread
// gets the signal handling event.
+//
//go:nosplit
//go:nowritebarrierrec
func (p *cpuProfile) addNonGo(stk []uintptr) {
// For Plan 9 shared environment semantics, instead of Getenv(key) and
// Setenv(key, value), one can use os.ReadFile("/env/" + key) and
// os.WriteFile("/env/" + key, value, 0666) respectively.
+//
//go:nosplit
func goenvs() {
buf := make([]byte, envBufSize)
// Dofiles reads the directory opened with file descriptor fd, applying function f
// to each filename in it.
+//
//go:nosplit
func dofiles(dirfd int32, f func([]byte)) {
dirbuf := new([dirBufSize]byte)
// Gdirname returns the first filename from a buffer of directory entries,
// and a slice containing the remaining directory entries.
// If the buffer doesn't start with a valid directory entry, the returned name is nil.
+//
//go:nosplit
func gdirname(buf []byte) (name []byte, rest []byte) {
if 2+nameOffset+2 > len(buf) {
// Gbit16 reads a 16-bit little-endian binary number from b and returns it
// with the remaining slice of b.
+//
//go:nosplit
func gbit16(b []byte) (int, []byte) {
return int(b[0]) | int(b[1])<<8, b[2:]
// Update the C environment if cgo is loaded.
// Called from syscall.Setenv.
+//
//go:linkname syscall_setenv_c syscall.setenv_c
func syscall_setenv_c(k string, v string) {
if _cgo_setenv == nil {
// Update the C environment if cgo is loaded.
// Called from syscall.unsetenv.
+//
//go:linkname syscall_unsetenv_c syscall.unsetenv_c
func syscall_unsetenv_c(k string) {
if _cgo_unsetenv == nil {
}
// mspan wrapper for testing.
+//
//go:notinheap
type MSpan mspan
//
// Disallow preemptions and stack growths because this function
// may run in sensitive locations.
+//
//go:nosplit
func (h *timeHistogram) record(duration int64) {
if duration < 0 {
import "unsafe"
// Export some functions via linkname to assembly in sync/atomic.
+//
//go:linkname Load
//go:linkname Loadp
import "unsafe"
// Export some functions via linkname to assembly in sync/atomic.
+//
//go:linkname Load
//go:linkname Loadp
//go:linkname Load64
)
// Export some functions via linkname to assembly in sync/atomic.
+//
//go:linkname Xchg
//go:linkname Xchguintptr
}
// Atomic add and return new value.
+//
//go:nosplit
func Xadd(val *uint32, delta int32) uint32 {
for {
//go:build mips || mipsle
// Export some functions via linkname to assembly in sync/atomic.
+//
//go:linkname Xadd64
//go:linkname Xchg64
//go:linkname Cas64
import "unsafe"
// Export some functions via linkname to assembly in sync/atomic.
+//
//go:linkname Load
//go:linkname Loadp
//go:linkname Load64
// See https://github.com/WebAssembly/design/issues/1073
// Export some functions via linkname to assembly in sync/atomic.
+//
//go:linkname Load
//go:linkname Loadp
//go:linkname Load64
// affect mutex's state.
// We use the uintptr mutex.key and note.key as a uint32.
+//
//go:nosplit
func key32(p *uintptr) *uint32 {
return (*uint32)(unsafe.Pointer(p))
}
// This function may be called in nosplit context and thus must be nosplit.
+//
//go:nosplit
func acquireLockRank(rank lockRank) {
}
}
// This function may be called in nosplit context and thus must be nosplit.
+//
//go:nosplit
func releaseLockRank(rank lockRank) {
}
}
// nosplit to ensure it can be called in as many contexts as possible.
+//
//go:nosplit
func printHeldLocks(gp *g) {
if gp.m.locksHeldLen == 0 {
// acquireLockRank acquires a rank which is not associated with a mutex lock
//
// This function may be called in nosplit context and thus must be nosplit.
+//
//go:nosplit
func acquireLockRank(rank lockRank) {
gp := getg()
// releaseLockRank releases a rank which is not associated with a mutex lock
//
// This function may be called in nosplit context and thus must be nosplit.
+//
//go:nosplit
func releaseLockRank(rank lockRank) {
gp := getg()
}
// nosplit to ensure it can be called in as many contexts as possible.
+//
//go:nosplit
func checkLockHeld(gp *g, l *mutex) bool {
for i := gp.m.locksHeldLen - 1; i >= 0; i-- {
// assertLockHeld throws if l is not held by the caller.
//
// nosplit to ensure it can be called in as many contexts as possible.
+//
//go:nosplit
func assertLockHeld(l *mutex) {
gp := getg()
// pointer to the exact mutex is not available.
//
// nosplit to ensure it can be called in as many contexts as possible.
+//
//go:nosplit
func assertRankHeld(r lockRank) {
gp := getg()
// Caller must hold worldsema.
//
// nosplit to ensure it can be called in as many contexts as possible.
+//
//go:nosplit
func worldStopped() {
if stopped := atomic.Xadd(&worldIsStopped, 1); stopped != 1 {
// Caller must hold worldsema.
//
// nosplit to ensure it can be called in as many contexts as possible.
+//
//go:nosplit
func worldStarted() {
if stopped := atomic.Xadd(&worldIsStopped, -1); stopped != 0 {
}
// nosplit to ensure it can be called in as many contexts as possible.
+//
//go:nosplit
func checkWorldStopped() bool {
stopped := atomic.Load(&worldIsStopped)
// which M stopped the world.
//
// nosplit to ensure it can be called in as many contexts as possible.
+//
//go:nosplit
func assertWorldStopped() {
if checkWorldStopped() {
// passed lock is not held.
//
// nosplit to ensure it can be called in as many contexts as possible.
+//
//go:nosplit
func assertWorldStoppedOrLockHeld(l *mutex) {
if checkWorldStopped() {
// Must run on system stack because stack growth can (re)invoke it.
// See issue 9174.
+//
//go:systemstack
func persistentalloc1(size, align uintptr, sysStat *sysMemStat) *notInHeap {
const (
// inPersistentAlloc reports whether p points to memory allocated by
// persistentalloc. This must be nosplit because it is called by the
// cgo checker code, which is called by the write barrier code.
+//
//go:nosplit
func inPersistentAlloc(p uintptr) bool {
chunk := atomic.Loaduintptr((*uintptr)(unsafe.Pointer(&persistentChunks)))
// typedmemmovepartial is like typedmemmove but assumes that
// dst and src point off bytes into the value and only copies size bytes.
// off must be a multiple of goarch.PtrSize.
+//
//go:linkname reflect_typedmemmovepartial reflect.typedmemmovepartial
func reflect_typedmemmovepartial(typ *_type, dst, src unsafe.Pointer, off, size uintptr) {
if writeBarrier.needed && typ.ptrdata > off && size >= goarch.PtrSize {
)
// addb returns the byte pointer p+n.
+//
//go:nowritebarrier
//go:nosplit
func addb(p *byte, n uintptr) *byte {
}
// subtractb returns the byte pointer p-n.
+//
//go:nowritebarrier
//go:nosplit
func subtractb(p *byte, n uintptr) *byte {
}
// add1 returns the byte pointer p+1.
+//
//go:nowritebarrier
//go:nosplit
func add1(p *byte) *byte {
}
// subtract1 returns the byte pointer p-1.
+//
//go:nowritebarrier
//
// nosplit because it is used during write barriers and must not be preempted.
+//
//go:nosplit
func subtract1(p *byte) *byte {
// Note: wrote out full expression instead of calling subtractb(p, 1)
// In particular, be careful not to point past the end of an object.
//
// nosplit because it is used during write barriers and must not be preempted.
+//
//go:nosplit
func heapBitsForAddr(addr uintptr) (h heapBits) {
// 2 bits per word, 4 pairs per byte, and a mask is hard coded.
//
// It is nosplit so it is safe for p to be a pointer to the current goroutine's stack.
// Since p is a uintptr, it would not be adjusted if the stack were to move.
+//
//go:nosplit
func findObject(p, refBase, refOff uintptr) (base uintptr, s *mspan, objIndex uintptr) {
s = spanOf(p)
}
// verifyNotInHeapPtr reports whether converting the not-in-heap pointer into a unsafe.Pointer is ok.
+//
//go:linkname reflect_verifyNotInHeapPtr reflect.verifyNotInHeapPtr
func reflect_verifyNotInHeapPtr(p uintptr) bool {
// Conversion to a pointer is ok as long as findObject above does not call badPointer.
// Note that next does not modify h. The caller must record the result.
//
// nosplit because it is used during write barriers and must not be preempted.
+//
//go:nosplit
func (h heapBits) next() heapBits {
if h.shift < 3*heapBitsShift {
// h.forward(1) is equivalent to h.next(), just slower.
// Note that forward does not modify h. The caller must record the result.
// bits returns the heap bits for the current word.
+//
//go:nosplit
func (h heapBits) forward(n uintptr) heapBits {
n += uintptr(h.shift) / heapBitsShift
// described by the same bitmap byte.
//
// nosplit because it is used during write barriers and must not be preempted.
+//
//go:nosplit
func (h heapBits) bits() uint32 {
// The (shift & 31) eliminates a test and conditional branch
// isPointer reports whether the heap bits describe a pointer word.
//
// nosplit because it is used during write barriers and must not be preempted.
+//
//go:nosplit
func (h heapBits) isPointer() bool {
return h.bits()&bitPointer != 0
//
// This is used for special cases where e.g. dst was just
// created and zeroed with malloc.
+//
//go:nosplit
func bulkBarrierPreWriteSrcOnly(dst, src, size uintptr) {
if (dst|src|size)&(goarch.PtrSize-1) != 0 {
// gcbits returns the GC type info for x, for testing.
// The result is the bitmap entries (0 or 1), one entry per byte.
+//
//go:linkname reflect_gcbits reflect.gcbits
func reflect_gcbits(x any) []byte {
ret := getgcmask(x)
// Don't split the stack as this method may be invoked without a valid G, which
// prevents us from allocating more stack.
+//
//go:nosplit
func sysAllocOS(n uintptr) unsafe.Pointer {
p, err := mmap(nil, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_PRIVATE, -1, 0)
// Don't split the stack as this function may be invoked without a valid G,
// which prevents us from allocating more stack.
+//
//go:nosplit
func sysFreeOS(v unsafe.Pointer, n uintptr) {
munmap(v, n)
// Don't split the stack as this function may be invoked without a valid G,
// which prevents us from allocating more stack.
+//
//go:nosplit
func sysAllocOS(n uintptr) unsafe.Pointer {
v, err := mmap(nil, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_PRIVATE, -1, 0)
// Don't split the stack as this function may be invoked without a valid G,
// which prevents us from allocating more stack.
+//
//go:nosplit
func sysFreeOS(v unsafe.Pointer, n uintptr) {
munmap(v, n)
// Don't split the stack as this function may be invoked without a valid G,
// which prevents us from allocating more stack.
+//
//go:nosplit
func sysAllocOS(n uintptr) unsafe.Pointer {
v, err := mmap(nil, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_PRIVATE, -1, 0)
// Don't split the stack as this function may be invoked without a valid G,
// which prevents us from allocating more stack.
+//
//go:nosplit
func sysFreeOS(v unsafe.Pointer, n uintptr) {
munmap(v, n)
// Don't split the stack as this function may be invoked without a valid G,
// which prevents us from allocating more stack.
+//
//go:nosplit
func sysAllocOS(n uintptr) unsafe.Pointer {
p := sysReserveOS(nil, n)
// Don't split the stack as this function may be invoked without a valid G,
// which prevents us from allocating more stack.
+//
//go:nosplit
func sysFreeOS(v unsafe.Pointer, n uintptr) {
}
// Don't split the stack as this method may be invoked without a valid G, which
// prevents us from allocating more stack.
+//
//go:nosplit
func sysAllocOS(n uintptr) unsafe.Pointer {
p, err := mmap(nil, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_PRIVATE, -1, 0)
// Don't split the stack as this function may be invoked without a valid G,
// which prevents us from allocating more stack.
+//
//go:nosplit
func sysFreeOS(v unsafe.Pointer, n uintptr) {
munmap(v, n)
// Don't split the stack as this function may be invoked without a valid G,
// which prevents us from allocating more stack.
+//
//go:nosplit
func sysAllocOS(n uintptr) unsafe.Pointer {
return unsafe.Pointer(stdcall4(_VirtualAlloc, 0, n, _MEM_COMMIT|_MEM_RESERVE, _PAGE_READWRITE))
// Don't split the stack as this function may be invoked without a valid G,
// which prevents us from allocating more stack.
+//
//go:nosplit
func sysFreeOS(v unsafe.Pointer, n uintptr) {
r := stdcall3(_VirtualFree, uintptr(v), 0, _MEM_RELEASE)
}
// Mark KeepAlive as noinline so that it is easily detectable as an intrinsic.
+//
//go:noinline
// KeepAlive marks its argument as currently reachable.
}
// Scan a stack frame: local variables and function arguments/results.
+//
//go:nowritebarrier
func scanframeworker(frame *stkframe, state *stackScanState, gcw *gcWork) {
if _DebugGC > 1 && frame.continpc != 0 {
// gcw.bytesMarked or gcw.heapScanWork.
//
// If stk != nil, possible stack pointers are also reported to stk.putPtr.
+//
//go:nowritebarrier
func scanblock(b0, n0 uintptr, ptrmask *uint8, gcw *gcWork, stk *stackScanState) {
// Use local copies of original parameters, so that a stack trace
// Shade the object if it isn't already.
// The object is not nil and known to be in the heap.
// Preemption must be disabled.
+//
//go:nowritebarrier
func shade(b uintptr) {
if obj, span, objIndex := findObject(b, 0, 0); obj != 0 {
}
// obj.r = r, but with no write barrier.
+//
//go:nowritebarrier
func (obj *stackObject) setRecord(r *stackObjectRecord) {
// Types of stack objects are always in read-only memory, not the heap.
}
// Returns only when span s has been swept.
+//
//go:nowritebarrier
func (s *mspan) ensureSwept() {
// Caller must disable preemption.
// put enqueues a pointer for the garbage collector to trace.
// obj must point to the beginning of a heap object or an oblet.
+//
//go:nowritebarrierrec
func (w *gcWork) put(obj uintptr) {
flushed := false
// putFast does a put and reports whether it can be done quickly
// otherwise it returns false and the caller needs to call put.
+//
//go:nowritebarrierrec
func (w *gcWork) putFast(obj uintptr) bool {
wbuf := w.wbuf1
// If there are no pointers remaining in this gcWork or in the global
// queue, tryGet returns 0. Note that there may still be pointers in
// other gcWork instances or other caches.
+//
//go:nowritebarrierrec
func (w *gcWork) tryGet() uintptr {
wbuf := w.wbuf1
// tryGetFast dequeues a pointer for the garbage collector to trace
// if one is readily available. Otherwise it returns 0 and
// the caller is expected to call tryGet().
+//
//go:nowritebarrierrec
func (w *gcWork) tryGetFast() uintptr {
wbuf := w.wbuf1
// balance moves some work that's cached in this gcWork back on the
// global queue.
+//
//go:nowritebarrierrec
func (w *gcWork) balance() {
if w.wbuf1 == nil {
}
// empty reports whether w has no mark work available.
+//
//go:nowritebarrierrec
func (w *gcWork) empty() bool {
return w.wbuf1 == nil || (w.wbuf1.nobj == 0 && w.wbuf2.nobj == 0)
// getempty pops an empty work buffer off the work.empty list,
// allocating new buffers if none are available.
+//
//go:nowritebarrier
func getempty() *workbuf {
var b *workbuf
// putempty puts a workbuf onto the work.empty list.
// Upon entry this goroutine owns b. The lfstack.push relinquishes ownership.
+//
//go:nowritebarrier
func putempty(b *workbuf) {
b.checkempty()
// putfull puts the workbuf on the work.full list for the GC.
// putfull accepts partially full buffers so the GC can avoid competing
// with the mutators for ownership of partially full buffers.
+//
//go:nowritebarrier
func putfull(b *workbuf) {
b.checknonempty()
// trygetfull tries to get a full or partially empty workbuffer.
// If one is not immediately available return nil
+//
//go:nowritebarrier
func trygetfull() *workbuf {
b := (*workbuf)(work.full.pop())
// inheap reports whether b is a pointer into a (potentially dead) heap object.
// It returns false for pointers into mSpanManual spans.
// Non-preemptible because it is used by write barriers.
+//
//go:nowritebarrier
//go:nosplit
func inheap(b uintptr) bool {
func msanmove(dst, src unsafe.Pointer, sz uintptr)
// These are called from msan_GOARCH.s
+//
//go:cgo_import_static __msan_read_go
//go:cgo_import_static __msan_write_go
//go:cgo_import_static __msan_malloc_go
// readGCStats_m must be called on the system stack because it acquires the heap
// lock. See mheap for details.
+//
//go:systemstack
func readGCStats_m(pauses *[]uint64) {
p := *pauses
// load atomically reads the value of the stat.
//
// Must be nosplit as it is called in runtime initialization, e.g. newosproc0.
+//
//go:nosplit
func (s *sysMemStat) load() uint64 {
return atomic.Load64((*uint64)(s))
// add atomically adds the sysMemStat by n.
//
// Must be nosplit as it is called in runtime initialization, e.g. newosproc0.
+//
//go:nosplit
func (s *sysMemStat) add(n int64) {
if s == nil {
// poll_runtime_pollReset, which is internal/poll.runtime_pollReset,
// prepares a descriptor for polling in mode, which is 'r' or 'w'.
// This returns an error code; the codes are defined above.
+//
//go:linkname poll_runtime_pollReset internal/poll.runtime_pollReset
func poll_runtime_pollReset(pd *pollDesc, mode int) int {
errcode := netpollcheckerr(pd, int32(mode))
// waits for a descriptor to be ready for reading or writing,
// according to mode, which is 'r' or 'w'.
// This returns an error code; the codes are defined above.
+//
//go:linkname poll_runtime_pollWait internal/poll.runtime_pollWait
func poll_runtime_pollWait(pd *pollDesc, mode int) int {
errcode := netpollcheckerr(pd, int32(mode))
// whether the fd is ready for reading or writing or both.
//
// This may run while the world is stopped, so write barriers are not allowed.
+//
//go:nowritebarrier
func netpollready(toRun *gList, pd *pollDesc, mode int32) {
var rg, wg *g
// delay < 0: blocks indefinitely
// delay == 0: does not block, just polls
// delay > 0: block for up to that many nanoseconds
+//
//go:nowritebarrierrec
func netpoll(delay int64) gList {
var timeout uintptr
// license that can be found in the LICENSE file.
// The file contains tests that cannot run under race detector for some reason.
+//
//go:build !race
package runtime_test
// license that can be found in the LICENSE file.
// The file contains tests that cannot run under race detector for some reason.
+//
//go:build !race
package runtime_test
// assembly routine; the higher bits (if required), should be provided
// by the assembly routine as 0.
// The err result is an OS error code such as ENOMEM.
+//
//go:nosplit
func mmap(addr unsafe.Pointer, n uintptr, prot, flags, fd int32, off uint32) (unsafe.Pointer, int) {
r, err0 := syscall6(&libc_mmap, uintptr(addr), uintptr(n), uintptr(prot), uintptr(flags), uintptr(fd), uintptr(off))
func tstart_sysvicall(newm *m) uint32
// May run with m.p==nil, so write barriers are not allowed.
+//
//go:nowritebarrier
func newosproc(mp *m) {
var (
}
// setSignaltstackSP sets the ss_sp field of a stackt.
+//
//go:nosplit
func setSignalstackSP(s *stackt, sp uintptr) {
*(*uintptr)(unsafe.Pointer(&s.ss_sp)) = sp
// Called to do synchronous initialization of Go code built with
// -buildmode=c-archive or -buildmode=c-shared.
// None of the Go runtime is initialized.
+//
//go:nosplit
//go:nowritebarrierrec
func libpreinit() {
}
// setSignaltstackSP sets the ss_sp field of a stackt.
+//
//go:nosplit
func setSignalstackSP(s *stackt, sp uintptr) {
*(*uintptr)(unsafe.Pointer(&s.ss_sp)) = sp
}
// May run with m.p==nil, so write barriers are not allowed.
+//
//go:nowritebarrierrec
func newosproc(mp *m) {
stk := unsafe.Pointer(mp.g0.stack.hi)
// Called to do synchronous initialization of Go code built with
// -buildmode=c-archive or -buildmode=c-shared.
// None of the Go runtime is initialized.
+//
//go:nosplit
//go:nowritebarrierrec
func libpreinit() {
}
// Called from dropm to undo the effect of an minit.
+//
//go:nosplit
func unminit() {
// iOS does not support alternate signal stack.
}
// setSignaltstackSP sets the ss_sp field of a stackt.
+//
//go:nosplit
func setSignalstackSP(s *stackt, sp uintptr) {
*(*uintptr)(unsafe.Pointer(&s.ss_sp)) = sp
func lwp_start(uintptr)
// May run with m.p==nil, so write barriers are not allowed.
+//
//go:nowritebarrier
func newosproc(mp *m) {
stk := unsafe.Pointer(mp.g0.stack.hi)
}
// Called from dropm to undo the effect of an minit.
+//
//go:nosplit
func unminit() {
unminitSignals()
}
// setSignaltstackSP sets the ss_sp field of a stackt.
+//
//go:nosplit
func setSignalstackSP(s *stackt, sp uintptr) {
s.ss_sp = sp
func thr_start()
// May run with m.p==nil, so write barriers are not allowed.
+//
//go:nowritebarrier
func newosproc(mp *m) {
stk := unsafe.Pointer(mp.g0.stack.hi)
}
// Version of newosproc that doesn't require a valid G.
+//
//go:nosplit
func newosproc0(stacksize uintptr, fn unsafe.Pointer) {
stack := sysAlloc(stacksize, &memstats.stacks_sys)
// Called to do synchronous initialization of Go code built with
// -buildmode=c-archive or -buildmode=c-shared.
// None of the Go runtime is initialized.
+//
//go:nosplit
//go:nowritebarrierrec
func libpreinit() {
}
// Called from dropm to undo the effect of an minit.
+//
//go:nosplit
func unminit() {
unminitSignals()
}
// setSignaltstackSP sets the ss_sp field of a stackt.
+//
//go:nosplit
func setSignalstackSP(s *stackt, sp uintptr) {
s.ss_sp = sp
}
// sysSigaction calls the sigaction system call.
+//
//go:nosplit
func sysSigaction(sig uint32, new, old *sigactiont) {
// Use system stack to avoid split stack overflow on amd64
}
// asmSigaction is implemented in assembly.
+//
//go:noescape
func asmSigaction(sig uintptr, new, old *sigactiont) int32
}
// May run with m.p==nil, so write barriers are not allowed.
+//
//go:nowritebarrier
func newosproc(mp *m) {
panic("newosproc: not implemented")
// if(*addr == val) sleep
// Might be woken up spuriously; that's allowed.
// Don't sleep longer than ns; ns < 0 means forever.
+//
//go:nosplit
func futexsleep(addr *uint32, val uint32, ns int64) {
// Some Linux kernels have a bug where futex of
}
// If any procs are sleeping on addr, wake up at most cnt.
+//
//go:nosplit
func futexwakeup(addr *uint32, cnt uint32) {
ret := futex(unsafe.Pointer(addr), _FUTEX_WAKE_PRIVATE, cnt, nil, nil, 0)
func clone(flags int32, stk, mp, gp, fn unsafe.Pointer) int32
// May run with m.p==nil, so write barriers are not allowed.
+//
//go:nowritebarrier
func newosproc(mp *m) {
stk := unsafe.Pointer(mp.g0.stack.hi)
}
// Version of newosproc that doesn't require a valid G.
+//
//go:nosplit
func newosproc0(stacksize uintptr, fn unsafe.Pointer) {
stack := sysAlloc(stacksize, &memstats.stacks_sys)
// Called to do synchronous initialization of Go code built with
// -buildmode=c-archive or -buildmode=c-shared.
// None of the Go runtime is initialized.
+//
//go:nosplit
//go:nowritebarrierrec
func libpreinit() {
}
// Called from dropm to undo the effect of an minit.
+//
//go:nosplit
func unminit() {
unminitSignals()
}
// setSignaltstackSP sets the ss_sp field of a stackt.
+//
//go:nosplit
func setSignalstackSP(s *stackt, sp uintptr) {
*(*uintptr)(unsafe.Pointer(&s.ss_sp)) = sp
}
// sysSigaction calls the rt_sigaction system call.
+//
//go:nosplit
func sysSigaction(sig uint32, new, old *sigactiont) {
if rt_sigaction(uintptr(sig), new, old, unsafe.Sizeof(sigactiont{}.sa_mask)) != 0 {
}
// rt_sigaction is implemented in assembly.
+//
//go:noescape
func rt_sigaction(sig uintptr, new, old *sigactiont, size uintptr) int32
}
// May run with m.p==nil, so write barriers are not allowed.
+//
//go:nowritebarrier
func newosproc(mp *m) {
stk := unsafe.Pointer(mp.g0.stack.hi)
// baroque to remove a signal stack here only to add one in minit, but
// it's a simple change that keeps NetBSD working like other OS's.
// At this point all signals are blocked, so there is no race.
+//
//go:nosplit
func netbsdMstart0() {
st := stackt{ss_flags: _SS_DISABLE}
}
// Called from dropm to undo the effect of an minit.
+//
//go:nosplit
func unminit() {
unminitSignals()
}
// setSignaltstackSP sets the ss_sp field of a stackt.
+//
//go:nosplit
func setSignalstackSP(s *stackt, sp uintptr) {
s.ss_sp = sp
}
// Called from dropm to undo the effect of an minit.
+//
//go:nosplit
func unminit() {
unminitSignals()
}
// setSignaltstackSP sets the ss_sp field of a stackt.
+//
//go:nosplit
func setSignalstackSP(s *stackt, sp uintptr) {
s.ss_sp = sp
func mstart_stub()
// May run with m.p==nil, so write barriers are not allowed.
+//
//go:nowritebarrierrec
func newosproc(mp *m) {
if false {
func tfork(param *tforkt, psize uintptr, mm *m, gg *g, fn uintptr) int32
// May run with m.p==nil, so write barriers are not allowed.
+//
//go:nowritebarrier
func newosproc(mp *m) {
stk := unsafe.Pointer(mp.g0.stack.hi)
// write calls the write system call.
// It returns a non-negative number of bytes written or a negative errno value.
+//
//go:noescape
func write1(fd uintptr, p unsafe.Pointer, n int32) int32
}
// May run with m.p==nil, so write barriers are not allowed.
+//
//go:nowritebarrier
func newosproc(mp *m) {
if false {
var _badsignal = []byte("runtime: signal received on thread not created by Go.\n")
// This runs on a foreign stack, without an m or a g. No stack split.
+//
//go:nosplit
func badsignal2() {
pwrite(2, unsafe.Pointer(&_badsignal[0]), int32(len(_badsignal)), -1)
// May run with m.p==nil, so write barriers are not allowed. This
// function is called by newosproc0, so it is also required to
// operate without stack guards.
+//
//go:nowritebarrierrec
//go:nosplit
func newosproc(mp *m) {
// Used by the C library build mode. On Linux this function would allocate a
// stack, but that's not necessary for Windows. No stack guards are present
// and the GC has not been initialized, so write barriers will fail.
+//
//go:nowritebarrierrec
//go:nosplit
func newosproc0(mp *m, stk unsafe.Pointer) {
}
// Called from dropm to undo the effect of an minit.
+//
//go:nosplit
func unminit() {
mp := getg().m
// Called from exitm, but not from drop, to undo the effect of thread-owned
// resources in minit, semacreate, or elsewhere. Do not take locks after calling this.
+//
//go:nosplit
func mdestroy(mp *m) {
if mp.highResTimer != 0 {
// Calling stdcall on os stack.
// May run during STW, so write barriers are not allowed.
+//
//go:nowritebarrier
//go:nosplit
func stdcall(fn stdFunction) uintptr {
// getargp returns the location where the caller
// writes outgoing function call arguments.
+//
//go:nosplit
//go:noinline
func getargp() uintptr {
//
// TODO(rsc): Once we commit to CopyStackAlways,
// this doesn't need to be nosplit.
+//
//go:nosplit
func gorecover(argp uintptr) any {
// Must be in a function running as part of a deferred call during the panic.
}
// Used by TestBlockProfileBias.
+//
//go:linkname blockevent runtime.blockevent
func blockevent(cycles int64, skip int)
}
// os_beforeExit is called from os.Exit(0).
+//
//go:linkname os_beforeExit os.runtime_beforeExit
func os_beforeExit() {
if raceenabled {
// goschedguarded yields the processor like gosched, but also checks
// for forbidden states and opts out of the yield in those cases.
+//
//go:nosplit
func goschedguarded() {
mcall(goschedguarded_m)
// All reads and writes of g's status go through readgstatus, casgstatus
// castogscanstatus, casfrom_Gscanstatus.
+//
//go:nosplit
func readgstatus(gp *g) uint32 {
return atomic.Load(&gp.atomicstatus)
// and casfrom_Gscanstatus instead.
// casgstatus will loop if the g->atomicstatus is in a Gscan status until the routine that
// put it in the Gscan state is finished.
+//
//go:nosplit
func casgstatus(gp *g, oldval, newval uint32) {
if (oldval&_Gscan != 0) || (newval&_Gscan != 0) || oldval == newval {
// async wakeup that might come in from netpoll. If we see Gwaiting from the readgstatus,
// it might have become Grunnable by the time we get to the cas. If we called casgstatus,
// it would loop waiting for the status to go back to Gwaiting, which it never will.
+//
//go:nosplit
func casgcopystack(gp *g) uint32 {
for {
// The go:noinline is to guarantee the getcallerpc/getcallersp below are safe,
// so that we can set up g0.sched to return to the call of mstart1 above.
+//
//go:noinline
func mstart1() {
_g_ := getg()
}
// mPark causes a thread to park itself, returning once woken.
+//
//go:nosplit
func mPark() {
gp := getg()
//
// When the callback is done with the m, it calls dropm to
// put the m back on the list.
+//
//go:nosplit
func needm() {
if (iscgo || GOOS == "windows") && !cgoHasExtraM {
// to extram. If nilokay is true, then lockextra will
// return a nil list head if that's what it finds. If nilokay is false,
// lockextra will keep waiting until the list head is no longer nil.
+//
//go:nosplit
func lockextra(nilokay bool) *m {
const locked = 1
// May run with m.p==nil, so write barriers are not allowed.
//
// id is optional pre-allocated m ID. Omit by passing -1.
+//
//go:nowritebarrierrec
func newm(fn func(), _p_ *p, id int64) {
// allocm adds a new M to allm, but they do not start until created by
// comment on acquirem below.
//
// Must not have write barriers because this may be called without a P.
+//
//go:nowritebarrierrec
func startm(_p_ *p, spinning bool) {
// Disable preemption.
// Hands off P from syscall or locked M.
// Always runs without a P, so write barriers are not allowed.
+//
//go:nowritebarrierrec
func handoffp(_p_ *p) {
// handoffp must start an M in any situation where
// Schedules the locked m to run the locked gp.
// May run during STW, so write barriers are not allowed.
+//
//go:nowritebarrierrec
func startlockedm(gp *g) {
_g_ := getg()
// If the time when the next timer should run is not 0,
// it is always larger than the returned time.
// We pass now in and out to avoid extra calls of nanotime.
+//
//go:yeswritebarrierrec
func checkTimers(pp *p, now int64) (rnow, pollUntil int64, ran bool) {
// If it's not yet time for the first timer, or the first adjusted
}
// The same as entersyscall(), but with a hint that the syscall is blocking.
+//
//go:nosplit
func entersyscallblock() {
_g_ := getg()
}
// Called from syscall package before fork.
+//
//go:linkname syscall_runtime_BeforeFork syscall.runtime_BeforeFork
//go:nosplit
func syscall_runtime_BeforeFork() {
}
// Called from syscall package after fork in parent.
+//
//go:linkname syscall_runtime_AfterFork syscall.runtime_AfterFork
//go:nosplit
func syscall_runtime_AfterFork() {
var pendingPreemptSignals uint32
// Called from syscall package before Exec.
+//
//go:linkname syscall_runtime_BeforeExec syscall.runtime_BeforeExec
func syscall_runtime_BeforeExec() {
// Prevent thread creation during exec.
}
// Called from syscall package after Exec.
+//
//go:linkname syscall_runtime_AfterExec syscall.runtime_AfterExec
func syscall_runtime_AfterExec() {
execLock.unlock()
// dolockOSThread is called by LockOSThread and lockOSThread below
// after they modify m.locked. Do not allow preemption during this call,
// or else the m might be different in this function than in the caller.
+//
//go:nosplit
func dolockOSThread() {
if GOARCH == "wasm" {
// dounlockOSThread is called by UnlockOSThread and unlockOSThread below
// after they update m->locked. Do not allow preemption during this call,
// or else the m might be in different in this function than in the caller.
+//
//go:nosplit
func dounlockOSThread() {
if GOARCH == "wasm" {
// Called if we receive a SIGPROF signal.
// Called by the signal handler, may run during STW.
+//
//go:nowritebarrierrec
func sigprof(pc, sp, lr uintptr, gp *g, mp *m) {
if prof.hz == 0 {
// Put mp on midle list.
// sched.lock must be held.
// May run during STW, so write barriers are not allowed.
+//
//go:nowritebarrierrec
func mput(mp *m) {
assertLockHeld(&sched.lock)
// Try to get an m from midle list.
// sched.lock must be held.
// May run during STW, so write barriers are not allowed.
+//
//go:nowritebarrierrec
func mget() *m {
assertLockHeld(&sched.lock)
// Put gp on the global runnable queue.
// sched.lock must be held.
// May run during STW, so write barriers are not allowed.
+//
//go:nowritebarrierrec
func globrunqput(gp *g) {
assertLockHeld(&sched.lock)
// Put gp at the head of the global runnable queue.
// sched.lock must be held.
// May run during STW, so write barriers are not allowed.
+//
//go:nowritebarrierrec
func globrunqputhead(gp *g) {
assertLockHeld(&sched.lock)
// This clears *batch.
// sched.lock must be held.
// May run during STW, so write barriers are not allowed.
+//
//go:nowritebarrierrec
func globrunqputbatch(batch *gQueue, n int32) {
assertLockHeld(&sched.lock)
// sched.lock must be held.
//
// May run during STW, so write barriers are not allowed.
+//
//go:nowritebarrierrec
func pidleput(_p_ *p) {
assertLockHeld(&sched.lock)
// sched.lock must be held.
//
// May run during STW, so write barriers are not allowed.
+//
//go:nowritebarrierrec
func pidleget() *p {
assertLockHeld(&sched.lock)
}
// Active spinning for sync.Mutex.
+//
//go:linkname sync_runtime_canSpin sync.runtime_canSpin
//go:nosplit
func sync_runtime_canSpin(i int) bool {
}
// fakeSyscall emulates a system call.
+//
//go:nosplit
func fakeSyscall(duration time.Duration) {
runtime.Entersyscall()
}
// Race runtime functions called via runtime·racecall.
+//
//go:linkname __tsan_init __tsan_init
var __tsan_init byte
var __tsan_report_count byte
// Mimic what cmd/cgo would do.
+//
//go:cgo_import_static __tsan_init
//go:cgo_import_static __tsan_fini
//go:cgo_import_static __tsan_proc_create
//go:cgo_import_static __tsan_report_count
// These are called from race_amd64.s.
+//
//go:cgo_import_static __tsan_read
//go:cgo_import_static __tsan_read_pc
//go:cgo_import_static __tsan_read_range
func racecall(fn *byte, arg0, arg1, arg2, arg3 uintptr)
// checks if the address has shadow (i.e. heap or data/bss)
+//
//go:nosplit
func isvalidaddr(addr unsafe.Pointer) bool {
return racearenastart <= uintptr(addr) && uintptr(addr) < racearenaend ||
)
// nosplit for use in linux startup sysargs
+//
//go:nosplit
func argv_index(argv **byte, i int32) *byte {
return *(**byte)(add(unsafe.Pointer(argv), uintptr(i)*goarch.PtrSize))
// int64 division is lowered into _divv() call on 386, which does not fit into nosplit functions.
// Handles overflow in a time-specific manner.
// This keeps us within no-split stack limits on 32-bit processors.
+//
//go:nosplit
func timediv(v int64, div int32, rem *int32) int32 {
res := int32(0)
}
// reflect_resolveNameOff resolves a name offset from a base pointer.
+//
//go:linkname reflect_resolveNameOff reflect.resolveNameOff
func reflect_resolveNameOff(ptrInModule unsafe.Pointer, off int32) unsafe.Pointer {
return unsafe.Pointer(resolveNameOff(ptrInModule, nameOff(off)).bytes)
}
// reflect_resolveTypeOff resolves an *rtype offset from a base type.
+//
//go:linkname reflect_resolveTypeOff reflect.resolveTypeOff
func reflect_resolveTypeOff(rtype unsafe.Pointer, off int32) unsafe.Pointer {
return unsafe.Pointer((*_type)(rtype).typeOff(typeOff(off)))
}
// reflect_resolveTextOff resolves a function pointer offset from a base type.
+//
//go:linkname reflect_resolveTextOff reflect.resolveTextOff
func reflect_resolveTextOff(rtype unsafe.Pointer, off int32) unsafe.Pointer {
return (*_type)(rtype).textOff(textOff(off))
}
// reflectlite_resolveNameOff resolves a name offset from a base pointer.
+//
//go:linkname reflectlite_resolveNameOff internal/reflectlite.resolveNameOff
func reflectlite_resolveNameOff(ptrInModule unsafe.Pointer, off int32) unsafe.Pointer {
return unsafe.Pointer(resolveNameOff(ptrInModule, nameOff(off)).bytes)
}
// reflectlite_resolveTypeOff resolves an *rtype offset from a base type.
+//
//go:linkname reflectlite_resolveTypeOff internal/reflectlite.resolveTypeOff
func reflectlite_resolveTypeOff(rtype unsafe.Pointer, off int32) unsafe.Pointer {
return unsafe.Pointer((*_type)(rtype).typeOff(typeOff(off)))
}
// reflect_addReflectOff adds a pointer to the reflection offset lookup map.
+//
//go:linkname reflect_addReflectOff reflect.addReflectOff
func reflect_addReflectOff(ptr unsafe.Pointer) int32 {
reflectOffsLock()
// setGNoWB performs *gp = new without a write barrier.
// For times when it's impractical to use a guintptr.
+//
//go:nosplit
//go:nowritebarrier
func setGNoWB(gp **g, new *g) {
// setMNoWB performs *mp = new without a write barrier.
// For times when it's impractical to use an muintptr.
+//
//go:nosplit
//go:nowritebarrier
func setMNoWB(mp **m, new *m) {
// testSetPanicOnFault tests one potentially faulting address.
// It deliberately constructs and uses an invalid pointer,
// so mark it as nocheckptr.
+//
//go:nocheckptr
func testSetPanicOnFault(t *testing.T, addr uintptr, nfault *int) {
if GOOS == "js" {
// notifyListAdd adds the caller to a notify list such that it can receive
// notifications. The caller must eventually call notifyListWait to wait for
// such a notification, passing the returned ticket number.
+//
//go:linkname notifyListAdd sync.runtime_notifyListAdd
func notifyListAdd(l *notifyList) uint32 {
// This may be called concurrently, for example, when called from
// notifyListWait waits for a notification. If one has been sent since
// notifyListAdd was called, it returns immediately. Otherwise, it blocks.
+//
//go:linkname notifyListWait sync.runtime_notifyListWait
func notifyListWait(l *notifyList, t uint32) {
lockWithRank(&l.lock, lockRankNotifyList)
}
// notifyListNotifyAll notifies all entries in the list.
+//
//go:linkname notifyListNotifyAll sync.runtime_notifyListNotifyAll
func notifyListNotifyAll(l *notifyList) {
// Fast-path: if there are no new waiters since the last notification
}
// notifyListNotifyOne notifies one entry in the list.
+//
//go:linkname notifyListNotifyOne sync.runtime_notifyListNotifyOne
func notifyListNotifyOne(l *notifyList) {
// Fast-path: if there are no new waiters since the last notification
// Initialize signals.
// Called by libpreinit so runtime may not be initialized.
+//
//go:nosplit
//go:nowritebarrierrec
func initsig(preinit bool) {
// back to the default. This is called by the child after a fork, so that
// we can enable the signal mask for the exec without worrying about
// running a signal handler in the child.
+//
//go:nosplit
//go:nowritebarrierrec
func clearSignalHandlers() {
// sigprofNonGoPC is called when a profiling signal arrived on a
// non-Go thread and we have a single PC value, not a stack trace.
// g is nil, and what we can do is very limited.
+//
//go:nosplit
//go:nowritebarrierrec
func sigprofNonGoPC(pc uintptr) {
// We do this in case some non-Go code called sigaltstack.
// This reports whether the stack was adjusted, and if so stores the old
// signal stack in *gsigstack.
+//
//go:nosplit
func adjustSignalStack(sig uint32, mp *m, gsigStack *gsignalStack) bool {
sp := uintptr(unsafe.Pointer(&sig))
// getg().throwsplit, since sigpanic may need to grow the stack.
//
// This is exported via linkname to assembly in runtime/cgo.
+//
//go:linkname sigpanic
func sigpanic() {
g := getg()
// dieFromSignal kills the program with a signal.
// This provides the expected exit status for the shell.
// This is only called with fatal signals expected to kill the process.
+//
//go:nosplit
//go:nowritebarrierrec
func dieFromSignal(sig uint32) {
var badginsignalMsg = "fatal: bad g in signal handler\n"
// This runs on a foreign stack, without an m or a g. No stack split.
+//
//go:nosplit
//go:norace
//go:nowritebarrierrec
// signal to the handler that was installed before Go's. Returns whether the
// signal was forwarded.
// This is called by the signal handler, and the world may be stopped.
+//
//go:nosplit
//go:nowritebarrierrec
func sigfwdgo(sig uint32, info *siginfo, ctx unsafe.Pointer) bool {
// thread calls a Go function.
// This is nosplit and nowritebarrierrec because it is called by needm
// which may be called on a non-Go thread with no g available.
+//
//go:nosplit
//go:nowritebarrierrec
func sigsave(p *sigset) {
// calls a Go function.
// This is nosplit and nowritebarrierrec because it is called by dropm
// after g has been cleared.
+//
//go:nosplit
//go:nowritebarrierrec
func msigrestore(sigmask sigset) {
// definition of sigset_all is used.
// This is nosplit and nowritebarrierrec because it is called by needm
// which may be called on a non-Go thread with no g available.
+//
//go:nosplit
//go:nowritebarrierrec
func sigblock(exiting bool) {
// This is nosplit and nowritebarrierrec because it is called from
// dieFromSignal, which can be called by sigfwdgo while running in the
// signal handler, on the signal stack, with no g available.
+//
//go:nosplit
//go:nowritebarrierrec
func unblocksig(sig uint32) {
// unminitSignals is called from dropm, via unminit, to undo the
// effect of calling minit on a non-Go thread.
+//
//go:nosplit
func unminitSignals() {
if getg().m.newSigstack {
// It saves the old values in *old for use by restoreGsignalStack.
// This is used when handling a signal if non-Go code has set the
// alternate signal stack.
+//
//go:nosplit
//go:nowritebarrierrec
func setGsignalStack(st *stackt, old *gsignalStack) {
// restoreGsignalStack restores the gsignal stack to the value it had
// before entering the signal handler.
+//
//go:nosplit
//go:nowritebarrierrec
func restoreGsignalStack(st *gsignalStack) {
}
// signalstack sets the current thread's alternate signal stack to s.
+//
//go:nosplit
func signalstack(s *stack) {
st := stackt{ss_size: s.hi - s.lo}
// Called to receive the next queued signal.
// Must only be called from a single goroutine at a time.
+//
//go:linkname signal_recv os/signal.signal_recv
func signal_recv() uint32 {
for {
// the signal(s) in question, and here we are just waiting to make sure
// that all the signals have been delivered to the user channels
// by the os/signal package.
+//
//go:linkname signalWaitUntilIdle os/signal.signalWaitUntilIdle
func signalWaitUntilIdle() {
// Although the signals we care about have been removed from
}
// Must only be called from a single goroutine at a time.
+//
//go:linkname signal_enable os/signal.signal_enable
func signal_enable(s uint32) {
if !sig.inuse {
}
// Must only be called from a single goroutine at a time.
+//
//go:linkname signal_disable os/signal.signal_disable
func signal_disable(s uint32) {
if s >= uint32(len(sig.wanted)*32) {
}
// Must only be called from a single goroutine at a time.
+//
//go:linkname signal_ignore os/signal.signal_ignore
func signal_ignore(s uint32) {
if s >= uint32(len(sig.wanted)*32) {
// sigInitIgnored marks the signal as already ignored. This is called at
// program start by initsig. In a shared library initsig is called by
// libpreinit, so the runtime may not be initialized yet.
+//
//go:nosplit
func sigInitIgnored(s uint32) {
i := sig.ignored[s/32]
}
// Checked by signal handlers.
+//
//go:linkname signal_ignored os/signal.signal_ignored
func signal_ignored(s uint32) bool {
i := atomic.Load(&sig.ignored[s/32])
// Called to receive the next queued signal.
// Must only be called from a single goroutine at a time.
+//
//go:linkname signal_recv os/signal.signal_recv
func signal_recv() string {
for {
// the signal(s) in question, and here we are just waiting to make sure
// that all the signals have been delivered to the user channels
// by the os/signal package.
+//
//go:linkname signalWaitUntilIdle os/signal.signalWaitUntilIdle
func signalWaitUntilIdle() {
for {
}
// Must only be called from a single goroutine at a time.
+//
//go:linkname signal_enable os/signal.signal_enable
func signal_enable(s uint32) {
if !sig.inuse {
}
// Must only be called from a single goroutine at a time.
+//
//go:linkname signal_disable os/signal.signal_disable
func signal_disable(s uint32) {
}
// Must only be called from a single goroutine at a time.
+//
//go:linkname signal_ignore os/signal.signal_ignore
func signal_ignore(s uint32) {
}
}
// This is exported via linkname to assembly in syscall (for Plan9).
+//
//go:linkname gostring
func gostring(p *byte) string {
l := findnull(p)
)
// Should be a built-in for unsafe.Pointer?
+//
//go:nosplit
func add(p unsafe.Pointer, x uintptr) unsafe.Pointer {
return unsafe.Pointer(uintptr(p) + x)
func memmove(to, from unsafe.Pointer, n uintptr)
// Outside assembly calls memmove. Make sure it has ABI wrappers.
+//
//go:linkname memmove
//go:linkname reflect_memmove reflect.memmove
func os_fastrand() uint32 { return fastrand() }
// in internal/bytealg/equal_*.s
+//
//go:noescape
func memequal(a, b unsafe.Pointer, size uintptr) bool
// output depends on the input. noescape is inlined and currently
// compiles down to zero instructions.
// USE CAREFULLY!
+//
//go:nosplit
func noescape(p unsafe.Pointer) unsafe.Pointer {
x := uintptr(p)
// Arguments passed through to reflectcall do not escape. The type is used
// only in a very limited callee of reflectcall, the stackArgs are copied, and
// regArgs is only used in the reflectcall frame.
+//
//go:noescape
func reflectcall(stackArgsType *_type, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
// write calls the write system call.
// It returns a non-negative number of bytes written or a negative errno value.
+//
//go:noescape
func write1(fd uintptr, p unsafe.Pointer, n int32) int32
// Called from write_err_android.go only, but defined in sys_linux_*.s;
// declared here (instead of in write_err_android.go) for go vet on non-android builds.
// The return value is the raw syscall result, which may encode an error number.
+//
//go:noescape
func access(name *byte, mode int32) int32
func connect(fd int32, addr unsafe.Pointer, len int32) int32
package runtime
// This is needed for vet
+//
//go:noescape
func callCgoSigaction(sig uintptr, new, old *sigactiont) int32
//
// This is nosplit/nowritebarrier because it is called by the
// cgo pointer checking code.
+//
//go:nosplit
//go:nowritebarrier
func activeModules() []*moduledata {
// relocated baseaddr to compute the function address.
//
// It is nosplit because it is part of the findfunc implementation.
+//
//go:nosplit
func (md *moduledata) textAddr(off32 uint32) uintptr {
off := uintptr(off32)
// to md.text, and returns if the PC is in any Go text section.
//
// It is nosplit because it is part of the findfunc implementation.
+//
//go:nosplit
func (md *moduledata) textOff(pc uintptr) (uint32, bool) {
res := uint32(pc - md.text)
// These are marked noinline so that we can use FuncForPC
// in testCallerBar.
+//
//go:noinline
func testCallerFoo(t *testing.T) {
testCallerBar(t)
// mmap is used to do low-level memory allocation via mmap. Don't allow stack
// splits, since this function (used by sysAlloc) is called in a lot of low-level
// parts of the runtime and callers often assume it won't acquire any locks.
+//
//go:nosplit
func mmap(addr unsafe.Pointer, n uintptr, prot, flags, fd int32, off uint32) (unsafe.Pointer, int) {
args := struct {
//go:cgo_unsafe_args
//
// This is exported via linkname to assembly in runtime/cgo.
+//
//go:linkname exit
func exit(code int32) {
libcCall(unsafe.Pointer(abi.FuncPCABI0(exit_trampoline)), unsafe.Pointer(&code))
// fn is the raw pc value of the entry point of the desired function.
// Switches to the system stack, if not already there.
// Preserves the calling point as the location where a profiler traceback will begin.
+//
//go:nosplit
func libcCall(fn, arg unsafe.Pointer) int32 {
// Leave caller's PC/SP/G around for traceback.
)
// This is exported via linkname to assembly in runtime/cgo.
+//
//go:linkname exit
//go:nosplit
//go:cgo_unsafe_args
// mmap is used to do low-level memory allocation via mmap. Don't allow stack
// splits, since this function (used by sysAlloc) is called in a lot of low-level
// parts of the runtime and callers often assume it won't acquire any locks.
+//
//go:nosplit
func mmap(addr unsafe.Pointer, n uintptr, prot, flags, fd int32, off uint32) (unsafe.Pointer, int) {
args := struct {
}
// like close, but must not split stack, for fork.
+//
//go:linkname syscall_close syscall.close
//go:nosplit
func syscall_close(fd int32) int32 {
}
// like exit, but must not split stack, for fork.
+//
//go:linkname syscall_exit syscall.exit
//go:nosplit
func syscall_exit(code uintptr) {
}
// like close, but must not split stack, for forkx.
+//
//go:nosplit
//go:linkname syscall_close
func syscall_close(fd int32) int32 {
}
// like exit, but must not split stack, for forkx.
+//
//go:nosplit
//go:linkname syscall_exit
func syscall_exit(code uintptr) {
// parameter and the important SEARCH_SYSTEM32 argument. But on systems that
// do not have that option, absoluteFilepath should contain a fallback
// to the full path inside of system32 for use with vanilla LoadLibrary.
+//
//go:linkname syscall_loadsystemlibrary syscall.loadsystemlibrary
//go:nosplit
//go:cgo_unsafe_args
// that insufficient spill slots allocated (according to the ABI)
// may cause compiler-generated spills to clobber the return PC.
// Then, the GC stack scanning will catch that.
+//
//go:registerparams
func sum9andGC(i1, i2, i3, i4, i5, i6, i7, i8, i9 uint32) uintptr {
runtime.GC()
import _ "unsafe" // for go:linkname
// Defined in the runtime package.
+//
//go:linkname runtime_getm_for_test runtime.getm
func runtime_getm_for_test() uintptr
// spin does CPU bound spinning and allocating for a millisecond,
// to get a SIGURG.
+//
//go:noinline
func spin() (float64, []byte) {
stop := time.Now().Add(time.Millisecond)
// time.now is implemented in assembly.
// timeSleep puts the current goroutine to sleep for at least ns nanoseconds.
+//
//go:linkname timeSleep time.Sleep
func timeSleep(ns int64) {
if ns <= 0 {
}
// startTimer adds t to the timer heap.
+//
//go:linkname startTimer time.startTimer
func startTimer(t *timer) {
if raceenabled {
// stopTimer stops a timer.
// It reports whether t was stopped before being run.
+//
//go:linkname stopTimer time.stopTimer
func stopTimer(t *timer) bool {
return deltimer(t)
}
// resetTimer resets an inactive timer, adding it to the heap.
+//
//go:linkname resetTimer time.resetTimer
// Reports whether the timer was modified before it was run.
func resetTimer(t *timer, when int64) bool {
}
// modTimer modifies an existing timer.
+//
//go:linkname modTimer time.modTimer
func modTimer(t *timer, when, period int64, f func(any, uintptr), arg any, seq uintptr) {
modtimer(t, when, period, f, arg, seq)
// should wake up the netpoller. It returns 0 if there are no timers.
// This function is invoked when dropping a P, and must run without
// any write barriers.
+//
//go:nowritebarrierrec
func nobarrierWakeTime(pp *p) int64 {
next := int64(atomic.Load64(&pp.timer0When))
// when the first timer should run.
// The caller must have locked the timers for pp.
// If a timer is run, this will temporarily unlock the timers.
+//
//go:systemstack
func runtimer(pp *p, now int64) int64 {
for {
// runOneTimer runs a single timer.
// The caller must have locked the timers for pp.
// This will temporarily unlock the timers while running the timer function.
+//
//go:systemstack
func runOneTimer(pp *p, t *timer, now int64) {
if raceenabled {
// write is like the Unix write system call.
// We have to avoid write barriers to avoid potential deadlock
// on write calls.
+//
//go:nowritebarrierrec
func write(fd uintptr, p unsafe.Pointer, n int32) int32 {
if !(fd == 1 || fd == 2) {
}
// vdsoMarker reports whether PC is on the VDSO page.
+//
//go:nosplit
func inVDSOPage(pc uintptr) bool {
for _, k := range vdsoSymbolKeys {
// noescape is inlined and currently compiles down to zero instructions.
// USE CAREFULLY!
// This was copied from the runtime; see issues 23382 and 7921.
+//
//go:nosplit
//go:nocheckptr
func noescape(p unsafe.Pointer) unsafe.Pointer {
// license that can be found in the LICENSE file.
// Pool is no-op under race detector, so all these tests do not work.
+//
//go:build !race
package sync_test
}
// gbit16 reads a 16-bit number in little-endian order from b and returns it with the remaining slice of b.
+//
//go:nosplit
func gbit16(b []byte) (uint16, []byte) {
return uint16(b[0]) | uint16(b[1])<<8, b[2:]
// For the same reason compiler does not race instrument it.
// The calls to RawSyscall are okay because they are assembly
// functions that do not grow the stack.
+//
//go:norace
func forkAndExecInChild(argv0 *byte, argv, envv []*byte, chroot, dir *byte, attr *ProcAttr, sys *SysProcAttr, pipe int) (pid int, err Errno) {
// Declare all variables at top in case any
// For the same reason compiler does not race instrument it.
// The calls to RawSyscall are okay because they are assembly
// functions that do not grow the stack.
+//
//go:norace
func forkAndExecInChild(argv0 *byte, argv, envv []*byte, chroot, dir *byte, attr *ProcAttr, sys *SysProcAttr, pipe int) (pid int, err Errno) {
// Declare all variables at top in case any
// because we need to avoid lazy-loading the functions (might malloc,
// split the stack, or acquire mutexes). We can't call RawSyscall
// because it's not safe even for BSD-subsystem calls.
+//
//go:norace
func forkAndExecInChild(argv0 *byte, argv, envv []*byte, chroot, dir *byte, attr *ProcAttr, sys *SysProcAttr, pipe int) (pid int, err Errno) {
// Declare all variables at top in case any
// For the same reason compiler does not race instrument it.
// The calls to rawSyscall are okay because they are assembly
// functions that do not grow the stack.
+//
//go:norace
func forkAndExecInChild(argv0 *byte, argv, envv []*byte, chroot, dir *byte, attr *ProcAttr, sys *SysProcAttr, pipe int) (pid int, err Errno) {
// Declare all variables at top in case any
// For the same reason compiler does not race instrument it.
// The calls to RawSyscall are okay because they are assembly
// functions that do not grow the stack.
+//
//go:norace
func forkAndExecInChild(argv0 *byte, argv, envv []*byte, chroot, dir *byte, attr *ProcAttr, sys *SysProcAttr, pipe int) (pid int, err Errno) {
// Set up and fork. This returns immediately in the parent or
// gstringb reads a non-empty string from b, prefixed with a 16-bit length in little-endian order.
// It returns the string as a byte slice, or nil if b is too short to contain the length or
// the full string.
+//
//go:nosplit
func gstringb(b []byte) []byte {
if len(b) < 2 {
// gdirname returns the first filename from a buffer of directory entries,
// and a slice containing the remaining directory entries.
// If the buffer doesn't start with a valid directory entry, the returned name is nil.
+//
//go:nosplit
func gdirname(buf []byte) (name []byte, rest []byte) {
if len(buf) < 2 {
// no rescheduling, no malloc calls, and no new stack segments.
// The calls to RawSyscall are okay because they are assembly
// functions that do not grow the stack.
+//
//go:norace
func forkAndExecInChild(argv0 *byte, argv []*byte, envv []envItem, dir *byte, attr *ProcAttr, pipe int, rflag int) (pid int, err error) {
// Declare all variables at top in case any
}
// close the numbered file descriptor, unless it is fd1, fd2, or a member of fds.
+//
//go:nosplit
func closeFdExcept(n int, fd1 int, fd2 int, fds []int) {
if n == fd1 || n == fd2 {
// Provided by runtime.syscall_runtime_doAllThreadsSyscall which stops the
// world and invokes the syscall on each OS thread. Once this function returns,
// all threads are in sync.
+//
//go:uintptrescapes
func runtime_doAllThreadsSyscall(trap, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, err uintptr)
// AllThreadsSyscall is unaware of any threads that are launched
// explicitly by cgo linked code, so the function always returns
// ENOTSUP in binaries that use cgo.
+//
//go:uintptrescapes
func AllThreadsSyscall(trap, a1, a2, a3 uintptr) (r1, r2 uintptr, err Errno) {
if cgo_libc_setegid != nil {
// AllThreadsSyscall6 is like AllThreadsSyscall, but extended to six
// arguments.
+//
//go:uintptrescapes
func AllThreadsSyscall6(trap, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err Errno) {
if cgo_libc_setegid != nil {
}
// linked by runtime.cgocall.go
+//
//go:uintptrescapes
func cgocaller(unsafe.Pointer, ...uintptr) uintptr
func now() (sec int64, nsec int32, mono int64)
// runtimeNano returns the current value of the runtime clock in nanoseconds.
+//
//go:linkname runtimeNano runtime.nanotime
func runtimeNano() int64
)
// registerLoadFromEmbeddedTZData is defined in package time.
+//
//go:linkname registerLoadFromEmbeddedTZData time.registerLoadFromEmbeddedTZData
func registerLoadFromEmbeddedTZData(func(string) (string, error))