From: Ian Lance Taylor Date: Fri, 14 Apr 2023 21:36:12 +0000 (-0700) Subject: runtime: change lfstack support to taggedPointer X-Git-Tag: go1.21rc1~860 X-Git-Url: http://www.git.cypherpunks.su/?a=commitdiff_plain;h=d959dd97b12f4067c8511fb11493e3bf47eb9737;p=gostls13.git runtime: change lfstack support to taggedPointer This is a refactoring with no change in behavior, in preparation for future netpoll work. For #59545 Change-Id: I493c5fd0f49f31b75787f7b5b89c544bed73f64f Reviewed-on: https://go-review.googlesource.com/c/go/+/484836 TryBot-Result: Gopher Robot Reviewed-by: Bryan Mills Auto-Submit: Ian Lance Taylor Reviewed-by: Michael Knyszek Run-TryBot: Ian Lance Taylor Reviewed-by: Orlando Labao --- diff --git a/src/runtime/lfstack.go b/src/runtime/lfstack.go index 306a8e888a..a91ae64e53 100644 --- a/src/runtime/lfstack.go +++ b/src/runtime/lfstack.go @@ -67,3 +67,11 @@ func lfnodeValidate(node *lfnode) { throw("bad lfnode address") } } + +func lfstackPack(node *lfnode, cnt uintptr) uint64 { + return uint64(taggedPointerPack(unsafe.Pointer(node), cnt)) +} + +func lfstackUnpack(val uint64) *lfnode { + return (*lfnode)(taggedPointer(val).pointer()) +} diff --git a/src/runtime/lfstack_32bit.go b/src/runtime/lfstack_32bit.go deleted file mode 100644 index 405923cc34..0000000000 --- a/src/runtime/lfstack_32bit.go +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright 2014 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build 386 || arm || mips || mipsle - -package runtime - -import "unsafe" - -// On 32-bit systems, the stored uint64 has a 32-bit pointer and 32-bit count. - -func lfstackPack(node *lfnode, cnt uintptr) uint64 { - return uint64(uintptr(unsafe.Pointer(node)))<<32 | uint64(cnt) -} - -func lfstackUnpack(val uint64) *lfnode { - return (*lfnode)(unsafe.Pointer(uintptr(val >> 32))) -} diff --git a/src/runtime/malloc.go b/src/runtime/malloc.go index bababe042e..de83722fff 100644 --- a/src/runtime/malloc.go +++ b/src/runtime/malloc.go @@ -405,6 +405,10 @@ func mallocinit() { throw("bad pagesPerReclaimerChunk") } + if minTagBits > taggedPointerBits { + throw("taggedPointerbits too small") + } + // Initialize the heap. mheap_.init() mcache0 = allocmcache() diff --git a/src/runtime/tagptr.go b/src/runtime/tagptr.go new file mode 100644 index 0000000000..0e17a1598f --- /dev/null +++ b/src/runtime/tagptr.go @@ -0,0 +1,14 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +// taggedPointer is a pointer with a numeric tag. +// The size of the numeric tag is GOARCH-dependent, +// currently at least 10 bits. +// This should only be used with pointers allocated outside the Go heap. +type taggedPointer uint64 + +// minTagBits is the minimum number of tag bits that we expect. +const minTagBits = 10 diff --git a/src/runtime/tagptr_32bit.go b/src/runtime/tagptr_32bit.go new file mode 100644 index 0000000000..f79e1821a1 --- /dev/null +++ b/src/runtime/tagptr_32bit.go @@ -0,0 +1,30 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build 386 || arm || mips || mipsle + +package runtime + +import "unsafe" + +// The number of bits stored in the numeric tag of a taggedPointer +const taggedPointerBits = 32 + +// On 32-bit systems, taggedPointer has a 32-bit pointer and 32-bit count. + +// taggedPointerPack created a taggedPointer from a pointer and a tag. +// Tag bits that don't fit in the result are discarded. +func taggedPointerPack(ptr unsafe.Pointer, tag uintptr) taggedPointer { + return taggedPointer(uintptr(ptr))<<32 | taggedPointer(tag) +} + +// Pointer returns the pointer from a taggedPointer. +func (tp taggedPointer) pointer() unsafe.Pointer { + return unsafe.Pointer(uintptr(tp >> 32)) +} + +// Tag returns the tag from a taggedPointer. +func (tp taggedPointer) tag() uintptr { + return uintptr(tp) +} diff --git a/src/runtime/lfstack_64bit.go b/src/runtime/tagptr_64bit.go similarity index 51% rename from src/runtime/lfstack_64bit.go rename to src/runtime/tagptr_64bit.go index 88cbd3bcc7..9ff11ccd16 100644 --- a/src/runtime/lfstack_64bit.go +++ b/src/runtime/tagptr_64bit.go @@ -6,7 +6,11 @@ package runtime -import "unsafe" +import ( + "internal/goarch" + "internal/goos" + "unsafe" +) const ( // addrBits is the number of bits needed to represent a virtual address. @@ -27,44 +31,59 @@ const ( // In addition to the 16 bits taken from the top, we can take 3 from the // bottom, because node must be pointer-aligned, giving a total of 19 bits // of count. - cntBits = 64 - addrBits + 3 + tagBits = 64 - addrBits + 3 // On AIX, 64-bit addresses are split into 36-bit segment number and 28-bit // offset in segment. Segment numbers in the range 0x0A0000000-0x0AFFFFFFF(LSA) // are available for mmap. - // We assume all lfnode addresses are from memory allocated with mmap. + // We assume all tagged addresses are from memory allocated with mmap. // We use one bit to distinguish between the two ranges. aixAddrBits = 57 - aixCntBits = 64 - aixAddrBits + 3 + aixTagBits = 64 - aixAddrBits + 3 // riscv64 SV57 mode gives 56 bits of userspace VA. - // lfstack code supports it, but broader support for SV57 mode is incomplete, + // tagged pointer code supports it, + // but broader support for SV57 mode is incomplete, // and there may be other issues (see #54104). riscv64AddrBits = 56 - riscv64CntBits = 64 - riscv64AddrBits + 3 + riscv64TagBits = 64 - riscv64AddrBits + 3 ) -func lfstackPack(node *lfnode, cnt uintptr) uint64 { - if GOARCH == "ppc64" && GOOS == "aix" { - return uint64(uintptr(unsafe.Pointer(node)))<<(64-aixAddrBits) | uint64(cnt&(1<> cntBits << 3))) + return unsafe.Pointer(uintptr(int64(tp) >> tagBits << 3)) } - if GOARCH == "ppc64" && GOOS == "aix" { - return (*lfnode)(unsafe.Pointer(uintptr((val >> aixCntBits << 3) | 0xa<<56))) + if GOOS == "aix" { + return unsafe.Pointer(uintptr((tp >> aixTagBits << 3) | 0xa<<56)) } if GOARCH == "riscv64" { - return (*lfnode)(unsafe.Pointer(uintptr(val >> riscv64CntBits << 3))) + return unsafe.Pointer(uintptr(tp >> riscv64TagBits << 3)) } - return (*lfnode)(unsafe.Pointer(uintptr(val >> cntBits << 3))) + return unsafe.Pointer(uintptr(tp >> tagBits << 3)) +} + +// Tag returns the tag from a taggedPointer. +func (tp taggedPointer) tag() uintptr { + return uintptr(tp & (1<