runtime: change lfstack support to taggedPointer

This is a refactoring with no change in behavior, in preparation
for future netpoll work.

For #59545

Change-Id: I493c5fd0f49f31b75787f7b5b89c544bed73f64f
Reviewed-on: https://go-review.googlesource.com/c/go/+/484836
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: Bryan Mills <bcmills@google.com>
Auto-Submit: Ian Lance Taylor <iant@golang.org>
Reviewed-by: Michael Knyszek <mknyszek@google.com>
Run-TryBot: Ian Lance Taylor <iant@golang.org>
Reviewed-by: Orlando Labao <orlando.labao43@gmail.com>

diff --git a/src/runtime/lfstack.go b/src/runtime/lfstack.go
index 306a8e888a9bf313d084f341f0c6df3a31bd66ab..a91ae64e5383ad8bfdd5c0c32099d8c11e4c591a 100644 (file)
--- a/src/runtime/lfstack.go
+++ b/src/runtime/lfstack.go
@@ -67,3 +67,11 @@ func lfnodeValidate(node *lfnode) {
                throw("bad lfnode address")
        }
 }
+
+func lfstackPack(node *lfnode, cnt uintptr) uint64 {
+       return uint64(taggedPointerPack(unsafe.Pointer(node), cnt))
+}
+
+func lfstackUnpack(val uint64) *lfnode {
+       return (*lfnode)(taggedPointer(val).pointer())
+}

diff --git a/src/runtime/lfstack_32bit.go b/src/runtime/lfstack_32bit.go
deleted file mode 100644 (file)
index 405923c..0000000
--- a/src/runtime/lfstack_32bit.go
+++ /dev/null
@@ -1,19 +0,0 @@
-// Copyright 2014 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-//go:build 386 || arm || mips || mipsle
-
-package runtime
-
-import "unsafe"
-
-// On 32-bit systems, the stored uint64 has a 32-bit pointer and 32-bit count.
-
-func lfstackPack(node *lfnode, cnt uintptr) uint64 {
-       return uint64(uintptr(unsafe.Pointer(node)))<<32 | uint64(cnt)
-}
-
-func lfstackUnpack(val uint64) *lfnode {
-       return (*lfnode)(unsafe.Pointer(uintptr(val >> 32)))
-}

diff --git a/src/runtime/malloc.go b/src/runtime/malloc.go
index bababe042e222bf575a30f511b346df509a4b968..de83722fffa836de1e87235c9f2aae48824b9de2 100644 (file)
--- a/src/runtime/malloc.go
+++ b/src/runtime/malloc.go
@@ -405,6 +405,10 @@ func mallocinit() {
                throw("bad pagesPerReclaimerChunk")
        }
 
+       if minTagBits > taggedPointerBits {
+               throw("taggedPointerbits too small")
+       }
+
        // Initialize the heap.
        mheap_.init()
        mcache0 = allocmcache()

diff --git a/src/runtime/tagptr.go b/src/runtime/tagptr.go
new file mode 100644 (file)
index 0000000..0e17a15
--- /dev/null
+++ b/src/runtime/tagptr.go
@@ -0,0 +1,14 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package runtime
+
+// taggedPointer is a pointer with a numeric tag.
+// The size of the numeric tag is GOARCH-dependent,
+// currently at least 10 bits.
+// This should only be used with pointers allocated outside the Go heap.
+type taggedPointer uint64
+
+// minTagBits is the minimum number of tag bits that we expect.
+const minTagBits = 10

diff --git a/src/runtime/tagptr_32bit.go b/src/runtime/tagptr_32bit.go
new file mode 100644 (file)
index 0000000..f79e182
--- /dev/null
+++ b/src/runtime/tagptr_32bit.go
@@ -0,0 +1,30 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build 386 || arm || mips || mipsle
+
+package runtime
+
+import "unsafe"
+
+// The number of bits stored in the numeric tag of a taggedPointer
+const taggedPointerBits = 32
+
+// On 32-bit systems, taggedPointer has a 32-bit pointer and 32-bit count.
+
+// taggedPointerPack created a taggedPointer from a pointer and a tag.
+// Tag bits that don't fit in the result are discarded.
+func taggedPointerPack(ptr unsafe.Pointer, tag uintptr) taggedPointer {
+       return taggedPointer(uintptr(ptr))<<32 | taggedPointer(tag)
+}
+
+// Pointer returns the pointer from a taggedPointer.
+func (tp taggedPointer) pointer() unsafe.Pointer {
+       return unsafe.Pointer(uintptr(tp >> 32))
+}
+
+// Tag returns the tag from a taggedPointer.
+func (tp taggedPointer) tag() uintptr {
+       return uintptr(tp)
+}

diff --git a/src/runtime/lfstack_64bit.go b/src/runtime/tagptr_64bit.go
similarity index 51%
rename from src/runtime/lfstack_64bit.go
rename to src/runtime/tagptr_64bit.go
index 88cbd3bcc7e4ac187e920ec70dab9fa959fa26be..9ff11ccd16211c1c6e77409d35351022513bd24d 100644 (file)
--- a/src/runtime/lfstack_64bit.go
+++ b/src/runtime/tagptr_64bit.go
@@ -6,7 +6,11 @@
 
 package runtime
 
-import "unsafe"
+import (
+       "internal/goarch"
+       "internal/goos"
+       "unsafe"
+)
 
 const (
        // addrBits is the number of bits needed to represent a virtual address.
@@ -27,44 +31,59 @@ const (
        // In addition to the 16 bits taken from the top, we can take 3 from the
        // bottom, because node must be pointer-aligned, giving a total of 19 bits
        // of count.
-       cntBits = 64 - addrBits + 3
+       tagBits = 64 - addrBits + 3
 
        // On AIX, 64-bit addresses are split into 36-bit segment number and 28-bit
        // offset in segment.  Segment numbers in the range 0x0A0000000-0x0AFFFFFFF(LSA)
        // are available for mmap.
-       // We assume all lfnode addresses are from memory allocated with mmap.
+       // We assume all tagged addresses are from memory allocated with mmap.
        // We use one bit to distinguish between the two ranges.
        aixAddrBits = 57
-       aixCntBits  = 64 - aixAddrBits + 3
+       aixTagBits  = 64 - aixAddrBits + 3
 
        // riscv64 SV57 mode gives 56 bits of userspace VA.
-       // lfstack code supports it, but broader support for SV57 mode is incomplete,
+       // tagged pointer code supports it,
+       // but broader support for SV57 mode is incomplete,
        // and there may be other issues (see #54104).
        riscv64AddrBits = 56
-       riscv64CntBits  = 64 - riscv64AddrBits + 3
+       riscv64TagBits  = 64 - riscv64AddrBits + 3
 )
 
-func lfstackPack(node *lfnode, cnt uintptr) uint64 {
-       if GOARCH == "ppc64" && GOOS == "aix" {
-               return uint64(uintptr(unsafe.Pointer(node)))<<(64-aixAddrBits) | uint64(cnt&(1<<aixCntBits-1))
+// The number of bits stored in the numeric tag of a taggedPointer
+const taggedPointerBits = (goos.IsAix * aixTagBits) + (goarch.IsRiscv64 * riscv64TagBits) + ((1 - goos.IsAix) * (1 - goarch.IsRiscv64) * tagBits)
+
+// taggedPointerPack created a taggedPointer from a pointer and a tag.
+// Tag bits that don't fit in the result are discarded.
+func taggedPointerPack(ptr unsafe.Pointer, tag uintptr) taggedPointer {
+       if GOOS == "aix" {
+               if GOARCH != "ppc64" {
+                       throw("check this code for aix on non-ppc64")
+               }
+               return taggedPointer(uint64(uintptr(ptr))<<(64-aixAddrBits) | uint64(tag&(1<<aixTagBits-1)))
        }
        if GOARCH == "riscv64" {
-               return uint64(uintptr(unsafe.Pointer(node)))<<(64-riscv64AddrBits) | uint64(cnt&(1<<riscv64CntBits-1))
+               return taggedPointer(uint64(uintptr(ptr))<<(64-riscv64AddrBits) | uint64(tag&(1<<riscv64TagBits-1)))
        }
-       return uint64(uintptr(unsafe.Pointer(node)))<<(64-addrBits) | uint64(cnt&(1<<cntBits-1))
+       return taggedPointer(uint64(uintptr(ptr))<<(64-addrBits) | uint64(tag&(1<<tagBits-1)))
 }
 
-func lfstackUnpack(val uint64) *lfnode {
+// Pointer returns the pointer from a taggedPointer.
+func (tp taggedPointer) pointer() unsafe.Pointer {
        if GOARCH == "amd64" {
                // amd64 systems can place the stack above the VA hole, so we need to sign extend
                // val before unpacking.
-               return (*lfnode)(unsafe.Pointer(uintptr(int64(val) >> cntBits << 3)))
+               return unsafe.Pointer(uintptr(int64(tp) >> tagBits << 3))
        }
-       if GOARCH == "ppc64" && GOOS == "aix" {
-               return (*lfnode)(unsafe.Pointer(uintptr((val >> aixCntBits << 3) | 0xa<<56)))
+       if GOOS == "aix" {
+               return unsafe.Pointer(uintptr((tp >> aixTagBits << 3) | 0xa<<56))
        }
        if GOARCH == "riscv64" {
-               return (*lfnode)(unsafe.Pointer(uintptr(val >> riscv64CntBits << 3)))
+               return unsafe.Pointer(uintptr(tp >> riscv64TagBits << 3))
        }
-       return (*lfnode)(unsafe.Pointer(uintptr(val >> cntBits << 3)))
+       return unsafe.Pointer(uintptr(tp >> tagBits << 3))
+}
+
+// Tag returns the tag from a taggedPointer.
+func (tp taggedPointer) tag() uintptr {
+       return uintptr(tp & (1<<taggedPointerBits - 1))
 }