runtime: reusable intrusive doubly-linked list

Unfortunately we have two nearly identical types. One for standard types
and one for sys.NotInHeap types or cases that must avoid write barriers.
The latter must use uintptr fields, as assignment to unsafe.Pointer
fields generates a write barrier.

Change-Id: I6a6a636c62d83fa93b991033c7108d3b934412ac
Reviewed-on: https://go-review.googlesource.com/c/go/+/714020
Reviewed-by: Michael Knyszek <mknyszek@google.com>
Commit-Queue: Michael Pratt <mpratt@google.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Auto-Submit: Michael Pratt <mpratt@google.com>

diff --git a/src/runtime/export_test.go b/src/runtime/export_test.go
index d6fce72bf001fcc6b7b77f39dd2f4c338fc1938c..d17984881d4323de3e0d47b65cbdf15c2f6575e7 100644 (file)
--- a/src/runtime/export_test.go
+++ b/src/runtime/export_test.go
@@ -1938,3 +1938,51 @@ func TraceStack(gp *G, tab *TraceStackTable) {
 var DebugDecorateMappings = &debug.decoratemappings
 
 func SetVMANameSupported() bool { return setVMANameSupported() }
+
+type ListHead struct {
+       l listHead
+}
+
+func (head *ListHead) Init(off uintptr) {
+       head.l.init(off)
+}
+
+type ListNode struct {
+       l listNode
+}
+
+func (head *ListHead) Push(p unsafe.Pointer) {
+       head.l.push(p)
+}
+
+func (head *ListHead) Pop() unsafe.Pointer {
+       return head.l.pop()
+}
+
+func (head *ListHead) Remove(p unsafe.Pointer) {
+       head.l.remove(p)
+}
+
+type ListHeadManual struct {
+       l listHeadManual
+}
+
+func (head *ListHeadManual) Init(off uintptr) {
+       head.l.init(off)
+}
+
+type ListNodeManual struct {
+       l listNodeManual
+}
+
+func (head *ListHeadManual) Push(p unsafe.Pointer) {
+       head.l.push(p)
+}
+
+func (head *ListHeadManual) Pop() unsafe.Pointer {
+       return head.l.pop()
+}
+
+func (head *ListHeadManual) Remove(p unsafe.Pointer) {
+       head.l.remove(p)
+}

diff --git a/src/runtime/list.go b/src/runtime/list.go
new file mode 100644 (file)
index 0000000..c900ad7
--- /dev/null
+++ b/src/runtime/list.go
@@ -0,0 +1,136 @@
+// Copyright 2025 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package runtime
+
+import (
+       "unsafe"
+)
+
+// listHead points to the head of an intrusive doubly-linked list.
+//
+// Prior to use, you must call init to store the offset of listNode fields.
+//
+// Every object in the list should be the same type.
+type listHead struct {
+       obj unsafe.Pointer
+
+       initialized bool
+       nodeOffset  uintptr
+}
+
+// init initializes the list head. off is the offset (via unsafe.Offsetof) of
+// the listNode field in the objects in the list.
+func (head *listHead) init(off uintptr) {
+       head.initialized = true
+       head.nodeOffset = off
+}
+
+// listNode is the linked list node for objects in a listHead list.
+//
+// listNode must be stored as a field in objects placed in the linked list. The
+// offset of the field is registered via listHead.init.
+//
+// For example:
+//
+// type foo struct {
+//     val int
+//
+//     node listNode
+// }
+//
+// var fooHead listHead
+// fooHead.init(unsafe.Offsetof(foo{}.node))
+type listNode struct {
+       prev unsafe.Pointer
+       next unsafe.Pointer
+}
+
+func (head *listHead) getNode(p unsafe.Pointer) *listNode {
+       if !head.initialized {
+               throw("runtime: uninitialized listHead")
+       }
+
+       if p == nil {
+               return nil
+       }
+       return (*listNode)(unsafe.Add(p, head.nodeOffset))
+}
+
+// Returns true if the list is empty.
+func (head *listHead) empty() bool {
+       return head.obj == nil
+}
+
+// Returns the head of the list without removing it.
+func (head *listHead) head() unsafe.Pointer {
+       return head.obj
+}
+
+// Push p onto the front of the list.
+func (head *listHead) push(p unsafe.Pointer) {
+       // p becomes the head of the list.
+
+       // ... so p's next is the current head.
+       pNode := head.getNode(p)
+       pNode.next = head.obj
+
+       // ... and the current head's prev is p.
+       if head.obj != nil {
+               headNode := head.getNode(head.obj)
+               headNode.prev = p
+       }
+
+       head.obj = p
+}
+
+// Pop removes the head of the list.
+func (head *listHead) pop() unsafe.Pointer {
+       if head.obj == nil {
+               return nil
+       }
+
+       // Return the head of the list.
+       p := head.obj
+
+       // ... so the new head is p's next.
+       pNode := head.getNode(p)
+       head.obj = pNode.next
+       // p is no longer on the list. Clear next to remove unused references.
+       // N.B. as the head, prev must already be nil.
+       pNode.next = nil
+
+       // ... and the new head no longer has a prev.
+       if head.obj != nil {
+               headNode := head.getNode(head.obj)
+               headNode.prev = nil
+       }
+
+       return p
+}
+
+// Remove p from the middle of the list.
+func (head *listHead) remove(p unsafe.Pointer) {
+       if head.obj == p {
+               // Use pop to ensure head is updated when removing the head.
+               head.pop()
+               return
+       }
+
+       pNode := head.getNode(p)
+       prevNode := head.getNode(pNode.prev)
+       nextNode := head.getNode(pNode.next)
+
+       // Link prev to next.
+       if prevNode != nil {
+               prevNode.next = pNode.next
+       }
+       // Link next to prev.
+       if nextNode != nil {
+               nextNode.prev = pNode.prev
+       }
+
+       pNode.prev = nil
+       pNode.next = nil
+}

diff --git a/src/runtime/list_manual.go b/src/runtime/list_manual.go
new file mode 100644 (file)
index 0000000..af0ae6b
--- /dev/null
+++ b/src/runtime/list_manual.go
@@ -0,0 +1,143 @@
+// Copyright 2025 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package runtime
+
+import (
+       "unsafe"
+)
+
+// The types in this file are exact copies of the types in list.go, but with
+// unsafe.Pointer replaced with uintptr for use where write barriers must be
+// avoided, such as uses of muintptr, puintptr, guintptr.
+//
+// Objects in these lists must be kept alive via another real reference.
+
+// listHeadManual points to the head of an intrusive doubly-linked list of
+// objects.
+//
+// Prior to use, you must call init to store the offset of listNodeManual fields.
+//
+// Every object in the list should be the same type.
+type listHeadManual struct {
+       obj uintptr
+
+       initialized bool
+       nodeOffset  uintptr
+}
+
+// init initializes the list head. off is the offset (via unsafe.Offsetof) of
+// the listNodeManual field in the objects in the list.
+func (head *listHeadManual) init(off uintptr) {
+       head.initialized = true
+       head.nodeOffset = off
+}
+
+// listNodeManual is the linked list node for objects in a listHeadManual list.
+//
+// listNodeManual must be stored as a field in objects placed in the linked list.
+// The offset of the field is registered via listHeadManual.init.
+//
+// For example:
+//
+// type foo struct {
+//     val int
+//
+//     node listNodeManual
+// }
+//
+// var fooHead listHeadManual
+// fooHead.init(unsafe.Offsetof(foo{}.node))
+type listNodeManual struct {
+       prev uintptr
+       next uintptr
+}
+
+func (head *listHeadManual) getNode(p unsafe.Pointer) *listNodeManual {
+       if !head.initialized {
+               throw("runtime: uninitialized listHead")
+       }
+
+       if p == nil {
+               return nil
+       }
+       return (*listNodeManual)(unsafe.Add(p, head.nodeOffset))
+}
+
+// Returns true if the list is empty.
+func (head *listHeadManual) empty() bool {
+       return head.obj == 0
+}
+
+// Returns the head of the list without removing it.
+func (head *listHeadManual) head() unsafe.Pointer {
+       return unsafe.Pointer(head.obj)
+}
+
+// Push p onto the front of the list.
+func (head *listHeadManual) push(p unsafe.Pointer) {
+       // p becomes the head of the list.
+
+       // ... so p's next is the current head.
+       pNode := head.getNode(p)
+       pNode.next = head.obj
+
+       // ... and the current head's prev is p.
+       if head.obj != 0 {
+               headNode := head.getNode(unsafe.Pointer(head.obj))
+               headNode.prev = uintptr(p)
+       }
+
+       head.obj = uintptr(p)
+}
+
+// Pop removes the head of the list.
+func (head *listHeadManual) pop() unsafe.Pointer {
+       if head.obj == 0 {
+               return nil
+       }
+
+       // Return the head of the list.
+       p := unsafe.Pointer(head.obj)
+
+       // ... so the new head is p's next.
+       pNode := head.getNode(p)
+       head.obj = pNode.next
+       // p is no longer on the list. Clear next to remove unused references.
+       // N.B. as the head, prev must already be nil.
+       pNode.next = 0
+
+       // ... and the new head no longer has a prev.
+       if head.obj != 0 {
+               headNode := head.getNode(unsafe.Pointer(head.obj))
+               headNode.prev = 0
+       }
+
+       return p
+}
+
+// Remove p from the middle of the list.
+func (head *listHeadManual) remove(p unsafe.Pointer) {
+       if unsafe.Pointer(head.obj) == p {
+               // Use pop to ensure head is updated when removing the head.
+               head.pop()
+               return
+       }
+
+       pNode := head.getNode(p)
+       prevNode := head.getNode(unsafe.Pointer(pNode.prev))
+       nextNode := head.getNode(unsafe.Pointer(pNode.next))
+
+       // Link prev to next.
+       if prevNode != nil {
+               prevNode.next = pNode.next
+       }
+       // Link next to prev.
+       if nextNode != nil {
+               nextNode.prev = pNode.prev
+       }
+
+       pNode.prev = 0
+       pNode.next = 0
+}

diff --git a/src/runtime/list_manual_test.go b/src/runtime/list_manual_test.go
new file mode 100644 (file)
index 0000000..281a06a
--- /dev/null
+++ b/src/runtime/list_manual_test.go
@@ -0,0 +1,407 @@
+// Copyright 2025 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package runtime_test
+
+import (
+       "internal/runtime/sys"
+       "runtime"
+       "testing"
+       "unsafe"
+)
+
+// The tests in this file are identical to list_test.go, but for the
+// manually-managed variants.
+
+type listedValManual struct {
+       val int
+
+       aNode runtime.ListNodeManual
+       bNode runtime.ListNodeManual
+}
+
+func newListedValManual(v int) *listedValManual {
+       return &listedValManual{
+               val: v,
+       }
+}
+
+func TestListManualPush(t *testing.T) {
+       var headA runtime.ListHeadManual
+       headA.Init(unsafe.Offsetof(listedValManual{}.aNode))
+
+       one := newListedValManual(1)
+       headA.Push(unsafe.Pointer(one))
+
+       two := newListedValManual(2)
+       headA.Push(unsafe.Pointer(two))
+
+       three := newListedValManual(3)
+       headA.Push(unsafe.Pointer(three))
+
+       p := headA.Pop()
+       v := (*listedValManual)(p)
+       if v == nil {
+               t.Fatalf("pop got nil want 3")
+       }
+       if v.val != 3 {
+               t.Errorf("pop got %d want 3", v.val)
+       }
+
+       p = headA.Pop()
+       v = (*listedValManual)(p)
+       if v == nil {
+               t.Fatalf("pop got nil want 2")
+       }
+       if v.val != 2 {
+               t.Errorf("pop got %d want 2", v.val)
+       }
+
+       p = headA.Pop()
+       v = (*listedValManual)(p)
+       if v == nil {
+               t.Fatalf("pop got nil want 1")
+       }
+       if v.val != 1 {
+               t.Errorf("pop got %d want 1", v.val)
+       }
+
+       p = headA.Pop()
+       v = (*listedValManual)(p)
+       if v != nil {
+               t.Fatalf("pop got %+v want nil", v)
+       }
+
+       runtime.KeepAlive(one)
+       runtime.KeepAlive(two)
+       runtime.KeepAlive(three)
+}
+
+func wantValManual(t *testing.T, v *listedValManual, i int) {
+       t.Helper()
+       if v == nil {
+               t.Fatalf("listedVal got nil want %d", i)
+       }
+       if v.val != i {
+               t.Errorf("pop got %d want %d", v.val, i)
+       }
+}
+
+func TestListManualRemoveHead(t *testing.T) {
+       var headA runtime.ListHeadManual
+       headA.Init(unsafe.Offsetof(listedValManual{}.aNode))
+
+       one := newListedValManual(1)
+       headA.Push(unsafe.Pointer(one))
+
+       two := newListedValManual(2)
+       headA.Push(unsafe.Pointer(two))
+
+       three := newListedValManual(3)
+       headA.Push(unsafe.Pointer(three))
+
+       headA.Remove(unsafe.Pointer(three))
+
+       p := headA.Pop()
+       v := (*listedValManual)(p)
+       wantValManual(t, v, 2)
+
+       p = headA.Pop()
+       v = (*listedValManual)(p)
+       wantValManual(t, v, 1)
+
+       p = headA.Pop()
+       v = (*listedValManual)(p)
+       if v != nil {
+               t.Fatalf("pop got %+v want nil", v)
+       }
+
+       runtime.KeepAlive(one)
+       runtime.KeepAlive(two)
+       runtime.KeepAlive(three)
+}
+
+func TestListManualRemoveMiddle(t *testing.T) {
+       var headA runtime.ListHeadManual
+       headA.Init(unsafe.Offsetof(listedValManual{}.aNode))
+
+       one := newListedValManual(1)
+       headA.Push(unsafe.Pointer(one))
+
+       two := newListedValManual(2)
+       headA.Push(unsafe.Pointer(two))
+
+       three := newListedValManual(3)
+       headA.Push(unsafe.Pointer(three))
+
+       headA.Remove(unsafe.Pointer(two))
+
+       p := headA.Pop()
+       v := (*listedValManual)(p)
+       wantValManual(t, v, 3)
+
+       p = headA.Pop()
+       v = (*listedValManual)(p)
+       wantValManual(t, v, 1)
+
+       p = headA.Pop()
+       v = (*listedValManual)(p)
+       if v != nil {
+               t.Fatalf("pop got %+v want nil", v)
+       }
+
+       runtime.KeepAlive(one)
+       runtime.KeepAlive(two)
+       runtime.KeepAlive(three)
+}
+
+func TestListManualRemoveTail(t *testing.T) {
+       var headA runtime.ListHeadManual
+       headA.Init(unsafe.Offsetof(listedValManual{}.aNode))
+
+       one := newListedValManual(1)
+       headA.Push(unsafe.Pointer(one))
+
+       two := newListedValManual(2)
+       headA.Push(unsafe.Pointer(two))
+
+       three := newListedValManual(3)
+       headA.Push(unsafe.Pointer(three))
+
+       headA.Remove(unsafe.Pointer(one))
+
+       p := headA.Pop()
+       v := (*listedValManual)(p)
+       wantValManual(t, v, 3)
+
+       p = headA.Pop()
+       v = (*listedValManual)(p)
+       wantValManual(t, v, 2)
+
+       p = headA.Pop()
+       v = (*listedValManual)(p)
+       if v != nil {
+               t.Fatalf("pop got %+v want nil", v)
+       }
+
+       runtime.KeepAlive(one)
+       runtime.KeepAlive(two)
+       runtime.KeepAlive(three)
+}
+
+func TestListManualRemoveAll(t *testing.T) {
+       var headA runtime.ListHeadManual
+       headA.Init(unsafe.Offsetof(listedValManual{}.aNode))
+
+       one := newListedValManual(1)
+       headA.Push(unsafe.Pointer(one))
+
+       two := newListedValManual(2)
+       headA.Push(unsafe.Pointer(two))
+
+       three := newListedValManual(3)
+       headA.Push(unsafe.Pointer(three))
+
+       headA.Remove(unsafe.Pointer(one))
+       headA.Remove(unsafe.Pointer(two))
+       headA.Remove(unsafe.Pointer(three))
+
+       p := headA.Pop()
+       v := (*listedValManual)(p)
+       if v != nil {
+               t.Fatalf("pop got %+v want nil", v)
+       }
+
+       runtime.KeepAlive(one)
+       runtime.KeepAlive(two)
+       runtime.KeepAlive(three)
+}
+
+// The tests below are identical, but used with a sys.NotInHeap type.
+
+type listedValNIH struct {
+       _ sys.NotInHeap
+       listedValManual
+}
+
+func newListedValNIH(v int) *listedValNIH {
+       l := (*listedValNIH)(runtime.PersistentAlloc(unsafe.Sizeof(listedValNIH{}), unsafe.Alignof(listedValNIH{})))
+       l.val = v
+       return l
+}
+
+func newListHeadNIH() *runtime.ListHeadManual {
+       return (*runtime.ListHeadManual)(runtime.PersistentAlloc(unsafe.Sizeof(runtime.ListHeadManual{}), unsafe.Alignof(runtime.ListHeadManual{})))
+}
+
+func TestListNIHPush(t *testing.T) {
+       headA := newListHeadNIH()
+       headA.Init(unsafe.Offsetof(listedValNIH{}.aNode))
+
+       one := newListedVal(1)
+       headA.Push(unsafe.Pointer(one))
+
+       two := newListedVal(2)
+       headA.Push(unsafe.Pointer(two))
+
+       three := newListedVal(3)
+       headA.Push(unsafe.Pointer(three))
+
+       p := headA.Pop()
+       v := (*listedValNIH)(p)
+       if v == nil {
+               t.Fatalf("pop got nil want 3")
+       }
+       if v.val != 3 {
+               t.Errorf("pop got %d want 3", v.val)
+       }
+
+       p = headA.Pop()
+       v = (*listedValNIH)(p)
+       if v == nil {
+               t.Fatalf("pop got nil want 2")
+       }
+       if v.val != 2 {
+               t.Errorf("pop got %d want 2", v.val)
+       }
+
+       p = headA.Pop()
+       v = (*listedValNIH)(p)
+       if v == nil {
+               t.Fatalf("pop got nil want 1")
+       }
+       if v.val != 1 {
+               t.Errorf("pop got %d want 1", v.val)
+       }
+
+       p = headA.Pop()
+       v = (*listedValNIH)(p)
+       if v != nil {
+               t.Fatalf("pop got %+v want nil", v)
+       }
+}
+
+func wantValNIH(t *testing.T, v *listedValNIH, i int) {
+       t.Helper()
+       if v == nil {
+               t.Fatalf("listedVal got nil want %d", i)
+       }
+       if v.val != i {
+               t.Errorf("pop got %d want %d", v.val, i)
+       }
+}
+
+func TestListNIHRemoveHead(t *testing.T) {
+       headA := newListHeadNIH()
+       headA.Init(unsafe.Offsetof(listedValNIH{}.aNode))
+
+       one := newListedVal(1)
+       headA.Push(unsafe.Pointer(one))
+
+       two := newListedVal(2)
+       headA.Push(unsafe.Pointer(two))
+
+       three := newListedVal(3)
+       headA.Push(unsafe.Pointer(three))
+
+       headA.Remove(unsafe.Pointer(three))
+
+       p := headA.Pop()
+       v := (*listedValNIH)(p)
+       wantValNIH(t, v, 2)
+
+       p = headA.Pop()
+       v = (*listedValNIH)(p)
+       wantValNIH(t, v, 1)
+
+       p = headA.Pop()
+       v = (*listedValNIH)(p)
+       if v != nil {
+               t.Fatalf("pop got %+v want nil", v)
+       }
+}
+
+func TestListNIHRemoveMiddle(t *testing.T) {
+       headA := newListHeadNIH()
+       headA.Init(unsafe.Offsetof(listedValNIH{}.aNode))
+
+       one := newListedVal(1)
+       headA.Push(unsafe.Pointer(one))
+
+       two := newListedVal(2)
+       headA.Push(unsafe.Pointer(two))
+
+       three := newListedVal(3)
+       headA.Push(unsafe.Pointer(three))
+
+       headA.Remove(unsafe.Pointer(two))
+
+       p := headA.Pop()
+       v := (*listedValNIH)(p)
+       wantValNIH(t, v, 3)
+
+       p = headA.Pop()
+       v = (*listedValNIH)(p)
+       wantValNIH(t, v, 1)
+
+       p = headA.Pop()
+       v = (*listedValNIH)(p)
+       if v != nil {
+               t.Fatalf("pop got %+v want nil", v)
+       }
+}
+
+func TestListNIHRemoveTail(t *testing.T) {
+       headA := newListHeadNIH()
+       headA.Init(unsafe.Offsetof(listedValNIH{}.aNode))
+
+       one := newListedVal(1)
+       headA.Push(unsafe.Pointer(one))
+
+       two := newListedVal(2)
+       headA.Push(unsafe.Pointer(two))
+
+       three := newListedVal(3)
+       headA.Push(unsafe.Pointer(three))
+
+       headA.Remove(unsafe.Pointer(one))
+
+       p := headA.Pop()
+       v := (*listedValNIH)(p)
+       wantValNIH(t, v, 3)
+
+       p = headA.Pop()
+       v = (*listedValNIH)(p)
+       wantValNIH(t, v, 2)
+
+       p = headA.Pop()
+       v = (*listedValNIH)(p)
+       if v != nil {
+               t.Fatalf("pop got %+v want nil", v)
+       }
+}
+
+func TestListNIHRemoveAll(t *testing.T) {
+       headA := newListHeadNIH()
+       headA.Init(unsafe.Offsetof(listedValNIH{}.aNode))
+
+       one := newListedVal(1)
+       headA.Push(unsafe.Pointer(one))
+
+       two := newListedVal(2)
+       headA.Push(unsafe.Pointer(two))
+
+       three := newListedVal(3)
+       headA.Push(unsafe.Pointer(three))
+
+       headA.Remove(unsafe.Pointer(one))
+       headA.Remove(unsafe.Pointer(two))
+       headA.Remove(unsafe.Pointer(three))
+
+       p := headA.Pop()
+       v := (*listedValNIH)(p)
+       if v != nil {
+               t.Fatalf("pop got %+v want nil", v)
+       }
+}

diff --git a/src/runtime/list_test.go b/src/runtime/list_test.go
new file mode 100644 (file)
index 0000000..5839bf6
--- /dev/null
+++ b/src/runtime/list_test.go
@@ -0,0 +1,215 @@
+// Copyright 2025 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package runtime_test
+
+import (
+       "runtime"
+       "testing"
+       "unsafe"
+)
+
+type listedVal struct {
+       val int
+
+       aNode runtime.ListNode
+       bNode runtime.ListNode
+}
+
+func newListedVal(v int) *listedVal {
+       return &listedVal{
+               val: v,
+       }
+}
+
+func TestListPush(t *testing.T) {
+       var headA runtime.ListHead
+       headA.Init(unsafe.Offsetof(listedVal{}.aNode))
+
+       one := newListedVal(1)
+       headA.Push(unsafe.Pointer(one))
+
+       two := newListedVal(2)
+       headA.Push(unsafe.Pointer(two))
+
+       three := newListedVal(3)
+       headA.Push(unsafe.Pointer(three))
+
+       p := headA.Pop()
+       v := (*listedVal)(p)
+       if v == nil {
+               t.Fatalf("pop got nil want 3")
+       }
+       if v.val != 3 {
+               t.Errorf("pop got %d want 3", v.val)
+       }
+
+       p = headA.Pop()
+       v = (*listedVal)(p)
+       if v == nil {
+               t.Fatalf("pop got nil want 2")
+       }
+       if v.val != 2 {
+               t.Errorf("pop got %d want 2", v.val)
+       }
+
+       p = headA.Pop()
+       v = (*listedVal)(p)
+       if v == nil {
+               t.Fatalf("pop got nil want 1")
+       }
+       if v.val != 1 {
+               t.Errorf("pop got %d want 1", v.val)
+       }
+
+       p = headA.Pop()
+       v = (*listedVal)(p)
+       if v != nil {
+               t.Fatalf("pop got %+v want nil", v)
+       }
+}
+
+func wantVal(t *testing.T, v *listedVal, i int) {
+       t.Helper()
+       if v == nil {
+               t.Fatalf("listedVal got nil want %d", i)
+       }
+       if v.val != i {
+               t.Errorf("pop got %d want %d", v.val, i)
+       }
+}
+
+func TestListRemoveHead(t *testing.T) {
+       var headA runtime.ListHead
+       headA.Init(unsafe.Offsetof(listedVal{}.aNode))
+
+       one := newListedVal(1)
+       headA.Push(unsafe.Pointer(one))
+
+       two := newListedVal(2)
+       headA.Push(unsafe.Pointer(two))
+
+       three := newListedVal(3)
+       headA.Push(unsafe.Pointer(three))
+
+       headA.Remove(unsafe.Pointer(three))
+
+       p := headA.Pop()
+       v := (*listedVal)(p)
+       wantVal(t, v, 2)
+
+       p = headA.Pop()
+       v = (*listedVal)(p)
+       wantVal(t, v, 1)
+
+       p = headA.Pop()
+       v = (*listedVal)(p)
+       if v != nil {
+               t.Fatalf("pop got %+v want nil", v)
+       }
+}
+
+func TestListRemoveMiddle(t *testing.T) {
+       var headA runtime.ListHead
+       headA.Init(unsafe.Offsetof(listedVal{}.aNode))
+
+       one := newListedVal(1)
+       headA.Push(unsafe.Pointer(one))
+
+       two := newListedVal(2)
+       headA.Push(unsafe.Pointer(two))
+
+       three := newListedVal(3)
+       headA.Push(unsafe.Pointer(three))
+
+       headA.Remove(unsafe.Pointer(two))
+
+       p := headA.Pop()
+       v := (*listedVal)(p)
+       wantVal(t, v, 3)
+
+       p = headA.Pop()
+       v = (*listedVal)(p)
+       wantVal(t, v, 1)
+
+       p = headA.Pop()
+       v = (*listedVal)(p)
+       if v != nil {
+               t.Fatalf("pop got %+v want nil", v)
+       }
+}
+
+func TestListRemoveTail(t *testing.T) {
+       var headA runtime.ListHead
+       headA.Init(unsafe.Offsetof(listedVal{}.aNode))
+
+       one := newListedVal(1)
+       headA.Push(unsafe.Pointer(one))
+
+       two := newListedVal(2)
+       headA.Push(unsafe.Pointer(two))
+
+       three := newListedVal(3)
+       headA.Push(unsafe.Pointer(three))
+
+       headA.Remove(unsafe.Pointer(one))
+
+       p := headA.Pop()
+       v := (*listedVal)(p)
+       wantVal(t, v, 3)
+
+       p = headA.Pop()
+       v = (*listedVal)(p)
+       wantVal(t, v, 2)
+
+       p = headA.Pop()
+       v = (*listedVal)(p)
+       if v != nil {
+               t.Fatalf("pop got %+v want nil", v)
+       }
+}
+
+func TestListRemoveAll(t *testing.T) {
+       var headA runtime.ListHead
+       headA.Init(unsafe.Offsetof(listedVal{}.aNode))
+
+       one := newListedVal(1)
+       headA.Push(unsafe.Pointer(one))
+
+       two := newListedVal(2)
+       headA.Push(unsafe.Pointer(two))
+
+       three := newListedVal(3)
+       headA.Push(unsafe.Pointer(three))
+
+       headA.Remove(unsafe.Pointer(one))
+       headA.Remove(unsafe.Pointer(two))
+       headA.Remove(unsafe.Pointer(three))
+
+       p := headA.Pop()
+       v := (*listedVal)(p)
+       if v != nil {
+               t.Fatalf("pop got %+v want nil", v)
+       }
+}
+
+func BenchmarkListPushPop(b *testing.B) {
+       var head runtime.ListHead
+       head.Init(unsafe.Offsetof(listedVal{}.aNode))
+
+       vals := make([]listedVal, 10000)
+       i := 0
+       for b.Loop() {
+               if i == len(vals) {
+                       for range len(vals) {
+                               head.Pop()
+                       }
+                       i = 0
+               }
+
+               head.Push(unsafe.Pointer(&vals[i]))
+
+               i++
+       }
+}