runtime: use typedmemclr for typed memory

The hybrid barrier requires distinguishing typed and untyped memory
even when zeroing because the *current* contents of the memory matters
even when overwriting.

This commit introduces runtime.typedmemclr and runtime.memclrHasPointers
as a typed memory clearing functions parallel to runtime.typedmemmove.
Currently these simply call memclr, but with the hybrid barrier we'll
need to shade any pointers we're overwriting. These will provide us
with the necessary hooks to do so.

Updates #17503.

Change-Id: I74478619f8907825898092aaa204d6e4690f27e6
Reviewed-on: https://go-review.googlesource.com/31366
Reviewed-by: Keith Randall <khr@golang.org>
Reviewed-by: Rick Hudson <rlh@golang.org>

diff --git a/src/reflect/value.go b/src/reflect/value.go
index fa1b3e3b51f3aa8d4ac8141ad26f20ab05ade8bb..283fbd3c53fe92bd6966b8f8e80faa1f89a8b5d7 100644 (file)
--- a/src/reflect/value.go
+++ b/src/reflect/value.go
@@ -440,6 +440,8 @@ func (v Value) call(op string, in []Value) []Value {
 
        var ret []Value
        if nout == 0 {
+               // This is untyped because the frame is really a
+               // stack, even though it's a heap object.
                memclr(args, frametype.size)
                framePool.Put(args)
        } else {
@@ -644,6 +646,8 @@ func callMethod(ctxt *methodValue, frame unsafe.Pointer) {
                retOffset,
                frametype.size-retOffset)
 
+       // This is untyped because the frame is really a stack, even
+       // though it's a heap object.
        memclr(args, frametype.size)
        framePool.Put(args)
 }

diff --git a/src/runtime/chan.go b/src/runtime/chan.go
index 261d37d4eea6b00a9fb5baf7b684ee91958f0804..ac81cc74dcfc7946ebbfda10cb742b2689102701 100644 (file)
--- a/src/runtime/chan.go
+++ b/src/runtime/chan.go
@@ -334,7 +334,7 @@ func closechan(c *hchan) {
                        break
                }
                if sg.elem != nil {
-                       memclr(sg.elem, uintptr(c.elemsize))
+                       typedmemclr(c.elemtype, sg.elem)
                        sg.elem = nil
                }
                if sg.releasetime != 0 {
@@ -443,7 +443,7 @@ func chanrecv(t *chantype, c *hchan, ep unsafe.Pointer, block bool) (selected, r
                }
                unlock(&c.lock)
                if ep != nil {
-                       memclr(ep, uintptr(c.elemsize))
+                       typedmemclr(c.elemtype, ep)
                }
                return true, false
        }
@@ -467,7 +467,7 @@ func chanrecv(t *chantype, c *hchan, ep unsafe.Pointer, block bool) (selected, r
                if ep != nil {
                        typedmemmove(c.elemtype, ep, qp)
                }
-               memclr(qp, uintptr(c.elemsize))
+               typedmemclr(c.elemtype, qp)
                c.recvx++
                if c.recvx == c.dataqsiz {
                        c.recvx = 0

diff --git a/src/runtime/hashmap.go b/src/runtime/hashmap.go
index 68f4c8b84199ac456b4eaaceec96dd9974b83e12..86d3b37ff1ac3874d1746be108f35db5956ec569 100644 (file)
--- a/src/runtime/hashmap.go
+++ b/src/runtime/hashmap.go
@@ -637,9 +637,17 @@ func mapdelete(t *maptype, h *hmap, key unsafe.Pointer) {
                        if !alg.equal(key, k2) {
                                continue
                        }
-                       memclr(k, uintptr(t.keysize))
+                       if t.indirectkey {
+                               *(*unsafe.Pointer)(k) = nil
+                       } else {
+                               typedmemclr(t.key, k)
+                       }
                        v := unsafe.Pointer(uintptr(unsafe.Pointer(b)) + dataOffset + bucketCnt*uintptr(t.keysize) + i*uintptr(t.valuesize))
-                       memclr(v, uintptr(t.valuesize))
+                       if t.indirectvalue {
+                               *(*unsafe.Pointer)(v) = nil
+                       } else {
+                               typedmemclr(t.elem, v)
+                       }
                        b.tophash[i] = empty
                        h.count--
                        goto done
@@ -1079,7 +1087,11 @@ func evacuate(t *maptype, h *hmap, oldbucket uintptr) {
                        b = (*bmap)(add(h.oldbuckets, oldbucket*uintptr(t.bucketsize)))
                        // Preserve b.tophash because the evacuation
                        // state is maintained there.
-                       memclr(add(unsafe.Pointer(b), dataOffset), uintptr(t.bucketsize)-dataOffset)
+                       if t.bucket.kind&kindNoPointers == 0 {
+                               memclrHasPointers(add(unsafe.Pointer(b), dataOffset), uintptr(t.bucketsize)-dataOffset)
+                       } else {
+                               memclr(add(unsafe.Pointer(b), dataOffset), uintptr(t.bucketsize)-dataOffset)
+                       }
                }
        }
 

diff --git a/src/runtime/iface.go b/src/runtime/iface.go
index f7ad40d1c024b734056b70e15a52d40d77618208..721ac6924f1daf1387a64c464bff5cddad961167 100644 (file)
--- a/src/runtime/iface.go
+++ b/src/runtime/iface.go
@@ -222,7 +222,7 @@ func assertI2T(t *_type, i iface, r unsafe.Pointer) {
 func assertI2T2(t *_type, i iface, r unsafe.Pointer) bool {
        tab := i.tab
        if tab == nil || tab._type != t {
-               memclr(r, t.size)
+               typedmemclr(t, r)
                return false
        }
        if isDirectIface(t) {
@@ -257,7 +257,7 @@ func assertE2T2(t *_type, e eface, r unsafe.Pointer) bool {
                GC()
        }
        if e._type != t {
-               memclr(r, t.size)
+               typedmemclr(t, r)
                return false
        }
        if isDirectIface(t) {

diff --git a/src/runtime/mbarrier.go b/src/runtime/mbarrier.go
index 90f730ee12184ee50fc82119ee914233e50d482c..1a7bef4fa11fdf884025e45789b74c3fd4cf5798 100644 (file)
--- a/src/runtime/mbarrier.go
+++ b/src/runtime/mbarrier.go
@@ -331,3 +331,24 @@ func reflect_typedslicecopy(elemType *_type, dst, src slice) int {
        }
        return typedslicecopy(elemType, dst, src)
 }
+
+// typedmemclr clears the typed memory at ptr with type typ. The
+// memory at ptr must already be type-safe.
+//
+// If the caller knows that typ has pointers, it can alternatively
+// call memclrHasPointers.
+//
+//go:nosplit
+func typedmemclr(typ *_type, ptr unsafe.Pointer) {
+       memclr(ptr, typ.size)
+}
+
+// memclrHasPointers clears n bytes of typed memory starting at ptr.
+// The caller must ensure that the type of the object at ptr has
+// pointers, usually by checking typ.kind&kindNoPointers. However, ptr
+// does not have to point to the start of the allocation.
+//
+//go:nosplit
+func memclrHasPointers(ptr unsafe.Pointer, n uintptr) {
+       memclr(ptr, n)
+}

diff --git a/src/runtime/select.go b/src/runtime/select.go
index 1aaafff19866c9542b10d1c1a37f76e4cde4e89d..03e9e4a30ad93c679ea2342d78a72fc3527bd9e6 100644 (file)
--- a/src/runtime/select.go
+++ b/src/runtime/select.go
@@ -518,7 +518,7 @@ bufrecv:
        if cas.elem != nil {
                typedmemmove(c.elemtype, cas.elem, qp)
        }
-       memclr(qp, uintptr(c.elemsize))
+       typedmemclr(c.elemtype, qp)
        c.recvx++
        if c.recvx == c.dataqsiz {
                c.recvx = 0
@@ -564,7 +564,7 @@ rclose:
                *cas.receivedp = false
        }
        if cas.elem != nil {
-               memclr(cas.elem, uintptr(c.elemsize))
+               typedmemclr(c.elemtype, cas.elem)
        }
        if raceenabled {
                raceacquire(unsafe.Pointer(c))

diff --git a/src/runtime/stubs.go b/src/runtime/stubs.go
index b73a97f73567fd0318efaa424bc6009f49c2f502..693a3445c292de6eac1ef5c95931dff2c96add6e 100644 (file)
--- a/src/runtime/stubs.go
+++ b/src/runtime/stubs.go
@@ -61,6 +61,12 @@ func badsystemstack() {
 }
 
 // memclr clears n bytes starting at ptr.
+//
+// Usually you should use typedmemclr. memclr should be used only when
+// the caller knows that *ptr contains no heap pointers or to
+// initialize memory to a type-safe state when allocation reuses dead
+// memory.
+//
 // in memclr_*.s
 //go:noescape
 func memclr(ptr unsafe.Pointer, n uintptr)