runtime: Improve scanning performance

To achieve a 2% improvement in the garbage benchmark this CL removes
an unneeded assert and avoids one hbits.next() call per object
being scanned.

Change-Id: Ibd542d01e9c23eace42228886f9edc488354df0d
Reviewed-on: https://go-review.googlesource.com/9244
Reviewed-by: Austin Clements <austin@google.com>

diff --git a/src/runtime/mgcmark.go b/src/runtime/mgcmark.go
index c53747c893c872d38e99568b8c7fcb6004ba8d77..4afdca432bd4c043b039bb77d52b886f7abde14f 100644 (file)
--- a/src/runtime/mgcmark.go
+++ b/src/runtime/mgcmark.go
@@ -596,11 +596,14 @@ func scanobject(b, n uintptr, ptrmask *uint8, gcw *gcWork) {
                        // dense mask (stack or data)
                        bits = (uintptr(*(*byte)(add(unsafe.Pointer(ptrmask), (i/ptrSize)/4))) >> (((i / ptrSize) % 4) * typeBitsWidth)) & typeMask
                } else {
+                       if i != 0 {
+                               // Avoid needless hbits.next() on last iteration.
+                               hbits = hbits.next()
+                       }
                        bits = uintptr(hbits.typeBits())
                        if bits == typeDead {
                                break // no more pointers in this object
                        }
-                       hbits = hbits.next()
                }
 
                if bits <= typeScalar { // typeScalar, typeDead, typeScalarMarked
@@ -647,9 +650,6 @@ func scanobject(b, n uintptr, ptrmask *uint8, gcw *gcWork) {
 // The object is not nil and known to be in the heap.
 //go:nowritebarrier
 func shade(b uintptr) {
-       if !inheap(b) {
-               throw("shade: passed an address not in the heap")
-       }
        if obj, hbits, span := heapBitsForObject(b); obj != 0 {
                // TODO: this would be a great place to put a check to see
                // if we are harvesting and if we are then we should