[dev.garbage] runtime: use sys.Ctz64 intrinsic

Our compilers now provides instrinsics including
sys.Ctz64 that support CTZ (count trailing zero)
instructions. This CL replaces the Go versions
of CTZ with the compiler intrinsic.

Count trailing zeros CTZ finds the least
significant 1 in a word and returns the number
of less significant 0s in the word.

Allocation uses the bitmap created by the garbage
collector to locate an unmarked object. The logic
takes a word of the bitmap, complements, and then
caches it. It then uses CTZ to locate an available
unmarked object. It then shifts marked bits out of
the bitmap word preparing it for the next search.
Once all the unmarked objects are used in the
cached work the bitmap gets another word and
repeats the process.

Change-Id: Id2fc42d1d4b9893efaa2e1bd01896985b7e42f82
Reviewed-on: https://go-review.googlesource.com/21366
Reviewed-by: Austin Clements <austin@google.com>

diff --git a/src/runtime/malloc.go b/src/runtime/malloc.go
index 86fdb3fdbbf0c62ea94067f4cdc687bfec5434c0..ec4939f1dd4e10e0bb950434185fa0e66c315f15 100644 (file)
--- a/src/runtime/malloc.go
+++ b/src/runtime/malloc.go
@@ -491,14 +491,13 @@ var zerobase uintptr
 // Otherwise it returns 0.
 func (c *mcache) nextFreeFast(sizeclass int8) gclinkptr {
        s := c.alloc[sizeclass]
-       ctzIndex := uint8(s.allocCache)
-       if ctzIndex != 0 {
-               theBit := uint64(ctzVals[ctzIndex])
-               freeidx := s.freeindex // help the pre ssa compiler out here with cse.
-               result := freeidx + uintptr(theBit)
+
+       theBit := sys.Ctz64(s.allocCache) // Is there a free object in the allocCache?
+       if theBit < 64 {
+               result := s.freeindex + uintptr(theBit)
                if result < s.nelems {
                        s.allocCache >>= (theBit + 1)
-                       freeidx = result + 1
+                       freeidx := result + 1
                        if freeidx%64 == 0 && freeidx != s.nelems {
                                // We just incremented s.freeindex so it isn't 0
                                // so we are moving to the next aCache.

diff --git a/src/runtime/mbitmap.go b/src/runtime/mbitmap.go
index 387fb8535d2754234f08a729007ecd3c57ad8a58..f2a5238c311c52616019220cbbfa4da32fceaf92 100644 (file)
--- a/src/runtime/mbitmap.go
+++ b/src/runtime/mbitmap.go
@@ -190,62 +190,7 @@ func (s *mspan) allocBitsForIndex(allocBitIndex uintptr) markBits {
        return markBits{bytePtr, uint8(1 << whichBit), allocBitIndex}
 }
 
-// ctzVals contains the count of trailing zeros for the
-// index. 0 returns 8 indicating 8 zeros.
-var ctzVals = [256]int8{
-       8, 0, 1, 0, 2, 0, 1, 0,
-       3, 0, 1, 0, 2, 0, 1, 0,
-       4, 0, 1, 0, 2, 0, 1, 0,
-       3, 0, 1, 0, 2, 0, 1, 0,
-       5, 0, 1, 0, 2, 0, 1, 0,
-       3, 0, 1, 0, 2, 0, 1, 0,
-       4, 0, 1, 0, 2, 0, 1, 0,
-       3, 0, 1, 0, 2, 0, 1, 0,
-       6, 0, 1, 0, 2, 0, 1, 0,
-       3, 0, 1, 0, 2, 0, 1, 0,
-       4, 0, 1, 0, 2, 0, 1, 0,
-       3, 0, 1, 0, 2, 0, 1, 0,
-       5, 0, 1, 0, 2, 0, 1, 0,
-       3, 0, 1, 0, 2, 0, 1, 0,
-       4, 0, 1, 0, 2, 0, 1, 0,
-       3, 0, 1, 0, 2, 0, 1, 0,
-       7, 0, 1, 0, 2, 0, 1, 0,
-       3, 0, 1, 0, 2, 0, 1, 0,
-       4, 0, 1, 0, 2, 0, 1, 0,
-       3, 0, 1, 0, 2, 0, 1, 0,
-       5, 0, 1, 0, 2, 0, 1, 0,
-       3, 0, 1, 0, 2, 0, 1, 0,
-       4, 0, 1, 0, 2, 0, 1, 0,
-       3, 0, 1, 0, 2, 0, 1, 0,
-       6, 0, 1, 0, 2, 0, 1, 0,
-       3, 0, 1, 0, 2, 0, 1, 0,
-       4, 0, 1, 0, 2, 0, 1, 0,
-       3, 0, 1, 0, 2, 0, 1, 0,
-       5, 0, 1, 0, 2, 0, 1, 0,
-       3, 0, 1, 0, 2, 0, 1, 0,
-       4, 0, 1, 0, 2, 0, 1, 0,
-       3, 0, 1, 0, 2, 0, 1, 0}
-
-// A temporary stand in for the count trailing zero ctz instruction.
-// IA bsf works on 64 bit non-zero word.
-func ctz64(markBits uint64) uint64 {
-       ctz8 := ctzVals[markBits&0xff]
-       if ctz8 != 8 {
-               return uint64(ctz8)
-       } else if markBits == 0 { // low byte is zero check fill word.
-               return 64 // bits in 64 bit word, ensures loop terminates
-       }
-       result := uint64(8)
-       markBits >>= 8
-       for ctz8 = ctzVals[markBits&0xff]; ctz8 == 8; ctz8 = ctzVals[markBits&0xff] {
-               result += 8
-               markBits >>= 8
-       }
-       result += uint64(ctz8)
-       return result
-}
-
-// refillAllocCache takes 8 bytes s.allocBits starting at whichByte
+// refillaCache takes 8 bytes s.allocBits starting at whichByte
 // and negates them so that ctz (count trailing zeros) instructions
 // can be used. It then places these 8 bytes into the cached 64 bit
 // s.allocCache.
@@ -278,7 +223,8 @@ func (s *mspan) nextFreeIndex() uintptr {
        }
 
        aCache := s.allocCache
-       bitIndex := ctz64(aCache)
+
+       bitIndex := sys.Ctz64(aCache)
        for bitIndex == 64 {
                // Move index to start of next cached bits.
                sfreeindex = (sfreeindex + 64) &^ (64 - 1)
@@ -290,8 +236,9 @@ func (s *mspan) nextFreeIndex() uintptr {
                // Refill s.allocCache with the next 64 alloc bits.
                s.refillAllocCache(whichByte)
                aCache = s.allocCache
-               bitIndex = ctz64(aCache)
-               // Nothing was available try again now allocCache has been refilled.
+               bitIndex = sys.Ctz64(aCache)
+               // nothing available in cached bits
+               // grab the next 8 bytes and try again.
        }
        result := sfreeindex + uintptr(bitIndex)
        if result >= snelems {

diff --git a/src/runtime/mheap.go b/src/runtime/mheap.go
index 7d85891617c29b008b5ae66b7b3554094760d6e7..e4946ff8e951b05b99f8fa76c6c38c9a7336db51 100644 (file)
--- a/src/runtime/mheap.go
+++ b/src/runtime/mheap.go
@@ -145,7 +145,7 @@ type mspan struct {
        // Cache of the allocBits at freeindex. allocCache is shifted
        // such that the lowest bit corresponds to the bit freeindex.
        // allocCache holds the complement of allocBits, thus allowing
-       // ctz64 (count trailing zero) to use it directly.
+       // ctz (count trailing zero) to use it directly.
        // allocCache may contain bits beyond s.nelems; the caller must ignore
        // these.
        allocCache uint64