runtime: mask shifts in map implementation on x86

This slightly improves the generated code on x86 architectures,
including on many hot paths.

It is a no-op on other architectures.

Change-Id: I86336fd846bc5805a27bbec572e8c73dcbd0d567
Reviewed-on: https://go-review.googlesource.com/57411
Run-TryBot: Josh Bleecher Snyder <josharian@gmail.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Keith Randall <khr@golang.org>

diff --git a/src/runtime/hashmap.go b/src/runtime/hashmap.go
index 60af870fac2db6a26c9e0ee4ae6c377cec702ddb..df4df053d159036edd9830e09c52202b89d84846 100644 (file)
--- a/src/runtime/hashmap.go
+++ b/src/runtime/hashmap.go
@@ -170,6 +170,19 @@ type hiter struct {
        checkBucket uintptr
 }
 
+// bucketShift returns 1<<b, optimized for code generation.
+func bucketShift(b uint8) uintptr {
+       if sys.GoarchAmd64|sys.GoarchAmd64p32|sys.Goarch386 != 0 {
+               b &= sys.PtrSize*8 - 1 // help x86 archs remove shift overflow checks
+       }
+       return uintptr(1) << b
+}
+
+// bucketMask returns 1<<b - 1, optimized for code generation.
+func bucketMask(b uint8) uintptr {
+       return bucketShift(b) - 1
+}
+
 // tophash calculates the tophash value for hash.
 func tophash(hash uintptr) uint8 {
        top := uint8(hash >> (sys.PtrSize*8 - 8))
@@ -374,7 +387,7 @@ func mapaccess1(t *maptype, h *hmap, key unsafe.Pointer) unsafe.Pointer {
        }
        alg := t.key.alg
        hash := alg.hash(key, uintptr(h.hash0))
-       m := uintptr(1)<<h.B - 1
+       m := bucketMask(h.B)
        b := (*bmap)(add(h.buckets, (hash&m)*uintptr(t.bucketsize)))
        if c := h.oldbuckets; c != nil {
                if !h.sameSizeGrow() {
@@ -429,7 +442,7 @@ func mapaccess2(t *maptype, h *hmap, key unsafe.Pointer) (unsafe.Pointer, bool)
        }
        alg := t.key.alg
        hash := alg.hash(key, uintptr(h.hash0))
-       m := uintptr(1)<<h.B - 1
+       m := bucketMask(h.B)
        b := (*bmap)(unsafe.Pointer(uintptr(h.buckets) + (hash&m)*uintptr(t.bucketsize)))
        if c := h.oldbuckets; c != nil {
                if !h.sameSizeGrow() {
@@ -473,7 +486,7 @@ func mapaccessK(t *maptype, h *hmap, key unsafe.Pointer) (unsafe.Pointer, unsafe
        }
        alg := t.key.alg
        hash := alg.hash(key, uintptr(h.hash0))
-       m := uintptr(1)<<h.B - 1
+       m := bucketMask(h.B)
        b := (*bmap)(unsafe.Pointer(uintptr(h.buckets) + (hash&m)*uintptr(t.bucketsize)))
        if c := h.oldbuckets; c != nil {
                if !h.sameSizeGrow() {
@@ -555,7 +568,7 @@ func mapassign(t *maptype, h *hmap, key unsafe.Pointer) unsafe.Pointer {
        }
 
 again:
-       bucket := hash & (uintptr(1)<<h.B - 1)
+       bucket := hash & bucketMask(h.B)
        if h.growing() {
                growWork(t, h, bucket)
        }
@@ -662,7 +675,7 @@ func mapdelete(t *maptype, h *hmap, key unsafe.Pointer) {
        // in which case we have not actually done a write (delete).
        h.flags |= hashWriting
 
-       bucket := hash & (uintptr(1)<<h.B - 1)
+       bucket := hash & bucketMask(h.B)
        if h.growing() {
                growWork(t, h, bucket)
        }
@@ -758,7 +771,7 @@ func mapiterinit(t *maptype, h *hmap, it *hiter) {
        if h.B > 31-bucketCntBits {
                r += uintptr(fastrand()) << 31
        }
-       it.startBucket = r & (uintptr(1)<<h.B - 1)
+       it.startBucket = r & bucketMask(h.B)
        it.offset = uint8(r >> h.B & (bucketCnt - 1))
 
        // iterator state
@@ -817,7 +830,7 @@ next:
                        checkBucket = noCheck
                }
                bucket++
-               if bucket == uintptr(1)<<it.B {
+               if bucket == bucketShift(it.B) {
                        bucket = 0
                        it.wrapped = true
                }
@@ -845,7 +858,7 @@ next:
                                // If the item in the oldbucket is not destined for
                                // the current new bucket in the iteration, skip it.
                                hash := alg.hash(k, uintptr(h.hash0))
-                               if hash&(uintptr(1)<<it.B-1) != checkBucket {
+                               if hash&bucketMask(it.B) != checkBucket {
                                        continue
                                }
                        } else {
@@ -901,7 +914,7 @@ next:
 }
 
 func makeBucketArray(t *maptype, b uint8) (buckets unsafe.Pointer, nextOverflow *bmap) {
-       base := uintptr(1 << b)
+       base := bucketShift(b)
        nbuckets := base
        // For small b, overflow buckets are unlikely.
        // Avoid the overhead of the calculation.
@@ -909,7 +922,7 @@ func makeBucketArray(t *maptype, b uint8) (buckets unsafe.Pointer, nextOverflow
                // Add on the estimated number of overflow buckets
                // required to insert the median number of elements
                // used with this value of b.
-               nbuckets += 1 << (b - 4)
+               nbuckets += bucketShift(b - 4)
                sz := t.bucket.size * nbuckets
                up := roundupsize(sz)
                if up != sz {
@@ -975,7 +988,7 @@ func hashGrow(t *maptype, h *hmap) {
 
 // overLoadFactor reports whether count items placed in 1<<B buckets is over loadFactor.
 func overLoadFactor(count int, B uint8) bool {
-       return count >= bucketCnt && uint64(count) >= loadFactorNum*((uint64(1)<<B)/loadFactorDen)
+       return count >= bucketCnt && uintptr(count) >= loadFactorNum*(bucketShift(B)/loadFactorDen)
 }
 
 // tooManyOverflowBuckets reports whether noverflow buckets is too many for a map with 1<<B buckets.
@@ -1009,7 +1022,7 @@ func (h *hmap) noldbuckets() uintptr {
        if !h.sameSizeGrow() {
                oldB--
        }
-       return uintptr(1) << oldB
+       return bucketShift(oldB)
 }
 
 // oldbucketmask provides a mask that can be applied to calculate n % noldbuckets().

diff --git a/src/runtime/hashmap_fast.go b/src/runtime/hashmap_fast.go
index 64f9dc4c2ad647711160a6de48fca705ce02e528..27e5f50f87c2791a804e9463013ee814cac764cd 100644 (file)
--- a/src/runtime/hashmap_fast.go
+++ b/src/runtime/hashmap_fast.go
@@ -26,7 +26,7 @@ func mapaccess1_fast32(t *maptype, h *hmap, key uint32) unsafe.Pointer {
                b = (*bmap)(h.buckets)
        } else {
                hash := t.key.alg.hash(noescape(unsafe.Pointer(&key)), uintptr(h.hash0))
-               m := uintptr(1)<<h.B - 1
+               m := bucketMask(h.B)
                b = (*bmap)(add(h.buckets, (hash&m)*uintptr(t.bucketsize)))
                if c := h.oldbuckets; c != nil {
                        if !h.sameSizeGrow() {
@@ -69,7 +69,7 @@ func mapaccess2_fast32(t *maptype, h *hmap, key uint32) (unsafe.Pointer, bool) {
                b = (*bmap)(h.buckets)
        } else {
                hash := t.key.alg.hash(noescape(unsafe.Pointer(&key)), uintptr(h.hash0))
-               m := uintptr(1)<<h.B - 1
+               m := bucketMask(h.B)
                b = (*bmap)(add(h.buckets, (hash&m)*uintptr(t.bucketsize)))
                if c := h.oldbuckets; c != nil {
                        if !h.sameSizeGrow() {
@@ -112,7 +112,7 @@ func mapaccess1_fast64(t *maptype, h *hmap, key uint64) unsafe.Pointer {
                b = (*bmap)(h.buckets)
        } else {
                hash := t.key.alg.hash(noescape(unsafe.Pointer(&key)), uintptr(h.hash0))
-               m := uintptr(1)<<h.B - 1
+               m := bucketMask(h.B)
                b = (*bmap)(add(h.buckets, (hash&m)*uintptr(t.bucketsize)))
                if c := h.oldbuckets; c != nil {
                        if !h.sameSizeGrow() {
@@ -155,7 +155,7 @@ func mapaccess2_fast64(t *maptype, h *hmap, key uint64) (unsafe.Pointer, bool) {
                b = (*bmap)(h.buckets)
        } else {
                hash := t.key.alg.hash(noescape(unsafe.Pointer(&key)), uintptr(h.hash0))
-               m := uintptr(1)<<h.B - 1
+               m := bucketMask(h.B)
                b = (*bmap)(add(h.buckets, (hash&m)*uintptr(t.bucketsize)))
                if c := h.oldbuckets; c != nil {
                        if !h.sameSizeGrow() {
@@ -243,7 +243,7 @@ func mapaccess1_faststr(t *maptype, h *hmap, ky string) unsafe.Pointer {
        }
 dohash:
        hash := t.key.alg.hash(noescape(unsafe.Pointer(&ky)), uintptr(h.hash0))
-       m := uintptr(1)<<h.B - 1
+       m := bucketMask(h.B)
        b := (*bmap)(add(h.buckets, (hash&m)*uintptr(t.bucketsize)))
        if c := h.oldbuckets; c != nil {
                if !h.sameSizeGrow() {
@@ -335,7 +335,7 @@ func mapaccess2_faststr(t *maptype, h *hmap, ky string) (unsafe.Pointer, bool) {
        }
 dohash:
        hash := t.key.alg.hash(noescape(unsafe.Pointer(&ky)), uintptr(h.hash0))
-       m := uintptr(1)<<h.B - 1
+       m := bucketMask(h.B)
        b := (*bmap)(add(h.buckets, (hash&m)*uintptr(t.bucketsize)))
        if c := h.oldbuckets; c != nil {
                if !h.sameSizeGrow() {
@@ -386,7 +386,7 @@ func mapassign_fast32(t *maptype, h *hmap, key uint32) unsafe.Pointer {
        }
 
 again:
-       bucket := hash & (uintptr(1)<<h.B - 1)
+       bucket := hash & bucketMask(h.B)
        if h.growing() {
                growWork(t, h, bucket)
        }
@@ -471,7 +471,7 @@ func mapassign_fast64(t *maptype, h *hmap, key uint64) unsafe.Pointer {
        }
 
 again:
-       bucket := hash & (uintptr(1)<<h.B - 1)
+       bucket := hash & bucketMask(h.B)
        if h.growing() {
                growWork(t, h, bucket)
        }
@@ -557,7 +557,7 @@ func mapassign_faststr(t *maptype, h *hmap, ky string) unsafe.Pointer {
        }
 
 again:
-       bucket := hash & (uintptr(1)<<h.B - 1)
+       bucket := hash & bucketMask(h.B)
        if h.growing() {
                growWork(t, h, bucket)
        }
@@ -642,7 +642,7 @@ func mapdelete_fast32(t *maptype, h *hmap, key uint32) {
        // Set hashWriting after calling alg.hash for consistency with mapdelete
        h.flags |= hashWriting
 
-       bucket := hash & (uintptr(1)<<h.B - 1)
+       bucket := hash & bucketMask(h.B)
        if h.growing() {
                growWork(t, h, bucket)
        }
@@ -695,7 +695,7 @@ func mapdelete_fast64(t *maptype, h *hmap, key uint64) {
        // Set hashWriting after calling alg.hash for consistency with mapdelete
        h.flags |= hashWriting
 
-       bucket := hash & (uintptr(1)<<h.B - 1)
+       bucket := hash & bucketMask(h.B)
        if h.growing() {
                growWork(t, h, bucket)
        }
@@ -749,7 +749,7 @@ func mapdelete_faststr(t *maptype, h *hmap, ky string) {
        // Set hashWriting after calling alg.hash for consistency with mapdelete
        h.flags |= hashWriting
 
-       bucket := hash & (uintptr(1)<<h.B - 1)
+       bucket := hash & bucketMask(h.B)
        if h.growing() {
                growWork(t, h, bucket)
        }

diff --git a/src/runtime/runtime_test.go b/src/runtime/runtime_test.go
index e1f1be7196bb6ec6b5db1f529258c82b10606225..752cd21c92f9df8ad01c6006521bff7f1366e35f 100644 (file)
--- a/src/runtime/runtime_test.go
+++ b/src/runtime/runtime_test.go
@@ -369,7 +369,7 @@ func TestIntendedInlining(t *testing.T) {
        t.Parallel()
 
        // want is the list of function names that should be inlined.
-       want := []string{"tophash", "add", "(*bmap).keys"}
+       want := []string{"tophash", "add", "(*bmap).keys", "bucketShift", "bucketMask"}
 
        m := make(map[string]bool, len(want))
        for _, s := range want {