runtime: strength reduce key pointer calculations in mapaccess*_fast*

While we're here, check string length before checking b.tophash.

name                     old time/op  new time/op  delta
MapStringKeysEight_16-8  11.4ns ±10%   7.0ns ± 2%  -38.27%  (p=0.000 n=29+28)
MapStringKeysEight_32-8  10.9ns ± 2%   6.3ns ± 3%  -41.89%  (p=0.000 n=26+30)
MapStringKeysEight_64-8  10.8ns ± 3%   6.3ns ± 2%  -41.52%  (p=0.000 n=28+27)
MapStringKeysEight_1M-8  10.9ns ± 4%   6.3ns ± 2%  -41.91%  (p=0.000 n=29+29)
IntMap-8                 7.05ns ± 4%  6.77ns ± 3%   -3.94%  (p=0.000 n=29+30)

Change-Id: I0f3dc3301bdf550e4ac5250e1e64e7f2a0ffb269
Reviewed-on: https://go-review.googlesource.com/57590
Run-TryBot: Josh Bleecher Snyder <josharian@gmail.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Keith Randall <khr@golang.org>

diff --git a/src/runtime/hashmap.go b/src/runtime/hashmap.go
index 77a09b676ae843869da5422c7b986e0911a18808..9456500f001c392f15e54afb60cc5e7e8d1e22ff 100644 (file)
--- a/src/runtime/hashmap.go
+++ b/src/runtime/hashmap.go
@@ -192,6 +192,10 @@ func (b *bmap) setoverflow(t *maptype, ovf *bmap) {
        *(**bmap)(add(unsafe.Pointer(b), uintptr(t.bucketsize)-sys.PtrSize)) = ovf
 }
 
+func (b *bmap) keys() unsafe.Pointer {
+       return add(unsafe.Pointer(b), dataOffset)
+}
+
 // incrnoverflow increments h.noverflow.
 // noverflow counts the number of overflow buckets.
 // This is used to trigger same-size map growth.

diff --git a/src/runtime/hashmap_fast.go b/src/runtime/hashmap_fast.go
index e83c72d0f903e3105c55b1cfa237017ff549ca6d..de52f2cc04f973c166dc52bff216c71e924d45b3 100644 (file)
--- a/src/runtime/hashmap_fast.go
+++ b/src/runtime/hashmap_fast.go
@@ -40,15 +40,10 @@ func mapaccess1_fast32(t *maptype, h *hmap, key uint32) unsafe.Pointer {
                }
        }
        for {
-               for i := uintptr(0); i < bucketCnt; i++ {
-                       k := *((*uint32)(add(unsafe.Pointer(b), dataOffset+i*4)))
-                       if k != key {
-                               continue
+               for i, k := uintptr(0), b.keys(); i < bucketCnt; i, k = i+1, add(k, 4) {
+                       if *(*uint32)(k) == key && b.tophash[i] != empty {
+                               return add(unsafe.Pointer(b), dataOffset+bucketCnt*4+i*uintptr(t.valuesize))
                        }
-                       if b.tophash[i] == empty {
-                               continue
-                       }
-                       return add(unsafe.Pointer(b), dataOffset+bucketCnt*4+i*uintptr(t.valuesize))
                }
                b = b.overflow(t)
                if b == nil {
@@ -88,15 +83,10 @@ func mapaccess2_fast32(t *maptype, h *hmap, key uint32) (unsafe.Pointer, bool) {
                }
        }
        for {
-               for i := uintptr(0); i < bucketCnt; i++ {
-                       k := *((*uint32)(add(unsafe.Pointer(b), dataOffset+i*4)))
-                       if k != key {
-                               continue
+               for i, k := uintptr(0), b.keys(); i < bucketCnt; i, k = i+1, add(k, 4) {
+                       if *(*uint32)(k) == key && b.tophash[i] != empty {
+                               return add(unsafe.Pointer(b), dataOffset+bucketCnt*4+i*uintptr(t.valuesize)), true
                        }
-                       if b.tophash[i] == empty {
-                               continue
-                       }
-                       return add(unsafe.Pointer(b), dataOffset+bucketCnt*4+i*uintptr(t.valuesize)), true
                }
                b = b.overflow(t)
                if b == nil {
@@ -136,15 +126,10 @@ func mapaccess1_fast64(t *maptype, h *hmap, key uint64) unsafe.Pointer {
                }
        }
        for {
-               for i := uintptr(0); i < bucketCnt; i++ {
-                       k := *((*uint64)(add(unsafe.Pointer(b), dataOffset+i*8)))
-                       if k != key {
-                               continue
+               for i, k := uintptr(0), b.keys(); i < bucketCnt; i, k = i+1, add(k, 8) {
+                       if *(*uint64)(k) == key && b.tophash[i] != empty {
+                               return add(unsafe.Pointer(b), dataOffset+bucketCnt*8+i*uintptr(t.valuesize))
                        }
-                       if b.tophash[i] == empty {
-                               continue
-                       }
-                       return add(unsafe.Pointer(b), dataOffset+bucketCnt*8+i*uintptr(t.valuesize))
                }
                b = b.overflow(t)
                if b == nil {
@@ -184,15 +169,10 @@ func mapaccess2_fast64(t *maptype, h *hmap, key uint64) (unsafe.Pointer, bool) {
                }
        }
        for {
-               for i := uintptr(0); i < bucketCnt; i++ {
-                       k := *((*uint64)(add(unsafe.Pointer(b), dataOffset+i*8)))
-                       if k != key {
-                               continue
+               for i, k := uintptr(0), b.keys(); i < bucketCnt; i, k = i+1, add(k, 8) {
+                       if *(*uint64)(k) == key && b.tophash[i] != empty {
+                               return add(unsafe.Pointer(b), dataOffset+bucketCnt*8+i*uintptr(t.valuesize)), true
                        }
-                       if b.tophash[i] == empty {
-                               continue
-                       }
-                       return add(unsafe.Pointer(b), dataOffset+bucketCnt*8+i*uintptr(t.valuesize)), true
                }
                b = b.overflow(t)
                if b == nil {
@@ -218,12 +198,9 @@ func mapaccess1_faststr(t *maptype, h *hmap, ky string) unsafe.Pointer {
                b := (*bmap)(h.buckets)
                if key.len < 32 {
                        // short key, doing lots of comparisons is ok
-                       for i := uintptr(0); i < bucketCnt; i++ {
-                               if b.tophash[i] == empty {
-                                       continue
-                               }
-                               k := (*stringStruct)(add(unsafe.Pointer(b), dataOffset+i*2*sys.PtrSize))
-                               if k.len != key.len {
+                       for i, kptr := uintptr(0), b.keys(); i < bucketCnt; i, kptr = i+1, add(kptr, 2*sys.PtrSize) {
+                               k := (*stringStruct)(kptr)
+                               if k.len != key.len || b.tophash[i] == empty {
                                        continue
                                }
                                if k.str == key.str || memequal(k.str, key.str, uintptr(key.len)) {
@@ -234,12 +211,9 @@ func mapaccess1_faststr(t *maptype, h *hmap, ky string) unsafe.Pointer {
                }
                // long key, try not to do more comparisons than necessary
                keymaybe := uintptr(bucketCnt)
-               for i := uintptr(0); i < bucketCnt; i++ {
-                       if b.tophash[i] == empty {
-                               continue
-                       }
-                       k := (*stringStruct)(add(unsafe.Pointer(b), dataOffset+i*2*sys.PtrSize))
-                       if k.len != key.len {
+               for i, kptr := uintptr(0), b.keys(); i < bucketCnt; i, kptr = i+1, add(kptr, 2*sys.PtrSize) {
+                       k := (*stringStruct)(kptr)
+                       if k.len != key.len || b.tophash[i] == empty {
                                continue
                        }
                        if k.str == key.str {
@@ -283,12 +257,9 @@ dohash:
        }
        top := tophash(hash)
        for {
-               for i := uintptr(0); i < bucketCnt; i++ {
-                       if b.tophash[i] != top {
-                               continue
-                       }
-                       k := (*stringStruct)(add(unsafe.Pointer(b), dataOffset+i*2*sys.PtrSize))
-                       if k.len != key.len {
+               for i, kptr := uintptr(0), b.keys(); i < bucketCnt; i, kptr = i+1, add(kptr, 2*sys.PtrSize) {
+                       k := (*stringStruct)(kptr)
+                       if k.len != key.len || b.tophash[i] != top {
                                continue
                        }
                        if k.str == key.str || memequal(k.str, key.str, uintptr(key.len)) {
@@ -319,12 +290,9 @@ func mapaccess2_faststr(t *maptype, h *hmap, ky string) (unsafe.Pointer, bool) {
                b := (*bmap)(h.buckets)
                if key.len < 32 {
                        // short key, doing lots of comparisons is ok
-                       for i := uintptr(0); i < bucketCnt; i++ {
-                               if b.tophash[i] == empty {
-                                       continue
-                               }
-                               k := (*stringStruct)(add(unsafe.Pointer(b), dataOffset+i*2*sys.PtrSize))
-                               if k.len != key.len {
+                       for i, kptr := uintptr(0), b.keys(); i < bucketCnt; i, kptr = i+1, add(kptr, 2*sys.PtrSize) {
+                               k := (*stringStruct)(kptr)
+                               if k.len != key.len || b.tophash[i] == empty {
                                        continue
                                }
                                if k.str == key.str || memequal(k.str, key.str, uintptr(key.len)) {
@@ -335,12 +303,9 @@ func mapaccess2_faststr(t *maptype, h *hmap, ky string) (unsafe.Pointer, bool) {
                }
                // long key, try not to do more comparisons than necessary
                keymaybe := uintptr(bucketCnt)
-               for i := uintptr(0); i < bucketCnt; i++ {
-                       if b.tophash[i] == empty {
-                               continue
-                       }
-                       k := (*stringStruct)(add(unsafe.Pointer(b), dataOffset+i*2*sys.PtrSize))
-                       if k.len != key.len {
+               for i, kptr := uintptr(0), b.keys(); i < bucketCnt; i, kptr = i+1, add(kptr, 2*sys.PtrSize) {
+                       k := (*stringStruct)(kptr)
+                       if k.len != key.len || b.tophash[i] == empty {
                                continue
                        }
                        if k.str == key.str {
@@ -384,12 +349,9 @@ dohash:
        }
        top := tophash(hash)
        for {
-               for i := uintptr(0); i < bucketCnt; i++ {
-                       if b.tophash[i] != top {
-                               continue
-                       }
-                       k := (*stringStruct)(add(unsafe.Pointer(b), dataOffset+i*2*sys.PtrSize))
-                       if k.len != key.len {
+               for i, kptr := uintptr(0), b.keys(); i < bucketCnt; i, kptr = i+1, add(kptr, 2*sys.PtrSize) {
+                       k := (*stringStruct)(kptr)
+                       if k.len != key.len || b.tophash[i] != top {
                                continue
                        }
                        if k.str == key.str || memequal(k.str, key.str, uintptr(key.len)) {

diff --git a/src/runtime/runtime_test.go b/src/runtime/runtime_test.go
index 25dfe921fecbaf9950ade52e1356bcafdcc75ee6..2355da977436f0adb987effa9ae57e0e699da68c 100644 (file)
--- a/src/runtime/runtime_test.go
+++ b/src/runtime/runtime_test.go
@@ -369,7 +369,7 @@ func TestIntendedInlining(t *testing.T) {
        t.Parallel()
 
        // want is the list of function names that should be inlined.
-       want := []string{"tophash", "add"}
+       want := []string{"tophash", "add", "(*bmap).keys"}
 
        m := make(map[string]bool, len(want))
        for _, s := range want {