runtime: exit early when scanning map buckets

author Keith Randall <khr@google.com>

Mon, 15 Oct 2018 22:14:48 +0000 (15:14 -0700)

committer Keith Randall <khr@golang.org>

Wed, 31 Oct 2018 22:48:13 +0000 (22:48 +0000)
author Keith Randall <khr@google.com>
Mon, 15 Oct 2018 22:14:48 +0000 (15:14 -0700)
committer Keith Randall <khr@golang.org>
Wed, 31 Oct 2018 22:48:13 +0000 (22:48 +0000)
diff --git a/src/runtime/map.go b/src/runtime/map.go

index 3e368f929fa33c09ed065bd3ddc7359bfc1f0423..617e88faa45a32b329e3b93138e95815a2e718bd 100644 (file)
--- a/src/runtime/map.go
+++ b/src/runtime/map.go
@@ -89,11 +89,12 @@ const (
         // Each bucket (including its overflow buckets, if any) will have either all or none of its
         // entries in the evacuated* states (except during the evacuate() method, which only happens
         // during map writes and thus no one else can observe the map during that time).
-       empty          = 0 // cell is empty
-       evacuatedEmpty = 1 // cell is empty, bucket is evacuated.
+       emptyRest      = 0 // this cell is empty, and there are no more non-empty cells at higher indexes or overflows.
+       emptyOne       = 1 // this cell is empty
         evacuatedX     = 2 // key/value is valid.  Entry has been evacuated to first half of larger table.
         evacuatedY     = 3 // same as above, but evacuated to second half of larger table.
-       minTopHash     = 4 // minimum tophash for a normal filled cell.
+       evacuatedEmpty = 4 // cell is empty, bucket is evacuated.
+       minTopHash     = 5 // minimum tophash for a normal filled cell.
  
         // flags
         iterator     = 1 // there may be an iterator using buckets
@@ -105,6 +106,11 @@ const (
         noCheck = 1<<(8*sys.PtrSize) - 1
  )
  
+// isEmpty reports whether the given tophash array entry represents an empty bucket entry.
+func isEmpty(x uint8) bool {
+       return x <= emptyOne
+}
+
  // A header for a Go map.
  type hmap struct {
         // Note: the format of the hmap is also encoded in cmd/compile/internal/gc/reflect.go.
@@ -197,7 +203,7 @@ func tophash(hash uintptr) uint8 {
  
  func evacuated(b *bmap) bool {
         h := b.tophash[0]
-       return h > empty && h < minTopHash
+       return h > emptyOne && h < minTopHash
  }
  
  func (b *bmap) overflow(t *maptype) *bmap {
@@ -418,9 +424,13 @@ func mapaccess1(t *maptype, h *hmap, key unsafe.Pointer) unsafe.Pointer {
                 }
         }
         top := tophash(hash)
+bucketloop:
         for ; b != nil; b = b.overflow(t) {
                 for i := uintptr(0); i < bucketCnt; i++ {
                         if b.tophash[i] != top {
+                               if b.tophash[i] == emptyRest {
+                                       break bucketloop
+                               }
                                 continue
                         }
                         k := add(unsafe.Pointer(b), dataOffset+i*uintptr(t.keysize))
@@ -470,9 +480,13 @@ func mapaccess2(t *maptype, h *hmap, key unsafe.Pointer) (unsafe.Pointer, bool)
                 }
         }
         top := tophash(hash)
+bucketloop:
         for ; b != nil; b = b.overflow(t) {
                 for i := uintptr(0); i < bucketCnt; i++ {
                         if b.tophash[i] != top {
+                               if b.tophash[i] == emptyRest {
+                                       break bucketloop
+                               }
                                 continue
                         }
                         k := add(unsafe.Pointer(b), dataOffset+i*uintptr(t.keysize))
@@ -511,9 +525,13 @@ func mapaccessK(t *maptype, h *hmap, key unsafe.Pointer) (unsafe.Pointer, unsafe
                 }
         }
         top := tophash(hash)
+bucketloop:
         for ; b != nil; b = b.overflow(t) {
                 for i := uintptr(0); i < bucketCnt; i++ {
                         if b.tophash[i] != top {
+                               if b.tophash[i] == emptyRest {
+                                       break bucketloop
+                               }
                                 continue
                         }
                         k := add(unsafe.Pointer(b), dataOffset+i*uintptr(t.keysize))
@@ -587,14 +605,18 @@ again:
         var inserti *uint8
         var insertk unsafe.Pointer
         var val unsafe.Pointer
+bucketloop:
         for {
                 for i := uintptr(0); i < bucketCnt; i++ {
                         if b.tophash[i] != top {
-                               if b.tophash[i] == empty && inserti == nil {
+                               if isEmpty(b.tophash[i]) && inserti == nil {
                                         inserti = &b.tophash[i]
                                         insertk = add(unsafe.Pointer(b), dataOffset+i*uintptr(t.keysize))
                                         val = add(unsafe.Pointer(b), dataOffset+bucketCnt*uintptr(t.keysize)+i*uintptr(t.valuesize))
                                 }
+                               if b.tophash[i] == emptyRest {
+                                       break bucketloop
+                               }
                                 continue
                         }
                         k := add(unsafe.Pointer(b), dataOffset+i*uintptr(t.keysize))
@@ -694,6 +716,9 @@ search:
         for ; b != nil; b = b.overflow(t) {
                 for i := uintptr(0); i < bucketCnt; i++ {
                         if b.tophash[i] != top {
+                               if b.tophash[i] == emptyRest {
+                                       break search
+                               }
                                 continue
                         }
                         k := add(unsafe.Pointer(b), dataOffset+i*uintptr(t.keysize))
@@ -718,7 +743,8 @@ search:
                         } else {
                                 memclrNoHeapPointers(v, t.elem.size)
                         }
-                       b.tophash[i] = empty
+                       b.tophash[i] = emptyOne
+                       // TODO: set up emptyRest here.
                         h.count--
                         break search
                 }
@@ -833,7 +859,9 @@ next:
         }
         for ; i < bucketCnt; i++ {
                 offi := (i + it.offset) & (bucketCnt - 1)
-               if b.tophash[offi] == empty || b.tophash[offi] == evacuatedEmpty {
+               if isEmpty(b.tophash[offi]) || b.tophash[offi] == evacuatedEmpty {
+                       // TODO: emptyRest is hard to use here, as we start iterating
+                       // in the middle of a bucket. It's feasible, just tricky.
                         continue
                 }
                 k := add(unsafe.Pointer(b), dataOffset+uintptr(offi)*uintptr(t.keysize))
@@ -1092,7 +1120,7 @@ func evacuate(t *maptype, h *hmap, oldbucket uintptr) {
                         v := add(k, bucketCnt*uintptr(t.keysize))
                         for i := 0; i < bucketCnt; i, k, v = i+1, add(k, uintptr(t.keysize)), add(v, uintptr(t.valuesize)) {
                                 top := b.tophash[i]
-                               if top == empty {
+                               if isEmpty(top) {
                                         b.tophash[i] = evacuatedEmpty
                                         continue
                                 }
@@ -1129,7 +1157,7 @@ func evacuate(t *maptype, h *hmap, oldbucket uintptr) {
                                         }
                                 }
  
-                               if evacuatedX+1 != evacuatedY {
+                               if evacuatedX+1 != evacuatedY || evacuatedX^1 != evacuatedY {
                                         throw("bad evacuatedN")
                                 }
  
diff --git a/src/runtime/map_benchmark_test.go b/src/runtime/map_benchmark_test.go

index 5681d5eeb8cfb721f8b093b82d933bf9742b0219..d37dadcb5694d55bfef8798459cd83cacc7c1d63 100644 (file)
--- a/src/runtime/map_benchmark_test.go
+++ b/src/runtime/map_benchmark_test.go
@@ -5,6 +5,7 @@ package runtime_test
  
  import (
         "fmt"
+       "math/rand"
         "strconv"
         "strings"
         "testing"
@@ -206,6 +207,67 @@ func BenchmarkIntMap(b *testing.B) {
         }
  }
  
+func BenchmarkMapFirst(b *testing.B) {
+       for n := 1; n <= 16; n++ {
+               b.Run(fmt.Sprintf("%d", n), func(b *testing.B) {
+                       m := make(map[int]bool)
+                       for i := 0; i < n; i++ {
+                               m[i] = true
+                       }
+                       b.ResetTimer()
+                       for i := 0; i < b.N; i++ {
+                               _ = m[0]
+                       }
+               })
+       }
+}
+func BenchmarkMapMid(b *testing.B) {
+       for n := 1; n <= 16; n++ {
+               b.Run(fmt.Sprintf("%d", n), func(b *testing.B) {
+                       m := make(map[int]bool)
+                       for i := 0; i < n; i++ {
+                               m[i] = true
+                       }
+                       b.ResetTimer()
+                       for i := 0; i < b.N; i++ {
+                               _ = m[n>>1]
+                       }
+               })
+       }
+}
+func BenchmarkMapLast(b *testing.B) {
+       for n := 1; n <= 16; n++ {
+               b.Run(fmt.Sprintf("%d", n), func(b *testing.B) {
+                       m := make(map[int]bool)
+                       for i := 0; i < n; i++ {
+                               m[i] = true
+                       }
+                       b.ResetTimer()
+                       for i := 0; i < b.N; i++ {
+                               _ = m[n-1]
+                       }
+               })
+       }
+}
+
+func BenchmarkMapCycle(b *testing.B) {
+       // Arrange map entries to be a permuation, so that
+       // we hit all entries, and one lookup is data dependent
+       // on the previous lookup.
+       const N = 3127
+       p := rand.New(rand.NewSource(1)).Perm(N)
+       m := map[int]int{}
+       for i := 0; i < N; i++ {
+               m[i] = p[i]
+       }
+       b.ResetTimer()
+       j := 0
+       for i := 0; i < b.N; i++ {
+               j = m[j]
+       }
+       sink = uint64(j)
+}
+
  // Accessing the same keys in a row.
  func benchmarkRepeatedLookup(b *testing.B, lookupKeySize int) {
         m := make(map[string]bool)
diff --git a/src/runtime/map_fast32.go b/src/runtime/map_fast32.go

index 671558545a2ba462f3eb69a2988ab9613ef789a7..063a5cbe3a6e01e146b34e4fcaf8beeb7388dde1 100644 (file)
--- a/src/runtime/map_fast32.go
+++ b/src/runtime/map_fast32.go
@@ -41,7 +41,7 @@ func mapaccess1_fast32(t *maptype, h *hmap, key uint32) unsafe.Pointer {
         }
         for ; b != nil; b = b.overflow(t) {
                 for i, k := uintptr(0), b.keys(); i < bucketCnt; i, k = i+1, add(k, 4) {
-                       if *(*uint32)(k) == key && b.tophash[i] != empty {
+                       if *(*uint32)(k) == key && !isEmpty(b.tophash[i]) {
                                 return add(unsafe.Pointer(b), dataOffset+bucketCnt*4+i*uintptr(t.valuesize))
                         }
                 }
@@ -81,7 +81,7 @@ func mapaccess2_fast32(t *maptype, h *hmap, key uint32) (unsafe.Pointer, bool) {
         }
         for ; b != nil; b = b.overflow(t) {
                 for i, k := uintptr(0), b.keys(); i < bucketCnt; i, k = i+1, add(k, 4) {
-                       if *(*uint32)(k) == key && b.tophash[i] != empty {
+                       if *(*uint32)(k) == key && !isEmpty(b.tophash[i]) {
                                 return add(unsafe.Pointer(b), dataOffset+bucketCnt*4+i*uintptr(t.valuesize)), true
                         }
                 }
@@ -120,13 +120,17 @@ again:
         var inserti uintptr
         var insertk unsafe.Pointer
  
+bucketloop:
         for {
                 for i := uintptr(0); i < bucketCnt; i++ {
-                       if b.tophash[i] == empty {
+                       if isEmpty(b.tophash[i]) {
                                 if insertb == nil {
                                         inserti = i
                                         insertb = b
                                 }
+                               if b.tophash[i] == emptyRest {
+                                       break bucketloop
+                               }
                                 continue
                         }
                         k := *((*uint32)(add(unsafe.Pointer(b), dataOffset+i*4)))
@@ -206,13 +210,17 @@ again:
         var inserti uintptr
         var insertk unsafe.Pointer
  
+bucketloop:
         for {
                 for i := uintptr(0); i < bucketCnt; i++ {
-                       if b.tophash[i] == empty {
+                       if isEmpty(b.tophash[i]) {
                                 if insertb == nil {
                                         inserti = i
                                         insertb = b
                                 }
+                               if b.tophash[i] == emptyRest {
+                                       break bucketloop
+                               }
                                 continue
                         }
                         k := *((*unsafe.Pointer)(add(unsafe.Pointer(b), dataOffset+i*4)))
@@ -286,7 +294,7 @@ func mapdelete_fast32(t *maptype, h *hmap, key uint32) {
  search:
         for ; b != nil; b = b.overflow(t) {
                 for i, k := uintptr(0), b.keys(); i < bucketCnt; i, k = i+1, add(k, 4) {
-                       if key != *(*uint32)(k) || b.tophash[i] == empty {
+                       if key != *(*uint32)(k) || isEmpty(b.tophash[i]) {
                                 continue
                         }
                         // Only clear key if there are pointers in it.
@@ -299,7 +307,8 @@ search:
                         } else {
                                 memclrNoHeapPointers(v, t.elem.size)
                         }
-                       b.tophash[i] = empty
+                       b.tophash[i] = emptyOne
+                       // TODO: emptyRest?
                         h.count--
                         break search
                 }
@@ -350,7 +359,7 @@ func evacuate_fast32(t *maptype, h *hmap, oldbucket uintptr) {
                         v := add(k, bucketCnt*4)
                         for i := 0; i < bucketCnt; i, k, v = i+1, add(k, 4), add(v, uintptr(t.valuesize)) {
                                 top := b.tophash[i]
-                               if top == empty {
+                               if isEmpty(top) {
                                         b.tophash[i] = evacuatedEmpty
                                         continue
                                 }
diff --git a/src/runtime/map_fast64.go b/src/runtime/map_fast64.go

index 164a4dd1cef7189b553e033589545f21b4460b81..8270cf7b7d5b095bb299b96ec2cee52921e9eec7 100644 (file)
--- a/src/runtime/map_fast64.go
+++ b/src/runtime/map_fast64.go
@@ -41,7 +41,7 @@ func mapaccess1_fast64(t *maptype, h *hmap, key uint64) unsafe.Pointer {
         }
         for ; b != nil; b = b.overflow(t) {
                 for i, k := uintptr(0), b.keys(); i < bucketCnt; i, k = i+1, add(k, 8) {
-                       if *(*uint64)(k) == key && b.tophash[i] != empty {
+                       if *(*uint64)(k) == key && !isEmpty(b.tophash[i]) {
                                 return add(unsafe.Pointer(b), dataOffset+bucketCnt*8+i*uintptr(t.valuesize))
                         }
                 }
@@ -81,7 +81,7 @@ func mapaccess2_fast64(t *maptype, h *hmap, key uint64) (unsafe.Pointer, bool) {
         }
         for ; b != nil; b = b.overflow(t) {
                 for i, k := uintptr(0), b.keys(); i < bucketCnt; i, k = i+1, add(k, 8) {
-                       if *(*uint64)(k) == key && b.tophash[i] != empty {
+                       if *(*uint64)(k) == key && !isEmpty(b.tophash[i]) {
                                 return add(unsafe.Pointer(b), dataOffset+bucketCnt*8+i*uintptr(t.valuesize)), true
                         }
                 }
@@ -120,13 +120,17 @@ again:
         var inserti uintptr
         var insertk unsafe.Pointer
  
+bucketloop:
         for {
                 for i := uintptr(0); i < bucketCnt; i++ {
-                       if b.tophash[i] == empty {
+                       if isEmpty(b.tophash[i]) {
                                 if insertb == nil {
                                         insertb = b
                                         inserti = i
                                 }
+                               if b.tophash[i] == emptyRest {
+                                       break bucketloop
+                               }
                                 continue
                         }
                         k := *((*uint64)(add(unsafe.Pointer(b), dataOffset+i*8)))
@@ -206,13 +210,17 @@ again:
         var inserti uintptr
         var insertk unsafe.Pointer
  
+bucketloop:
         for {
                 for i := uintptr(0); i < bucketCnt; i++ {
-                       if b.tophash[i] == empty {
+                       if isEmpty(b.tophash[i]) {
                                 if insertb == nil {
                                         insertb = b
                                         inserti = i
                                 }
+                               if b.tophash[i] == emptyRest {
+                                       break bucketloop
+                               }
                                 continue
                         }
                         k := *((*unsafe.Pointer)(add(unsafe.Pointer(b), dataOffset+i*8)))
@@ -286,7 +294,7 @@ func mapdelete_fast64(t *maptype, h *hmap, key uint64) {
  search:
         for ; b != nil; b = b.overflow(t) {
                 for i, k := uintptr(0), b.keys(); i < bucketCnt; i, k = i+1, add(k, 8) {
-                       if key != *(*uint64)(k) || b.tophash[i] == empty {
+                       if key != *(*uint64)(k) || isEmpty(b.tophash[i]) {
                                 continue
                         }
                         // Only clear key if there are pointers in it.
@@ -299,7 +307,8 @@ search:
                         } else {
                                 memclrNoHeapPointers(v, t.elem.size)
                         }
-                       b.tophash[i] = empty
+                       b.tophash[i] = emptyOne
+                       //TODO: emptyRest
                         h.count--
                         break search
                 }
@@ -350,7 +359,7 @@ func evacuate_fast64(t *maptype, h *hmap, oldbucket uintptr) {
                         v := add(k, bucketCnt*8)
                         for i := 0; i < bucketCnt; i, k, v = i+1, add(k, 8), add(v, uintptr(t.valuesize)) {
                                 top := b.tophash[i]
-                               if top == empty {
+                               if isEmpty(top) {
                                         b.tophash[i] = evacuatedEmpty
                                         continue
                                 }
diff --git a/src/runtime/map_faststr.go b/src/runtime/map_faststr.go

index bee62dfb03b98fd30ac1c2b51aa41628035047f8..8f505f90a609cb6cf147239720e6e562ba772c01 100644 (file)
--- a/src/runtime/map_faststr.go
+++ b/src/runtime/map_faststr.go
@@ -28,7 +28,10 @@ func mapaccess1_faststr(t *maptype, h *hmap, ky string) unsafe.Pointer {
                         // short key, doing lots of comparisons is ok
                         for i, kptr := uintptr(0), b.keys(); i < bucketCnt; i, kptr = i+1, add(kptr, 2*sys.PtrSize) {
                                 k := (*stringStruct)(kptr)
-                               if k.len != key.len || b.tophash[i] == empty {
+                               if k.len != key.len || isEmpty(b.tophash[i]) {
+                                       if b.tophash[i] == emptyRest {
+                                               break
+                                       }
                                         continue
                                 }
                                 if k.str == key.str || memequal(k.str, key.str, uintptr(key.len)) {
@@ -41,7 +44,10 @@ func mapaccess1_faststr(t *maptype, h *hmap, ky string) unsafe.Pointer {
                 keymaybe := uintptr(bucketCnt)
                 for i, kptr := uintptr(0), b.keys(); i < bucketCnt; i, kptr = i+1, add(kptr, 2*sys.PtrSize) {
                         k := (*stringStruct)(kptr)
-                       if k.len != key.len || b.tophash[i] == empty {
+                       if k.len != key.len || isEmpty(b.tophash[i]) {
+                               if b.tophash[i] == emptyRest {
+                                       break
+                               }
                                 continue
                         }
                         if k.str == key.str {
@@ -117,7 +123,10 @@ func mapaccess2_faststr(t *maptype, h *hmap, ky string) (unsafe.Pointer, bool) {
                         // short key, doing lots of comparisons is ok
                         for i, kptr := uintptr(0), b.keys(); i < bucketCnt; i, kptr = i+1, add(kptr, 2*sys.PtrSize) {
                                 k := (*stringStruct)(kptr)
-                               if k.len != key.len || b.tophash[i] == empty {
+                               if k.len != key.len || isEmpty(b.tophash[i]) {
+                                       if b.tophash[i] == emptyRest {
+                                               break
+                                       }
                                         continue
                                 }
                                 if k.str == key.str || memequal(k.str, key.str, uintptr(key.len)) {
@@ -130,7 +139,10 @@ func mapaccess2_faststr(t *maptype, h *hmap, ky string) (unsafe.Pointer, bool) {
                 keymaybe := uintptr(bucketCnt)
                 for i, kptr := uintptr(0), b.keys(); i < bucketCnt; i, kptr = i+1, add(kptr, 2*sys.PtrSize) {
                         k := (*stringStruct)(kptr)
-                       if k.len != key.len || b.tophash[i] == empty {
+                       if k.len != key.len || isEmpty(b.tophash[i]) {
+                               if b.tophash[i] == emptyRest {
+                                       break
+                               }
                                 continue
                         }
                         if k.str == key.str {
@@ -220,13 +232,17 @@ again:
         var inserti uintptr
         var insertk unsafe.Pointer
  
+bucketloop:
         for {
                 for i := uintptr(0); i < bucketCnt; i++ {
                         if b.tophash[i] != top {
-                               if b.tophash[i] == empty && insertb == nil {
+                               if isEmpty(b.tophash[i]) && insertb == nil {
                                         insertb = b
                                         inserti = i
                                 }
+                               if b.tophash[i] == emptyRest {
+                                       break bucketloop
+                               }
                                 continue
                         }
                         k := (*stringStruct)(add(unsafe.Pointer(b), dataOffset+i*2*sys.PtrSize))
@@ -320,7 +336,8 @@ search:
                         } else {
                                 memclrNoHeapPointers(v, t.elem.size)
                         }
-                       b.tophash[i] = empty
+                       b.tophash[i] = emptyOne
+                       // TODO: emptyRest
                         h.count--
                         break search
                 }
@@ -371,7 +388,7 @@ func evacuate_faststr(t *maptype, h *hmap, oldbucket uintptr) {
                         v := add(k, bucketCnt*2*sys.PtrSize)
                         for i := 0; i < bucketCnt; i, k, v = i+1, add(k, 2*sys.PtrSize), add(v, uintptr(t.valuesize)) {
                                 top := b.tophash[i]
-                               if top == empty {
+                               if isEmpty(top) {
                                         b.tophash[i] = evacuatedEmpty
                                         continue
                                 }
author	Keith Randall <khr@google.com>
	Mon, 15 Oct 2018 22:14:48 +0000 (15:14 -0700)
committer	Keith Randall <khr@golang.org>
	Wed, 31 Oct 2018 22:48:13 +0000 (22:48 +0000)
src/runtime/map.go		patch \| blob \| history
src/runtime/map_benchmark_test.go		patch \| blob \| history
src/runtime/map_fast32.go		patch \| blob \| history
src/runtime/map_fast64.go		patch \| blob \| history
src/runtime/map_faststr.go		patch \| blob \| history