internal/runtime/maps: loop invariant code motion with h2(hash) by hand

Change-Id: I0cd9763aeedfe326bc566da39b8be0d0ebd113ec
GitHub-Last-Rev: 1ec88644148deb41eea2ae89f7f1fb1759b37c27
GitHub-Pull-Request: golang/go#74952
Reviewed-on: https://go-review.googlesource.com/c/go/+/694016
Auto-Submit: Michael Pratt <mpratt@google.com>
Reviewed-by: Keith Randall <khr@google.com>
Reviewed-by: Dmitri Shuralyov <dmitshur@google.com>
Reviewed-by: Michael Pratt <mpratt@google.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Keith Randall <khr@golang.org>

diff --git a/src/internal/runtime/maps/runtime.go b/src/internal/runtime/maps/runtime.go
index ff8a7482494aef7bedea8af9b0081fff9477777e..8bba23f07003bd036b75a2ac473775240998d0ab 100644 (file)
--- a/src/internal/runtime/maps/runtime.go
+++ b/src/internal/runtime/maps/runtime.go
@@ -94,10 +94,11 @@ func runtime_mapaccess1(typ *abi.MapType, m *Map, key unsafe.Pointer) unsafe.Poi
 
        // Probe table.
        seq := makeProbeSeq(h1(hash), t.groups.lengthMask)
+       h2Hash := h2(hash)
        for ; ; seq = seq.next() {
                g := t.groups.group(typ, seq.offset)
 
-               match := g.ctrls().matchH2(h2(hash))
+               match := g.ctrls().matchH2(h2Hash)
 
                for match != 0 {
                        i := match.first()
@@ -168,10 +169,11 @@ func runtime_mapaccess2(typ *abi.MapType, m *Map, key unsafe.Pointer) (unsafe.Po
 
        // Probe table.
        seq := makeProbeSeq(h1(hash), t.groups.lengthMask)
+       h2Hash := h2(hash)
        for ; ; seq = seq.next() {
                g := t.groups.group(typ, seq.offset)
 
-               match := g.ctrls().matchH2(h2(hash))
+               match := g.ctrls().matchH2(h2Hash)
 
                for match != 0 {
                        i := match.first()
@@ -262,9 +264,10 @@ outer:
                var firstDeletedGroup groupReference
                var firstDeletedSlot uintptr
 
+               h2Hash := h2(hash)
                for ; ; seq = seq.next() {
                        g := t.groups.group(typ, seq.offset)
-                       match := g.ctrls().matchH2(h2(hash))
+                       match := g.ctrls().matchH2(h2Hash)
 
                        // Look for an existing slot containing this key.
                        for match != 0 {
@@ -329,7 +332,7 @@ outer:
                                                slotElem = emem
                                        }
 
-                                       g.ctrls().set(i, ctrl(h2(hash)))
+                                       g.ctrls().set(i, ctrl(h2Hash))
                                        t.growthLeft--
                                        t.used++
                                        m.used++

diff --git a/src/internal/runtime/maps/runtime_fast32.go b/src/internal/runtime/maps/runtime_fast32.go
index beed67ce286aa1999d9f1efd79e846e015c94b11..d5be04afd450c0e1a2b65f881705b4849b36febe 100644 (file)
--- a/src/internal/runtime/maps/runtime_fast32.go
+++ b/src/internal/runtime/maps/runtime_fast32.go
@@ -55,10 +55,11 @@ func runtime_mapaccess1_fast32(typ *abi.MapType, m *Map, key uint32) unsafe.Poin
 
        // Probe table.
        seq := makeProbeSeq(h1(hash), t.groups.lengthMask)
+       h2Hash := h2(hash)
        for ; ; seq = seq.next() {
                g := t.groups.group(typ, seq.offset)
 
-               match := g.ctrls().matchH2(h2(hash))
+               match := g.ctrls().matchH2(h2Hash)
 
                for match != 0 {
                        i := match.first()
@@ -124,10 +125,11 @@ func runtime_mapaccess2_fast32(typ *abi.MapType, m *Map, key uint32) (unsafe.Poi
 
        // Probe table.
        seq := makeProbeSeq(h1(hash), t.groups.lengthMask)
+       h2Hash := h2(hash)
        for ; ; seq = seq.next() {
                g := t.groups.group(typ, seq.offset)
 
-               match := g.ctrls().matchH2(h2(hash))
+               match := g.ctrls().matchH2(h2Hash)
 
                for match != 0 {
                        i := match.first()
@@ -245,9 +247,10 @@ outer:
                var firstDeletedGroup groupReference
                var firstDeletedSlot uintptr
 
+               h2Hash := h2(hash)
                for ; ; seq = seq.next() {
                        g := t.groups.group(typ, seq.offset)
-                       match := g.ctrls().matchH2(h2(hash))
+                       match := g.ctrls().matchH2(h2Hash)
 
                        // Look for an existing slot containing this key.
                        for match != 0 {
@@ -302,7 +305,7 @@ outer:
 
                                slotElem = g.elem(typ, i)
 
-                               g.ctrls().set(i, ctrl(h2(hash)))
+                               g.ctrls().set(i, ctrl(h2Hash))
                                t.growthLeft--
                                t.used++
                                m.used++
@@ -383,9 +386,10 @@ outer:
                var firstDeletedGroup groupReference
                var firstDeletedSlot uintptr
 
+               h2Hash := h2(hash)
                for ; ; seq = seq.next() {
                        g := t.groups.group(typ, seq.offset)
-                       match := g.ctrls().matchH2(h2(hash))
+                       match := g.ctrls().matchH2(h2Hash)
 
                        // Look for an existing slot containing this key.
                        for match != 0 {
@@ -435,7 +439,7 @@ outer:
 
                                slotElem = g.elem(typ, i)
 
-                               g.ctrls().set(i, ctrl(h2(hash)))
+                               g.ctrls().set(i, ctrl(h2Hash))
                                t.growthLeft--
                                t.used++
                                m.used++

diff --git a/src/internal/runtime/maps/runtime_fast64.go b/src/internal/runtime/maps/runtime_fast64.go
index 2f9cf28daafdb4004cf8056f36b509e707922f88..2bee2d4be019b3febfa84abb8bf708784048729a 100644 (file)
--- a/src/internal/runtime/maps/runtime_fast64.go
+++ b/src/internal/runtime/maps/runtime_fast64.go
@@ -55,10 +55,11 @@ func runtime_mapaccess1_fast64(typ *abi.MapType, m *Map, key uint64) unsafe.Poin
 
        // Probe table.
        seq := makeProbeSeq(h1(hash), t.groups.lengthMask)
+       h2Hash := h2(hash)
        for ; ; seq = seq.next() {
                g := t.groups.group(typ, seq.offset)
 
-               match := g.ctrls().matchH2(h2(hash))
+               match := g.ctrls().matchH2(h2Hash)
 
                for match != 0 {
                        i := match.first()
@@ -124,10 +125,12 @@ func runtime_mapaccess2_fast64(typ *abi.MapType, m *Map, key uint64) (unsafe.Poi
 
        // Probe table.
        seq := makeProbeSeq(h1(hash), t.groups.lengthMask)
+
+       h2Hash := h2(hash)
        for ; ; seq = seq.next() {
                g := t.groups.group(typ, seq.offset)
 
-               match := g.ctrls().matchH2(h2(hash))
+               match := g.ctrls().matchH2(h2Hash)
 
                for match != 0 {
                        i := match.first()
@@ -245,9 +248,10 @@ outer:
                var firstDeletedGroup groupReference
                var firstDeletedSlot uintptr
 
+               h2Hash := h2(hash)
                for ; ; seq = seq.next() {
                        g := t.groups.group(typ, seq.offset)
-                       match := g.ctrls().matchH2(h2(hash))
+                       match := g.ctrls().matchH2(h2Hash)
 
                        // Look for an existing slot containing this key.
                        for match != 0 {
@@ -302,7 +306,7 @@ outer:
 
                                slotElem = g.elem(typ, i)
 
-                               g.ctrls().set(i, ctrl(h2(hash)))
+                               g.ctrls().set(i, ctrl(h2Hash))
                                t.growthLeft--
                                t.used++
                                m.used++
@@ -422,9 +426,10 @@ outer:
                var firstDeletedGroup groupReference
                var firstDeletedSlot uintptr
 
+               h2Hash := h2(hash)
                for ; ; seq = seq.next() {
                        g := t.groups.group(typ, seq.offset)
-                       match := g.ctrls().matchH2(h2(hash))
+                       match := g.ctrls().matchH2(h2Hash)
 
                        // Look for an existing slot containing this key.
                        for match != 0 {
@@ -474,7 +479,7 @@ outer:
 
                                slotElem = g.elem(typ, i)
 
-                               g.ctrls().set(i, ctrl(h2(hash)))
+                               g.ctrls().set(i, ctrl(h2Hash))
                                t.growthLeft--
                                t.used++
                                m.used++

diff --git a/src/internal/runtime/maps/runtime_faststr.go b/src/internal/runtime/maps/runtime_faststr.go
index ddac7eacc52ece00fd2189b6962bd53fb81f3e56..374468b66438a6c02691e89998f8c0d300a71bf1 100644 (file)
--- a/src/internal/runtime/maps/runtime_faststr.go
+++ b/src/internal/runtime/maps/runtime_faststr.go
@@ -131,10 +131,11 @@ func runtime_mapaccess1_faststr(typ *abi.MapType, m *Map, key string) unsafe.Poi
 
        // Probe table.
        seq := makeProbeSeq(h1(hash), t.groups.lengthMask)
+       h2Hash := h2(hash)
        for ; ; seq = seq.next() {
                g := t.groups.group(typ, seq.offset)
 
-               match := g.ctrls().matchH2(h2(hash))
+               match := g.ctrls().matchH2(h2Hash)
 
                for match != 0 {
                        i := match.first()
@@ -190,10 +191,11 @@ func runtime_mapaccess2_faststr(typ *abi.MapType, m *Map, key string) (unsafe.Po
 
        // Probe table.
        seq := makeProbeSeq(h1(hash), t.groups.lengthMask)
+       h2Hash := h2(hash)
        for ; ; seq = seq.next() {
                g := t.groups.group(typ, seq.offset)
 
-               match := g.ctrls().matchH2(h2(hash))
+               match := g.ctrls().matchH2(h2Hash)
 
                for match != 0 {
                        i := match.first()
@@ -313,9 +315,10 @@ outer:
                var firstDeletedGroup groupReference
                var firstDeletedSlot uintptr
 
+               h2Hash := h2(hash)
                for ; ; seq = seq.next() {
                        g := t.groups.group(typ, seq.offset)
-                       match := g.ctrls().matchH2(h2(hash))
+                       match := g.ctrls().matchH2(h2Hash)
 
                        // Look for an existing slot containing this key.
                        for match != 0 {
@@ -373,7 +376,7 @@ outer:
 
                                slotElem = g.elem(typ, i)
 
-                               g.ctrls().set(i, ctrl(h2(hash)))
+                               g.ctrls().set(i, ctrl(h2Hash))
                                t.growthLeft--
                                t.used++
                                m.used++

diff --git a/src/internal/runtime/maps/table.go b/src/internal/runtime/maps/table.go
index d4b9276b57078f733a68113fb973698fb81f2621..7e2c6e31bcaa1ec1f92888cb14a7814aa60c1183 100644 (file)
--- a/src/internal/runtime/maps/table.go
+++ b/src/internal/runtime/maps/table.go
@@ -192,10 +192,11 @@ func (t *table) getWithKey(typ *abi.MapType, hash uintptr, key unsafe.Pointer) (
        // load factors, k is less than 32, meaning that the number of false
        // positive comparisons we must perform is less than 1/8 per find.
        seq := makeProbeSeq(h1(hash), t.groups.lengthMask)
+       h2Hash := h2(hash)
        for ; ; seq = seq.next() {
                g := t.groups.group(typ, seq.offset)
 
-               match := g.ctrls().matchH2(h2(hash))
+               match := g.ctrls().matchH2(h2Hash)
 
                for match != 0 {
                        i := match.first()
@@ -225,10 +226,11 @@ func (t *table) getWithKey(typ *abi.MapType, hash uintptr, key unsafe.Pointer) (
 
 func (t *table) getWithoutKey(typ *abi.MapType, hash uintptr, key unsafe.Pointer) (unsafe.Pointer, bool) {
        seq := makeProbeSeq(h1(hash), t.groups.lengthMask)
+       h2Hash := h2(hash)
        for ; ; seq = seq.next() {
                g := t.groups.group(typ, seq.offset)
 
-               match := g.ctrls().matchH2(h2(hash))
+               match := g.ctrls().matchH2(h2Hash)
 
                for match != 0 {
                        i := match.first()
@@ -271,9 +273,10 @@ func (t *table) PutSlot(typ *abi.MapType, m *Map, hash uintptr, key unsafe.Point
        var firstDeletedGroup groupReference
        var firstDeletedSlot uintptr
 
+       h2Hash := h2(hash)
        for ; ; seq = seq.next() {
                g := t.groups.group(typ, seq.offset)
-               match := g.ctrls().matchH2(h2(hash))
+               match := g.ctrls().matchH2(h2Hash)
 
                // Look for an existing slot containing this key.
                for match != 0 {
@@ -348,7 +351,7 @@ func (t *table) PutSlot(typ *abi.MapType, m *Map, hash uintptr, key unsafe.Point
                                slotElem = emem
                        }
 
-                       g.ctrls().set(i, ctrl(h2(hash)))
+                       g.ctrls().set(i, ctrl(h2Hash))
                        t.growthLeft--
                        t.used++
                        m.used++
@@ -420,9 +423,10 @@ func (t *table) uncheckedPutSlot(typ *abi.MapType, hash uintptr, key, elem unsaf
 // Delete returns true if it put a tombstone in t.
 func (t *table) Delete(typ *abi.MapType, m *Map, hash uintptr, key unsafe.Pointer) bool {
        seq := makeProbeSeq(h1(hash), t.groups.lengthMask)
+       h2Hash := h2(hash)
        for ; ; seq = seq.next() {
                g := t.groups.group(typ, seq.offset)
-               match := g.ctrls().matchH2(h2(hash))
+               match := g.ctrls().matchH2(h2Hash)
 
                for match != 0 {
                        i := match.first()