runtime,hash/maphash: reuse hashSets to save memory pressure

Might help with OOMs on 32-bit platforms

Change-Id: Idd5129c61ecdfeedd5a9a18fce85dbba27cab946
Reviewed-on: https://go-review.googlesource.com/c/go/+/574475
Reviewed-by: Emmanuel Odeke <emmanuel@orijtech.com>
Reviewed-by: Keith Randall <khr@google.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Than McIntosh <thanm@google.com>

diff --git a/src/hash/maphash/smhasher_test.go b/src/hash/maphash/smhasher_test.go
index f34cea8e8054f957ef6f6fc614b0d739e61127d5..28cdae0444d986b7203002f51de654de836f4a00 100644 (file)
--- a/src/hash/maphash/smhasher_test.go
+++ b/src/hash/maphash/smhasher_test.go
@@ -112,6 +112,8 @@ func (s *hashSet) check(t *testing.T) {
        if float64(collisions) > expected+SLOP*(3*stddev+1) {
                t.Errorf("unexpected number of collisions: got=%d mean=%f stddev=%f", collisions, expected, stddev)
        }
+       // Reset for reuse
+       s.list = s.list[:0]
 }
 
 // a string plus adding zeros must make distinct hashes
@@ -212,8 +214,8 @@ func TestSmhasherCyclic(t *testing.T) {
        r := rand.New(rand.NewSource(1234))
        const REPEAT = 8
        const N = 1000000
+       h := newHashSet()
        for n := 4; n <= 12; n++ {
-               h := newHashSet()
                b := make([]byte, REPEAT*n)
                for i := 0; i < N; i++ {
                        b[0] = byte(i * 79 % 97)
@@ -238,18 +240,18 @@ func TestSmhasherSparse(t *testing.T) {
        if testing.Short() {
                t.Skip("Skipping in short mode")
        }
-       sparse(t, 32, 6)
-       sparse(t, 40, 6)
-       sparse(t, 48, 5)
-       sparse(t, 56, 5)
-       sparse(t, 64, 5)
-       sparse(t, 96, 4)
-       sparse(t, 256, 3)
-       sparse(t, 2048, 2)
-}
-func sparse(t *testing.T, n int, k int) {
-       b := make([]byte, n/8)
        h := newHashSet()
+       sparse(t, h, 32, 6)
+       sparse(t, h, 40, 6)
+       sparse(t, h, 48, 5)
+       sparse(t, h, 56, 5)
+       sparse(t, h, 64, 5)
+       sparse(t, h, 96, 4)
+       sparse(t, h, 256, 3)
+       sparse(t, h, 2048, 2)
+}
+func sparse(t *testing.T, h *hashSet, n int, k int) {
+       b := make([]byte, n/8)
        setbits(h, b, 0, k)
        h.check(t)
 }
@@ -276,15 +278,15 @@ func TestSmhasherPermutation(t *testing.T) {
        if testing.Short() {
                t.Skip("Skipping in short mode")
        }
-       permutation(t, []uint32{0, 1, 2, 3, 4, 5, 6, 7}, 8)
-       permutation(t, []uint32{0, 1 << 29, 2 << 29, 3 << 29, 4 << 29, 5 << 29, 6 << 29, 7 << 29}, 8)
-       permutation(t, []uint32{0, 1}, 20)
-       permutation(t, []uint32{0, 1 << 31}, 20)
-       permutation(t, []uint32{0, 1, 2, 3, 4, 5, 6, 7, 1 << 29, 2 << 29, 3 << 29, 4 << 29, 5 << 29, 6 << 29, 7 << 29}, 6)
+       h := newHashSet()
+       permutation(t, h, []uint32{0, 1, 2, 3, 4, 5, 6, 7}, 8)
+       permutation(t, h, []uint32{0, 1 << 29, 2 << 29, 3 << 29, 4 << 29, 5 << 29, 6 << 29, 7 << 29}, 8)
+       permutation(t, h, []uint32{0, 1}, 20)
+       permutation(t, h, []uint32{0, 1 << 31}, 20)
+       permutation(t, h, []uint32{0, 1, 2, 3, 4, 5, 6, 7, 1 << 29, 2 << 29, 3 << 29, 4 << 29, 5 << 29, 6 << 29, 7 << 29}, 6)
 }
-func permutation(t *testing.T, s []uint32, n int) {
+func permutation(t *testing.T, h *hashSet, s []uint32, n int) {
        b := make([]byte, n*4)
-       h := newHashSet()
        genPerm(h, b, s, 0)
        h.check(t)
 }
@@ -418,8 +420,8 @@ func windowed(t *testing.T, k key) {
        }
        const BITS = 16
 
+       h := newHashSet()
        for r := 0; r < k.bits(); r++ {
-               h := newHashSet()
                for i := 0; i < 1<<BITS; i++ {
                        k.clear()
                        for j := 0; j < BITS; j++ {
@@ -438,18 +440,18 @@ func TestSmhasherText(t *testing.T) {
        if testing.Short() {
                t.Skip("Skipping in short mode")
        }
-       text(t, "Foo", "Bar")
-       text(t, "FooBar", "")
-       text(t, "", "FooBar")
+       h := newHashSet()
+       text(t, h, "Foo", "Bar")
+       text(t, h, "FooBar", "")
+       text(t, h, "", "FooBar")
 }
-func text(t *testing.T, prefix, suffix string) {
+func text(t *testing.T, h *hashSet, prefix, suffix string) {
        const N = 4
        const S = "ABCDEFGHIJKLMNOPQRSTabcdefghijklmnopqrst0123456789"
        const L = len(S)
        b := make([]byte, len(prefix)+N+len(suffix))
        copy(b, prefix)
        copy(b[len(prefix)+N:], suffix)
-       h := newHashSet()
        c := b[len(prefix):]
        for i := 0; i < L; i++ {
                c[0] = S[i]

diff --git a/src/runtime/hash_test.go b/src/runtime/hash_test.go
index 86bc55fb140d5892416ac90b35757fc3cc6047c9..08672dbe8af521a6a22c06202a01010cb537f70c 100644 (file)
--- a/src/runtime/hash_test.go
+++ b/src/runtime/hash_test.go
@@ -121,6 +121,8 @@ func (s *HashSet) check(t *testing.T) {
        if float64(collisions) > expected+SLOP*(3*stddev+1) {
                t.Errorf("unexpected number of collisions: got=%d mean=%f stddev=%f threshold=%f", collisions, expected, stddev, expected+SLOP*(3*stddev+1))
        }
+       // Reset for reuse
+       s.list = s.list[:0]
 }
 
 // a string plus adding zeros must make distinct hashes
@@ -230,8 +232,8 @@ func TestSmhasherCyclic(t *testing.T) {
        r := rand.New(rand.NewSource(1234))
        const REPEAT = 8
        const N = 1000000
+       h := newHashSet()
        for n := 4; n <= 12; n++ {
-               h := newHashSet()
                b := make([]byte, REPEAT*n)
                for i := 0; i < N; i++ {
                        b[0] = byte(i * 79 % 97)
@@ -256,18 +258,18 @@ func TestSmhasherSparse(t *testing.T) {
        if testing.Short() {
                t.Skip("Skipping in short mode")
        }
-       sparse(t, 32, 6)
-       sparse(t, 40, 6)
-       sparse(t, 48, 5)
-       sparse(t, 56, 5)
-       sparse(t, 64, 5)
-       sparse(t, 96, 4)
-       sparse(t, 256, 3)
-       sparse(t, 2048, 2)
-}
-func sparse(t *testing.T, n int, k int) {
-       b := make([]byte, n/8)
        h := newHashSet()
+       sparse(t, h, 32, 6)
+       sparse(t, h, 40, 6)
+       sparse(t, h, 48, 5)
+       sparse(t, h, 56, 5)
+       sparse(t, h, 64, 5)
+       sparse(t, h, 96, 4)
+       sparse(t, h, 256, 3)
+       sparse(t, h, 2048, 2)
+}
+func sparse(t *testing.T, h *HashSet, n int, k int) {
+       b := make([]byte, n/8)
        setbits(h, b, 0, k)
        h.check(t)
 }
@@ -297,15 +299,15 @@ func TestSmhasherPermutation(t *testing.T) {
        if race.Enabled {
                t.Skip("Too long for race mode")
        }
-       permutation(t, []uint32{0, 1, 2, 3, 4, 5, 6, 7}, 8)
-       permutation(t, []uint32{0, 1 << 29, 2 << 29, 3 << 29, 4 << 29, 5 << 29, 6 << 29, 7 << 29}, 8)
-       permutation(t, []uint32{0, 1}, 20)
-       permutation(t, []uint32{0, 1 << 31}, 20)
-       permutation(t, []uint32{0, 1, 2, 3, 4, 5, 6, 7, 1 << 29, 2 << 29, 3 << 29, 4 << 29, 5 << 29, 6 << 29, 7 << 29}, 6)
+       h := newHashSet()
+       permutation(t, h, []uint32{0, 1, 2, 3, 4, 5, 6, 7}, 8)
+       permutation(t, h, []uint32{0, 1 << 29, 2 << 29, 3 << 29, 4 << 29, 5 << 29, 6 << 29, 7 << 29}, 8)
+       permutation(t, h, []uint32{0, 1}, 20)
+       permutation(t, h, []uint32{0, 1 << 31}, 20)
+       permutation(t, h, []uint32{0, 1, 2, 3, 4, 5, 6, 7, 1 << 29, 2 << 29, 3 << 29, 4 << 29, 5 << 29, 6 << 29, 7 << 29}, 6)
 }
-func permutation(t *testing.T, s []uint32, n int) {
+func permutation(t *testing.T, h *HashSet, s []uint32, n int) {
        b := make([]byte, n*4)
-       h := newHashSet()
        genPerm(h, b, s, 0)
        h.check(t)
 }
@@ -542,14 +544,15 @@ func TestSmhasherWindowed(t *testing.T) {
        if race.Enabled {
                t.Skip("Too long for race mode")
        }
+       h := newHashSet()
        t.Logf("32 bit keys")
-       windowed(t, &Int32Key{})
+       windowed(t, h, &Int32Key{})
        t.Logf("64 bit keys")
-       windowed(t, &Int64Key{})
+       windowed(t, h, &Int64Key{})
        t.Logf("string keys")
-       windowed(t, &BytesKey{make([]byte, 128)})
+       windowed(t, h, &BytesKey{make([]byte, 128)})
 }
-func windowed(t *testing.T, k Key) {
+func windowed(t *testing.T, h *HashSet, k Key) {
        if GOARCH == "wasm" {
                t.Skip("Too slow on wasm")
        }
@@ -566,7 +569,6 @@ func windowed(t *testing.T, k Key) {
        const BITS = 16
 
        for r := 0; r < k.bits(); r++ {
-               h := newHashSet()
                for i := 0; i < 1<<BITS; i++ {
                        k.clear()
                        for j := 0; j < BITS; j++ {
@@ -585,18 +587,18 @@ func TestSmhasherText(t *testing.T) {
        if testing.Short() {
                t.Skip("Skipping in short mode")
        }
-       text(t, "Foo", "Bar")
-       text(t, "FooBar", "")
-       text(t, "", "FooBar")
+       h := newHashSet()
+       text(t, h, "Foo", "Bar")
+       text(t, h, "FooBar", "")
+       text(t, h, "", "FooBar")
 }
-func text(t *testing.T, prefix, suffix string) {
+func text(t *testing.T, h *HashSet, prefix, suffix string) {
        const N = 4
        const S = "ABCDEFGHIJKLMNOPQRSTabcdefghijklmnopqrst0123456789"
        const L = len(S)
        b := make([]byte, len(prefix)+N+len(suffix))
        copy(b, prefix)
        copy(b[len(prefix)+N:], suffix)
-       h := newHashSet()
        c := b[len(prefix):]
        for i := 0; i < L; i++ {
                c[0] = S[i]