This lets the inliner do a better job optimizing the mapKeyError call.
goos: linux
goarch: amd64
pkg: runtime
cpu: AMD Ryzen 5 4600G with Radeon Graphics
│ /tmp/before2 │ /tmp/after3 │
│ sec/op │ sec/op vs base │
MapAccessZero/Key=int64-12 1.875n ± 0% 1.875n ± 0% ~ (p=0.506 n=25)
MapAccessZero/Key=int32-12 1.875n ± 0% 1.875n ± 0% ~ (p=0.082 n=25)
MapAccessZero/Key=string-12 1.902n ± 1% 1.902n ± 1% ~ (p=0.256 n=25)
MapAccessZero/Key=mediumType-12 2.816n ± 0% 1.958n ± 0% -30.47% (p=0.000 n=25)
MapAccessZero/Key=bigType-12 2.815n ± 0% 1.935n ± 0% -31.26% (p=0.000 n=25)
MapAccessEmpty/Key=int64-12 1.942n ± 0% 2.109n ± 0% +8.60% (p=0.000 n=25)
MapAccessEmpty/Key=int32-12 2.110n ± 0% 1.940n ± 0% -8.06% (p=0.000 n=25)
MapAccessEmpty/Key=string-12 2.024n ± 0% 2.109n ± 0% +4.20% (p=0.000 n=25)
MapAccessEmpty/Key=mediumType-12 3.157n ± 0% 2.344n ± 0% -25.75% (p=0.000 n=25)
MapAccessEmpty/Key=bigType-12 3.054n ± 0% 2.115n ± 0% -30.75% (p=0.000 n=25)
geomean 2.305n 2.011n -12.75%
Change-Id: Iee83930884dc4c8a791a711aa189a1c93b68d536
Reviewed-on: https://go-review.googlesource.com/c/go/+/663495
Reviewed-by: Dmitri Shuralyov <dmitshur@google.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Michael Pratt <mpratt@google.com>
"crypto/internal/sysrand.fatal": {"crypto/internal/sysrand"},
"crypto/rand.fatal": {"crypto/rand"},
"internal/runtime/maps.errNilAssign": {"internal/runtime/maps"},
+ "internal/runtime/maps.typeString": {"internal/runtime/maps"},
"internal/runtime/maps.fatal": {"internal/runtime/maps"},
- "internal/runtime/maps.mapKeyError": {"internal/runtime/maps"},
"internal/runtime/maps.newarray": {"internal/runtime/maps"},
"internal/runtime/maps.newobject": {"internal/runtime/maps"},
"internal/runtime/maps.typedmemclr": {"internal/runtime/maps"},
Type *Type
Data unsafe.Pointer
}
+
+// EmptyInterface describes the layout of an interface that contains any methods.
+type NonEmptyInterface struct {
+ ITab *ITab
+ Data unsafe.Pointer
+}
return m
}
+
+func OldMapKeyError(t *abi.OldMapType, p unsafe.Pointer) error {
+ if !t.HashMightPanic() {
+ return nil
+ }
+ return mapKeyError2(t.Key, p)
+}
+
+func mapKeyError(t *abi.SwissMapType, p unsafe.Pointer) error {
+ if !t.HashMightPanic() {
+ return nil
+ }
+ return mapKeyError2(t.Key, p)
+}
+
+func mapKeyError2(t *abi.Type, p unsafe.Pointer) error {
+ if t.TFlag&abi.TFlagRegularMemory != 0 {
+ return nil
+ }
+ switch t.Kind() {
+ case abi.Float32, abi.Float64, abi.Complex64, abi.Complex128, abi.String:
+ return nil
+ case abi.Interface:
+ i := (*abi.InterfaceType)(unsafe.Pointer(t))
+ var t *abi.Type
+ var pdata *unsafe.Pointer
+ if len(i.Methods) == 0 {
+ a := (*abi.EmptyInterface)(p)
+ t = a.Type
+ if t == nil {
+ return nil
+ }
+ pdata = &a.Data
+ } else {
+ a := (*abi.NonEmptyInterface)(p)
+ if a.ITab == nil {
+ return nil
+ }
+ t = a.ITab.Type
+ pdata = &a.Data
+ }
+
+ if t.Equal == nil {
+ return unhashableTypeError{t}
+ }
+
+ if t.Kind_&abi.KindDirectIface != 0 {
+ return mapKeyError2(t, unsafe.Pointer(pdata))
+ } else {
+ return mapKeyError2(t, *pdata)
+ }
+ case abi.Array:
+ a := (*abi.ArrayType)(unsafe.Pointer(t))
+ for i := uintptr(0); i < a.Len; i++ {
+ if err := mapKeyError2(a.Elem, unsafe.Pointer(uintptr(p)+i*a.Elem.Size_)); err != nil {
+ return err
+ }
+ }
+ return nil
+ case abi.Struct:
+ s := (*abi.StructType)(unsafe.Pointer(t))
+ for _, f := range s.Fields {
+ if f.Name.IsBlank() {
+ continue
+ }
+ if err := mapKeyError2(f.Typ, unsafe.Pointer(uintptr(p)+f.Offset)); err != nil {
+ return err
+ }
+ }
+ return nil
+ default:
+ // Should never happen, keep this case for robustness.
+ return unhashableTypeError{t}
+ }
+}
+
+type unhashableTypeError struct{ typ *abi.Type }
+
+func (unhashableTypeError) RuntimeError() {}
+
+func (e unhashableTypeError) Error() string { return "hash of unhashable type: " + typeString(e.typ) }
+
+// Pushed from runtime
+//
+//go:linkname typeString
+func typeString(typ *abi.Type) string
+++ /dev/null
-// Copyright 2024 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-//go:build !goexperiment.swissmap
-
-package maps
-
-import (
- "internal/abi"
- "unsafe"
-)
-
-// For testing, we don't ever need key errors.
-func mapKeyError(typ *abi.SwissMapType, p unsafe.Pointer) error {
- return nil
-}
// Functions below pushed from runtime.
-//go:linkname mapKeyError
-func mapKeyError(typ *abi.SwissMapType, p unsafe.Pointer) error
-
// Pushed from runtime in order to use runtime.plainError
//
//go:linkname errNilAssign
}
}
-func mapKeyError(t *maptype, p unsafe.Pointer) error {
- if !t.HashMightPanic() {
- return nil
- }
- return mapKeyError2(t.Key, p)
-}
-
-func mapKeyError2(t *_type, p unsafe.Pointer) error {
- if t.TFlag&abi.TFlagRegularMemory != 0 {
- return nil
- }
- switch t.Kind_ & abi.KindMask {
- case abi.Float32, abi.Float64, abi.Complex64, abi.Complex128, abi.String:
- return nil
- case abi.Interface:
- i := (*interfacetype)(unsafe.Pointer(t))
- var t *_type
- var pdata *unsafe.Pointer
- if len(i.Methods) == 0 {
- a := (*eface)(p)
- t = a._type
- if t == nil {
- return nil
- }
- pdata = &a.data
- } else {
- a := (*iface)(p)
- if a.tab == nil {
- return nil
- }
- t = a.tab.Type
- pdata = &a.data
- }
-
- if t.Equal == nil {
- return errorString("hash of unhashable type " + toRType(t).string())
- }
-
- if isDirectIface(t) {
- return mapKeyError2(t, unsafe.Pointer(pdata))
- } else {
- return mapKeyError2(t, *pdata)
- }
- case abi.Array:
- a := (*arraytype)(unsafe.Pointer(t))
- for i := uintptr(0); i < a.Len; i++ {
- if err := mapKeyError2(a.Elem, add(p, i*a.Elem.Size_)); err != nil {
- return err
- }
- }
- return nil
- case abi.Struct:
- s := (*structtype)(unsafe.Pointer(t))
- for _, f := range s.Fields {
- if f.Name.IsBlank() {
- continue
- }
- if err := mapKeyError2(f.Typ, add(p, f.Offset)); err != nil {
- return err
- }
- }
- return nil
- default:
- // Should never happen, keep this case for robustness.
- return errorString("hash of unhashable type " + toRType(t).string())
- }
-}
-
//go:linkname reflect_typehash reflect.typehash
func reflect_typehash(t *_type, p unsafe.Pointer, h uintptr) uintptr {
return typehash(t, p, h)
b.Run("Key=string/Elem=string", smallBenchSizes(benchmarkMapAccessMiss[string, string]))
b.Run("Key=smallType/Elem=int32", smallBenchSizes(benchmarkMapAccessMiss[smallType, int32]))
}
+
+func mapAccessZeroBenchmark[K comparable](b *testing.B) {
+ var m map[K]uint64
+ var key K
+ for i := 0; i < b.N; i++ {
+ sink = m[key]
+ }
+}
+
+func BenchmarkMapAccessZero(b *testing.B) {
+ b.Run("Key=int64", mapAccessZeroBenchmark[int64])
+ b.Run("Key=int32", mapAccessZeroBenchmark[int32])
+ b.Run("Key=string", mapAccessZeroBenchmark[string])
+ b.Run("Key=mediumType", mapAccessZeroBenchmark[mediumType])
+ b.Run("Key=bigType", mapAccessZeroBenchmark[bigType])
+}
+
+func mapAccessEmptyBenchmark[K mapBenchmarkKeyType](b *testing.B) {
+ m := make(map[K]uint64)
+ for i, v := range genValues[K](0, 1000) {
+ m[v] = uint64(i)
+ }
+ clear(m)
+ var key K
+ for i := 0; i < b.N; i++ {
+ sink = m[key]
+ }
+}
+
+func BenchmarkMapAccessEmpty(b *testing.B) {
+ b.Run("Key=int64", mapAccessEmptyBenchmark[int64])
+ b.Run("Key=int32", mapAccessEmptyBenchmark[int32])
+ b.Run("Key=string", mapAccessEmptyBenchmark[string])
+ b.Run("Key=mediumType", mapAccessEmptyBenchmark[mediumType])
+ b.Run("Key=bigType", mapAccessEmptyBenchmark[bigType])
+}
"internal/abi"
"internal/goarch"
"internal/runtime/atomic"
+ "internal/runtime/maps"
"internal/runtime/math"
"internal/runtime/sys"
"unsafe"
asanread(key, t.Key.Size_)
}
if h == nil || h.count == 0 {
- if err := mapKeyError(t, key); err != nil {
+ if err := maps.OldMapKeyError(t, key); err != nil {
panic(err) // see issue 23734
}
return unsafe.Pointer(&zeroVal[0])
asanread(key, t.Key.Size_)
}
if h == nil || h.count == 0 {
- if err := mapKeyError(t, key); err != nil {
+ if err := maps.OldMapKeyError(t, key); err != nil {
panic(err) // see issue 23734
}
return unsafe.Pointer(&zeroVal[0]), false
asanread(key, t.Key.Size_)
}
if h == nil || h.count == 0 {
- if err := mapKeyError(t, key); err != nil {
+ if err := maps.OldMapKeyError(t, key); err != nil {
panic(err) // see issue 23734
}
return
//go:linkname maps_errNilAssign internal/runtime/maps.errNilAssign
var maps_errNilAssign error = plainError("assignment to entry in nil map")
-//go:linkname maps_mapKeyError internal/runtime/maps.mapKeyError
-func maps_mapKeyError(t *abi.SwissMapType, p unsafe.Pointer) error {
- return mapKeyError(t, p)
-}
-
func makemap64(t *abi.SwissMapType, hint int64, m *maps.Map) *maps.Map {
if int64(int(hint)) != hint {
hint = 0
"unsafe"
)
+//go:linkname maps_typeString internal/runtime/maps.typeString
+func maps_typeString(typ *abi.Type) string {
+ return toRType(typ).string()
+}
+
type nameOff = abi.NameOff
type typeOff = abi.TypeOff
type textOff = abi.TextOff