From: Michael Pratt Date: Mon, 7 Oct 2024 21:07:34 +0000 (-0400) Subject: internal/runtime/maps: merge Iter.groupIdx and Iter.slotIdx X-Git-Tag: go1.24rc1~569 X-Git-Url: http://www.git.cypherpunks.su/?a=commitdiff_plain;h=7de87ebd59f7667f6b27d635a380ea0d9d3dabf5;p=gostls13.git internal/runtime/maps: merge Iter.groupIdx and Iter.slotIdx For #54766. Cq-Include-Trybots: luci.golang.try:gotip-linux-amd64-longtest-swissmap Change-Id: Ie21ef0f33f42735eadccd75eeebb3b5e81c2f459 Reviewed-on: https://go-review.googlesource.com/c/go/+/618535 Auto-Submit: Michael Pratt Reviewed-by: Michael Knyszek Commit-Queue: Michael Pratt Reviewed-by: David Chase LUCI-TryBot-Result: Go LUCI --- diff --git a/src/cmd/compile/internal/reflectdata/map_swiss.go b/src/cmd/compile/internal/reflectdata/map_swiss.go index 50e123ddb0..2037d0473f 100644 --- a/src/cmd/compile/internal/reflectdata/map_swiss.go +++ b/src/cmd/compile/internal/reflectdata/map_swiss.go @@ -95,6 +95,7 @@ func swissTableType() *types.Type { // groups_typ unsafe.Pointer // *abi.SwissMapType // groups_data unsafe.Pointer // groups_lengthMask uint64 + // groups_entryMask uint64 // } // must match internal/runtime/maps/table.go:table. fields := []*types.Field{ @@ -108,6 +109,7 @@ func swissTableType() *types.Type { makefield("groups_typ", types.Types[types.TUNSAFEPTR]), makefield("groups_data", types.Types[types.TUNSAFEPTR]), makefield("groups_lengthMask", types.Types[types.TUINT64]), + makefield("groups_entryMask", types.Types[types.TUINT64]), } n := ir.NewDeclNameAt(src.NoXPos, ir.OTYPE, ir.Pkgs.InternalMaps.Lookup("table")) @@ -118,9 +120,9 @@ func swissTableType() *types.Type { table.SetUnderlying(types.NewStruct(fields)) types.CalcSize(table) - // The size of table should be 56 bytes on 64 bit - // and 36 bytes on 32 bit platforms. - if size := int64(3*2 + 2*1 /* one extra for padding */ + 1*8 + 5*types.PtrSize); table.Size() != size { + // The size of table should be 64 bytes on 64 bit + // and 44 bytes on 32 bit platforms. + if size := int64(3*2 + 2*1 /* one extra for padding */ + 2*8 + 5*types.PtrSize); table.Size() != size { base.Fatalf("internal/runtime/maps.table size not correct: got %d, want %d", table.Size(), size) } @@ -204,10 +206,7 @@ func SwissMapIterType() *types.Type { // // tab *table // - // groupIdx uint64 - // slotIdx uint32 - // - // // 4 bytes of padding on 64-bit arches. + // entryIdx uint64 // } // must match internal/runtime/maps/table.go:Iter. fields := []*types.Field{ @@ -221,8 +220,7 @@ func SwissMapIterType() *types.Type { makefield("globalDepth", types.Types[types.TUINT8]), makefield("dirIdx", types.Types[types.TINT]), makefield("tab", types.NewPtr(swissTableType())), - makefield("groupIdx", types.Types[types.TUINT64]), - makefield("slotIdx", types.Types[types.TUINT32]), + makefield("entryIdx", types.Types[types.TUINT64]), } // build iterator struct hswissing the above fields @@ -233,12 +231,11 @@ func SwissMapIterType() *types.Type { iter.SetUnderlying(types.NewStruct(fields)) types.CalcSize(iter) - want := 7*types.PtrSize + 4*8 + 1*4 - if types.PtrSize == 8 { - want += 4 // tailing padding - } - if iter.Size() != int64(want) { - base.Fatalf("internal/runtime/maps.Iter size not correct: got %d, want %d", iter.Size(), want) + + // The size of Iter should be 88 bytes on 64 bit + // and 60 bytes on 32 bit platforms. + if size := 7*types.PtrSize /* one extra for globalDepth + padding */ + 4*8; iter.Size() != int64(size) { + base.Fatalf("internal/runtime/maps.Iter size not correct: got %d, want %d", iter.Size(), size) } cachedSwissIterType = iter diff --git a/src/internal/abi/map_swiss.go b/src/internal/abi/map_swiss.go index d69aefbb29..3eeb9ffa57 100644 --- a/src/internal/abi/map_swiss.go +++ b/src/internal/abi/map_swiss.go @@ -11,8 +11,11 @@ import ( // Map constants common to several packages // runtime/runtime-gdb.py:MapTypePrinter contains its own copy const ( + // Number of bits in the group.slot count. + SwissMapGroupSlotsBits = 3 + // Number of slots in a group. - SwissMapGroupSlots = 8 + SwissMapGroupSlots = 1 << SwissMapGroupSlotsBits // 8 ) type SwissMapType struct { diff --git a/src/internal/runtime/maps/group.go b/src/internal/runtime/maps/group.go index e03ed98c94..ed66b5e1f2 100644 --- a/src/internal/runtime/maps/group.go +++ b/src/internal/runtime/maps/group.go @@ -230,6 +230,9 @@ type groupsReference struct { // length must be a power of two). This allows computing i%length // quickly using bitwise AND. lengthMask uint64 + + // entryMask is the total number of slots in the groups minus one. + entryMask uint64 } // newGroups allocates a new array of length groups. @@ -241,6 +244,7 @@ func newGroups(typ *abi.SwissMapType, length uint64) groupsReference { // TODO: make the length type the same throughout. data: newarray(typ.Group, int(length)), lengthMask: length - 1, + entryMask: (length * abi.SwissMapGroupSlots) - 1, } } diff --git a/src/internal/runtime/maps/table.go b/src/internal/runtime/maps/table.go index 6f66e1fa38..801479ba88 100644 --- a/src/internal/runtime/maps/table.go +++ b/src/internal/runtime/maps/table.go @@ -431,8 +431,8 @@ type Iter struct { // Randomize iteration order by starting iteration at a random slot // offset. The offset into the directory uses a separate offset, as it // must adjust when the directory grows. - groupSlotOffset uint64 - dirOffset uint64 + entryOffset uint64 + dirOffset uint64 // Snapshot of Map.clearSeq at iteration initialization time. Used to // detect clear during iteration. @@ -449,12 +449,10 @@ type Iter struct { // tab is the table at dirIdx during the previous call to Next. tab *table - // TODO: these could be merged into a single counter (and pre-offset - // with offset). - groupIdx uint64 - slotIdx uint32 - - // 4 bytes of padding on 64-bit arches. + // entryIdx is the current entry index, prior to adjustment by entryOffset. + // The lower 3 bits of the index are the slot index, and the upper bits + // are the group index. + entryIdx uint64 } // Init initializes Iter for iteration. @@ -466,7 +464,7 @@ func (it *Iter) Init(typ *abi.SwissMapType, m *Map) { it.typ = m.typ it.m = m - it.groupSlotOffset = rand() + it.entryOffset = rand() it.dirOffset = rand() it.globalDepth = m.globalDepth it.clearSeq = m.clearSeq @@ -579,88 +577,92 @@ func (it *Iter) Next() { it.tab = newTab } + var g groupReference + // N.B. Use it.tab, not newTab. It is important to use the old // table for key selection if the table has grown. See comment // on grown below. - for ; it.groupIdx <= it.tab.groups.lengthMask; it.groupIdx++ { - g := it.tab.groups.group((it.groupIdx + it.groupSlotOffset) & it.tab.groups.lengthMask) + for ; it.entryIdx <= it.tab.groups.entryMask; it.entryIdx++ { + entryIdx := (it.entryIdx + it.entryOffset) & it.tab.groups.entryMask + slotIdx := uint32(entryIdx & (abi.SwissMapGroupSlots - 1)) + + if slotIdx == 0 || g.data == nil { + // Only compute the group (a) when we switch + // groups (slotIdx rolls over) and (b) on the + // first iteration in this table (slotIdx may + // not be zero due to entryOffset). + groupIdx := entryIdx >> abi.SwissMapGroupSlotsBits + g = it.tab.groups.group(groupIdx) + } // TODO(prattmic): Skip over groups that are composed of only empty // or deleted slots using matchEmptyOrDeleted() and counting the // number of bits set. - for ; it.slotIdx < abi.SwissMapGroupSlots; it.slotIdx++ { - k := (it.slotIdx + uint32(it.groupSlotOffset)) % abi.SwissMapGroupSlots - if (g.ctrls().get(k) & ctrlEmpty) == ctrlEmpty { - // Empty or deleted. - continue - } + if (g.ctrls().get(slotIdx) & ctrlEmpty) == ctrlEmpty { + // Empty or deleted. + continue + } - key := g.key(k) + key := g.key(slotIdx) - // If the table has changed since the last - // call, then it has grown or split. In this - // case, further mutations (changes to - // key->elem or deletions) will not be visible - // in our snapshot table. Instead we must - // consult the new table by doing a full - // lookup. - // - // We still use our old table to decide which - // keys to lookup in order to avoid returning - // the same key twice. - grown := it.tab != newTab - var elem unsafe.Pointer - if grown { - var ok bool - newKey, newElem, ok := it.m.getWithKey(key) - if !ok { - // Key has likely been deleted, and - // should be skipped. - // - // One exception is keys that don't - // compare equal to themselves (e.g., - // NaN). These keys cannot be looked - // up, so getWithKey will fail even if - // the key exists. - // - // However, we are in luck because such - // keys cannot be updated and they - // cannot be deleted except with clear. - // Thus if no clear has occurted, the - // key/elem must still exist exactly as - // in the old groups, so we can return - // them from there. - // - // TODO(prattmic): Consider checking - // clearSeq early. If a clear occurred, - // Next could always return - // immediately, as iteration doesn't - // need to return anything added after - // clear. - if it.clearSeq == it.m.clearSeq && !it.m.typ.Key.Equal(key, key) { - elem = g.elem(k) - } else { - continue - } + // If the table has changed since the last + // call, then it has grown or split. In this + // case, further mutations (changes to + // key->elem or deletions) will not be visible + // in our snapshot table. Instead we must + // consult the new table by doing a full + // lookup. + // + // We still use our old table to decide which + // keys to lookup in order to avoid returning + // the same key twice. + grown := it.tab != newTab + var elem unsafe.Pointer + if grown { + var ok bool + newKey, newElem, ok := it.m.getWithKey(key) + if !ok { + // Key has likely been deleted, and + // should be skipped. + // + // One exception is keys that don't + // compare equal to themselves (e.g., + // NaN). These keys cannot be looked + // up, so getWithKey will fail even if + // the key exists. + // + // However, we are in luck because such + // keys cannot be updated and they + // cannot be deleted except with clear. + // Thus if no clear has occurted, the + // key/elem must still exist exactly as + // in the old groups, so we can return + // them from there. + // + // TODO(prattmic): Consider checking + // clearSeq early. If a clear occurred, + // Next could always return + // immediately, as iteration doesn't + // need to return anything added after + // clear. + if it.clearSeq == it.m.clearSeq && !it.m.typ.Key.Equal(key, key) { + elem = g.elem(slotIdx) } else { - key = newKey - elem = newElem + continue } } else { - elem = g.elem(k) - } - - it.slotIdx++ - if it.slotIdx >= abi.SwissMapGroupSlots { - it.groupIdx++ - it.slotIdx = 0 + key = newKey + elem = newElem } - it.key = key - it.elem = elem - return + } else { + elem = g.elem(slotIdx) } - it.slotIdx = 0 + + it.entryIdx++ + it.key = key + it.elem = elem + return } // Skip other entries in the directory that refer to the same @@ -692,7 +694,7 @@ func (it *Iter) Next() { entries := 1 << (it.m.globalDepth - it.tab.localDepth) it.dirIdx += entries it.tab = nil - it.groupIdx = 0 + it.entryIdx = 0 } it.key = nil