return it.elem
}
+func (it *Iter) nextDirIdx() {
+ // Skip other entries in the directory that refer to the same
+ // logical table. There are two cases of this:
+ //
+ // Consider this directory:
+ //
+ // - 0: *t1
+ // - 1: *t1
+ // - 2: *t2a
+ // - 3: *t2b
+ //
+ // At some point, the directory grew to accomodate a split of
+ // t2. t1 did not split, so entries 0 and 1 both point to t1.
+ // t2 did split, so the two halves were installed in entries 2
+ // and 3.
+ //
+ // If dirIdx is 0 and it.tab is t1, then we should skip past
+ // entry 1 to avoid repeating t1.
+ //
+ // If dirIdx is 2 and it.tab is t2 (pre-split), then we should
+ // skip past entry 3 because our pre-split t2 already covers
+ // all keys from t2a and t2b (except for new insertions, which
+ // iteration need not return).
+ //
+ // We can achieve both of these by using to difference between
+ // the directory and table depth to compute how many entries
+ // the table covers.
+ entries := 1 << (it.m.globalDepth - it.tab.localDepth)
+ it.dirIdx += entries
+ it.tab = nil
+ it.group = groupReference{}
+ it.entryIdx = 0
+}
+
+// Return the appropriate key/elem for key at slotIdx index within it.group, if
+// any.
+func (it *Iter) grownKeyElem(key unsafe.Pointer, slotIdx uintptr) (unsafe.Pointer, unsafe.Pointer, bool) {
+ newKey, newElem, ok := it.m.getWithKey(it.typ, key)
+ if !ok {
+ // Key has likely been deleted, and
+ // should be skipped.
+ //
+ // One exception is keys that don't
+ // compare equal to themselves (e.g.,
+ // NaN). These keys cannot be looked
+ // up, so getWithKey will fail even if
+ // the key exists.
+ //
+ // However, we are in luck because such
+ // keys cannot be updated and they
+ // cannot be deleted except with clear.
+ // Thus if no clear has occurred, the
+ // key/elem must still exist exactly as
+ // in the old groups, so we can return
+ // them from there.
+ //
+ // TODO(prattmic): Consider checking
+ // clearSeq early. If a clear occurred,
+ // Next could always return
+ // immediately, as iteration doesn't
+ // need to return anything added after
+ // clear.
+ if it.clearSeq == it.m.clearSeq && !it.typ.Key.Equal(key, key) {
+ elem := it.group.elem(it.typ, slotIdx)
+ if it.typ.IndirectElem() {
+ elem = *((*unsafe.Pointer)(elem))
+ }
+ return key, elem, true
+ }
+
+ // This entry doesn't exist anymore.
+ return nil, nil, false
+ }
+
+ return newKey, newElem, true
+}
+
// Next proceeds to the next element in iteration, which can be accessed via
// the Key and Elem methods.
//
}
// Continue iteration until we find a full slot.
- for it.dirIdx < it.m.dirLen {
- // Find next table.
+ for ; it.dirIdx < it.m.dirLen; it.nextDirIdx() {
+ // Resolve the table.
if it.tab == nil {
dirIdx := int((uint64(it.dirIdx) + it.dirOffset) & uint64(it.m.dirLen-1))
newTab := it.m.directoryAt(uintptr(dirIdx))
// N.B. Use it.tab, not newTab. It is important to use the old
// table for key selection if the table has grown. See comment
// on grown below.
- for ; it.entryIdx <= it.tab.groups.entryMask; it.entryIdx++ {
+
+ if it.entryIdx > it.tab.groups.entryMask {
+ // Continue to next table.
+ continue
+ }
+
+ // Fast path: skip matching and directly check if entryIdx is a
+ // full slot.
+ //
+ // In the slow path below, we perform an 8-slot match check to
+ // look for full slots within the group.
+ //
+ // However, with a max load factor of 7/8, each slot in a
+ // mostly full map has a high probability of being full. Thus
+ // it is cheaper to check a single slot than do a full control
+ // match.
+
+ entryIdx := (it.entryIdx + it.entryOffset) & it.tab.groups.entryMask
+ slotIdx := uintptr(entryIdx & (abi.SwissMapGroupSlots - 1))
+ if slotIdx == 0 || it.group.data == nil {
+ // Only compute the group (a) when we switch
+ // groups (slotIdx rolls over) and (b) on the
+ // first iteration in this table (slotIdx may
+ // not be zero due to entryOffset).
+ groupIdx := entryIdx >> abi.SwissMapGroupSlotsBits
+ it.group = it.tab.groups.group(it.typ, groupIdx)
+ }
+
+ if (it.group.ctrls().get(slotIdx) & ctrlEmpty) == 0 {
+ // Slot full.
+
+ key := it.group.key(it.typ, slotIdx)
+ if it.typ.IndirectKey() {
+ key = *((*unsafe.Pointer)(key))
+ }
+
+ grown := it.tab.index == -1
+ var elem unsafe.Pointer
+ if grown {
+ newKey, newElem, ok := it.grownKeyElem(key, slotIdx)
+ if !ok {
+ // This entry doesn't exist
+ // anymore. Continue to the
+ // next one.
+ goto next
+ } else {
+ key = newKey
+ elem = newElem
+ }
+ } else {
+ elem = it.group.elem(it.typ, slotIdx)
+ if it.typ.IndirectElem() {
+ elem = *((*unsafe.Pointer)(elem))
+ }
+ }
+
+ it.entryIdx++
+ it.key = key
+ it.elem = elem
+ return
+ }
+
+next:
+ it.entryIdx++
+
+ // Slow path: use a match on the control word to jump ahead to
+ // the next full slot.
+ //
+ // This is highly effective for maps with particularly low load
+ // (e.g., map allocated with large hint but few insertions).
+ //
+ // For maps with medium load (e.g., 3-4 empty slots per group)
+ // it also tends to work pretty well. Since slots within a
+ // group are filled in order, then if there have been no
+ // deletions, a match will allow skipping past all empty slots
+ // at once.
+ //
+ // Note: it is tempting to cache the group match result in the
+ // iterator to use across Next calls. However because entries
+ // may be deleted between calls later calls would still need to
+ // double-check the control value.
+
+ var groupMatch bitset
+ for it.entryIdx <= it.tab.groups.entryMask {
entryIdx := (it.entryIdx + it.entryOffset) & it.tab.groups.entryMask
slotIdx := uintptr(entryIdx & (abi.SwissMapGroupSlots - 1))
it.group = it.tab.groups.group(it.typ, groupIdx)
}
- // TODO(prattmic): Skip over groups that are composed of only empty
- // or deleted slots using matchEmptyOrDeleted() and counting the
- // number of bits set.
+ if groupMatch == 0 {
+ groupMatch = it.group.ctrls().matchFull()
- if (it.group.ctrls().get(slotIdx) & ctrlEmpty) == ctrlEmpty {
- // Empty or deleted.
- continue
+ if slotIdx != 0 {
+ // Starting in the middle of the group.
+ // Ignore earlier groups.
+ groupMatch = groupMatch.removeBelow(slotIdx)
+ }
+
+ // Skip over groups that are composed of only empty or
+ // deleted slots.
+ if groupMatch == 0 {
+ // Jump past remaining slots in this
+ // group.
+ it.entryIdx += abi.SwissMapGroupSlots - uint64(slotIdx)
+ continue
+ }
+
+ i := groupMatch.first()
+ it.entryIdx += uint64(i - slotIdx)
+ if it.entryIdx > it.tab.groups.entryMask {
+ // Past the end of this table's iteration.
+ continue
+ }
+ entryIdx += uint64(i - slotIdx)
+ slotIdx = i
}
key := it.group.key(it.typ, slotIdx)
grown := it.tab.index == -1
var elem unsafe.Pointer
if grown {
- var ok bool
- newKey, newElem, ok := it.m.getWithKey(it.typ, key)
+ newKey, newElem, ok := it.grownKeyElem(key, slotIdx)
if !ok {
- // Key has likely been deleted, and
- // should be skipped.
- //
- // One exception is keys that don't
- // compare equal to themselves (e.g.,
- // NaN). These keys cannot be looked
- // up, so getWithKey will fail even if
- // the key exists.
- //
- // However, we are in luck because such
- // keys cannot be updated and they
- // cannot be deleted except with clear.
- // Thus if no clear has occurted, the
- // key/elem must still exist exactly as
- // in the old groups, so we can return
- // them from there.
- //
- // TODO(prattmic): Consider checking
- // clearSeq early. If a clear occurred,
- // Next could always return
- // immediately, as iteration doesn't
- // need to return anything added after
- // clear.
- if it.clearSeq == it.m.clearSeq && !it.typ.Key.Equal(key, key) {
- elem = it.group.elem(it.typ, slotIdx)
- if it.typ.IndirectElem() {
- elem = *((*unsafe.Pointer)(elem))
- }
- } else {
+ // This entry doesn't exist anymore.
+ // Continue to the next one.
+ groupMatch = groupMatch.removeFirst()
+ if groupMatch == 0 {
+ // No more entries in this
+ // group. Continue to next
+ // group.
+ it.entryIdx += abi.SwissMapGroupSlots - uint64(slotIdx)
continue
}
+
+ // Next full slot.
+ i := groupMatch.first()
+ it.entryIdx += uint64(i - slotIdx)
+ continue
} else {
key = newKey
elem = newElem
}
}
- it.entryIdx++
+ // Jump ahead to the next full slot or next group.
+ groupMatch = groupMatch.removeFirst()
+ if groupMatch == 0 {
+ // No more entries in
+ // this group. Continue
+ // to next group.
+ it.entryIdx += abi.SwissMapGroupSlots - uint64(slotIdx)
+ } else {
+ // Next full slot.
+ i := groupMatch.first()
+ it.entryIdx += uint64(i - slotIdx)
+ }
+
it.key = key
it.elem = elem
return
}
- // Skip other entries in the directory that refer to the same
- // logical table. There are two cases of this:
- //
- // Consider this directory:
- //
- // - 0: *t1
- // - 1: *t1
- // - 2: *t2a
- // - 3: *t2b
- //
- // At some point, the directory grew to accomodate a split of
- // t2. t1 did not split, so entries 0 and 1 both point to t1.
- // t2 did split, so the two halves were installed in entries 2
- // and 3.
- //
- // If dirIdx is 0 and it.tab is t1, then we should skip past
- // entry 1 to avoid repeating t1.
- //
- // If dirIdx is 2 and it.tab is t2 (pre-split), then we should
- // skip past entry 3 because our pre-split t2 already covers
- // all keys from t2a and t2b (except for new insertions, which
- // iteration need not return).
- //
- // We can achieve both of these by using to difference between
- // the directory and table depth to compute how many entries
- // the table covers.
- entries := 1 << (it.m.globalDepth - it.tab.localDepth)
- it.dirIdx += entries
- it.tab = nil
- it.group = groupReference{}
- it.entryIdx = 0
+ // Continue to next table.
}
it.key = nil