--- /dev/null
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Page heap.
+//
+// See malloc.go for the general overview.
+//
+// Large spans are the subject of this file. Spans consisting of less than
+// _MaxMHeapLists are held in lists of like sized spans. Larger spans
+// are held in a treap. See https://en.wikipedia.org/wiki/Treap or
+// http://faculty.washington.edu/aragon/pubs/rst89.pdf for an overview.
+// sema.go also holds an implementation of a treap.
+//
+// Each treapNode holds a single span. The treap is sorted by page size
+// and for spans of the same size a secondary sort based on start address
+// is done.
+// Spans are returned based on a best fit algorithm and for spans of the same
+// size the one at the lowest address is selected.
+//
+// The primary routines are
+// insert: adds a span to the treap
+// remove: removes the span from that treap that best fits the required size
+// removeSpan: which removes a specific span from the treap
+//
+// _mheap.lock must be held when manipulating this data structure.
+//
+package runtime
+
+import (
+ "unsafe"
+)
+
+//go:notinheap
+type mTreap struct {
+ treap *treapNode
+}
+
+//go:notinheap
+type treapNode struct {
+ right *treapNode // all treapNodes > this treap node
+ left *treapNode // all treapNodes < this treap node
+ parent *treapNode // direct parent of this node, nil if root
+ npagesKey uintptr // number of pages in spanKey, used as primary sort key
+ spanKey *mspan // span of size npagesKey, used as secondary sort key
+ priority uint32 // random number used by treap algorithm keep tree probablistically balanced
+}
+
+func (t *treapNode) init() {
+ t.right = nil
+ t.left = nil
+ t.parent = nil
+ t.spanKey = nil
+ t.npagesKey = 0
+ t.priority = 0
+}
+
+// isSpanInTreap is handy for debugging. One should hold the heap lock, usually
+// mheap_.lock().
+func (t *treapNode) isSpanInTreap(s *mspan) bool {
+ if t == nil {
+ return false
+ }
+ return t.spanKey == s || t.left.isSpanInTreap(s) || t.right.isSpanInTreap(s)
+}
+
+// walkTreap is handy for debugging.
+// Starting at some treapnode t, for example the root, do a depth first preorder walk of
+// the tree executing fn at each treap node. One should hold the heap lock, usually
+// mheap_.lock().
+func (t *treapNode) walkTreap(fn func(tn *treapNode)) {
+ if t == nil {
+ return
+ }
+ fn(t)
+ t.left.walkTreap(fn)
+ t.right.walkTreap(fn)
+}
+
+// checkTreapNode when used in conjunction with walkTreap can usually detect a
+// poorly formed treap.
+func checkTreapNode(t *treapNode) {
+ // lessThan is used to order the treap.
+ // npagesKey and npages are the primary keys.
+ // spanKey and span are the secondary keys.
+ // span == nil (0) will always be lessThan all
+ // spans of the same size.
+ lessThan := func(npages uintptr, s *mspan) bool {
+ if t.npagesKey != npages {
+ return t.npagesKey < npages
+ }
+ // t.npagesKey == npages
+ return uintptr(unsafe.Pointer(t.spanKey)) < uintptr(unsafe.Pointer(s))
+ }
+
+ if t == nil {
+ return
+ }
+ if t.spanKey.npages != t.npagesKey || t.spanKey.next != nil {
+ println("runtime: checkTreapNode treapNode t=", t, " t.npagesKey=", t.npagesKey,
+ "t.spanKey.npages=", t.spanKey.npages)
+ throw("why does span.npages and treap.ngagesKey do not match?")
+ }
+ if t.left != nil && lessThan(t.left.npagesKey, t.left.spanKey) {
+ throw("t.lessThan(t.left.npagesKey, t.left.spanKey) is not false")
+ }
+ if t.right != nil && !lessThan(t.right.npagesKey, t.right.spanKey) {
+ throw("!t.lessThan(t.left.npagesKey, t.left.spanKey) is not false")
+ }
+}
+
+// insert adds span to the large span treap.
+func (root *mTreap) insert(span *mspan) {
+ npages := span.npages
+ var last *treapNode
+ pt := &root.treap
+ for t := *pt; t != nil; t = *pt {
+ last = t
+ if t.npagesKey < npages {
+ pt = &t.right
+ } else if t.npagesKey > npages {
+ pt = &t.left
+ } else if uintptr(unsafe.Pointer(t.spanKey)) < uintptr(unsafe.Pointer(span)) {
+ // t.npagesKey == npages, so sort on span addresses.
+ pt = &t.right
+ } else if uintptr(unsafe.Pointer(t.spanKey)) > uintptr(unsafe.Pointer(span)) {
+ pt = &t.left
+ } else {
+ throw("inserting span already in treap")
+ }
+ }
+
+ // Add t as new leaf in tree of span size and unique addrs.
+ // The balanced tree is a treap using priority as the random heap priority.
+ // That is, it is a binary tree ordered according to the npagesKey,
+ // but then among the space of possible binary trees respecting those
+ // npagesKeys, it is kept balanced on average by maintaining a heap ordering
+ // on the priority: s.priority <= both s.right.priority and s.right.priority.
+ // https://en.wikipedia.org/wiki/Treap
+ // http://faculty.washington.edu/aragon/pubs/rst89.pdf
+
+ t := (*treapNode)(mheap_.treapalloc.alloc())
+ t.init()
+ t.npagesKey = span.npages
+ t.priority = fastrand()
+ t.spanKey = span
+ t.parent = last
+ *pt = t // t now at a leaf.
+ // Rotate up into tree according to priority.
+ for t.parent != nil && t.parent.priority > t.priority {
+ if t != nil && t.spanKey.npages != t.npagesKey {
+ println("runtime: insert t=", t, "t.npagesKey=", t.npagesKey)
+ println("runtime: t.spanKey=", t.spanKey, "t.spanKey.npages=", t.spanKey.npages)
+ throw("span and treap sizes do not match?")
+ }
+ if t.parent.left == t {
+ root.rotateRight(t.parent)
+ } else {
+ if t.parent.right != t {
+ throw("treap insert finds a broken treap")
+ }
+ root.rotateLeft(t.parent)
+ }
+ }
+}
+
+func (root *mTreap) removeNode(t *treapNode) *mspan {
+ if t.spanKey.npages != t.npagesKey {
+ throw("span and treap node npages do not match")
+ }
+ result := t.spanKey
+
+ // Rotate t down to be leaf of tree for removal, respecting priorities.
+ for t.right != nil || t.left != nil {
+ if t.right == nil || t.left != nil && t.left.priority < t.right.priority {
+ root.rotateRight(t)
+ } else {
+ root.rotateLeft(t)
+ }
+ }
+ // Remove t, now a leaf.
+ if t.parent != nil {
+ if t.parent.left == t {
+ t.parent.left = nil
+ } else {
+ t.parent.right = nil
+ }
+ } else {
+ root.treap = nil
+ }
+ // Return the found treapNode's span after freeing the treapNode.
+ t.spanKey = nil
+ t.npagesKey = 0
+ mheap_.treapalloc.free(unsafe.Pointer(t))
+ return result
+}
+
+// remove searches for, finds, removes from the treap, and returns the smallest
+// span that can hold npages. If no span has at least npages return nil.
+// This is slightly more complicated than a simple binary tree search
+// since if an exact match is not found the next larger node is
+// returned.
+// If the last node inspected > npagesKey not holding
+// a left node (a smaller npages) is the "best fit" node.
+func (root *mTreap) remove(npages uintptr) *mspan {
+ t := root.treap
+ for t != nil {
+ if t.spanKey == nil {
+ throw("treap node with nil spanKey found")
+ }
+ if t.npagesKey < npages {
+ t = t.right
+ } else if t.left != nil && t.left.npagesKey >= npages {
+ t = t.left
+ } else {
+ result := t.spanKey
+ root.removeNode(t)
+ return result
+ }
+ }
+ return nil
+}
+
+// removeSpan searches for, finds, deletes span along with
+// the associated treap node. If the span is not in the treap
+// then t will eventually be set to nil and the t.spanKey
+// will throw.
+func (root *mTreap) removeSpan(span *mspan) {
+ npages := span.npages
+ t := root.treap
+ for t.spanKey != span {
+ if t.npagesKey < npages {
+ t = t.right
+ } else if t.npagesKey > npages {
+ t = t.left
+ } else if uintptr(unsafe.Pointer(t.spanKey)) < uintptr(unsafe.Pointer(span)) {
+ t = t.right
+ } else if uintptr(unsafe.Pointer(t.spanKey)) > uintptr(unsafe.Pointer(span)) {
+ t = t.left
+ }
+ }
+ root.removeNode(t)
+ return
+}
+
+// scavengetreap visits each node in the treap and scavenges the
+// treapNode's span.
+func scavengetreap(treap *treapNode, now, limit uint64) uintptr {
+ if treap == nil {
+ return 0
+ }
+ return scavengeTreapNode(treap, now, limit) +
+ scavengetreap(treap.left, now, limit) +
+ scavengetreap(treap.right, now, limit)
+}
+
+// rotateLeft rotates the tree rooted at node x.
+// turning (x a (y b c)) into (y (x a b) c).
+func (root *mTreap) rotateLeft(x *treapNode) {
+ // p -> (x a (y b c))
+ p := x.parent
+ a, y := x.left, x.right
+ b, c := y.left, y.right
+
+ y.left = x
+ x.parent = y
+ y.right = c
+ if c != nil {
+ c.parent = y
+ }
+ x.left = a
+ if a != nil {
+ a.parent = x
+ }
+ x.right = b
+ if b != nil {
+ b.parent = x
+ }
+
+ y.parent = p
+ if p == nil {
+ root.treap = y
+ } else if p.left == x {
+ p.left = y
+ } else {
+ if p.right != x {
+ throw("large span treap rotateLeft")
+ }
+ p.right = y
+ }
+}
+
+// rotateRight rotates the tree rooted at node y.
+// turning (y (x a b) c) into (x a (y b c)).
+func (root *mTreap) rotateRight(y *treapNode) {
+ // p -> (y (x a b) c)
+ p := y.parent
+ x, c := y.left, y.right
+ a, b := x.left, x.right
+
+ x.left = a
+ if a != nil {
+ a.parent = x
+ }
+ x.right = y
+ y.parent = x
+ y.left = b
+ if b != nil {
+ b.parent = y
+ }
+ y.right = c
+ if c != nil {
+ c.parent = y
+ }
+
+ x.parent = p
+ if p == nil {
+ root.treap = x
+ } else if p.left == y {
+ p.left = x
+ } else {
+ if p.right != y {
+ throw("large span treap rotateRight")
+ }
+ p.right = x
+ }
+}
//go:notinheap
type mheap struct {
lock mutex
- free [_MaxMHeapList]mSpanList // free lists of given length
- freelarge mSpanList // free lists length >= _MaxMHeapList
- busy [_MaxMHeapList]mSpanList // busy lists of large objects of given length
- busylarge mSpanList // busy lists of large objects length >= _MaxMHeapList
+ free [_MaxMHeapList]mSpanList // free lists of given length up to _MaxMHeapList
+ freelarge mTreap // free treap of length >= _MaxMHeapList
+ busy [_MaxMHeapList]mSpanList // busy lists of large spans of given length
+ busylarge mSpanList // busy lists of large spans length >= _MaxMHeapList
sweepgen uint32 // sweep generation, see comment in mspan
sweepdone uint32 // all spans are swept
// on the swept stack.
sweepSpans [2]gcSweepBuf
- _ uint32 // align uint64 fields on 32-bit for atomics
+ // _ uint32 // align uint64 fields on 32-bit for atomics
// Proportional sweep
pagesInUse uint64 // pages of spans in stats _MSpanInUse; R/W with mheap.lock
spanalloc fixalloc // allocator for span*
cachealloc fixalloc // allocator for mcache*
+ treapalloc fixalloc // allocator for treapNodes* used by large objects
specialfinalizeralloc fixalloc // allocator for specialfinalizer*
specialprofilealloc fixalloc // allocator for specialprofile*
speciallock mutex // lock for special record allocators.
// Initialize the heap.
func (h *mheap) init(spansStart, spansBytes uintptr) {
+ h.treapalloc.init(unsafe.Sizeof(treapNode{}), nil, nil, &memstats.other_sys)
h.spanalloc.init(unsafe.Sizeof(mspan{}), recordspan, unsafe.Pointer(h), &memstats.mspan_sys)
h.cachealloc.init(unsafe.Sizeof(mcache{}), nil, nil, &memstats.mcache_sys)
h.specialfinalizeralloc.init(unsafe.Sizeof(specialfinalizer{}), nil, nil, &memstats.other_sys)
h.busy[i].init()
}
- h.freelarge.init()
h.busylarge.init()
for i := range h.central {
h.central[i].mcentral.init(int32(i))
if s.sweepgen == sg-2 && atomic.Cas(&s.sweepgen, sg-2, sg-1) {
list.remove(s)
// swept spans are at the end of the list
- list.insertBack(s)
+ list.insertBack(s) // Puts it back on a busy list. s is not in the treap at this point.
unlock(&h.lock)
snpages := s.npages
if s.sweep(false) {
// This unlock acts as a release barrier. See mHeap_Alloc_m.
unlock(&h.lock)
+
return s
}
list = &h.free[i]
if !list.isEmpty() {
s = list.first
+ list.remove(s)
goto HaveSpan
}
}
-
// Best fit in list of large spans.
- list = &h.freelarge
- s = h.allocLarge(npage)
+ s = h.allocLarge(npage) // allocLarge removed s from h.freelarge for us
if s == nil {
if !h.grow(npage) {
return nil
if s.npages < npage {
throw("MHeap_AllocLocked - bad npages")
}
- list.remove(s)
- if s.inList() {
- throw("still in list")
- }
if s.npreleased > 0 {
sysUsed(unsafe.Pointer(s.base()), s.npages<<_PageShift)
memstats.heap_released -= uint64(s.npreleased << _PageShift)
return s
}
-// Allocate a span of exactly npage pages from the list of large spans.
+// Large spans have a minimum size of 1MByte. The maximum number of large spans to support
+// 1TBytes is 1 million, experimentation using random sizes indicates that the depth of
+// the tree is less that 2x that of a perfectly balanced tree. For 1TByte can be referenced
+// by a perfectly balanced tree with a a depth of 20. Twice that is an acceptable 40.
+func (h *mheap) isLargeSpan(npages uintptr) bool {
+ return npages >= uintptr(len(h.free))
+}
+
+// Allocate a span of exactly npage pages from the treap of large spans.
func (h *mheap) allocLarge(npage uintptr) *mspan {
- return bestFit(&h.freelarge, npage, nil)
+ return bestFitTreap(&h.freelarge, npage, nil)
}
-// Search list for smallest span with >= npage pages.
-// If there are multiple smallest spans, take the one
+// Search treap for smallest span with >= npage pages.
+// If there are multiple smallest spans, select the one
// with the earliest starting address.
-func bestFit(list *mSpanList, npage uintptr, best *mspan) *mspan {
- for s := list.first; s != nil; s = s.next {
- if s.npages < npage {
- continue
- }
- if best == nil || s.npages < best.npages || (s.npages == best.npages && s.base() < best.base()) {
- best = s
- }
- }
- return best
+func bestFitTreap(treap *mTreap, npage uintptr, best *mspan) *mspan {
+ return treap.remove(npage)
}
// Try to add at least npage pages of memory to the heap,
// Coalesce with earlier, later spans.
p := (s.base() - h.arena_start) >> _PageShift
if p > 0 {
- t := h.spans[p-1]
- if t != nil && t.state == _MSpanFree {
- s.startAddr = t.startAddr
- s.npages += t.npages
- s.npreleased = t.npreleased // absorb released pages
- s.needzero |= t.needzero
- p -= t.npages
+ before := h.spans[p-1]
+ if before != nil && before.state == _MSpanFree {
+ // Now adjust s.
+ s.startAddr = before.startAddr
+ s.npages += before.npages
+ s.npreleased = before.npreleased // absorb released pages
+ s.needzero |= before.needzero
+ p -= before.npages
h.spans[p] = s
- h.freeList(t.npages).remove(t)
- t.state = _MSpanDead
- h.spanalloc.free(unsafe.Pointer(t))
+ // The size is potentially changing so the treap needs to delete adjacent nodes and
+ // insert back as a combined node.
+ if h.isLargeSpan(before.npages) {
+ // We have a t, it is large so it has to be in the treap so we can remove it.
+ h.freelarge.removeSpan(before)
+ } else {
+ h.freeList(before.npages).remove(before)
+ }
+ before.state = _MSpanDead
+ h.spanalloc.free(unsafe.Pointer(before))
}
}
+
+ // Now check to see if next (greater addresses) span is free and can be coalesced.
if (p + s.npages) < uintptr(len(h.spans)) {
- t := h.spans[p+s.npages]
- if t != nil && t.state == _MSpanFree {
- s.npages += t.npages
- s.npreleased += t.npreleased
- s.needzero |= t.needzero
+ after := h.spans[p+s.npages]
+ if after != nil && after.state == _MSpanFree {
+ s.npages += after.npages
+ s.npreleased += after.npreleased
+ s.needzero |= after.needzero
h.spans[p+s.npages-1] = s
- h.freeList(t.npages).remove(t)
- t.state = _MSpanDead
- h.spanalloc.free(unsafe.Pointer(t))
+ if h.isLargeSpan(after.npages) {
+ h.freelarge.removeSpan(after)
+ } else {
+ h.freeList(after.npages).remove(after)
+ }
+ after.state = _MSpanDead
+ h.spanalloc.free(unsafe.Pointer(after))
}
}
- // Insert s into appropriate list.
- h.freeList(s.npages).insert(s)
+ // Insert s into appropriate list or treap.
+ if h.isLargeSpan(s.npages) {
+ h.freelarge.insert(s)
+ } else {
+ h.freeList(s.npages).insert(s)
+ }
}
func (h *mheap) freeList(npages uintptr) *mSpanList {
- if npages < uintptr(len(h.free)) {
- return &h.free[npages]
- }
- return &h.freelarge
+ return &h.free[npages]
}
func (h *mheap) busyList(npages uintptr) *mSpanList {
return &h.busylarge
}
+func scavengeTreapNode(t *treapNode, now, limit uint64) uintptr {
+ s := t.spanKey
+ var sumreleased uintptr
+ if (now-uint64(s.unusedsince)) > limit && s.npreleased != s.npages {
+ start := s.base()
+ end := start + s.npages<<_PageShift
+ if physPageSize > _PageSize {
+ // We can only release pages in
+ // physPageSize blocks, so round start
+ // and end in. (Otherwise, madvise
+ // will round them *out* and release
+ // more memory than we want.)
+ start = (start + physPageSize - 1) &^ (physPageSize - 1)
+ end &^= physPageSize - 1
+ if end <= start {
+ // start and end don't span a
+ // whole physical page.
+ return sumreleased
+ }
+ }
+ len := end - start
+ released := len - (s.npreleased << _PageShift)
+ if physPageSize > _PageSize && released == 0 {
+ return sumreleased
+ }
+ memstats.heap_released += uint64(released)
+ sumreleased += released
+ s.npreleased = len >> _PageShift
+ sysUnused(unsafe.Pointer(start), len)
+ }
+ return sumreleased
+}
+
func scavengelist(list *mSpanList, now, limit uint64) uintptr {
if list.isEmpty() {
return 0
for i := 0; i < len(h.free); i++ {
sumreleased += scavengelist(&h.free[i], now, limit)
}
- sumreleased += scavengelist(&h.freelarge, now, limit)
+ sumreleased += scavengetreap(h.freelarge.treap, now, limit)
unlock(&h.lock)
gp.m.mallocing--
func (list *mSpanList) remove(span *mspan) {
if span.list != list {
- println("runtime: failed MSpanList_Remove", span, span.prev, span.list, list)
+ print("runtime: failed MSpanList_Remove span.npages=", span.npages,
+ " span=", span, " prev=", span.prev, " span.list=", span.list, " list=", list, "\n")
throw("MSpanList_Remove")
}
if list.first == span {
func (list *mSpanList) insertBack(span *mspan) {
if span.next != nil || span.prev != nil || span.list != nil {
- println("failed MSpanList_InsertBack", span, span.next, span.prev, span.list)
+ println("runtime: failed MSpanList_InsertBack", span, span.next, span.prev, span.list)
throw("MSpanList_InsertBack")
}
span.prev = list.last