runtime: fix memory allocator on Plan 9

Previously, the memory allocator on Plan 9 did
not free memory properly. It was only able to
free the last allocated block.

This change implements a variant of the
Kernighan & Ritchie memory allocator with
coalescing and splitting.

The most notable differences are:

- no header is prefixing the allocated blocks, since
  the size is always specified when calling sysFree,
- the free list is nil-terminated instead of circular.

Fixes #9736.
Fixes #9803.
Fixes #9952.

Change-Id: I00d533714e4144a0012f69820d31cbb0253031a3
Reviewed-on: https://go-review.googlesource.com/5524
Reviewed-by: Dmitry Vyukov <dvyukov@google.com>

diff --git a/src/runtime/mem_plan9.go b/src/runtime/mem_plan9.go
index 6ceed25d87972b85489a5a5cbc3b009773f10c67..bf7d23881f974d0d951ab00c051a562f7d735223 100644 (file)
--- a/src/runtime/mem_plan9.go
+++ b/src/runtime/mem_plan9.go
@@ -6,9 +6,111 @@ package runtime
 
 import "unsafe"
 
+const memDebug = false
+
 var bloc uintptr
 var memlock mutex
 
+type memHdr struct {
+       next *memHdr
+       size uintptr
+}
+
+var memFreelist *memHdr // sorted in ascending order
+
+func memAlloc(n uintptr) unsafe.Pointer {
+       n = memRound(n)
+       var prevp *memHdr
+       for p := memFreelist; p != nil; p = p.next {
+               if p.size >= n {
+                       if p.size == n {
+                               if prevp != nil {
+                                       prevp.next = p.next
+                               } else {
+                                       memFreelist = p.next
+                               }
+                       } else {
+                               p.size -= n
+                               p = (*memHdr)(add(unsafe.Pointer(p), p.size))
+                       }
+                       memclr(unsafe.Pointer(p), unsafe.Sizeof(memHdr{}))
+                       return unsafe.Pointer(p)
+               }
+               prevp = p
+       }
+       return sbrk(n)
+}
+
+func memFree(ap unsafe.Pointer, n uintptr) {
+       n = memRound(n)
+       memclr(ap, n)
+       bp := (*memHdr)(ap)
+       bp.size = n
+       bpn := uintptr(ap)
+       if memFreelist == nil {
+               bp.next = nil
+               memFreelist = bp
+               return
+       }
+       p := memFreelist
+       if bpn < uintptr(unsafe.Pointer(p)) {
+               memFreelist = bp
+               if bpn+bp.size == uintptr(unsafe.Pointer(p)) {
+                       bp.size += p.size
+                       bp.next = p.next
+                       memclr(unsafe.Pointer(p), unsafe.Sizeof(memHdr{}))
+               } else {
+                       bp.next = p
+               }
+               return
+       }
+       for ; p.next != nil; p = p.next {
+               if bpn > uintptr(unsafe.Pointer(p)) && bpn < uintptr(unsafe.Pointer(p.next)) {
+                       break
+               }
+       }
+       if bpn+bp.size == uintptr(unsafe.Pointer(p.next)) {
+               bp.size += p.next.size
+               bp.next = p.next.next
+               memclr(unsafe.Pointer(p.next), unsafe.Sizeof(memHdr{}))
+       } else {
+               bp.next = p.next
+       }
+       if uintptr(unsafe.Pointer(p))+p.size == bpn {
+               p.size += bp.size
+               p.next = bp.next
+               memclr(unsafe.Pointer(bp), unsafe.Sizeof(memHdr{}))
+       } else {
+               p.next = bp
+       }
+}
+
+func memCheck() {
+       if memDebug == false {
+               return
+       }
+       for p := memFreelist; p != nil && p.next != nil; p = p.next {
+               if uintptr(unsafe.Pointer(p)) == uintptr(unsafe.Pointer(p.next)) {
+                       print("runtime: ", unsafe.Pointer(p), " == ", unsafe.Pointer(p.next), "\n")
+                       throw("mem: infinite loop")
+               }
+               if uintptr(unsafe.Pointer(p)) > uintptr(unsafe.Pointer(p.next)) {
+                       print("runtime: ", unsafe.Pointer(p), " > ", unsafe.Pointer(p.next), "\n")
+                       throw("mem: unordered list")
+               }
+               if uintptr(unsafe.Pointer(p))+p.size > uintptr(unsafe.Pointer(p.next)) {
+                       print("runtime: ", unsafe.Pointer(p), "+", p.size, " > ", unsafe.Pointer(p.next), "\n")
+                       throw("mem: overlapping blocks")
+               }
+               for b := add(unsafe.Pointer(p), unsafe.Sizeof(memHdr{})); uintptr(b) < uintptr(unsafe.Pointer(p))+p.size; b = add(b, 1) {
+                       if *(*byte)(b) != 0 {
+                               print("runtime: value at addr ", b, " with offset ", uintptr(b)-uintptr(unsafe.Pointer(p)), " in block ", p, " of size ", p.size, " is not zero\n")
+                               throw("mem: uninitialised memory")
+                       }
+               }
+       }
+}
+
 func memRound(p uintptr) uintptr {
        return (p + _PAGESIZE - 1) &^ (_PAGESIZE - 1)
 }
@@ -18,21 +120,21 @@ func initBloc() {
 }
 
 func sbrk(n uintptr) unsafe.Pointer {
-       lock(&memlock)
        // Plan 9 sbrk from /sys/src/libc/9sys/sbrk.c
        bl := bloc
        n = memRound(n)
        if brk_(unsafe.Pointer(bl+n)) < 0 {
-               unlock(&memlock)
                return nil
        }
        bloc += n
-       unlock(&memlock)
        return unsafe.Pointer(bl)
 }
 
 func sysAlloc(n uintptr, stat *uint64) unsafe.Pointer {
-       p := sbrk(n)
+       lock(&memlock)
+       p := memAlloc(n)
+       memCheck()
+       unlock(&memlock)
        if p != nil {
                xadd64(stat, int64(n))
        }
@@ -42,14 +144,8 @@ func sysAlloc(n uintptr, stat *uint64) unsafe.Pointer {
 func sysFree(v unsafe.Pointer, n uintptr, stat *uint64) {
        xadd64(stat, -int64(n))
        lock(&memlock)
-       // from tiny/mem.c
-       // Push pointer back if this is a free
-       // of the most recent sysAlloc.
-       n = memRound(n)
-       if bloc == uintptr(v)+n {
-               bloc -= n
-               memclr(unsafe.Pointer(bloc), n)
-       }
+       memFree(v, n)
+       memCheck()
        unlock(&memlock)
 }
 
@@ -70,5 +166,9 @@ func sysFault(v unsafe.Pointer, n uintptr) {
 
 func sysReserve(v unsafe.Pointer, n uintptr, reserved *bool) unsafe.Pointer {
        *reserved = true
-       return sbrk(n)
+       lock(&memlock)
+       p := memAlloc(n)
+       memCheck()
+       unlock(&memlock)
+       return p
 }