--- /dev/null
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// General C malloc/free, but intended for Go.
+// Same design as tcmalloc:
+// see https://www/eng/designdocs/tcmalloc/tcmalloc.html
+
+// TODO:
+// * Central free lists.
+// * Thread cache stealing.
+// * Return memory to the OS.
+// * Memory footprint during testrandom is too big.
+// * Need to coalesce adjacent free spans.
+//
+// *** Some way to avoid the ``malloc overflows the stack
+// during the stack overflow malloc'' problem.
+
+#include "malloc.h"
+
+typedef struct Span Span;
+typedef struct Central Central;
+
+// A Span contains metadata about a range of pages.
+enum {
+ SpanInUse = 0, // span has been handed out by allocator
+ SpanFree = 1, // span is in central free list
+};
+struct Span
+{
+ Span *next; // in free lists
+ byte *base; // first byte in span
+ uintptr length; // number of pages in span
+ int32 cl;
+ int32 state; // state (enum above)
+// int ref; // reference count if state == SpanInUse (for GC)
+// void *type; // object type if state == SpanInUse (for GC)
+};
+
+// The Central cache contains a list of free spans,
+// as well as free lists of small blocks.
+struct Central
+{
+ Lock;
+ Span *free[256];
+ Span *large; // free spans >= MaxPage pages
+};
+
+static Central central;
+static PageMap spanmap;
+
+// Insert a new span into the map.
+static void
+insertspan(Span *s)
+{
+ int32 i;
+ uintptr base;
+
+ // TODO: This is likely too slow for large spans.
+ base = (uintptr)s->base >> PageShift;
+ for(i=0; i<s->length; i++)
+ pminsert(&spanmap, base+i, s);
+}
+
+// Record that a span has gotten smaller.
+static void
+shrinkspan(Span *s, int32 newlength)
+{
+ int32 i;
+ uintptr base;
+
+ // TODO: This is unnecessary, because an insertspan is next.
+ base = (uintptr)s->base >> PageShift;
+ for(i=newlength; i<s->length; i++)
+ pminsert(&spanmap, base+i, nil);
+
+ s->length = newlength;
+}
+
+// Find the span for a given pointer.
+static Span*
+spanofptr(void *v)
+{
+ return pmlookup(&spanmap, (uintptr)v >> PageShift);
+}
+
+static void freespan(Span*);
+
+// Allocate a span of at least n pages.
+static Span*
+allocspan(int32 npage)
+{
+ Span *s, **l, *s1;
+ int32 allocnpage, i;
+
+ // Look in the n-page free lists for big enough n.
+ for(i=npage; i<nelem(central.free); i++) {
+ s = central.free[i];
+ if(s != nil) {
+ central.free[i] = s->next;
+ goto havespan;
+ }
+ }
+
+ // Look in the large list, which has large runs of pages.
+ for(l=¢ral.large; (s=*l) != nil; l=&s->next) {
+ if(s->length >= npage) {
+ *l = s->next;
+ s->next = nil;
+if(s->length > npage) {
+prints("Chop span");
+sys·printint(s->length);
+prints(" for ");
+sys·printint(npage);
+prints("\n");
+}
+ goto havespan;
+ }
+ }
+
+ // Otherwise we need more memory.
+ // TODO: Could try to release this lock while asking for memory.
+ s = trivalloc(sizeof *s);
+ allocnpage = npage;
+ if(allocnpage < (1<<20>>PageShift)) // TODO: Tune
+ allocnpage = (1<<20>>PageShift);
+ s->length = allocnpage;
+prints("New span ");
+sys·printint(allocnpage);
+prints(" for ");
+sys·printint(npage);
+prints("\n");
+ s->base = trivalloc(allocnpage<<PageShift);
+ insertspan(s);
+
+havespan:
+ // If span is bigger than needed, redistribute the remainder.
+ if(s->length > npage) {
+ s1 = trivalloc(sizeof *s);
+ s1->base = s->base + (npage << PageShift);
+ s1->length = s->length - npage;
+ shrinkspan(s, npage);
+ insertspan(s1);
+ freespan(s1);
+ }
+ s->state = SpanInUse;
+ return s;
+}
+
+// Free a span.
+static void
+freespan(Span *s)
+{
+ Span **l;
+ Span *ss;
+
+ s->state = SpanFree;
+ if(s->length < nelem(central.free)) {
+ s->next = central.free[s->length];
+ central.free[s->length] = s;
+ } else {
+ // Keep central.large sorted in
+ // increasing size for best-fit allocation.
+ for(l = ¢ral.large; (ss=*l) != nil; l=&ss->next)
+ if(ss->length >= s->length)
+ break;
+ s->next = *l;
+ *l = s;
+ }
+}
+
+// Small objects are kept on per-size free lists in the M.
+// There are SmallFreeClasses (defined in runtime.h) different lists.
+static int32 classtosize[SmallFreeClasses] = {
+ /*
+ seq 8 8 127 | sed 's/$/,/' | fmt
+ seq 128 16 255 | sed 's/$/,/' | fmt
+ seq 256 32 511 | sed 's/$/,/' | fmt
+ seq 512 64 1023 | sed 's/$/,/' | fmt
+ seq 1024 128 2047 | sed 's/$/,/' | fmt
+ seq 2048 256 32768 | sed 's/$/,/' | fmt
+ */
+ 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120,
+ 128, 144, 160, 176, 192, 208, 224, 240,
+ 256, 288, 320, 352, 384, 416, 448, 480,
+ 512, 576, 640, 704, 768, 832, 896, 960,
+ 1024, 1152, 1280, 1408, 1536, 1664, 1792, 1920,
+ 2048, 2304, 2560, 2816, 3072, 3328, 3584, 3840, 4096, 4352, 4608,
+ 4864, 5120, 5376, 5632, 5888, 6144, 6400, 6656, 6912, 7168, 7424,
+ 7680, 7936, 8192, 8448, 8704, 8960, 9216, 9472, 9728, 9984, 10240,
+ 10496, 10752, 11008, 11264, 11520, 11776, 12032, 12288, 12544,
+ 12800, 13056, 13312, 13568, 13824, 14080, 14336, 14592, 14848,
+ 15104, 15360, 15616, 15872, 16128, 16384, 16640, 16896, 17152,
+ 17408, 17664, 17920, 18176, 18432, 18688, 18944, 19200, 19456,
+ 19712, 19968, 20224, 20480, 20736, 20992, 21248, 21504, 21760,
+ 22016, 22272, 22528, 22784, 23040, 23296, 23552, 23808, 24064,
+ 24320, 24576, 24832, 25088, 25344, 25600, 25856, 26112, 26368,
+ 26624, 26880, 27136, 27392, 27648, 27904, 28160, 28416, 28672,
+ 28928, 29184, 29440, 29696, 29952, 30208, 30464, 30720, 30976,
+ 31232, 31488, 31744, 32000, 32256, 32512, 32768,
+};
+enum {
+ LargeSize = 32768
+};
+
+// Trigger compile error if nelem(classtosize) != SmallFreeClasses.
+static int32 zzz1[SmallFreeClasses-nelem(classtosize)+1];
+static int32 zzz2[nelem(classtosize)-SmallFreeClasses+1];
+
+static int32
+sizetoclass(int32 siz)
+{
+ if(siz <= 0)
+ return 0;
+ if(siz <= 128)
+ return (siz-1) >> 3;
+ if(siz <= 256)
+ return ((siz-1) >> 4) + 8;
+ if(siz <= 512)
+ return ((siz-1) >> 5) + 16;
+ if(siz <= 1024)
+ return ((siz-1) >> 6) + 24;
+ if(siz <= 2048)
+ return ((siz-1) >> 7) + 32;
+ if(siz <= 32768)
+ return ((siz-1) >> 8) + 40;
+ throw("sizetoclass - invalid size");
+ return -1;
+}
+
+void
+allocator·testsizetoclass(void)
+{
+ int32 i, n;
+
+ n = 0;
+ for(i=0; i<nelem(classtosize); i++) {
+ for(; n <= classtosize[i]; n++) {
+ if(sizetoclass(n) != i) {
+ prints("sizetoclass ");
+ sys·printint(n);
+ prints(" = ");
+ sys·printint(sizetoclass(n));
+ prints(" want ");
+ sys·printint(i);
+ prints("\n");
+ throw("testsizetoclass");
+ }
+ }
+ }
+ if (n != 32768+1) {
+ prints("testsizetoclass stopped at ");
+ sys·printint(n);
+ prints("\n");
+ throw("testsizetoclass");
+ }
+}
+
+// Grab a bunch of objects of size class cl off the central free list.
+// Set *pn to the number of objects returned.
+static void*
+centralgrab(int32 cl, int32 *pn)
+{
+ byte *p;
+ Span *s;
+ int32 chunk, i, n, siz;
+
+ // For now there is no central free list.
+ // Fall back to allocating a new span
+ // and chopping it up.
+ chunk = classtosize[cl] * 1024;
+ if(chunk > 1<<20) {
+ chunk = 1<<20;
+ }
+ chunk = (chunk+PageMask) & ~PageMask;
+ s = allocspan(chunk>>PageShift);
+prints("New Class ");
+sys·printint(cl);
+prints("\n");
+ s->state = SpanInUse;
+ s->cl = cl;
+ siz = classtosize[cl];
+ n = chunk/siz;
+ p = s->base;
+ for(i=0; i<n-1; i++) {
+ *(void**)p = p+siz;
+ p += siz;
+ }
+ *pn = n;
+ return p;
+}
+
+// Allocate a small object of size class cl.
+void*
+allocsmall(int32 cl)
+{
+ void **p;
+ int32 n;
+
+ if(cl < 0 || cl >= SmallFreeClasses)
+ throw("allocsmall - invalid class");
+
+ // try m-local cache.
+ p = m->freelist[cl];
+ if(p == nil) {
+ // otherwise grab some blocks from central cache.
+ lock(¢ral);
+ p = centralgrab(cl, &n);
+ // TODO: update local counters using n
+ unlock(¢ral);
+ }
+
+ // advance linked list.
+ m->freelist[cl] = *p;
+
+ // Blocks on free list are zeroed except for
+ // the linked list pointer that we just used. Zero it.
+ *p = 0;
+
+ return p;
+}
+
+// Allocate large object of np pages.
+void*
+alloclarge(int32 np)
+{
+ Span *s;
+
+ lock(¢ral);
+//prints("Alloc span ");
+//sys·printint(np);
+//prints("\n");
+ s = allocspan(np);
+ unlock(¢ral);
+ s->state = SpanInUse;
+ s->cl = -1;
+ return s->base;
+}
+
+// Allocate object of n bytes.
+void*
+alloc(int32 n)
+{
+ int32 cl, np;
+
+ if(n < LargeSize) {
+ cl = sizetoclass(n);
+ if(cl < 0 || cl >= SmallFreeClasses) {
+ sys·printint(n);
+ prints(" -> ");
+ sys·printint(cl);
+ prints("\n");
+ throw("alloc - logic error");
+ }
+ return allocsmall(sizetoclass(n));
+ }
+
+ // count number of pages; careful about overflow for big n.
+ np = (n>>PageShift) + (((n&PageMask)+PageMask)>>PageShift);
+ return alloclarge(np);
+}
+
+void
+allocator·malloc(int32 n, byte *out)
+{
+ out = alloc(n);
+ FLUSH(&out);
+}
+
+// Free object with base pointer v.
+void
+free(void *v)
+{
+ void **p;
+ Span *s;
+ int32 siz, off;
+
+ s = spanofptr(v);
+ if(s->state != SpanInUse)
+ throw("free - invalid pointer1");
+
+ // Big object should be s->base.
+ if(s->cl < 0) {
+ if(v != s->base)
+ throw("free - invalid pointer2");
+ // TODO: For large spans, maybe just return the
+ // memory to the operating system and let it zero it.
+ sys·memclr(s->base, s->length << PageShift);
+//prints("Free big ");
+//sys·printint(s->length);
+//prints("\n");
+ lock(¢ral);
+ freespan(s);
+ unlock(¢ral);
+ return;
+ }
+
+ // Small object should be aligned properly.
+ siz = classtosize[s->cl];
+ off = (byte*)v - (byte*)s->base;
+ if(off%siz)
+ throw("free - invalid pointer3");
+
+ // Zero and add to free list.
+ sys·memclr(v, siz);
+ p = v;
+ *p = m->freelist[s->cl];
+ m->freelist[s->cl] = p;
+}
+
+void
+allocator·free(byte *v)
+{
+ free(v);
+}
+
+void
+allocator·memset(byte *v, int32 c, int32 n)
+{
+ int32 i;
+
+ for(i=0; i<n; i++)
+ v[i] = c;
+}
+
+// Allocate stack segment.
+// Must be done without holding locks, because
+// calling any function might trigger another stack segment allocation.
+void*
+allocstack(int32 n)
+{
+ // TODO
+ USED(n);
+ return nil;
+}
+
+void
+freestack(void *v)
+{
+ // TODO
+ USED(v);
+}
+