out of malloc proper.
TBR=r
OCL=26689
CL=26689
int32 MCentral_AllocList(MCentral *c, int32 n, MLink **first);
void MCentral_FreeList(MCentral *c, int32 n, MLink *first);
-
-// Free(v) must be able to determine the MSpan containing v.
-// The MHeapMap is a 3-level radix tree mapping page numbers to MSpans.
-//
-// NOTE(rsc): On a 32-bit platform (= 20-bit page numbers),
-// we can swap in a 2-level radix tree.
-//
-// NOTE(rsc): We use a 3-level tree because tcmalloc does, but
-// having only three levels requires approximately 1 MB per node
-// in the tree, making the minimum map footprint 3 MB.
-// Using a 4-level tree would cut the minimum footprint to 256 kB.
-// On the other hand, it's just virtual address space: most of
-// the memory is never going to be touched, thus never paged in.
-
-typedef struct MHeapMapNode2 MHeapMapNode2;
-typedef struct MHeapMapNode3 MHeapMapNode3;
-
-enum
-{
- // 64 bit address - 12 bit page size = 52 bits to map
- MHeapMap_Level1Bits = 18,
- MHeapMap_Level2Bits = 18,
- MHeapMap_Level3Bits = 16,
-
- MHeapMap_TotalBits =
- MHeapMap_Level1Bits +
- MHeapMap_Level2Bits +
- MHeapMap_Level3Bits,
-
- MHeapMap_Level1Mask = (1<<MHeapMap_Level1Bits) - 1,
- MHeapMap_Level2Mask = (1<<MHeapMap_Level2Bits) - 1,
- MHeapMap_Level3Mask = (1<<MHeapMap_Level3Bits) - 1,
-};
-
-struct MHeapMap
-{
- void *(*allocator)(uintptr);
- MHeapMapNode2 *p[1<<MHeapMap_Level1Bits];
-};
-
-struct MHeapMapNode2
-{
- MHeapMapNode3 *p[1<<MHeapMap_Level2Bits];
-};
-
-struct MHeapMapNode3
-{
- MSpan *s[1<<MHeapMap_Level3Bits];
-};
-
-void MHeapMap_Init(MHeapMap *m, void *(*allocator)(uintptr));
-bool MHeapMap_Preallocate(MHeapMap *m, PageID k, uintptr npages);
-MSpan* MHeapMap_Get(MHeapMap *m, PageID k);
-MSpan* MHeapMap_GetMaybe(MHeapMap *m, PageID k);
-void MHeapMap_Set(MHeapMap *m, PageID k, MSpan *v);
-
-
-// Much of the time, free(v) needs to know only the size class for v,
-// not which span it came from. The MHeapMap finds the size class
-// by looking up the span.
-//
-// An MHeapMapCache is a simple direct-mapped cache translating
-// page numbers to size classes. It avoids the expensive MHeapMap
-// lookup for hot pages.
-//
-// The cache entries are 64 bits, with the page number in the low part
-// and the value at the top.
-//
-// NOTE(rsc): On a machine with 32-bit addresses (= 20-bit page numbers),
-// we can use a 16-bit cache entry by not storing the redundant 12 bits
-// of the key that are used as the entry index. Here in 64-bit land,
-// that trick won't work unless the hash table has 2^28 entries.
-enum
-{
- MHeapMapCache_HashBits = 12
-};
-
-struct MHeapMapCache
-{
- uintptr array[1<<MHeapMapCache_HashBits];
-};
-
-// All macros for speed (sorry).
-#define HMASK ((1<<MHeapMapCache_HashBits)-1)
-#define KBITS MHeapMap_TotalBits
-#define KMASK ((1LL<<KBITS)-1)
-
-#define MHeapMapCache_SET(cache, key, value) \
- ((cache)->array[(key) & HMASK] = (key) | ((uintptr)(value) << KBITS))
-
-#define MHeapMapCache_GET(cache, key, tmp) \
- (tmp = (cache)->array[(key) & HMASK], \
- (tmp & KMASK) == (key) ? (tmp >> KBITS) : 0)
-
+#include "mheapmap64.h"
// Main malloc heap.
// The heap itself is the "free[]" and "large" arrays,
// TODO(rsc): IncrementalScavenge() to return memory to OS.
}
-// 3-level radix tree mapping page ids to Span*.
-void
-MHeapMap_Init(MHeapMap *m, void *(*allocator)(size_t))
-{
- m->allocator = allocator;
-}
-
-MSpan*
-MHeapMap_Get(MHeapMap *m, PageID k)
-{
- int32 i1, i2, i3;
-
- i3 = k & MHeapMap_Level3Mask;
- k >>= MHeapMap_Level3Bits;
- i2 = k & MHeapMap_Level2Mask;
- k >>= MHeapMap_Level2Bits;
- i1 = k & MHeapMap_Level1Mask;
- k >>= MHeapMap_Level1Bits;
- if(k != 0)
- throw("MHeapMap_Get");
-
- return m->p[i1]->p[i2]->s[i3];
-}
-
-MSpan*
-MHeapMap_GetMaybe(MHeapMap *m, PageID k)
-{
- int32 i1, i2, i3;
- MHeapMapNode2 *p2;
- MHeapMapNode3 *p3;
-
- i3 = k & MHeapMap_Level3Mask;
- k >>= MHeapMap_Level3Bits;
- i2 = k & MHeapMap_Level2Mask;
- k >>= MHeapMap_Level2Bits;
- i1 = k & MHeapMap_Level1Mask;
- k >>= MHeapMap_Level1Bits;
- if(k != 0)
- throw("MHeapMap_Get");
-
- p2 = m->p[i1];
- if(p2 == nil)
- return nil;
- p3 = p2->p[i2];
- if(p3 == nil)
- return nil;
- return p3->s[i3];
-}
-
-void
-MHeapMap_Set(MHeapMap *m, PageID k, MSpan *s)
-{
- int32 i1, i2, i3;
-
- i3 = k & MHeapMap_Level3Mask;
- k >>= MHeapMap_Level3Bits;
- i2 = k & MHeapMap_Level2Mask;
- k >>= MHeapMap_Level2Bits;
- i1 = k & MHeapMap_Level1Mask;
- k >>= MHeapMap_Level1Bits;
- if(k != 0)
- throw("MHeapMap_Set");
-
- m->p[i1]->p[i2]->s[i3] = s;
-}
-
-// Allocate the storage required for entries [k, k+1, ..., k+len-1]
-// so that Get and Set calls need not check for nil pointers.
-bool
-MHeapMap_Preallocate(MHeapMap *m, PageID k, uintptr len)
-{
- uintptr end;
- int32 i1, i2;
- MHeapMapNode2 *p2;
- MHeapMapNode3 *p3;
-
- end = k+len;
- while(k < end) {
- if((k >> MHeapMap_TotalBits) != 0)
- return false;
- i2 = (k >> MHeapMap_Level3Bits) & MHeapMap_Level2Mask;
- i1 = (k >> (MHeapMap_Level3Bits + MHeapMap_Level2Bits)) & MHeapMap_Level1Mask;
-
- // first-level pointer
- if((p2 = m->p[i1]) == nil) {
- p2 = m->allocator(sizeof *p2);
- if(p2 == nil)
- return false;
- sys_memclr((byte*)p2, sizeof *p2);
- m->p[i1] = p2;
- }
-
- // second-level pointer
- if(p2->p[i2] == nil) {
- p3 = m->allocator(sizeof *p3);
- if(p3 == nil)
- return false;
- sys_memclr((byte*)p3, sizeof *p3);
- p2->p[i2] = p3;
- }
-
- // advance key past this leaf node
- k = ((k >> MHeapMap_Level3Bits) + 1) << MHeapMap_Level3Bits;
- }
- return true;
-}
-
// Initialize a new span with the given start and npages.
void
MSpan_Init(MSpan *span, PageID start, uintptr npages)
--- /dev/null
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Heap map, 64-bit version
+// See malloc.h and mheap.c for overview.
+
+#include "runtime.h"
+#include "malloc.h"
+
+// 3-level radix tree mapping page ids to Span*.
+void
+MHeapMap_Init(MHeapMap *m, void *(*allocator)(size_t))
+{
+ m->allocator = allocator;
+}
+
+MSpan*
+MHeapMap_Get(MHeapMap *m, PageID k)
+{
+ int32 i1, i2, i3;
+
+ i3 = k & MHeapMap_Level3Mask;
+ k >>= MHeapMap_Level3Bits;
+ i2 = k & MHeapMap_Level2Mask;
+ k >>= MHeapMap_Level2Bits;
+ i1 = k & MHeapMap_Level1Mask;
+ k >>= MHeapMap_Level1Bits;
+ if(k != 0)
+ throw("MHeapMap_Get");
+
+ return m->p[i1]->p[i2]->s[i3];
+}
+
+MSpan*
+MHeapMap_GetMaybe(MHeapMap *m, PageID k)
+{
+ int32 i1, i2, i3;
+ MHeapMapNode2 *p2;
+ MHeapMapNode3 *p3;
+
+ i3 = k & MHeapMap_Level3Mask;
+ k >>= MHeapMap_Level3Bits;
+ i2 = k & MHeapMap_Level2Mask;
+ k >>= MHeapMap_Level2Bits;
+ i1 = k & MHeapMap_Level1Mask;
+ k >>= MHeapMap_Level1Bits;
+ if(k != 0)
+ throw("MHeapMap_Get");
+
+ p2 = m->p[i1];
+ if(p2 == nil)
+ return nil;
+ p3 = p2->p[i2];
+ if(p3 == nil)
+ return nil;
+ return p3->s[i3];
+}
+
+void
+MHeapMap_Set(MHeapMap *m, PageID k, MSpan *s)
+{
+ int32 i1, i2, i3;
+
+ i3 = k & MHeapMap_Level3Mask;
+ k >>= MHeapMap_Level3Bits;
+ i2 = k & MHeapMap_Level2Mask;
+ k >>= MHeapMap_Level2Bits;
+ i1 = k & MHeapMap_Level1Mask;
+ k >>= MHeapMap_Level1Bits;
+ if(k != 0)
+ throw("MHeapMap_Set");
+
+ m->p[i1]->p[i2]->s[i3] = s;
+}
+
+// Allocate the storage required for entries [k, k+1, ..., k+len-1]
+// so that Get and Set calls need not check for nil pointers.
+bool
+MHeapMap_Preallocate(MHeapMap *m, PageID k, uintptr len)
+{
+ uintptr end;
+ int32 i1, i2;
+ MHeapMapNode2 *p2;
+ MHeapMapNode3 *p3;
+
+ end = k+len;
+ while(k < end) {
+ if((k >> MHeapMap_TotalBits) != 0)
+ return false;
+ i2 = (k >> MHeapMap_Level3Bits) & MHeapMap_Level2Mask;
+ i1 = (k >> (MHeapMap_Level3Bits + MHeapMap_Level2Bits)) & MHeapMap_Level1Mask;
+
+ // first-level pointer
+ if((p2 = m->p[i1]) == nil) {
+ p2 = m->allocator(sizeof *p2);
+ if(p2 == nil)
+ return false;
+ sys_memclr((byte*)p2, sizeof *p2);
+ m->p[i1] = p2;
+ }
+
+ // second-level pointer
+ if(p2->p[i2] == nil) {
+ p3 = m->allocator(sizeof *p3);
+ if(p3 == nil)
+ return false;
+ sys_memclr((byte*)p3, sizeof *p3);
+ p2->p[i2] = p3;
+ }
+
+ // advance key past this leaf node
+ k = ((k >> MHeapMap_Level3Bits) + 1) << MHeapMap_Level3Bits;
+ }
+ return true;
+}
+
--- /dev/null
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Free(v) must be able to determine the MSpan containing v.
+// The MHeapMap is a 3-level radix tree mapping page numbers to MSpans.
+//
+// NOTE(rsc): On a 32-bit platform (= 20-bit page numbers),
+// we can swap in a 2-level radix tree.
+//
+// NOTE(rsc): We use a 3-level tree because tcmalloc does, but
+// having only three levels requires approximately 1 MB per node
+// in the tree, making the minimum map footprint 3 MB.
+// Using a 4-level tree would cut the minimum footprint to 256 kB.
+// On the other hand, it's just virtual address space: most of
+// the memory is never going to be touched, thus never paged in.
+
+typedef struct MHeapMapNode2 MHeapMapNode2;
+typedef struct MHeapMapNode3 MHeapMapNode3;
+
+enum
+{
+ // 64 bit address - 12 bit page size = 52 bits to map
+ MHeapMap_Level1Bits = 18,
+ MHeapMap_Level2Bits = 18,
+ MHeapMap_Level3Bits = 16,
+
+ MHeapMap_TotalBits =
+ MHeapMap_Level1Bits +
+ MHeapMap_Level2Bits +
+ MHeapMap_Level3Bits,
+
+ MHeapMap_Level1Mask = (1<<MHeapMap_Level1Bits) - 1,
+ MHeapMap_Level2Mask = (1<<MHeapMap_Level2Bits) - 1,
+ MHeapMap_Level3Mask = (1<<MHeapMap_Level3Bits) - 1,
+};
+
+struct MHeapMap
+{
+ void *(*allocator)(uintptr);
+ MHeapMapNode2 *p[1<<MHeapMap_Level1Bits];
+};
+
+struct MHeapMapNode2
+{
+ MHeapMapNode3 *p[1<<MHeapMap_Level2Bits];
+};
+
+struct MHeapMapNode3
+{
+ MSpan *s[1<<MHeapMap_Level3Bits];
+};
+
+void MHeapMap_Init(MHeapMap *m, void *(*allocator)(uintptr));
+bool MHeapMap_Preallocate(MHeapMap *m, PageID k, uintptr npages);
+MSpan* MHeapMap_Get(MHeapMap *m, PageID k);
+MSpan* MHeapMap_GetMaybe(MHeapMap *m, PageID k);
+void MHeapMap_Set(MHeapMap *m, PageID k, MSpan *v);
+
+
+// Much of the time, free(v) needs to know only the size class for v,
+// not which span it came from. The MHeapMap finds the size class
+// by looking up the span.
+//
+// An MHeapMapCache is a simple direct-mapped cache translating
+// page numbers to size classes. It avoids the expensive MHeapMap
+// lookup for hot pages.
+//
+// The cache entries are 64 bits, with the page number in the low part
+// and the value at the top.
+//
+// NOTE(rsc): On a machine with 32-bit addresses (= 20-bit page numbers),
+// we can use a 16-bit cache entry by not storing the redundant 12 bits
+// of the key that are used as the entry index. Here in 64-bit land,
+// that trick won't work unless the hash table has 2^28 entries.
+enum
+{
+ MHeapMapCache_HashBits = 12
+};
+
+struct MHeapMapCache
+{
+ uintptr array[1<<MHeapMapCache_HashBits];
+};
+
+// All macros for speed (sorry).
+#define HMASK ((1<<MHeapMapCache_HashBits)-1)
+#define KBITS MHeapMap_TotalBits
+#define KMASK ((1LL<<KBITS)-1)
+
+#define MHeapMapCache_SET(cache, key, value) \
+ ((cache)->array[(key) & HMASK] = (key) | ((uintptr)(value) << KBITS))
+
+#define MHeapMapCache_GET(cache, key, tmp) \
+ (tmp = (cache)->array[(key) & HMASK], \
+ (tmp & KMASK) == (key) ? (tmp >> KBITS) : 0)