int8 runtime·size_to_class8[1024/8 + 1];
int8 runtime·size_to_class128[(MaxSmallSize-1024)/128 + 1];
-static int32
-SizeToClass(int32 size)
+void runtime·testdefersizes(void);
+
+int32
+runtime·SizeToClass(int32 size)
{
if(size > MaxSmallSize)
runtime·throw("SizeToClass - invalid size");
// Double-check SizeToClass.
if(0) {
for(n=0; n < MaxSmallSize; n++) {
- sizeclass = SizeToClass(n);
+ sizeclass = runtime·SizeToClass(n);
if(sizeclass < 1 || sizeclass >= NumSizeClasses || runtime·class_to_size[sizeclass] < n) {
runtime·printf("size=%d sizeclass=%d runtime·class_to_size=%d\n", n, sizeclass, runtime·class_to_size[sizeclass]);
runtime·printf("incorrect SizeToClass");
}
}
+ runtime·testdefersizes();
+
// Copy out for statistics table.
for(i=0; i<nelem(runtime·class_to_size); i++)
mstats.by_size[i].size = runtime·class_to_size[i];
uint32 runtime·panicking;
static Lock paniclk;
-enum
-{
- DeferChunkSize = 2048
-};
+// Each P holds pool for defers with arg sizes 8, 24, 40, 56 and 72 bytes.
+// Memory block is 40 (24 for 32 bits) bytes larger due to Defer header.
+// This maps exactly to malloc size classes.
+
+// defer size class for arg size sz
+#define DEFERCLASS(sz) (((sz)+7)>>4)
+// total size of memory block for defer with arg size sz
+#define TOTALSIZE(sz) (sizeof(Defer) - sizeof(((Defer*)nil)->args) + ROUND(sz, sizeof(uintptr)))
-// Allocate a Defer, usually as part of the larger frame of deferred functions.
-// Each defer must be released with both popdefer and freedefer.
+// Allocate a Defer, usually using per-P pool.
+// Each defer must be released with freedefer.
static Defer*
newdefer(int32 siz)
{
- int32 total;
- DeferChunk *c;
+ int32 total, sc;
Defer *d;
-
- c = g->dchunk;
- total = sizeof(*d) + ROUND(siz, sizeof(uintptr)) - sizeof(d->args);
- if(c == nil || total > DeferChunkSize - c->off) {
- if(total > DeferChunkSize / 2) {
- // Not worth putting in any chunk.
- // Allocate a separate block.
- d = runtime·malloc(total);
- d->siz = siz;
- d->special = 1;
- d->free = 1;
- d->link = g->defer;
- g->defer = d;
- return d;
- }
-
- // Cannot fit in current chunk.
- // Switch to next chunk, allocating if necessary.
- c = g->dchunknext;
- if(c == nil)
- c = runtime·malloc(DeferChunkSize);
- c->prev = g->dchunk;
- c->off = sizeof(*c);
- g->dchunk = c;
- g->dchunknext = nil;
+ P *p;
+
+ d = nil;
+ sc = DEFERCLASS(siz);
+ if(sc < nelem(p->deferpool)) {
+ p = m->p;
+ d = p->deferpool[sc];
+ if(d)
+ p->deferpool[sc] = d->link;
+ }
+ if(d == nil) {
+ // deferpool is empty or just a big defer
+ total = TOTALSIZE(siz);
+ d = runtime·malloc(total);
}
-
- d = (Defer*)((byte*)c + c->off);
- c->off += total;
d->siz = siz;
d->special = 0;
- d->free = 0;
d->link = g->defer;
g->defer = d;
- return d;
-}
-
-// Pop the current defer from the defer stack.
-// Its contents are still valid until the goroutine begins executing again.
-// In particular it is safe to call reflect.call(d->fn, d->argp, d->siz) after
-// popdefer returns.
-static void
-popdefer(void)
-{
- Defer *d;
- DeferChunk *c;
- int32 total;
-
- d = g->defer;
- if(d == nil)
- runtime·throw("runtime: popdefer nil");
- g->defer = d->link;
- if(d->special) {
- // Nothing else to do.
- return;
- }
- total = sizeof(*d) + ROUND(d->siz, sizeof(uintptr)) - sizeof(d->args);
- c = g->dchunk;
- if(c == nil || (byte*)d+total != (byte*)c+c->off)
- runtime·throw("runtime: popdefer phase error");
- c->off -= total;
- if(c->off == sizeof(*c)) {
- // Chunk now empty, so pop from stack.
- // Save in dchunknext both to help with pingponging between frames
- // and to make sure d is still valid on return.
- if(g->dchunknext != nil)
- runtime·free(g->dchunknext);
- g->dchunknext = c;
- g->dchunk = c->prev;
- }
+ return d;
}
// Free the given defer.
-// For defers in the per-goroutine chunk this just clears the saved arguments.
-// For large defers allocated on the heap, this frees them.
// The defer cannot be used after this call.
static void
freedefer(Defer *d)
{
- int32 total;
+ int32 sc;
+ P *p;
- if(d->special) {
- if(d->free)
- runtime·free(d);
- } else {
- // Wipe out any possible pointers in argp/pc/fn/args.
- total = sizeof(*d) + ROUND(d->siz, sizeof(uintptr)) - sizeof(d->args);
- runtime·memclr((byte*)d, total);
- }
+ if(d->special)
+ return;
+ sc = DEFERCLASS(d->siz);
+ if(sc < nelem(p->deferpool)) {
+ p = m->p;
+ d->link = p->deferpool[sc];
+ p->deferpool[sc] = d;
+ // No need to wipe out pointers in argp/pc/fn/args,
+ // because we empty the pool before GC.
+ } else
+ runtime·free(d);
}
// Create a new deferred function fn with siz bytes of arguments.
m->locks++;
runtime·memmove(argp, d->args, d->siz);
fn = d->fn;
- popdefer();
+ g->defer = d->link;
freedefer(d);
m->locks--;
if(m->locks == 0 && g->preempt)
runtime·jmpdefer(fn, argp);
}
+// Ensure that defer arg sizes that map to the same defer size class
+// also map to the same malloc size class.
+void
+runtime·testdefersizes(void)
+{
+ P *p;
+ int32 i, siz, defersc, mallocsc;
+ int32 map[nelem(p->deferpool)];
+
+ for(i=0; i<nelem(p->deferpool); i++)
+ map[i] = -1;
+ for(i=0;; i++) {
+ defersc = DEFERCLASS(i);
+ if(defersc >= nelem(p->deferpool))
+ break;
+ siz = TOTALSIZE(i);
+ mallocsc = runtime·SizeToClass(siz);
+ siz = runtime·class_to_size[mallocsc];
+ // runtime·printf("defer class %d: arg size %d, block size %d(%d)\n", defersc, i, siz, mallocsc);
+ if(map[defersc] < 0) {
+ map[defersc] = mallocsc;
+ continue;
+ }
+ if(map[defersc] != mallocsc) {
+ runtime·printf("bad defer size class: i=%d siz=%d mallocsc=%d/%d\n",
+ i, siz, map[defersc], mallocsc);
+ runtime·throw("bad defer size class");
+ }
+ }
+}
+
// Run all deferred functions for the current goroutine.
static void
rundefer(void)
Defer *d;
while((d = g->defer) != nil) {
- popdefer();
+ g->defer = d->link;
reflect·call(d->fn, (byte*)d->args, d->siz);
freedefer(d);
}
if(d == nil)
break;
// take defer off list in case of recursive panic
- popdefer();
+ g->defer = d->link;
g->ispanic = true; // rock for newstack, where reflect.newstackcall ends up
argp = d->argp;
pc = d->pc;