runtime: inline size to class conversion in malloc()

Also change table type from int32[] to int8[] to save space in L1$.

benchmark          old ns/op    new ns/op    delta
BenchmarkMalloc           42           40   -4.68%

R=golang-dev, bradfitz, r
CC=golang-dev
https://golang.org/cl/9199044

diff --git a/src/pkg/runtime/malloc.goc b/src/pkg/runtime/malloc.goc
index f1d25a793f82b67f957a6cca34cbb345ae6276d3..5326551fee4f99e39d6663bc37632d714ab8771a 100644 (file)
--- a/src/pkg/runtime/malloc.goc
+++ b/src/pkg/runtime/malloc.goc
@@ -50,7 +50,11 @@ runtime·mallocgc(uintptr size, uint32 flag, int32 dogc, int32 zeroed)
        c->local_nmalloc++;
        if(size <= MaxSmallSize) {
                // Allocate from mcache free lists.
-               sizeclass = runtime·SizeToClass(size);
+               // Inlined version of SizeToClass().
+               if(size <= 1024-8)
+                       sizeclass = runtime·size_to_class8[(size+7)>>3];
+               else
+                       sizeclass = runtime·size_to_class128[(size-1024+127) >> 7];
                size = runtime·class_to_size[sizeclass];
                v = runtime·MCache_Alloc(c, sizeclass, size, zeroed);
                if(v == nil)

diff --git a/src/pkg/runtime/malloc.h b/src/pkg/runtime/malloc.h
index 52b76d5574157309ece8f185043faa3b00b1b406..7474f85258837783d83835ad76a36e6d46f4d71c 100644 (file)
--- a/src/pkg/runtime/malloc.h
+++ b/src/pkg/runtime/malloc.h
@@ -275,6 +275,8 @@ int32       runtime·SizeToClass(int32);
 extern int32   runtime·class_to_size[NumSizeClasses];
 extern int32   runtime·class_to_allocnpages[NumSizeClasses];
 extern int32   runtime·class_to_transfercount[NumSizeClasses];
+extern int8    runtime·size_to_class8[1024/8 + 1];
+extern int8    runtime·size_to_class128[(MaxSmallSize-1024)/128 + 1];
 extern void    runtime·InitSizes(void);
 
 

diff --git a/src/pkg/runtime/msize.c b/src/pkg/runtime/msize.c
index e6cfcdb02d8eb7cc0aa374af7ed2897c2acb58af..a81bc11aaed686aa15ba4707d910a8cbf1aee753 100644 (file)
--- a/src/pkg/runtime/msize.c
+++ b/src/pkg/runtime/msize.c
@@ -42,17 +42,17 @@ int32 runtime·class_to_transfercount[NumSizeClasses];
 // size divided by 128 (rounded up).  The arrays are filled in
 // by InitSizes.
 
-static int32 size_to_class8[1024/8 + 1];
-static int32 size_to_class128[(MaxSmallSize-1024)/128 + 1];
+int8 runtime·size_to_class8[1024/8 + 1];
+int8 runtime·size_to_class128[(MaxSmallSize-1024)/128 + 1];
 
-int32
-runtime·SizeToClass(int32 size)
+static int32
+SizeToClass(int32 size)
 {
        if(size > MaxSmallSize)
                runtime·throw("SizeToClass - invalid size");
        if(size > 1024-8)
-               return size_to_class128[(size-1024+127) >> 7];
-       return size_to_class8[(size+7)>>3];
+               return runtime·size_to_class128[(size-1024+127) >> 7];
+       return runtime·size_to_class8[(size+7)>>3];
 }
 
 void
@@ -111,16 +111,16 @@ runtime·InitSizes(void)
        nextsize = 0;
        for (sizeclass = 1; sizeclass < NumSizeClasses; sizeclass++) {
                for(; nextsize < 1024 && nextsize <= runtime·class_to_size[sizeclass]; nextsize+=8)
-                       size_to_class8[nextsize/8] = sizeclass;
+                       runtime·size_to_class8[nextsize/8] = sizeclass;
                if(nextsize >= 1024)
                        for(; nextsize <= runtime·class_to_size[sizeclass]; nextsize += 128)
-                               size_to_class128[(nextsize-1024)/128] = sizeclass;
+                               runtime·size_to_class128[(nextsize-1024)/128] = sizeclass;
        }
 
        // Double-check SizeToClass.
        if(0) {
                for(n=0; n < MaxSmallSize; n++) {
-                       sizeclass = runtime·SizeToClass(n);
+                       sizeclass = SizeToClass(n);
                        if(sizeclass < 1 || sizeclass >= NumSizeClasses || runtime·class_to_size[sizeclass] < n) {
                                runtime·printf("size=%d sizeclass=%d runtime·class_to_size=%d\n", n, sizeclass, runtime·class_to_size[sizeclass]);
                                runtime·printf("incorrect SizeToClass");
@@ -157,12 +157,14 @@ dump:
                        runtime·printf(" %d", runtime·class_to_size[sizeclass]);
                runtime·printf("\n\n");
                runtime·printf("size_to_class8:");
-               for(i=0; i<nelem(size_to_class8); i++)
-                       runtime·printf(" %d=>%d(%d)\n", i*8, size_to_class8[i], runtime·class_to_size[size_to_class8[i]]);
+               for(i=0; i<nelem(runtime·size_to_class8); i++)
+                       runtime·printf(" %d=>%d(%d)\n", i*8, runtime·size_to_class8[i],
+                               runtime·class_to_size[runtime·size_to_class8[i]]);
                runtime·printf("\n");
                runtime·printf("size_to_class128:");
-               for(i=0; i<nelem(size_to_class128); i++)
-                       runtime·printf(" %d=>%d(%d)\n", i*128, size_to_class128[i], runtime·class_to_size[size_to_class128[i]]);
+               for(i=0; i<nelem(runtime·size_to_class128); i++)
+                       runtime·printf(" %d=>%d(%d)\n", i*128, runtime·size_to_class128[i],
+                               runtime·class_to_size[runtime·size_to_class128[i]]);
                runtime·printf("\n");
        }
        runtime·throw("InitSizes failed");