From: Keith Randall Date: Fri, 2 Aug 2013 20:03:14 +0000 (-0700) Subject: runtime: reimplement reflect.call to not use stack splitting. X-Git-Tag: go1.2rc2~814 X-Git-Url: http://www.git.cypherpunks.su/?a=commitdiff_plain;h=9cd570680bd1d6ea23e4f5da1fe3a50c6927d6d5;p=gostls13.git runtime: reimplement reflect.call to not use stack splitting. R=golang-dev, r, khr, rsc CC=golang-dev https://golang.org/cl/12053043 --- diff --git a/misc/cgo/test/callback.go b/misc/cgo/test/callback.go index 838105155a..467edfbae5 100644 --- a/misc/cgo/test/callback.go +++ b/misc/cgo/test/callback.go @@ -151,6 +151,7 @@ func testCallbackCallers(t *testing.T) { n := 0 name := []string{ "test.goCallback", + "runtime.call16", "runtime.cgocallbackg1", "runtime.cgocallbackg", "runtime.cgocallback_gofunc", diff --git a/src/cmd/ld/lib.c b/src/cmd/ld/lib.c index c5af7fed88..e3bf3a2354 100644 --- a/src/cmd/ld/lib.c +++ b/src/cmd/ld/lib.c @@ -1522,7 +1522,7 @@ pctospadj(Sym *sym, int32 oldval, Prog *p, int32 phase, int32 arg) oldval = 0; if(phase == 0) return oldval; - if(oldval + p->spadj < -10000 || oldval + p->spadj > 1000000000) { + if(oldval + p->spadj < -10000 || oldval + p->spadj > 1100000000) { diag("overflow in spadj: %d + %d = %d", oldval, p->spadj, oldval + p->spadj); errorexit(); } diff --git a/src/pkg/reflect/all_test.go b/src/pkg/reflect/all_test.go index 93df4d1365..b905f93436 100644 --- a/src/pkg/reflect/all_test.go +++ b/src/pkg/reflect/all_test.go @@ -3509,3 +3509,14 @@ func (x *exhaustive) Choose(max int) int { func (x *exhaustive) Maybe() bool { return x.Choose(2) == 1 } + +func bigArgFunc(v [(1<<30)+64]byte) { +} + +func TestBigArgs(t *testing.T) { + if !testing.Short() && ^uint(0)>>32 != 0 { // test on 64-bit only + v := new([(1<<30)+64]byte) + bigArgFunc(*v) // regular calls are ok + shouldPanic(func() {ValueOf(bigArgFunc).Call([]Value{ValueOf(*v)})}) // ... just not reflect calls + } +} diff --git a/src/pkg/runtime/asm_386.s b/src/pkg/runtime/asm_386.s index 65b18cbf32..f6204acdb6 100644 --- a/src/pkg/runtime/asm_386.s +++ b/src/pkg/runtime/asm_386.s @@ -244,12 +244,12 @@ TEXT runtime·morestack(SB),7,$0-0 MOVL $0, 0x1003 // crash if newstack returns RET -// Called from reflection library. Mimics morestack, +// Called from panic. Mimics morestack, // reuses stack growth code to create a frame // with the desired args running the desired function. // // func call(fn *byte, arg *byte, argsize uint32). -TEXT reflect·call(SB), 7, $0-12 +TEXT runtime·newstackcall(SB), 7, $0-12 get_tls(CX) MOVL m(CX), BX @@ -264,12 +264,12 @@ TEXT reflect·call(SB), 7, $0-12 // Save our own state as the PC and SP to restore // if this goroutine needs to be restarted. - MOVL $reflect·call(SB), (g_sched+gobuf_pc)(AX) + MOVL $runtime·newstackcall(SB), (g_sched+gobuf_pc)(AX) MOVL SP, (g_sched+gobuf_sp)(AX) // Set up morestack arguments to call f on a new stack. // We set f's frame size to 1, as a hint to newstack - // that this is a call from reflect·call. + // that this is a call from runtime·newstackcall. // If it turns out that f needs a larger frame than // the default stack, f's usual stack growth prolog will // allocate a new segment (and recopy the arguments). @@ -291,6 +291,95 @@ TEXT reflect·call(SB), 7, $0-12 MOVL $0, 0x1103 // crash if newstack returns RET +// reflect·call: call a function with the given argument list +// func call(f *FuncVal, arg *byte, argsize uint32). +// we don't have variable-sized frames, so we use a small number +// of constant-sized-frame functions to encode a few bits of size in the pc. +// Caution: ugly multiline assembly macros in your future! + +#define DISPATCH(NAME,MAXSIZE) \ + CMPL CX, $MAXSIZE; \ + JA 3(PC); \ + MOVL $runtime·NAME(SB), AX; \ + JMP AX +// Note: can't just "JMP runtime·NAME(SB)" - bad inlining results. + +TEXT reflect·call(SB), 7, $0-12 + MOVL argsize+8(FP), CX + DISPATCH(call16, 16) + DISPATCH(call32, 32) + DISPATCH(call64, 64) + DISPATCH(call128, 128) + DISPATCH(call256, 256) + DISPATCH(call512, 512) + DISPATCH(call1024, 1024) + DISPATCH(call2048, 2048) + DISPATCH(call4096, 4096) + DISPATCH(call8192, 8192) + DISPATCH(call16384, 16384) + DISPATCH(call32768, 32768) + DISPATCH(call65536, 65536) + DISPATCH(call131072, 131072) + DISPATCH(call262144, 262144) + DISPATCH(call524288, 524288) + DISPATCH(call1048576, 1048576) + DISPATCH(call2097152, 2097152) + DISPATCH(call4194304, 4194304) + DISPATCH(call8388608, 8388608) + DISPATCH(call16777216, 16777216) + DISPATCH(call33554432, 33554432) + DISPATCH(call67108864, 67108864) + DISPATCH(call134217728, 134217728) + DISPATCH(call268435456, 268435456) + DISPATCH(call536870912, 536870912) + DISPATCH(call1073741824, 1073741824) + MOVL $runtime·badreflectcall(SB), AX + JMP AX + +#define CALLFN(NAME,MAXSIZE,FLAGS) \ +TEXT runtime·NAME(SB), FLAGS, $MAXSIZE-12; \ + /* copy arguments to stack */ \ + MOVL argptr+4(FP), SI; \ + MOVL argsize+8(FP), CX; \ + MOVL SP, DI; \ + REP;MOVSB; \ + /* call function */ \ + MOVL f+0(FP), DX; \ + CALL (DX); \ + /* copy return values back */ \ + MOVL argptr+4(FP), DI; \ + MOVL argsize+8(FP), CX; \ + MOVL SP, SI; \ + REP;MOVSB; \ + RET + +CALLFN(call16, 16, 7) +CALLFN(call32, 32, 7) +CALLFN(call64, 64, 7) +CALLFN(call128, 128, 0) +CALLFN(call256, 256, 0) +CALLFN(call512, 512, 0) +CALLFN(call1024, 1024, 0) +CALLFN(call2048, 2048, 0) +CALLFN(call4096, 4096, 0) +CALLFN(call8192, 8192, 0) +CALLFN(call16384, 16384, 0) +CALLFN(call32768, 32768, 0) +CALLFN(call65536, 65536, 0) +CALLFN(call131072, 131072, 0) +CALLFN(call262144, 262144, 0) +CALLFN(call524288, 524288, 0) +CALLFN(call1048576, 1048576, 0) +CALLFN(call2097152, 2097152, 0) +CALLFN(call4194304, 4194304, 0) +CALLFN(call8388608, 8388608, 0) +CALLFN(call16777216, 16777216, 0) +CALLFN(call33554432, 33554432, 0) +CALLFN(call67108864, 67108864, 0) +CALLFN(call134217728, 134217728, 0) +CALLFN(call268435456, 268435456, 0) +CALLFN(call536870912, 536870912, 0) +CALLFN(call1073741824, 1073741824, 0) // Return point when leaving stack. // diff --git a/src/pkg/runtime/asm_amd64.s b/src/pkg/runtime/asm_amd64.s index 9425fa99e0..d22c645740 100644 --- a/src/pkg/runtime/asm_amd64.s +++ b/src/pkg/runtime/asm_amd64.s @@ -225,12 +225,12 @@ TEXT runtime·morestack(SB),7,$0-0 MOVQ $0, 0x1003 // crash if newstack returns RET -// Called from reflection library. Mimics morestack, +// Called from panic. Mimics morestack, // reuses stack growth code to create a frame // with the desired args running the desired function. // // func call(fn *byte, arg *byte, argsize uint32). -TEXT reflect·call(SB), 7, $0-20 +TEXT runtime·newstackcall(SB), 7, $0-20 get_tls(CX) MOVQ m(CX), BX @@ -245,12 +245,12 @@ TEXT reflect·call(SB), 7, $0-20 // Save our own state as the PC and SP to restore // if this goroutine needs to be restarted. - MOVQ $reflect·call(SB), (g_sched+gobuf_pc)(AX) + MOVQ $runtime·newstackcall(SB), (g_sched+gobuf_pc)(AX) MOVQ SP, (g_sched+gobuf_sp)(AX) // Set up morestack arguments to call f on a new stack. // We set f's frame size to 1, as a hint to newstack - // that this is a call from reflect·call. + // that this is a call from runtime·newstackcall. // If it turns out that f needs a larger frame than // the default stack, f's usual stack growth prolog will // allocate a new segment (and recopy the arguments). @@ -272,6 +272,96 @@ TEXT reflect·call(SB), 7, $0-20 MOVQ $0, 0x1103 // crash if newstack returns RET +// reflect·call: call a function with the given argument list +// func call(f *FuncVal, arg *byte, argsize uint32). +// we don't have variable-sized frames, so we use a small number +// of constant-sized-frame functions to encode a few bits of size in the pc. +// Caution: ugly multiline assembly macros in your future! + +#define DISPATCH(NAME,MAXSIZE) \ + CMPQ CX, $MAXSIZE; \ + JA 3(PC); \ + MOVQ $runtime·NAME(SB), AX; \ + JMP AX +// Note: can't just "JMP runtime·NAME(SB)" - bad inlining results. + +TEXT reflect·call(SB), 7, $0-20 + MOVLQZX argsize+16(FP), CX + DISPATCH(call16, 16) + DISPATCH(call32, 32) + DISPATCH(call64, 64) + DISPATCH(call128, 128) + DISPATCH(call256, 256) + DISPATCH(call512, 512) + DISPATCH(call1024, 1024) + DISPATCH(call2048, 2048) + DISPATCH(call4096, 4096) + DISPATCH(call8192, 8192) + DISPATCH(call16384, 16384) + DISPATCH(call32768, 32768) + DISPATCH(call65536, 65536) + DISPATCH(call131072, 131072) + DISPATCH(call262144, 262144) + DISPATCH(call524288, 524288) + DISPATCH(call1048576, 1048576) + DISPATCH(call2097152, 2097152) + DISPATCH(call4194304, 4194304) + DISPATCH(call8388608, 8388608) + DISPATCH(call16777216, 16777216) + DISPATCH(call33554432, 33554432) + DISPATCH(call67108864, 67108864) + DISPATCH(call134217728, 134217728) + DISPATCH(call268435456, 268435456) + DISPATCH(call536870912, 536870912) + DISPATCH(call1073741824, 1073741824) + MOVQ $runtime·badreflectcall(SB), AX + JMP AX + +#define CALLFN(NAME,MAXSIZE,FLAGS) \ +TEXT runtime·NAME(SB), FLAGS, $MAXSIZE-20; \ + /* copy arguments to stack */ \ + MOVQ argptr+8(FP), SI; \ + MOVLQZX argsize+16(FP), CX; \ + MOVQ SP, DI; \ + REP;MOVSB; \ + /* call function */ \ + MOVQ f+0(FP), DX; \ + CALL (DX); \ + /* copy return values back */ \ + MOVQ argptr+8(FP), DI; \ + MOVLQZX argsize+16(FP), CX; \ + MOVQ SP, SI; \ + REP;MOVSB; \ + RET + +CALLFN(call16, 16, 7) +CALLFN(call32, 32, 7) +CALLFN(call64, 64, 7) +CALLFN(call128, 128, 0) +CALLFN(call256, 256, 0) +CALLFN(call512, 512, 0) +CALLFN(call1024, 1024, 0) +CALLFN(call2048, 2048, 0) +CALLFN(call4096, 4096, 0) +CALLFN(call8192, 8192, 0) +CALLFN(call16384, 16384, 0) +CALLFN(call32768, 32768, 0) +CALLFN(call65536, 65536, 0) +CALLFN(call131072, 131072, 0) +CALLFN(call262144, 262144, 0) +CALLFN(call524288, 524288, 0) +CALLFN(call1048576, 1048576, 0) +CALLFN(call2097152, 2097152, 0) +CALLFN(call4194304, 4194304, 0) +CALLFN(call8388608, 8388608, 0) +CALLFN(call16777216, 16777216, 0) +CALLFN(call33554432, 33554432, 0) +CALLFN(call67108864, 67108864, 0) +CALLFN(call134217728, 134217728, 0) +CALLFN(call268435456, 268435456, 0) +CALLFN(call536870912, 536870912, 0) +CALLFN(call1073741824, 1073741824, 0) + // Return point when leaving stack. // // Lessstack can appear in stack traces for the same reason diff --git a/src/pkg/runtime/asm_arm.s b/src/pkg/runtime/asm_arm.s index e3331b0d14..b7c95fed0d 100644 --- a/src/pkg/runtime/asm_arm.s +++ b/src/pkg/runtime/asm_arm.s @@ -208,12 +208,12 @@ TEXT runtime·morestack(SB),7,$-4-0 // is still in this function, and not the beginning of the next. RET -// Called from reflection library. Mimics morestack, +// Called from panic. Mimics morestack, // reuses stack growth code to create a frame // with the desired args running the desired function. // // func call(fn *byte, arg *byte, argsize uint32). -TEXT reflect·call(SB), 7, $-4-12 +TEXT runtime·newstackcall(SB), 7, $-4-12 // Save our caller's state as the PC and SP to // restore when returning from f. MOVW LR, (m_morebuf+gobuf_pc)(m) // our caller's PC @@ -222,14 +222,14 @@ TEXT reflect·call(SB), 7, $-4-12 // Save our own state as the PC and SP to restore // if this goroutine needs to be restarted. - MOVW $reflect·call(SB), R11 + MOVW $runtime·newstackcall(SB), R11 MOVW R11, (g_sched+gobuf_pc)(g) MOVW LR, (g_sched+gobuf_lr)(g) MOVW SP, (g_sched+gobuf_sp)(g) // Set up morestack arguments to call f on a new stack. // We set f's frame size to 1, as a hint to newstack - // that this is a call from reflect·call. + // that this is a call from runtime·newstackcall. // If it turns out that f needs a larger frame than // the default stack, f's usual stack growth prolog will // allocate a new segment (and recopy the arguments). @@ -248,6 +248,105 @@ TEXT reflect·call(SB), 7, $-4-12 MOVW (g_sched+gobuf_sp)(g), SP B runtime·newstack(SB) +// reflect·call: call a function with the given argument list +// func call(f *FuncVal, arg *byte, argsize uint32). +// we don't have variable-sized frames, so we use a small number +// of constant-sized-frame functions to encode a few bits of size in the pc. +// Caution: ugly multiline assembly macros in your future! + +#define DISPATCH(NAME,MAXSIZE) \ + CMP $MAXSIZE, R0; \ + B.HI 3(PC); \ + MOVW $runtime·NAME(SB), R1; \ + B (R1) + +TEXT reflect·call(SB), 7, $-4-12 + MOVW argsize+8(FP), R0 + DISPATCH(call16, 16) + DISPATCH(call32, 32) + DISPATCH(call64, 64) + DISPATCH(call128, 128) + DISPATCH(call256, 256) + DISPATCH(call512, 512) + DISPATCH(call1024, 1024) + DISPATCH(call2048, 2048) + DISPATCH(call4096, 4096) + DISPATCH(call8192, 8192) + DISPATCH(call16384, 16384) + DISPATCH(call32768, 32768) + DISPATCH(call65536, 65536) + DISPATCH(call131072, 131072) + DISPATCH(call262144, 262144) + DISPATCH(call524288, 524288) + DISPATCH(call1048576, 1048576) + DISPATCH(call2097152, 2097152) + DISPATCH(call4194304, 4194304) + DISPATCH(call8388608, 8388608) + DISPATCH(call16777216, 16777216) + DISPATCH(call33554432, 33554432) + DISPATCH(call67108864, 67108864) + DISPATCH(call134217728, 134217728) + DISPATCH(call268435456, 268435456) + DISPATCH(call536870912, 536870912) + DISPATCH(call1073741824, 1073741824) + MOVW $runtime·badreflectcall(SB), R1 + B (R1) + +#define CALLFN(NAME,MAXSIZE,FLAGS) \ +TEXT runtime·NAME(SB), FLAGS, $MAXSIZE-12; \ + /* copy arguments to stack */ \ + MOVW argptr+4(FP), R0; \ + MOVW argsize+8(FP), R2; \ + ADD $4, SP, R1; \ + CMP $0, R2; \ + B.EQ 5(PC); \ + MOVBU.P 1(R0), R5; \ + MOVBU.P R5, 1(R1); \ + SUB $1, R2, R2; \ + B -5(PC); \ + /* call function */ \ + MOVW f+0(FP), R7; \ + MOVW (R7), R0; \ + BL (R0); \ + /* copy return values back */ \ + MOVW argptr+4(FP), R0; \ + MOVW argsize+8(FP), R2; \ + ADD $4, SP, R1; \ + CMP $0, R2; \ + RET.EQ ; \ + MOVBU.P 1(R1), R5; \ + MOVBU.P R5, 1(R0); \ + SUB $1, R2, R2; \ + B -5(PC) \ + +CALLFN(call16, 16, 7) +CALLFN(call32, 32, 7) +CALLFN(call64, 64, 7) +CALLFN(call128, 128, 0) +CALLFN(call256, 256, 0) +CALLFN(call512, 512, 0) +CALLFN(call1024, 1024, 0) +CALLFN(call2048, 2048, 0) +CALLFN(call4096, 4096, 0) +CALLFN(call8192, 8192, 0) +CALLFN(call16384, 16384, 0) +CALLFN(call32768, 32768, 0) +CALLFN(call65536, 65536, 0) +CALLFN(call131072, 131072, 0) +CALLFN(call262144, 262144, 0) +CALLFN(call524288, 524288, 0) +CALLFN(call1048576, 1048576, 0) +CALLFN(call2097152, 2097152, 0) +CALLFN(call4194304, 4194304, 0) +CALLFN(call8388608, 8388608, 0) +CALLFN(call16777216, 16777216, 0) +CALLFN(call33554432, 33554432, 0) +CALLFN(call67108864, 67108864, 0) +CALLFN(call134217728, 134217728, 0) +CALLFN(call268435456, 268435456, 0) +CALLFN(call536870912, 536870912, 0) +CALLFN(call1073741824, 1073741824, 0) + // Return point when leaving stack. // using frame size $-4 means do not save LR on stack. // diff --git a/src/pkg/runtime/panic.c b/src/pkg/runtime/panic.c index 5692c537a0..36a3c41ba7 100644 --- a/src/pkg/runtime/panic.c +++ b/src/pkg/runtime/panic.c @@ -241,10 +241,10 @@ runtime·panic(Eface e) break; // take defer off list in case of recursive panic popdefer(); - g->ispanic = true; // rock for newstack, where reflect.call ends up + g->ispanic = true; // rock for newstack, where reflect.newstackcall ends up argp = d->argp; pc = d->pc; - reflect·call(d->fn, (byte*)d->args, d->siz); + runtime·newstackcall(d->fn, (byte*)d->args, d->siz); freedefer(d); if(p->recovered) { g->panic = p->link; diff --git a/src/pkg/runtime/proc.c b/src/pkg/runtime/proc.c index 79568503b7..52849809d0 100644 --- a/src/pkg/runtime/proc.c +++ b/src/pkg/runtime/proc.c @@ -1926,6 +1926,12 @@ runtime·badmcall2(void) // called from assembly runtime·throw("runtime: mcall function returned"); } +void +runtime·badreflectcall(void) // called from assembly +{ + runtime·panicstring("runtime: arg size to reflect.call more than 1GB"); +} + static struct { Lock; void (*fn)(uintptr*, int32); diff --git a/src/pkg/runtime/runtime.h b/src/pkg/runtime/runtime.h index cfb4793af5..705845d3f1 100644 --- a/src/pkg/runtime/runtime.h +++ b/src/pkg/runtime/runtime.h @@ -985,6 +985,7 @@ void runtime·printuint(uint64); void runtime·printhex(uint64); void runtime·printslice(Slice); void runtime·printcomplex(Complex128); +void runtime·newstackcall(FuncVal*, byte*, uint32); void reflect·call(FuncVal*, byte*, uint32); void runtime·panic(Eface); void runtime·panicindex(void); diff --git a/src/pkg/runtime/stack.c b/src/pkg/runtime/stack.c index 71c4c4caee..812ba17e2d 100644 --- a/src/pkg/runtime/stack.c +++ b/src/pkg/runtime/stack.c @@ -183,7 +183,7 @@ runtime·oldstack(void) runtime·gogo(&gp->sched); } -// Called from reflect·call or from runtime·morestack when a new +// Called from runtime·newstackcall or from runtime·morestack when a new // stack segment is needed. Allocate a new stack big enough for // m->moreframesize bytes, copy m->moreargsize bytes to the new frame, // and then act as though runtime·lessstack called the function at @@ -198,7 +198,7 @@ runtime·newstack(void) uintptr *src, *dst, *dstend; G *gp; Gobuf label; - bool reflectcall; + bool newstackcall; uintptr free; if(m->morebuf.g != m->curg) { @@ -217,12 +217,12 @@ runtime·newstack(void) argsize = m->moreargsize; gp->status = Gwaiting; gp->waitreason = "stack split"; - reflectcall = framesize==1; - if(reflectcall) + newstackcall = framesize==1; + if(newstackcall) framesize = 0; - // For reflectcall the context already points to beginning of reflect·call. - if(!reflectcall) + // For newstackcall the context already points to beginning of runtime·newstackcall. + if(!newstackcall) runtime·rewindmorestack(&gp->sched); sp = gp->sched.sp; @@ -269,8 +269,8 @@ runtime·newstack(void) runtime·gosched0(gp); // never return } - if(reflectcall && m->morebuf.sp - sizeof(Stktop) - argsize - 32 > gp->stackguard) { - // special case: called from reflect.call (framesize==1) + if(newstackcall && m->morebuf.sp - sizeof(Stktop) - argsize - 32 > gp->stackguard) { + // special case: called from runtime.newstackcall (framesize==1) // to call code with an arbitrary argument size, // and we have enough space on the current stack. // the new Stktop* is necessary to unwind, but @@ -334,7 +334,7 @@ runtime·newstack(void) label.sp = sp; label.pc = (uintptr)runtime·lessstack; label.g = m->curg; - if(reflectcall) + if(newstackcall) runtime·gostartcallfn(&label, (FuncVal*)m->cret); else { runtime·gostartcall(&label, (void(*)(void))gp->sched.pc, gp->sched.ctxt);