func TestBoolAlign(t *testing.T) { testBoolAlign(t) }
func Test3729(t *testing.T) { test3729(t) }
func Test3775(t *testing.T) { test3775(t) }
+func TestCthread(t *testing.T) { testCthread(t) }
func BenchmarkCgoCall(b *testing.B) { benchCgoCall(b) }
--- /dev/null
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cgotest
+
+// extern void doAdd(int, int);
+import "C"
+
+import (
+ "runtime"
+ "sync"
+ "testing"
+)
+
+var sum struct {
+ sync.Mutex
+ i int
+}
+
+//export Add
+func Add(x int) {
+ defer func() {
+ recover()
+ }()
+ sum.Lock()
+ sum.i += x
+ sum.Unlock()
+ var p *int
+ *p = 2
+}
+
+func testCthread(t *testing.T) {
+ if runtime.GOARCH == "arm" {
+ t.Skip("testCthread disabled on arm")
+ }
+
+ C.doAdd(10, 6)
+
+ want := 10 * (10 - 1) / 2 * 6
+ if sum.i != want {
+ t.Fatalf("sum=%d, want %d", sum.i, want)
+ }
+}
--- /dev/null
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build darwin freebsd linux netbsd openbsd
+
+#include <pthread.h>
+#include "_cgo_export.h"
+
+static void*
+addThread(void *p)
+{
+ int i, max;
+
+ max = *(int*)p;
+ for(i=0; i<max; i++)
+ Add(i);
+ return 0;
+}
+
+void
+doAdd(int max, int nthread)
+{
+ enum { MaxThread = 20 };
+ int i;
+ pthread_t thread_id[MaxThread];
+
+ if(nthread > MaxThread)
+ nthread = MaxThread;
+ for(i=0; i<nthread; i++)
+ pthread_create(&thread_id[i], 0, addThread, &max);
+ for(i=0; i<nthread; i++)
+ pthread_join(thread_id[i], 0);
+}
--- /dev/null
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+#define WIN32_LEAN_AND_MEAN
+#include <windows.h>
+#include <process.h>
+#include "_cgo_export.h"
+
+__stdcall
+static unsigned int
+addThread(void *p)
+{
+ int i, max;
+
+ max = *(int*)p;
+ for(i=0; i<max; i++)
+ Add(i);
+ return 0;
+}
+
+void
+doAdd(int max, int nthread)
+{
+ enum { MaxThread = 20 };
+ int i;
+ uintptr_t thread_id[MaxThread];
+
+ if(nthread > MaxThread)
+ nthread = MaxThread;
+ for(i=0; i<nthread; i++)
+ thread_id[i] = _beginthreadex(0, 0, addThread, &max, 0, 0);
+ for(i=0; i<nthread; i++) {
+ WaitForSingleObject((HANDLE)thread_id[i], INFINITE);
+ CloseHandle((HANDLE)thread_id[i]);
+ }
+}
// cgocallback(void (*fn)(void*), void *frame, uintptr framesize)
// See cgocall.c for more details.
TEXT runtime·cgocallback(SB),7,$12
- MOVL fn+0(FP), AX
- MOVL frame+4(FP), BX
- MOVL framesize+8(FP), DX
-
- // Save current m->g0->sched.sp on stack and then set it to SP.
+ // If m is nil, Go did not create the current thread.
+ // Call needm to obtain one for temporary use.
+ // In this case, we're running on the thread stack, so there's
+ // lots of space, but the linker doesn't know. Hide the call from
+ // the linker analysis by using an indirect call through AX.
get_tls(CX)
+#ifdef GOOS_windows
+ CMPL CX, $0
+ JNE 3(PC)
+ PUSHL $0
+ JMP needm
+#endif
MOVL m(CX), BP
-
- // If m is nil, it is almost certainly because we have been called
- // on a thread that Go did not create. We're going to crash as
- // soon as we try to use m; instead, try to print a nice error and exit.
+ PUSHL BP
CMPL BP, $0
- JNE 2(PC)
- CALL runtime·badcallback(SB)
+ JNE havem
+needm:
+ MOVL $runtime·needm(SB), AX
+ CALL AX
+ get_tls(CX)
+ MOVL m(CX), BP
+havem:
+ // Now there's a valid m, and we're running on its m->g0.
+ // Save current m->g0->sched.sp on stack and then set it to SP.
+ // Save current sp in m->g0->sched.sp in preparation for
+ // switch back to m->curg stack.
MOVL m_g0(BP), SI
PUSHL (g_sched+gobuf_sp)(SI)
MOVL SP, (g_sched+gobuf_sp)(SI)
// a frame size of 12, the same amount that we use below),
// so that the traceback will seamlessly trace back into
// the earlier calls.
+ MOVL fn+0(FP), AX
+ MOVL frame+4(FP), BX
+ MOVL framesize+8(FP), DX
+
MOVL m_curg(BP), SI
MOVL SI, g(CX)
MOVL (g_sched+gobuf_sp)(SI), DI // prepare stack as DI
MOVL SI, g(CX)
MOVL (g_sched+gobuf_sp)(SI), SP
POPL (g_sched+gobuf_sp)(SI)
+
+ // If the m on entry was nil, we called needm above to borrow an m
+ // for the duration of the call. Since the call is over, return it with dropm.
+ POPL BP
+ CMPL BP, $0
+ JNE 3(PC)
+ MOVL $runtime·dropm(SB), AX
+ CALL AX
// Done!
RET
+// void setmg(M*, G*); set m and g. for use by needm.
+TEXT runtime·setmg(SB), 7, $0
+#ifdef GOOS_windows
+ MOVL mm+0(FP), AX
+ CMPL AX, $0
+ JNE settls
+ MOVL $0, 0x14(FS)
+ RET
+settls:
+ LEAL m_tls(AX), AX
+ MOVL AX, 0x14(FS)
+#endif
+ MOVL mm+0(FP), AX
+ get_tls(CX)
+ MOVL mm+0(FP), AX
+ MOVL AX, m(CX)
+ MOVL gg+4(FP), BX
+ MOVL BX, g(CX)
+ RET
+
// check that SP is in range [g->stackbase, g->stackguard)
TEXT runtime·stackcheck(SB), 7, $0
get_tls(CX)
// cgocallback(void (*fn)(void*), void *frame, uintptr framesize)
// See cgocall.c for more details.
TEXT runtime·cgocallback(SB),7,$24
- MOVQ fn+0(FP), AX
- MOVQ frame+8(FP), BX
- MOVQ framesize+16(FP), DX
-
- // Save current m->g0->sched.sp on stack and then set it to SP.
+ // If m is nil, Go did not create the current thread.
+ // Call needm to obtain one for temporary use.
+ // In this case, we're running on the thread stack, so there's
+ // lots of space, but the linker doesn't know. Hide the call from
+ // the linker analysis by using an indirect call through AX.
get_tls(CX)
+#ifdef GOOS_windows
+ CMPQ CX, $0
+ JNE 3(PC)
+ PUSHQ $0
+ JMP needm
+#endif
MOVQ m(CX), BP
-
- // If m is nil, it is almost certainly because we have been called
- // on a thread that Go did not create. We're going to crash as
- // soon as we try to use m; instead, try to print a nice error and exit.
+ PUSHQ BP
CMPQ BP, $0
- JNE 2(PC)
- CALL runtime·badcallback(SB)
+ JNE havem
+needm:
+ MOVQ $runtime·needm(SB), AX
+ CALL AX
+ get_tls(CX)
+ MOVQ m(CX), BP
+havem:
+ // Now there's a valid m, and we're running on its m->g0.
+ // Save current m->g0->sched.sp on stack and then set it to SP.
+ // Save current sp in m->g0->sched.sp in preparation for
+ // switch back to m->curg stack.
MOVQ m_g0(BP), SI
PUSHQ (g_sched+gobuf_sp)(SI)
MOVQ SP, (g_sched+gobuf_sp)(SI)
// a frame size of 24, the same amount that we use below),
// so that the traceback will seamlessly trace back into
// the earlier calls.
+ MOVQ fn+0(FP), AX
+ MOVQ frame+8(FP), BX
+ MOVQ framesize+16(FP), DX
+
MOVQ m_curg(BP), SI
MOVQ SI, g(CX)
MOVQ (g_sched+gobuf_sp)(SI), DI // prepare stack as DI
MOVQ SI, g(CX)
MOVQ (g_sched+gobuf_sp)(SI), SP
POPQ (g_sched+gobuf_sp)(SI)
+
+ // If the m on entry was nil, we called needm above to borrow an m
+ // for the duration of the call. Since the call is over, return it with dropm.
+ POPQ BP
+ CMPQ BP, $0
+ JNE 3(PC)
+ MOVQ $runtime·dropm(SB), AX
+ CALL AX
// Done!
RET
+// void setmg(M*, G*); set m and g. for use by needm.
+TEXT runtime·setmg(SB), 7, $0
+ MOVQ mm+0(FP), AX
+#ifdef GOOS_windows
+ CMPQ AX, $0
+ JNE settls
+ MOVQ $0, 0x28(GS)
+ RET
+settls:
+ LEAQ m_tls(AX), AX
+ MOVQ AX, 0x28(GS)
+#endif
+ get_tls(CX)
+ MOVQ mm+0(FP), AX
+ MOVQ AX, m(CX)
+ MOVQ gg+8(FP), BX
+ MOVQ BX, g(CX)
+ RET
+
// check that SP is in range [g->stackbase, g->stackguard)
TEXT runtime·stackcheck(SB), 7, $0
get_tls(CX)
// cgocallback(void (*fn)(void*), void *frame, uintptr framesize)
// See cgocall.c for more details.
TEXT runtime·cgocallback(SB),7,$16
+ // Load m and g from thread-local storage.
+ MOVW cgo_load_gm(SB), R0
+ CMP $0, R0
+ BL.NE (R0)
+
+ // If m is nil, Go did not create the current thread.
+ // Call needm to obtain one for temporary use.
+ // In this case, we're running on the thread stack, so there's
+ // lots of space, but the linker doesn't know. Hide the call from
+ // the linker analysis by using an indirect call.
+ MOVW m, savedm-16(SP)
+ CMP $0, m
+ B.NE havem
+ MOVW $runtime·needm(SB), R0
+ BL (R0)
+
+havem:
+ // Now there's a valid m, and we're running on its m->g0.
+ // Save current m->g0->sched.sp on stack and then set it to SP.
+ // Save current sp in m->g0->sched.sp in preparation for
+ // switch back to m->curg stack.
MOVW fn+0(FP), R0
MOVW frame+4(FP), R1
MOVW framesize+8(FP), R2
- // Save current m->g0->sched.sp on stack and then set it to SP.
MOVW m_g0(m), R3
MOVW (g_sched+gobuf_sp)(R3), R4
MOVW.W R4, -4(R13)
// a frame size of 16, the same amount that we use below),
// so that the traceback will seamlessly trace back into
// the earlier calls.
+
+ // Save current m->g0->sched.sp on stack and then set it to SP.
MOVW m_curg(m), g
MOVW (g_sched+gobuf_sp)(g), R4 // prepare stack as R4
ADD $4, R13
MOVW R6, (g_sched+gobuf_sp)(g)
+ // If the m on entry was nil, we called needm above to borrow an m
+ // for the duration of the call. Since the call is over, return it with dropm.
+ MOVW savedm-16(SP), R6
+ CMP $0, R6
+ B.NE 3(PC)
+ MOVW $runtime·dropm(SB), R0
+ BL (R0)
+
// Done!
RET
+// void setmg(M*, G*); set m and g. for use by needm.
+TEXT runtime·setmg(SB), 7, $-4
+ MOVW mm+0(FP), m
+ MOVW gg+4(FP), g
+
+ // Save m and g to thread-local storage.
+ MOVW cgo_save_gm(SB), R0
+ CMP $0, R0
+ BL.NE (R0)
+
+ RET
+
TEXT runtime·getcallerpc(SB),7,$-4
MOVW 0(SP), R0
RET
runtime·exitsyscall(); // coming out of cgo call
+ if(m->needextram) {
+ m->needextram = 0;
+ runtime·newextram();
+ }
+
// Add entry to defer stack in case of panic.
d.fn = (byte*)unwindm;
d.siz = 0;
void *runtime·callbackasm(void);
void runtime·install_exception_handler(void);
+void runtime·remove_exception_handler(void);
// TODO(brainman): should not need those
#define NSIG 65
G* runtime·allg;
G* runtime·lastg;
M* runtime·allm;
+M* runtime·extram;
int8* runtime·goos;
int32 runtime·ncpu;
// Install signal handlers; after minit so that minit can
// prepare the thread to be able to handle the signals.
- if(m == &runtime·m0)
+ if(m == &runtime·m0) {
runtime·initsig();
+ if(runtime·iscgo)
+ runtime·newextram();
+ }
schedule(nil);
}
}
-// Create a new m. It will start off with a call to runtime·mstart.
+// Allocate a new m unassociated with any thread.
M*
-runtime·newm(void)
+runtime·allocm(void)
{
M *mp;
static Type *mtype; // The Go type M
mp = runtime·cnew(mtype);
mcommoninit(mp);
+ if(runtime·iscgo || Windows)
+ mp->g0 = runtime·malg(-1);
+ else
+ mp->g0 = runtime·malg(8192);
+
+ return mp;
+}
+
+static M* lockextra(bool nilokay);
+static void unlockextra(M*);
+
+// needm is called when a cgo callback happens on a
+// thread without an m (a thread not created by Go).
+// In this case, needm is expected to find an m to use
+// and return with m, g initialized correctly.
+// Since m and g are not set now (likely nil, but see below)
+// needm is limited in what routines it can call. In particular
+// it can only call nosplit functions (textflag 7) and cannot
+// do any scheduling that requires an m.
+//
+// In order to avoid needing heavy lifting here, we adopt
+// the following strategy: there is a stack of available m's
+// that can be stolen. Using compare-and-swap
+// to pop from the stack has ABA races, so we simulate
+// a lock by doing an exchange (via casp) to steal the stack
+// head and replace the top pointer with MLOCKED (1).
+// This serves as a simple spin lock that we can use even
+// without an m. The thread that locks the stack in this way
+// unlocks the stack by storing a valid stack head pointer.
+//
+// In order to make sure that there is always an m structure
+// available to be stolen, we maintain the invariant that there
+// is always one more than needed. At the beginning of the
+// program (if cgo is in use) the list is seeded with a single m.
+// If needm finds that it has taken the last m off the list, its job
+// is - once it has installed its own m so that it can do things like
+// allocate memory - to create a spare m and put it on the list.
+//
+// Each of these extra m's also has a g0 and a curg that are
+// pressed into service as the scheduling stack and current
+// goroutine for the duration of the cgo callback.
+//
+// When the callback is done with the m, it calls dropm to
+// put the m back on the list.
+#pragma textflag 7
+void
+runtime·needm(byte x)
+{
+ M *mp;
+
+ // Lock extra list, take head, unlock popped list.
+ // nilokay=false is safe here because of the invariant above,
+ // that the extra list always contains or will soon contain
+ // at least one m.
+ mp = lockextra(false);
+
+ // Set needextram when we've just emptied the list,
+ // so that the eventual call into cgocallbackg will
+ // allocate a new m for the extra list. We delay the
+ // allocation until then so that it can be done
+ // after exitsyscall makes sure it is okay to be
+ // running at all (that is, there's no garbage collection
+ // running right now).
+ mp->needextram = mp->schedlink == nil;
+ unlockextra(mp->schedlink);
+
+ // Install m and g (= m->g0) and set the stack bounds
+ // to match the current stack. We don't actually know
+ // how big the stack is, like we don't know how big any
+ // scheduling stack is, but we assume there's at least 32 kB,
+ // which is more than enough for us.
+ runtime·setmg(mp, mp->g0);
+ g->stackbase = (uintptr)(&x + 1024);
+ g->stackguard = (uintptr)(&x - 32*1024);
+
+ // On windows/386, we need to put an SEH frame (two words)
+ // somewhere on the current stack. We are called
+ // from needm, and we know there is some available
+ // space one word into the argument frame. Use that.
+ m->seh = (SEH*)((uintptr*)&x + 1);
+
+ // Initialize this thread to use the m.
+ runtime·asminit();
+ runtime·minit();
+}
+
+// newextram allocates an m and puts it on the extra list.
+// It is called with a working local m, so that it can do things
+// like call schedlock and allocate.
+void
+runtime·newextram(void)
+{
+ M *mp, *mnext;
+ G *gp;
+
+ // Scheduler protects allocation of new m's and g's.
+ // Create extra goroutine locked to extra m.
+ // The goroutine is the context in which the cgo callback will run.
+ // The sched.pc will never be returned to, but setting it to
+ // runtime.goexit makes clear to the traceback routines where
+ // the goroutine stack ends.
+ schedlock();
+ mp = runtime·allocm();
+ gp = runtime·malg(4096);
+ gp->sched.pc = (void*)runtime·goexit;
+ gp->sched.sp = gp->stackbase;
+ gp->sched.g = gp;
+ gp->status = Gsyscall;
+ mp->curg = gp;
+ mp->locked = LockInternal;
+ mp->lockedg = gp;
+ gp->lockedm = mp;
+ schedunlock();
+
+ // Add m to the extra list.
+ mnext = lockextra(true);
+ mp->schedlink = mnext;
+ unlockextra(mp);
+}
+
+// dropm is called when a cgo callback has called needm but is now
+// done with the callback and returning back into the non-Go thread.
+// It puts the current m back onto the extra list.
+//
+// The main expense here is the call to signalstack to release the
+// m's signal stack, and then the call to needm on the next callback
+// from this thread. It is tempting to try to save the m for next time,
+// which would eliminate both these costs, but there might not be
+// a next time: the current thread (which Go does not control) might exit.
+// If we saved the m for that thread, there would be an m leak each time
+// such a thread exited. Instead, we acquire and release an m on each
+// call. These should typically not be scheduling operations, just a few
+// atomics, so the cost should be small.
+//
+// TODO(rsc): An alternative would be to allocate a dummy pthread per-thread
+// variable using pthread_key_create. Unlike the pthread keys we already use
+// on OS X, this dummy key would never be read by Go code. It would exist
+// only so that we could register at thread-exit-time destructor.
+// That destructor would put the m back onto the extra list.
+// This is purely a performance optimization. The current version,
+// in which dropm happens on each cgo call, is still correct too.
+// We may have to keep the current version on systems with cgo
+// but without pthreads, like Windows.
+void
+runtime·dropm(void)
+{
+ M *mp, *mnext;
+
+ // Undo whatever initialization minit did during needm.
+ runtime·unminit();
+
+ // Clear m and g, and return m to the extra list.
+ // After the call to setmg we can only call nosplit functions.
+ mp = m;
+ runtime·setmg(nil, nil);
+
+ mnext = lockextra(true);
+ mp->schedlink = mnext;
+ unlockextra(mp);
+}
+
+#define MLOCKED ((M*)1)
+
+// lockextra locks the extra list and returns the list head.
+// The caller must unlock the list by storing a new list head
+// to runtime.extram. If nilokay is true, then lockextra will
+// return a nil list head if that's what it finds. If nilokay is false,
+// lockextra will keep waiting until the list head is no longer nil.
+#pragma textflag 7
+static M*
+lockextra(bool nilokay)
+{
+ M *mp;
+ void (*yield)(void);
+
+ for(;;) {
+ mp = runtime·atomicloadp(&runtime·extram);
+ if(mp == MLOCKED) {
+ yield = runtime·osyield;
+ yield();
+ continue;
+ }
+ if(mp == nil && !nilokay) {
+ runtime·usleep(1);
+ continue;
+ }
+ if(!runtime·casp(&runtime·extram, mp, MLOCKED)) {
+ yield = runtime·osyield;
+ yield();
+ continue;
+ }
+ break;
+ }
+ return mp;
+}
+
+#pragma textflag 7
+static void
+unlockextra(M *mp)
+{
+ runtime·atomicstorep(&runtime·extram, mp);
+}
+
+
+// Create a new m. It will start off with a call to runtime·mstart.
+M*
+runtime·newm(void)
+{
+ M *mp;
+
+ mp = runtime·allocm();
+
if(runtime·iscgo) {
CgoThreadStart ts;
if(libcgo_thread_start == nil)
runtime·throw("libcgo_thread_start missing");
- // pthread_create will make us a stack.
- mp->g0 = runtime·malg(-1);
ts.m = mp;
ts.g = mp->g0;
ts.fn = runtime·mstart;
runtime·asmcgocall(libcgo_thread_start, &ts);
} else {
- if(Windows)
- // windows will layout sched stack on os stack
- mp->g0 = runtime·malg(-1);
- else
- mp->g0 = runtime·malg(8192);
runtime·newosproc(mp, mp->g0, (byte*)mp->g0->stackbase, runtime·mstart);
}
uint32 waitsemalock;
GCStats gcstats;
bool racecall;
+ bool needextram;
void* racepc;
uint32 moreframesize_minalloc;
byte* runtime·getenv(int8*);
int32 runtime·atoi(byte*);
void runtime·newosproc(M *mp, G *gp, void *stk, void (*fn)(void));
-void runtime·signalstack(byte*, int32);
G* runtime·malg(int32);
void runtime·asminit(void);
void runtime·minit(void);
+void runtime·unminit(void);
+void runtime·signalstack(byte*, int32);
Func* runtime·findfunc(uintptr);
int32 runtime·funcline(Func*, uintptr);
void* runtime·stackalloc(uint32);
void runtime·mcall(void(*)(G*));
uint32 runtime·fastrand1(void);
+void runtime·setmg(M*, G*);
+void runtime·newextram(void);
void runtime·exit(int32);
void runtime·breakpoint(void);
void runtime·gosched(void);
MOVL DX, 0(FS)
RET
+
+// void remove_exception_handler()
+TEXT runtime·remove_exception_handler(SB),7,$0
+ get_tls(CX)
+ MOVL m(CX), CX // m
+
+ // Remove SEH frame
+ MOVL m_seh(CX), DX
+ MOVL seh_prev(DX), AX
+ MOVL AX, 0(FS)
+
+ RET
TEXT runtime·install_exception_handler(SB),7,$0
CALL runtime·setstacklimits(SB)
RET
+
+TEXT runtime·remove_exception_handler(SB),7,$0
+ RET
runtime·setprof(m->profilehz > 0);
}
+// Called from dropm to undo the effect of an minit.
+void
+runtime·unminit(void)
+{
+ runtime·signalstack(nil, 0);
+}
+
// Mach IPC, to get at semaphores
// Definitions are in /usr/include/mach on a Mac.
runtime·sigprocmask(&sigset_none, nil);
}
+// Called from dropm to undo the effect of an minit.
+void
+runtime·unminit(void)
+{
+ runtime·signalstack(nil, 0);
+}
+
void
runtime·sigpanic(void)
{
runtime·rtsigprocmask(SIG_SETMASK, &sigset_none, nil, sizeof(Sigset));
}
+// Called from dropm to undo the effect of an minit.
+void
+runtime·unminit(void)
+{
+ runtime·signalstack(nil, 0);
+}
+
void
runtime·sigpanic(void)
{
runtime·sigprocmask(SIG_SETMASK, &sigset_none, nil);
}
+// Called from dropm to undo the effect of an minit.
+void
+runtime·unminit(void)
+{
+ runtime·signalstack(nil, 0);
+}
+
void
runtime·sigpanic(void)
{
runtime·sigprocmask(SIG_SETMASK, sigset_none);
}
+// Called from dropm to undo the effect of an minit.
+void
+runtime·unminit(void)
+{
+ runtime·signalstack(nil, 0);
+}
+
void
runtime·sigpanic(void)
{
runtime·setfpmasks();
}
+// Called from dropm to undo the effect of an minit.
+void
+runtime·unminit(void)
+{
+}
+
+
static int32
getproccount(void)
{
{
}
+#pragma textflag 7
void
runtime·osyield(void)
{
return written;
}
+#pragma textflag 7
void
runtime·osyield(void)
{
runtime·install_exception_handler();
}
+// Called from dropm to undo the effect of an minit.
+void
+runtime·unminit(void)
+{
+ runtime·remove_exception_handler();
+}
+
int64
runtime·nanotime(void)
{