From 4f34a5291317fe959074fb40b85d9be242b8a2fc Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Fri, 16 Jun 2017 16:21:12 -0400 Subject: [PATCH] runtime: terminate locked OS thread if its goroutine exits runtime.LockOSThread is sometimes used when the caller intends to put the OS thread into an unusual state. In this case, we never want to return this thread to the runtime thread pool. However, currently exiting the goroutine implicitly unlocks its OS thread. Fix this by terminating the locked OS thread when its goroutine exits, rather than simply returning it to the pool. Fixes #20395. Change-Id: I3dcec63b200957709965f7240dc216fa84b62ad9 Reviewed-on: https://go-review.googlesource.com/46038 Run-TryBot: Austin Clements TryBot-Result: Gobot Gobot Reviewed-by: Keith Randall --- src/runtime/crash_cgo_test.go | 9 ++ src/runtime/crash_test.go | 3 +- src/runtime/proc.go | 14 ++- src/runtime/proc_test.go | 17 +++ src/runtime/testdata/testprog/gettid.go | 29 +++++ src/runtime/testdata/testprog/gettid_none.go | 15 +++ src/runtime/testdata/testprog/lockosthread.go | 94 +++++++++++++++ .../testdata/testprogcgo/lockosthread.c | 13 ++ .../testdata/testprogcgo/lockosthread.go | 111 ++++++++++++++++++ 9 files changed, 303 insertions(+), 2 deletions(-) create mode 100644 src/runtime/testdata/testprog/gettid.go create mode 100644 src/runtime/testdata/testprog/gettid_none.go create mode 100644 src/runtime/testdata/testprog/lockosthread.go create mode 100644 src/runtime/testdata/testprogcgo/lockosthread.c create mode 100644 src/runtime/testdata/testprogcgo/lockosthread.go diff --git a/src/runtime/crash_cgo_test.go b/src/runtime/crash_cgo_test.go index ae083ef8e8..d1c8d37b2f 100644 --- a/src/runtime/crash_cgo_test.go +++ b/src/runtime/crash_cgo_test.go @@ -443,3 +443,12 @@ func TestCatchPanic(t *testing.T) { } } } + +func TestCgoLockOSThreadExit(t *testing.T) { + switch runtime.GOOS { + case "plan9", "windows": + t.Skipf("no pthreads on %s", runtime.GOOS) + } + t.Parallel() + testLockOSThreadExit(t, "testprogcgo") +} diff --git a/src/runtime/crash_test.go b/src/runtime/crash_test.go index 2962fbd082..0f11150f18 100644 --- a/src/runtime/crash_test.go +++ b/src/runtime/crash_test.go @@ -43,7 +43,7 @@ type buildexe struct { err error } -func runTestProg(t *testing.T, binary, name string) string { +func runTestProg(t *testing.T, binary, name string, env ...string) string { testenv.MustHaveGoBuild(t) exe, err := buildTestProg(t, binary) @@ -52,6 +52,7 @@ func runTestProg(t *testing.T, binary, name string) string { } cmd := testenv.CleanCmdEnv(exec.Command(exe, name)) + cmd.Env = append(cmd.Env, env...) var b bytes.Buffer cmd.Stdout = &b cmd.Stderr = &b diff --git a/src/runtime/proc.go b/src/runtime/proc.go index d8ed1e9185..1e41a020bb 100644 --- a/src/runtime/proc.go +++ b/src/runtime/proc.go @@ -2633,6 +2633,7 @@ func goexit0(gp *g) { atomic.Xadd(&sched.ngsys, -1) } gp.m = nil + locked := gp.lockedm != 0 gp.lockedm = 0 _g_.m.lockedg = 0 gp.paniconfault = false @@ -2655,6 +2656,15 @@ func goexit0(gp *g) { } _g_.m.lockedExt = 0 gfput(_g_.m.p.ptr(), gp) + if locked { + // The goroutine may have locked this thread because + // it put it in an unusual kernel state. Kill it + // rather than returning it to the thread pool. + + // Return to mstart, which will release the P and exit + // the thread. + gogo(&_g_.m.g0.sched) + } schedule() } @@ -3419,8 +3429,10 @@ func dolockOSThread() { // LockOSThread wires the calling goroutine to its current operating system thread. // The calling goroutine will always execute in that thread, // and no other goroutine will execute in it, -// until the calling goroutine exits or has made as many calls to +// until the calling goroutine has made as many calls to // UnlockOSThread as to LockOSThread. +// If the calling goroutine exits without unlocking the thread, +// the thread will be terminated. func LockOSThread() { if atomic.Load(&newmHandoff.haveTemplateThread) == 0 { // If we need to start a new thread from the locked diff --git a/src/runtime/proc_test.go b/src/runtime/proc_test.go index 835b548742..c6ecc2a472 100644 --- a/src/runtime/proc_test.go +++ b/src/runtime/proc_test.go @@ -746,3 +746,20 @@ func TestLockOSThreadNesting(t *testing.T) { } }() } + +func TestLockOSThreadExit(t *testing.T) { + testLockOSThreadExit(t, "testprog") +} + +func testLockOSThreadExit(t *testing.T, prog string) { + output := runTestProg(t, prog, "LockOSThreadMain", "GOMAXPROCS=1") + want := "OK\n" + if output != want { + t.Errorf("want %s, got %s\n", want, output) + } + + output = runTestProg(t, prog, "LockOSThreadAlt") + if output != want { + t.Errorf("want %s, got %s\n", want, output) + } +} diff --git a/src/runtime/testdata/testprog/gettid.go b/src/runtime/testdata/testprog/gettid.go new file mode 100644 index 0000000000..1b3e29ab08 --- /dev/null +++ b/src/runtime/testdata/testprog/gettid.go @@ -0,0 +1,29 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build linux + +package main + +import ( + "bytes" + "fmt" + "io/ioutil" + "os" + "syscall" +) + +func gettid() int { + return syscall.Gettid() +} + +func tidExists(tid int) (exists, supported bool) { + stat, err := ioutil.ReadFile(fmt.Sprintf("/proc/self/task/%d/stat", tid)) + if os.IsNotExist(err) { + return false, true + } + // Check if it's a zombie thread. + state := bytes.Fields(stat)[2] + return !(len(state) == 1 && state[0] == 'Z'), true +} diff --git a/src/runtime/testdata/testprog/gettid_none.go b/src/runtime/testdata/testprog/gettid_none.go new file mode 100644 index 0000000000..036db87e10 --- /dev/null +++ b/src/runtime/testdata/testprog/gettid_none.go @@ -0,0 +1,15 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !linux + +package main + +func gettid() int { + return 0 +} + +func tidExists(tid int) (exists, supported bool) { + return false, false +} diff --git a/src/runtime/testdata/testprog/lockosthread.go b/src/runtime/testdata/testprog/lockosthread.go new file mode 100644 index 0000000000..88c0d12e4c --- /dev/null +++ b/src/runtime/testdata/testprog/lockosthread.go @@ -0,0 +1,94 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "os" + "runtime" + "time" +) + +var mainTID int + +func init() { + registerInit("LockOSThreadMain", func() { + // init is guaranteed to run on the main thread. + mainTID = gettid() + }) + register("LockOSThreadMain", LockOSThreadMain) + + registerInit("LockOSThreadAlt", func() { + // Lock the OS thread now so main runs on the main thread. + runtime.LockOSThread() + }) + register("LockOSThreadAlt", LockOSThreadAlt) +} + +func LockOSThreadMain() { + // gettid only works on Linux, so on other platforms this just + // checks that the runtime doesn't do anything terrible. + + // This requires GOMAXPROCS=1 from the beginning to reliably + // start a goroutine on the main thread. + if runtime.GOMAXPROCS(-1) != 1 { + println("requires GOMAXPROCS=1") + os.Exit(1) + } + + ready := make(chan bool, 1) + go func() { + // Because GOMAXPROCS=1, this *should* be on the main + // thread. Stay there. + runtime.LockOSThread() + if mainTID != 0 && gettid() != mainTID { + println("failed to start goroutine on main thread") + os.Exit(1) + } + // Exit with the thread locked, which should exit the + // main thread. + ready <- true + }() + <-ready + time.Sleep(1 * time.Millisecond) + // Check that this goroutine is still running on a different + // thread. + if mainTID != 0 && gettid() == mainTID { + println("goroutine migrated to locked thread") + os.Exit(1) + } + println("OK") +} + +func LockOSThreadAlt() { + // This is running locked to the main OS thread. + + var subTID int + ready := make(chan bool, 1) + go func() { + // This goroutine must be running on a new thread. + runtime.LockOSThread() + subTID = gettid() + ready <- true + // Exit with the thread locked. + }() + <-ready + runtime.UnlockOSThread() + for i := 0; i < 100; i++ { + time.Sleep(1 * time.Millisecond) + // Check that this goroutine is running on a different thread. + if subTID != 0 && gettid() == subTID { + println("locked thread reused") + os.Exit(1) + } + exists, supported := tidExists(subTID) + if !supported || !exists { + goto ok + } + } + println("sub thread", subTID, "still running") + return +ok: + println("OK") +} diff --git a/src/runtime/testdata/testprogcgo/lockosthread.c b/src/runtime/testdata/testprogcgo/lockosthread.c new file mode 100644 index 0000000000..b10cc4f3b9 --- /dev/null +++ b/src/runtime/testdata/testprogcgo/lockosthread.c @@ -0,0 +1,13 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !plan9,!windows + +#include + +uint32_t threadExited; + +void setExited(void *x) { + __sync_fetch_and_add(&threadExited, 1); +} diff --git a/src/runtime/testdata/testprogcgo/lockosthread.go b/src/runtime/testdata/testprogcgo/lockosthread.go new file mode 100644 index 0000000000..36423d9eb0 --- /dev/null +++ b/src/runtime/testdata/testprogcgo/lockosthread.go @@ -0,0 +1,111 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !plan9,!windows + +package main + +import ( + "os" + "runtime" + "sync/atomic" + "time" + "unsafe" +) + +/* +#include +#include + +extern uint32_t threadExited; + +void setExited(void *x); +*/ +import "C" + +var mainThread C.pthread_t + +func init() { + registerInit("LockOSThreadMain", func() { + // init is guaranteed to run on the main thread. + mainThread = C.pthread_self() + }) + register("LockOSThreadMain", LockOSThreadMain) + + registerInit("LockOSThreadAlt", func() { + // Lock the OS thread now so main runs on the main thread. + runtime.LockOSThread() + }) + register("LockOSThreadAlt", LockOSThreadAlt) +} + +func LockOSThreadMain() { + // This requires GOMAXPROCS=1 from the beginning to reliably + // start a goroutine on the main thread. + if runtime.GOMAXPROCS(-1) != 1 { + println("requires GOMAXPROCS=1") + os.Exit(1) + } + + ready := make(chan bool, 1) + go func() { + // Because GOMAXPROCS=1, this *should* be on the main + // thread. Stay there. + runtime.LockOSThread() + self := C.pthread_self() + if C.pthread_equal(mainThread, self) == 0 { + println("failed to start goroutine on main thread") + os.Exit(1) + } + // Exit with the thread locked, which should exit the + // main thread. + ready <- true + }() + <-ready + time.Sleep(1 * time.Millisecond) + // Check that this goroutine is still running on a different + // thread. + self := C.pthread_self() + if C.pthread_equal(mainThread, self) != 0 { + println("goroutine migrated to locked thread") + os.Exit(1) + } + println("OK") +} + +func LockOSThreadAlt() { + // This is running locked to the main OS thread. + + var subThread C.pthread_t + ready := make(chan bool, 1) + C.threadExited = 0 + go func() { + // This goroutine must be running on a new thread. + runtime.LockOSThread() + subThread = C.pthread_self() + // Register a pthread destructor so we can tell this + // thread has exited. + var key C.pthread_key_t + C.pthread_key_create(&key, (*[0]byte)(unsafe.Pointer(C.setExited))) + C.pthread_setspecific(key, unsafe.Pointer(new(int))) + ready <- true + // Exit with the thread locked. + }() + <-ready + for i := 0; i < 100; i++ { + time.Sleep(1 * time.Millisecond) + // Check that this goroutine is running on a different thread. + self := C.pthread_self() + if C.pthread_equal(subThread, self) != 0 { + println("locked thread reused") + os.Exit(1) + } + if atomic.LoadUint32((*uint32)(&C.threadExited)) != 0 { + println("OK") + return + } + } + println("sub thread still running") + os.Exit(1) +} -- 2.48.1