//go:noescape
func racewritepc(addr unsafe.Pointer, callpc, pc uintptr)
-type symbolizeContext struct {
+type symbolizeCodeContext struct {
pc uintptr
fn *byte
file *byte
var qq = [...]byte{'?', '?', 0}
var dash = [...]byte{'-', 0}
+const (
+ raceGetProcCmd = iota
+ raceSymbolizeCodeCmd
+ raceSymbolizeDataCmd
+)
+
// Callback from C into Go, runs on g0.
-func racesymbolize(ctx *symbolizeContext) {
+func racecallback(cmd uintptr, ctx unsafe.Pointer) {
+ switch cmd {
+ case raceGetProcCmd:
+ throw("should have been handled by racecallbackthunk")
+ case raceSymbolizeCodeCmd:
+ raceSymbolizeCode((*symbolizeCodeContext)(ctx))
+ case raceSymbolizeDataCmd:
+ raceSymbolizeData((*symbolizeDataContext)(ctx))
+ default:
+ throw("unknown command")
+ }
+}
+
+func raceSymbolizeCode(ctx *symbolizeCodeContext) {
f := findfunc(ctx.pc)
if f == nil {
ctx.fn = &qq[0]
return
}
+type symbolizeDataContext struct {
+ addr uintptr
+ heap uintptr
+ start uintptr
+ size uintptr
+ name *byte
+ file *byte
+ line uintptr
+ res uintptr
+}
+
+func raceSymbolizeData(ctx *symbolizeDataContext) {
+ if _, x, n := findObject(unsafe.Pointer(ctx.addr)); x != nil {
+ ctx.heap = 1
+ ctx.start = uintptr(x)
+ ctx.size = n
+ ctx.res = 1
+ }
+}
+
// Race runtime functions called via runtime·racecall.
//go:linkname __tsan_init __tsan_init
var __tsan_init byte
//go:linkname __tsan_fini __tsan_fini
var __tsan_fini byte
+//go:linkname __tsan_proc_create __tsan_proc_create
+var __tsan_proc_create byte
+
+//go:linkname __tsan_proc_destroy __tsan_proc_destroy
+var __tsan_proc_destroy byte
+
//go:linkname __tsan_map_shadow __tsan_map_shadow
var __tsan_map_shadow byte
//go:linkname __tsan_malloc __tsan_malloc
var __tsan_malloc byte
+//go:linkname __tsan_free __tsan_free
+var __tsan_free byte
+
//go:linkname __tsan_acquire __tsan_acquire
var __tsan_acquire byte
// Mimic what cmd/cgo would do.
//go:cgo_import_static __tsan_init
//go:cgo_import_static __tsan_fini
+//go:cgo_import_static __tsan_proc_create
+//go:cgo_import_static __tsan_proc_destroy
//go:cgo_import_static __tsan_map_shadow
//go:cgo_import_static __tsan_finalizer_goroutine
//go:cgo_import_static __tsan_go_start
//go:cgo_import_static __tsan_go_end
//go:cgo_import_static __tsan_malloc
+//go:cgo_import_static __tsan_free
//go:cgo_import_static __tsan_acquire
//go:cgo_import_static __tsan_release
//go:cgo_import_static __tsan_release_merge
func racefuncexit()
func racereadrangepc1(uintptr, uintptr, uintptr)
func racewriterangepc1(uintptr, uintptr, uintptr)
-func racesymbolizethunk(uintptr)
+func racecallbackthunk(uintptr)
// racecall allows calling an arbitrary function f from C race runtime
// with up to 4 uintptr arguments.
}
//go:nosplit
-func raceinit() uintptr {
+func raceinit() (gctx, pctx uintptr) {
// cgo is required to initialize libc, which is used by race runtime
if !iscgo {
throw("raceinit: race build must use cgo")
}
- var racectx uintptr
- racecall(&__tsan_init, uintptr(unsafe.Pointer(&racectx)), funcPC(racesymbolizethunk), 0, 0)
+ racecall(&__tsan_init, uintptr(unsafe.Pointer(&gctx)), uintptr(unsafe.Pointer(&pctx)), funcPC(racecallbackthunk), 0)
// Round data segment to page boundaries, because it's used in mmap().
start := ^uintptr(0)
racedatastart = start
racedataend = start + size
- return racectx
+ return
}
//go:nosplit
racecall(&__tsan_fini, 0, 0, 0, 0)
}
+//go:nosplit
+func raceproccreate() uintptr {
+ var ctx uintptr
+ racecall(&__tsan_proc_create, uintptr(unsafe.Pointer(&ctx)), 0, 0, 0)
+ return ctx
+}
+
+//go:nosplit
+func raceprocdestroy(ctx uintptr) {
+ racecall(&__tsan_proc_destroy, ctx, 0, 0, 0)
+}
+
//go:nosplit
func racemapshadow(addr unsafe.Pointer, size uintptr) {
if racearenastart == 0 {
//go:nosplit
func racemalloc(p unsafe.Pointer, sz uintptr) {
- racecall(&__tsan_malloc, uintptr(p), sz, 0, 0)
+ racecall(&__tsan_malloc, 0, 0, uintptr(p), sz)
+}
+
+//go:nosplit
+func racefree(p unsafe.Pointer, sz uintptr) {
+ racecall(&__tsan_free, uintptr(p), sz, 0, 0)
}
//go:nosplit
//go:nosplit
func racerelease(addr unsafe.Pointer) {
- _g_ := getg()
- if _g_.raceignore != 0 || !isvalidaddr(addr) {
- return
- }
- racereleaseg(_g_, addr)
+ racereleaseg(getg(), addr)
}
//go:nosplit
func raceReadObjectPC(t *_type, addr unsafe.Pointer, callerpc, pc uintptr) { throw("race") }
func raceWriteObjectPC(t *_type, addr unsafe.Pointer, callerpc, pc uintptr) { throw("race") }
-func raceinit() uintptr { throw("race"); return 0 }
+func raceinit() (uintptr, uintptr) { throw("race"); return 0, 0 }
func racefini() { throw("race") }
+func raceproccreate() uintptr { throw("race"); return 0 }
+func raceprocdestroy(ctx uintptr) { throw("race") }
func racemapshadow(addr unsafe.Pointer, size uintptr) { throw("race") }
func racewritepc(addr unsafe.Pointer, callerpc, pc uintptr) { throw("race") }
func racereadpc(addr unsafe.Pointer, callerpc, pc uintptr) { throw("race") }
func racereleasemergeg(gp *g, addr unsafe.Pointer) { throw("race") }
func racefingo() { throw("race") }
func racemalloc(p unsafe.Pointer, sz uintptr) { throw("race") }
+func racefree(p unsafe.Pointer, sz uintptr) { throw("race") }
func racegostart(pc uintptr) uintptr { throw("race"); return 0 }
func racegoend() { throw("race") }
// C->Go callback thunk that allows to call runtime·racesymbolize from C code.
// Direct Go->C race call has only switched SP, finish g->g0 switch by setting correct g.
// The overall effect of Go->C->Go call chain is similar to that of mcall.
-TEXT runtime·racesymbolizethunk(SB), NOSPLIT, $56-8
+// RARG0 contains command code. RARG1 contains command-specific context.
+// See racecallback for command codes.
+TEXT runtime·racecallbackthunk(SB), NOSPLIT, $56-8
+ // Handle command raceGetProcCmd (0) here.
+ // First, code below assumes that we are on curg, while raceGetProcCmd
+ // can be executed on g0. Second, it is called frequently, so will
+ // benefit from this fast path.
+ CMPQ RARG0, $0
+ JNE rest
+ get_tls(RARG0)
+ MOVQ g(RARG0), RARG0
+ MOVQ g_m(RARG0), RARG0
+ MOVQ m_p(RARG0), RARG0
+ MOVQ p_racectx(RARG0), RARG0
+ MOVQ RARG0, (RARG1)
+ RET
+
+rest:
// Save callee-saved registers (Go code won't respect that).
// This is superset of darwin/linux/windows registers.
PUSHQ BX
MOVQ g_m(R13), R13
MOVQ m_g0(R13), R14
MOVQ R14, g(R12) // g = m->g0
+ PUSHQ RARG1 // func arg
PUSHQ RARG0 // func arg
- CALL runtime·racesymbolize(SB)
+ CALL runtime·racecallback(SB)
+ POPQ R12
POPQ R12
// All registers are smashed after Go code, reload.
get_tls(R12)