runtime/race: implement race detector for ppc64le

This adds the support to enable the race detector for ppc64le.

Added runtime/race_ppc64le.s to manage the calls from Go to the
LLVM tsan functions, mostly converting from the Go ABI to the
PPC64 ABI expected by Clang generated code.

Changed racewalk.go to call racefuncenterfp instead of racefuncenter
on ppc64le to allow the caller pc to be obtained in the asm code
before calling the tsan version.

Changed the set up code for racecallbackthunk so it doesn't use
the autogenerated save and restore of the link register since that
sequence uses registers inconsistent with the normal ppc64 ABI.

Made various changes to recognize that race is supported for
ppc64le.

Ensured that tls_g is updated and accessible from race_linux_ppc64le.s
so that the race ctx can be obtained and passed to tsan functions.

This enables the race tests for ppc64le in cmd/dist/test.go and
increases the timeout when running the benchmarks with the -race
option to avoid timing out.

Updates #24354, #23731

Change-Id: Ib97dc7ac313e6313c836dc7d2fb698f9d8fba3ef
Reviewed-on: https://go-review.googlesource.com/107935
Run-TryBot: Lynn Boger <laboger@linux.vnet.ibm.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Bryan C. Mills <bcmills@google.com>
Reviewed-by: Brad Fitzpatrick <bradfitz@golang.org>

diff --git a/src/cmd/compile/internal/gc/builtin.go b/src/cmd/compile/internal/gc/builtin.go
index 6b416c8a5c99e355f49273f5d03629efc0627945..ec8f1093b6f5aab8d2064a1bca48b706992f60c5 100644 (file)
--- a/src/cmd/compile/internal/gc/builtin.go
+++ b/src/cmd/compile/internal/gc/builtin.go
@@ -141,6 +141,7 @@ var runtimeDecls = [...]struct {
        {"uint32tofloat64", funcTag, 109},
        {"complex128div", funcTag, 110},
        {"racefuncenter", funcTag, 111},
+       {"racefuncenterfp", funcTag, 5},
        {"racefuncexit", funcTag, 5},
        {"raceread", funcTag, 111},
        {"racewrite", funcTag, 111},

diff --git a/src/cmd/compile/internal/gc/builtin/runtime.go b/src/cmd/compile/internal/gc/builtin/runtime.go
index d459c07cbeca8e0ac6f8c2822732f14b1c711277..140b7f3b2d0dc6953d33eac5b1788d67b3efd55b 100644 (file)
--- a/src/cmd/compile/internal/gc/builtin/runtime.go
+++ b/src/cmd/compile/internal/gc/builtin/runtime.go
@@ -183,6 +183,7 @@ func complex128div(num complex128, den complex128) (quo complex128)
 
 // race detection
 func racefuncenter(uintptr)
+func racefuncenterfp()
 func racefuncexit()
 func raceread(uintptr)
 func racewrite(uintptr)

diff --git a/src/cmd/compile/internal/gc/racewalk.go b/src/cmd/compile/internal/gc/racewalk.go
index 8ae080ab40861f343e13467a12ea28a0fcb8ecf7..df0e5f4059035f4c9f18c3c09d58cbbeeac00586 100644 (file)
--- a/src/cmd/compile/internal/gc/racewalk.go
+++ b/src/cmd/compile/internal/gc/racewalk.go
@@ -7,6 +7,7 @@ package gc
 import (
        "cmd/compile/internal/types"
        "cmd/internal/src"
+       "cmd/internal/sys"
 )
 
 // The racewalk pass is currently handled in two parts.
@@ -58,17 +59,23 @@ func instrument(fn *Node) {
                lno := lineno
                lineno = src.NoXPos
 
-               // nodpc is the PC of the caller as extracted by
-               // getcallerpc. We use -widthptr(FP) for x86.
-               // BUG: this will not work on arm.
-               nodpc := nodfp.copy()
-               nodpc.Type = types.Types[TUINTPTR]
-               nodpc.Xoffset = int64(-Widthptr)
-               fn.Func.Dcl = append(fn.Func.Dcl, nodpc)
-
-               fn.Func.Enter.Prepend(mkcall("racefuncenter", nil, nil, nodpc))
-               fn.Func.Exit.Append(mkcall("racefuncexit", nil, nil))
+               if thearch.LinkArch.Arch == sys.ArchPPC64LE {
+                       fn.Func.Enter.Prepend(mkcall("racefuncenterfp", nil, nil))
+                       fn.Func.Exit.Append(mkcall("racefuncexit", nil, nil))
+               } else {
 
+                       // nodpc is the PC of the caller as extracted by
+                       // getcallerpc. We use -widthptr(FP) for x86.
+                       // BUG: This only works for amd64. This will not
+                       // work on arm or others that might support
+                       // race in the future.
+                       nodpc := nodfp.copy()
+                       nodpc.Type = types.Types[TUINTPTR]
+                       nodpc.Xoffset = int64(-Widthptr)
+                       fn.Func.Dcl = append(fn.Func.Dcl, nodpc)
+                       fn.Func.Enter.Prepend(mkcall("racefuncenter", nil, nil, nodpc))
+                       fn.Func.Exit.Append(mkcall("racefuncexit", nil, nil))
+               }
                lineno = lno
        }
 }

diff --git a/src/cmd/dist/test.go b/src/cmd/dist/test.go
index a1c470cc97cc5562e3598647b52a2fb0accc84e9..ac43701d888abf0010947b37a167c256abfed65c 100644 (file)
--- a/src/cmd/dist/test.go
+++ b/src/cmd/dist/test.go
@@ -312,7 +312,6 @@ func (t *tester) registerStdTest(pkg string) {
                                        break
                                }
                        }
-
                        args := []string{
                                "test",
                                short(),
@@ -355,7 +354,8 @@ func (t *tester) registerRaceBenchTest(pkg string) {
                                "test",
                                short(),
                                "-race",
-                               "-run=^$", // nothing. only benchmarks.
+                               t.timeout(1200), // longer timeout for race with benchmarks
+                               "-run=^$",       // nothing. only benchmarks.
                                "-benchtime=.1s",
                                "-cpu=4",
                        }
@@ -1318,7 +1318,7 @@ func (t *tester) raceDetectorSupported() bool {
        case "linux", "darwin", "freebsd", "windows":
                // The race detector doesn't work on Alpine Linux:
                // golang.org/issue/14481
-               return t.cgoEnabled && goarch == "amd64" && gohostos == goos && !isAlpineLinux()
+               return t.cgoEnabled && (goarch == "amd64" || goarch == "ppc64le") && gohostos == goos && !isAlpineLinux()
        }
        return false
 }

diff --git a/src/cmd/go/internal/work/init.go b/src/cmd/go/internal/work/init.go
index 4d3c5cbd17a01c4a069af4b9dab3366f5c77c716..1081e5147e06dd5f65187b52539597762211adeb 100644 (file)
--- a/src/cmd/go/internal/work/init.go
+++ b/src/cmd/go/internal/work/init.go
@@ -43,11 +43,16 @@ func instrumentInit() {
                fmt.Fprintf(os.Stderr, "-msan is not supported on %s/%s\n", cfg.Goos, cfg.Goarch)
                os.Exit(2)
        }
-       if cfg.BuildRace && (cfg.Goarch != "amd64" || cfg.Goos != "linux" && cfg.Goos != "freebsd" && cfg.Goos != "darwin" && cfg.Goos != "windows") {
-               fmt.Fprintf(os.Stderr, "go %s: -race is only supported on linux/amd64, freebsd/amd64, darwin/amd64 and windows/amd64\n", flag.Args()[0])
-               os.Exit(2)
+       if cfg.BuildRace {
+               platform := cfg.Goos + "/" + cfg.Goarch
+               switch platform {
+               default:
+                       fmt.Fprintf(os.Stderr, "go %s: -race is only supported on linux/amd64, linux/ppc64le, freebsd/amd64, darwin/amd64 and windows/amd64\n", flag.Args()[0])
+                       os.Exit(2)
+               case "linux/amd64", "linux/ppc64le", "freebsd/amd64", "darwin/amd64", "windows/amd64":
+                       // race supported on these platforms
+               }
        }
-
        mode := "race"
        if cfg.BuildMSan {
                mode = "msan"

diff --git a/src/cmd/internal/obj/ppc64/obj9.go b/src/cmd/internal/obj/ppc64/obj9.go
index 7bb21ab439363e318a26667a1e8b518b368ac250..f42d6758050ba80fa713497ee81069efc6cd79ec 100644 (file)
--- a/src/cmd/internal/obj/ppc64/obj9.go
+++ b/src/cmd/internal/obj/ppc64/obj9.go
@@ -502,7 +502,10 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) {
                                q = c.stacksplit(q, autosize) // emit split check
                        }
 
-                       if autosize != 0 {
+                       // Special handling of the racecall thunk. Assume that its asm code will
+                       // save the link register and update the stack, since that code is
+                       // called directly from C/C++ and can't clobber REGTMP (R31).
+                       if autosize != 0 && c.cursym.Name != "runtime.racecallbackthunk" {
                                // Save the link register and update the SP.  MOVDU is used unless
                                // the frame size is too large.  The link register must be saved
                                // even for non-empty leaf functions so that traceback works.
@@ -678,7 +681,7 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) {
                        retTarget := p.To.Sym
 
                        if c.cursym.Func.Text.Mark&LEAF != 0 {
-                               if autosize == 0 {
+                               if autosize == 0 || c.cursym.Name == "runtime.racecallbackthunk" {
                                        p.As = ABR
                                        p.From = obj.Addr{}
                                        if retTarget == nil {
@@ -747,8 +750,8 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) {
                                p.Link = q
                                p = q
                        }
-
-                       if autosize != 0 {
+                       prev := p
+                       if autosize != 0 && c.cursym.Name != "runtime.racecallbackthunk" {
                                q = c.newprog()
                                q.As = AADD
                                q.Pos = p.Pos
@@ -759,7 +762,8 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) {
                                q.Spadj = -autosize
 
                                q.Link = p.Link
-                               p.Link = q
+                               prev.Link = q
+                               prev = q
                        }
 
                        q1 = c.newprog()
@@ -776,7 +780,7 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) {
                        q1.Spadj = +autosize
 
                        q1.Link = q.Link
-                       q.Link = q1
+                       prev.Link = q1
                case AADD:
                        if p.To.Type == obj.TYPE_REG && p.To.Reg == REGSP && p.From.Type == obj.TYPE_CONST {
                                p.Spadj = int32(-p.From.Offset)

diff --git a/src/cmd/link/internal/ld/config.go b/src/cmd/link/internal/ld/config.go
index 6685ad50ac052915c951537f90ce85932336079b..18fbea62eef98d1fcfef3d0f1ffaf299d2913d1f 100644 (file)
--- a/src/cmd/link/internal/ld/config.go
+++ b/src/cmd/link/internal/ld/config.go
@@ -196,6 +196,13 @@ func mustLinkExternal(ctxt *Link) (res bool, reason string) {
                return true, objabi.GOARCH + " does not support internal cgo"
        }
 
+       // When the race flag is set, the LLVM tsan relocatable file is linked
+       // into the final binary, which means external linking is required because
+       // internal linking does not support it.
+       if *flagRace && ctxt.Arch.InFamily(sys.PPC64) {
+               return true, "race on ppc64le"
+       }
+
        // Some build modes require work the internal linker cannot do (yet).
        switch ctxt.BuildMode {
        case BuildModeCArchive:

diff --git a/src/race.bash b/src/race.bash
index cafd834777c6826f06f77052258d09afba186611..73cb1e583bc23ea0eac28445e59611c86ba394e6 100755 (executable)
--- a/src/race.bash
+++ b/src/race.bash
@@ -9,7 +9,7 @@
 set -e
 
 function usage {
-       echo 'race detector is only supported on linux/amd64, freebsd/amd64 and darwin/amd64' 1>&2
+       echo 'race detector is only supported on linux/amd64, linux/ppc64le, freebsd/amd64 and darwin/amd64' 1>&2
        exit 1
 }
 
@@ -21,7 +21,7 @@ case $(uname) in
        fi
        ;;
 "Linux")
-       if [ $(uname -m) != "x86_64" ]; then
+       if [ $(uname -m) != "x86_64" ] && [ $(uname -m) != "ppc64le" ]; then
                usage
        fi
        ;;

diff --git a/src/runtime/asm_ppc64x.s b/src/runtime/asm_ppc64x.s
index 3708961d76db705e2560fcd6967973291086ad8b..b6a797640df217cfebb1fa361a3c6bc98ead651f 100644 (file)
--- a/src/runtime/asm_ppc64x.s
+++ b/src/runtime/asm_ppc64x.s
@@ -24,6 +24,7 @@ TEXT runtime·rt0_go(SB),NOSPLIT,$0
        // create istack out of the given (operating system) stack.
        // _cgo_init may update stackguard.
        MOVD    $runtime·g0(SB), g
+       BL      runtime·save_g(SB)
        MOVD    $(-64*1024), R31
        ADD     R31, R1, R3
        MOVD    R3, g_stackguard0(g)

diff --git a/src/runtime/race.go b/src/runtime/race.go
index 09a83567706ae1b1b849a305a5e6864b11c0486b..0124e231fa89f27d099acd7a2bcb8e52b778597c 100644 (file)
--- a/src/runtime/race.go
+++ b/src/runtime/race.go
@@ -292,6 +292,7 @@ var racearenastart uintptr
 var racearenaend uintptr
 
 func racefuncenter(uintptr)
+func racefuncenterfp()
 func racefuncexit()
 func racereadrangepc1(uintptr, uintptr, uintptr)
 func racewriterangepc1(uintptr, uintptr, uintptr)

diff --git a/src/runtime/race/race.go b/src/runtime/race/race.go
index 15e20112a8d36b7e53ce5a2f9b089817ecfd3d80..f702c7a5d408b06ab10c77f26c80139cd5cd9c4d 100644 (file)
--- a/src/runtime/race/race.go
+++ b/src/runtime/race/race.go
@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build race,linux,amd64 race,freebsd,amd64 race,darwin,amd64 race,windows,amd64
+// +build race,linux,amd64 race,freebsd,amd64 race,darwin,amd64 race,windows,amd64 race,linux,ppc64le
 
 package race
 

diff --git a/src/runtime/race_ppc64le.s b/src/runtime/race_ppc64le.s
new file mode 100644 (file)
index 0000000..5c723e0
--- /dev/null
+++ b/src/runtime/race_ppc64le.s
@@ -0,0 +1,568 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build race
+
+#include "go_asm.h"
+#include "go_tls.h"
+#include "funcdata.h"
+#include "textflag.h"
+
+// The following functions allow calling the clang-compiled race runtime directly
+// from Go code without going all the way through cgo.
+// First, it's much faster (up to 50% speedup for real Go programs).
+// Second, it eliminates race-related special cases from cgocall and scheduler.
+// Third, in long-term it will allow to remove cyclic runtime/race dependency on cmd/go.
+
+// A brief recap of the ppc64le calling convention.
+// Arguments are passed in R3, R4, R5 ...
+// SP must be 16-byte aligned.
+
+// Note that for ppc64x, LLVM follows the standard ABI and
+// expects arguments in registers, so these functions move
+// the arguments from storage to the registers expected
+// by the ABI.
+
+// When calling from Go to Clang tsan code:
+// R3 is the 1st argument and is usually the ThreadState*
+// R4-? are the 2nd, 3rd, 4th, etc. arguments
+
+// When calling racecalladdr:
+// R8 is the call target address
+
+// The race ctx is passed in R3 and loaded in
+// racecalladdr.
+//
+// The sequence used to get the race ctx:
+//    MOVD    runtime·tls_g(SB), R10  // offset to TLS
+//    MOVD    0(R13)(R10*1), g         // R13=TLS for this thread, g = R30
+//    MOVD    g_racectx(g), R3         // racectx == ThreadState
+
+// func runtime·RaceRead(addr uintptr)
+// Called from instrumented Go code
+TEXT   runtime·raceread(SB), NOSPLIT, $0-8
+       MOVD    addr+0(FP), R4
+       MOVD    LR, R5 // caller of this?
+       // void __tsan_read(ThreadState *thr, void *addr, void *pc);
+       MOVD    $__tsan_read(SB), R8
+       BR      racecalladdr<>(SB)
+
+TEXT    runtime·RaceRead(SB), NOSPLIT, $0-8
+       BR      runtime·raceread(SB)
+
+// void runtime·racereadpc(void *addr, void *callpc, void *pc)
+TEXT   runtime·racereadpc(SB), NOSPLIT, $0-24
+       MOVD    addr+0(FP), R4
+       MOVD    callpc+8(FP), R5
+       MOVD    pc+16(FP), R6
+       // void __tsan_read_pc(ThreadState *thr, void *addr, void *callpc, void *pc);
+       MOVD    $__tsan_read_pc(SB), R8
+       BR      racecalladdr<>(SB)
+
+// func runtime·RaceWrite(addr uintptr)
+// Called from instrumented Go code
+TEXT   runtime·racewrite(SB), NOSPLIT, $0-8
+       MOVD    addr+0(FP), R4
+       MOVD    LR, R5 // caller has set LR via BL inst
+       // void __tsan_write(ThreadState *thr, void *addr, void *pc);
+       MOVD    $__tsan_write(SB), R8
+       BR      racecalladdr<>(SB)
+
+TEXT    runtime·RaceWrite(SB), NOSPLIT, $0-8
+       JMP     runtime·racewrite(SB)
+
+// void runtime·racewritepc(void *addr, void *callpc, void *pc)
+TEXT   runtime·racewritepc(SB), NOSPLIT, $0-24
+       MOVD    addr+0(FP), R4
+       MOVD    callpc+8(FP), R5
+       MOVD    pc+16(FP), R6
+       // void __tsan_write_pc(ThreadState *thr, void *addr, void *callpc, void *pc);
+       MOVD    $__tsan_write_pc(SB), R8
+       BR      racecalladdr<>(SB)
+
+// func runtime·RaceReadRange(addr, size uintptr)
+// Called from instrumented Go code.
+TEXT   runtime·racereadrange(SB), NOSPLIT, $0-16
+       MOVD    addr+0(FP), R4
+       MOVD    size+8(FP), R5
+       MOVD    LR, R6
+       // void __tsan_read_range(ThreadState *thr, void *addr, uintptr size, void *pc);
+       MOVD    $__tsan_read_range(SB), R8
+       BR      racecalladdr<>(SB)
+
+// void runtime·racereadrangepc1(void *addr, uintptr sz, void *pc)
+TEXT   runtime·racereadrangepc1(SB), NOSPLIT, $0-24
+       MOVD    addr+0(FP), R4
+       MOVD    size+8(FP), R5
+       MOVD    pc+16(FP), R6
+       ADD     $4, R6          // tsan wants return addr
+        // void __tsan_read_range(ThreadState *thr, void *addr, uintptr size, void *pc);
+        MOVD    $__tsan_read_range(SB), R8
+        BR     racecalladdr<>(SB)
+
+TEXT    runtime·RaceReadRange(SB), NOSPLIT, $0-24
+       BR      runtime·racereadrange(SB)
+
+// func runtime·RaceWriteRange(addr, size uintptr)
+// Called from instrumented Go code.
+TEXT   runtime·racewriterange(SB), NOSPLIT, $0-16
+       MOVD    addr+0(FP), R4
+       MOVD    size+8(FP), R5
+       MOVD    LR, R6
+       // void __tsan_write_range(ThreadState *thr, void *addr, uintptr size, void *pc);
+       MOVD    $__tsan_write_range(SB), R8
+       BR      racecalladdr<>(SB)
+
+TEXT    runtime·RaceWriteRange(SB), NOSPLIT, $0-16
+       BR      runtime·racewriterange(SB)
+
+// void runtime·racewriterangepc1(void *addr, uintptr sz, void *pc)
+// Called from instrumented Go code
+TEXT   runtime·racewriterangepc1(SB), NOSPLIT, $0-24
+       MOVD    addr+0(FP), R4
+       MOVD    size+8(FP), R5
+       MOVD    pc+16(FP), R6
+       ADD     $4, R6                  // add 4 to inst offset?
+       // void __tsan_write_range(ThreadState *thr, void *addr, uintptr size, void *pc);
+       MOVD    $__tsan_write_range(SB), R8
+       BR      racecalladdr<>(SB)
+
+// Call a __tsan function from Go code.
+// R8 = tsan function address
+// R3 = *ThreadState a.k.a. g_racectx from g
+// R4 = addr passed to __tsan function
+//
+// Otherwise, setup goroutine context and invoke racecall. Other arguments already set.
+TEXT   racecalladdr<>(SB), NOSPLIT, $0-0
+       MOVD    runtime·tls_g(SB), R10
+       MOVD    0(R13)(R10*1), g
+       MOVD    g_racectx(g), R3        // goroutine context
+       // Check that addr is within [arenastart, arenaend) or within [racedatastart, racedataend).
+       MOVD    runtime·racearenastart(SB), R9
+       CMP     R4, R9
+       BLT     data
+       MOVD    runtime·racearenaend(SB), R9
+       CMP     R4, R9
+       BLT     call
+data:
+       MOVD    runtime·racedatastart(SB), R9
+       CMP     R4, R9
+       BLT     ret
+       MOVD    runtime·racedataend(SB), R9
+       CMP     R4, R9
+       BGT     ret
+call:
+       // Careful!! racecall will save LR on its
+       // stack, which is OK as long as racecalladdr
+       // doesn't change in a way that generates a stack.
+       // racecall should return to the caller of
+       // recalladdr.
+       BR      racecall<>(SB)
+ret:
+       RET
+
+// func runtime·racefuncenterfp()
+// Called from instrumented Go code.
+// Like racefuncenter but doesn't pass an arg, uses the caller pc
+// from the first slot on the stack.
+TEXT   runtime·racefuncenterfp(SB), NOSPLIT, $0-0
+       MOVD    0(R1), R8
+       BR      racefuncenter<>(SB)
+
+// func runtime·racefuncenter(pc uintptr)
+// Called from instrumented Go code.
+// Not used now since gc/racewalk.go doesn't pass the
+// correct caller pc and racefuncenterfp can do it.
+TEXT   runtime·racefuncenter(SB), NOSPLIT, $0-8
+       MOVD    callpc+0(FP), R8
+       BR      racefuncenter<>(SB)
+
+// Common code for racefuncenter/racefuncenterfp
+// R11 = caller's return address
+TEXT   racefuncenter<>(SB), NOSPLIT, $0-0
+       MOVD    runtime·tls_g(SB), R10
+       MOVD    0(R13)(R10*1), g
+       MOVD    g_racectx(g), R3        // goroutine racectx aka *ThreadState
+       MOVD    R8, R4                  // caller pc set by caller in R8
+       // void __tsan_func_enter(ThreadState *thr, void *pc);
+       MOVD    $__tsan_func_enter(SB), R8
+       BR      racecall<>(SB)
+       RET
+
+// func runtime·racefuncexit()
+// Called from Go instrumented code.
+TEXT   runtime·racefuncexit(SB), NOSPLIT, $0-0
+       MOVD    runtime·tls_g(SB), R10
+       MOVD    0(R13)(R10*1), g
+       MOVD    g_racectx(g), R3        // goroutine racectx aka *ThreadState
+       // void __tsan_func_exit(ThreadState *thr);
+       MOVD    $__tsan_func_exit(SB), R8
+       BR      racecall<>(SB)
+
+// Atomic operations for sync/atomic package.
+// Some use the __tsan versions instead
+// R6 = addr of arguments passed to this function
+// R3, R4, R5 set in racecallatomic
+
+// Load atomic in tsan
+TEXT   sync∕atomic·LoadInt32(SB), NOSPLIT, $0-0
+       // void __tsan_go_atomic32_load(ThreadState *thr, uptr cpc, uptr pc, u8 *a);
+       MOVD    $__tsan_go_atomic32_load(SB), R8
+       ADD     $32, R1, R6     // addr of caller's 1st arg
+       BR      racecallatomic<>(SB)
+       RET
+
+TEXT   sync∕atomic·LoadInt64(SB), NOSPLIT, $0-0
+       // void __tsan_go_atomic64_load(ThreadState *thr, uptr cpc, uptr pc, u8 *a);
+       MOVD    $__tsan_go_atomic64_load(SB), R8
+       ADD     $32, R1, R6     // addr of caller's 1st arg
+       BR      racecallatomic<>(SB)
+       RET
+
+TEXT   sync∕atomic·LoadUint32(SB), NOSPLIT, $0-0
+       BR      sync∕atomic·LoadInt32(SB)
+
+TEXT   sync∕atomic·LoadUint64(SB), NOSPLIT, $0-0
+       BR      sync∕atomic·LoadInt64(SB)
+
+TEXT   sync∕atomic·LoadUintptr(SB), NOSPLIT, $0-0
+       BR      sync∕atomic·LoadInt64(SB)
+
+TEXT   sync∕atomic·LoadPointer(SB), NOSPLIT, $0-0
+       BR      sync∕atomic·LoadInt64(SB)
+
+// Store atomic in tsan
+TEXT   sync∕atomic·StoreInt32(SB), NOSPLIT, $0-0
+       // void __tsan_go_atomic32_store(ThreadState *thr, uptr cpc, uptr pc, u8 *a);
+       MOVD    $__tsan_go_atomic32_store(SB), R8
+       ADD     $32, R1, R6     // addr of caller's 1st arg
+       BR      racecallatomic<>(SB)
+
+TEXT   sync∕atomic·StoreInt64(SB), NOSPLIT, $0-0
+       // void __tsan_go_atomic64_store(ThreadState *thr, uptr cpc, uptr pc, u8 *a);
+       MOVD    $__tsan_go_atomic64_store(SB), R8
+       ADD     $32, R1, R6     // addr of caller's 1st arg
+       BR      racecallatomic<>(SB)
+
+TEXT   sync∕atomic·StoreUint32(SB), NOSPLIT, $0-0
+       BR      sync∕atomic·StoreInt32(SB)
+
+TEXT   sync∕atomic·StoreUint64(SB), NOSPLIT, $0-0
+       BR      sync∕atomic·StoreInt64(SB)
+
+TEXT   sync∕atomic·StoreUintptr(SB), NOSPLIT, $0-0
+       BR      sync∕atomic·StoreInt64(SB)
+
+// Swap in tsan
+TEXT   sync∕atomic·SwapInt32(SB), NOSPLIT, $0-0
+       // void __tsan_go_atomic32_exchange(ThreadState *thr, uptr cpc, uptr pc, u8 *a);
+       MOVD    $__tsan_go_atomic32_exchange(SB), R8
+       ADD     $32, R1, R6     // addr of caller's 1st arg
+       BR      racecallatomic<>(SB)
+
+TEXT   sync∕atomic·SwapInt64(SB), NOSPLIT, $0-0
+       // void __tsan_go_atomic64_exchange(ThreadState *thr, uptr cpc, uptr pc, u8 *a)
+       MOVD    $__tsan_go_atomic64_exchange(SB), R8
+       ADD     $32, R1, R6     // addr of caller's 1st arg
+       BR      racecallatomic<>(SB)
+
+TEXT   sync∕atomic·SwapUint32(SB), NOSPLIT, $0-0
+       BR      sync∕atomic·SwapInt32(SB)
+
+TEXT   sync∕atomic·SwapUint64(SB), NOSPLIT, $0-0
+       BR      sync∕atomic·SwapInt64(SB)
+
+TEXT   sync∕atomic·SwapUintptr(SB), NOSPLIT, $0-0
+       BR      sync∕atomic·SwapInt64(SB)
+
+// Add atomic in tsan
+TEXT   sync∕atomic·AddInt32(SB), NOSPLIT, $0-0
+       // void __tsan_go_atomic32_fetch_add(ThreadState *thr, uptr cpc, uptr pc, u8 *a);
+       MOVD    $__tsan_go_atomic32_fetch_add(SB), R8
+       ADD     $64, R1, R6     // addr of caller's 1st arg
+       BL      racecallatomic<>(SB)
+       // The tsan fetch_add result is not as expected by Go,
+       // so the 'add' must be added to the result.
+       MOVW    add+8(FP), R3   // The tsa fetch_add does not return the
+       MOVW    ret+16(FP), R4  // result as expected by go, so fix it.
+       ADD     R3, R4, R3
+       MOVW    R3, ret+16(FP)
+       RET
+
+TEXT   sync∕atomic·AddInt64(SB), NOSPLIT, $0-0
+       // void __tsan_go_atomic64_fetch_add(ThreadState *thr, uptr cpc, uptr pc, u8 *a);
+       MOVD    $__tsan_go_atomic64_fetch_add(SB), R8
+       ADD     $64, R1, R6     // addr of caller's 1st arg
+       BL      racecallatomic<>(SB)
+       // The tsan fetch_add result is not as expected by Go,
+       // so the 'add' must be added to the result.
+       MOVD    add+8(FP), R3
+       MOVD    ret+16(FP), R4
+       ADD     R3, R4, R3
+       MOVD    R3, ret+16(FP)
+       RET
+
+TEXT   sync∕atomic·AddUint32(SB), NOSPLIT, $0-0
+       BR      sync∕atomic·AddInt32(SB)
+
+TEXT   sync∕atomic·AddUint64(SB), NOSPLIT, $0-0
+       BR      sync∕atomic·AddInt64(SB)
+
+TEXT   sync∕atomic·AddUintptr(SB), NOSPLIT, $0-0
+       BR      sync∕atomic·AddInt64(SB)
+
+// CompareAndSwap in tsan
+TEXT   sync∕atomic·CompareAndSwapInt32(SB), NOSPLIT, $0-0
+       // void __tsan_go_atomic32_compare_exchange(
+       //   ThreadState *thr, uptr cpc, uptr pc, u8 *a)
+       MOVD    $__tsan_go_atomic32_compare_exchange(SB), R8
+       ADD     $32, R1, R6     // addr of caller's 1st arg
+       BR      racecallatomic<>(SB)
+
+TEXT   sync∕atomic·CompareAndSwapInt64(SB), NOSPLIT, $0-0
+       // void __tsan_go_atomic32_compare_exchange(
+       //   ThreadState *thr, uptr cpc, uptr pc, u8 *a)
+       MOVD    $__tsan_go_atomic64_compare_exchange(SB), R8
+       ADD     $32, R1, R6     // addr of caller's 1st arg
+       BR      racecallatomic<>(SB)
+
+TEXT   sync∕atomic·CompareAndSwapUint32(SB), NOSPLIT, $0-0
+       BR      sync∕atomic·CompareAndSwapInt32(SB)
+
+TEXT   sync∕atomic·CompareAndSwapUint64(SB), NOSPLIT, $0-0
+       BR      sync∕atomic·CompareAndSwapInt64(SB)
+
+TEXT   sync∕atomic·CompareAndSwapUintptr(SB), NOSPLIT, $0-0
+       BR      sync∕atomic·CompareAndSwapInt64(SB)
+
+// Common function used to call tsan's atomic functions
+// R3 = *ThreadState
+// R4 = TODO: What's this supposed to be?
+// R5 = caller pc
+// R6 = addr of incoming arg list
+// R8 contains addr of target function.
+TEXT   racecallatomic<>(SB), NOSPLIT, $0-0
+       // Trigger SIGSEGV early if address passed to atomic function is bad.
+       MOVD    (R6), R7        // 1st arg is addr
+       MOVD    (R7), R9        // segv here if addr is bad
+       // Check that addr is within [arenastart, arenaend) or within [racedatastart, racedataend).
+       MOVD    runtime·racearenastart(SB), R9
+       CMP     R7, R9
+       BLT     racecallatomic_data
+       MOVD    runtime·racearenaend(SB), R9
+       CMP     R7, R9
+       BLT     racecallatomic_ok
+racecallatomic_data:
+       MOVD    runtime·racedatastart(SB), R9
+       CMP     R7, R9
+       BLT     racecallatomic_ignore
+       MOVD    runtime·racedataend(SB), R9
+       CMP     R7, R9
+       BGE     racecallatomic_ignore
+racecallatomic_ok:
+       // Addr is within the good range, call the atomic function.
+       MOVD    runtime·tls_g(SB), R10
+       MOVD    0(R13)(R10*1), g
+       MOVD    g_racectx(g), R3        // goroutine racectx aka *ThreadState
+       MOVD    R8, R5                  // pc is the function called
+       MOVD    (R1), R4                // caller pc from stack
+       BL      racecall<>(SB)          // BL needed to maintain stack consistency
+       RET                             //
+racecallatomic_ignore:
+       // Addr is outside the good range.
+       // Call __tsan_go_ignore_sync_begin to ignore synchronization during the atomic op.
+       // An attempt to synchronize on the address would cause crash.
+       MOVD    R8, R15 // save the original function
+       MOVD    R6, R17 // save the original arg list addr
+       MOVD    $__tsan_go_ignore_sync_begin(SB), R8 // func addr to call
+       MOVD    runtime·tls_g(SB), R10
+       MOVD    0(R13)(R10*1), g
+       MOVD    g_racectx(g), R3        // goroutine context
+       BL      racecall<>(SB)
+       MOVD    R15, R8 // restore the original function
+       MOVD    R17, R6 // restore arg list addr
+       // Call the atomic function.
+       // racecall will call LLVM race code which might clobber r30 (g)
+       MOVD    runtime·tls_g(SB), R10
+        MOVD    0(R13)(R10*1), g
+
+       MOVD    g_racectx(g), R3
+       MOVD    R8, R4          // pc being called same TODO as above
+       MOVD    (R1), R5        // caller pc from latest LR
+       BL      racecall<>(SB)
+       // Call __tsan_go_ignore_sync_end.
+       MOVD    $__tsan_go_ignore_sync_end(SB), R8
+       MOVD    g_racectx(g), R3        // goroutine context g should sitll be good?
+       BL      racecall<>(SB)
+       RET
+
+// void runtime·racecall(void(*f)(...), ...)
+// Calls C function f from race runtime and passes up to 4 arguments to it.
+// The arguments are never heap-object-preserving pointers, so we pretend there are no arguments.
+TEXT   runtime·racecall(SB), NOSPLIT, $0-0
+       MOVD    fn+0(FP), R8
+       MOVD    arg0+8(FP), R3
+       MOVD    arg1+16(FP), R4
+       MOVD    arg2+24(FP), R5
+       MOVD    arg3+32(FP), R6
+       JMP     racecall<>(SB)
+
+// Finds g0 and sets its stack
+// Arguments were loaded for call from Go to C
+TEXT   racecall<>(SB), NOSPLIT, $0-0
+       // Set the LR slot for the ppc64 ABI
+       MOVD    LR, R10
+       MOVD    R10, 0(R1)      // Go expectation
+       MOVD    R10, 16(R1)     // C ABI
+       // Get info from the current goroutine
+       MOVD    runtime·tls_g(SB), R10 // g offset in TLS
+       MOVD    0(R13)(R10*1), g        // R13 = current TLS
+       MOVD    g_m(g), R7              // m for g
+       MOVD    R1, R16                 // callee-saved, preserved across C call
+       MOVD    m_g0(R7), R10           // g0 for m
+       CMP     R10, g                  // same g0?
+       BEQ     call                    // already on g0
+       MOVD    (g_sched+gobuf_sp)(R10), R1 // switch R1
+call:
+       MOVD    R8, CTR                 // R8 = caller addr
+       MOVD    R8, R12                 // expected by PPC64 ABI
+       BL      (CTR)
+       XOR     R0, R0                  // clear R0 on return from Clang
+       MOVD    R16, R1                 // restore R1; R16 nonvol in Clang
+       MOVD    runtime·tls_g(SB), R10 // find correct g
+       MOVD    0(R13)(R10*1), g
+       MOVD    16(R1), R10             // LR was saved away, restore for return
+       MOVD    R10, LR
+       RET
+
+// C->Go callback thunk that allows to call runtime·racesymbolize from C code.
+// Direct Go->C race call has only switched SP, finish g->g0 switch by setting correct g.
+// The overall effect of Go->C->Go call chain is similar to that of mcall.
+// RARG0 contains command code. RARG1 contains command-specific context.
+// See racecallback for command codes.
+TEXT   runtime·racecallbackthunk(SB), NOSPLIT, $-8
+       // Handle command raceGetProcCmd (0) here.
+       // First, code below assumes that we are on curg, while raceGetProcCmd
+       // can be executed on g0. Second, it is called frequently, so will
+       // benefit from this fast path.
+       XOR     R0, R0          // clear R0 since we came from C code
+       CMP     R3, $0
+       BNE     rest
+       // g0 TODO: Don't modify g here since R30 is nonvolatile
+       MOVD    g, R9
+       MOVD    runtime·tls_g(SB), R10
+       MOVD    0(R13)(R10*1), g
+       MOVD    g_m(g), R3
+       MOVD    m_p(R3), R3
+       MOVD    p_racectx(R3), R3
+       MOVD    R3, (R4)
+       MOVD    R9, g           // restore R30 ??
+       RET
+
+       // This is all similar to what cgo does
+       // Save registers according to the ppc64 ABI
+rest:
+       MOVD    LR, R10 // save link register
+       MOVD    R10, 16(R1)
+       MOVW    CR, R10
+       MOVW    R10, 8(R1)
+       MOVDU   R1, -336(R1) // Allocate frame needed for register save area
+
+       MOVD    R14, 40(R1)
+       MOVD    R15, 48(R1)
+       MOVD    R16, 56(R1)
+       MOVD    R17, 64(R1)
+       MOVD    R18, 72(R1)
+       MOVD    R19, 80(R1)
+       MOVD    R20, 88(R1)
+       MOVD    R21, 96(R1)
+       MOVD    R22, 104(R1)
+       MOVD    R23, 112(R1)
+       MOVD    R24, 120(R1)
+       MOVD    R25, 128(R1)
+       MOVD    R26, 136(R1)
+       MOVD    R27, 144(R1)
+       MOVD    R28, 152(R1)
+       MOVD    R29, 160(R1)
+       MOVD    g, 168(R1) // R30
+       MOVD    R31, 176(R1)
+       FMOVD   F14, 184(R1)
+       FMOVD   F15, 192(R1)
+       FMOVD   F16, 200(R1)
+       FMOVD   F17, 208(R1)
+       FMOVD   F18, 216(R1)
+       FMOVD   F19, 224(R1)
+       FMOVD   F20, 232(R1)
+       FMOVD   F21, 240(R1)
+       FMOVD   F22, 248(R1)
+       FMOVD   F23, 256(R1)
+       FMOVD   F24, 264(R1)
+       FMOVD   F25, 272(R1)
+       FMOVD   F26, 280(R1)
+       FMOVD   F27, 288(R1)
+       FMOVD   F28, 296(R1)
+       FMOVD   F29, 304(R1)
+       FMOVD   F30, 312(R1)
+       FMOVD   F31, 320(R1)
+
+       MOVD    runtime·tls_g(SB), R10
+       MOVD    0(R13)(R10*1), g
+
+       MOVD    g_m(g), R7
+       MOVD    m_g0(R7), g // set g = m-> g0
+       MOVD    R3, cmd+0(FP) // can't use R1 here ?? use input args and assumer caller expects those?
+       MOVD    R4, ctx+8(FP) // can't use R1 here ??
+       BL      runtime·racecallback(SB)
+       // All registers are clobbered after Go code, reload.
+       MOVD    runtime·tls_g(SB), R10
+        MOVD    0(R13)(R10*1), g
+
+       MOVD    g_m(g), R7
+       MOVD    m_curg(R7), g // restore g = m->curg
+       MOVD    40(R1), R14
+       MOVD    48(R1), R15
+       MOVD    56(R1), R16
+       MOVD    64(R1), R17
+       MOVD    72(R1), R18
+       MOVD    80(R1), R19
+       MOVD    88(R1), R20
+       MOVD    96(R1), R21
+       MOVD    104(R1), R22
+       MOVD    112(R1), R23
+       MOVD    120(R1), R24
+       MOVD    128(R1), R25
+       MOVD    136(R1), R26
+       MOVD    144(R1), R27
+       MOVD    152(R1), R28
+       MOVD    160(R1), R29
+       MOVD    168(R1), g // R30
+       MOVD    176(R1), R31
+       FMOVD   184(R1), F14
+       FMOVD   192(R1), F15
+       FMOVD   200(R1), F16
+       FMOVD   208(R1), F17
+       FMOVD   216(R1), F18
+       FMOVD   224(R1), F19
+       FMOVD   232(R1), F20
+       FMOVD   240(R1), F21
+       FMOVD   248(R1), F22
+       FMOVD   256(R1), F23
+       FMOVD   264(R1), F24
+       FMOVD   272(R1), F25
+       FMOVD   280(R1), F26
+       FMOVD   288(R1), F27
+       FMOVD   296(R1), F28
+       FMOVD   304(R1), F29
+       FMOVD   312(R1), F30
+       FMOVD   320(R1), F31
+
+       ADD     $336, R1
+       MOVD    8(R1), R10
+       MOVFL   R10, $0xff // Restore of CR
+       MOVD    16(R1), R10     // needed?
+       MOVD    R10, LR
+       RET
+
+// tls_g, g value for each thread in TLS
+GLOBL runtime·tls_g+0(SB), TLSBSS+DUPOK, $8

diff --git a/src/runtime/tls_ppc64x.s b/src/runtime/tls_ppc64x.s
index 69c0d9eb99ac89c19600ed145a297b5af8403af0..ed94989b6914671da8c953785334e5c14386026d 100644 (file)
--- a/src/runtime/tls_ppc64x.s
+++ b/src/runtime/tls_ppc64x.s
@@ -46,4 +46,4 @@ TEXT runtime·load_g(SB),NOSPLIT|NOFRAME,$0-0
        MOVD    0(R13)(R31*1), g
        RET
 
-GLOBL runtime·tls_g+0(SB), TLSBSS, $8
+GLOBL runtime·tls_g+0(SB), TLSBSS+DUPOK, $8