runtime: linux/arm64 cgo support

Change-Id: I309e3df7608b9eef9339196fdc50dedf5f9439f3
Reviewed-on: https://go-review.googlesource.com/8450
Reviewed-by: Aram Hăvărneanu <aram@mgk.ro>

diff --git a/src/runtime/asm_arm64.s b/src/runtime/asm_arm64.s
index 0b21a1da2f42313934a5911545e0180357f96426..68d0447f037ff1a073daa15b5556ef8520729519 100644 (file)
--- a/src/runtime/asm_arm64.s
+++ b/src/runtime/asm_arm64.s
@@ -4,6 +4,7 @@
 
 #include "go_asm.h"
 #include "go_tls.h"
+#include "tls_arm64.h"
 #include "funcdata.h"
 #include "textflag.h"
 
@@ -32,7 +33,16 @@ TEXT runtime·rt0_go(SB),NOSPLIT,$0
        CMP     $0, R12
        BEQ     nocgo
 
-       BL      runtime·abort(SB)
+       MRS_TPIDR_R0                    // load TLS base pointer
+       MOVD    R0, R3                  // arg 3: TLS base pointer
+       //MOVD  $runtime·tlsg(SB), R2  // arg 2: tlsg
+       MOVD    $0x10, R2               // arg 2: tlsg TODO(minux): hardcoded for linux
+       MOVD    $setg_gcc<>(SB), R1     // arg 1: setg
+       MOVD    g, R0                   // arg 0: G
+       BL      (R12)
+       MOVD    _cgo_init(SB), R12
+       CMP     $0, R12
+       BEQ     nocgo
 
 nocgo:
        // update stackguard after _cgo_init
@@ -504,62 +514,61 @@ TEXT gosave<>(SB),NOSPLIT,$-8
 // asmcgocall(void(*fn)(void*), void *arg)
 // Call fn(arg) on the scheduler stack,
 // aligned appropriately for the gcc ABI.
-// See cgocall.c for more details.
+// See cgocall.go for more details.
 TEXT ·asmcgocall(SB),NOSPLIT,$0-16
-       MOVD    fn+0(FP), R3
-       MOVD    arg+8(FP), R4
+       MOVD    fn+0(FP), R1
+       MOVD    arg+8(FP), R0
        BL      asmcgocall<>(SB)
        RET
 
-TEXT ·asmcgocall_errno(SB),NOSPLIT,$0-20
-       MOVD    fn+0(FP), R3
-       MOVD    arg+8(FP), R4
+TEXT ·asmcgocall_errno(SB),NOSPLIT,$0-24
+       MOVD    fn+0(FP), R1
+       MOVD    arg+8(FP), R0
        BL      asmcgocall<>(SB)
-       MOVW    R0, ret+16(FP)
+       MOVD    R0, ret+16(FP)
        RET
 
-// asmcgocall common code. fn in R3, arg in R4. returns errno in R0.
+// asmcgocall common code. fn in R1, arg in R0. returns errno in R0.
 TEXT asmcgocall<>(SB),NOSPLIT,$0-0
        MOVD    RSP, R2         // save original stack pointer
-       MOVD    g, R5
+       MOVD    g, R4
 
        // Figure out if we need to switch to m->g0 stack.
        // We get called to create new OS threads too, and those
        // come in on the m->g0 stack already.
-       MOVD    g_m(g), R6
-       MOVD    m_g0(R6), R6
-       CMP     R6, g
+       MOVD    g_m(g), R8
+       MOVD    m_g0(R8), R3
+       CMP     R3, g
        BEQ     g0
+       MOVD    R0, R9  // gosave<> and save_g might clobber R0
        BL      gosave<>(SB)
-       MOVD    R6, g
+       MOVD    R3, g
        BL      runtime·save_g(SB)
-       MOVD    (g_sched+gobuf_sp)(g), R13
-       MOVD    R13, RSP
+       MOVD    (g_sched+gobuf_sp)(g), R0
+       MOVD    R0, RSP
+       MOVD    R9, R0
 
        // Now on a scheduling stack (a pthread-created stack).
 g0:
-       // Save room for two of our pointers, plus 32 bytes of callee
-       // save area that lives on the caller stack.
+       // Save room for two of our pointers /*, plus 32 bytes of callee
+       // save area that lives on the caller stack. */
        MOVD    RSP, R13
-       SUB     $48, R13
-       AND     $~15, R13       // 16-byte alignment for gcc ABI
+       SUB     $16, R13
        MOVD    R13, RSP
-       MOVD    R5, 40(RSP)     // save old g on stack
-       MOVD    (g_stack+stack_hi)(R5), R5
-       SUB     R2, R5
-       MOVD    R5, 32(RSP)     // save depth in old g stack (can't just save RSP, as stack might be copied during a callback)
-       MOVD    R0, 0(RSP)      // clear back chain pointer (TODO can we give it real back trace information?)
-       // This is a "global call", so put the global entry point in r12
-       MOVD    R3, R12
-       MOVD    R4, R0
-       BL      (R12)
+       MOVD    R4, 0(RSP)      // save old g on stack
+       MOVD    (g_stack+stack_hi)(R4), R4
+       SUB     R2, R4
+       MOVD    R4, 8(RSP)      // save depth in old g stack (can't just save SP, as stack might be copied during a callback)
+       BL      (R1)
+       MOVD    R0, R9
 
        // Restore g, stack pointer.  R0 is errno, so don't touch it
-       MOVD    40(RSP), g
+       MOVD    0(RSP), g
        BL      runtime·save_g(SB)
        MOVD    (g_stack+stack_hi)(g), R5
-       MOVD    32(RSP), R6
+       MOVD    8(RSP), R6
        SUB     R6, R5
+       MOVD    R9, R0
        MOVD    R5, RSP
        RET
 
@@ -567,27 +576,26 @@ g0:
 // Turn the fn into a Go func (by taking its address) and call
 // cgocallback_gofunc.
 TEXT runtime·cgocallback(SB),NOSPLIT,$24-24
-       MOVD    $fn+0(FP), R3
-       MOVD    R3, 8(RSP)
-       MOVD    frame+8(FP), R3
-       MOVD    R3, 16(RSP)
-       MOVD    framesize+16(FP), R3
-       MOVD    R3, 24(RSP)
-       MOVD    $runtime·cgocallback_gofunc(SB), R3
-       BL      (R3)
+       MOVD    $fn+0(FP), R0
+       MOVD    R0, 8(RSP)
+       MOVD    frame+8(FP), R0
+       MOVD    R0, 16(RSP)
+       MOVD    framesize+16(FP), R0
+       MOVD    R0, 24(RSP)
+       MOVD    $runtime·cgocallback_gofunc(SB), R0
+       BL      (R0)
        RET
 
 // cgocallback_gofunc(FuncVal*, void *frame, uintptr framesize)
-// See cgocall.c for more details.
+// See cgocall.go for more details.
 TEXT ·cgocallback_gofunc(SB),NOSPLIT,$16-24
        NO_LOCAL_POINTERS
 
-       // Load m and g from thread-local storage.
+       // Load g from thread-local storage.
        MOVB    runtime·iscgo(SB), R3
-       CMP     $0, R3 
+       CMP     $0, R3
        BEQ     nocgo
-       // TODO(aram):
-       BL runtime·abort(SB)
+       BL      runtime·load_g(SB)
 nocgo:
 
        // If g is nil, Go did not create the current thread.
@@ -598,8 +606,8 @@ nocgo:
        CMP     $0, g
        BNE     havem
        MOVD    g, savedm-8(SP) // g is zero, so is m.
-       MOVD    $runtime·needm(SB), R3
-       BL      (R3)
+       MOVD    $runtime·needm(SB), R0
+       BL      (R0)
 
        // Set m->sched.sp = SP, so that if a panic happens
        // during the function we are about to execute, it will
@@ -612,8 +620,8 @@ nocgo:
        // and then systemstack will try to use it. If we don't set it here,
        // that restored SP will be uninitialized (typically 0) and
        // will not be usable.
-       MOVD    g_m(g), R3
-       MOVD    m_g0(R3), R3
+       MOVD    g_m(g), R8
+       MOVD    m_g0(R8), R3
        MOVD    RSP, R0
        MOVD    R0, (g_sched+gobuf_sp)(R3)
 
@@ -624,7 +632,8 @@ havem:
        // Save current m->g0->sched.sp on stack and then set it to SP.
        // Save current sp in m->g0->sched.sp in preparation for
        // switch back to m->curg stack.
-       // NOTE: unwindm knows that the saved g->sched.sp is at 8(R1) aka savedsp-16(SP).
+       // NOTE: unwindm knows that the saved g->sched.sp is at 16(RSP) aka savedsp-16(SP).
+       // Beware that the frame size is actually 32.
        MOVD    m_g0(R8), R3
        MOVD    (g_sched+gobuf_sp)(R3), R4
        MOVD    R4, savedsp-16(SP)
@@ -650,15 +659,16 @@ havem:
        BL      runtime·save_g(SB)
        MOVD    (g_sched+gobuf_sp)(g), R4 // prepare stack as R4
        MOVD    (g_sched+gobuf_pc)(g), R5
-       MOVD    R5, -24(R4)
-       MOVD    $-24(R4), R0
+       MOVD    R5, -(24+8)(R4) // maintain 16-byte SP alignment
+       MOVD    $-(24+8)(R4), R0
        MOVD    R0, RSP
        BL      runtime·cgocallbackg(SB)
 
        // Restore g->sched (== m->curg->sched) from saved values.
        MOVD    0(RSP), R5
        MOVD    R5, (g_sched+gobuf_pc)(g)
-       MOVD    $24(RSP), R4
+       MOVD    RSP, R4
+       ADD     $(24+8), R4, R4
        MOVD    R4, (g_sched+gobuf_sp)(g)
 
        // Switch back to m->g0's stack and restore m->g0->sched.sp.
@@ -677,13 +687,30 @@ havem:
        MOVD    savedm-8(SP), R6
        CMP     $0, R6
        BNE     droppedm
-       MOVD    $runtime·dropm(SB), R3
-       BL      (R3)
+       MOVD    $runtime·dropm(SB), R0
+       BL      (R0)
 droppedm:
 
        // Done!
        RET
 
+// Called from cgo wrappers, this function returns g->m->curg.stack.hi.
+// Must obey the gcc calling convention.
+TEXT _cgo_topofstack(SB),NOSPLIT,$16
+       // g (R28) and REGTMP (R27)  might be clobbered by load_g. They
+       // are callee-save in the gcc calling convention, so save them.
+       MOVD    R27, savedR27-8(SP)
+       MOVD    g, saveG-16(SP)
+
+       BL      runtime·load_g(SB)
+       MOVD    g_m(g), R0
+       MOVD    m_curg(R0), R0
+       MOVD    (g_stack+stack_hi)(R0), R0
+
+       MOVD    saveG-16(SP), g
+       MOVD    savedR28-8(SP), R27
+       RET
+
 // void setg(G*); set g. for use by needm.
 TEXT runtime·setg(SB), NOSPLIT, $0-8
        MOVD    gg+0(FP), g
@@ -691,23 +718,14 @@ TEXT runtime·setg(SB), NOSPLIT, $0-8
        BL      runtime·save_g(SB)
        RET
 
-// save_g saves the g register into pthread-provided
-// thread-local memory, so that we can call externally compiled
-// ppc64 code that will overwrite this register.
-//
-// If !iscgo, this is a no-op.
-TEXT runtime·save_g(SB),NOSPLIT,$-8-0
-       MOVB    runtime·iscgo(SB), R0
-       CMP     $0, R0
-       BEQ     nocgo
-
-       // TODO: implement cgo.
-       BL      runtime·abort(SB)
-
-nocgo:
+// void setg_gcc(G*); set g called from gcc
+TEXT setg_gcc<>(SB),NOSPLIT,$8
+       MOVD    R0, g
+       MOVD    R27, savedR27-8(SP)
+       BL      runtime·save_g(SB)
+       MOVD    savedR27-8(SP), R27
        RET
 
-
 TEXT runtime·getcallerpc(SB),NOSPLIT,$-8-16
        MOVD    0(RSP), R0
        MOVD    R0, ret+8(FP)

diff --git a/src/runtime/cgocall.go b/src/runtime/cgocall.go
index 052830de5ad1674dcfc4d10565835edb89b156d9..5b24304c1d515224d0237804d1d3eca00cf7d76f 100644 (file)
--- a/src/runtime/cgocall.go
+++ b/src/runtime/cgocall.go
@@ -218,6 +218,10 @@ func cgocallbackg1() {
                // On arm, stack frame is two words and there's a saved LR between
                // SP and the stack frame and between the stack frame and the arguments.
                cb = (*args)(unsafe.Pointer(sp + 4*ptrSize))
+       case "arm64":
+               // On arm64, stack frame is four words and there's a saved LR between
+               // SP and the stack frame and between the stack frame and the arguments.
+               cb = (*args)(unsafe.Pointer(sp + 5*ptrSize))
        case "amd64":
                // On amd64, stack frame is one word, plus caller PC.
                if framepointer_enabled {
@@ -268,6 +272,8 @@ func unwindm(restore *bool) {
                sched.sp = *(*uintptr)(unsafe.Pointer(sched.sp))
        case "arm":
                sched.sp = *(*uintptr)(unsafe.Pointer(sched.sp + 4))
+       case "arm64":
+               sched.sp = *(*uintptr)(unsafe.Pointer(sched.sp + 16))
        case "ppc64", "ppc64le":
                sched.sp = *(*uintptr)(unsafe.Pointer(sched.sp + 8))
        }

diff --git a/src/runtime/sys_linux_arm64.s b/src/runtime/sys_linux_arm64.s
index 28d813f849d8d416f05d6680ddf076f8aeb9b876..df7a9b49422355225ad68254227bb8c0b3f34d0c 100644 (file)
--- a/src/runtime/sys_linux_arm64.s
+++ b/src/runtime/sys_linux_arm64.s
@@ -217,7 +217,10 @@ TEXT runtime·sigtramp(SB),NOSPLIT,$64
        // where g is not set.
        // first save R0, because runtime·load_g will clobber it
        MOVW    R0, 8(RSP)
-       // TODO(minux): iscgo & load_g
+       MOVBU   runtime·iscgo(SB), R0
+       CMP     $0, R0
+       BEQ     2(PC)
+       BL      runtime·load_g(SB)
 
        // check that g exists
        CMP     g, ZR

diff --git a/src/runtime/tls_arm64.h b/src/runtime/tls_arm64.h
new file mode 100644 (file)
index 0000000..5416306
--- /dev/null
+++ b/src/runtime/tls_arm64.h
@@ -0,0 +1,14 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+#ifdef GOOS_linux
+#define TPIDR TPIDR_EL0
+#define MRS_TPIDR_R0 WORD $0xd53bd040
+#endif
+
+// Define something that will break the build if
+// the GOOS is unknown.
+#ifndef TPIDR
+#define MRS_TPIDR_R0 TPIDR_UNKNOWN
+#endif

diff --git a/src/runtime/tls_arm64.s b/src/runtime/tls_arm64.s
new file mode 100644 (file)
index 0000000..3ab087a
--- /dev/null
+++ b/src/runtime/tls_arm64.s
@@ -0,0 +1,31 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+#include "go_asm.h"
+#include "go_tls.h"
+#include "funcdata.h"
+#include "textflag.h"
+#include "tls_arm64.h"
+
+TEXT runtime·load_g(SB),NOSPLIT,$0
+       MOVB    runtime·iscgo(SB), R0
+       CMP     $0, R0
+       BEQ     nocgo
+
+       MRS_TPIDR_R0
+       MOVD    0x10(R0), g
+
+nocgo:
+       RET
+
+TEXT runtime·save_g(SB),NOSPLIT,$0
+       MOVB    runtime·iscgo(SB), R0
+       CMP     $0, R0
+       BEQ     nocgo
+
+       MRS_TPIDR_R0
+       MOVD    g, 0x10(R0)
+
+nocgo:
+       RET