runtime/internal/atomic: set SP delta correctly for 64-bit atomic functions on ARM

64-bit atomic functions on ARM have the following structure:
- check if the address is 64-bit aligned, if not, prepare a frame
  and call panicUnaligned
- tail call armXXX or goXXX depending on GOARM

The alignment check calls panicUnaligned after preparing a frame,
so the stack can be unwound. The call never returns, so the SP is
not set back. However, the assembler assigns SP delta following
the instruction stream order, not the control flow. So it leaves
a nonzero SP delta after the check, to the tail call instructions,
which is wrong because when they are executed the SP is not
decremented. This CL fixes this by adding the SP back (the
instruction never executes, just tells the assembler to set the
SP delta back).

Should fix #51353.

Change-Id: I976cb1cfb0e9008b13538765cbc7eea0c19c7130
Reviewed-on: https://go-review.googlesource.com/c/go/+/388014
Trust: Cherry Mui <cherryyz@google.com>
Run-TryBot: Cherry Mui <cherryyz@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: Michael Pratt <mpratt@google.com>

diff --git a/src/runtime/internal/atomic/atomic_arm.s b/src/runtime/internal/atomic/atomic_arm.s
index be3fd3a395061714aff2ce73aa4bdd7d73e83fc3..92cbe8a34f68d2250102ad29398ac0df8b71e77c 100644 (file)
--- a/src/runtime/internal/atomic/atomic_arm.s
+++ b/src/runtime/internal/atomic/atomic_arm.s
@@ -229,16 +229,22 @@ store64loop:
 // functions tail-call into the appropriate implementation, which
 // means they must not open a frame. Hence, when they go down the
 // panic path, at that point they push the LR to create a real frame
-// (they don't need to pop it because panic won't return).
+// (they don't need to pop it because panic won't return; however, we
+// do need to set the SP delta back).
+
+// Check if R1 is 8-byte aligned, panic if not.
+// Clobbers R2.
+#define CHECK_ALIGN \
+       AND.S   $7, R1, R2 \
+       BEQ     4(PC) \
+       MOVW.W  R14, -4(R13) /* prepare a real frame */ \
+       BL      ·panicUnaligned(SB) \
+       ADD     $4, R13 /* compensate SP delta */
 
 TEXT ·Cas64(SB),NOSPLIT,$-4-21
        NO_LOCAL_POINTERS
        MOVW    addr+0(FP), R1
-       // make unaligned atomic access panic
-       AND.S   $7, R1, R2
-       BEQ     3(PC)
-       MOVW.W  R14, -4(R13) // prepare a real frame
-       BL      ·panicUnaligned(SB)
+       CHECK_ALIGN
 
        MOVB    runtime·goarm(SB), R11
        CMP     $7, R11
@@ -249,11 +255,7 @@ TEXT ·Cas64(SB),NOSPLIT,$-4-21
 TEXT ·Xadd64(SB),NOSPLIT,$-4-20
        NO_LOCAL_POINTERS
        MOVW    addr+0(FP), R1
-       // make unaligned atomic access panic
-       AND.S   $7, R1, R2
-       BEQ     3(PC)
-       MOVW.W  R14, -4(R13) // prepare a real frame
-       BL      ·panicUnaligned(SB)
+       CHECK_ALIGN
 
        MOVB    runtime·goarm(SB), R11
        CMP     $7, R11
@@ -264,11 +266,7 @@ TEXT ·Xadd64(SB),NOSPLIT,$-4-20
 TEXT ·Xchg64(SB),NOSPLIT,$-4-20
        NO_LOCAL_POINTERS
        MOVW    addr+0(FP), R1
-       // make unaligned atomic access panic
-       AND.S   $7, R1, R2
-       BEQ     3(PC)
-       MOVW.W  R14, -4(R13) // prepare a real frame
-       BL      ·panicUnaligned(SB)
+       CHECK_ALIGN
 
        MOVB    runtime·goarm(SB), R11
        CMP     $7, R11
@@ -279,11 +277,7 @@ TEXT ·Xchg64(SB),NOSPLIT,$-4-20
 TEXT ·Load64(SB),NOSPLIT,$-4-12
        NO_LOCAL_POINTERS
        MOVW    addr+0(FP), R1
-       // make unaligned atomic access panic
-       AND.S   $7, R1, R2
-       BEQ     3(PC)
-       MOVW.W  R14, -4(R13) // prepare a real frame
-       BL      ·panicUnaligned(SB)
+       CHECK_ALIGN
 
        MOVB    runtime·goarm(SB), R11
        CMP     $7, R11
@@ -294,11 +288,7 @@ TEXT ·Load64(SB),NOSPLIT,$-4-12
 TEXT ·Store64(SB),NOSPLIT,$-4-12
        NO_LOCAL_POINTERS
        MOVW    addr+0(FP), R1
-       // make unaligned atomic access panic
-       AND.S   $7, R1, R2
-       BEQ     3(PC)
-       MOVW.W  R14, -4(R13) // prepare a real frame
-       BL      ·panicUnaligned(SB)
+       CHECK_ALIGN
 
        MOVB    runtime·goarm(SB), R11
        CMP     $7, R11