From 4a0115c886101717dfc6186a3296b69fc4415487 Mon Sep 17 00:00:00 2001 From: qmuntal Date: Tue, 26 Aug 2025 16:20:56 +0200 Subject: [PATCH] runtime,syscall: implement and use syscalln on darwin All darwin syscall implementations can be consolidated into a single syscalln function, as already happens on Windows. This reduces duplication and allows moving some logic from runtime to syscall. Updates #699135 Cq-Include-Trybots: luci.golang.try:gotip-darwin-arm64-longtest,gotip-darwin-amd64-longtest,x_sys-gotip-darwin-arm64-longtest,x_sys-gotip-darwin-amd64-longtest Change-Id: If5de80442b1d4a1123258401a3ae21695e7c8f6b Reviewed-on: https://go-review.googlesource.com/c/go/+/699177 LUCI-TryBot-Result: Go LUCI Reviewed-by: David Chase Reviewed-by: Michael Pratt --- src/internal/trace/trace_test.go | 3 + src/runtime/os_darwin.go | 6 + src/runtime/sys_darwin.go | 195 ++++++---------------------- src/runtime/sys_darwin_amd64.s | 161 +++++++++-------------- src/runtime/sys_darwin_arm64.s | 186 +++++++++++--------------- src/syscall/syscall_darwin.go | 128 +++++++++++++++++- src/syscall/syscall_darwin_amd64.go | 3 - src/syscall/syscall_darwin_arm64.go | 3 - 8 files changed, 304 insertions(+), 381 deletions(-) diff --git a/src/internal/trace/trace_test.go b/src/internal/trace/trace_test.go index 25f671d440..479411548f 100644 --- a/src/internal/trace/trace_test.go +++ b/src/internal/trace/trace_test.go @@ -448,6 +448,9 @@ func TestTraceStacks(t *testing.T) { {"main.main.func11", 0}, }}, }...) + if runtime.GOOS == "darwin" { + want[len(want)-1].frames = append([]frame{{"syscall.syscall", 0}}, want[len(want)-1].frames...) + } } stackMatches := func(stk trace.Stack, frames []frame) bool { for i, f := range slices.Collect(stk.Frames()) { diff --git a/src/runtime/os_darwin.go b/src/runtime/os_darwin.go index 0c7144e9d0..c908809c81 100644 --- a/src/runtime/os_darwin.go +++ b/src/runtime/os_darwin.go @@ -15,6 +15,11 @@ type mOS struct { mutex pthreadmutex cond pthreadcond count int + + // address of errno variable for this thread. + // This is an optimization to avoid calling libc_error + // on every syscall_rawsyscalln. + errnoAddr *int32 } func unimplemented(name string) { @@ -330,6 +335,7 @@ func minit() { } minitSignalMask() getg().m.procid = uint64(pthread_self()) + libc_error_addr(&getg().m.errnoAddr) } // Called from dropm to undo the effect of an minit. diff --git a/src/runtime/sys_darwin.go b/src/runtime/sys_darwin.go index aa628021a0..48ad5afd8a 100644 --- a/src/runtime/sys_darwin.go +++ b/src/runtime/sys_darwin.go @@ -10,186 +10,65 @@ import ( "unsafe" ) -//go:nosplit -func libcError() uintptr { - errPtr, _ := syscall(abi.FuncPCABI0(libc_error_trampoline), 0, 0, 0) - return errPtr -} func libc_error_trampoline() -// The X versions of syscall expect the libc call to return a 64-bit result. -// Otherwise (the non-X version) expects a 32-bit result. -// This distinction is required because an error is indicated by returning -1, -// and we need to know whether to check 32 or 64 bits of the result. -// (Some libc functions that return 32 bits put junk in the upper 32 bits of AX.) - -//go:nosplit -func syscall(fn, a1, a2, a3 uintptr) (r1, r2 uintptr) { - args := struct{ fn, a1, a2, a3, r1, r2 uintptr }{fn, a1, a2, a3, r1, r2} - libcCall(unsafe.Pointer(abi.FuncPCABI0(syscall_trampoline)), unsafe.Pointer(&args)) - return args.r1, args.r2 -} -func syscall_trampoline() - -// golang.org/x/sys linknames syscall_syscall -// (in addition to standard package syscall). -// Do not remove or change the type signature. -// -//go:linkname syscall_syscall syscall.syscall -//go:nosplit -func syscall_syscall(fn, a1, a2, a3 uintptr) (r1, r2, err uintptr) { - entersyscall() - r1, r2, err = syscall_rawSyscall(fn, a1, a2, a3) - exitsyscall() - return r1, r2, err -} - -//go:linkname syscall_syscallX syscall.syscallX -//go:nosplit -func syscall_syscallX(fn, a1, a2, a3 uintptr) (r1, r2, err uintptr) { - entersyscall() - r1, r2, err = syscall_rawSyscallX(fn, a1, a2, a3) - exitsyscall() - return r1, r2, err -} - -//go:nosplit -func syscall6(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr) { - args := struct{ fn, a1, a2, a3, a4, a5, a6, r1, r2 uintptr }{fn, a1, a2, a3, a4, a5, a6, r1, r2} - libcCall(unsafe.Pointer(abi.FuncPCABI0(syscall6_trampoline)), unsafe.Pointer(&args)) - return args.r1, args.r2 -} -func syscall6_trampoline() - -// golang.org/x/sys linknames syscall.syscall6 -// (in addition to standard package syscall). -// Do not remove or change the type signature. -// -// syscall.syscall6 is meant for package syscall (and x/sys), -// but widely used packages access it using linkname. -// Notable members of the hall of shame include: -// - github.com/tetratelabs/wazero -// -// See go.dev/issue/67401. +// libc_error_addr puts the libc error +// address into addr. // -//go:linkname syscall_syscall6 syscall.syscall6 //go:nosplit -func syscall_syscall6(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, err uintptr) { - entersyscall() - r1, r2, err = syscall_rawSyscall6(fn, a1, a2, a3, a4, a5, a6) - exitsyscall() - return r1, r2, err -} - -//go:linkname syscall_syscall6X syscall.syscall6X -//go:nosplit -func syscall_syscall6X(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, err uintptr) { - entersyscall() - r1, r2, err = syscall_rawSyscall6X(fn, a1, a2, a3, a4, a5, a6) - exitsyscall() - return r1, r2, err +//go:cgo_unsafe_args +func libc_error_addr(addr **int32) { + libcCall(unsafe.Pointer(abi.FuncPCABI0(libc_error_trampoline)), unsafe.Pointer(&addr)) } -//go:nosplit -func syscall9(fn, a1, a2, a3, a4, a5, a6, a7, a8, a9 uintptr) (r1, r2 uintptr) { - args := struct{ fn, a1, a2, a3, a4, a5, a6, a7, a8, a9, r1, r2 uintptr }{fn, a1, a2, a3, a4, a5, a6, a7, a8, a9, r1, r2} - libcCall(unsafe.Pointer(abi.FuncPCABI0(syscall9_trampoline)), unsafe.Pointer(&args)) - return args.r1, args.r2 +// libcCallInfo is a structure used to pass parameters to the system call. +type libcCallInfo struct { + fn uintptr + n uintptr // number of parameters + args uintptr // parameters + r1, r2 uintptr // return values } -func syscall9_trampoline() -// golang.org/x/sys linknames syscall.syscall9 -// (in addition to standard package syscall). -// Do not remove or change the type signature. +// syscall_syscalln is a wrapper around the libc call with variable arguments. // -//go:linkname syscall_syscall9 syscall.syscall9 +//go:linkname syscall_syscalln syscall.syscalln //go:nosplit -func syscall_syscall9(fn, a1, a2, a3, a4, a5, a6, a7, a8, a9 uintptr) (r1, r2, err uintptr) { +//go:uintptrkeepalive +func syscall_syscalln(fn uintptr, args ...uintptr) (r1, r2, err uintptr) { entersyscall() - r1, r2, err = syscall_rawSyscall9(fn, a1, a2, a3, a4, a5, a6, a7, a8, a9) + r1, r2, err = syscall_rawsyscalln(fn, args...) exitsyscall() return r1, r2, err } -// golang.org/x/sys linknames syscall.syscallPtr -// (in addition to standard package syscall). -// Do not remove or change the type signature. +// syscall_rawsyscalln is a wrapper around the libc call with variable arguments. +// The scheduler is not notified about the system call. +// The syscall is executed on the current goroutine thread rather than on a +// dedicated syscall thread. // -//go:linkname syscall_syscallPtr syscall.syscallPtr +//go:linkname syscall_rawsyscalln syscall.rawsyscalln //go:nosplit -func syscall_syscallPtr(fn, a1, a2, a3 uintptr) (r1, r2, err uintptr) { - entersyscall() - r1, r2, err = syscall_rawSyscallPtr(fn, a1, a2, a3) - exitsyscall() - return r1, r2, err -} - -// golang.org/x/sys linknames syscall_rawSyscall -// (in addition to standard package syscall). -// Do not remove or change the type signature. -// -//go:linkname syscall_rawSyscall syscall.rawSyscall -//go:nosplit -func syscall_rawSyscall(fn, a1, a2, a3 uintptr) (r1, r2, err uintptr) { - r1, r2 = syscall(fn, a1, a2, a3) - // Check if r1 low 32 bits is -1, indicating an error. - if int32(r1) == -1 { - err = libcError() +//go:uintptrkeepalive +func syscall_rawsyscalln(fn uintptr, args ...uintptr) (r1, r2, err uintptr) { + c := &libcCallInfo{ + fn: fn, + n: uintptr(len(args)), } - return r1, r2, err -} - -//go:nosplit -func syscall_rawSyscallX(fn, a1, a2, a3 uintptr) (r1, r2, err uintptr) { - r1, r2 = syscall(fn, a1, a2, a3) - if r1 == ^uintptr(0) { - err = libcError() + if c.n != 0 { + c.args = uintptr(noescape(unsafe.Pointer(&args[0]))) } - return r1, r2, err -} - -//go:nosplit -func syscall_rawSyscallPtr(fn, a1, a2, a3 uintptr) (r1, r2, err uintptr) { - r1, r2 = syscall(fn, a1, a2, a3) - if r1 == 0 { - err = libcError() - } - return r1, r2, err -} - -// golang.org/x/sys linknames syscall_rawSyscall6 -// (in addition to standard package syscall). -// Do not remove or change the type signature. -// -//go:linkname syscall_rawSyscall6 syscall.rawSyscall6 -//go:nosplit -func syscall_rawSyscall6(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, err uintptr) { - r1, r2 = syscall6(fn, a1, a2, a3, a4, a5, a6) - // Check if r1 low 32 bits is -1, indicating an error. - if int32(r1) == -1 { - err = libcError() - } - return r1, r2, err -} - -//go:nosplit -func syscall_rawSyscall6X(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, err uintptr) { - r1, r2 = syscall6(fn, a1, a2, a3, a4, a5, a6) - if r1 == ^uintptr(0) { - err = libcError() + libcCall(unsafe.Pointer(abi.FuncPCABI0(syscallN_trampoline)), unsafe.Pointer(c)) + if gp := getg(); gp != nil && gp.m != nil && gp.m.errnoAddr != nil { + err = uintptr(*gp.m.errnoAddr) + } else { + var errnoAddr *int32 + libc_error_addr(&errnoAddr) + err = uintptr(*errnoAddr) } - return r1, r2, err + return c.r1, c.r2, err } -//go:nosplit -func syscall_rawSyscall9(fn, a1, a2, a3, a4, a5, a6, a7, a8, a9 uintptr) (r1, r2, err uintptr) { - r1, r2 = syscall9(fn, a1, a2, a3, a4, a5, a6, a7, a8, a9) - // Check if r1 low 32 bits is -1, indicating an error. - if int32(r1) == -1 { - err = libcError() - } - return r1, r2, err -} +func syscallN_trampoline() // crypto_x509_syscall is used in crypto/x509/internal/macos to call into Security.framework and CF. diff --git a/src/runtime/sys_darwin_amd64.s b/src/runtime/sys_darwin_amd64.s index e746257967..99d67a9cfd 100644 --- a/src/runtime/sys_darwin_amd64.s +++ b/src/runtime/sys_darwin_amd64.s @@ -506,118 +506,73 @@ TEXT runtime·arc4random_buf_trampoline(SB),NOSPLIT,$0 CALL libc_arc4random_buf(SB) RET -// syscall_trampoline calls a function in libc on behalf of the syscall package. -// syscall_trampoline takes a pointer to a struct like: -// struct { -// fn uintptr -// a1 uintptr -// a2 uintptr -// a3 uintptr -// r1 uintptr -// r2 uintptr -// } -// syscall_trampoline must be called on the g0 stack with the -// C calling convention (use libcCall). -TEXT runtime·syscall_trampoline(SB),NOSPLIT,$16 - MOVQ (0*8)(DI), CX // fn - MOVQ (2*8)(DI), SI // a2 - MOVQ (3*8)(DI), DX // a3 - MOVQ DI, (SP) - MOVQ (1*8)(DI), DI // a1 - XORL AX, AX // vararg: say "no float args" - - CALL CX - - MOVQ (SP), DI - MOVQ AX, (4*8)(DI) // r1 - MOVQ DX, (5*8)(DI) // r2 - - XORL AX, AX // no error (it's ignored anyway) - RET -// syscall6_trampoline calls a function in libc on behalf of the syscall package. -// syscall6_trampoline takes a pointer to a struct like: -// struct { -// fn uintptr -// a1 uintptr -// a2 uintptr -// a3 uintptr -// a4 uintptr -// a5 uintptr -// a6 uintptr -// r1 uintptr -// r2 uintptr -// } -// syscall6_trampoline must be called on the g0 stack with the -// C calling convention (use libcCall). -TEXT runtime·syscall6_trampoline(SB),NOSPLIT,$16 - MOVQ (0*8)(DI), R11// fn - MOVQ (2*8)(DI), SI // a2 - MOVQ (3*8)(DI), DX // a3 - MOVQ (4*8)(DI), CX // a4 - MOVQ (5*8)(DI), R8 // a5 - MOVQ (6*8)(DI), R9 // a6 - MOVQ DI, (SP) - MOVQ (1*8)(DI), DI // a1 - XORL AX, AX // vararg: say "no float args" +TEXT runtime·syscallN_trampoline(SB),NOSPLIT,$0 + // store argument and original SP in a callee-saved register + MOVQ DI, R13 + MOVQ SP, R14 + + MOVQ libcCallInfo_fn(R13), R11 + MOVQ libcCallInfo_n(R13), CX + MOVQ libcCallInfo_args(R13), R10 + + // Fast version, do not store args on the stack. + CMPL CX, $0; JE _0args + CMPL CX, $1; JE _1args + CMPL CX, $2; JE _2args + CMPL CX, $3; JE _3args + CMPL CX, $4; JE _4args + CMPL CX, $5; JE _5args + CMPL CX, $6; JE _6args + + // Reserve stack space for remaining args + MOVQ CX, R12 + SUBQ $6, R12 + ADDQ $1, R12 // make even number of words for stack alignment + ANDQ $~1, R12 + SHLQ $3, R12 + SUBQ R12, SP + + // Copy args to the stack. + // CX: count of stack arguments (n-6) + // SI: &args[6] + // DI: copy of RSP + SUBQ $6, CX + MOVQ R10, SI + ADDQ $(8*6), SI + MOVQ SP, DI + CLD + REP; MOVSQ + +_6args: + MOVQ (5*8)(R10), R9 +_5args: + MOVQ (4*8)(R10), R8 +_4args: + MOVQ (3*8)(R10), CX +_3args: + MOVQ (2*8)(R10), DX +_2args: + MOVQ (1*8)(R10), SI +_1args: + MOVQ (0*8)(R10), DI +_0args: + + XORL AX, AX // vararg: say "no float args" CALL R11 - MOVQ (SP), DI - MOVQ AX, (7*8)(DI) // r1 - MOVQ DX, (8*8)(DI) // r2 + MOVQ R14, SP // free stack space - XORL AX, AX // no error (it's ignored anyway) - RET - -// syscall9_trampoline calls a function in libc on behalf of the syscall package. -// syscall9_trampoline takes a pointer to a struct like: -// struct { -// fn uintptr -// a1 uintptr -// a2 uintptr -// a3 uintptr -// a4 uintptr -// a5 uintptr -// a6 uintptr -// a7 uintptr -// a8 uintptr -// a9 uintptr -// r1 uintptr -// r2 uintptr -// err uintptr -// } -// syscall9_trampoline must be called on the g0 stack with the -// C calling convention (use libcCall). -TEXT runtime·syscall9_trampoline(SB),NOSPLIT,$32 - MOVQ (0*8)(DI), R13// fn - MOVQ (2*8)(DI), SI // a2 - MOVQ (3*8)(DI), DX // a3 - MOVQ (4*8)(DI), CX // a4 - MOVQ (5*8)(DI), R8 // a5 - MOVQ (6*8)(DI), R9 // a6 - MOVQ (7*8)(DI), R10 // a7 - MOVQ R10, 0(SP) - MOVQ (8*8)(DI), R11 // a8 - MOVQ R11, 8(SP) - MOVQ (9*8)(DI), R12 // a9 - MOVQ R12, 16(SP) - MOVQ DI, 24(SP) - MOVQ (1*8)(DI), DI // a1 - XORL AX, AX // vararg: say "no float args" - - CALL R13 - - MOVQ 24(SP), DI - MOVQ AX, (10*8)(DI) // r1 - MOVQ DX, (11*8)(DI) // r2 - - XORL AX, AX // no error (it's ignored anyway) + // Return result. + MOVQ AX, libcCallInfo_r1(R13) + MOVQ DX, libcCallInfo_r2(R13) RET TEXT runtime·libc_error_trampoline(SB),NOSPLIT,$0 + MOVQ 0(DI), R14 CALL libc_error(SB) - MOVLQSX (AX), AX + MOVQ AX, (R14) RET // syscall_x509 is for crypto/x509. It is like syscall6 but does not check for errors, diff --git a/src/runtime/sys_darwin_arm64.s b/src/runtime/sys_darwin_arm64.s index adbb2adafe..7bbe965c15 100644 --- a/src/runtime/sys_darwin_arm64.s +++ b/src/runtime/sys_darwin_arm64.s @@ -481,128 +481,98 @@ TEXT runtime·arc4random_buf_trampoline(SB),NOSPLIT,$0 BL libc_arc4random_buf(SB) RET -// syscall_trampoline calls a function in libc on behalf of the syscall package. -// syscall_trampoline takes a pointer to a struct like: -// struct { -// fn uintptr -// a1 uintptr -// a2 uintptr -// a3 uintptr -// r1 uintptr -// r2 uintptr -// } -// syscall_trampoline must be called on the g0 stack with the -// C calling convention (use libcCall). -TEXT runtime·syscall_trampoline(SB),NOSPLIT,$0 - SUB $16, RSP // push structure pointer - MOVD R0, 8(RSP) - - MOVD 0(R0), R12 // fn - MOVD 16(R0), R1 // a2 - MOVD 24(R0), R2 // a3 - MOVD 8(R0), R0 // a1 +TEXT runtime·syscallN_trampoline(SB),NOSPLIT,$16 + STP (R19, R20), 16(RSP) // save old R19, R20 + MOVD R0, R19 // save struct pointer + MOVD RSP, R20 // save stack pointer + SUB $16, RSP // reserve 16 bytes for sp-8 where fp may be saved. + + MOVD libcCallInfo_args(R19), R12 + // Do we have more than 8 arguments? + MOVD libcCallInfo_n(R19), R0 + CMP $0, R0; BEQ _0args + CMP $1, R0; BEQ _1args + CMP $2, R0; BEQ _2args + CMP $3, R0; BEQ _3args + CMP $4, R0; BEQ _4args + CMP $5, R0; BEQ _5args + CMP $6, R0; BEQ _6args + CMP $7, R0; BEQ _7args + CMP $8, R0; BEQ _8args + + // Reserve stack space for remaining args + SUB $8, R0, R2 + ADD $1, R2, R3 // make even number of words for stack alignment + AND $~1, R3 + LSL $3, R3 + SUB R3, RSP + + // R4: size of stack arguments (n-8)*8 + // R5: &args[8] + // R6: loop counter, from 0 to (n-8)*8 + // R7: scratch + // R8: copy of RSP - (R2)(RSP) assembles as (R2)(ZR) + SUB $8, R0, R4 + LSL $3, R4 + ADD $(8*8), R12, R5 + MOVD $0, R6 + MOVD RSP, R8 +stackargs: + MOVD (R6)(R5), R7 + MOVD R7, (R6)(R8) + ADD $8, R6 + CMP R6, R4 + BNE stackargs + +_8args: + MOVD (7*8)(R12), R7 +_7args: + MOVD (6*8)(R12), R6 +_6args: + MOVD (5*8)(R12), R5 +_5args: + MOVD (4*8)(R12), R4 +_4args: + MOVD (3*8)(R12), R3 +_3args: + MOVD (2*8)(R12), R2 +_2args: + MOVD (1*8)(R12), R1 +_1args: + MOVD (0*8)(R12), R0 +_0args: // If fn is declared as vararg, we have to pass the vararg arguments on the stack. // (Because ios decided not to adhere to the standard arm64 calling convention, sigh...) - // The only libSystem calls we support that are vararg are open, fcntl, and ioctl, - // which are all of the form fn(x, y, ...). So we just need to put the 3rd arg - // on the stack as well. + // The only libSystem calls we support with vararg are open, fcntl, ioctl, + // which are all of the form fn(x, y, ...), and openat, which is of the form fn(x, y, z, ...). + // So we just need to put the 3rd and the 4th arg on the stack as well. + // Note that historically openat has been called with syscall6, so we need to handle that case too. // If we ever have other vararg libSystem calls, we might need to handle more cases. + MOVD libcCallInfo_n(R19), R12 + CMP $3, R12; BNE 2(PC); MOVD R2, (RSP) - - BL (R12) - - MOVD 8(RSP), R2 // pop structure pointer - ADD $16, RSP - MOVD R0, 32(R2) // save r1 - MOVD R1, 40(R2) // save r2 - RET - -// syscall6_trampoline calls a function in libc on behalf of the syscall package. -// syscall6_trampoline takes a pointer to a struct like: -// struct { -// fn uintptr -// a1 uintptr -// a2 uintptr -// a3 uintptr -// a4 uintptr -// a5 uintptr -// a6 uintptr -// r1 uintptr -// r2 uintptr -// } -// syscall6_trampoline must be called on the g0 stack with the -// C calling convention (use libcCall). -TEXT runtime·syscall6_trampoline(SB),NOSPLIT,$0 - SUB $16, RSP // push structure pointer - MOVD R0, 8(RSP) - - MOVD 0(R0), R12 // fn - MOVD 16(R0), R1 // a2 - MOVD 24(R0), R2 // a3 - MOVD 32(R0), R3 // a4 - MOVD 40(R0), R4 // a5 - MOVD 48(R0), R5 // a6 - MOVD 8(R0), R0 // a1 - - // If fn is declared as vararg, we have to pass the vararg arguments on the stack. - // See syscall_trampoline above. The only function this applies to is openat, for which the 4th - // arg must be on the stack. + CMP $4, R12; BNE 2(PC); + MOVD R3, (RSP) + CMP $6, R12; BNE 2(PC); MOVD R3, (RSP) + MOVD libcCallInfo_fn(R19), R12 BL (R12) - MOVD 8(RSP), R2 // pop structure pointer - ADD $16, RSP - MOVD R0, 56(R2) // save r1 - MOVD R1, 64(R2) // save r2 - RET - -// syscall9_trampoline calls a function in libc on behalf of the syscall package. -// syscall9_trampoline takes a pointer to a struct like: -// struct { -// fn uintptr -// a1 uintptr -// a2 uintptr -// a3 uintptr -// a4 uintptr -// a5 uintptr -// a6 uintptr -// a7 uintptr -// a8 uintptr -// a9 uintptr -// r1 uintptr -// r2 uintptr -// } -// syscall9_trampoline must be called on the g0 stack with the -// C calling convention (use libcCall). -TEXT runtime·syscall9_trampoline(SB),NOSPLIT,$0 - SUB $16, RSP // push structure pointer - MOVD R0, 8(RSP) + MOVD R20, RSP // free stack space - MOVD 0(R0), R12 // fn - MOVD 16(R0), R1 // a2 - MOVD 24(R0), R2 // a3 - MOVD 32(R0), R3 // a4 - MOVD 40(R0), R4 // a5 - MOVD 48(R0), R5 // a6 - MOVD 56(R0), R6 // a7 - MOVD 64(R0), R7 // a8 - MOVD 72(R0), R8 // a9 - MOVD R8, 0(RSP) // the 9th arg and onwards must be passed on the stack - MOVD 8(R0), R0 // a1 - - BL (R12) + MOVD R0, libcCallInfo_r1(R19) + MOVD R1, libcCallInfo_r2(R19) - MOVD 8(RSP), R2 // pop structure pointer - ADD $16, RSP - MOVD R0, 80(R2) // save r1 - MOVD R1, 88(R2) // save r2 - RET + // Restore callee-saved registers. + LDP 16(RSP), (R19, R20) + RET TEXT runtime·libc_error_trampoline(SB),NOSPLIT,$0 + MOVD 0(R0), R20 BL libc_error(SB) - MOVW (R0), R0 + MOVD R0, (R20) RET // syscall_x509 is for crypto/x509. It is like syscall6 but does not check for errors, diff --git a/src/syscall/syscall_darwin.go b/src/syscall/syscall_darwin.go index 7b4f0d4fb7..ca76cc2962 100644 --- a/src/syscall/syscall_darwin.go +++ b/src/syscall/syscall_darwin.go @@ -323,10 +323,126 @@ func Getdirentries(fd int, buf []byte, basep *uintptr) (n int, err error) { return n, nil } +// errno return e if int32(r) is -1, else it returns 0. +// +//go:nosplit +func errno(r uintptr, e Errno) Errno { + if int32(r) == -1 { + return e + } + return 0 +} + +// errnoX return e if r is -1, else it returns 0. +// +//go:nosplit +func errnoX(r uintptr, e Errno) Errno { + if r == ^uintptr(0) { + return e + } + return 0 +} + +// errnoPtr return e if r is 0, else it returns 0. +// +//go:nosplit +func errnoPtr(r uintptr, e Errno) Errno { + if r == 0 { + return e + } + return 0 +} + +//go:cgo_import_dynamic libc_error __error "/usr/lib/libSystem.B.dylib" + +// golang.org/x/sys linknames the following syscalls. +// Do not remove or change the type signature. + +//go:linkname syscall +//go:nosplit +//go:uintptrkeepalive +func syscall(fn, a1, a2, a3 uintptr) (r1, r2 uintptr, err Errno) { + r1, r2, err = syscalln(fn, a1, a2, a3) + return r1, r2, errno(r1, err) +} + +//go:linkname syscallX +//go:nosplit +//go:uintptrkeepalive +func syscallX(fn, a1, a2, a3 uintptr) (r1, r2 uintptr, err Errno) { + r1, r2, err = syscalln(fn, a1, a2, a3) + return r1, r2, errnoX(r1, err) +} + +// syscall.syscall6 is meant for package syscall (and x/sys), +// but widely used packages access it using linkname. +// Notable members of the hall of shame include: +// - github.com/tetratelabs/wazero +// +// See go.dev/issue/67401. +// +//go:linkname syscall6 +//go:nosplit +//go:uintptrkeepalive +func syscall6(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err Errno) { + r1, r2, err = syscalln(fn, a1, a2, a3, a4, a5, a6) + return r1, r2, errno(r1, err) +} + +//go:linkname syscall6X +//go:nosplit +//go:uintptrkeepalive +func syscall6X(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err Errno) { + r1, r2, err = syscalln(fn, a1, a2, a3, a4, a5, a6) + return r1, r2, errnoX(r1, err) +} + +// syscall9 is used in [internal/syscall/unix]. +// +//go:linkname syscall9 +//go:nosplit +//go:uintptrkeepalive +func syscall9(fn, a1, a2, a3, a4, a5, a6, a7, a8, a9 uintptr) (r1, r2 uintptr, err Errno) { + r1, r2, err = syscalln(fn, a1, a2, a3, a4, a5, a6, a7, a8, a9) + return r1, r2, errno(r1, err) +} + +//go:linkname rawSyscall +//go:nosplit +//go:uintptrkeepalive +func rawSyscall(fn, a1, a2, a3 uintptr) (r1, r2 uintptr, err Errno) { + r1, r2, err = rawsyscalln(fn, a1, a2, a3) + return r1, r2, errno(r1, err) +} + +//go:linkname rawSyscall6 +//go:nosplit +//go:uintptrkeepalive +func rawSyscall6(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err Errno) { + r1, r2, err = rawsyscalln(fn, a1, a2, a3, a4, a5, a6) + return r1, r2, errno(r1, err) +} + +//go:linkname rawSyscall9 +//go:nosplit +//go:uintptrkeepalive +func rawSyscall9(fn, a1, a2, a3, a4, a5, a6, a7, a8, a9 uintptr) (r1, r2 uintptr, err Errno) { + r1, r2, err = rawsyscalln(fn, a1, a2, a3, a4, a5, a6, a7, a8, a9) + return r1, r2, errno(r1, err) +} + +//go:linkname syscallPtr +//go:nosplit +//go:uintptrkeepalive +func syscallPtr(fn, a1, a2, a3 uintptr) (r1, r2 uintptr, err Errno) { + r1, r2, e1 := syscalln(fn, a1, a2, a3) + return r1, r2, errnoPtr(r1, e1) +} + // Implemented in the runtime package (runtime/sys_darwin.go) -func syscall(fn, a1, a2, a3 uintptr) (r1, r2 uintptr, err Errno) -func syscall6(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err Errno) -func syscall6X(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err Errno) -func rawSyscall(fn, a1, a2, a3 uintptr) (r1, r2 uintptr, err Errno) -func rawSyscall6(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err Errno) -func syscallPtr(fn, a1, a2, a3 uintptr) (r1, r2 uintptr, err Errno) + +//go:noescape +func syscalln(fn uintptr, args ...uintptr) (r1, r2 uintptr, err Errno) + +//go:noescape +func rawsyscalln(fn uintptr, args ...uintptr) (r1, r2 uintptr, err Errno) diff --git a/src/syscall/syscall_darwin_amd64.go b/src/syscall/syscall_darwin_amd64.go index 64e54ad730..d4de4c25a4 100644 --- a/src/syscall/syscall_darwin_amd64.go +++ b/src/syscall/syscall_darwin_amd64.go @@ -61,7 +61,4 @@ func libc_sendfile_trampoline() //go:cgo_import_dynamic libc_sendfile sendfile "/usr/lib/libSystem.B.dylib" -// Implemented in the runtime package (runtime/sys_darwin_64.go) -func syscallX(fn, a1, a2, a3 uintptr) (r1, r2 uintptr, err Errno) - func Syscall9(trap, a1, a2, a3, a4, a5, a6, a7, a8, a9 uintptr) (r1, r2 uintptr, err Errno) diff --git a/src/syscall/syscall_darwin_arm64.go b/src/syscall/syscall_darwin_arm64.go index 913c748374..4a3dce8088 100644 --- a/src/syscall/syscall_darwin_arm64.go +++ b/src/syscall/syscall_darwin_arm64.go @@ -61,7 +61,4 @@ func libc_sendfile_trampoline() //go:cgo_import_dynamic libc_sendfile sendfile "/usr/lib/libSystem.B.dylib" -// Implemented in the runtime package (runtime/sys_darwin_64.go) -func syscallX(fn, a1, a2, a3 uintptr) (r1, r2 uintptr, err Errno) - func Syscall9(num, a1, a2, a3, a4, a5, a6, a7, a8, a9 uintptr) (r1, r2 uintptr, err Errno) // sic -- 2.52.0