From c7ed3a1c5a152d0e610bbbf104dba30099e6942a Mon Sep 17 00:00:00 2001 From: qmuntal Date: Mon, 21 Jul 2025 14:39:04 +0200 Subject: [PATCH] internal/runtime/syscall/windows: factor out code from runtime Factor out the code related to doing calls using the Windows stdcall calling convention into a separate package. This will allow us to reuse it in other low-level packages that can't depend on syscall. Updates #51087. Cq-Include-Trybots: luci.golang.try:gotip-windows-arm64,gotip-windows-amd64-longtest,gotip-solaris-amd64 Change-Id: I68640b07091183b50da6bef17406c10a397896e9 Reviewed-on: https://go-review.googlesource.com/c/go/+/689156 Reviewed-by: Michael Pratt LUCI-TryBot-Result: Go LUCI Reviewed-by: Michael Knyszek --- src/cmd/internal/objabi/pkgspecial.go | 2 + src/go/build/deps_test.go | 1 + src/internal/coverage/pkid.go | 1 + .../runtime/syscall/windows/asm_windows_386.s | 48 ++++++++++ .../syscall/windows/asm_windows_amd64.s | 84 +++++++++++++++++ .../runtime/syscall/windows/asm_windows_arm.s | 77 ++++++++++++++++ .../syscall/windows/asm_windows_arm64.s | 90 +++++++++++++++++++ .../syscall/windows/syscall_windows.go | 44 +++++++++ src/runtime/cgocall.go | 4 +- src/runtime/export_windows_test.go | 2 - src/runtime/os_aix.go | 1 + src/runtime/os_solaris.go | 5 +- src/runtime/os_windows.go | 69 +++++++------- src/runtime/runtime2.go | 4 +- src/runtime/sys_aix_ppc64.s | 20 ++--- src/runtime/sys_solaris_amd64.s | 4 +- src/runtime/sys_windows_386.s | 43 --------- src/runtime/sys_windows_amd64.s | 79 ---------------- src/runtime/sys_windows_arm.s | 70 --------------- src/runtime/sys_windows_arm64.s | 82 ----------------- src/runtime/syscall_windows.go | 20 ++--- src/runtime/syscall_windows_test.go | 3 +- 22 files changed, 407 insertions(+), 346 deletions(-) create mode 100644 src/internal/runtime/syscall/windows/asm_windows_386.s create mode 100644 src/internal/runtime/syscall/windows/asm_windows_amd64.s create mode 100644 src/internal/runtime/syscall/windows/asm_windows_arm.s create mode 100644 src/internal/runtime/syscall/windows/asm_windows_arm64.s create mode 100644 src/internal/runtime/syscall/windows/syscall_windows.go diff --git a/src/cmd/internal/objabi/pkgspecial.go b/src/cmd/internal/objabi/pkgspecial.go index 522337f583..fe510160b3 100644 --- a/src/cmd/internal/objabi/pkgspecial.go +++ b/src/cmd/internal/objabi/pkgspecial.go @@ -57,6 +57,7 @@ var runtimePkgs = []string{ "internal/runtime/strconv", "internal/runtime/sys", "internal/runtime/syscall/linux", + "internal/runtime/syscall/windows", "internal/abi", "internal/bytealg", @@ -95,6 +96,7 @@ var allowAsmABIPkgs = []string{ "internal/bytealg", "internal/chacha8rand", "internal/runtime/syscall/linux", + "internal/runtime/syscall/windows", "internal/runtime/startlinetest", } diff --git a/src/go/build/deps_test.go b/src/go/build/deps_test.go index b3d8f66bc7..e8bdfe7c12 100644 --- a/src/go/build/deps_test.go +++ b/src/go/build/deps_test.go @@ -92,6 +92,7 @@ var depsRules = ` < internal/asan < internal/runtime/sys < internal/runtime/syscall/linux + < internal/runtime/syscall/windows < internal/runtime/atomic < internal/runtime/exithook < internal/runtime/gc diff --git a/src/internal/coverage/pkid.go b/src/internal/coverage/pkid.go index c9bcce2b6c..09501e6bd2 100644 --- a/src/internal/coverage/pkid.go +++ b/src/internal/coverage/pkid.go @@ -67,6 +67,7 @@ var rtPkgs = [...]string{ "internal/runtime/sys", "internal/runtime/maps", "internal/runtime/syscall/linux", + "internal/runtime/syscall/windows", "internal/runtime/cgroup", "internal/stringslite", "runtime", diff --git a/src/internal/runtime/syscall/windows/asm_windows_386.s b/src/internal/runtime/syscall/windows/asm_windows_386.s new file mode 100644 index 0000000000..29cce00309 --- /dev/null +++ b/src/internal/runtime/syscall/windows/asm_windows_386.s @@ -0,0 +1,48 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "go_asm.h" +#include "textflag.h" + +TEXT ·StdCall(SB),NOSPLIT,$0 + JMP ·asmstdcall(SB) + +TEXT ·asmstdcall(SB),NOSPLIT,$0 + MOVL fn+0(FP), BX + MOVL SP, BP // save stack pointer + + // SetLastError(0). + MOVL $0, 0x34(FS) + + MOVL StdCallInfo_N(BX), CX + + // Fast version, do not store args on the stack. + CMPL CX, $0 + JE docall + + // Copy args to the stack. + MOVL CX, AX + SALL $2, AX + SUBL AX, SP // room for args + MOVL SP, DI + MOVL StdCallInfo_Args(BX), SI + CLD + REP; MOVSL + +docall: + // Call stdcall or cdecl function. + // DI SI BP BX are preserved, SP is not + CALL StdCallInfo_Fn(BX) + MOVL BP, SP + + // Return result. + MOVL fn+0(FP), BX + MOVL AX, StdCallInfo_R1(BX) + MOVL DX, StdCallInfo_R2(BX) + + // GetLastError(). + MOVL 0x34(FS), AX + MOVL AX, StdCallInfo_Err(BX) + + RET diff --git a/src/internal/runtime/syscall/windows/asm_windows_amd64.s b/src/internal/runtime/syscall/windows/asm_windows_amd64.s new file mode 100644 index 0000000000..c31cbcdd14 --- /dev/null +++ b/src/internal/runtime/syscall/windows/asm_windows_amd64.s @@ -0,0 +1,84 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "go_asm.h" +#include "textflag.h" + +TEXT ·StdCall(SB),NOSPLIT,$0 + MOVQ AX, CX + JMP ·asmstdcall(SB) + +TEXT ·asmstdcall(SB),NOSPLIT,$16 + MOVQ SP, AX + ANDQ $~15, SP // alignment as per Windows requirement + MOVQ AX, 8(SP) + MOVQ CX, 0(SP) // asmcgocall will put first argument into CX. + + MOVQ StdCallInfo_Fn(CX), AX + MOVQ StdCallInfo_Args(CX), SI + MOVQ StdCallInfo_N(CX), CX + + // SetLastError(0). + MOVQ 0x30(GS), DI + MOVL $0, 0x68(DI) + + SUBQ $(const_MaxArgs*8), SP // room for args + + // Fast version, do not store args on the stack. + CMPL CX, $0; JE _0args + CMPL CX, $1; JE _1args + CMPL CX, $2; JE _2args + CMPL CX, $3; JE _3args + CMPL CX, $4; JE _4args + + // Check we have enough room for args. + CMPL CX, $const_MaxArgs + JLE 2(PC) + INT $3 // not enough room -> crash + + // Copy args to the stack. + MOVQ SP, DI + CLD + REP; MOVSQ + MOVQ SP, SI + + // Load first 4 args into correspondent registers. + // Floating point arguments are passed in the XMM + // registers. Set them here in case any of the arguments + // are floating point values. For details see + // https://learn.microsoft.com/en-us/cpp/build/x64-calling-convention?view=msvc-170 +_4args: + MOVQ 24(SI), R9 + MOVQ R9, X3 +_3args: + MOVQ 16(SI), R8 + MOVQ R8, X2 +_2args: + MOVQ 8(SI), DX + MOVQ DX, X1 +_1args: + MOVQ 0(SI), CX + MOVQ CX, X0 +_0args: + + // Call stdcall function. + CALL AX + + ADDQ $(const_MaxArgs*8), SP + + // Return result. + MOVQ 0(SP), CX + MOVQ 8(SP), SP + MOVQ AX, StdCallInfo_R1(CX) + // Floating point return values are returned in XMM0. Setting r2 to this + // value in case this call returned a floating point value. For details, + // see https://docs.microsoft.com/en-us/cpp/build/x64-calling-convention + MOVQ X0, StdCallInfo_R2(CX) + + // GetLastError(). + MOVQ 0x30(GS), DI + MOVL 0x68(DI), AX + MOVQ AX, StdCallInfo_Err(CX) + + RET diff --git a/src/internal/runtime/syscall/windows/asm_windows_arm.s b/src/internal/runtime/syscall/windows/asm_windows_arm.s new file mode 100644 index 0000000000..8cc4d5c9aa --- /dev/null +++ b/src/internal/runtime/syscall/windows/asm_windows_arm.s @@ -0,0 +1,77 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "go_asm.h" +#include "go_tls.h" +#include "textflag.h" +#include "time_windows.h" + +TEXT ·StdCall(SB),NOSPLIT,$0 + B ·asmstdcall(SB) + +TEXT ·asmstdcall(SB),NOSPLIT|NOFRAME,$0 + MOVM.DB.W [R4, R5, R14], (R13) // push {r4, r5, lr} + MOVW R0, R4 // put fn * in r4 + MOVW R13, R5 // save stack pointer in r5 + + // SetLastError(0) + MOVW $0, R0 + MRC 15, 0, R1, C13, C0, 2 + MOVW R0, 0x34(R1) + + MOVW 8(R4), R12 // fn->Args + + // Do we have more than 4 arguments? + MOVW 4(R4), R0 // fn->n + SUB.S $4, R0, R2 + BLE loadregs + + // Reserve stack space for remaining args + SUB R2<<2, R13 + BIC $0x7, R13 // alignment for ABI + + // R0: count of arguments + // R1: + // R2: loop counter, from 0 to (n-4) + // R3: scratch + // R4: pointer to StdCallInfo struct + // R12: fn->args + MOVW $0, R2 +stackargs: + ADD $4, R2, R3 // r3 = args[4 + i] + MOVW R3<<2(R12), R3 + MOVW R3, R2<<2(R13) // stack[i] = r3 + + ADD $1, R2 // i++ + SUB $4, R0, R3 // while (i < (n - 4)) + CMP R3, R2 + BLT stackargs + +loadregs: + CMP $3, R0 + MOVW.GT 12(R12), R3 + + CMP $2, R0 + MOVW.GT 8(R12), R2 + + CMP $1, R0 + MOVW.GT 4(R12), R1 + + CMP $0, R0 + MOVW.GT 0(R12), R0 + + BIC $0x7, R13 // alignment for ABI + MOVW 0(R4), R12 // branch to fn->fn + BL (R12) + + MOVW R5, R13 // free stack space + MOVW R0, 12(R4) // save return value to fn->r1 + MOVW R1, 16(R4) + + // GetLastError + MRC 15, 0, R1, C13, C0, 2 + MOVW 0x34(R1), R0 + MOVW R0, 20(R4) // store in fn->err + + MOVM.IA.W (R13), [R4, R5, R15] diff --git a/src/internal/runtime/syscall/windows/asm_windows_arm64.s b/src/internal/runtime/syscall/windows/asm_windows_arm64.s new file mode 100644 index 0000000000..fb4cda0f83 --- /dev/null +++ b/src/internal/runtime/syscall/windows/asm_windows_arm64.s @@ -0,0 +1,90 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "go_asm.h" +#include "textflag.h" + +// Offsets into Thread Environment Block (pointer in R18) +#define TEB_error 0x68 + +TEXT ·StdCall(SB),NOSPLIT,$0 + B ·asmstdcall(SB) + +TEXT ·asmstdcall(SB),NOSPLIT,$16 + STP (R19, R20), 16(RSP) // save old R19, R20 + MOVD R0, R19 // save fn pointer + MOVD RSP, R20 // save stack pointer + + // SetLastError(0) + MOVD $0, TEB_error(R18_PLATFORM) + MOVD StdCallInfo_Args(R19), R12 + + // Do we have more than 8 arguments? + MOVD StdCallInfo_N(R19), R0 + CMP $0, R0; BEQ _0args + CMP $1, R0; BEQ _1args + CMP $2, R0; BEQ _2args + CMP $3, R0; BEQ _3args + CMP $4, R0; BEQ _4args + CMP $5, R0; BEQ _5args + CMP $6, R0; BEQ _6args + CMP $7, R0; BEQ _7args + CMP $8, R0; BEQ _8args + + // Reserve stack space for remaining args + SUB $8, R0, R2 + ADD $1, R2, R3 // make even number of words for stack alignment + AND $~1, R3 + LSL $3, R3 + SUB R3, RSP + + // R4: size of stack arguments (n-8)*8 + // R5: &args[8] + // R6: loop counter, from 0 to (n-8)*8 + // R7: scratch + // R8: copy of RSP - (R2)(RSP) assembles as (R2)(ZR) + SUB $8, R0, R4 + LSL $3, R4 + ADD $(8*8), R12, R5 + MOVD $0, R6 + MOVD RSP, R8 +stackargs: + MOVD (R6)(R5), R7 + MOVD R7, (R6)(R8) + ADD $8, R6 + CMP R6, R4 + BNE stackargs + +_8args: + MOVD (7*8)(R12), R7 +_7args: + MOVD (6*8)(R12), R6 +_6args: + MOVD (5*8)(R12), R5 +_5args: + MOVD (4*8)(R12), R4 +_4args: + MOVD (3*8)(R12), R3 +_3args: + MOVD (2*8)(R12), R2 +_2args: + MOVD (1*8)(R12), R1 +_1args: + MOVD (0*8)(R12), R0 +_0args: + + MOVD StdCallInfo_Fn(R19), R12 + BL (R12) + + MOVD R20, RSP // free stack space + MOVD R0, StdCallInfo_R1(R19) // save return value + // TODO(rsc) floating point like amd64 in StdCallInfo_R2? + + // GetLastError + MOVD TEB_error(R18_PLATFORM), R0 + MOVD R0, StdCallInfo_Err(R19) + + // Restore callee-saved registers. + LDP 16(RSP), (R19, R20) + RET diff --git a/src/internal/runtime/syscall/windows/syscall_windows.go b/src/internal/runtime/syscall/windows/syscall_windows.go new file mode 100644 index 0000000000..0d350f0d7f --- /dev/null +++ b/src/internal/runtime/syscall/windows/syscall_windows.go @@ -0,0 +1,44 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package windows provides the syscall primitives required for the runtime. + +package windows + +import ( + "internal/abi" +) + +// MaxArgs should be divisible by 2, as Windows stack +// must be kept 16-byte aligned on syscall entry. +// +// Although it only permits maximum 42 parameters, it +// is arguably large enough. +const MaxArgs = 42 + +// StdCallInfo is a structure used to pass parameters to the system call. +type StdCallInfo struct { + Fn uintptr + N uintptr // number of parameters + Args uintptr // parameters + R1 uintptr // return values + R2 uintptr + Err uintptr // error number +} + +// StdCall calls a function using Windows' stdcall convention. +// +//go:noescape +func StdCall(fn *StdCallInfo) + +// asmstdcall is the function pointer for [AsmStdCallAddr]. +func asmstdcall(fn *StdCallInfo) + +// AsmStdCallAddr is the address of a function that accepts a pointer +// to [StdCallInfo] stored on the stack following the C calling convention, +// and calls the function using Windows' stdcall calling convention. +// Shouldn't be called directly from Go. +func AsmStdCallAddr() uintptr { + return abi.FuncPCABI0(asmstdcall) +} diff --git a/src/runtime/cgocall.go b/src/runtime/cgocall.go index b046ab960f..5bb6f58bc2 100644 --- a/src/runtime/cgocall.go +++ b/src/runtime/cgocall.go @@ -191,8 +191,8 @@ func cgocall(fn, arg unsafe.Pointer) int32 { osPreemptExtExit(mp) - // Save current syscall parameters, so m.winsyscall can be - // used again if callback decide to make syscall. + // After exitsyscall we can be rescheduled on a different M, + // so we need to restore the original M's winsyscall. winsyscall := mp.winsyscall exitsyscall() diff --git a/src/runtime/export_windows_test.go b/src/runtime/export_windows_test.go index 13d30d4bc4..2467632670 100644 --- a/src/runtime/export_windows_test.go +++ b/src/runtime/export_windows_test.go @@ -11,8 +11,6 @@ import ( "unsafe" ) -const MaxArgs = maxArgs - var ( OsYield = osyield TimeBeginPeriodRetValue = &timeBeginPeriodRetValue diff --git a/src/runtime/os_aix.go b/src/runtime/os_aix.go index 9d63c4d39d..4bb8576f42 100644 --- a/src/runtime/os_aix.go +++ b/src/runtime/os_aix.go @@ -27,6 +27,7 @@ type funcDescriptor struct { type mOS struct { waitsema uintptr // semaphore for parking on locks perrno uintptr // pointer to tls errno + libcall libcall } //go:nosplit diff --git a/src/runtime/os_solaris.go b/src/runtime/os_solaris.go index 5f6163f131..42b7e4a6bc 100644 --- a/src/runtime/os_solaris.go +++ b/src/runtime/os_solaris.go @@ -21,9 +21,8 @@ type mscratch struct { type mOS struct { waitsema uintptr // semaphore for parking on locks perrno *int32 // pointer to tls errno - // these are here because they are too large to be on the stack - // of low-level NOSPLIT functions. - //LibCall libcall; + // This is here to avoid using the G stack so the stack can move during the call. + libcall libcall ts mts scratch mscratch } diff --git a/src/runtime/os_windows.go b/src/runtime/os_windows.go index 04752f2603..88d730aa02 100644 --- a/src/runtime/os_windows.go +++ b/src/runtime/os_windows.go @@ -8,6 +8,7 @@ import ( "internal/abi" "internal/runtime/atomic" "internal/runtime/sys" + "internal/runtime/syscall/windows" "unsafe" ) @@ -160,6 +161,9 @@ func tstart_stdcall(newm *m) func wintls() type mOS struct { + // This is here to avoid using the G stack so the stack can move during the call. + stdCallInfo windows.StdCallInfo + threadLock mutex // protects "thread" and prevents closing thread uintptr // thread handle @@ -210,13 +214,9 @@ func read(fd int32, p unsafe.Pointer, n int32) int32 { type sigset struct{} -// Call a Windows function with stdcall conventions, -// and switch to os stack during the call. -func asmstdcall(fn unsafe.Pointer) - var asmstdcallAddr unsafe.Pointer -type winlibcall libcall +type winlibcall windows.StdCallInfo func windowsFindfunc(lib uintptr, name []byte) stdFunction { if name[len(name)-1] != 0 { @@ -472,7 +472,7 @@ func initLongPathSupport() { } func osinit() { - asmstdcallAddr = unsafe.Pointer(abi.FuncPCABI0(asmstdcall)) + asmstdcallAddr = unsafe.Pointer(windows.AsmStdCallAddr()) loadOptionalSyscalls() @@ -935,20 +935,17 @@ func mdestroy(mp *m) { } } -// asmstdcall_trampoline calls asmstdcall converting from Go to C calling convention. -func asmstdcall_trampoline(args unsafe.Pointer) - // stdcall_no_g calls asmstdcall on os stack without using g. // //go:nosplit func stdcall_no_g(fn stdFunction, n int, args uintptr) uintptr { - libcall := libcall{ - fn: uintptr(unsafe.Pointer(fn)), - n: uintptr(n), - args: args, + call := windows.StdCallInfo{ + Fn: uintptr(unsafe.Pointer(fn)), + N: uintptr(n), + Args: args, } - asmstdcall_trampoline(noescape(unsafe.Pointer(&libcall))) - return libcall.r1 + windows.StdCall(&call) + return call.R1 } // Calling stdcall on os stack. @@ -959,7 +956,7 @@ func stdcall_no_g(fn stdFunction, n int, args uintptr) uintptr { func stdcall(fn stdFunction) uintptr { gp := getg() mp := gp.m - mp.libcall.fn = uintptr(unsafe.Pointer(fn)) + mp.stdCallInfo.Fn = uintptr(unsafe.Pointer(fn)) resetLibcall := false if mp.profilehz != 0 && mp.libcallsp == 0 { // leave pc/sp for cpu profiler @@ -970,18 +967,18 @@ func stdcall(fn stdFunction) uintptr { mp.libcallsp = sys.GetCallerSP() resetLibcall = true // See comment in sys_darwin.go:libcCall } - asmcgocall(asmstdcallAddr, unsafe.Pointer(&mp.libcall)) + asmcgocall(asmstdcallAddr, unsafe.Pointer(&mp.stdCallInfo)) if resetLibcall { mp.libcallsp = 0 } - return mp.libcall.r1 + return mp.stdCallInfo.R1 } //go:nosplit func stdcall0(fn stdFunction) uintptr { mp := getg().m - mp.libcall.n = 0 - mp.libcall.args = 0 + mp.stdCallInfo.N = 0 + mp.stdCallInfo.Args = 0 return stdcall(fn) } @@ -989,8 +986,8 @@ func stdcall0(fn stdFunction) uintptr { //go:cgo_unsafe_args func stdcall1(fn stdFunction, a0 uintptr) uintptr { mp := getg().m - mp.libcall.n = 1 - mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0))) + mp.stdCallInfo.N = 1 + mp.stdCallInfo.Args = uintptr(noescape(unsafe.Pointer(&a0))) return stdcall(fn) } @@ -998,8 +995,8 @@ func stdcall1(fn stdFunction, a0 uintptr) uintptr { //go:cgo_unsafe_args func stdcall2(fn stdFunction, a0, a1 uintptr) uintptr { mp := getg().m - mp.libcall.n = 2 - mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0))) + mp.stdCallInfo.N = 2 + mp.stdCallInfo.Args = uintptr(noescape(unsafe.Pointer(&a0))) return stdcall(fn) } @@ -1007,8 +1004,8 @@ func stdcall2(fn stdFunction, a0, a1 uintptr) uintptr { //go:cgo_unsafe_args func stdcall3(fn stdFunction, a0, a1, a2 uintptr) uintptr { mp := getg().m - mp.libcall.n = 3 - mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0))) + mp.stdCallInfo.N = 3 + mp.stdCallInfo.Args = uintptr(noescape(unsafe.Pointer(&a0))) return stdcall(fn) } @@ -1016,8 +1013,8 @@ func stdcall3(fn stdFunction, a0, a1, a2 uintptr) uintptr { //go:cgo_unsafe_args func stdcall4(fn stdFunction, a0, a1, a2, a3 uintptr) uintptr { mp := getg().m - mp.libcall.n = 4 - mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0))) + mp.stdCallInfo.N = 4 + mp.stdCallInfo.Args = uintptr(noescape(unsafe.Pointer(&a0))) return stdcall(fn) } @@ -1025,8 +1022,8 @@ func stdcall4(fn stdFunction, a0, a1, a2, a3 uintptr) uintptr { //go:cgo_unsafe_args func stdcall5(fn stdFunction, a0, a1, a2, a3, a4 uintptr) uintptr { mp := getg().m - mp.libcall.n = 5 - mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0))) + mp.stdCallInfo.N = 5 + mp.stdCallInfo.Args = uintptr(noescape(unsafe.Pointer(&a0))) return stdcall(fn) } @@ -1034,8 +1031,8 @@ func stdcall5(fn stdFunction, a0, a1, a2, a3, a4 uintptr) uintptr { //go:cgo_unsafe_args func stdcall6(fn stdFunction, a0, a1, a2, a3, a4, a5 uintptr) uintptr { mp := getg().m - mp.libcall.n = 6 - mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0))) + mp.stdCallInfo.N = 6 + mp.stdCallInfo.Args = uintptr(noescape(unsafe.Pointer(&a0))) return stdcall(fn) } @@ -1043,8 +1040,8 @@ func stdcall6(fn stdFunction, a0, a1, a2, a3, a4, a5 uintptr) uintptr { //go:cgo_unsafe_args func stdcall7(fn stdFunction, a0, a1, a2, a3, a4, a5, a6 uintptr) uintptr { mp := getg().m - mp.libcall.n = 7 - mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0))) + mp.stdCallInfo.N = 7 + mp.stdCallInfo.Args = uintptr(noescape(unsafe.Pointer(&a0))) return stdcall(fn) } @@ -1052,8 +1049,8 @@ func stdcall7(fn stdFunction, a0, a1, a2, a3, a4, a5, a6 uintptr) uintptr { //go:cgo_unsafe_args func stdcall8(fn stdFunction, a0, a1, a2, a3, a4, a5, a6, a7 uintptr) uintptr { mp := getg().m - mp.libcall.n = 8 - mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0))) + mp.stdCallInfo.N = 8 + mp.stdCallInfo.Args = uintptr(noescape(unsafe.Pointer(&a0))) return stdcall(fn) } diff --git a/src/runtime/runtime2.go b/src/runtime/runtime2.go index ee07c1ed93..29e9b8a7b9 100644 --- a/src/runtime/runtime2.go +++ b/src/runtime/runtime2.go @@ -593,9 +593,7 @@ type m struct { freelink *m // on sched.freem trace mTraceState - // these are here because they are too large to be on the stack - // of low-level NOSPLIT functions. - libcall libcall + // These are here to avoid using the G stack so the stack can move during the call. libcallpc uintptr // for cpu profiler libcallsp uintptr libcallg guintptr diff --git a/src/runtime/sys_aix_ppc64.s b/src/runtime/sys_aix_ppc64.s index 66081977b1..a0ef7e111a 100644 --- a/src/runtime/sys_aix_ppc64.s +++ b/src/runtime/sys_aix_ppc64.s @@ -130,15 +130,15 @@ TEXT sigtramp<>(SB),NOSPLIT|NOFRAME|TOPFRAME,$0 // Save m->libcall. We need to do this because we // might get interrupted by a signal in runtime·asmcgocall. - MOVD (m_libcall+libcall_fn)(R6), R7 + MOVD (m_mOS+mOS_libcall+libcall_fn)(R6), R7 MOVD R7, 96(R1) - MOVD (m_libcall+libcall_args)(R6), R7 + MOVD (m_mOS+mOS_libcall+libcall_args)(R6), R7 MOVD R7, 104(R1) - MOVD (m_libcall+libcall_n)(R6), R7 + MOVD (m_mOS+mOS_libcall+libcall_n)(R6), R7 MOVD R7, 112(R1) - MOVD (m_libcall+libcall_r1)(R6), R7 + MOVD (m_mOS+mOS_libcall+libcall_r1)(R6), R7 MOVD R7, 120(R1) - MOVD (m_libcall+libcall_r2)(R6), R7 + MOVD (m_mOS+mOS_libcall+libcall_r2)(R6), R7 MOVD R7, 128(R1) // save errno, it might be EINTR; stuff we do here might reset it. @@ -162,15 +162,15 @@ sigtramp: // restore libcall MOVD 96(R1), R7 - MOVD R7, (m_libcall+libcall_fn)(R6) + MOVD R7, (m_mOS+mOS_libcall+libcall_fn)(R6) MOVD 104(R1), R7 - MOVD R7, (m_libcall+libcall_args)(R6) + MOVD R7, (m_mOS+mOS_libcall+libcall_args)(R6) MOVD 112(R1), R7 - MOVD R7, (m_libcall+libcall_n)(R6) + MOVD R7, (m_mOS+mOS_libcall+libcall_n)(R6) MOVD 120(R1), R7 - MOVD R7, (m_libcall+libcall_r1)(R6) + MOVD R7, (m_mOS+mOS_libcall+libcall_r1)(R6) MOVD 128(R1), R7 - MOVD R7, (m_libcall+libcall_r2)(R6) + MOVD R7, (m_mOS+mOS_libcall+libcall_r2)(R6) // restore errno MOVD (m_mOS+mOS_perrno)(R6), R7 diff --git a/src/runtime/sys_solaris_amd64.s b/src/runtime/sys_solaris_amd64.s index 7a80020ba3..9235cad391 100644 --- a/src/runtime/sys_solaris_amd64.s +++ b/src/runtime/sys_solaris_amd64.s @@ -155,7 +155,7 @@ allgood: // save m->libcall MOVQ g_m(R10), BP - LEAQ m_libcall(BP), R11 + LEAQ (m_mOS+mOS_libcall)(BP), R11 MOVQ libcall_fn(R11), R10 MOVQ R10, 72(SP) MOVQ libcall_args(R11), R10 @@ -197,7 +197,7 @@ allgood: MOVQ g(BX), BP MOVQ g_m(BP), BP // restore libcall - LEAQ m_libcall(BP), R11 + LEAQ (m_mOS+mOS_libcall)(BP), R11 MOVQ 72(SP), R10 MOVQ R10, libcall_fn(R11) MOVQ 80(SP), R10 diff --git a/src/runtime/sys_windows_386.s b/src/runtime/sys_windows_386.s index e71fda78ae..4030e4c38b 100644 --- a/src/runtime/sys_windows_386.s +++ b/src/runtime/sys_windows_386.s @@ -11,49 +11,6 @@ #define TEB_TlsSlots 0xE10 #define TEB_ArbitraryPtr 0x14 -TEXT runtime·asmstdcall_trampoline(SB),NOSPLIT,$0 - JMP runtime·asmstdcall(SB) - -// void runtime·asmstdcall(void *c); -TEXT runtime·asmstdcall(SB),NOSPLIT,$0 - MOVL fn+0(FP), BX - MOVL SP, BP // save stack pointer - - // SetLastError(0). - MOVL $0, 0x34(FS) - - MOVL libcall_n(BX), CX - - // Fast version, do not store args on the stack. - CMPL CX, $0 - JE docall - - // Copy args to the stack. - MOVL CX, AX - SALL $2, AX - SUBL AX, SP // room for args - MOVL SP, DI - MOVL libcall_args(BX), SI - CLD - REP; MOVSL - -docall: - // Call stdcall or cdecl function. - // DI SI BP BX are preserved, SP is not - CALL libcall_fn(BX) - MOVL BP, SP - - // Return result. - MOVL fn+0(FP), BX - MOVL AX, libcall_r1(BX) - MOVL DX, libcall_r2(BX) - - // GetLastError(). - MOVL 0x34(FS), AX - MOVL AX, libcall_err(BX) - - RET - // faster get/set last error TEXT runtime·getlasterror(SB),NOSPLIT,$0 MOVL 0x34(FS), AX diff --git a/src/runtime/sys_windows_amd64.s b/src/runtime/sys_windows_amd64.s index 56a2dc0bcf..e438599910 100644 --- a/src/runtime/sys_windows_amd64.s +++ b/src/runtime/sys_windows_amd64.s @@ -12,85 +12,6 @@ #define TEB_TlsSlots 0x1480 #define TEB_ArbitraryPtr 0x28 -TEXT runtime·asmstdcall_trampoline(SB),NOSPLIT,$0 - MOVQ AX, CX - JMP runtime·asmstdcall(SB) - -// void runtime·asmstdcall(void *c); -TEXT runtime·asmstdcall(SB),NOSPLIT,$16 - MOVQ SP, AX - ANDQ $~15, SP // alignment as per Windows requirement - MOVQ AX, 8(SP) - MOVQ CX, 0(SP) // asmcgocall will put first argument into CX. - - MOVQ libcall_fn(CX), AX - MOVQ libcall_args(CX), SI - MOVQ libcall_n(CX), CX - - // SetLastError(0). - MOVQ 0x30(GS), DI - MOVL $0, 0x68(DI) - - SUBQ $(const_maxArgs*8), SP // room for args - - // Fast version, do not store args on the stack. - CMPL CX, $0; JE _0args - CMPL CX, $1; JE _1args - CMPL CX, $2; JE _2args - CMPL CX, $3; JE _3args - CMPL CX, $4; JE _4args - - // Check we have enough room for args. - CMPL CX, $const_maxArgs - JLE 2(PC) - INT $3 // not enough room -> crash - - // Copy args to the stack. - MOVQ SP, DI - CLD - REP; MOVSQ - MOVQ SP, SI - - // Load first 4 args into correspondent registers. - // Floating point arguments are passed in the XMM - // registers. Set them here in case any of the arguments - // are floating point values. For details see - // https://learn.microsoft.com/en-us/cpp/build/x64-calling-convention?view=msvc-170 -_4args: - MOVQ 24(SI), R9 - MOVQ R9, X3 -_3args: - MOVQ 16(SI), R8 - MOVQ R8, X2 -_2args: - MOVQ 8(SI), DX - MOVQ DX, X1 -_1args: - MOVQ 0(SI), CX - MOVQ CX, X0 -_0args: - - // Call stdcall function. - CALL AX - - ADDQ $(const_maxArgs*8), SP - - // Return result. - MOVQ 0(SP), CX - MOVQ 8(SP), SP - MOVQ AX, libcall_r1(CX) - // Floating point return values are returned in XMM0. Setting r2 to this - // value in case this call returned a floating point value. For details, - // see https://docs.microsoft.com/en-us/cpp/build/x64-calling-convention - MOVQ X0, libcall_r2(CX) - - // GetLastError(). - MOVQ 0x30(GS), DI - MOVL 0x68(DI), AX - MOVQ AX, libcall_err(CX) - - RET - // faster get/set last error TEXT runtime·getlasterror(SB),NOSPLIT,$0 MOVQ 0x30(GS), AX diff --git a/src/runtime/sys_windows_arm.s b/src/runtime/sys_windows_arm.s index 99f33cf07d..c7f2369e57 100644 --- a/src/runtime/sys_windows_arm.s +++ b/src/runtime/sys_windows_arm.s @@ -9,76 +9,6 @@ // Note: For system ABI, R0-R3 are args, R4-R11 are callee-save. -TEXT runtime·asmstdcall_trampoline(SB),NOSPLIT,$0 - B runtime·asmstdcall(SB) - -// void runtime·asmstdcall(void *c); -TEXT runtime·asmstdcall(SB),NOSPLIT|NOFRAME,$0 - MOVM.DB.W [R4, R5, R14], (R13) // push {r4, r5, lr} - MOVW R0, R4 // put libcall * in r4 - MOVW R13, R5 // save stack pointer in r5 - - // SetLastError(0) - MOVW $0, R0 - MRC 15, 0, R1, C13, C0, 2 - MOVW R0, 0x34(R1) - - MOVW 8(R4), R12 // libcall->args - - // Do we have more than 4 arguments? - MOVW 4(R4), R0 // libcall->n - SUB.S $4, R0, R2 - BLE loadregs - - // Reserve stack space for remaining args - SUB R2<<2, R13 - BIC $0x7, R13 // alignment for ABI - - // R0: count of arguments - // R1: - // R2: loop counter, from 0 to (n-4) - // R3: scratch - // R4: pointer to libcall struct - // R12: libcall->args - MOVW $0, R2 -stackargs: - ADD $4, R2, R3 // r3 = args[4 + i] - MOVW R3<<2(R12), R3 - MOVW R3, R2<<2(R13) // stack[i] = r3 - - ADD $1, R2 // i++ - SUB $4, R0, R3 // while (i < (n - 4)) - CMP R3, R2 - BLT stackargs - -loadregs: - CMP $3, R0 - MOVW.GT 12(R12), R3 - - CMP $2, R0 - MOVW.GT 8(R12), R2 - - CMP $1, R0 - MOVW.GT 4(R12), R1 - - CMP $0, R0 - MOVW.GT 0(R12), R0 - - BIC $0x7, R13 // alignment for ABI - MOVW 0(R4), R12 // branch to libcall->fn - BL (R12) - - MOVW R5, R13 // free stack space - MOVW R0, 12(R4) // save return value to libcall->r1 - MOVW R1, 16(R4) - - // GetLastError - MRC 15, 0, R1, C13, C0, 2 - MOVW 0x34(R1), R0 - MOVW R0, 20(R4) // store in libcall->err - - MOVM.IA.W (R13), [R4, R5, R15] - TEXT runtime·getlasterror(SB),NOSPLIT,$0 MRC 15, 0, R0, C13, C0, 2 MOVW 0x34(R0), R0 diff --git a/src/runtime/sys_windows_arm64.s b/src/runtime/sys_windows_arm64.s index 1f6d411b07..da3cb7e546 100644 --- a/src/runtime/sys_windows_arm64.s +++ b/src/runtime/sys_windows_arm64.s @@ -19,88 +19,6 @@ // // load_g and save_g (in tls_arm64.s) clobber R27 (REGTMP) and R0. -TEXT runtime·asmstdcall_trampoline(SB),NOSPLIT,$0 - B runtime·asmstdcall(SB) - -// void runtime·asmstdcall(void *c); -TEXT runtime·asmstdcall(SB),NOSPLIT,$16 - STP (R19, R20), 16(RSP) // save old R19, R20 - MOVD R0, R19 // save libcall pointer - MOVD RSP, R20 // save stack pointer - - // SetLastError(0) - MOVD $0, TEB_error(R18_PLATFORM) - MOVD libcall_args(R19), R12 // libcall->args - - // Do we have more than 8 arguments? - MOVD libcall_n(R19), R0 - CMP $0, R0; BEQ _0args - CMP $1, R0; BEQ _1args - CMP $2, R0; BEQ _2args - CMP $3, R0; BEQ _3args - CMP $4, R0; BEQ _4args - CMP $5, R0; BEQ _5args - CMP $6, R0; BEQ _6args - CMP $7, R0; BEQ _7args - CMP $8, R0; BEQ _8args - - // Reserve stack space for remaining args - SUB $8, R0, R2 - ADD $1, R2, R3 // make even number of words for stack alignment - AND $~1, R3 - LSL $3, R3 - SUB R3, RSP - - // R4: size of stack arguments (n-8)*8 - // R5: &args[8] - // R6: loop counter, from 0 to (n-8)*8 - // R7: scratch - // R8: copy of RSP - (R2)(RSP) assembles as (R2)(ZR) - SUB $8, R0, R4 - LSL $3, R4 - ADD $(8*8), R12, R5 - MOVD $0, R6 - MOVD RSP, R8 -stackargs: - MOVD (R6)(R5), R7 - MOVD R7, (R6)(R8) - ADD $8, R6 - CMP R6, R4 - BNE stackargs - -_8args: - MOVD (7*8)(R12), R7 -_7args: - MOVD (6*8)(R12), R6 -_6args: - MOVD (5*8)(R12), R5 -_5args: - MOVD (4*8)(R12), R4 -_4args: - MOVD (3*8)(R12), R3 -_3args: - MOVD (2*8)(R12), R2 -_2args: - MOVD (1*8)(R12), R1 -_1args: - MOVD (0*8)(R12), R0 -_0args: - - MOVD libcall_fn(R19), R12 // branch to libcall->fn - BL (R12) - - MOVD R20, RSP // free stack space - MOVD R0, libcall_r1(R19) // save return value to libcall->r1 - // TODO(rsc) floating point like amd64 in libcall->r2? - - // GetLastError - MOVD TEB_error(R18_PLATFORM), R0 - MOVD R0, libcall_err(R19) - - // Restore callee-saved registers. - LDP 16(RSP), (R19, R20) - RET - TEXT runtime·getlasterror(SB),NOSPLIT,$0 MOVD TEB_error(R18_PLATFORM), R0 MOVD R0, ret+0(FP) diff --git a/src/runtime/syscall_windows.go b/src/runtime/syscall_windows.go index 85b1b8c902..a9c0588a0a 100644 --- a/src/runtime/syscall_windows.go +++ b/src/runtime/syscall_windows.go @@ -7,6 +7,7 @@ package runtime import ( "internal/abi" "internal/goarch" + "internal/runtime/syscall/windows" "unsafe" ) @@ -487,13 +488,6 @@ func syscall_Syscall18(fn, nargs, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, return syscall_syscalln(fn, nargs, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15, a16, a17, a18) } -// maxArgs should be divisible by 2, as Windows stack -// must be kept 16-byte aligned on syscall entry. -// -// Although it only permits maximum 42 parameters, it -// is arguably large enough. -const maxArgs = 42 - //go:linkname syscall_SyscallN syscall.SyscallN //go:nosplit func syscall_SyscallN(fn uintptr, args ...uintptr) (r1, r2, err uintptr) { @@ -505,7 +499,7 @@ func syscall_syscalln(fn, n uintptr, args ...uintptr) (r1, r2, err uintptr) { if n > uintptr(len(args)) { panic("syscall: n > len(args)") // should not be reachable from user code } - if n > maxArgs { + if n > windows.MaxArgs { panic("runtime: SyscallN has too many arguments") } @@ -513,15 +507,15 @@ func syscall_syscalln(fn, n uintptr, args ...uintptr) (r1, r2, err uintptr) { // the stack because the stack can move during fn if it // calls back into Go. c := &getg().m.winsyscall - c.fn = fn - c.n = n - if c.n != 0 { - c.args = uintptr(noescape(unsafe.Pointer(&args[0]))) + c.Fn = fn + c.N = n + if c.N != 0 { + c.Args = uintptr(noescape(unsafe.Pointer(&args[0]))) } cgocall(asmstdcallAddr, unsafe.Pointer(c)) // cgocall may reschedule us on to a different M, // but it copies the return values into the new M's // so we can read them from there. c = &getg().m.winsyscall - return c.r1, c.r2, c.err + return c.R1, c.R2, c.Err } diff --git a/src/runtime/syscall_windows_test.go b/src/runtime/syscall_windows_test.go index ad9bfb464b..6a9b165d62 100644 --- a/src/runtime/syscall_windows_test.go +++ b/src/runtime/syscall_windows_test.go @@ -8,6 +8,7 @@ import ( "fmt" "internal/abi" "internal/race" + "internal/runtime/syscall/windows" "internal/syscall/windows/sysdll" "internal/testenv" "io" @@ -776,7 +777,7 @@ func TestSyscallN(t *testing.T) { t.Skipf("skipping test: GOARCH=%s", runtime.GOARCH) } - for arglen := 0; arglen <= runtime.MaxArgs; arglen++ { + for arglen := 0; arglen <= windows.MaxArgs; arglen++ { arglen := arglen t.Run(fmt.Sprintf("arg-%d", arglen), func(t *testing.T) { t.Parallel() -- 2.51.0