syscall: use CLONE_VFORK and CLONE_VM

This greatly improves the latency of starting a child process when
the Go process is using a lot of memory. Even though the kernel uses
copy-on-write, preparation for that can take up to several 100ms under
certain conditions. All other goroutines are suspended while starting
a subprocess so this latency directly affects total throughput.

With CLONE_VM the child process shares the same memory with the parent
process. On its own this would lead to conflicting use of the same
memory, so CLONE_VFORK is used to suspend the parent process until the
child releases the memory when switching to to the new program binary
via the exec syscall. When the parent process continues to run, one
has to consider the changes to memory that the child process did,
namely the return address of the syscall function needs to be restored
from a register.

A simple benchmark has shown a difference in latency of 16ms vs. 0.5ms
at 10GB memory usage. However, much higher latencies of several 100ms
have been observed in real world scenarios. For more information see
comments on #5838.

Fixes #5838

Change-Id: I6377d7bd8dcd00c85ca0c52b6683e70ce2174ba6
Reviewed-on: https://go-review.googlesource.com/37439
Reviewed-by: Ian Lance Taylor <iant@golang.org>
Run-TryBot: Ian Lance Taylor <iant@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>

diff --git a/src/syscall/asm_linux_amd64.s b/src/syscall/asm_linux_amd64.s
index 6634875f6a1927fb48d6ec449dc3079eccc92794..b7cd5416fea7ac467f671473477368d3ba3c946b 100644 (file)
--- a/src/syscall/asm_linux_amd64.s
+++ b/src/syscall/asm_linux_amd64.s
@@ -111,6 +111,29 @@ ok2:
        MOVQ    $0, err+72(FP)
        RET
 
+// func rawVforkSyscall(trap, a1 uintptr) (r1, err uintptr)
+TEXT ·rawVforkSyscall(SB),NOSPLIT,$0-32
+       MOVQ    a1+8(FP), DI
+       MOVQ    $0, SI
+       MOVQ    $0, DX
+       MOVQ    $0, R10
+       MOVQ    $0, R8
+       MOVQ    $0, R9
+       MOVQ    trap+0(FP), AX  // syscall entry
+       POPQ    R12 // preserve return address
+       SYSCALL
+       PUSHQ   R12
+       CMPQ    AX, $0xfffffffffffff001
+       JLS     ok2
+       MOVQ    $-1, r1+16(FP)
+       NEGQ    AX
+       MOVQ    AX, err+24(FP)
+       RET
+ok2:
+       MOVQ    AX, r1+16(FP)
+       MOVQ    $0, err+24(FP)
+       RET
+
 // func gettimeofday(tv *Timeval) (err uintptr)
 TEXT ·gettimeofday(SB),NOSPLIT,$0-16
        MOVQ    tv+0(FP), DI

diff --git a/src/syscall/exec_linux.go b/src/syscall/exec_linux.go
index 6ad20f6af10cb2082ae1fd643d395625a3914714..1ed10dd9150d301c12830773aa6faec40a72deee 100644 (file)
--- a/src/syscall/exec_linux.go
+++ b/src/syscall/exec_linux.go
@@ -95,9 +95,12 @@ func forkAndExecInChild(argv0 *byte, argv, envv []*byte, chroot, dir *byte, attr
        // About to call fork.
        // No more allocation or calls of non-assembly functions.
        runtime_BeforeFork()
-       if runtime.GOARCH == "s390x" {
+       switch {
+       case runtime.GOARCH == "amd64" && sys.Cloneflags&CLONE_NEWUSER == 0:
+               r1, err1 = rawVforkSyscall(SYS_CLONE, uintptr(SIGCHLD|CLONE_VFORK|CLONE_VM)|sys.Cloneflags)
+       case runtime.GOARCH == "s390x":
                r1, _, err1 = RawSyscall6(SYS_CLONE, 0, uintptr(SIGCHLD)|sys.Cloneflags, 0, 0, 0, 0)
-       } else {
+       default:
                r1, _, err1 = RawSyscall6(SYS_CLONE, uintptr(SIGCHLD)|sys.Cloneflags, 0, 0, 0, 0, 0)
        }
        if err1 != 0 {

diff --git a/src/syscall/syscall_linux_386.go b/src/syscall/syscall_linux_386.go
index d08338b1dbcd86baa1cdf065bbe502f27e04901c..2c5d9a3eee79a797c445f6c45502afae3a5a237c 100644 (file)
--- a/src/syscall/syscall_linux_386.go
+++ b/src/syscall/syscall_linux_386.go
@@ -375,3 +375,7 @@ func (msghdr *Msghdr) SetControllen(length int) {
 func (cmsg *Cmsghdr) SetLen(length int) {
        cmsg.Len = uint32(length)
 }
+
+func rawVforkSyscall(trap, a1 uintptr) (r1 uintptr, err Errno) {
+       panic("not implemented")
+}

diff --git a/src/syscall/syscall_linux_amd64.go b/src/syscall/syscall_linux_amd64.go
index 0184d7d850b377365e5d4b3ecd037f8d79774d7c..eaba868f89491c6574addf62a99bf00f8718ed1b 100644 (file)
--- a/src/syscall/syscall_linux_amd64.go
+++ b/src/syscall/syscall_linux_amd64.go
@@ -134,3 +134,5 @@ func (msghdr *Msghdr) SetControllen(length int) {
 func (cmsg *Cmsghdr) SetLen(length int) {
        cmsg.Len = uint64(length)
 }
+
+func rawVforkSyscall(trap, a1 uintptr) (r1 uintptr, err Errno)

diff --git a/src/syscall/syscall_linux_arm.go b/src/syscall/syscall_linux_arm.go
index a58ebd4ed1e5773645c9b3b1572890da10c5cffc..5c652b2e5bfce582ec79a1c76e64cdbd532538dd 100644 (file)
--- a/src/syscall/syscall_linux_arm.go
+++ b/src/syscall/syscall_linux_arm.go
@@ -215,3 +215,7 @@ func (msghdr *Msghdr) SetControllen(length int) {
 func (cmsg *Cmsghdr) SetLen(length int) {
        cmsg.Len = uint32(length)
 }
+
+func rawVforkSyscall(trap, a1 uintptr) (r1 uintptr, err Errno) {
+       panic("not implemented")
+}

diff --git a/src/syscall/syscall_linux_arm64.go b/src/syscall/syscall_linux_arm64.go
index 4462139c8b3376cf769c9231a3133a8ca667bfbe..4e81673109053e404b37435b1481b5caefb5da2b 100644 (file)
--- a/src/syscall/syscall_linux_arm64.go
+++ b/src/syscall/syscall_linux_arm64.go
@@ -139,3 +139,7 @@ const (
        SYS_EPOLL_CREATE = 1042
        SYS_EPOLL_WAIT   = 1069
 )
+
+func rawVforkSyscall(trap, a1 uintptr) (r1 uintptr, err Errno) {
+       panic("not implemented")
+}

diff --git a/src/syscall/syscall_linux_mips64x.go b/src/syscall/syscall_linux_mips64x.go
index 9fd7982d2fcc3d78d3f54ca0caa0315d443080a7..671cfe6d8abe7391bddc12df6621a55adb949e60 100644 (file)
--- a/src/syscall/syscall_linux_mips64x.go
+++ b/src/syscall/syscall_linux_mips64x.go
@@ -198,3 +198,7 @@ func (msghdr *Msghdr) SetControllen(length int) {
 func (cmsg *Cmsghdr) SetLen(length int) {
        cmsg.Len = uint64(length)
 }
+
+func rawVforkSyscall(trap, a1 uintptr) (r1 uintptr, err Errno) {
+       panic("not implemented")
+}

diff --git a/src/syscall/syscall_linux_mipsx.go b/src/syscall/syscall_linux_mipsx.go
index 1b072c13803355080f16b6c97ee5d1acfdbb59b0..1da265d3c450b5e1375d47b5e3caf535f6608fd3 100644 (file)
--- a/src/syscall/syscall_linux_mipsx.go
+++ b/src/syscall/syscall_linux_mipsx.go
@@ -218,3 +218,7 @@ func (msghdr *Msghdr) SetControllen(length int) {
 func (cmsg *Cmsghdr) SetLen(length int) {
        cmsg.Len = uint32(length)
 }
+
+func rawVforkSyscall(trap, a1 uintptr) (r1 uintptr, err Errno) {
+       panic("not implemented")
+}

diff --git a/src/syscall/syscall_linux_ppc64x.go b/src/syscall/syscall_linux_ppc64x.go
index 307abc9f53d287ae6be533f78faef06f1d3ee1f6..53086f90347c463538a0493e40ffbb9e9ecd0312 100644 (file)
--- a/src/syscall/syscall_linux_ppc64x.go
+++ b/src/syscall/syscall_linux_ppc64x.go
@@ -115,3 +115,7 @@ func (msghdr *Msghdr) SetControllen(length int) {
 func (cmsg *Cmsghdr) SetLen(length int) {
        cmsg.Len = uint64(length)
 }
+
+func rawVforkSyscall(trap, a1 uintptr) (r1 uintptr, err Errno) {
+       panic("not implemented")
+}

diff --git a/src/syscall/syscall_linux_s390x.go b/src/syscall/syscall_linux_s390x.go
index a1dd5d6fdb6bfff50b8213c91b9eb0e2af6d4026..8f3bbfc6f74c08a0782aeb71c69109d2cbac1c87 100644 (file)
--- a/src/syscall/syscall_linux_s390x.go
+++ b/src/syscall/syscall_linux_s390x.go
@@ -286,3 +286,7 @@ func (msghdr *Msghdr) SetControllen(length int) {
 func (cmsg *Cmsghdr) SetLen(length int) {
        cmsg.Len = uint64(length)
 }
+
+func rawVforkSyscall(trap, a1 uintptr) (r1 uintptr, err Errno) {
+       panic("not implemented")
+}