net: handle >=2GiB files with sendfile on Windows

CL 187037 applied a fix to handle the case where
files larger than 2GiB were not being sendfile-d,
in one shot, rejecting any files whose size was
larger than the 2GiB.

This CL allows files that are larger than limit
by SendFile-ing in chunks of upto 2GiB per chunk.

The test has been excluded as testing with 3GB
requires creating a local file, flushing it
and then doing sendfile which takes a while
and could cause flakes on computers without capacity,
but the test can be retroactively accessed at:
https://go-review.googlesource.com/c/go/+/192518/8/src/net/sendfile_windows_test.go

Fixes #33193.

Change-Id: If57c25bc289aec82b748890ac1ac4f55798d6a5e
Reviewed-on: https://go-review.googlesource.com/c/go/+/192518
Run-TryBot: Emmanuel Odeke <emm.odeke@gmail.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Alex Brainman <alex.brainman@gmail.com>

diff --git a/src/net/sendfile_windows.go b/src/net/sendfile_windows.go
index a223e2562ee7e52e84ba7d3e33fbbf3cab6b77ac..4e187205d965604ce2d091198c03b524e8716e9d 100644 (file)
--- a/src/net/sendfile_windows.go
+++ b/src/net/sendfile_windows.go
@@ -18,10 +18,8 @@ import (
 // non-EOF error.
 //
 // if handled == false, sendFile performed no work.
-//
-// Note that sendfile for windows does not support >2GB file.
 func sendFile(fd *netFD, r io.Reader) (written int64, err error, handled bool) {
-       var n int64 = 0 // by default, copy until EOF
+       var n int64 = 0 // by default, copy until EOF.
 
        lr, ok := r.(*io.LimitedReader)
        if ok {
@@ -29,26 +27,53 @@ func sendFile(fd *netFD, r io.Reader) (written int64, err error, handled bool) {
                if n <= 0 {
                        return 0, nil, true
                }
-               // TransmitFile can be invoked in one call with at most
-               // 2,147,483,646 bytes: the maximum value for a 32-bit integer minus 1.
-               // See https://docs.microsoft.com/en-us/windows/win32/api/mswsock/nf-mswsock-transmitfile
-               const maxSendBytes = 0x7fffffff - 1
-               if n > maxSendBytes {
-                       return 0, nil, false
-               }
        }
+
        f, ok := r.(*os.File)
        if !ok {
                return 0, nil, false
        }
 
-       done, err := poll.SendFile(&fd.pfd, syscall.Handle(f.Fd()), n)
+       // TransmitFile can be invoked in one call with at most
+       // 2,147,483,646 bytes: the maximum value for a 32-bit integer minus 1.
+       // See https://docs.microsoft.com/en-us/windows/win32/api/mswsock/nf-mswsock-transmitfile
+       const maxChunkSizePerCall = int64(0x7fffffff - 1)
+
+       switch {
+       case n <= maxChunkSizePerCall:
+               // The file is within sendfile's limits.
+               written, err = doSendFile(fd, lr, f, n)
+
+       default:
+               // Now invoke doSendFile on the file in chunks of upto 2GiB per chunk.
+               for lr.N > 0 { // lr.N is decremented in every successful invocation of doSendFile.
+                       chunkSize := maxChunkSizePerCall
+                       if chunkSize > lr.N {
+                               chunkSize = lr.N
+                       }
+                       var nw int64
+                       nw, err = doSendFile(fd, lr, f, chunkSize)
+                       if err != nil {
+                               break
+                       }
+                       written += nw
+               }
+       }
+
+       // If any byte was copied, regardless of any error
+       // encountered mid-way, handled must be set to true.
+       return written, err, written > 0
+}
 
+// doSendFile is a helper to invoke poll.SendFile.
+// It will decrement lr.N by the number of written bytes.
+func doSendFile(fd *netFD, lr *io.LimitedReader, f *os.File, remain int64) (written int64, err error) {
+       done, err := poll.SendFile(&fd.pfd, syscall.Handle(f.Fd()), remain)
        if err != nil {
-               return 0, wrapSyscallError("transmitfile", err), false
+               return 0, wrapSyscallError("transmitfile", err)
        }
        if lr != nil {
                lr.N -= int64(done)
        }
-       return int64(done), nil, true
+       return int64(done), nil
 }