internal/poll: make SendFile work with large files on Windows

CL 192518 was a minimal simplification to get sendfile
on Windows to work with chunked files, but as I had mentioned,
I would add even more improvements.

This CL improves it by:
* If the reader is not an *io.LimitedReader, since the underlying
reader is anyways an *os.File, we fallback and stat that
file to determine the file size and then also invoke the chunked
sendFile on the underlying reader. This issue existed even
before the prior CL.
* Extracting the chunked TransmitFile logic and moving it directly
into internal/poll.SendFile.

Thus if the callers of net.sendFile don't use *io.LimitedReader,
but have a huge file (>2GiB), we can still invoke the chunked
internal/poll.SendFile on it directly.

The test case is not included in this patch as it requires
creating a 3GiB file, but that if anyone wants to view it, they
can find it at
    https://go-review.googlesource.com/c/go/+/194218/13/src/net/sendfile_windows_test.go

Updates #33193.

Change-Id: I97a67c712d558c84ced716d8df98b040cd7ed7f7
Reviewed-on: https://go-review.googlesource.com/c/go/+/194218
Run-TryBot: Emmanuel Odeke <emm.odeke@gmail.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Alex Brainman <alex.brainman@gmail.com>

diff --git a/src/internal/poll/sendfile_windows.go b/src/internal/poll/sendfile_windows.go
index 0fe9b9b420922f8acedf5a9dd5c801e675da9071..5674af418952578f25551230ebe3cd42df2f8a45 100644 (file)
--- a/src/internal/poll/sendfile_windows.go
+++ b/src/internal/poll/sendfile_windows.go
@@ -4,10 +4,13 @@
 
 package poll
 
-import "syscall"
+import (
+       "io"
+       "syscall"
+)
 
 // SendFile wraps the TransmitFile call.
-func SendFile(fd *FD, src syscall.Handle, n int64) (int64, error) {
+func SendFile(fd *FD, src syscall.Handle, n int64) (written int64, err error) {
        if fd.kind == kindPipe {
                // TransmitFile does not work with pipes
                return 0, syscall.ESPIPE
@@ -19,26 +22,60 @@ func SendFile(fd *FD, src syscall.Handle, n int64) (int64, error) {
        defer fd.writeUnlock()
 
        o := &fd.wop
-       o.qty = uint32(n)
        o.handle = src
 
        // TODO(brainman): skip calling syscall.Seek if OS allows it
-       curpos, err := syscall.Seek(o.handle, 0, 1)
+       curpos, err := syscall.Seek(o.handle, 0, io.SeekCurrent)
        if err != nil {
                return 0, err
        }
 
-       o.o.Offset = uint32(curpos)
-       o.o.OffsetHigh = uint32(curpos >> 32)
+       if n <= 0 { // We don't know the size of the file so infer it.
+               // Find the number of bytes offset from curpos until the end of the file.
+               n, err = syscall.Seek(o.handle, -curpos, io.SeekEnd)
+               if err != nil {
+                       return
+               }
+               // Now seek back to the original position.
+               if _, err = syscall.Seek(o.handle, curpos, io.SeekStart); err != nil {
+                       return
+               }
+       }
+
+       // TransmitFile can be invoked in one call with at most
+       // 2,147,483,646 bytes: the maximum value for a 32-bit integer minus 1.
+       // See https://docs.microsoft.com/en-us/windows/win32/api/mswsock/nf-mswsock-transmitfile
+       const maxChunkSizePerCall = int64(0x7fffffff - 1)
+
+       for n > 0 {
+               chunkSize := maxChunkSizePerCall
+               if chunkSize > n {
+                       chunkSize = n
+               }
+
+               o.qty = uint32(chunkSize)
+               o.o.Offset = uint32(curpos)
+               o.o.OffsetHigh = uint32(curpos >> 32)
+
+               nw, err := wsrv.ExecIO(o, func(o *operation) error {
+                       return syscall.TransmitFile(o.fd.Sysfd, o.handle, o.qty, 0, &o.o, nil, syscall.TF_WRITE_BEHIND)
+               })
+               if err != nil {
+                       return written, err
+               }
+
+               curpos += int64(nw)
 
-       done, err := wsrv.ExecIO(o, func(o *operation) error {
-               return syscall.TransmitFile(o.fd.Sysfd, o.handle, o.qty, 0, &o.o, nil, syscall.TF_WRITE_BEHIND)
-       })
-       if err == nil {
                // Some versions of Windows (Windows 10 1803) do not set
                // file position after TransmitFile completes.
                // So just use Seek to set file position.
-               _, err = syscall.Seek(o.handle, curpos+int64(done), 0)
+               if _, err = syscall.Seek(o.handle, curpos, io.SeekStart); err != nil {
+                       return written, err
+               }
+
+               n -= int64(nw)
+               written += int64(nw)
        }
-       return int64(done), err
+
+       return
 }

diff --git a/src/net/sendfile_windows.go b/src/net/sendfile_windows.go
index 4e187205d965604ce2d091198c03b524e8716e9d..59b1b0d5c1dd85b0cd540f463184ed7295d3ddc8 100644 (file)
--- a/src/net/sendfile_windows.go
+++ b/src/net/sendfile_windows.go
@@ -34,46 +34,14 @@ func sendFile(fd *netFD, r io.Reader) (written int64, err error, handled bool) {
                return 0, nil, false
        }
 
-       // TransmitFile can be invoked in one call with at most
-       // 2,147,483,646 bytes: the maximum value for a 32-bit integer minus 1.
-       // See https://docs.microsoft.com/en-us/windows/win32/api/mswsock/nf-mswsock-transmitfile
-       const maxChunkSizePerCall = int64(0x7fffffff - 1)
-
-       switch {
-       case n <= maxChunkSizePerCall:
-               // The file is within sendfile's limits.
-               written, err = doSendFile(fd, lr, f, n)
-
-       default:
-               // Now invoke doSendFile on the file in chunks of upto 2GiB per chunk.
-               for lr.N > 0 { // lr.N is decremented in every successful invocation of doSendFile.
-                       chunkSize := maxChunkSizePerCall
-                       if chunkSize > lr.N {
-                               chunkSize = lr.N
-                       }
-                       var nw int64
-                       nw, err = doSendFile(fd, lr, f, chunkSize)
-                       if err != nil {
-                               break
-                       }
-                       written += nw
-               }
+       written, err = poll.SendFile(&fd.pfd, syscall.Handle(f.Fd()), n)
+       if err != nil {
+               err = wrapSyscallError("transmitfile", err)
        }
 
        // If any byte was copied, regardless of any error
        // encountered mid-way, handled must be set to true.
-       return written, err, written > 0
-}
+       handled = written > 0
 
-// doSendFile is a helper to invoke poll.SendFile.
-// It will decrement lr.N by the number of written bytes.
-func doSendFile(fd *netFD, lr *io.LimitedReader, f *os.File, remain int64) (written int64, err error) {
-       done, err := poll.SendFile(&fd.pfd, syscall.Handle(f.Fd()), remain)
-       if err != nil {
-               return 0, wrapSyscallError("transmitfile", err)
-       }
-       if lr != nil {
-               lr.N -= int64(done)
-       }
-       return int64(done), nil
+       return
 }