From b2f40019da3e7c0cf1e79b6f5e039c8bb5a877f3 Mon Sep 17 00:00:00 2001 From: Emmanuel T Odeke Date: Sun, 8 Sep 2019 12:03:42 -0700 Subject: [PATCH] internal/poll: make SendFile work with large files on Windows CL 192518 was a minimal simplification to get sendfile on Windows to work with chunked files, but as I had mentioned, I would add even more improvements. This CL improves it by: * If the reader is not an *io.LimitedReader, since the underlying reader is anyways an *os.File, we fallback and stat that file to determine the file size and then also invoke the chunked sendFile on the underlying reader. This issue existed even before the prior CL. * Extracting the chunked TransmitFile logic and moving it directly into internal/poll.SendFile. Thus if the callers of net.sendFile don't use *io.LimitedReader, but have a huge file (>2GiB), we can still invoke the chunked internal/poll.SendFile on it directly. The test case is not included in this patch as it requires creating a 3GiB file, but that if anyone wants to view it, they can find it at https://go-review.googlesource.com/c/go/+/194218/13/src/net/sendfile_windows_test.go Updates #33193. Change-Id: I97a67c712d558c84ced716d8df98b040cd7ed7f7 Reviewed-on: https://go-review.googlesource.com/c/go/+/194218 Run-TryBot: Emmanuel Odeke TryBot-Result: Gobot Gobot Reviewed-by: Alex Brainman --- src/internal/poll/sendfile_windows.go | 61 +++++++++++++++++++++------ src/net/sendfile_windows.go | 42 +++--------------- 2 files changed, 54 insertions(+), 49 deletions(-) diff --git a/src/internal/poll/sendfile_windows.go b/src/internal/poll/sendfile_windows.go index 0fe9b9b420..5674af4189 100644 --- a/src/internal/poll/sendfile_windows.go +++ b/src/internal/poll/sendfile_windows.go @@ -4,10 +4,13 @@ package poll -import "syscall" +import ( + "io" + "syscall" +) // SendFile wraps the TransmitFile call. -func SendFile(fd *FD, src syscall.Handle, n int64) (int64, error) { +func SendFile(fd *FD, src syscall.Handle, n int64) (written int64, err error) { if fd.kind == kindPipe { // TransmitFile does not work with pipes return 0, syscall.ESPIPE @@ -19,26 +22,60 @@ func SendFile(fd *FD, src syscall.Handle, n int64) (int64, error) { defer fd.writeUnlock() o := &fd.wop - o.qty = uint32(n) o.handle = src // TODO(brainman): skip calling syscall.Seek if OS allows it - curpos, err := syscall.Seek(o.handle, 0, 1) + curpos, err := syscall.Seek(o.handle, 0, io.SeekCurrent) if err != nil { return 0, err } - o.o.Offset = uint32(curpos) - o.o.OffsetHigh = uint32(curpos >> 32) + if n <= 0 { // We don't know the size of the file so infer it. + // Find the number of bytes offset from curpos until the end of the file. + n, err = syscall.Seek(o.handle, -curpos, io.SeekEnd) + if err != nil { + return + } + // Now seek back to the original position. + if _, err = syscall.Seek(o.handle, curpos, io.SeekStart); err != nil { + return + } + } + + // TransmitFile can be invoked in one call with at most + // 2,147,483,646 bytes: the maximum value for a 32-bit integer minus 1. + // See https://docs.microsoft.com/en-us/windows/win32/api/mswsock/nf-mswsock-transmitfile + const maxChunkSizePerCall = int64(0x7fffffff - 1) + + for n > 0 { + chunkSize := maxChunkSizePerCall + if chunkSize > n { + chunkSize = n + } + + o.qty = uint32(chunkSize) + o.o.Offset = uint32(curpos) + o.o.OffsetHigh = uint32(curpos >> 32) + + nw, err := wsrv.ExecIO(o, func(o *operation) error { + return syscall.TransmitFile(o.fd.Sysfd, o.handle, o.qty, 0, &o.o, nil, syscall.TF_WRITE_BEHIND) + }) + if err != nil { + return written, err + } + + curpos += int64(nw) - done, err := wsrv.ExecIO(o, func(o *operation) error { - return syscall.TransmitFile(o.fd.Sysfd, o.handle, o.qty, 0, &o.o, nil, syscall.TF_WRITE_BEHIND) - }) - if err == nil { // Some versions of Windows (Windows 10 1803) do not set // file position after TransmitFile completes. // So just use Seek to set file position. - _, err = syscall.Seek(o.handle, curpos+int64(done), 0) + if _, err = syscall.Seek(o.handle, curpos, io.SeekStart); err != nil { + return written, err + } + + n -= int64(nw) + written += int64(nw) } - return int64(done), err + + return } diff --git a/src/net/sendfile_windows.go b/src/net/sendfile_windows.go index 4e187205d9..59b1b0d5c1 100644 --- a/src/net/sendfile_windows.go +++ b/src/net/sendfile_windows.go @@ -34,46 +34,14 @@ func sendFile(fd *netFD, r io.Reader) (written int64, err error, handled bool) { return 0, nil, false } - // TransmitFile can be invoked in one call with at most - // 2,147,483,646 bytes: the maximum value for a 32-bit integer minus 1. - // See https://docs.microsoft.com/en-us/windows/win32/api/mswsock/nf-mswsock-transmitfile - const maxChunkSizePerCall = int64(0x7fffffff - 1) - - switch { - case n <= maxChunkSizePerCall: - // The file is within sendfile's limits. - written, err = doSendFile(fd, lr, f, n) - - default: - // Now invoke doSendFile on the file in chunks of upto 2GiB per chunk. - for lr.N > 0 { // lr.N is decremented in every successful invocation of doSendFile. - chunkSize := maxChunkSizePerCall - if chunkSize > lr.N { - chunkSize = lr.N - } - var nw int64 - nw, err = doSendFile(fd, lr, f, chunkSize) - if err != nil { - break - } - written += nw - } + written, err = poll.SendFile(&fd.pfd, syscall.Handle(f.Fd()), n) + if err != nil { + err = wrapSyscallError("transmitfile", err) } // If any byte was copied, regardless of any error // encountered mid-way, handled must be set to true. - return written, err, written > 0 -} + handled = written > 0 -// doSendFile is a helper to invoke poll.SendFile. -// It will decrement lr.N by the number of written bytes. -func doSendFile(fd *netFD, lr *io.LimitedReader, f *os.File, remain int64) (written int64, err error) { - done, err := poll.SendFile(&fd.pfd, syscall.Handle(f.Fd()), remain) - if err != nil { - return 0, wrapSyscallError("transmitfile", err) - } - if lr != nil { - lr.N -= int64(done) - } - return int64(done), nil + return } -- 2.48.1