From 57c79febda993fe2f4b43db07476219e7b07b452 Mon Sep 17 00:00:00 2001 From: Joe Tsai Date: Thu, 31 Aug 2017 14:52:20 -0700 Subject: [PATCH] archive/tar: add Reader.WriteTo and Writer.ReadFrom To support the efficient packing and extracting of sparse files, add two new methods: func Reader.WriteTo(io.Writer) (int64, error) func Writer.ReadFrom(io.Reader) (int64, error) If the current archive entry is sparse and the provided io.{Reader,Writer} is also an io.Seeker, then use Seek to skip past the holes. If the last region in a file entry is a hole, then we seek to 1 byte before the EOF: * for Reader.WriteTo to write a single byte to ensure that the resulting filesize is correct. * for Writer.ReadFrom to read a single byte to verify that the input filesize is correct. The downside of this approach is when the last region in the sparse file is a hole. In the case of Reader.WriteTo, the 1-byte write will cause the last fragment to have a single chunk allocated. However, the goal of ReadFrom/WriteTo is *not* the ability to exactly reproduce sparse files (in terms of the location of sparse holes), but rather to provide an efficient way to create them. File systems already impose their own restrictions on how the sparse file will be created. Some filesystems (e.g., HFS+) don't support sparseness and seeking forward simply causes the FS to write zeros. Other filesystems have different chunk sizes, which will cause chunk allocations at boundaries different from what was in the original sparse file. In either case, it should not be a normal expectation of users that the location of holes in sparse files exactly matches the source. For users that really desire to have exact reproduction of sparse holes, they can wrap os.File with their own io.WriteSeeker that discards the final 1-byte write and uses File.Truncate to resize the file to the correct size. Other reasons we choose this approach over special-casing *os.File because: * The Reader already has special-case logic for io.Seeker * As much as possible, we want to decouple OS-specific logic from Reader and Writer. * This allows other abstractions over *os.File to also benefit from the "skip past holes" logic. * It is easier to test, since it is harder to mock an *os.File. Updates #13548 Change-Id: I0a4f293bd53d13d154a946bc4a2ade28a6646f6a Reviewed-on: https://go-review.googlesource.com/60872 Run-TryBot: Joe Tsai TryBot-Result: Gobot Gobot Reviewed-by: Ian Lance Taylor --- src/archive/tar/common.go | 9 +- src/archive/tar/reader.go | 155 ++++++++++--------- src/archive/tar/reader_test.go | 134 +++++++++------- src/archive/tar/tar_test.go | 178 ++++++++++++++++++++++ src/archive/tar/writer.go | 103 ++++++++----- src/archive/tar/writer_test.go | 270 +++++++++++++++++---------------- 6 files changed, 558 insertions(+), 291 deletions(-) diff --git a/src/archive/tar/common.go b/src/archive/tar/common.go index c67115e49d..8813d089d0 100644 --- a/src/archive/tar/common.go +++ b/src/archive/tar/common.go @@ -323,10 +323,13 @@ func invertSparseEntries(src []SparseEntry, size int64) []SparseEntry { return append(dst, pre) } +// fileState tracks the number of logical (includes sparse holes) and physical +// (actual in tar archive) bytes remaining for the current file. +// +// Invariant: LogicalRemaining >= PhysicalRemaining type fileState interface { - // Remaining reports the number of remaining bytes in the current file. - // This count includes any sparse holes that may exist. - Remaining() int64 + LogicalRemaining() int64 + PhysicalRemaining() int64 } // allowedFormats determines which formats can be used. diff --git a/src/archive/tar/reader.go b/src/archive/tar/reader.go index dc0c95a00a..c2e3041d92 100644 --- a/src/archive/tar/reader.go +++ b/src/archive/tar/reader.go @@ -32,7 +32,7 @@ type fileReader interface { io.Reader fileState - Discard(n int64) (int64, error) + WriteTo(io.Writer) (int64, error) } // NewReader creates a new Reader reading from r. @@ -67,7 +67,7 @@ func (tr *Reader) next() (*Header, error) { loop: for { // Discard the remainder of the file and any padding. - if _, err := tr.curr.Discard(tr.curr.Remaining()); err != nil { + if err := discard(tr.r, tr.curr.PhysicalRemaining()); err != nil { return nil, err } if _, err := tryReadFull(tr.r, tr.blk[:tr.pad]); err != nil { @@ -625,21 +625,19 @@ func (tr *Reader) Read(b []byte) (int, error) { return n, err } -// TODO(dsnet): Export the Reader.Discard method to assist in quickly -// skipping over sections of a file. This is especially useful: -// * when skipping through an underlying io.Reader that is also an io.Seeker. -// * when skipping over large holes in a sparse file. - -// discard skips the next n bytes in the current file, -// returning the number of bytes discarded. -// If fewer than n bytes are discarded, it returns an non-nil error, -// which may be io.EOF if there are no more remaining bytes in the current file. -func (tr *Reader) discard(n int64) (int64, error) { +// WriteTo writes the content of the current file to w. +// The bytes written matches the number of remaining bytes in the current file. +// +// If the current file is sparse and w is an io.WriteSeeker, +// then WriteTo uses Seek to skip past holes defined in Header.SparseHoles, +// assuming that skipped regions are filled with NULs. +// This always writes the last byte to ensure w is the right size. +func (tr *Reader) WriteTo(w io.Writer) (int64, error) { if tr.err != nil { return 0, tr.err } - n, err := tr.curr.Discard(n) - if err != nil && err != io.EOF { + n, err := tr.curr.WriteTo(w) + if err != nil { tr.err = err } return n, err @@ -667,47 +665,14 @@ func (fr *regFileReader) Read(b []byte) (int, error) { } } -func (fr *regFileReader) Discard(n int64) (int64, error) { - overread := n > fr.Remaining() - if overread { - n = fr.Remaining() - } - - // If possible, Seek to the last byte before the end of the data section. - // Do this because Seek is often lazy about reporting errors; this will mask - // the fact that the stream may be truncated. We can rely on the - // io.CopyN done shortly afterwards to trigger any IO errors. - var seekSkipped int64 // Number of bytes skipped via Seek - if sr, ok := fr.r.(io.Seeker); ok && n > 1 { - // Not all io.Seeker can actually Seek. For example, os.Stdin implements - // io.Seeker, but calling Seek always returns an error and performs - // no action. Thus, we try an innocent seek to the current position - // to see if Seek is really supported. - pos1, err := sr.Seek(0, io.SeekCurrent) - if pos1 >= 0 && err == nil { - // Seek seems supported, so perform the real Seek. - pos2, err := sr.Seek(n-1, io.SeekCurrent) - if pos2 < 0 || err != nil { - return 0, err - } - seekSkipped = pos2 - pos1 - } - } - - copySkipped, err := io.CopyN(ioutil.Discard, fr.r, n-seekSkipped) - discarded := seekSkipped + copySkipped - fr.nb -= discarded - switch { - case err == io.EOF && discarded < n: - return discarded, io.ErrUnexpectedEOF - case err == nil && overread: - return discarded, io.EOF - default: - return discarded, err - } +func (fr *regFileReader) WriteTo(w io.Writer) (int64, error) { + return io.Copy(w, struct{ io.Reader }{fr}) } -func (rf regFileReader) Remaining() int64 { +func (rf regFileReader) LogicalRemaining() int64 { + return rf.nb +} +func (rf regFileReader) PhysicalRemaining() int64 { return rf.nb } @@ -719,9 +684,9 @@ type sparseFileReader struct { } func (sr *sparseFileReader) Read(b []byte) (n int, err error) { - finished := int64(len(b)) >= sr.Remaining() + finished := int64(len(b)) >= sr.LogicalRemaining() if finished { - b = b[:sr.Remaining()] + b = b[:sr.LogicalRemaining()] } b0 := b @@ -749,7 +714,7 @@ func (sr *sparseFileReader) Read(b []byte) (n int, err error) { return n, errMissData // Less data in dense file than sparse file case err != nil: return n, err - case sr.Remaining() == 0 && sr.fr.Remaining() > 0: + case sr.LogicalRemaining() == 0 && sr.PhysicalRemaining() > 0: return n, errUnrefData // More data in dense file than sparse file case finished: return n, io.EOF @@ -758,22 +723,32 @@ func (sr *sparseFileReader) Read(b []byte) (n int, err error) { } } -func (sr *sparseFileReader) Discard(n int64) (int64, error) { - overread := n > sr.Remaining() - if overread { - n = sr.Remaining() +func (sr *sparseFileReader) WriteTo(w io.Writer) (n int64, err error) { + ws, ok := w.(io.WriteSeeker) + if ok { + if _, err := ws.Seek(0, io.SeekCurrent); err != nil { + ok = false // Not all io.Seeker can really seek + } + } + if !ok { + return io.Copy(w, struct{ io.Reader }{sr}) } - var realDiscard int64 // Number of real data bytes to discard - endPos := sr.pos + n - for endPos > sr.pos { + var writeLastByte bool + pos0 := sr.pos + for sr.LogicalRemaining() > 0 && !writeLastByte && err == nil { var nf int64 // Size of fragment holeStart, holeEnd := sr.sp[0].Offset, sr.sp[0].endOffset() if sr.pos < holeStart { // In a data fragment - nf = min(endPos-sr.pos, holeStart-sr.pos) - realDiscard += nf + nf = holeStart - sr.pos + nf, err = io.CopyN(ws, sr.fr, nf) } else { // In a hole fragment - nf = min(endPos-sr.pos, holeEnd-sr.pos) + nf = holeEnd - sr.pos + if sr.PhysicalRemaining() == 0 { + writeLastByte = true + nf-- + } + _, err = ws.Seek(nf, io.SeekCurrent) } sr.pos += nf if sr.pos >= holeEnd && len(sr.sp) > 1 { @@ -781,24 +756,32 @@ func (sr *sparseFileReader) Discard(n int64) (int64, error) { } } - _, err := sr.fr.Discard(realDiscard) + // If the last fragment is a hole, then seek to 1-byte before EOF, and + // write a single byte to ensure the file is the right size. + if writeLastByte && err == nil { + _, err = ws.Write([]byte{0}) + sr.pos++ + } + + n = sr.pos - pos0 switch { case err == io.EOF: return n, errMissData // Less data in dense file than sparse file case err != nil: return n, err - case sr.Remaining() == 0 && sr.fr.Remaining() > 0: + case sr.LogicalRemaining() == 0 && sr.PhysicalRemaining() > 0: return n, errUnrefData // More data in dense file than sparse file - case overread: - return n, io.EOF default: return n, nil } } -func (sr sparseFileReader) Remaining() int64 { +func (sr sparseFileReader) LogicalRemaining() int64 { return sr.sp[len(sr.sp)-1].endOffset() - sr.pos } +func (sr sparseFileReader) PhysicalRemaining() int64 { + return sr.fr.PhysicalRemaining() +} type zeroReader struct{} @@ -832,3 +815,33 @@ func tryReadFull(r io.Reader, b []byte) (n int, err error) { } return n, err } + +// discard skips n bytes in r, reporting an error if unable to do so. +func discard(r io.Reader, n int64) error { + // If possible, Seek to the last byte before the end of the data section. + // Do this because Seek is often lazy about reporting errors; this will mask + // the fact that the stream may be truncated. We can rely on the + // io.CopyN done shortly afterwards to trigger any IO errors. + var seekSkipped int64 // Number of bytes skipped via Seek + if sr, ok := r.(io.Seeker); ok && n > 1 { + // Not all io.Seeker can actually Seek. For example, os.Stdin implements + // io.Seeker, but calling Seek always returns an error and performs + // no action. Thus, we try an innocent seek to the current position + // to see if Seek is really supported. + pos1, err := sr.Seek(0, io.SeekCurrent) + if pos1 >= 0 && err == nil { + // Seek seems supported, so perform the real Seek. + pos2, err := sr.Seek(n-1, io.SeekCurrent) + if pos2 < 0 || err != nil { + return err + } + seekSkipped = pos2 - pos1 + } + } + + copySkipped, err := io.CopyN(ioutil.Discard, r, n-seekSkipped) + if err == io.EOF && seekSkipped+copySkipped < n { + err = io.ErrUnexpectedEOF + } + return err +} diff --git a/src/archive/tar/reader_test.go b/src/archive/tar/reader_test.go index 3b9903eb7a..256f0eaca1 100644 --- a/src/archive/tar/reader_test.go +++ b/src/archive/tar/reader_test.go @@ -923,17 +923,17 @@ func TestReadTruncation(t *testing.T) { } cnt++ if s2 == "manual" { - if _, err = io.Copy(ioutil.Discard, tr); err != nil { + if _, err = tr.WriteTo(ioutil.Discard); err != nil { break } } } if err != v.err { - t.Errorf("test %d, NewReader(%s(...)) with %s discard: got %v, want %v", + t.Errorf("test %d, NewReader(%s) with %s discard: got %v, want %v", i, s1, s2, err, v.err) } if cnt != v.cnt { - t.Errorf("test %d, NewReader(%s(...)) with %s discard: got %d headers, want %d headers", + t.Errorf("test %d, NewReader(%s) with %s discard: got %d headers, want %d headers", i, s1, s2, cnt, v.cnt) } } @@ -1402,15 +1402,16 @@ func TestFileReader(t *testing.T) { wantStr string wantErr error } - testDiscard struct { // Discard(cnt) == (wantCnt, wantErr) - cnt int64 + testWriteTo struct { // WriteTo(testFile{ops}) == (wantCnt, wantErr) + ops fileOps wantCnt int64 wantErr error } - testRemaining struct { // Remaining() == wantCnt - wantCnt int64 + testRemaining struct { // LogicalRemaining() == wantLCnt, PhysicalRemaining() == wantPCnt + wantLCnt int64 + wantPCnt int64 } - testFnc interface{} // testRead | testDiscard | testRemaining + testFnc interface{} // testRead | testWriteTo | testRemaining ) type ( @@ -1432,102 +1433,112 @@ func TestFileReader(t *testing.T) { }{{ maker: makeReg{"", 0}, tests: []testFnc{ - testRemaining{0}, + testRemaining{0, 0}, testRead{0, "", io.EOF}, testRead{1, "", io.EOF}, - testDiscard{0, 0, nil}, - testDiscard{1, 0, io.EOF}, - testRemaining{0}, + testWriteTo{nil, 0, nil}, + testRemaining{0, 0}, }, }, { maker: makeReg{"", 1}, tests: []testFnc{ - testRemaining{1}, + testRemaining{1, 1}, testRead{0, "", io.ErrUnexpectedEOF}, testRead{5, "", io.ErrUnexpectedEOF}, - testDiscard{0, 0, nil}, - testDiscard{1, 0, io.ErrUnexpectedEOF}, - testRemaining{1}, + testWriteTo{nil, 0, io.ErrUnexpectedEOF}, + testRemaining{1, 1}, }, }, { maker: makeReg{"hello", 5}, tests: []testFnc{ - testRemaining{5}, + testRemaining{5, 5}, testRead{5, "hello", io.EOF}, - testRemaining{0}, + testRemaining{0, 0}, }, }, { maker: makeReg{"hello, world", 50}, tests: []testFnc{ - testRemaining{50}, - testDiscard{7, 7, nil}, - testRemaining{43}, + testRemaining{50, 50}, + testRead{7, "hello, ", nil}, + testRemaining{43, 43}, testRead{5, "world", nil}, - testRemaining{38}, - testDiscard{1, 0, io.ErrUnexpectedEOF}, + testRemaining{38, 38}, + testWriteTo{nil, 0, io.ErrUnexpectedEOF}, testRead{1, "", io.ErrUnexpectedEOF}, - testRemaining{38}, + testRemaining{38, 38}, }, }, { maker: makeReg{"hello, world", 5}, tests: []testFnc{ - testRemaining{5}, + testRemaining{5, 5}, testRead{0, "", nil}, testRead{4, "hell", nil}, - testRemaining{1}, - testDiscard{5, 1, io.EOF}, - testRemaining{0}, - testDiscard{5, 0, io.EOF}, + testRemaining{1, 1}, + testWriteTo{fileOps{"o"}, 1, nil}, + testRemaining{0, 0}, + testWriteTo{nil, 0, nil}, testRead{0, "", io.EOF}, }, }, { maker: makeSparse{makeReg{"abcde", 5}, sparseDatas{{0, 2}, {5, 3}}, 8}, tests: []testFnc{ - testRemaining{8}, + testRemaining{8, 5}, testRead{3, "ab\x00", nil}, testRead{10, "\x00\x00cde", io.EOF}, - testRemaining{0}, + testRemaining{0, 0}, }, }, { maker: makeSparse{makeReg{"abcde", 5}, sparseDatas{{0, 2}, {5, 3}}, 8}, tests: []testFnc{ - testRemaining{8}, - testDiscard{100, 8, io.EOF}, - testRemaining{0}, + testRemaining{8, 5}, + testWriteTo{fileOps{"ab", int64(3), "cde"}, 8, nil}, + testRemaining{0, 0}, }, }, { maker: makeSparse{makeReg{"abcde", 5}, sparseDatas{{0, 2}, {5, 3}}, 10}, tests: []testFnc{ - testRemaining{10}, + testRemaining{10, 5}, testRead{100, "ab\x00\x00\x00cde\x00\x00", io.EOF}, - testRemaining{0}, + testRemaining{0, 0}, }, }, { maker: makeSparse{makeReg{"abc", 5}, sparseDatas{{0, 2}, {5, 3}}, 10}, tests: []testFnc{ - testRemaining{10}, + testRemaining{10, 5}, testRead{100, "ab\x00\x00\x00c", io.ErrUnexpectedEOF}, - testRemaining{4}, + testRemaining{4, 2}, }, }, { maker: makeSparse{makeReg{"abcde", 5}, sparseDatas{{1, 3}, {6, 2}}, 8}, tests: []testFnc{ - testRemaining{8}, + testRemaining{8, 5}, testRead{8, "\x00abc\x00\x00de", io.EOF}, - testRemaining{0}, + testRemaining{0, 0}, }, }, { maker: makeSparse{makeReg{"abcde", 5}, sparseDatas{{1, 3}, {6, 0}, {6, 0}, {6, 2}}, 8}, tests: []testFnc{ - testRemaining{8}, + testRemaining{8, 5}, testRead{8, "\x00abc\x00\x00de", io.EOF}, - testRemaining{0}, + testRemaining{0, 0}, + }, + }, { + maker: makeSparse{makeReg{"abcde", 5}, sparseDatas{{1, 3}, {6, 0}, {6, 0}, {6, 2}}, 8}, + tests: []testFnc{ + testRemaining{8, 5}, + testWriteTo{fileOps{int64(1), "abc", int64(2), "de"}, 8, nil}, + testRemaining{0, 0}, }, }, { maker: makeSparse{makeReg{"abcde", 5}, sparseDatas{{1, 3}, {6, 2}}, 10}, tests: []testFnc{ testRead{100, "\x00abc\x00\x00de\x00\x00", io.EOF}, }, + }, { + maker: makeSparse{makeReg{"abcde", 5}, sparseDatas{{1, 3}, {6, 2}}, 10}, + tests: []testFnc{ + testWriteTo{fileOps{int64(1), "abc", int64(2), "de", int64(1), "\x00"}, 10, nil}, + }, }, { maker: makeSparse{makeReg{"abcde", 5}, sparseDatas{{1, 3}, {6, 2}, {8, 0}, {8, 0}, {8, 0}, {8, 0}}, 10}, tests: []testFnc{ @@ -1568,6 +1579,11 @@ func TestFileReader(t *testing.T) { tests: []testFnc{ testRead{100, "\x00abc\x00\x00de", errMissData}, }, + }, { + maker: makeSparse{makeReg{"abcde", 5}, sparseDatas{{1, 3}, {6, 5}}, 15}, + tests: []testFnc{ + testWriteTo{fileOps{int64(1), "abc", int64(2), "de"}, 8, errMissData}, + }, }, { maker: makeSparse{makeReg{"abcde", 8}, sparseDatas{{1, 3}, {6, 5}}, 15}, tests: []testFnc{ @@ -1576,18 +1592,18 @@ func TestFileReader(t *testing.T) { }, { maker: makeSparse{makeReg{"abcdefghEXTRA", 13}, sparseDatas{{1, 3}, {6, 5}}, 15}, tests: []testFnc{ - testRemaining{15}, + testRemaining{15, 13}, testRead{100, "\x00abc\x00\x00defgh\x00\x00\x00\x00", errUnrefData}, - testDiscard{100, 0, errUnrefData}, - testRemaining{0}, + testWriteTo{nil, 0, errUnrefData}, + testRemaining{0, 5}, }, }, { maker: makeSparse{makeReg{"abcdefghEXTRA", 13}, sparseDatas{{1, 3}, {6, 5}}, 15}, tests: []testFnc{ - testRemaining{15}, - testDiscard{100, 15, errUnrefData}, + testRemaining{15, 13}, + testWriteTo{fileOps{int64(1), "abc", int64(2), "defgh", int64(4)}, 15, errUnrefData}, testRead{100, "", errUnrefData}, - testRemaining{0}, + testRemaining{0, 5}, }, }} @@ -1617,15 +1633,23 @@ func TestFileReader(t *testing.T) { if got := string(b[:n]); got != tf.wantStr || err != tf.wantErr { t.Errorf("test %d.%d, Read(%d):\ngot (%q, %v)\nwant (%q, %v)", i, j, tf.cnt, got, err, tf.wantStr, tf.wantErr) } - case testDiscard: - got, err := fr.Discard(tf.cnt) - if got != tf.wantCnt || err != tf.wantErr { - t.Errorf("test %d.%d, Discard(%d) = (%d, %v), want (%d, %v)", i, j, tf.cnt, got, err, tf.wantCnt, tf.wantErr) + case testWriteTo: + f := &testFile{ops: tf.ops} + got, err := fr.WriteTo(f) + if _, ok := err.(testError); ok { + t.Errorf("test %d.%d, WriteTo(): %v", i, j, err) + } else if got != tf.wantCnt || err != tf.wantErr { + t.Errorf("test %d.%d, WriteTo() = (%d, %v), want (%d, %v)", i, j, got, err, tf.wantCnt, tf.wantErr) + } + if len(f.ops) > 0 { + t.Errorf("test %d.%d, expected %d more operations", i, j, len(f.ops)) } case testRemaining: - got := fr.Remaining() - if got != tf.wantCnt { - t.Errorf("test %d.%d, Remaining() = %d, want %d", i, j, got, tf.wantCnt) + if got := fr.LogicalRemaining(); got != tf.wantLCnt { + t.Errorf("test %d.%d, LogicalRemaining() = %d, want %d", i, j, got, tf.wantLCnt) + } + if got := fr.PhysicalRemaining(); got != tf.wantPCnt { + t.Errorf("test %d.%d, PhysicalRemaining() = %d, want %d", i, j, got, tf.wantPCnt) } default: t.Fatalf("test %d.%d, unknown test operation: %T", i, j, tf) diff --git a/src/archive/tar/tar_test.go b/src/archive/tar/tar_test.go index 04b0a48027..f2e1d75c78 100644 --- a/src/archive/tar/tar_test.go +++ b/src/archive/tar/tar_test.go @@ -6,6 +6,8 @@ package tar import ( "bytes" + "errors" + "fmt" "internal/testenv" "io" "io/ioutil" @@ -19,6 +21,83 @@ import ( "time" ) +type testError struct{ error } + +type fileOps []interface{} // []T where T is (string | int64) + +// testFile is an io.ReadWriteSeeker where the IO operations performed +// on it must match the list of operations in ops. +type testFile struct { + ops fileOps + pos int64 +} + +func (f *testFile) Read(b []byte) (int, error) { + if len(b) == 0 { + return 0, nil + } + if len(f.ops) == 0 { + return 0, io.EOF + } + s, ok := f.ops[0].(string) + if !ok { + return 0, errors.New("unexpected Read operation") + } + + n := copy(b, s) + if len(s) > n { + f.ops[0] = s[n:] + } else { + f.ops = f.ops[1:] + } + f.pos += int64(len(b)) + return n, nil +} + +func (f *testFile) Write(b []byte) (int, error) { + if len(b) == 0 { + return 0, nil + } + if len(f.ops) == 0 { + return 0, errors.New("unexpected Write operation") + } + s, ok := f.ops[0].(string) + if !ok { + return 0, errors.New("unexpected Write operation") + } + + if !strings.HasPrefix(s, string(b)) { + return 0, testError{fmt.Errorf("got Write(%q), want Write(%q)", b, s)} + } + if len(s) > len(b) { + f.ops[0] = s[len(b):] + } else { + f.ops = f.ops[1:] + } + f.pos += int64(len(b)) + return len(b), nil +} + +func (f *testFile) Seek(pos int64, whence int) (int64, error) { + if pos == 0 && whence == io.SeekCurrent { + return f.pos, nil + } + if len(f.ops) == 0 { + return 0, errors.New("unexpected Seek operation") + } + s, ok := f.ops[0].(int64) + if !ok { + return 0, errors.New("unexpected Seek operation") + } + + if s != pos || whence != io.SeekCurrent { + return 0, testError{fmt.Errorf("got Seek(%d, %d), want Seek(%d, %d)", pos, whence, s, io.SeekCurrent)} + } + f.pos += s + f.ops = f.ops[1:] + return f.pos, nil +} + func equalSparseEntries(x, y []SparseEntry) bool { return (len(x) == 0 && len(y) == 0) || reflect.DeepEqual(x, y) } @@ -687,6 +766,105 @@ func TestHeaderAllowedFormats(t *testing.T) { } } +func TestSparseFiles(t *testing.T) { + vectors := []struct { + label string + sparseMap sparseHoles + }{ + {"EmptyFile", sparseHoles{{0, 0}}}, + {"BigData", sparseHoles{{1e6, 0}}}, + {"BigHole", sparseHoles{{0, 1e6}}}, + {"DataFront", sparseHoles{{1e3, 1e6 - 1e3}}}, + {"HoleFront", sparseHoles{{0, 1e6 - 1e3}, {1e6, 0}}}, + {"DataMiddle", sparseHoles{{0, 5e5 - 1e3}, {5e5, 5e5}}}, + {"HoleMiddle", sparseHoles{{1e3, 1e6 - 2e3}, {1e6, 0}}}, + {"Multiple", func() (sph []SparseEntry) { + for i := 0; i < 20; i++ { + sph = append(sph, SparseEntry{1e6 * int64(i), 1e6 - 1e3}) + } + sph = append(sph, SparseEntry{20e6, 0}) + return + }()}, + } + + for _, v := range vectors { + sph := v.sparseMap + t.Run(v.label, func(t *testing.T) { + src, err := ioutil.TempFile("", "") + if err != nil { + t.Fatalf("unexpected TempFile error: %v", err) + } + defer os.Remove(src.Name()) + dst, err := ioutil.TempFile("", "") + if err != nil { + t.Fatalf("unexpected TempFile error: %v", err) + } + defer os.Remove(dst.Name()) + + // Create the source sparse file. + hdr := Header{ + Typeflag: TypeReg, + Name: "sparse.db", + Size: sph[len(sph)-1].endOffset(), + SparseHoles: sph, + } + // TODO: Explicitly punch holes in the sparse file. + if err := src.Truncate(hdr.Size); err != nil { + t.Fatalf("unexpected Truncate error: %v", err) + } + var pos int64 + for _, s := range sph { + b := bytes.Repeat([]byte{'Y'}, int(s.Offset-pos)) + if _, err := src.WriteAt(b, pos); err != nil { + t.Fatalf("unexpected WriteAt error: %v", err) + } + pos = s.endOffset() + } + + // Round-trip the sparse file to/from a tar archive. + b := new(bytes.Buffer) + tw := NewWriter(b) + if err := tw.WriteHeader(&hdr); err != nil { + t.Fatalf("unexpected WriteHeader error: %v", err) + } + if _, err := tw.ReadFrom(src); err != nil { + t.Fatalf("unexpected ReadFrom error: %v", err) + } + if err := tw.Close(); err != nil { + t.Fatalf("unexpected Close error: %v", err) + } + tr := NewReader(b) + if _, err := tr.Next(); err != nil { + t.Fatalf("unexpected Next error: %v", err) + } + // TODO: Explicitly punch holes in the sparse file. + if err := dst.Truncate(hdr.Size); err != nil { + t.Fatalf("unexpected Truncate error: %v", err) + } + if _, err := tr.WriteTo(dst); err != nil { + t.Fatalf("unexpected Copy error: %v", err) + } + + // Verify the sparse file matches. + // Even if the OS and underlying FS do not support sparse files, + // the content should still match (i.e., holes read as zeros). + got, err := ioutil.ReadFile(dst.Name()) + if err != nil { + t.Fatalf("unexpected ReadFile error: %v", err) + } + want, err := ioutil.ReadFile(src.Name()) + if err != nil { + t.Fatalf("unexpected ReadFile error: %v", err) + } + if !bytes.Equal(got, want) { + t.Fatal("sparse files mismatch") + } + + // TODO: Actually check that the file is sparse. + }) + } +} + func Benchmark(b *testing.B) { type file struct { hdr *Header diff --git a/src/archive/tar/writer.go b/src/archive/tar/writer.go index 26491ddc07..0772d8b206 100644 --- a/src/archive/tar/writer.go +++ b/src/archive/tar/writer.go @@ -40,7 +40,7 @@ type fileWriter interface { io.Writer fileState - FillZeros(n int64) (int64, error) + ReadFrom(io.Reader) (int64, error) } // Flush finishes writing the current file's block padding. @@ -52,7 +52,7 @@ func (tw *Writer) Flush() error { if tw.err != nil { return tw.err } - if nb := tw.curr.Remaining(); nb > 0 { + if nb := tw.curr.LogicalRemaining(); nb > 0 { return fmt.Errorf("tar: missed writing %d bytes", nb) } if _, tw.err = tw.w.Write(zeroBlock[:tw.pad]); tw.err != nil { @@ -406,19 +406,18 @@ func (tw *Writer) Write(b []byte) (int, error) { return n, err } -// TODO(dsnet): Export the Writer.FillZeros method to assist in quickly zeroing -// out sections of a file. This is especially useful for efficiently -// skipping over large holes in a sparse file. - -// fillZeros writes n bytes of zeros to the current file, -// returning the number of bytes written. -// If fewer than n bytes are discarded, it returns an non-nil error, -// which may be ErrWriteTooLong if the current file is complete. -func (tw *Writer) fillZeros(n int64) (int64, error) { +// ReadFrom populates the content of the current file by reading from r. +// The bytes read must match the number of remaining bytes in the current file. +// +// If the current file is sparse and r is an io.ReadSeeker, +// then ReadFrom uses Seek to skip past holes defined in Header.SparseHoles, +// assuming that skipped regions are all NULs. +// This always reads the last byte to ensure r is the right size. +func (tw *Writer) ReadFrom(r io.Reader) (int64, error) { if tw.err != nil { return 0, tw.err } - n, err := tw.curr.FillZeros(n) + n, err := tw.curr.ReadFrom(r) if err != nil && err != ErrWriteTooLong { tw.err = err } @@ -470,11 +469,14 @@ func (fw *regFileWriter) Write(b []byte) (int, error) { } } -func (fw *regFileWriter) FillZeros(n int64) (int64, error) { - return io.CopyN(fw, zeroReader{}, n) +func (fw *regFileWriter) ReadFrom(r io.Reader) (int64, error) { + return io.Copy(struct{ io.Writer }{fw}, r) } -func (fw regFileWriter) Remaining() int64 { +func (fw regFileWriter) LogicalRemaining() int64 { + return fw.nb +} +func (fw regFileWriter) PhysicalRemaining() int64 { return fw.nb } @@ -486,9 +488,9 @@ type sparseFileWriter struct { } func (sw *sparseFileWriter) Write(b []byte) (n int, err error) { - overwrite := int64(len(b)) > sw.Remaining() + overwrite := int64(len(b)) > sw.LogicalRemaining() if overwrite { - b = b[:sw.Remaining()] + b = b[:sw.LogicalRemaining()] } b0 := b @@ -516,7 +518,7 @@ func (sw *sparseFileWriter) Write(b []byte) (n int, err error) { return n, errMissData // Not possible; implies bug in validation logic case err != nil: return n, err - case sw.Remaining() == 0 && sw.fw.Remaining() > 0: + case sw.LogicalRemaining() == 0 && sw.PhysicalRemaining() > 0: return n, errUnrefData // Not possible; implies bug in validation logic case overwrite: return n, ErrWriteTooLong @@ -525,22 +527,32 @@ func (sw *sparseFileWriter) Write(b []byte) (n int, err error) { } } -func (sw *sparseFileWriter) FillZeros(n int64) (int64, error) { - overwrite := n > sw.Remaining() - if overwrite { - n = sw.Remaining() +func (sw *sparseFileWriter) ReadFrom(r io.Reader) (n int64, err error) { + rs, ok := r.(io.ReadSeeker) + if ok { + if _, err := rs.Seek(0, io.SeekCurrent); err != nil { + ok = false // Not all io.Seeker can really seek + } + } + if !ok { + return io.Copy(struct{ io.Writer }{sw}, r) } - var realFill int64 // Number of real data bytes to fill - endPos := sw.pos + n - for endPos > sw.pos { + var readLastByte bool + pos0 := sw.pos + for sw.LogicalRemaining() > 0 && !readLastByte && err == nil { var nf int64 // Size of fragment dataStart, dataEnd := sw.sp[0].Offset, sw.sp[0].endOffset() if sw.pos < dataStart { // In a hole fragment - nf = min(endPos-sw.pos, dataStart-sw.pos) + nf = dataStart - sw.pos + if sw.PhysicalRemaining() == 0 { + readLastByte = true + nf-- + } + _, err = rs.Seek(nf, io.SeekCurrent) } else { // In a data fragment - nf = min(endPos-sw.pos, dataEnd-sw.pos) - realFill += nf + nf = dataEnd - sw.pos + nf, err = io.CopyN(sw.fw, rs, nf) } sw.pos += nf if sw.pos >= dataEnd && len(sw.sp) > 1 { @@ -548,24 +560,34 @@ func (sw *sparseFileWriter) FillZeros(n int64) (int64, error) { } } - _, err := sw.fw.FillZeros(realFill) + // If the last fragment is a hole, then seek to 1-byte before EOF, and + // read a single byte to ensure the file is the right size. + if readLastByte && err == nil { + _, err = mustReadFull(rs, []byte{0}) + sw.pos++ + } + + n = sw.pos - pos0 switch { + case err == io.EOF: + return n, io.ErrUnexpectedEOF case err == ErrWriteTooLong: return n, errMissData // Not possible; implies bug in validation logic case err != nil: return n, err - case sw.Remaining() == 0 && sw.fw.Remaining() > 0: + case sw.LogicalRemaining() == 0 && sw.PhysicalRemaining() > 0: return n, errUnrefData // Not possible; implies bug in validation logic - case overwrite: - return n, ErrWriteTooLong default: - return n, nil + return n, ensureEOF(rs) } } -func (sw sparseFileWriter) Remaining() int64 { +func (sw sparseFileWriter) LogicalRemaining() int64 { return sw.sp[len(sw.sp)-1].endOffset() - sw.pos } +func (sw sparseFileWriter) PhysicalRemaining() int64 { + return sw.fw.PhysicalRemaining() +} // zeroWriter may only be written with NULs, otherwise it returns errWriteHole. type zeroWriter struct{} @@ -578,3 +600,16 @@ func (zeroWriter) Write(b []byte) (int, error) { } return len(b), nil } + +// ensureEOF checks whether r is at EOF, reporting ErrWriteTooLong if not so. +func ensureEOF(r io.Reader) error { + n, err := tryReadFull(r, []byte{0}) + switch { + case n > 0: + return ErrWriteTooLong + case err == io.EOF: + return nil + default: + return err + } +} diff --git a/src/archive/tar/writer_test.go b/src/archive/tar/writer_test.go index 42b0a201ca..122ec7d3d9 100644 --- a/src/archive/tar/writer_test.go +++ b/src/archive/tar/writer_test.go @@ -50,24 +50,24 @@ func bytediff(a, b []byte) string { func TestWriter(t *testing.T) { type ( - testHeader struct { // WriteHeader(&hdr) == wantErr + testHeader struct { // WriteHeader(hdr) == wantErr hdr Header wantErr error } - testWrite struct { // Write([]byte(str)) == (wantCnt, wantErr) + testWrite struct { // Write(str) == (wantCnt, wantErr) str string wantCnt int wantErr error } - testFill struct { // fillZeros(cnt) == (wantCnt, wantErr) - cnt int64 + testReadFrom struct { // ReadFrom(testFile{ops}) == (wantCnt, wantErr) + ops fileOps wantCnt int64 wantErr error } testClose struct { // Close() == wantErr wantErr error } - testFnc interface{} // testHeader | testWrite | testFill | testClose + testFnc interface{} // testHeader | testWrite | testReadFrom | testClose ) vectors := []struct { @@ -402,20 +402,20 @@ func TestWriter(t *testing.T) { {Offset: 5e10, Length: 1e10 - 100}, }, }, nil}, - testFill{1e10 - 100, 1e10 - 100, nil}, - testWrite{strings.Repeat("0123456789", 10), 100, nil}, - testFill{1e10 - 100, 1e10 - 100, nil}, - testWrite{strings.Repeat("0123456789", 10), 100, nil}, - testFill{1e10 - 100, 1e10 - 100, nil}, - testWrite{strings.Repeat("0123456789", 10), 100, nil}, - testFill{1e10 - 100, 1e10 - 100, nil}, - testWrite{strings.Repeat("0123456789", 10), 100, nil}, - testFill{1e10 - 100, 1e10 - 100, nil}, - testWrite{strings.Repeat("0123456789", 10), 100, nil}, - testFill{1e10 - 100, 1e10 - 100, nil}, - testWrite{strings.Repeat("0123456789", 10), 100, nil}, - testFill{1e10 - 100, 0, ErrWriteTooLong}, - testWrite{strings.Repeat("0123456789", 10), 0, ErrWriteTooLong}, + testReadFrom{fileOps{ + int64(1e10 - blockSize), + strings.Repeat("\x00", blockSize-100) + strings.Repeat("0123456789", 10), + int64(1e10 - blockSize), + strings.Repeat("\x00", blockSize-100) + strings.Repeat("0123456789", 10), + int64(1e10 - blockSize), + strings.Repeat("\x00", blockSize-100) + strings.Repeat("0123456789", 10), + int64(1e10 - blockSize), + strings.Repeat("\x00", blockSize-100) + strings.Repeat("0123456789", 10), + int64(1e10 - blockSize), + strings.Repeat("\x00", blockSize-100) + strings.Repeat("0123456789", 10), + int64(1e10 - blockSize), + strings.Repeat("\x00", blockSize-100) + strings.Repeat("0123456789", 10), + }, 6e10, nil}, testClose{nil}, }, }, { @@ -434,20 +434,20 @@ func TestWriter(t *testing.T) { {Offset: 5e10, Length: 1e10 - 100}, }, }, nil}, - testFill{1e10 - 100, 1e10 - 100, nil}, - testWrite{strings.Repeat("0123456789", 10), 100, nil}, - testFill{1e10 - 100, 1e10 - 100, nil}, - testWrite{strings.Repeat("0123456789", 10), 100, nil}, - testFill{1e10 - 100, 1e10 - 100, nil}, - testWrite{strings.Repeat("0123456789", 10), 100, nil}, - testFill{1e10 - 100, 1e10 - 100, nil}, - testWrite{strings.Repeat("0123456789", 10), 100, nil}, - testFill{1e10 - 100, 1e10 - 100, nil}, - testWrite{strings.Repeat("0123456789", 10), 100, nil}, - testFill{1e10 - 100, 1e10 - 100, nil}, - testWrite{strings.Repeat("0123456789", 10), 100, nil}, - testFill{1e10 - 100, 0, ErrWriteTooLong}, - testWrite{strings.Repeat("0123456789", 10), 0, ErrWriteTooLong}, + testReadFrom{fileOps{ + int64(1e10 - blockSize), + strings.Repeat("\x00", blockSize-100) + strings.Repeat("0123456789", 10), + int64(1e10 - blockSize), + strings.Repeat("\x00", blockSize-100) + strings.Repeat("0123456789", 10), + int64(1e10 - blockSize), + strings.Repeat("\x00", blockSize-100) + strings.Repeat("0123456789", 10), + int64(1e10 - blockSize), + strings.Repeat("\x00", blockSize-100) + strings.Repeat("0123456789", 10), + int64(1e10 - blockSize), + strings.Repeat("\x00", blockSize-100) + strings.Repeat("0123456789", 10), + int64(1e10 - blockSize), + strings.Repeat("\x00", blockSize-100) + strings.Repeat("0123456789", 10), + }, 6e10, nil}, testClose{nil}, }, }} @@ -478,10 +478,16 @@ func TestWriter(t *testing.T) { if got != tf.wantCnt || !equalError(err, tf.wantErr) { t.Fatalf("test %d, Write() = (%d, %v), want (%d, %v)", i, got, err, tf.wantCnt, tf.wantErr) } - case testFill: - got, err := tw.fillZeros(tf.cnt) - if got != tf.wantCnt || !equalError(err, tf.wantErr) { - t.Fatalf("test %d, fillZeros() = (%d, %v), want (%d, %v)", i, got, err, tf.wantCnt, tf.wantErr) + case testReadFrom: + f := &testFile{ops: tf.ops} + got, err := tw.ReadFrom(f) + if _, ok := err.(testError); ok { + t.Errorf("test %d, ReadFrom(): %v", i, err) + } else if got != tf.wantCnt || !equalError(err, tf.wantErr) { + t.Errorf("test %d, ReadFrom() = (%d, %v), want (%d, %v)", i, got, err, tf.wantCnt, tf.wantErr) + } + if len(f.ops) > 0 { + t.Errorf("test %d, expected %d more operations", i, len(f.ops)) } case testClose: err := tw.Close() @@ -988,15 +994,16 @@ func TestFileWriter(t *testing.T) { wantCnt int wantErr error } - testFill struct { // FillZeros(cnt) == (wantCnt, wantErr) - cnt int64 + testReadFrom struct { // ReadFrom(testFile{ops}) == (wantCnt, wantErr) + ops fileOps wantCnt int64 wantErr error } - testRemaining struct { // Remaining() == wantCnt - wantCnt int64 + testRemaining struct { // LogicalRemaining() == wantLCnt, PhysicalRemaining() == wantPCnt + wantLCnt int64 + wantPCnt int64 } - testFnc interface{} // testWrite | testFill | testRemaining + testFnc interface{} // testWrite | testReadFrom | testRemaining ) type ( @@ -1018,149 +1025,160 @@ func TestFileWriter(t *testing.T) { }{{ maker: makeReg{0, ""}, tests: []testFnc{ - testRemaining{0}, + testRemaining{0, 0}, testWrite{"", 0, nil}, testWrite{"a", 0, ErrWriteTooLong}, - testFill{0, 0, nil}, - testFill{1, 0, ErrWriteTooLong}, - testRemaining{0}, + testReadFrom{fileOps{""}, 0, nil}, + testReadFrom{fileOps{"a"}, 0, ErrWriteTooLong}, + testRemaining{0, 0}, }, }, { maker: makeReg{1, "a"}, tests: []testFnc{ - testRemaining{1}, + testRemaining{1, 1}, testWrite{"", 0, nil}, testWrite{"a", 1, nil}, testWrite{"bcde", 0, ErrWriteTooLong}, testWrite{"", 0, nil}, - testFill{0, 0, nil}, - testFill{1, 0, ErrWriteTooLong}, - testRemaining{0}, + testReadFrom{fileOps{""}, 0, nil}, + testReadFrom{fileOps{"a"}, 0, ErrWriteTooLong}, + testRemaining{0, 0}, }, }, { maker: makeReg{5, "hello"}, tests: []testFnc{ - testRemaining{5}, + testRemaining{5, 5}, testWrite{"hello", 5, nil}, - testRemaining{0}, + testRemaining{0, 0}, }, }, { maker: makeReg{5, "\x00\x00\x00\x00\x00"}, tests: []testFnc{ - testRemaining{5}, - testFill{5, 5, nil}, - testRemaining{0}, + testRemaining{5, 5}, + testReadFrom{fileOps{"\x00\x00\x00\x00\x00"}, 5, nil}, + testRemaining{0, 0}, }, }, { maker: makeReg{5, "\x00\x00\x00\x00\x00"}, tests: []testFnc{ - testRemaining{5}, - testFill{10, 5, ErrWriteTooLong}, - testRemaining{0}, + testRemaining{5, 5}, + testReadFrom{fileOps{"\x00\x00\x00\x00\x00extra"}, 5, ErrWriteTooLong}, + testRemaining{0, 0}, }, }, { maker: makeReg{5, "abc\x00\x00"}, tests: []testFnc{ - testRemaining{5}, + testRemaining{5, 5}, testWrite{"abc", 3, nil}, - testRemaining{2}, - testFill{2, 2, nil}, - testRemaining{0}, + testRemaining{2, 2}, + testReadFrom{fileOps{"\x00\x00"}, 2, nil}, + testRemaining{0, 0}, }, }, { maker: makeReg{5, "\x00\x00abc"}, tests: []testFnc{ - testRemaining{5}, - testFill{2, 2, nil}, - testRemaining{3}, + testRemaining{5, 5}, + testWrite{"\x00\x00", 2, nil}, + testRemaining{3, 3}, testWrite{"abc", 3, nil}, - testFill{1, 0, ErrWriteTooLong}, + testReadFrom{fileOps{"z"}, 0, ErrWriteTooLong}, testWrite{"z", 0, ErrWriteTooLong}, - testRemaining{0}, + testRemaining{0, 0}, }, }, { maker: makeSparse{makeReg{5, "abcde"}, sparseHoles{{2, 3}}, 8}, tests: []testFnc{ - testRemaining{8}, + testRemaining{8, 5}, testWrite{"ab\x00\x00\x00cde", 8, nil}, testWrite{"a", 0, ErrWriteTooLong}, - testRemaining{0}, + testRemaining{0, 0}, }, }, { maker: makeSparse{makeReg{5, "abcde"}, sparseHoles{{2, 3}}, 8}, tests: []testFnc{ testWrite{"ab\x00\x00\x00cdez", 8, ErrWriteTooLong}, - testRemaining{0}, + testRemaining{0, 0}, }, }, { maker: makeSparse{makeReg{5, "abcde"}, sparseHoles{{2, 3}}, 8}, tests: []testFnc{ testWrite{"ab\x00", 3, nil}, - testRemaining{5}, + testRemaining{5, 3}, testWrite{"\x00\x00cde", 5, nil}, testWrite{"a", 0, ErrWriteTooLong}, - testRemaining{0}, + testRemaining{0, 0}, }, }, { maker: makeSparse{makeReg{5, "abcde"}, sparseHoles{{2, 3}}, 8}, tests: []testFnc{ testWrite{"ab", 2, nil}, - testRemaining{6}, - testFill{3, 3, nil}, - testRemaining{3}, - testWrite{"cde", 3, nil}, - testRemaining{0}, + testRemaining{6, 3}, + testReadFrom{fileOps{int64(3), "cde"}, 6, nil}, + testRemaining{0, 0}, }, }, { - maker: makeSparse{makeReg{5, "\x00\x00\x00\x00\x00"}, sparseHoles{{2, 3}}, 8}, + maker: makeSparse{makeReg{5, "abcde"}, sparseHoles{{2, 3}}, 8}, tests: []testFnc{ - testFill{8, 8, nil}, - testRemaining{0}, + testReadFrom{fileOps{"ab", int64(3), "cde"}, 8, nil}, + testRemaining{0, 0}, }, }, { - maker: makeSparse{makeReg{5, "\x00\x00\x00\x00\x00"}, sparseHoles{{2, 3}}, 8}, + maker: makeSparse{makeReg{5, "abcde"}, sparseHoles{{2, 3}}, 8}, tests: []testFnc{ - testFill{9, 8, ErrWriteTooLong}, - testRemaining{0}, + testReadFrom{fileOps{"ab", int64(3), "cdeX"}, 8, ErrWriteTooLong}, + testRemaining{0, 0}, }, }, { - maker: makeSparse{makeReg{4, "\x00\x00\x00\x00"}, sparseHoles{{2, 3}}, 8}, + maker: makeSparse{makeReg{4, "abcd"}, sparseHoles{{2, 3}}, 8}, tests: []testFnc{ - testFill{9, 8, errMissData}, - testRemaining{0}, + testReadFrom{fileOps{"ab", int64(3), "cd"}, 7, io.ErrUnexpectedEOF}, + testRemaining{1, 0}, }, }, { - maker: makeSparse{makeReg{6, "\x00\x00\x00\x00\x00"}, sparseHoles{{2, 3}}, 8}, + maker: makeSparse{makeReg{4, "abcd"}, sparseHoles{{2, 3}}, 8}, + tests: []testFnc{ + testReadFrom{fileOps{"ab", int64(3), "cde"}, 7, errMissData}, + testRemaining{1, 0}, + }, + }, { + maker: makeSparse{makeReg{6, "abcde"}, sparseHoles{{2, 3}}, 8}, tests: []testFnc{ - testFill{9, 8, errUnrefData}, - testRemaining{0}, + testReadFrom{fileOps{"ab", int64(3), "cde"}, 8, errUnrefData}, + testRemaining{0, 1}, }, }, { maker: makeSparse{makeReg{4, "abcd"}, sparseHoles{{2, 3}}, 8}, tests: []testFnc{ testWrite{"ab", 2, nil}, - testRemaining{6}, - testFill{3, 3, nil}, - testRemaining{3}, + testRemaining{6, 2}, + testWrite{"\x00\x00\x00", 3, nil}, + testRemaining{3, 2}, testWrite{"cde", 2, errMissData}, - testRemaining{1}, + testRemaining{1, 0}, }, }, { maker: makeSparse{makeReg{6, "abcde"}, sparseHoles{{2, 3}}, 8}, tests: []testFnc{ testWrite{"ab", 2, nil}, - testRemaining{6}, - testFill{3, 3, nil}, - testRemaining{3}, + testRemaining{6, 4}, + testWrite{"\x00\x00\x00", 3, nil}, + testRemaining{3, 4}, testWrite{"cde", 3, errUnrefData}, - testRemaining{0}, + testRemaining{0, 1}, }, }, { maker: makeSparse{makeReg{3, "abc"}, sparseHoles{{0, 2}, {5, 2}}, 7}, tests: []testFnc{ - testRemaining{7}, + testRemaining{7, 3}, testWrite{"\x00\x00abc\x00\x00", 7, nil}, - testRemaining{0}, + testRemaining{0, 0}, + }, + }, { + maker: makeSparse{makeReg{3, "abc"}, sparseHoles{{0, 2}, {5, 2}}, 7}, + tests: []testFnc{ + testRemaining{7, 3}, + testReadFrom{fileOps{int64(2), "abc", int64(1), "\x00"}, 7, nil}, + testRemaining{0, 0}, }, }, { maker: makeSparse{makeReg{3, ""}, sparseHoles{{0, 2}, {5, 2}}, 7}, @@ -1176,43 +1194,31 @@ func TestFileWriter(t *testing.T) { maker: makeSparse{makeReg{3, "abc"}, sparseHoles{{0, 2}, {5, 2}}, 7}, tests: []testFnc{ testWrite{"\x00\x00abc\x00\x00z", 7, ErrWriteTooLong}, - testRemaining{0}, - }, - }, { - maker: makeSparse{makeReg{3, "\x00\x00\x00"}, sparseHoles{{0, 2}, {5, 2}}, 7}, - tests: []testFnc{ - testFill{7, 7, nil}, - testFill{1, 0, ErrWriteTooLong}, - }, - }, { - maker: makeSparse{makeReg{3, "\x00\x00\x00"}, sparseHoles{{0, 2}, {5, 2}}, 7}, - tests: []testFnc{ - testFill{4, 4, nil}, - testFill{8, 3, ErrWriteTooLong}, + testRemaining{0, 0}, }, }, { maker: makeSparse{makeReg{3, "abc"}, sparseHoles{{0, 2}, {5, 2}}, 7}, tests: []testFnc{ - testFill{2, 2, nil}, - testRemaining{5}, + testWrite{"\x00\x00", 2, nil}, + testRemaining{5, 3}, testWrite{"abc", 3, nil}, - testRemaining{2}, - testFill{2, 2, nil}, - testRemaining{0}, + testRemaining{2, 0}, + testWrite{"\x00\x00", 2, nil}, + testRemaining{0, 0}, }, }, { maker: makeSparse{makeReg{2, "ab"}, sparseHoles{{0, 2}, {5, 2}}, 7}, tests: []testFnc{ - testFill{2, 2, nil}, + testWrite{"\x00\x00", 2, nil}, testWrite{"abc", 2, errMissData}, - testFill{2, 2, errMissData}, + testWrite{"\x00\x00", 0, errMissData}, }, }, { maker: makeSparse{makeReg{4, "abc"}, sparseHoles{{0, 2}, {5, 2}}, 7}, tests: []testFnc{ - testFill{2, 2, nil}, + testWrite{"\x00\x00", 2, nil}, testWrite{"abc", 3, nil}, - testFill{2, 2, errUnrefData}, + testWrite{"\x00\x00", 2, errUnrefData}, }, }} @@ -1243,15 +1249,23 @@ func TestFileWriter(t *testing.T) { if got != tf.wantCnt || err != tf.wantErr { t.Errorf("test %d.%d, Write(%s):\ngot (%d, %v)\nwant (%d, %v)", i, j, tf.str, got, err, tf.wantCnt, tf.wantErr) } - case testFill: - got, err := fw.FillZeros(tf.cnt) - if got != tf.wantCnt || err != tf.wantErr { - t.Errorf("test %d.%d, FillZeros(%d) = (%d, %v), want (%d, %v)", i, j, tf.cnt, got, err, tf.wantCnt, tf.wantErr) + case testReadFrom: + f := &testFile{ops: tf.ops} + got, err := fw.ReadFrom(f) + if _, ok := err.(testError); ok { + t.Errorf("test %d.%d, ReadFrom(): %v", i, j, err) + } else if got != tf.wantCnt || err != tf.wantErr { + t.Errorf("test %d.%d, ReadFrom() = (%d, %v), want (%d, %v)", i, j, got, err, tf.wantCnt, tf.wantErr) + } + if len(f.ops) > 0 { + t.Errorf("test %d.%d, expected %d more operations", i, j, len(f.ops)) } case testRemaining: - got := fw.Remaining() - if got != tf.wantCnt { - t.Errorf("test %d.%d, Remaining() = %d, want %d", i, j, got, tf.wantCnt) + if got := fw.LogicalRemaining(); got != tf.wantLCnt { + t.Errorf("test %d.%d, LogicalRemaining() = %d, want %d", i, j, got, tf.wantLCnt) + } + if got := fw.PhysicalRemaining(); got != tf.wantPCnt { + t.Errorf("test %d.%d, PhysicalRemaining() = %d, want %d", i, j, got, tf.wantPCnt) } default: t.Fatalf("test %d.%d, unknown test operation: %T", i, j, tf) -- 2.48.1