From 7ae9561610200beab11a1a6380a1d0f245262296 Mon Sep 17 00:00:00 2001 From: Joe Tsai Date: Fri, 11 Aug 2017 18:58:58 -0700 Subject: [PATCH] archive/tar: implement specialized logic for PAX format Rather than going through writeHeader, which attempts to handle all formats, implement writePAXHeader, which only has an understanding of the PAX format. In PAX, the USTAR header is filled out in a best-effort manner. Thus, we change logic of formatString and formatOctal to try their best to output something (possibly truncated) in the event of an error. The new implementation of PAX headers causes several tests to fail. An investigation into the new output reveals that the new behavior is correct, while the tests had actually locked in incorrect behavior before. A dump of the differences is listed below (-before, +after): << writer-big.tar >> This change is due to fact that we changed the Header.Devminor to force the tar.Writer to choose the GNU format over the PAX one. The ability to control the output is an open issue (see #18710). - 00000150 00 30 30 30 30 30 30 30 00 00 00 00 00 00 00 00 |.0000000........| + 00000150 00 ff ff ff ff ff ff ff ff 00 00 00 00 00 00 00 |................| << writer-big-long.tar>> The previous logic generated the GNU magic values for a PAX file. The new logic correctly uses the USTAR magic values. - 00000100 00 75 73 74 61 72 20 20 00 00 00 00 00 00 00 00 |.ustar ........| - 00000500 00 75 73 74 61 72 20 20 00 67 75 69 6c 6c 61 75 |.ustar .guillau| + 00000100 00 75 73 74 61 72 00 30 30 00 00 00 00 00 00 00 |.ustar.00.......| + 00000500 00 75 73 74 61 72 00 30 30 67 75 69 6c 6c 61 75 |.ustar.00guillau| The previous logic tried to use the specified timestmap in the PAX headers file, but this is problematic as this timestamp can overflow, defeating the point of using PAX, which is intended to extend tar. The new logic uses the zero timestamp similar to what GNU and BSD tar do. - 00000080 30 30 30 30 32 33 32 00 31 32 33 33 32 37 37 30 |0000232.12332770| + 00000080 30 30 30 30 32 35 36 00 30 30 30 30 30 30 30 30 |0000256.00000000| The previous logic populated the devminor and devmajor fields. The new logic leaves them zeroed just like what GNU and BSD tar do. - 00000140 00 00 00 00 00 00 00 00 00 30 30 30 30 30 30 30 |.........0000000| - 00000150 00 30 30 30 30 30 30 30 00 00 00 00 00 00 00 00 |.0000000........| + 00000140 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| + 00000150 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| The previous logic uses PAX headers, but fails to add a record for the size. The new logic does properly add a record for the size. - 00000290 31 36 67 69 67 2e 74 78 74 0a 00 00 00 00 00 00 |16gig.txt.......| - 000002a0 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| + 00000290 31 36 67 69 67 2e 74 78 74 0a 32 30 20 73 69 7a |16gig.txt.20 siz| + 000002a0 65 3d 31 37 31 37 39 38 36 39 31 38 34 0a 00 00 |e=17179869184...| The previous logic encoded the size as a base-256 field, which is only valid in GNU, but the previous PAX headers implies this should be a PAX file. This result in a strange hybrid that is neither GNU nor PAX. The new logic uses PAX headers to store the size. - 00000470 37 35 30 00 30 30 30 31 37 35 30 00 80 00 00 00 |750.0001750.....| - 00000480 00 00 00 04 00 00 00 00 31 32 33 33 32 37 37 30 |........12332770| + 00000470 37 35 30 00 30 30 30 31 37 35 30 00 30 30 30 30 |750.0001750.0000| + 00000480 30 30 30 30 30 30 30 00 31 32 33 33 32 37 37 30 |0000000.12332770| << ustar.issue12594.tar >> The previous logic used the specified timestamp for the PAX headers file. The new logic just uses the zero timestmap. - 00000080 30 30 30 30 32 33 31 00 31 32 31 30 34 34 30 32 |0000231.12104402| + 00000080 30 30 30 30 32 33 31 00 30 30 30 30 30 30 30 30 |0000231.00000000| The previous logic populated the devminor and devmajor fields. The new logic leaves them zeroed just like what GNU and BSD tar do. - 00000140 00 00 00 00 00 00 00 00 00 30 30 30 30 30 30 30 |.........0000000| - 00000150 00 30 30 30 30 30 30 30 00 00 00 00 00 00 00 00 |.0000000........| + 00000140 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| + 00000150 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| Change-Id: I33419eb1124951968e9d5a10d50027e03133c811 Reviewed-on: https://go-review.googlesource.com/55231 Reviewed-by: Ian Lance Taylor Run-TryBot: Ian Lance Taylor TryBot-Result: Gobot Gobot --- src/archive/tar/strconv.go | 16 ++-- src/archive/tar/testdata/ustar.issue12594.tar | Bin 3072 -> 3072 bytes src/archive/tar/testdata/writer-big-long.tar | Bin 4096 -> 4096 bytes src/archive/tar/testdata/writer-big.tar | Bin 4096 -> 4096 bytes src/archive/tar/writer.go | 86 ++++++++++++++++-- src/archive/tar/writer_test.go | 1 + 6 files changed, 90 insertions(+), 13 deletions(-) diff --git a/src/archive/tar/strconv.go b/src/archive/tar/strconv.go index 16d060c231..6aad6805aa 100644 --- a/src/archive/tar/strconv.go +++ b/src/archive/tar/strconv.go @@ -53,16 +53,15 @@ func (*parser) parseString(b []byte) string { return string(b) } -// Write s into b, terminating it with a NUL if there is room. +// formatString copies s into b, NUL-terminating if possible. func (f *formatter) formatString(b []byte, s string) { if len(s) > len(b) { f.err = ErrFieldTooLong - return } - ascii := toASCII(s) - copy(b, ascii) - if len(ascii) < len(b) { - b[len(ascii)] = 0 + s = toASCII(s) // TODO(dsnet): Remove this for UTF-8 support in GNU format + copy(b, s) + if len(s) < len(b) { + b[len(s)] = 0 } } @@ -162,6 +161,11 @@ func (p *parser) parseOctal(b []byte) int64 { } func (f *formatter) formatOctal(b []byte, x int64) { + if !fitsInOctal(len(b), x) { + x = 0 // Last resort, just write zero + f.err = ErrFieldTooLong + } + s := strconv.FormatInt(x, 8) // Add leading zeros, but leave room for a NUL. if n := len(b) - len(s) - 1; n > 0 { diff --git a/src/archive/tar/testdata/ustar.issue12594.tar b/src/archive/tar/testdata/ustar.issue12594.tar index 50fcd00976066625be6d9aef68aa7b24a6a99c29..64931bfbe17187372266b90aee763b6a8d5dd803 100644 GIT binary patch delta 43 qcmZpWXporDA!>jC3iEH#P512YB%1CV|WsD6gcoUGru03Pic!2kdN diff --git a/src/archive/tar/testdata/writer-big.tar b/src/archive/tar/testdata/writer-big.tar index 753e883cebf52ac1291f1b7bf1b7a37ae517b2d9..f838ada81b100f1daf84937e84579a4971cfd158 100644 GIT binary patch delta 39 pcmZorXi%6ijor-H+?YYZU~(d3%49~y0FM7qusM-&KHo$I4gehv5J~_5 delta 38 pcmZorXi%6ijosML$e2OFU~(d3%49~y0Coci*qq2XpKqc9Cji0|377x? diff --git a/src/archive/tar/writer.go b/src/archive/tar/writer.go index 28ce13ea72..8572c46335 100644 --- a/src/archive/tar/writer.go +++ b/src/archive/tar/writer.go @@ -83,12 +83,11 @@ func (tw *Writer) WriteHeader(hdr *Header) error { hdrCpy := *hdr hdrCpy.ModTime = hdrCpy.ModTime.Truncate(time.Second) - switch allowedFormats, _ := hdrCpy.allowedFormats(); { + switch allowedFormats, paxHdrs := hdrCpy.allowedFormats(); { case allowedFormats&formatUSTAR != 0: return tw.writeUSTARHeader(&hdrCpy) case allowedFormats&formatPAX != 0: - // TODO(dsnet): Implement and call specialized writePAXHeader. - return tw.writeHeader(&hdrCpy, true) + return tw.writePAXHeader(&hdrCpy, paxHdrs) case allowedFormats&formatGNU != 0: // TODO(dsnet): Implement and call specialized writeGNUHeader. return tw.writeHeader(&hdrCpy, true) @@ -111,6 +110,45 @@ func (tw *Writer) writeUSTARHeader(hdr *Header) error { return tw.writeRawHeader(blk, hdr.Size) } +func (tw *Writer) writePAXHeader(hdr *Header, paxHdrs map[string]string) error { + // Write PAX records to the output. + if len(paxHdrs) > 0 { + // Sort keys for deterministic ordering. + var keys []string + for k := range paxHdrs { + keys = append(keys, k) + } + sort.Strings(keys) + + // Write each record to a buffer. + var buf bytes.Buffer + for _, k := range keys { + rec, err := formatPAXRecord(k, paxHdrs[k]) + if err != nil { + return err + } + buf.WriteString(rec) + } + + // Write the extended header file. + dir, file := path.Split(hdr.Name) + name := path.Join(dir, "PaxHeaders.0", file) + data := buf.String() + if err := tw.writeRawFile(name, data, TypeXHeader, formatPAX); err != nil { + return err + } + } + + // Pack the main header. + var f formatter + blk := tw.templateV7Plus(hdr, &f) + blk.SetFormat(formatPAX) + if f.err != nil && len(paxHdrs) == 0 { + return f.err // Should never happen, otherwise PAX headers would be used + } + return tw.writeRawHeader(blk, hdr.Size) +} + // templateV7Plus fills out the V7 fields of a block using values from hdr. // It also fills out fields (uname, gname, devmajor, devminor) that are // shared in the USTAR, PAX, and GNU formats. @@ -143,6 +181,40 @@ func (tw *Writer) templateV7Plus(hdr *Header, f *formatter) *block { return &tw.blk } +// writeRawFile writes a minimal file with the given name and flag type. +// It uses format to encode the header format and will write data as the body. +// It uses default values for all of the other fields (as BSD and GNU tar does). +func (tw *Writer) writeRawFile(name, data string, flag byte, format int) error { + tw.blk.Reset() + + // Best effort for the filename. + name = toASCII(name) + if len(name) > nameSize { + name = name[:nameSize] + } + + var f formatter + v7 := tw.blk.V7() + v7.TypeFlag()[0] = flag + f.formatString(v7.Name(), name) + f.formatOctal(v7.Mode(), 0) + f.formatOctal(v7.UID(), 0) + f.formatOctal(v7.GID(), 0) + f.formatOctal(v7.Size(), int64(len(data))) // Must be < 8GiB + f.formatOctal(v7.ModTime(), 0) + tw.blk.SetFormat(format) + if f.err != nil { + return f.err // Only occurs if size condition is violated + } + + // Write the header and data. + if err := tw.writeRawHeader(&tw.blk, int64(len(data))); err != nil { + return err + } + _, err := io.WriteString(tw, data) + return err +} + // writeRawHeader writes the value of blk, regardless of its value. // It sets up the Writer such that it can accept a file of the given size. func (tw *Writer) writeRawHeader(blk *block, size int64) error { @@ -185,7 +257,7 @@ func (tw *Writer) writeHeader(hdr *Header, allowPax bool) error { // We need to select which scratch buffer to use carefully, // since this method is called recursively to write PAX headers. // If allowPax is true, this is the non-recursive call, and we will use hdrBuff. - // If allowPax is false, we are being called by writePAXHeader, and hdrBuff is + // If allowPax is false, we are being called by writePAXHeaderLegacy, and hdrBuff is // already being used by the non-recursive call, so we must use paxHdrBuff. header := &tw.hdrBuff if !allowPax { @@ -318,7 +390,7 @@ func (tw *Writer) writeHeader(hdr *Header, allowPax bool) error { if !allowPax { return errInvalidHeader } - if err := tw.writePAXHeader(hdr, paxHeaders); err != nil { + if err := tw.writePAXHeaderLegacy(hdr, paxHeaders); err != nil { return err } } @@ -350,9 +422,9 @@ func splitUSTARPath(name string) (prefix, suffix string, ok bool) { return name[:i], name[i+1:], true } -// writePaxHeader writes an extended pax header to the +// writePAXHeaderLegacy writes an extended pax header to the // archive. -func (tw *Writer) writePAXHeader(hdr *Header, paxHeaders map[string]string) error { +func (tw *Writer) writePAXHeaderLegacy(hdr *Header, paxHeaders map[string]string) error { // Prepare extended header ext := new(Header) ext.Typeflag = TypeXHeader diff --git a/src/archive/tar/writer_test.go b/src/archive/tar/writer_test.go index a246b9387d..f37d4fdcee 100644 --- a/src/archive/tar/writer_test.go +++ b/src/archive/tar/writer_test.go @@ -116,6 +116,7 @@ func TestWriter(t *testing.T) { Typeflag: '0', Uname: "dsymonds", Gname: "eng", + Devminor: -1, // Force use of GNU format }, // fake contents contents: strings.Repeat("\x00", 4<<10), -- 2.48.1