+

+// isHeaderOnlyType checks if the given type flag is of the type that has no

+// data section even if a size is specified.

+func isHeaderOnlyType(flag byte) bool {

+ switch flag {

+ case TypeLink, TypeSymlink, TypeChar, TypeBlock, TypeDir, TypeFifo:

+ return true

+ default:

+ return false

+ }

+}

+ "math"

+type parser struct {

+ err error // Last error seen

+}

+

+// A sparseFileReader is a numBytesReader for reading sparse file data from a

+// tar archive.

+ rfr numBytesReader // Reads the sparse-encoded file data

+ sp []sparseEntry // The sparse map for the file

+ pos int64 // Keeps track of file position

+ total int64 // Total size of the file

+}

+

+// A sparseEntry holds a single entry in a sparse file's sparse map.

+//

+// Sparse files are represented using a series of sparseEntrys.

+// Despite the name, a sparseEntry represents an actual data fragment that

+// references data found in the underlying archive stream. All regions not

+// covered by a sparseEntry are logically filled with zeros.

+//

+// For example, if the underlying raw file contains the 10-byte data:

+// var compactData = "abcdefgh"

+//

+// And the sparse map has the following entries:

+// var sp = []sparseEntry{

+// {offset: 2, numBytes: 5} // Data fragment for [2..7]

+// {offset: 18, numBytes: 3} // Data fragment for [18..21]

+// }

+//

+// Then the content of the resulting sparse file with a "real" size of 25 is:

+// var sparseData = "\x00"*2 + "abcde" + "\x00"*11 + "fgh" + "\x00"*4

+type sparseEntry struct {

+ offset int64 // Starting position of the fragment

+ numBytes int64 // Length of the fragment

+ return nil, tr.err

+

+ var hdr *Header

+ var extHdrs map[string]string

+

+ // Externally, Next iterates through the tar archive as if it is a series of

+ // files. Internally, the tar format often uses fake "files" to add meta

+ // data that describes the next file. These meta data "files" should not

+ // normally be visible to the outside. As such, this loop iterates through

+ // one or more "header files" until it finds a "normal file".

+loop:

+ for {

+ tr.err = tr.skipUnread()

+

+ if tr.err != nil {

+ // Check for PAX/GNU special headers and files.

+ switch hdr.Typeflag {

+ case TypeXHeader:

+ extHdrs, tr.err = parsePAX(tr)

+ if tr.err != nil {

+ return nil, tr.err

+ }

+ continue loop // This is a meta header affecting the next header

+ case TypeGNULongName, TypeGNULongLink:

+ var realname []byte

+ realname, tr.err = ioutil.ReadAll(tr)

+ if tr.err != nil {

+ return nil, tr.err

+ }

+

+ // Convert GNU extensions to use PAX headers.

+ if extHdrs == nil {

+ extHdrs = make(map[string]string)

+ }

+ var p parser

+ switch hdr.Typeflag {

+ case TypeGNULongName:

+ extHdrs[paxPath] = p.parseString(realname)

+ case TypeGNULongLink:

+ extHdrs[paxLinkpath] = p.parseString(realname)

+ }

+ if p.err != nil {

+ tr.err = p.err

+ return nil, tr.err

+ }

+ continue loop // This is a meta header affecting the next header

+ default:

+ mergePAX(hdr, extHdrs)

+

+ // Check for a PAX format sparse file

+ sp, err := tr.checkForGNUSparsePAXHeaders(hdr, extHdrs)

+ if err != nil {

+ tr.err = err

+ return nil, err

+ }

+ if sp != nil {

+ // Current file is a PAX format GNU sparse file.

+ // Set the current file reader to a sparse file reader.

+ tr.curr, tr.err = newSparseFileReader(tr.curr, sp, hdr.Size)

+ if tr.err != nil {

+ return nil, tr.err

+ }

+ }

+ break loop // This is a file, so stop

+ return hdr, nil

+ sbuf := string(buf)

+ for len(sbuf) > 0 {

+ key, value, residual, err := parsePAXRecord(sbuf)

+ if err != nil {

+ sbuf = residual

+ sparseMap.WriteString(value)

+// parsePAXRecord parses the input PAX record string into a key-value pair.

+// If parsing is successful, it will slice off the currently read record and

+// return the remainder as r.

+//

+// A PAX record is of the following form:

+// "%d %s=%s\n" % (size, key, value)

+func parsePAXRecord(s string) (k, v, r string, err error) {

+ // The size field ends at the first space.

+ sp := strings.IndexByte(s, ' ')

+ if sp == -1 {

+ return "", "", s, ErrHeader

+ }

+

+ // Parse the first token as a decimal integer.

+ n, perr := strconv.ParseInt(s[:sp], 10, 0) // Intentionally parse as native int

+ if perr != nil || n < 5 || int64(len(s)) < n {

+ return "", "", s, ErrHeader

+ }

+

+ // Extract everything between the space and the final newline.

+ rec, nl, rem := s[sp+1:n-1], s[n-1:n], s[n:]

+ if nl != "\n" {

+ return "", "", s, ErrHeader

+ }

+

+ // The first equals separates the key from the value.

+ eq := strings.IndexByte(rec, '=')

+ if eq == -1 {

+ return "", "", s, ErrHeader

+ }

+ return rec[:eq], rec[eq+1:], rem, nil

+}

+

+// parseString parses bytes as a NUL-terminated C-style string.

+func (*parser) parseString(b []byte) string {

+// parseNumeric parses the input as being encoded in either base-256 or octal.

+// This function may return negative numbers.

+// If parsing fails or an integer overflow occurs, err will be set.

+func (p *parser) parseNumeric(b []byte) int64 {

+ // Check for base-256 (binary) format first.

+ // If the first bit is set, then all following bits constitute a two's

+ // complement encoded number in big-endian byte order.

+ // Handling negative numbers relies on the following identity:

+ // -a-1 == ^a

+ //

+ // If the number is negative, we use an inversion mask to invert the

+ // data bytes and treat the value as an unsigned number.

+ var inv byte // 0x00 if positive or zero, 0xff if negative

+ if b[0]&0x40 != 0 {

+ inv = 0xff

+ }

+

+ var x uint64

+ c ^= inv // Inverts c only if inv is 0xff, otherwise does nothing

+ c &= 0x7f // Ignore signal bit in first byte

+ }

+ if (x >> 56) > 0 {

+ p.err = ErrHeader // Integer overflow

+ return 0

+ x = x<<8 | uint64(c)

+ }

+ if (x >> 63) > 0 {

+ p.err = ErrHeader // Integer overflow

+ return 0

+ }

+ if inv == 0xff {

+ return ^int64(x)

+ return int64(x)

+ // Normal case is base-8 (octal) format.

+ return p.parseOctal(b)

+}

+

+func (p *parser) parseOctal(b []byte) int64 {

+ x, perr := strconv.ParseUint(p.parseString(b), 8, 64)

+ if perr != nil {

+ p.err = ErrHeader

+// skipUnread skips any unread bytes in the existing file entry, as well as any

+// alignment padding. It returns io.ErrUnexpectedEOF if any io.EOF is

+// encountered in the data portion; it is okay to hit io.EOF in the padding.

+//

+// Note that this function still works properly even when sparse files are being

+// used since numBytes returns the bytes remaining in the underlying io.Reader.

+func (tr *Reader) skipUnread() error {

+ dataSkip := tr.numBytes() // Number of data bytes to skip

+ totalSkip := dataSkip + tr.pad // Total number of bytes to skip

+

+ // If possible, Seek to the last byte before the end of the data section.

+ // Do this because Seek is often lazy about reporting errors; this will mask

+ // the fact that the tar stream may be truncated. We can rely on the

+ // io.CopyN done shortly afterwards to trigger any IO errors.

+ var seekSkipped int64 // Number of bytes skipped via Seek

+ if sr, ok := tr.r.(io.Seeker); ok && dataSkip > 1 {

+ // Not all io.Seeker can actually Seek. For example, os.Stdin implements

+ // io.Seeker, but calling Seek always returns an error and performs

+ // no action. Thus, we try an innocent seek to the current position

+ // to see if Seek is really supported.

+ pos1, err := sr.Seek(0, os.SEEK_CUR)

+ if err == nil {

+ // Seek seems supported, so perform the real Seek.

+ pos2, err := sr.Seek(dataSkip-1, os.SEEK_CUR)

+ if err != nil {

+ tr.err = err

+ return tr.err

+ }

+ seekSkipped = pos2 - pos1

+

+ var copySkipped int64 // Number of bytes skipped via CopyN

+ copySkipped, tr.err = io.CopyN(ioutil.Discard, tr.r, totalSkip-seekSkipped)

+ if tr.err == io.EOF && seekSkipped+copySkipped < dataSkip {

+ tr.err = io.ErrUnexpectedEOF

+ }

+ return tr.err

+ var p parser

+ given := p.parseOctal(header[148:156])

+ return p.err == nil && (given == unsigned || given == signed)

+// readHeader reads the next block header and assumes that the underlying reader

+// is already aligned to a block boundary.

+//

+// The err will be set to io.EOF only when one of the following occurs:

+// * Exactly 0 bytes are read and EOF is hit.

+// * Exactly 1 block of zeros is read and EOF is hit.

+// * At least 2 blocks of zeros are read.

+ return nil // io.EOF is okay here

+ return nil // io.EOF is okay here

+ var p parser

+ hdr.Name = p.parseString(s.next(100))

+ hdr.Mode = p.parseNumeric(s.next(8))

+ hdr.Uid = int(p.parseNumeric(s.next(8)))

+ hdr.Gid = int(p.parseNumeric(s.next(8)))

+ hdr.Size = p.parseNumeric(s.next(12))

+ hdr.ModTime = time.Unix(p.parseNumeric(s.next(12)), 0)

+ hdr.Linkname = p.parseString(s.next(100))

+ hdr.Uname = p.parseString(s.next(32))

+ hdr.Gname = p.parseString(s.next(32))

+ hdr.Devmajor = p.parseNumeric(devmajor)

+ hdr.Devminor = p.parseNumeric(devminor)

+ prefix = p.parseString(s.next(155))

+ prefix = p.parseString(s.next(131))

+ hdr.AccessTime = time.Unix(p.parseNumeric(s.next(12)), 0)

+ hdr.ChangeTime = time.Unix(p.parseNumeric(s.next(12)), 0)

+ if p.err != nil {

+ tr.err = p.err

+ nb := hdr.Size

+ if isHeaderOnlyType(hdr.Typeflag) {

+ nb = 0

+ }

+ if nb < 0 {

+ tr.err = ErrHeader

+ return nil

+ }

+ tr.pad = -nb & (blockSize - 1) // blockSize is a power of two

+ hdr.Size = p.parseNumeric(header[483:495])

+ if p.err != nil {

+ tr.err = p.err

+ return nil

+ }

+

+ tr.curr, tr.err = newSparseFileReader(tr.curr, sp, hdr.Size)

+ if tr.err != nil {

+ return nil

+ }

+ var p parser

+ offset := p.parseNumeric(s.next(oldGNUSparseOffsetSize))

+ numBytes := p.parseNumeric(s.next(oldGNUSparseNumBytesSize))

+ if p.err != nil {

+ tr.err = p.err

+ offset := p.parseNumeric(s.next(oldGNUSparseOffsetSize))

+ numBytes := p.parseNumeric(s.next(oldGNUSparseNumBytesSize))

+ if p.err != nil {

+ tr.err = p.err

+// readGNUSparseMap1x0 reads the sparse map as stored in GNU's PAX sparse format

+// version 1.0. The format of the sparse map consists of a series of

+// newline-terminated numeric fields. The first field is the number of entries

+// and is always present. Following this are the entries, consisting of two

+// fields (offset, numBytes). This function must stop reading at the end

+// boundary of the block containing the last newline.

+//

+// Note that the GNU manual says that numeric values should be encoded in octal

+// format. However, the GNU tar utility itself outputs these values in decimal.

+// As such, this library treats values as being encoded in decimal.

+ var cntNewline int64

+ var buf bytes.Buffer

+ var blk = make([]byte, blockSize)

+

+ // feedTokens copies data in numBlock chunks from r into buf until there are

+ // at least cnt newlines in buf. It will not read more blocks than needed.

+ var feedTokens = func(cnt int64) error {

+ for cntNewline < cnt {

+ if _, err := io.ReadFull(r, blk); err != nil {

+ if err == io.EOF {

+ err = io.ErrUnexpectedEOF

+ }

+ return err

+ buf.Write(blk)

+ for _, c := range blk {

+ if c == '\n' {

+ cntNewline++

+ }

+ return nil

+ }

+ // nextToken gets the next token delimited by a newline. This assumes that

+ // at least one newline exists in the buffer.

+ var nextToken = func() string {

+ cntNewline--

+ tok, _ := buf.ReadString('\n')

+ return tok[:len(tok)-1] // Cut off newline

+ // Parse for the number of entries.

+ // Use integer overflow resistant math to check this.

+ if err := feedTokens(1); err != nil {

+ numEntries, err := strconv.ParseInt(nextToken(), 10, 0) // Intentionally parse as native int

+ if err != nil || numEntries < 0 || int(2*numEntries) < int(numEntries) {

+ return nil, ErrHeader

+ }

+ // Parse for all member entries.

+ // numEntries is trusted after this since a potential attacker must have

+ // committed resources proportional to what this library used.

+ if err := feedTokens(2 * numEntries); err != nil {

+ offset, err := strconv.ParseInt(nextToken(), 10, 64)

+ return nil, ErrHeader

+ numBytes, err := strconv.ParseInt(nextToken(), 10, 64)

+ return nil, ErrHeader

+// readGNUSparseMap0x1 reads the sparse map as stored in GNU's PAX sparse format

+// version 0.1. The sparse map is stored in the PAX headers.

+func readGNUSparseMap0x1(extHdrs map[string]string) ([]sparseEntry, error) {

+ // Get number of entries.

+ // Use integer overflow resistant math to check this.

+ numEntriesStr := extHdrs[paxGNUSparseNumBlocks]

+ numEntries, err := strconv.ParseInt(numEntriesStr, 10, 0) // Intentionally parse as native int

+ if err != nil || numEntries < 0 || int(2*numEntries) < int(numEntries) {

+ // There should be two numbers in sparseMap for each entry.

+ sparseMap := strings.Split(extHdrs[paxGNUSparseMap], ",")

+ // Loop through the entries in the sparse map.

+ // numEntries is trusted now.

+ offset, err := strconv.ParseInt(sparseMap[2*i], 10, 64)

+ numBytes, err := strconv.ParseInt(sparseMap[2*i+1], 10, 64)

+//

+// Calling Read on special types like TypeLink, TypeSymLink, TypeChar,

+// TypeBlock, TypeDir, and TypeFifo returns 0, io.EOF regardless of what

+// the Header.Size claims.

+ if tr.err != nil {

+ return 0, tr.err

+ }

+

+// newSparseFileReader creates a new sparseFileReader, but validates all of the

+// sparse entries before doing so.

+func newSparseFileReader(rfr numBytesReader, sp []sparseEntry, total int64) (*sparseFileReader, error) {

+ if total < 0 {

+ return nil, ErrHeader // Total size cannot be negative

+ }

+

+ // Validate all sparse entries. These are the same checks as performed by

+ // the BSD tar utility.

+ for i, s := range sp {

+ switch {

+ case s.offset < 0 || s.numBytes < 0:

+ return nil, ErrHeader // Negative values are never okay

+ case s.offset > math.MaxInt64-s.numBytes:

+ return nil, ErrHeader // Integer overflow with large length

+ case s.offset+s.numBytes > total:

+ return nil, ErrHeader // Region extends beyond the "real" size

+ case i > 0 && sp[i-1].offset+sp[i-1].numBytes > s.offset:

+ return nil, ErrHeader // Regions can't overlap and must be in order

+ }

+ }

+ return &sparseFileReader{rfr: rfr, sp: sp, total: total}, nil

+}

+

+// readHole reads a sparse hole ending at endOffset.

+func (sfr *sparseFileReader) readHole(b []byte, endOffset int64) int {

+ n64 := endOffset - sfr.pos

+ // Skip past all empty fragments.

+ for len(sfr.sp) > 0 && sfr.sp[0].numBytes == 0 {

+ sfr.sp = sfr.sp[1:]

+ }

+

+ // If there are no more fragments, then it is possible that there

+ // is one last sparse hole.

+ // This behavior matches the BSD tar utility.

+ // However, GNU tar stops returning data even if sfr.total is unmet.

+ if sfr.pos < sfr.total {

+ return sfr.readHole(b, sfr.total), nil

+

+ // In front of a data fragment, so read a hole.

+ return sfr.readHole(b, sfr.sp[0].offset), nil

+ // In a data fragment, so read from it.

+ // This math is overflow free since we verify that offset and numBytes can

+ // be safely added when creating the sparseFileReader.

+ endPos := sfr.sp[0].offset + sfr.sp[0].numBytes // End offset of fragment

+ bytesLeft := endPos - sfr.pos // Bytes left in fragment

+ b = b[:bytesLeft]

+ if err == io.EOF {

+ if sfr.pos < endPos {

+ err = io.ErrUnexpectedEOF // There was supposed to be more data

+ } else if sfr.pos < sfr.total {

+ err = nil // There is still an implicit sparse hole at the end

+ }

+ if sfr.pos == endPos {

+ sfr.sp = sfr.sp[1:] // We are done with this fragment, so pop it

+ return n, err

+ return sfr.rfr.numBytes()

+ "math"

+ file string // Test input file

+ headers []*Header // Expected output headers

+ chksums []string // MD5 checksum of files, leave as nil if not checked

+ err error // Expected error to occur

+ chksums: []string{

+ chksums: []string{

+ {

+ // Matches the behavior of GNU, BSD, and STAR tar utilities.

+ file: "testdata/gnu-multi-hdrs.tar",

+ headers: []*Header{

+ {

+ Name: "GNU2/GNU2/long-path-name",

+ Linkname: "GNU4/GNU4/long-linkpath-name",

+ ModTime: time.Unix(0, 0),

+ Typeflag: '2',

+ },

+ },

+ },

+ {

+ // Matches the behavior of GNU and BSD tar utilities.

+ file: "testdata/pax-multi-hdrs.tar",

+ headers: []*Header{

+ {

+ Name: "bar",

+ Linkname: "PAX4/PAX4/long-linkpath-name",

+ ModTime: time.Unix(0, 0),

+ Typeflag: '2',

+ },

+ },

+ },

+ {

+ file: "testdata/neg-size.tar",

+ err: ErrHeader,

+ },

+ {

+ file: "testdata/issue10968.tar",

+ err: ErrHeader,

+ },

+ {

+ file: "testdata/issue11169.tar",

+ err: ErrHeader,

+ },

+ {

+ file: "testdata/issue12435.tar",

+ err: ErrHeader,

+ },

+ for i, v := range untarTests {

+ f, err := os.Open(v.file)

+ t.Errorf("file %s, test %d: unexpected error: %v", v.file, i, err)

+

+ // Capture all headers and checksums.

+ var (

+ tr = NewReader(f)

+ hdrs []*Header

+ chksums []string

+ rdbuf = make([]byte, 8)

+ )

+ for {

+ var hdr *Header

+ hdr, err = tr.Next()

+ if err != nil {

+ if err == io.EOF {

+ err = nil // Expected error

+ }

+ break

+ hdrs = append(hdrs, hdr)

+

+ if v.chksums == nil {

+ continue

+ }

+ h := md5.New()

+ _, err = io.CopyBuffer(h, tr, rdbuf) // Effectively an incremental read

+ if err != nil {

+ break

+ chksums = append(chksums, fmt.Sprintf("%x", h.Sum(nil)))

+

+ for j, hdr := range hdrs {

+ if j >= len(v.headers) {

+ t.Errorf("file %s, test %d, entry %d: unexpected header:\ngot %+v",

+ v.file, i, j, *hdr)

+ continue

+ }

+ if !reflect.DeepEqual(*hdr, *v.headers[j]) {

+ t.Errorf("file %s, test %d, entry %d: incorrect header:\ngot %+v\nwant %+v",

+ v.file, i, j, *hdr, *v.headers[j])

+ }

+ }

+ if len(hdrs) != len(v.headers) {

+ t.Errorf("file %s, test %d: got %d headers, want %d headers",

+ v.file, i, len(hdrs), len(v.headers))

+ }

+

+ for j, sum := range chksums {

+ if j >= len(v.chksums) {

+ t.Errorf("file %s, test %d, entry %d: unexpected sum: got %s",

+ v.file, i, j, sum)

+ continue

+ }

+ if sum != v.chksums[j] {

+ t.Errorf("file %s, test %d, entry %d: incorrect checksum: got %s, want %s",

+ v.file, i, j, sum, v.chksums[j])

+ }

+

+ if err != v.err {

+ t.Errorf("file %s, test %d: unexpected error: got %v, want %v",

+ v.file, i, err, v.err)

+ f.Close()

+func TestSparseFileReader(t *testing.T) {

+ var vectors = []struct {

+ realSize int64 // Real size of the output file

+ sparseMap []sparseEntry // Input sparse map

+ sparseData string // Input compact data

+ expected string // Expected output data

+ err error // Expected error outcome

+ }{{

+ realSize: 8,

+ sparseData: "abcde",

+ expected: "ab\x00\x00\x00cde",

+ }, {

+ realSize: 10,

+ sparseData: "abcde",

+ expected: "ab\x00\x00\x00cde\x00\x00",

+ }, {

+ realSize: 8,

+ sparseData: "abcde",

+ expected: "\x00abc\x00\x00de",

+ }, {

+ {offset: 6, numBytes: 0},

+ {offset: 6, numBytes: 0},

+ sparseData: "abcde",

+ expected: "\x00abc\x00\x00de",

+ }, {

+ sparseMap: []sparseEntry{

+ {offset: 1, numBytes: 3},

+ {offset: 6, numBytes: 2},

+ },

+ sparseData: "abcde",

+ expected: "\x00abc\x00\x00de\x00\x00",

+ }, {

+ realSize: 10,

+ sparseMap: []sparseEntry{

+ {offset: 1, numBytes: 3},

+ {offset: 6, numBytes: 2},

+ {offset: 8, numBytes: 0},

+ {offset: 8, numBytes: 0},

+ {offset: 8, numBytes: 0},

+ {offset: 8, numBytes: 0},

+ },

+ sparseData: "abcde",

+ expected: "\x00abc\x00\x00de\x00\x00",

+ }, {

+ sparseMap: []sparseEntry{},

+ sparseData: "",

+ expected: "\x00\x00",

+ }, {

+ realSize: -2,

+ sparseMap: []sparseEntry{},

+ err: ErrHeader,

+ }, {

+ realSize: -10,

+ sparseMap: []sparseEntry{

+ {offset: 1, numBytes: 3},

+ {offset: 6, numBytes: 2},

+ },

+ sparseData: "abcde",

+ err: ErrHeader,

+ }, {

+ realSize: 10,

+ sparseMap: []sparseEntry{

+ {offset: 1, numBytes: 3},

+ {offset: 6, numBytes: 5},

+ },

+ sparseData: "abcde",

+ err: ErrHeader,

+ }, {

+ realSize: 35,

+ sparseMap: []sparseEntry{

+ {offset: 1, numBytes: 3},

+ {offset: 6, numBytes: 5},

+ },

+ sparseData: "abcde",

+ err: io.ErrUnexpectedEOF,

+ }, {

+ realSize: 35,

+ sparseMap: []sparseEntry{

+ {offset: 1, numBytes: 3},

+ {offset: 6, numBytes: -5},

+ },

+ sparseData: "abcde",

+ err: ErrHeader,

+ }, {

+ realSize: 35,

+ sparseMap: []sparseEntry{

+ {offset: math.MaxInt64, numBytes: 3},

+ {offset: 6, numBytes: -5},

+ },

+ sparseData: "abcde",

+ err: ErrHeader,

+ }, {

+ realSize: 10,

+ sparseMap: []sparseEntry{

+ {offset: 1, numBytes: 3},

+ {offset: 2, numBytes: 2},

+ },

+ sparseData: "abcde",

+ err: ErrHeader,

+ }}

+ for i, v := range vectors {

+ r := bytes.NewReader([]byte(v.sparseData))

+ rfr := &regFileReader{r: r, nb: int64(len(v.sparseData))}

+

+ var sfr *sparseFileReader

+ var err error

+ var buf []byte

+

+ sfr, err = newSparseFileReader(rfr, v.sparseMap, v.realSize)

+ if err != nil {

+ goto fail

+ }

+ if sfr.numBytes() != int64(len(v.sparseData)) {

+ t.Errorf("test %d, numBytes() before reading: got %d, want %d", i, sfr.numBytes(), len(v.sparseData))

+ }

+ buf, err = ioutil.ReadAll(sfr)

+ goto fail

+ if string(buf) != v.expected {

+ t.Errorf("test %d, ReadAll(): got %q, want %q", i, string(buf), v.expected)

+ t.Errorf("test %d, numBytes() after reading: got %d, want %d", i, sfr.numBytes(), 0)

+ }

+

+ fail:

+ if err != v.err {

+ t.Errorf("test %d, unexpected error: got %v, want %v", i, err, v.err)

+func TestReadGNUSparseMap0x1(t *testing.T) {

+ const (

+ maxUint = ^uint(0)

+ maxInt = int(maxUint >> 1)

+ )

+ var (

+ big1 = fmt.Sprintf("%d", int64(maxInt))

+ big2 = fmt.Sprintf("%d", (int64(maxInt)/2)+1)

+ big3 = fmt.Sprintf("%d", (int64(maxInt) / 3))

+ )

+ var vectors = []struct {

+ extHdrs map[string]string // Input data

+ sparseMap []sparseEntry // Expected sparse entries to be outputted

+ err error // Expected errors that may be raised

+ }{{

+ extHdrs: map[string]string{paxGNUSparseNumBlocks: "-4"},

+ err: ErrHeader,

+ }, {

+ extHdrs: map[string]string{paxGNUSparseNumBlocks: "fee "},

+ err: ErrHeader,

+ }, {

+ extHdrs: map[string]string{

+ paxGNUSparseNumBlocks: big1,

+ paxGNUSparseMap: "0,5,10,5,20,5,30,5",

+ },

+ err: ErrHeader,

+ }, {

+ extHdrs: map[string]string{

+ paxGNUSparseNumBlocks: big2,

+ paxGNUSparseMap: "0,5,10,5,20,5,30,5",

+ },

+ err: ErrHeader,

+ }, {

+ extHdrs: map[string]string{

+ paxGNUSparseNumBlocks: big3,

+ paxGNUSparseMap: "0,5,10,5,20,5,30,5",

+ },

+ err: ErrHeader,

+ }, {

+ extHdrs: map[string]string{

+ paxGNUSparseNumBlocks: "4",

+ paxGNUSparseMap: "0.5,5,10,5,20,5,30,5",

+ },

+ err: ErrHeader,

+ }, {

+ extHdrs: map[string]string{

+ paxGNUSparseNumBlocks: "4",

+ paxGNUSparseMap: "0,5.5,10,5,20,5,30,5",

+ },

+ err: ErrHeader,

+ }, {

+ extHdrs: map[string]string{

+ paxGNUSparseNumBlocks: "4",

+ paxGNUSparseMap: "0,fewafewa.5,fewafw,5,20,5,30,5",

+ },

+ err: ErrHeader,

+ }, {

+ extHdrs: map[string]string{

+ paxGNUSparseNumBlocks: "4",

+ paxGNUSparseMap: "0,5,10,5,20,5,30,5",

+ },

+ sparseMap: []sparseEntry{{0, 5}, {10, 5}, {20, 5}, {30, 5}},

+ }}

+ for i, v := range vectors {

+ sp, err := readGNUSparseMap0x1(v.extHdrs)

+ if !reflect.DeepEqual(sp, v.sparseMap) && !(len(sp) == 0 && len(v.sparseMap) == 0) {

+ t.Errorf("test %d, readGNUSparseMap0x1(...): got %v, want %v", i, sp, v.sparseMap)

+ if err != v.err {

+ t.Errorf("test %d, unexpected error: got %v, want %v", i, err, v.err)

+func TestReadGNUSparseMap1x0(t *testing.T) {

+ var sp = []sparseEntry{{1, 2}, {3, 4}}

+ for i := 0; i < 98; i++ {

+ sp = append(sp, sparseEntry{54321, 12345})

+ var vectors = []struct {

+ input string // Input data

+ sparseMap []sparseEntry // Expected sparse entries to be outputted

+ cnt int // Expected number of bytes read

+ err error // Expected errors that may be raised

+ }{{

+ input: "",

+ cnt: 0,

+ err: io.ErrUnexpectedEOF,

+ }, {

+ input: "ab",

+ cnt: 2,

+ err: io.ErrUnexpectedEOF,

+ }, {

+ input: strings.Repeat("\x00", 512),

+ cnt: 512,

+ err: io.ErrUnexpectedEOF,

+ }, {

+ input: strings.Repeat("\x00", 511) + "\n",

+ cnt: 512,

+ err: ErrHeader,

+ }, {

+ input: strings.Repeat("\n", 512),

+ cnt: 512,

+ err: ErrHeader,

+ }, {

+ input: "0\n" + strings.Repeat("\x00", 510) + strings.Repeat("a", 512),

+ sparseMap: []sparseEntry{},

+ cnt: 512,

+ }, {

+ input: strings.Repeat("0", 512) + "0\n" + strings.Repeat("\x00", 510),

+ sparseMap: []sparseEntry{},

+ cnt: 1024,

+ }, {

+ input: strings.Repeat("0", 1024) + "1\n2\n3\n" + strings.Repeat("\x00", 506),

+ sparseMap: []sparseEntry{{2, 3}},

+ cnt: 1536,

+ }, {

+ input: strings.Repeat("0", 1024) + "1\n2\n\n" + strings.Repeat("\x00", 509),

+ cnt: 1536,

+ err: ErrHeader,

+ }, {

+ input: strings.Repeat("0", 1024) + "1\n2\n" + strings.Repeat("\x00", 508),

+ cnt: 1536,

+ err: io.ErrUnexpectedEOF,

+ }, {

+ input: "-1\n2\n\n" + strings.Repeat("\x00", 506),

+ cnt: 512,

+ err: ErrHeader,

+ }, {

+ input: "1\nk\n2\n" + strings.Repeat("\x00", 506),

+ cnt: 512,

+ err: ErrHeader,

+ }, {

+ input: "100\n1\n2\n3\n4\n" + strings.Repeat("54321\n0000000000000012345\n", 98) + strings.Repeat("\x00", 512),

+ cnt: 2560,

+ sparseMap: sp,

+ }}

+ for i, v := range vectors {

+ r := strings.NewReader(v.input)

+ sp, err := readGNUSparseMap1x0(r)

+ if !reflect.DeepEqual(sp, v.sparseMap) && !(len(sp) == 0 && len(v.sparseMap) == 0) {

+ t.Errorf("test %d, readGNUSparseMap1x0(...): got %v, want %v", i, sp, v.sparseMap)

+ }

+ if numBytes := len(v.input) - r.Len(); numBytes != v.cnt {

+ t.Errorf("test %d, bytes read: got %v, want %v", i, numBytes, v.cnt)

+ }

+ if err != v.err {

+ t.Errorf("test %d, unexpected error: got %v, want %v", i, err, v.err)

+ }

+type reader struct{ io.Reader }

+type readSeeker struct{ io.ReadSeeker }

+type readBadSeeker struct{ io.ReadSeeker }

+

+func (rbs *readBadSeeker) Seek(int64, int) (int64, error) { return 0, fmt.Errorf("illegal seek") }

+

+// TestReadTruncation test the ending condition on various truncated files and

+// that truncated files are still detected even if the underlying io.Reader

+// satisfies io.Seeker.

+func TestReadTruncation(t *testing.T) {

+ var ss []string

+ for _, p := range []string{

+ "testdata/gnu.tar",

+ "testdata/ustar-file-reg.tar",

+ "testdata/pax-path-hdr.tar",

+ "testdata/sparse-formats.tar",

+ } {

+ buf, err := ioutil.ReadFile(p)

+ if err != nil {

+ t.Fatalf("unexpected error: %v", err)

+ }

+ ss = append(ss, string(buf))

+

+ data1, data2, pax, sparse := ss[0], ss[1], ss[2], ss[3]

+ data2 += strings.Repeat("\x00", 10*512)

+ trash := strings.Repeat("garbage ", 64) // Exactly 512 bytes

+

+ var vectors = []struct {

+ input string // Input stream

+ cnt int // Expected number of headers read

+ err error // Expected error outcome

+ }{

+ {"", 0, io.EOF}, // Empty file is a "valid" tar file

+ {data1[:511], 0, io.ErrUnexpectedEOF},

+ {data1[:512], 1, io.ErrUnexpectedEOF},

+ {data1[:1024], 1, io.EOF},

+ {data1[:1536], 2, io.ErrUnexpectedEOF},

+ {data1[:2048], 2, io.EOF},

+ {data1, 2, io.EOF},

+ {data1[:2048] + data2[:1536], 3, io.EOF},

+ {data2[:511], 0, io.ErrUnexpectedEOF},

+ {data2[:512], 1, io.ErrUnexpectedEOF},

+ {data2[:1195], 1, io.ErrUnexpectedEOF},

+ {data2[:1196], 1, io.EOF}, // Exact end of data and start of padding

+ {data2[:1200], 1, io.EOF},

+ {data2[:1535], 1, io.EOF},

+ {data2[:1536], 1, io.EOF}, // Exact end of padding

+ {data2[:1536] + trash[:1], 1, io.ErrUnexpectedEOF},

+ {data2[:1536] + trash[:511], 1, io.ErrUnexpectedEOF},

+ {data2[:1536] + trash, 1, ErrHeader},

+ {data2[:2048], 1, io.EOF}, // Exactly 1 empty block

+ {data2[:2048] + trash[:1], 1, io.ErrUnexpectedEOF},

+ {data2[:2048] + trash[:511], 1, io.ErrUnexpectedEOF},

+ {data2[:2048] + trash, 1, ErrHeader},

+ {data2[:2560], 1, io.EOF}, // Exactly 2 empty blocks (normal end-of-stream)

+ {data2[:2560] + trash[:1], 1, io.EOF},

+ {data2[:2560] + trash[:511], 1, io.EOF},

+ {data2[:2560] + trash, 1, io.EOF},

+ {data2[:3072], 1, io.EOF},

+ {pax, 0, io.EOF}, // PAX header without data is a "valid" tar file

+ {pax + trash[:1], 0, io.ErrUnexpectedEOF},

+ {pax + trash[:511], 0, io.ErrUnexpectedEOF},

+ {sparse[:511], 0, io.ErrUnexpectedEOF},

+ // TODO(dsnet): This should pass, but currently fails.

+ // {sparse[:512], 0, io.ErrUnexpectedEOF},

+ {sparse[:3584], 1, io.EOF},

+ {sparse[:9200], 1, io.EOF}, // Terminate in padding of sparse header

+ {sparse[:9216], 1, io.EOF},

+ {sparse[:9728], 2, io.ErrUnexpectedEOF},

+ {sparse[:10240], 2, io.EOF},

+ {sparse[:11264], 2, io.ErrUnexpectedEOF},

+ {sparse, 5, io.EOF},

+ {sparse + trash, 5, io.EOF},

+ }

+

+ for i, v := range vectors {

+ for j := 0; j < 6; j++ {

+ var tr *Reader

+ var s1, s2 string

+

+ switch j {

+ case 0:

+ tr = NewReader(&reader{strings.NewReader(v.input)})

+ s1, s2 = "io.Reader", "auto"

+ case 1:

+ tr = NewReader(&reader{strings.NewReader(v.input)})

+ s1, s2 = "io.Reader", "manual"

+ case 2:

+ tr = NewReader(&readSeeker{strings.NewReader(v.input)})

+ s1, s2 = "io.ReadSeeker", "auto"

+ case 3:

+ tr = NewReader(&readSeeker{strings.NewReader(v.input)})

+ s1, s2 = "io.ReadSeeker", "manual"

+ case 4:

+ tr = NewReader(&readBadSeeker{strings.NewReader(v.input)})

+ s1, s2 = "ReadBadSeeker", "auto"

+ case 5:

+ tr = NewReader(&readBadSeeker{strings.NewReader(v.input)})

+ s1, s2 = "ReadBadSeeker", "manual"

+ }

+

+ var cnt int

+ var err error

+ for {

+ if _, err = tr.Next(); err != nil {

+ break

+ }

+ cnt++

+ if s2 == "manual" {

+ if _, err = io.Copy(ioutil.Discard, tr); err != nil {

+ break

+ }

+ }

+ }

+ if err != v.err {

+ t.Errorf("test %d, NewReader(%s(...)) with %s discard: got %v, want %v",

+ i, s1, s2, err, v.err)

+ }

+ if cnt != v.cnt {

+ t.Errorf("test %d, NewReader(%s(...)) with %s discard: got %d headers, want %d headers",

+ i, s1, s2, cnt, v.cnt)

+ }

+ }

+// TestReadHeaderOnly tests that Reader does not attempt to read special

+// header-only files.

+func TestReadHeaderOnly(t *testing.T) {

+ f, err := os.Open("testdata/hdr-only.tar")

+ t.Fatalf("unexpected error: %v", err)

+

+ var hdrs []*Header

+ tr := NewReader(f)

+ for {

+ hdr, err := tr.Next()

+ if err == io.EOF {

+ break

+ }

+ if err != nil {

+ t.Errorf("Next(): got %v, want %v", err, nil)

+ continue

+ }

+ hdrs = append(hdrs, hdr)

+

+ // If a special flag, we should read nothing.

+ cnt, _ := io.ReadFull(tr, []byte{0})

+ if cnt > 0 && hdr.Typeflag != TypeReg {

+ t.Errorf("ReadFull(...): got %d bytes, want 0 bytes", cnt)

+ }

+ }

+

+ // File is crafted with 16 entries. The later 8 are identical to the first

+ // 8 except that the size is set.

+ if len(hdrs) != 16 {

+ t.Fatalf("len(hdrs): got %d, want %d", len(hdrs), 16)

+ for i := 0; i < 8; i++ {

+ var hdr1, hdr2 = hdrs[i+0], hdrs[i+8]

+ hdr1.Size, hdr2.Size = 0, 0

+ if !reflect.DeepEqual(*hdr1, *hdr2) {

+ t.Errorf("incorrect header:\ngot %+v\nwant %+v", *hdr1, *hdr2)

+ }

+func TestParsePAXRecord(t *testing.T) {

+ var medName = strings.Repeat("CD", 50)

+ var longName = strings.Repeat("AB", 100)

+

+ var vectors = []struct {

+ input string

+ residual string

+ outputKey string

+ outputVal string

+ ok bool

+ }{

+ {"6 k=v\n\n", "\n", "k", "v", true},

+ {"19 path=/etc/hosts\n", "", "path", "/etc/hosts", true},

+ {"210 path=" + longName + "\nabc", "abc", "path", longName, true},

+ {"110 path=" + medName + "\n", "", "path", medName, true},

+ {"9 foo=ba\n", "", "foo", "ba", true},

+ {"11 foo=bar\n\x00", "\x00", "foo", "bar", true},

+ {"18 foo=b=\nar=\n==\x00\n", "", "foo", "b=\nar=\n==\x00", true},

+ {"27 foo=hello9 foo=ba\nworld\n", "", "foo", "hello9 foo=ba\nworld", true},

+ {"27 ☺☻☹=日a本b語ç\nmeow mix", "meow mix", "☺☻☹", "日a本b語ç", true},

+ {"17 \x00hello=\x00world\n", "", "\x00hello", "\x00world", true},

+ {"1 k=1\n", "1 k=1\n", "", "", false},

+ {"6 k~1\n", "6 k~1\n", "", "", false},

+ {"6_k=1\n", "6_k=1\n", "", "", false},

+ {"6 k=1 ", "6 k=1 ", "", "", false},

+ {"632 k=1\n", "632 k=1\n", "", "", false},

+ {"16 longkeyname=hahaha\n", "16 longkeyname=hahaha\n", "", "", false},

+ {"3 somelongkey=\n", "3 somelongkey=\n", "", "", false},

+ {"50 tooshort=\n", "50 tooshort=\n", "", "", false},

+

+ for _, v := range vectors {

+ key, val, res, err := parsePAXRecord(v.input)

+ ok := (err == nil)

+ if v.ok != ok {

+ if v.ok {

+ t.Errorf("parsePAXRecord(%q): got parsing failure, want success", v.input)

+ } else {

+ t.Errorf("parsePAXRecord(%q): got parsing success, want failure", v.input)

+ }

+ }

+ if ok && (key != v.outputKey || val != v.outputVal) {

+ t.Errorf("parsePAXRecord(%q): got (%q: %q), want (%q: %q)",

+ v.input, key, val, v.outputKey, v.outputVal)

+ }

+ if res != v.residual {

+ t.Errorf("parsePAXRecord(%q): got residual %q, want residual %q",

+ v.input, res, v.residual)

+ }

+ }

+}

+

+func TestParseNumeric(t *testing.T) {

+ var vectors = []struct {

+ input string

+ output int64

+ ok bool

+ }{

+ // Test base-256 (binary) encoded values.

+ {"", 0, true},

+ {"\x80", 0, true},

+ {"\x80\x00", 0, true},

+ {"\x80\x00\x00", 0, true},

+ {"\xbf", (1 << 6) - 1, true},

+ {"\xbf\xff", (1 << 14) - 1, true},

+ {"\xbf\xff\xff", (1 << 22) - 1, true},

+ {"\xff", -1, true},

+ {"\xff\xff", -1, true},

+ {"\xff\xff\xff", -1, true},

+ {"\xc0", -1 * (1 << 6), true},

+ {"\xc0\x00", -1 * (1 << 14), true},

+ {"\xc0\x00\x00", -1 * (1 << 22), true},

+ {"\x87\x76\xa2\x22\xeb\x8a\x72\x61", 537795476381659745, true},

+ {"\x80\x00\x00\x00\x07\x76\xa2\x22\xeb\x8a\x72\x61", 537795476381659745, true},

+ {"\xf7\x76\xa2\x22\xeb\x8a\x72\x61", -615126028225187231, true},

+ {"\xff\xff\xff\xff\xf7\x76\xa2\x22\xeb\x8a\x72\x61", -615126028225187231, true},

+ {"\x80\x7f\xff\xff\xff\xff\xff\xff\xff", math.MaxInt64, true},

+ {"\x80\x80\x00\x00\x00\x00\x00\x00\x00", 0, false},

+ {"\xff\x80\x00\x00\x00\x00\x00\x00\x00", math.MinInt64, true},

+ {"\xff\x7f\xff\xff\xff\xff\xff\xff\xff", 0, false},

+ {"\xf5\xec\xd1\xc7\x7e\x5f\x26\x48\x81\x9f\x8f\x9b", 0, false},

+

+ // Test base-8 (octal) encoded values.

+ {"0000000\x00", 0, true},

+ {" \x0000000\x00", 0, true},

+ {" \x0000003\x00", 3, true},

+ {"00000000227\x00", 0227, true},

+ {"032033\x00 ", 032033, true},

+ {"320330\x00 ", 0320330, true},

+ {"0000660\x00 ", 0660, true},

+ {"\x00 0000660\x00 ", 0660, true},

+ {"0123456789abcdef", 0, false},

+ {"0123456789\x00abcdef", 0, false},

+ {"01234567\x0089abcdef", 342391, true},

+ {"0123\x7e\x5f\x264123", 0, false},

+ }

+

+ for _, v := range vectors {

+ var p parser

+ num := p.parseNumeric([]byte(v.input))

+ ok := (p.err == nil)

+ if v.ok != ok {

+ if v.ok {

+ t.Errorf("parseNumeric(%q): got parsing failure, want success", v.input)

+ } else {

+ t.Errorf("parseNumeric(%q): got parsing success, want failure", v.input)

+ }

+ }

+ if ok && num != v.output {

+ t.Errorf("parseNumeric(%q): got %d, want %d", v.input, num, v.output)

+ }

+ "sort"

+type formatter struct {

+ err error // Last error seen

+}

+

+func (f *formatter) formatString(b []byte, s string) {

+ f.err = ErrFieldTooLong

+func (f *formatter) formatOctal(b []byte, x int64) {

+ f.formatString(b, s)

+// fitsInBase256 reports whether x can be encoded into n bytes using base-256

+// encoding. Unlike octal encoding, base-256 encoding does not require that the

+// string ends with a NUL character. Thus, all n bytes are available for output.

+//

+// If operating in binary mode, this assumes strict GNU binary mode; which means

+// that the first byte can only be either 0x80 or 0xff. Thus, the first byte is

+// equivalent to the sign bit in two's complement form.

+func fitsInBase256(n int, x int64) bool {

+ var binBits = uint(n-1) * 8

+ return n >= 9 || (x >= -1<<binBits && x < 1<<binBits)

+}

+// Write x into b, as binary (GNUtar/star extension).

+func (f *formatter) formatNumeric(b []byte, x int64) {

+ if fitsInBase256(len(b), x) {

+ for i := len(b) - 1; i >= 0; i-- {

+ b[i] = byte(x)

+ x >>= 8

+ }

+ b[0] |= 0x80 // Highest bit indicates binary format

+ f.formatOctal(b, 0) // Last resort, just write zero

+ f.err = ErrFieldTooLong

+ var f formatter

+ // Wrappers around formatter that automatically sets paxHeaders if the

+ // argument extends beyond the capacity of the input byte slice.

+ var formatString = func(b []byte, s string, paxKeyword string) {

+ needsPaxHeader := paxKeyword != paxNone && len(s) > len(b) || !isASCII(s)

+ if needsPaxHeader {

+ paxHeaders[paxKeyword] = s

+ return

+ }

+ f.formatString(b, s)

+ }

+ var formatNumeric = func(b []byte, x int64, paxKeyword string) {

+ // Try octal first.

+ s := strconv.FormatInt(x, 8)

+ if len(s) < len(b) {

+ f.formatOctal(b, x)

+ return

+ }

+

+ // If it is too long for octal, and PAX is preferred, use a PAX header.

+ if paxKeyword != paxNone && tw.preferPax {

+ f.formatOctal(b, 0)

+ s := strconv.FormatInt(x, 10)

+ paxHeaders[paxKeyword] = s

+ return

+ }

+

+ tw.usedBinary = true

+ f.formatNumeric(b, x)

+ }

+

+ formatString(pathHeaderBytes, hdr.Name, paxPath)

+ f.formatOctal(s.next(8), hdr.Mode) // 100:108

+ formatNumeric(s.next(8), int64(hdr.Uid), paxUid) // 108:116

+ formatNumeric(s.next(8), int64(hdr.Gid), paxGid) // 116:124

+ formatNumeric(s.next(12), hdr.Size, paxSize) // 124:136

+ formatNumeric(s.next(12), modTime, paxNone) // 136:148 --- consider using pax for finer granularity

+ s.next(8) // chksum (148:156)

+ s.next(1)[0] = hdr.Typeflag // 156:157

+ formatString(s.next(100), hdr.Linkname, paxLinkpath)

+ copy(s.next(8), []byte("ustar\x0000")) // 257:265

+ formatString(s.next(32), hdr.Uname, paxUname) // 265:297

+ formatString(s.next(32), hdr.Gname, paxGname) // 297:329

+ formatNumeric(s.next(8), hdr.Devmajor, paxNone) // 329:337

+ formatNumeric(s.next(8), hdr.Devminor, paxNone) // 337:345

+ formatString(prefixHeaderBytes, "", paxNone) // 345:500 prefix

+ prefix, suffix, ok := splitUSTARPath(hdr.Name)

+ if ok {

+ // Since we can encode in USTAR format, disable PAX header.

+ delete(paxHeaders, paxPath)

+

+ // Update the path fields

+ formatString(pathHeaderBytes, suffix, paxNone)

+ formatString(prefixHeaderBytes, prefix, paxNone)

+ f.formatOctal(header[148:155], chksum) // Never fails

+ // Check if there were any formatting errors.

+ if f.err != nil {

+ tw.err = f.err

+// splitUSTARPath splits a path according to USTAR prefix and suffix rules.

+// If the path is not splittable, then it will return ("", "", false).

+func splitUSTARPath(name string) (prefix, suffix string, ok bool) {

+ if length <= fileNameSize || !isASCII(name) {

+ return "", "", false

+ } else if length > fileNamePrefixSize+1 {

+

+ nlen := len(name) - i - 1 // nlen is length of suffix

+ plen := i // plen is length of prefix

+ return "", "", false

+ return name[:i], name[i+1:], true

+ // with the current pid. However, this results in differing outputs

+ // for identical inputs. As such, the constant 0 is now used instead.

+ // golang.org/issue/12358

+ fullName := path.Join(dir, "PaxHeaders.0", file)

+ // Keys are sorted before writing to body to allow deterministic output.

+ var keys []string

+ for k := range paxHeaders {

+ keys = append(keys, k)

+ }

+ sort.Strings(keys)

+

+ for _, k := range keys {

+ fmt.Fprint(&buf, formatPAXRecord(k, paxHeaders[k]))

+// formatPAXRecord formats a single PAX record, prefixing it with the

+// appropriate length.

+func formatPAXRecord(k, v string) string {

+ const padding = 3 // Extra padding for ' ', '=', and '\n'

+ size := len(k) + len(v) + padding

+ record := fmt.Sprintf("%d %s=%s\n", size, k, v)

+

+ // Final adjustment if adding size field increased the record size.

+ record = fmt.Sprintf("%d %s=%s\n", size, k, v)

+ "math"

+ "sort"

+ if !bytes.Contains(buf.Bytes(), []byte("PaxHeaders.0")) {

+ if !bytes.Contains(buf.Bytes(), []byte("PaxHeaders.0")) {

+ if !bytes.Contains(buf.Bytes(), []byte("PaxHeaders.0")) {

+func TestPaxHeadersSorted(t *testing.T) {

+ fileinfo, err := os.Stat("testdata/small.txt")

+ if err != nil {

+ t.Fatal(err)

+ }

+ hdr, err := FileInfoHeader(fileinfo, "")

+ if err != nil {

+ t.Fatalf("os.Stat: %v", err)

+ }

+ contents := strings.Repeat(" ", int(hdr.Size))

+

+ hdr.Xattrs = map[string]string{

+ "foo": "foo",

+ "bar": "bar",

+ "baz": "baz",

+ "qux": "qux",

+ }

+

+ var buf bytes.Buffer

+ writer := NewWriter(&buf)

+ if err := writer.WriteHeader(hdr); err != nil {

+ t.Fatal(err)

+ }

+ if _, err = writer.Write([]byte(contents)); err != nil {

+ t.Fatal(err)

+ }

+ if err := writer.Close(); err != nil {

+ t.Fatal(err)

+ }

+ // Simple test to make sure PAX extensions are in effect

+ if !bytes.Contains(buf.Bytes(), []byte("PaxHeaders.0")) {

+ t.Fatal("Expected at least one PAX header to be written.")

+ }

+

+ // xattr bar should always appear before others

+ indices := []int{

+ bytes.Index(buf.Bytes(), []byte("bar=bar")),

+ bytes.Index(buf.Bytes(), []byte("baz=baz")),

+ bytes.Index(buf.Bytes(), []byte("foo=foo")),

+ bytes.Index(buf.Bytes(), []byte("qux=qux")),

+ }

+ if !sort.IntsAreSorted(indices) {

+ t.Fatal("PAX headers are not sorted")

+

+func TestSplitUSTARPath(t *testing.T) {

+ var sr = strings.Repeat

+

+ var vectors = []struct {

+ input string // Input path

+ prefix string // Expected output prefix

+ suffix string // Expected output suffix

+ ok bool // Split success?

+ }{

+ {"", "", "", false},

+ {"abc", "", "", false},

+ {"用戶名", "", "", false},

+ {sr("a", fileNameSize), "", "", false},

+ {sr("a", fileNameSize) + "/", "", "", false},

+ {sr("a", fileNameSize) + "/a", sr("a", fileNameSize), "a", true},

+ {sr("a", fileNamePrefixSize) + "/", "", "", false},

+ {sr("a", fileNamePrefixSize) + "/a", sr("a", fileNamePrefixSize), "a", true},

+ {sr("a", fileNameSize+1), "", "", false},

+ {sr("/", fileNameSize+1), sr("/", fileNameSize-1), "/", true},

+ {sr("a", fileNamePrefixSize) + "/" + sr("b", fileNameSize),

+ sr("a", fileNamePrefixSize), sr("b", fileNameSize), true},

+ {sr("a", fileNamePrefixSize) + "//" + sr("b", fileNameSize), "", "", false},

+ {sr("a/", fileNameSize), sr("a/", 77) + "a", sr("a/", 22), true},

+ }

+

+ for _, v := range vectors {

+ prefix, suffix, ok := splitUSTARPath(v.input)

+ if prefix != v.prefix || suffix != v.suffix || ok != v.ok {

+ t.Errorf("splitUSTARPath(%q):\ngot (%q, %q, %v)\nwant (%q, %q, %v)",

+ v.input, prefix, suffix, ok, v.prefix, v.suffix, v.ok)

+ }

+ }

+}

+

+func TestFormatPAXRecord(t *testing.T) {

+ var medName = strings.Repeat("CD", 50)

+ var longName = strings.Repeat("AB", 100)

+

+ var vectors = []struct {

+ inputKey string

+ inputVal string

+ output string

+ }{

+ {"k", "v", "6 k=v\n"},

+ {"path", "/etc/hosts", "19 path=/etc/hosts\n"},

+ {"path", longName, "210 path=" + longName + "\n"},

+ {"path", medName, "110 path=" + medName + "\n"},

+ {"foo", "ba", "9 foo=ba\n"},

+ {"foo", "bar", "11 foo=bar\n"},

+ {"foo", "b=\nar=\n==\x00", "18 foo=b=\nar=\n==\x00\n"},

+ {"foo", "hello9 foo=ba\nworld", "27 foo=hello9 foo=ba\nworld\n"},

+ {"☺☻☹", "日a本b語ç", "27 ☺☻☹=日a本b語ç\n"},

+ {"\x00hello", "\x00world", "17 \x00hello=\x00world\n"},

+ }

+

+ for _, v := range vectors {

+ output := formatPAXRecord(v.inputKey, v.inputVal)

+ if output != v.output {

+ t.Errorf("formatPAXRecord(%q, %q): got %q, want %q",

+ v.inputKey, v.inputVal, output, v.output)

+ }

+ }

+}

+

+func TestFitsInBase256(t *testing.T) {

+ var vectors = []struct {

+ input int64

+ width int

+ ok bool

+ }{

+ {+1, 8, true},

+ {0, 8, true},

+ {-1, 8, true},

+ {1 << 56, 8, false},

+ {(1 << 56) - 1, 8, true},

+ {-1 << 56, 8, true},

+ {(-1 << 56) - 1, 8, false},

+ {121654, 8, true},

+ {-9849849, 8, true},

+ {math.MaxInt64, 9, true},

+ {0, 9, true},

+ {math.MinInt64, 9, true},

+ {math.MaxInt64, 12, true},

+ {0, 12, true},

+ {math.MinInt64, 12, true},

+ }

+

+ for _, v := range vectors {

+ ok := fitsInBase256(v.width, v.input)

+ if ok != v.ok {

+ t.Errorf("checkNumeric(%d, %d): got %v, want %v", v.input, v.width, ok, v.ok)

+ }

+ }

+}

+

+func TestFormatNumeric(t *testing.T) {

+ var vectors = []struct {

+ input int64

+ output string

+ ok bool

+ }{

+ // Test base-256 (binary) encoded values.

+ {-1, "\xff", true},

+ {-1, "\xff\xff", true},

+ {-1, "\xff\xff\xff", true},

+ {(1 << 0), "0", false},

+ {(1 << 8) - 1, "\x80\xff", true},

+ {(1 << 8), "0\x00", false},

+ {(1 << 16) - 1, "\x80\xff\xff", true},

+ {(1 << 16), "00\x00", false},

+ {-1 * (1 << 0), "\xff", true},

+ {-1*(1<<0) - 1, "0", false},

+ {-1 * (1 << 8), "\xff\x00", true},

+ {-1*(1<<8) - 1, "0\x00", false},

+ {-1 * (1 << 16), "\xff\x00\x00", true},

+ {-1*(1<<16) - 1, "00\x00", false},

+ {537795476381659745, "0000000\x00", false},

+ {537795476381659745, "\x80\x00\x00\x00\x07\x76\xa2\x22\xeb\x8a\x72\x61", true},

+ {-615126028225187231, "0000000\x00", false},

+ {-615126028225187231, "\xff\xff\xff\xff\xf7\x76\xa2\x22\xeb\x8a\x72\x61", true},

+ {math.MaxInt64, "0000000\x00", false},

+ {math.MaxInt64, "\x80\x00\x00\x00\x7f\xff\xff\xff\xff\xff\xff\xff", true},

+ {math.MinInt64, "0000000\x00", false},

+ {math.MinInt64, "\xff\xff\xff\xff\x80\x00\x00\x00\x00\x00\x00\x00", true},

+ {math.MaxInt64, "\x80\x7f\xff\xff\xff\xff\xff\xff\xff", true},

+ {math.MinInt64, "\xff\x80\x00\x00\x00\x00\x00\x00\x00", true},

+ }

+

+ for _, v := range vectors {

+ var f formatter

+ output := make([]byte, len(v.output))

+ f.formatNumeric(output, v.input)

+ ok := (f.err == nil)

+ if ok != v.ok {

+ if v.ok {

+ t.Errorf("formatNumeric(%d): got formatting failure, want success", v.input)

+ } else {

+ t.Errorf("formatNumeric(%d): got formatting success, want failure", v.input)

+ }

+ }

+ if string(output) != v.output {

+ t.Errorf("formatNumeric(%d): got %q, want %q", v.input, output, v.output)

+ }

+ }

+}

+ r io.ReaderAt

+ File []*File

+ Comment string

+ decompressors map[uint16]Decompressor

+ zip *Reader

+ f := &File{zip: z, zipr: r, zipsize: size}

+// RegisterDecompressor registers or overrides a custom decompressor for a

+// specific method ID. If a decompressor for a given method is not found,

+// Reader will default to looking up the decompressor at the package level.

+func (z *Reader) RegisterDecompressor(method uint16, dcomp Decompressor) {

+ if z.decompressors == nil {

+ z.decompressors = make(map[uint16]Decompressor)

+ }

+ z.decompressors[method] = dcomp

+}

+

+func (z *Reader) decompressor(method uint16) Decompressor {

+ dcomp := z.decompressors[method]

+ if dcomp == nil {

+ dcomp = decompressor(method)

+ }

+ return dcomp

+}

+

+ dcomp := f.zip.decompressor(f.Method)

+ needUSize := f.UncompressedSize == ^uint32(0)

+ needCSize := f.CompressedSize == ^uint32(0)

+ needHeaderOffset := f.headerOffset == int64(^uint32(0))

+

+ // Best effort to find what we need.

+ // Other zip authors might not even follow the basic format,

+ // and we'll just ignore the Extra content in that case.

+ break

+ // update directory values from the zip64 extra block.

+ // They should only be consulted if the sizes read earlier

+ // are maxed out.

+ // See golang.org/issue/13367.

+

+ if needUSize {

+ needUSize = false

+ if len(eb) < 8 {

+ return ErrFormat

+ }

+ if needCSize {

+ needCSize = false

+ if len(eb) < 8 {

+ return ErrFormat

+ }

+ if needHeaderOffset {

+ needHeaderOffset = false

+ if len(eb) < 8 {

+ return ErrFormat

+ }

+ break

+

+ if needUSize || needCSize || needHeaderOffset {

+ return ErrFormat

+ }

+

+ // These values mean that the file can be a zip64 file

+ if d.directoryRecords == 0xffff || d.directorySize == 0xffff || d.directoryOffset == 0xffffffff {

+ p, err := findDirectory64End(r, directoryEndOffset)

+ if err == nil && p >= 0 {

+ err = readDirectory64End(r, p, d)

+ }

+ if err != nil {

+ return nil, err

+ }

+ if b.uint32() != 0 { // number of the disk with the start of the zip64 end of central directory

+ return -1, nil // the file is not a valid zip64-file

+ }

+ p := b.uint64() // relative offset of the zip64 end of central directory record

+ if b.uint32() != 1 { // total number of disks

+ return -1, nil // the file is not a valid zip64-file

+ }

+

+// Verify we do not treat non-zip64 archives as zip64

+func TestIssue12449(t *testing.T) {

+ data := []byte{

+ 0x50, 0x4b, 0x03, 0x04, 0x14, 0x00, 0x08, 0x00,

+ 0x00, 0x00, 0x6b, 0xb4, 0xba, 0x46, 0x00, 0x00,

+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,

+ 0x00, 0x00, 0x03, 0x00, 0x18, 0x00, 0xca, 0x64,

+ 0x55, 0x75, 0x78, 0x0b, 0x00, 0x50, 0x4b, 0x05,

+ 0x06, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x01,

+ 0x00, 0x49, 0x00, 0x00, 0x00, 0x44, 0x00, 0x00,

+ 0x00, 0x31, 0x31, 0x31, 0x32, 0x32, 0x32, 0x0a,

+ 0x50, 0x4b, 0x07, 0x08, 0x1d, 0x88, 0x77, 0xb0,

+ 0x07, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00,

+ 0x50, 0x4b, 0x01, 0x02, 0x14, 0x03, 0x14, 0x00,

+ 0x08, 0x00, 0x00, 0x00, 0x6b, 0xb4, 0xba, 0x46,

+ 0x1d, 0x88, 0x77, 0xb0, 0x07, 0x00, 0x00, 0x00,

+ 0x07, 0x00, 0x00, 0x00, 0x03, 0x00, 0x18, 0x00,

+ 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,

+ 0xa0, 0x81, 0x00, 0x00, 0x00, 0x00, 0xca, 0x64,

+ 0x55, 0x75, 0x78, 0x0b, 0x00, 0x50, 0x4b, 0x05,

+ 0x06, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x01,

+ 0x00, 0x49, 0x00, 0x00, 0x00, 0x44, 0x00, 0x00,

+ 0x00, 0x97, 0x2b, 0x49, 0x23, 0x05, 0xc5, 0x0b,

+ 0xa7, 0xd1, 0x52, 0xa2, 0x9c, 0x50, 0x4b, 0x06,

+ 0x07, 0xc8, 0x19, 0xc1, 0xaf, 0x94, 0x9c, 0x61,

+ 0x44, 0xbe, 0x94, 0x19, 0x42, 0x58, 0x12, 0xc6,

+ 0x5b, 0x50, 0x4b, 0x05, 0x06, 0x00, 0x00, 0x00,

+ 0x00, 0x01, 0x00, 0x01, 0x00, 0x69, 0x00, 0x00,

+ 0x00, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00,

+ }

+ // Read in the archive.

+ _, err := NewReader(bytes.NewReader([]byte(data)), int64(len(data)))

+ if err != nil {

+ t.Errorf("Error reading the archive: %v", err)

+ }

+}

+// A Compressor returns a new compressing writer, writing to w.

+// The WriteCloser's Close method must be used to flush pending data to w.

+// The Compressor itself must be safe to invoke from multiple goroutines

+// simultaneously, but each returned writer will be used only by

+// one goroutine at a time.

+type Compressor func(w io.Writer) (io.WriteCloser, error)

+

+// A Decompressor returns a new decompressing reader, reading from r.

+// The ReadCloser's Close method must be used to release associated resources.

+// The Decompressor itself must be safe to invoke from multiple goroutines

+// simultaneously, but each returned reader will be used only by

+// one goroutine at a time.

+type Decompressor func(r io.Reader) io.ReadCloser

+// The common methods Store and Deflate are built in.

+func RegisterDecompressor(method uint16, dcomp Decompressor) {

+ decompressors[method] = dcomp

+ return fh.CompressedSize64 >= uint32max || fh.UncompressedSize64 >= uint32max

+ cw *countWriter

+ dir []*header

+ last *fileWriter

+ closed bool

+ compressors map[uint16]Compressor

+ if h.isZip64() || h.offset >= uint32max {

+ comp := w.compressor(fh.Method)

+// RegisterCompressor registers or overrides a custom compressor for a specific

+// method ID. If a compressor for a given method is not found, Writer will

+// default to looking up the compressor at the package level.

+func (w *Writer) RegisterCompressor(method uint16, comp Compressor) {

+ if w.compressors == nil {

+ w.compressors = make(map[uint16]Compressor)

+ }

+ w.compressors[method] = comp

+}

+

+func (w *Writer) compressor(method uint16) Compressor {

+ comp := w.compressors[method]

+ if comp == nil {

+ comp = compressor(method)

+ }

+ return comp

+}

+

+ "internal/testenv"

+ if testing.Short() && testenv.Builder() == "" {

+ t.Skip("skipping in short mode")

+ }

+func TestZip64EdgeCase(t *testing.T) {

+ if testing.Short() {

+ t.Skip("slow test; skipping")

+ }

+ // Test a zip file with uncompressed size 0xFFFFFFFF.

+ // That's the magic marker for a 64-bit file, so even though

+ // it fits in a 32-bit field we must use the 64-bit field.

+ // Go 1.5 and earlier got this wrong,

+ // writing an invalid zip file.

+ const size = 1<<32 - 1 - int64(len("END\n")) // before the "END\n" part

+ buf := testZip64(t, size)

+ testZip64DirectoryRecordLength(buf, t)

+}

+

+ // write size bytes plus "END\n" to a zip file

+ if frag := int(size % chunkSize); frag > 0 {

+ _, err := f.Write(chunk[:frag])

+ if err != nil {

+ t.Fatal("write chunk:", err)

+ }

+ }

+ if frag := int(size % chunkSize); frag > 0 {

+ _, err := io.ReadFull(rc, chunk[:frag])

+ if err != nil {

+ t.Fatal("read:", err)

+ }

+ }

+ if size+int64(len("END\n")) >= 1<<32-1 {

+ t.Errorf("UncompressedSize %#x, want %#x", got, want)

+ t.Errorf("UncompressedSize64 %#x, want %#x", got, want)

+ zf, err := NewReader(bytes.NewReader(b), int64(len(b)))

+ if err != nil {

+ zh := zf.File[0].FileHeader

+ if zh.Name != h.Name || zh.Method != h.Method || zh.UncompressedSize64 != uint64(len("hi")) {

+ t.Fatalf("got %q/%d/%d expected %q/%d/%d", zh.Name, zh.Method, zh.UncompressedSize64, h.Name, h.Method, len("hi"))

+ }

+ Extra: []byte(ts.Format(time.RFC3339Nano)), // missing tag and len, but Extra is best-effort parsing

+ testValidHeader(&h, t)

+ Extra: []byte{zip64ExtraId}, // missing size and second half of tag, but Extra is best-effort parsing

+ testValidHeader(&h, t)

+}

+

+func TestHeaderIgnoredSize(t *testing.T) {

+ h := FileHeader{

+ Name: "foo.txt",

+ Method: Deflate,

+ Extra: []byte{zip64ExtraId & 0xFF, zip64ExtraId >> 8, 24, 0, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8}, // bad size but shouldn't be consulted

+ }

+ testValidHeader(&h, t)