compress/flate: fix overflow on 2GB input. Reset hashOffset every 16 MB.

This bug has been introduced in the following revision:

changeset:   11404:26dceba5c610
user:        Ivan Krasin <krasin@golang.org>
date:        Mon Jan 23 09:19:39 2012 -0500
summary:     compress/flate: reduce memory pressure at cost of additional arithmetic operation.

This is the review page for that CL: https://golang.org/cl/5555070/

R=rsc, imkrasin
CC=golang-dev
https://golang.org/cl/6249067

diff --git a/src/pkg/compress/flate/deflate.go b/src/pkg/compress/flate/deflate.go
index 20408409c8ecb4ab4b416c2558b75af6c163d186..e511b50fd1b079066382eaf59b5de29c53a978e8 100644 (file)
--- a/src/pkg/compress/flate/deflate.go
+++ b/src/pkg/compress/flate/deflate.go
@@ -32,6 +32,7 @@ const (
        hashSize            = 1 << hashBits
        hashMask            = (1 << hashBits) - 1
        hashShift           = (hashBits + minMatchLength - 1) / minMatchLength
+       maxHashOffset       = 1 << 24
 
        skipNever = math.MaxInt32
 )
@@ -106,6 +107,25 @@ func (d *compressor) fillDeflate(b []byte) int {
                        d.blockStart = math.MaxInt32
                }
                d.hashOffset += windowSize
+               if d.hashOffset > maxHashOffset {
+                       delta := d.hashOffset - 1
+                       d.hashOffset -= delta
+                       d.chainHead -= delta
+                       for i, v := range d.hashPrev {
+                               if v > delta {
+                                       d.hashPrev[i] -= delta
+                               } else {
+                                       d.hashPrev[i] = 0
+                               }
+                       }
+                       for i, v := range d.hashHead {
+                               if v > delta {
+                                       d.hashHead[i] -= delta
+                               } else {
+                                       d.hashHead[i] = 0
+                               }
+                       }
+               }
        }
        n := copy(d.window[d.windowEnd:], b)
        d.windowEnd += n

diff --git a/src/pkg/compress/flate/deflate_test.go b/src/pkg/compress/flate/deflate_test.go
index 267366772d69dada63d763e192731a87df176798..e0b225e298835aeb2563dbdff851c7057f5562f9 100644 (file)
--- a/src/pkg/compress/flate/deflate_test.go
+++ b/src/pkg/compress/flate/deflate_test.go
@@ -94,6 +94,50 @@ func TestDeflate(t *testing.T) {
        }
 }
 
+// A sparseReader returns a stream consisting of 0s followed by 1<<16 1s.
+// This tests missing hash references in a very large input.
+type sparseReader struct {
+       l   int64
+       cur int64
+}
+
+func (r *sparseReader) Read(b []byte) (n int, err error) {
+       if r.cur >= r.l {
+               return 0, io.EOF
+       }
+       n = len(b)
+       cur := r.cur + int64(n)
+       if cur > r.l {
+               n -= int(cur - r.l)
+               cur = r.l
+       }
+       for i := range b[0:n] {
+               if r.cur+int64(i) >= r.l-1<<16 {
+                       b[i] = 1
+               } else {
+                       b[i] = 0
+               }
+       }
+       r.cur = cur
+       return
+}
+
+func TestVeryLongSparseChunk(t *testing.T) {
+       if testing.Short() {
+               t.Logf("skipping sparse chunk during short test")
+               return
+       }
+       w, err := NewWriter(ioutil.Discard, 1)
+       if err != nil {
+               t.Errorf("NewWriter: %v", err)
+               return
+       }
+       if _, err = io.Copy(w, &sparseReader{l: 23E8}); err != nil {
+               t.Errorf("Compress failed: %v", err)
+               return
+       }
+}
+
 type syncBuffer struct {
        buf    bytes.Buffer
        mu     sync.RWMutex