Use circular buffer to reduce data movements.
The CL also increases size of bigData to make changes of benchmark results apparent.
goos: linux
goarch: amd64
pkg: internal/zstd
│ /tmp/BenchmarkLarge.old │ /tmp/BenchmarkLarge.new │
│ sec/op │ sec/op vs base │
Large-8 12.672m ± 1% 9.521m ± 0% -24.87% (p=0.000 n=10)
│ /tmp/BenchmarkLarge.old │ /tmp/BenchmarkLarge.new │
│ B/s │ B/s vs base │
Large-8 13.43Mi ± 1% 17.88Mi ± 0% +33.08% (p=0.000 n=10)
│ /tmp/BenchmarkLarge.old │ /tmp/BenchmarkLarge.new │
│ B/op │ B/op vs base │
Large-8 58.23Ki ± 5% 41.72Ki ± 1% -28.35% (p=0.000 n=10)
│ /tmp/BenchmarkLarge.old │ /tmp/BenchmarkLarge.new │
│ allocs/op │ allocs/op vs base │
Large-8 0.000 ± 0% 0.000 ± 0% ~ (p=1.000 n=10)
Change-Id: Ic03fabfc575c5e6d18bcd5ba1c845aa502c12497
GitHub-Last-Rev:
16cb1e13ff61f0fc8c9209cb034f31ec6c37f596
GitHub-Pull-Request: golang/go#62625
Reviewed-on: https://go-review.googlesource.com/c/go/+/528318
Reviewed-by: Ian Lance Taylor <iant@google.com>
Reviewed-by: Matthew Dempsky <mdempsky@google.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Auto-Submit: Ian Lance Taylor <iant@google.com>
lenBlock := uint32(len(r.buffer))
if lenBlock < offset {
- lenWindow := uint32(len(r.window))
+ lenWindow := r.window.len()
windowOffset := offset - lenBlock
if windowOffset > lenWindow {
return rbr.makeError("offset past window")
}
from := lenWindow - windowOffset
if from+match <= lenWindow {
- r.buffer = append(r.buffer, r.window[from:from+match]...)
+ r.buffer = r.window.appendTo(r.buffer, from, from+match)
return nil
}
- r.buffer = append(r.buffer, r.window[from:]...)
+ r.buffer = r.window.appendTo(r.buffer, from, lenWindow)
copied := lenWindow - from
offset -= copied
match -= copied
--- /dev/null
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package zstd
+
+// window stores up to size bytes of data.
+// It is implemented as a circular buffer:
+// sequential save calls append to the data slice until
+// its length reaches configured size and after that,
+// save calls overwrite previously saved data at off
+// and update off such that it always points at
+// the byte stored before others.
+type window struct {
+ size int
+ data []byte
+ off int
+}
+
+// reset clears stored data and configures window size.
+func (w *window) reset(size int) {
+ w.data = w.data[:0]
+ w.off = 0
+ w.size = size
+}
+
+// len returns the number of stored bytes.
+func (w *window) len() uint32 {
+ return uint32(len(w.data))
+}
+
+// save stores up to size last bytes from the buf.
+func (w *window) save(buf []byte) {
+ if w.size == 0 {
+ return
+ }
+ if len(buf) == 0 {
+ return
+ }
+
+ if len(buf) >= w.size {
+ from := len(buf) - w.size
+ w.data = append(w.data[:0], buf[from:]...)
+ w.off = 0
+ return
+ }
+
+ // Update off to point to the oldest remaining byte.
+ free := w.size - len(w.data)
+ if free == 0 {
+ n := copy(w.data[w.off:], buf)
+ if n == len(buf) {
+ w.off += n
+ } else {
+ w.off = copy(w.data, buf[n:])
+ }
+ } else {
+ if free >= len(buf) {
+ w.data = append(w.data, buf...)
+ } else {
+ w.data = append(w.data, buf[:free]...)
+ w.off = copy(w.data, buf[free:])
+ }
+ }
+}
+
+// appendTo appends stored bytes between from and to indices to the buf.
+// Index from must be less or equal to index to and to must be less or equal to w.len().
+func (w *window) appendTo(buf []byte, from, to uint32) []byte {
+ dataLen := uint32(len(w.data))
+ from += uint32(w.off)
+ to += uint32(w.off)
+
+ wrap := false
+ if from > dataLen {
+ from -= dataLen
+ wrap = !wrap
+ }
+ if to > dataLen {
+ to -= dataLen
+ wrap = !wrap
+ }
+
+ if wrap {
+ buf = append(buf, w.data[from:]...)
+ return append(buf, w.data[:to]...)
+ } else {
+ return append(buf, w.data[from:to]...)
+ }
+}
--- /dev/null
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package zstd
+
+import (
+ "bytes"
+ "fmt"
+ "testing"
+)
+
+func makeSequence(start, n int) (seq []byte) {
+ for i := 0; i < n; i++ {
+ seq = append(seq, byte(start+i))
+ }
+ return
+}
+
+func TestWindow(t *testing.T) {
+ for size := 0; size <= 3; size++ {
+ for i := 0; i <= 2*size; i++ {
+ a := makeSequence('a', i)
+ for j := 0; j <= 2*size; j++ {
+ b := makeSequence('a'+i, j)
+ for k := 0; k <= 2*size; k++ {
+ c := makeSequence('a'+i+j, k)
+
+ t.Run(fmt.Sprintf("%d-%d-%d-%d", size, i, j, k), func(t *testing.T) {
+ testWindow(t, size, a, b, c)
+ })
+ }
+ }
+ }
+ }
+}
+
+// testWindow tests window by saving three sequences of bytes to it.
+// Third sequence tests read offset that can become non-zero only after second save.
+func testWindow(t *testing.T, size int, a, b, c []byte) {
+ var w window
+ w.reset(size)
+
+ w.save(a)
+ w.save(b)
+ w.save(c)
+
+ var tail []byte
+ tail = append(tail, a...)
+ tail = append(tail, b...)
+ tail = append(tail, c...)
+
+ if len(tail) > size {
+ tail = tail[len(tail)-size:]
+ }
+
+ if w.len() != uint32(len(tail)) {
+ t.Errorf("wrong data length: got: %d, want: %d", w.len(), len(tail))
+ }
+
+ var from, to uint32
+ for from = 0; from <= uint32(len(tail)); from++ {
+ for to = from; to <= uint32(len(tail)); to++ {
+ got := w.appendTo(nil, from, to)
+ want := tail[from:to]
+
+ if !bytes.Equal(got, want) {
+ t.Errorf("wrong data at [%d:%d]: got %q, want %q", from, to, got, want)
+ }
+ }
+ }
+}
huffmanTableBits int
// The window for back references.
- windowSize int // maximum required window size
- window []byte // window data
+ window window
// A buffer available to hold a compressed block.
compressedBuf []byte
// repeatedOffset3
// huffmanTable
// huffmanTableBits
- // windowSize
// window
// compressedBuf
// literals
// Figure out the maximum amount of data we need to retain
// for backreferences.
-
+ var windowSize int
if singleSegment {
// No window required, as all the data is in a single buffer.
- r.windowSize = 0
+ windowSize = 0
} else {
// Window descriptor. RFC 3.1.1.1.2.
windowDescriptor := r.scratch[0]
windowLog := exponent + 10
windowBase := uint64(1) << windowLog
windowAdd := (windowBase / 8) * mantissa
- windowSize := windowBase + windowAdd
+ windowSize = int(windowBase + windowAdd)
// Default zstd sets limits on the window size.
if fuzzing && (windowLog > 31 || windowSize > 1<<27) {
if windowSize > 8<<20 {
windowSize = 8 << 20
}
-
- r.windowSize = int(windowSize)
}
// Frame_Content_Size. RFC 3.1.1.4.
r.repeatedOffset2 = 4
r.repeatedOffset3 = 8
r.huffmanTableBits = 0
- r.window = r.window[:0]
+ r.window.reset(windowSize)
r.seqTables[0] = nil
r.seqTables[1] = nil
r.seqTables[2] = nil
// Maximum block size is smaller of window size and 128K.
// We don't record the window size for a single segment frame,
// so just use 128K. RFC 3.1.1.2.3, 3.1.1.2.4.
- if blockSize > 128<<10 || (r.windowSize > 0 && blockSize > r.windowSize) {
+ if blockSize > 128<<10 || (r.window.size > 0 && blockSize > r.window.size) {
return r.makeError(relativeOffset, "block size too large")
}
}
if !lastBlock {
- r.saveWindow(r.buffer)
+ r.window.save(r.buffer)
} else {
if !r.frameSizeUnknown && r.remainingFrameSize != 0 {
return r.makeError(relativeOffset, "not enough uncompressed bytes for frame")
r.buffer = r.buffer[:size]
}
-// saveWindow saves bytes in the backreference window.
-// TODO: use a circular buffer for less data movement.
-func (r *Reader) saveWindow(buf []byte) {
- if r.windowSize == 0 {
- return
- }
-
- if len(buf) >= r.windowSize {
- from := len(buf) - r.windowSize
- r.window = append(r.window[:0], buf[from:]...)
- return
- }
-
- keep := r.windowSize - len(buf) // must be positive
- if keep < len(r.window) {
- remove := len(r.window) - keep
- copy(r.window[:], r.window[remove:])
- r.window = r.window[:keep]
- }
-
- r.window = append(r.window, buf...)
-}
-
// zstdError is an error while decompressing.
type zstdError struct {
offset int64
bigDataOnce.Do(func() {
bigDataBytes, bigDataErr = os.ReadFile("../../testdata/Isaac.Newton-Opticks.txt")
if bigDataErr == nil {
- bigDataBytes = bytes.Repeat(bigDataBytes, 3)
+ bigDataBytes = bytes.Repeat(bigDataBytes, 20)
}
})
if bigDataErr != nil {