From 4a310877f23140b45bd86a45b8c1fc7d69ba2d29 Mon Sep 17 00:00:00 2001
From: Alexander Yastrebov <yastrebov.alex@gmail.com>
Date: Wed, 27 Sep 2023 01:53:38 +0000
Subject: [PATCH] internal/zstd: configure window size for single segment
 frames

Set window size to frame content size when single segment flag is set.

For #62513

Change-Id: I2a60c33123aca4f6a631e6d625f4582ff31a63cb
GitHub-Last-Rev: 9bafe01e45aad6a9f22abca08b25b2b8d9107040
GitHub-Pull-Request: golang/go#63224
Reviewed-on: https://go-review.googlesource.com/c/go/+/531075
Auto-Submit: Ian Lance Taylor <iant@google.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Ian Lance Taylor <iant@google.com>
Reviewed-by: Bryan Mills <bcmills@google.com>
---
 .../testdata/1890a371.gettysburg.txt-100x.zst | Bin 0 -> 826 bytes
 src/internal/zstd/testdata/README             |  10 +++++
 src/internal/zstd/zstd.go                     |  22 ++++++-----
 src/internal/zstd/zstd_test.go                |  35 ++++++++++++++++++
 4 files changed, 58 insertions(+), 9 deletions(-)
 create mode 100644 src/internal/zstd/testdata/1890a371.gettysburg.txt-100x.zst
 create mode 100644 src/internal/zstd/testdata/README

diff --git a/src/internal/zstd/testdata/1890a371.gettysburg.txt-100x.zst b/src/internal/zstd/testdata/1890a371.gettysburg.txt-100x.zst
new file mode 100644
index 0000000000000000000000000000000000000000..afb4a2769b638989740ed52b9186e457abcc9c9c
GIT binary patch
literal 826
zcmV-A1I7F(wJ-gouv`KF{1^ZR`ivh?4VwT!gpv~anb2El?b2SA$MLc4BrJB*6=IR1
zGzWhGf&ha6navA4YaYGYtHuegZC@k32(!hk1n*rFUvdfljhl3^PXN1R;!fYY9=@LO
ziWxiasWs->)l!kF@YOo^r0*RoJ)y<wjn!aWPG{B+%)<zC?V+2a&uH<u7V~ghY7eH>
zY9d~hEUAW@UhN^)C7sE+>3v~5^X7!-YaY$Iq=Fp?ulUTV;VTIH1SDKi69c$O#!gv}
zH_YPzcoW|kz2Teni5^3{AlN6MEw#?%#FkXeZxY@vB*|sxw!92>z^Y^^Zf6#=uB-K#
zOMJa@s>W}r)|!Za98>6FhYRt!osp)(=k=95eCCv1V<er}7Xom$dq#S%MA#?bx+MHb
zFq!cVqlOb)xK6oRqTOEY$#qIU@R)R7FeC9~Yl~XMsTy@G)mTEaak^>?7lO<24d)uE
zQCqBuoPd?QxY~W9iCE8kmv=GF;+o%BZzxx;)<mOkqR)00Gh?f9Vw#PwAWpy@r~710
zr*zJ>zDb2OQ*kDZvmRo!%jwK1qjZ)ei*7^do|wfP=N2Br77g(aMu{#)TkL+pXn>$V
zz{o&=Ne3fjdEjcWa_ZDLI43}@-rbgJORrcMMrVq=!e6|CdL|P9E17;pj4b1zlnd4;
zTyPz;(mBJ}spdD9?dk<n$x@9E0Fu5g?H0t_j`_6Z4(}!N<n(se1D7vx>o-pKbka33
zmEW$`N-E;Lw)mRJ174_c5n+ZYL?WahDM<nc5Jr%L&<+v^=c#Av#{U2R8;j3>F#V(A
z#6?LWAgc4<2r=t_jv(7<`$$|;XwV+pVcvg;Wu6q9U}=$yN1$W?nMY9`p#WLaBi{wF
zu)}O2!$V+Tg+i)8#}ZxOEJQ3IS&Gs>uor+HxIbj|TN@YWn4VZZwke~%1<f}i7#nNC
za&K)Kzy!GF9(99Kt<jr3&Y&RhmQJS1>4pVbPke&zaUp1Utr_R!pPU4w77%DZAq>`k
z%5c1{0GvIFP5s)AI$)u~I4Z~>>zV(U6QV`rfiIP?cvLakeN{yO00<xf0Ib{&gj{W+
EI$2(o82|tP

literal 0
HcmV?d00001

diff --git a/src/internal/zstd/testdata/README b/src/internal/zstd/testdata/README
new file mode 100644
index 0000000000..1a6dbb3a8f
--- /dev/null
+++ b/src/internal/zstd/testdata/README
@@ -0,0 +1,10 @@
+This directory holds files for testing zstd.NewReader.
+
+Each one is a Zstandard compressed file named as hash.arbitrary-name.zst,
+where hash is the first eight hexadecimal digits of the SHA256 hash
+of the expected uncompressed content:
+
+	zstd -d < 1890a371.gettysburg.txt-100x.zst | sha256sum | head -c 8
+	1890a371
+
+The test uses hash value to verify decompression result.
diff --git a/src/internal/zstd/zstd.go b/src/internal/zstd/zstd.go
index 60551a4371..1a7a0a381b 100644
--- a/src/internal/zstd/zstd.go
+++ b/src/internal/zstd/zstd.go
@@ -235,10 +235,7 @@ retry:
 	// Figure out the maximum amount of data we need to retain
 	// for backreferences.
 	var windowSize int
-	if singleSegment {
-		// No window required, as all the data is in a single buffer.
-		windowSize = 0
-	} else {
+	if !singleSegment {
 		// Window descriptor. RFC 3.1.1.1.2.
 		windowDescriptor := r.scratch[0]
 		exponent := uint64(windowDescriptor >> 3)
@@ -252,11 +249,6 @@ retry:
 		if fuzzing && (windowLog > 31 || windowSize > 1<<27) {
 			return r.makeError(relativeOffset, "windowSize too large")
 		}
-
-		// RFC 8878 permits us to set an 8M max on window size.
-		if windowSize > 8<<20 {
-			windowSize = 8 << 20
-		}
 	}
 
 	// Frame_Content_Size. RFC 3.1.1.4.
@@ -278,6 +270,18 @@ retry:
 		panic("unreachable")
 	}
 
+	// RFC 3.1.1.1.2.
+	// When Single_Segment_Flag is set, Window_Descriptor is not present.
+	// In this case, Window_Size is Frame_Content_Size.
+	if singleSegment {
+		windowSize = int(r.remainingFrameSize)
+	}
+
+	// RFC 8878 3.1.1.1.1.2. permits us to set an 8M max on window size.
+	if windowSize > 8<<20 {
+		windowSize = 8 << 20
+	}
+
 	relativeOffset += headerSize
 
 	r.sawFrameHeader = true
diff --git a/src/internal/zstd/zstd_test.go b/src/internal/zstd/zstd_test.go
index c2914bb1f4..e5c964c307 100644
--- a/src/internal/zstd/zstd_test.go
+++ b/src/internal/zstd/zstd_test.go
@@ -6,12 +6,14 @@ package zstd
 
 import (
 	"bytes"
+	"crypto/sha256"
 	"fmt"
 	"internal/race"
 	"internal/testenv"
 	"io"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"strings"
 	"sync"
 	"testing"
@@ -237,6 +239,39 @@ func TestAlloc(t *testing.T) {
 	}
 }
 
+func TestFileSamples(t *testing.T) {
+	samples, err := os.ReadDir("testdata")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, sample := range samples {
+		name := sample.Name()
+		if !strings.HasSuffix(name, ".zst") {
+			continue
+		}
+
+		t.Run(name, func(t *testing.T) {
+			f, err := os.Open(filepath.Join("testdata", name))
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			r := NewReader(f)
+			h := sha256.New()
+			if _, err := io.Copy(h, r); err != nil {
+				t.Fatal(err)
+			}
+			got := fmt.Sprintf("%x", h.Sum(nil))[:8]
+
+			want, _, _ := strings.Cut(name, ".")
+			if got != want {
+				t.Errorf("Wrong uncompressed content hash: got %s, want %s", got, want)
+			}
+		})
+	}
+}
+
 func BenchmarkLarge(b *testing.B) {
 	b.StopTimer()
 	b.ReportAllocs()
-- 
2.50.0