From: Damien Neil Date: Thu, 16 Mar 2023 23:56:12 +0000 (-0700) Subject: [release-branch.go1.19] net/textproto, mime/multipart: improve accounting of non... X-Git-Tag: go1.19.8~4 X-Git-Url: http://www.git.cypherpunks.su/?a=commitdiff_plain;h=7a359a651c7ebdb29e0a1c03102fce793e9f58f0;p=gostls13.git [release-branch.go1.19] net/textproto, mime/multipart: improve accounting of non-file data For requests containing large numbers of small parts, memory consumption of a parsed form could be about 250% over the estimated size. When considering the size of parsed forms, account for the size of FileHeader structs and increase the estimate of memory consumed by map entries. Thanks to Jakob Ackermann (@das7pad) for reporting this issue. For CVE-2023-24536 For #59153 For #59269 Reviewed-on: https://team-review.git.corp.google.com/c/golang/go-private/+/1802454 Run-TryBot: Damien Neil Reviewed-by: Roland Shoemaker Reviewed-by: Julie Qiu Reviewed-on: https://team-review.git.corp.google.com/c/golang/go-private/+/1802396 Run-TryBot: Roland Shoemaker Reviewed-by: Damien Neil Change-Id: I31bc50e9346b4eee6fbe51a18c3c57230cc066db Reviewed-on: https://go-review.googlesource.com/c/go/+/481984 Reviewed-by: Matthew Dempsky Auto-Submit: Michael Knyszek TryBot-Result: Gopher Robot Run-TryBot: Michael Knyszek --- diff --git a/src/mime/multipart/formdata.go b/src/mime/multipart/formdata.go index 975dcb6b26..3f6ff697ca 100644 --- a/src/mime/multipart/formdata.go +++ b/src/mime/multipart/formdata.go @@ -103,8 +103,9 @@ func (r *Reader) readForm(maxMemory int64) (_ *Form, err error) { // Multiple values for the same key (one map entry, longer slice) are cheaper // than the same number of values for different keys (many map entries), but // using a consistent per-value cost for overhead is simpler. + const mapEntryOverhead = 200 maxMemoryBytes -= int64(len(name)) - maxMemoryBytes -= 100 // map overhead + maxMemoryBytes -= mapEntryOverhead if maxMemoryBytes < 0 { // We can't actually take this path, since nextPart would already have // rejected the MIME headers for being too large. Check anyway. @@ -128,7 +129,10 @@ func (r *Reader) readForm(maxMemory int64) (_ *Form, err error) { } // file, store in memory or on disk + const fileHeaderSize = 100 maxMemoryBytes -= mimeHeaderSize(p.Header) + maxMemoryBytes -= mapEntryOverhead + maxMemoryBytes -= fileHeaderSize if maxMemoryBytes < 0 { return nil, ErrMessageTooLarge } @@ -183,9 +187,10 @@ func (r *Reader) readForm(maxMemory int64) (_ *Form, err error) { } func mimeHeaderSize(h textproto.MIMEHeader) (size int64) { + size = 400 for k, vs := range h { size += int64(len(k)) - size += 100 // map entry overhead + size += 200 // map entry overhead for _, v := range vs { size += int64(len(v)) } diff --git a/src/mime/multipart/formdata_test.go b/src/mime/multipart/formdata_test.go index f5b56083b2..8ed26e0c34 100644 --- a/src/mime/multipart/formdata_test.go +++ b/src/mime/multipart/formdata_test.go @@ -192,10 +192,10 @@ func (r *failOnReadAfterErrorReader) Read(p []byte) (n int, err error) { // TestReadForm_NonFileMaxMemory asserts that the ReadForm maxMemory limit is applied // while processing non-file form data as well as file form data. func TestReadForm_NonFileMaxMemory(t *testing.T) { - n := 10<<20 + 25 if testing.Short() { - n = 10<<10 + 25 + t.Skip("skipping in -short mode") } + n := 10 << 20 largeTextValue := strings.Repeat("1", n) message := `--MyBoundary Content-Disposition: form-data; name="largetext" @@ -203,38 +203,29 @@ Content-Disposition: form-data; name="largetext" ` + largeTextValue + ` --MyBoundary-- ` - testBody := strings.ReplaceAll(message, "\n", "\r\n") - testCases := []struct { - name string - maxMemory int64 - err error - }{ - {"smaller", 50 + int64(len("largetext")) + 100, nil}, - {"exact-fit", 25 + int64(len("largetext")) + 100, nil}, - {"too-large", 0, ErrMessageTooLarge}, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - if tc.maxMemory == 0 && testing.Short() { - t.Skip("skipping in -short mode") - } - b := strings.NewReader(testBody) - r := NewReader(b, boundary) - f, err := r.ReadForm(tc.maxMemory) - if err == nil { - defer f.RemoveAll() - } - if tc.err != err { - t.Fatalf("ReadForm error - got: %v; expected: %v", err, tc.err) - } - if err == nil { - if g := f.Value["largetext"][0]; g != largeTextValue { - t.Errorf("largetext mismatch: got size: %v, expected size: %v", len(g), len(largeTextValue)) - } - } - }) + // Try parsing the form with increasing maxMemory values. + // Changes in how we account for non-file form data may cause the exact point + // where we change from rejecting the form as too large to accepting it to vary, + // but we should see both successes and failures. + const failWhenMaxMemoryLessThan = 128 + for maxMemory := int64(0); maxMemory < failWhenMaxMemoryLessThan*2; maxMemory += 16 { + b := strings.NewReader(testBody) + r := NewReader(b, boundary) + f, err := r.ReadForm(maxMemory) + if err != nil { + continue + } + if g := f.Value["largetext"][0]; g != largeTextValue { + t.Errorf("largetext mismatch: got size: %v, expected size: %v", len(g), len(largeTextValue)) + } + f.RemoveAll() + if maxMemory < failWhenMaxMemoryLessThan { + t.Errorf("ReadForm(%v): no error, expect to hit memory limit when maxMemory < %v", maxMemory, failWhenMaxMemoryLessThan) + } + return } + t.Errorf("ReadForm(x) failed for x < 1024, expect success") } // TestReadForm_MetadataTooLarge verifies that we account for the size of field names, diff --git a/src/net/textproto/reader.go b/src/net/textproto/reader.go index 9a21777df8..c1284fde25 100644 --- a/src/net/textproto/reader.go +++ b/src/net/textproto/reader.go @@ -503,6 +503,12 @@ func readMIMEHeader(r *Reader, lim int64) (MIMEHeader, error) { m := make(MIMEHeader, hint) + // Account for 400 bytes of overhead for the MIMEHeader, plus 200 bytes per entry. + // Benchmarking map creation as of go1.20, a one-entry MIMEHeader is 416 bytes and large + // MIMEHeaders average about 200 bytes per entry. + lim -= 400 + const mapEntryOverhead = 200 + // The first line cannot start with a leading space. if buf, err := r.R.Peek(1); err == nil && (buf[0] == ' ' || buf[0] == '\t') { line, err := r.readLineSlice() @@ -538,7 +544,7 @@ func readMIMEHeader(r *Reader, lim int64) (MIMEHeader, error) { vv := m[key] if vv == nil { lim -= int64(len(key)) - lim -= 100 // map entry overhead + lim -= mapEntryOverhead } lim -= int64(len(value)) if lim < 0 {