net/http: remove discrepancies between the MIME Sniffing Spec and its implementation

The change fixes the following deviations between the existing implementation and the Spec:
1. Using pattern instead of "mask" for assertion and iteration in the Pattern Matching Algorithm.
2. Rename "image/vnd.microsoft.icon" to "image/x-icon" and add another signature for the same.
3. Using named strings instead of hexadecimal representation in "application/zip" and "application/x-rar-compressed".
4. Reordering "sniffSignatures" in accordance with the Spec section "Identifying a resource with an unknown MIME type".

In addition to the above fixes, unit tests for Image MIME type group are added.

Fixes #30570

Change-Id: I97d2ae22b426c3c57bf8efd2ed9396c0be983688
Reviewed-on: https://go-review.googlesource.com/c/go/+/165277
Reviewed-by: Emmanuel Odeke <emm.odeke@gmail.com>
Run-TryBot: Emmanuel Odeke <emm.odeke@gmail.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>

diff --git a/src/net/http/sniff.go b/src/net/http/sniff.go
index c1494abb4c837930dc94831ac1a00671e81f0964..f03f723542ef304d57cde02655cfa5945df7e1f8 100644 (file)
--- a/src/net/http/sniff.go
+++ b/src/net/http/sniff.go
@@ -37,6 +37,8 @@ func DetectContentType(data []byte) string {
        return "application/octet-stream" // fallback
 }
 
+// isWS reports whether the provided byte is a whitespace byte (0xWS)
+// as defined in https://mimesniff.spec.whatwg.org/#terminology.
 func isWS(b byte) bool {
        switch b {
        case '\t', '\n', '\x0c', '\r', ' ':
@@ -45,6 +47,16 @@ func isWS(b byte) bool {
        return false
 }
 
+// isTT reports whether the provided byte is a tag-terminating byte (0xTT)
+// as defined in https://mimesniff.spec.whatwg.org/#terminology.
+func isTT(b byte) bool {
+       switch b {
+       case ' ', '>':
+               return true
+       }
+       return false
+}
+
 type sniffSig interface {
        // match returns the MIME type of the data, or "" if unknown.
        match(data []byte, firstNonWS int) string
@@ -69,33 +81,57 @@ var sniffSignatures = []sniffSig{
        htmlSig("<BR"),
        htmlSig("<P"),
        htmlSig("<!--"),
-
-       &maskedSig{mask: []byte("\xFF\xFF\xFF\xFF\xFF"), pat: []byte("<?xml"), skipWS: true, ct: "text/xml; charset=utf-8"},
-
+       &maskedSig{
+               mask:   []byte("\xFF\xFF\xFF\xFF\xFF"),
+               pat:    []byte("<?xml"),
+               skipWS: true,
+               ct:     "text/xml; charset=utf-8"},
        &exactSig{[]byte("%PDF-"), "application/pdf"},
        &exactSig{[]byte("%!PS-Adobe-"), "application/postscript"},
 
        // UTF BOMs.
-       &maskedSig{mask: []byte("\xFF\xFF\x00\x00"), pat: []byte("\xFE\xFF\x00\x00"), ct: "text/plain; charset=utf-16be"},
-       &maskedSig{mask: []byte("\xFF\xFF\x00\x00"), pat: []byte("\xFF\xFE\x00\x00"), ct: "text/plain; charset=utf-16le"},
-       &maskedSig{mask: []byte("\xFF\xFF\xFF\x00"), pat: []byte("\xEF\xBB\xBF\x00"), ct: "text/plain; charset=utf-8"},
+       &maskedSig{
+               mask: []byte("\xFF\xFF\x00\x00"),
+               pat:  []byte("\xFE\xFF\x00\x00"),
+               ct:   "text/plain; charset=utf-16be",
+       },
+       &maskedSig{
+               mask: []byte("\xFF\xFF\x00\x00"),
+               pat:  []byte("\xFF\xFE\x00\x00"),
+               ct:   "text/plain; charset=utf-16le",
+       },
+       &maskedSig{
+               mask: []byte("\xFF\xFF\xFF\x00"),
+               pat:  []byte("\xEF\xBB\xBF\x00"),
+               ct:   "text/plain; charset=utf-8",
+       },
 
+       // Image types
+       // For posterity, we originally returned "image/vnd.microsoft.icon" from
+       // https://tools.ietf.org/html/draft-ietf-websec-mime-sniff-03#section-7
+       // https://codereview.appspot.com/4746042
+       // but that has since been replaced with "image/x-icon" in Section 6.2
+       // of https://mimesniff.spec.whatwg.org/#matching-an-image-type-pattern
+       &exactSig{[]byte("\x00\x00\x01\x00"), "image/x-icon"},
+       &exactSig{[]byte("\x00\x00\x02\x00"), "image/x-icon"},
+       &exactSig{[]byte("BM"), "image/bmp"},
        &exactSig{[]byte("GIF87a"), "image/gif"},
        &exactSig{[]byte("GIF89a"), "image/gif"},
-       &exactSig{[]byte("\x89\x50\x4E\x47\x0D\x0A\x1A\x0A"), "image/png"},
-       &exactSig{[]byte("\xFF\xD8\xFF"), "image/jpeg"},
-       &exactSig{[]byte("BM"), "image/bmp"},
        &maskedSig{
                mask: []byte("\xFF\xFF\xFF\xFF\x00\x00\x00\x00\xFF\xFF\xFF\xFF\xFF\xFF"),
                pat:  []byte("RIFF\x00\x00\x00\x00WEBPVP"),
                ct:   "image/webp",
        },
-       &exactSig{[]byte("\x00\x00\x01\x00"), "image/vnd.microsoft.icon"},
+       &exactSig{[]byte("\x89PNG\x0D\x0A\x1A\x0A"), "image/png"},
+       &exactSig{[]byte("\xFF\xD8\xFF"), "image/jpeg"},
 
+       // Audio and Video types
+       // Enforce the pattern match ordering as prescribed in
+       // https://mimesniff.spec.whatwg.org/#matching-an-audio-or-video-type-pattern
        &maskedSig{
-               mask: []byte("\xFF\xFF\xFF\xFF\x00\x00\x00\x00\xFF\xFF\xFF\xFF"),
-               pat:  []byte("RIFF\x00\x00\x00\x00WAVE"),
-               ct:   "audio/wave",
+               mask: []byte("\xFF\xFF\xFF\xFF"),
+               pat:  []byte(".snd"),
+               ct:   "audio/basic",
        },
        &maskedSig{
                mask: []byte("\xFF\xFF\xFF\xFF\x00\x00\x00\x00\xFF\xFF\xFF\xFF"),
@@ -103,9 +139,9 @@ var sniffSignatures = []sniffSig{
                ct:   "audio/aiff",
        },
        &maskedSig{
-               mask: []byte("\xFF\xFF\xFF\xFF"),
-               pat:  []byte(".snd"),
-               ct:   "audio/basic",
+               mask: []byte("\xFF\xFF\xFF"),
+               pat:  []byte("ID3"),
+               ct:   "audio/mpeg",
        },
        &maskedSig{
                mask: []byte("\xFF\xFF\xFF\xFF\xFF"),
@@ -117,21 +153,25 @@ var sniffSignatures = []sniffSig{
                pat:  []byte("MThd\x00\x00\x00\x06"),
                ct:   "audio/midi",
        },
-       &maskedSig{
-               mask: []byte("\xFF\xFF\xFF"),
-               pat:  []byte("ID3"),
-               ct:   "audio/mpeg",
-       },
        &maskedSig{
                mask: []byte("\xFF\xFF\xFF\xFF\x00\x00\x00\x00\xFF\xFF\xFF\xFF"),
                pat:  []byte("RIFF\x00\x00\x00\x00AVI "),
                ct:   "video/avi",
        },
+       &maskedSig{
+               mask: []byte("\xFF\xFF\xFF\xFF\x00\x00\x00\x00\xFF\xFF\xFF\xFF"),
+               pat:  []byte("RIFF\x00\x00\x00\x00WAVE"),
+               ct:   "audio/wave",
+       },
+       // 6.2.0.2. video/mp4
+       mp4Sig{},
+       // 6.2.0.3. video/webm
+       &exactSig{[]byte("\x1A\x45\xDF\xA3"), "video/webm"},
 
-       // Fonts
+       // Font types
        &maskedSig{
                // 34 NULL bytes followed by the string "LP"
-               pat: []byte("\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x4C\x50"),
+               pat: []byte("\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00LP"),
                // 34 NULL bytes followed by \xF\xF
                mask: []byte("\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xFF\xFF"),
                ct:   "application/vnd.ms-fontobject",
@@ -142,15 +182,13 @@ var sniffSignatures = []sniffSig{
        &exactSig{[]byte("wOFF"), "font/woff"},
        &exactSig{[]byte("wOF2"), "font/woff2"},
 
-       &exactSig{[]byte("\x1A\x45\xDF\xA3"), "video/webm"},
-       &exactSig{[]byte("\x52\x61\x72\x20\x1A\x07\x00"), "application/x-rar-compressed"},
-       &exactSig{[]byte("\x50\x4B\x03\x04"), "application/zip"},
+       // Archive types
        &exactSig{[]byte("\x1F\x8B\x08"), "application/x-gzip"},
+       &exactSig{[]byte("PK\x03\x04"), "application/zip"},
+       &exactSig{[]byte("Rar \x1A\x07\x00"), "application/x-rar-compressed"},
 
        &exactSig{[]byte("\x00\x61\x73\x6D"), "application/wasm"},
 
-       mp4Sig{},
-
        textSig{}, // should be last
 }
 
@@ -182,12 +220,12 @@ func (m *maskedSig) match(data []byte, firstNonWS int) string {
        if len(m.pat) != len(m.mask) {
                return ""
        }
-       if len(data) < len(m.mask) {
+       if len(data) < len(m.pat) {
                return ""
        }
-       for i, mask := range m.mask {
-               db := data[i] & mask
-               if db != m.pat[i] {
+       for i, pb := range m.pat {
+               maskedData := data[i] & m.mask[i]
+               if maskedData != pb {
                        return ""
                }
        }
@@ -210,8 +248,8 @@ func (h htmlSig) match(data []byte, firstNonWS int) string {
                        return ""
                }
        }
-       // Next byte must be space or right angle bracket.
-       if db := data[len(h)]; db != ' ' && db != '>' {
+       // Next byte must be a tag-terminating byte(0xTT).
+       if !isTT(data[len(h)]) {
                return ""
        }
        return "text/html; charset=utf-8"
@@ -229,7 +267,7 @@ func (mp4Sig) match(data []byte, firstNonWS int) string {
                return ""
        }
        boxSize := int(binary.BigEndian.Uint32(data[:4]))
-       if boxSize%4 != 0 || len(data) < boxSize {
+       if len(data) < boxSize || boxSize%4 != 0 {
                return ""
        }
        if !bytes.Equal(data[4:8], mp4ftype) {
@@ -237,7 +275,7 @@ func (mp4Sig) match(data []byte, firstNonWS int) string {
        }
        for st := 8; st < boxSize; st += 4 {
                if st == 12 {
-                       // minor version number
+                       // Ignores the four bytes that correspond to the version number of the "major brand".
                        continue
                }
                if bytes.Equal(data[st:st+3], mp4) {

diff --git a/src/net/http/sniff_test.go b/src/net/http/sniff_test.go
index b4d3c9f0cc84a594c08ce9059f9b78f2c4584054..b752f2338210b3e3e063e89fe3ac615702a7e6b7 100644 (file)
--- a/src/net/http/sniff_test.go
+++ b/src/net/http/sniff_test.go
@@ -36,8 +36,14 @@ var sniffTests = []struct {
        {"XML", []byte("\n<?xml!"), "text/xml; charset=utf-8"},
 
        // Image types.
+       {"Windows icon", []byte("\x00\x00\x01\x00"), "image/x-icon"},
+       {"Windows cursor", []byte("\x00\x00\x02\x00"), "image/x-icon"},
+       {"BMP image", []byte("BM..."), "image/bmp"},
        {"GIF 87a", []byte(`GIF87a`), "image/gif"},
        {"GIF 89a", []byte(`GIF89a...`), "image/gif"},
+       {"WEBP image", []byte("RIFF\x00\x00\x00\x00WEBPVP"), "image/webp"},
+       {"PNG image", []byte("\x89PNG\x0D\x0A\x1A\x0A"), "image/png"},
+       {"JPEG image", []byte("\xFF\xD8\xFF"), "image/jpeg"},
 
        // Audio types.
        {"MIDI audio", []byte("MThd\x00\x00\x00\x06\x00\x01"), "audio/midi"},