net/http: add more IDNA2008 tests and fix some omissions

It wasn't lowercasing the string, folding widths, and putting strings
into NFC form. Do those.

Fixes #13835

Change-Id: Ia3de6159417cacec203b48e206e51d79f945df58
Reviewed-on: https://go-review.googlesource.com/29860
Run-TryBot: Brad Fitzpatrick <bradfitz@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Marcel van Lohuizen <mpvl@golang.org>

diff --git a/src/go/build/deps_test.go b/src/go/build/deps_test.go
index c7e11498ddb7246cdf99393207d77ba461c30fea..fb31ac31c3c5a74cc61f3a3610cfbc3b118b88fc 100644 (file)
--- a/src/go/build/deps_test.go
+++ b/src/go/build/deps_test.go
@@ -382,6 +382,8 @@ var pkgDeps = map[string][]string{
                "golang_org/x/net/http2/hpack",
                "golang_org/x/net/idna",
                "golang_org/x/net/lex/httplex",
+               "golang_org/x/text/unicode/norm",
+               "golang_org/x/text/width",
                "internal/nettrace",
                "net/http/httptrace",
        },

diff --git a/src/net/http/http.go b/src/net/http/http.go
index b34ae41ec517b8e9ff34b8f5872f1e70cb37f2af..258efbb152a623da791e6f9efd8c17b3f7665918 100644 (file)
--- a/src/net/http/http.go
+++ b/src/net/http/http.go
@@ -6,6 +6,7 @@ package http
 
 import (
        "strings"
+       "unicode/utf8"
 
        "golang_org/x/net/lex/httplex"
 )
@@ -41,3 +42,12 @@ func removeEmptyPort(host string) string {
 func isNotToken(r rune) bool {
        return !httplex.IsTokenRune(r)
 }
+
+func isASCII(s string) bool {
+       for i := 0; i < len(s); i++ {
+               if s[i] >= utf8.RuneSelf {
+                       return false
+               }
+       }
+       return true
+}

diff --git a/src/net/http/http_test.go b/src/net/http/http_test.go
index b95dab053c7f815318cde6fc021c756238bd4e33..c6c38ffcaedaeade9fef9e14adfd1b2470db3a05 100644 (file)
--- a/src/net/http/http_test.go
+++ b/src/net/http/http_test.go
@@ -51,10 +51,18 @@ func TestCleanHost(t *testing.T) {
                {"www.google.com foo", "www.google.com"},
                {"www.google.com/foo", "www.google.com"},
                {" first character is a space", ""},
+               {"[1::6]:8080", "[1::6]:8080"},
+
+               // Punycode:
                {"гофер.рф/foo", "xn--c1ae0ajs.xn--p1ai"},
                {"bücher.de", "xn--bcher-kva.de"},
                {"bücher.de:8080", "xn--bcher-kva.de:8080"},
-               {"[1::6]:8080", "[1::6]:8080"},
+               // Verify we convert to lowercase before punycode:
+               {"BÜCHER.de", "xn--bcher-kva.de"},
+               {"BÜCHER.de:8080", "xn--bcher-kva.de:8080"},
+               // Verify we normalize to NFC before punycode:
+               {"gophér.nfc", "xn--gophr-esa.nfc"},            // NFC input; no work needed
+               {"goph\u0065\u0301r.nfd", "xn--gophr-esa.nfd"}, // NFD input
        }
        for _, tt := range tests {
                got := cleanHost(tt.in)

diff --git a/src/net/http/request.go b/src/net/http/request.go
index bebf55ccc4faf89c4801ae69e43e216e7fa4b3ea..a27d13cb9853ffc0e43ecdfe90996e1d8f3b1375 100644 (file)
--- a/src/net/http/request.go
+++ b/src/net/http/request.go
@@ -27,6 +27,8 @@ import (
        "sync"
 
        "golang_org/x/net/idna"
+       "golang_org/x/text/unicode/norm"
+       "golang_org/x/text/width"
 )
 
 const (
@@ -581,6 +583,19 @@ func (req *Request) write(w io.Writer, usingProxy bool, extraHeaders Header, wai
        return nil
 }
 
+func idnaASCII(v string) (string, error) {
+       if isASCII(v) {
+               return v, nil
+       }
+       // The idna package doesn't do everything from
+       // https://tools.ietf.org/html/rfc5895 so we do it here.
+       // TODO(bradfitz): should the idna package do this instead?
+       v = strings.ToLower(v)
+       v = width.Fold.String(v)
+       v = norm.NFC.String(v)
+       return idna.ToASCII(v)
+}
+
 // cleanHost cleans up the host sent in request's Host header.
 //
 // It both strips anything after '/' or ' ', and puts the value
@@ -600,13 +615,13 @@ func cleanHost(in string) string {
        }
        host, port, err := net.SplitHostPort(in)
        if err != nil { // input was just a host
-               a, err := idna.ToASCII(in)
+               a, err := idnaASCII(in)
                if err != nil {
                        return in // garbage in, garbage out
                }
                return a
        }
-       a, err := idna.ToASCII(host)
+       a, err := idnaASCII(host)
        if err != nil {
                return in // garbage in, garbage out
        }

diff --git a/src/net/http/transport.go b/src/net/http/transport.go
index cde7acac31063c5e2226bb9020768a9770d4db46..ed2b3a26edaca7bd9087aba1de217ad38929b316 100644 (file)
--- a/src/net/http/transport.go
+++ b/src/net/http/transport.go
@@ -27,7 +27,6 @@ import (
        "sync"
        "time"
 
-       "golang_org/x/net/idna"
        "golang_org/x/net/lex/httplex"
 )
 
@@ -1945,7 +1944,7 @@ var portMap = map[string]string{
 // canonicalAddr returns url.Host but always with a ":port" suffix
 func canonicalAddr(url *url.URL) string {
        addr := url.Hostname()
-       if v, err := idna.ToASCII(addr); err == nil {
+       if v, err := idnaASCII(addr); err == nil {
                addr = v
        }
        port := url.Port()