net/http: fix Content-Length/Transfer-Encoding on HEAD requests

net/http currently assumes that the response to a HEAD request
    will always have a Content-Length header. This is incorrect.

RFC2616 says: "The HEAD method is identical to GET except that
the server MUST NOT return a message-body in the response. The
metainformation contained in the HTTP headers in response to a
HEAD request SHOULD be identical to the information sent in
response to a GET request. This method can be used for
obtaining metainformation about the entity implied by the
request without transferring the entity-body itself. This
method is often used for testing hypertext links for validity,
accessibility, and recent modification."

This means that three cases are possible: a Content-Length
header, a Transfer-Encoding header or neither. In the wild the
following sites exhibit these behaviours (curl -I):

HEAD on http://www.google.co.uk/ has Transfer-Encoding: chunked
HEAD on http://www.bbc.co.uk/    has Content-Length: 45247
HEAD on http://edition.cnn.com/  has neither header

This patch does not remove the ErrMissingContentLength error
for compatibility reasons, but it is no longer used.

R=golang-dev, bradfitz
CC=golang-dev
https://golang.org/cl/7182045

diff --git a/src/pkg/net/http/response_test.go b/src/pkg/net/http/response_test.go
index f31e5d09fe5d9133edc10d7fe299255bc79319e1..a00a4ae0a9b246aba4e0aa185735a4e57065e03a 100644 (file)
--- a/src/pkg/net/http/response_test.go
+++ b/src/pkg/net/http/response_test.go
@@ -124,7 +124,7 @@ var respTests = []respTest{
 
        // Chunked response without Content-Length.
        {
-               "HTTP/1.0 200 OK\r\n" +
+               "HTTP/1.1 200 OK\r\n" +
                        "Transfer-Encoding: chunked\r\n" +
                        "\r\n" +
                        "0a\r\n" +
@@ -137,12 +137,12 @@ var respTests = []respTest{
                Response{
                        Status:           "200 OK",
                        StatusCode:       200,
-                       Proto:            "HTTP/1.0",
+                       Proto:            "HTTP/1.1",
                        ProtoMajor:       1,
-                       ProtoMinor:       0,
+                       ProtoMinor:       1,
                        Request:          dummyReq("GET"),
                        Header:           Header{},
-                       Close:            true,
+                       Close:            false,
                        ContentLength:    -1,
                        TransferEncoding: []string{"chunked"},
                },
@@ -152,7 +152,7 @@ var respTests = []respTest{
 
        // Chunked response with Content-Length.
        {
-               "HTTP/1.0 200 OK\r\n" +
+               "HTTP/1.1 200 OK\r\n" +
                        "Transfer-Encoding: chunked\r\n" +
                        "Content-Length: 10\r\n" +
                        "\r\n" +
@@ -164,12 +164,12 @@ var respTests = []respTest{
                Response{
                        Status:           "200 OK",
                        StatusCode:       200,
-                       Proto:            "HTTP/1.0",
+                       Proto:            "HTTP/1.1",
                        ProtoMajor:       1,
-                       ProtoMinor:       0,
+                       ProtoMinor:       1,
                        Request:          dummyReq("GET"),
                        Header:           Header{},
-                       Close:            true,
+                       Close:            false,
                        ContentLength:    -1, // TODO(rsc): Fix?
                        TransferEncoding: []string{"chunked"},
                },
@@ -177,23 +177,88 @@ var respTests = []respTest{
                "Body here\n",
        },
 
-       // Chunked response in response to a HEAD request (the "chunked" should
-       // be ignored, as HEAD responses never have bodies)
+       // Chunked response in response to a HEAD request
        {
-               "HTTP/1.0 200 OK\r\n" +
+               "HTTP/1.1 200 OK\r\n" +
                        "Transfer-Encoding: chunked\r\n" +
                        "\r\n",
 
                Response{
-                       Status:        "200 OK",
-                       StatusCode:    200,
-                       Proto:         "HTTP/1.0",
-                       ProtoMajor:    1,
-                       ProtoMinor:    0,
-                       Request:       dummyReq("HEAD"),
-                       Header:        Header{},
-                       Close:         true,
-                       ContentLength: -1,
+                       Status:           "200 OK",
+                       StatusCode:       200,
+                       Proto:            "HTTP/1.1",
+                       ProtoMajor:       1,
+                       ProtoMinor:       1,
+                       Request:          dummyReq("HEAD"),
+                       Header:           Header{},
+                       TransferEncoding: []string{"chunked"},
+                       Close:            false,
+                       ContentLength:    -1,
+               },
+
+               "",
+       },
+
+       // Content-Length in response to a HEAD request
+       {
+               "HTTP/1.0 200 OK\r\n" +
+                       "Content-Length: 256\r\n" +
+                       "\r\n",
+
+               Response{
+                       Status:           "200 OK",
+                       StatusCode:       200,
+                       Proto:            "HTTP/1.0",
+                       ProtoMajor:       1,
+                       ProtoMinor:       0,
+                       Request:          dummyReq("HEAD"),
+                       Header:           Header{"Content-Length": {"256"}},
+                       TransferEncoding: nil,
+                       Close:            true,
+                       ContentLength:    256,
+               },
+
+               "",
+       },
+
+       // Content-Length in response to a HEAD request with HTTP/1.1
+       {
+               "HTTP/1.1 200 OK\r\n" +
+                       "Content-Length: 256\r\n" +
+                       "\r\n",
+
+               Response{
+                       Status:           "200 OK",
+                       StatusCode:       200,
+                       Proto:            "HTTP/1.1",
+                       ProtoMajor:       1,
+                       ProtoMinor:       1,
+                       Request:          dummyReq("HEAD"),
+                       Header:           Header{"Content-Length": {"256"}},
+                       TransferEncoding: nil,
+                       Close:            false,
+                       ContentLength:    256,
+               },
+
+               "",
+       },
+
+       // No Content-Length or Chunked in response to a HEAD request
+       {
+               "HTTP/1.0 200 OK\r\n" +
+                       "\r\n",
+
+               Response{
+                       Status:           "200 OK",
+                       StatusCode:       200,
+                       Proto:            "HTTP/1.0",
+                       ProtoMajor:       1,
+                       ProtoMinor:       0,
+                       Request:          dummyReq("HEAD"),
+                       Header:           Header{},
+                       TransferEncoding: nil,
+                       Close:            true,
+                       ContentLength:    -1,
                },
 
                "",

diff --git a/src/pkg/net/http/transfer.go b/src/pkg/net/http/transfer.go
index 70ea15b8e4a346c8e255a102b0fb43347e80c104..25b34addec77fe9b57af79cc22bafe134653af67 100644 (file)
--- a/src/pkg/net/http/transfer.go
+++ b/src/pkg/net/http/transfer.go
@@ -87,10 +87,8 @@ func newTransferWriter(r interface{}) (t *transferWriter, err error) {
        // Sanitize Body,ContentLength,TransferEncoding
        if t.ResponseToHEAD {
                t.Body = nil
-               t.TransferEncoding = nil
-               // ContentLength is expected to hold Content-Length
-               if t.ContentLength < 0 {
-                       return nil, ErrMissingContentLength
+               if chunked(t.TransferEncoding) {
+                       t.ContentLength = -1
                }
        } else {
                if !atLeastHTTP11 || t.Body == nil {
@@ -122,9 +120,6 @@ func (t *transferWriter) shouldSendContentLength() bool {
        if t.ContentLength > 0 {
                return true
        }
-       if t.ResponseToHEAD {
-               return true
-       }
        // Many servers expect a Content-Length for these methods
        if t.Method == "POST" || t.Method == "PUT" {
                return true
@@ -380,12 +375,6 @@ func fixTransferEncoding(requestMethod string, header Header) ([]string, error)
 
        delete(header, "Transfer-Encoding")
 
-       // Head responses have no bodies, so the transfer encoding
-       // should be ignored.
-       if requestMethod == "HEAD" {
-               return nil, nil
-       }
-
        encodings := strings.Split(raw[0], ",")
        te := make([]string, 0, len(encodings))
        // TODO: Even though we only support "identity" and "chunked"