From 914c626cae8aaa205198dba839cb8526de3a2046 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Mon, 18 Jan 2010 21:46:59 -0800 Subject: [PATCH] Significant extension to http.Response, which now adheres to the usage pattern of http.Request and paves the way to persistent connection handling. R=rsc CC=golang-dev https://golang.org/cl/185043 --- src/pkg/http/Makefile | 2 + src/pkg/http/chunked.go | 56 +++++ src/pkg/http/client.go | 95 +------ src/pkg/http/request.go | 9 +- src/pkg/http/response.go | 480 ++++++++++++++++++++++++++++++++++++ src/pkg/rpc/client.go | 2 +- src/pkg/websocket/client.go | 2 +- 7 files changed, 550 insertions(+), 96 deletions(-) create mode 100644 src/pkg/http/chunked.go create mode 100644 src/pkg/http/response.go diff --git a/src/pkg/http/Makefile b/src/pkg/http/Makefile index 93852fe711..7654de807f 100644 --- a/src/pkg/http/Makefile +++ b/src/pkg/http/Makefile @@ -6,9 +6,11 @@ include ../../Make.$(GOARCH) TARG=http GOFILES=\ + chunked.go\ client.go\ fs.go\ request.go\ + response.go\ server.go\ status.go\ url.go\ diff --git a/src/pkg/http/chunked.go b/src/pkg/http/chunked.go new file mode 100644 index 0000000000..66195f06b9 --- /dev/null +++ b/src/pkg/http/chunked.go @@ -0,0 +1,56 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http + +import ( + "io" + "os" + "strconv" +) + +// NewChunkedWriter returns a new writer that translates writes into HTTP +// "chunked" format before writing them to w. Closing the returned writer +// sends the final 0-length chunk that marks the end of the stream. +func NewChunkedWriter(w io.Writer) io.WriteCloser { + return &chunkedWriter{w} +} + +// Writing to ChunkedWriter translates to writing in HTTP chunked Transfer +// Encoding wire format to the undering Wire writer. +type chunkedWriter struct { + Wire io.Writer +} + +// Write the contents of data as one chunk to Wire. +// NOTE: Note that the corresponding chunk-writing procedure in Conn.Write has +// a bug since it does not check for success of io.WriteString +func (cw *chunkedWriter) Write(data []byte) (n int, err os.Error) { + + // Don't send 0-length data. It looks like EOF for chunked encoding. + if len(data) == 0 { + return 0, nil + } + + head := strconv.Itob(len(data), 16) + "\r\n" + + if _, err = io.WriteString(cw.Wire, head); err != nil { + return 0, err + } + if n, err = cw.Wire.Write(data); err != nil { + return + } + if n != len(data) { + err = io.ErrShortWrite + return + } + _, err = io.WriteString(cw.Wire, "\r\n") + + return +} + +func (cw *chunkedWriter) Close() os.Error { + _, err := io.WriteString(cw.Wire, "0\r\n") + return err +} diff --git a/src/pkg/http/client.go b/src/pkg/http/client.go index af11a4b745..24758eee1b 100644 --- a/src/pkg/http/client.go +++ b/src/pkg/http/client.go @@ -13,49 +13,9 @@ import ( "io" "net" "os" - "strconv" "strings" ) -// Response represents the response from an HTTP request. -type Response struct { - Status string // e.g. "200 OK" - StatusCode int // e.g. 200 - - // Header maps header keys to values. If the response had multiple - // headers with the same key, they will be concatenated, with comma - // delimiters. (Section 4.2 of RFC 2616 requires that multiple headers - // be semantically equivalent to a comma-delimited sequence.) - // - // Keys in the map are canonicalized (see CanonicalHeaderKey). - Header map[string]string - - // Stream from which the response body can be read. - Body io.ReadCloser -} - -// GetHeader returns the value of the response header with the given -// key, and true. If there were multiple headers with this key, their -// values are concatenated, with a comma delimiter. If there were no -// response headers with the given key, it returns the empty string and -// false. Keys are not case sensitive. -func (r *Response) GetHeader(key string) (value string) { - value, _ = r.Header[CanonicalHeaderKey(key)] - return -} - -// AddHeader adds a value under the given key. Keys are not case sensitive. -func (r *Response) AddHeader(key, value string) { - key = CanonicalHeaderKey(key) - - oldValues, oldValuesPresent := r.Header[key] - if oldValuesPresent { - r.Header[key] = oldValues + "," + value - } else { - r.Header[key] = value - } -} - // Given a string of the form "host", "host:port", or "[ipv6::address]:port", // return true if the string includes a port. func hasPort(s string) bool { return strings.LastIndex(s, ":") > strings.LastIndex(s, "]") } @@ -68,43 +28,6 @@ type readClose struct { io.Closer } -// ReadResponse reads and returns an HTTP response from r. -func ReadResponse(r *bufio.Reader) (*Response, os.Error) { - resp := new(Response) - - // Parse the first line of the response. - resp.Header = make(map[string]string) - - line, err := readLine(r) - if err != nil { - return nil, err - } - f := strings.Split(line, " ", 3) - if len(f) < 3 { - return nil, &badStringError{"malformed HTTP response", line} - } - resp.Status = f[1] + " " + f[2] - resp.StatusCode, err = strconv.Atoi(f[1]) - if err != nil { - return nil, &badStringError{"malformed HTTP status code", f[1]} - } - - // Parse the response headers. - for { - key, value, err := readKeyValue(r) - if err != nil { - return nil, err - } - if key == "" { - break // end of response header - } - resp.AddHeader(key, value) - } - - return resp, nil -} - - // Send issues an HTTP request. Caller should close resp.Body when done reading it. // // TODO: support persistent connections (multiple requests on a single connection). @@ -141,23 +64,13 @@ func send(req *Request) (resp *Response, err os.Error) { } reader := bufio.NewReader(conn) - resp, err = ReadResponse(reader) + resp, err = ReadResponse(reader, req.Method) if err != nil { conn.Close() return nil, err } - r := io.Reader(reader) - if v := resp.GetHeader("Transfer-Encoding"); v == "chunked" { - r = newChunkedReader(reader) - } else if v := resp.GetHeader("Content-Length"); v != "" { - n, err := strconv.Atoi64(v) - if err != nil { - return nil, &badStringError{"invalid Content-Length", v} - } - r = io.LimitReader(r, n) - } - resp.Body = readClose{r, conn} + resp.Body = readClose{resp.Body, conn} return } @@ -180,8 +93,8 @@ func shouldRedirect(statusCode int) bool { // 303 (See Other) // 307 (Temporary Redirect) // -// finalURL is the URL from which the response was fetched -- identical to the input -// URL unless redirects were followed. +// finalURL is the URL from which the response was fetched -- identical to the +// input URL unless redirects were followed. // // Caller should close r.Body when done reading it. func Get(url string) (r *Response, finalURL string, err os.Error) { diff --git a/src/pkg/http/request.go b/src/pkg/http/request.go index 884fe48fa7..2ade5b7661 100644 --- a/src/pkg/http/request.go +++ b/src/pkg/http/request.go @@ -34,9 +34,12 @@ type ProtocolError struct { } var ( - ErrLineTooLong = &ProtocolError{"header line too long"} - ErrHeaderTooLong = &ProtocolError{"header too long"} - ErrShortBody = &ProtocolError{"entity body too short"} + ErrLineTooLong = &ProtocolError{"header line too long"} + ErrHeaderTooLong = &ProtocolError{"header too long"} + ErrShortBody = &ProtocolError{"entity body too short"} + ErrNotSupported = &ProtocolError{"feature not supported"} + ErrUnexpectedTrailer = &ProtocolError{"trailer header without chunked transfer encoding"} + ErrMissingContentLength = &ProtocolError{"missing ContentLength in HEAD response"} ) type badStringError struct { diff --git a/src/pkg/http/response.go b/src/pkg/http/response.go new file mode 100644 index 0000000000..eec9486c61 --- /dev/null +++ b/src/pkg/http/response.go @@ -0,0 +1,480 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// HTTP Response reading and parsing. + +package http + +import ( + "bufio" + "io" + "os" + "strconv" + "strings" +) + +// Response represents the response from an HTTP request. +// +type Response struct { + Status string // e.g. "200 OK" + StatusCode int // e.g. 200 + Proto string // e.g. "HTTP/1.0" + ProtoMajor int // e.g. 1 + ProtoMinor int // e.g. 0 + + // RequestMethod records the method used in the HTTP request. + // Header fields such as Content-Length have method-specific meaning. + RequestMethod string // e.g. "HEAD", "CONNECT", "GET", etc. + + // Header maps header keys to values. If the response had multiple + // headers with the same key, they will be concatenated, with comma + // delimiters. (Section 4.2 of RFC 2616 requires that multiple headers + // be semantically equivalent to a comma-delimited sequence.) Values + // duplicated by other fields in this struct (e.g., ContentLength) are + // omitted from Header. + // + // Keys in the map are canonicalized (see CanonicalHeaderKey). + Header map[string]string + + // Body represents the response body. + Body io.ReadCloser + + // ContentLength records the length of the associated content. The + // value -1 indicates that the length is unknown. Unless RequestMethod + // is "HEAD", values >= 0 indicate that the given number of bytes may + // be read from Body. + ContentLength int64 + + // Contains transfer encodings from outer-most to inner-most. Value is + // nil, means that "identity" encoding is used. + TransferEncoding []string + + // Close records whether the header directed that the connection be + // closed after reading Body. The value is advice for clients: neither + // ReadResponse nor Response.Write ever closes a connection. + Close bool + + // Trailer maps trailer keys to values. Like for Header, if the + // response has multiple trailer lines with the same key, they will be + // concatenated, delimited by commas. + Trailer map[string]string +} + +// ReadResponse reads and returns an HTTP response from r. The RequestMethod +// parameter specifies the method used in the corresponding request (e.g., +// "GET", "HEAD"). Clients must call resp.Body.Close when finished reading +// resp.Body. After that call, clients can inspect resp.Trailer to find +// key/value pairs included in the response trailer. +func ReadResponse(r *bufio.Reader, requestMethod string) (resp *Response, err os.Error) { + + resp = new(Response) + + resp.RequestMethod = strings.ToUpper(requestMethod) + + // Parse the first line of the response. + line, err := readLine(r) + if err != nil { + return nil, err + } + f := strings.Split(line, " ", 3) + if len(f) < 3 { + return nil, &badStringError{"malformed HTTP response", line} + } + resp.Status = f[1] + " " + f[2] + resp.StatusCode, err = strconv.Atoi(f[1]) + if err != nil { + return nil, &badStringError{"malformed HTTP status code", f[1]} + } + + resp.Proto = f[0] + var ok bool + if resp.ProtoMajor, resp.ProtoMinor, ok = parseHTTPVersion(resp.Proto); !ok { + return nil, &badStringError{"malformed HTTP version", resp.Proto} + } + + // Parse the response headers. + nheader := 0 + resp.Header = make(map[string]string) + for { + key, value, err := readKeyValue(r) + if err != nil { + return nil, err + } + if key == "" { + break // end of response header + } + if nheader++; nheader >= maxHeaderLines { + return nil, ErrHeaderTooLong + } + resp.AddHeader(key, value) + } + + fixPragmaCacheControl(resp.Header) + + resp.TransferEncoding, err = fixTransferEncoding(resp.Header) + if err != nil { + return nil, err + } + + resp.ContentLength, err = fixLength(resp.StatusCode, resp.RequestMethod, + resp.Header, resp.TransferEncoding) + if err != nil { + return nil, err + } + + resp.Close = shouldClose(resp.ProtoMajor, resp.ProtoMinor, resp.Header) + + resp.Trailer, err = fixTrailer(resp.Header, resp.TransferEncoding) + if err != nil { + return nil, err + } + + // Prepare body reader. ContentLength < 0 means chunked encoding, + // since multipart is not supported yet + if resp.ContentLength < 0 { + resp.Body = &body{newChunkedReader(r), resp, r} + } else { + resp.Body = &body{io.LimitReader(r, resp.ContentLength), nil, nil} + } + + return resp, nil +} + +// ffwdClose (fast-forward close) adds a Close method to a Reader which skips +// ahead until EOF +type body struct { + io.Reader + resp *Response // non-nil value means read trailer + r *bufio.Reader // underlying wire-format reader for the trailer +} + +func (b *body) Close() os.Error { + trashBuf := make([]byte, 1024) // local for thread safety + for { + _, err := b.Read(trashBuf) + if err == nil { + continue + } + if err == os.EOF { + break + } + return err + } + if b.resp == nil { // not reading trailer + return nil + } + + // TODO(petar): Put trailer reader code here + + return nil +} + +// RFC2616: Should treat +// Pragma: no-cache +// like +// Cache-Control: no-cache +func fixPragmaCacheControl(header map[string]string) { + if v, present := header["Pragma"]; present && v == "no-cache" { + if _, presentcc := header["Cache-Control"]; !presentcc { + header["Cache-Control"] = "no-cache" + } + } +} + +// Parse the trailer header +func fixTrailer(header map[string]string, te []string) (map[string]string, os.Error) { + raw, present := header["Trailer"] + if !present { + return nil, nil + } + + header["Trailer"] = "", false + trailer := make(map[string]string) + keys := strings.Split(raw, ",", 0) + for _, key := range keys { + key = CanonicalHeaderKey(strings.TrimSpace(key)) + switch key { + case "Transfer-Encoding", "Trailer", "Content-Length": + return nil, &badStringError{"bad trailer key", key} + } + trailer[key] = "" + } + if len(trailer) == 0 { + return nil, nil + } + if !chunked(te) { + // Trailer and no chunking + return nil, ErrUnexpectedTrailer + } + return trailer, nil +} + +// Sanitize transfer encoding +func fixTransferEncoding(header map[string]string) ([]string, os.Error) { + raw, present := header["Transfer-Encoding"] + if !present { + return nil, nil + } + + header["Transfer-Encoding"] = "", false + encodings := strings.Split(raw, ",", 0) + te := make([]string, 0, len(encodings)) + // TODO: Even though we only support "identity" and "chunked" + // encodings, the loop below is designed with foresight. One + // invariant that must be maintained is that, if present, + // chunked encoding must always come first. + for _, encoding := range encodings { + encoding = strings.ToLower(strings.TrimSpace(encoding)) + // "identity" encoding is not recored + if encoding == "identity" { + break + } + if encoding != "chunked" { + return nil, &badStringError{"unsupported transfer encoding", encoding} + } + te = te[0 : len(te)+1] + te[len(te)-1] = encoding + } + if len(te) > 1 { + return nil, &badStringError{"too many transfer encodings", strings.Join(te, ",")} + } + if len(te) > 0 { + // Chunked encoding trumps Content-Length. See RFC 2616 + // Section 4.4. Currently len(te) > 0 implies chunked + // encoding. + header["Content-Length"] = "", false + return te, nil + } + + return nil, nil +} + +func noBodyExpected(requestMethod string) bool { + return requestMethod == "HEAD" +} + +// Determine the expected body length, using RFC 2616 Section 4.4. This +// function is not a method, because ultimately it should be shared by +// ReadResponse and ReadRequest. +func fixLength(status int, requestMethod string, header map[string]string, te []string) (int64, os.Error) { + + // Logic based on response type or status + if noBodyExpected(requestMethod) { + return 0, nil + } + if status/100 == 1 { + return 0, nil + } + switch status { + case 204, 304: + return 0, nil + } + + // Logic based on Transfer-Encoding + if chunked(te) { + return -1, nil + } + + // Logic based on Content-Length + if cl, present := header["Content-Length"]; present { + cl = strings.TrimSpace(cl) + if cl != "" { + n, err := strconv.Atoi64(cl) + if err != nil || n < 0 { + return -1, &badStringError{"bad Content-Length", cl} + } + return n, nil + } else { + header["Content-Length"] = "", false + } + } + + // Logic based on media type. The purpose of the following code is just + // to detect whether the unsupported "multipart/byteranges" is being + // used. A proper Content-Type parser is needed in the future. + if ct, present := header["Content-Type"]; present { + ct = strings.ToLower(ct) + if strings.Index(ct, "multipart/byteranges") >= 0 { + return -1, ErrNotSupported + } + } + + + // Logic based on close + return -1, nil +} + +// Determine whether to hang up after sending a request and body, or +// receiving a response and body +func shouldClose(major, minor int, header map[string]string) bool { + if major < 1 || (major == 1 && minor < 1) { + return true + } else if v, present := header["Connection"]; present { + // TODO: Should split on commas, toss surrounding white space, + // and check each field. + if v == "close" { + return true + } + } + return false +} + +// Checks whether chunked is part of the encodings stack +func chunked(te []string) bool { return len(te) > 0 && te[0] == "chunked" } + +// AddHeader adds a value under the given key. Keys are not case sensitive. +func (r *Response) AddHeader(key, value string) { + key = CanonicalHeaderKey(key) + + oldValues, oldValuesPresent := r.Header[key] + if oldValuesPresent { + r.Header[key] = oldValues + "," + value + } else { + r.Header[key] = value + } +} + +// GetHeader returns the value of the response header with the given +// key, and true. If there were multiple headers with this key, their +// values are concatenated, with a comma delimiter. If there were no +// response headers with the given key, it returns the empty string and +// false. Keys are not case sensitive. +func (r *Response) GetHeader(key string) (value string) { + value, _ = r.Header[CanonicalHeaderKey(key)] + return +} + +// ProtoAtLeast returns whether the HTTP protocol used +// in the response is at least major.minor. +func (r *Response) ProtoAtLeast(major, minor int) bool { + return r.ProtoMajor > major || + r.ProtoMajor == major && r.ProtoMinor >= minor +} + +// Writes the response (header, body and trailer) in wire format. This method +// consults the following fields of resp: +// +// StatusCode +// ProtoMajor +// ProtoMinor +// RequestMethod +// TransferEncoding +// Trailer +// Body +// ContentLength +// Header, values for non-canonical keys will have unpredictable behavior +// +func (resp *Response) Write(w io.Writer) os.Error { + + // RequestMethod should be lower-case + resp.RequestMethod = strings.ToUpper(resp.RequestMethod) + + // Status line + text, ok := statusText[resp.StatusCode] + if !ok { + text = "status code " + strconv.Itoa(resp.StatusCode) + } + io.WriteString(w, "HTTP/"+strconv.Itoa(resp.ProtoMajor)+".") + io.WriteString(w, strconv.Itoa(resp.ProtoMinor)+" ") + io.WriteString(w, strconv.Itoa(resp.StatusCode)+" "+text+"\r\n") + + // Sanitize the field triple (Body, ContentLength, TransferEncoding) + if noBodyExpected(resp.RequestMethod) { + resp.Body = nil + resp.TransferEncoding = nil + // resp.ContentLength is expected to hold Content-Length + if resp.ContentLength < 0 { + return ErrMissingContentLength + } + } else { + if !resp.ProtoAtLeast(1, 1) || resp.Body == nil { + resp.TransferEncoding = nil + } + if chunked(resp.TransferEncoding) { + resp.ContentLength = -1 + } else if resp.Body != nil { + // For safety, consider sending a 0-length body an + // error + if resp.ContentLength <= 0 { + return &ProtocolError{"zero body length"} + } + } else { // no chunking, no body + resp.ContentLength = 0 + } + } + + // Write Content-Length and/or Transfer-Encoding whose values are a + // function of the sanitized field triple (Body, ContentLength, + // TransferEncoding) + if chunked(resp.TransferEncoding) { + io.WriteString(w, "Transfer-Encoding: chunked\r\n") + } else { + io.WriteString(w, "Content-Length: ") + io.WriteString(w, strconv.Itoa64(resp.ContentLength)+"\r\n") + } + if resp.Header != nil { + resp.Header["Content-Length"] = "", false + resp.Header["Transfer-Encoding"] = "", false + } + + // Sanitize Trailer + if !chunked(resp.TransferEncoding) { + resp.Trailer = nil + } else if resp.Trailer != nil { + // TODO: At some point, there should be a generic mechanism for + // writing long headers, using HTTP line splitting + io.WriteString(w, "Trailer: ") + needComma := false + for k, _ := range resp.Trailer { + k = CanonicalHeaderKey(k) + switch k { + case "Transfer-Encoding", "Trailer", "Content-Length": + return &badStringError{"invalid Trailer key", k} + } + if needComma { + io.WriteString(w, ",") + } + io.WriteString(w, k) + needComma = true + } + io.WriteString(w, "\r\n") + } + if resp.Header != nil { + resp.Header["Trailer"] = "", false + } + + // Rest of header + for k, v := range resp.Header { + io.WriteString(w, k+": "+v+"\r\n") + } + + // End-of-header + io.WriteString(w, "\r\n") + + // Write body + if resp.Body != nil { + var err os.Error + if chunked(resp.TransferEncoding) { + cw := NewChunkedWriter(w) + _, err = io.Copy(cw, resp.Body) + if err == nil { + err = cw.Close() + } + } else { + _, err = io.Copy(w, io.LimitReader(resp.Body, resp.ContentLength)) + } + if err != nil { + return err + } + } + + // TODO(petar): Place trailer writer code here. + if chunked(resp.TransferEncoding) { + // Last chunk, empty trailer + io.WriteString(w, "\r\n") + } + + // Success + return nil +} diff --git a/src/pkg/rpc/client.go b/src/pkg/rpc/client.go index 673283be3d..153c56d831 100644 --- a/src/pkg/rpc/client.go +++ b/src/pkg/rpc/client.go @@ -126,7 +126,7 @@ func DialHTTP(network, address string) (*Client, os.Error) { // Require successful HTTP response // before switching to RPC protocol. - resp, err := http.ReadResponse(bufio.NewReader(conn)) + resp, err := http.ReadResponse(bufio.NewReader(conn), "CONNECT") if err == nil && resp.Status == connected { return NewClient(conn), nil } diff --git a/src/pkg/websocket/client.go b/src/pkg/websocket/client.go index c81f4f440f..c5dde4b799 100644 --- a/src/pkg/websocket/client.go +++ b/src/pkg/websocket/client.go @@ -90,7 +90,7 @@ func handshake(resourceName, host, origin, location, protocol string, br *bufio. } bw.WriteString("\r\n") bw.Flush() - resp, err := http.ReadResponse(br) + resp, err := http.ReadResponse(br, "GET") if err != nil { return } -- 2.48.1