"io"
"net"
"os"
- "strconv"
"strings"
)
-// Response represents the response from an HTTP request.
-type Response struct {
- Status string // e.g. "200 OK"
- StatusCode int // e.g. 200
-
- // Header maps header keys to values. If the response had multiple
- // headers with the same key, they will be concatenated, with comma
- // delimiters. (Section 4.2 of RFC 2616 requires that multiple headers
- // be semantically equivalent to a comma-delimited sequence.)
- //
- // Keys in the map are canonicalized (see CanonicalHeaderKey).
- Header map[string]string
-
- // Stream from which the response body can be read.
- Body io.ReadCloser
-}
-
-// GetHeader returns the value of the response header with the given
-// key, and true. If there were multiple headers with this key, their
-// values are concatenated, with a comma delimiter. If there were no
-// response headers with the given key, it returns the empty string and
-// false. Keys are not case sensitive.
-func (r *Response) GetHeader(key string) (value string) {
- value, _ = r.Header[CanonicalHeaderKey(key)]
- return
-}
-
-// AddHeader adds a value under the given key. Keys are not case sensitive.
-func (r *Response) AddHeader(key, value string) {
- key = CanonicalHeaderKey(key)
-
- oldValues, oldValuesPresent := r.Header[key]
- if oldValuesPresent {
- r.Header[key] = oldValues + "," + value
- } else {
- r.Header[key] = value
- }
-}
-
// Given a string of the form "host", "host:port", or "[ipv6::address]:port",
// return true if the string includes a port.
func hasPort(s string) bool { return strings.LastIndex(s, ":") > strings.LastIndex(s, "]") }
io.Closer
}
-// ReadResponse reads and returns an HTTP response from r.
-func ReadResponse(r *bufio.Reader) (*Response, os.Error) {
- resp := new(Response)
-
- // Parse the first line of the response.
- resp.Header = make(map[string]string)
-
- line, err := readLine(r)
- if err != nil {
- return nil, err
- }
- f := strings.Split(line, " ", 3)
- if len(f) < 3 {
- return nil, &badStringError{"malformed HTTP response", line}
- }
- resp.Status = f[1] + " " + f[2]
- resp.StatusCode, err = strconv.Atoi(f[1])
- if err != nil {
- return nil, &badStringError{"malformed HTTP status code", f[1]}
- }
-
- // Parse the response headers.
- for {
- key, value, err := readKeyValue(r)
- if err != nil {
- return nil, err
- }
- if key == "" {
- break // end of response header
- }
- resp.AddHeader(key, value)
- }
-
- return resp, nil
-}
-
-
// Send issues an HTTP request. Caller should close resp.Body when done reading it.
//
// TODO: support persistent connections (multiple requests on a single connection).
}
reader := bufio.NewReader(conn)
- resp, err = ReadResponse(reader)
+ resp, err = ReadResponse(reader, req.Method)
if err != nil {
conn.Close()
return nil, err
}
- r := io.Reader(reader)
- if v := resp.GetHeader("Transfer-Encoding"); v == "chunked" {
- r = newChunkedReader(reader)
- } else if v := resp.GetHeader("Content-Length"); v != "" {
- n, err := strconv.Atoi64(v)
- if err != nil {
- return nil, &badStringError{"invalid Content-Length", v}
- }
- r = io.LimitReader(r, n)
- }
- resp.Body = readClose{r, conn}
+ resp.Body = readClose{resp.Body, conn}
return
}
// 303 (See Other)
// 307 (Temporary Redirect)
//
-// finalURL is the URL from which the response was fetched -- identical to the input
-// URL unless redirects were followed.
+// finalURL is the URL from which the response was fetched -- identical to the
+// input URL unless redirects were followed.
//
// Caller should close r.Body when done reading it.
func Get(url string) (r *Response, finalURL string, err os.Error) {
--- /dev/null
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// HTTP Response reading and parsing.
+
+package http
+
+import (
+ "bufio"
+ "io"
+ "os"
+ "strconv"
+ "strings"
+)
+
+// Response represents the response from an HTTP request.
+//
+type Response struct {
+ Status string // e.g. "200 OK"
+ StatusCode int // e.g. 200
+ Proto string // e.g. "HTTP/1.0"
+ ProtoMajor int // e.g. 1
+ ProtoMinor int // e.g. 0
+
+ // RequestMethod records the method used in the HTTP request.
+ // Header fields such as Content-Length have method-specific meaning.
+ RequestMethod string // e.g. "HEAD", "CONNECT", "GET", etc.
+
+ // Header maps header keys to values. If the response had multiple
+ // headers with the same key, they will be concatenated, with comma
+ // delimiters. (Section 4.2 of RFC 2616 requires that multiple headers
+ // be semantically equivalent to a comma-delimited sequence.) Values
+ // duplicated by other fields in this struct (e.g., ContentLength) are
+ // omitted from Header.
+ //
+ // Keys in the map are canonicalized (see CanonicalHeaderKey).
+ Header map[string]string
+
+ // Body represents the response body.
+ Body io.ReadCloser
+
+ // ContentLength records the length of the associated content. The
+ // value -1 indicates that the length is unknown. Unless RequestMethod
+ // is "HEAD", values >= 0 indicate that the given number of bytes may
+ // be read from Body.
+ ContentLength int64
+
+ // Contains transfer encodings from outer-most to inner-most. Value is
+ // nil, means that "identity" encoding is used.
+ TransferEncoding []string
+
+ // Close records whether the header directed that the connection be
+ // closed after reading Body. The value is advice for clients: neither
+ // ReadResponse nor Response.Write ever closes a connection.
+ Close bool
+
+ // Trailer maps trailer keys to values. Like for Header, if the
+ // response has multiple trailer lines with the same key, they will be
+ // concatenated, delimited by commas.
+ Trailer map[string]string
+}
+
+// ReadResponse reads and returns an HTTP response from r. The RequestMethod
+// parameter specifies the method used in the corresponding request (e.g.,
+// "GET", "HEAD"). Clients must call resp.Body.Close when finished reading
+// resp.Body. After that call, clients can inspect resp.Trailer to find
+// key/value pairs included in the response trailer.
+func ReadResponse(r *bufio.Reader, requestMethod string) (resp *Response, err os.Error) {
+
+ resp = new(Response)
+
+ resp.RequestMethod = strings.ToUpper(requestMethod)
+
+ // Parse the first line of the response.
+ line, err := readLine(r)
+ if err != nil {
+ return nil, err
+ }
+ f := strings.Split(line, " ", 3)
+ if len(f) < 3 {
+ return nil, &badStringError{"malformed HTTP response", line}
+ }
+ resp.Status = f[1] + " " + f[2]
+ resp.StatusCode, err = strconv.Atoi(f[1])
+ if err != nil {
+ return nil, &badStringError{"malformed HTTP status code", f[1]}
+ }
+
+ resp.Proto = f[0]
+ var ok bool
+ if resp.ProtoMajor, resp.ProtoMinor, ok = parseHTTPVersion(resp.Proto); !ok {
+ return nil, &badStringError{"malformed HTTP version", resp.Proto}
+ }
+
+ // Parse the response headers.
+ nheader := 0
+ resp.Header = make(map[string]string)
+ for {
+ key, value, err := readKeyValue(r)
+ if err != nil {
+ return nil, err
+ }
+ if key == "" {
+ break // end of response header
+ }
+ if nheader++; nheader >= maxHeaderLines {
+ return nil, ErrHeaderTooLong
+ }
+ resp.AddHeader(key, value)
+ }
+
+ fixPragmaCacheControl(resp.Header)
+
+ resp.TransferEncoding, err = fixTransferEncoding(resp.Header)
+ if err != nil {
+ return nil, err
+ }
+
+ resp.ContentLength, err = fixLength(resp.StatusCode, resp.RequestMethod,
+ resp.Header, resp.TransferEncoding)
+ if err != nil {
+ return nil, err
+ }
+
+ resp.Close = shouldClose(resp.ProtoMajor, resp.ProtoMinor, resp.Header)
+
+ resp.Trailer, err = fixTrailer(resp.Header, resp.TransferEncoding)
+ if err != nil {
+ return nil, err
+ }
+
+ // Prepare body reader. ContentLength < 0 means chunked encoding,
+ // since multipart is not supported yet
+ if resp.ContentLength < 0 {
+ resp.Body = &body{newChunkedReader(r), resp, r}
+ } else {
+ resp.Body = &body{io.LimitReader(r, resp.ContentLength), nil, nil}
+ }
+
+ return resp, nil
+}
+
+// ffwdClose (fast-forward close) adds a Close method to a Reader which skips
+// ahead until EOF
+type body struct {
+ io.Reader
+ resp *Response // non-nil value means read trailer
+ r *bufio.Reader // underlying wire-format reader for the trailer
+}
+
+func (b *body) Close() os.Error {
+ trashBuf := make([]byte, 1024) // local for thread safety
+ for {
+ _, err := b.Read(trashBuf)
+ if err == nil {
+ continue
+ }
+ if err == os.EOF {
+ break
+ }
+ return err
+ }
+ if b.resp == nil { // not reading trailer
+ return nil
+ }
+
+ // TODO(petar): Put trailer reader code here
+
+ return nil
+}
+
+// RFC2616: Should treat
+// Pragma: no-cache
+// like
+// Cache-Control: no-cache
+func fixPragmaCacheControl(header map[string]string) {
+ if v, present := header["Pragma"]; present && v == "no-cache" {
+ if _, presentcc := header["Cache-Control"]; !presentcc {
+ header["Cache-Control"] = "no-cache"
+ }
+ }
+}
+
+// Parse the trailer header
+func fixTrailer(header map[string]string, te []string) (map[string]string, os.Error) {
+ raw, present := header["Trailer"]
+ if !present {
+ return nil, nil
+ }
+
+ header["Trailer"] = "", false
+ trailer := make(map[string]string)
+ keys := strings.Split(raw, ",", 0)
+ for _, key := range keys {
+ key = CanonicalHeaderKey(strings.TrimSpace(key))
+ switch key {
+ case "Transfer-Encoding", "Trailer", "Content-Length":
+ return nil, &badStringError{"bad trailer key", key}
+ }
+ trailer[key] = ""
+ }
+ if len(trailer) == 0 {
+ return nil, nil
+ }
+ if !chunked(te) {
+ // Trailer and no chunking
+ return nil, ErrUnexpectedTrailer
+ }
+ return trailer, nil
+}
+
+// Sanitize transfer encoding
+func fixTransferEncoding(header map[string]string) ([]string, os.Error) {
+ raw, present := header["Transfer-Encoding"]
+ if !present {
+ return nil, nil
+ }
+
+ header["Transfer-Encoding"] = "", false
+ encodings := strings.Split(raw, ",", 0)
+ te := make([]string, 0, len(encodings))
+ // TODO: Even though we only support "identity" and "chunked"
+ // encodings, the loop below is designed with foresight. One
+ // invariant that must be maintained is that, if present,
+ // chunked encoding must always come first.
+ for _, encoding := range encodings {
+ encoding = strings.ToLower(strings.TrimSpace(encoding))
+ // "identity" encoding is not recored
+ if encoding == "identity" {
+ break
+ }
+ if encoding != "chunked" {
+ return nil, &badStringError{"unsupported transfer encoding", encoding}
+ }
+ te = te[0 : len(te)+1]
+ te[len(te)-1] = encoding
+ }
+ if len(te) > 1 {
+ return nil, &badStringError{"too many transfer encodings", strings.Join(te, ",")}
+ }
+ if len(te) > 0 {
+ // Chunked encoding trumps Content-Length. See RFC 2616
+ // Section 4.4. Currently len(te) > 0 implies chunked
+ // encoding.
+ header["Content-Length"] = "", false
+ return te, nil
+ }
+
+ return nil, nil
+}
+
+func noBodyExpected(requestMethod string) bool {
+ return requestMethod == "HEAD"
+}
+
+// Determine the expected body length, using RFC 2616 Section 4.4. This
+// function is not a method, because ultimately it should be shared by
+// ReadResponse and ReadRequest.
+func fixLength(status int, requestMethod string, header map[string]string, te []string) (int64, os.Error) {
+
+ // Logic based on response type or status
+ if noBodyExpected(requestMethod) {
+ return 0, nil
+ }
+ if status/100 == 1 {
+ return 0, nil
+ }
+ switch status {
+ case 204, 304:
+ return 0, nil
+ }
+
+ // Logic based on Transfer-Encoding
+ if chunked(te) {
+ return -1, nil
+ }
+
+ // Logic based on Content-Length
+ if cl, present := header["Content-Length"]; present {
+ cl = strings.TrimSpace(cl)
+ if cl != "" {
+ n, err := strconv.Atoi64(cl)
+ if err != nil || n < 0 {
+ return -1, &badStringError{"bad Content-Length", cl}
+ }
+ return n, nil
+ } else {
+ header["Content-Length"] = "", false
+ }
+ }
+
+ // Logic based on media type. The purpose of the following code is just
+ // to detect whether the unsupported "multipart/byteranges" is being
+ // used. A proper Content-Type parser is needed in the future.
+ if ct, present := header["Content-Type"]; present {
+ ct = strings.ToLower(ct)
+ if strings.Index(ct, "multipart/byteranges") >= 0 {
+ return -1, ErrNotSupported
+ }
+ }
+
+
+ // Logic based on close
+ return -1, nil
+}
+
+// Determine whether to hang up after sending a request and body, or
+// receiving a response and body
+func shouldClose(major, minor int, header map[string]string) bool {
+ if major < 1 || (major == 1 && minor < 1) {
+ return true
+ } else if v, present := header["Connection"]; present {
+ // TODO: Should split on commas, toss surrounding white space,
+ // and check each field.
+ if v == "close" {
+ return true
+ }
+ }
+ return false
+}
+
+// Checks whether chunked is part of the encodings stack
+func chunked(te []string) bool { return len(te) > 0 && te[0] == "chunked" }
+
+// AddHeader adds a value under the given key. Keys are not case sensitive.
+func (r *Response) AddHeader(key, value string) {
+ key = CanonicalHeaderKey(key)
+
+ oldValues, oldValuesPresent := r.Header[key]
+ if oldValuesPresent {
+ r.Header[key] = oldValues + "," + value
+ } else {
+ r.Header[key] = value
+ }
+}
+
+// GetHeader returns the value of the response header with the given
+// key, and true. If there were multiple headers with this key, their
+// values are concatenated, with a comma delimiter. If there were no
+// response headers with the given key, it returns the empty string and
+// false. Keys are not case sensitive.
+func (r *Response) GetHeader(key string) (value string) {
+ value, _ = r.Header[CanonicalHeaderKey(key)]
+ return
+}
+
+// ProtoAtLeast returns whether the HTTP protocol used
+// in the response is at least major.minor.
+func (r *Response) ProtoAtLeast(major, minor int) bool {
+ return r.ProtoMajor > major ||
+ r.ProtoMajor == major && r.ProtoMinor >= minor
+}
+
+// Writes the response (header, body and trailer) in wire format. This method
+// consults the following fields of resp:
+//
+// StatusCode
+// ProtoMajor
+// ProtoMinor
+// RequestMethod
+// TransferEncoding
+// Trailer
+// Body
+// ContentLength
+// Header, values for non-canonical keys will have unpredictable behavior
+//
+func (resp *Response) Write(w io.Writer) os.Error {
+
+ // RequestMethod should be lower-case
+ resp.RequestMethod = strings.ToUpper(resp.RequestMethod)
+
+ // Status line
+ text, ok := statusText[resp.StatusCode]
+ if !ok {
+ text = "status code " + strconv.Itoa(resp.StatusCode)
+ }
+ io.WriteString(w, "HTTP/"+strconv.Itoa(resp.ProtoMajor)+".")
+ io.WriteString(w, strconv.Itoa(resp.ProtoMinor)+" ")
+ io.WriteString(w, strconv.Itoa(resp.StatusCode)+" "+text+"\r\n")
+
+ // Sanitize the field triple (Body, ContentLength, TransferEncoding)
+ if noBodyExpected(resp.RequestMethod) {
+ resp.Body = nil
+ resp.TransferEncoding = nil
+ // resp.ContentLength is expected to hold Content-Length
+ if resp.ContentLength < 0 {
+ return ErrMissingContentLength
+ }
+ } else {
+ if !resp.ProtoAtLeast(1, 1) || resp.Body == nil {
+ resp.TransferEncoding = nil
+ }
+ if chunked(resp.TransferEncoding) {
+ resp.ContentLength = -1
+ } else if resp.Body != nil {
+ // For safety, consider sending a 0-length body an
+ // error
+ if resp.ContentLength <= 0 {
+ return &ProtocolError{"zero body length"}
+ }
+ } else { // no chunking, no body
+ resp.ContentLength = 0
+ }
+ }
+
+ // Write Content-Length and/or Transfer-Encoding whose values are a
+ // function of the sanitized field triple (Body, ContentLength,
+ // TransferEncoding)
+ if chunked(resp.TransferEncoding) {
+ io.WriteString(w, "Transfer-Encoding: chunked\r\n")
+ } else {
+ io.WriteString(w, "Content-Length: ")
+ io.WriteString(w, strconv.Itoa64(resp.ContentLength)+"\r\n")
+ }
+ if resp.Header != nil {
+ resp.Header["Content-Length"] = "", false
+ resp.Header["Transfer-Encoding"] = "", false
+ }
+
+ // Sanitize Trailer
+ if !chunked(resp.TransferEncoding) {
+ resp.Trailer = nil
+ } else if resp.Trailer != nil {
+ // TODO: At some point, there should be a generic mechanism for
+ // writing long headers, using HTTP line splitting
+ io.WriteString(w, "Trailer: ")
+ needComma := false
+ for k, _ := range resp.Trailer {
+ k = CanonicalHeaderKey(k)
+ switch k {
+ case "Transfer-Encoding", "Trailer", "Content-Length":
+ return &badStringError{"invalid Trailer key", k}
+ }
+ if needComma {
+ io.WriteString(w, ",")
+ }
+ io.WriteString(w, k)
+ needComma = true
+ }
+ io.WriteString(w, "\r\n")
+ }
+ if resp.Header != nil {
+ resp.Header["Trailer"] = "", false
+ }
+
+ // Rest of header
+ for k, v := range resp.Header {
+ io.WriteString(w, k+": "+v+"\r\n")
+ }
+
+ // End-of-header
+ io.WriteString(w, "\r\n")
+
+ // Write body
+ if resp.Body != nil {
+ var err os.Error
+ if chunked(resp.TransferEncoding) {
+ cw := NewChunkedWriter(w)
+ _, err = io.Copy(cw, resp.Body)
+ if err == nil {
+ err = cw.Close()
+ }
+ } else {
+ _, err = io.Copy(w, io.LimitReader(resp.Body, resp.ContentLength))
+ }
+ if err != nil {
+ return err
+ }
+ }
+
+ // TODO(petar): Place trailer writer code here.
+ if chunked(resp.TransferEncoding) {
+ // Last chunk, empty trailer
+ io.WriteString(w, "\r\n")
+ }
+
+ // Success
+ return nil
+}