http: client gzip support

R=adg, rsc, bradfitzwork
CC=golang-dev
https://golang.org/cl/4389048

diff --git a/src/pkg/http/transport.go b/src/pkg/http/transport.go
index 797d134aa8511b645783d4c62807fb14e4f509d1..7fa37af3b60d5160090641c7039d8ba2d6a4347d 100644 (file)
--- a/src/pkg/http/transport.go
+++ b/src/pkg/http/transport.go
@@ -6,6 +6,7 @@ package http
 
 import (
        "bufio"
+       "compress/gzip"
        "crypto/tls"
        "encoding/base64"
        "fmt"
@@ -39,8 +40,9 @@ type Transport struct {
        // TODO: tunable on timeout on cached connections
        // TODO: optional pipelining
 
-       IgnoreEnvironment bool // don't look at environment variables for proxy configuration
-       DisableKeepAlives bool
+       IgnoreEnvironment  bool // don't look at environment variables for proxy configuration
+       DisableKeepAlives  bool
+       DisableCompression bool
 
        // MaxIdleConnsPerHost, if non-zero, controls the maximum idle
        // (keep-alive) to keep to keep per-host.  If zero,
@@ -474,6 +476,19 @@ func (pc *persistConn) roundTrip(req *Request) (resp *Response, err os.Error) {
                pc.mutateRequestFunc(req)
        }
 
+       // Ask for a compressed version if the caller didn't set their
+       // own value for Accept-Encoding. We only attempted to
+       // uncompress the gzip stream if we were the layer that
+       // requested it.
+       requestedGzip := false
+       if !pc.t.DisableCompression && req.Header.Get("Accept-Encoding") == "" {
+               // Request gzip only, not deflate. Deflate is ambiguous and 
+               // as universally supported anyway.
+               // See: http://www.gzip.org/zlib/zlib_faq.html#faq38
+               requestedGzip = true
+               req.Header.Set("Accept-Encoding", "gzip")
+       }
+
        pc.lk.Lock()
        pc.numExpectedResponses++
        pc.lk.Unlock()
@@ -490,6 +505,19 @@ func (pc *persistConn) roundTrip(req *Request) (resp *Response, err os.Error) {
        pc.lk.Lock()
        pc.numExpectedResponses--
        pc.lk.Unlock()
+
+       if re.err == nil && requestedGzip && re.res.Header.Get("Content-Encoding") == "gzip" {
+               re.res.Header.Del("Content-Encoding")
+               re.res.Header.Del("Content-Length")
+               re.res.ContentLength = -1
+               var err os.Error
+               re.res.Body, err = gzip.NewReader(re.res.Body)
+               if err != nil {
+                       pc.close()
+                       return nil, err
+               }
+       }
+
        return re.res, re.err
 }
 

diff --git a/src/pkg/http/transport_test.go b/src/pkg/http/transport_test.go
index 05328fe5b0a4d2e57eaaf3942e086c5a4e9b381b..f83deedfc4adf1d0b7fe868c98c4d68926da4a48 100644 (file)
--- a/src/pkg/http/transport_test.go
+++ b/src/pkg/http/transport_test.go
@@ -7,6 +7,8 @@
 package http_test
 
 import (
+       "bytes"
+       "compress/gzip"
        "fmt"
        . "http"
        "http/httptest"
@@ -24,7 +26,7 @@ var hostPortHandler = HandlerFunc(func(w ResponseWriter, r *Request) {
        if r.FormValue("close") == "true" {
                w.Header().Set("Connection", "close")
        }
-       fmt.Fprintf(w, "%s", r.RemoteAddr)
+       w.Write([]byte(r.RemoteAddr))
 })
 
 // Two subsequent requests and verify their response is the same.
@@ -179,7 +181,7 @@ func TestTransportIdleCacheKeys(t *testing.T) {
 func TestTransportMaxPerHostIdleConns(t *testing.T) {
        ch := make(chan string)
        ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) {
-               fmt.Fprintf(w, "%s", <-ch)
+               w.Write([]byte(<-ch))
        }))
        defer ts.Close()
        maxIdleConns := 2
@@ -338,6 +340,7 @@ func TestTransportNilURL(t *testing.T) {
        req.Proto = "HTTP/1.1"
        req.ProtoMajor = 1
        req.ProtoMinor = 1
+       req.Header = make(Header)
 
        tr := &Transport{}
        res, err := tr.RoundTrip(req)
@@ -349,3 +352,99 @@ func TestTransportNilURL(t *testing.T) {
                t.Fatalf("Expected response body of %q; got %q", e, g)
        }
 }
+
+func TestTransportGzip(t *testing.T) {
+       const testString = "The test string aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+       ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) {
+               if g, e := r.Header.Get("Accept-Encoding"), "gzip"; g != e {
+                       t.Errorf("Accept-Encoding = %q, want %q", g, e)
+               }
+               w.Header().Set("Content-Encoding", "gzip")
+               gz, _ := gzip.NewWriter(w)
+               defer gz.Close()
+               gz.Write([]byte(testString))
+
+       }))
+       defer ts.Close()
+
+       c := &Client{Transport: &Transport{}}
+       res, _, err := c.Get(ts.URL)
+       if err != nil {
+               t.Fatal(err)
+       }
+       body, err := ioutil.ReadAll(res.Body)
+       if err != nil {
+               t.Fatal(err)
+       }
+       if g, e := string(body), testString; g != e {
+               t.Fatalf("body = %q; want %q", g, e)
+       }
+       if g, e := res.Header.Get("Content-Encoding"), ""; g != e {
+               t.Fatalf("Content-Encoding = %q; want %q", g, e)
+       }
+}
+
+// TestTransportGzipRecursive sends a gzip quine and checks that the
+// client gets the same value back. This is more cute than anything,
+// but checks that we don't recurse forever, and checks that
+// Content-Encoding is removed.
+func TestTransportGzipRecursive(t *testing.T) {
+       ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) {
+               w.Header().Set("Content-Encoding", "gzip")
+               w.Write(rgz)
+       }))
+       defer ts.Close()
+
+       c := &Client{Transport: &Transport{}}
+       res, _, err := c.Get(ts.URL)
+       if err != nil {
+               t.Fatal(err)
+       }
+       body, err := ioutil.ReadAll(res.Body)
+       if err != nil {
+               t.Fatal(err)
+       }
+       if !bytes.Equal(body, rgz) {
+               t.Fatalf("Incorrect result from recursive gz:\nhave=%x\nwant=%x",
+                       body, rgz)
+       }
+       if g, e := res.Header.Get("Content-Encoding"), ""; g != e {
+               t.Fatalf("Content-Encoding = %q; want %q", g, e)
+       }
+}
+
+// rgz is a gzip quine that uncompresses to itself.
+var rgz = []byte{
+       0x1f, 0x8b, 0x08, 0x08, 0x00, 0x00, 0x00, 0x00,
+       0x00, 0x00, 0x72, 0x65, 0x63, 0x75, 0x72, 0x73,
+       0x69, 0x76, 0x65, 0x00, 0x92, 0xef, 0xe6, 0xe0,
+       0x60, 0x00, 0x83, 0xa2, 0xd4, 0xe4, 0xd2, 0xa2,
+       0xe2, 0xcc, 0xb2, 0x54, 0x06, 0x00, 0x00, 0x17,
+       0x00, 0xe8, 0xff, 0x92, 0xef, 0xe6, 0xe0, 0x60,
+       0x00, 0x83, 0xa2, 0xd4, 0xe4, 0xd2, 0xa2, 0xe2,
+       0xcc, 0xb2, 0x54, 0x06, 0x00, 0x00, 0x17, 0x00,
+       0xe8, 0xff, 0x42, 0x12, 0x46, 0x16, 0x06, 0x00,
+       0x05, 0x00, 0xfa, 0xff, 0x42, 0x12, 0x46, 0x16,
+       0x06, 0x00, 0x05, 0x00, 0xfa, 0xff, 0x00, 0x05,
+       0x00, 0xfa, 0xff, 0x00, 0x14, 0x00, 0xeb, 0xff,
+       0x42, 0x12, 0x46, 0x16, 0x06, 0x00, 0x05, 0x00,
+       0xfa, 0xff, 0x00, 0x05, 0x00, 0xfa, 0xff, 0x00,
+       0x14, 0x00, 0xeb, 0xff, 0x42, 0x88, 0x21, 0xc4,
+       0x00, 0x00, 0x14, 0x00, 0xeb, 0xff, 0x42, 0x88,
+       0x21, 0xc4, 0x00, 0x00, 0x14, 0x00, 0xeb, 0xff,
+       0x42, 0x88, 0x21, 0xc4, 0x00, 0x00, 0x14, 0x00,
+       0xeb, 0xff, 0x42, 0x88, 0x21, 0xc4, 0x00, 0x00,
+       0x14, 0x00, 0xeb, 0xff, 0x42, 0x88, 0x21, 0xc4,
+       0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0x00, 0x00,
+       0x00, 0xff, 0xff, 0x00, 0x17, 0x00, 0xe8, 0xff,
+       0x42, 0x88, 0x21, 0xc4, 0x00, 0x00, 0x00, 0x00,
+       0xff, 0xff, 0x00, 0x00, 0x00, 0xff, 0xff, 0x00,
+       0x17, 0x00, 0xe8, 0xff, 0x42, 0x12, 0x46, 0x16,
+       0x06, 0x00, 0x00, 0x00, 0xff, 0xff, 0x01, 0x08,
+       0x00, 0xf7, 0xff, 0x3d, 0xb1, 0x20, 0x85, 0xfa,
+       0x00, 0x00, 0x00, 0x42, 0x12, 0x46, 0x16, 0x06,
+       0x00, 0x00, 0x00, 0xff, 0xff, 0x01, 0x08, 0x00,
+       0xf7, 0xff, 0x3d, 0xb1, 0x20, 0x85, 0xfa, 0x00,
+       0x00, 0x00, 0x3d, 0xb1, 0x20, 0x85, 0xfa, 0x00,
+       0x00, 0x00,
+}