From 92210eefb2fad766f4fe1664495a3c351a32eb89 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Tue, 12 Apr 2011 09:35:07 -0700 Subject: [PATCH] http: client gzip support R=adg, rsc, bradfitzwork CC=golang-dev https://golang.org/cl/4389048 --- src/pkg/http/transport.go | 32 +++++++++- src/pkg/http/transport_test.go | 103 ++++++++++++++++++++++++++++++++- 2 files changed, 131 insertions(+), 4 deletions(-) diff --git a/src/pkg/http/transport.go b/src/pkg/http/transport.go index 797d134aa8..7fa37af3b6 100644 --- a/src/pkg/http/transport.go +++ b/src/pkg/http/transport.go @@ -6,6 +6,7 @@ package http import ( "bufio" + "compress/gzip" "crypto/tls" "encoding/base64" "fmt" @@ -39,8 +40,9 @@ type Transport struct { // TODO: tunable on timeout on cached connections // TODO: optional pipelining - IgnoreEnvironment bool // don't look at environment variables for proxy configuration - DisableKeepAlives bool + IgnoreEnvironment bool // don't look at environment variables for proxy configuration + DisableKeepAlives bool + DisableCompression bool // MaxIdleConnsPerHost, if non-zero, controls the maximum idle // (keep-alive) to keep to keep per-host. If zero, @@ -474,6 +476,19 @@ func (pc *persistConn) roundTrip(req *Request) (resp *Response, err os.Error) { pc.mutateRequestFunc(req) } + // Ask for a compressed version if the caller didn't set their + // own value for Accept-Encoding. We only attempted to + // uncompress the gzip stream if we were the layer that + // requested it. + requestedGzip := false + if !pc.t.DisableCompression && req.Header.Get("Accept-Encoding") == "" { + // Request gzip only, not deflate. Deflate is ambiguous and + // as universally supported anyway. + // See: http://www.gzip.org/zlib/zlib_faq.html#faq38 + requestedGzip = true + req.Header.Set("Accept-Encoding", "gzip") + } + pc.lk.Lock() pc.numExpectedResponses++ pc.lk.Unlock() @@ -490,6 +505,19 @@ func (pc *persistConn) roundTrip(req *Request) (resp *Response, err os.Error) { pc.lk.Lock() pc.numExpectedResponses-- pc.lk.Unlock() + + if re.err == nil && requestedGzip && re.res.Header.Get("Content-Encoding") == "gzip" { + re.res.Header.Del("Content-Encoding") + re.res.Header.Del("Content-Length") + re.res.ContentLength = -1 + var err os.Error + re.res.Body, err = gzip.NewReader(re.res.Body) + if err != nil { + pc.close() + return nil, err + } + } + return re.res, re.err } diff --git a/src/pkg/http/transport_test.go b/src/pkg/http/transport_test.go index 05328fe5b0..f83deedfc4 100644 --- a/src/pkg/http/transport_test.go +++ b/src/pkg/http/transport_test.go @@ -7,6 +7,8 @@ package http_test import ( + "bytes" + "compress/gzip" "fmt" . "http" "http/httptest" @@ -24,7 +26,7 @@ var hostPortHandler = HandlerFunc(func(w ResponseWriter, r *Request) { if r.FormValue("close") == "true" { w.Header().Set("Connection", "close") } - fmt.Fprintf(w, "%s", r.RemoteAddr) + w.Write([]byte(r.RemoteAddr)) }) // Two subsequent requests and verify their response is the same. @@ -179,7 +181,7 @@ func TestTransportIdleCacheKeys(t *testing.T) { func TestTransportMaxPerHostIdleConns(t *testing.T) { ch := make(chan string) ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { - fmt.Fprintf(w, "%s", <-ch) + w.Write([]byte(<-ch)) })) defer ts.Close() maxIdleConns := 2 @@ -338,6 +340,7 @@ func TestTransportNilURL(t *testing.T) { req.Proto = "HTTP/1.1" req.ProtoMajor = 1 req.ProtoMinor = 1 + req.Header = make(Header) tr := &Transport{} res, err := tr.RoundTrip(req) @@ -349,3 +352,99 @@ func TestTransportNilURL(t *testing.T) { t.Fatalf("Expected response body of %q; got %q", e, g) } } + +func TestTransportGzip(t *testing.T) { + const testString = "The test string aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + if g, e := r.Header.Get("Accept-Encoding"), "gzip"; g != e { + t.Errorf("Accept-Encoding = %q, want %q", g, e) + } + w.Header().Set("Content-Encoding", "gzip") + gz, _ := gzip.NewWriter(w) + defer gz.Close() + gz.Write([]byte(testString)) + + })) + defer ts.Close() + + c := &Client{Transport: &Transport{}} + res, _, err := c.Get(ts.URL) + if err != nil { + t.Fatal(err) + } + body, err := ioutil.ReadAll(res.Body) + if err != nil { + t.Fatal(err) + } + if g, e := string(body), testString; g != e { + t.Fatalf("body = %q; want %q", g, e) + } + if g, e := res.Header.Get("Content-Encoding"), ""; g != e { + t.Fatalf("Content-Encoding = %q; want %q", g, e) + } +} + +// TestTransportGzipRecursive sends a gzip quine and checks that the +// client gets the same value back. This is more cute than anything, +// but checks that we don't recurse forever, and checks that +// Content-Encoding is removed. +func TestTransportGzipRecursive(t *testing.T) { + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + w.Header().Set("Content-Encoding", "gzip") + w.Write(rgz) + })) + defer ts.Close() + + c := &Client{Transport: &Transport{}} + res, _, err := c.Get(ts.URL) + if err != nil { + t.Fatal(err) + } + body, err := ioutil.ReadAll(res.Body) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(body, rgz) { + t.Fatalf("Incorrect result from recursive gz:\nhave=%x\nwant=%x", + body, rgz) + } + if g, e := res.Header.Get("Content-Encoding"), ""; g != e { + t.Fatalf("Content-Encoding = %q; want %q", g, e) + } +} + +// rgz is a gzip quine that uncompresses to itself. +var rgz = []byte{ + 0x1f, 0x8b, 0x08, 0x08, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x72, 0x65, 0x63, 0x75, 0x72, 0x73, + 0x69, 0x76, 0x65, 0x00, 0x92, 0xef, 0xe6, 0xe0, + 0x60, 0x00, 0x83, 0xa2, 0xd4, 0xe4, 0xd2, 0xa2, + 0xe2, 0xcc, 0xb2, 0x54, 0x06, 0x00, 0x00, 0x17, + 0x00, 0xe8, 0xff, 0x92, 0xef, 0xe6, 0xe0, 0x60, + 0x00, 0x83, 0xa2, 0xd4, 0xe4, 0xd2, 0xa2, 0xe2, + 0xcc, 0xb2, 0x54, 0x06, 0x00, 0x00, 0x17, 0x00, + 0xe8, 0xff, 0x42, 0x12, 0x46, 0x16, 0x06, 0x00, + 0x05, 0x00, 0xfa, 0xff, 0x42, 0x12, 0x46, 0x16, + 0x06, 0x00, 0x05, 0x00, 0xfa, 0xff, 0x00, 0x05, + 0x00, 0xfa, 0xff, 0x00, 0x14, 0x00, 0xeb, 0xff, + 0x42, 0x12, 0x46, 0x16, 0x06, 0x00, 0x05, 0x00, + 0xfa, 0xff, 0x00, 0x05, 0x00, 0xfa, 0xff, 0x00, + 0x14, 0x00, 0xeb, 0xff, 0x42, 0x88, 0x21, 0xc4, + 0x00, 0x00, 0x14, 0x00, 0xeb, 0xff, 0x42, 0x88, + 0x21, 0xc4, 0x00, 0x00, 0x14, 0x00, 0xeb, 0xff, + 0x42, 0x88, 0x21, 0xc4, 0x00, 0x00, 0x14, 0x00, + 0xeb, 0xff, 0x42, 0x88, 0x21, 0xc4, 0x00, 0x00, + 0x14, 0x00, 0xeb, 0xff, 0x42, 0x88, 0x21, 0xc4, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0x00, 0x00, + 0x00, 0xff, 0xff, 0x00, 0x17, 0x00, 0xe8, 0xff, + 0x42, 0x88, 0x21, 0xc4, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0x00, 0x00, 0x00, 0xff, 0xff, 0x00, + 0x17, 0x00, 0xe8, 0xff, 0x42, 0x12, 0x46, 0x16, + 0x06, 0x00, 0x00, 0x00, 0xff, 0xff, 0x01, 0x08, + 0x00, 0xf7, 0xff, 0x3d, 0xb1, 0x20, 0x85, 0xfa, + 0x00, 0x00, 0x00, 0x42, 0x12, 0x46, 0x16, 0x06, + 0x00, 0x00, 0x00, 0xff, 0xff, 0x01, 0x08, 0x00, + 0xf7, 0xff, 0x3d, 0xb1, 0x20, 0x85, 0xfa, 0x00, + 0x00, 0x00, 0x3d, 0xb1, 0x20, 0x85, 0xfa, 0x00, + 0x00, 0x00, +} -- 2.50.0