From: Steve Newman <devnull@localhost>
Date: Tue, 9 Jun 2009 17:58:58 +0000 (-0700)
Subject: Basic HTTP client.
X-Git-Tag: weekly.2009-11-06~1414
X-Git-Url: http://www.git.cypherpunks.su/?a=commitdiff_plain;h=f315fb3d56746ddd14dbfeeea106564349bb5ce9;p=gostls13.git

Basic HTTP client.

R=rsc
APPROVED=rsc
DELTA=392  (386 added, 2 deleted, 4 changed)
OCL=29963
CL=30107
---

diff --git a/src/pkg/http/Makefile b/src/pkg/http/Makefile
index 0a029497c9..34445b5770 100644
--- a/src/pkg/http/Makefile
+++ b/src/pkg/http/Makefile
@@ -39,6 +39,7 @@ O2=\
 	request.$O\
 
 O3=\
+	client.$O\
 	server.$O\
 
 O4=\
@@ -57,7 +58,7 @@ a2: $(O2)
 	rm -f $(O2)
 
 a3: $(O3)
-	$(AR) grc _obj$D/http.a server.$O
+	$(AR) grc _obj$D/http.a client.$O server.$O
 	rm -f $(O3)
 
 a4: $(O4)
diff --git a/src/pkg/http/client.go b/src/pkg/http/client.go
new file mode 100644
index 0000000000..61ec56d06a
--- /dev/null
+++ b/src/pkg/http/client.go
@@ -0,0 +1,227 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Primitive HTTP client.  See RFC 2616.
+
+package http
+
+import (
+	"bufio";
+	"fmt";
+	"http";
+	"io";
+	"log";
+	"net";
+	"os";
+	"strings";
+	"strconv";
+)
+
+// Response represents the response from an HTTP request.
+type Response struct {
+	Status string; // e.g. "200 OK"
+	StatusCode int; // e.g. 200
+
+	// Header maps header keys to values.  If the response had multiple
+	// headers with the same key, they will be concatenated, with comma
+	// delimiters.  (Section 4.2 of RFC 2616 requires that multiple headers
+	// be semantically equivalent to a comma-delimited sequence.)
+	//
+	// Keys in the map are canonicalized (see CanonicalHeaderKey).
+	Header map [string] string;
+
+	// Stream from which the response body can be read.
+	Body io.ReadCloser;
+}
+
+// GetHeader returns the value of the response header with the given
+// key, and true.  If there were multiple headers with this key, their
+// values are concatenated, with a comma delimiter.  If there were no
+// response headers with the given key, it returns the empty string and
+// false.  Keys are not case sensitive.
+func (r *Response) GetHeader(key string) (value string) {
+	value, present := r.Header[CanonicalHeaderKey(key)];
+	return;
+}
+
+// AddHeader adds a value under the given key.  Keys are not case sensitive.
+func (r *Response) AddHeader(key, value string) {
+	key = CanonicalHeaderKey(key);
+
+	oldValues, oldValuesPresent := r.Header[key];
+	if oldValuesPresent {
+		r.Header[key] = oldValues + "," + value;
+	} else {
+		r.Header[key] = value;
+	}
+}
+
+// Given a string of the form "host", "host:port", or "[ipv6::address]:port",
+// return true if the string includes a port.
+func hasPort(s string) bool {
+	return strings.LastIndex(s, ":") > strings.LastIndex(s, "]");
+}
+
+// Used in Send to implement io.ReadCloser by bundling together the
+// io.BufReader through which we read the response, and the underlying
+// network connection.
+type readClose struct {
+	io.Reader;
+	io.Closer;
+}
+
+// Send issues an HTTP request.  Caller should close resp.Body when done reading it.
+//
+// This method consults the following fields of req:
+//
+//	Url
+//	Method (defaults to "GET")
+//	Proto (defaults to "HTTP/1.0")
+//	UserAgent (if empty, currently defaults to http.Client; may change)
+//	Referer (if empty, no Referer header will be supplied)
+//	Header
+//	Body (if nil, defaults to empty body)
+//
+// The following fields are redundant and are ignored:
+//
+//	RawUrl
+//	ProtoMajor
+//	ProtoMinor
+//	Close
+//	Host
+//
+// TODO: support persistent connections (multiple requests on a single connection).
+// send() method is nonpublic because, when we refactor the code for persistent
+// connections, it may no longer make sense to have a method with this signature.
+func send(req *Request) (resp *Response, err os.Error) {
+	if req.Url.Scheme != "http" {
+		return nil, os.ErrorString("Unsupported protocol: " + req.Url.Scheme);
+	}
+
+	addr := req.Url.Host;
+	if !hasPort(addr) {
+		addr += ":http";
+	}
+	conn, err := net.Dial("tcp", "", addr);
+	if err != nil {
+		return nil, os.ErrorString("Error dialing " + addr + ": " + err.String());
+	}
+
+	// Close the connection if we encounter an error during header parsing.  We'll
+	// cancel this when we hand the connection off to our caller.
+	defer func() { if conn != nil { conn.Close() } }();
+
+	err = req.write(conn);
+	if err != nil {
+		return nil, err;
+	}
+
+	// Parse the first line of the response.
+	resp = new(Response);
+	resp.Header = make(map[string] string);
+	reader := bufio.NewReader(conn);
+
+	line, err := readLine(reader);
+	if err != nil {
+		return nil, err;
+	}
+	ss := strings.Split(line, " ");
+	if len(ss) != 3 {
+		return nil, os.ErrorString(fmt.Sprintf("Invalid first line in HTTP response: %q", line));
+	}
+	resp.Status = ss[1] + " " + ss[2];
+	resp.StatusCode, err = strconv.Atoi(ss[1]);
+	if err != nil {
+		return nil, os.ErrorString(fmt.Sprintf("Invalid status code in HTTP response %q", line));
+	}
+
+	// Parse the response headers.
+	for {
+		key, value, err := readKeyValue(reader);
+		if err != nil {
+			return nil, err;
+		}
+		if key == "" {
+			break; // end of response header
+		}
+		resp.AddHeader(key, value);
+	}
+
+	resp.Body = readClose{reader, conn};
+	conn = nil; // so that defered func won't close it
+	err = nil;
+	return;
+}
+
+// True if the specified HTTP status code is one for which the Get utility should
+// automatically redirect.
+func shouldRedirect(statusCode int) bool {
+	switch statusCode {
+	case StatusMovedPermanently, StatusFound, StatusSeeOther, StatusTemporaryRedirect:
+		return true;
+	}
+	return false;
+}
+
+// Get issues a GET to the specified URL.  If the response is one of the following
+// redirect codes, it follows the redirect, up to a maximum of 10 redirects:
+//
+//    301 (Moved Permanently)
+//    302 (Found)
+//    303 (See Other)
+//    307 (Temporary Redirect)
+//
+// finalUrl is the URL from which the response was fetched -- identical to the input
+// URL unless redirects were followed.
+//
+// Caller should close r.Body when done reading it.
+func Get(url string) (r *Response, finalUrl string, err os.Error) {
+	// TODO: if/when we add cookie support, the redirected request shouldn't
+	// necessarily supply the same cookies as the original.
+	// TODO: adjust referrer header on redirects.
+	for redirectCount := 0; redirectCount < 10; redirectCount++ {
+		var req Request;
+		req.Url, err = ParseURL(url);
+		if err != nil {
+			return nil, url, err;
+		}
+
+		r, err := send(&req);
+		if err != nil {
+			return nil, url, err;
+		}
+
+		if !shouldRedirect(r.StatusCode) {
+			return r, url, nil;
+		}
+
+		r.Body.Close();
+		url := r.GetHeader("Location");
+		if url == "" {
+			return r, url, os.ErrorString("302 result with no Location header");
+		}
+	}
+
+	return nil, url, os.ErrorString("Too many redirects");
+}
+
+
+// Post issues a POST to the specified URL.
+//
+// Caller should close resp.Body when done reading it.
+func Post(url string, requestBody io.Reader) (r *Response, err os.Error) {
+	// NOTE TO REVIEWER: this could share more code with Get, waiting for API to settle
+	// down before cleaning up that detail.
+
+	var req Request;
+	req.Method = "POST";
+	req.Body = requestBody;
+	req.Url, err = ParseURL(url);
+	if err != nil {
+		return nil, err;
+	}
+
+	return send(&req);
+}
+
diff --git a/src/pkg/http/client_test.go b/src/pkg/http/client_test.go
new file mode 100644
index 0000000000..e9354fc40d
--- /dev/null
+++ b/src/pkg/http/client_test.go
@@ -0,0 +1,40 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Tests for client.go
+
+package http
+
+import (
+	"fmt";
+	"http";
+	"io";
+	"strings";
+	"testing";
+)
+
+func TestClient(t *testing.T) {
+	// TODO: add a proper test suite.  Current test merely verifies that
+	// we can retrieve the Google home page.
+	
+	r, url, err := Get("http://www.google.com");
+	var b []byte;
+	if err == nil {
+		b, err = io.ReadAll(r.Body);
+		r.Body.Close();
+	}
+
+	// TODO: io.ErrEOF check is needed because we're sometimes getting
+	// this error when nothing is actually wrong.  rsc suspects a bug
+	// in bufio.  Can remove the ErrEOF check once the bug is fixed
+	// (expected to occur within a few weeks of this writing, 6/9/09).
+	if err != nil && err != io.ErrEOF {
+		t.Errorf("Error fetching URL: %v", err);
+	} else {
+		s := string(b);
+		if (!strings.HasPrefix(s, "<html>")) {
+			t.Errorf("Incorrect page body (did not begin with <html>): %q", s);
+		}
+	}
+}
diff --git a/src/pkg/http/request.go b/src/pkg/http/request.go
index 76dd6f30c1..f8c37ec1ea 100644
--- a/src/pkg/http/request.go
+++ b/src/pkg/http/request.go
@@ -4,11 +4,9 @@
 
 // HTTP Request reading and parsing.
 
-// The http package implements parsing of HTTP requests and URLs
-// and provides an extensible HTTP server.
-//
-// In the future it should also implement parsing of HTTP replies
-// and provide methods to fetch URLs via HTTP.
+// The http package implements parsing of HTTP requests, replies,
+// and URLs and provides an extensible HTTP server and a basic
+// HTTP client.
 package http
 
 import (
@@ -106,6 +104,56 @@ func (r *Request) ProtoAtLeast(major, minor int) bool {
 		r.ProtoMajor == major && r.ProtoMinor >= minor
 }
 
+// Return value if nonempty, def otherwise.
+func valueOrDefault(value, def string) string {
+	if value != "" {
+		return value;
+	}
+	return def;
+}
+
+// TODO(rsc): Change default UserAgent before open-source release.
+const defaultUserAgent = "http.Client";
+
+// Write an HTTP request -- header and body -- in wire format.
+// See Send for a list of which Request fields we use.
+func (req *Request) write(w io.Writer) os.Error {
+	uri := "/" + URLEscape(req.Url.Path);
+	if req.Url.RawQuery != "" {
+		uri += "?" + req.Url.RawQuery;
+	}
+
+	fmt.Fprintf(w, "%s %s %s\r\n", valueOrDefault(req.Method, "GET"), uri, valueOrDefault(req.Proto, "HTTP/1.0"));
+	fmt.Fprintf(w, "Host: %s\r\n", req.Url.Host);
+	fmt.Fprintf(w, "User-Agent: %s\r\n", valueOrDefault(req.UserAgent, defaultUserAgent));
+
+	if (req.Referer != "") {
+		fmt.Fprintf(w, "Referer: %s\r\n", req.Referer);
+	}
+
+	// TODO: split long values?  (If so, should share code with Conn.Write)
+	// TODO: if Header includes values for Host, User-Agent, or Referer, this
+	// may conflict with the User-Agent or Referer headers we add manually.
+	// One solution would be to remove the Host, UserAgent, and Referer fields
+	// from Request, and introduce Request methods along the lines of
+	// Response.{GetHeader,AddHeader} and string constants for "Host",
+	// "User-Agent" and "Referer".
+	for k, v := range req.Header {
+		io.WriteString(w, k + ": " + v + "\r\n");
+	}
+
+	io.WriteString(w, "\r\n");
+
+	if req.Body != nil {
+		nCopied, err := io.Copy(req.Body, w);
+		if err != nil && err != io.ErrEOF {
+			return err;
+		}
+	}
+
+	return nil;
+}
+
 // Read a line of bytes (up to \n) from b.
 // Give up if the line exceeds maxLineLength.
 // The returned bytes are a pointer into storage in
diff --git a/src/pkg/strings/strings.go b/src/pkg/strings/strings.go
index 2e3dc0215e..0350907771 100644
--- a/src/pkg/strings/strings.go
+++ b/src/pkg/strings/strings.go
@@ -53,6 +53,21 @@ func Index(s, sep string) int {
 	return -1
 }
 
+// Index returns the index of the last instance of sep in s, or -1 if sep is not present in s.
+func LastIndex(s, sep string) int {
+	n := len(sep);
+	if n == 0 {
+		return len(s)
+	}
+	c := sep[0];
+	for i := len(s)-n; i >= 0; i-- {
+		if s[i] == c && (n == 1 || s[i:i+n] == sep) {
+			return i
+		}
+	}
+	return -1
+}
+
 // Split returns the array representing the substrings of s separated by string sep. Adjacent
 // occurrences of sep produce empty substrings.  If sep is empty, it is the same as Explode.
 func Split(s, sep string) []string {
diff --git a/src/pkg/strings/strings_test.go b/src/pkg/strings/strings_test.go
index 05e6620321..6464ca3992 100644
--- a/src/pkg/strings/strings_test.go
+++ b/src/pkg/strings/strings_test.go
@@ -26,6 +26,61 @@ var faces = "☺☻☹";
 var commas = "1,2,3,4";
 var dots = "1....2....3....4";
 
+type IndexTest struct {
+	s string;
+	sep string;
+	out int;
+}
+
+var indexTests = []IndexTest {
+	IndexTest{"", "", 0},
+	IndexTest{"", "a", -1},
+	IndexTest{"", "foo", -1},
+	IndexTest{"fo", "foo", -1},
+	IndexTest{"foo", "foo", 0},
+	IndexTest{"oofofoofooo", "f", 2},
+	IndexTest{"oofofoofooo", "foo", 4},
+	IndexTest{"barfoobarfoo", "foo", 3},
+	IndexTest{"foo", "", 0},
+	IndexTest{"foo", "o", 1},
+	IndexTest{"abcABCabc", "A", 3},
+}
+
+var lastIndexTests = []IndexTest {
+	IndexTest{"", "", 0},
+	IndexTest{"", "a", -1},
+	IndexTest{"", "foo", -1},
+	IndexTest{"fo", "foo", -1},
+	IndexTest{"foo", "foo", 0},
+	IndexTest{"oofofoofooo", "f", 7},
+	IndexTest{"oofofoofooo", "foo", 7},
+	IndexTest{"barfoobarfoo", "foo", 9},
+	IndexTest{"foo", "", 3},
+	IndexTest{"foo", "o", 2},
+	IndexTest{"abcABCabc", "A", 3},
+	IndexTest{"abcABCabc", "a", 6},
+}
+
+// Execute f on each test case.  funcName should be the name of f; it's used
+// in failure reports.
+func runIndexTests(t *testing.T, f func(s, sep string) int, funcName string, testCases []IndexTest) {
+	for i,test := range testCases {
+		actual := f(test.s, test.sep);
+		if (actual != test.out) {
+			t.Errorf("%s(%q,%q) = %v; want %v", funcName, test.s, test.sep, actual, test.out);
+		}
+	}
+}
+
+func TestIndex(t *testing.T) {
+	runIndexTests(t, Index, "Index", indexTests);
+}
+
+func TestLastIndex(t *testing.T) {
+	runIndexTests(t, LastIndex, "LastIndex", lastIndexTests);
+}
+
+
 type ExplodeTest struct {
 	s string;
 	a []string;