return 0
}
+// Return true if the specified character should be escaped when appearing in a
+// URL string.
+//
+// TODO: for now, this is a hack; it only flags a few common characters that have
+// special meaning in URLs. That will get the job done in the common cases.
+func shouldEscape(c byte) bool {
+ switch c {
+ case ' ', '?', '&', '=', '#', '+', '%':
+ return true;
+ }
+ return false;
+}
+
// URLUnescape unescapes a URL-encoded string,
-// converting %AB into the byte 0xAB.
+// converting %AB into the byte 0xAB and '+' into ' ' (space).
// It returns a BadURL error if any % is not followed
// by two hexadecimal digits.
func URLUnescape(s string) (string, os.Error) {
// Count %, check that they're well-formed.
n := 0;
+ anyPlusses := false;
for i := 0; i < len(s); {
- if s[i] == '%' {
+ switch s[i] {
+ case '%':
n++;
- if !ishex(s[i+1]) || !ishex(s[i+2]) {
+ if i+2 >= len(s) || !ishex(s[i+1]) || !ishex(s[i+2]) {
return "", BadURL{"invalid hexadecimal escape"}
}
- i += 3
- } else {
+ i += 3;
+ case '+':
+ anyPlusses = true;
+ i++;
+ default:
i++
}
}
- if n == 0 {
+ if n == 0 && !anyPlusses {
return s, nil
}
t := make([]byte, len(s)-2*n);
j := 0;
for i := 0; i < len(s); {
- if s[i] == '%' {
+ switch s[i] {
+ case '%':
t[j] = unhex(s[i+1]) << 4 | unhex(s[i+2]);
j++;
i += 3;
- } else {
+ case '+':
+ t[j] = ' ';
+ j++;
+ i++;
+ default:
t[j] = s[i];
j++;
i++;
return string(t), nil;
}
+// URLEscape converts a string into URL-encoded form.
+func URLEscape(s string) string {
+ spaceCount, hexCount := 0, 0;
+ for i := 0; i < len(s); i++ {
+ c := s[i];
+ if (shouldEscape(c)) {
+ if (c == ' ') {
+ spaceCount++;
+ } else {
+ hexCount++;
+ }
+ }
+ }
+
+ if spaceCount == 0 && hexCount == 0 {
+ return s;
+ }
+
+ t := make([]byte, len(s)+2*hexCount);
+ j := 0;
+ for i := 0; i < len(s); i++ {
+ c := s[i];
+ if !shouldEscape(c) {
+ t[j] = s[i];
+ j++;
+ } else if (c == ' ') {
+ t[j] = '+';
+ j++;
+ } else {
+ t[j] = '%';
+ t[j+1] = "0123456789abcdef"[c>>4];
+ t[j+2] = "0123456789abcdef"[c&15];
+ j += 3;
+ }
+ }
+ return string(t);
+}
+
// A URL represents a parsed URL (technically, a URI reference).
// The general form represented is:
// scheme://[userinfo@]host/path[?query][#fragment]
+// The Raw, RawPath, and RawQuery fields are in "wire format" (special
+// characters must be hex-escaped if not meant to have special meaning).
+// All other fields are logical values; '+' or '%' represent themselves.
+//
+// Note, the reason for using wire format for the query is that it needs
+// to be split into key/value pairs before decoding.
type URL struct {
Raw string; // the original string
Scheme string; // scheme
Userinfo string; // userinfo
Host string; // host
Path string; // /path
- Query string; // query
+ RawQuery string; // query
Fragment string; // fragment
}
// RFC 2396: a relative URI (no scheme) has a ?query,
// but absolute URIs only have query if path begins with /
if url.Scheme == "" || len(path) > 0 && path[0] == '/' {
- path, url.Query = split(path, '?', true);
- if url.Query, err = URLUnescape(url.Query); err != nil {
- return nil, err
- }
+ path, url.RawQuery = split(path, '?', true);
}
// Maybe path is //authority/path
return nil, err
}
+ // Remove escapes from the Authority and Userinfo fields, and verify
+ // that Scheme and Host contain no escapes (that would be illegal).
+ if url.Authority, err = URLUnescape(url.Authority); err != nil {
+ return nil, err
+ }
+ if url.Userinfo, err = URLUnescape(url.Userinfo); err != nil {
+ return nil, err
+ }
+ if (strings.Index(url.Scheme, "%") >= 0) {
+ return nil, BadURL{"hexadecimal escape in scheme"}
+ }
+ if (strings.Index(url.Host, "%") >= 0) {
+ return nil, BadURL{"hexadecimal escape in host"}
+ }
+
return url, nil
}
//
// There are redundant fields stored in the URL structure:
// the String method consults Scheme, Path, Host, Userinfo,
-// Query, and Fragment, but not RawPath or Authority.
+// RawQuery, and Fragment, but not Raw, RawPath or Authority.
func (url *URL) String() string {
result := "";
if url.Scheme != "" {
if url.Host != "" || url.Userinfo != "" {
result += "//";
if url.Userinfo != "" {
- result += url.Userinfo + "@";
+ result += URLEscape(url.Userinfo) + "@";
}
result += url.Host;
}
- result += url.Path;
- if url.Query != "" {
- result += "?" + url.Query;
+ result += URLEscape(url.Path);
+ if url.RawQuery != "" {
+ result += "?" + url.RawQuery;
}
if url.Fragment != "" {
- result += "#" + url.Fragment;
+ result += "#" + URLEscape(url.Fragment);
}
return result;
}
type URLTest struct {
in string;
out *URL;
+ roundtrip string; // expected result of reserializing the URL; empty means same as "in".
}
var urltests = []URLTest {
"http", "//www.google.com",
"www.google.com", "", "www.google.com",
"", "", ""
- }
+ },
+ ""
},
// path
URLTest{
"http", "//www.google.com/",
"www.google.com", "", "www.google.com",
"/", "", ""
- }
+ },
+ ""
+ },
+ // path with hex escaping... note that space roundtrips to +
+ URLTest{
+ "http://www.google.com/file%20one%26two",
+ &URL{
+ "http://www.google.com/file%20one%26two",
+ "http", "//www.google.com/file%20one%26two",
+ "www.google.com", "", "www.google.com",
+ "/file one&two", "", ""
+ },
+ "http://www.google.com/file+one%26two"
},
// user
URLTest{
"ftp", "//webmaster@www.google.com/",
"webmaster@www.google.com", "webmaster", "www.google.com",
"/", "", ""
- }
+ },
+ ""
+ },
+ // escape sequence in username
+ URLTest{
+ "ftp://john%20doe@www.google.com/",
+ &URL{
+ "ftp://john%20doe@www.google.com/",
+ "ftp", "//john%20doe@www.google.com/",
+ "john doe@www.google.com", "john doe", "www.google.com",
+ "/", "", ""
+ },
+ "ftp://john+doe@www.google.com/"
},
// query
URLTest{
"http", "//www.google.com/?q=go+language",
"www.google.com", "", "www.google.com",
"/", "q=go+language", ""
- }
+ },
+ ""
+ },
+ // query with hex escaping: NOT parsed
+ URLTest{
+ "http://www.google.com/?q=go%20language",
+ &URL{
+ "http://www.google.com/?q=go%20language",
+ "http", "//www.google.com/?q=go%20language",
+ "www.google.com", "", "www.google.com",
+ "/", "q=go%20language", ""
+ },
+ ""
},
// path without /, so no query parsing
URLTest{
"http:www.google.com/?q=go+language",
"http", "www.google.com/?q=go+language",
"", "", "",
- "www.google.com/?q=go+language", "", ""
- }
+ "www.google.com/?q=go language", "", ""
+ },
+ "http:www.google.com/%3fq%3dgo+language"
},
// non-authority
URLTest{
"mailto", "/webmaster@golang.org",
"", "", "",
"/webmaster@golang.org", "", ""
- }
+ },
+ ""
},
// non-authority
URLTest{
"mailto", "webmaster@golang.org",
"", "", "",
"webmaster@golang.org", "", ""
- }
+ },
+ ""
},
}
"http", "//www.google.com/?q=go+language#foo",
"www.google.com", "", "www.google.com",
"/", "q=go+language#foo", ""
- }
+ },
+ ""
},
}
"http", "//www.google.com/?q=go+language",
"www.google.com", "", "www.google.com",
"/", "q=go+language", "foo"
- }
+ },
+ ""
+ },
+ URLTest{
+ "http://www.google.com/?q=go+language#foo%26bar",
+ &URL{
+ "http://www.google.com/?q=go+language",
+ "http", "//www.google.com/?q=go+language",
+ "www.google.com", "", "www.google.com",
+ "/", "q=go+language", "foo&bar"
+ },
+ ""
},
}
func ufmt(u *URL) string {
return fmt.Sprintf("%q, %q, %q, %q, %q, %q, %q, %q, %q",
u.Raw, u.Scheme, u.RawPath, u.Authority, u.Userinfo,
- u.Host, u.Path, u.Query, u.Fragment);
+ u.Host, u.Path, u.RawQuery, u.Fragment);
}
func DoTest(t *testing.T, parse func(string) (*URL, os.Error), name string, tests []URLTest) {
continue;
}
s := u.String();
- if s != tt.in {
- t.Errorf("%s(%q).String() == %q", tt.in, s);
+ expected := tt.in;
+ if len(tt.roundtrip) > 0 {
+ expected = tt.roundtrip;
+ }
+ if s != expected {
+ t.Errorf("%s(%q).String() == %q (expected %q)", name, tt.in, s, expected);
}
}
}
DoTestString(t, ParseURLReference, "ParseURLReference", urlfragtests);
DoTestString(t, ParseURLReference, "ParseURLReference", urlnofragtests);
}
+
+type URLEscapeTest struct {
+ in string;
+ out string;
+ err os.Error;
+}
+
+var unescapeTests = []URLEscapeTest {
+ URLEscapeTest{
+ "",
+ "",
+ nil
+ },
+ URLEscapeTest{
+ "abc",
+ "abc",
+ nil
+ },
+ URLEscapeTest{
+ "1%41",
+ "1A",
+ nil
+ },
+ URLEscapeTest{
+ "1%41%42%43",
+ "1ABC",
+ nil
+ },
+ URLEscapeTest{
+ "%4a",
+ "J",
+ nil
+ },
+ URLEscapeTest{
+ "%6F",
+ "o",
+ nil
+ },
+ URLEscapeTest{
+ "%", // not enough characters after %
+ "",
+ BadURL{"invalid hexadecimal escape"}
+ },
+ URLEscapeTest{
+ "%a", // not enough characters after %
+ "",
+ BadURL{"invalid hexadecimal escape"}
+ },
+ URLEscapeTest{
+ "%1", // not enough characters after %
+ "",
+ BadURL{"invalid hexadecimal escape"}
+ },
+ URLEscapeTest{
+ "123%45%6", // not enough characters after %
+ "",
+ BadURL{"invalid hexadecimal escape"}
+ },
+ URLEscapeTest{
+ "%zz", // invalid hex digits
+ "",
+ BadURL{"invalid hexadecimal escape"}
+ },
+}
+
+func TestURLUnescape(t *testing.T) {
+ for i, tt := range unescapeTests {
+ actual, err := URLUnescape(tt.in);
+ if actual != tt.out || (err != nil) != (tt.err != nil) {
+ t.Errorf("URLUnescape(%q) = %q, %s; want %q, %s", tt.in, actual, err, tt.out, tt.err);
+ }
+ }
+}
+
+var escapeTests = []URLEscapeTest {
+ URLEscapeTest{
+ "",
+ "",
+ nil
+ },
+ URLEscapeTest{
+ "abc",
+ "abc",
+ nil
+ },
+ URLEscapeTest{
+ "one two",
+ "one+two",
+ nil
+ },
+ URLEscapeTest{
+ "10%",
+ "10%25",
+ nil
+ },
+ URLEscapeTest{
+ " ?&=#+%!",
+ "+%3f%26%3d%23%2b%25!",
+ nil
+ },
+}
+
+func TestURLEscape(t *testing.T) {
+ for i, tt := range escapeTests {
+ actual := URLEscape(tt.in);
+ if tt.out != actual {
+ t.Errorf("URLEscape(%q) = %q, want %q", tt.in, actual, tt.out);
+ }
+
+ // for bonus points, verify that escape:unescape is an identity.
+ roundtrip, err := URLUnescape(actual);
+ if roundtrip != tt.in || err != nil {
+ t.Errorf("URLUnescape(%q) = %q, %s; want %q, %s", actual, roundtrip, err, tt.in, "[no error]");
+ }
+ }
+}
+