UTFMax = 4 // maximum number of bytes of a UTF-8 encoded Unicode character.
)
+// Code points in the surrogate range are not valid for UTF-8.
+const (
+ surrogateMin = 0xD800
+ surrogateMax = 0xDFFF
+)
+
const (
t1 = 0x00 // 0000 0000
tx = 0x80 // 1000 0000
rune1Max = 1<<7 - 1
rune2Max = 1<<11 - 1
rune3Max = 1<<16 - 1
- rune4Max = 1<<21 - 1
)
func decodeRuneInternal(p []byte) (r rune, size int, short bool) {
if r <= rune2Max {
return RuneError, 1, false
}
+ if surrogateMin <= r && r <= surrogateMax {
+ return RuneError, 1, false
+ }
return r, 3, false
}
if r <= rune2Max {
return RuneError, 1, false
}
+ if surrogateMin <= r && r <= surrogateMax {
+ return RuneError, 1, false
+ }
return r, 3, false
}
}
// RuneLen returns the number of bytes required to encode the rune.
+// It returns -1 if the rune is not a valid value to encode in UTF-8.
func RuneLen(r rune) int {
switch {
+ case r < 0:
+ return -1
case r <= rune1Max:
return 1
case r <= rune2Max:
return 2
+ case surrogateMin <= r && r <= surrogateMax:
+ return -1
case r <= rune3Max:
return 3
- case r <= rune4Max:
+ case r <= MaxRune:
return 4
}
return -1
r = RuneError
}
+ if surrogateMin <= r && r <= surrogateMax {
+ r = RuneError
+ }
+
if uint32(r) <= rune3Max {
p[0] = t3 | byte(r>>12)
p[1] = tx | byte(r>>6)&maskx
}
return true
}
+
+// ValidRune reports whether r can be legally encoded as UTF-8.
+// Code points that are out of range or a surrogate half are illegal.
+func ValidRune(r rune) bool {
+ switch {
+ case r < 0:
+ return false
+ case surrogateMin <= r && r <= surrogateMax:
+ return false
+ case r > MaxRune:
+ return false
+ }
+ return true
+}
{0x07ff, "\xdf\xbf"},
{0x0800, "\xe0\xa0\x80"},
{0x0801, "\xe0\xa0\x81"},
+ {0xd7ff, "\xed\x9f\xbf"}, // last code point before surrogate half.
+ {0xe000, "\xee\x80\x80"}, // first code point after surrogate half.
{0xfffe, "\xef\xbf\xbe"},
{0xffff, "\xef\xbf\xbf"},
{0x10000, "\xf0\x90\x80\x80"},
{0xFFFD, "\xef\xbf\xbd"},
}
+var surrogateMap = []Utf8Map{
+ {0xd800, "\xed\xa0\x80"}, // surrogate min decodes to (RuneError, 1)
+ {0xdfff, "\xed bf bf"}, // surrogate max decodes to (RuneError, 1)
+}
+
var testStrings = []string{
"",
"abcd",
}
func TestFullRune(t *testing.T) {
- for i := 0; i < len(utf8map); i++ {
- m := utf8map[i]
+ for _, m := range utf8map {
b := []byte(m.str)
if !FullRune(b) {
t.Errorf("FullRune(%q) (%U) = false, want true", b, m.r)
}
func TestEncodeRune(t *testing.T) {
- for i := 0; i < len(utf8map); i++ {
- m := utf8map[i]
+ for _, m := range utf8map {
b := []byte(m.str)
var buf [10]byte
n := EncodeRune(buf[0:], m.r)
}
func TestDecodeRune(t *testing.T) {
- for i := 0; i < len(utf8map); i++ {
- m := utf8map[i]
+ for _, m := range utf8map {
b := []byte(m.str)
r, size := DecodeRune(b)
if r != m.r || size != len(b) {
}
}
+func TestDecodeSurrogateRune(t *testing.T) {
+ for _, m := range surrogateMap {
+ b := []byte(m.str)
+ r, size := DecodeRune(b)
+ if r != RuneError || size != 1 {
+ t.Errorf("DecodeRune(%q) = %x, %d want %x, %d", b, r, size, RuneError, 1)
+ }
+ s := m.str
+ r, size = DecodeRuneInString(s)
+ if r != RuneError || size != 1 {
+ t.Errorf("DecodeRune(%q) = %x, %d want %x, %d", b, r, size, RuneError, 1)
+ }
+ }
+}
+
// Check that DecodeRune and DecodeLastRune correspond to
// the equivalent range loop.
func TestSequencing(t *testing.T) {
}
func TestRuneCount(t *testing.T) {
- for i := 0; i < len(runecounttests); i++ {
- tt := runecounttests[i]
+ for _, tt := range runecounttests {
if out := RuneCountInString(tt.in); out != tt.out {
t.Errorf("RuneCountInString(%q) = %d, want %d", tt.in, out, tt.out)
}
}
}
+type RuneLenTest struct {
+ r rune
+ size int
+}
+
+var runelentests = []RuneLenTest{
+ {0, 1},
+ {'e', 1},
+ {'é', 2},
+ {'☺', 3},
+ {RuneError, 3},
+ {MaxRune, 4},
+ {0xD800, -1},
+ {0xDFFF, -1},
+ {MaxRune + 1, -1},
+ {-1, -1},
+}
+
+func TestRuneLen(t *testing.T) {
+ for _, tt := range runelentests {
+ if size := RuneLen(tt.r); size != tt.size {
+ t.Errorf("RuneLen(%#U) = %d, want %d", tt.r, size, tt.size)
+ }
+ }
+}
+
type ValidTest struct {
in string
out bool
{string("\xF7\xBF\xBF\xBF"), true}, // U+1FFFFF
{string("\xFB\xBF\xBF\xBF\xBF"), false}, // 0x3FFFFFF; out of range
{string("\xc0\x80"), false}, // U+0000 encoded in two bytes: incorrect
- // TODO {string("\xed\xa0\x80"), false }, // U+D800 high surrogate (sic)
- // TODO {string("\xed\xbf\xbf"), false }, // U+DFFF low surrogate (sic)
+ {string("\xed\xa0\x80"), false}, // U+D800 high surrogate (sic)
+ {string("\xed\xbf\xbf"), false}, // U+DFFF low surrogate (sic)
}
func TestValid(t *testing.T) {
- for i, tt := range validTests {
+ for _, tt := range validTests {
if Valid([]byte(tt.in)) != tt.out {
- t.Errorf("%d. Valid(%q) = %v; want %v", i, tt.in, !tt.out, tt.out)
+ t.Errorf("Valid(%q) = %v; want %v", tt.in, !tt.out, tt.out)
}
if ValidString(tt.in) != tt.out {
- t.Errorf("%d. ValidString(%q) = %v; want %v", i, tt.in, !tt.out, tt.out)
+ t.Errorf("ValidString(%q) = %v; want %v", tt.in, !tt.out, tt.out)
+ }
+ }
+}
+
+type ValidRuneTest struct {
+ r rune
+ ok bool
+}
+
+var validrunetests = []ValidRuneTest{
+ {0, true},
+ {'e', true},
+ {'é', true},
+ {'☺', true},
+ {RuneError, true},
+ {MaxRune, true},
+ {0xD7FF, true},
+ {0xD800, false},
+ {0xDFFF, false},
+ {0xE000, true},
+ {MaxRune + 1, false},
+ {-1, false},
+}
+
+func TestValidRune(t *testing.T) {
+ for _, tt := range validrunetests {
+ if ok := ValidRune(tt.r); ok != tt.ok {
+ t.Errorf("ValidRune(%#U) = %t, want %t", tt.r, ok, tt.ok)
}
}
}