strings.utflen -> utf8.RuneCount, RuneCountInString

R=r
DELTA=94  (52 added, 33 deleted, 9 changed)
OCL=20547
CL=20552

diff --git a/src/lib/Makefile b/src/lib/Makefile
index 7079433897b4bc7182d985a87f279ac517a5972f..b920aa6198f2a4939670e61c32e33111f85f3f45 100644 (file)
--- a/src/lib/Makefile
+++ b/src/lib/Makefile
@@ -88,6 +88,7 @@ bignum.6: fmt.dirinstall
 bufio.6: io.dirinstall os.dirinstall
 flag.6: fmt.dirinstall
 testing.6: flag.install fmt.dirinstall
+strings.6: utf8.install
 
 fmt.dirinstall: io.dirinstall reflect.dirinstall strconv.dirinstall
 hash.dirinstall: os.dirinstall

diff --git a/src/lib/strings.go b/src/lib/strings.go
index 433e500640e74b21a2e12d2c067bad7227960403..c171214db214c3ed039c58d78b983444216e680a 100644 (file)
--- a/src/lib/strings.go
+++ b/src/lib/strings.go
@@ -4,30 +4,17 @@
 
 package strings
 
-// Count UTF-8 sequences in s.
-// Assumes s is well-formed.
-export func utflen(s string) int {
-       n := 0;
-       for i := 0; i < len(s); i++ {
-               if s[i]&0xC0 != 0x80 {
-                       n++
-               }
-       }
-       return n
-}
+import "utf8"
 
 // Split string into array of UTF-8 sequences (still strings)
 export func explode(s string) *[]string {
-       a := new([]string, utflen(s));
+       a := new([]string, utf8.RuneCountInString(s, 0, len(s)));
        j := 0;
+       var size, rune int;
        for i := 0; i < len(a); i++ {
-               ej := j;
-               ej++;
-               for ej < len(s) && (s[ej]&0xC0) == 0x80 {
-                       ej++
-               }
-               a[i] = s[j:ej];
-               j = ej
+               rune, size = utf8.DecodeRuneInString(s, j);
+               a[i] = string(rune);
+               j += size;
        }
        return a
 }
@@ -35,7 +22,7 @@ export func explode(s string) *[]string {
 // Count non-overlapping instances of sep in s.
 export func count(s, sep string) int {
        if sep == "" {
-               return utflen(s)+1
+               return utf8.RuneCountInString(s, 0, len(s))+1
        }
        c := sep[0];
        n := 0;
@@ -83,7 +70,7 @@ export func split(s, sep string) *[]string {
        a[na] = s[start:len(s)];
        return a
 }
-       
+
 // Join list of strings with separators between them.
 export func join(a *[]string, sep string) string {
        if len(a) == 0 {

diff --git a/src/lib/strings_test.go b/src/lib/strings_test.go
index a7b63738b5df9e49ec818f8b7b9dbc00f4c1636e..50ad30cc36befe0979f4262ed3b49c8cfb21a239 100644 (file)
--- a/src/lib/strings_test.go
+++ b/src/lib/strings_test.go
@@ -79,21 +79,3 @@ export func TestSplit(t *testing.T) {
        }
 }
 
-// TODO: utflen shouldn't even be in strings.
-type UtflenTest struct {
-       in string;
-       out int;
-}
-var utflentests = []UtflenTest {
-       UtflenTest{ abcd, 4 },
-       UtflenTest{ faces, 3 },
-       UtflenTest{ commas, 7 },
-}
-export func TestUtflen(t *testing.T) {
-       for i := 0; i < len(utflentests); i++ {
-               tt := utflentests[i];
-               if out := strings.utflen(tt.in); out != tt.out {
-                       t.Errorf("utflen(%q) = %d, want %d", tt.in, out, tt.out);
-               }
-       }
-}

diff --git a/src/lib/utf8.go b/src/lib/utf8.go
index 9ece25f6a5052891a7d6781fc0dffc0298a4f062..7c1c8fbe5afb5b59680fb78dc5d9ed5424f6d424 100644 (file)
--- a/src/lib/utf8.go
+++ b/src/lib/utf8.go
@@ -107,8 +107,7 @@ func DecodeRuneInternal(p *[]byte) (rune, size int, short bool) {
        return RuneError, 1, false
 }
 
-func DecodeRuneInStringInternal(s string, i int) (rune, size int, short bool) {
-       n := len(s) - i;
+func DecodeRuneInStringInternal(s string, i int, n int) (rune, size int, short bool) {
        if n < 1 {
                return RuneError, 0, true;
        }
@@ -188,7 +187,7 @@ export func FullRune(p *[]byte) bool {
 }
 
 export func FullRuneInString(s string, i int) bool {
-       rune, size, short := DecodeRuneInStringInternal(s, i);
+       rune, size, short := DecodeRuneInStringInternal(s, i, len(s) - i);
        return !short
 }
 
@@ -200,7 +199,7 @@ export func DecodeRune(p *[]byte) (rune, size int) {
 
 export func DecodeRuneInString(s string, i int) (rune, size int) {
        var short bool;
-       rune, size, short = DecodeRuneInStringInternal(s, i);
+       rune, size, short = DecodeRuneInStringInternal(s, i, len(s) - i);
        return;
 }
 
@@ -248,3 +247,31 @@ export func EncodeRune(rune int, p *[]byte) int {
        return 4;
 }
 
+export func RuneCount(p *[]byte) int {
+       i := 0;
+       var n int;
+       for n = 0; i < len(p); n++ {
+               if p[i] < RuneSelf {
+                       i++;
+               } else {
+                       rune, size := DecodeRune(p[i:len(p)]);
+                       i += size;
+               }
+       }
+       return n;
+}
+
+export func RuneCountInString(s string, i int, l int) int {
+       ei := i + l;
+       n := 0;
+       for n = 0; i < ei; n++ {
+               if s[i] < RuneSelf {
+                       i++;
+               } else {
+                       rune, size, short := DecodeRuneInStringInternal(s, i, ei - i);
+                       i += size;
+               }
+       }
+       return n;
+}
+

diff --git a/src/lib/utf8_test.go b/src/lib/utf8_test.go
index 18c06c2ce5d1dcf346450ba4c1c603b6e352dba1..31118dd30cb826c2b9faab51e633d54c2104a5e5 100644 (file)
--- a/src/lib/utf8_test.go
+++ b/src/lib/utf8_test.go
@@ -156,3 +156,25 @@ export func TestDecodeRune(t *testing.T) {
                }
        }
 }
+
+type RuneCountTest struct {
+       in string;
+       out int;
+}
+var runecounttests = []RuneCountTest {
+       RuneCountTest{ "abcd", 4 },
+       RuneCountTest{ "☺☻☹", 3 },
+       RuneCountTest{ "1,2,3,4", 7 },
+       RuneCountTest{ "\xe2\x00", 2 },
+}
+export func TestRuneCount(t *testing.T) {
+       for i := 0; i < len(runecounttests); i++ {
+               tt := runecounttests[i];
+               if out := utf8.RuneCountInString(tt.in, 0, len(tt.in)); out != tt.out {
+                       t.Errorf("RuneCountInString(%q) = %d, want %d", tt.in, out, tt.out);
+               }
+               if out := utf8.RuneCount(Bytes(tt.in)); out != tt.out {
+                       t.Errorf("RuneCount(%q) = %d, want %d", tt.in, out, tt.out);
+               }
+       }
+}