exp/locale/collate: add code to ignore tests with (unpaired) surrogates.

In the regtest data, surrogates are assigned primary weights based on
the surrogate code point value.  Go now converts surrogates to FFFD, however,
meaning that the primary weight is based on this code point instead.
This change drops tests with surrogates and lets the tests pass.

R=r
CC=golang-dev
https://golang.org/cl/6461100

diff --git a/src/pkg/exp/locale/collate/regtest.go b/src/pkg/exp/locale/collate/regtest.go
index 38c5783aca7d599b580f410806987263808c7031..14a447c1e4e7cf5c8b7c63b58236f1d96b073cee 100644 (file)
--- a/src/pkg/exp/locale/collate/regtest.go
+++ b/src/pkg/exp/locale/collate/regtest.go
@@ -24,6 +24,7 @@ import (
        "strconv"
        "strings"
        "unicode"
+       "unicode/utf8"
 )
 
 // This regression test runs tests for the test files in CollationTest.zip
@@ -53,7 +54,7 @@ var localFiles = flag.Bool("local",
 
 type Test struct {
        name    string
-       str     []string
+       str     [][]byte
        comment []string
 }
 
@@ -186,14 +187,23 @@ func loadTestData() []Test {
                        if m == nil || len(m) < 3 {
                                log.Fatalf(`Failed to parse: "%s" result: %#v`, line, m)
                        }
-                       str := ""
+                       str := []byte{}
+                       // In the regression test data (unpaired) surrogates are assigned a weight
+                       // corresponding to their code point value.  However, utf8.DecodeRune,
+                       // which is used to compute the implicit weight, assigns FFFD to surrogates.
+                       // We therefore skip tests with surrogates.  This skips about 35 entries
+                       // per test.
+                       valid := true
                        for _, split := range strings.Split(m[1], " ") {
                                r, err := strconv.ParseUint(split, 16, 64)
                                Error(err)
-                               str += string(rune(r))
+                               valid = valid && utf8.ValidRune(rune(r))
+                               str = append(str, string(rune(r))...)
+                       }
+                       if valid {
+                               test.str = append(test.str, str)
+                               test.comment = append(test.comment, m[2])
                        }
-                       test.str = append(test.str, str)
-                       test.comment = append(test.comment, m[2])
                }
                tests = append(tests, test)
        }
@@ -227,13 +237,13 @@ func doTest(t Test) {
                c.Alternate = collate.AltNonIgnorable
        }
 
-       prev := []byte(t.str[0])
+       prev := t.str[0]
        for i := 1; i < len(t.str); i++ {
-               s := []byte(t.str[i])
+               s := t.str[i]
                ka := c.Key(b, prev)
                kb := c.Key(b, s)
                if r := bytes.Compare(ka, kb); r == 1 {
-                       fail(t, "%d: Key(%.4X) < Key(%.4X) (%X < %X) == %d; want -1 or 0", i, runes(prev), runes(s), ka, kb, r)
+                       fail(t, "%d: Key(%.4X) < Key(%.4X) (%X < %X) == %d; want -1 or 0", i, []rune(string(prev)), []rune(string(s)), ka, kb, r)
                        prev = s
                        continue
                }