- allow unicode digits in identifiers

- fixed a bug with character escapes (before: allowed arbitrary long sequences)

R=r
DELTA=63  (33 added, 19 deleted, 11 changed)
OCL=26010
CL=26070

diff --git a/src/lib/go/scanner.go b/src/lib/go/scanner.go
index 94391c04461fec8a0c5229766a0e22e3920ff0df..90078e0dd9eabf995850a6658c5c55c8863af5a6 100644 (file)
--- a/src/lib/go/scanner.go
+++ b/src/lib/go/scanner.go
@@ -60,25 +60,6 @@ type Scanner struct {
 }
 
 
-func isLetter(ch int) bool {
-       return
-               'a' <= ch && ch <= 'z' ||
-               'A' <= ch && ch <= 'Z' ||
-               ch == '_' ||
-               ch >= 0x80 && unicode.IsLetter(ch);
-}
-
-
-func digitVal(ch int) int {
-       switch {
-       case '0' <= ch && ch <= '9': return ch - '0';
-       case 'a' <= ch && ch <= 'f': return ch - 'a' + 10;
-       case 'A' <= ch && ch <= 'F': return ch - 'A' + 10;
-       }
-       return 16;  // larger than any legal digit val
-}
-
-
 // Read the next Unicode char into S.ch.
 // S.ch < 0 means end-of-file.
 func (S *Scanner) next() {
@@ -195,9 +176,25 @@ func (S *Scanner) scanComment() []byte {
 }
 
 
+func isLetter(ch int) bool {
+       return
+               'a' <= ch && ch <= 'z' ||
+               'A' <= ch && ch <= 'Z' ||
+               ch == '_' ||
+               ch >= 0x80 && unicode.IsLetter(ch);
+}
+
+
+func isDigit(ch int) bool {
+       return
+               '0' <= ch && ch <= '9' ||
+               ch >= 0x80 && unicode.IsDecimalDigit(ch);
+}
+
+
 func (S *Scanner) scanIdentifier() (tok int, lit []byte) {
        pos := S.chpos;
-       for isLetter(S.ch) || digitVal(S.ch) < 10 {
+       for isLetter(S.ch) || isDigit(S.ch) {
                S.next();
        }
        lit = S.src[pos : S.chpos];
@@ -205,6 +202,16 @@ func (S *Scanner) scanIdentifier() (tok int, lit []byte) {
 }
 
 
+func digitVal(ch int) int {
+       switch {
+       case '0' <= ch && ch <= '9': return ch - '0';
+       case 'a' <= ch && ch <= 'f': return ch - 'a' + 10;
+       case 'A' <= ch && ch <= 'F': return ch - 'A' + 10;
+       }
+       return 16;  // larger than any legal digit val
+}
+
+
 func (S *Scanner) scanMantissa(base int) {
        for digitVal(S.ch) < base {
                S.next();
@@ -270,12 +277,12 @@ exit:
 }
 
 
-func (S *Scanner) scanDigits(n int, base int) {
-       for digitVal(S.ch) < base {
+func (S *Scanner) scanDigits(base, length int) {
+       for length > 0 && digitVal(S.ch) < base {
                S.next();
-               n--;
+               length--;
        }
-       if n > 0 {
+       if length > 0 {
                S.error(S.chpos, "illegal char escape");
        }
 }
@@ -289,13 +296,13 @@ func (S *Scanner) scanEscape(quote int) {
        case 'a', 'b', 'f', 'n', 'r', 't', 'v', '\\', quote:
                // nothing to do
        case '0', '1', '2', '3', '4', '5', '6', '7':
-               S.scanDigits(3 - 1, 8);  // 1 char read already
+               S.scanDigits(8, 3 - 1);  // 1 char read already
        case 'x':
-               S.scanDigits(2, 16);
+               S.scanDigits(16, 2);
        case 'u':
-               S.scanDigits(4, 16);
+               S.scanDigits(16, 4);
        case 'U':
-               S.scanDigits(8, 16);
+               S.scanDigits(16, 8);
        default:
                S.error(pos, "illegal char escape");
        }

diff --git a/src/lib/go/scanner_test.go b/src/lib/go/scanner_test.go
index 136677cd0a87acfea7ac2cc5eb11ba7b65f52e73..9cad23914f2ec5b6bedaf1bd248e2871235165e7 100644 (file)
--- a/src/lib/go/scanner_test.go
+++ b/src/lib/go/scanner_test.go
@@ -45,6 +45,9 @@ var tokens = [...]elt{
 
        // Identifiers and basic type literals
        elt{ 0, token.IDENT, "foobar", literal },
+       elt{ 0, token.IDENT, "a۰۱۸", literal },
+       elt{ 0, token.IDENT, "foo६४", literal },
+       elt{ 0, token.IDENT, "bar９８７６", literal },
        elt{ 0, token.INT, "0", literal },
        elt{ 0, token.INT, "01234567", literal },
        elt{ 0, token.INT, "0xcafebabe", literal },
@@ -56,6 +59,10 @@ var tokens = [...]elt{
        elt{ 0, token.FLOAT, "1e-100", literal },
        elt{ 0, token.FLOAT, "2.71828e-1000", literal },
        elt{ 0, token.CHAR, "'a'", literal },
+       elt{ 0, token.CHAR, "'\\000'", literal },
+       elt{ 0, token.CHAR, "'\\xFF'", literal },
+       elt{ 0, token.CHAR, "'\\uff16'", literal },
+       elt{ 0, token.CHAR, "'\\U0000ff16'", literal },
        elt{ 0, token.STRING, "`foobar`", literal },
 
        // Operators and delimitors