report an error for illegal octal numbers instead of treating them as floats

added more test cases
some capitalization cleanups

R=rsc
CC=golang-dev
https://golang.org/cl/180085

diff --git a/src/pkg/go/scanner/scanner.go b/src/pkg/go/scanner/scanner.go
index 4735cbd3b5dfd349dece11ad3bfd71918af2fd74..7a21205a95bdc033afbf04ff93dfab06840e2223 100644 (file)
--- a/src/pkg/go/scanner/scanner.go
+++ b/src/pkg/go/scanner/scanner.go
@@ -271,10 +271,11 @@ func (S *Scanner) scanMantissa(base int) {
 }
 
 
-func (S *Scanner) scanNumber(seen_decimal_point bool) token.Token {
+func (S *Scanner) scanNumber(pos token.Position, seenDecimalPoint bool) token.Token {
+       // digitVal(S.ch) < 10
        tok := token.INT
 
-       if seen_decimal_point {
+       if seenDecimalPoint {
                tok = token.FLOAT
                S.scanMantissa(10)
                goto exponent
@@ -289,23 +290,29 @@ func (S *Scanner) scanNumber(seen_decimal_point bool) token.Token {
                        S.scanMantissa(16)
                } else {
                        // octal int or float
+                       seenDecimalDigit := false
                        S.scanMantissa(8)
-                       if digitVal(S.ch) < 10 || S.ch == '.' || S.ch == 'e' || S.ch == 'E' {
-                               // float
-                               tok = token.FLOAT
-                               goto mantissa
+                       if S.ch == '8' || S.ch == '9' {
+                               // illegal octal int or float
+                               seenDecimalDigit = true
+                               S.scanMantissa(10)
+                       }
+                       if S.ch == '.' || S.ch == 'e' || S.ch == 'E' {
+                               goto fraction
                        }
                        // octal int
+                       if seenDecimalDigit {
+                               S.error(pos, "illegal octal number")
+                       }
                }
                goto exit
        }
 
-mantissa:
        // decimal int or float
        S.scanMantissa(10)
 
+fraction:
        if S.ch == '.' {
-               // float
                tok = token.FLOAT
                S.next()
                S.scanMantissa(10)
@@ -313,7 +320,6 @@ mantissa:
 
 exponent:
        if S.ch == 'e' || S.ch == 'E' {
-               // float
                tok = token.FLOAT
                S.next()
                if S.ch == '-' || S.ch == '+' {
@@ -503,7 +509,7 @@ scanAgain:
                }
        case digitVal(ch) < 10:
                insertSemi = true
-               tok = S.scanNumber(false)
+               tok = S.scanNumber(pos, false)
        default:
                S.next() // always make progress
                switch ch {
@@ -532,7 +538,7 @@ scanAgain:
                case '.':
                        if digitVal(S.ch) < 10 {
                                insertSemi = true
-                               tok = S.scanNumber(true)
+                               tok = S.scanNumber(pos, true)
                        } else if S.ch == '.' {
                                S.next()
                                if S.ch == '.' {

diff --git a/src/pkg/go/scanner/scanner_test.go b/src/pkg/go/scanner/scanner_test.go
index 83314a3aa1882da787618b5b29a3d1d4e5d954a1..56091a39f8dbd972febc64263d4dc35f14f989e1 100644 (file)
--- a/src/pkg/go/scanner/scanner_test.go
+++ b/src/pkg/go/scanner/scanner_test.go
@@ -164,16 +164,16 @@ var tokens = [...]elt{
 
 const whitespace = "  \t  \n\n\n" // to separate tokens
 
-type TestErrorHandler struct {
+type testErrorHandler struct {
        t *testing.T
 }
 
-func (h *TestErrorHandler) Error(pos token.Position, msg string) {
+func (h *testErrorHandler) Error(pos token.Position, msg string) {
        h.t.Errorf("Error() called (msg = %s)", msg)
 }
 
 
-func NewlineCount(s string) int {
+func newlineCount(s string) int {
        n := 0
        for i := 0; i < len(s); i++ {
                if s[i] == '\n' {
@@ -207,12 +207,12 @@ func TestScan(t *testing.T) {
        for _, e := range tokens {
                src += e.lit + whitespace
        }
-       whitespace_linecount := NewlineCount(whitespace)
+       whitespace_linecount := newlineCount(whitespace)
 
        // verify scan
        index := 0
        epos := token.Position{"", 0, 1, 1}
-       nerrors := Tokenize("", strings.Bytes(src), &TestErrorHandler{t}, ScanComments,
+       nerrors := Tokenize("", strings.Bytes(src), &testErrorHandler{t}, ScanComments,
                func(pos token.Position, tok token.Token, litb []byte) bool {
                        e := elt{token.EOF, "", special}
                        if index < len(tokens) {
@@ -234,7 +234,7 @@ func TestScan(t *testing.T) {
                                t.Errorf("bad class for %q: got %d, expected %d", lit, tokenclass(tok), e.class)
                        }
                        epos.Offset += len(lit) + len(whitespace)
-                       epos.Line += NewlineCount(lit) + whitespace_linecount
+                       epos.Line += newlineCount(lit) + whitespace_linecount
                        if tok == token.COMMENT && litb[1] == '/' {
                                // correct for unaccounted '/n' in //-style comment
                                epos.Offset++
@@ -249,11 +249,6 @@ func TestScan(t *testing.T) {
 }
 
 
-func getTok(_ token.Position, tok token.Token, _ []byte) token.Token {
-       return tok
-}
-
-
 func checkSemi(t *testing.T, line string, mode uint) {
        var S Scanner
        S.Init("TestSemis", strings.Bytes(line), nil, mode)
@@ -485,7 +480,7 @@ func TestIllegalChars(t *testing.T) {
        var s Scanner
 
        const src = "*?*$*@*"
-       s.Init("", strings.Bytes(src), &TestErrorHandler{t}, AllowIllegalChars)
+       s.Init("", strings.Bytes(src), &testErrorHandler{t}, AllowIllegalChars)
        for offs, ch := range src {
                pos, tok, lit := s.Scan()
                if pos.Offset != offs {
@@ -540,3 +535,74 @@ func TestStdErrorHander(t *testing.T) {
                t.Errorf("found %d errors, expected %d", v.ErrorCount(), nerrors)
        }
 }
+
+
+type errorCollector struct {
+       cnt int            // number of errors encountered
+       msg string         // last error message encountered
+       pos token.Position // last error position encountered
+}
+
+
+func (h *errorCollector) Error(pos token.Position, msg string) {
+       h.cnt++
+       h.msg = msg
+       h.pos = pos
+}
+
+
+func checkError(t *testing.T, src string, tok token.Token, err string) {
+       var s Scanner
+       var h errorCollector
+       s.Init("", strings.Bytes(src), &h, ScanComments)
+       _, tok0, _ := s.Scan()
+       _, tok1, _ := s.Scan()
+       if tok0 != tok {
+               t.Errorf("%q: got %s, expected %s", src, tok0, tok)
+       }
+       if tok1 != token.EOF {
+               t.Errorf("%q: got %s, expected EOF", src, tok1)
+       }
+       cnt := 0
+       if err != "" {
+               cnt = 1
+       }
+       if h.cnt != cnt {
+               t.Errorf("%q: got cnt %d, expected %d", src, h.cnt, cnt)
+       }
+       if h.msg != err {
+               t.Errorf("%q: got msg %q, expected %q", src, h.msg, err)
+       }
+       if h.pos.Offset != 0 {
+               t.Errorf("%q: got offset %d, expected 0", src, h.pos.Offset)
+       }
+}
+
+
+type srcerr struct {
+       src string
+       tok token.Token
+       err string
+}
+
+var errors = []srcerr{
+       srcerr{"\"\"", token.STRING, ""},
+       srcerr{"\"", token.STRING, "string not terminated"},
+       srcerr{"/**/", token.COMMENT, ""},
+       srcerr{"/*", token.COMMENT, "comment not terminated"},
+       srcerr{"//\n", token.COMMENT, ""},
+       srcerr{"//", token.COMMENT, "comment not terminated"},
+       srcerr{"077", token.INT, ""},
+       srcerr{"078.", token.FLOAT, ""},
+       srcerr{"07801234567.", token.FLOAT, ""},
+       srcerr{"078e0", token.FLOAT, ""},
+       srcerr{"078", token.INT, "illegal octal number"},
+       srcerr{"07800000009", token.INT, "illegal octal number"},
+}
+
+
+func TestScanErrors(t *testing.T) {
+       for _, e := range errors {
+               checkError(t, e.src, e.tok, e.err)
+       }
+}