From 8458a387e3a0aa29449767f633514747a1ee7e72 Mon Sep 17 00:00:00 2001 From: Robert Griesemer Date: Thu, 8 Sep 2016 21:39:33 -0700 Subject: [PATCH] cmd/compile: make fmt_test work on entire compiler - process all directories recursively Change-Id: I27a737013d17fd3c2cc8ae9de4722dcbe989e6e5 Reviewed-on: https://go-review.googlesource.com/28789 Reviewed-by: Matthew Dempsky --- src/cmd/compile/fmt_test.go | 780 ++++++++++++++++++++++++ src/cmd/compile/internal/gc/fmt_test.go | 600 ------------------ 2 files changed, 780 insertions(+), 600 deletions(-) create mode 100644 src/cmd/compile/fmt_test.go delete mode 100644 src/cmd/compile/internal/gc/fmt_test.go diff --git a/src/cmd/compile/fmt_test.go b/src/cmd/compile/fmt_test.go new file mode 100644 index 0000000000..80a96021af --- /dev/null +++ b/src/cmd/compile/fmt_test.go @@ -0,0 +1,780 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements TestFormats; a test that verifies +// format strings in the compiler (this directory and all +// subdirectories, recursively). +// +// TestFormats finds potential (Printf, etc.) format strings. +// If they are used in a call, the format verbs are verified +// based on the matching argument type against a precomputed +// table of valid formats. The knownFormats table can be used +// to automatically rewrite format strings with the -u flag. +// +// A new knownFormats table based on the found formats is printed +// when the test is run in verbose mode (-v flag). The table +// needs to be updated whenever a new (type, format) combination +// is found and the format verb is not 'v' (as in "%v"). +// +// Run as: go test -run Formats [-u][-v] +// +// Known bugs: +// - indexed format strings ("%[2]s", etc.) are not supported +// (the test will fail) +// - format strings that are not simple string literals cannot +// be updated automatically +// (the test will fail with respective warnings) +// - format strings in _test packages outside the current +// package are not processed +// (the test will report those files) +// +package main_test + +import ( + "bytes" + "flag" + "fmt" + "go/ast" + "go/build" + "go/constant" + "go/format" + "go/importer" + "go/parser" + "go/token" + "go/types" + "internal/testenv" + "io/ioutil" + "log" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "testing" + "unicode/utf8" +) + +var update = flag.Bool("u", false, "update format strings") + +// The following variables collect information across all processed files. +var ( + fset = token.NewFileSet() + formatStrings = make(map[*ast.BasicLit]bool) // set of all potential format strings found + foundFormats = make(map[string]bool) // set of all formats found + callSites = make(map[*ast.CallExpr]*callSite) // set of all calls using format strings +) + +// A File is a corresponding (filename, ast) pair. +type File struct { + name string + ast *ast.File +} + +func TestFormats(t *testing.T) { + testenv.MustHaveGoBuild(t) // more restrictive than necessary, but that's ok + + // process all directories + filepath.Walk(".", func(path string, info os.FileInfo, err error) error { + if info.IsDir() { + if info.Name() == "testdata" { + return filepath.SkipDir + } + + importPath := filepath.Join("cmd/compile", path) + pkg, err := build.Import(importPath, path, 0) + if err != nil { + if _, ok := err.(*build.NoGoError); ok { + return nil // nothing to do here + } + t.Fatal(err) + } + collectPkgFormats(t, pkg) + } + return nil + }) + + // test and rewrite formats + updatedFiles := make(map[string]File) // files that were rewritten + for _, p := range callSites { + // test current format literal and determine updated one + out := formatReplace(p.str, func(index int, in string) string { + if in == "*" { + return in // cannot rewrite '*' (as in "%*d") + } + // in != '*' + typ := p.types[index] + format := typ + " " + in // e.g., "*Node %n" + + // check if format is known + out, known := knownFormats[format] + + // record format if not yet found + _, found := foundFormats[format] + if !found { + foundFormats[format] = true + } + + // report an error if the format is unknown and this is the first + // time we see it; ignore "%v" which is always considered valid + if !known && !found && in != "%v" { + t.Errorf("%s: unknown format %q for %s argument", posString(p.arg), in, typ) + } + + if out == "" { + out = in + } + return out + }) + + // replace existing format literal if it changed + if out != p.str { + // we cannot replace the argument if it's not a string literal for now + // (e.g., it may be "foo" + "bar") + lit, ok := p.arg.(*ast.BasicLit) + if !ok { + delete(callSites, p.call) // treat as if we hadn't found this site + continue + } + + if testing.Verbose() { + fmt.Printf("%s:\n\t- %q\n\t+ %q\n", posString(p.arg), p.str, out) + } + + // find argument index of format argument + index := -1 + for i, arg := range p.call.Args { + if p.arg == arg { + index = i + break + } + } + if index < 0 { + // we may have processed the same call site twice, + // but that shouldn't happen + panic("internal error: matching argument not found") + } + + // replace literal + new := *lit // make a copy + new.Value = strconv.Quote(out) // this may introduce "-quotes where there were `-quotes + p.call.Args[index] = &new + updatedFiles[p.file.name] = p.file + } + } + + // write dirty files back + var filesUpdated bool + if len(updatedFiles) > 0 && *update { + for _, file := range updatedFiles { + var buf bytes.Buffer + if err := format.Node(&buf, fset, file.ast); err != nil { + t.Errorf("WARNING: formatting %s failed: %v", file.name, err) + continue + } + if err := ioutil.WriteFile(file.name, buf.Bytes(), 0x666); err != nil { + t.Errorf("WARNING: writing %s failed: %v", file.name, err) + continue + } + fmt.Printf("updated %s\n", file.name) + filesUpdated = true + } + } + + // report all function names containing a format string + if len(callSites) > 0 && testing.Verbose() { + set := make(map[string]bool) + for _, p := range callSites { + set[nodeString(p.call.Fun)] = true + } + var list []string + for s := range set { + list = append(list, s) + } + fmt.Println("\nFunctions") + printList(list) + } + + // report all formats found + if len(foundFormats) > 0 && testing.Verbose() { + var list []string + for s := range foundFormats { + list = append(list, fmt.Sprintf("%q: \"\",", s)) + } + fmt.Println("\nvar knownFormats = map[string]string{") + printList(list) + fmt.Println("}") + } + + // all format strings of calls must be in the formatStrings set (self-verification) + for _, p := range callSites { + if lit, ok := p.arg.(*ast.BasicLit); ok && lit.Kind == token.STRING { + if formatStrings[lit] { + // ok + delete(formatStrings, lit) + } else { + // this should never happen + panic(fmt.Sprintf("internal error: format string not found (%s)", posString(lit))) + } + } + } + + // if we have any strings left, we may need to update them manually + if len(formatStrings) > 0 && filesUpdated { + var list []string + for lit := range formatStrings { + list = append(list, fmt.Sprintf("%s: %s", posString(lit), nodeString(lit))) + } + fmt.Println("\nWARNING: Potentially missed format strings") + printList(list) + t.Fail() + } + + fmt.Println() +} + +// A callSite describes a function call that appears to contain +// a format string. +type callSite struct { + file File + call *ast.CallExpr // call containing the format string + arg ast.Expr // format argument (string literal or constant) + str string // unquoted format string + types []string // argument types +} + +func collectPkgFormats(t *testing.T, pkg *build.Package) { + // collect all files + var filenames []string + filenames = append(filenames, pkg.GoFiles...) + filenames = append(filenames, pkg.CgoFiles...) + filenames = append(filenames, pkg.TestGoFiles...) + + // TODO(gri) verify _test files outside package + for _, name := range pkg.XTestGoFiles { + // don't process this test itself + if name != "fmt_test.go" && testing.Verbose() { + fmt.Printf("WARNING: %s not processed\n", filepath.Join(pkg.Dir, name)) + } + } + + // make filenames relative to . + for i, name := range filenames { + filenames[i] = filepath.Join(pkg.Dir, name) + } + + // parse all files + files := make([]*ast.File, len(filenames)) + for i, filename := range filenames { + f, err := parser.ParseFile(fset, filename, nil, parser.ParseComments) + if err != nil { + t.Fatal(err) + } + files[i] = f + } + + // typecheck package + conf := types.Config{Importer: importer.Default()} + etypes := make(map[ast.Expr]types.TypeAndValue) + if _, err := conf.Check(pkg.ImportPath, fset, files, &types.Info{Types: etypes}); err != nil { + t.Fatal(err) + } + + // collect all potential format strings (for extra verification later) + for _, file := range files { + ast.Inspect(file, func(n ast.Node) bool { + if s, ok := stringLit(n); ok && isFormat(s) { + formatStrings[n.(*ast.BasicLit)] = true + } + return true + }) + } + + // collect all formats/arguments of calls with format strings + for index, file := range files { + ast.Inspect(file, func(n ast.Node) bool { + if call, ok := n.(*ast.CallExpr); ok { + // ignore blacklisted functions + if functionBlacklisted[nodeString(call.Fun)] { + return true + } + // look for an arguments that might be a format string + for i, arg := range call.Args { + if s, ok := stringVal(etypes[arg]); ok && isFormat(s) { + // make sure we have enough arguments + n := numFormatArgs(s) + if i+1+n > len(call.Args) { + t.Errorf("%s: not enough format args (blacklist %s?)", posString(call), nodeString(call.Fun)) + break // ignore this call + } + // assume last n arguments are to be formatted; + // determine their types + argTypes := make([]string, n) + for i, arg := range call.Args[len(call.Args)-n:] { + if tv, ok := etypes[arg]; ok { + argTypes[i] = typeString(tv.Type) + } + } + // collect call site + if callSites[call] != nil { + panic("internal error: file processed twice?") + } + callSites[call] = &callSite{ + file: File{filenames[index], file}, + call: call, + arg: arg, + str: s, + types: argTypes, + } + break // at most one format per argument list + } + } + } + return true + }) + } +} + +// printList prints list in sorted order. +func printList(list []string) { + sort.Strings(list) + for _, s := range list { + fmt.Println("\t", s) + } +} + +// posString returns a string representation of n's position +// in the form filename:line:col: . +func posString(n ast.Node) string { + if n == nil { + return "" + } + return fset.Position(n.Pos()).String() +} + +// nodeString returns a string representation of n. +func nodeString(n ast.Node) string { + var buf bytes.Buffer + if err := format.Node(&buf, fset, n); err != nil { + log.Fatal(err) // should always succeed + } + return buf.String() +} + +// typeString returns a string representation of n. +func typeString(typ types.Type) string { + s := typ.String() + // canonicalize path separators + if filepath.Separator != '/' { + s = strings.Replace(s, string(filepath.Separator), "/", -1) + } + return s +} + +// stringLit returns the unquoted string value and true if +// n represents a string literal; otherwise it returns "" +// and false. +func stringLit(n ast.Node) (string, bool) { + if lit, ok := n.(*ast.BasicLit); ok && lit.Kind == token.STRING { + s, err := strconv.Unquote(lit.Value) + if err != nil { + log.Fatal(err) // should not happen with correct ASTs + } + return s, true + } + return "", false +} + +// stringVal returns the (unquoted) string value and true if +// tv is a string constant; otherwise it returns "" and false. +func stringVal(tv types.TypeAndValue) (string, bool) { + if tv.IsValue() && tv.Value != nil && tv.Value.Kind() == constant.String { + return constant.StringVal(tv.Value), true + } + return "", false +} + +// formatIter iterates through the string s in increasing +// index order and calls f for each format specifier '%..v'. +// The arguments for f describe the specifier's index range. +// If a format specifier contains a "*", f is called with +// the index range for "*" alone, before being called for +// the entire specifier. The result of f is the index of +// the rune at which iteration continues. +func formatIter(s string, f func(i, j int) int) { + i := 0 // index after current rune + var r rune // current rune + + next := func() { + r1, w := utf8.DecodeRuneInString(s[i:]) + if w == 0 { + r1 = -1 // signal end-of-string + } + r = r1 + i += w + } + + flags := func() { + for r == ' ' || r == '#' || r == '+' || r == '-' || r == '0' { + next() + } + } + + index := func() { + if r == '[' { + log.Fatalf("cannot handle indexed arguments: %s", s) + } + } + + digits := func() { + index() + if r == '*' { + i = f(i-1, i) + next() + return + } + for '0' <= r && r <= '9' { + next() + } + } + + for next(); r >= 0; next() { + if r == '%' { + i0 := i + next() + flags() + digits() + if r == '.' { + next() + digits() + } + index() + // accept any char except for % as format flag + if r == '%' { + if i-i0 == 1 { + continue // skip "%%" + } + log.Fatalf("incorrect format string: %s", s) + } + if r >= 0 { + i = f(i0-1, i) + } + } + } +} + +// isFormat reports whether s contains format specifiers. +func isFormat(s string) (yes bool) { + formatIter(s, func(i, j int) int { + yes = true + return len(s) // stop iteration + }) + return +} + +// oneFormat reports whether s is exactly one format specifier. +func oneFormat(s string) (yes bool) { + formatIter(s, func(i, j int) int { + yes = i == 0 && j == len(s) + return j + }) + return +} + +// numFormatArgs returns the number of format specifiers in s. +func numFormatArgs(s string) int { + count := 0 + formatIter(s, func(i, j int) int { + count++ + return j + }) + return count +} + +// formatReplace replaces the i'th format specifier s in the incoming +// string in with the result of f(i, s) and returns the new string. +func formatReplace(in string, f func(i int, s string) string) string { + var buf []byte + i0 := 0 + index := 0 + formatIter(in, func(i, j int) int { + if sub := in[i:j]; sub != "*" { // ignore calls for "*" width/length specifiers + buf = append(buf, in[i0:i]...) + buf = append(buf, f(index, sub)...) + i0 = j + } + index++ + return j + }) + return string(append(buf, in[i0:]...)) +} + +// functionBlacklisted is the set of functions which may have +// format-like arguments but which don't do any formatting and +// thus may be ignored. +var functionBlacklisted = map[string]bool{ + "len": true, + "strings.ContainsRune": true, + "w.WriteString": true, +} + +func init() { + // verify that knownFormats entries are correctly formatted + for key, val := range knownFormats { + // key must be "typename format", and format starts with a '%' + // (formats containing '*' alone are not collected in this table) + i := strings.Index(key, "%") + if i < 0 || !oneFormat(key[i:]) { + log.Fatalf("incorrect knownFormats key: %q", key) + } + // val must be "format" or "" + if val != "" && !oneFormat(val) { + log.Fatalf("incorrect knownFormats value: %q (key = %q)", val, key) + } + } +} + +// knownFormats entries are of the form "typename oldformat" -> "newformat". +// An absent entry means that the format is not recognized as valid. +// An empty new format means that the existing format should remain unchanged. +// To print out a new table, run: go test -run Formats -v. +var knownFormats = map[string]string{ + "**cmd/compile/internal/big.Rat %v": "", + "*bytes.Buffer %s": "", + "*cmd/compile/internal/big.Float %5s": "", + "*cmd/compile/internal/big.Float %s": "", + "*cmd/compile/internal/big.Float %v": "", + "*cmd/compile/internal/big.Int %#x": "", + "*cmd/compile/internal/big.Int %d": "", + "*cmd/compile/internal/big.Int %s": "", + "*cmd/compile/internal/big.Int %v": "", + "*cmd/compile/internal/big.Int %x": "", + "*cmd/compile/internal/big.Rat %p": "", + "*cmd/compile/internal/big.Rat %s": "", + "*cmd/compile/internal/big.Rat %v": "", + "*cmd/compile/internal/big.matrix %s": "", + "*cmd/compile/internal/gc.Bits %v": "", + "*cmd/compile/internal/gc.Field %p": "", + "*cmd/compile/internal/gc.Field %v": "", + "*cmd/compile/internal/gc.Mpflt %v": "", + "*cmd/compile/internal/gc.Mpint %v": "", + "*cmd/compile/internal/gc.Node %#v": "", + "*cmd/compile/internal/gc.Node %+1v": "", + "*cmd/compile/internal/gc.Node %+v": "", + "*cmd/compile/internal/gc.Node %0j": "", + "*cmd/compile/internal/gc.Node %1v": "", + "*cmd/compile/internal/gc.Node %2v": "", + "*cmd/compile/internal/gc.Node %j": "", + "*cmd/compile/internal/gc.Node %p": "", + "*cmd/compile/internal/gc.Node %s": "", + "*cmd/compile/internal/gc.Node %v": "", + "*cmd/compile/internal/gc.Sym % v": "", + "*cmd/compile/internal/gc.Sym %+v": "", + "*cmd/compile/internal/gc.Sym %-v": "", + "*cmd/compile/internal/gc.Sym %01v": "", + "*cmd/compile/internal/gc.Sym %1v": "", + "*cmd/compile/internal/gc.Sym %p": "", + "*cmd/compile/internal/gc.Sym %s": "", + "*cmd/compile/internal/gc.Sym %v": "", + "*cmd/compile/internal/gc.Type % -v": "", + "*cmd/compile/internal/gc.Type %#v": "", + "*cmd/compile/internal/gc.Type %+v": "", + "*cmd/compile/internal/gc.Type %- v": "", + "*cmd/compile/internal/gc.Type %-1v": "", + "*cmd/compile/internal/gc.Type %-v": "", + "*cmd/compile/internal/gc.Type %01v": "", + "*cmd/compile/internal/gc.Type %1v": "", + "*cmd/compile/internal/gc.Type %2v": "", + "*cmd/compile/internal/gc.Type %p": "", + "*cmd/compile/internal/gc.Type %s": "", + "*cmd/compile/internal/gc.Type %v": "", + "*cmd/compile/internal/ssa.Block %s": "", + "*cmd/compile/internal/ssa.Block %v": "", + "*cmd/compile/internal/ssa.Func %s": "", + "*cmd/compile/internal/ssa.SparseTreeNode %v": "", + "*cmd/compile/internal/ssa.Value %s": "", + "*cmd/compile/internal/ssa.Value %v": "", + "*cmd/compile/internal/ssa.sparseTreeMapEntry %v": "", + "*cmd/internal/obj.Addr %v": "", + "*cmd/internal/obj.Prog %p": "", + "*cmd/internal/obj.Prog %s": "", + "*cmd/internal/obj.Prog %v": "", + "[16]byte %x": "", + "[]*cmd/compile/internal/big.Int %s": "", + "[]*cmd/compile/internal/big.Rat %s": "", + "[]*cmd/compile/internal/gc.Node %v": "", + "[]*cmd/compile/internal/gc.Sig %#v": "", + "[]*cmd/compile/internal/ssa.Value %v": "", + "[]byte %q": "", + "[]byte %s": "", + "[]byte %x": "", + "[]cmd/compile/internal/ssa.Edge %v": "", + "[]cmd/compile/internal/ssa.ID %v": "", + "[]string %v": "", + "bool %t": "", + "bool %v": "", + "byte %02x": "", + "byte %08b": "", + "byte %c": "", + "byte %d": "", + "byte %q": "", + "cmd/compile/internal/arm.shift %d": "", + "cmd/compile/internal/big.Accuracy %d": "", + "cmd/compile/internal/big.Accuracy %s": "", + "cmd/compile/internal/big.Bits %v": "", + "cmd/compile/internal/big.ErrNaN %v": "", + "cmd/compile/internal/big.Int %v": "", + "cmd/compile/internal/big.RoundingMode %d": "", + "cmd/compile/internal/big.RoundingMode %s": "", + "cmd/compile/internal/big.RoundingMode %v": "", + "cmd/compile/internal/big.Word %#x": "", + "cmd/compile/internal/big.Word %d": "", + "cmd/compile/internal/big.Word %x": "", + "cmd/compile/internal/big.argNN %+v": "", + "cmd/compile/internal/big.argVV %+v": "", + "cmd/compile/internal/big.argVW %+v": "", + "cmd/compile/internal/big.argVWW %+v": "", + "cmd/compile/internal/big.argWVW %+v": "", + "cmd/compile/internal/big.argWW %+v": "", + "cmd/compile/internal/big.argZZ %+v": "", + "cmd/compile/internal/big.decimal %v": "", + "cmd/compile/internal/big.nat %v": "", + "cmd/compile/internal/gc.Class %d": "", + "cmd/compile/internal/gc.Ctype %d": "", + "cmd/compile/internal/gc.Ctype %v": "", + "cmd/compile/internal/gc.EType %d": "", + "cmd/compile/internal/gc.EType %s": "", + "cmd/compile/internal/gc.EType %v": "", + "cmd/compile/internal/gc.Level %d": "", + "cmd/compile/internal/gc.Level %v": "", + "cmd/compile/internal/gc.Node %#v": "", + "cmd/compile/internal/gc.Nodes %#s": "", + "cmd/compile/internal/gc.Nodes %#v": "", + "cmd/compile/internal/gc.Nodes %+v": "", + "cmd/compile/internal/gc.Nodes %.v": "", + "cmd/compile/internal/gc.Nodes %v": "", + "cmd/compile/internal/gc.Op %#v": "", + "cmd/compile/internal/gc.Op %d": "", + "cmd/compile/internal/gc.Op %s": "", + "cmd/compile/internal/gc.Op %v": "", + "cmd/compile/internal/gc.Val %#v": "", + "cmd/compile/internal/gc.Val %s": "", + "cmd/compile/internal/gc.Val %v": "", + "cmd/compile/internal/gc.initKind %d": "", + "cmd/compile/internal/ssa.BlockKind %s": "", + "cmd/compile/internal/ssa.BranchPrediction %d": "", + "cmd/compile/internal/ssa.Edge %v": "", + "cmd/compile/internal/ssa.GCNode %s": "", + "cmd/compile/internal/ssa.ID %d": "", + "cmd/compile/internal/ssa.LocalSlot %s": "", + "cmd/compile/internal/ssa.Location %v": "", + "cmd/compile/internal/ssa.Op %s": "", + "cmd/compile/internal/ssa.Op %v": "", + "cmd/compile/internal/ssa.SizeAndAlign %s": "", + "cmd/compile/internal/ssa.Type %s": "", + "cmd/compile/internal/ssa.ValAndOff %s": "", + "cmd/compile/internal/ssa.markKind %d": "", + "cmd/compile/internal/ssa.rbrank %d": "", + "cmd/compile/internal/ssa.regMask %d": "", + "cmd/compile/internal/ssa.register %d": "", + "cmd/compile/internal/syntax.Expr %#v": "", + "cmd/compile/internal/syntax.Expr %s": "", + "cmd/compile/internal/syntax.Node %T": "", + "cmd/compile/internal/syntax.Operator %d": "", + "cmd/compile/internal/syntax.Operator %s": "", + "cmd/compile/internal/syntax.token %d": "", + "cmd/compile/internal/syntax.token %q": "", + "cmd/compile/internal/syntax.token %s": "", + "cmd/internal/obj.As %v": "", + "error %q": "", + "error %s": "", + "error %v": "", + "float32 %b": "", + "float32 %g": "", + "float64 %.2f": "", + "float64 %.3f": "", + "float64 %.5g": "", + "float64 %.6g": "", + "float64 %5g": "", + "float64 %b": "", + "float64 %g": "", + "float64 %v": "", + "fmt.Stringer %T": "", + "int %#x": "", + "int %-12d": "", + "int %-2d": "", + "int %-6d": "", + "int %-8o": "", + "int %2d": "", + "int %3d": "", + "int %5d": "", + "int %6d": "", + "int %c": "", + "int %d": "", + "int %v": "", + "int %x": "", + "int16 %2d": "", + "int16 %d": "", + "int16 %x": "", + "int32 %4d": "", + "int32 %5d": "", + "int32 %d": "", + "int32 %v": "", + "int32 %x": "", + "int64 %#x": "", + "int64 %+d": "", + "int64 %-10d": "", + "int64 %X": "", + "int64 %d": "", + "int64 %v": "", + "int64 %x": "", + "int8 %d": "", + "int8 %x": "", + "interface{} %#v": "", + "interface{} %T": "", + "interface{} %q": "", + "interface{} %s": "", + "interface{} %v": "", + "map[*cmd/compile/internal/gc.Node]*cmd/compile/internal/ssa.Value %v": "", + "reflect.Type %s": "", + "rune %#U": "", + "rune %c": "", + "rune %d": "", + "rune %q": "", + "string %-16s": "", + "string %.*s": "", + "string %q": "", + "string %s": "", + "string %v": "", + "struct{format string; value interface{}; want string} %v": "", + "struct{in string; out string; base int; val int64; ok bool} %v": "", + "struct{s string; base int; frac bool; x cmd/compile/internal/big.nat; b int; count int; ok bool; next rune} %+v": "", + "struct{x cmd/compile/internal/big.nat; b int; s string} %+v": "", + "struct{x float64; format byte; prec int; want string} %v": "", + "struct{x string; prec uint; format byte; digits int; want string} %v": "", + "time.Duration %10s": "", + "time.Duration %4d": "", + "time.Duration %d": "", + "time.Duration %v": "", + "uint %.4d": "", + "uint %04x": "", + "uint %d": "", + "uint %v": "", + "uint16 %d": "", + "uint16 %v": "", + "uint16 %x": "", + "uint32 %#08x": "", + "uint32 %#x": "", + "uint32 %08x": "", + "uint32 %d": "", + "uint32 %x": "", + "uint64 %#016x": "", + "uint64 %#x": "", + "uint64 %016x": "", + "uint64 %08x": "", + "uint64 %d": "", + "uint64 %x": "", + "uint8 %d": "", + "uint8 %x": "", + "uintptr %d": "", +} diff --git a/src/cmd/compile/internal/gc/fmt_test.go b/src/cmd/compile/internal/gc/fmt_test.go deleted file mode 100644 index 06026ac153..0000000000 --- a/src/cmd/compile/internal/gc/fmt_test.go +++ /dev/null @@ -1,600 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// This file implements TestFormats; a test that verifies -// all format strings in the compiler. -// -// TestFormats finds potential (Printf, etc.) format strings. -// If they are used in a call, the format verbs are verified -// based on the matching argument type against a precomputed -// table of valid formats (formatMapping, below). The table -// can be used to automatically rewrite format strings with -// the -u flag. -// -// Run as: go test -run Formats [-u] -// -// A formatMapping based on the existing formats is printed -// when the test is run in verbose mode (-v flag). The table -// needs to be updated whenever a new (type, format) combination -// is found and the format verb is not 'v' (as in "%v"). -// -// Known bugs: -// - indexed format strings ("%[2]s", etc.) are not supported -// (the test will fail) -// - format strings that are not simple string literals cannot -// be updated automatically -// (the test will fail with respective warnings) -// -package gc_test - -import ( - "bytes" - "flag" - "fmt" - "go/ast" - "go/build" - "go/constant" - "go/format" - "go/importer" - "go/parser" - "go/token" - "go/types" - "internal/testenv" - "io/ioutil" - "log" - "sort" - "strconv" - "strings" - "testing" - "unicode/utf8" -) - -var update = flag.Bool("u", false, "update format strings") - -var fset = token.NewFileSet() // file set for all processed files -var typedPkg *types.Package // the package we are type-checking - -// A CallSite describes a function call that appears to contain -// a format string. -type CallSite struct { - file int // buildPkg.GoFiles index - call *ast.CallExpr // call containing the format string - arg ast.Expr // format argument (string literal or constant) - str string // unquoted format string - types []string // argument types -} - -func TestFormats(t *testing.T) { - testenv.MustHaveGoBuild(t) // more restrictive than necessary, but that's ok - - // determine .go files - buildPkg, err := build.ImportDir(".", 0) - if err != nil { - t.Fatal(err) - } - - // parse all files - files := make([]*ast.File, len(buildPkg.GoFiles)) - for i, filename := range buildPkg.GoFiles { - f, err := parser.ParseFile(fset, filename, nil, parser.ParseComments) - if err != nil { - t.Fatal(err) - } - files[i] = f - } - - // typecheck package - conf := types.Config{Importer: importer.Default()} - etypes := make(map[ast.Expr]types.TypeAndValue) - typedPkg, err = conf.Check("some_path", fset, files, &types.Info{Types: etypes}) - if err != nil { - t.Fatal(err) - } - - // collect all potential format strings (for extra verification later) - potentialFormats := make(map[*ast.BasicLit]bool) - for _, file := range files { - ast.Inspect(file, func(n ast.Node) bool { - if s, ok := stringLit(n); ok && isFormat(s) { - potentialFormats[n.(*ast.BasicLit)] = true - } - return true - }) - } - - // collect all formats/arguments of calls with format strings - var callSites []*CallSite - for index, file := range files { - ast.Inspect(file, func(n ast.Node) bool { - if call, ok := n.(*ast.CallExpr); ok { - // ignore blackisted functions - if functionBlacklisted[nodeString(call.Fun)] { - return true - } - // look for an arguments that might be a format string - for i, arg := range call.Args { - if s, ok := stringVal(etypes[arg]); ok && isFormat(s) { - // make sure we have enough arguments - n := formatArgs(s) - if i+1+n > len(call.Args) { - t.Errorf("%s: not enough format args (blacklist %s?)", posString(call), nodeString(call.Fun)) - break // ignore this call - } - // assume last n arguments are to be formatted; - // determine their types - argTypes := make([]string, n) - for i, arg := range call.Args[len(call.Args)-n:] { - if tv, ok := etypes[arg]; ok { - argTypes[i] = typeString(tv.Type) - } - } - // collect call site - callSites = append(callSites, &CallSite{ - file: index, - call: call, - arg: arg, - str: s, - types: argTypes, - }) - break // at most one format per argument list - } - } - } - return true - }) - } - - // test and rewrite formats - keys := make(map[string]bool) - dirty := false // dirty means some files have been modified - dirtyFiles := make([]bool, len(files)) // dirtyFiles[i] means file i has been modified - for _, p := range callSites { - // test current format literal and determine updated one - out := formatReplace(p.str, func(index int, in string) string { - if in == "*" { - return in - } - typ := p.types[index] - key := typ + " " + in // e.g., "*Node %n" - keys[key] = true - out, found := formatMapping[key] - if !found && in != "%v" { // always accept simple "%v" - t.Errorf("%s: unknown format %q for %s argument", posString(p.arg), in, typ) - } - if out == "" { - out = in - } - return out - }) - - // replace existing format literal if it changed - if out != p.str { - if testing.Verbose() { - fmt.Printf("%s:\n\t %q\n\t=> %q\n", posString(p.arg), p.str, out) - } - // find argument index of format argument - index := -1 - for i, arg := range p.call.Args { - if p.arg == arg { - index = i - break - } - } - if index < 0 { - panic("internal error: matching argument not found") - } - // we cannot replace the argument if it's not a string literal for now - // (e.g., it may be "foo" + "bar") - lit, ok := p.arg.(*ast.BasicLit) - if !ok { - continue // we issue a warning about missed format strings at the end - } - new := *lit // make a copy - new.Value = strconv.Quote(out) // this may introduce "-quotes where there were `-quotes - p.call.Args[index] = &new - dirty = true - dirtyFiles[p.file] = true // mark file as changed - } - } - - // write dirty files back - if dirty && *update { - for i, d := range dirtyFiles { - if !d { - continue - } - filename := buildPkg.GoFiles[i] - var buf bytes.Buffer - if err := format.Node(&buf, fset, files[i]); err != nil { - t.Errorf("WARNING: formatting %s failed: %v", filename, err) - continue - } - if err := ioutil.WriteFile(filename, buf.Bytes(), 0x666); err != nil { - t.Errorf("WARNING: writing %s failed: %v", filename, err) - continue - } - fmt.Println(filename) - } - } - - // report all function names containing a format string - if len(callSites) > 0 && testing.Verbose() { - set := make(map[string]bool) - for _, p := range callSites { - set[nodeString(p.call.Fun)] = true - } - var list []string - for s := range set { - list = append(list, s) - } - fmt.Println("\nFunctions") - printList(list) - } - - // report all keys found - if len(keys) > 0 && testing.Verbose() { - var list []string - for s := range keys { - list = append(list, fmt.Sprintf("%q: \"\",", s)) - } - fmt.Println("\nvar formatMapping = map[string]string{") - printList(list) - fmt.Println("}") - } - - // all format literals must in the potentialFormats set (self-verification) - for _, p := range callSites { - if lit, ok := p.arg.(*ast.BasicLit); ok && lit.Kind == token.STRING { - if potentialFormats[lit] { - // ok - delete(potentialFormats, lit) - } else { - // this should never happen - panic(fmt.Sprintf("internal error: format string not found (%s)", posString(lit))) - } - } - } - - // if we have any strings left, we may need to update them manually - if len(potentialFormats) > 0 && dirty && *update { - var list []string - for lit := range potentialFormats { - list = append(list, fmt.Sprintf("%s: %s", posString(lit), nodeString(lit))) - } - fmt.Println("\nWARNING: Potentially missed format strings") - printList(list) - t.Fail() - } -} - -// printList prints list in sorted order. -func printList(list []string) { - sort.Strings(list) - for _, s := range list { - fmt.Println("\t", s) - } -} - -// posString returns a string representation of n's position -// in the form filename:line:col: . -func posString(n ast.Node) string { - if n == nil { - return "" - } - return fset.Position(n.Pos()).String() -} - -// nodeString returns a string representation of n. -func nodeString(n ast.Node) string { - var buf bytes.Buffer - if err := format.Node(&buf, fset, n); err != nil { - log.Fatal(err) // should always succeed - } - return buf.String() -} - -// typeString returns a string representation of t. -func typeString(t types.Type) string { - return types.TypeString(t, func(pkg *types.Package) string { - // don't qualify type names of the typed package - if pkg == typedPkg { - return "" - } - return pkg.Path() - }) -} - -// stringLit returns the unquoted string value and true if -// n represents a string literal; otherwise it returns "" -// and false. -func stringLit(n ast.Node) (string, bool) { - if lit, ok := n.(*ast.BasicLit); ok && lit.Kind == token.STRING { - s, err := strconv.Unquote(lit.Value) - if err != nil { - log.Fatal(err) // should not happen with correct ASTs - } - return s, true - } - return "", false -} - -// stringVal returns the (unquoted) string value and true if -// tv is a string constant; otherwise it returns "" and false. -func stringVal(tv types.TypeAndValue) (string, bool) { - if tv.IsValue() && tv.Value != nil && tv.Value.Kind() == constant.String { - return constant.StringVal(tv.Value), true - } - return "", false -} - -// formatIter iterates through the string s in increasing -// index order and calls f for each format specifier '%..v'. -// The arguments for f describe the specifier's index range. -// If a format specifier contains a "*", f is called with -// the index range for "*" alone, before being called for -// the entire specifier. The result of f is the index of -// the rune at which iteration continues. -func formatIter(s string, f func(i, j int) int) { - i := 0 // index after current rune - var r rune // current rune - - next := func() { - r1, w := utf8.DecodeRuneInString(s[i:]) - if w == 0 { - r1 = -1 // signal end-of-string - } - r = r1 - i += w - } - - flags := func() { - for r == ' ' || r == '#' || r == '+' || r == '-' || r == '0' { - next() - } - } - - index := func() { - if r == '[' { - log.Fatalf("cannot handle indexed arguments: %s", s) - } - } - - digits := func() { - index() - if r == '*' { - i = f(i-1, i) - next() - return - } - for '0' <= r && r <= '9' { - next() - } - } - - for next(); r >= 0; next() { - if r == '%' { - i0 := i - next() - flags() - digits() - if r == '.' { - next() - digits() - } - index() - // accept any char except for % as format flag - if r == '%' { - if i-i0 == 1 { - continue // skip "%%" - } - log.Fatalf("incorrect format string: %s", s) - } - if r >= 0 { - i = f(i0-1, i) - } - } - } -} - -// isFormat reports whether s contains format specifiers. -func isFormat(s string) (yes bool) { - formatIter(s, func(i, j int) int { - yes = true - return len(s) // stop iteration - }) - return -} - -// oneFormat reports whether s is exactly one format specifier. -func oneFormat(s string) (yes bool) { - formatIter(s, func(i, j int) int { - yes = i == 0 && j == len(s) - return j - }) - return -} - -// formatArgs counts the number of format specifiers in s. -func formatArgs(s string) int { - count := 0 - formatIter(s, func(i, j int) int { - count++ - return j - }) - return count -} - -// formatReplace replaces the i'th format specifier s in the incoming -// string in with the result of f(i, s) and returns the new string. -func formatReplace(in string, f func(i int, s string) string) string { - var buf []byte - i0 := 0 - index := 0 - formatIter(in, func(i, j int) int { - if sub := in[i:j]; sub != "*" { // ignore calls for "*" width/length specifiers - buf = append(buf, in[i0:i]...) - buf = append(buf, f(index, sub)...) - i0 = j - } - index++ - return j - }) - return string(append(buf, in[i0:]...)) -} - -func init() { - // verify that formatMapping entries are correctly formatted - for key, val := range formatMapping { - // key must be "typename format" (format may be "*") - i := strings.Index(key, " ") - if i < 0 || key[i+1:] != "*" && !oneFormat(key[i+1:]) { - log.Fatalf("incorrect formatMapping key: %q", key) - } - // val must be "format" or "" - if val != "" && !oneFormat(val) { - log.Fatalf("incorrect formatMapping key: %q", key) - } - } -} - -// functionBlacklisted is the set of functions which are known to -// have format-like arguments but which don't do any formatting -// and thus can be ignored. -var functionBlacklisted = map[string]bool{ - "len": true, - "strings.ContainsRune": true, -} - -// formatMapping entries are of the form "typename oldformat" -> "newformat". -// An absent entry means that the format is not recognized as correct in test -// mode. -// An empty new format means that the existing format should remain unchanged. -// To generate a new table, run: go test -run Formats -v. -var formatMapping = map[string]string{ - "*Bits %v": "", - "*Field %p": "", - "*Field %v": "", - "*Mpflt %v": "", - "*Mpint %v": "", - "*Node %#v": "", - "*Node %+1v": "", - "*Node %+v": "", - "*Node %0j": "", - "*Node %1v": "", - "*Node %2v": "", - "*Node %j": "", - "*Node %p": "", - "*Node %s": "", - "*Node %v": "", - "*Sym % v": "", - "*Sym %+v": "", - "*Sym %-v": "", - "*Sym %01v": "", - "*Sym %1v": "", - "*Sym %p": "", - "*Sym %s": "", - "*Sym %v": "", - "*Type % -v": "", - "*Type %#v": "", - "*Type %+v": "", - "*Type %- v": "", - "*Type %-1v": "", - "*Type %-v": "", - "*Type %01v": "", - "*Type %1v": "", - "*Type %2v": "", - "*Type %p": "", - "*Type %s": "", - "*Type %v": "", - "*cmd/compile/internal/big.Int %#x": "", - "*cmd/compile/internal/ssa.Block %v": "", - "*cmd/compile/internal/ssa.Func %s": "", - "*cmd/compile/internal/ssa.Value %s": "", - "*cmd/compile/internal/ssa.Value %v": "", - "*cmd/internal/obj.Addr %v": "", - "*cmd/internal/obj.Prog %p": "", - "*cmd/internal/obj.Prog %s": "", - "*cmd/internal/obj.Prog %v": "", - "Class %d": "", - "Ctype %d": "", - "Ctype %v": "", - "EType %d": "", - "EType %s": "", - "EType %v": "", - "Level %d": "", - "Level %v": "", - "Nodes %#s": "", - "Nodes %#v": "", - "Nodes %+v": "", - "Nodes %.v": "", - "Nodes %v": "", - "Op %#v": "", - "Op %d": "", - "Op %s": "", - "Op %v": "", - "Val %#v": "", - "Val %s": "", - "Val %v": "", - "[16]byte %x": "", - "[]*Node %v": "", - "[]byte %s": "", - "[]byte %x": "", - "bool %t": "", - "bool %v": "", - "byte %02x": "", - "byte %c": "", - "cmd/compile/internal/ssa.Location %v": "", - "cmd/compile/internal/ssa.Type %s": "", - "cmd/compile/internal/syntax.Expr %#v": "", - "error %v": "", - "float64 %.2f": "", - "float64 %.6g": "", - "float64 %g": "", - "fmt.Stringer %T": "", - "initKind %d": "", - "int %-12d": "", - "int %-2d": "", - "int %-6d": "", - "int %-8o": "", - "int %2d": "", - "int %6d": "", - "int %c": "", - "int %d": "", - "int %v": "", - "int %x": "", - "int16 %2d": "", - "int16 %d": "", - "int32 %4d": "", - "int32 %5d": "", - "int32 %d": "", - "int32 %v": "", - "int64 %+d": "", - "int64 %-10d": "", - "int64 %d": "", - "int64 %v": "", - "int8 %d": "", - "interface{} %#v": "", - "interface{} %T": "", - "interface{} %v": "", - "map[*Node]*cmd/compile/internal/ssa.Value %v": "", - "reflect.Type %s": "", - "rune %#U": "", - "rune %c": "", - "rune %d": "", - "string %-16s": "", - "string %.*s": "", - "string %q": "", - "string %s": "", - "string %v": "", - "time.Duration %d": "", - "uint %.4d": "", - "uint %04x": "", - "uint16 %d": "", - "uint16 %v": "", - "uint32 %#x": "", - "uint32 %d": "", - "uint64 %#x": "", - "uint64 %08x": "", - "uint8 %d": "", -} -- 2.48.1