go/types: add UsesCgo config to support _cgo_gotypes.go

(Reland of golang.org/cl/33677.)

This CL adds a UsesCgo config setting to go/types to specify that the
_cgo_gotypes.go file generated by cmd/cgo has been provided as a
source file. The type checker then internally resolves C.bar qualified
identifiers to _Cfoo_bar as appropriate.

It also adds support to srcimporter to automatically run cgo.
Unfortunately, this functionality is not compatible with overriding
OpenFile, because cmd/cgo and gcc will directly open files.

Updates #16623.
Updates #35721.

Change-Id: Ib179d55c8c589916f98ceeae0b9a3e746157253a
Reviewed-on: https://go-review.googlesource.com/c/go/+/231459
Run-TryBot: Matthew Dempsky <mdempsky@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Robert Griesemer <gri@golang.org>

diff --git a/src/go/internal/srcimporter/srcimporter.go b/src/go/internal/srcimporter/srcimporter.go
index 2a6c274424d15b81a7feb5713a3fcac1a300ddca..daef27c8b9dc86be5e83e0f143e727813a1f068b 100644 (file)
--- a/src/go/internal/srcimporter/srcimporter.go
+++ b/src/go/internal/srcimporter/srcimporter.go
@@ -14,8 +14,11 @@ import (
        "go/token"
        "go/types"
        "io"
+       "io/ioutil"
        "os"
+       "os/exec"
        "path/filepath"
+       "strings"
        "sync"
 )
 
@@ -115,7 +118,6 @@ func (p *Importer) ImportFrom(path, srcDir string, mode types.ImportMode) (*type
        var firstHardErr error
        conf := types.Config{
                IgnoreFuncBodies: true,
-               FakeImportC:      true,
                // continue type-checking after the first error
                Error: func(err error) {
                        if firstHardErr == nil && !err.(types.Error).Soft {
@@ -125,6 +127,21 @@ func (p *Importer) ImportFrom(path, srcDir string, mode types.ImportMode) (*type
                Importer: p,
                Sizes:    p.sizes,
        }
+       if len(bp.CgoFiles) > 0 {
+               if p.ctxt.OpenFile != nil {
+                       // cgo, gcc, pkg-config, etc. do not support
+                       // build.Context's VFS.
+                       conf.FakeImportC = true
+               } else {
+                       conf.UsesCgo = true
+                       file, err := p.cgo(bp)
+                       if err != nil {
+                               return nil, err
+                       }
+                       files = append(files, file)
+               }
+       }
+
        pkg, err = conf.Check(bp.ImportPath, p.fset, files, nil)
        if err != nil {
                // If there was a hard error it is possibly unsafe
@@ -181,6 +198,47 @@ func (p *Importer) parseFiles(dir string, filenames []string) ([]*ast.File, erro
        return files, nil
 }
 
+func (p *Importer) cgo(bp *build.Package) (*ast.File, error) {
+       tmpdir, err := ioutil.TempDir("", "srcimporter")
+       if err != nil {
+               return nil, err
+       }
+       defer os.RemoveAll(tmpdir)
+
+       args := []string{"go", "tool", "cgo", "-objdir", tmpdir}
+       if bp.Goroot {
+               switch bp.ImportPath {
+               case "runtime/cgo":
+                       args = append(args, "-import_runtime_cgo=false", "-import_syscall=false")
+               case "runtime/race":
+                       args = append(args, "-import_syscall=false")
+               }
+       }
+       args = append(args, "--")
+       args = append(args, strings.Fields(os.Getenv("CGO_CPPFLAGS"))...)
+       args = append(args, bp.CgoCPPFLAGS...)
+       if len(bp.CgoPkgConfig) > 0 {
+               cmd := exec.Command("pkg-config", append([]string{"--cflags"}, bp.CgoPkgConfig...)...)
+               out, err := cmd.CombinedOutput()
+               if err != nil {
+                       return nil, err
+               }
+               args = append(args, strings.Fields(string(out))...)
+       }
+       args = append(args, "-I", tmpdir)
+       args = append(args, strings.Fields(os.Getenv("CGO_CFLAGS"))...)
+       args = append(args, bp.CgoCFLAGS...)
+       args = append(args, bp.CgoFiles...)
+
+       cmd := exec.Command(args[0], args[1:]...)
+       cmd.Dir = bp.Dir
+       if err := cmd.Run(); err != nil {
+               return nil, err
+       }
+
+       return parser.ParseFile(p.fset, filepath.Join(tmpdir, "_cgo_gotypes.go"), nil, 0)
+}
+
 // context-controlled file system operations
 
 func (p *Importer) absPath(path string) (string, error) {

diff --git a/src/go/internal/srcimporter/srcimporter_test.go b/src/go/internal/srcimporter/srcimporter_test.go
index 56549434d14d3ec15611d46a8b265d1fc212b7e2..c456b8e26a7823f355213ca2fcc9c4b910789c9d 100644 (file)
--- a/src/go/internal/srcimporter/srcimporter_test.go
+++ b/src/go/internal/srcimporter/srcimporter_test.go
@@ -232,3 +232,14 @@ func TestIssue23092(t *testing.T) {
 func TestIssue24392(t *testing.T) {
        testImportPath(t, "go/internal/srcimporter/testdata/issue24392")
 }
+
+func TestCgo(t *testing.T) {
+       testenv.MustHaveGoBuild(t)
+       testenv.MustHaveCGO(t)
+
+       importer := New(&build.Default, token.NewFileSet(), make(map[string]*types.Package))
+       _, err := importer.ImportFrom("./misc/cgo/test", runtime.GOROOT(), 0)
+       if err != nil {
+               t.Fatalf("Import failed: %v", err)
+       }
+}

diff --git a/src/go/types/api.go b/src/go/types/api.go
index 2a21ad0c53f2e36c9ff5e1822424a112ee0c63d1..7787b88906e3586fd1524d728bfbe3fee56fb6a4 100644 (file)
--- a/src/go/types/api.go
+++ b/src/go/types/api.go
@@ -105,6 +105,15 @@ type Config struct {
        //          Do not use casually!
        FakeImportC bool
 
+       // If UsesCgo is set, the type checker expects the
+       // _cgo_gotypes.go file generated by running cmd/cgo to be
+       // provided as a package source file. Qualified identifiers
+       // referring to package C will be resolved to cgo-provided
+       // declarations within _cgo_gotypes.go.
+       //
+       // It is an error to set both FakeImportC and UsesCgo.
+       UsesCgo bool
+
        // If Error != nil, it is called with each error found
        // during type checking; err has dynamic type Error.
        // Secondary errors (for instance, to enumerate all types
@@ -281,7 +290,7 @@ func (tv TypeAndValue) IsBuiltin() bool {
 // nil Value.
 func (tv TypeAndValue) IsValue() bool {
        switch tv.mode {
-       case constant_, variable, mapindex, value, commaok:
+       case constant_, variable, mapindex, value, commaok, commaerr:
                return true
        }
        return false

diff --git a/src/go/types/assignments.go b/src/go/types/assignments.go
index efa0cbba50270a73ba317fd7272bc8ced1ab017d..34a9d7843d94388ecc4419f4174b559195ee181b 100644 (file)
--- a/src/go/types/assignments.go
+++ b/src/go/types/assignments.go
@@ -22,7 +22,7 @@ func (check *Checker) assignment(x *operand, T Type, context string) {
        switch x.mode {
        case invalid:
                return // error reported before
-       case constant_, variable, mapindex, value, commaok:
+       case constant_, variable, mapindex, value, commaok, commaerr:
                // ok
        default:
                unreachable()

diff --git a/src/go/types/call.go b/src/go/types/call.go
index 9ea6a6551d4e95d678975ef0959f0d544b933acc..be4cfdf9eb458e0433f0bfc201273467c5313698 100644 (file)
--- a/src/go/types/call.go
+++ b/src/go/types/call.go
@@ -9,6 +9,7 @@ package types
 import (
        "go/ast"
        "go/token"
+       "strings"
        "unicode"
 )
 
@@ -55,6 +56,8 @@ func (check *Checker) call(x *operand, e *ast.CallExpr) exprKind {
 
        default:
                // function/method call
+               cgocall := x.mode == cgofunc
+
                sig, _ := x.typ.Underlying().(*Signature)
                if sig == nil {
                        check.invalidOp(x.pos(), "cannot call non-function %s", x)
@@ -75,7 +78,11 @@ func (check *Checker) call(x *operand, e *ast.CallExpr) exprKind {
                case 0:
                        x.mode = novalue
                case 1:
-                       x.mode = value
+                       if cgocall {
+                               x.mode = commaerr
+                       } else {
+                               x.mode = value
+                       }
                        x.typ = sig.results.vars[0].typ // unpack tuple
                default:
                        x.mode = value
@@ -193,10 +200,13 @@ func unpack(get getter, n int, allowCommaOk bool) (getter, int, bool) {
                }, t.Len(), false
        }
 
-       if x0.mode == mapindex || x0.mode == commaok {
+       if x0.mode == mapindex || x0.mode == commaok || x0.mode == commaerr {
                // comma-ok value
                if allowCommaOk {
                        a := [2]Type{x0.typ, Typ[UntypedBool]}
+                       if x0.mode == commaerr {
+                               a[1] = universeError
+                       }
                        return func(x *operand, i int) {
                                x.mode = value
                                x.expr = x0.expr
@@ -303,6 +313,17 @@ func (check *Checker) argument(sig *Signature, i int, x *operand, ellipsis token
        check.assignment(x, typ, context)
 }
 
+var cgoPrefixes = [...]string{
+       "_Ciconst_",
+       "_Cfconst_",
+       "_Csconst_",
+       "_Ctype_",
+       "_Cvar_", // actually a pointer to the var
+       "_Cfpvar_fp_",
+       "_Cfunc_",
+       "_Cmacro_", // function to evaluate the expanded expression
+}
+
 func (check *Checker) selector(x *operand, e *ast.SelectorExpr) {
        // these must be declared before the "goto Error" statements
        var (
@@ -323,16 +344,43 @@ func (check *Checker) selector(x *operand, e *ast.SelectorExpr) {
                        check.recordUse(ident, pname)
                        pname.used = true
                        pkg := pname.imported
-                       exp := pkg.scope.Lookup(sel)
-                       if exp == nil {
-                               if !pkg.fake {
-                                       check.errorf(e.Sel.Pos(), "%s not declared by package %s", sel, pkg.name)
+
+                       var exp Object
+                       funcMode := value
+                       if pkg.cgo {
+                               // cgo special cases C.malloc: it's
+                               // rewritten to _CMalloc and does not
+                               // support two-result calls.
+                               if sel == "malloc" {
+                                       sel = "_CMalloc"
+                               } else {
+                                       funcMode = cgofunc
+                               }
+                               for _, prefix := range cgoPrefixes {
+                                       // cgo objects are part of the current package (in file
+                                       // _cgo_gotypes.go). Use regular lookup.
+                                       _, exp = check.scope.LookupParent(prefix+sel, check.pos)
+                                       if exp != nil {
+                                               break
+                                       }
+                               }
+                               if exp == nil {
+                                       check.errorf(e.Sel.Pos(), "%s not declared by package C", sel)
+                                       goto Error
+                               }
+                               check.objDecl(exp, nil)
+                       } else {
+                               exp = pkg.scope.Lookup(sel)
+                               if exp == nil {
+                                       if !pkg.fake {
+                                               check.errorf(e.Sel.Pos(), "%s not declared by package %s", sel, pkg.name)
+                                       }
+                                       goto Error
+                               }
+                               if !exp.Exported() {
+                                       check.errorf(e.Sel.Pos(), "%s not exported by package %s", sel, pkg.name)
+                                       // ok to continue
                                }
-                               goto Error
-                       }
-                       if !exp.Exported() {
-                               check.errorf(e.Sel.Pos(), "%s not exported by package %s", sel, pkg.name)
-                               // ok to continue
                        }
                        check.recordUse(e.Sel, exp)
 
@@ -350,9 +398,16 @@ func (check *Checker) selector(x *operand, e *ast.SelectorExpr) {
                        case *Var:
                                x.mode = variable
                                x.typ = exp.typ
+                               if pkg.cgo && strings.HasPrefix(exp.name, "_Cvar_") {
+                                       x.typ = x.typ.(*Pointer).base
+                               }
                        case *Func:
-                               x.mode = value
+                               x.mode = funcMode
                                x.typ = exp.typ
+                               if pkg.cgo && strings.HasPrefix(exp.name, "_Cmacro_") {
+                                       x.mode = value
+                                       x.typ = x.typ.(*Signature).results.vars[0].typ
+                               }
                        case *Builtin:
                                x.mode = builtin
                                x.typ = exp.typ

diff --git a/src/go/types/check.go b/src/go/types/check.go
index 71d49ad83d583dbf8d61be6e5e2044e6cf643e93..a94770ffef3ed67b3f9e352e4e8ee03836252a75 100644 (file)
--- a/src/go/types/check.go
+++ b/src/go/types/check.go
@@ -7,6 +7,7 @@
 package types
 
 import (
+       "errors"
        "go/ast"
        "go/constant"
        "go/token"
@@ -247,7 +248,13 @@ func (check *Checker) handleBailout(err *error) {
 // Files checks the provided files as part of the checker's package.
 func (check *Checker) Files(files []*ast.File) error { return check.checkFiles(files) }
 
+var errBadCgo = errors.New("cannot use FakeImportC and UsesCgo together")
+
 func (check *Checker) checkFiles(files []*ast.File) (err error) {
+       if check.conf.FakeImportC && check.conf.UsesCgo {
+               return errBadCgo
+       }
+
        defer check.handleBailout(&err)
 
        check.initFiles(files)
@@ -348,7 +355,7 @@ func (check *Checker) recordCommaOkTypes(x ast.Expr, a [2]Type) {
        if a[0] == nil || a[1] == nil {
                return
        }
-       assert(isTyped(a[0]) && isTyped(a[1]) && isBoolean(a[1]))
+       assert(isTyped(a[0]) && isTyped(a[1]) && (isBoolean(a[1]) || a[1] == universeError))
        if m := check.Types; m != nil {
                for {
                        tv := m[x]

diff --git a/src/go/types/operand.go b/src/go/types/operand.go
index 43b7385ad86dea9b5470cc297bc26a5860bc82ab..80d11e2f2153019a225b126dd0c12cbf9aafd612 100644 (file)
--- a/src/go/types/operand.go
+++ b/src/go/types/operand.go
@@ -27,6 +27,8 @@ const (
        mapindex                     // operand is a map index expression (acts like a variable on lhs, commaok on rhs of an assignment)
        value                        // operand is a computed value
        commaok                      // like value, but operand may be used in a comma,ok expression
+       commaerr                     // like commaok, but second value is error, not boolean
+       cgofunc                      // operand is a cgo function
 )
 
 var operandModeString = [...]string{
@@ -39,6 +41,8 @@ var operandModeString = [...]string{
        mapindex:  "map index expression",
        value:     "value",
        commaok:   "comma, ok expression",
+       commaerr:  "comma, error expression",
+       cgofunc:   "cgo function",
 }
 
 // An operand represents an intermediate value during type checking.
@@ -94,6 +98,12 @@ func (x *operand) pos() token.Pos {
 // commaok    <expr> (<untyped kind> <mode>                    )
 // commaok    <expr> (               <mode>       of type <typ>)
 //
+// commaerr   <expr> (<untyped kind> <mode>                    )
+// commaerr   <expr> (               <mode>       of type <typ>)
+//
+// cgofunc    <expr> (<untyped kind> <mode>                    )
+// cgofunc    <expr> (               <mode>       of type <typ>)
+//
 func operandString(x *operand, qf Qualifier) string {
        var buf bytes.Buffer
 

diff --git a/src/go/types/package.go b/src/go/types/package.go
index cd202a0ed9f789c558c69c60ef6ed0ecbab966a9..7b89def1b5d5137c5572131cf336cb12f97e3257 100644 (file)
--- a/src/go/types/package.go
+++ b/src/go/types/package.go
@@ -17,6 +17,7 @@ type Package struct {
        complete bool
        imports  []*Package
        fake     bool // scope lookup errors are silently dropped if package is fake (internal use only)
+       cgo      bool // uses of this package will be rewritten into uses of declarations from _cgo_gotypes.go
 }
 
 // NewPackage returns a new Package for the given package path and name.

diff --git a/src/go/types/resolver.go b/src/go/types/resolver.go
index 839d076e361447223b7c821ad07fe07128dccc33..f80b4ec7841f17d79c2c9ced9b6581c44e02645a 100644 (file)
--- a/src/go/types/resolver.go
+++ b/src/go/types/resolver.go
@@ -141,9 +141,10 @@ func (check *Checker) importPackage(pos token.Pos, path, dir string) *Package {
        }
 
        // no package yet => import it
-       if path == "C" && check.conf.FakeImportC {
+       if path == "C" && (check.conf.FakeImportC || check.conf.UsesCgo) {
                imp = NewPackage("C", "C")
-               imp.fake = true
+               imp.fake = true // package scope is not populated
+               imp.cgo = check.conf.UsesCgo
        } else {
                // ordinary import
                var err error

diff --git a/src/go/types/universe.go b/src/go/types/universe.go
index 7af6dab320edd1beb0b865fe58e80bdbcd51e5d8..ff5b89118ad9ef940ae1fbe1e00276190e6f0476 100644 (file)
--- a/src/go/types/universe.go
+++ b/src/go/types/universe.go
@@ -21,9 +21,10 @@ var Universe *Scope
 var Unsafe *Package
 
 var (
-       universeIota *Const
-       universeByte *Basic // uint8 alias, but has name "byte"
-       universeRune *Basic // int32 alias, but has name "rune"
+       universeIota  *Const
+       universeByte  *Basic // uint8 alias, but has name "byte"
+       universeRune  *Basic // int32 alias, but has name "rune"
+       universeError *Named
 )
 
 // Typ contains the predeclared *Basic types indexed by their
@@ -200,6 +201,7 @@ func init() {
        universeIota = Universe.Lookup("iota").(*Const)
        universeByte = Universe.Lookup("byte").(*TypeName).typ.(*Basic)
        universeRune = Universe.Lookup("rune").(*TypeName).typ.(*Basic)
+       universeError = Universe.Lookup("error").(*TypeName).typ.(*Named)
 }
 
 // Objects with names containing blanks are internal and not entered into