cmd/vet: avoid crash in cgo test on recursive type

This CL also re-enables the cgo tests that were accidentally disabled
in CL 32754.

Fixes #18389.

Change-Id: I2fdc4fe3ec1f92b7da3db3fa66f4e0f806fc899f
Reviewed-on: https://go-review.googlesource.com/34660
Reviewed-by: Brad Fitzpatrick <bradfitz@golang.org>

diff --git a/src/cmd/vet/cgo.go b/src/cmd/vet/cgo.go
index d233e9a96020bebffc71518359473df0abc837db..984911c489f9d70c8c71e371ccbc6c1ac452146f 100644 (file)
--- a/src/cmd/vet/cgo.go
+++ b/src/cmd/vet/cgo.go
@@ -44,7 +44,7 @@ func checkCgoCall(f *File, node ast.Node) {
        }
 
        for _, arg := range x.Args {
-               if !typeOKForCgoCall(cgoBaseType(f, arg)) {
+               if !typeOKForCgoCall(cgoBaseType(f, arg), make(map[types.Type]bool)) {
                        f.Badf(arg.Pos(), "possibly passing Go type with embedded pointer to C")
                }
 
@@ -53,7 +53,7 @@ func checkCgoCall(f *File, node ast.Node) {
                        arg = conv.Args[0]
                }
                if u, ok := arg.(*ast.UnaryExpr); ok && u.Op == token.AND {
-                       if !typeOKForCgoCall(cgoBaseType(f, u.X)) {
+                       if !typeOKForCgoCall(cgoBaseType(f, u.X), make(map[types.Type]bool)) {
                                f.Badf(arg.Pos(), "possibly passing Go type with embedded pointer to C")
                        }
                }
@@ -110,23 +110,24 @@ func cgoBaseType(f *File, arg ast.Expr) types.Type {
        return f.pkg.types[arg].Type
 }
 
-// typeOKForCgoCall returns true if the type of arg is OK to pass to a
+// typeOKForCgoCall reports whether the type of arg is OK to pass to a
 // C function using cgo. This is not true for Go types with embedded
-// pointers.
-func typeOKForCgoCall(t types.Type) bool {
-       if t == nil {
+// pointers. m is used to avoid infinite recursion on recursive types.
+func typeOKForCgoCall(t types.Type, m map[types.Type]bool) bool {
+       if t == nil || m[t] {
                return true
        }
+       m[t] = true
        switch t := t.Underlying().(type) {
        case *types.Chan, *types.Map, *types.Signature, *types.Slice:
                return false
        case *types.Pointer:
-               return typeOKForCgoCall(t.Elem())
+               return typeOKForCgoCall(t.Elem(), m)
        case *types.Array:
-               return typeOKForCgoCall(t.Elem())
+               return typeOKForCgoCall(t.Elem(), m)
        case *types.Struct:
                for i := 0; i < t.NumFields(); i++ {
-                       if !typeOKForCgoCall(t.Field(i).Type()) {
+                       if !typeOKForCgoCall(t.Field(i).Type(), m) {
                                return false
                        }
                }

diff --git a/src/cmd/vet/testdata/cgo/cgo.go b/src/cmd/vet/testdata/cgo/cgo.go
index 25d395b1ea89008cb38c62c11a1cae12285696d0..d0df7cf6787fe00f12054f9fed434a8b8e1edad7 100644 (file)
--- a/src/cmd/vet/testdata/cgo/cgo.go
+++ b/src/cmd/vet/testdata/cgo/cgo.go
@@ -52,5 +52,8 @@ func CgoTests() {
        C.f(*(*unsafe.Pointer)(unsafe.Pointer(&st2)))
        C.f(unsafe.Pointer(&st2))
 
+       type cgoStruct struct{ p *cgoStruct }
+       C.f(unsafe.Pointer(&cgoStruct{}))
+
        C.CBytes([]byte("hello"))
 }

diff --git a/src/cmd/vet/vet_test.go b/src/cmd/vet/vet_test.go
index 725f013a7dfc71c6c5a364343de7dcf8c5d2261b..b3d5c663a714c755503a254f74f41a2dc5aff312 100644 (file)
--- a/src/cmd/vet/vet_test.go
+++ b/src/cmd/vet/vet_test.go
@@ -143,6 +143,7 @@ func TestVetDirs(t *testing.T) {
                "divergent",
                "buildtag",
                "incomplete", // incomplete examples
+               "cgo",
        } {
                dir := dir
                t.Run(dir, func(t *testing.T) {