encoding/gob: fix mutually recursive slices of structs

Fix by setting the element type if we discover it's zero while building.
We could have fixed this better with foresight by doing the id setting in a
different sequence, but doing that now would break binary compatibility.

Fixes #2995.

R=golang-dev, dsymonds
CC=golang-dev
https://golang.org/cl/5675083

diff --git a/src/pkg/encoding/gob/encoder_test.go b/src/pkg/encoding/gob/encoder_test.go
index 9a62cf9c2ade63cf7aa658fe4c20facd5c8d4394..7911dad90da14bccd204f7eb6ade2f5dd479902f 100644 (file)
--- a/src/pkg/encoding/gob/encoder_test.go
+++ b/src/pkg/encoding/gob/encoder_test.go
@@ -685,3 +685,30 @@ func TestSliceIncompatibility(t *testing.T) {
                t.Error("expected compatibility error")
        }
 }
+
+// Mutually recursive slices of structs caused problems.
+type Bug3 struct {
+       Num      int
+       Children []*Bug3
+}
+
+func TestGobPtrSlices(t *testing.T) {
+       in := []*Bug3{
+               &Bug3{1, nil},
+               &Bug3{2, nil},
+       }
+       b := new(bytes.Buffer)
+       err := NewEncoder(b).Encode(&in)
+       if err != nil {
+               t.Fatal("encode:", err)
+       }
+
+       var out []*Bug3
+       err = NewDecoder(b).Decode(&out)
+       if err != nil {
+               t.Fatal("decode:", err)
+       }
+       if !reflect.DeepEqual(in, out) {
+               t.Fatal("got %v; wanted %v", out, in)
+       }
+}

diff --git a/src/pkg/encoding/gob/type.go b/src/pkg/encoding/gob/type.go
index 39006efdb2d596a553899fa778df42cb1e91cd84..0dd7a0a770ea137956683f9e69c4ced123d1f2d3 100644 (file)
--- a/src/pkg/encoding/gob/type.go
+++ b/src/pkg/encoding/gob/type.go
@@ -152,6 +152,10 @@ var idToType = make(map[typeId]gobType)
 var builtinIdToType map[typeId]gobType // set in init() after builtins are established
 
 func setTypeId(typ gobType) {
+       // When building recursive types, someone may get there before us.
+       if typ.id() != 0 {
+               return
+       }
        nextId++
        typ.setId(nextId)
        idToType[nextId] = typ
@@ -346,6 +350,11 @@ func newSliceType(name string) *sliceType {
 func (s *sliceType) init(elem gobType) {
        // Set our type id before evaluating the element's, in case it's our own.
        setTypeId(s)
+       // See the comments about ids in newTypeObject. Only slices and
+       // structs have mutual recursion.
+       if elem.id() == 0 {
+               setTypeId(elem)
+       }
        s.Elem = elem.id()
 }
 
@@ -503,6 +512,13 @@ func newTypeObject(name string, ut *userTypeInfo, rt reflect.Type) (gobType, err
                        if err != nil {
                                return nil, err
                        }
+                       // Some mutually recursive types can cause us to be here while
+                       // still defining the element. Fix the element type id here.
+                       // We could do this more neatly by setting the id at the start of
+                       // building every type, but that would break binary compatibility.
+                       if gt.id() == 0 {
+                               setTypeId(gt)
+                       }
                        st.Field = append(st.Field, &fieldType{f.Name, gt.id()})
                }
                return st, nil