net: deflake lookup tests

The build dashboard is dotted with net test failures.
We cannot declare all builders to have flaky networks,
although all fundamentally do.

Instead, add a simple retry/backoff loop to the ones that
show up most commonly on the dashboard at this moment.

If this approach works well in practice, we can
incrementally apply it to other flaky net tests.

Change-Id: I69c1ca6ce5b347ad549c7eb18d0438373f6e2489
Reviewed-on: https://go-review.googlesource.com/102397
Run-TryBot: Josh Bleecher Snyder <josharian@gmail.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Brad Fitzpatrick <bradfitz@golang.org>

diff --git a/src/net/lookup_test.go b/src/net/lookup_test.go
index ca1d804d5084041a616c4ea985a43d869f45bcc6..217a24705b820d729a38939ec78cb9b20e791f0e 100644 (file)
--- a/src/net/lookup_test.go
+++ b/src/net/lookup_test.go
@@ -60,6 +60,8 @@ var lookupGoogleSRVTests = []struct {
        },
 }
 
+var backoffDuration = [...]time.Duration{time.Second, 5 * time.Second, 30 * time.Second}
+
 func TestLookupGoogleSRV(t *testing.T) {
        if testenv.Builder() == "" {
                testenv.MustHaveExternalNetwork(t)
@@ -69,10 +71,20 @@ func TestLookupGoogleSRV(t *testing.T) {
                t.Skip("IPv4 is required")
        }
 
-       for _, tt := range lookupGoogleSRVTests {
+       attempts := 0
+       for i := 0; i < len(lookupGoogleSRVTests); i++ {
+               tt := lookupGoogleSRVTests[i]
                cname, srvs, err := LookupSRV(tt.service, tt.proto, tt.name)
                if err != nil {
                        testenv.SkipFlakyNet(t)
+                       if attempts < len(backoffDuration) {
+                               dur := backoffDuration[attempts]
+                               t.Logf("backoff %v after failure %v\n", dur, err)
+                               time.Sleep(dur)
+                               attempts++
+                               i--
+                               continue
+                       }
                        t.Fatal(err)
                }
                if len(srvs) == 0 {
@@ -107,9 +119,20 @@ func TestLookupGmailMX(t *testing.T) {
 
        defer dnsWaitGroup.Wait()
 
-       for _, tt := range lookupGmailMXTests {
+       attempts := 0
+       for i := 0; i < len(lookupGmailMXTests); i++ {
+               tt := lookupGmailMXTests[i]
                mxs, err := LookupMX(tt.name)
                if err != nil {
+                       testenv.SkipFlakyNet(t)
+                       if attempts < len(backoffDuration) {
+                               dur := backoffDuration[attempts]
+                               t.Logf("backoff %v after failure %v\n", dur, err)
+                               time.Sleep(dur)
+                               attempts++
+                               i--
+                               continue
+                       }
                        t.Fatal(err)
                }
                if len(mxs) == 0 {
@@ -176,9 +199,20 @@ func TestLookupGmailTXT(t *testing.T) {
 
        defer dnsWaitGroup.Wait()
 
-       for _, tt := range lookupGmailTXTTests {
+       attempts := 0
+       for i := 0; i < len(lookupGmailTXTTests); i++ {
+               tt := lookupGmailTXTTests[i]
                txts, err := LookupTXT(tt.name)
                if err != nil {
+                       testenv.SkipFlakyNet(t)
+                       if attempts < len(backoffDuration) {
+                               dur := backoffDuration[attempts]
+                               t.Logf("backoff %v after failure %v\n", dur, err)
+                               time.Sleep(dur)
+                               attempts++
+                               i--
+                               continue
+                       }
                        t.Fatal(err)
                }
                if len(txts) == 0 {