From 5ce92d0384ecc8aa0247a9c280acbb6a65e4585e Mon Sep 17 00:00:00 2001 From: Josh Bleecher Snyder Date: Sat, 24 Mar 2018 22:09:51 -0700 Subject: [PATCH] net: deflake lookup tests The build dashboard is dotted with net test failures. We cannot declare all builders to have flaky networks, although all fundamentally do. Instead, add a simple retry/backoff loop to the ones that show up most commonly on the dashboard at this moment. If this approach works well in practice, we can incrementally apply it to other flaky net tests. Change-Id: I69c1ca6ce5b347ad549c7eb18d0438373f6e2489 Reviewed-on: https://go-review.googlesource.com/102397 Run-TryBot: Josh Bleecher Snyder TryBot-Result: Gobot Gobot Reviewed-by: Brad Fitzpatrick --- src/net/lookup_test.go | 40 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/src/net/lookup_test.go b/src/net/lookup_test.go index ca1d804d50..217a24705b 100644 --- a/src/net/lookup_test.go +++ b/src/net/lookup_test.go @@ -60,6 +60,8 @@ var lookupGoogleSRVTests = []struct { }, } +var backoffDuration = [...]time.Duration{time.Second, 5 * time.Second, 30 * time.Second} + func TestLookupGoogleSRV(t *testing.T) { if testenv.Builder() == "" { testenv.MustHaveExternalNetwork(t) @@ -69,10 +71,20 @@ func TestLookupGoogleSRV(t *testing.T) { t.Skip("IPv4 is required") } - for _, tt := range lookupGoogleSRVTests { + attempts := 0 + for i := 0; i < len(lookupGoogleSRVTests); i++ { + tt := lookupGoogleSRVTests[i] cname, srvs, err := LookupSRV(tt.service, tt.proto, tt.name) if err != nil { testenv.SkipFlakyNet(t) + if attempts < len(backoffDuration) { + dur := backoffDuration[attempts] + t.Logf("backoff %v after failure %v\n", dur, err) + time.Sleep(dur) + attempts++ + i-- + continue + } t.Fatal(err) } if len(srvs) == 0 { @@ -107,9 +119,20 @@ func TestLookupGmailMX(t *testing.T) { defer dnsWaitGroup.Wait() - for _, tt := range lookupGmailMXTests { + attempts := 0 + for i := 0; i < len(lookupGmailMXTests); i++ { + tt := lookupGmailMXTests[i] mxs, err := LookupMX(tt.name) if err != nil { + testenv.SkipFlakyNet(t) + if attempts < len(backoffDuration) { + dur := backoffDuration[attempts] + t.Logf("backoff %v after failure %v\n", dur, err) + time.Sleep(dur) + attempts++ + i-- + continue + } t.Fatal(err) } if len(mxs) == 0 { @@ -176,9 +199,20 @@ func TestLookupGmailTXT(t *testing.T) { defer dnsWaitGroup.Wait() - for _, tt := range lookupGmailTXTTests { + attempts := 0 + for i := 0; i < len(lookupGmailTXTTests); i++ { + tt := lookupGmailTXTTests[i] txts, err := LookupTXT(tt.name) if err != nil { + testenv.SkipFlakyNet(t) + if attempts < len(backoffDuration) { + dur := backoffDuration[attempts] + t.Logf("backoff %v after failure %v\n", dur, err) + time.Sleep(dur) + attempts++ + i-- + continue + } t.Fatal(err) } if len(txts) == 0 { -- 2.50.0