From 6b1ac8290f189709273c9d4b402e14421f21eda0 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Sun, 5 May 2019 02:25:42 +0000 Subject: [PATCH] cmd/dist: allow builders to control granularity of test/ directory sharding Even with 10 shards on builders, it still takes about ~2.5 minutes per shard (and getting slower all the time as the test/ directory grows). I'm currently experimenting with massively sharding out testing on Cloud Run (each dist test & normal TestFoo func all running in parallel), and in such a setup, 2.5 minutes is an eternity. I'd like to increase that dist test's sharding from 10 to more like 1,000. Updates golang/go#31834 Change-Id: I8b02989727793b5b5b2013d67e1eb01ef4786e28 Reviewed-on: https://go-review.googlesource.com/c/go/+/175297 Reviewed-by: Dmitri Shuralyov Reviewed-by: Ian Lance Taylor --- src/cmd/dist/test.go | 5 ++++- src/run.bash | 11 +++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/cmd/dist/test.go b/src/cmd/dist/test.go index e7fd155ee4..9e259f1835 100644 --- a/src/cmd/dist/test.go +++ b/src/cmd/dist/test.go @@ -709,11 +709,14 @@ func (t *tester) registerTests() { if goos != "android" && !t.iOS() { // Only start multiple test dir shards on builders, // where they get distributed to multiple machines. - // See issue 20141. + // See issues 20141 and 31834. nShards := 1 if os.Getenv("GO_BUILDER_NAME") != "" { nShards = 10 } + if n, err := strconv.Atoi(os.Getenv("GO_TEST_SHARDS")); err == nil { + nShards = n + } for shard := 0; shard < nShards; shard++ { shard := shard t.tests = append(t.tests, distTest{ diff --git a/src/run.bash b/src/run.bash index 1acd0194b6..e18540c990 100755 --- a/src/run.bash +++ b/src/run.bash @@ -3,6 +3,17 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. +# Environment variables that control run.bash: +# +# GO_TEST_SHARDS: number of "dist test" test shards that the +# $GOROOT/test directory will be sliced up into for parallel +# execution. Defaults to 1, unless GO_BUILDER_NAME is also specified, +# in which case it defaults to 10. +# +# GO_BUILDER_NAME: the name of the Go builder that's running the tests. +# Some tests are conditionally enabled or disabled based on the builder +# name or the builder name being non-empty. + set -e eval $(go env) -- 2.50.0