From 2ff5fbfbd4539d321043e6b84691ecb131402030 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Mon, 19 Sep 2022 21:48:10 -0400 Subject: [PATCH] cmd/compile: limit goroutine count to parallelism When the compiler crashes, it is not uncommon to see many hundreds of goroutines all blocked waiting their turn to be one of the nWorkers goroutines that is allowed to run. All these goroutine stacks are not a terribly efficient use of memory, and they also make the crash dumps hard to read. Introduce a manager goroutine to hand out work to at most nWorker goroutines, maintaining pending work in a local slice, rather than having all those blocked goroutines hanging around waiting to run. Change-Id: I46cb4e1afd6392805f359e14554ebc17d538bcba Reviewed-on: https://go-review.googlesource.com/c/go/+/431956 Reviewed-by: Cuong Manh Le Reviewed-by: Cherry Mui Run-TryBot: Russ Cox Auto-Submit: Russ Cox TryBot-Result: Gopher Robot --- src/cmd/compile/internal/gc/compile.go | 44 ++++++++++++++++++-------- 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/src/cmd/compile/internal/gc/compile.go b/src/cmd/compile/internal/gc/compile.go index 00504451a8..cbd48e0956 100644 --- a/src/cmd/compile/internal/gc/compile.go +++ b/src/cmd/compile/internal/gc/compile.go @@ -126,20 +126,38 @@ func compileFunctions() { } if nWorkers := base.Flag.LowerC; nWorkers > 1 { - // For concurrent builds, we create a goroutine per task, but - // require them to hold a unique worker ID while performing work - // to limit parallelism. - workerIDs := make(chan int, nWorkers) - for i := 0; i < nWorkers; i++ { - workerIDs <- i - } - + // For concurrent builds, we allow the work queue + // to grow arbitrarily large, but only nWorkers work items + // can be running concurrently. + workq := make(chan func(int)) + done := make(chan int) + go func() { + ids := make([]int, nWorkers) + for i := range ids { + ids[i] = i + } + var pending []func(int) + for { + select { + case work := <-workq: + pending = append(pending, work) + case id := <-done: + ids = append(ids, id) + } + for len(pending) > 0 && len(ids) > 0 { + work := pending[len(pending)-1] + id := ids[len(ids)-1] + pending = pending[:len(pending)-1] + ids = ids[:len(ids)-1] + go func() { + work(id) + done <- id + }() + } + } + }() queue = func(work func(int)) { - go func() { - worker := <-workerIDs - work(worker) - workerIDs <- worker - }() + workq <- work } } -- 2.50.0