runtime/metrics: add metric for current Go-owned thread count

Fixes #15490.

Change-Id: I6ce9edc46398030ff639e22d4ca4adebccdfe1b7
Reviewed-on: https://go-review.googlesource.com/c/go/+/690399
Auto-Submit: Michael Knyszek <mknyszek@google.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Michael Pratt <mpratt@google.com>

diff --git a/src/runtime/metrics.go b/src/runtime/metrics.go
index 028de6b2e1d69438487c622990e168c9ef93aa32..36efef39c032c7d22aeb2db4f79e6c22c9a5ef4f 100644 (file)
--- a/src/runtime/metrics.go
+++ b/src/runtime/metrics.go
@@ -532,6 +532,13 @@ func initMetrics() {
                                sched.stwTotalTimeOther.write(out)
                        },
                },
+               "/sched/threads/total:threads": {
+                       deps: makeStatDepSet(schedStatsDep),
+                       compute: func(in *statAggregate, out *metricValue) {
+                               out.kind = metricKindUint64
+                               out.scalar = uint64(in.schedStats.threads)
+                       },
+               },
                "/sync/mutex/wait/total:seconds": {
                        compute: func(_ *statAggregate, out *metricValue) {
                                out.kind = metricKindFloat64
@@ -787,6 +794,7 @@ type schedStatsAggregate struct {
        gNonGo    uint64
        gWaiting  uint64
        gCreated  uint64
+       threads   uint64
 }
 
 // compute populates the schedStatsAggregate with values from the runtime.
@@ -797,6 +805,10 @@ func (a *schedStatsAggregate) compute() {
        // approximate.
        lock(&sched.lock)
 
+       // The total count of threads owned by Go is the number of Ms
+       // minus extra Ms on the list or in use.
+       a.threads = uint64(mcount()) - uint64(extraMInUse.Load()) - uint64(extraMLength.Load())
+
        // Collect running/runnable from per-P run queues.
        a.gCreated += sched.goroutinesCreated.Load()
        for _, p := range allp {

diff --git a/src/runtime/metrics/description.go b/src/runtime/metrics/description.go
index dd0b485cf8098e1c64e7d0367f85e0b411e5283a..c8c5bf98881a779e53ce88acbc70f04bcfaa7103 100644 (file)
--- a/src/runtime/metrics/description.go
+++ b/src/runtime/metrics/description.go
@@ -498,6 +498,11 @@ var allDesc = []Description{
                Kind:        KindFloat64Histogram,
                Cumulative:  true,
        },
+       {
+               Name:        "/sched/threads/total:threads",
+               Description: "The current count of live threads that are owned by the Go runtime.",
+               Kind:        KindUint64,
+       },
        {
                Name:        "/sync/mutex/wait/total:seconds",
                Description: "Approximate cumulative time goroutines have spent blocked on a sync.Mutex, sync.RWMutex, or runtime-internal lock. This metric is useful for identifying global changes in lock contention. Collect a mutex or block profile using the runtime/pprof package for more detailed contention data.",

diff --git a/src/runtime/metrics/doc.go b/src/runtime/metrics/doc.go
index 2d3b716a3c1d227b328e13551253e75c8182790a..00ce60dde1a7f5212e2da10b67b671ad3969aa31 100644 (file)
--- a/src/runtime/metrics/doc.go
+++ b/src/runtime/metrics/doc.go
@@ -572,6 +572,10 @@ Below is the full list of supported metrics, ordered lexicographically.
                /sched/pauses/stopping/other:seconds). Bucket counts increase
                monotonically.
 
+       /sched/threads/total:threads
+               The current count of live threads that are owned by the Go
+               runtime.
+
        /sync/mutex/wait/total:seconds
                Approximate cumulative time goroutines have spent blocked on a
                sync.Mutex, sync.RWMutex, or runtime-internal lock. This metric

diff --git a/src/runtime/metrics_test.go b/src/runtime/metrics_test.go
index 77223a37a77488c58e257083e9965cc81f6a565c..385891d7d9fe005dd46ed8c8978dd76f6ed87bf3 100644 (file)
--- a/src/runtime/metrics_test.go
+++ b/src/runtime/metrics_test.go
@@ -1584,13 +1584,16 @@ func TestReadMetricsSched(t *testing.T) {
                running
                waiting
                created
+               threads
+               numSamples
        )
-       var s [5]metrics.Sample
+       var s [numSamples]metrics.Sample
        s[notInGo].Name = "/sched/goroutines/not-in-go:goroutines"
        s[runnable].Name = "/sched/goroutines/runnable:goroutines"
        s[running].Name = "/sched/goroutines/running:goroutines"
        s[waiting].Name = "/sched/goroutines/waiting:goroutines"
        s[created].Name = "/sched/goroutines-created:goroutines"
+       s[threads].Name = "/sched/threads/total:threads"
 
        logMetrics := func(t *testing.T, s []metrics.Sample) {
                for i := range s {
@@ -1608,6 +1611,10 @@ func TestReadMetricsSched(t *testing.T) {
        // goroutines.
        const waitingSlack = 100
 
+       // threadsSlack is the maximum number of threads left over
+       // from other tests and the runtime (sysmon, the template thread, etc.)
+       const threadsSlack = 20
+
        // Make sure GC isn't running, since GC workers interfere with
        // expected counts.
        defer debug.SetGCPercent(debug.SetGCPercent(-1))
@@ -1694,6 +1701,7 @@ func TestReadMetricsSched(t *testing.T) {
                        }, time.Second)
                        logMetrics(t, s[:])
                        check(t, &s[running], count, count+4)
+                       check(t, &s[threads], count, count+4+threadsSlack)
                })
 
                // Force runnable count to be high.
@@ -1724,6 +1732,7 @@ func TestReadMetricsSched(t *testing.T) {
                t.Run("running", func(t *testing.T) {
                        logMetrics(t, s[:])
                        checkEq(t, &s[running], 1)
+                       checkEq(t, &s[threads], 1)
                })
                t.Run("runnable", func(t *testing.T) {
                        logMetrics(t, s[:])

diff --git a/src/runtime/proc.go b/src/runtime/proc.go
index 329d1b26242f16d314898c67920d40976f41b731..68647d771fe9528f87261c50ff0686f2a12900e0 100644 (file)
--- a/src/runtime/proc.go
+++ b/src/runtime/proc.go
@@ -1007,7 +1007,7 @@ func mcommoninit(mp *m, id int64) {
        // when it is just in a register or thread-local storage.
        mp.alllink = allm
 
-       // NumCgoCall() and others iterate over allm w/o schedlock,
+       // NumCgoCall and others iterate over allm w/o schedlock,
        // so we need to publish it safely.
        atomicstorep(unsafe.Pointer(&allm), unsafe.Pointer(mp))
        unlock(&sched.lock)

diff --git a/src/runtime/runtime2.go b/src/runtime/runtime2.go
index a80a34a18ecdd95f39f02ad007c5f26c6072cee0..042c3137cd03dfa2af31a6567299be75e3d3be82 100644 (file)
--- a/src/runtime/runtime2.go
+++ b/src/runtime/runtime2.go
@@ -1225,7 +1225,9 @@ var isIdleInSynctest = [len(waitReasonStrings)]bool{
 }
 
 var (
-       allm          *m
+       // Linked-list of all Ms. Written under sched.lock, read atomically.
+       allm *m
+
        gomaxprocs    int32
        numCPUStartup int32
        forcegc       forcegcstate