runtime/metrics: add metric for total goroutines created

For #15490.

Change-Id: Ic587dda1f42d613ea131a6b53ce6ba6e6cadf4c7
Reviewed-on: https://go-review.googlesource.com/c/go/+/690398
Reviewed-by: Michael Pratt <mpratt@google.com>
Auto-Submit: Michael Knyszek <mknyszek@google.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>

diff --git a/src/runtime/metrics.go b/src/runtime/metrics.go
index 47b1b891e140e13274f837bbebe2fc93ce4ba689..028de6b2e1d69438487c622990e168c9ef93aa32 100644 (file)
--- a/src/runtime/metrics.go
+++ b/src/runtime/metrics.go
@@ -500,6 +500,13 @@ func initMetrics() {
                                out.scalar = uint64(in.schedStats.gWaiting)
                        },
                },
+               "/sched/goroutines-created:goroutines": {
+                       deps: makeStatDepSet(schedStatsDep),
+                       compute: func(in *statAggregate, out *metricValue) {
+                               out.kind = metricKindUint64
+                               out.scalar = uint64(in.schedStats.gCreated)
+                       },
+               },
                "/sched/latencies:seconds": {
                        compute: func(_ *statAggregate, out *metricValue) {
                                sched.timeToRun.write(out)
@@ -779,6 +786,7 @@ type schedStatsAggregate struct {
        gRunnable uint64
        gNonGo    uint64
        gWaiting  uint64
+       gCreated  uint64
 }
 
 // compute populates the schedStatsAggregate with values from the runtime.
@@ -790,10 +798,12 @@ func (a *schedStatsAggregate) compute() {
        lock(&sched.lock)
 
        // Collect running/runnable from per-P run queues.
+       a.gCreated += sched.goroutinesCreated.Load()
        for _, p := range allp {
                if p == nil || p.status == _Pdead {
                        break
                }
+               a.gCreated += p.goroutinesCreated
                switch p.status {
                case _Prunning:
                        a.gRunning++

diff --git a/src/runtime/metrics/description.go b/src/runtime/metrics/description.go
index cf22bb73adc3c2bf2a524ca7efe629ade15ef934..dd0b485cf8098e1c64e7d0367f85e0b411e5283a 100644 (file)
--- a/src/runtime/metrics/description.go
+++ b/src/runtime/metrics/description.go
@@ -437,6 +437,12 @@ var allDesc = []Description{
                Description: "The current runtime.GOMAXPROCS setting, or the number of operating system threads that can execute user-level Go code simultaneously.",
                Kind:        KindUint64,
        },
+       {
+               Name:        "/sched/goroutines-created:goroutines",
+               Description: "Count of goroutines created since program start.",
+               Cumulative:  true,
+               Kind:        KindUint64,
+       },
        {
                Name:        "/sched/goroutines/not-in-go:goroutines",
                Description: "Approximate count of goroutines running or blocked in a system call or cgo call. Not guaranteed to add up to /sched/goroutines:goroutines with other goroutine metrics.",

diff --git a/src/runtime/metrics/doc.go b/src/runtime/metrics/doc.go
index c379b201b4b79b91ef801fab77cdb06db228088b..2d3b716a3c1d227b328e13551253e75c8182790a 100644 (file)
--- a/src/runtime/metrics/doc.go
+++ b/src/runtime/metrics/doc.go
@@ -509,6 +509,9 @@ Below is the full list of supported metrics, ordered lexicographically.
                operating system threads that can execute user-level Go code
                simultaneously.
 
+       /sched/goroutines-created:goroutines
+               Count of goroutines created since program start.
+
        /sched/goroutines/not-in-go:goroutines
                Approximate count of goroutines running or blocked in
                a system call or cgo call. Not guaranteed to add up to

diff --git a/src/runtime/metrics_test.go b/src/runtime/metrics_test.go
index 5b16cbcb22d92e26cebb9c88448566752ecc655c..77223a37a77488c58e257083e9965cc81f6a565c 100644 (file)
--- a/src/runtime/metrics_test.go
+++ b/src/runtime/metrics_test.go
@@ -1583,12 +1583,14 @@ func TestReadMetricsSched(t *testing.T) {
                runnable
                running
                waiting
+               created
        )
-       var s [4]metrics.Sample
-       s[0].Name = "/sched/goroutines/not-in-go:goroutines"
-       s[1].Name = "/sched/goroutines/runnable:goroutines"
-       s[2].Name = "/sched/goroutines/running:goroutines"
-       s[3].Name = "/sched/goroutines/waiting:goroutines"
+       var s [5]metrics.Sample
+       s[notInGo].Name = "/sched/goroutines/not-in-go:goroutines"
+       s[runnable].Name = "/sched/goroutines/runnable:goroutines"
+       s[running].Name = "/sched/goroutines/running:goroutines"
+       s[waiting].Name = "/sched/goroutines/waiting:goroutines"
+       s[created].Name = "/sched/goroutines-created:goroutines"
 
        logMetrics := func(t *testing.T, s []metrics.Sample) {
                for i := range s {
@@ -1645,6 +1647,9 @@ func TestReadMetricsSched(t *testing.T) {
                check(t, &s[waiting], 0, waitingSlack)
        })
 
+       metrics.Read(s[:])
+       createdAfterBase := s[created].Value.Uint64()
+
        // Force Running count to be high. We'll use these goroutines
        // for Runnable, too.
        const count = 10
@@ -1673,6 +1678,11 @@ func TestReadMetricsSched(t *testing.T) {
        // of runnable goroutines all spinning. We cannot write anything
        // out.
        if testenv.HasParallelism() {
+               t.Run("created", func(t *testing.T) {
+                       metrics.Read(s[:])
+                       logMetrics(t, s[:])
+                       checkEq(t, &s[created], createdAfterBase+count)
+               })
                t.Run("running", func(t *testing.T) {
                        defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(count + 4))
                        // It can take a little bit for the scheduler to
@@ -1706,6 +1716,11 @@ func TestReadMetricsSched(t *testing.T) {
                exit.Store(1)
 
                // Now we can check our invariants.
+               t.Run("created", func(t *testing.T) {
+                       // Look for count-1 goroutines because we read metrics
+                       // *before* t.Run goroutine was created for this sub-test.
+                       checkEq(t, &s[created], createdAfterBase+count-1)
+               })
                t.Run("running", func(t *testing.T) {
                        logMetrics(t, s[:])
                        checkEq(t, &s[running], 1)

diff --git a/src/runtime/proc.go b/src/runtime/proc.go
index 15f10f087ec395e71f0f1d50a6a6867981e0fb31..329d1b26242f16d314898c67920d40976f41b731 100644 (file)
--- a/src/runtime/proc.go
+++ b/src/runtime/proc.go
@@ -5262,6 +5262,7 @@ func newproc1(fn *funcval, callergp *g, callerpc uintptr, parked bool, waitreaso
                        racereleasemergeg(newg, unsafe.Pointer(&labelSync))
                }
        }
+       pp.goroutinesCreated++
        releasem(mp)
 
        return newg
@@ -5841,6 +5842,8 @@ func (pp *p) destroy() {
        pp.gcAssistTime = 0
        gcCleanups.queued += pp.cleanupsQueued
        pp.cleanupsQueued = 0
+       sched.goroutinesCreated.Add(int64(pp.goroutinesCreated))
+       pp.goroutinesCreated = 0
        pp.xRegs.free()
        pp.status = _Pdead
 }

diff --git a/src/runtime/runtime2.go b/src/runtime/runtime2.go
index c5d15754ec93c70cbed015b269e81bbecb47879b..a80a34a18ecdd95f39f02ad007c5f26c6072cee0 100644 (file)
--- a/src/runtime/runtime2.go
+++ b/src/runtime/runtime2.go
@@ -764,6 +764,9 @@ type p struct {
        // gcStopTime is the nanotime timestamp that this P last entered _Pgcstop.
        gcStopTime int64
 
+       // goroutinesCreated is the total count of goroutines created by this P.
+       goroutinesCreated uint64
+
        // xRegs is the per-P extended register state used by asynchronous
        // preemption. This is an empty struct on platforms that don't use extended
        // register state.
@@ -892,6 +895,10 @@ type schedt struct {
        // M, but waiting for locks within the runtime. This field stores the value
        // for Ms that have exited.
        totalRuntimeLockWaitTime atomic.Int64
+
+       // goroutinesCreated (plus the value of goroutinesCreated on each P in allp)
+       // is the sum of all goroutines created by the program.
+       goroutinesCreated atomic.Uint64
 }
 
 // Values for the flags field of a sigTabT.