runtime: dynamically allocate allp

This makes it possible to eliminate the hard cap on GOMAXPROCS.

Updates #15131.

Change-Id: I4c422b340791621584c118a6be1b38e8a44f8b70
Reviewed-on: https://go-review.googlesource.com/45573
Run-TryBot: Austin Clements <austin@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Rick Hudson <rlh@golang.org>

diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go
index 48ccfe8df2b964e0a8d31e6e9ceb51f20fb7a8cf..23dc79d79a8c86bafe28d09e9a9c2349770f10be 100644 (file)
--- a/src/runtime/mgc.go
+++ b/src/runtime/mgc.go
@@ -465,7 +465,7 @@ func (c *gcControllerState) startCycle() {
        }
 
        // Clear per-P state
-       for _, p := range &allp {
+       for _, p := range allp {
                if p == nil {
                        break
                }
@@ -1662,7 +1662,7 @@ func gcMarkTermination(nextTriggerRatio float64) {
 func gcBgMarkStartWorkers() {
        // Background marking is performed by per-P G's. Ensure that
        // each P has a background GC G.
-       for _, p := range &allp {
+       for _, p := range allp {
                if p == nil || p.status == _Pdead {
                        break
                }

diff --git a/src/runtime/mgcmark.go b/src/runtime/mgcmark.go
index 9029d19d4347ae7c213d16ff584dadbd09ea443b..efc1a042f92b3a4697c18365a18b18dfe75f83c8 100644 (file)
--- a/src/runtime/mgcmark.go
+++ b/src/runtime/mgcmark.go
@@ -1356,7 +1356,7 @@ func gcmarknewobject(obj, size, scanSize uintptr) {
 //
 // The world must be stopped.
 func gcMarkTinyAllocs() {
-       for _, p := range &allp {
+       for _, p := range allp {
                if p == nil || p.status == _Pdead {
                        break
                }

diff --git a/src/runtime/mstats.go b/src/runtime/mstats.go
index 1cb44a15ddb3e9dfd8a684cb7bd67ef6d5b0c9e1..8538bad0dbafde0486dfb4f8eaec8bc79f5fb07b 100644 (file)
--- a/src/runtime/mstats.go
+++ b/src/runtime/mstats.go
@@ -589,9 +589,13 @@ func updatememstats() {
        memstats.heap_objects = memstats.nmalloc - memstats.nfree
 }
 
+// cachestats flushes all mcache stats.
+//
+// The world must be stopped.
+//
 //go:nowritebarrier
 func cachestats() {
-       for _, p := range &allp {
+       for _, p := range allp {
                if p == nil {
                        break
                }

diff --git a/src/runtime/proc.go b/src/runtime/proc.go
index c58e806e0d884ea9fb9030a6e7155306bf6c4d30..188c897723498a200723e717b1b2ddb61048cb13 100644 (file)
--- a/src/runtime/proc.go
+++ b/src/runtime/proc.go
@@ -3231,7 +3231,7 @@ func badunlockosthread() {
 
 func gcount() int32 {
        n := int32(allglen) - sched.ngfree - int32(atomic.Load(&sched.ngsys))
-       for _, _p_ := range &allp {
+       for _, _p_ := range allp {
                if _p_ == nil {
                        break
                }
@@ -3543,6 +3543,23 @@ func procresize(nprocs int32) *p {
        }
        sched.procresizetime = now
 
+       // Grow allp if necessary.
+       if nprocs > int32(len(allp)) {
+               // Synchronize with retake, which could be running
+               // concurrently since it doesn't run on a P.
+               lock(&allpLock)
+               if nprocs <= int32(cap(allp)) {
+                       allp = allp[:nprocs]
+               } else {
+                       nallp := make([]*p, nprocs)
+                       // Copy everything up to allp's cap so we
+                       // never lose old allocated Ps.
+                       copy(nallp, allp[:cap(allp)])
+                       allp = nallp
+               }
+               unlock(&allpLock)
+       }
+
        // initialize new P's
        for i := int32(0); i < nprocs; i++ {
                pp := allp[i]
@@ -3631,6 +3648,13 @@ func procresize(nprocs int32) *p {
                // can't free P itself because it can be referenced by an M in syscall
        }
 
+       // Trim allp.
+       if int32(len(allp)) != nprocs {
+               lock(&allpLock)
+               allp = allp[:nprocs]
+               unlock(&allpLock)
+       }
+
        _g_ := getg()
        if _g_.m.p != 0 && _g_.m.p.ptr().id < nprocs {
                // continue to use the current P
@@ -3956,7 +3980,10 @@ const forcePreemptNS = 10 * 1000 * 1000 // 10ms
 
 func retake(now int64) uint32 {
        n := 0
-       for i := int32(0); i < gomaxprocs; i++ {
+       // Prevent allp slice changes. This lock will be completely
+       // uncontended unless we're already stopping the world.
+       lock(&allpLock)
+       for i := 0; i < len(allp); i++ {
                _p_ := allp[i]
                if _p_ == nil {
                        continue
@@ -3977,6 +4004,8 @@ func retake(now int64) uint32 {
                        if runqempty(_p_) && atomic.Load(&sched.nmspinning)+atomic.Load(&sched.npidle) > 0 && pd.syscallwhen+10*1000*1000 > now {
                                continue
                        }
+                       // Drop allpLock so we can take sched.lock.
+                       unlock(&allpLock)
                        // Need to decrement number of idle locked M's
                        // (pretending that one more is running) before the CAS.
                        // Otherwise the M from which we retake can exit the syscall,
@@ -3992,6 +4021,7 @@ func retake(now int64) uint32 {
                                handoffp(_p_)
                        }
                        incidlelocked(1)
+                       lock(&allpLock)
                } else if s == _Prunning {
                        // Preempt G if it's running for too long.
                        t := int64(_p_.schedtick)
@@ -4006,6 +4036,7 @@ func retake(now int64) uint32 {
                        preemptone(_p_)
                }
        }
+       unlock(&allpLock)
        return uint32(n)
 }
 

diff --git a/src/runtime/runtime2.go b/src/runtime/runtime2.go
index 27b1e378035e2c0fa16444c67b688d330277c861..269c5b1c4d39e3ac5f8c506245d50a3bd78c8038 100644 (file)
--- a/src/runtime/runtime2.go
+++ b/src/runtime/runtime2.go
@@ -722,7 +722,8 @@ const _TracebackMaxFrames = 100
 var (
        allglen    uintptr
        allm       *m
-       allp       [_MaxGomaxprocs + 1]*p
+       allp       []*p  // len(allp) == gomaxprocs; may change at safe points, otherwise immutable
+       allpLock   mutex // Protects P-less reads of allp and all writes
        gomaxprocs int32
        ncpu       int32
        forcegc    forcegcstate

diff --git a/src/runtime/trace.go b/src/runtime/trace.go
index 50e4c73c838162c1aa297dbcf1d7493b03206b14..398d0449b4b497bc85d8a07f5d36b808f641a72c 100644 (file)
--- a/src/runtime/trace.go
+++ b/src/runtime/trace.go
@@ -277,7 +277,9 @@ func StopTrace() {
 
        traceGoSched()
 
-       for _, p := range &allp {
+       // Loop over all allocated Ps because dead Ps may still have
+       // trace buffers.
+       for _, p := range allp[:cap(allp)] {
                if p == nil {
                        break
                }
@@ -320,7 +322,7 @@ func StopTrace() {
 
        // The lock protects us from races with StartTrace/StopTrace because they do stop-the-world.
        lock(&trace.lock)
-       for _, p := range &allp {
+       for _, p := range allp[:cap(allp)] {
                if p == nil {
                        break
                }