From: Austin Clements Date: Fri, 10 Aug 2018 03:47:37 +0000 (-0400) Subject: runtime: replace manually managed G dequeues with a type X-Git-Tag: go1.12beta1~1433 X-Git-Url: http://www.git.cypherpunks.su/?a=commitdiff_plain;h=3578918b6614effaeaa581687d810b74e342e0f8;p=gostls13.git runtime: replace manually managed G dequeues with a type There are two manually managed G dequeues. Abstract these both into a shared gQueue type. This also introduces a gList type, which we'll use to replace several manually-managed G lists in follow-up CLs. This makes the code more readable and maintainable. gcFlushBgCredit in particular becomes much easier to follow. It also makes it easier to introduce more G queues in the future. Finally, the gList type clearly distinguishes between lists of Gs and individual Gs; currently both are represented by a *g, which can easily lead to confusion and bugs. Change-Id: Ic7798841b405d311fc8b6aa5a958ffa4c7993c6c Reviewed-on: https://go-review.googlesource.com/129396 Run-TryBot: Austin Clements TryBot-Result: Gobot Gobot Reviewed-by: Rick Hudson --- diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go index 6a3219de73..7d4ba9f9cd 100644 --- a/src/runtime/mgc.go +++ b/src/runtime/mgc.go @@ -1023,8 +1023,8 @@ var work struct { // there was neither enough credit to steal or enough work to // do. assistQueue struct { - lock mutex - head, tail guintptr + lock mutex + q gQueue } // sweepWaiters is a list of blocked goroutines to wake when diff --git a/src/runtime/mgcmark.go b/src/runtime/mgcmark.go index e8cfdce4fc..7850f86bb2 100644 --- a/src/runtime/mgcmark.go +++ b/src/runtime/mgcmark.go @@ -602,9 +602,7 @@ func gcAssistAlloc1(gp *g, scanWork int64) { // new assists from going to sleep after this point. func gcWakeAllAssists() { lock(&work.assistQueue.lock) - injectglist(work.assistQueue.head.ptr()) - work.assistQueue.head.set(nil) - work.assistQueue.tail.set(nil) + injectglist(work.assistQueue.q.popList().head.ptr()) unlock(&work.assistQueue.lock) } @@ -625,24 +623,17 @@ func gcParkAssist() bool { } gp := getg() - oldHead, oldTail := work.assistQueue.head, work.assistQueue.tail - if oldHead == 0 { - work.assistQueue.head.set(gp) - } else { - oldTail.ptr().schedlink.set(gp) - } - work.assistQueue.tail.set(gp) - gp.schedlink.set(nil) + oldList := work.assistQueue.q + work.assistQueue.q.pushBack(gp) // Recheck for background credit now that this G is in // the queue, but can still back out. This avoids a // race in case background marking has flushed more // credit since we checked above. if atomic.Loadint64(&gcController.bgScanCredit) > 0 { - work.assistQueue.head = oldHead - work.assistQueue.tail = oldTail - if oldTail != 0 { - oldTail.ptr().schedlink.set(nil) + work.assistQueue.q = oldList + if oldList.tail != 0 { + oldList.tail.ptr().schedlink.set(nil) } unlock(&work.assistQueue.lock) return false @@ -663,7 +654,7 @@ func gcParkAssist() bool { // //go:nowritebarrierrec func gcFlushBgCredit(scanWork int64) { - if work.assistQueue.head == 0 { + if work.assistQueue.q.empty() { // Fast path; there are no blocked assists. There's a // small window here where an assist may add itself to // the blocked queue and park. If that happens, we'll @@ -675,23 +666,21 @@ func gcFlushBgCredit(scanWork int64) { scanBytes := int64(float64(scanWork) * gcController.assistBytesPerWork) lock(&work.assistQueue.lock) - gp := work.assistQueue.head.ptr() - for gp != nil && scanBytes > 0 { + for !work.assistQueue.q.empty() && scanBytes > 0 { + gp := work.assistQueue.q.pop() // Note that gp.gcAssistBytes is negative because gp // is in debt. Think carefully about the signs below. if scanBytes+gp.gcAssistBytes >= 0 { // Satisfy this entire assist debt. scanBytes += gp.gcAssistBytes gp.gcAssistBytes = 0 - xgp := gp - gp = gp.schedlink.ptr() - // It's important that we *not* put xgp in + // It's important that we *not* put gp in // runnext. Otherwise, it's possible for user // code to exploit the GC worker's high // scheduler priority to get itself always run // before other goroutines and always in the // fresh quantum started by GC. - ready(xgp, 0, false) + ready(gp, 0, false) } else { // Partially satisfy this assist. gp.gcAssistBytes += scanBytes @@ -700,23 +689,10 @@ func gcFlushBgCredit(scanWork int64) { // back of the queue so that large assists // can't clog up the assist queue and // substantially delay small assists. - xgp := gp - gp = gp.schedlink.ptr() - if gp == nil { - // gp is the only assist in the queue. - gp = xgp - } else { - xgp.schedlink = 0 - work.assistQueue.tail.ptr().schedlink.set(xgp) - work.assistQueue.tail.set(xgp) - } + work.assistQueue.q.pushBack(gp) break } } - work.assistQueue.head.set(gp) - if gp == nil { - work.assistQueue.tail.set(nil) - } if scanBytes > 0 { // Convert from scan bytes back to work. diff --git a/src/runtime/proc.go b/src/runtime/proc.go index f82014eb92..fbb1ce1750 100644 --- a/src/runtime/proc.go +++ b/src/runtime/proc.go @@ -4667,13 +4667,7 @@ func mget() *m { // May run during STW, so write barriers are not allowed. //go:nowritebarrierrec func globrunqput(gp *g) { - gp.schedlink = 0 - if sched.runqtail != 0 { - sched.runqtail.ptr().schedlink.set(gp) - } else { - sched.runqhead.set(gp) - } - sched.runqtail.set(gp) + sched.runq.pushBack(gp) sched.runqsize++ } @@ -4682,25 +4676,17 @@ func globrunqput(gp *g) { // May run during STW, so write barriers are not allowed. //go:nowritebarrierrec func globrunqputhead(gp *g) { - gp.schedlink = sched.runqhead - sched.runqhead.set(gp) - if sched.runqtail == 0 { - sched.runqtail.set(gp) - } + sched.runq.push(gp) sched.runqsize++ } // Put a batch of runnable goroutines on the global runnable queue. +// This clears *batch. // Sched must be locked. -func globrunqputbatch(ghead *g, gtail *g, n int32) { - gtail.schedlink = 0 - if sched.runqtail != 0 { - sched.runqtail.ptr().schedlink.set(ghead) - } else { - sched.runqhead.set(ghead) - } - sched.runqtail.set(gtail) +func globrunqputbatch(batch *gQueue, n int32) { + sched.runq.pushBackAll(*batch) sched.runqsize += n + *batch = gQueue{} } // Try get a batch of G's from the global runnable queue. @@ -4722,16 +4708,11 @@ func globrunqget(_p_ *p, max int32) *g { } sched.runqsize -= n - if sched.runqsize == 0 { - sched.runqtail = 0 - } - gp := sched.runqhead.ptr() - sched.runqhead = gp.schedlink + gp := sched.runq.pop() n-- for ; n > 0; n-- { - gp1 := sched.runqhead.ptr() - sched.runqhead = gp1.schedlink + gp1 := sched.runq.pop() runqput(_p_, gp1, false) } return gp @@ -4859,10 +4840,13 @@ func runqputslow(_p_ *p, gp *g, h, t uint32) bool { for i := uint32(0); i < n; i++ { batch[i].schedlink.set(batch[i+1]) } + var q gQueue + q.head.set(batch[0]) + q.tail.set(batch[n]) // Now put the batch on global queue. lock(&sched.lock) - globrunqputbatch(batch[0], batch[n], int32(n+1)) + globrunqputbatch(&q, int32(n+1)) unlock(&sched.lock) return true } @@ -4974,6 +4958,107 @@ func runqsteal(_p_, p2 *p, stealRunNextG bool) *g { return gp } +// A gQueue is a dequeue of Gs linked through g.schedlink. A G can only +// be on one gQueue or gList at a time. +type gQueue struct { + head guintptr + tail guintptr +} + +// empty returns true if q is empty. +func (q *gQueue) empty() bool { + return q.head == 0 +} + +// push adds gp to the head of q. +func (q *gQueue) push(gp *g) { + gp.schedlink = q.head + q.head.set(gp) + if q.tail == 0 { + q.tail.set(gp) + } +} + +// pushBack adds gp to the tail of q. +func (q *gQueue) pushBack(gp *g) { + gp.schedlink = 0 + if q.tail != 0 { + q.tail.ptr().schedlink.set(gp) + } else { + q.head.set(gp) + } + q.tail.set(gp) +} + +// pushBackAll adds all Gs in l2 to the tail of q. After this q2 must +// not be used. +func (q *gQueue) pushBackAll(q2 gQueue) { + if q2.tail == 0 { + return + } + q2.tail.ptr().schedlink = 0 + if q.tail != 0 { + q.tail.ptr().schedlink = q2.head + } else { + q.head = q2.head + } + q.tail = q2.tail +} + +// pop removes and returns the head of queue q. It returns nil if +// q is empty. +func (q *gQueue) pop() *g { + gp := q.head.ptr() + if gp != nil { + q.head = gp.schedlink + if q.head == 0 { + q.tail = 0 + } + } + return gp +} + +// popList takes all Gs in q and returns them as a gList. +func (q *gQueue) popList() gList { + stack := gList{q.head} + *q = gQueue{} + return stack +} + +// A gList is a list of Gs linked through g.schedlink. A G can only be +// on one gQueue or gList at a time. +type gList struct { + head guintptr +} + +// empty returns true if l is empty. +func (l *gList) empty() bool { + return l.head == 0 +} + +// push adds gp to the head of l. +func (l *gList) push(gp *g) { + gp.schedlink = l.head + l.head.set(gp) +} + +// pushAll prepends all Gs in q to l. +func (l *gList) pushAll(q gQueue) { + if !q.empty() { + q.tail.ptr().schedlink = l.head + l.head = q.head + } +} + +// pop removes and returns the head of l. If l is empty, it returns nil. +func (l *gList) pop() *g { + gp := l.head.ptr() + if gp != nil { + l.head = gp.schedlink + } + return gp +} + //go:linkname setMaxThreads runtime/debug.setMaxThreads func setMaxThreads(in int) (out int) { lock(&sched.lock) diff --git a/src/runtime/runtime2.go b/src/runtime/runtime2.go index ad47d1275e..5bd37e49be 100644 --- a/src/runtime/runtime2.go +++ b/src/runtime/runtime2.go @@ -574,8 +574,7 @@ type schedt struct { nmspinning uint32 // See "Worker thread parking/unparking" comment in proc.go. // Global runnable queue. - runqhead guintptr - runqtail guintptr + runq gQueue runqsize int32 // Global cache of dead G's.