From: Dmitriy Vyukov Date: Mon, 25 Aug 2014 16:12:26 +0000 (+0400) Subject: runtime: convert async semaphores to Go X-Git-Tag: go1.4beta1~714 X-Git-Url: http://www.git.cypherpunks.su/?a=commitdiff_plain;h=ebac0e6f302c52e97a212fab8837a73f31edc884;p=gostls13.git runtime: convert async semaphores to Go LGTM=rsc R=golang-codereviews, rsc CC=golang-codereviews, khr https://golang.org/cl/126210046 --- diff --git a/src/cmd/gc/select.c b/src/cmd/gc/select.c index a8caefbb53..8cf9926c17 100644 --- a/src/cmd/gc/select.c +++ b/src/cmd/gc/select.c @@ -337,6 +337,7 @@ selecttype(int32 size) sudog->list = list(sudog->list, nod(ODCLFIELD, newname(lookup("g")), typenod(ptrto(types[TUINT8])))); sudog->list = list(sudog->list, nod(ODCLFIELD, newname(lookup("selectdone")), typenod(ptrto(types[TUINT8])))); sudog->list = list(sudog->list, nod(ODCLFIELD, newname(lookup("link")), typenod(ptrto(types[TUINT8])))); + sudog->list = list(sudog->list, nod(ODCLFIELD, newname(lookup("prev")), typenod(ptrto(types[TUINT8])))); sudog->list = list(sudog->list, nod(ODCLFIELD, newname(lookup("elem")), typenod(ptrto(types[TUINT8])))); sudog->list = list(sudog->list, nod(ODCLFIELD, newname(lookup("releasetime")), typenod(types[TUINT64]))); sudog->list = list(sudog->list, nod(ODCLFIELD, newname(lookup("nrelease")), typenod(types[TINT32]))); diff --git a/src/pkg/runtime/arch_386.go b/src/pkg/runtime/arch_386.go new file mode 100644 index 0000000000..32eafb549b --- /dev/null +++ b/src/pkg/runtime/arch_386.go @@ -0,0 +1,9 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +const ( + cacheLineSize = 64 +) diff --git a/src/pkg/runtime/arch_amd64.go b/src/pkg/runtime/arch_amd64.go new file mode 100644 index 0000000000..32eafb549b --- /dev/null +++ b/src/pkg/runtime/arch_amd64.go @@ -0,0 +1,9 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +const ( + cacheLineSize = 64 +) diff --git a/src/pkg/runtime/arch_amd64p32.go b/src/pkg/runtime/arch_amd64p32.go new file mode 100644 index 0000000000..32eafb549b --- /dev/null +++ b/src/pkg/runtime/arch_amd64p32.go @@ -0,0 +1,9 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +const ( + cacheLineSize = 64 +) diff --git a/src/pkg/runtime/arch_arm.go b/src/pkg/runtime/arch_arm.go new file mode 100644 index 0000000000..7faeb94049 --- /dev/null +++ b/src/pkg/runtime/arch_arm.go @@ -0,0 +1,9 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +const ( + cacheLineSize = 32 +) diff --git a/src/pkg/runtime/chan.go b/src/pkg/runtime/chan.go index f9a540af3d..bbf5e7aa83 100644 --- a/src/pkg/runtime/chan.go +++ b/src/pkg/runtime/chan.go @@ -168,7 +168,7 @@ func chansend(t *chantype, c *hchan, ep unsafe.Pointer, block bool, callerpc uin if t0 != 0 { mysg.releasetime = -1 } - mysg.elem = (*uint8)(ep) + mysg.elem = ep mysg.waitlink = nil gp.waiting = mysg mysg.g = gp @@ -257,13 +257,13 @@ func chansend(t *chantype, c *hchan, ep unsafe.Pointer, block bool, callerpc uin } func (q *waitq) enqueue(sgp *sudog) { - sgp.link = nil + sgp.next = nil if q.first == nil { q.first = sgp q.last = sgp return } - q.last.link = sgp + q.last.next = sgp q.last = sgp } @@ -273,7 +273,7 @@ func (q *waitq) dequeue() *sudog { if sgp == nil { return nil } - q.first = sgp.link + q.first = sgp.next if q.last == sgp { q.last = nil } diff --git a/src/pkg/runtime/chan.goc b/src/pkg/runtime/chan.goc index 4d4f366efa..7d6ee1be2b 100644 --- a/src/pkg/runtime/chan.goc +++ b/src/pkg/runtime/chan.goc @@ -1089,7 +1089,7 @@ loop: sgp = q->first; if(sgp == nil) return nil; - q->first = sgp->link; + q->first = sgp->next; if(q->last == sgp) q->last = nil; @@ -1109,9 +1109,9 @@ dequeueg(WaitQ *q) SudoG **l, *sgp, *prevsgp; prevsgp = nil; - for(l=&q->first; (sgp=*l) != nil; l=&sgp->link, prevsgp=sgp) { + for(l=&q->first; (sgp=*l) != nil; l=&sgp->next, prevsgp=sgp) { if(sgp->g == g) { - *l = sgp->link; + *l = sgp->next; if(q->last == sgp) q->last = prevsgp; break; @@ -1122,13 +1122,13 @@ dequeueg(WaitQ *q) static void enqueue(WaitQ *q, SudoG *sgp) { - sgp->link = nil; + sgp->next = nil; if(q->first == nil) { q->first = sgp; q->last = sgp; return; } - q->last->link = sgp; + q->last->next = sgp; q->last = sgp; } diff --git a/src/pkg/runtime/proc.go b/src/pkg/runtime/proc.go index 7a3864f3bc..1ed6c26eba 100644 --- a/src/pkg/runtime/proc.go +++ b/src/pkg/runtime/proc.go @@ -68,7 +68,7 @@ func acquireSudog() *sudog { c := gomcache() s := c.sudogcache if s != nil { - c.sudogcache = s.link + c.sudogcache = s.next return s } return new(sudog) @@ -77,6 +77,6 @@ func acquireSudog() *sudog { //go:nosplit func releaseSudog(s *sudog) { c := gomcache() - s.link = c.sudogcache + s.next = c.sudogcache c.sudogcache = s } diff --git a/src/pkg/runtime/runtime.h b/src/pkg/runtime/runtime.h index c96fc81417..d7709ae3c1 100644 --- a/src/pkg/runtime/runtime.h +++ b/src/pkg/runtime/runtime.h @@ -224,8 +224,9 @@ struct SudoG { G* g; uint32* selectdone; - SudoG* link; - byte* elem; // data element + SudoG* next; + SudoG* prev; + void* elem; // data element int64 releasetime; int32 nrelease; // -1 for acquire SudoG* waitlink; // G.waiting list diff --git a/src/pkg/runtime/sema.go b/src/pkg/runtime/sema.go index 1d591ffeba..a9ed7150d8 100644 --- a/src/pkg/runtime/sema.go +++ b/src/pkg/runtime/sema.go @@ -21,6 +21,159 @@ package runtime import "unsafe" +// Asynchronous semaphore for sync.Mutex. + +type semaRoot struct { + lock + head *sudog + tail *sudog + nwait uint32 // Number of waiters. Read w/o the lock. +} + +// Prime to not correlate with any user patterns. +const semTabSize = 251 + +var semtable [semTabSize]struct { + root semaRoot + pad [cacheLineSize - unsafe.Sizeof(semaRoot{})]byte +} + +// Called from sync/net packages. +func asyncsemacquire(addr *uint32) { + semacquire(addr, true) +} + +func asyncsemrelease(addr *uint32) { + semrelease(addr) +} + +// Called from runtime. +func semacquire(addr *uint32, profile bool) { + // Easy case. + if cansemacquire(addr) { + return + } + + // Harder case: + // increment waiter count + // try cansemacquire one more time, return if succeeded + // enqueue itself as a waiter + // sleep + // (waiter descriptor is dequeued by signaler) + s := acquireSudog() + root := semroot(addr) + t0 := int64(0) + s.releasetime = 0 + if profile && blockprofilerate > 0 { + t0 = gocputicks() + s.releasetime = -1 + } + for { + golock(&root.lock) + // Add ourselves to nwait to disable "easy case" in semrelease. + goxadd(&root.nwait, 1) + // Check cansemacquire to avoid missed wakeup. + if cansemacquire(addr) { + goxadd(&root.nwait, ^uint32(0)) + gounlock(&root.lock) + break + } + // Any semrelease after the cansemacquire knows we're waiting + // (we set nwait above), so go to sleep. + root.queue(addr, s) + goparkunlock(&root.lock, "semacquire") + if cansemacquire(addr) { + break + } + } + if s.releasetime > 0 { + goblockevent(int64(s.releasetime)-t0, 4) + } + releaseSudog(s) +} + +func semrelease(addr *uint32) { + root := semroot(addr) + goxadd(addr, 1) + + // Easy case: no waiters? + // This check must happen after the xadd, to avoid a missed wakeup + // (see loop in semacquire). + if goatomicload(&root.nwait) == 0 { + return + } + + // Harder case: search for a waiter and wake it. + golock(&root.lock) + if goatomicload(&root.nwait) == 0 { + // The count is already consumed by another goroutine, + // so no need to wake up another goroutine. + gounlock(&root.lock) + return + } + s := root.head + for ; s != nil; s = s.next { + if s.elem == unsafe.Pointer(addr) { + goxadd(&root.nwait, ^uint32(0)) + root.dequeue(s) + break + } + } + gounlock(&root.lock) + if s != nil { + if s.releasetime != 0 { + // TODO: Remove use of unsafe here. + releasetimep := (*int64)(unsafe.Pointer(&s.releasetime)) + *releasetimep = gocputicks() + } + goready(s.g) + } +} + +func semroot(addr *uint32) *semaRoot { + return &semtable[(uintptr(unsafe.Pointer(addr))>>3)%semTabSize].root +} + +func cansemacquire(addr *uint32) bool { + for { + v := goatomicload(addr) + if v == 0 { + return false + } + if gocas(addr, v, v-1) { + return true + } + } +} + +func (root *semaRoot) queue(addr *uint32, s *sudog) { + s.g = getg() + s.elem = unsafe.Pointer(addr) + s.next = nil + s.prev = root.tail + if root.tail != nil { + root.tail.next = s + } else { + root.head = s + } + root.tail = s +} + +func (root *semaRoot) dequeue(s *sudog) { + if s.next != nil { + s.next.prev = s.prev + } else { + root.tail = s.prev + } + if s.prev != nil { + s.prev.next = s.next + } else { + root.head = s.next + } + s.next = nil + s.prev = nil +} + // Synchronous semaphore for sync.Cond. type syncSema struct { lock lock @@ -37,7 +190,7 @@ func syncsemacquire(s *syncSema) { s.head.nrelease-- if s.head.nrelease == 0 { wake = s.head - s.head = wake.link + s.head = wake.next if s.head == nil { s.tail = nil } @@ -51,7 +204,7 @@ func syncsemacquire(s *syncSema) { w := acquireSudog() w.g = getg() w.nrelease = -1 - w.link = nil + w.next = nil w.releasetime = 0 t0 := int64(0) if blockprofilerate > 0 { @@ -61,7 +214,7 @@ func syncsemacquire(s *syncSema) { if s.tail == nil { s.head = w } else { - s.tail.link = w + s.tail.next = w } s.tail = w goparkunlock(&s.lock, "semacquire") @@ -78,7 +231,7 @@ func syncsemrelease(s *syncSema, n uint32) { for n > 0 && s.head != nil && s.head.nrelease < 0 { // Have pending acquire, satisfy it. wake := s.head - s.head = wake.link + s.head = wake.next if s.head == nil { s.tail = nil } @@ -95,12 +248,12 @@ func syncsemrelease(s *syncSema, n uint32) { w := acquireSudog() w.g = getg() w.nrelease = int32(n) - w.link = nil + w.next = nil w.releasetime = 0 if s.tail == nil { s.head = w } else { - s.tail.link = w + s.tail.next = w } s.tail = w goparkunlock(&s.lock, "semarelease") diff --git a/src/pkg/runtime/sema.goc b/src/pkg/runtime/sema.goc deleted file mode 100644 index 7724566a1e..0000000000 --- a/src/pkg/runtime/sema.goc +++ /dev/null @@ -1,204 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Semaphore implementation exposed to Go. -// Intended use is provide a sleep and wakeup -// primitive that can be used in the contended case -// of other synchronization primitives. -// Thus it targets the same goal as Linux's futex, -// but it has much simpler semantics. -// -// That is, don't think of these as semaphores. -// Think of them as a way to implement sleep and wakeup -// such that every sleep is paired with a single wakeup, -// even if, due to races, the wakeup happens before the sleep. -// -// See Mullender and Cox, ``Semaphores in Plan 9,'' -// http://swtch.com/semaphore.pdf - -package sync -#include "runtime.h" -#include "arch_GOARCH.h" -#include "../../cmd/ld/textflag.h" - -typedef struct SemaWaiter SemaWaiter; -struct SemaWaiter -{ - uint32 volatile* addr; - G* g; - int64 releasetime; - int32 nrelease; // -1 for acquire - SemaWaiter* prev; - SemaWaiter* next; -}; - -typedef struct SemaRoot SemaRoot; -struct SemaRoot -{ - Lock lock; - SemaWaiter* head; - SemaWaiter* tail; - // Number of waiters. Read w/o the lock. - uint32 volatile nwait; -}; - -// Prime to not correlate with any user patterns. -#define SEMTABLESZ 251 - -struct semtable -{ - SemaRoot root; - uint8 pad[CacheLineSize-sizeof(SemaRoot)]; -}; -#pragma dataflag NOPTR /* mark semtable as 'no pointers', hiding from garbage collector */ -static struct semtable semtable[SEMTABLESZ]; - -static SemaRoot* -semroot(uint32 *addr) -{ - return &semtable[((uintptr)addr >> 3) % SEMTABLESZ].root; -} - -static void -semqueue(SemaRoot *root, uint32 volatile *addr, SemaWaiter *s) -{ - s->g = g; - s->addr = addr; - s->next = nil; - s->prev = root->tail; - if(root->tail) - root->tail->next = s; - else - root->head = s; - root->tail = s; -} - -static void -semdequeue(SemaRoot *root, SemaWaiter *s) -{ - if(s->next) - s->next->prev = s->prev; - else - root->tail = s->prev; - if(s->prev) - s->prev->next = s->next; - else - root->head = s->next; - s->prev = nil; - s->next = nil; -} - -static int32 -cansemacquire(uint32 *addr) -{ - uint32 v; - - while((v = runtime·atomicload(addr)) > 0) - if(runtime·cas(addr, v, v-1)) - return 1; - return 0; -} - -void -runtime·semacquire(uint32 volatile *addr, bool profile) -{ - SemaWaiter s; // Needs to be allocated on stack, otherwise garbage collector could deallocate it - SemaRoot *root; - int64 t0; - - // Easy case. - if(cansemacquire(addr)) - return; - - // Harder case: - // increment waiter count - // try cansemacquire one more time, return if succeeded - // enqueue itself as a waiter - // sleep - // (waiter descriptor is dequeued by signaler) - root = semroot(addr); - t0 = 0; - s.releasetime = 0; - if(profile && runtime·blockprofilerate > 0) { - t0 = runtime·cputicks(); - s.releasetime = -1; - } - for(;;) { - runtime·lock(&root->lock); - // Add ourselves to nwait to disable "easy case" in semrelease. - runtime·xadd(&root->nwait, 1); - // Check cansemacquire to avoid missed wakeup. - if(cansemacquire(addr)) { - runtime·xadd(&root->nwait, -1); - runtime·unlock(&root->lock); - return; - } - // Any semrelease after the cansemacquire knows we're waiting - // (we set nwait above), so go to sleep. - semqueue(root, addr, &s); - runtime·parkunlock(&root->lock, runtime·gostringnocopy((byte*)"semacquire")); - if(cansemacquire(addr)) { - if(t0) - runtime·blockevent(s.releasetime - t0, 3); - return; - } - } -} - -void -runtime·semrelease(uint32 volatile *addr) -{ - SemaWaiter *s; - SemaRoot *root; - - root = semroot(addr); - runtime·xadd(addr, 1); - - // Easy case: no waiters? - // This check must happen after the xadd, to avoid a missed wakeup - // (see loop in semacquire). - if(runtime·atomicload(&root->nwait) == 0) - return; - - // Harder case: search for a waiter and wake it. - runtime·lock(&root->lock); - if(runtime·atomicload(&root->nwait) == 0) { - // The count is already consumed by another goroutine, - // so no need to wake up another goroutine. - runtime·unlock(&root->lock); - return; - } - for(s = root->head; s; s = s->next) { - if(s->addr == addr) { - runtime·xadd(&root->nwait, -1); - semdequeue(root, s); - break; - } - } - runtime·unlock(&root->lock); - if(s) { - if(s->releasetime) - s->releasetime = runtime·cputicks(); - runtime·ready(s->g); - } -} - -// TODO(dvyukov): move to netpoll.goc once it's used by all OSes. -void net·runtime_Semacquire(uint32 *addr) -{ - runtime·semacquire(addr, true); -} - -void net·runtime_Semrelease(uint32 *addr) -{ - runtime·semrelease(addr); -} - -func runtime_Semacquire(addr *uint32) { - runtime·semacquire(addr, true); -} - -func runtime_Semrelease(addr *uint32) { - runtime·semrelease(addr); -} diff --git a/src/pkg/runtime/stubs.go b/src/pkg/runtime/stubs.go index 37e768a4ec..6cabed063e 100644 --- a/src/pkg/runtime/stubs.go +++ b/src/pkg/runtime/stubs.go @@ -116,7 +116,9 @@ const ( // Atomic operations to read/write a pointer. // in stubs.goc +func goatomicload(p *uint32) uint32 // return *p func goatomicloadp(p unsafe.Pointer) unsafe.Pointer // return *p +func goatomicstore(p *uint32, v uint32) // *p = v func goatomicstorep(p unsafe.Pointer, v unsafe.Pointer) // *p = v // in stubs.goc @@ -124,6 +126,9 @@ func goatomicstorep(p unsafe.Pointer, v unsafe.Pointer) // *p = v //go:noescape func gocas(p *uint32, x uint32, y uint32) bool +//go:noescape +func goxadd(p *uint32, x uint32) uint32 + //go:noescape func gocasx(p *uintptr, x uintptr, y uintptr) bool @@ -151,8 +156,6 @@ func gothrow(s string) func golock(x *lock) func gounlock(x *lock) -func semacquire(*uint32, bool) -func semrelease(*uint32) // Return the Go equivalent of the C Alg structure. // TODO: at some point Go will hold the truth for the layout diff --git a/src/pkg/runtime/stubs.goc b/src/pkg/runtime/stubs.goc index 74762dab8a..b30c40286c 100644 --- a/src/pkg/runtime/stubs.goc +++ b/src/pkg/runtime/stubs.goc @@ -48,16 +48,31 @@ func gonanotime() (r int64) { r = runtime·nanotime(); } +#pragma textflag NOSPLIT +func goatomicload(p *uint32) (v uint32) { + v = runtime·atomicload(p); +} + #pragma textflag NOSPLIT func goatomicloadp(p **byte) (v *byte) { v = runtime·atomicloadp(p); } +#pragma textflag NOSPLIT +func goatomicstore(p *uint32, v uint32) { + runtime·atomicstore(p, v); +} + #pragma textflag NOSPLIT func goatomicstorep(p **byte, v *byte) { runtime·atomicstorep(p, v); } +#pragma textflag NOSPLIT +func runtime·goxadd(p *uint32, x uint32) (ret uint32) { + ret = runtime·xadd(p, x); +} + #pragma textflag NOSPLIT func runtime·gocas(p *uint32, x uint32, y uint32) (ret bool) { ret = runtime·cas(p, x, y); diff --git a/src/pkg/runtime/thunk.s b/src/pkg/runtime/thunk.s index 2a48f5ff23..b54d9eded9 100644 --- a/src/pkg/runtime/thunk.s +++ b/src/pkg/runtime/thunk.s @@ -19,3 +19,15 @@ TEXT sync·runtime_Syncsemrelease(SB),NOSPLIT,$0-0 TEXT sync·runtime_Syncsemcheck(SB),NOSPLIT,$0-0 JMP runtime·syncsemcheck(SB) + +TEXT sync·runtime_Semacquire(SB),NOSPLIT,$0-0 + JMP runtime·asyncsemacquire(SB) + +TEXT sync·runtime_Semrelease(SB),NOSPLIT,$0-0 + JMP runtime·asyncsemrelease(SB) + +TEXT net·runtime_Semacquire(SB),NOSPLIT,$0-0 + JMP runtime·asyncsemacquire(SB) + +TEXT net·runtime_Semrelease(SB),NOSPLIT,$0-0 + JMP runtime·asyncsemrelease(SB)