runtime: replace lock() with casp() in the GC

Note: BitTarget will be removed by a forthcoming changeset.

R=golang-dev, dvyukov
CC=golang-dev, rsc
https://golang.org/cl/7837044

diff --git a/src/pkg/runtime/mgc0.c b/src/pkg/runtime/mgc0.c
index 6ec9706f49a092f782271a53cd8574d8ea9268ef..4cafda4439ded561680722a0a142db449ca1e123 100644 (file)
--- a/src/pkg/runtime/mgc0.c
+++ b/src/pkg/runtime/mgc0.c
@@ -191,7 +191,7 @@ static struct {
 
 // markonly marks an object. It returns true if the object
 // has been marked by this function, false otherwise.
-// This function isn't thread-safe and doesn't append the object to any buffer.
+// This function doesn't append the object to any buffer.
 static bool
 markonly(void *obj)
 {
@@ -254,7 +254,17 @@ found:
        // Only care about allocated and not marked.
        if((bits & (bitAllocated|bitMarked)) != bitAllocated)
                return false;
-       *bitp |= bitMarked<<shift;
+       if(work.nproc == 1)
+               *bitp |= bitMarked<<shift;
+       else {
+               for(;;) {
+                       x = *bitp;
+                       if(x & (bitMarked<<shift))
+                               return false;
+                       if(runtime·casp((void**)bitp, (void*)x, (void*)(x|(bitMarked<<shift))))
+                               break;
+               }
+       }
 
        // The object is now marked
        return true;
@@ -325,7 +335,6 @@ flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf
        Obj *wp;
        Workbuf *wbuf;
        PtrTarget *ptrbuf_end;
-       BitTarget *bitbufpos, *bt;
 
        arena_start = runtime·mheap->arena_start;
 
@@ -359,8 +368,6 @@ flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf
        {
                // Multi-threaded version.
 
-               bitbufpos = bitbuf;
-
                while(ptrbuf < ptrbuf_end) {
                        obj = ptrbuf->p;
                        ti = ptrbuf->ti;
@@ -438,26 +445,22 @@ flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf
                        // Only care about allocated and not marked.
                        if((bits & (bitAllocated|bitMarked)) != bitAllocated)
                                continue;
-
-                       *bitbufpos++ = (BitTarget){obj, ti, bitp, shift};
-               }
-
-               runtime·lock(&lock);
-               for(bt=bitbuf; bt<bitbufpos; bt++){
-                       xbits = *bt->bitp;
-                       bits = xbits >> bt->shift;
-                       if((bits & bitMarked) != 0)
-                               continue;
-
-                       // Mark the block
-                       *bt->bitp = xbits | (bitMarked << bt->shift);
+                       if(work.nproc == 1)
+                               *bitp |= bitMarked<<shift;
+                       else {
+                               for(;;) {
+                                       x = *bitp;
+                                       if(x & (bitMarked<<shift))
+                                               goto continue_obj;
+                                       if(runtime·casp((void**)bitp, (void*)x, (void*)(x|(bitMarked<<shift))))
+                                               break;
+                               }
+                       }
 
                        // If object has no pointers, don't need to scan further.
                        if((bits & bitNoPointers) != 0)
                                continue;
 
-                       obj = bt->p;
-
                        // Ask span about size class.
                        // (Manually inlined copy of MHeap_Lookup.)
                        x = (uintptr)obj >> PageShift;
@@ -467,11 +470,11 @@ flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf
 
                        PREFETCH(obj);
 
-                       *wp = (Obj){obj, s->elemsize, bt->ti};
+                       *wp = (Obj){obj, s->elemsize, ti};
                        wp++;
                        nobj++;
+               continue_obj:;
                }
-               runtime·unlock(&lock);
 
                // If another proc wants a pointer, give it some.
                if(work.nwait > 0 && nobj > handoffThreshold && work.full == 0) {
@@ -588,7 +591,7 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
        Iface *iface;
        Hmap *hmap;
        MapType *maptype;
-       bool didmark, mapkey_kind, mapval_kind;
+       bool mapkey_kind, mapval_kind;
        struct hash_gciter map_iter;
        struct hash_gciter_data d;
        Hchan *chan;
@@ -894,10 +897,7 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
                                pc += 3;
                                continue;
                        }
-                       runtime·lock(&lock);
-                       didmark = markonly(hmap);
-                       runtime·unlock(&lock);
-                       if(didmark) {
+                       if(markonly(hmap)) {
                                maptype = (MapType*)pc[2];
                                if(hash_gciter_init(hmap, &map_iter)) {
                                        mapkey_size = maptype->key->size;
@@ -927,11 +927,9 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
                                if(objbufpos+2 >= objbuf_end)
                                        flushobjbuf(objbuf, &objbufpos, &wp, &wbuf, &nobj);
 
-                               if(d.st != nil) {
-                                       runtime·lock(&lock);
+                               if(d.st != nil)
                                        markonly(d.st);
-                                       runtime·unlock(&lock);
-                               }
+
                                if(d.key_data != nil) {
                                        if(!(mapkey_kind & KindNoPointers) || d.indirectkey) {
                                                if(!d.indirectkey)