[dev.garbage] runtime: add gc work buffer tryGet and put fast paths

The complexity of the GC work buffers put and tryGet
prevented them from being inlined. This CL simplifies
the fast path thus enabling inlining. If the fast
path does not succeed the previous put and tryGet
functions are called.

Change-Id: I6da6495d0dadf42bd0377c110b502274cc01acf5
Reviewed-on: https://go-review.googlesource.com/20704
Reviewed-by: Austin Clements <austin@google.com>

diff --git a/src/runtime/mgcmark.go b/src/runtime/mgcmark.go
index 47456857e9c45da2ae84e519fbc9e84d251c6f73..d05ad6549fcb8b5b8e24808bbcc2c49a696532df 100644 (file)
--- a/src/runtime/mgcmark.go
+++ b/src/runtime/mgcmark.go
@@ -831,7 +831,10 @@ func gcDrain(gcw *gcWork, flags gcDrainFlags) {
                if blocking {
                        b = gcw.get()
                } else {
-                       b = gcw.tryGet()
+                       b = gcw.tryGetFast()
+                       if b == 0 {
+                               b = gcw.tryGet()
+                       }
                }
                if b == 0 {
                        // work barrier reached or tryGet failed.
@@ -894,7 +897,11 @@ func gcDrainN(gcw *gcWork, scanWork int64) int64 {
                //         PREFETCH(wbuf->obj[wbuf.nobj - 3];
                //  }
                //
-               b := gcw.tryGet()
+               b := gcw.tryGetFast()
+               if b == 0 {
+                       b = gcw.tryGet()
+               }
+
                if b == 0 {
                        break
                }
@@ -1087,8 +1094,9 @@ func greyobject(obj, base, off uintptr, hbits heapBits, span *mspan, gcw *gcWork
        // Previously we put the obj in an 8 element buffer that is drained at a rate
        // to give the PREFETCH time to do its work.
        // Use of PREFETCHNTA might be more appropriate than PREFETCH
-
-       gcw.put(obj)
+       if !gcw.putFast(obj) {
+               gcw.put(obj)
+       }
 }
 
 // gcDumpObject dumps the contents of obj for debugging and marks the

diff --git a/src/runtime/mgcwork.go b/src/runtime/mgcwork.go
index 63a3ade3a663f1158b44ae53985db37a6aba2654..d04840b68671a6ada200d600c55ba20e8d70a7b3 100644 (file)
--- a/src/runtime/mgcwork.go
+++ b/src/runtime/mgcwork.go
@@ -116,6 +116,22 @@ func (w *gcWork) put(obj uintptr) {
        wbuf.nobj++
 }
 
+// putFast does a put and returns true if it can be done quickly
+// otherwise it returns false and the caller needs to call put.
+//go:nowritebarrier
+func (w *gcWork) putFast(obj uintptr) bool {
+       wbuf := w.wbuf1.ptr()
+       if wbuf == nil {
+               return false
+       } else if wbuf.nobj == len(wbuf.obj) {
+               return false
+       }
+
+       wbuf.obj[wbuf.nobj] = obj
+       wbuf.nobj++
+       return true
+}
+
 // tryGet dequeues a pointer for the garbage collector to trace.
 //
 // If there are no pointers remaining in this gcWork or in the global
@@ -147,6 +163,23 @@ func (w *gcWork) tryGet() uintptr {
        return wbuf.obj[wbuf.nobj]
 }
 
+// tryGetFast dequeues a pointer for the garbage collector to trace
+// if one is readily available. Otherwise it returns 0 and
+// the caller is expected to call tryGet().
+//go:nowritebarrier
+func (w *gcWork) tryGetFast() uintptr {
+       wbuf := w.wbuf1.ptr()
+       if wbuf == nil {
+               return 0
+       }
+       if wbuf.nobj == 0 {
+               return 0
+       }
+
+       wbuf.nobj--
+       return wbuf.obj[wbuf.nobj]
+}
+
 // get dequeues a pointer for the garbage collector to trace, blocking
 // if necessary to ensure all pointers from all queues and caches have
 // been retrieved.  get returns 0 if there are no pointers remaining.