runtime: make cas64 like cas32 and casp

The current cas64 definition hard-codes the x86 behavior
of updating *old with the new value when the cas fails.
This is inconsistent with cas32 and casp.
Make it consistent.

This means that the cas64 uses will be epsilon less efficient
than they might be, because they have to do an unnecessary
memory load on x86. But so be it. Code clarity and consistency
is more important.

R=golang-dev, bradfitz
CC=golang-dev
https://golang.org/cl/10909045

diff --git a/src/pkg/runtime/asm_amd64.s b/src/pkg/runtime/asm_amd64.s
index d43eb0283574d6dcf7616fb9530a2aaae319353e..363e680db92145991421d53e72aac9370753c588 100644 (file)
--- a/src/pkg/runtime/asm_amd64.s
+++ b/src/pkg/runtime/asm_amd64.s
@@ -374,13 +374,11 @@ TEXT runtime·cas(SB), 7, $0
 //             *val = new;
 //             return 1;
 //     } else {
-//             *old = *val
 //             return 0;
 //     }
 TEXT runtime·cas64(SB), 7, $0
        MOVQ    8(SP), BX
-       MOVQ    16(SP), BP
-       MOVQ    0(BP), AX
+       MOVQ    16(SP), AX
        MOVQ    24(SP), CX
        LOCK
        CMPXCHGQ        CX, 0(BX)
@@ -388,7 +386,6 @@ TEXT runtime·cas64(SB), 7, $0
        MOVL    $1, AX
        RET
 cas64_fail:
-       MOVQ    AX, 0(BP)
        MOVL    $0, AX
        RET
 

diff --git a/src/pkg/runtime/atomic_386.c b/src/pkg/runtime/atomic_386.c
index 1046eb81e3f196d00714b40edd9cea82ec51f1ac..fbbe9b58e91e49d76abf323a7d0a648b6d4b24cb 100644 (file)
--- a/src/pkg/runtime/atomic_386.c
+++ b/src/pkg/runtime/atomic_386.c
@@ -24,10 +24,10 @@ runtime·xadd64(uint64 volatile* addr, int64 v)
 {
        uint64 old;
 
-       old = *addr;
-       while(!runtime·cas64(addr, &old, old+v)) {
-               // nothing
-       }
+       do
+               old = *addr;
+       while(!runtime·cas64(addr, old, old+v));
+
        return old+v;
 }
 
@@ -37,9 +37,9 @@ runtime·xchg64(uint64 volatile* addr, uint64 v)
 {
        uint64 old;
 
-       old = *addr;
-       while(!runtime·cas64(addr, &old, v)) {
-               // nothing
-       }
+       do
+               old = addr;
+       while(!runtime·cas64(addr, old, v));
+
        return old;
 }

diff --git a/src/pkg/runtime/atomic_arm.c b/src/pkg/runtime/atomic_arm.c
index a78b1dfe2ce80740dece04abedc1db2f64ffb2d6..b186d1b7041fd5ed4ad48d532b7792199c650ed4 100644 (file)
--- a/src/pkg/runtime/atomic_arm.c
+++ b/src/pkg/runtime/atomic_arm.c
@@ -92,16 +92,15 @@ runtime·atomicstore(uint32 volatile* addr, uint32 v)
 
 #pragma textflag 7
 bool
-runtime·cas64(uint64 volatile *addr, uint64 *old, uint64 new)
+runtime·cas64(uint64 volatile *addr, uint64 old, uint64 new)
 {
        bool res;
        
        runtime·lock(LOCK(addr));
-       if(*addr == *old) {
+       if(*addr == old) {
                *addr = new;
                res = true;
        } else {
-               *old = *addr;
                res = false;
        }
        runtime·unlock(LOCK(addr));

diff --git a/src/pkg/runtime/lfstack.c b/src/pkg/runtime/lfstack.c
index 1d48491aac782df124c70050b687f85552db9dce..140384d3dc91469058f70378bb8fa0adac70bee5 100644 (file)
--- a/src/pkg/runtime/lfstack.c
+++ b/src/pkg/runtime/lfstack.c
@@ -29,10 +29,10 @@ runtime·lfstackpush(uint64 *head, LFNode *node)
 
        node->pushcnt++;
        new = (uint64)(uintptr)node|(((uint64)node->pushcnt&CNT_MASK)<<PTR_BITS);
-       old = runtime·atomicload64(head);
        for(;;) {
+               old = runtime·atomicload64(head);
                node->next = (LFNode*)(uintptr)(old&PTR_MASK);
-               if(runtime·cas64(head, &old, new))
+               if(runtime·cas64(head, old, new))
                        break;
        }
 }
@@ -43,8 +43,8 @@ runtime·lfstackpop(uint64 *head)
        LFNode *node, *node2;
        uint64 old, new;
 
-       old = runtime·atomicload64(head);
        for(;;) {
+               old = runtime·atomicload64(head);
                if(old == 0)
                        return nil;
                node = (LFNode*)(uintptr)(old&PTR_MASK);
@@ -52,7 +52,7 @@ runtime·lfstackpop(uint64 *head)
                new = 0;
                if(node2 != nil)
                        new = (uint64)(uintptr)node2|(((uint64)node2->pushcnt&CNT_MASK)<<PTR_BITS);
-               if(runtime·cas64(head, &old, new))
+               if(runtime·cas64(head, old, new))
                        return node;
        }
 }

diff --git a/src/pkg/runtime/parfor.c b/src/pkg/runtime/parfor.c
index a4468c2afc3b121b322e6376a0df5a97c9ada41c..ceaac8bc924bb0ac157e85d08da67984e77f3c98 100644 (file)
--- a/src/pkg/runtime/parfor.c
+++ b/src/pkg/runtime/parfor.c
@@ -144,9 +144,9 @@ runtime·parfordo(ParFor *desc)
                        if(victim >= tid)
                                victim++;
                        victimpos = &desc->thr[victim].pos;
-                       pos = runtime·atomicload64(victimpos);
                        for(;;) {
                                // See if it has any work.
+                               pos = runtime·atomicload64(victimpos);
                                begin = (uint32)pos;
                                end = (uint32)(pos>>32);
                                if(begin+1 >= end) {
@@ -159,7 +159,7 @@ runtime·parfordo(ParFor *desc)
                                }
                                begin2 = begin + (end-begin)/2;
                                newpos = (uint64)begin | (uint64)begin2<<32;
-                               if(runtime·cas64(victimpos, &pos, newpos)) {
+                               if(runtime·cas64(victimpos, pos, newpos)) {
                                        begin = begin2;
                                        break;
                                }

diff --git a/src/pkg/runtime/runtime.c b/src/pkg/runtime/runtime.c
index f59a3f4e80a664f84b7a2758eb63ada32f9bf59d..f0571f189956baa52467ee9ed6ac1dc1125b8b85 100644 (file)
--- a/src/pkg/runtime/runtime.c
+++ b/src/pkg/runtime/runtime.c
@@ -157,11 +157,12 @@ TestAtomic64(void)
        z64 = 42;
        x64 = 0;
        PREFETCH(&z64);
-       if(runtime·cas64(&z64, &x64, 1))
+       if(runtime·cas64(&z64, x64, 1))
                runtime·throw("cas64 failed");
-       if(x64 != 42)
+       if(x64 != 0)
                runtime·throw("cas64 failed");
-       if(!runtime·cas64(&z64, &x64, 1))
+       x64 = 42;
+       if(!runtime·cas64(&z64, x64, 1))
                runtime·throw("cas64 failed");
        if(x64 != 42 || z64 != 1)
                runtime·throw("cas64 failed");
@@ -193,7 +194,7 @@ runtime·check(void)
        uint64 h;
        float32 i, i1;
        float64 j, j1;
-       void* k;
+       byte *k, *k1;
        uint16* l;
        struct x1 {
                byte x;
@@ -232,6 +233,17 @@ runtime·check(void)
        if(z != 4)
                runtime·throw("cas4");
 
+       k = (byte*)0xfedcb123;
+       if(sizeof(void*) == 8)
+               k = (byte*)((uintptr)k<<10);
+       if(runtime·casp((void**)&k, nil, nil))
+               runtime·throw("casp1");
+       k1 = k+1;
+       if(!runtime·casp((void**)&k, k, k1))
+               runtime·throw("casp2");
+       if(k != k1)
+               runtime·throw("casp3");
+
        *(uint64*)&j = ~0ULL;
        if(j == j)
                runtime·throw("float64nan");

diff --git a/src/pkg/runtime/runtime.h b/src/pkg/runtime/runtime.h
index 6c590be633573a197f513c92973c7839e81a7f43..49503ab41babd85396e9e19cc14ec2eb543ed638 100644 (file)
--- a/src/pkg/runtime/runtime.h
+++ b/src/pkg/runtime/runtime.h
@@ -761,7 +761,7 @@ int32       runtime·write(int32, void*, int32);
 int32  runtime·close(int32);
 int32  runtime·mincore(void*, uintptr, byte*);
 bool   runtime·cas(uint32*, uint32, uint32);
-bool   runtime·cas64(uint64*, uint64*, uint64);
+bool   runtime·cas64(uint64*, uint64, uint64);
 bool   runtime·casp(void**, void*, void*);
 // Don't confuse with XADD x86 instruction,
 // this one is actually 'addx', that is, add-and-fetch.