math/big: fix binaryGCD

R=rsc
CC=golang-dev
https://golang.org/cl/6297085

diff --git a/src/pkg/math/big/int.go b/src/pkg/math/big/int.go
index 16fd9bfa98bdaf6d586f323e028c3b27b59575ba..74e5c2313b1117553a7fd2ccfe4165d4143174b7 100644 (file)
--- a/src/pkg/math/big/int.go
+++ b/src/pkg/math/big/int.go
@@ -643,12 +643,13 @@ func (z *Int) GCD(x, y, a, b *Int) *Int {
        return z
 }
 
-// binaryGCD sets z to the greatest common divisor of a and b, which must be
-// positive, and returns z.
+// binaryGCD sets z to the greatest common divisor of a and b, which both must
+// be > 0, and returns z.
 // See Knuth, The Art of Computer Programming, Vol. 2, Section 4.5.2, Algorithm B.
 func (z *Int) binaryGCD(a, b *Int) *Int {
        u := z
        v := new(Int)
+
        // use one Euclidean iteration to ensure that u and v are approx. the same size
        switch {
        case len(a.abs) > len(b.abs):
@@ -662,6 +663,12 @@ func (z *Int) binaryGCD(a, b *Int) *Int {
                v.Set(b)
        }
 
+       // v might be 0 now
+       if len(v.abs) == 0 {
+               return u
+       }
+       // u > 0 && v > 0
+
        // determine largest k such that u = u' << k, v = v' << k
        k := u.abs.trailingZeroBits()
        if vk := v.abs.trailingZeroBits(); vk < k {