math/big: allow all values for GCD

Allow the inputs a and b to be zero or negative to GCD
with the following definitions.

If x or y are not nil, GCD sets their value such that z = a*x + b*y.
Regardless of the signs of a and b, z is always >= 0.
If a == b == 0, GCD sets z = x = y = 0.
If a == 0 and b != 0, GCD sets z = |b|, x = 0, y = sign(b) * 1.
If a != 0 and b == 0, GCD sets z = |a|, x = sign(a) * 1, y = 0.

Fixes #28878

Change-Id: Ia83fce66912a96545c95cd8df0549bfd852652f3
Reviewed-on: https://go-review.googlesource.com/c/go/+/164972
Run-TryBot: Brian Kessler <brian.m.kessler@gmail.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Robert Griesemer <gri@golang.org>

diff --git a/src/math/big/int.go b/src/math/big/int.go
index f4d9a08d72916be704f312270ffd9f93bd63fbe9..bf1fa73cce34863ba90ac0004560dbcd93e1ee8d 100644 (file)
--- a/src/math/big/int.go
+++ b/src/math/big/int.go
@@ -502,18 +502,36 @@ func (z *Int) Exp(x, y, m *Int) *Int {
        return z
 }
 
-// GCD sets z to the greatest common divisor of a and b, which both must
-// be > 0, and returns z.
+// GCD sets z to the greatest common divisor of a and b and returns z.
 // If x or y are not nil, GCD sets their value such that z = a*x + b*y.
-// If either a or b is <= 0, GCD sets z = x = y = 0.
+// Regardless of the signs of a and b, z is always >= 0.
+// If a == b == 0, GCD sets z = x = y = 0.
+// If a == 0 and b != 0, GCD sets z = |b|, x = 0, y = sign(b) * 1.
+// If a != 0 and b == 0, GCD sets z = |a|, x = sign(a) * 1, y = 0.
 func (z *Int) GCD(x, y, a, b *Int) *Int {
-       if a.Sign() <= 0 || b.Sign() <= 0 {
-               z.SetInt64(0)
+       if len(a.abs) == 0 || len(b.abs) == 0 {
+               lenA, lenB, negA, negB := len(a.abs), len(b.abs), a.neg, b.neg
+               if lenA == 0 {
+                       z.Set(b)
+               } else {
+                       z.Set(a)
+               }
+               z.neg = false
                if x != nil {
-                       x.SetInt64(0)
+                       if lenA == 0 {
+                               x.SetUint64(0)
+                       } else {
+                               x.SetUint64(1)
+                               x.neg = negA
+                       }
                }
                if y != nil {
-                       y.SetInt64(0)
+                       if lenB == 0 {
+                               y.SetUint64(0)
+                       } else {
+                               y.SetUint64(1)
+                               y.neg = negB
+                       }
                }
                return z
        }
@@ -621,7 +639,7 @@ func euclidUpdate(A, B, Ua, Ub, q, r, s, t *Int, extended bool) {
 }
 
 // lehmerGCD sets z to the greatest common divisor of a and b,
-// which both must be > 0, and returns z.
+// which both must be != 0, and returns z.
 // If x or y are not nil, their values are set such that z = a*x + b*y.
 // See Knuth, The Art of Computer Programming, Vol. 2, Section 4.5.2, Algorithm L.
 // This implementation uses the improved condition by Collins requiring only one
@@ -633,8 +651,8 @@ func euclidUpdate(A, B, Ua, Ub, q, r, s, t *Int, extended bool) {
 func (z *Int) lehmerGCD(x, y, a, b *Int) *Int {
        var A, B, Ua, Ub *Int
 
-       A = new(Int).Set(a)
-       B = new(Int).Set(b)
+       A = new(Int).Abs(a)
+       B = new(Int).Abs(b)
 
        extended := x != nil || y != nil
 
@@ -720,7 +738,7 @@ func (z *Int) lehmerGCD(x, y, a, b *Int) *Int {
                        A.abs[0] = aWord
                }
        }
-
+       negA := a.neg
        if y != nil {
                // avoid aliasing b needed in the division below
                if y == b {
@@ -730,12 +748,18 @@ func (z *Int) lehmerGCD(x, y, a, b *Int) *Int {
                }
                // y = (z - a*x)/b
                y.Mul(a, Ua) // y can safely alias a
+               if negA {
+                       y.neg = !y.neg
+               }
                y.Sub(A, y)
                y.Div(y, B)
        }
 
        if x != nil {
                *x = *Ua
+               if negA {
+                       x.neg = !x.neg
+               }
        }
 
        *z = *A

diff --git a/src/math/big/int_test.go b/src/math/big/int_test.go
index da12a4b00162a29335717e976516ba7cbb30df44..a4285f3239c7c2aa18a231a414ac5b9b188f821d 100644 (file)
--- a/src/math/big/int_test.go
+++ b/src/math/big/int_test.go
@@ -757,11 +757,13 @@ var gcdTests = []struct {
 }{
        // a <= 0 || b <= 0
        {"0", "0", "0", "0", "0"},
-       {"0", "0", "0", "0", "7"},
-       {"0", "0", "0", "11", "0"},
-       {"0", "0", "0", "-77", "35"},
-       {"0", "0", "0", "64515", "-24310"},
-       {"0", "0", "0", "-64515", "-24310"},
+       {"7", "0", "1", "0", "7"},
+       {"7", "0", "-1", "0", "-7"},
+       {"11", "1", "0", "11", "0"},
+       {"7", "-1", "-2", "-77", "35"},
+       {"935", "-3", "8", "64515", "24310"},
+       {"935", "-3", "-8", "64515", "-24310"},
+       {"935", "3", "-8", "-64515", "-24310"},
 
        {"1", "-9", "47", "120", "23"},
        {"7", "1", "-2", "77", "35"},