math/big: clean up Float.SetPrec, use shorter internal representation

Change-Id: I9b78085adc12cbd240d0b8b48db6810ddb2aeadd
Reviewed-on: https://go-review.googlesource.com/5991
Reviewed-by: Alan Donovan <adonovan@google.com>

diff --git a/src/math/big/float.go b/src/math/big/float.go
index adb914d30e59785c1e38b9f7d95321357016178e..3dedf1db05221306538f2df930e1b0fe147cb370 100644 (file)
--- a/src/math/big/float.go
+++ b/src/math/big/float.go
@@ -61,7 +61,7 @@ type Float struct {
        neg  bool
        mant nat
        exp  int32
-       prec uint // TODO(gri) make this a 32bit field
+       prec uint32
 }
 
 // TODO(gri) provide a couple of Example tests showing typical Float intialization
@@ -77,8 +77,9 @@ type Float struct {
 // values have an empty mantissa and a 0 or infExp exponent, respectively.
 
 const (
-       MaxExp = math.MaxInt32 // largest supported exponent magnitude
-       infExp = -MaxExp - 1   // exponent for Inf values
+       MaxExp  = math.MaxInt32  // largest supported exponent magnitude
+       infExp  = -MaxExp - 1    // exponent for Inf values
+       MaxPrec = math.MaxUint32 // largest (theoretically) supported precision; likely memory-limited
 )
 
 // NewInf returns a new infinite Float value with value +Inf (sign >= 0),
@@ -150,11 +151,32 @@ func (mode RoundingMode) String() string {
 // SetPrec sets z's precision to prec and returns the (possibly) rounded
 // value of z. Rounding occurs according to z's rounding mode if the mantissa
 // cannot be represented in prec bits without loss of precision.
+// If prec == 0, the result is ±0 for finite z, and ±Inf for infinite z,
+// with the sign set according to z. If prec > MaxPrec, it is set to MaxPrec.
 func (z *Float) SetPrec(prec uint) *Float {
+       z.acc = Exact // optimistically assume no rounding is needed
+       // handle special case
+       if prec == 0 {
+               z.prec = 0
+               if len(z.mant) != 0 {
+                       // truncate and compute accuracy
+                       z.mant = z.mant[:0]
+                       z.exp = 0
+                       acc := Below
+                       if z.neg {
+                               acc = Above
+                       }
+                       z.acc = acc
+               }
+               return z
+       }
+       // general case
+       if prec > MaxPrec {
+               prec = MaxPrec
+       }
        old := z.prec
-       z.acc = Exact
-       z.prec = prec
-       if prec < old {
+       z.prec = uint32(prec)
+       if z.prec < old {
                z.round(0)
        }
        return z
@@ -259,7 +281,7 @@ func (x *Float) IsInt() bool {
                return len(x.mant) == 0 && x.exp != infExp
        }
        // x.exp > 0
-       return x.prec <= uint(x.exp) || x.minPrec() <= uint(x.exp) // not enough bits for fractional mantissa
+       return x.prec <= uint32(x.exp) || x.minPrec() <= uint(x.exp) // not enough bits for fractional mantissa
 }
 
 // IsInf reports whether x is an infinity, according to sign.
@@ -320,7 +342,7 @@ func (z *Float) round(sbit uint) {
        z.acc = Exact
 
        // handle zero and Inf
-       m := uint(len(z.mant)) // present mantissa length in words
+       m := uint32(len(z.mant)) // present mantissa length in words
        if m == 0 {
                if z.exp != infExp {
                        z.exp = 0
@@ -351,8 +373,8 @@ func (z *Float) round(sbit uint) {
        //   1     1        >  0.5, < 1.0
 
        // bits > z.prec: mantissa too large => round
-       r := bits - z.prec - 1 // rounding bit position; r >= 0
-       rbit := z.mant.bit(r)  // rounding bit
+       r := uint(bits - z.prec - 1) // rounding bit position; r >= 0
+       rbit := z.mant.bit(r)        // rounding bit
        if sbit == 0 {
                sbit = z.mant.sticky(r)
        }
@@ -567,9 +589,9 @@ func (z *Float) SetInt(x *Int) *Float {
        // TODO(gri) can be more efficient if z.prec > 0
        // but small compared to the size of x, or if there
        // are many trailing 0's.
-       bits := uint(x.BitLen())
+       bits := uint32(x.BitLen())
        if z.prec == 0 {
-               z.prec = umax(bits, 64)
+               z.prec = umax32(bits, 64)
        }
        z.acc = Exact
        z.neg = x.neg
@@ -599,7 +621,7 @@ func (z *Float) SetRat(x *Rat) *Float {
        a.SetInt(x.Num())
        b.SetInt(x.Denom())
        if z.prec == 0 {
-               z.prec = umax(a.prec, b.prec)
+               z.prec = umax32(a.prec, b.prec)
        }
        return z.Quo(&a, &b)
 }
@@ -1126,7 +1148,7 @@ func (z *Float) Add(x, y *Float) *Float {
        }
 
        if z.prec == 0 {
-               z.prec = umax(x.prec, y.prec)
+               z.prec = umax32(x.prec, y.prec)
        }
 
        // TODO(gri) what about -0?
@@ -1167,7 +1189,7 @@ func (z *Float) Sub(x, y *Float) *Float {
        }
 
        if z.prec == 0 {
-               z.prec = umax(x.prec, y.prec)
+               z.prec = umax32(x.prec, y.prec)
        }
 
        // TODO(gri) what about -0?
@@ -1207,7 +1229,7 @@ func (z *Float) Mul(x, y *Float) *Float {
        }
 
        if z.prec == 0 {
-               z.prec = umax(x.prec, y.prec)
+               z.prec = umax32(x.prec, y.prec)
        }
 
        // TODO(gri) handle Inf
@@ -1236,7 +1258,7 @@ func (z *Float) Quo(x, y *Float) *Float {
        }
 
        if z.prec == 0 {
-               z.prec = umax(x.prec, y.prec)
+               z.prec = umax32(x.prec, y.prec)
        }
 
        // TODO(gri) handle Inf
@@ -1337,7 +1359,7 @@ func (x *Float) Cmp(y *Float) int {
        return 0
 }
 
-func umax(x, y uint) uint {
+func umax32(x, y uint32) uint32 {
        if x > y {
                return x
        }

diff --git a/src/math/big/float_test.go b/src/math/big/float_test.go
index 6391beea0bb156b98080b6bb4390b0452a75b87f..bbab7676bb0b8205d7c2a23361790952bd8a0e5c 100644 (file)
--- a/src/math/big/float_test.go
+++ b/src/math/big/float_test.go
@@ -104,6 +104,58 @@ func makeFloat(s string) *Float {
        return &x
 }
 
+func TestFloatSetPrec(t *testing.T) {
+       for _, test := range []struct {
+               x    string
+               prec uint
+               want string
+               acc  Accuracy
+       }{
+               // prec 0
+               {"0", 0, "0", Exact},
+               {"-0", 0, "-0", Exact},
+               {"-Inf", 0, "-Inf", Exact},
+               {"+Inf", 0, "+Inf", Exact},
+               {"123", 0, "0", Below},
+               {"-123", 0, "-0", Above},
+
+               // prec at upper limit
+               {"0", MaxPrec, "0", Exact},
+               {"0", MaxPrec + 1, "0", Exact},
+               {"-0", MaxPrec, "-0", Exact},
+               {"-0", MaxPrec + 1, "-0", Exact},
+               {"-Inf", MaxPrec, "-Inf", Exact},
+               {"+Inf", MaxPrec + 1, "+Inf", Exact},
+               {"-Inf", MaxPrec, "-Inf", Exact},
+               {"+Inf", MaxPrec + 1, "+Inf", Exact},
+
+               // just a few regular cases - general rounding is tested elsewhere
+               {"1.5", 1, "2", Above},
+               {"-1.5", 1, "-2", Below},
+               {"123", 1e6, "123", Exact},
+               {"-123", 1e6, "-123", Exact},
+       } {
+               x := makeFloat(test.x).SetPrec(test.prec)
+               prec := test.prec
+               if prec > MaxPrec {
+                       prec = MaxPrec
+               }
+               if got := x.Prec(); got != prec {
+                       t.Errorf("%s.SetPrec(%d).Prec() == %d; want %d", test.x, test.prec, got, prec)
+               }
+               if got, acc := x.String(), x.Acc(); got != test.want || acc != test.acc {
+                       t.Errorf("%s.SetPrec(%d) = %s (%s); want %s (%s)", test.x, test.prec, got, acc, test.want, test.acc)
+               }
+               // look inside x and check correct value for x.exp
+               if len(x.mant) == 0 {
+                       // ±0 or ±Inf
+                       if x.exp != 0 && x.exp != infExp {
+                               t.Errorf("%s.SetPrec(%d): incorrect exponent %d", test.x, test.prec, x.exp)
+                       }
+               }
+       }
+}
+
 func TestFloatSign(t *testing.T) {
        for _, test := range []struct {
                x string

diff --git a/src/math/big/floatconv.go b/src/math/big/floatconv.go
index 96ccd601da6e2b0eef87c211a02b365df9d1300e..b1b028c235d25465f56bc0abb93f5f4a7d52b582 100644 (file)
--- a/src/math/big/floatconv.go
+++ b/src/math/big/floatconv.go
@@ -277,11 +277,11 @@ func (x *Float) bstring(buf []byte) []byte {
 
        // adjust mantissa to use exactly x.prec bits
        m := x.mant
-       switch w := uint(len(x.mant)) * _W; {
+       switch w := uint32(len(x.mant)) * _W; {
        case w < x.prec:
-               m = nat(nil).shl(m, x.prec-w)
+               m = nat(nil).shl(m, uint(x.prec-w))
        case w > x.prec:
-               m = nat(nil).shr(m, w-x.prec)
+               m = nat(nil).shr(m, uint(w-x.prec))
        }
 
        buf = append(buf, m.decimalString()...)