py: Improve handling of long-int overflow.

This removes mpz_as_int, since that was a terrible function (it
implemented saturating conversion).

Use mpz_as_int_checked and mpz_as_uint_checked.  These now work
correctly (they previously had wrong overflow checking, eg
print(chr(10000000000000)) on 32-bit machine would incorrectly convert
this large number to a small int).

1 files changed, 17 insertions, 25 deletions

diff --git a/py/mpz.c b/py/mpz.c
index 24c83d913..cf02aa242 100644
--- a/py/mpz.c
+++ b/py/mpz.c
@@ -1229,49 +1229,41 @@ mp_int_t mpz_hash(const mpz_t *z) {
     return val;
 }
 
-// TODO check that this correctly handles overflow in all cases
-mp_int_t mpz_as_int(const mpz_t *i) {
+bool mpz_as_int_checked(const mpz_t *i, mp_int_t *value) {
     mp_int_t val = 0;
     mpz_dig_t *d = i->dig + i->len;
 
     while (--d >= i->dig) {
-        mp_int_t oldval = val;
-        val = (val << DIG_SIZE) | *d;
-        if (val < oldval) {
-            // overflow, return +/- "infinity"
-            if (i->neg == 0) {
-                // +infinity
-                return ~WORD_MSBIT_HIGH;
-            } else {
-                // -infinity
-                return WORD_MSBIT_HIGH;
-            }
+        if (val > (~(WORD_MSBIT_HIGH) >> DIG_SIZE)) {
+            // will overflow
+            return false;
         }
+        val = (val << DIG_SIZE) | *d;
     }
 
     if (i->neg != 0) {
         val = -val;
     }
 
-    return val;
+    *value = val;
+    return true;
 }
 
-// TODO check that this correctly handles overflow in all cases
-bool mpz_as_int_checked(const mpz_t *i, mp_int_t *value) {
-    mp_int_t val = 0;
+bool mpz_as_uint_checked(const mpz_t *i, mp_uint_t *value) {
+    if (i->neg != 0) {
+        // can't represent signed values
+        return false;
+    }
+
+    mp_uint_t val = 0;
     mpz_dig_t *d = i->dig + i->len;
 
     while (--d >= i->dig) {
-        mp_int_t oldval = val;
-        val = (val << DIG_SIZE) | *d;
-        if (val < oldval) {
-            // overflow
+        if (val > ((~0) >> DIG_SIZE)) {
+            // will overflow
             return false;
         }
-    }
-
-    if (i->neg != 0) {
-        val = -val;
+        val = (val << DIG_SIZE) | *d;
     }
 
     *value = val;