py/mpz: Avoid undefined behavior decrementing NULL.

In the case where an mpz number is zero, its `len` is 0 and its `dig` is
NULL.  In that case, decrementing NULL via `d--` is undefined behavior
according to the C specification.

Restructuring the loops in this way avoids undefined behavior.

Also, ensure that these cases are tested in the coverage test.  This
doesn't make much difference now, but would otherwise cause errors later
when the undefined behavior sanitizer is employed in CI.

Signed-off-by: Jeff Epler <jepler@gmail.com>

1 files changed, 8 insertions, 4 deletions

diff --git a/py/mpz.c b/py/mpz.c
index 471bd1598..5a4d7d27d 100644
--- a/py/mpz.c
+++ b/py/mpz.c
@@ -1537,7 +1537,8 @@ mp_int_t mpz_hash(const mpz_t *z) {
     mp_uint_t val = 0;
     mpz_dig_t *d = z->dig + z->len;
 
-    while (d-- > z->dig) {
+    while (d > z->dig) {
+        d--;
         val = (val << DIG_SIZE) | *d;
     }
 
@@ -1552,11 +1553,12 @@ bool mpz_as_int_checked(const mpz_t *i, mp_int_t *value) {
     mp_uint_t val = 0;
     mpz_dig_t *d = i->dig + i->len;
 
-    while (d-- > i->dig) {
+    while (d > i->dig) {
         if (val > (~(MP_OBJ_WORD_MSBIT_HIGH) >> DIG_SIZE)) {
             // will overflow
             return false;
         }
+        d--;
         val = (val << DIG_SIZE) | *d;
     }
 
@@ -1577,11 +1579,12 @@ bool mpz_as_uint_checked(const mpz_t *i, mp_uint_t *value) {
     mp_uint_t val = 0;
     mpz_dig_t *d = i->dig + i->len;
 
-    while (d-- > i->dig) {
+    while (d > i->dig) {
         if (val > (~(MP_OBJ_WORD_MSBIT_HIGH) >> (DIG_SIZE - 1))) {
             // will overflow
             return false;
         }
+        d--;
         val = (val << DIG_SIZE) | *d;
     }
 
@@ -1642,7 +1645,8 @@ mp_float_t mpz_as_float(const mpz_t *i) {
     mp_float_t val = 0;
     mpz_dig_t *d = i->dig + i->len;
 
-    while (d-- > i->dig) {
+    while (d > i->dig) {
+        d--;
         val = val * DIG_BASE + *d;
     }