Misc BN fixes

Never output -0; make "negative zero" an impossibility.
Do better checking on BN_rand top/bottom requirements and #bits.
Update doc.
Ignoring trailing garbage in BN_asc2bn.

Port this commit from boringSSL: https://boringssl.googlesource.com/boringssl/+/899b9b19a4cd3fe526aaf5047ab9234cdca19f7d%5E!/
        Ensure |BN_div| never gives negative zero in the no_branch code.

        Have |bn_correct_top| fix |bn->neg| if the input is zero so that we
        don't have negative zeros lying around.

        Thanks to Brian Smith for noticing.

Reviewed-by: Richard Levitte <levitte@openssl.org>

3 files changed, 23 insertions, 15 deletions

diff --git a/crypto/bn/bn_lib.c b/crypto/bn/bn_lib.c
index 0be42f8..17d34c3 100644
--- a/crypto/bn/bn_lib.c
+++ b/crypto/bn/bn_lib.c
@@ -1031,5 +1031,7 @@ void bn_correct_top(BIGNUM *a)
         }
         a->top = tmp_top;
     }
+    if (a->top == 0)
+        a->neg = 0;
     bn_pollute(a);
 }
diff --git a/crypto/bn/bn_print.c b/crypto/bn/bn_print.c
index 39fb034..a16bde8 100644
--- a/crypto/bn/bn_print.c
+++ b/crypto/bn/bn_print.c
@@ -23,12 +23,9 @@ char *BN_bn2hex(const BIGNUM *a)
     char *buf;
     char *p;
 
-    if (a->neg && BN_is_zero(a)) {
-        /* "-0" == 3 bytes including NULL terminator */
-        buf = OPENSSL_malloc(3);
-    } else {
-        buf = OPENSSL_malloc(a->top * BN_BYTES * 2 + 2);
-    }
+    if (BN_is_zero(a))
+        return OPENSSL_strdup("0");
+    buf = OPENSSL_malloc(a->top * BN_BYTES * 2 + 2);
     if (buf == NULL) {
         BNerr(BN_F_BN_BN2HEX, ERR_R_MALLOC_FAILURE);
         goto err;
@@ -186,10 +183,12 @@ int BN_hex2bn(BIGNUM **bn, const char *a)
     }
     ret->top = h;
     bn_correct_top(ret);
-    ret->neg = neg;
 
     *bn = ret;
     bn_check_top(ret);
+    /* Don't set the negative flag if it's zero. */
+    if (ret->top != 0)
+        ret->neg = neg;
     return (num);
  err:
     if (*bn == NULL)
@@ -241,7 +240,7 @@ int BN_dec2bn(BIGNUM **bn, const char *a)
     if (j == BN_DEC_NUM)
         j = 0;
     l = 0;
-    while (*a) {
+    while (--i >= 0) {
         l *= 10;
         l += *a - '0';
         a++;
@@ -253,11 +252,13 @@ int BN_dec2bn(BIGNUM **bn, const char *a)
             j = 0;
         }
     }
-    ret->neg = neg;
 
     bn_correct_top(ret);
     *bn = ret;
     bn_check_top(ret);
+    /* Don't set the negative flag if it's zero. */
+    if (ret->top != 0)
+        ret->neg = neg;
     return (num);
  err:
     if (*bn == NULL)
@@ -268,6 +269,7 @@ int BN_dec2bn(BIGNUM **bn, const char *a)
 int BN_asc2bn(BIGNUM **bn, const char *a)
 {
     const char *p = a;
+
     if (*p == '-')
         p++;
 
@@ -278,7 +280,8 @@ int BN_asc2bn(BIGNUM **bn, const char *a)
         if (!BN_dec2bn(bn, p))
             return 0;
     }
-    if (*a == '-')
+    /* Don't set the negative flag if it's zero. */
+    if (*a == '-' && (*bn)->top != 0)
         (*bn)->neg = 1;
     return 1;
 }
diff --git a/crypto/bn/bn_rand.c b/crypto/bn/bn_rand.c
index 5ad8050..c577fd1 100644
--- a/crypto/bn/bn_rand.c
+++ b/crypto/bn/bn_rand.c
@@ -20,15 +20,14 @@ static int bnrand(int pseudorand, BIGNUM *rnd, int bits, int top, int bottom)
     int ret = 0, bit, bytes, mask;
     time_t tim;
 
-    if (bits < 0 || (bits == 1 && top > 0)) {
-        BNerr(BN_F_BNRAND, BN_R_BITS_TOO_SMALL);
-        return 0;
-    }
-
     if (bits == 0) {
+        if (top != BN_RAND_TOP_ANY || bottom != BN_RAND_BOTTOM_ANY)
+            goto toosmall;
         BN_zero(rnd);
         return 1;
     }
+    if (bits < 0 || (bits == 1 && top > 0))
+        goto toosmall;
 
     bytes = (bits + 7) / 8;
     bit = (bits - 1) % 8;
@@ -88,6 +87,10 @@ static int bnrand(int pseudorand, BIGNUM *rnd, int bits, int top, int bottom)
     OPENSSL_clear_free(buf, bytes);
     bn_check_top(rnd);
     return (ret);
+
+toosmall:
+    BNerr(BN_F_BNRAND, BN_R_BITS_TOO_SMALL);
+    return 0;
 }
 
 int BN_rand(BIGNUM *rnd, int bits, int top, int bottom)