crypto: updates to pass size_t to RAND_bytes_ex()

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Shane Lontis <shane.lontis@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/15540)

10 files changed, 28 insertions, 11 deletions

diff --git a/crypto/asn1/p5_pbev2.c b/crypto/asn1/p5_pbev2.c
index d16fb8c..162e31d 100644
--- a/crypto/asn1/p5_pbev2.c
+++ b/crypto/asn1/p5_pbev2.c
@@ -45,7 +45,7 @@ X509_ALGOR *PKCS5_pbe2_set_iv_ex(const EVP_CIPHER *cipher, int iter,
                                  OSSL_LIB_CTX *libctx)
 {
     X509_ALGOR *scheme = NULL, *ret = NULL;
-    int alg_nid, keylen;
+    int alg_nid, keylen, ivlen;
     EVP_CIPHER_CTX *ctx = NULL;
     unsigned char iv[EVP_MAX_IV_LENGTH];
     PBE2PARAM *pbe2 = NULL;
@@ -66,11 +66,11 @@ X509_ALGOR *PKCS5_pbe2_set_iv_ex(const EVP_CIPHER *cipher, int iter,
         goto merr;
 
     /* Create random IV */
-    if (EVP_CIPHER_iv_length(cipher)) {
+    ivlen = EVP_CIPHER_iv_length(cipher);
+    if (ivlen > 0) {
         if (aiv)
-            memcpy(iv, aiv, EVP_CIPHER_iv_length(cipher));
-        else if (RAND_bytes_ex(libctx, iv, EVP_CIPHER_iv_length(cipher),
-                               0) <= 0)
+            memcpy(iv, aiv, ivlen);
+        else if (RAND_bytes_ex(libctx, iv, ivlen, 0) <= 0)
             goto err;
     }
 
diff --git a/crypto/crmf/crmf_pbm.c b/crypto/crmf/crmf_pbm.c
index 21808d0..5641bee 100644
--- a/crypto/crmf/crmf_pbm.c
+++ b/crypto/crmf/crmf_pbm.c
@@ -55,7 +55,7 @@ OSSL_CRMF_PBMPARAMETER *OSSL_CRMF_pbmp_new(OSSL_LIB_CTX *libctx, size_t slen,
      */
     if ((salt = OPENSSL_malloc(slen)) == NULL)
         goto err;
-    if (RAND_bytes_ex(libctx, salt, (int)slen, 0) <= 0) {
+    if (RAND_bytes_ex(libctx, salt, slen, 0) <= 0) {
         ERR_raise(ERR_LIB_CRMF, CRMF_R_FAILURE_OBTAINING_RANDOM);
         goto err;
     }
diff --git a/crypto/ec/ec_err.c b/crypto/ec/ec_err.c
index 9e21968..9dc143c 100644
--- a/crypto/ec/ec_err.c
+++ b/crypto/ec/ec_err.c
@@ -1,6 +1,6 @@
 /*
  * Generated by util/mkerr.pl DO NOT EDIT
- * Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved.
  *
  * Licensed under the Apache License 2.0 (the "License").  You may not use
  * this file except in compliance with the License.  You can obtain a copy
@@ -62,6 +62,7 @@ static const ERR_STRING_DATA EC_str_reasons[] = {
     {ERR_PACK(ERR_LIB_EC, 0, EC_R_INVALID_GENERATOR), "invalid generator"},
     {ERR_PACK(ERR_LIB_EC, 0, EC_R_INVALID_GROUP_ORDER), "invalid group order"},
     {ERR_PACK(ERR_LIB_EC, 0, EC_R_INVALID_KEY), "invalid key"},
+    {ERR_PACK(ERR_LIB_EC, 0, EC_R_INVALID_LENGTH), "invalid length"},
     {ERR_PACK(ERR_LIB_EC, 0, EC_R_INVALID_NAMED_GROUP_CONVERSION),
     "invalid named group conversion"},
     {ERR_PACK(ERR_LIB_EC, 0, EC_R_INVALID_OUTPUT_LENGTH),
diff --git a/crypto/ec/ecp_s390x_nistp.c b/crypto/ec/ecp_s390x_nistp.c
index 4a676c3..5c70b2d 100644
--- a/crypto/ec/ecp_s390x_nistp.c
+++ b/crypto/ec/ecp_s390x_nistp.c
@@ -173,6 +173,10 @@ static ECDSA_SIG *ecdsa_s390x_nistp_sign_sig(const unsigned char *dgst,
     }
 
     if (r == NULL || kinv == NULL) {
+        if (len < 0) {
+            ERR_raise(ERR_LIB_EC, EC_R_INVALID_LENGTH);
+            goto ret;
+        }
         /*
          * Generate random k and copy to param param block. RAND_priv_bytes_ex
          * is used instead of BN_priv_rand_range or BN_generate_dsa_nonce
@@ -180,7 +184,7 @@ static ECDSA_SIG *ecdsa_s390x_nistp_sign_sig(const unsigned char *dgst,
          * internally implementing counter-measures for RNG weakness.
          */
          if (RAND_priv_bytes_ex(eckey->libctx, param + S390X_OFF_RN(len),
-                                len, 0) != 1) {
+                                (size_t)len, 0) != 1) {
              ERR_raise(ERR_LIB_EC, EC_R_RANDOM_NUMBER_GENERATION_FAILED);
              goto ret;
          }
diff --git a/crypto/err/openssl.txt b/crypto/err/openssl.txt
index 48d1175..eb0ace5 100644
--- a/crypto/err/openssl.txt
+++ b/crypto/err/openssl.txt
@@ -555,6 +555,7 @@ EC_R_INVALID_FORM:104:invalid form
 EC_R_INVALID_GENERATOR:173:invalid generator
 EC_R_INVALID_GROUP_ORDER:122:invalid group order
 EC_R_INVALID_KEY:116:invalid key
+EC_R_INVALID_LENGTH:117:invalid length
 EC_R_INVALID_NAMED_GROUP_CONVERSION:174:invalid named group conversion
 EC_R_INVALID_OUTPUT_LENGTH:161:invalid output length
 EC_R_INVALID_P:172:invalid p
@@ -1152,6 +1153,7 @@ RSA_R_INVALID_HEADER:137:invalid header
 RSA_R_INVALID_KEYPAIR:171:invalid keypair
 RSA_R_INVALID_KEY_LENGTH:173:invalid key length
 RSA_R_INVALID_LABEL:160:invalid label
+RSA_R_INVALID_LENGTH:181:invalid length
 RSA_R_INVALID_MESSAGE_LENGTH:131:invalid message length
 RSA_R_INVALID_MGF1_MD:156:invalid mgf1 md
 RSA_R_INVALID_MODULUS:174:invalid modulus
diff --git a/crypto/ffc/ffc_params_generate.c b/crypto/ffc/ffc_params_generate.c
index 3c6f789..85ae524 100644
--- a/crypto/ffc/ffc_params_generate.c
+++ b/crypto/ffc/ffc_params_generate.c
@@ -329,7 +329,7 @@ static int generate_q_fips186_4(BN_CTX *ctx, BIGNUM *q, const EVP_MD *evpmd,
 
         /* A.1.1.2 Step (5) : generate seed with size seed_len */
         if (generate_seed
-                && RAND_bytes_ex(libctx, seed, (int)seedlen, 0) < 0)
+                && RAND_bytes_ex(libctx, seed, seedlen, 0) < 0)
             goto err;
         /*
          * A.1.1.2 Step (6) AND
@@ -399,7 +399,7 @@ static int generate_q_fips186_2(BN_CTX *ctx, BIGNUM *q, const EVP_MD *evpmd,
         if (!BN_GENCB_call(cb, 0, m++))
             goto err;
 
-        if (generate_seed && RAND_bytes_ex(libctx, seed, (int)qsize, 0) <= 0)
+        if (generate_seed && RAND_bytes_ex(libctx, seed, qsize, 0) <= 0)
             goto err;
 
         memcpy(buf, seed, qsize);
diff --git a/crypto/pkcs12/p12_mutl.c b/crypto/pkcs12/p12_mutl.c
index 041711d..be4ed16 100644
--- a/crypto/pkcs12/p12_mutl.c
+++ b/crypto/pkcs12/p12_mutl.c
@@ -259,8 +259,10 @@ int PKCS12_setup_mac(PKCS12 *p12, int iter, unsigned char *salt, int saltlen,
     }
     p12->mac->salt->length = saltlen;
     if (!salt) {
+        if (saltlen < 0)
+            return 0;
         if (RAND_bytes_ex(p12->authsafes->ctx.libctx, p12->mac->salt->data,
-                          saltlen, 0) <= 0)
+                          (size_t)saltlen, 0) <= 0)
             return 0;
     } else
         memcpy(p12->mac->salt->data, salt, saltlen);
diff --git a/crypto/rsa/rsa_err.c b/crypto/rsa/rsa_err.c
index 85bee96..269971c 100644
--- a/crypto/rsa/rsa_err.c
+++ b/crypto/rsa/rsa_err.c
@@ -57,6 +57,7 @@ static const ERR_STRING_DATA RSA_str_reasons[] = {
     {ERR_PACK(ERR_LIB_RSA, 0, RSA_R_INVALID_KEYPAIR), "invalid keypair"},
     {ERR_PACK(ERR_LIB_RSA, 0, RSA_R_INVALID_KEY_LENGTH), "invalid key length"},
     {ERR_PACK(ERR_LIB_RSA, 0, RSA_R_INVALID_LABEL), "invalid label"},
+    {ERR_PACK(ERR_LIB_RSA, 0, RSA_R_INVALID_LENGTH), "invalid length"},
     {ERR_PACK(ERR_LIB_RSA, 0, RSA_R_INVALID_MESSAGE_LENGTH),
     "invalid message length"},
     {ERR_PACK(ERR_LIB_RSA, 0, RSA_R_INVALID_MGF1_MD), "invalid mgf1 md"},
diff --git a/crypto/rsa/rsa_oaep.c b/crypto/rsa/rsa_oaep.c
index 5068057..0064664 100644
--- a/crypto/rsa/rsa_oaep.c
+++ b/crypto/rsa/rsa_oaep.c
@@ -77,6 +77,10 @@ int ossl_rsa_padding_add_PKCS1_OAEP_mgf1_ex(OSSL_LIB_CTX *libctx,
         mgf1md = md;
 
     mdlen = EVP_MD_size(md);
+    if (mdlen <= 0) {
+        ERR_raise(ERR_LIB_RSA, RSA_R_INVALID_LENGTH);
+        return 0;
+    }
 
     /* step 2b: check KLen > nLen - 2 HLen - 2 */
     if (flen > emlen - 2 * mdlen - 1) {
diff --git a/crypto/rsa/rsa_pk1.c b/crypto/rsa/rsa_pk1.c
index 9094b1a..f1eabf1 100644
--- a/crypto/rsa/rsa_pk1.c
+++ b/crypto/rsa/rsa_pk1.c
@@ -128,6 +128,9 @@ int ossl_rsa_padding_add_PKCS1_type_2_ex(OSSL_LIB_CTX *libctx, unsigned char *to
     if (flen > (tlen - RSA_PKCS1_PADDING_SIZE)) {
         ERR_raise(ERR_LIB_RSA, RSA_R_DATA_TOO_LARGE_FOR_KEY_SIZE);
         return 0;
+    } else if (flen < 0) {
+        ERR_raise(ERR_LIB_RSA, RSA_R_INVALID_LENGTH);
+        return 0;
     }
 
     p = (unsigned char *)to;