Add fix for RSA keygen in FIPS using keysizes 2048 < bits < 3072

Fixes #11863

Reviewed-by: Matt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/12162)

4 files changed, 42 insertions, 22 deletions

diff --git a/apps/dgst.c b/apps/dgst.c
index da162e6..0654d4c 100644
--- a/apps/dgst.c
+++ b/apps/dgst.c
@@ -293,7 +293,7 @@ int dgst_main(int argc, char **argv)
     if (mac_name != NULL) {
         EVP_PKEY_CTX *mac_ctx = NULL;
         int r = 0;
-        if (!init_gen_str(&mac_ctx, mac_name, impl, 0))
+        if (!init_gen_str(&mac_ctx, mac_name, impl, 0, NULL, NULL))
             goto mac_end;
         if (macopts != NULL) {
             char *macopt;
diff --git a/apps/genpkey.c b/apps/genpkey.c
index 389f0e6..9da5b55 100644
--- a/apps/genpkey.c
+++ b/apps/genpkey.c
@@ -21,13 +21,15 @@
 # include <openssl/engine.h>
 #endif
 
-static int init_keygen_file(EVP_PKEY_CTX **pctx, const char *file, ENGINE *e);
+static int init_keygen_file(EVP_PKEY_CTX **pctx, const char *file, ENGINE *e,
+                            OPENSSL_CTX *libctx, const char *propq);
 static int genpkey_cb(EVP_PKEY_CTX *ctx);
 
 typedef enum OPTION_choice {
     OPT_ERR = -1, OPT_EOF = 0, OPT_HELP,
     OPT_ENGINE, OPT_OUTFORM, OPT_OUT, OPT_PASS, OPT_PARAMFILE,
     OPT_ALGORITHM, OPT_PKEYOPT, OPT_GENPARAM, OPT_TEXT, OPT_CIPHER,
+    OPT_CONFIG,
     OPT_PROV_ENUM
 } OPTION_CHOICE;
 
@@ -41,6 +43,7 @@ const OPTIONS genpkey_options[] = {
     {"algorithm", OPT_ALGORITHM, 's', "The public key algorithm"},
     {"pkeyopt", OPT_PKEYOPT, 's',
      "Set the public key algorithm option as opt:value"},
+     OPT_CONFIG_OPTION,
 
     OPT_SECTION("Output"),
     {"out", OPT_OUT, '>', "Output file"},
@@ -60,6 +63,7 @@ const OPTIONS genpkey_options[] = {
 
 int genpkey_main(int argc, char **argv)
 {
+    CONF *conf = NULL;
     BIO *in = NULL, *out = NULL;
     ENGINE *e = NULL;
     EVP_PKEY *pkey = NULL;
@@ -69,6 +73,8 @@ int genpkey_main(int argc, char **argv)
     OPTION_CHOICE o;
     int outformat = FORMAT_PEM, text = 0, ret = 1, rv, do_param = 0;
     int private = 0;
+    OPENSSL_CTX *libctx = app_get0_libctx();
+    const char *propq = app_get0_propq();
 
     prog = opt_init(argc, argv, genpkey_options);
     while ((o = opt_next()) != OPT_EOF) {
@@ -98,11 +104,11 @@ int genpkey_main(int argc, char **argv)
         case OPT_PARAMFILE:
             if (do_param == 1)
                 goto opthelp;
-            if (!init_keygen_file(&ctx, opt_arg(), e))
+            if (!init_keygen_file(&ctx, opt_arg(), e, libctx, propq))
                 goto end;
             break;
         case OPT_ALGORITHM:
-            if (!init_gen_str(&ctx, opt_arg(), e, do_param))
+            if (!init_gen_str(&ctx, opt_arg(), e, do_param, libctx, propq))
                 goto end;
             break;
         case OPT_PKEYOPT:
@@ -138,6 +144,11 @@ int genpkey_main(int argc, char **argv)
                 goto end;
             }
             break;
+        case OPT_CONFIG:
+            conf = app_load_config_modules(opt_arg());
+            if (conf == NULL)
+                goto end;
+            break;
         case OPT_PROV_CASES:
             if (!opt_provider(o))
                 goto end;
@@ -220,10 +231,12 @@ int genpkey_main(int argc, char **argv)
     BIO_free(in);
     release_engine(e);
     OPENSSL_free(pass);
+    NCONF_free(conf);
     return ret;
 }
 
-static int init_keygen_file(EVP_PKEY_CTX **pctx, const char *file, ENGINE *e)
+static int init_keygen_file(EVP_PKEY_CTX **pctx, const char *file, ENGINE *e,
+                            OPENSSL_CTX *libctx, const char *propq)
 {
     BIO *pbio;
     EVP_PKEY *pkey = NULL;
@@ -247,7 +260,10 @@ static int init_keygen_file(EVP_PKEY_CTX **pctx, const char *file, ENGINE *e)
         return 0;
     }
 
-    ctx = EVP_PKEY_CTX_new(pkey, e);
+    if (e != NULL)
+        ctx = EVP_PKEY_CTX_new(pkey, e);
+    else
+        ctx = EVP_PKEY_CTX_new_from_pkey(libctx, pkey, propq);
     if (ctx == NULL)
         goto err;
     if (EVP_PKEY_keygen_init(ctx) <= 0)
@@ -266,7 +282,8 @@ static int init_keygen_file(EVP_PKEY_CTX **pctx, const char *file, ENGINE *e)
 }
 
 int init_gen_str(EVP_PKEY_CTX **pctx,
-                 const char *algname, ENGINE *e, int do_param)
+                 const char *algname, ENGINE *e, int do_param,
+                 OPENSSL_CTX *libctx, const char *propq)
 {
     EVP_PKEY_CTX *ctx = NULL;
     const EVP_PKEY_ASN1_METHOD *ameth;
@@ -278,25 +295,27 @@ int init_gen_str(EVP_PKEY_CTX **pctx,
         return 0;
     }
 
-    ameth = EVP_PKEY_asn1_find_str(&tmpeng, algname, -1);
+    if (libctx == NULL || e != NULL) {
+        ameth = EVP_PKEY_asn1_find_str(&tmpeng, algname, -1);
 
 #if !defined(OPENSSL_NO_ENGINE) && !defined(OPENSSL_NO_DEPRECATED_3_0)
-    if (!ameth && e)
-        ameth = ENGINE_get_pkey_asn1_meth_str(e, algname, -1);
+        if (ameth == NULL && e != NULL)
+            ameth = ENGINE_get_pkey_asn1_meth_str(e, algname, -1);
 #endif
+        if (ameth == NULL) {
+            BIO_printf(bio_err, "Algorithm %s not found\n", algname);
+            return 0;
+        }
+        ERR_clear_error();
 
-    if (!ameth) {
-        BIO_printf(bio_err, "Algorithm %s not found\n", algname);
-        return 0;
-    }
-
-    ERR_clear_error();
-
-    EVP_PKEY_asn1_get0_info(&pkey_id, NULL, NULL, NULL, NULL, ameth);
+        EVP_PKEY_asn1_get0_info(&pkey_id, NULL, NULL, NULL, NULL, ameth);
 #if !defined(OPENSSL_NO_ENGINE) && !defined(OPENSSL_NO_DEPRECATED_3_0)
-    ENGINE_finish(tmpeng);
+        ENGINE_finish(tmpeng);
 #endif
-    ctx = EVP_PKEY_CTX_new_id(pkey_id, e);
+        ctx = EVP_PKEY_CTX_new_id(pkey_id, e);
+    } else {
+        ctx = EVP_PKEY_CTX_new_from_name(libctx, algname, propq);
+    }
 
     if (!ctx)
         goto err;
diff --git a/apps/genrsa.c b/apps/genrsa.c
index 9a91301..4f589e9 100644
--- a/apps/genrsa.c
+++ b/apps/genrsa.c
@@ -169,7 +169,7 @@ opthelp:
     if (out == NULL)
         goto end;
 
-    if (!init_gen_str(&ctx, "RSA", eng, 0))
+    if (!init_gen_str(&ctx, "RSA", eng, 0, NULL, NULL))
         goto end;
 
     EVP_PKEY_CTX_set_cb(ctx, genrsa_cb);
diff --git a/apps/include/apps.h b/apps/include/apps.h
index 554d33e..0ee8e07 100644
--- a/apps/include/apps.h
+++ b/apps/include/apps.h
@@ -209,7 +209,8 @@ int pkey_ctrl_string(EVP_PKEY_CTX *ctx, const char *value);
 int x509_ctrl_string(X509 *x, const char *value);
 int x509_req_ctrl_string(X509_REQ *x, const char *value);
 int init_gen_str(EVP_PKEY_CTX **pctx,
-                 const char *algname, ENGINE *e, int do_param);
+                 const char *algname, ENGINE *e, int do_param,
+                 OPENSSL_CTX *libctx, const char *propq);
 int do_X509_sign(X509 *x, EVP_PKEY *pkey, const EVP_MD *md,
                  STACK_OF(OPENSSL_STRING) *sigopts);
 int do_X509_verify(X509 *x, EVP_PKEY *pkey, STACK_OF(OPENSSL_STRING) *vfyopts);