RSA: Add a less loaded PSS-parameter structure

RSA_PSS_PARAMS carries with it a lot of baggage in form of X509_ALGOR
and ASN1_INTEGER, which we would rather avoid in our providers.
Therefore, we create a parallell structure - RSA_PSS_PARAMS_30 - that
contains the same information, but uses numeric identities (*) and C
integers (**).  This makes it simpler to handle.

Note that neither this structure nor its contents are passed between
libcrypto and the providers.  Instead, the numeric identities are
translated to and from names, which are then passed over that
boundary.

For future considerations, we might consider dropping RSA_PSS_PARAMS
entirely.  For now, it's still reserved for EVP_PKEY_ASN1_METHOD code,
which RSA_PSS_PARAMS_30 is (almost entirely) reserved for use in our
providers.

(*) We use NIDs in this case, because we already have them and because
only algorithms that libcrypto knows about are permitted in PSS
restrictions.  We could use any number series we want, as long as we
know for sure what they represent.

(**) That's for saltlen and for trailerfield, which are never expect
to surpass the set of numbers that fit in a regular 'int'.

Reviewed-by: Shane Lontis <shane.lontis@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/11710)

2 files changed, 43 insertions, 3 deletions

diff --git a/include/crypto/rsa.h b/include/crypto/rsa.h
index fb7f80da3..69fe4c7 100644
--- a/include/crypto/rsa.h
+++ b/include/crypto/rsa.h
@@ -13,6 +13,37 @@
 #include <openssl/core.h>
 #include <openssl/rsa.h>
 
+typedef struct rsa_pss_params_30_st {
+    int hash_algorithm_nid;
+    struct {
+        int algorithm_nid;       /* Currently always NID_mgf1 */
+        int hash_algorithm_nid;
+    } mask_gen;
+    unsigned int salt_len;
+    unsigned int trailer_field;
+} RSA_PSS_PARAMS_30;
+
+RSA_PSS_PARAMS_30 *rsa_get0_pss_params_30(RSA *r);
+int rsa_pss_params_30_set_defaults(RSA_PSS_PARAMS_30 *rsa_pss_params);
+int rsa_pss_params_30_copy(RSA_PSS_PARAMS_30 *to,
+                           const RSA_PSS_PARAMS_30 *from);
+int rsa_pss_params_30_is_unrestricted(const RSA_PSS_PARAMS_30 *rsa_pss_params);
+int rsa_pss_params_30_set_hashalg(RSA_PSS_PARAMS_30 *rsa_pss_params,
+                                  int hashalg_nid);
+int rsa_pss_params_30_set_maskgenalg(RSA_PSS_PARAMS_30 *rsa_pss_params,
+                                     int maskgenalg_nid);
+int rsa_pss_params_30_set_maskgenhashalg(RSA_PSS_PARAMS_30 *rsa_pss_params,
+                                         int maskgenhashalg_nid);
+int rsa_pss_params_30_set_saltlen(RSA_PSS_PARAMS_30 *rsa_pss_params,
+                                  int saltlen);
+int rsa_pss_params_30_set_trailerfield(RSA_PSS_PARAMS_30 *rsa_pss_params,
+                                       int trailerfield);
+int rsa_pss_params_30_hashalg(const RSA_PSS_PARAMS_30 *rsa_pss_params);
+int rsa_pss_params_30_maskgenalg(const RSA_PSS_PARAMS_30 *rsa_pss_params);
+int rsa_pss_params_30_maskgenhashalg(const RSA_PSS_PARAMS_30 *rsa_pss_params);
+int rsa_pss_params_30_saltlen(const RSA_PSS_PARAMS_30 *rsa_pss_params);
+int rsa_pss_params_30_trailerfield(const RSA_PSS_PARAMS_30 *rsa_pss_params);
+
 const char *rsa_mgf_nid2name(int mgf);
 int rsa_oaeppss_md2nid(const EVP_MD *md);
 const char *rsa_oaeppss_nid2name(int md);
@@ -28,6 +59,10 @@ int rsa_get0_all_params(RSA *r, STACK_OF(BIGNUM_const) *primes,
 
 int rsa_todata(RSA *rsa, OSSL_PARAM_BLD *bld, OSSL_PARAM params[]);
 int rsa_fromdata(RSA *rsa, const OSSL_PARAM params[]);
+int rsa_pss_params_30_todata(const RSA_PSS_PARAMS_30 *pss, const char *propq,
+                             OSSL_PARAM_BLD *bld, OSSL_PARAM params[]);
+int rsa_pss_params_30_fromdata(RSA_PSS_PARAMS_30 *pss_params,
+                               const OSSL_PARAM params[], OPENSSL_CTX *libctx);
 
 int rsa_padding_check_PKCS1_type_2_TLS(OPENSSL_CTX *ctx, unsigned char *to,
                                        size_t tlen, const unsigned char *from,
diff --git a/include/openssl/core_names.h b/include/openssl/core_names.h
index 6e93738..1bd1224 100644
--- a/include/openssl/core_names.h
+++ b/include/openssl/core_names.h
@@ -182,6 +182,7 @@ extern "C" {
 #define OSSL_PKEY_PARAM_DIGEST              OSSL_ALG_PARAM_DIGEST
 #define OSSL_PKEY_PARAM_PROPERTIES          OSSL_ALG_PARAM_PROPERTIES
 #define OSSL_PKEY_PARAM_DIGEST_SIZE         "digest-size"
+#define OSSL_PKEY_PARAM_MASKGENFUNC         "mgf"
 #define OSSL_PKEY_PARAM_MGF1_DIGEST         "mgf1-digest"
 #define OSSL_PKEY_PARAM_MGF1_PROPERTIES     "mgf1-properties"
 
@@ -271,8 +272,12 @@ extern "C" {
 
 
 /* Key generation parameters */
-#define OSSL_PKEY_PARAM_RSA_BITS        OSSL_PKEY_PARAM_BITS
-#define OSSL_PKEY_PARAM_RSA_PRIMES      "primes"
+#define OSSL_PKEY_PARAM_RSA_BITS            OSSL_PKEY_PARAM_BITS
+#define OSSL_PKEY_PARAM_RSA_PRIMES          "primes"
+#define OSSL_PKEY_PARAM_RSA_DIGEST          OSSL_PKEY_PARAM_DIGEST
+#define OSSL_PKEY_PARAM_RSA_MASKGENFUNC     OSSL_PKEY_PARAM_MASKGENFUNC
+#define OSSL_PKEY_PARAM_RSA_MGF1_DIGEST     OSSL_PKEY_PARAM_MGF1_DIGEST
+#define OSSL_PKEY_PARAM_RSA_PSS_SALTLEN     "saltlen"
 
 /* Key generation parameters */
 #define OSSL_PKEY_PARAM_FFC_TYPE         "type"
@@ -310,7 +315,7 @@ extern "C" {
 #define OSSL_SIGNATURE_PARAM_PAD_MODE           OSSL_PKEY_PARAM_PAD_MODE
 #define OSSL_SIGNATURE_PARAM_DIGEST             OSSL_PKEY_PARAM_DIGEST
 #define OSSL_SIGNATURE_PARAM_PROPERTIES         OSSL_PKEY_PARAM_PROPERTIES
-#define OSSL_SIGNATURE_PARAM_PSS_SALTLEN        "pss-saltlen"
+#define OSSL_SIGNATURE_PARAM_PSS_SALTLEN        "saltlen"
 #define OSSL_SIGNATURE_PARAM_MGF1_DIGEST        OSSL_PKEY_PARAM_MGF1_DIGEST
 #define OSSL_SIGNATURE_PARAM_MGF1_PROPERTIES    \
     OSSL_PKEY_PARAM_MGF1_PROPERTIES