Add EVP_PKEY_supports_digest_nid()

Rather than relying only on mandatory default digests, add a way for
the EVP_PKEY to individually report whether each digest algorithm is
supported.

Reviewed-by: Nicola Tuveri <nic.tuv@gmail.com>
Reviewed-by: Richard Levitte <levitte@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/7408)

7 files changed, 109 insertions, 26 deletions

diff --git a/crypto/evp/p_lib.c b/crypto/evp/p_lib.c
index 154ef78..c8f3264 100644
--- a/crypto/evp/p_lib.c
+++ b/crypto/evp/p_lib.c
@@ -667,6 +667,26 @@ int EVP_PKEY_get_default_digest_nid(EVP_PKEY *pkey, int *pnid)
     return evp_pkey_asn1_ctrl(pkey, ASN1_PKEY_CTRL_DEFAULT_MD_NID, 0, pnid);
 }
 
+int EVP_PKEY_supports_digest_nid(EVP_PKEY *pkey, int nid)
+{
+    int rv, default_nid;
+
+    rv = evp_pkey_asn1_ctrl(pkey, ASN1_PKEY_CTRL_SUPPORTS_MD_NID, nid, NULL);
+    if (rv == -2) {
+        /*
+         * If there is a mandatory default digest and this isn't it, then
+         * the answer is 'no'.
+         */
+        rv = EVP_PKEY_get_default_digest_nid(pkey, &default_nid);
+        if (rv == 2)
+            return (nid == default_nid);
+        /* zero is an error from EVP_PKEY_get_default_digest_nid() */
+        if (rv == 0)
+            return -1;
+    }
+    return rv;
+}
+
 int EVP_PKEY_set1_tls_encodedpoint(EVP_PKEY *pkey,
                                const unsigned char *pt, size_t ptlen)
 {
diff --git a/doc/man3/EVP_PKEY_ASN1_METHOD.pod b/doc/man3/EVP_PKEY_ASN1_METHOD.pod
index 3c2ffd9..ed8c24b 100644
--- a/doc/man3/EVP_PKEY_ASN1_METHOD.pod
+++ b/doc/man3/EVP_PKEY_ASN1_METHOD.pod
@@ -257,6 +257,7 @@ L<EVP_PKEY_set_type_str(3)>, and L<EVP_PKEY_assign(3)>.
 
 The pkey_ctrl() method adds extra algorithm specific control.
 It's called by L<EVP_PKEY_get_default_digest_nid(3)>,
+L<EVP_PKEY_supports_digest_nid(3)>,
 L<EVP_PKEY_set1_tls_encodedpoint(3)>,
 L<EVP_PKEY_get1_tls_encodedpoint(3)>, L<PKCS7_SIGNER_INFO_set(3)>,
 L<PKCS7_RECIP_INFO_set(3)>, ...
diff --git a/doc/man3/EVP_PKEY_get_default_digest_nid.pod b/doc/man3/EVP_PKEY_get_default_digest_nid.pod
index da76677..02d25d0 100644
--- a/doc/man3/EVP_PKEY_get_default_digest_nid.pod
+++ b/doc/man3/EVP_PKEY_get_default_digest_nid.pod
@@ -18,7 +18,7 @@ a digest during signing. In this case B<pnid> will be set to NID_undef.
 
 =head1 NOTES
 
-For all current standard OpenSSL public key algorithms SHA1 is returned.
+For all current standard OpenSSL public key algorithms SHA256 is returned.
 
 =head1 RETURN VALUES
 
@@ -32,6 +32,7 @@ public key algorithm.
 
 L<EVP_PKEY_CTX_new(3)>,
 L<EVP_PKEY_sign(3)>,
+L<EVP_PKEY_supports_digest_nid(3)>,
 L<EVP_PKEY_verify(3)>,
 L<EVP_PKEY_verify_recover(3)>,
 
diff --git a/doc/man3/EVP_PKEY_supports_digest_nid.pod b/doc/man3/EVP_PKEY_supports_digest_nid.pod
new file mode 100644
index 0000000..4f0882c
--- /dev/null
+++ b/doc/man3/EVP_PKEY_supports_digest_nid.pod
@@ -0,0 +1,53 @@
+=pod
+
+=head1 NAME
+
+EVP_PKEY_supports_digest_nid - indicate support for signature digest
+
+=head1 SYNOPSIS
+
+ #include <openssl/evp.h>
+ int EVP_PKEY_supports_digest_nid(EVP_PKEY *pkey, int nid);
+
+=head1 DESCRIPTION
+
+The EVP_PKEY_supports_digest_nid() function queries whether the message digest
+NID B<nid> is supported for public key signature operations associated with key
+B<pkey>.
+
+=head1 NOTES
+
+If the EVP_PKEY implementation does not explicitly support this method, but
+L<EVP_PKEY_get_default_digest_nid(3)> returns a mandatory digest result, then
+only that mandatory digest will be supported.
+
+=head1 RETURN VALUES
+
+The EVP_PKEY_supports_digest_nid() function returns 1 if the message digest
+algorithm identified by B<nid> can be used for public key signature operations
+associated with key B<pkey> and 0 if it cannot be used. It returns a negative
+value for failure. In particular a return value of -2 indicates the query
+operation is not supported by the public key algorithm.
+
+=head1 SEE ALSO
+
+L<EVP_PKEY_CTX_new(3)>,
+L<EVP_PKEY_get_default_digest_nid(3)>,
+L<EVP_PKEY_sign(3)>,
+L<EVP_PKEY_verify(3)>,
+L<EVP_PKEY_verify_recover(3)>,
+
+=head1 HISTORY
+
+This function was first added to OpenSSL 1.1.2.
+
+=head1 COPYRIGHT
+
+Copyright 2006-2018 The OpenSSL Project Authors. All Rights Reserved.
+
+Licensed under the OpenSSL license (the "License").  You may not use
+this file except in compliance with the License.  You can obtain a copy
+in the file LICENSE in the source distribution or at
+L<https://www.openssl.org/source/license.html>.
+
+=cut
diff --git a/include/openssl/evp.h b/include/openssl/evp.h
index e803fa8..a0b7a54 100644
--- a/include/openssl/evp.h
+++ b/include/openssl/evp.h
@@ -1111,6 +1111,7 @@ int EVP_PKEY_print_params(BIO *out, const EVP_PKEY *pkey,
                           int indent, ASN1_PCTX *pctx);
 
 int EVP_PKEY_get_default_digest_nid(EVP_PKEY *pkey, int *pnid);
+int EVP_PKEY_supports_digest_nid(EVP_PKEY *pkey, int nid);
 
 int EVP_PKEY_set1_tls_encodedpoint(EVP_PKEY *pkey,
                                    const unsigned char *pt, size_t ptlen);
@@ -1187,6 +1188,7 @@ int EVP_PBE_get(int *ptype, int *ppbe_nid, size_t num);
 
 # define ASN1_PKEY_CTRL_SET1_TLS_ENCPT   0x9
 # define ASN1_PKEY_CTRL_GET1_TLS_ENCPT   0xa
+# define ASN1_PKEY_CTRL_SUPPORTS_MD_NID  0xb
 
 int EVP_PKEY_asn1_get_count(void);
 const EVP_PKEY_ASN1_METHOD *EVP_PKEY_asn1_get0(int idx);
diff --git a/ssl/t1_lib.c b/ssl/t1_lib.c
index 2e785a9..91353e7 100644
--- a/ssl/t1_lib.c
+++ b/ssl/t1_lib.c
@@ -2496,8 +2496,7 @@ static int tls12_get_cert_sigalg_idx(const SSL *s, const SIGALG_LOOKUP *lu)
 static int has_usable_cert(SSL *s, const SIGALG_LOOKUP *sig, int idx)
 {
     const SIGALG_LOOKUP *lu;
-    int mdnid, pknid, default_mdnid;
-    int mandatory_md = 0;
+    int mdnid, pknid, supported;
     size_t i;
 
     /* TLS 1.2 callers can override lu->sig_idx, but not TLS 1.3 callers. */
@@ -2505,39 +2504,45 @@ static int has_usable_cert(SSL *s, const SIGALG_LOOKUP *sig, int idx)
         idx = sig->sig_idx;
     if (!ssl_has_cert(s, idx))
         return 0;
-    /* If the EVP_PKEY reports a mandatory digest, allow nothing else. */
-    ERR_set_mark();
-    switch (EVP_PKEY_get_default_digest_nid(s->cert->pkeys[idx].privatekey,
-                                            &default_mdnid)) {
-    case 2:
-        mandatory_md = 1;
-        break;
-    case 1:
-        break;
-    default: /* If it didn't report a mandatory NID, for whatever reasons,
-              * just clear the error and allow all hashes to be used. */
-        ERR_pop_to_mark();
-    }
     if (s->s3->tmp.peer_cert_sigalgs != NULL) {
         for (i = 0; i < s->s3->tmp.peer_cert_sigalgslen; i++) {
             lu = tls1_lookup_sigalg(s->s3->tmp.peer_cert_sigalgs[i]);
             if (lu == NULL
                 || !X509_get_signature_info(s->cert->pkeys[idx].x509, &mdnid,
                                             &pknid, NULL, NULL)
-                || (mandatory_md && mdnid != default_mdnid))
+                /*
+                 * TODO this does not differentiate between the
+                 * rsa_pss_pss_* and rsa_pss_rsae_* schemes since we do not
+                 * have a chain here that lets us look at the key OID in the
+                 * signing certificate.
+                 */
+                || mdnid != lu->hash
+                || pknid != lu->sig)
                 continue;
-            /*
-             * TODO this does not differentiate between the
-             * rsa_pss_pss_* and rsa_pss_rsae_* schemes since we do not
-             * have a chain here that lets us look at the key OID in the
-             * signing certificate.
-             */
-            if (mdnid == lu->hash && pknid == lu->sig)
-                return 1;
+
+            ERR_set_mark();
+            supported = EVP_PKEY_supports_digest_nid(s->cert->pkeys[idx].privatekey,
+                                                     mdnid);
+            if (supported == 0)
+                continue;
+            else if (supported < 0)
+            {
+                /* If it didn't report a mandatory NID, for whatever reasons,
+                 * just clear the error and allow all hashes to be used. */
+                ERR_pop_to_mark();
+            }
+            return 1;
         }
         return 0;
     }
-    return !mandatory_md || sig->hash == default_mdnid;
+    supported = EVP_PKEY_supports_digest_nid(s->cert->pkeys[idx].privatekey,
+                                             sig->hash);
+    if (supported == 0)
+        return 0;
+    else if (supported < 0)
+        ERR_clear_error();
+
+    return 1;
 }
 
 /*
diff --git a/util/libcrypto.num b/util/libcrypto.num
index f159a40..c6de172 100644
--- a/util/libcrypto.num
+++ b/util/libcrypto.num
@@ -4597,3 +4597,4 @@ EVP_MAC_do_all                          4550	1_1_2	EXIST::FUNCTION:
 EVP_MAC_do_all_sorted                   4551	1_1_2	EXIST::FUNCTION:
 EVP_str2ctrl                            4552	1_1_2	EXIST::FUNCTION:
 EVP_hex2ctrl                            4553	1_1_2	EXIST::FUNCTION:
+EVP_PKEY_supports_digest_nid            4554	1_1_2	EXIST::FUNCTION: