Fix PKINIT CMS error checking for older OpenSSL

Commit 70f61d417261ca17efe3d60d180033bea2da60b0 updated the
CMS_verify() error code checks, using two error codes new to OpenSSL
3.0 (RSA_R_DIGEST_NOT_ALLOWED and CMS_R_UNKNOWN_DIGEST_ALGORITHM).
This change broke the build for OpenSSL 1.0 and 1.1.

Instead of looking for codes indicating an algorithm issue and
assuming that everything else is an invalid signature, check for the
code indicating an invalid signature and assume that everything else
is an algorithm issue.

(cherry picked from commit e48e2e56a05a47fd932a941ac82c1131ceed47d0)

ticket: 9069
version_fixed: 1.20.2

1 files changed, 3 insertions, 11 deletions

diff --git a/src/plugins/preauth/pkinit/pkinit_crypto_openssl.c b/src/plugins/preauth/pkinit/pkinit_crypto_openssl.c
index 4f29510..a3afbc5 100644
--- a/src/plugins/preauth/pkinit/pkinit_crypto_openssl.c
+++ b/src/plugins/preauth/pkinit/pkinit_crypto_openssl.c
@@ -2102,18 +2102,10 @@ cms_signeddata_verify(krb5_context context,
             goto cleanup;
         out = BIO_new(BIO_s_mem());
         if (CMS_verify(cms, NULL, store, NULL, out, flags) == 0) {
-            unsigned long err = ERR_peek_last_error();
-            switch(ERR_GET_REASON(err)) {
-            case RSA_R_DIGEST_NOT_ALLOWED:
-            case CMS_R_UNKNOWN_DIGEST_ALGORITHM:
-            case CMS_R_NO_MATCHING_DIGEST:
-            case CMS_R_NO_MATCHING_SIGNATURE:
-                retval = KRB5KDC_ERR_DIGEST_IN_SIGNED_DATA_NOT_ACCEPTED;
-                break;
-            case CMS_R_VERIFICATION_FAILURE:
-            default:
+            if (ERR_peek_last_error() == CMS_R_VERIFICATION_FAILURE)
                 retval = KRB5KDC_ERR_INVALID_SIG;
-            }
+            else
+                retval = KRB5KDC_ERR_DIGEST_IN_SIGNED_DATA_NOT_ACCEPTED;
             (void)oerr(context, retval, _("Failed to verify CMS message"));
             goto cleanup;
         }