TLS client: allow cert verify callback return -1 for SSL_ERROR_WANT_RETRY_VERIFY

The client-side cert verification callback function may not only return
as usual for success or 0 for failure, but also -1,
typically on failure verifying the server certificate.
This makes the handshake suspend and return control to the calling application
with SSL_ERROR_WANT_RETRY_VERIFY.
The app can for instance fetch further certificates or cert status information
needed for the verification.
Calling SSL_connect() again resumes the connection attempt
by retrying the server certificate verification step.
This process may even be repeated if need be.

The core implementation of the feature is in ssl/statem/statem_clnt.c,
splitting tls_process_server_certificate() into a preparation step
that just copies the certificates received from the server to s->session->peer_chain
(rather than having them in a local variable at first) and returns to the state machine,
and a post-processing step in tls_post_process_server_certificate() that can be repeated:
Try verifying the current contents of s->session->peer_chain basically as before,
but give the verification callback function the chance to pause connecting and
make the TLS state machine later call tls_post_process_server_certificate() again.
Otherwise processing continues as usual.

The documentation of the new feature is added to SSL_CTX_set_cert_verify_callback.pod
and SSL_want.pod.

This adds two tests:
* A generic test in test/helpers/handshake.c
  on the usability of the new server cert verification retry feature.
  It is triggered via test/ssl-tests/03-custom_verify.cnf.in (while the bulky auto-
  generated changes to test/ssl-tests/03-custom_verify.cnf can be basically ignored).
* A test in test/sslapitest.c that demonstrates the effectiveness of the approach
  for augmenting the cert chain provided by the server in between SSL_connect() calls.

Reviewed-by: Matt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/13906)

2 files changed, 103 insertions, 61 deletions

diff --git a/test/ssl-tests/03-custom_verify.cnf b/test/ssl-tests/03-custom_verify.cnf
index 8dca715..e107b93 100644
--- a/test/ssl-tests/03-custom_verify.cnf
+++ b/test/ssl-tests/03-custom_verify.cnf
@@ -1,16 +1,17 @@
 # Generated with generate_ssl_tests.pl
 
-num_tests = 9
+num_tests = 10
 
 test-0 = 0-verify-success
 test-1 = 1-verify-custom-reject
 test-2 = 2-verify-custom-allow
-test-3 = 3-noverify-success
-test-4 = 4-noverify-ignore-custom-reject
-test-5 = 5-noverify-accept-custom-allow
-test-6 = 6-verify-fail-no-root
-test-7 = 7-verify-custom-success-no-root
-test-8 = 8-verify-custom-fail-no-root
+test-3 = 3-verify-custom-retry
+test-4 = 4-noverify-success
+test-5 = 5-noverify-ignore-custom-reject
+test-6 = 6-noverify-accept-custom-allow
+test-7 = 7-verify-fail-no-root
+test-8 = 8-verify-custom-success-no-root
+test-9 = 9-verify-custom-fail-no-root
 # ===========================================================
 
 [0-verify-success]
@@ -91,148 +92,175 @@ VerifyCallback = AcceptAll
 
 # ===========================================================
 
-[3-noverify-success]
-ssl_conf = 3-noverify-success-ssl
+[3-verify-custom-retry]
+ssl_conf = 3-verify-custom-retry-ssl
 
-[3-noverify-success-ssl]
-server = 3-noverify-success-server
-client = 3-noverify-success-client
+[3-verify-custom-retry-ssl]
+server = 3-verify-custom-retry-server
+client = 3-verify-custom-retry-client
 
-[3-noverify-success-server]
+[3-verify-custom-retry-server]
 Certificate = ${ENV::TEST_CERTS_DIR}/servercert.pem
 CipherString = DEFAULT
 PrivateKey = ${ENV::TEST_CERTS_DIR}/serverkey.pem
 
-[3-noverify-success-client]
+[3-verify-custom-retry-client]
 CipherString = DEFAULT
+VerifyCAFile = ${ENV::TEST_CERTS_DIR}/rootcert.pem
+VerifyMode = Peer
 
 [test-3]
 ExpectedResult = Success
+client = 3-verify-custom-retry-client-extra
+
+[3-verify-custom-retry-client-extra]
+VerifyCallback = RetryOnce
 
 
 # ===========================================================
 
-[4-noverify-ignore-custom-reject]
-ssl_conf = 4-noverify-ignore-custom-reject-ssl
+[4-noverify-success]
+ssl_conf = 4-noverify-success-ssl
 
-[4-noverify-ignore-custom-reject-ssl]
-server = 4-noverify-ignore-custom-reject-server
-client = 4-noverify-ignore-custom-reject-client
+[4-noverify-success-ssl]
+server = 4-noverify-success-server
+client = 4-noverify-success-client
 
-[4-noverify-ignore-custom-reject-server]
+[4-noverify-success-server]
 Certificate = ${ENV::TEST_CERTS_DIR}/servercert.pem
 CipherString = DEFAULT
 PrivateKey = ${ENV::TEST_CERTS_DIR}/serverkey.pem
 
-[4-noverify-ignore-custom-reject-client]
+[4-noverify-success-client]
 CipherString = DEFAULT
 
 [test-4]
 ExpectedResult = Success
-client = 4-noverify-ignore-custom-reject-client-extra
 
-[4-noverify-ignore-custom-reject-client-extra]
+
+# ===========================================================
+
+[5-noverify-ignore-custom-reject]
+ssl_conf = 5-noverify-ignore-custom-reject-ssl
+
+[5-noverify-ignore-custom-reject-ssl]
+server = 5-noverify-ignore-custom-reject-server
+client = 5-noverify-ignore-custom-reject-client
+
+[5-noverify-ignore-custom-reject-server]
+Certificate = ${ENV::TEST_CERTS_DIR}/servercert.pem
+CipherString = DEFAULT
+PrivateKey = ${ENV::TEST_CERTS_DIR}/serverkey.pem
+
+[5-noverify-ignore-custom-reject-client]
+CipherString = DEFAULT
+
+[test-5]
+ExpectedResult = Success
+client = 5-noverify-ignore-custom-reject-client-extra
+
+[5-noverify-ignore-custom-reject-client-extra]
 VerifyCallback = RejectAll
 
 
 # ===========================================================
 
-[5-noverify-accept-custom-allow]
-ssl_conf = 5-noverify-accept-custom-allow-ssl
+[6-noverify-accept-custom-allow]
+ssl_conf = 6-noverify-accept-custom-allow-ssl
 
-[5-noverify-accept-custom-allow-ssl]
-server = 5-noverify-accept-custom-allow-server
-client = 5-noverify-accept-custom-allow-client
+[6-noverify-accept-custom-allow-ssl]
+server = 6-noverify-accept-custom-allow-server
+client = 6-noverify-accept-custom-allow-client
 
-[5-noverify-accept-custom-allow-server]
+[6-noverify-accept-custom-allow-server]
 Certificate = ${ENV::TEST_CERTS_DIR}/servercert.pem
 CipherString = DEFAULT
 PrivateKey = ${ENV::TEST_CERTS_DIR}/serverkey.pem
 
-[5-noverify-accept-custom-allow-client]
+[6-noverify-accept-custom-allow-client]
 CipherString = DEFAULT
 
-[test-5]
+[test-6]
 ExpectedResult = Success
-client = 5-noverify-accept-custom-allow-client-extra
+client = 6-noverify-accept-custom-allow-client-extra
 
-[5-noverify-accept-custom-allow-client-extra]
+[6-noverify-accept-custom-allow-client-extra]
 VerifyCallback = AcceptAll
 
 
 # ===========================================================
 
-[6-verify-fail-no-root]
-ssl_conf = 6-verify-fail-no-root-ssl
+[7-verify-fail-no-root]
+ssl_conf = 7-verify-fail-no-root-ssl
 
-[6-verify-fail-no-root-ssl]
-server = 6-verify-fail-no-root-server
-client = 6-verify-fail-no-root-client
+[7-verify-fail-no-root-ssl]
+server = 7-verify-fail-no-root-server
+client = 7-verify-fail-no-root-client
 
-[6-verify-fail-no-root-server]
+[7-verify-fail-no-root-server]
 Certificate = ${ENV::TEST_CERTS_DIR}/servercert.pem
 CipherString = DEFAULT
 PrivateKey = ${ENV::TEST_CERTS_DIR}/serverkey.pem
 
-[6-verify-fail-no-root-client]
+[7-verify-fail-no-root-client]
 CipherString = DEFAULT
 VerifyMode = Peer
 
-[test-6]
+[test-7]
 ExpectedClientAlert = UnknownCA
 ExpectedResult = ClientFail
 
 
 # ===========================================================
 
-[7-verify-custom-success-no-root]
-ssl_conf = 7-verify-custom-success-no-root-ssl
+[8-verify-custom-success-no-root]
+ssl_conf = 8-verify-custom-success-no-root-ssl
 
-[7-verify-custom-success-no-root-ssl]
-server = 7-verify-custom-success-no-root-server
-client = 7-verify-custom-success-no-root-client
+[8-verify-custom-success-no-root-ssl]
+server = 8-verify-custom-success-no-root-server
+client = 8-verify-custom-success-no-root-client
 
-[7-verify-custom-success-no-root-server]
+[8-verify-custom-success-no-root-server]
 Certificate = ${ENV::TEST_CERTS_DIR}/servercert.pem
 CipherString = DEFAULT
 PrivateKey = ${ENV::TEST_CERTS_DIR}/serverkey.pem
 
-[7-verify-custom-success-no-root-client]
+[8-verify-custom-success-no-root-client]
 CipherString = DEFAULT
 VerifyMode = Peer
 
-[test-7]
+[test-8]
 ExpectedResult = Success
-client = 7-verify-custom-success-no-root-client-extra
+client = 8-verify-custom-success-no-root-client-extra
 
-[7-verify-custom-success-no-root-client-extra]
+[8-verify-custom-success-no-root-client-extra]
 VerifyCallback = AcceptAll
 
 
 # ===========================================================
 
-[8-verify-custom-fail-no-root]
-ssl_conf = 8-verify-custom-fail-no-root-ssl
+[9-verify-custom-fail-no-root]
+ssl_conf = 9-verify-custom-fail-no-root-ssl
 
-[8-verify-custom-fail-no-root-ssl]
-server = 8-verify-custom-fail-no-root-server
-client = 8-verify-custom-fail-no-root-client
+[9-verify-custom-fail-no-root-ssl]
+server = 9-verify-custom-fail-no-root-server
+client = 9-verify-custom-fail-no-root-client
 
-[8-verify-custom-fail-no-root-server]
+[9-verify-custom-fail-no-root-server]
 Certificate = ${ENV::TEST_CERTS_DIR}/servercert.pem
 CipherString = DEFAULT
 PrivateKey = ${ENV::TEST_CERTS_DIR}/serverkey.pem
 
-[8-verify-custom-fail-no-root-client]
+[9-verify-custom-fail-no-root-client]
 CipherString = DEFAULT
 VerifyMode = Peer
 
-[test-8]
+[test-9]
 ExpectedClientAlert = HandshakeFailure
 ExpectedResult = ClientFail
-client = 8-verify-custom-fail-no-root-client-extra
+client = 9-verify-custom-fail-no-root-client-extra
 
-[8-verify-custom-fail-no-root-client-extra]
+[9-verify-custom-fail-no-root-client-extra]
 VerifyCallback = RejectAll
 
 
diff --git a/test/ssl-tests/03-custom_verify.cnf.in b/test/ssl-tests/03-custom_verify.cnf.in
index 28b5721..a6b33ba 100644
--- a/test/ssl-tests/03-custom_verify.cnf.in
+++ b/test/ssl-tests/03-custom_verify.cnf.in
@@ -51,6 +51,20 @@ our @tests = (
         },
     },
 
+    # Same test as above but with a custom callback that requests retry once.
+    {
+        name => "verify-custom-retry",
+        server => { },
+        client => {
+            extra => {
+                "VerifyCallback" => "RetryOnce",
+            },
+        },
+        test   => {
+            "ExpectedResult" => "Success",
+        },
+    },
+
     # Sanity-check that verification indeed succeeds if peer verification
     # is not requested.
     {