Generalize the HTTP client so far implemented mostly in crypto/ocsp/ocsp_ht.c

The new client has become an independent libcrpyto module in crypto/http/ and
* can handle any types of requests and responses (ASN.1-encoded and plain)
* does not include potentially busy loops when waiting for responses but
* makes use of a new timeout mechanism integrated with socket-based BIO
* supports the use of HTTP proxies and TLS, including HTTPS over proxies
* supports HTTP redirection via codes 301 and 302 for GET requests
* returns more useful diagnostics in various error situations
Also adapts - and strongly simplifies - hitherto uses of HTTP in crypto/ocsp/,
crypto/x509/x_all.c, apps/lib/apps.c, and apps/{ocsp,s_client,s_server}.c

Reviewed-by: Matt Caswell <matt@openssl.org>
Reviewed-by: David von Oheimb <david.von.oheimb@siemens.com>
(Merged from https://github.com/openssl/openssl/pull/10667)

9 files changed, 165 insertions, 30 deletions

diff --git a/include/openssl/cmp.h b/include/openssl/cmp.h
index d778842..7876324 100644
--- a/include/openssl/cmp.h
+++ b/include/openssl/cmp.h
@@ -18,6 +18,7 @@
 #  include <openssl/crmf.h>
 #  include <openssl/cmperr.h>
 #  include <openssl/cmp_util.h>
+#  include <openssl/http.h>
 
 /* explicit #includes not strictly needed since implied by the above: */
 #  include <openssl/types.h>
@@ -274,14 +275,11 @@ int OSSL_CMP_CTX_set_serverPort(OSSL_CMP_CTX *ctx, int port);
 int OSSL_CMP_CTX_set1_proxyName(OSSL_CMP_CTX *ctx, const char *name);
 int OSSL_CMP_CTX_set_proxyPort(OSSL_CMP_CTX *ctx, int port);
 #  define OSSL_CMP_DEFAULT_PORT 80
-typedef BIO *(*OSSL_cmp_http_cb_t) (OSSL_CMP_CTX *ctx, BIO *hbio,
-                                    unsigned long detail);
-int OSSL_CMP_CTX_set_http_cb(OSSL_CMP_CTX *ctx, OSSL_cmp_http_cb_t cb);
+int OSSL_CMP_CTX_set_http_cb(OSSL_CMP_CTX *ctx, OSSL_HTTP_bio_cb_t cb);
 int OSSL_CMP_CTX_set_http_cb_arg(OSSL_CMP_CTX *ctx, void *arg);
 void *OSSL_CMP_CTX_get_http_cb_arg(const OSSL_CMP_CTX *ctx);
-typedef int (*OSSL_cmp_transfer_cb_t) (OSSL_CMP_CTX *ctx,
-                                       const OSSL_CMP_MSG *req,
-                                       OSSL_CMP_MSG **res);
+typedef OSSL_CMP_MSG *(*OSSL_cmp_transfer_cb_t) (OSSL_CMP_CTX *ctx,
+                                                 const OSSL_CMP_MSG *req);
 int OSSL_CMP_CTX_set_transfer_cb(OSSL_CMP_CTX *ctx, OSSL_cmp_transfer_cb_t cb);
 int OSSL_CMP_CTX_set_transfer_cb_arg(OSSL_CMP_CTX *ctx, void *arg);
 void *OSSL_CMP_CTX_get_transfer_cb_arg(const OSSL_CMP_CTX *ctx);
diff --git a/include/openssl/cmperr.h b/include/openssl/cmperr.h
index b21db04..cd962fb 100644
--- a/include/openssl/cmperr.h
+++ b/include/openssl/cmperr.h
@@ -63,6 +63,7 @@ int ERR_load_CMP_strings(void);
 #  define CMP_R_NO_STDIO                                   194
 #  define CMP_R_NULL_ARGUMENT                              103
 #  define CMP_R_PKISTATUSINFO_NOT_FOUND                    132
+#  define CMP_R_POTENTIALLY_INVALID_CERTIFICATE            139
 #  define CMP_R_UNEXPECTED_PKIBODY                         133
 #  define CMP_R_UNKNOWN_ALGORITHM_ID                       134
 #  define CMP_R_UNKNOWN_CERT_TYPE                          135
diff --git a/include/openssl/err.h b/include/openssl/err.h
index 96b6088..17a248c 100644
--- a/include/openssl/err.h
+++ b/include/openssl/err.h
@@ -113,6 +113,7 @@ struct err_state_st {
 # define ERR_LIB_PROV            57
 # define ERR_LIB_CMP             58
 # define ERR_LIB_OSSL_SERIALIZER 59
+# define ERR_LIB_HTTP            60
 
 # define ERR_LIB_USER            128
 
@@ -140,6 +141,7 @@ struct err_state_st {
 #  define EVPerr(f, r) ERR_raise_data(ERR_LIB_EVP, (r), NULL)
 #  define FIPSerr(f, r) ERR_raise_data(ERR_LIB_FIPS, (r), NULL)
 #  define HMACerr(f, r) ERR_raise_data(ERR_LIB_HMAC, (r), NULL)
+#  define HTTPerr(f, r) ERR_raise_data(ERR_LIB_HTTP, (r), NULL)
 #  define KDFerr(f, r) ERR_raise_data(ERR_LIB_KDF, (r), NULL)
 #  define OBJerr(f, r) ERR_raise_data(ERR_LIB_OBJ, (r), NULL)
 #  define OCSPerr(f, r) ERR_raise_data(ERR_LIB_OCSP, (r), NULL)
diff --git a/include/openssl/http.h b/include/openssl/http.h
new file mode 100644
index 0000000..4201d98
--- /dev/null
+++ b/include/openssl/http.h
@@ -0,0 +1,72 @@
+/*
+ * Copyright 2000-2020 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright Siemens AG 2018-2020
+ *
+ * Licensed under the Apache License 2.0 (the "License").  You may not use
+ * this file except in compliance with the License.  You can obtain a copy
+ * in the file LICENSE in the source distribution or at
+ * https://www.openssl.org/source/license.html
+ */
+
+#ifndef OPENSSL_HTTP_H
+# define OPENSSL_HTTP_H
+# pragma once
+
+# include <openssl/opensslconf.h>
+
+# include <openssl/bio.h>
+# include <openssl/asn1.h>
+# include <openssl/conf.h>
+
+
+# ifdef __cplusplus
+extern "C" {
+# endif
+
+typedef BIO *(*OSSL_HTTP_bio_cb_t)(BIO *bio, void *arg, int connect, int detail);
+# ifndef OPENSSL_NO_SOCK
+BIO *OSSL_HTTP_get(const char *url, const char *proxy, const char *proxy_port,
+                   BIO *bio, BIO *rbio,
+                   OSSL_HTTP_bio_cb_t bio_update_fn, void *arg,
+                   const STACK_OF(CONF_VALUE) *headers,
+                   int maxline, unsigned long max_resp_len, int timeout,
+                   const char *expected_content_type, int expect_asn1);
+ASN1_VALUE *OSSL_HTTP_get_asn1(const char *url,
+                               const char *proxy, const char *proxy_port,
+                               BIO *bio, BIO *rbio,
+                               OSSL_HTTP_bio_cb_t bio_update_fn, void *arg,
+                               const STACK_OF(CONF_VALUE) *headers,
+                               int maxline, unsigned long max_resp_len,
+                               int timeout, const char *expected_content_type,
+                               const ASN1_ITEM *it);
+ASN1_VALUE *OSSL_HTTP_post_asn1(const char *server, const char *port,
+                                const char *path, int use_ssl,
+                                const char *proxy, const char *proxy_port,
+                                BIO *bio, BIO *rbio,
+                                OSSL_HTTP_bio_cb_t bio_update_fn, void *arg,
+                                const STACK_OF(CONF_VALUE) *headers,
+                                const char *content_type,
+                                ASN1_VALUE *req, const ASN1_ITEM *req_it,
+                                int maxline, unsigned long max_resp_len,
+                                int timeout, const char *expected_ct,
+                                const ASN1_ITEM *rsp_it);
+BIO *OSSL_HTTP_transfer(const char *server, const char *port, const char *path,
+                        int use_ssl, const char *proxy, const char *proxy_port,
+                        BIO *bio, BIO *rbio,
+                        OSSL_HTTP_bio_cb_t bio_update_fn, void *arg,
+                        const STACK_OF(CONF_VALUE) *headers,
+                        const char *content_type, BIO *req_mem,
+                        int maxline, unsigned long max_resp_len, int timeout,
+                        const char *expected_ct, int expect_asn1,
+                        char **redirection_url);
+int OSSL_HTTP_proxy_connect(BIO *bio, const char *server, const char *port,
+                            const char *proxyuser, const char *proxypass,
+                            int timeout, BIO *bio_err, const char *prog);
+# endif
+int OSSL_HTTP_parse_url(const char *url, char **phost, char **pport,
+                        char **ppath, int *pssl);
+
+# ifdef  __cplusplus
+}
+# endif
+#endif /* !defined OPENSSL_HTTP_H */
diff --git a/include/openssl/httperr.h b/include/openssl/httperr.h
new file mode 100644
index 0000000..36dd7cb
--- /dev/null
+++ b/include/openssl/httperr.h
@@ -0,0 +1,55 @@
+/*
+ * Generated by util/mkerr.pl DO NOT EDIT
+ * Copyright 1995-2019 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Licensed under the Apache License 2.0 (the "License").  You may not use
+ * this file except in compliance with the License.  You can obtain a copy
+ * in the file LICENSE in the source distribution or at
+ * https://www.openssl.org/source/license.html
+ */
+
+#ifndef OPENSSL_HTTPERR_H
+# define OPENSSL_HTTPERR_H
+
+# include <openssl/opensslconf.h>
+# include <openssl/symhacks.h>
+
+
+# ifdef  __cplusplus
+extern "C"
+# endif
+int ERR_load_HTTP_strings(void);
+
+/*
+ * HTTP function codes.
+ */
+# ifndef OPENSSL_NO_DEPRECATED_3_0
+# endif
+
+/*
+ * HTTP reason codes.
+ */
+# define HTTP_R_ASN1_LEN_EXCEEDS_MAX_RESP_LEN             108
+# define HTTP_R_CONNECT_FAILURE                           100
+# define HTTP_R_ERROR_PARSING_ASN1_LENGTH                 109
+# define HTTP_R_ERROR_PARSING_CONTENT_LENGTH              119
+# define HTTP_R_ERROR_PARSING_URL                         101
+# define HTTP_R_ERROR_RECEIVING                           103
+# define HTTP_R_ERROR_SENDING                             102
+# define HTTP_R_INCONSISTENT_CONTENT_LENGTH               120
+# define HTTP_R_MAX_RESP_LEN_EXCEEDED                     117
+# define HTTP_R_MISSING_ASN1_ENCODING                     110
+# define HTTP_R_MISSING_CONTENT_TYPE                      121
+# define HTTP_R_MISSING_REDIRECT_LOCATION                 111
+# define HTTP_R_REDIRECTION_FROM_HTTPS_TO_HTTP            112
+# define HTTP_R_REDIRECTION_NOT_ENABLED                   116
+# define HTTP_R_RESPONSE_LINE_TOO_LONG                    113
+# define HTTP_R_SERVER_RESPONSE_PARSE_ERROR               104
+# define HTTP_R_SERVER_SENT_ERROR                         105
+# define HTTP_R_SERVER_SENT_WRONG_HTTP_VERSION            106
+# define HTTP_R_STATUS_CODE_UNSUPPORTED                   114
+# define HTTP_R_TLS_NOT_ENABLED                           107
+# define HTTP_R_TOO_MANY_REDIRECTIONS                     115
+# define HTTP_R_UNEXPECTED_CONTENT_TYPE                   118
+
+#endif
diff --git a/include/openssl/ocsp.h b/include/openssl/ocsp.h
index b9dc988..b15d747 100644
--- a/include/openssl/ocsp.h
+++ b/include/openssl/ocsp.h
@@ -1,5 +1,5 @@
 /*
- * Copyright 2000-2018 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright 2000-2020 The OpenSSL Project Authors. All Rights Reserved.
  *
  * Licensed under the Apache License 2.0 (the "License").  You may not use
  * this file except in compliance with the License.  You can obtain a copy
@@ -162,20 +162,32 @@ DECLARE_ASN1_DUP_FUNCTION(OCSP_CERTID)
 OCSP_RESPONSE *OCSP_sendreq_bio(BIO *b, const char *path, OCSP_REQUEST *req);
 OCSP_REQ_CTX *OCSP_sendreq_new(BIO *io, const char *path, OCSP_REQUEST *req,
                                int maxline);
-int OCSP_REQ_CTX_nbio(OCSP_REQ_CTX *rctx);
+#  ifndef OPENSSL_NO_SOCK
 int OCSP_sendreq_nbio(OCSP_RESPONSE **presp, OCSP_REQ_CTX *rctx);
-OCSP_REQ_CTX *OCSP_REQ_CTX_new(BIO *io, int maxline);
+#  endif
+/* The following functions are used only internally */
+int OCSP_REQ_CTX_nbio(OCSP_REQ_CTX *rctx);
+OCSP_REQ_CTX *OCSP_REQ_CTX_new(BIO *wbio, BIO *rbio,
+                               int method_GET, int maxline,
+                               unsigned long max_resp_len, int timeout,
+                               const char *expected_content_type,
+                               int expect_asn1);
 void OCSP_REQ_CTX_free(OCSP_REQ_CTX *rctx);
 void OCSP_set_max_response_length(OCSP_REQ_CTX *rctx, unsigned long len);
-int OCSP_REQ_CTX_i2d(OCSP_REQ_CTX *rctx, const ASN1_ITEM *it,
-                     ASN1_VALUE *val);
-int OCSP_REQ_CTX_nbio_d2i(OCSP_REQ_CTX *rctx, ASN1_VALUE **pval,
-                          const ASN1_ITEM *it);
+int OCSP_REQ_CTX_i2d(OCSP_REQ_CTX *rctx, const char *content_type,
+                     const ASN1_ITEM *it, ASN1_VALUE *req);
+#  ifndef OPENSSL_NO_SOCK
+ASN1_VALUE *OCSP_REQ_CTX_nbio_d2i(OCSP_REQ_CTX *rctx, const ASN1_ITEM *it);
+#  endif
 BIO *OCSP_REQ_CTX_get0_mem_bio(OCSP_REQ_CTX *rctx);
-int OCSP_REQ_CTX_http(OCSP_REQ_CTX *rctx, const char *op, const char *path);
-int OCSP_REQ_CTX_set1_req(OCSP_REQ_CTX *rctx, OCSP_REQUEST *req);
+int OCSP_REQ_CTX_http(OCSP_REQ_CTX *rctx,
+                      const char *server, const char *port, const char *path);
 int OCSP_REQ_CTX_add1_header(OCSP_REQ_CTX *rctx,
                              const char *name, const char *value);
+/* End of functions used only internally */
+
+/* TODO: remove this (documented but) meanwhile obsolete function? */
+int OCSP_REQ_CTX_set1_req(OCSP_REQ_CTX *rctx, const OCSP_REQUEST *req);
 
 OCSP_CERTID *OCSP_cert_to_id(const EVP_MD *dgst, const X509 *subject,
                              const X509 *issuer);
@@ -237,8 +249,7 @@ int OCSP_check_validity(ASN1_GENERALIZEDTIME *thisupd,
 int OCSP_request_verify(OCSP_REQUEST *req, STACK_OF(X509) *certs,
                         X509_STORE *store, unsigned long flags);
 
-int OCSP_parse_url(const char *url, char **phost, char **pport, char **ppath,
-                   int *pssl);
+#  define OCSP_parse_url OSSL_HTTP_parse_url /* for backward compatibility */
 
 int OCSP_id_issuer_cmp(const OCSP_CERTID *a, const OCSP_CERTID *b);
 int OCSP_id_cmp(const OCSP_CERTID *a, const OCSP_CERTID *b);
@@ -359,5 +370,5 @@ int OCSP_basic_verify(OCSP_BASICRESP *bs, STACK_OF(X509) *certs,
 #  ifdef  __cplusplus
 }
 #  endif
-# endif
+# endif /* !defined OPENSSL_NO_OCSP */
 #endif
diff --git a/include/openssl/ocsperr.h b/include/openssl/ocsperr.h
index 57f0a90..7e3fd0f 100644
--- a/include/openssl/ocsperr.h
+++ b/include/openssl/ocsperr.h
@@ -45,11 +45,9 @@ int ERR_load_OCSP_strings(void);
 #   define OCSP_F_OCSP_CHECK_ISSUER                         0
 #   define OCSP_F_OCSP_CHECK_VALIDITY                       0
 #   define OCSP_F_OCSP_MATCH_ISSUERID                       0
-#   define OCSP_F_OCSP_PARSE_URL                            0
 #   define OCSP_F_OCSP_REQUEST_SIGN                         0
 #   define OCSP_F_OCSP_REQUEST_VERIFY                       0
 #   define OCSP_F_OCSP_RESPONSE_GET1_BASIC                  0
-#   define OCSP_F_PARSE_HTTP_LINE1                          0
 # endif
 
 /*
@@ -59,7 +57,6 @@ int ERR_load_OCSP_strings(void);
 #  define OCSP_R_DIGEST_ERR                                102
 #  define OCSP_R_ERROR_IN_NEXTUPDATE_FIELD                 122
 #  define OCSP_R_ERROR_IN_THISUPDATE_FIELD                 123
-#  define OCSP_R_ERROR_PARSING_URL                         121
 #  define OCSP_R_MISSING_OCSPSIGNING_USAGE                 103
 #  define OCSP_R_NEXTUPDATE_BEFORE_THISUPDATE              124
 #  define OCSP_R_NOT_BASIC_RESPONSE                        104
@@ -71,8 +68,6 @@ int ERR_load_OCSP_strings(void);
 #  define OCSP_R_REQUEST_NOT_SIGNED                        128
 #  define OCSP_R_RESPONSE_CONTAINS_NO_REVOCATION_DATA      111
 #  define OCSP_R_ROOT_CA_NOT_TRUSTED                       112
-#  define OCSP_R_SERVER_RESPONSE_ERROR                     114
-#  define OCSP_R_SERVER_RESPONSE_PARSE_ERROR               115
 #  define OCSP_R_SIGNATURE_FAILURE                         117
 #  define OCSP_R_SIGNER_CERTIFICATE_NOT_FOUND              118
 #  define OCSP_R_STATUS_EXPIRED                            125
diff --git a/include/openssl/types.h b/include/openssl/types.h
index a497637..5761afc 100644
--- a/include/openssl/types.h
+++ b/include/openssl/types.h
@@ -184,7 +184,7 @@ typedef struct NAME_CONSTRAINTS_st NAME_CONSTRAINTS;
 
 typedef struct crypto_ex_data_st CRYPTO_EX_DATA;
 
-typedef struct ocsp_req_ctx_st OCSP_REQ_CTX;
+typedef struct ossl_http_req_ctx_st OCSP_REQ_CTX; /* backward compatibility */
 typedef struct ocsp_response_st OCSP_RESPONSE;
 typedef struct ocsp_responder_id_st OCSP_RESPID;
 
diff --git a/include/openssl/x509.h b/include/openssl/x509.h
index 5e553ef..2b4d14e 100644
--- a/include/openssl/x509.h
+++ b/include/openssl/x509.h
@@ -372,16 +372,10 @@ int X509_signature_print(BIO *bp, const X509_ALGOR *alg,
 
 int X509_sign(X509 *x, EVP_PKEY *pkey, const EVP_MD *md);
 int X509_sign_ctx(X509 *x, EVP_MD_CTX *ctx);
-# ifndef OPENSSL_NO_OCSP
-int X509_http_nbio(OCSP_REQ_CTX *rctx, X509 **pcert);
-# endif
 int X509_REQ_sign(X509_REQ *x, EVP_PKEY *pkey, const EVP_MD *md);
 int X509_REQ_sign_ctx(X509_REQ *x, EVP_MD_CTX *ctx);
 int X509_CRL_sign(X509_CRL *x, EVP_PKEY *pkey, const EVP_MD *md);
 int X509_CRL_sign_ctx(X509_CRL *x, EVP_MD_CTX *ctx);
-# ifndef OPENSSL_NO_OCSP
-int X509_CRL_http_nbio(OCSP_REQ_CTX *rctx, X509_CRL **pcrl);
-# endif
 int NETSCAPE_SPKI_sign(NETSCAPE_SPKI *x, EVP_PKEY *pkey, const EVP_MD *md);
 
 int X509_pubkey_digest(const X509 *data, const EVP_MD *type,
@@ -395,6 +389,13 @@ int X509_REQ_digest(const X509_REQ *data, const EVP_MD *type,
 int X509_NAME_digest(const X509_NAME *data, const EVP_MD *type,
                      unsigned char *md, unsigned int *len);
 
+# if !defined(OPENSSL_NO_SOCK)
+X509 *X509_load_http(const char *url, BIO *bio, BIO *rbio, int timeout);
+#  define X509_http_nbio(url) X509_load_http(url, NULL, NULL, 0)
+X509_CRL *X509_CRL_load_http(const char *url, BIO *bio, BIO *rbio, int timeout);
+#  define X509_CRL_http_nbio(url) X509_CRL_load_http(url, NULL,  NULL, 0)
+# endif
+
 # ifndef OPENSSL_NO_STDIO
 X509 *d2i_X509_fp(FILE *fp, X509 **x509);
 int i2d_X509_fp(FILE *fp, const X509 *x509);