Further libssl size_t-ify of reading

Writing still to be done

Reviewed-by: Rich Salz <rsalz@openssl.org>

6 files changed, 44 insertions, 51 deletions

diff --git a/ssl/statem/statem.c b/ssl/statem/statem.c
index 786eb24..901a3f2 100644
--- a/ssl/statem/statem.c
+++ b/ssl/statem/statem.c
@@ -490,12 +490,12 @@ static SUB_STATE_RETURN read_state_machine(SSL *s)
 {
     OSSL_STATEM *st = &s->statem;
     int ret, mt;
-    unsigned long len = 0;
+    size_t len = 0;
     int (*transition) (SSL *s, int mt);
     PACKET pkt;
     MSG_PROCESS_RETURN(*process_message) (SSL *s, PACKET *pkt);
     WORK_STATE(*post_process_message) (SSL *s, WORK_STATE wst);
-    unsigned long (*max_message_size) (SSL *s);
+    size_t (*max_message_size) (SSL *s);
     void (*cb) (const SSL *ssl, int type, int val) = NULL;
 
     cb = get_callback(s);
diff --git a/ssl/statem/statem_clnt.c b/ssl/statem/statem_clnt.c
index 51513d5..5704654 100644
--- a/ssl/statem/statem_clnt.c
+++ b/ssl/statem/statem_clnt.c
@@ -568,7 +568,7 @@ int ossl_statem_client_construct_message(SSL *s, WPACKET *pkt,
  * Returns the maximum allowed length for the current message that we are
  * reading. Excludes the message header.
  */
-unsigned long ossl_statem_client_max_message_size(SSL *s)
+size_t ossl_statem_client_max_message_size(SSL *s)
 {
     OSSL_STATEM *st = &s->statem;
 
diff --git a/ssl/statem/statem_dtls.c b/ssl/statem/statem_dtls.c
index 5b90c56..8281188 100644
--- a/ssl/statem/statem_dtls.c
+++ b/ssl/statem/statem_dtls.c
@@ -124,7 +124,7 @@ int dtls1_do_write(SSL *s, int type)
 
     if (s->init_off == 0 && type == SSL3_RT_HANDSHAKE)
         OPENSSL_assert(s->init_num ==
-                       (int)s->d1->w_msg_hdr.msg_len + DTLS1_HM_HEADER_LENGTH);
+                       s->d1->w_msg_hdr.msg_len + DTLS1_HM_HEADER_LENGTH);
 
     if (s->write_hash) {
         if (s->enc_write_ctx
@@ -295,7 +295,7 @@ int dtls1_do_write(SSL *s, int type)
                     return -1;
             }
 
-            if (ret == s->init_num) {
+            if (ret == (int)s->init_num) {
                 if (s->msg_callback)
                     s->msg_callback(1, s->version, type, s->init_buf->data,
                                     (size_t)(s->init_off + s->init_num), s,
@@ -323,7 +323,7 @@ int dtls1_do_write(SSL *s, int type)
     return (0);
 }
 
-int dtls_get_message(SSL *s, int *mt, unsigned long *len)
+int dtls_get_message(SSL *s, int *mt, size_t *len)
 {
     struct hm_header_st *msg_hdr;
     unsigned char *p;
@@ -516,6 +516,7 @@ dtls1_reassemble_fragment(SSL *s, const struct hm_header_st *msg_hdr, int *ok)
     int i = -1, is_complete;
     unsigned char seq64be[8];
     unsigned long frag_len = msg_hdr->frag_len;
+    size_t read;
 
     if ((msg_hdr->frag_off + frag_len) > msg_hdr->msg_len ||
         msg_hdr->msg_len > dtls1_max_handshake_message_len(s))
@@ -559,10 +560,10 @@ dtls1_reassemble_fragment(SSL *s, const struct hm_header_st *msg_hdr, int *ok)
                                           devnull,
                                           frag_len >
                                           sizeof(devnull) ? sizeof(devnull) :
-                                          frag_len, 0);
+                                          frag_len, 0, &read);
             if (i <= 0)
                 goto err;
-            frag_len -= i;
+            frag_len -= read;
         }
         return DTLS1_HM_FRAGMENT_RETRY;
     }
@@ -570,8 +571,8 @@ dtls1_reassemble_fragment(SSL *s, const struct hm_header_st *msg_hdr, int *ok)
     /* read the body of the fragment (header has already been read */
     i = s->method->ssl_read_bytes(s, SSL3_RT_HANDSHAKE, NULL,
                                   frag->fragment + msg_hdr->frag_off,
-                                  frag_len, 0);
-    if ((unsigned long)i != frag_len)
+                                  frag_len, 0, &read);
+    if (i <= 0 || read != frag_len)
         i = -1;
     if (i <= 0)
         goto err;
@@ -622,6 +623,7 @@ dtls1_process_out_of_seq_message(SSL *s, const struct hm_header_st *msg_hdr,
     pitem *item = NULL;
     unsigned char seq64be[8];
     unsigned long frag_len = msg_hdr->frag_len;
+    size_t read;
 
     if ((msg_hdr->frag_off + frag_len) > msg_hdr->msg_len)
         goto err;
@@ -654,10 +656,10 @@ dtls1_process_out_of_seq_message(SSL *s, const struct hm_header_st *msg_hdr,
                                           devnull,
                                           frag_len >
                                           sizeof(devnull) ? sizeof(devnull) :
-                                          frag_len, 0);
+                                          frag_len, 0, &read);
             if (i <= 0)
                 goto err;
-            frag_len -= i;
+            frag_len -= read;
         }
     } else {
         if (frag_len != msg_hdr->msg_len)
@@ -677,8 +679,8 @@ dtls1_process_out_of_seq_message(SSL *s, const struct hm_header_st *msg_hdr,
              * read the body of the fragment (header has already been read
              */
             i = s->method->ssl_read_bytes(s, SSL3_RT_HANDSHAKE, NULL,
-                                          frag->fragment, frag_len, 0);
-            if ((unsigned long)i != frag_len)
+                                          frag->fragment, frag_len, 0, &read);
+            if (i<=0 || read != frag_len)
                 i = -1;
             if (i <= 0)
                 goto err;
@@ -716,6 +718,7 @@ static int dtls_get_reassembled_message(SSL *s, long *len)
     int i, al, recvd_type;
     struct hm_header_st msg_hdr;
     int ok;
+    size_t read;
 
  redo:
     /* see if we have the required fragment already */
@@ -728,7 +731,7 @@ static int dtls_get_reassembled_message(SSL *s, long *len)
 
     /* read handshake message header */
     i = s->method->ssl_read_bytes(s, SSL3_RT_HANDSHAKE, &recvd_type, wire,
-                                  DTLS1_HM_HEADER_LENGTH, 0);
+                                  DTLS1_HM_HEADER_LENGTH, 0, &read);
     if (i <= 0) {               /* nbio, or an error */
         s->rwstate = SSL_READING;
         *len = i;
@@ -742,17 +745,17 @@ static int dtls_get_reassembled_message(SSL *s, long *len)
             goto f_err;
         }
 
-        memcpy(s->init_buf->data, wire, i);
-        s->init_num = i - 1;
+        memcpy(s->init_buf->data, wire, read);
+        s->init_num = read - 1;
         s->init_msg = s->init_buf->data + 1;
         s->s3->tmp.message_type = SSL3_MT_CHANGE_CIPHER_SPEC;
-        s->s3->tmp.message_size = i - 1;
-        *len = i - 1;
+        s->s3->tmp.message_size = read - 1;
+        *len = read - 1;
         return 1;
     }
 
     /* Handshake fails if message header is incomplete */
-    if (i != DTLS1_HM_HEADER_LENGTH) {
+    if (read != DTLS1_HM_HEADER_LENGTH) {
         al = SSL_AD_UNEXPECTED_MESSAGE;
         SSLerr(SSL_F_DTLS_GET_REASSEMBLED_MESSAGE, SSL_R_UNEXPECTED_MESSAGE);
         goto f_err;
@@ -823,7 +826,7 @@ static int dtls_get_reassembled_message(SSL *s, long *len)
             (unsigned char *)s->init_buf->data + DTLS1_HM_HEADER_LENGTH;
 
         i = s->method->ssl_read_bytes(s, SSL3_RT_HANDSHAKE, NULL,
-                                      &p[frag_off], frag_len, 0);
+                                      &p[frag_off], frag_len, 0, &read);
 
         /*
          * This shouldn't ever fail due to NBIO because we already checked
@@ -835,13 +838,13 @@ static int dtls_get_reassembled_message(SSL *s, long *len)
             return 0;
         }
     } else
-        i = 0;
+        read = 0;
 
     /*
      * XDTLS: an incorrectly formatted fragment should cause the handshake
      * to fail
      */
-    if (i != (int)frag_len) {
+    if (read != frag_len) {
         al = SSL3_AD_ILLEGAL_PARAMETER;
         SSLerr(SSL_F_DTLS_GET_REASSEMBLED_MESSAGE, SSL3_AD_ILLEGAL_PARAMETER);
         goto f_err;
diff --git a/ssl/statem/statem_lib.c b/ssl/statem/statem_lib.c
index a3d8d1e..e7ea4c6 100644
--- a/ssl/statem/statem_lib.c
+++ b/ssl/statem/statem_lib.c
@@ -45,7 +45,7 @@ int ssl3_do_write(SSL *s, int type)
                              ret))
             return -1;
 
-    if (ret == s->init_num) {
+    if (ret == (int)s->init_num) {
         if (s->msg_callback)
             s->msg_callback(1, s->version, type, s->init_buf->data,
                             (size_t)(s->init_off + s->init_num), s,
@@ -357,7 +357,7 @@ int tls_get_message_header(SSL *s, int *mt)
     /* s->init_num < SSL3_HM_HEADER_LENGTH */
     int skip_message, i, recvd_type, al;
     unsigned char *p;
-    unsigned long l;
+    size_t l, read;
 
     p = (unsigned char *)s->init_buf->data;
 
@@ -366,7 +366,7 @@ int tls_get_message_header(SSL *s, int *mt)
             i = s->method->ssl_read_bytes(s, SSL3_RT_HANDSHAKE, &recvd_type,
                                           &p[s->init_num],
                                           SSL3_HM_HEADER_LENGTH - s->init_num,
-                                          0);
+                                          0, &read);
             if (i <= 0) {
                 s->rwstate = SSL_READING;
                 return 0;
@@ -376,22 +376,22 @@ int tls_get_message_header(SSL *s, int *mt)
                  * A ChangeCipherSpec must be a single byte and may not occur
                  * in the middle of a handshake message.
                  */
-                if (s->init_num != 0 || i != 1 || p[0] != SSL3_MT_CCS) {
+                if (s->init_num != 0 || read != 1 || p[0] != SSL3_MT_CCS) {
                     al = SSL_AD_UNEXPECTED_MESSAGE;
                     SSLerr(SSL_F_TLS_GET_MESSAGE_HEADER,
                            SSL_R_BAD_CHANGE_CIPHER_SPEC);
                     goto f_err;
                 }
                 s->s3->tmp.message_type = *mt = SSL3_MT_CHANGE_CIPHER_SPEC;
-                s->init_num = i - 1;
-                s->s3->tmp.message_size = i;
+                s->init_num = read - 1;
+                s->s3->tmp.message_size = read;
                 return 1;
             } else if (recvd_type != SSL3_RT_HANDSHAKE) {
                 al = SSL_AD_UNEXPECTED_MESSAGE;
                 SSLerr(SSL_F_TLS_GET_MESSAGE_HEADER, SSL_R_CCS_RECEIVED_EARLY);
                 goto f_err;
             }
-            s->init_num += i;
+            s->init_num += read;
         }
 
         skip_message = 0;
@@ -452,9 +452,9 @@ int tls_get_message_header(SSL *s, int *mt)
     return 0;
 }
 
-int tls_get_message_body(SSL *s, unsigned long *len)
+int tls_get_message_body(SSL *s, size_t *len)
 {
-    long n;
+    size_t n, read;
     unsigned char *p;
     int i;
 
@@ -468,14 +468,14 @@ int tls_get_message_body(SSL *s, unsigned long *len)
     n = s->s3->tmp.message_size - s->init_num;
     while (n > 0) {
         i = s->method->ssl_read_bytes(s, SSL3_RT_HANDSHAKE, NULL,
-                                      &p[s->init_num], n, 0);
+                                      &p[s->init_num], n, 0, &read);
         if (i <= 0) {
             s->rwstate = SSL_READING;
             *len = 0;
             return 0;
         }
-        s->init_num += i;
-        n -= i;
+        s->init_num += read;
+        n -= read;
     }
 
 #ifndef OPENSSL_NO_NEXTPROTONEG
@@ -513,17 +513,7 @@ int tls_get_message_body(SSL *s, unsigned long *len)
                             s->msg_callback_arg);
     }
 
-    /*
-     * init_num should never be negative...should probably be declared
-     * unsigned
-     */
-    if (s->init_num < 0) {
-        SSLerr(SSL_F_TLS_GET_MESSAGE_BODY, ERR_R_INTERNAL_ERROR);
-        ssl3_send_alert(s, SSL3_AL_FATAL, SSL_AD_INTERNAL_ERROR);
-        *len = 0;
-        return 0;
-    }
-    *len = (unsigned long)s->init_num;
+    *len = s->init_num;
     return 1;
 }
 
diff --git a/ssl/statem/statem_locl.h b/ssl/statem/statem_locl.h
index 6b57b25..a360fc9 100644
--- a/ssl/statem/statem_locl.h
+++ b/ssl/statem/statem_locl.h
@@ -54,7 +54,7 @@ WORK_STATE ossl_statem_client_pre_work(SSL *s, WORK_STATE wst);
 WORK_STATE ossl_statem_client_post_work(SSL *s, WORK_STATE wst);
 int ossl_statem_client_construct_message(SSL *s, WPACKET *pkt,
                                          confunc_f *confunc, int *mt);
-unsigned long ossl_statem_client_max_message_size(SSL *s);
+size_t ossl_statem_client_max_message_size(SSL *s);
 MSG_PROCESS_RETURN ossl_statem_client_process_message(SSL *s, PACKET *pkt);
 WORK_STATE ossl_statem_client_post_process_message(SSL *s, WORK_STATE wst);
 
@@ -67,14 +67,14 @@ WORK_STATE ossl_statem_server_pre_work(SSL *s, WORK_STATE wst);
 WORK_STATE ossl_statem_server_post_work(SSL *s, WORK_STATE wst);
 int ossl_statem_server_construct_message(SSL *s, WPACKET *pkt,
                                          confunc_f *confunc,int *mt);
-unsigned long ossl_statem_server_max_message_size(SSL *s);
+size_t ossl_statem_server_max_message_size(SSL *s);
 MSG_PROCESS_RETURN ossl_statem_server_process_message(SSL *s, PACKET *pkt);
 WORK_STATE ossl_statem_server_post_process_message(SSL *s, WORK_STATE wst);
 
 /* Functions for getting new message data */
 __owur int tls_get_message_header(SSL *s, int *mt);
-__owur int tls_get_message_body(SSL *s, unsigned long *len);
-__owur int dtls_get_message(SSL *s, int *mt, unsigned long *len);
+__owur int tls_get_message_body(SSL *s, size_t *len);
+__owur int dtls_get_message(SSL *s, int *mt, size_t *len);
 
 /* Message construction and processing functions */
 __owur MSG_PROCESS_RETURN tls_process_change_cipher_spec(SSL *s, PACKET *pkt);
diff --git a/ssl/statem/statem_srvr.c b/ssl/statem/statem_srvr.c
index 009d06c..a243161 100644
--- a/ssl/statem/statem_srvr.c
+++ b/ssl/statem/statem_srvr.c
@@ -717,7 +717,7 @@ int ossl_statem_server_construct_message(SSL *s, WPACKET *pkt,
  * Returns the maximum allowed length for the current message that we are
  * reading. Excludes the message header.
  */
-unsigned long ossl_statem_server_max_message_size(SSL *s)
+size_t ossl_statem_server_max_message_size(SSL *s)
 {
     OSSL_STATEM *st = &s->statem;