Convert record layer to use size_t

Reviewed-by: Rich Salz <rsalz@openssl.org>

4 files changed, 55 insertions, 42 deletions

diff --git a/ssl/record/rec_layer_s3.c b/ssl/record/rec_layer_s3.c
index 28de7c3..001fcc6 100644
--- a/ssl/record/rec_layer_s3.c
+++ b/ssl/record/rec_layer_s3.c
@@ -179,11 +179,12 @@ const char *SSL_rstate_string(const SSL *s)
 
 /*
  * Return values are as per SSL_read(), i.e.
- * >0 The number of read bytes
+ *  1 Success
  *  0 Failure (not retryable)
  * <0 Failure (may be retryable)
  */
-int ssl3_read_n(SSL *s, int n, int max, int extend, int clearold)
+int ssl3_read_n(SSL *s, size_t n, size_t max, int extend, int clearold,
+                size_t *read)
 {
     /*
      * If extend == 0, obtain new n-byte packet; if extend == 1, increase
@@ -194,13 +195,12 @@ int ssl3_read_n(SSL *s, int n, int max, int extend, int clearold)
      * if clearold == 1, move the packet to the start of the buffer; if
      * clearold == 0 then leave any old packets where they were
      */
-    int i, len, left;
-    size_t align = 0;
+    size_t len, left, align = 0;
     unsigned char *pkt;
     SSL3_BUFFER *rb;
 
-    if (n <= 0)
-        return n;
+    if (n == 0)
+        return 0;
 
     rb = &s->rlayer.rbuf;
     if (rb->buf == NULL)
@@ -270,12 +270,13 @@ int ssl3_read_n(SSL *s, int n, int max, int extend, int clearold)
         s->rlayer.packet_length += n;
         rb->left = left - n;
         rb->offset += n;
-        return (n);
+        *read = n;
+        return 1;
     }
 
     /* else we need to read more data */
 
-    if (n > (int)(rb->len - rb->offset)) { /* does not happen */
+    if (n > (rb->len - rb->offset)) { /* does not happen */
         SSLerr(SSL_F_SSL3_READ_N, ERR_R_INTERNAL_ERROR);
         return -1;
     }
@@ -287,11 +288,14 @@ int ssl3_read_n(SSL *s, int n, int max, int extend, int clearold)
     else {
         if (max < n)
             max = n;
-        if (max > (int)(rb->len - rb->offset))
+        if (max > (rb->len - rb->offset))
             max = rb->len - rb->offset;
     }
 
     while (left < n) {
+        size_t bioread;
+        int ret;
+
         /*
          * Now we have len+left bytes at the front of s->s3->rbuf.buf and
          * need to read in more until we have len+n (up to len+max if
@@ -301,20 +305,23 @@ int ssl3_read_n(SSL *s, int n, int max, int extend, int clearold)
         clear_sys_error();
         if (s->rbio != NULL) {
             s->rwstate = SSL_READING;
-            i = BIO_read(s->rbio, pkt + len + left, max - left);
+            /* TODO(size_t): Convert this function */
+            ret = BIO_read(s->rbio, pkt + len + left, max - left);
+            if (ret >= 0)
+                bioread = ret;
         } else {
             SSLerr(SSL_F_SSL3_READ_N, SSL_R_READ_BIO_NOT_SET);
-            i = -1;
+            ret = -1;
         }
 
-        if (i <= 0) {
+        if (ret <= 0) {
             rb->left = left;
             if (s->mode & SSL_MODE_RELEASE_BUFFERS && !SSL_IS_DTLS(s))
                 if (len + left == 0)
                     ssl3_release_read_buffer(s);
             return -1;
         }
-        left += i;
+        left += bioread;
         /*
          * reads should *never* span multiple packets for DTLS because the
          * underlying transport protocol is message oriented as opposed to
@@ -331,7 +338,8 @@ int ssl3_read_n(SSL *s, int n, int max, int extend, int clearold)
     rb->left = left - n;
     s->rlayer.packet_length += n;
     s->rwstate = SSL_NOTHING;
-    return (n);
+    *read = n;
+    return 1;
 }
 
 /*
@@ -411,7 +419,7 @@ int ssl3_write_bytes(SSL *s, int type, const void *buf_, int len)
         EVP_CIPH_FLAG_TLS1_1_MULTIBLOCK) {
         unsigned char aad[13];
         EVP_CTRL_TLS1_1_MULTIBLOCK_PARAM mb_param;
-        int packlen;
+        size_t packlen;
 
         /* minimize address aliasing conflicts */
         if ((max_send_fragment & 0xfff) == 0)
@@ -470,11 +478,12 @@ int ssl3_write_bytes(SSL *s, int type, const void *buf_, int len)
             mb_param.inp = aad;
             mb_param.len = nw;
 
+            /* TODO: CHECK ME - CAN THIS EVER BE NEGATIVE???? */
             packlen = EVP_CIPHER_CTX_ctrl(s->enc_write_ctx,
                                           EVP_CTRL_TLS1_1_MULTIBLOCK_AAD,
                                           sizeof(mb_param), &mb_param);
 
-            if (packlen <= 0 || packlen > (int)wb->len) { /* never happens */
+            if (packlen <= 0 || packlen > wb->len) { /* never happens */
                 /* free jumbo buffer */
                 ssl3_release_write_buffer(s);
                 break;
@@ -921,7 +930,7 @@ int ssl3_write_pending(SSL *s, int type, const unsigned char *buf,
             SSLerr(SSL_F_SSL3_WRITE_PENDING, SSL_R_BIO_NOT_SET);
             i = -1;
         }
-        if (i == SSL3_BUFFER_get_left(&wb[currbuf])) {
+        if (i == (int)SSL3_BUFFER_get_left(&wb[currbuf])) {
             SSL3_BUFFER_set_left(&wb[currbuf], 0);
             SSL3_BUFFER_add_offset(&wb[currbuf], i);
             if (currbuf + 1 < s->rlayer.numwpipes)
diff --git a/ssl/record/record.h b/ssl/record/record.h
index 3e1530f..2ed8ab9 100644
--- a/ssl/record/record.h
+++ b/ssl/record/record.h
@@ -22,9 +22,9 @@ typedef struct ssl3_buffer_st {
     /* buffer size */
     size_t len;
     /* where to 'copy from' */
-    int offset;
+    size_t offset;
     /* how many bytes left */
-    int left;
+    size_t left;
 } SSL3_BUFFER;
 
 #define SEQ_NUM_SIZE                            8
@@ -154,7 +154,7 @@ typedef struct record_layer_st {
     SSL3_RECORD rrec[SSL_MAX_PIPELINES];
     /* used internally to point at a raw packet */
     unsigned char *packet;
-    unsigned int packet_length;
+    size_t packet_length;
     /* number of bytes sent so far */
     unsigned int wnum;
     /*
diff --git a/ssl/record/record_locl.h b/ssl/record/record_locl.h
index b69afd8..2bb0738 100644
--- a/ssl/record/record_locl.h
+++ b/ssl/record/record_locl.h
@@ -38,7 +38,8 @@
 #define RECORD_LAYER_clear_first_record(rl)     ((rl)->is_first_record = 0)
 #define DTLS_RECORD_LAYER_get_r_epoch(rl)       ((rl)->d->r_epoch)
 
-__owur int ssl3_read_n(SSL *s, int n, int max, int extend, int clearold);
+__owur int ssl3_read_n(SSL *s, size_t n, size_t max, int extend, int clearold,
+                       size_t *read);
 
 void RECORD_LAYER_set_write_sequence(RECORD_LAYER *rl, const unsigned char *ws);
 DTLS1_BITMAP *dtls1_get_bitmap(SSL *s, SSL3_RECORD *rr,
diff --git a/ssl/record/ssl3_record.c b/ssl/record/ssl3_record.c
index 3236166..94f140f 100644
--- a/ssl/record/ssl3_record.c
+++ b/ssl/record/ssl3_record.c
@@ -68,7 +68,7 @@ void SSL3_RECORD_set_seq_num(SSL3_RECORD *r, const unsigned char *seq_num)
 static int ssl3_record_app_data_waiting(SSL *s)
 {
     SSL3_BUFFER *rbuf;
-    int left, len;
+    size_t left, len;
     unsigned char *p;
 
     rbuf = RECORD_LAYER_get_rbuf(&s->rlayer);
@@ -125,7 +125,9 @@ static int ssl3_record_app_data_waiting(SSL *s)
 int ssl3_get_record(SSL *s)
 {
     int ssl_major, ssl_minor, al;
-    int enc_err, n, i, ret = -1;
+    int enc_err, rret, ret = -1;
+    int i;
+    size_t more, n;
     SSL3_RECORD *rr;
     SSL3_BUFFER *rbuf;
     SSL_SESSION *sess;
@@ -149,11 +151,11 @@ int ssl3_get_record(SSL *s)
         if ((RECORD_LAYER_get_rstate(&s->rlayer) != SSL_ST_READ_BODY) ||
             (RECORD_LAYER_get_packet_length(&s->rlayer)
              < SSL3_RT_HEADER_LENGTH)) {
-            n = ssl3_read_n(s, SSL3_RT_HEADER_LENGTH,
-                            SSL3_BUFFER_get_len(rbuf), 0,
-                            num_recs == 0 ? 1 : 0);
-            if (n <= 0)
-                return (n);     /* error or non-blocking */
+            rret = ssl3_read_n(s, SSL3_RT_HEADER_LENGTH,
+                               SSL3_BUFFER_get_len(rbuf), 0,
+                               num_recs == 0 ? 1 : 0, &n);
+            if (rret <= 0)
+                return rret;     /* error or non-blocking */
             RECORD_LAYER_set_rstate(&s->rlayer, SSL_ST_READ_BODY);
 
             p = RECORD_LAYER_get_packet(&s->rlayer);
@@ -274,17 +276,17 @@ int ssl3_get_record(SSL *s)
          * record
          */
         if (rr[num_recs].rec_version == SSL2_VERSION) {
-            i = rr[num_recs].length + SSL2_RT_HEADER_LENGTH
+            more = rr[num_recs].length + SSL2_RT_HEADER_LENGTH
                 - SSL3_RT_HEADER_LENGTH;
         } else {
-            i = rr[num_recs].length;
+            more = rr[num_recs].length;
         }
-        if (i > 0) {
+        if (more > 0) {
             /* now s->packet_length == SSL3_RT_HEADER_LENGTH */
 
-            n = ssl3_read_n(s, i, i, 1, 0);
-            if (n <= 0)
-                return (n);     /* error or non-blocking io */
+            rret = ssl3_read_n(s, more, more, 1, 0, &n);
+            if (rret <= 0)
+                return rret;     /* error or non-blocking io */
         }
 
         /* set state for later operations */
@@ -1482,7 +1484,8 @@ int dtls1_process_record(SSL *s, DTLS1_BITMAP *bitmap)
 int dtls1_get_record(SSL *s)
 {
     int ssl_major, ssl_minor;
-    int i, n;
+    int rret;
+    size_t more, n;
     SSL3_RECORD *rr;
     unsigned char *p = NULL;
     unsigned short version;
@@ -1508,11 +1511,11 @@ int dtls1_get_record(SSL *s)
     /* check if we have the header */
     if ((RECORD_LAYER_get_rstate(&s->rlayer) != SSL_ST_READ_BODY) ||
         (RECORD_LAYER_get_packet_length(&s->rlayer) < DTLS1_RT_HEADER_LENGTH)) {
-        n = ssl3_read_n(s, DTLS1_RT_HEADER_LENGTH,
-                        SSL3_BUFFER_get_len(&s->rlayer.rbuf), 0, 1);
+        rret = ssl3_read_n(s, DTLS1_RT_HEADER_LENGTH,
+                           SSL3_BUFFER_get_len(&s->rlayer.rbuf), 0, 1, &n);
         /* read timeout is handled by dtls1_read_bytes */
-        if (n <= 0)
-            return (n);         /* error or non-blocking */
+        if (rret <= 0)
+            return rret;         /* error or non-blocking */
 
         /* this packet contained a partial record, dump it */
         if (RECORD_LAYER_get_packet_length(&s->rlayer) !=
@@ -1575,10 +1578,10 @@ int dtls1_get_record(SSL *s)
     if (rr->length >
         RECORD_LAYER_get_packet_length(&s->rlayer) - DTLS1_RT_HEADER_LENGTH) {
         /* now s->packet_length == DTLS1_RT_HEADER_LENGTH */
-        i = rr->length;
-        n = ssl3_read_n(s, i, i, 1, 1);
+        more = rr->length;
+        rret = ssl3_read_n(s, more, more, 1, 1, &n);
         /* this packet contained a partial record, dump it */
-        if (n != i) {
+        if (rret <= 0 || n != more) {
             rr->length = 0;
             RECORD_LAYER_reset_packet_length(&s->rlayer);
             goto again;