Speed up the per-block loops of AES, DES3, and DES IOV encryption by

avoiding function calls and copies in the case where the next block
is wholly contained within the current buffer.  To do this, introduce
two new inline functions in aead.h called iov_next_block and
iov_store_block.

git-svn-id: svn://anonsvn.mit.edu/krb5/trunk@23430 dc483132-0cff-0310-8789-dd5450dbe970

5 files changed, 160 insertions, 197 deletions

diff --git a/src/lib/crypto/builtin/des/d3_aead.c b/src/lib/crypto/builtin/des/d3_aead.c
index e018895..549a27f 100644
--- a/src/lib/crypto/builtin/des/d3_aead.c
+++ b/src/lib/crypto/builtin/des/d3_aead.c
@@ -37,71 +37,55 @@ krb5int_des3_cbc_encrypt_iov(krb5_crypto_iov *data,
     unsigned DES_INT32 left, right;
     const unsigned DES_INT32 *kp1, *kp2, *kp3;
     const unsigned char *ip;
-    unsigned char *op;
     struct iov_block_state input_pos, output_pos;
-    unsigned char iblock[MIT_DES_BLOCK_LENGTH];
-    unsigned char oblock[MIT_DES_BLOCK_LENGTH];
+    unsigned char storage[MIT_DES_BLOCK_LENGTH], *block = NULL, *ptr;
 
     IOV_BLOCK_STATE_INIT(&input_pos);
     IOV_BLOCK_STATE_INIT(&output_pos);
 
-    /*
-     * Get key pointer here.  This won't need to be reinitialized
-     */
+    /* Get key pointers here.  These won't need to be reinitialized. */
     kp1 = (const unsigned DES_INT32 *)ks1;
     kp2 = (const unsigned DES_INT32 *)ks2;
     kp3 = (const unsigned DES_INT32 *)ks3;
 
-    /*
-     * Initialize left and right with the contents of the initial
-     * vector.
-     */
-    if (ivec != NULL)
-        ip = ivec;
-    else
-        ip = mit_des_zeroblock;
+    /* Initialize left and right with the contents of the initial vector. */
+    ip = (ivec != NULL) ? ivec : mit_des_zeroblock;
     GET_HALF_BLOCK(left, ip);
     GET_HALF_BLOCK(right, ip);
 
-    /*
-     * Suitably initialized, now work the length down 8 bytes
-     * at a time.
-     */
+    /* Work the length down 8 bytes at a time. */
     for (;;) {
         unsigned DES_INT32 temp;
 
-        ip = iblock;
-        op = oblock;
-
-        if (!krb5int_c_iov_get_block(iblock, MIT_DES_BLOCK_LENGTH, data, num_data, &input_pos))
-            break;
-
-        if (input_pos.iov_pos == num_data)
+        ptr = iov_next_block(storage, MIT_DES_BLOCK_LENGTH, data, num_data,
+                             &input_pos);
+        if (ptr == NULL)
             break;
+        block = ptr;
 
-        GET_HALF_BLOCK(temp, ip);
+        /* Decompose this block and xor it with the previous ciphertext. */
+        GET_HALF_BLOCK(temp, ptr);
         left  ^= temp;
-        GET_HALF_BLOCK(temp, ip);
+        GET_HALF_BLOCK(temp, ptr);
         right ^= temp;
 
-        /*
-         * Encrypt what we have
-         */
+        /* Encrypt what we have and store it back into block. */
         DES_DO_ENCRYPT(left, right, kp1);
         DES_DO_DECRYPT(left, right, kp2);
         DES_DO_ENCRYPT(left, right, kp3);
+        ptr = block;
+        PUT_HALF_BLOCK(left, ptr);
+        PUT_HALF_BLOCK(right, ptr);
 
-        /*
-         * Copy the results out
-         */
-        PUT_HALF_BLOCK(left, op);
-        PUT_HALF_BLOCK(right, op);
-
-        krb5int_c_iov_put_block(data, num_data, oblock, MIT_DES_BLOCK_LENGTH, &output_pos);
+        iov_store_block(data, num_data, block, storage, MIT_DES_BLOCK_LENGTH,
+                        &output_pos);
     }
 
-    if (ivec != NULL)
-        memcpy(ivec, oblock, MIT_DES_BLOCK_LENGTH);
+    if (ivec != NULL && block != NULL) {
+        ptr = ivec;
+        PUT_HALF_BLOCK(left, ptr);
+        PUT_HALF_BLOCK(right, ptr);
+    }
 }
 
 void
@@ -117,17 +101,13 @@ krb5int_des3_cbc_decrypt_iov(krb5_crypto_iov *data,
     const unsigned char *ip;
     unsigned DES_INT32 ocipherl, ocipherr;
     unsigned DES_INT32 cipherl, cipherr;
-    unsigned char *op;
     struct iov_block_state input_pos, output_pos;
-    unsigned char iblock[MIT_DES_BLOCK_LENGTH];
-    unsigned char oblock[MIT_DES_BLOCK_LENGTH];
+    unsigned char storage[MIT_DES_BLOCK_LENGTH], *block = NULL, *ptr;
 
     IOV_BLOCK_STATE_INIT(&input_pos);
     IOV_BLOCK_STATE_INIT(&output_pos);
 
-    /*
-     * Get key pointer here.  This won't need to be reinitialized
-     */
+    /* Get key pointers here.  These won't need to be reinitialized. */
     kp1 = (const unsigned DES_INT32 *)ks1;
     kp2 = (const unsigned DES_INT32 *)ks2;
     kp3 = (const unsigned DES_INT32 *)ks3;
@@ -138,71 +118,48 @@ krb5int_des3_cbc_decrypt_iov(krb5_crypto_iov *data,
      * Should think about this a little more...
      */
 
-    if (num_data == 0)
-        return;
-
-    /*
-     * Prime the old cipher with ivec.
-     */
-    if (ivec != NULL)
-        ip = ivec;
-    else
-        ip = mit_des_zeroblock;
+    /* Prime the old cipher with ivec.*/
+    ip = (ivec != NULL) ? ivec : mit_des_zeroblock;
     GET_HALF_BLOCK(ocipherl, ip);
     GET_HALF_BLOCK(ocipherr, ip);
 
-    /*
-     * Now do this in earnest until we run out of length.
-     */
+    /* Work the length down 8 bytes at a time. */
     for (;;) {
-        /*
-         * Read a block from the input into left and
-         * right.  Save this cipher block for later.
-         */
-
-        if (!krb5int_c_iov_get_block(iblock, MIT_DES_BLOCK_LENGTH, data, num_data, &input_pos))
-            break;
-
-        if (input_pos.iov_pos == num_data)
+        ptr = iov_next_block(storage, MIT_DES_BLOCK_LENGTH, data, num_data,
+                             &input_pos);
+        if (ptr == NULL)
             break;
+        block = ptr;
 
-        ip = iblock;
-        op = oblock;
-
-        GET_HALF_BLOCK(left, ip);
-        GET_HALF_BLOCK(right, ip);
+        /* Split this block into left and right. */
+        GET_HALF_BLOCK(left, ptr);
+        GET_HALF_BLOCK(right, ptr);
         cipherl = left;
         cipherr = right;
 
-        /*
-         * Decrypt this.
-         */
+        /* Decrypt and xor with the old cipher to get plain text. */
         DES_DO_DECRYPT(left, right, kp3);
         DES_DO_ENCRYPT(left, right, kp2);
         DES_DO_DECRYPT(left, right, kp1);
-
-        /*
-         * Xor with the old cipher to get plain
-         * text.  Output 8 or less bytes of this.
-         */
         left ^= ocipherl;
         right ^= ocipherr;
 
-        PUT_HALF_BLOCK(left, op);
-        PUT_HALF_BLOCK(right, op);
+        /* Store the encrypted halves back into block. */
+        ptr = block;
+        PUT_HALF_BLOCK(left, ptr);
+        PUT_HALF_BLOCK(right, ptr);
 
-        /*
-         * Save current cipher block here
-         */
+        /* Save current cipher block halves. */
         ocipherl = cipherl;
         ocipherr = cipherr;
 
-        krb5int_c_iov_put_block(data, num_data, oblock, MIT_DES_BLOCK_LENGTH, &output_pos);
+        iov_store_block(data, num_data, block, storage, MIT_DES_BLOCK_LENGTH,
+                        &output_pos);
     }
 
-    if (ivec != NULL) {
-        op = ivec;
-        PUT_HALF_BLOCK(ocipherl,op);
-        PUT_HALF_BLOCK(ocipherr, op);
+    if (ivec != NULL && block != NULL) {
+        ptr = ivec;
+        PUT_HALF_BLOCK(ocipherl, ptr);
+        PUT_HALF_BLOCK(ocipherr, ptr);
     }
 }
diff --git a/src/lib/crypto/builtin/des/f_aead.c b/src/lib/crypto/builtin/des/f_aead.c
index 1f4d94a..1b92e05 100644
--- a/src/lib/crypto/builtin/des/f_aead.c
+++ b/src/lib/crypto/builtin/des/f_aead.c
@@ -35,67 +35,51 @@ krb5int_des_cbc_encrypt_iov(krb5_crypto_iov *data,
     unsigned DES_INT32 left, right;
     const unsigned DES_INT32 *kp;
     const unsigned char *ip;
-    unsigned char *op;
     struct iov_block_state input_pos, output_pos;
-    unsigned char iblock[MIT_DES_BLOCK_LENGTH];
-    unsigned char oblock[MIT_DES_BLOCK_LENGTH];
+    unsigned char storage[MIT_DES_BLOCK_LENGTH], *block = NULL, *ptr;
 
     IOV_BLOCK_STATE_INIT(&input_pos);
     IOV_BLOCK_STATE_INIT(&output_pos);
 
-    /*
-     * Get key pointer here.  This won't need to be reinitialized
-     */
+    /* Get key pointer here.  This won't need to be reinitialized. */
     kp = (const unsigned DES_INT32 *)schedule;
 
-    /*
-     * Initialize left and right with the contents of the initial
-     * vector.
-     */
-    if (ivec != NULL)
-        ip = ivec;
-    else
-        ip = mit_des_zeroblock;
+    /* Initialize left and right with the contents of the initial vector. */
+    ip = (ivec != NULL) ? ivec : mit_des_zeroblock;
     GET_HALF_BLOCK(left, ip);
     GET_HALF_BLOCK(right, ip);
 
-    /*
-     * Suitably initialized, now work the length down 8 bytes
-     * at a time.
-     */
+    /* Work the length down 8 bytes at a time. */
     for (;;) {
         unsigned DES_INT32 temp;
 
-        ip = iblock;
-        op = oblock;
-
-        if (!krb5int_c_iov_get_block(iblock, MIT_DES_BLOCK_LENGTH, data, num_data, &input_pos))
-            break;
-
-        if (input_pos.iov_pos == num_data)
+        ptr = iov_next_block(storage, MIT_DES_BLOCK_LENGTH, data, num_data,
+                             &input_pos);
+        if (ptr == NULL)
             break;
+        block = ptr;
 
-        GET_HALF_BLOCK(temp, ip);
+        /* Decompose this block and xor it with the previous ciphertext. */
+        GET_HALF_BLOCK(temp, ptr);
         left  ^= temp;
-        GET_HALF_BLOCK(temp, ip);
+        GET_HALF_BLOCK(temp, ptr);
         right ^= temp;
 
-        /*
-         * Encrypt what we have
-         */
+        /* Encrypt what we have and store back into block. */
         DES_DO_ENCRYPT(left, right, kp);
+        ptr = block;
+        PUT_HALF_BLOCK(left, ptr);
+        PUT_HALF_BLOCK(right, ptr);
 
-        /*
-         * Copy the results out
-         */
-        PUT_HALF_BLOCK(left, op);
-        PUT_HALF_BLOCK(right, op);
-
-        krb5int_c_iov_put_block(data, num_data, oblock, MIT_DES_BLOCK_LENGTH, &output_pos);
+        iov_store_block(data, num_data, block, storage, MIT_DES_BLOCK_LENGTH,
+                        &output_pos);
     }
 
-    if (ivec != NULL)
-        memcpy(ivec, oblock, MIT_DES_BLOCK_LENGTH);
+    if (ivec != NULL && block != NULL) {
+        ptr = ivec;
+        PUT_HALF_BLOCK(left, ptr);
+        PUT_HALF_BLOCK(right, ptr);
+    }
 }
 
 void
@@ -109,17 +93,13 @@ krb5int_des_cbc_decrypt_iov(krb5_crypto_iov *data,
     const unsigned char *ip;
     unsigned DES_INT32 ocipherl, ocipherr;
     unsigned DES_INT32 cipherl, cipherr;
-    unsigned char *op;
     struct iov_block_state input_pos, output_pos;
-    unsigned char iblock[MIT_DES_BLOCK_LENGTH];
-    unsigned char oblock[MIT_DES_BLOCK_LENGTH];
+    unsigned char storage[MIT_DES_BLOCK_LENGTH], *block = NULL, *ptr;
 
     IOV_BLOCK_STATE_INIT(&input_pos);
     IOV_BLOCK_STATE_INIT(&output_pos);
 
-    /*
-     * Get key pointer here.  This won't need to be reinitialized
-     */
+    /* Get key pointer here.  This won't need to be reinitialized. */
     kp = (const unsigned DES_INT32 *)schedule;
 
     /*
@@ -128,69 +108,46 @@ krb5int_des_cbc_decrypt_iov(krb5_crypto_iov *data,
      * Should think about this a little more...
      */
 
-    if (num_data == 0)
-        return;
-
-    /*
-     * Prime the old cipher with ivec.
-     */
-    if (ivec != NULL)
-        ip = ivec;
-    else
-        ip = mit_des_zeroblock;
+    /* Prime the old cipher with ivec. */
+    ip = (ivec != NULL) ? ivec : mit_des_zeroblock;
     GET_HALF_BLOCK(ocipherl, ip);
     GET_HALF_BLOCK(ocipherr, ip);
 
-    /*
-     * Now do this in earnest until we run out of length.
-     */
+    /* Work the length down 8 bytes at a time. */
     for (;;) {
-        /*
-         * Read a block from the input into left and
-         * right.  Save this cipher block for later.
-         */
-
-        if (!krb5int_c_iov_get_block(iblock, MIT_DES_BLOCK_LENGTH, data, num_data, &input_pos))
-            break;
-
-        if (input_pos.iov_pos == num_data)
+        ptr = iov_next_block(storage, MIT_DES_BLOCK_LENGTH, data, num_data,
+                             &input_pos);
+        if (ptr == NULL)
             break;
+        block = ptr;
 
-        ip = iblock;
-        op = oblock;
-
-        GET_HALF_BLOCK(left, ip);
-        GET_HALF_BLOCK(right, ip);
+        /* Split this block into left and right. */
+        GET_HALF_BLOCK(left, ptr);
+        GET_HALF_BLOCK(right, ptr);
         cipherl = left;
         cipherr = right;
 
-        /*
-         * Decrypt this.
-         */
+        /* Decrypt and xor with the old cipher to get plain text. */
         DES_DO_DECRYPT(left, right, kp);
-
-        /*
-         * Xor with the old cipher to get plain
-         * text.  Output 8 or less bytes of this.
-         */
         left ^= ocipherl;
         right ^= ocipherr;
 
-        PUT_HALF_BLOCK(left, op);
-        PUT_HALF_BLOCK(right, op);
+        /* Store the encrypted halves back into block. */
+        ptr = block;
+        PUT_HALF_BLOCK(left, ptr);
+        PUT_HALF_BLOCK(right, ptr);
 
-        /*
-         * Save current cipher block here
-         */
+        /* Save current cipher block halves. */
         ocipherl = cipherl;
         ocipherr = cipherr;
 
-        krb5int_c_iov_put_block(data, num_data, oblock, MIT_DES_BLOCK_LENGTH, &output_pos);
+        iov_store_block(data, num_data, block, storage, MIT_DES_BLOCK_LENGTH,
+                        &output_pos);
     }
 
-    if (ivec != NULL) {
-        op = ivec;
-        PUT_HALF_BLOCK(ocipherl, op);
-        PUT_HALF_BLOCK(ocipherr, op);
+    if (ivec != NULL && block != NULL) {
+        ptr = ivec;
+        PUT_HALF_BLOCK(ocipherl, ptr);
+        PUT_HALF_BLOCK(ocipherr, ptr);
     }
 }
diff --git a/src/lib/crypto/builtin/enc_provider/aes.c b/src/lib/crypto/builtin/enc_provider/aes.c
index cfa3643..396f653 100644
--- a/src/lib/crypto/builtin/enc_provider/aes.c
+++ b/src/lib/crypto/builtin/enc_provider/aes.c
@@ -214,17 +214,17 @@ krb5int_aes_encrypt_iov(krb5_key key,
         IOV_BLOCK_STATE_INIT(&output_pos);
 
         for (blockno = 0; blockno < nblocks - 2; blockno++) {
-            unsigned char blockN[BLOCK_SIZE];
+            unsigned char blockN[BLOCK_SIZE], *block;
 
-            krb5int_c_iov_get_block(blockN, BLOCK_SIZE, data, num_data,
-                                    &input_pos);
-            xorblock(tmp, blockN);
-            enc(tmp2, tmp, &ctx);
-            krb5int_c_iov_put_block(data, num_data, tmp2, BLOCK_SIZE,
-                                    &output_pos);
+            block = iov_next_block(blockN, BLOCK_SIZE, data, num_data,
+                                   &input_pos);
+            xorblock(tmp, block);
+            enc(block, tmp, &ctx);
+            iov_store_block(data, num_data, block, blockN, BLOCK_SIZE,
+                            &output_pos);
 
             /* Set up for next block.  */
-            memcpy(tmp, tmp2, BLOCK_SIZE);
+            memcpy(tmp, block, BLOCK_SIZE);
         }
 
         /* Do final CTS step for last two blocks (the second of which
@@ -304,15 +304,16 @@ krb5int_aes_decrypt_iov(krb5_key key,
         IOV_BLOCK_STATE_INIT(&output_pos);
 
         for (blockno = 0; blockno < nblocks - 2; blockno++) {
-            unsigned char blockN[BLOCK_SIZE];
+            unsigned char blockN[BLOCK_SIZE], *block;
 
-            krb5int_c_iov_get_block(blockN, BLOCK_SIZE, data, num_data,
-                                    &input_pos);
-            dec(tmp2, blockN, &ctx);
-            xorblock(tmp2, tmp);
-            krb5int_c_iov_put_block(data, num_data, tmp2, BLOCK_SIZE,
-                                    &output_pos);
-            memcpy(tmp, blockN, BLOCK_SIZE);
+            block = iov_next_block(blockN, BLOCK_SIZE, data, num_data,
+                                   &input_pos);
+            memcpy(tmp2, block, BLOCK_SIZE);
+            dec(block, block, &ctx);
+            xorblock(block, tmp);
+            memcpy(tmp, tmp2, BLOCK_SIZE);
+            iov_store_block(data, num_data, block, blockN, BLOCK_SIZE,
+                            &output_pos);
         }
 
         /* Do last two blocks, the second of which (next-to-last block
diff --git a/src/lib/crypto/krb/aead.c b/src/lib/crypto/krb/aead.c
index 353cd17..646c73c 100644
--- a/src/lib/crypto/krb/aead.c
+++ b/src/lib/crypto/krb/aead.c
@@ -274,6 +274,8 @@ krb5int_c_iov_get_block(unsigned char *block,
     }
 
     iov_state->iov_pos = i;
+    if (i == num_data)
+        return FALSE;
 
     if (j != block_size)
         memset(block + j, 0, block_size - j);
@@ -282,7 +284,7 @@ krb5int_c_iov_get_block(unsigned char *block,
     dump_block("get_block", i, j, block, block_size);
 #endif
 
-    return (iov_state->iov_pos < num_data);
+    return TRUE;
 }
 
 krb5_boolean
diff --git a/src/lib/crypto/krb/aead.h b/src/lib/crypto/krb/aead.h
index dcadfee..cc1e66a 100644
--- a/src/lib/crypto/krb/aead.h
+++ b/src/lib/crypto/krb/aead.h
@@ -127,3 +127,49 @@ krb5int_c_padding_length(const struct krb5_aead_provider *aead,
                          const struct krb5_hash_provider *hash,
                          size_t data_length,
                          unsigned int *pad_length);
+
+/*
+ * Returns an alias into the current buffer if the next block is fully
+ * contained within; otherwise makes a copy of the next block and returns an
+ * alias to storage.  After calling this function, encrypt the returned block
+ * in place and then call iov_store_block (with a separate output cursor) to
+ * store the result back into the iov if necessary.  Returns NULL if there
+ * is no next block.
+ */
+static inline unsigned char *
+iov_next_block(unsigned char *storage, size_t len,
+               const krb5_crypto_iov *data, size_t num_data,
+               struct iov_block_state *pos)
+{
+    const krb5_crypto_iov *iov = &data[pos->iov_pos];
+    unsigned char *block;
+
+    if (pos->iov_pos < num_data && iov->data.length - pos->data_pos >= len) {
+        /* Return an alias to memory inside the current iov. */
+        block = (unsigned char *) iov->data.data + pos->data_pos;
+        pos->data_pos += len;
+        return block;
+    }
+    /* Do it the slow way and return a copy into storage. */
+    if (krb5int_c_iov_get_block(storage, len, data, num_data, pos))
+        return storage;
+    return NULL;
+}
+
+/*
+ * Store a block retrieved with iov_next_block if necessary, and advance the
+ * output cursor.
+ */
+static inline void
+iov_store_block(const krb5_crypto_iov *data, size_t num_data,
+                unsigned char *block, unsigned char *storage, size_t len,
+                struct iov_block_state *pos)
+{
+    if (block == storage) {
+        /* We got the block the slow way; put it back that way too. */
+        krb5int_c_iov_put_block(data, num_data, storage, len, pos);
+    } else {
+        /* It's already stored; we just have to advance the output cursor. */
+        pos->data_pos += len;
+    }
+}