Speed up the per-block loops of AES, DES3, and DES IOV encryption by

avoiding function calls and copies in the case where the next block
is wholly contained within the current buffer.  To do this, introduce
two new inline functions in aead.h called iov_next_block and
iov_store_block.

git-svn-id: svn://anonsvn.mit.edu/krb5/trunk@23430 dc483132-0cff-0310-8789-dd5450dbe970

2 files changed, 94 insertions, 180 deletions

diff --git a/src/lib/crypto/builtin/des/d3_aead.c b/src/lib/crypto/builtin/des/d3_aead.c
index e018895..549a27f 100644
--- a/src/lib/crypto/builtin/des/d3_aead.c
+++ b/src/lib/crypto/builtin/des/d3_aead.c
@@ -37,71 +37,55 @@ krb5int_des3_cbc_encrypt_iov(krb5_crypto_iov *data,
     unsigned DES_INT32 left, right;
     const unsigned DES_INT32 *kp1, *kp2, *kp3;
     const unsigned char *ip;
-    unsigned char *op;
     struct iov_block_state input_pos, output_pos;
-    unsigned char iblock[MIT_DES_BLOCK_LENGTH];
-    unsigned char oblock[MIT_DES_BLOCK_LENGTH];
+    unsigned char storage[MIT_DES_BLOCK_LENGTH], *block = NULL, *ptr;
 
     IOV_BLOCK_STATE_INIT(&input_pos);
     IOV_BLOCK_STATE_INIT(&output_pos);
 
-    /*
-     * Get key pointer here.  This won't need to be reinitialized
-     */
+    /* Get key pointers here.  These won't need to be reinitialized. */
     kp1 = (const unsigned DES_INT32 *)ks1;
     kp2 = (const unsigned DES_INT32 *)ks2;
     kp3 = (const unsigned DES_INT32 *)ks3;
 
-    /*
-     * Initialize left and right with the contents of the initial
-     * vector.
-     */
-    if (ivec != NULL)
-        ip = ivec;
-    else
-        ip = mit_des_zeroblock;
+    /* Initialize left and right with the contents of the initial vector. */
+    ip = (ivec != NULL) ? ivec : mit_des_zeroblock;
     GET_HALF_BLOCK(left, ip);
     GET_HALF_BLOCK(right, ip);
 
-    /*
-     * Suitably initialized, now work the length down 8 bytes
-     * at a time.
-     */
+    /* Work the length down 8 bytes at a time. */
     for (;;) {
         unsigned DES_INT32 temp;
 
-        ip = iblock;
-        op = oblock;
-
-        if (!krb5int_c_iov_get_block(iblock, MIT_DES_BLOCK_LENGTH, data, num_data, &input_pos))
-            break;
-
-        if (input_pos.iov_pos == num_data)
+        ptr = iov_next_block(storage, MIT_DES_BLOCK_LENGTH, data, num_data,
+                             &input_pos);
+        if (ptr == NULL)
             break;
+        block = ptr;
 
-        GET_HALF_BLOCK(temp, ip);
+        /* Decompose this block and xor it with the previous ciphertext. */
+        GET_HALF_BLOCK(temp, ptr);
         left  ^= temp;
-        GET_HALF_BLOCK(temp, ip);
+        GET_HALF_BLOCK(temp, ptr);
         right ^= temp;
 
-        /*
-         * Encrypt what we have
-         */
+        /* Encrypt what we have and store it back into block. */
         DES_DO_ENCRYPT(left, right, kp1);
         DES_DO_DECRYPT(left, right, kp2);
         DES_DO_ENCRYPT(left, right, kp3);
+        ptr = block;
+        PUT_HALF_BLOCK(left, ptr);
+        PUT_HALF_BLOCK(right, ptr);
 
-        /*
-         * Copy the results out
-         */
-        PUT_HALF_BLOCK(left, op);
-        PUT_HALF_BLOCK(right, op);
-
-        krb5int_c_iov_put_block(data, num_data, oblock, MIT_DES_BLOCK_LENGTH, &output_pos);
+        iov_store_block(data, num_data, block, storage, MIT_DES_BLOCK_LENGTH,
+                        &output_pos);
     }
 
-    if (ivec != NULL)
-        memcpy(ivec, oblock, MIT_DES_BLOCK_LENGTH);
+    if (ivec != NULL && block != NULL) {
+        ptr = ivec;
+        PUT_HALF_BLOCK(left, ptr);
+        PUT_HALF_BLOCK(right, ptr);
+    }
 }
 
 void
@@ -117,17 +101,13 @@ krb5int_des3_cbc_decrypt_iov(krb5_crypto_iov *data,
     const unsigned char *ip;
     unsigned DES_INT32 ocipherl, ocipherr;
     unsigned DES_INT32 cipherl, cipherr;
-    unsigned char *op;
     struct iov_block_state input_pos, output_pos;
-    unsigned char iblock[MIT_DES_BLOCK_LENGTH];
-    unsigned char oblock[MIT_DES_BLOCK_LENGTH];
+    unsigned char storage[MIT_DES_BLOCK_LENGTH], *block = NULL, *ptr;
 
     IOV_BLOCK_STATE_INIT(&input_pos);
     IOV_BLOCK_STATE_INIT(&output_pos);
 
-    /*
-     * Get key pointer here.  This won't need to be reinitialized
-     */
+    /* Get key pointers here.  These won't need to be reinitialized. */
     kp1 = (const unsigned DES_INT32 *)ks1;
     kp2 = (const unsigned DES_INT32 *)ks2;
     kp3 = (const unsigned DES_INT32 *)ks3;
@@ -138,71 +118,48 @@ krb5int_des3_cbc_decrypt_iov(krb5_crypto_iov *data,
      * Should think about this a little more...
      */
 
-    if (num_data == 0)
-        return;
-
-    /*
-     * Prime the old cipher with ivec.
-     */
-    if (ivec != NULL)
-        ip = ivec;
-    else
-        ip = mit_des_zeroblock;
+    /* Prime the old cipher with ivec.*/
+    ip = (ivec != NULL) ? ivec : mit_des_zeroblock;
     GET_HALF_BLOCK(ocipherl, ip);
     GET_HALF_BLOCK(ocipherr, ip);
 
-    /*
-     * Now do this in earnest until we run out of length.
-     */
+    /* Work the length down 8 bytes at a time. */
     for (;;) {
-        /*
-         * Read a block from the input into left and
-         * right.  Save this cipher block for later.
-         */
-
-        if (!krb5int_c_iov_get_block(iblock, MIT_DES_BLOCK_LENGTH, data, num_data, &input_pos))
-            break;
-
-        if (input_pos.iov_pos == num_data)
+        ptr = iov_next_block(storage, MIT_DES_BLOCK_LENGTH, data, num_data,
+                             &input_pos);
+        if (ptr == NULL)
             break;
+        block = ptr;
 
-        ip = iblock;
-        op = oblock;
-
-        GET_HALF_BLOCK(left, ip);
-        GET_HALF_BLOCK(right, ip);
+        /* Split this block into left and right. */
+        GET_HALF_BLOCK(left, ptr);
+        GET_HALF_BLOCK(right, ptr);
         cipherl = left;
         cipherr = right;
 
-        /*
-         * Decrypt this.
-         */
+        /* Decrypt and xor with the old cipher to get plain text. */
         DES_DO_DECRYPT(left, right, kp3);
         DES_DO_ENCRYPT(left, right, kp2);
         DES_DO_DECRYPT(left, right, kp1);
-
-        /*
-         * Xor with the old cipher to get plain
-         * text.  Output 8 or less bytes of this.
-         */
         left ^= ocipherl;
         right ^= ocipherr;
 
-        PUT_HALF_BLOCK(left, op);
-        PUT_HALF_BLOCK(right, op);
+        /* Store the encrypted halves back into block. */
+        ptr = block;
+        PUT_HALF_BLOCK(left, ptr);
+        PUT_HALF_BLOCK(right, ptr);
 
-        /*
-         * Save current cipher block here
-         */
+        /* Save current cipher block halves. */
         ocipherl = cipherl;
         ocipherr = cipherr;
 
-        krb5int_c_iov_put_block(data, num_data, oblock, MIT_DES_BLOCK_LENGTH, &output_pos);
+        iov_store_block(data, num_data, block, storage, MIT_DES_BLOCK_LENGTH,
+                        &output_pos);
     }
 
-    if (ivec != NULL) {
-        op = ivec;
-        PUT_HALF_BLOCK(ocipherl,op);
-        PUT_HALF_BLOCK(ocipherr, op);
+    if (ivec != NULL && block != NULL) {
+        ptr = ivec;
+        PUT_HALF_BLOCK(ocipherl, ptr);
+        PUT_HALF_BLOCK(ocipherr, ptr);
     }
 }
diff --git a/src/lib/crypto/builtin/des/f_aead.c b/src/lib/crypto/builtin/des/f_aead.c
index 1f4d94a..1b92e05 100644
--- a/src/lib/crypto/builtin/des/f_aead.c
+++ b/src/lib/crypto/builtin/des/f_aead.c
@@ -35,67 +35,51 @@ krb5int_des_cbc_encrypt_iov(krb5_crypto_iov *data,
     unsigned DES_INT32 left, right;
     const unsigned DES_INT32 *kp;
     const unsigned char *ip;
-    unsigned char *op;
     struct iov_block_state input_pos, output_pos;
-    unsigned char iblock[MIT_DES_BLOCK_LENGTH];
-    unsigned char oblock[MIT_DES_BLOCK_LENGTH];
+    unsigned char storage[MIT_DES_BLOCK_LENGTH], *block = NULL, *ptr;
 
     IOV_BLOCK_STATE_INIT(&input_pos);
     IOV_BLOCK_STATE_INIT(&output_pos);
 
-    /*
-     * Get key pointer here.  This won't need to be reinitialized
-     */
+    /* Get key pointer here.  This won't need to be reinitialized. */
     kp = (const unsigned DES_INT32 *)schedule;
 
-    /*
-     * Initialize left and right with the contents of the initial
-     * vector.
-     */
-    if (ivec != NULL)
-        ip = ivec;
-    else
-        ip = mit_des_zeroblock;
+    /* Initialize left and right with the contents of the initial vector. */
+    ip = (ivec != NULL) ? ivec : mit_des_zeroblock;
     GET_HALF_BLOCK(left, ip);
     GET_HALF_BLOCK(right, ip);
 
-    /*
-     * Suitably initialized, now work the length down 8 bytes
-     * at a time.
-     */
+    /* Work the length down 8 bytes at a time. */
     for (;;) {
         unsigned DES_INT32 temp;
 
-        ip = iblock;
-        op = oblock;
-
-        if (!krb5int_c_iov_get_block(iblock, MIT_DES_BLOCK_LENGTH, data, num_data, &input_pos))
-            break;
-
-        if (input_pos.iov_pos == num_data)
+        ptr = iov_next_block(storage, MIT_DES_BLOCK_LENGTH, data, num_data,
+                             &input_pos);
+        if (ptr == NULL)
             break;
+        block = ptr;
 
-        GET_HALF_BLOCK(temp, ip);
+        /* Decompose this block and xor it with the previous ciphertext. */
+        GET_HALF_BLOCK(temp, ptr);
         left  ^= temp;
-        GET_HALF_BLOCK(temp, ip);
+        GET_HALF_BLOCK(temp, ptr);
         right ^= temp;
 
-        /*
-         * Encrypt what we have
-         */
+        /* Encrypt what we have and store back into block. */
         DES_DO_ENCRYPT(left, right, kp);
+        ptr = block;
+        PUT_HALF_BLOCK(left, ptr);
+        PUT_HALF_BLOCK(right, ptr);
 
-        /*
-         * Copy the results out
-         */
-        PUT_HALF_BLOCK(left, op);
-        PUT_HALF_BLOCK(right, op);
-
-        krb5int_c_iov_put_block(data, num_data, oblock, MIT_DES_BLOCK_LENGTH, &output_pos);
+        iov_store_block(data, num_data, block, storage, MIT_DES_BLOCK_LENGTH,
+                        &output_pos);
     }
 
-    if (ivec != NULL)
-        memcpy(ivec, oblock, MIT_DES_BLOCK_LENGTH);
+    if (ivec != NULL && block != NULL) {
+        ptr = ivec;
+        PUT_HALF_BLOCK(left, ptr);
+        PUT_HALF_BLOCK(right, ptr);
+    }
 }
 
 void
@@ -109,17 +93,13 @@ krb5int_des_cbc_decrypt_iov(krb5_crypto_iov *data,
     const unsigned char *ip;
     unsigned DES_INT32 ocipherl, ocipherr;
     unsigned DES_INT32 cipherl, cipherr;
-    unsigned char *op;
     struct iov_block_state input_pos, output_pos;
-    unsigned char iblock[MIT_DES_BLOCK_LENGTH];
-    unsigned char oblock[MIT_DES_BLOCK_LENGTH];
+    unsigned char storage[MIT_DES_BLOCK_LENGTH], *block = NULL, *ptr;
 
     IOV_BLOCK_STATE_INIT(&input_pos);
     IOV_BLOCK_STATE_INIT(&output_pos);
 
-    /*
-     * Get key pointer here.  This won't need to be reinitialized
-     */
+    /* Get key pointer here.  This won't need to be reinitialized. */
     kp = (const unsigned DES_INT32 *)schedule;
 
     /*
@@ -128,69 +108,46 @@ krb5int_des_cbc_decrypt_iov(krb5_crypto_iov *data,
      * Should think about this a little more...
      */
 
-    if (num_data == 0)
-        return;
-
-    /*
-     * Prime the old cipher with ivec.
-     */
-    if (ivec != NULL)
-        ip = ivec;
-    else
-        ip = mit_des_zeroblock;
+    /* Prime the old cipher with ivec. */
+    ip = (ivec != NULL) ? ivec : mit_des_zeroblock;
     GET_HALF_BLOCK(ocipherl, ip);
     GET_HALF_BLOCK(ocipherr, ip);
 
-    /*
-     * Now do this in earnest until we run out of length.
-     */
+    /* Work the length down 8 bytes at a time. */
     for (;;) {
-        /*
-         * Read a block from the input into left and
-         * right.  Save this cipher block for later.
-         */
-
-        if (!krb5int_c_iov_get_block(iblock, MIT_DES_BLOCK_LENGTH, data, num_data, &input_pos))
-            break;
-
-        if (input_pos.iov_pos == num_data)
+        ptr = iov_next_block(storage, MIT_DES_BLOCK_LENGTH, data, num_data,
+                             &input_pos);
+        if (ptr == NULL)
             break;
+        block = ptr;
 
-        ip = iblock;
-        op = oblock;
-
-        GET_HALF_BLOCK(left, ip);
-        GET_HALF_BLOCK(right, ip);
+        /* Split this block into left and right. */
+        GET_HALF_BLOCK(left, ptr);
+        GET_HALF_BLOCK(right, ptr);
         cipherl = left;
         cipherr = right;
 
-        /*
-         * Decrypt this.
-         */
+        /* Decrypt and xor with the old cipher to get plain text. */
         DES_DO_DECRYPT(left, right, kp);
-
-        /*
-         * Xor with the old cipher to get plain
-         * text.  Output 8 or less bytes of this.
-         */
         left ^= ocipherl;
         right ^= ocipherr;
 
-        PUT_HALF_BLOCK(left, op);
-        PUT_HALF_BLOCK(right, op);
+        /* Store the encrypted halves back into block. */
+        ptr = block;
+        PUT_HALF_BLOCK(left, ptr);
+        PUT_HALF_BLOCK(right, ptr);
 
-        /*
-         * Save current cipher block here
-         */
+        /* Save current cipher block halves. */
         ocipherl = cipherl;
         ocipherr = cipherr;
 
-        krb5int_c_iov_put_block(data, num_data, oblock, MIT_DES_BLOCK_LENGTH, &output_pos);
+        iov_store_block(data, num_data, block, storage, MIT_DES_BLOCK_LENGTH,
+                        &output_pos);
     }
 
-    if (ivec != NULL) {
-        op = ivec;
-        PUT_HALF_BLOCK(ocipherl, op);
-        PUT_HALF_BLOCK(ocipherr, op);
+    if (ivec != NULL && block != NULL) {
+        ptr = ivec;
+        PUT_HALF_BLOCK(ocipherl, ptr);
+        PUT_HALF_BLOCK(ocipherr, ptr);
     }
 }