Move validation of CTR mode cipher state up to CCM layer. This allows

us to re-use the CTR implementation for other compositions such as
GCM, which have differently formatted counter blocks. This is similar
to the CTR implementation in OpenSSL, which requires the caller to
manage counter overflow. It also simplifies the code and specification
somewhat. The CTR initial cipher state is now 16 bytes of zeros; the
number of bits used for the counter is up to the composition.

git-svn-id: svn://anonsvn.mit.edu/krb5/users/lhoward/camellia-ccm@24047 dc483132-0cff-0310-8789-dd5450dbe970

5 files changed, 31 insertions, 116 deletions

diff --git a/src/lib/crypto/builtin/enc_provider/aes_ctr.c b/src/lib/crypto/builtin/enc_provider/aes_ctr.c
index e03bfad..7f527a7 100644
--- a/src/lib/crypto/builtin/enc_provider/aes_ctr.c
+++ b/src/lib/crypto/builtin/enc_provider/aes_ctr.c
@@ -94,9 +94,6 @@ static inline void putctrblockno(krb5_ui_8 blockno,
     }
 }
 
-/* Maximum number of invocations with a given nonce and key */
-#define maxblocks(q)	    (1UL << (8 * (q)))
-
 /*
  * ivec must be a correctly formatted counter block per SP800-38C A.3
  */
@@ -135,9 +132,6 @@ krb5int_aes_encrypt_ctr(krb5_key key,
 	unsigned char storage[BLOCK_SIZE], *block;
 	unsigned char ectr[BLOCK_SIZE];
 
-	if (blockno >= maxblocks(ctr[0] + 1))
-	    return KRB5_CRYPTO_INTERNAL;
-
         block = iov_next_block(storage, BLOCK_SIZE, data, num_data, &input_pos);
         if (block == NULL)
 	    break;
@@ -212,24 +206,7 @@ static krb5_error_code
 krb5int_aes_init_state_ctr (const krb5_keyblock *key, krb5_keyusage usage,
 			    krb5_data *state)
 {
-    unsigned int n, q;
-    krb5_error_code code;
-
-    code = krb5_c_crypto_length(NULL, key->enctype, KRB5_CRYPTO_TYPE_HEADER, &n);
-    if (code != 0)
-	return code;
-
-    assert(n >= 7 && n <= 13);
-
-    state->length = 16;
-    state->data = k5alloc(state->length, &code);
-    if (code != 0)
-	return code;
-
-    q = 15 - n;
-    state->data[0] = q - 1;
-
-    return 0;
+    return alloc_data(state, 16);
 }
 
 const struct krb5_enc_provider krb5int_enc_aes128_ctr = {
diff --git a/src/lib/crypto/builtin/enc_provider/camellia_ctr.c b/src/lib/crypto/builtin/enc_provider/camellia_ctr.c
index a6d8224..00208c1 100644
--- a/src/lib/crypto/builtin/enc_provider/camellia_ctr.c
+++ b/src/lib/crypto/builtin/enc_provider/camellia_ctr.c
@@ -94,9 +94,6 @@ static inline void putctrblockno(krb5_ui_8 blockno,
     }
 }
 
-/* Maximum number of invocations with a given nonce and key */
-#define maxblocks(q)            (1UL << (8 * (q)))
-
 /*
  * ivec must be a correctly formatted counter block per SP800-38C A.3
  */
@@ -135,9 +132,6 @@ krb5int_camellia_encrypt_ctr(krb5_key key,
         unsigned char storage[BLOCK_SIZE], *block;
         unsigned char ectr[BLOCK_SIZE];
 
-        if (blockno >= maxblocks(ctr[0] + 1))
-            return KRB5_CRYPTO_INTERNAL;
-
         block = iov_next_block(storage, BLOCK_SIZE, data, num_data, &input_pos);
         if (block == NULL)
             break;
@@ -212,24 +206,7 @@ static krb5_error_code
 krb5int_camellia_init_state_ctr (const krb5_keyblock *key, krb5_keyusage usage,
                                  krb5_data *state)
 {
-    unsigned int n, q;
-    krb5_error_code code;
-
-    code = krb5_c_crypto_length(NULL, key->enctype, KRB5_CRYPTO_TYPE_HEADER, &n);
-    if (code != 0)
-        return code;
-
-    assert(n >= 7 && n <= 13);
-
-    state->length = 16;
-    state->data = k5alloc(state->length, &code);
-    if (code != 0)
-        return code;
-
-    q = 15 - n;
-    state->data[0] = q - 1;
-
-    return 0;
+    return alloc_data(state, 16);
 }
 
 const struct krb5_enc_provider krb5int_enc_camellia128_ctr = {
diff --git a/src/lib/crypto/krb/dk/dk_ccm.c b/src/lib/crypto/krb/dk/dk_ccm.c
index 6f6f675..628e9b0 100644
--- a/src/lib/crypto/krb/dk/dk_ccm.c
+++ b/src/lib/crypto/krb/dk/dk_ccm.c
@@ -207,23 +207,6 @@ format_B0(krb5_data *B0,            /* B0 */
 }
 
 /*
- * Returns TRUE if the cipher state is valid.
- */
-static krb5_boolean
-valid_cipher_state_p(const krb5_data *state, unsigned int n)
-{
-    if (state != NULL) {
-        if (state->length != 16 ||
-            state->data[0] & ~(CCM_FLAG_MASK_Q) ||
-            14 - (unsigned)state->data[0] != n) {
-            return FALSE;
-        }
-    }
-
-    return TRUE;
-}
-
-/*
  * Format initial counter block. Counter may be chained
  * across invocations.
  */
@@ -237,9 +220,6 @@ format_Ctr0(krb5_data *counter,
 
     assert(n >= 7 && n <= 13);
 
-    if (!valid_cipher_state_p(state, n))
-        return KRB5_BAD_MSIZE;
-
     q = 15 - n;
     counter->data[0] = q - 1;
     memcpy(&counter->data[1], nonce->data, n);
@@ -252,6 +232,27 @@ format_Ctr0(krb5_data *counter,
     return 0;
 }
 
+static krb5_boolean
+valid_payload_length_p(const struct krb5_keytypes *ktp,
+                       unsigned int n,
+                       unsigned int payload_len)
+{
+    unsigned int block_size = ktp->enc->block_size;
+    unsigned int nblocks, maxblocks;
+    krb5_octet q;
+
+    assert(n >= 7 && n <= 13);
+
+    q = 15 - n;
+
+    maxblocks = (1UL << (8 * q));
+
+    nblocks = 1; /* tag */
+    nblocks += (payload_len + block_size - 1) / block_size;
+
+    return (nblocks <= maxblocks);
+}
+
 krb5_error_code
 krb5int_ccm_encrypt(const struct krb5_keytypes *ktp,
                     krb5_key key,
@@ -304,6 +305,9 @@ krb5int_ccm_encrypt(const struct krb5_keytypes *ktp,
         }
     }
 
+    if (!valid_payload_length_p(ktp, header_len, payload_len))
+        return KRB5_BAD_MSIZE;
+
     header->data.length = header_len;
 
     /* Initialize nonce */
@@ -450,6 +454,9 @@ krb5int_ccm_decrypt(const struct krb5_keytypes *ktp,
         }
     }
 
+    if (!valid_payload_length_p(ktp, header_len, payload_len))
+        return KRB5_BAD_MSIZE;
+
     /* Initialize counter block */
     ret = format_Ctr0(&counter, &header->data, state, header_len);
     if (ret != 0)
diff --git a/src/lib/crypto/openssl/enc_provider/aes_ctr.c b/src/lib/crypto/openssl/enc_provider/aes_ctr.c
index c9e8907..31d9c55 100644
--- a/src/lib/crypto/openssl/enc_provider/aes_ctr.c
+++ b/src/lib/crypto/openssl/enc_provider/aes_ctr.c
@@ -64,9 +64,6 @@ xorblock(unsigned char *out, const unsigned char *in)
     }
 }
 
-/* Maximum number of invocations with a given nonce and key */
-#define maxblocks(q)            (1UL << (8 * (q)))
-
 /*
  * ivec must be a correctly formatted counter block per SP800-38C A.3
  */
@@ -103,9 +100,6 @@ krb5int_aes_encrypt_ctr(krb5_key key,
         unsigned char ectr[AES_BLOCK_SIZE];
         unsigned int num = 0;
 
-        if (blockno >= maxblocks(ctr[0] + 1))
-            return KRB5_CRYPTO_INTERNAL;
-
         block = iov_next_block(storage, AES_BLOCK_SIZE, data, num_data, &input_pos);
         if (block == NULL)
             break;
@@ -176,24 +170,7 @@ static krb5_error_code
 krb5int_aes_init_state_ctr (const krb5_keyblock *key, krb5_keyusage usage,
                             krb5_data *state)
 {
-    unsigned int n, q;
-    krb5_error_code code;
-
-    code = krb5_c_crypto_length(NULL, key->enctype, KRB5_CRYPTO_TYPE_HEADER, &n);
-    if (code != 0)
-        return code;
-
-    assert(n >= 7 && n <= 13);
-
-    state->length = 16;
-    state->data = k5alloc(state->length, &code);
-    if (code != 0)
-        return code;
-
-    q = 15 - n;
-    state->data[0] = q - 1;
-
-    return 0;
+    return alloc_data(state, 16);
 }
 
 const struct krb5_enc_provider krb5int_enc_aes128_ctr = {
diff --git a/src/lib/crypto/openssl/enc_provider/camellia_ctr.c b/src/lib/crypto/openssl/enc_provider/camellia_ctr.c
index 2812c05..2dfb8c5 100644
--- a/src/lib/crypto/openssl/enc_provider/camellia_ctr.c
+++ b/src/lib/crypto/openssl/enc_provider/camellia_ctr.c
@@ -63,9 +63,6 @@ xorblock(unsigned char *out, const unsigned char *in)
     }
 }
 
-/* Maximum number of invocations with a given nonce and key */
-#define maxblocks(q)            (1UL << (8 * (q)))
-
 /*
  * ivec must be a correctly formatted counter block per SP800-38C A.3
  */
@@ -102,9 +99,6 @@ krb5int_camellia_encrypt_ctr(krb5_key key,
         unsigned char ectr[CAMELLIA_BLOCK_SIZE];
         unsigned int num = 0;
 
-        if (blockno >= maxblocks(ctr[0] + 1))
-            return KRB5_CRYPTO_INTERNAL;
-
         block = iov_next_block(storage, CAMELLIA_BLOCK_SIZE, data, num_data, &input_pos);
         if (block == NULL)
             break;
@@ -175,24 +169,7 @@ static krb5_error_code
 krb5int_camellia_init_state_ctr (const krb5_keyblock *key, krb5_keyusage usage,
                                  krb5_data *state)
 {
-    unsigned int n, q;
-    krb5_error_code code;
-
-    code = krb5_c_crypto_length(NULL, key->enctype, KRB5_CRYPTO_TYPE_HEADER, &n);
-    if (code != 0)
-        return code;
-
-    assert(n >= 7 && n <= 13);
-
-    state->length = 16;
-    state->data = k5alloc(state->length, &code);
-    if (code != 0)
-        return code;
-
-    q = 15 - n;
-    state->data[0] = q - 1;
-
-    return 0;
+    return alloc_data(state, 16);
 }
 
 const struct krb5_enc_provider krb5int_enc_camellia128_ctr = {