crypto: avoid undefined behavior in nettle calls

Calling a function pointer that was cast from an incompatible function
results in undefined behavior.  'void *' isn't compatible with 'struct
XXX *', so we can't cast to nettle_cipher_func, but have to provide a
wrapper.  (Conversion from 'void *' to 'struct XXX *' might require
computation, which won't be done if we drop argument's true type, and
pointers can have different sizes so passing arguments on stack would
bug.)

Having two different prototypes based on nettle version doesn't make
this solution any nicer.

Reported-by: Peter Maydell <peter.maydell@linaro.org>
Signed-off-by: Radim Krčmář <rkrcmar@redhat.com>
Message-Id: <1437062641-12684-3-git-send-email-rkrcmar@redhat.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>

1 files changed, 39 insertions, 4 deletions

diff --git a/crypto/cipher-nettle.c b/crypto/cipher-nettle.c
index e61aaa2..a55a8e8 100644
--- a/crypto/cipher-nettle.c
+++ b/crypto/cipher-nettle.c
@@ -25,8 +25,43 @@
 
 #if CONFIG_NETTLE_VERSION_MAJOR < 3
 typedef nettle_crypt_func nettle_cipher_func;
+
+typedef void *       cipher_ctx_t;
+typedef unsigned     cipher_length_t;
+#else
+typedef const void * cipher_ctx_t;
+typedef size_t       cipher_length_t;
 #endif
 
+static nettle_cipher_func aes_encrypt_wrapper;
+static nettle_cipher_func aes_decrypt_wrapper;
+static nettle_cipher_func des_encrypt_wrapper;
+static nettle_cipher_func des_decrypt_wrapper;
+
+static void aes_encrypt_wrapper(cipher_ctx_t ctx, cipher_length_t length,
+                                uint8_t *dst, const uint8_t *src)
+{
+    aes_encrypt(ctx, length, dst, src);
+}
+
+static void aes_decrypt_wrapper(cipher_ctx_t ctx, cipher_length_t length,
+                                uint8_t *dst, const uint8_t *src)
+{
+    aes_encrypt(ctx, length, dst, src);
+}
+
+static void des_encrypt_wrapper(cipher_ctx_t ctx, cipher_length_t length,
+                                uint8_t *dst, const uint8_t *src)
+{
+    des_encrypt(ctx, length, dst, src);
+}
+
+static void des_decrypt_wrapper(cipher_ctx_t ctx, cipher_length_t length,
+                                uint8_t *dst, const uint8_t *src)
+{
+    des_decrypt(ctx, length, dst, src);
+}
+
 typedef struct QCryptoCipherNettle QCryptoCipherNettle;
 struct QCryptoCipherNettle {
     void *ctx_encrypt;
@@ -87,8 +122,8 @@ QCryptoCipher *qcrypto_cipher_new(QCryptoCipherAlgorithm alg,
         des_set_key(ctx->ctx_encrypt, rfbkey);
         g_free(rfbkey);
 
-        ctx->alg_encrypt = (nettle_cipher_func *)des_encrypt;
-        ctx->alg_decrypt = (nettle_cipher_func *)des_decrypt;
+        ctx->alg_encrypt = des_encrypt_wrapper;
+        ctx->alg_decrypt = des_decrypt_wrapper;
 
         ctx->niv = DES_BLOCK_SIZE;
         break;
@@ -102,8 +137,8 @@ QCryptoCipher *qcrypto_cipher_new(QCryptoCipherAlgorithm alg,
         aes_set_encrypt_key(ctx->ctx_encrypt, nkey, key);
         aes_set_decrypt_key(ctx->ctx_decrypt, nkey, key);
 
-        ctx->alg_encrypt = (nettle_cipher_func *)aes_encrypt;
-        ctx->alg_decrypt = (nettle_cipher_func *)aes_decrypt;
+        ctx->alg_encrypt = aes_encrypt_wrapper;
+        ctx->alg_decrypt = aes_decrypt_wrapper;
 
         ctx->niv = AES_BLOCK_SIZE;
         break;