crypto: switch hash code to use nettle/gcrypt directly

Currently the internal hash code is using the gnutls hash APIs.
GNUTLS in turn is wrapping either nettle or gcrypt. Not only
were the GNUTLS hash APIs not added until GNUTLS 2.9.10, but
they don't expose support for all the algorithms QEMU needs
to use with LUKS.

Address this by directly wrapping nettle/gcrypt in QEMU and
avoiding GNUTLS's extra layer of indirection. This gives us
support for hash functions on a much wider range of platforms
and opens up ability to support more hash functions. It also
avoids a GNUTLS bug which would not correctly handle hashing
of large data blocks if int != size_t.

Signed-off-by: Daniel P. Berrange <berrange@redhat.com>

5 files changed, 276 insertions, 105 deletions

diff --git a/crypto/Makefile.objs b/crypto/Makefile.objs
index 0737f48..1f86f4f 100644
--- a/crypto/Makefile.objs
+++ b/crypto/Makefile.objs
@@ -1,5 +1,7 @@
 crypto-obj-y = init.o
 crypto-obj-y += hash.o
+crypto-obj-$(CONFIG_NETTLE) += hash-nettle.o
+crypto-obj-$(if $(CONFIG_NETTLE),n,$(CONFIG_GCRYPT)) += hash-gcrypt.o
 crypto-obj-y += aes.o
 crypto-obj-y += desrfb.o
 crypto-obj-y += cipher.o
@@ -28,3 +30,4 @@ crypto-aes-obj-y = aes.o
 
 stub-obj-y += random-stub.o
 stub-obj-y += pbkdf-stub.o
+stub-obj-y += hash-stub.o
diff --git a/crypto/hash-gcrypt.c b/crypto/hash-gcrypt.c
new file mode 100644
index 0000000..e045689
--- /dev/null
+++ b/crypto/hash-gcrypt.c
@@ -0,0 +1,106 @@
+/*
+ * QEMU Crypto hash algorithms
+ *
+ * Copyright (c) 2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "crypto/hash.h"
+#include "gcrypt.h"
+
+
+static int qcrypto_hash_alg_map[QCRYPTO_HASH_ALG__MAX] = {
+    [QCRYPTO_HASH_ALG_MD5] = GCRY_MD_MD5,
+    [QCRYPTO_HASH_ALG_SHA1] = GCRY_MD_SHA1,
+    [QCRYPTO_HASH_ALG_SHA256] = GCRY_MD_SHA256,
+};
+
+gboolean qcrypto_hash_supports(QCryptoHashAlgorithm alg)
+{
+    if (alg < G_N_ELEMENTS(qcrypto_hash_alg_map) &&
+        qcrypto_hash_alg_map[alg] != GCRY_MD_NONE) {
+        return true;
+    }
+    return false;
+}
+
+
+int qcrypto_hash_bytesv(QCryptoHashAlgorithm alg,
+                        const struct iovec *iov,
+                        size_t niov,
+                        uint8_t **result,
+                        size_t *resultlen,
+                        Error **errp)
+{
+    int i, ret;
+    gcry_md_hd_t md;
+    unsigned char *digest;
+
+    if (alg >= G_N_ELEMENTS(qcrypto_hash_alg_map) ||
+        qcrypto_hash_alg_map[alg] == GCRY_MD_NONE) {
+        error_setg(errp,
+                   "Unknown hash algorithm %d",
+                   alg);
+        return -1;
+    }
+
+    ret = gcry_md_open(&md, qcrypto_hash_alg_map[alg], 0);
+
+    if (ret < 0) {
+        error_setg(errp,
+                   "Unable to initialize hash algorithm: %s",
+                   gcry_strerror(ret));
+        return -1;
+    }
+
+    for (i = 0; i < niov; i++) {
+        gcry_md_write(md, iov[i].iov_base, iov[i].iov_len);
+    }
+
+    ret = gcry_md_get_algo_dlen(qcrypto_hash_alg_map[alg]);
+    if (ret <= 0) {
+        error_setg(errp,
+                   "Unable to get hash length: %s",
+                   gcry_strerror(ret));
+        goto error;
+    }
+    if (*resultlen == 0) {
+        *resultlen = ret;
+        *result = g_new0(uint8_t, *resultlen);
+    } else if (*resultlen != ret) {
+        error_setg(errp,
+                   "Result buffer size %zu is smaller than hash %d",
+                   *resultlen, ret);
+        goto error;
+    }
+
+    digest = gcry_md_read(md, 0);
+    if (!digest) {
+        error_setg(errp,
+                   "No digest produced");
+        goto error;
+    }
+    memcpy(*result, digest, *resultlen);
+
+    gcry_md_close(md);
+    return 0;
+
+ error:
+    gcry_md_close(md);
+    return -1;
+}
diff --git a/crypto/hash-nettle.c b/crypto/hash-nettle.c
new file mode 100644
index 0000000..8ec5572
--- /dev/null
+++ b/crypto/hash-nettle.c
@@ -0,0 +1,126 @@
+/*
+ * QEMU Crypto hash algorithms
+ *
+ * Copyright (c) 2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "crypto/hash.h"
+#include <nettle/md5.h>
+#include <nettle/sha.h>
+
+typedef void (*qcrypto_nettle_init)(void *ctx);
+typedef void (*qcrypto_nettle_write)(void *ctx,
+                                     unsigned int len,
+                                     const uint8_t *buf);
+typedef void (*qcrypto_nettle_result)(void *ctx,
+                                      unsigned int len,
+                                      uint8_t *buf);
+
+union qcrypto_hash_ctx {
+    struct md5_ctx md5;
+    struct sha1_ctx sha1;
+    struct sha256_ctx sha256;
+};
+
+struct qcrypto_hash_alg {
+    qcrypto_nettle_init init;
+    qcrypto_nettle_write write;
+    qcrypto_nettle_result result;
+    size_t len;
+} qcrypto_hash_alg_map[] = {
+    [QCRYPTO_HASH_ALG_MD5] = {
+        .init = (qcrypto_nettle_init)md5_init,
+        .write = (qcrypto_nettle_write)md5_update,
+        .result = (qcrypto_nettle_result)md5_digest,
+        .len = MD5_DIGEST_SIZE,
+    },
+    [QCRYPTO_HASH_ALG_SHA1] = {
+        .init = (qcrypto_nettle_init)sha1_init,
+        .write = (qcrypto_nettle_write)sha1_update,
+        .result = (qcrypto_nettle_result)sha1_digest,
+        .len = SHA1_DIGEST_SIZE,
+    },
+    [QCRYPTO_HASH_ALG_SHA256] = {
+        .init = (qcrypto_nettle_init)sha256_init,
+        .write = (qcrypto_nettle_write)sha256_update,
+        .result = (qcrypto_nettle_result)sha256_digest,
+        .len = SHA256_DIGEST_SIZE,
+    },
+};
+
+gboolean qcrypto_hash_supports(QCryptoHashAlgorithm alg)
+{
+    if (alg < G_N_ELEMENTS(qcrypto_hash_alg_map) &&
+        qcrypto_hash_alg_map[alg].init != NULL) {
+        return true;
+    }
+    return false;
+}
+
+
+int qcrypto_hash_bytesv(QCryptoHashAlgorithm alg,
+                        const struct iovec *iov,
+                        size_t niov,
+                        uint8_t **result,
+                        size_t *resultlen,
+                        Error **errp)
+{
+    int i;
+    union qcrypto_hash_ctx ctx;
+
+    if (alg >= G_N_ELEMENTS(qcrypto_hash_alg_map) ||
+        qcrypto_hash_alg_map[alg].init == NULL) {
+        error_setg(errp,
+                   "Unknown hash algorithm %d",
+                   alg);
+        return -1;
+    }
+
+    qcrypto_hash_alg_map[alg].init(&ctx);
+
+    for (i = 0; i < niov; i++) {
+        /* Some versions of nettle have functions
+         * declared with 'int' instead of 'size_t'
+         * so to be safe avoid writing more than
+         * UINT_MAX bytes at a time
+         */
+        size_t len = iov[i].iov_len;
+        uint8_t *base = iov[i].iov_base;
+        while (len) {
+            size_t shortlen = MIN(len, UINT_MAX);
+            qcrypto_hash_alg_map[alg].write(&ctx, len, base);
+            len -= shortlen;
+            base += len;
+        }
+    }
+
+    if (*resultlen == 0) {
+        *resultlen = qcrypto_hash_alg_map[alg].len;
+        *result = g_new0(uint8_t, *resultlen);
+    } else if (*resultlen != qcrypto_hash_alg_map[alg].len) {
+        error_setg(errp,
+                   "Result buffer size %zu is smaller than hash %zu",
+                   *resultlen, qcrypto_hash_alg_map[alg].len);
+        return -1;
+    }
+
+    qcrypto_hash_alg_map[alg].result(&ctx, *resultlen, *result);
+
+    return 0;
+}
diff --git a/crypto/hash-stub.c b/crypto/hash-stub.c
new file mode 100644
index 0000000..8a9b8d4
--- /dev/null
+++ b/crypto/hash-stub.c
@@ -0,0 +1,41 @@
+/*
+ * QEMU Crypto hash algorithms
+ *
+ * Copyright (c) 2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "crypto/hash.h"
+
+gboolean qcrypto_hash_supports(QCryptoHashAlgorithm alg G_GNUC_UNUSED)
+{
+    return false;
+}
+
+int qcrypto_hash_bytesv(QCryptoHashAlgorithm alg,
+                        const struct iovec *iov G_GNUC_UNUSED,
+                        size_t niov G_GNUC_UNUSED,
+                        uint8_t **result G_GNUC_UNUSED,
+                        size_t *resultlen G_GNUC_UNUSED,
+                        Error **errp)
+{
+    error_setg(errp,
+               "Hash algorithm %d not supported without GNUTLS",
+               alg);
+    return -1;
+}
diff --git a/crypto/hash.c b/crypto/hash.c
index 2907bff..aa2a26e 100644
--- a/crypto/hash.c
+++ b/crypto/hash.c
@@ -22,12 +22,6 @@
 #include "qapi/error.h"
 #include "crypto/hash.h"
 
-#ifdef CONFIG_GNUTLS_HASH
-#include <gnutls/gnutls.h>
-#include <gnutls/crypto.h>
-#endif
-
-
 static size_t qcrypto_hash_alg_size[QCRYPTO_HASH_ALG__MAX] = {
     [QCRYPTO_HASH_ALG_MD5] = 16,
     [QCRYPTO_HASH_ALG_SHA1] = 20,
@@ -41,105 +35,6 @@ size_t qcrypto_hash_digest_len(QCryptoHashAlgorithm alg)
 }
 
 
-#ifdef CONFIG_GNUTLS_HASH
-static int qcrypto_hash_alg_map[QCRYPTO_HASH_ALG__MAX] = {
-    [QCRYPTO_HASH_ALG_MD5] = GNUTLS_DIG_MD5,
-    [QCRYPTO_HASH_ALG_SHA1] = GNUTLS_DIG_SHA1,
-    [QCRYPTO_HASH_ALG_SHA256] = GNUTLS_DIG_SHA256,
-};
-
-gboolean qcrypto_hash_supports(QCryptoHashAlgorithm alg)
-{
-    if (alg < G_N_ELEMENTS(qcrypto_hash_alg_map)) {
-        return true;
-    }
-    return false;
-}
-
-
-int qcrypto_hash_bytesv(QCryptoHashAlgorithm alg,
-                        const struct iovec *iov,
-                        size_t niov,
-                        uint8_t **result,
-                        size_t *resultlen,
-                        Error **errp)
-{
-    int i, ret;
-    gnutls_hash_hd_t dig;
-
-    if (alg >= G_N_ELEMENTS(qcrypto_hash_alg_map)) {
-        error_setg(errp,
-                   "Unknown hash algorithm %d",
-                   alg);
-        return -1;
-    }
-
-    ret = gnutls_hash_init(&dig, qcrypto_hash_alg_map[alg]);
-
-    if (ret < 0) {
-        error_setg(errp,
-                   "Unable to initialize hash algorithm: %s",
-                   gnutls_strerror(ret));
-        return -1;
-    }
-
-    for (i = 0; i < niov; i++) {
-        ret = gnutls_hash(dig, iov[i].iov_base, iov[i].iov_len);
-        if (ret < 0) {
-            error_setg(errp,
-                       "Unable process hash data: %s",
-                       gnutls_strerror(ret));
-            goto error;
-        }
-    }
-
-    ret = gnutls_hash_get_len(qcrypto_hash_alg_map[alg]);
-    if (ret <= 0) {
-        error_setg(errp,
-                   "Unable to get hash length: %s",
-                   gnutls_strerror(ret));
-        goto error;
-    }
-    if (*resultlen == 0) {
-        *resultlen = ret;
-        *result = g_new0(uint8_t, *resultlen);
-    } else if (*resultlen != ret) {
-        error_setg(errp,
-                   "Result buffer size %zu is smaller than hash %d",
-                   *resultlen, ret);
-        goto error;
-    }
-
-    gnutls_hash_deinit(dig, *result);
-    return 0;
-
- error:
-    gnutls_hash_deinit(dig, NULL);
-    return -1;
-}
-
-#else /* ! CONFIG_GNUTLS_HASH */
-
-gboolean qcrypto_hash_supports(QCryptoHashAlgorithm alg G_GNUC_UNUSED)
-{
-    return false;
-}
-
-int qcrypto_hash_bytesv(QCryptoHashAlgorithm alg,
-                        const struct iovec *iov G_GNUC_UNUSED,
-                        size_t niov G_GNUC_UNUSED,
-                        uint8_t **result G_GNUC_UNUSED,
-                        size_t *resultlen G_GNUC_UNUSED,
-                        Error **errp)
-{
-    error_setg(errp,
-               "Hash algorithm %d not supported without GNUTLS",
-               alg);
-    return -1;
-}
-
-#endif /* ! CONFIG_GNUTLS_HASH */
-
 int qcrypto_hash_bytes(QCryptoHashAlgorithm alg,
                        const char *buf,
                        size_t len,