Add "random malloc failure" tooling

Still needs to be documented, somehow/somewhere.

The env var OPENSSL_MALLOC_FAILURES controls how often malloc/realloc
should fail.  It's a set of fields separated by semicolons.  Each field
is a count and optional percentage (separated by @) which defaults to 100.
If count is zero then it lasts "forever."  For example: 100;@25 means the
first 100 allocations pass, then the rest have a 25% chance of failing
until the program exits or crashes.

If env var OPENSSL_MALLOC_FD parses as a positive integer, a record
of all malloc "shouldfail" tests is written to that file descriptor.
If a malloc will fail, and OPENSSL_NO_CRYPTO_MDEBUG_BACKTRACE is not set
(platform specific), then a backtrace will be written to the descriptor
when a malloc fails.  This can be useful because a malloc may fail but
not be checked, and problems will only occur later.

Reviewed-by: Richard Levitte <levitte@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/1252)

3 files changed, 93 insertions, 0 deletions

diff --git a/crypto/include/internal/cryptlib_int.h b/crypto/include/internal/cryptlib_int.h
index 8e2a719..60241d1 100644
--- a/crypto/include/internal/cryptlib_int.h
+++ b/crypto/include/internal/cryptlib_int.h
@@ -29,3 +29,4 @@ int ossl_init_thread_start(uint64_t opts);
 # define OPENSSL_INIT_THREAD_ASYNC           0x01
 # define OPENSSL_INIT_THREAD_ERR_STATE       0x02
 
+void ossl_malloc_setup_failures(void);
diff --git a/crypto/init.c b/crypto/init.c
index 3f91119..8036654 100644
--- a/crypto/init.c
+++ b/crypto/init.c
@@ -68,6 +68,9 @@ DEFINE_RUN_ONCE_STATIC(ossl_init_base)
 #ifdef OPENSSL_INIT_DEBUG
     fprintf(stderr, "OPENSSL_INIT: ossl_init_base: Setting up stop handlers\n");
 #endif
+#ifndef OPENSSL_NO_CRYPTO_MDEBUG
+    ossl_malloc_setup_failures();
+#endif
     /*
      * We use a dummy thread local key here. We use the destructor to detect
      * when the thread is going to stop (where that feature is available)
diff --git a/crypto/mem.c b/crypto/mem.c
index 02aa43a..2e8a00c 100644
--- a/crypto/mem.c
+++ b/crypto/mem.c
@@ -12,6 +12,10 @@
 #include <limits.h>
 #include <openssl/crypto.h>
 #include "internal/cryptlib.h"
+#include "internal/cryptlib_int.h"
+#ifndef OPENSSL_NO_CRYPTO_MDEBUG_BACKTRACE
+# include <execinfo.h>
+#endif
 
 /*
  * the following pointers may be changed as long as 'allow_customize' is set
@@ -26,9 +30,21 @@ static void (*free_impl)(void *, const char *, int)
     = CRYPTO_free;
 
 #ifndef OPENSSL_NO_CRYPTO_MDEBUG
+static char *md_failstring;
+static long md_count;
+static int md_percent = 100;
+static int md_tracefd = -1;
 static int call_malloc_debug = 1;
+
+static void parseit(void);
+static int shouldfail(void);
+
+# define FAILTEST() if (shouldfail()) return NULL
+
 #else
 static int call_malloc_debug = 0;
+
+# define FAILTEST() /* empty */
 #endif
 
 int CRYPTO_set_mem_functions(
@@ -68,6 +84,76 @@ void CRYPTO_get_mem_functions(
         *f = free_impl;
 }
 
+#ifndef OPENSSL_NO_CRYPTO_MDEBUG
+/*
+ * Parse a "malloc failure spec" string.  This likes like a set of fields
+ * separated by semicolons.  Each field has a count and an optional failure
+ * percentage.  For example:
+ *          100;100@25;@100
+ * This means 100 mallocs succeed, then next 100 fail 25% of the time, and
+ * all remaining (count is zero) succeed.
+ */
+static void parseit(void)
+{
+    char *semi = strchr(md_failstring, ';');
+    char *atsign;
+
+    if (semi != NULL)
+        *semi++ = '\0';
+
+    /* Get the count (atol will stop at the @ if there), and percentage */
+    md_count = atol(md_failstring);
+    atsign = strchr(md_failstring, '@');
+    md_percent = atsign == NULL ? 100 : atoi(atsign + 1);
+
+    if (semi != NULL)
+        md_failstring = semi;
+}
+
+/*
+ * See if the current malloc should fail.
+ */
+static int shouldfail(void)
+{
+    int roll = (int)(random() % 100);
+    int shouldfail = roll > md_percent;
+    char buff[80];
+
+    if (md_tracefd > 0) {
+        BIO_snprintf(buff, sizeof(buff),
+                     "%c C%ld %%%d R%d\n",
+                     shouldfail ? '-' : '+', md_count, md_percent, roll);
+        write(md_tracefd, buff, strlen(buff));
+#ifndef OPENSSL_NO_CRYPTO_MDEBUG_BACKTRACE
+        if (shouldfail) {
+            void *addrs[30];
+            int num = backtrace(addrs, OSSL_NELEM(addrs));
+
+            backtrace_symbols_fd(addrs, num, md_tracefd);
+        }
+#endif
+    }
+
+    if (md_count) {
+        /* If we used up this one, go to the next. */
+        if (--md_count == 0)
+            parseit();
+    }
+
+    return shouldfail;
+}
+
+void ossl_malloc_setup_failures(void)
+{
+    const char *cp = getenv("OPENSSL_MALLOC_FAILURES");
+
+    if (cp != NULL && (md_failstring = strdup(cp)) != NULL)
+        parseit();
+    if ((cp = getenv("OPENSSL_MALLOC_FD")) != NULL)
+        md_tracefd = atoi(cp);
+}
+#endif
+
 void *CRYPTO_malloc(size_t num, const char *file, int line)
 {
     void *ret = NULL;
@@ -78,6 +164,7 @@ void *CRYPTO_malloc(size_t num, const char *file, int line)
     if (num <= 0)
         return NULL;
 
+    FAILTEST();
     allow_customize = 0;
 #ifndef OPENSSL_NO_CRYPTO_MDEBUG
     if (call_malloc_debug) {
@@ -99,6 +186,7 @@ void *CRYPTO_zalloc(size_t num, const char *file, int line)
 {
     void *ret = CRYPTO_malloc(num, file, line);
 
+    FAILTEST();
     if (ret != NULL)
         memset(ret, 0, num);
     return ret;
@@ -109,6 +197,7 @@ void *CRYPTO_realloc(void *str, size_t num, const char *file, int line)
     if (realloc_impl != NULL && realloc_impl != &CRYPTO_realloc)
         return realloc_impl(str, num, file, line);
 
+    FAILTEST();
     if (str == NULL)
         return CRYPTO_malloc(num, file, line);