Refactor the test framework testutil

It's now built as a static library, and greatly simplified for test
programs, which no longer need to include test_main_custom.h or
test_main.h and link with the corresponding object files.

Reviewed-by: Rich Salz <rsalz@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/3243)

5 files changed, 630 insertions, 0 deletions

diff --git a/test/testutil/basic_output.c b/test/testutil/basic_output.c
new file mode 100644
index 0000000..ac413a6
--- /dev/null
+++ b/test/testutil/basic_output.c
@@ -0,0 +1,80 @@
+/*
+ * Copyright 2017 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Licensed under the OpenSSL license (the "License").  You may not use
+ * this file except in compliance with the License.  You can obtain a copy
+ * in the file LICENSE in the source distribution or at
+ * https://www.openssl.org/source/license.html
+ */
+
+#include "../testutil.h"
+
+#include <openssl/crypto.h>
+#include <openssl/bio.h>
+
+BIO *bio_out = NULL;
+BIO *bio_err = NULL;
+
+#ifdef OPENSSL_USE_APPLINK
+/*
+ * Using BIO_new_fd() obligates the use of applinks on platforms where it's
+ * relevant.  Because it becomes a module of the libtestutil library and would
+ * be disregarded if not actively referred to, we have this dummy that does
+ * exactly this.  For any module that uses the rest of the routines here,
+ * OPENSSL_Applink should tag along for sure.
+ */
+void Applink_dummy(void);
+void Applink_dummy(void)
+{
+    OPENSSL_EXTERN void OPENSSL_Applink(void);
+
+    OPENSSL_Applink();
+}
+/* Generate an error for anyone who tries to actually use this dummy */
+# define Applink_dummy "DON'T USE THIS"
+#endif
+
+void test_open_streams(void)
+{
+    bio_out = BIO_new_fd(1, 0);
+    bio_err = BIO_new_fd(2, 0);
+
+    OPENSSL_assert(bio_out != NULL);
+    OPENSSL_assert(bio_err != NULL);
+}
+
+void test_close_streams(void)
+{
+    BIO_free(bio_out);
+    BIO_free(bio_err);
+}
+
+int test_puts_stdout(const char *str)
+{
+    return BIO_puts(bio_out, str);
+}
+
+int test_puts_stderr(const char *str)
+{
+    return BIO_puts(bio_err, str);
+}
+
+int test_vprintf_stdout(const char *fmt, va_list ap)
+{
+    return BIO_vprintf(bio_out, fmt, ap);
+}
+
+int test_vprintf_stderr(const char *fmt, va_list ap)
+{
+    return BIO_vprintf(bio_err, fmt, ap);
+}
+
+int test_flush_stdout(void)
+{
+    return BIO_flush(bio_out);
+}
+
+int test_flush_stderr(void)
+{
+    return BIO_flush(bio_err);
+}
diff --git a/test/testutil/driver.c b/test/testutil/driver.c
new file mode 100644
index 0000000..e70fd21
--- /dev/null
+++ b/test/testutil/driver.c
@@ -0,0 +1,167 @@
+/*
+ * Copyright 2016-2017 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Licensed under the OpenSSL license (the "License").  You may not use
+ * this file except in compliance with the License.  You can obtain a copy
+ * in the file LICENSE in the source distribution or at
+ * https://www.openssl.org/source/license.html
+ */
+
+#include "../testutil.h"
+
+#include <string.h>
+#include <assert.h>
+
+#include "../../e_os.h"
+#include <openssl/bio.h>
+
+/*
+ * Declares the structures needed to register each test case function.
+ */
+typedef struct test_info {
+    const char *test_case_name;
+    int (*test_fn) ();
+    int (*param_test_fn)(int idx);
+    int num;
+} TEST_INFO;
+
+static TEST_INFO all_tests[1024];
+static int num_tests = 0;
+/*
+ * A parameterised tests runs a loop of test cases.
+ * |num_test_cases| counts the total number of test cases
+ * across all tests.
+ */
+static int num_test_cases = 0;
+
+void add_test(const char *test_case_name, int (*test_fn) ())
+{
+    assert(num_tests != OSSL_NELEM(all_tests));
+    all_tests[num_tests].test_case_name = test_case_name;
+    all_tests[num_tests].test_fn = test_fn;
+    all_tests[num_tests].num = -1;
+    ++num_tests;
+    ++num_test_cases;
+}
+
+void add_all_tests(const char *test_case_name, int(*test_fn)(int idx),
+                   int num)
+{
+    assert(num_tests != OSSL_NELEM(all_tests));
+    all_tests[num_tests].test_case_name = test_case_name;
+    all_tests[num_tests].param_test_fn = test_fn;
+    all_tests[num_tests].num = num;
+    ++num_tests;
+    num_test_cases += num;
+}
+
+#ifndef OPENSSL_NO_CRYPTO_MDEBUG
+static int should_report_leaks()
+{
+    /*
+     * When compiled with enable-crypto-mdebug, OPENSSL_DEBUG_MEMORY=0
+     * can be used to disable leak checking at runtime.
+     * Note this only works when running the test binary manually;
+     * the test harness always enables OPENSSL_DEBUG_MEMORY.
+     */
+    char *mem_debug_env = getenv("OPENSSL_DEBUG_MEMORY");
+
+    return mem_debug_env == NULL
+        || (strcmp(mem_debug_env, "0") && strcmp(mem_debug_env, ""));
+}
+#endif
+
+
+static int err_cb(const char *str, size_t len, void *u)
+{
+    return test_puts_stderr(str);
+}
+
+void setup_test()
+{
+    test_open_streams();
+
+#ifndef OPENSSL_NO_CRYPTO_MDEBUG
+    if (should_report_leaks()) {
+        CRYPTO_set_mem_debug(1);
+        CRYPTO_mem_ctrl(CRYPTO_MEM_CHECK_ON);
+    }
+#endif
+}
+
+int finish_test(int ret)
+{
+#ifndef OPENSSL_NO_CRYPTO_MDEBUG
+    if (should_report_leaks() && CRYPTO_mem_leaks_cb(err_cb, NULL) <= 0)
+        return EXIT_FAILURE;
+#endif
+
+    test_close_streams();
+
+    return ret;
+}
+
+static void finalize(int success)
+{
+    if (success)
+        ERR_clear_error();
+    else
+        ERR_print_errors_cb(err_cb, NULL);
+}
+
+static void helper_printf_stdout(const char *fmt, ...)
+{
+    va_list ap;
+
+    va_start(ap, fmt);
+    test_vprintf_stdout(fmt, ap);
+    va_end(ap);
+}
+
+int run_tests(const char *test_prog_name)
+{
+    int num_failed = 0;
+    int i, j;
+
+    helper_printf_stdout("%s: %d test case%s\n", test_prog_name, num_test_cases,
+                         num_test_cases == 1 ? "" : "s");
+    test_flush_stdout();
+
+    for (i = 0; i != num_tests; ++i) {
+        if (all_tests[i].num == -1) {
+            int ret = all_tests[i].test_fn();
+
+            if (!ret) {
+                helper_printf_stdout("** %s failed **\n--------\n",
+                                     all_tests[i].test_case_name);
+                test_flush_stdout();
+                ++num_failed;
+            }
+            finalize(ret);
+        } else {
+            for (j = 0; j < all_tests[i].num; j++) {
+                int ret = all_tests[i].param_test_fn(j);
+
+                if (!ret) {
+                    helper_printf_stdout("** %s failed test %d\n--------\n",
+                                         all_tests[i].test_case_name, j);
+                    test_flush_stdout();
+                    ++num_failed;
+                }
+                finalize(ret);
+            }
+        }
+    }
+
+    if (num_failed != 0) {
+        helper_printf_stdout("%s: %d test%s failed (out of %d)\n",
+                             test_prog_name, num_failed,
+                             num_failed != 1 ? "s" : "", num_test_cases);
+        test_flush_stdout();
+        return EXIT_FAILURE;
+    }
+    helper_printf_stdout("  All tests passed.\n");
+    test_flush_stdout();
+    return EXIT_SUCCESS;
+}
+
diff --git a/test/testutil/main.c b/test/testutil/main.c
new file mode 100644
index 0000000..435a358
--- /dev/null
+++ b/test/testutil/main.c
@@ -0,0 +1,20 @@
+/*
+ * Copyright 2016-2017 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Licensed under the OpenSSL license (the "License").  You may not use
+ * this file except in compliance with the License.  You can obtain a copy
+ * in the file LICENSE in the source distribution or at
+ * https://www.openssl.org/source/license.html
+ */
+
+#include "../testutil.h"
+
+int main(int argc, char *argv[])
+{
+    int ret;
+    setup_test();
+
+    ret = test_main(argc, argv);
+
+    return finish_test(ret);
+}
diff --git a/test/testutil/test_main.c b/test/testutil/test_main.c
new file mode 100644
index 0000000..0152421
--- /dev/null
+++ b/test/testutil/test_main.c
@@ -0,0 +1,21 @@
+/*
+ * Copyright 2016-2017 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Licensed under the OpenSSL license (the "License").  You may not use
+ * this file except in compliance with the License.  You can obtain a copy
+ * in the file LICENSE in the source distribution or at
+ * https://www.openssl.org/source/license.html
+ */
+
+#include "../testutil.h"
+
+#include <stdio.h>
+
+int test_main(int argc, char *argv[])
+{
+    if (argc > 1)
+        test_puts_stderr("Warning: ignoring extra command-line arguments.\n");
+
+    register_tests();
+    return run_tests(argv[0]);
+}
diff --git a/test/testutil/tests.c b/test/testutil/tests.c
new file mode 100644
index 0000000..f00ec6c
--- /dev/null
+++ b/test/testutil/tests.c
@@ -0,0 +1,342 @@
+/*
+ * Copyright 2017 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Licensed under the OpenSSL license (the "License").  You may not use
+ * this file except in compliance with the License.  You can obtain a copy
+ * in the file LICENSE in the source distribution or at
+ * https://www.openssl.org/source/license.html
+ */
+
+#include "../testutil.h"
+
+#include <string.h>
+#include "../../e_os.h"
+
+/* The size of memory buffers to display on failure */
+#define MEM_BUFFER_SIZE     (21)
+
+/*
+ * A common routine to output test failure messages.  Generally this should not
+ * be called directly, rather it should be called by the following functions.
+ *
+ * |desc| is a printf formatted description with arguments |args| that is
+ * supplied by the user and |desc| can be NULL.  |type| is the data type
+ * that was tested (int, char, ptr, ...).  |fmt| is a system provided
+ * printf format with following arguments that spell out the failure
+ * details i.e. the actual values compared and the operator used.
+ *
+ * The typical use for this is from an utility test function:
+ *
+ * int test6(const char *file, int line, int n) {
+ *     if (n != 6) {
+ *         test_fail_message(1, file, line, "int", "value %d is not %d", n, 6);
+ *         return 0;
+ *     }
+ *     return 1;
+ * }
+ *
+ * calling test6(3, "oops") will return 0 and produce out along the lines of:
+ *      FAIL oops: (int) value 3 is not 6\n
+ *
+ * It general, test_fail_message should not be called directly.
+ */
+static void test_fail_message(const char *prefix, const char *file, int line,
+                              const char *type, const char *fmt, ...)
+            PRINTF_FORMAT(5, 6);
+
+static void helper_printf_stderr(const char *fmt, ...)
+{
+    va_list ap;
+
+    va_start(ap, fmt);
+    test_vprintf_stderr(fmt, ap);
+    va_end(ap);
+}
+
+static void test_fail_message_va(const char *prefix, const char *file, int line,
+                                 const char *type, const char *fmt, va_list ap)
+{
+    test_puts_stderr(prefix != NULL ? prefix : "ERROR");
+    test_puts_stderr(":");
+    if (type)
+        helper_printf_stderr(" (%s)", type);
+    if (fmt != NULL) {
+        test_puts_stderr(" ");
+        test_vprintf_stderr(fmt, ap);
+    }
+    if (file != NULL) {
+        helper_printf_stderr(" @ %s:%d", file, line);
+    }
+    test_puts_stderr("\n");
+    test_flush_stderr();
+}
+
+static void test_fail_message(const char *prefix, const char *file, int line,
+                              const char *type, const char *fmt, ...)
+{
+    va_list ap;
+
+    va_start(ap, fmt);
+    test_fail_message_va(prefix, file, line, type, fmt, ap);
+    va_end(ap);
+}
+
+void test_info_c90(const char *desc, ...)
+{
+    va_list ap;
+
+    va_start(ap, desc);
+    test_fail_message_va("INFO", NULL, -1, NULL, desc, ap);
+    va_end(ap);
+}
+
+void test_info(const char *file, int line, const char *desc, ...)
+{
+    va_list ap;
+
+    va_start(ap, desc);
+    test_fail_message_va("INFO", file, line, NULL, desc, ap);
+    va_end(ap);
+}
+
+void test_error_c90(const char *desc, ...)
+{
+    va_list ap;
+
+    va_start(ap, desc);
+    test_fail_message(NULL, NULL, -1, NULL, desc, ap);
+    va_end(ap);
+}
+
+void test_error(const char *file, int line, const char *desc, ...)
+{
+    va_list ap;
+
+    va_start(ap, desc);
+    test_fail_message_va(NULL, file, line, NULL, desc, ap);
+    va_end(ap);
+}
+
+/*
+ * Define some comparisons between pairs of various types.
+ * These functions return 1 if the test is true.
+ * Otherwise, they return 0 and pretty-print diagnostics.
+ *
+ * In each case the functions produced are:
+ *  int test_name_eq(const type t1, const type t2, const char *desc, ...);
+ *  int test_name_ne(const type t1, const type t2, const char *desc, ...);
+ *  int test_name_lt(const type t1, const type t2, const char *desc, ...);
+ *  int test_name_le(const type t1, const type t2, const char *desc, ...);
+ *  int test_name_gt(const type t1, const type t2, const char *desc, ...);
+ *  int test_name_ge(const type t1, const type t2, const char *desc, ...);
+ *
+ * The t1 and t2 arguments are to be compared for equality, inequality,
+ * less than, less than or equal to, greater than and greater than or
+ * equal to respectively.  If the specified condition holds, the functions
+ * return 1.  If the condition does not hold, the functions print a diagnostic
+ * message and return 0.
+ *
+ * The desc argument is a printf format string followed by its arguments and
+ * this is included in the output if the condition being tested for is false.
+ */
+#define DEFINE_COMPARISON(type, name, opname, op, fmt)                  \
+    int test_ ## name ## _ ## opname(const char *file, int line,        \
+                                     const char *s1, const char *s2,    \
+                                     const type t1, const type t2)      \
+    {                                                                   \
+        if (t1 op t2)                                                   \
+            return 1;                                                   \
+        test_fail_message(NULL, file, line, #type,                      \
+                          "%s [" fmt "] " #op " %s [" fmt "]",          \
+                          s1, t1, s2, t2);                              \
+        return 0;                                                       \
+    }
+
+#define DEFINE_COMPARISONS(type, name, fmt)                             \
+    DEFINE_COMPARISON(type, name, eq, ==, fmt)                          \
+    DEFINE_COMPARISON(type, name, ne, !=, fmt)                          \
+    DEFINE_COMPARISON(type, name, lt, <, fmt)                           \
+    DEFINE_COMPARISON(type, name, le, <=, fmt)                          \
+    DEFINE_COMPARISON(type, name, gt, >, fmt)                           \
+    DEFINE_COMPARISON(type, name, ge, >=, fmt)
+
+DEFINE_COMPARISONS(int, int, "%d")
+DEFINE_COMPARISONS(unsigned int, uint, "%u")
+DEFINE_COMPARISONS(char, char, "%c")
+DEFINE_COMPARISONS(unsigned char, uchar, "%u")
+DEFINE_COMPARISONS(long, long, "%ld")
+DEFINE_COMPARISONS(unsigned long, ulong, "%lu")
+DEFINE_COMPARISONS(size_t, size_t, "%zu")
+
+DEFINE_COMPARISON(void *, ptr, eq, ==, "%p")
+DEFINE_COMPARISON(void *, ptr, ne, !=, "%p")
+
+int test_ptr_null(const char *file, int line, const char *s, const void *p)
+{
+    if (p == NULL)
+        return 1;
+    test_fail_message(NULL, file, line, "ptr", "%s [%p] == NULL", s, p);
+    return 0;
+}
+
+int test_ptr(const char *file, int line, const char *s, const void *p)
+{
+    if (p != NULL)
+        return 1;
+    test_fail_message(NULL, file, line, "ptr", "%s [%p] != NULL", s, p);
+    return 0;
+}
+
+int test_true(const char *file, int line, const char *s, int b)
+{
+    if (b)
+        return 1;
+    test_fail_message(NULL, file, line, "bool", "%s [false] == true", s);
+    return 0;
+}
+
+int test_false(const char *file, int line, const char *s, int b)
+{
+    if (!b)
+        return 1;
+    test_fail_message(NULL, file, line, "bool", "%s [true] == false", s);
+    return 0;
+}
+
+static const char *print_string_maybe_null(const char *s)
+{
+    return s == NULL ? "(NULL)" : s;
+}
+
+int test_str_eq(const char *file, int line, const char *st1, const char *st2,
+                const char *s1, const char *s2)
+{
+    if (s1 == NULL && s2 == NULL)
+      return 1;
+    if (s1 == NULL || s2 == NULL || strcmp(s1, s2) != 0) {
+        test_fail_message(NULL, file, line, "string", "%s [%s] == %s [%s]",
+                          st1, print_string_maybe_null(s1),
+                          st2, print_string_maybe_null(s2));
+        return 0;
+    }
+    return 1;
+}
+
+int test_str_ne(const char *file, int line, const char *st1, const char *st2,
+                const char *s1, const char *s2)
+{
+    if ((s1 == NULL) ^ (s2 == NULL))
+      return 1;
+    if (s1 == NULL || strcmp(s1, s2) == 0) {
+        test_fail_message(NULL, file, line, "string", "%s [%s] != %s [%s]",
+                          st1, print_string_maybe_null(s1),
+                          st2, print_string_maybe_null(s2));
+        return 0;
+    }
+    return 1;
+}
+
+int test_strn_eq(const char *file, int line, const char *st1, const char *st2,
+                 const char *s1, const char *s2, size_t len)
+{
+    int prec = (int)len;
+
+    if (s1 == NULL && s2 == NULL)
+      return 1;
+    if (s1 == NULL || s2 == NULL || strncmp(s1, s2, len) != 0) {
+        test_fail_message(NULL, file, line, "string", "%.s [%.*s] == %s [%.*s]",
+                          st1, prec, print_string_maybe_null(s1),
+                          st2, prec, print_string_maybe_null(s2));
+        return 0;
+    }
+    return 1;
+}
+
+int test_strn_ne(const char *file, int line, const char *st1, const char *st2,
+                 const char *s1, const char *s2, size_t len)
+{
+    int prec = (int)len;
+
+    if ((s1 == NULL) ^ (s2 == NULL))
+      return 1;
+    if (s1 == NULL || strncmp(s1, s2, len) == 0) {
+        test_fail_message(NULL, file, line, "string", "%s [%.*s] != %s [%.*s]",
+                          st1, prec, print_string_maybe_null(s1),
+                          st2, prec, print_string_maybe_null(s2));
+        return 0;
+    }
+    return 1;
+}
+
+/*
+ * We could use OPENSSL_buf2hexstr() to do this but trying to allocate memory
+ * in a failure state isn't generally a great idea.
+ */
+static const char *print_mem_maybe_null(const void *s, size_t n,
+                                        char out[MEM_BUFFER_SIZE])
+{
+    size_t i;
+    const unsigned char *p = (const unsigned char *)s;
+    int pad = 2*n >= MEM_BUFFER_SIZE;
+
+    if (s == NULL)
+        return "(NULL)";
+    if (pad)
+        n = MEM_BUFFER_SIZE-4;
+    
+    for (i=0; i<2*n; i++) {
+        unsigned char c = (i & 1) != 0 ? p[i / 2] & 15 : p[i / 2] >> 4;
+        out[i] = "0123456789abcdef"[c];
+    }
+    if (pad) {
+        out[i++] = '.';
+        out[i++] = '.';
+        out[i++] = '.';
+    }
+    out[i] = '\0';
+        
+    return out;
+}
+
+int test_mem_eq(const char *file, int line, const char *st1, const char *st2,
+                const void *s1, size_t n1, const void *s2, size_t n2)
+{
+    char b1[MEM_BUFFER_SIZE], b2[MEM_BUFFER_SIZE];
+
+    if (s1 == NULL && s2 == NULL)
+        return 1;
+    if (n1 != n2) {
+        test_fail_message(NULL, file, line, "memory",
+                          "size mismatch %s %s [%zu] != %s %s [%zu]",
+                          st1, print_mem_maybe_null(s1, n1, b1), n1,
+                          st2, print_mem_maybe_null(s2, n2, b2), n2);
+        return 0;
+    }
+    if (s1 == NULL || s2 == NULL || memcmp(s1, s2, n1) != 0) {
+        test_fail_message(NULL, file, line, "memory",
+                          "%s %s [%zu] != %s %s [%zu]",
+                          st1, print_mem_maybe_null(s1, n1, b1), n1,
+                          st2, print_mem_maybe_null(s2, n2, b2), n2);
+        return 0;
+    }
+    return 1;
+}
+
+int test_mem_ne(const char *file, int line, const char *st1, const char *st2,
+                const void *s1, size_t n1, const void *s2, size_t n2)
+{
+    char b1[MEM_BUFFER_SIZE], b2[MEM_BUFFER_SIZE];
+
+    if ((s1 == NULL) ^ (s2 == NULL))
+      return 1;
+    if (n1 != n2)
+        return 1;
+    if (s1 == NULL || memcmp(s1, s2, n1) == 0) {
+        test_fail_message(NULL, file, line, "memory",
+                          "%s %s [%zu] != %s %s [%zu]",
+                          st1, print_mem_maybe_null(s1, n1, b1), n1,
+                          st2, print_mem_maybe_null(s2, n2, b2), n2);
+        return 0;
+    }
+    return 1;
+}