Test skip option.

Provide C test cases with the option to skip tests and subtests.

Reviewed-by: Matt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/8695)

3 files changed, 47 insertions, 33 deletions

diff --git a/test/testutil/driver.c b/test/testutil/driver.c
index 10d74e2..2da83ed 100644
--- a/test/testutil/driver.c
+++ b/test/testutil/driver.c
@@ -280,20 +280,20 @@ void set_test_title(const char *title)
     test_title = title == NULL ? NULL : strdup(title);
 }
 
-PRINTF_FORMAT(2, 3) static void test_verdict(int pass, const char *extra, ...)
+PRINTF_FORMAT(2, 3) static void test_verdict(int verdict,
+                                             const char *description, ...)
 {
     va_list ap;
 
     test_flush_stdout();
     test_flush_stderr();
 
-    test_printf_stdout("%*s%s", level, "", pass ? "ok" : "not ok");
-    if (extra != NULL) {
-        test_printf_stdout(" ");
-        va_start(ap, extra);
-        test_vprintf_stdout(extra, ap);
-        va_end(ap);
-    }
+    test_printf_stdout("%*s%s ", level, "", verdict != 0 ? "ok" : "not ok");
+    va_start(ap, description);
+    test_vprintf_stdout(description, ap);
+    va_end(ap);
+    if (verdict == TEST_SKIP_CODE)
+        test_printf_stdout(" # skipped");
     test_printf_stdout("\n");
     test_flush_stdout();
 }
@@ -349,20 +349,14 @@ int run_tests(const char *test_prog_name)
             }
             test_flush_stdout();
         } else if (all_tests[i].num == -1) {
-            int ret = 0;
-
             set_test_title(all_tests[i].test_case_name);
-            ret = all_tests[i].test_fn();
-            verdict = 1;
-            if (!ret) {
-                verdict = 0;
-                ++num_failed;
-            }
+            verdict = all_tests[i].test_fn();
             test_verdict(verdict, "%d - %s", ii + 1, test_title);
-            finalize(ret);
+            finalize(verdict != 0);
         } else {
             int num_failed_inner = 0;
 
+            verdict = TEST_SKIP_CODE;
             level += 4;
             if (all_tests[i].subtest && single_iter == -1) {
                 test_printf_stdout("%*s# Subtest: %s\n", level, "",
@@ -381,39 +375,34 @@ int run_tests(const char *test_prog_name)
                 while (jstep == 0 || gcd(all_tests[i].num, jstep) != 1);
 
             for (jj = 0; jj < all_tests[i].num; jj++) {
-                int ret;
+                int v;
 
                 j = (j + jstep) % all_tests[i].num;
                 if (single_iter != -1 && ((jj + 1) != single_iter))
                     continue;
                 set_test_title(NULL);
-                ret = all_tests[i].param_test_fn(j);
+                v = all_tests[i].param_test_fn(j);
 
-                if (!ret)
+                if (v == 0) {
                     ++num_failed_inner;
+                    verdict = 0;
+                } else if (v != TEST_SKIP_CODE && verdict != 0) {
+                    verdict = 1;
+                }
 
-                finalize(ret);
+                finalize(v != 0);
 
                 if (all_tests[i].subtest) {
-                    verdict = 1;
-                    if (!ret) {
-                        verdict = 0;
-                        ++num_failed_inner;
-                    }
                     if (test_title != NULL)
-                        test_verdict(verdict, "%d - %s", jj + 1, test_title);
+                        test_verdict(v, "%d - %s", jj + 1, test_title);
                     else
-                        test_verdict(verdict, "%d - iteration %d",
-                                     jj + 1, j + 1);
+                        test_verdict(v, "%d - iteration %d", jj + 1, j + 1);
                 }
             }
 
             level -= 4;
-            verdict = 1;
-            if (num_failed_inner) {
-                verdict = 0;
+            if (verdict == 0)
                 ++num_failed;
-            }
             test_verdict(verdict, "%d - %s", ii + 1,
                          all_tests[i].test_case_name);
         }
diff --git a/test/testutil/tests.c b/test/testutil/tests.c
index 05b0688..56177cd 100644
--- a/test/testutil/tests.c
+++ b/test/testutil/tests.c
@@ -157,6 +157,29 @@ void test_note(const char *fmt, ...)
     test_flush_stderr();
 }
 
+
+int test_skip(const char *file, int line, const char *desc, ...)
+{
+    va_list ap;
+
+    va_start(ap, desc);
+    test_fail_message_va("SKIP", file, line, NULL, NULL, NULL, NULL, desc, ap);
+    va_end(ap);
+    return TEST_SKIP_CODE;
+}
+
+int test_skip_c90(const char *desc, ...)
+{
+    va_list ap;
+
+    va_start(ap, desc);
+    test_fail_message_va("SKIP", NULL, -1, NULL, NULL, NULL, NULL, desc, ap);
+    va_end(ap);
+    test_printf_stderr("\n");
+    return TEST_SKIP_CODE;
+}
+
+
 void test_openssl_errors(void)
 {
     ERR_print_errors_cb(openssl_error_cb, NULL);
diff --git a/test/testutil/tu_local.h b/test/testutil/tu_local.h
index 049d7b1..ede4546 100644
--- a/test/testutil/tu_local.h
+++ b/test/testutil/tu_local.h
@@ -12,6 +12,8 @@
 #include <openssl/bio.h>
 #include "../testutil.h"
 
+#define TEST_SKIP_CODE  123
+
 int subtest_level(void);
 int openssl_error_cb(const char *str, size_t len, void *u);
 const BIO_METHOD *BIO_f_tap(void);