#!/bin/sh

### testtool -- Tool to run tests

# BSD Owl Scripts (https://github.com/michipili/bsdowl)
# This file is part of BSD Owl Scripts
#
# Copyright © 2002–2017 Michael Grünewald. All Rights Reserved.
#
# This file must be used under the terms of the BSD license.
# This source file is licensed as described in the file LICENSE, which
# you should have received as part of this distribution.

: ${BSDOWL_TOPLEVELDIR:=$(git rev-parse --show-toplevel)}
: ${BSDOWL_TESTDIR:=${HOME}/obj/bsdowl.testsuite}

unset EXTERNAL
unset MAKEFLAGS
unset MAKEOBJDIR
unset MAKEOBJDIRPREFIX
unset MODULE
unset OFFICER
unset PACKAGE
unset PACKAGEDIR
unset SRCDIR
unset VERSION
unset WRKDIR


# failwith [-x STATUS] PRINTF-LIKE-ARGV
#  Fail with the given diagnostic message
#
# The -x flag can be used to convey a custom exit status, instead of
# the value 1.  A newline is automatically added to the output.

failwith()
{
    local OPTIND OPTION OPTARG status

    status=1
    OPTIND=1

    while getopts 'x:' OPTION; do
        case ${OPTION} in
            x)	status="${OPTARG}";;
            *)	eprintf 'failwith: %s: Unsupported option.' "${OPTION}";;
        esac
    done

    shift $(expr ${OPTIND} - 1)
    {
        printf 'Failure: '
        printf "$@"
        printf '\n'
    } 1>&2
    exit "${status}"
}

eprintf()
{
    1>&2 printf "$@"
}


wlog()
{
    {
        printf "$@"
        printf '\n'
    } 1>&2
}

print_sep1()
{
    printf -- '================================================================================\n'
}

print_sep2()
{
    printf -- '--------------------------------------------------------------------------------\n'
}


#
# Pseudo-commands
#

testtool_make()
{
    if [ -z "${testtool_make__cmd}" ]; then
        case $(uname) in
            FreeBSD|NetBSD)
                testtool_make__cmd='make';;
            *)
                testtool_make__cmd='bmake';;
        esac
    fi
    "${testtool_make__cmd}" "$@"
}


# testtool_getvariable [-I INCLUDE-DIR][-f MAKEFILE] VARIABLE
#  Report the value of variables from make
#
# It is possible to retrieve several variables simultaneously. The
# tabular output consists has one column for each variable.

testtool_getvariable()
{
    local OPTIND OPTION OPTARG
    local query variable include makefile serial

    OPTIND=1
    makefile=''
    include=''
    variable=''

    while getopts "f:I:" OPTION; do
        case ${OPTION} in
            f)	makefile="${OPTARG}";;
            I)	include="${include}${include:+ }-I ${OPTARG}";;
            *)	failwith "getvariable: ${OPTION}: Unsupported option.";;
        esac
    done

    shift $(expr ${OPTIND} - 1)

    for variable in "$@"; do
        query="${query}${query:+ }-V ${variable}"
    done

    testtool_make\
        .MAKE.EXPAND_VARIABLES=1\
        ${makefile:+-f "${makefile}"}\
        ${include}\
        ${query}\
        | sed -n -e 'H;${x;s/\n/|/g;s/^|//;p;}'
}

#
# Prepare the test database
#

# testtool_db__find PATH
#  Find test-cases declarations in PATH

testtool_db__find()
{
    find "$@" -name '*.mk'\
        | xargs grep -l TEST_DESCRIPTION\
        | sort
}


# testtool_db__extract_index PATH
#  Prepare the index record corresponding to declrations found in PATH

testtool_db__extract_index()
{
    printf 'INDEX|%s|' "${file}";
    testtool_getvariable\
        -f "$1"\
        TEST_SOURCEDIR\
        TEST_SEQUENCE\
        TEST_DESCRIPTION
}

# testtool_db__extract_matrix PATH
#  Prepare the index record corresponding to declrations found in PATH

testtool_db__extract_matrix()
{
    local matrix var value

    matrix=$(testtool_getvariable -f "$1" TEST_MATRIX)
    if [ -n "${matrix}" ]; then
        for var in ${matrix}; do
            for value in $(testtool_getvariable -f "$1" "TEST_${var}"); do
                printf "MATRIX|${var}|%s\n" ${value}
            done
        done
    fi
}


# testtool_db__importscript
#  Filter reading files on stdin and writing importscript on stdout

testtool_db__importscript()
{
    (   for binding in $(testtool_run__macros_env); do
            export ${binding}
        done
        while read file; do
          testtool_db__extract_index "${file}"
          testtool_db__extract_matrix "${file}"
      done ) | awk -F '|' '
BEGIN{
  for(n=0;n<256;n++) {
    ord[sprintf("%c",n)] = n
  }
}

function hash(text, _prime, _modulo, _ax, _chars, _i)
{
  _prime = 104729;
  _modulo = 1048576;
  _ax = 0;
  split(text, _chars, "");
  for (_i=1; _i <= length(text); _i++) {
    _ax = (_ax * _prime + ord[_chars[_i]]) % _modulo;
  };
  return sprintf("%05x", _ax)
}

function q(text, _answer, _squote) {
  _squote = sprintf("%c", 39)
  _answer = text;
  gsub(_squote, _squote _squote, _answer);
  return sprintf("%s%s%s", _squote, _answer, _squote);
}

$1 == "INDEX" {
  currentid = hash($5);
  printf("INSERT INTO test_index VALUES(%s, %s, %s, %s, %s);\n", q(currentid), q($2), q($3), q($4),q($5));
}

$1 == "MATRIX" {
  printf("INSERT INTO test_matrix VALUES(%s, %s, %s);\n", q(currentid), q($2), q($3));
}
'
}

# testtool_dbinit
#  Initialise the test database

testtool_dbinit()
{
    rm -f "${testtool_database:?}"
    sqlite3 "${testtool_database:?}" <<EOF
CREATE TABLE test_index (
  id TEXT PRIMARY KEY,
  filename TEXT,
  sourcedir TEXT,
  sequence TEXT,
  description TEXT
);
CREATE TABLE test_matrix (
  id TEXT,
  name TEXT,
  value TEXT,
  FOREIGN KEY(id) REFERENCES test_index(id)
);
CREATE TABLE test_batch (
  id TEXT,
  timestamp TEXT,
  testid TEXT,
  status TEXT,
  env TEXT,
  FOREIGN KEY(testid) REFERENCES test_index(id)
);
EOF
}


# testtool_dbimport [-n]
#  Import data in the test database
#
# Options:
#  -n Dry run.

testtool_dbimport()
{
    local OPTIND OPTION OPTARG sink

    sink="sqlite3 ${testtool_database:?}"
    OPTIND=1

    while getopts 'n' OPTION; do
        case ${OPTION} in
            n)	sink='cat';;
            *)	eprintf 'failwith: %s: Unsupported option.' "${OPTION}";;
        esac
    done
    shift $(expr ${OPTIND} - 1)

    testtool_db__find "${BSDOWL_TOPLEVELDIR}/testsuite"\
        | testtool_db__importscript\
        | ${sink}
}


# testtool_batch [-n NAME] [-a] TEST1 ...
#  Require a batch of tests
#
# Options:
#  -a Require all tests instead of those provided by the arguments.
#  -n NAME
#     Use the batch NAME instead of 'current'.
#  -x EXCLUDE-LIST
#     Exclude given tests from the required batch.

testtool_batch()
{
    local OPTIND OPTION OPTARG batchall batchname timestamp batchexclude

    batchexclude=''
    batchall='no'
    batchname='current'
    timestamp=$(date '+%Y-%m-%dT%H:%M:%SZ')

    OPTIND=1

    while getopts 'an:x:' OPTION; do
        case ${OPTION} in
            a)	batchall='yes';;
            n)	batchname="${OPTARG}";;
            x)	batchexclude="${batchexclude}${batchexclude:+ }${OPTARG}";;
            *)	eprintf 'failwith: %s: Unsupported option.' "${OPTION}";;
        esac
    done
    shift $(expr ${OPTIND} - 1)

    sqlite3 ${testtool_database:?} <<EOF
DELETE FROM test_batch WHERE id = '${batchname}';
EOF

    ( if [ "${batchall}" = 'yes' ]; then
          sqlite3 "${testtool_database:?}" 'SELECT id FROM test_index WHERE sequence <> "IGNORE" ;'
      else
          printf '%s\n' "$@"
      fi )\
        | awk -v batchexclude="${batchexclude}" '
BEGIN {
  n = split(batchexclude, _excludelist, " ");
  for(i = 1; i <= n; ++i) {
    excludelist[_excludelist[i]];
  }
}
!($1 in excludelist) { print }
'\
        | ( while read id; do
                   testtool_batch__env "${id}"\
                       | awk\
                             -v timestamp="${timestamp}"\
                             -v id="${id}"\
                             -v batchname="${batchname}" '
function q(text, _answer, _squote) {
  _squote = sprintf("%c", 39)
  _answer = text;
  gsub(_squote, _squote _squote, _answer);
  return sprintf("%s%s%s", _squote, _answer, _squote);
}
{printf("INSERT INTO test_BATCH VALUES(%s, %s, %s, %s, %s);\n", q(batchname), q(timestamp), q(id), q("PENDING"),q($0));}
' | sqlite3 ${testtool_database:?}
               done )
}

testtool_batch__env()
{
    sqlite3\
        "${testtool_database:?}"\
        "SELECT name, value FROM test_matrix WHERE id = '$1';"\
        | awk -F '|' '
{
  size[$1] += 1;
  value[$1 "@" size[$1]] = $2;
}

END {
  total = 1;
  for(key in size) {
    total *= size[key];
  }
  repeat = 1;
  for(key in size) {
    for(i = 0; i < total / size[key] / repeat; ++i) {
      for(j = 0; j < size[key]; ++j) {
        for(k = 0; k < repeat; ++k) {
          pos = k + j*repeat + i*size[key]*repeat;
          a[pos] = a[pos] " " key "=" value[key "@" (j+1)];
        }
      }
    }
    repeat *= size[key];
  }
  for(i = 0; i < total; ++i) {
    gsub("^ ", "", a[i]);
    print(a[i]);
  }
}
'
}

testtool_makeobjdir()
{
    printf '%s/%s/%s/var/obj${CONFIGURATION:C@^.@/&@}${ARCHITECTURE:C@^.@/&@}${.CURDIR:S@^%s/%s/%s/var/src@@}'\
           "${BSDOWL_TESTDIR}"\
           "${batchname}"\
           "${batchoid}"\
           "${BSDOWL_TESTDIR}"\
           "${batchname}"\
           "${batchoid}"
}


testtool_makeobjdirprefix()
{
    printf '%s/%s/obj/%s' "${BSDOWL_TESTDIR}" "${batchname}" "${batchoid}"
}

# testtool_run_loop [-n][-k]
#  Run test specification read on stdin
#
# Option:
#  -n Dry run
#  -k Keep going
#  -I Use installed macros for the test
#
# The test specification has the following columns:
#
#   BATCHOID|BATCHNAME|TESTID|TESTFILE|SOURCEDIR|SEQUENCE|TESTENV|DESCRIPTION

testtool_run_loop()
{
    local OPTIND OPTION OPTARG dryrun keepgoing
    local batchoid batchname testid testfile sourcedir
    local sequence testenv description
    local installmacros testinclude
    local testrundir testprefixdir

    dryrun='no'
    keepgoing='no'
    installmacros='no'
    OPTIND=1
    while getopts 'knI' OPTION; do
        case ${OPTION} in
            k)	keepgoing='yes';;
            n)	dryrun='yes';;
            I)	installmacros='yes';;
            *)	eprintf 'testtool_run_loop: %s: Unsupported option.' "${OPTION}";;
        esac
    done
    shift $(expr ${OPTIND} - 1)

    if [ "${installmacros}" = 'yes' ]; then
        testinclude="-I ${BSDOWL_TESTDIR}/${testtool_batchname}/share/bsdowl"
        PATH="${BSDOWL_TESTDIR}/${testtool_batchname}/bin:${PATH}"
    else
        :
    fi


    while IFS='|' read batchoid batchname testid testfile sourcedir sequence testenv description; do

        testrundir="${BSDOWL_TESTDIR}/${testtool_batchname}/${batchoid}/var/src"
        testprefixdir="${BSDOWL_TESTDIR}/${testtool_batchname}/${batchoid}/opt/local"

        print_sep1;
        printf 'TEST-%03d %s\n' "${batchoid}" "${description}"
        if [ -n "${testenv}" ]; then
            printf '          %s\n' "${testenv}"
        fi
        printf '          %s\n' "${testfile}"
        print_sep1;
        cat <<EOF
batchoid: ${batchoid}
testid: ${testid}
testfile: ${testfile}
testdir: ${testrundir}
sourcedir: ${sourcedir}
sequence: ${sequence}
parallel: ${testtool_parallel}
EOF
        print_sep2

        if [ -n "${testenv}" ]; then
            printf '%s\n' ${testenv}
            print_sep2
        fi

        install -d "${testrundir}"\
            || failwith '%s: Cannot create test run directory.' "${batchoid}"

        install -d "${testprefixdir}"\
            || failwith '%s: Cannot create test output directory.' "${batchoid}"

        { tar cfC - "${BSDOWL_TOPLEVELDIR}/${sourcedir}" .\
                | tar xfC - "${testrundir}"; }\
            || failwith '%s: Cannot copy test source.' "${batchoid}"

        cp "${testfile}" "${testrundir}/Makefile.local"\
            || failwith '%s: Cannot copy test file.' "${batchoid}"

        ( set -e
          destdir="${BSDOWL_TESTDIR:?}/${testtool_batchname}"
          prefix="/opt/local/${batchoid}"

          if [ "${testtool_parallel}" = 'no' ]; then
              parallel=''
          else
              parallel="-j ${testtool_parallel}"
          fi

          if [ -n "${testenv}" ]; then
              for binding in ${testenv}; do
                  export ${binding}
              done
          fi

          case "${testtool_objdir}" in
              makeobjdir)
                  export MAKEOBJDIR="$(testtool_makeobjdir)";;
              makeobjdirprefix)
                  export MAKEOBJDIRPREFIX="$(testtool_makeobjdirprefix)";;
              no)
                  : ;;
              *)
                  failwith "Unsupported OBJDIR option ${testtool_objdir}";;
          esac

          cd "${testrundir}"

          for target in ${sequence} test; do
              wlog '====> Running test step %s' "${target}"
              testtool_make\
                  ${testinclude}\
                  CONFIGURE_ARGS=$(printf -- '--prefix="%s"' "${prefix}")\
                  BSDOWL_TESTDIR="${BSDOWL_TESTDIR}"\
                  BSDOWL_TESTID="${batchoid}"\
                  BSDOWL_BATCHNAME="${testtool_batchname}"\
                  prefix="${prefix}"\
                  exec_prefix="${prefix}"\
                  datarootdir="${prefix}/share"\
                  DESTDIR="${destdir}"\
                  ${parallel}\
                  ${target} || exit $?
          done;
          sqlite3 ${testtool_database:?} <<EOF
UPDATE test_batch SET status = 'DONE' WHERE oid = '${batchoid}';
EOF
        )\
            || {
            sqlite3 ${testtool_database:?} <<EOF
UPDATE test_batch SET status = 'FAILED' WHERE oid = '${batchoid}';
EOF
            if [ "${keepgoing}" = 'no' ]; then
                failwith '%s: Test failed.' "${batchoid}"
            fi; }
    done
}

# testtool_run [-n][-k][-r][-I]
#  Run pending tests
#
# Option:
#  -n Dry run
#  -k Keep going
#  -r Retry failed tests instead of running pending ones
#  -I Install macros to a special directory before running the tests

testtool_run()
{
    local OPTIND OPTION OPTARG dryrun keepgoing selectstate
    local installmacros testinclude

    selectstate='PENDING'
    dryrun='no'
    keepgoing='no'
    installmacros='no'
    OPTIND=1
    while getopts 'knrI' OPTION; do
        case ${OPTION} in
            k)	keepgoing='yes';;
            n)	dryrun='yes';;
            r)	selectstate='FAILED';;
            I)	installmacros='yes';;
            *)	eprintf 'testtool_action_run: %s: Unsupported option.' "${OPTION}";;
        esac
    done
    # shift $(expr ${OPTIND} - 1)

    rm -rf "${BSDOWL_TESTDIR:?}/${testtool_batchname:?}"
    install -d "${BSDOWL_TESTDIR:?}/${testtool_batchname:?}"

    if [ "${installmacros}" = 'yes' ]; then
        testtool_run__macros_install
    fi

    { sqlite3 ${testtool_database:?} <<EOF
SELECT
 test_batch.oid,
 test_batch.id,
 test_index.id,
 test_index.filename,
 test_index.sourcedir,
 test_index.sequence,
 test_batch.env,
 test_index.description
FROM
 test_index LEFT JOIN test_batch
WHERE
 test_batch.status = '${selectstate}'
AND
 test_batch.testid = test_index.id;
EOF
    } | { if [ "${dryrun}" = 'no' ]; then
              testtool_run_loop "$@"
          else
              cat
          fi; }
}

testtool_run__macros_install()
{
    local srcdir
    srcdir="${BSDOWL_TESTDIR}/${testtool_batchname}/var/src/bsdowl"

    install -d "${srcdir}"\
        || failwith '%s: Cannot create test source directory.' "${srcdir}"

    { tar cfC - "${BSDOWL_TOPLEVELDIR}" . | tar xfC - "${srcdir}"; }\
        || failwith '%s: Cannot copy sources to source directory.' "${srcdir}"

    ( cd "${srcdir}"\
            && testtool_run__macros_configure --prefix="${BSDOWL_TESTDIR}/${testtool_batchname}"\
            && testtool_make all\
            && testtool_make install )\
        || failwith '%s: Cannot install to prefix directory.' "${srcdir}"
}

testtool_run__macros_env()
{
    sed -n -e '
/^WITH_TESTSUITE_/{
  s/=[[:space:]]*/=/
  s/[?]=/=/
  p
}' "${BSDOWL_TOPLEVELDIR}/Makefile.config"
}

testtool_run__macros_configure_argv()
{
    eval $(testtool_run__macros_env)
    cat<<EOF
--enable-test-findlib=${WITH_TESTSUITE_FINDLIB}
--enable-test-py-setuptools=${WITH_TESTSUITE_PY_SETUPTOOLS}
--enable-test-mingw32=${WITH_TESTSUITE_MINGW32}
--enable-test-noweb=${WITH_TESTSUITE_NOWEB}
--enable-test-gpg=${WITH_TESTSUITE_GPG}
--enable-test-texmf=${WITH_TESTSUITE_TEXMF}
--enable-test-metapost=${WITH_TESTSUITE_METAPOST}
--with-credentials=no
EOF
}

testtool_run__macros_configure()
{
    autoconf && ./configure $(testtool_run__macros_configure_argv) "$@"
}


# testtool_status
#  Write a report for the selected batch of tests

testtool_status()
{
    local status

    for status in 'PENDING' 'DONE' 'FAILED'; do
        testtool_status__select "${status}" \
            | testtool_status__format "${status}"
    done

    sqlite3 ${testtool_database:?} "SELECT status UNIQ FROM test_batch WHERE test_batch.status = 'FAILED';"\
        | grep -q 'FAILED'

    if [ $? = 0 ]; then
        return 1
    else
        return 0
    fi
}

testtool_status__format()
{
    awk -F '|' -v status="$1" '
function print_sep(_i)
{
   for(_i = 0; _i < 80; ++ _i) {
     printf("=");
   }
   printf("\n");
}

function report_concise(testoid)
{
  printf("TEST-%03d %s\n", testoid, description[testoid]);
}

function report_detailed(testoid)
{
  printf("TEST-%03d %s (%s)\n          %s\n          %s\n\n", testoid, description[testoid], id[testoid], env[testoid], filename[testoid]);
}

function report(testoid)
{
  if(status == "FAILED") {
    report_detailed(testoid);
  } else {
    report_concise(testoid);
  }
}


{
 oid[NR] = $1;
 id[$1] = $2;
 filename[$1] = $3;
 sourcedir[$1] = $4;
 sequence[$1] = $5;
 env[$1] = $6;
 description[$1] = $7;
}

END {
  if(NR >= 1) {
    print_sep();
    printf("%s TEST REPORT\n", status);
    print_sep();
  }

  for(_index = 1; _index <= NR; ++_index) {
    report(oid[_index]);
  }

  if(NR >= 1) {
    printf("\n");
  }
}
'
}


testtool_status__select()
{
    sqlite3 ${testtool_database:?} <<EOF
SELECT
 test_batch.oid,
 test_index.id,
 test_index.filename,
 test_index.sourcedir,
 test_index.sequence,
 test_batch.env,
 test_index.description
FROM
 test_index LEFT JOIN test_batch
WHERE
 test_batch.id = '${testtool_batchname}'
AND
 test_batch.status = '$1'
AND
 test_batch.testid = test_index.id
ORDER BY test_batch.oid ASC;
EOF
}


testtool_usage()
{
    iconv -f utf-8 -c <<EOF
Usage: testtool [-j MAX-JOBS][-a][-DP][-l][-R] [TEST1 …]
 Run the testsuite
Options:
 -a Prepare a batch of tests made of all possible tests and run it.
 -f DATABASE-FILE
    Set the file used to store the test database.
 -h Display this help message.
 -j MAX-JOBS
    Run MAX-JOBS in parallel.  This affects individual tests, so that
    compilation jobs can run in parallel with in a test but tests
    still run sequentially.
 -l List available tests.
 -n BATCH-NAME
    Use the given BATCH-NAME instead of current to prepare test
    batches.
 -r Retry failed tests.
 -x TEST
    Exclude TEST from the list of tests to run. Can be used
    several times.
 -C Clean the test directory
 -D Run tests with MAKEOBJDIR.
 -P Run tests with MAKEOBJDIRPREFIX.
 -R Use rsync to copy the repository
 -S Show the current setup instead of testing.
EOF
}


testtool_assertdb()
{
    if [ -f "${BSDOWL_TESTDIR}/testtool.db" ]; then
        :
    else
        failwith '%s: Test database is missing.\n Please rebuild it with the -i option.'\
                 "${BSDOWL_TESTDIR}/testtool.db"
    fi
}

### Testtool actions

testtool_action_list()
{
    { sqlite3 ${testtool_database:?} <<EOF
SELECT
 id,
 description
FROM
 test_index
ORDER BY id ASC;
EOF
    } | awk -F '|' '{printf("%s %s\n", $1, $2)}'
}

testtool_action_clean()
{
    rm -Rf "${BSDOWL_TESTDIR:?}"
}

testtool_action_install()
{
    install -d "${BSDOWL_TESTDIR:?}"\
        && testtool_dbinit\
        && testtool_dbimport
}

testtool_action_main()
{
    testtool_assertdb
    testtool_batch -n "${testtool_batchname}" "$@"
    testtool_run ${testtool_macros}
    testtool_status
}

testtool_action_runall()
{
    testtool_assertdb
    testtool_batch -a -n "${testtool_batchname}" "${testtool_exclude:+-x}${testtool_exclude}"
    testtool_run -k ${testtool_macros}
    testtool_status
}

testtool_action_retry()
{
    testtool_assertdb
    testtool_run -r -k ${testtool_macros}
    testtool_status
}

testtool_action_status()
{
    testtool_assertdb
    testtool_status
}


testtool_action_help()
{
    testtool_usage
}

### Command line analysis

: ${testtool_database:=${BSDOWL_TESTDIR}/testtool.db}
: ${testtool_resume:=no}
: ${testtool_parallel:=no}
: ${testtool_objdir:=no}
: ${testtool_copy:=gitclone}
: ${testtool_exclude:=''}
: ${testtool_batchname:=current}
: ${testtool_action:=main}
: ${testtool_macros:=}

while getopts "af:hij:ln:rsx:CDIPRS" OPTION; do
    case "${OPTION}" in
        a)	testtool_action='runall';;
        f)	testtool_database="${OPTARG}";;
        h)	testtool_action='help';;
        i)	testtool_action='install';;
        j)	testtool_parallel="${OPTARG}";;
        l)	testtool_action='list';;
        n)	testtool_batchname="${OPTARG}";;
        r)	testtool_action='retry';;
        s)	testtool_action='status';;
        x)	testtool_exclude="${testtool_exclude}${testtool_exclude:+ }${OPTARG}";;
        C)	testtool_action='clean';;
        D)	testtool_objdir='makeobjdir';;
        I)	testtool_macros='-I' ;;
        P)	testtool_objdir='makeobjdirprefix';;
        R)	testtool_copy='rsync';;
        S)	testtool_action='show';;
        *)	testtool_action='usage';;
    esac
done

shift $(expr ${OPTIND} - 1)
testtool_action_${testtool_action} "$@"

### End of file `testtool'
