diff options
| -rw-r--r-- | ChangeLog | 9 | ||||
| -rw-r--r-- | contrib/mysql/README | 38 | ||||
| -rw-r--r-- | contrib/mysql/create-db.sql | 52 | ||||
| -rw-r--r-- | contrib/mysql/importxml.sh | 96 | ||||
| -rw-r--r-- | contrib/mysql/make-datafile.sh | 126 | ||||
| -rw-r--r-- | contrib/mysql/plot.sh | 129 | ||||
| -rw-r--r-- | contrib/mysql/sum2xml.sh | 104 |
7 files changed, 554 insertions, 0 deletions
@@ -1,3 +1,12 @@ +2016-04-02 Rob Savoye <rob.savoye@linaro.org> + + * contrib/mysql/create-db.sql: New file. + * contrib/mysql/importxml.sh: Likewise. + * contrib/mysql/make-datafile.sh: Likewise. + * contrib/mysql/plot.sh: Likewise. + * contrib/mysql/README: Likewise. + * contrib/mysql/sum2xml.sh: Likewise. + 2016-04-02 Ben Elliston <bje@gnu.org> * config.guess: Update to version 2016-04-02. diff --git a/contrib/mysql/README b/contrib/mysql/README new file mode 100644 index 0000000..503eefd --- /dev/null +++ b/contrib/mysql/README @@ -0,0 +1,38 @@ +This is a series of scripts to enable one to import into MySQL the +output of a test run, and then graph the data. These scripts will +likely require modification for different setups, which is why they're +in 'contrib' instead of DejaGnu. + + +sum2xml.sh +---------- + This script takes a standard DejaGnu .sum file, and converts + it to the XML format. Ideally the --xml option to runtest + should be used, but this is useful for importing historical + data. + +create-db.sql +------------- + This is an SQL input file to create the tables used for this + database. This assumes the 'dejagnu' database has been + created, but is empty. + +importxml.sh +------------ + This script imports the XML file into MySQL and populates the + tables with data. + +make-datafile.sh +---------------- + This script does an SQL query in the 'dejagnu' database, and + produces the data files that get used for graphing. This is + oriented towards GNU toolchain testing, and is probably the + main this any other user would need to modify for their + application. Currently the created data files are one for each + architecture. + +plot.sh +------- + This script reads the data files produced by make-datafile.sh, + and produces a command file for gnuplot, which then makes the + chart. diff --git a/contrib/mysql/create-db.sql b/contrib/mysql/create-db.sql new file mode 100644 index 0000000..34edd02 --- /dev/null +++ b/contrib/mysql/create-db.sql @@ -0,0 +1,52 @@ +-- Copyright (C) 2016 Free Software Foundation, Inc. + +-- This file is part of DejaGnu. + +-- DejaGnu is free software; you can redistribute it and/or modify it +-- under the terms of the GNU General Public License as published by +-- the Free Software Foundation; either version 3 of the License, or +-- (at your option) any later version. + +/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; +/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; +/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; +/*!40101 SET NAMES utf8 */; +/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; +/*!40103 SET TIME_ZONE='+00:00' */; +/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; +/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; +/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; +/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; + +-- +-- Table structure for table `test` +-- + +DROP TABLE IF EXISTS `test`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `test` ( + `testrun` int(9) NOT NULL DEFAULT '12345', + `input` varchar(128) NOT NULL, + `output` varchar(256) NOT NULL, + `result` enum('PASS','FAIL','XPASS','XFAIL','UNTESTED','UNRESOLVED', 'UNSUPPORTED') NOT NULL, + `name` varchar(128) NOT NULL, + `prmsid` int(11) NOT NULL, + KEY `testrun` (`testrun`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +DROP TABLE IF EXISTS `testruns`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `testruns` ( + `tool` varchar(72) NOT NULL, + `date` datetime NOT NULL, + `version` varchar(72) NOT NULL, + `branch` varchar(72) NOT NULL, + `testrun` int(20) NOT NULL, + `arch` varchar(72) NOT NULL + `build_machine` varchar(72) NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + diff --git a/contrib/mysql/importxml.sh b/contrib/mysql/importxml.sh new file mode 100644 index 0000000..9a194db --- /dev/null +++ b/contrib/mysql/importxml.sh @@ -0,0 +1,96 @@ +#!/bin/bash + +# importxml.sh -- import a .sum file into MySQL. +# +# Copyright (C) 2016 Free Software Foundation, Inc. +# +# This file is part of DejaGnu. +# +# DejaGnu is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. + +# This script takes a compressed or uncompressed sum file from a +# DejaGnu test run. It then extracts the relevant information about +# the build and writes that to the dejagnu.testruns control +# table. After that it converts the sum file to XML, and imports it +# into an SQL database, in this case MySQL. + +if test x"$1" = x; then + infile="/tmp/testrun.xml" + outfile="/tmp/testrun.xml" +else + infile=$1 + outfile=${1//\.sum.*/.xml} +fi + +if test ! -e "$infile"; then + echo "ERROR: no input file specified!" + exit +fi + +# These are required to access the database +user="USER" +passwd="PASSWD" + +# Check the environment for the database access information +if test x"${CBUILD_DBUSER}" != x; then + user="${CBUILD_DBUSER}" +fi +if test x"${CBUILD_DBPASSWD}" != x; then + passwd="${CBUILD_DBPASSWD}" +fi + +total=$(mysql -u"$user" -p"$passwd" -e "SELECT testrun from testruns ORDER BY testrun DESC LIMIT 1" dejagnu | tail -1) +total=$((total + 1)) + +type=$(file "$infile") + +build_machine=$(dirname "$infile" | cut -d '-' -f 8) + +# If compressed, uncompress it +count=$(echo "$type" | grep -c "XZ compressed data") +if test "$count" -gt 0; then + catprog="xzcat" + uncomp="xz -d" +else + count=$(echo "$type" | grep -c "XZ compressed data") + if test "$count" -gt 0; then + catprog="gzcat" + uncomp="gnzip" + else + catprog="cat" + uncomp="" + fi +fi + +# extract the build info from the sum file. +# This goes in the dejagnu.testruns table +tool=$(${catprog} "$infile" | grep "tests ===" | cut -d ' ' -f 2) +target=$(${catprog} "$infile" | head -20 | grep "configuration is " | cut -d ' ' -f 4) +version=$(${catprog} "$infile" | head -20 | grep "Running /" | cut -d '/' -f 7 | sed -e 's:^[a-z-]*-::' -e 's:_:.:' -e 's:-branch::' | tail -1) +date=$(${catprog} "$infile" | head -1 | sed -e 's:Test Run By [a-zA-Z]* on ::') +date=$(date -d "${date}" "+%Y-%m-%d %H:%M:%S") + +echo "Adding to DB: $target, $version, $date, $tool" + +# Add an entry in the testrun table for this sum file + +echo "Adding the test run into the testruns control table." +mysql -u"$user" -p"$passwd" --local -e "INSERT INTO dejagnu.testruns VALUES('${tool}','${date}','${version}','svn','${total}','${target}','${build_machine}');" dejagnu + +# Make the temp file copy +if test x"${uncomp}" != x; then + rm -f tmp.sum + ${catprog} "$infile" > tmp.sum +fi + +# convert the sum file to an xml file so it can be imported +echo "Converting the sum file to XML for importing into MySQL." +bash "$(dirname "$0")/sum2xml.sh" "$infile" + +# import the the xml file into MySQL. Note that we add the testrun index which is +# retreived from the database earlier in this script. +echo "Adding the sum file into the testruns control file." +mysql -u"$user" -p"$passwd" --local -e "LOAD XML LOCAL INFILE '${outfile}' INTO TABLE dejagnu.test ROWS IDENTIFIED BY '<test>' SET testrun = '${total}';" dejagnu diff --git a/contrib/mysql/make-datafile.sh b/contrib/mysql/make-datafile.sh new file mode 100644 index 0000000..dc40651 --- /dev/null +++ b/contrib/mysql/make-datafile.sh @@ -0,0 +1,126 @@ +#!/bin/bash + +# make-datafile.sh -- export data from MySQL testing database. +# +# Copyright (C) 2016 Free Software Foundation, Inc. +# +# This file is part of DejaGnu. +# +# DejaGnu is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. + +# This script executes a series of SQL calls to our testing database, +# and makes a data file for gnuplot. + +if test $# -eq 0; then + echo "ERROR: no testrun numbers specified!" + exit +fi + +# These are required to access the database +user="USER" +passwd="PASSWD" +outfile="testrun.data" + +# Check the environment for the database access information +if test x"${CBUILD_DBUSER}" != x; then + user="${CBUILD_DBUSER}" +fi +if test x"${CBUILD_DBPASSWD}" != x; then + passwd="${CBUILD_DBPASSWD}" +fi + +# Create the output file +rm -f ${outfile} + +# Add a header to make the file human readable +echo "# date tool version arch total PASS FAIL UNSUPPORTED UNTESTED UNRESOLVED XPASS XFAIL" > ${outfile} + +machines=$(mysql -u"$user" -p"$passwd" --local -e "SELECT DISTINCT(build_machine) FROM dejagnu.testruns" | sed -e 's/build_machine//' | tr '\n' ' ') +echo "Build machines are: ${machines}" + +tools="gcc g++ gfortran libstdc++ objc" +echo "Tools are: ${tools}" + +for testrun in "$@"; do + # Get the build info + binfo=$(mysql -u"$user" -p"$passwd" --local -e "SELECT tool,arch,date,version,branch,build_machine FROM dejagnu.testruns WHERE testrun=${testrun}" | tail -1) + + # Get rid of the embedded newlines + binfo=$(echo "$binfo" | tr -d '\n') + + # Split the query result into fields + tool=$(echo "$binfo" | cut -d ' ' -f 1) + arch=$(echo "$binfo" | cut -d ' ' -f 2) + date=$(echo "$binfo" | cut -d ' ' -f 3) + version=$(echo "$binfo" | cut -d ' ' -f 5) + build_machine=$(echo "$binfo" | cut -d ' ' -f 7) + + # Get the test counts +# total=`mysql -u"$user" -p${passwd} -e "SELECT count(*) FROM dejagnu.test WHERE result!=''" | tail -1` + passes=$(mysql -u"$user" -p"$passwd" --local -e "SELECT count(*) FROM dejagnu.test WHERE testrun=${testrun} AND result='PASS'" | tail -1) + fails=$(mysql -u"$user" -p"$passwd" --local -e "SELECT count(*) FROM dejagnu.test WHERE testrun=${testrun} AND result='FAIL'" | tail -1) + xpasses=$(mysql -u"$user" -p"$passwd" --local -e "SELECT count(*) FROM dejagnu.test WHERE testrun=${testrun} AND result='XPASS'" | tail -1) + xfails=$(mysql -u"$user" -p"$passwd" --local -e "SELECT count(*) FROM dejagnu.test WHERE testrun=${testrun} AND result='XFAIL'" | tail -1) + untested=$(mysql -u"$user" -p"$passwd" --local -e "SELECT count(*) FROM dejagnu.test WHERE testrun=${testrun} AND result='UNTESTED'" | tail -1) + unsupported=$(mysql -u"$user" -p"$passwd" --local -e "SELECT count(*) FROM dejagnu.test WHERE testrun=${testrun} AND result='UNSUPPORTED'" | tail -1) + unresolved=$(mysql -u"$user" -p"$passwd" --local -e "SELECT count(*) FROM dejagnu.test WHERE testrun=${testrun} AND result='UNRESOLVED'" | tail -1) + # total=$(mysql -u"$user" -p"$passwd" -e "SELECT count(*) FROM dejagnu.test WHERE testrun=${testrun} AND result!=''" | tail -1) + # passes=$(mysql -u"$user" -p"$passwd" --local -e "SELECT count(*) FROM dejagnu.test WHERE testrun=${testrun} AND result='PASS'" | tail -1) + # fails=$(mysql -u"$user" -p"$passwd" --local -e "SELECT count(*) FROM dejagnu.test WHERE testrun=${testrun} AND result='FAIL'" | tail -1) + # xpasses=$(mysql -u"$user" -p"$passwd" --local -e "SELECT count(*) FROM dejagnu.test WHERE testrun=${testrun} AND result='XPASS'" | tail -1) + # xfails=$(mysql -u"$user" -p"$passwd" --local -e "SELECT count(*) FROM dejagnu.test WHERE testrun=${testrun} AND result='XFAIL'" | tail -1) + # untested=$(mysql -u"$user" -p"$passwd" --local -e "SELECT count(*) FROM dejagnu.test WHERE testrun=${testrun} AND result='UNTESTED'" | tail -1) + # unsupported=$(mysql -u"$user" -p"$passwd" --local -e "SELECT count(*) FROM dejagnu.test WHERE testrun=${testrun} AND result='UNSUPPORTED'" | tail -1) + # unresolved=$(mysql -u"$user" -p"$passwd" --local -e "SELECT count(*) FROM dejagnu.test WHERE testrun=${testrun} AND result='UNRESOLVED'" | tail -1) + + # Write the data line + all_passes=$((passes+ xfails)) + all_failures=$((fails + xpasses + untested + unresolved + unsupported)) + all_tests=$((all_passes + all_failures)) + + echo "${date} ${tool} ${version} ${arch} ${all_tests} ${passes} ${fails} ${xpasses} ${xfails} ${untested} ${unresolved} ${unsupported} ${build_machine}" >> ${outfile} +done + +# Extract the list of architectures supported for these results +arches=$(grep -v '#' testrun.data | cut -d ' ' -f 13 | sort | uniq | tr '\n' ' ') +echo "Architectures for this set of results: ${arches}" + +# Make a separate data file for each architecture so gnuplot can keep them organized +for i in ${arches}; do + if test "$i" = '.'; then + continue + fi + # filter out bogus test results to avoid weird spikes in the data + if test "$passes" -eq 0 -a "$fails" -eq 0; then + continue + fi + rm -f "$i".data + grep "$i" testrun.data | sort -V -k 3 | uniq -u > "$i".data +done + +rm testrun.data + +files=$(find . -maxdepth 1 -name '*.data' | tr '\n' ' ') +# Get all the versions in the files, we'll pad rows so all the rows match +versions=$(cut -d ' ' -f 3 $files | sort -V | uniq | tr '\n' ' ') + +for i in ${files}; do + for j in ${versions}; do + cnt=$(grep -c "$j" "$i") + if test "$cnt" -eq 0; then + echo "Adding $j to $i" + machine=$(echo "$i" | cut -d '.' -f 1) + echo "${date} ${tool} $j ${arch} 0 0 0 0 0 0 0 0 $machine" >> "$i" + fi + done +done + +# Re-sort based on the versions we just added as padding so the rows line up +for j in ${files}; do + mv "$j" tmp.data + sort -V -k 3 < tmp.data > "$j" + rm tmp.data +done diff --git a/contrib/mysql/plot.sh b/contrib/mysql/plot.sh new file mode 100644 index 0000000..d6391de --- /dev/null +++ b/contrib/mysql/plot.sh @@ -0,0 +1,129 @@ +#!/bin/bash + +# plot.sh +# +# Copyright (C) 2016 Free Software Foundation, Inc. +# +# This file is part of DejaGnu. +# +# DejaGnu is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. + +# This script executes a series of SQL calls to our testing database, +# and makes a data file for gnuplot. + +if test "$#" -eq 0; then + echo "Need to supply a graph name!" + name="GCC" +else + name=$1 +fi + +# These are required to access the database +user="USER" +passwd="PASSWD" +outfile="testrun.data" + +# Check the environment for the database access information +if test x"${CBUILD_DBUSER}" != x; then + user="${CBUILD_DBUSER}" +fi +if test x"${CBUILD_DBPASSWD}" != x; then + passwd="${CBUILD_DBPASSWD}" +fi + +# setup an array of colors, since the number of data files varies +declare -a colors=('green' 'red' 'cyan' 'blue' 'purple' 'brown' 'coral' 'yellow' 'black') + +type="with lines" +type= + +machines=$(mysql -u"$user" -p"$passwd" --local -e "SELECT DISTINCT(build_machine) FROM dejagnu.testruns" | sed -e 's/build_machine//' | sed -e 's:\.::' | tr '\n' ' ') +echo "Build machines are: ${machines}" + +tools=$(mysql -u"$user" -p"$passwd" --local -e "SELECT DISTINCT(tool) FROM dejagnu.testruns" | sed -e 's/tool//' | tr '\n' ' ') + +# Only graph if there is a data file +#tools="`ls *.data | cut -d '.' -f 1`" +echo "Tools are: ${tools}" + +datafiles="${machines}" +if test "$#" -gt 1; then + case $2 in + m*) datafiles="echo ${machines}" + echo "Separating graphs by build machine name" + ;; + t*) datafiles="${tools}" + echo "Separating graphs by tool name" + ;; + *) + datafiles="${machines}" + ;; + esac +fi + +outfile="gnuplot.cmd" +cindex=0 +for j in ${datafiles}; do + # for now we don't want aarch64 results in the chart as they distort the data + # heavily. + if test "$(echo "$j" | grep -c aarch64)" -gt 0; then + continue + fi + if test -f "$j".data -a -s "$j".data; then + if test ${cindex} -eq 0; then + echo "Creating gnuplot comand file: ${outfile}" + cat <<EOF >${outfile} +set boxwidth 0.9 relative +set style data histograms +set style histogram cluster +set style fill solid 1.0 border lt -1 +set autoscale x +set autoscale y +#set yrange [97:100] +set title "Precentage PASSes" +set ylabel "Precent PASS" +set format y "%g%%" +#set xlabel "Architecture" + +# Rotate the X axis labels 90 degrees, so they all fit +set xtics border in scale 1,0.5 nomirror rotate by -90 offset character 0, 0, 0 + +# Out the key in out of the way +set key right top + +set term png size 1900,1024 +set output "testrun.png" + +set xlabel "${name} Versions" + +set grid ytics lt 0 lw 1 lc rgb "#bbbbbb" +#set grid xtics lt 0 lw 1 lc rgb "#bbbbbb" + +EOF +# echo -n "plot \"$j.data\" using (((\$6+\$9)/\$5)/100):xtic(3) title \"$j\" lt rgb \"${colors[$cindex]}\" ${type}" >> ${outfile} + echo -n "plot \"$j.data\" using ((\$6/\$5) * 100):xtic(3) title \"$j\" lt rgb \"${colors[$cindex]}\" ${type}" >> ${outfile} +# cindex=`expr $cindex + 1` +# echo -n ", \"\" using (((\$7+\$8+\$10+\$11+\$12)/\$5) * 100):xtic(3) title \"FAILS\" lt rgb \"${colors[$cindex]}\" ${type}" >> ${outfile} + else +# echo -n ", \"$j.data\" using ((\$6+\$9/\$5)/100):xtic(3) title \"$j\" lt rgb \"${colors[$cindex]}\" ${type}" >> ${outfile} + echo -n ", \"$j.data\" using ((\$6/\$5) * 100):xtic(3) title \"$j\" lt rgb \"${colors[$cindex]}\" ${type}" >> ${outfile} + fi + cindex=$((cindex + 1)) + fi +done + +#for j in ${datafiles}; do +# if test -f $j.data -a -s $j.data; then + cat <<EOF >>${outfile} + +set term x11 persist +replot + +EOF +# fi +#done + + diff --git a/contrib/mysql/sum2xml.sh b/contrib/mysql/sum2xml.sh new file mode 100644 index 0000000..f642a48 --- /dev/null +++ b/contrib/mysql/sum2xml.sh @@ -0,0 +1,104 @@ +#!/bin/bash + +# sum2xml.sh -- convert a .sum file into XML. +# +# Copyright (C) 2016 Free Software Foundation, Inc. +# +# This file is part of DejaGnu. +# +# DejaGnu is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. + + +if test x"$1" = x; then + outfile="/tmp/testrun.xml" + infile="/tmp/testrun.sum" +else + outfile=${1//\.sum.*/.xml} + infile=$1 +fi + +# Where to put the output file +if test x"$2" = x; then + outfile="$outfile" +else + outfile="/tmp/$outfile" +fi + +if test ! -e "$infile"; then + echo "ERROR: no input file specified!" + exit +fi + +# If compressed, uncompress it +type=$(file "$infile") +count=$(echo "$type" | grep -c "XZ compressed data") +if test "$count" -gt 0; then + decomp="xz -d" + comp="xz" +else + count=$(echo "$type" | grep -c "XZ compressed data") + if test "$count" -gt 0; then + decomp="gzip" + comp="gunzip" + fi +fi + +# +cat <<EOF > "$outfile" +<?xml version="1.0"?> +<!DOCTYPE testsuite [ +<!-- testsuite.dtd --> +<!ELEMENT testsuite (test | summary)+> +<!ELEMENT test (input, output, result, name, prms_id )> + <!ELEMENT input (#PCDATA)> + <!ELEMENT output (#PCDATA)> + <!ELEMENT result (#PCDATA)> + <!ELEMENT name (#PCDATA)> + <!ELEMENT prms_id (#PCDATA)> + <!ELEMENT summary (result, description, total)> + <!ELEMENT description (#PCDATA)> + <!ELEMENT total (#PCDATA)> +]> +EOF + +# Write the opening tag for the test results +echo "<testsuite>" >> "$outfile" + +${decomp} "$infile" +infile=$(echo "$infile" | sed -e 's:\.xz::' -e 's:\.gz::') + +while read line +do + # ignore blank lines + if test x"${line}" = x; then + continue + fi + # # ignore the test case name + # if test `echo ${line} | grep -c Running` -gt 0; then + # continue + # fi + # ignore the summary, we get this via SQL later + if test "$(echo "$line" | grep -c Summary)" -gt 0; then + break + fi + valid=$(echo "$line" | egrep -c 'PASS|FAIL|UNTESTED|UNSUPPORTED|UNRESOLVED') + if test "$valid" -eq 0; then + continue + fi + echo -n "." + { echo "<test>"; echo " <input></input>"; echo " <output></output>"; } >> "$outfile" + result=$(echo "$line" | sed -e 's/: .*$//') + echo " <result>${result}</result>" >> "$outfile" + name=${line/^[A-Z]*: /} + { echo " <name>${name}</name>"; echo " <prms_id></prms_id>"; echo "</test>"; } >> "$outfile" +done < "$infile" + +# Write the closing tag for the test results +echo "</testsuite>" >> "$outfile" + +# compress the file again +${comp} "$infile" + |