Merge remote-tracking branch 'remotes/famz/tags/docker-pull-request' into staging

# gpg: Signature made Wed 20 Jul 2016 12:19:56 BST
# gpg:                using RSA key 0xCA35624C6A9171C6
# gpg: Good signature from "Fam Zheng <famz@redhat.com>"
# gpg: WARNING: This key is not certified with a trusted signature!
# gpg:          There is no indication that the signature belongs to the owner.
# Primary key fingerprint: 5003 7CB7 9706 0F76 F021  AD56 CA35 624C 6A91 71C6

* remotes/famz/tags/docker-pull-request:
  docker: pass EXECUTABLE to build script
  docker: Don't start a container that doesn't exist
  docker: Add "images" subcommand to docker.py
  docker: Fix exit code if $CMD failed
  docker: More sensible run script
  tests/docker/docker.py: add update operation
  tests/docker/dockerfiles: new debian-bootstrap.docker
  tests/docker/docker.py: check and run .pre script
  tests/docker/docker.py: support --include-executable
  tests/docker/docker.py: docker_dir outside build

Signed-off-by: Peter Maydell <peter.maydell@linaro.org>

5 files changed, 280 insertions, 17 deletions

diff --git a/tests/docker/Makefile.include b/tests/docker/Makefile.include
index f88c0a7..78af468 100644
--- a/tests/docker/Makefile.include
+++ b/tests/docker/Makefile.include
@@ -46,7 +46,8 @@ docker-image: ${DOCKER_TARGETS}
 docker-image-%: $(DOCKER_FILES_DIR)/%.docker
 	$(call quiet-command,\
 		$(SRC_PATH)/tests/docker/docker.py build qemu:$* $< \
-		$(if $V,,--quiet) $(if $(NOCACHE),--no-cache),\
+		$(if $V,,--quiet) $(if $(NOCACHE),--no-cache) \
+		$(if $(EXECUTABLE),--include-executable=$(EXECUTABLE)),\
 		"  BUILD $*")
 
 # Expand all the pre-requistes for each docker image and test combination
@@ -95,6 +96,7 @@ docker:
 	@echo '    DEBUG=1              Stop and drop to shell in the created container'
 	@echo '                         before running the command.'
 	@echo '    NOCACHE=1            Ignore cache when build images.'
+	@echo '    EXECUTABLE=<path>    Include executable in image.'
 
 docker-run-%: CMD = $(shell echo '$@' | sed -e 's/docker-run-\([^@]*\)@\(.*\)/\1/')
 docker-run-%: IMAGE = $(shell echo '$@' | sed -e 's/docker-run-\([^@]*\)@\(.*\)/\2/')
@@ -105,7 +107,10 @@ docker-run-%: docker-qemu-src
 	fi
 	$(if $(filter $(TESTS),$(CMD)),$(if $(filter $(IMAGES),$(IMAGE)), \
 		$(call quiet-command,\
-			$(SRC_PATH)/tests/docker/docker.py run $(if $V,,--rm) \
+			if $(SRC_PATH)/tests/docker/docker.py images \
+				--format={{.Repository}}:{{.Tag}} | \
+					grep -qx qemu:$(IMAGE); then \
+				$(SRC_PATH)/tests/docker/docker.py run $(if $V,,--rm) \
 				-t \
 				$(if $(DEBUG),-i,--net=none) \
 				-e TARGET_LIST=$(TARGET_LIST) \
@@ -114,11 +119,10 @@ docker-run-%: docker-qemu-src
 				-e CCACHE_DIR=/var/tmp/ccache \
 				-v $$(realpath $(DOCKER_SRC_COPY)):/var/tmp/qemu:z$(COMMA)ro \
 				-v $(DOCKER_CCACHE_DIR):/var/tmp/ccache:z \
-				-w /var/tmp/qemu \
 				qemu:$(IMAGE) \
-				$(if $V,/bin/bash -x ,) \
-				./run \
+				/var/tmp/qemu/run \
 				$(CMD); \
+			fi \
 			, "  RUN $(CMD) in $(IMAGE)")))
 
 docker-clean:
diff --git a/tests/docker/docker.py b/tests/docker/docker.py
index 0151362..222a105 100755
--- a/tests/docker/docker.py
+++ b/tests/docker/docker.py
@@ -20,7 +20,10 @@ import atexit
 import uuid
 import argparse
 import tempfile
-from shutil import copy
+import re
+from tarfile import TarFile, TarInfo
+from StringIO import StringIO
+from shutil import copy, rmtree
 
 def _text_checksum(text):
     """Calculate a digest string unique to the text content"""
@@ -38,6 +41,54 @@ def _guess_docker_command():
     raise Exception("Cannot find working docker command. Tried:\n%s" % \
                     commands_txt)
 
+def _copy_with_mkdir(src, root_dir, sub_path):
+    """Copy src into root_dir, creating sub_path as needed."""
+    dest_dir = os.path.normpath("%s/%s" % (root_dir, sub_path))
+    try:
+        os.makedirs(dest_dir)
+    except OSError:
+        # we can safely ignore already created directories
+        pass
+
+    dest_file = "%s/%s" % (dest_dir, os.path.basename(src))
+    copy(src, dest_file)
+
+
+def _get_so_libs(executable):
+    """Return a list of libraries associated with an executable.
+
+    The paths may be symbolic links which would need to be resolved to
+    ensure theright data is copied."""
+
+    libs = []
+    ldd_re = re.compile(r"(/.*/)(\S*)")
+    try:
+        ldd_output = subprocess.check_output(["ldd", executable])
+        for line in ldd_output.split("\n"):
+            search = ldd_re.search(line)
+            if search and len(search.groups()) == 2:
+                so_path = search.groups()[0]
+                so_lib = search.groups()[1]
+                libs.append("%s/%s" % (so_path, so_lib))
+    except subprocess.CalledProcessError:
+        print "%s had no associated libraries (static build?)" % (executable)
+
+    return libs
+
+def _copy_binary_with_libs(src, dest_dir):
+    """Copy a binary executable and all its dependant libraries.
+
+    This does rely on the host file-system being fairly multi-arch
+    aware so the file don't clash with the guests layout."""
+
+    _copy_with_mkdir(src, dest_dir, "/usr/bin")
+
+    libs = _get_so_libs(src)
+    if libs:
+        for l in libs:
+            so_path = os.path.dirname(l)
+            _copy_with_mkdir(l , dest_dir, so_path)
+
 class Docker(object):
     """ Running Docker commands """
     def __init__(self):
@@ -45,9 +96,11 @@ class Docker(object):
         self._instances = []
         atexit.register(self._kill_instances)
 
-    def _do(self, cmd, quiet=True, **kwargs):
+    def _do(self, cmd, quiet=True, infile=None, **kwargs):
         if quiet:
             kwargs["stdout"] = subprocess.PIPE
+        if infile:
+            kwargs["stdin"] = infile
         return subprocess.call(self._command + cmd, **kwargs)
 
     def _do_kill_instances(self, only_known, only_active=True):
@@ -87,22 +140,27 @@ class Docker(object):
         labels = json.loads(resp)[0]["Config"].get("Labels", {})
         return labels.get("com.qemu.dockerfile-checksum", "")
 
-    def build_image(self, tag, dockerfile, df_path, quiet=True, argv=None):
+    def build_image(self, tag, docker_dir, dockerfile, quiet=True, argv=None):
         if argv == None:
             argv = []
-        tmp_dir = tempfile.mkdtemp(prefix="docker_build")
 
-        tmp_df = tempfile.NamedTemporaryFile(dir=tmp_dir, suffix=".docker")
+        tmp_df = tempfile.NamedTemporaryFile(dir=docker_dir, suffix=".docker")
         tmp_df.write(dockerfile)
 
         tmp_df.write("\n")
         tmp_df.write("LABEL com.qemu.dockerfile-checksum=%s" %
                      _text_checksum(dockerfile))
         tmp_df.flush()
+
         self._do(["build", "-t", tag, "-f", tmp_df.name] + argv + \
-                 [tmp_dir],
+                 [docker_dir],
                  quiet=quiet)
 
+    def update_image(self, tag, tarball, quiet=True):
+        "Update a tagged image using "
+
+        self._do(["build", "-t", tag, "-"], quiet=quiet, infile=tarball)
+
     def image_matches_dockerfile(self, tag, dockerfile):
         try:
             checksum = self.get_image_dockerfile_checksum(tag)
@@ -121,6 +179,9 @@ class Docker(object):
             self._instances.remove(label)
         return ret
 
+    def command(self, cmd, argv, quiet):
+        return self._do([cmd] + argv, quiet=quiet)
+
 class SubCommand(object):
     """A SubCommand template base class"""
     name = None # Subcommand name
@@ -151,6 +212,10 @@ class BuildCommand(SubCommand):
     """ Build docker image out of a dockerfile. Arguments: <tag> <dockerfile>"""
     name = "build"
     def args(self, parser):
+        parser.add_argument("--include-executable", "-e",
+                            help="""Specify a binary that will be copied to the
+                            container together with all its dependent
+                            libraries""")
         parser.add_argument("tag",
                             help="Image Tag")
         parser.add_argument("dockerfile",
@@ -164,10 +229,80 @@ class BuildCommand(SubCommand):
         if dkr.image_matches_dockerfile(tag, dockerfile):
             if not args.quiet:
                 print "Image is up to date."
-            return 0
+        else:
+            # Create a docker context directory for the build
+            docker_dir = tempfile.mkdtemp(prefix="docker_build")
+
+            # Is there a .pre file to run in the build context?
+            docker_pre = os.path.splitext(args.dockerfile)[0]+".pre"
+            if os.path.exists(docker_pre):
+                rc = subprocess.call(os.path.realpath(docker_pre),
+                                     cwd=docker_dir)
+                if rc == 3:
+                    print "Skip"
+                    return 0
+                elif rc != 0:
+                    print "%s exited with code %d" % (docker_pre, rc)
+                    return 1
+
+            # Do we include a extra binary?
+            if args.include_executable:
+                _copy_binary_with_libs(args.include_executable,
+                                       docker_dir)
+
+            dkr.build_image(tag, docker_dir, dockerfile,
+                            quiet=args.quiet, argv=argv)
+
+            rmtree(docker_dir)
+
+        return 0
+
+class UpdateCommand(SubCommand):
+    """ Update a docker image with new executables. Arguments: <tag> <executable>"""
+    name = "update"
+    def args(self, parser):
+        parser.add_argument("tag",
+                            help="Image Tag")
+        parser.add_argument("executable",
+                            help="Executable to copy")
+
+    def run(self, args, argv):
+        # Create a temporary tarball with our whole build context and
+        # dockerfile for the update
+        tmp = tempfile.NamedTemporaryFile(suffix="dckr.tar.gz")
+        tmp_tar = TarFile(fileobj=tmp, mode='w')
+
+        # Add the executable to the tarball
+        bn = os.path.basename(args.executable)
+        ff = "/usr/bin/%s" % bn
+        tmp_tar.add(args.executable, arcname=ff)
+
+        # Add any associated libraries
+        libs = _get_so_libs(args.executable)
+        if libs:
+            for l in libs:
+                tmp_tar.add(os.path.realpath(l), arcname=l)
+
+        # Create a Docker buildfile
+        df = StringIO()
+        df.write("FROM %s\n" % args.tag)
+        df.write("ADD . /\n")
+        df.seek(0)
+
+        df_tar = TarInfo(name="Dockerfile")
+        df_tar.size = len(df.buf)
+        tmp_tar.addfile(df_tar, fileobj=df)
+
+        tmp_tar.close()
+
+        # reset the file pointers
+        tmp.flush()
+        tmp.seek(0)
+
+        # Run the build with our tarball context
+        dkr = Docker()
+        dkr.update_image(args.tag, tmp, quiet=args.quiet)
 
-        dkr.build_image(tag, dockerfile, args.dockerfile,
-                        quiet=args.quiet, argv=argv)
         return 0
 
 class CleanCommand(SubCommand):
@@ -177,6 +312,12 @@ class CleanCommand(SubCommand):
         Docker().clean()
         return 0
 
+class ImagesCommand(SubCommand):
+    """Run "docker images" command"""
+    name = "images"
+    def run(self, args, argv):
+        return Docker().command("images", argv, args.quiet)
+
 def main():
     parser = argparse.ArgumentParser(description="A Docker helper",
             usage="%s <subcommand> ..." % os.path.basename(sys.argv[0]))
diff --git a/tests/docker/dockerfiles/debian-bootstrap.docker b/tests/docker/dockerfiles/debian-bootstrap.docker
new file mode 100644
index 0000000..3a9125e
--- /dev/null
+++ b/tests/docker/dockerfiles/debian-bootstrap.docker
@@ -0,0 +1,21 @@
+# Create Debian Bootstrap Image
+#
+# This is intended to be pre-poluated by:
+#  - a first stage debootstrap (see debian-bootstrap.pre)
+#  - a native qemu-$arch that binfmt_misc will run
+FROM scratch
+
+# Add everything from the context into the container
+ADD . /
+
+# Patch all mounts as docker already has stuff set up
+RUN sed -i 's/in_target mount/echo not for docker in_target mount/g' /debootstrap/functions
+
+# Run stage 2
+RUN /debootstrap/debootstrap --second-stage
+
+# At this point we can install additional packages if we want
+# Duplicate deb line as deb-src
+RUN cat /etc/apt/sources.list | sed "s/deb/deb-src/" >> /etc/apt/sources.list
+RUN apt-get update
+RUN apt-get -y build-dep qemu
diff --git a/tests/docker/dockerfiles/debian-bootstrap.pre b/tests/docker/dockerfiles/debian-bootstrap.pre
new file mode 100755
index 0000000..5d9c8d5
--- /dev/null
+++ b/tests/docker/dockerfiles/debian-bootstrap.pre
@@ -0,0 +1,87 @@
+#!/bin/sh
+#
+# Simple wrapper for debootstrap, run in the docker build context
+#
+FAKEROOT=`which fakeroot 2> /dev/null`
+
+exit_and_skip()
+{
+    exit 3
+}
+
+#
+# fakeroot is needed to run the bootstrap stage
+#
+if [ -z $FAKEROOT ]; then
+    echo "Please install fakeroot to enable bootstraping"
+    exit_and_skip
+fi
+
+# We check in order for
+#
+#  - DEBOOTSTRAP_DIR pointing at a development checkout
+#  - PATH for the debootstrap script (installed)
+#
+# If neither option works then we checkout debootstrap from its
+# upstream SCM and run it from there.
+#
+
+if [ -z $DEBOOTSTRAP_DIR ]; then
+    DEBOOTSTRAP=`which debootstrap 2> /dev/null`
+    if [ -z $DEBOOTSTRAP ]; then
+        echo "No debootstrap installed, attempting to install from SCM"
+        DEBOOTSTRAP_SOURCE=https://anonscm.debian.org/git/d-i/debootstrap.git
+        git clone ${DEBOOTSTRAP_SOURCE} ./debootstrap.git
+        export DEBOOTSTRAP_DIR=./debootstrap.git
+        DEBOOTSTRAP=./debootstrap.git/debootstrap
+    fi
+else
+    DEBOOTSTRAP=${DEBOOTSTRAP_DIR}/debootstrap
+    if [ ! -f $DEBOOTSTRAP ]; then
+        echo "Couldn't find script at ${DEBOOTSTRAP}"
+        exit_and_skip
+    fi
+fi
+
+#
+# Finally check to see if any qemu's are installed
+#
+BINFMT_DIR=/proc/sys/fs/binfmt_misc
+if [ ! -e $BINFMT_DIR ]; then
+   echo "binfmt_misc needs enabling for a QEMU bootstrap to work"
+   exit_and_skip
+else
+    # DEB_ARCH and QEMU arch names are not totally aligned
+    case "${DEB_ARCH}" in
+        amd64)
+            QEMU=qemu-i386
+            ;;
+        armel|armhf)
+            QEMU=qemu-arm
+            ;;
+        arm64)
+            QEMU=qemu-aarch64
+            ;;
+        powerpc)
+            QEMU=qemu-ppc
+            ;;
+        ppc64el)
+            QEMU=qemu-ppc64le
+            ;;
+        s390)
+            QEMU=qemu-s390x
+            ;;
+        *)
+            QEMU=qemu-${DEB_ARCH}
+        ;;
+    esac
+    if [ ! -e "${BINFMT_DIR}/$QEMU" ]; then
+        echo "No binfmt_misc rule to run $QEMU, can't bootstrap"
+        exit_and_skip
+    fi
+fi
+
+echo "Building a rootfs using ${FAKEROOT} and ${DEBOOTSTRAP} ${DEB_ARCH}/${DEB_TYPE}"
+
+${FAKEROOT} ${DEBOOTSTRAP} --variant=buildd --foreign --arch=$DEB_ARCH $DEB_TYPE . http://httpredir.debian.org/debian || exit 1
+exit 0
diff --git a/tests/docker/run b/tests/docker/run
index ec3d119..d85d49a 100755
--- a/tests/docker/run
+++ b/tests/docker/run
@@ -11,6 +11,14 @@
 # or (at your option) any later version. See the COPYING file in
 # the top-level directory.
 
+set -e
+
+if test -n "$V"; then
+    set -x
+fi
+
+BASE="$(dirname $(readlink -e $0))"
+
 # Prepare the environment
 . /etc/profile || true
 export PATH=/usr/lib/ccache:$PATH
@@ -24,10 +32,10 @@ export TEST_DIR=/tmp/qemu-test
 mkdir -p $TEST_DIR/{src,build,install}
 
 # Extract the source tarballs
-tar -C $TEST_DIR/src -xzf qemu.tgz
+tar -C $TEST_DIR/src -xzf $BASE/qemu.tgz
 for p in dtc pixman; do
-    if test -f $p.tgz; then
-        tar -C $TEST_DIR/src/$p -xzf $p.tgz
+    if test -f $BASE/$p.tgz; then
+        tar -C $TEST_DIR/src/$p -xzf $BASE/$p.tgz
         export FEATURES="$FEATURES $p"
     fi
 done
@@ -55,4 +63,6 @@ elif test -n "$DEBUG"; then
     echo
     # Force error after shell exits
     $SHELL && exit 1
+else
+    exit 1
 fi