diff options
author | Jussi Pakkanen <jpakkane@gmail.com> | 2020-04-04 20:39:00 +0300 |
---|---|---|
committer | GitHub <noreply@github.com> | 2020-04-04 20:39:00 +0300 |
commit | 69a03fa94bb03bb6527ca0ab5a7b519784d35f71 (patch) | |
tree | d38c18d3785c27d1b4e48107138fd4e16232a262 /ci | |
parent | 2cefe8fd41c5025fd4c1f17d63c7b36bc58d6e36 (diff) | |
parent | c17ed429be9575a99d893a31cd7bf32dc428ede9 (diff) | |
download | meson-69a03fa94bb03bb6527ca0ab5a7b519784d35f71.zip meson-69a03fa94bb03bb6527ca0ab5a7b519784d35f71.tar.gz meson-69a03fa94bb03bb6527ca0ab5a7b519784d35f71.tar.bz2 |
Merge pull request #6843 from mensinda/ciJSONSCHEMA
CI: Maintain docker images with GitHub Actions
Diffstat (limited to 'ci')
-rw-r--r-- | ci/ciimage/.gitignore | 2 | ||||
-rw-r--r-- | ci/ciimage/arch/Dockerfile | 4 | ||||
-rw-r--r-- | ci/ciimage/arch/image.json | 6 | ||||
-rwxr-xr-x | ci/ciimage/arch/install.sh | 8 | ||||
-rwxr-xr-x | ci/ciimage/build.py | 186 | ||||
-rw-r--r-- | ci/ciimage/common.sh | 22 | ||||
-rw-r--r-- | ci/ciimage/eoan/Dockerfile | 36 | ||||
-rw-r--r-- | ci/ciimage/eoan/image.json | 7 | ||||
-rwxr-xr-x | ci/ciimage/eoan/install.sh | 55 | ||||
-rwxr-xr-x | ci/ciimage/eoan/test.sh | 12 | ||||
-rw-r--r-- | ci/ciimage/fedora/Dockerfile | 4 | ||||
-rw-r--r-- | ci/ciimage/fedora/image.json | 8 | ||||
-rwxr-xr-x | ci/ciimage/fedora/install.sh | 4 | ||||
-rw-r--r-- | ci/ciimage/opensuse/Dockerfile | 4 | ||||
-rw-r--r-- | ci/ciimage/opensuse/image.json | 9 | ||||
-rwxr-xr-x | ci/ciimage/opensuse/install.sh | 30 |
16 files changed, 333 insertions, 64 deletions
diff --git a/ci/ciimage/.gitignore b/ci/ciimage/.gitignore new file mode 100644 index 0000000..02a0319 --- /dev/null +++ b/ci/ciimage/.gitignore @@ -0,0 +1,2 @@ +/build_* +/test_* diff --git a/ci/ciimage/arch/Dockerfile b/ci/ciimage/arch/Dockerfile deleted file mode 100644 index b8a36cd..0000000 --- a/ci/ciimage/arch/Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -FROM archlinux:latest - -ADD install.sh /usr/sbin/docker-arch-install -RUN docker-arch-install diff --git a/ci/ciimage/arch/image.json b/ci/ciimage/arch/image.json new file mode 100644 index 0000000..5254073 --- /dev/null +++ b/ci/ciimage/arch/image.json @@ -0,0 +1,6 @@ +{ + "base_image": "archlinux:latest", + "env": { + "CI": "1" + } +} diff --git a/ci/ciimage/arch/install.sh b/ci/ciimage/arch/install.sh index f7f0f6d..7b6eda9 100755 --- a/ci/ciimage/arch/install.sh +++ b/ci/ciimage/arch/install.sh @@ -1,9 +1,11 @@ #!/bin/bash +set -e + # Inspired by https://github.com/greyltc/docker-archlinux-aur/blob/master/add-aur.sh pkgs=( - python python-setuptools python-wheel python-pip python-pytest-xdist python-gobject + python python-setuptools python-wheel python-pip python-pytest-xdist python-gobject python-jsonschema ninja make git sudo fakeroot autoconf automake patch libelf gcc gcc-fortran gcc-objc vala rust bison flex cython go dlang-dmd mono boost qt5-base gtkmm3 gtest gmock protobuf wxgtk gobject-introspection @@ -13,7 +15,8 @@ pkgs=( # cuda ) -aur_pkgs=(hotdoc scalapack) +aur_pkgs=(scalapack) +pip_pkgs=(hotdoc) cleanup_pkgs=(go) AUR_USER=docker @@ -26,6 +29,7 @@ sed -i "s,PKGEXT='.pkg.tar.xz',PKGEXT='.pkg.tar',g" /etc/makepkg.conf # Install packages pacman -Syu $PACMAN_OPTS "${pkgs[@]}" +python -m pip install "${pip_pkgs[@]}" # Setup the user useradd -m $AUR_USER diff --git a/ci/ciimage/build.py b/ci/ciimage/build.py new file mode 100755 index 0000000..5e25e36 --- /dev/null +++ b/ci/ciimage/build.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python3 + +import json +import argparse +import stat +import textwrap +import shutil +import subprocess +from tempfile import TemporaryDirectory +from pathlib import Path +import typing as T + +image_namespace = 'mesonbuild' + +image_def_file = 'image.json' +install_script = 'install.sh' + +class ImageDef: + def __init__(self, image_dir: Path) -> None: + path = image_dir / image_def_file + data = json.loads(path.read_text()) + + assert isinstance(data, dict) + assert all([x in data for x in ['base_image', 'env']]) + assert isinstance(data['base_image'], str) + assert isinstance(data['env'], dict) + + self.base_image: str = data['base_image'] + self.env: T.Dict[str, str] = data['env'] + +class BuilderBase(): + def __init__(self, data_dir: Path, temp_dir: Path) -> None: + self.data_dir = data_dir + self.temp_dir = temp_dir + + self.common_sh = self.data_dir.parent / 'common.sh' + self.common_sh = self.common_sh.resolve(strict=True) + self.validate_data_dir() + + self.image_def = ImageDef(self.data_dir) + + self.docker = shutil.which('docker') + self.git = shutil.which('git') + if self.docker is None: + raise RuntimeError('Unable to find docker') + if self.git is None: + raise RuntimeError('Unable to find git') + + def validate_data_dir(self) -> None: + files = [ + self.data_dir / image_def_file, + self.data_dir / install_script, + ] + if not self.data_dir.exists(): + raise RuntimeError(f'{self.data_dir.as_posix()} does not exist') + for i in files: + if not i.exists(): + raise RuntimeError(f'{i.as_posix()} does not exist') + if not i.is_file(): + raise RuntimeError(f'{i.as_posix()} is not a regular file') + +class Builder(BuilderBase): + def gen_bashrc(self) -> None: + out_file = self.temp_dir / 'env_vars.sh' + out_data = '' + + for key, val in self.image_def.env.items(): + out_data += f'export {key}="{val}"\n' + + out_file.write_text(out_data) + + # make it executable + mode = out_file.stat().st_mode + out_file.chmod(mode | stat.S_IEXEC) + + def gen_dockerfile(self) -> None: + out_file = self.temp_dir / 'Dockerfile' + out_data = textwrap.dedent(f'''\ + FROM {self.image_def.base_image} + + ADD install.sh /ci/install.sh + ADD common.sh /ci/common.sh + ADD env_vars.sh /ci/env_vars.sh + RUN /ci/install.sh + ''') + + out_file.write_text(out_data) + + def do_build(self) -> None: + # copy files + for i in self.data_dir.iterdir(): + shutil.copy(str(i), str(self.temp_dir)) + shutil.copy(str(self.common_sh), str(self.temp_dir)) + + self.gen_bashrc() + self.gen_dockerfile() + + cmd_git = [self.git, 'rev-parse', '--short', 'HEAD'] + res = subprocess.run(cmd_git, cwd=self.data_dir, stdout=subprocess.PIPE) + if res.returncode != 0: + raise RuntimeError('Failed to get the current commit hash') + commit_hash = res.stdout.decode().strip() + + cmd = [ + self.docker, 'build', + '-t', f'{image_namespace}/{self.data_dir.name}:latest', + '-t', f'{image_namespace}/{self.data_dir.name}:{commit_hash}', + '--pull', + self.temp_dir.as_posix(), + ] + if subprocess.run(cmd).returncode != 0: + raise RuntimeError('Failde to build the docker image') + +class ImageTester(BuilderBase): + def __init__(self, data_dir: Path, temp_dir: Path, ci_root: Path) -> None: + super().__init__(data_dir, temp_dir) + self.meson_root = ci_root.parent.parent.resolve() + + def gen_dockerfile(self) -> None: + out_file = self.temp_dir / 'Dockerfile' + out_data = textwrap.dedent(f'''\ + FROM {image_namespace}/{self.data_dir.name} + + ADD meson /meson + ''') + + out_file.write_text(out_data) + + def copy_meson(self) -> None: + shutil.copytree( + self.meson_root, + self.temp_dir / 'meson', + ignore=shutil.ignore_patterns( + '.git', + '*_cache', + 'work area', + self.temp_dir.name, + ) + ) + + def do_test(self): + self.copy_meson() + self.gen_dockerfile() + + try: + build_cmd = [ + self.docker, 'build', + '-t', 'meson_test_image', + self.temp_dir.as_posix(), + ] + if subprocess.run(build_cmd).returncode != 0: + raise RuntimeError('Failde to build the test docker image') + + test_cmd = [ + self.docker, 'run', '--rm', '-t', 'meson_test_image', + '/usr/bin/bash', '-c', 'source /ci/env_vars.sh; cd meson; ./run_tests.py' + ] + if subprocess.run(test_cmd).returncode != 0: + raise RuntimeError('Running tests failed') + finally: + cleanup_cmd = [self.docker, 'rmi', '-f', 'meson_test_image'] + subprocess.run(cleanup_cmd).returncode + +def main() -> None: + parser = argparse.ArgumentParser(description='Meson CI image builder') + parser.add_argument('what', type=str, help='Which image to build / test') + parser.add_argument('-t', '--type', choices=['build', 'test'], help='What to do', required=True) + + args = parser.parse_args() + + ci_root = Path(__file__).parent + ci_data = ci_root / args.what + + with TemporaryDirectory(prefix=f'{args.type}_{args.what}_', dir=ci_root) as td: + ci_build = Path(td) + print(f'Build dir: {ci_build}') + + if args.type == 'build': + builder = Builder(ci_data, ci_build) + builder.do_build() + elif args.type == 'test': + tester = ImageTester(ci_data, ci_build, ci_root) + tester.do_test() + +if __name__ == '__main__': + main() diff --git a/ci/ciimage/common.sh b/ci/ciimage/common.sh new file mode 100644 index 0000000..c8940df --- /dev/null +++ b/ci/ciimage/common.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +### +### Common functions for CI builder files. +### All functions can be accessed in install.sh via: +### +### $ source /ci/common.sh +### + +set -e + +dub_fetch() { + set +e + for (( i=1; i<=24; ++i )); do + dub fetch "$@" + (( $? == 0 )) && break + + echo "Dub Fetch failed. Retrying in $((i*5))s" + sleep $((i*5)) + done + set -e +} diff --git a/ci/ciimage/eoan/Dockerfile b/ci/ciimage/eoan/Dockerfile deleted file mode 100644 index dcc8549..0000000 --- a/ci/ciimage/eoan/Dockerfile +++ /dev/null @@ -1,36 +0,0 @@ -FROM ubuntu:eoan - -ENV DEBIAN_FRONTEND noninteractive -ENV LANG='C.UTF-8' -ENV DC=gdc - -RUN sed -i '/^#\sdeb-src /s/^#//' "/etc/apt/sources.list" \ -&& apt-get -y update && apt-get -y upgrade \ -&& apt-get -y install eatmydata \ -&& eatmydata apt-get -y build-dep meson \ -&& eatmydata apt-get -y install python3-pytest-xdist \ -&& eatmydata apt-get -y install python3-pip libxml2-dev libxslt1-dev libyaml-dev libjson-glib-dev \ -&& eatmydata python3 -m pip install hotdoc codecov \ -&& eatmydata apt-get -y install wget unzip \ -&& eatmydata apt-get -y install qt5-default clang \ -&& eatmydata apt-get -y install pkg-config-arm-linux-gnueabihf \ -&& eatmydata apt-get -y install qt4-linguist-tools \ -&& eatmydata apt-get -y install python-dev \ -&& eatmydata apt-get -y install libomp-dev \ -&& eatmydata apt-get -y install dub ldc \ -&& eatmydata apt-get -y install mingw-w64 mingw-w64-tools nim \ -&& eatmydata apt-get -y install --no-install-recommends wine-stable \ -&& eatmydata apt-get -y install libclang-dev \ -&& eatmydata apt-get -y install libgcrypt20-dev \ -&& eatmydata apt-get -y install libgpgme-dev \ -&& eatmydata apt-get -y install libhdf5-dev \ -&& eatmydata apt-get -y install libboost-python-dev libboost-regex-dev \ -&& eatmydata apt-get -y install libblocksruntime-dev \ -&& eatmydata apt-get -y install libperl-dev \ -&& eatmydata apt-get -y install liblapack-dev libscalapack-mpi-dev \ -&& eatmydata dub fetch urld && dub build urld --compiler=gdc \ -&& eatmydata dub fetch dubtestproject \ -&& eatmydata dub build dubtestproject:test1 --compiler=ldc2 \ -&& eatmydata dub build dubtestproject:test2 --compiler=ldc2 -# OpenSSH client is needed to run openmpi binaries. - diff --git a/ci/ciimage/eoan/image.json b/ci/ciimage/eoan/image.json new file mode 100644 index 0000000..cbf86c3 --- /dev/null +++ b/ci/ciimage/eoan/image.json @@ -0,0 +1,7 @@ +{ + "base_image": "ubuntu:eoan", + "env": { + "CI": "1", + "DC": "gdc" + } +} diff --git a/ci/ciimage/eoan/install.sh b/ci/ciimage/eoan/install.sh new file mode 100755 index 0000000..4b3b746 --- /dev/null +++ b/ci/ciimage/eoan/install.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +set -e + +source /ci/common.sh + +export DEBIAN_FRONTEND=noninteractive +export LANG='C.UTF-8' +export DC=gdc + +pkgs=( + python3-pytest-xdist + python3-pip libxml2-dev libxslt1-dev libyaml-dev libjson-glib-dev + wget unzip + qt5-default clang + pkg-config-arm-linux-gnueabihf + qt4-linguist-tools + python-dev + libomp-dev + dub ldc + mingw-w64 mingw-w64-tools nim + libclang-dev + libgcrypt20-dev + libgpgme-dev + libhdf5-dev + libboost-python-dev libboost-regex-dev + libblocksruntime-dev + libperl-dev + liblapack-dev libscalapack-mpi-dev +) + +sed -i '/^#\sdeb-src /s/^#//' "/etc/apt/sources.list" +apt-get -y update +apt-get -y upgrade +apt-get -y install eatmydata + +# Base stuff +eatmydata apt-get -y build-dep meson + +# packages +eatmydata apt-get -y install "${pkgs[@]}" +eatmydata apt-get -y install --no-install-recommends wine-stable # Wine is special + +eatmydata python3 -m pip install hotdoc codecov jsonschema + +# dub stuff +dub_fetch urld +dub build urld --compiler=gdc +dub_fetch dubtestproject +dub build dubtestproject:test1 --compiler=ldc2 +dub build dubtestproject:test2 --compiler=ldc2 + +# cleanup +apt-get -y clean +apt-get -y autoclean diff --git a/ci/ciimage/eoan/test.sh b/ci/ciimage/eoan/test.sh new file mode 100755 index 0000000..d00df6f --- /dev/null +++ b/ci/ciimage/eoan/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +set -e + +testFN() { + set +e + false +} + +testFN +false +exit 0
\ No newline at end of file diff --git a/ci/ciimage/fedora/Dockerfile b/ci/ciimage/fedora/Dockerfile deleted file mode 100644 index fa552e4..0000000 --- a/ci/ciimage/fedora/Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -FROM fedora:latest - -ADD install.sh /usr/sbin/docker-fedora-install -RUN docker-fedora-install diff --git a/ci/ciimage/fedora/image.json b/ci/ciimage/fedora/image.json new file mode 100644 index 0000000..ae9ff4f --- /dev/null +++ b/ci/ciimage/fedora/image.json @@ -0,0 +1,8 @@ +{ + "base_image": "fedora:latest", + "env": { + "CI": "1", + "SKIP_SCIENTIFIC": "1", + "SKIP_STATIC_BOOST": "1" + } +} diff --git a/ci/ciimage/fedora/install.sh b/ci/ciimage/fedora/install.sh index fd4c558..242d677 100755 --- a/ci/ciimage/fedora/install.sh +++ b/ci/ciimage/fedora/install.sh @@ -1,8 +1,10 @@ #!/bin/bash +set -e + pkgs=( python python-setuptools python-wheel python-pip python-pytest-xdist pygobject3 python3-devel python2-devel - ninja-build make git autoconf automake patch python3-Cython python2-Cython + ninja-build make git autoconf automake patch python3-Cython python2-Cython python3-jsonschema elfutils gcc gcc-c++ gcc-fortran gcc-objc gcc-objc++ vala rust bison flex ldc libasan libasan-static mono-core boost-devel gtkmm30 gtest-devel gmock-devel protobuf-devel wxGTK3-devel gobject-introspection boost-python3-devel boost-python2-devel diff --git a/ci/ciimage/opensuse/Dockerfile b/ci/ciimage/opensuse/Dockerfile deleted file mode 100644 index 3d97bac..0000000 --- a/ci/ciimage/opensuse/Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -FROM opensuse/tumbleweed:latest - -ADD install.sh /usr/sbin/docker-opensuse-install -RUN docker-opensuse-install diff --git a/ci/ciimage/opensuse/image.json b/ci/ciimage/opensuse/image.json new file mode 100644 index 0000000..74acbe5 --- /dev/null +++ b/ci/ciimage/opensuse/image.json @@ -0,0 +1,9 @@ +{ + "base_image": "opensuse/tumbleweed:latest", + "env": { + "CI": "1", + "SKIP_SCIENTIFIC": "1", + "SKIP_STATIC_BOOST": "1", + "SINGLE_DUB_COMPILER": "1" + } +} diff --git a/ci/ciimage/opensuse/install.sh b/ci/ciimage/opensuse/install.sh index b30b208..5e35ef7 100755 --- a/ci/ciimage/opensuse/install.sh +++ b/ci/ciimage/opensuse/install.sh @@ -1,11 +1,15 @@ #!/bin/bash +set -e + +source /ci/common.sh + pkgs=( - python3-setuptools python3-wheel python3-pip python3-pytest-xdist python3 python2 - ninja make git autoconf automake patch python3-Cython python2-Cython + python3-setuptools python3-wheel python3-pip python3-pytest-xdist python3 + ninja make git autoconf automake patch python3-Cython python3-jsonschema elfutils gcc gcc-c++ gcc-fortran gcc-objc gcc-obj-c++ vala rust bison flex curl mono-core gtkmm3-devel gtest gmock protobuf-devel wxGTK3-3_2-devel gobject-introspection-devel - itstool gtk3-devel java-13-openjdk-devel gtk-doc llvm-devel clang-devel libSDL2-devel graphviz-devel zlib-devel zlib-devel-static + itstool gtk3-devel java-15-openjdk-devel gtk-doc llvm-devel clang-devel libSDL2-devel graphviz-devel zlib-devel zlib-devel-static #hdf5-devel netcdf-devel libscalapack2-openmpi3-devel libscalapack2-gnu-openmpi3-hpc-devel openmpi3-devel doxygen vulkan-devel vulkan-validationlayers openssh mercurial gtk-sharp3-complete gtk-sharp2-complete libpcap-devel libgpgme-devel libqt5-qtbase-devel libqt5-qttools-devel libqt5-linguist libqt5-qtbase-private-headers-devel @@ -13,32 +17,32 @@ pkgs=( libxml2-devel libxslt-devel libyaml-devel glib2-devel json-glib-devel boost-devel libboost_date_time-devel libboost_filesystem-devel libboost_locale-devel libboost_system-devel libboost_test-devel libboost_log-devel libboost_regex-devel - libboost_python-devel libboost_python-py2_7-1_71_0-devel libboost_python-py3-1_71_0-devel libboost_regex-devel + libboost_python-devel libboost_python-py3-1_71_0-devel libboost_regex-devel ) # Sys update -zypper patch --with-update --with-optional -zypper update +zypper --non-interactive patch --with-update --with-optional +zypper --non-interactive update # Install deps zypper install -y "${pkgs[@]}" python3 -m pip install hotdoc gobject PyGObject -echo 'export PKG_CONFIG_PATH="/usr/lib64/mpi/gcc/openmpi3/lib64/pkgconfig:$PKG_CONFIG_PATH"' >> ~/.bashrc +echo 'export PKG_CONFIG_PATH="/usr/lib64/mpi/gcc/openmpi3/lib64/pkgconfig:$PKG_CONFIG_PATH"' >> /env_vars.sh # dmd is very special on OpenSUSE (as in the packages do not work) # see https://bugzilla.opensuse.org/show_bug.cgi?id=1162408 curl -fsS https://dlang.org/install.sh | bash -s dmd | tee dmd_out.txt -cat dmd_out.txt | grep source | sed 's/^[^`]*`//g' | sed 's/`.*//g' >> ~/.bashrc -chmod +x ~/.bashrc +cat dmd_out.txt | grep source | sed 's/^[^`]*`//g' | sed 's/`.*//g' >> /env_vars.sh +chmod +x /env_vars.sh -source ~/.bashrc +source /env_vars.sh -dub fetch urld +dub_fetch urld dub build urld --compiler=dmd -dub fetch dubtestproject +dub_fetch dubtestproject dub build dubtestproject:test1 --compiler=dmd dub build dubtestproject:test2 --compiler=dmd # Cleanup -zypper clean --all +zypper --non-interactive clean --all |