From 67f059eaf5eb6cf9201ffe7776b48229a442d864 Mon Sep 17 00:00:00 2001 From: Stephen Kyle Date: Thu, 27 Sep 2018 10:00:33 +0100 Subject: Cross compilation support (#709) * build: add cross-compilation support to make Set CROSS_COMPILE when running make to use the selected cross compilation toolchain, such as arm-linux-gnueabihf, or aarch64-linux-gnu. Testing requires the presence of qemu - 'qemu-$(ARCH)' will be executed, where ARCH is the first part of the toolchain triplet. * build: add cross-compilation support to cmake If C_COMPILER/CXX_COMPILER/CC/CXX are found to have cross-compilation triplets in front of the compiler, then qemu will be used to execute the tests. * CI: add arm-linux-gnueabihf-gcc builder to Travis The version of qemu available in Ubuntu trusty (as provided by Travis) appears to have a bug in qemu-aarch64, which leads to the compatibility tests failing on some inputs, erroneously rejecting the input as corrupt. Once Travis supports xenial, we could add an aarch64-gnu-linux-gcc builder as well. * CI: propagate cmake errors out of .travis.sh Seems like even if cmake fails, the error isn't picked up by Travis. --- .travis.yml | 14 ++++++++++++++ CMakeLists.txt | 32 ++++++++++++++++++++++++++++---- Makefile | 16 ++++++++++++++-- scripts/.travis.sh | 6 +++--- tests/compatibility_test.sh | 5 ++++- tests/roundtrip_test.sh | 5 ++++- tests/run-compatibility-test.cmake | 2 ++ tests/run-roundtrip-test.cmake | 2 ++ 8 files changed, 71 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index fba9c55..f300519 100644 --- a/.travis.yml +++ b/.travis.yml @@ -76,6 +76,20 @@ matrix: - clang-3.5 ### + ## testing arm via qemu on Linux + ### + - os: linux + env: BUILD_SYSTEM=cmake C_COMPILER=arm-linux-gnueabihf-gcc CXX_COMPILER=arm-linux-gnueabihf-g++ + addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - qemu + - gcc-arm-linux-gnueabihf + - libc6-dev-armhf-cross + + ### ## PGI Community Edition on Linux ### - os: linux diff --git a/CMakeLists.txt b/CMakeLists.txt index 6f1c5a2..fc45f80 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -218,15 +218,37 @@ endif() # to run the tests... if(NOT BROTLI_DISABLE_TESTS) if(WIN32 AND NOT CMAKE_HOST_WIN32) - find_program(BROTLI_WINE NAMES wine) + find_program(BROTLI_WRAPPER NAMES wine) - if(NOT BROTLI_WINE) + if(NOT BROTLI_WRAPPER) message(STATUS "wine not found, disabling tests") set(BROTLI_DISABLE_TESTS TRUE) endif() endif() endif() +# If our compiler is a cross-compiler that we know about (arm/aarch64), +# then we need to use qemu to execute the tests. +if(NOT BROTLI_DISABLE_TESTS) + if ("${CMAKE_C_COMPILER}" MATCHES "^.*/arm-linux-gnueabihf-.*$") + message(STATUS "Detected arm-linux-gnueabihf cross-compilation") + set(BROTLI_WRAPPER "qemu-arm") + set(BROTLI_WRAPPER_LD_PREFIX "/usr/arm-linux-gnueabihf") + endif() + + if ("${CMAKE_C_COMPILER}" MATCHES "^.*/arm-linux-gnueabi-.*$") + message(STATUS "Detected arm-linux-gnueabi cross-compilation") + set(BROTLI_WRAPPER "qemu-arm") + set(BROTLI_WRAPPER_LD_PREFIX "/usr/arm-linux-gnueabi") + endif() + + if ("${CMAKE_C_COMPILER}" MATCHES "^.*/aarch64-linux-gnu-.*$") + message(STATUS "Detected aarch64-linux-gnu cross-compilation") + set(BROTLI_WRAPPER "qemu-aarch64") + set(BROTLI_WRAPPER_LD_PREFIX "/usr/aarch64-linux-gnu") + endif() +endif() + if(NOT BROTLI_DISABLE_TESTS) include(CTest) enable_testing() @@ -249,7 +271,8 @@ if(NOT BROTLI_DISABLE_TESTS) foreach(quality 1 6 9 11) add_test(NAME "${BROTLI_TEST_PREFIX}roundtrip/${INPUT}/${quality}" COMMAND "${CMAKE_COMMAND}" - -DBROTLI_WRAPPER=${BROTLI_WINE} + -DBROTLI_WRAPPER=${BROTLI_WRAPPER} + -DBROTLI_WRAPPER_LD_PREFIX=${BROTLI_WRAPPER_LD_PREFIX} -DBROTLI_CLI=$ -DQUALITY=${quality} -DINPUT=${INPUT_FILE} @@ -266,7 +289,8 @@ if(NOT BROTLI_DISABLE_TESTS) foreach(INPUT ${COMPATIBILITY_INPUTS}) add_test(NAME "${BROTLI_TEST_PREFIX}compatibility/${INPUT}" COMMAND "${CMAKE_COMMAND}" - -DBROTLI_WRAPPER=${BROTLI_WINE} + -DBROTLI_WRAPPER=${BROTLI_WRAPPER} + -DBROTLI_WRAPPER_LD_PREFIX=${BROTLI_WRAPPER_LD_PREFIX} -DBROTLI_CLI=$ -DINPUT=${CMAKE_CURRENT_SOURCE_DIR}/${INPUT} -P ${CMAKE_CURRENT_SOURCE_DIR}/tests/run-compatibility-test.cmake) diff --git a/Makefile b/Makefile index c48d798..58e563a 100644 --- a/Makefile +++ b/Makefile @@ -15,6 +15,18 @@ ifeq ($(os), Darwin) CPPFLAGS += -DOS_MACOSX endif +ifneq ($(strip $(CROSS_COMPILE)), ) + CC=$(CROSS_COMPILE)-gcc + ARCH=$(firstword $(subst -, ,$(CROSS_COMPILE))) + BROTLI_WRAPPER="qemu-$(ARCH) -L /usr/$(CROSS_COMPILE)" +endif + +# The arm-linux-gnueabi compiler defaults to Armv5. Since we only support Armv7 +# and beyond, we need to select Armv7 explicitly with march. +ifeq ($(ARCH), arm) + CFLAGS += -march=armv7-a +endif + all: test @: @@ -31,8 +43,8 @@ lib: $(LIBOBJECTS) ar -crs $(LIB_A) $(LIBOBJECTS) test: $(EXECUTABLE) - tests/compatibility_test.sh - tests/roundtrip_test.sh + tests/compatibility_test.sh $(BROTLI_WRAPPER) + tests/roundtrip_test.sh $(BROTLI_WRAPPER) clean: rm -rf $(BINDIR) $(LIB_A) diff --git a/scripts/.travis.sh b/scripts/.travis.sh index 6387e22..367cdfa 100755 --- a/scripts/.travis.sh +++ b/scripts/.travis.sh @@ -40,9 +40,9 @@ case "$1" in if [ "${CROSS_COMPILE}" = "yes" ]; then CMAKE_FLAGS="-DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER -DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=ONLY -DCMAKE_SYSTEM_NAME=Windows -DCMAKE_RC_COMPILER=${RC_COMPILER}" fi - cmake ${CMAKE_FLAGS} -DCMAKE_C_COMPILER="$CC" -DCMAKE_CXX_COMPILER="$CXX" -DENABLE_SANITIZER="${SANITIZER}" -DCMAKE_C_FLAGS="${CFLAGS}" .. - make VERBOSE=1 - ctest -V + cmake ${CMAKE_FLAGS} -DCMAKE_C_COMPILER="$CC" -DCMAKE_CXX_COMPILER="$CXX" -DENABLE_SANITIZER="${SANITIZER}" -DCMAKE_C_FLAGS="${CFLAGS}" .. || exit 1 + make VERBOSE=1 || exit 1 + ctest -V || exit 1 ;; "python") python setup.py test diff --git a/tests/compatibility_test.sh b/tests/compatibility_test.sh index 4026fd9..c5af8bf 100755 --- a/tests/compatibility_test.sh +++ b/tests/compatibility_test.sh @@ -2,10 +2,13 @@ # # Test that the brotli command-line tool can decompress old brotli-compressed # files. +# +# The first argument may be a wrapper for brotli, such as 'qemu-arm'. set -o errexit -BROTLI=bin/brotli +BROTLI_WRAPPER=$1 +BROTLI="${BROTLI_WRAPPER} bin/brotli" TMP_DIR=bin/tmp for file in tests/testdata/*.compressed*; do diff --git a/tests/roundtrip_test.sh b/tests/roundtrip_test.sh index 6c92418..90027af 100755 --- a/tests/roundtrip_test.sh +++ b/tests/roundtrip_test.sh @@ -1,10 +1,13 @@ #!/usr/bin/env bash # # Roundtrip test for the brotli command-line tool. +# +# The first argument may be a wrapper for brotli, such as 'qemu-arm'. set -o errexit -BROTLI=bin/brotli +BROTLI_WRAPPER=$1 +BROTLI="${BROTLI_WRAPPER} bin/brotli" TMP_DIR=bin/tmp INPUTS=""" tests/testdata/alice29.txt diff --git a/tests/run-compatibility-test.cmake b/tests/run-compatibility-test.cmake index fd49db8..f594d98 100644 --- a/tests/run-compatibility-test.cmake +++ b/tests/run-compatibility-test.cmake @@ -2,6 +2,8 @@ string(REGEX REPLACE "([a-zA-Z0-9\\.]+)\\.compressed(\\.[0-9]+)?$" "\\1" REFEREN string(REGEX REPLACE "\\.compressed" "" OUTPUT_FILE "${INPUT}") get_filename_component(OUTPUT_NAME "${OUTPUT_FILE}" NAME) +set(ENV{QEMU_LD_PREFIX} "${BROTLI_WRAPPER_LD_PREFIX}") + execute_process( WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" COMMAND ${BROTLI_WRAPPER} ${BROTLI_CLI} --force --decompress ${INPUT} --output=${CMAKE_CURRENT_BINARY_DIR}/${OUTPUT_NAME}.unbr diff --git a/tests/run-roundtrip-test.cmake b/tests/run-roundtrip-test.cmake index 08d4e01..39303d3 100644 --- a/tests/run-roundtrip-test.cmake +++ b/tests/run-roundtrip-test.cmake @@ -1,3 +1,5 @@ +set(ENV{QEMU_LD_PREFIX} "${BROTLI_WRAPPER_LD_PREFIX}") + execute_process( WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" COMMAND ${BROTLI_WRAPPER} ${BROTLI_CLI} --force --quality=${QUALITY} ${INPUT} --output=${OUTPUT}.br -- cgit v1.1