From ee9d6d7bf2bc91212e9d8f6906b09909b2481688 Mon Sep 17 00:00:00 2001 From: Cosmin Truta Date: Sun, 30 Jul 2023 17:07:25 +0300 Subject: [PATCH] ci: Improve the support for cross-build verifications Introduce CI_TARGET_TRIPLET and CI_TARGET_ABI. Rename CI_HOST_MACHINE and CI_TARGET_MACHINE, respectively, to CI_HOST_ARCH and CI_TARGET_ARCH, following on the conventional target triplet nomenclature. Introduce CI_BUILD_SYSTEM and CI_BUILD_ARCH, following on the GNU Autotools (host/build/target) practice and nomenclature. Ensure that CI_TARGET_SYSTEM, CI_TARGET_ARCH and CI_TARGET_ABI are all initialized when verifying a cross-platform build. Work around an obscure CMake error by ensuring that CMake variables (CMAKE_AR and CMAKE_RANLIB) are initialized to the full executable paths of their CI_* equivalents (CI_AR and CI_RANLIB). Implement other general-purpose improvements: * Check all CI_NO_* variables in an arithmetic context, to allow setting them explicitly to zero in external configurations. * Label the assertions with descriptions of what's being asserted. * Add more comments and tracing printouts. --- ci/ci_verify_cmake.sh | 56 +++++++++++++++++++++++++-------------- ci/ci_verify_configure.sh | 49 +++++++++++++++++++++++----------- ci/ci_verify_makefiles.sh | 20 +++++++++----- ci/lib/ci.lib.sh | 50 +++++++++++++++++++++++++++------- 4 files changed, 122 insertions(+), 53 deletions(-) diff --git a/ci/ci_verify_cmake.sh b/ci/ci_verify_cmake.sh index 5d53c4630..55967354a 100755 --- a/ci/ci_verify_cmake.sh +++ b/ci/ci_verify_cmake.sh @@ -16,16 +16,17 @@ cd "$CI_TOPLEVEL_DIR" CI_SRC_DIR="$CI_TOPLEVEL_DIR" CI_OUT_DIR="$CI_TOPLEVEL_DIR/out" -CI_BUILD_DIR="$CI_OUT_DIR/ci_verify_cmake.$CI_TARGET_SYSTEM.$CI_TARGET_MACHINE.build" -CI_INSTALL_DIR="$CI_OUT_DIR/ci_verify_cmake.$CI_TARGET_SYSTEM.$CI_TARGET_MACHINE.install" +CI_BUILD_DIR="$CI_OUT_DIR/ci_verify_cmake.$CI_TARGET_SYSTEM.$CI_TARGET_ARCH.build" +CI_INSTALL_DIR="$CI_OUT_DIR/ci_verify_cmake.$CI_TARGET_SYSTEM.$CI_TARGET_ARCH.install" # Keep the following relative paths in sync with the absolute paths. # We use them for the benefit of native Windows tools that might be # otherwise confused by the path encoding used by Bash-on-Windows. CI_BUILD_TO_SRC_RELDIR="../.." -CI_BUILD_TO_INSTALL_RELDIR="../ci_verify_cmake.$CI_TARGET_SYSTEM.$CI_TARGET_MACHINE.install" +CI_BUILD_TO_INSTALL_RELDIR="../ci_verify_cmake.$CI_TARGET_SYSTEM.$CI_TARGET_ARCH.install" function ci_init_build { + # Ensure that the mandatory variables are initialized. CI_CMAKE="${CI_CMAKE:-cmake}" CI_CTEST="${CI_CTEST:-ctest}" CI_CMAKE_BUILD_TYPE="${CI_CMAKE_BUILD_TYPE:-Release}" @@ -38,19 +39,20 @@ function ci_init_build { [[ $TEMP && ( $Temp || $temp ) ]] && unset TEMP [[ $TMP && ( $Tmp || $tmp ) ]] && unset TMP # Ensure that CI_CMAKE_GENERATOR_PLATFORM is initialized for this generator. - [[ $CI_CMAKE_GENERATOR_PLATFORM ]] || - ci_err "missing: \$CI_CMAKE_GENERATOR_PLATFORM" + ci_assert "checking CI_CMAKE_GENERATOR_PLATFORM" \ + -n "$CI_CMAKE_GENERATOR_PLATFORM" fi } function ci_trace_build { ci_info "## START OF CONFIGURATION ##" + ci_info "host arch: $CI_HOST_ARCH" ci_info "host system: $CI_HOST_SYSTEM" - ci_info "host machine hardware: $CI_HOST_MACHINE" - [[ "$CI_TARGET_SYSTEM" != "$CI_HOST_SYSTEM" ]] && + [[ "$CI_TARGET_SYSTEM.$CI_TARGET_ARCH" != "$CI_HOST_SYSTEM.$CI_HOST_ARCH" ]] && { + ci_info "target arch: $CI_TARGET_ARCH" ci_info "target system: $CI_TARGET_SYSTEM" - [[ "$CI_TARGET_MACHINE" != "$CI_HOST_MACHINE" ]] && - ci_info "target machine hardware: $CI_TARGET_MACHINE" + ci_info "target ABI: $CI_TARGET_ABI" + } ci_info "source directory: $CI_SRC_DIR" ci_info "build directory: $CI_BUILD_DIR" ci_info "install directory: $CI_INSTALL_DIR" @@ -110,13 +112,17 @@ function ci_build { ALL_CMAKE_VARS+=(-DCMAKE_C_COMPILER="$CI_CC") [[ $ALL_CC_FLAGS ]] && ALL_CMAKE_VARS+=(-DCMAKE_C_FLAGS="$ALL_CC_FLAGS") - [[ $CI_AR ]] && - ALL_CMAKE_VARS+=(-DCMAKE_AR="$CI_AR") - [[ $CI_RANLIB ]] && - ALL_CMAKE_VARS+=(-DCMAKE_RANLIB="$CI_RANLIB") + [[ $CI_AR ]] && { + # Use the full path of CI_AR to work around a CMake error. + ALL_CMAKE_VARS+=(-DCMAKE_AR="$(command -v "$CI_AR")") + } + [[ $CI_RANLIB ]] && { + # Use the full path of CI_RANLIB to work around a CMake error. + ALL_CMAKE_VARS+=(-DCMAKE_RANLIB="$(command -v "$CI_RANLIB")") + } ALL_CMAKE_VARS+=(-DCMAKE_BUILD_TYPE="$CI_CMAKE_BUILD_TYPE") ALL_CMAKE_VARS+=(-DCMAKE_VERBOSE_MAKEFILE=ON) - [[ $CI_NO_TEST ]] && + [[ $((CI_NO_TEST)) -ne 0 ]] && ALL_CMAKE_VARS+=(-DPNG_TESTS=OFF) ALL_CMAKE_VARS+=($CI_CMAKE_VARS) local ALL_CMAKE_BUILD_FLAGS=($CI_CMAKE_BUILD_FLAGS) @@ -126,33 +132,43 @@ function ci_build { ci_spawn export CMAKE_GENERATOR="$CI_CMAKE_GENERATOR" [[ $CI_CMAKE_GENERATOR_PLATFORM ]] && ci_spawn export CMAKE_GENERATOR_PLATFORM="$CI_CMAKE_GENERATOR_PLATFORM" - # Build and install. + # Build! # Use $CI_BUILD_TO_SRC_RELDIR and $CI_BUILD_TO_INSTALL_RELDIR # instead of $CI_SRC_DIR and $CI_INSTALL_DIR from this point onwards. ci_spawn mkdir -p "$CI_BUILD_DIR" ci_spawn cd "$CI_BUILD_DIR" - ci_assert "$CI_SRC_DIR" -ef "$CI_BUILD_TO_SRC_RELDIR" + ci_assert "checking CI_BUILD_TO_SRC_RELDIR" \ + "$CI_SRC_DIR" -ef "$CI_BUILD_TO_SRC_RELDIR" ci_spawn mkdir -p "$CI_INSTALL_DIR" - ci_assert "$CI_INSTALL_DIR" -ef "$CI_BUILD_TO_INSTALL_RELDIR" + ci_assert "checking CI_BUILD_TO_INSTALL_RELDIR" \ + "$CI_INSTALL_DIR" -ef "$CI_BUILD_TO_INSTALL_RELDIR" + # Spawn "cmake ...". ci_spawn "$CI_CMAKE" -DCMAKE_INSTALL_PREFIX="$CI_BUILD_TO_INSTALL_RELDIR" \ "${ALL_CMAKE_VARS[@]}" \ "$CI_BUILD_TO_SRC_RELDIR" + # Spawn "cmake --build ...". ci_spawn "$CI_CMAKE" --build . \ --config "$CI_CMAKE_BUILD_TYPE" \ "${ALL_CMAKE_BUILD_FLAGS[@]}" - [[ $CI_NO_TEST ]] || + [[ $((CI_NO_TEST)) -ne 0 ]] || { + # Spawn "ctest" if testing is not disabled. ci_spawn "$CI_CTEST" --build-config "$CI_CMAKE_BUILD_TYPE" \ "${ALL_CTEST_FLAGS[@]}" - [[ $CI_NO_INSTALL ]] || + } + [[ $((CI_NO_INSTALL)) -ne 0 ]] || { + # Spawn "cmake --build ... --target install" if installation is not disabled. ci_spawn "$CI_CMAKE" --build . \ --config "$CI_CMAKE_BUILD_TYPE" \ --target install \ "${ALL_CMAKE_BUILD_FLAGS[@]}" - [[ $CI_NO_CLEAN ]] || + } + [[ $((CI_NO_CLEAN)) -ne 0 ]] || { + # Spawn "make --build ... --target clean" if cleaning is not disabled. ci_spawn "$CI_CMAKE" --build . \ --config "$CI_CMAKE_BUILD_TYPE" \ --target clean \ "${ALL_CMAKE_BUILD_FLAGS[@]}" + } ci_info "## END OF BUILD ##" } diff --git a/ci/ci_verify_configure.sh b/ci/ci_verify_configure.sh index a85794118..9e0d77780 100755 --- a/ci/ci_verify_configure.sh +++ b/ci/ci_verify_configure.sh @@ -16,27 +16,33 @@ cd "$CI_TOPLEVEL_DIR" CI_SRC_DIR="$CI_TOPLEVEL_DIR" CI_OUT_DIR="$CI_TOPLEVEL_DIR/out" -CI_BUILD_DIR="$CI_OUT_DIR/ci_verify_configure.$CI_TARGET_SYSTEM.$CI_TARGET_MACHINE.build" -CI_INSTALL_DIR="$CI_OUT_DIR/ci_verify_configure.$CI_TARGET_SYSTEM.$CI_TARGET_MACHINE.install" +CI_BUILD_DIR="$CI_OUT_DIR/ci_verify_configure.$CI_TARGET_SYSTEM.$CI_TARGET_ARCH.build" +CI_INSTALL_DIR="$CI_OUT_DIR/ci_verify_configure.$CI_TARGET_SYSTEM.$CI_TARGET_ARCH.install" function ci_init_build { + # Ensure that the mandatory variables are initialized. CI_MAKE="${CI_MAKE:-make}" - # Set CI_CC to cc by default, if the cc command is available. - # The configure script defaults CC to gcc, which is not always a good idea. - [[ -x $(command -v cc) ]] && CI_CC="${CI_CC:-cc}" + [[ "$CI_TARGET_SYSTEM.$CI_TARGET_ARCH" != "$CI_HOST_SYSTEM.$CI_HOST_ARCH" ]] || { + # For native builds, set CI_CC to "cc" by default if the cc command is available. + # The configure script defaults CC to "gcc", which is not always a good idea. + [[ -x $(command -v cc) ]] && CI_CC="${CI_CC:-cc}" + } # Ensure that the CI_ variables that cannot be customized reliably are not initialized. - [[ ! $CI_CONFIGURE_VARS ]] || ci_err "unsupported: \$CI_CONFIGURE_VARS='$CI_CONFIGURE_VARS'" - [[ ! $CI_MAKE_VARS ]] || ci_err "unsupported: \$CI_MAKE_VARS='$CI_MAKE_VARS'" + [[ ! $CI_CONFIGURE_VARS ]] || + ci_err "unsupported: \$CI_CONFIGURE_VARS='$CI_CONFIGURE_VARS'" + [[ ! $CI_MAKE_VARS ]] || + ci_err "unsupported: \$CI_MAKE_VARS='$CI_MAKE_VARS'" } function ci_trace_build { ci_info "## START OF CONFIGURATION ##" + ci_info "host arch: $CI_HOST_ARCH" ci_info "host system: $CI_HOST_SYSTEM" - ci_info "host machine hardware: $CI_HOST_MACHINE" - [[ "$CI_TARGET_SYSTEM" != "$CI_HOST_SYSTEM" ]] && + [[ "$CI_TARGET_SYSTEM.$CI_TARGET_ARCH" != "$CI_HOST_SYSTEM.$CI_HOST_ARCH" ]] && { + ci_info "target arch: $CI_TARGET_ARCH" ci_info "target system: $CI_TARGET_SYSTEM" - [[ "$CI_TARGET_MACHINE" != "$CI_HOST_MACHINE" ]] && - ci_info "target machine hardware: $CI_TARGET_MACHINE" + ci_info "target ABI: $CI_TARGET_ABI" + } ci_info "source directory: $CI_SRC_DIR" ci_info "build directory: $CI_BUILD_DIR" ci_info "install directory: $CI_INSTALL_DIR" @@ -94,15 +100,26 @@ function ci_build { ci_spawn export CFLAGS="-fsanitize=$CI_SANITIZERS ${CFLAGS:-"-O2"}" ci_spawn export LDFLAGS="-fsanitize=$CI_SANITIZERS $LDFLAGS" } - # Build and install. + # Build! ci_spawn mkdir -p "$CI_BUILD_DIR" ci_spawn cd "$CI_BUILD_DIR" + # Spawn "configure". ci_spawn "$CI_SRC_DIR/configure" --prefix="$CI_INSTALL_DIR" $CI_CONFIGURE_FLAGS + # Spawn "make". ci_spawn "$CI_MAKE" $CI_MAKE_FLAGS - [[ $CI_NO_TEST ]] || ci_spawn "$CI_MAKE" $CI_MAKE_FLAGS test - [[ $CI_NO_INSTALL ]] || ci_spawn "$CI_MAKE" $CI_MAKE_FLAGS install - [[ $CI_NO_CLEAN ]] || ci_spawn "$CI_MAKE" $CI_MAKE_FLAGS clean - [[ $CI_NO_CLEAN ]] || ci_spawn "$CI_MAKE" $CI_MAKE_FLAGS distclean + [[ $((CI_NO_TEST)) -ne 0 ]] || { + # Spawn "make test" if testing is not disabled. + ci_spawn "$CI_MAKE" $CI_MAKE_FLAGS test + } + [[ $((CI_NO_INSTALL)) -ne 0 ]] || { + # Spawn "make install" if installation is not disabled. + ci_spawn "$CI_MAKE" $CI_MAKE_FLAGS install + } + [[ $((CI_NO_CLEAN)) -ne 0 ]] || { + # Spawn "make clean" and "make distclean" if cleaning is not disabled. + ci_spawn "$CI_MAKE" $CI_MAKE_FLAGS clean + ci_spawn "$CI_MAKE" $CI_MAKE_FLAGS distclean + } ci_info "## END OF BUILD ##" } diff --git a/ci/ci_verify_makefiles.sh b/ci/ci_verify_makefiles.sh index ac28c631b..ef6d7838b 100755 --- a/ci/ci_verify_makefiles.sh +++ b/ci/ci_verify_makefiles.sh @@ -17,6 +17,7 @@ cd "$CI_TOPLEVEL_DIR" CI_SRC_DIR="$CI_TOPLEVEL_DIR" function ci_init_build { + # Ensure that the mandatory variables are initialized. CI_MAKE="${CI_MAKE:-make}" case "$CI_CC" in ( *clang* ) @@ -30,12 +31,13 @@ function ci_init_build { function ci_trace_build { ci_info "## START OF CONFIGURATION ##" + ci_info "host arch: $CI_HOST_ARCH" ci_info "host system: $CI_HOST_SYSTEM" - ci_info "host machine hardware: $CI_HOST_MACHINE" - [[ "$CI_TARGET_SYSTEM" != "$CI_HOST_SYSTEM" ]] && + [[ "$CI_TARGET_SYSTEM.$CI_TARGET_ARCH" != "$CI_HOST_SYSTEM.$CI_HOST_ARCH" ]] && { + ci_info "target arch: $CI_TARGET_ARCH" ci_info "target system: $CI_TARGET_SYSTEM" - [[ "$CI_TARGET_MACHINE" != "$CI_HOST_MACHINE" ]] && - ci_info "target machine hardware: $CI_TARGET_MACHINE" + ci_info "target ABI: $CI_TARGET_ABI" + } ci_info "source directory: $CI_SRC_DIR" ci_info "environment option: \$CI_MAKEFILES: '$CI_MAKEFILES'" ci_info "environment option: \$CI_MAKE: '$CI_MAKE'" @@ -110,24 +112,28 @@ function ci_build { [[ $CI_LIBS ]] && ALL_MAKE_VARS+=(LIBS="$CI_LIBS") ALL_MAKE_VARS+=($CI_MAKE_VARS) # Build! - ci_assert "$CI_SRC_DIR" -ef . local MY_MAKEFILE for MY_MAKEFILE in $CI_MAKEFILES do ci_info "using makefile: $MY_MAKEFILE" + # Spawn "make". ci_spawn "$CI_MAKE" -f "$MY_MAKEFILE" \ "${ALL_MAKE_FLAGS[@]}" \ "${ALL_MAKE_VARS[@]}" - [[ $CI_NO_TEST ]] || + [[ $((CI_NO_TEST)) -ne 0 ]] || { + # Spawn "make test" if testing is not disabled. ci_spawn "$CI_MAKE" -f "$MY_MAKEFILE" \ "${ALL_MAKE_FLAGS[@]}" \ "${ALL_MAKE_VARS[@]}" \ test - [[ $CI_NO_CLEAN ]] || + } + [[ $((CI_NO_CLEAN)) -ne 0 ]] || { + # Spawn "make clean" if cleaning is not disabled. ci_spawn "$CI_MAKE" -f "$MY_MAKEFILE" \ "${ALL_MAKE_FLAGS[@]}" \ "${ALL_MAKE_VARS[@]}" \ clean + } done ci_info "## END OF BUILD ##" } diff --git a/ci/lib/ci.lib.sh b/ci/lib/ci.lib.sh index 7569a4693..29ab53ab8 100644 --- a/ci/lib/ci.lib.sh +++ b/ci/lib/ci.lib.sh @@ -25,11 +25,23 @@ CI_SCRIPT_NAME="$(basename -- "$0")" CI_SCRIPT_DIR="$(cd "$(dirname -- "$0")" && pwd)" CI_TOPLEVEL_DIR="$(cd "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)" -# Initialize the global constants CI_{HOST,TARGET}_{SYSTEM,MACHINE}. -CI_HOST_SYSTEM="${CI_HOST_SYSTEM:-"$(uname -s | tr 'A-Z/\.-' 'a-z____')"}" -CI_HOST_MACHINE="${CI_HOST_MACHINE:-"$(uname -m | tr 'A-Z/\.-' 'a-z____')"}" -CI_TARGET_SYSTEM="${CI_TARGET_SYSTEM:-"$CI_HOST_SYSTEM"}" -CI_TARGET_MACHINE="${CI_TARGET_MACHINE:-"$CI_HOST_MACHINE"}" +# Initialize the global constants CI_BUILD_{...} for the host build platform. +CI_BUILD_ARCH="${CI_BUILD_ARCH:-"$(uname -m | tr 'A-Z/\.-' 'a-z____')"}" +CI_BUILD_SYSTEM="${CI_BUILD_SYSTEM:-"$(uname -s | tr 'A-Z/\.-' 'a-z____')"}" + +# Initialize the global constants CI_TARGET_{...} for the target platform. +if [[ $CI_TARGET_ARCH && $CI_TARGET_SYSTEM && $CI_TARGET_ABI ]] +then + CI_TARGET_TRIPLET="${CI_TARGET_TRIPLET:-$CI_TARGET_ARCH-$CI_TARGET_SYSTEM-$CI_TARGET_ABI}" +elif [[ $CI_TARGET_TRIPLET ]] +then + CI_TARGET_ARCH="${CI_TARGET_ARCH:-${CI_TARGET_TRIPLET%%-*}}" + CI_TARGET_DOUBLET_IMPL="${CI_TARGET_TRIPLET#*-}" + CI_TARGET_SYSTEM="${CI_TARGET_SYSTEM:-${CI_TARGET_DOUBLET_IMPL%%-*}}" + CI_TARGET_ABI="${CI_TARGET_ABI:-${CI_TARGET_DOUBLET_IMPL#*-}}" +fi +CI_TARGET_ARCH="${CI_TARGET_ARCH:-"$CI_BUILD_ARCH"}" +CI_TARGET_SYSTEM="${CI_TARGET_SYSTEM:-"$CI_BUILD_SYSTEM"}" function ci_info { printf >&2 "%s: %s\\n" "$CI_SCRIPT_NAME" "$*" @@ -55,7 +67,12 @@ function ci_assert { # Use the "test" built-in command instead of the "[[ ]]" syntax, # to ensure the a-priori expansion of all assertion arguments. # (Consistently, both "ci_assert" and "test" have a command-like behavior.) - test "$@" || ci_err_internal "failed:" test "$@" + [[ $# -ge 2 ]] || + ci_err_internal "failed: ci_assert: bad or missing operands" + local label="$1" + shift + test "$@" || + ci_err_internal "failed: $label:" test "$@" } function ci_spawn { @@ -66,7 +83,20 @@ function ci_spawn { } # Ensure that the initialization is correct. -ci_assert "$CI_TOPLEVEL_DIR/ci/lib/ci.lib.sh" -ef "${BASH_SOURCE[0]}" -ci_assert "$CI_SCRIPT_DIR/$CI_SCRIPT_NAME" -ef "$0" -ci_assert -n "$CI_HOST_SYSTEM" -a -n "$CI_HOST_MACHINE" -ci_assert -n "$CI_TARGET_SYSTEM" -a -n "$CI_TARGET_MACHINE" +ci_assert "checking CI_TOPLEVEL_DIR" \ + "$CI_TOPLEVEL_DIR/ci/lib/ci.lib.sh" -ef "${BASH_SOURCE[0]}" +ci_assert "checking CI_SCRIPT_DIR and CI_SCRIPT_NAME" \ + "$CI_SCRIPT_DIR/$CI_SCRIPT_NAME" -ef "$0" +ci_assert "checking CI_BUILD_ARCH and CI_BUILD_SYSTEM" \ + -n "$CI_BUILD_ARCH" -a -n "$CI_BUILD_SYSTEM" +ci_assert "checking CI_TARGET_ARCH and CI_TARGET_SYSTEM" \ + -n "$CI_TARGET_ARCH" -a -n "$CI_TARGET_SYSTEM" +ci_assert "checking CI_TARGET_TRIPLET" \ + x"$CI_TARGET_TRIPLET" = x"" -o \ + x"$CI_TARGET_TRIPLET" = x"$CI_TARGET_ARCH-$CI_TARGET_SYSTEM-$CI_TARGET_ABI" +ci_assert "checking if CI_NO_TEST is boolean" \ + $((CI_NO_TEST)) -eq $((!!CI_NO_TEST)) +ci_assert "checking if CI_NO_INSTALL is boolean" \ + $((CI_NO_INSTALL)) -eq $((!!CI_NO_INSTALL)) +ci_assert "checking if CI_NO_CLEAN is boolean" \ + $((CI_NO_CLEAN)) -eq $((!!CI_NO_CLEAN))