ci.sh (49476B)
1 #!/usr/bin/env bash 2 # Copyright (c) the JPEG XL Project Authors. All rights reserved. 3 # 4 # Use of this source code is governed by a BSD-style 5 # license that can be found in the LICENSE file. 6 7 # Continuous integration helper module. This module is meant to be called from 8 # workflows during the continuous integration build, as well as from the 9 # command line for developers. 10 11 set -eu 12 13 OS=`uname -s` 14 15 MYDIR=$(dirname $(realpath "$0")) 16 17 ### Environment parameters: 18 TEST_STACK_LIMIT="${TEST_STACK_LIMIT:-256}" 19 CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-RelWithDebInfo} 20 CMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH:-} 21 CMAKE_C_COMPILER_LAUNCHER=${CMAKE_C_COMPILER_LAUNCHER:-} 22 CMAKE_CXX_COMPILER_LAUNCHER=${CMAKE_CXX_COMPILER_LAUNCHER:-} 23 CMAKE_MAKE_PROGRAM=${CMAKE_MAKE_PROGRAM:-} 24 SKIP_BUILD="${SKIP_BUILD:-0}" 25 SKIP_TEST="${SKIP_TEST:-0}" 26 FASTER_MSAN_BUILD="${FASTER_MSAN_BUILD:-0}" 27 TARGETS="${TARGETS:-all doc}" 28 TEST_SELECTOR="${TEST_SELECTOR:-}" 29 BUILD_TARGET="${BUILD_TARGET:-}" 30 ENABLE_WASM_SIMD="${ENABLE_WASM_SIMD:-0}" 31 if [[ -n "${BUILD_TARGET}" ]]; then 32 BUILD_DIR="${BUILD_DIR:-${MYDIR}/build-${BUILD_TARGET%%-*}}" 33 else 34 BUILD_DIR="${BUILD_DIR:-${MYDIR}/build}" 35 fi 36 # Whether we should post a message in the MR when the build fails. 37 POST_MESSAGE_ON_ERROR="${POST_MESSAGE_ON_ERROR:-1}" 38 # By default, do a lightweight debian HWY package build. 39 HWY_PKG_OPTIONS="${HWY_PKG_OPTIONS:---set-envvar=HWY_EXTRA_CONFIG=-DBUILD_TESTING=OFF -DHWY_ENABLE_EXAMPLES=OFF -DHWY_ENABLE_CONTRIB=OFF}" 40 41 # Set default compilers to clang if not already set 42 export CC=${CC:-clang} 43 export CXX=${CXX:-clang++} 44 45 # Time limit for the "fuzz" command in seconds (0 means no limit). 46 FUZZER_MAX_TIME="${FUZZER_MAX_TIME:-0}" 47 48 SANITIZER="none" 49 50 51 if [[ "${BUILD_TARGET%%-*}" == "x86_64" || 52 "${BUILD_TARGET%%-*}" == "i686" ]]; then 53 # Default to building all targets, even if compiler baseline is SSE4 54 HWY_BASELINE_TARGETS=${HWY_BASELINE_TARGETS:-HWY_EMU128} 55 else 56 HWY_BASELINE_TARGETS=${HWY_BASELINE_TARGETS:-} 57 fi 58 59 # Convenience flag to pass both CMAKE_C_FLAGS and CMAKE_CXX_FLAGS 60 CMAKE_FLAGS=${CMAKE_FLAGS:-} 61 CMAKE_C_FLAGS="${CMAKE_C_FLAGS:-} ${CMAKE_FLAGS}" 62 CMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS:-} ${CMAKE_FLAGS}" 63 64 CMAKE_CROSSCOMPILING_EMULATOR=${CMAKE_CROSSCOMPILING_EMULATOR:-} 65 CMAKE_EXE_LINKER_FLAGS=${CMAKE_EXE_LINKER_FLAGS:-} 66 CMAKE_FIND_ROOT_PATH=${CMAKE_FIND_ROOT_PATH:-} 67 CMAKE_MODULE_LINKER_FLAGS=${CMAKE_MODULE_LINKER_FLAGS:-} 68 CMAKE_SHARED_LINKER_FLAGS=${CMAKE_SHARED_LINKER_FLAGS:-} 69 CMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE:-} 70 71 if [[ "${ENABLE_WASM_SIMD}" -ne "0" ]]; then 72 CMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS} -msimd128" 73 CMAKE_C_FLAGS="${CMAKE_C_FLAGS} -msimd128" 74 CMAKE_EXE_LINKER_FLAGS="${CMAKE_EXE_LINKER_FLAGS} -msimd128" 75 fi 76 77 if [[ "${ENABLE_WASM_SIMD}" -eq "2" ]]; then 78 CMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS} -DHWY_WANT_WASM2" 79 CMAKE_C_FLAGS="${CMAKE_C_FLAGS} -DHWY_WANT_WASM2" 80 fi 81 82 if [[ ! -z "${HWY_BASELINE_TARGETS}" ]]; then 83 CMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS} -DHWY_BASELINE_TARGETS=${HWY_BASELINE_TARGETS}" 84 fi 85 86 # Version inferred from the CI variables. 87 CI_COMMIT_SHA=${GITHUB_SHA:-} 88 JPEGXL_VERSION=${JPEGXL_VERSION:-} 89 90 # Benchmark parameters 91 STORE_IMAGES=${STORE_IMAGES:-1} 92 BENCHMARK_CORPORA="${MYDIR}/third_party/corpora" 93 94 # Local flags passed to sanitizers. 95 UBSAN_FLAGS=( 96 -fsanitize=alignment 97 -fsanitize=bool 98 -fsanitize=bounds 99 -fsanitize=builtin 100 -fsanitize=enum 101 -fsanitize=float-cast-overflow 102 -fsanitize=float-divide-by-zero 103 -fsanitize=integer-divide-by-zero 104 -fsanitize=null 105 -fsanitize=object-size 106 -fsanitize=pointer-overflow 107 -fsanitize=return 108 -fsanitize=returns-nonnull-attribute 109 -fsanitize=shift-base 110 -fsanitize=shift-exponent 111 -fsanitize=unreachable 112 -fsanitize=vla-bound 113 114 -fno-sanitize-recover=undefined 115 # Brunsli uses unaligned accesses to uint32_t, so alignment is just a warning. 116 -fsanitize-recover=alignment 117 ) 118 # -fsanitize=function doesn't work on aarch64 and arm. 119 if [[ "${BUILD_TARGET%%-*}" != "aarch64" && 120 "${BUILD_TARGET%%-*}" != "arm" ]]; then 121 UBSAN_FLAGS+=( 122 -fsanitize=function 123 ) 124 fi 125 if [[ "${BUILD_TARGET%%-*}" != "arm" ]]; then 126 UBSAN_FLAGS+=( 127 -fsanitize=signed-integer-overflow 128 ) 129 fi 130 131 CLANG_TIDY_BIN=$(which clang-tidy-6.0 clang-tidy-7 clang-tidy-8 clang-tidy | head -n 1) 132 # Default to "cat" if "colordiff" is not installed or if stdout is not a tty. 133 if [[ -t 1 ]]; then 134 COLORDIFF_BIN=$(which colordiff cat | head -n 1) 135 else 136 COLORDIFF_BIN="cat" 137 fi 138 FIND_BIN=$(which gfind find | head -n 1) 139 # "false" will disable wine64 when not installed. This won't allow 140 # cross-compiling. 141 WINE_BIN=$(which wine64 false | head -n 1) 142 143 CLANG_VERSION="${CLANG_VERSION:-}" 144 # Detect the clang version suffix and store it in CLANG_VERSION. For example, 145 # "6.0" for clang 6 or "7" for clang 7. 146 detect_clang_version() { 147 if [[ -n "${CLANG_VERSION}" ]]; then 148 return 0 149 fi 150 local clang_version=$("${CC:-clang}" --version | head -n1) 151 clang_version=${clang_version#"Debian "} 152 clang_version=${clang_version#"Ubuntu "} 153 local llvm_tag 154 case "${clang_version}" in 155 "clang version 6."*) 156 CLANG_VERSION="6.0" 157 ;; 158 "clang version "*) 159 # Any other clang version uses just the major version number. 160 local suffix="${clang_version#clang version }" 161 CLANG_VERSION="${suffix%%.*}" 162 ;; 163 "emcc"*) 164 # We can't use asan or msan in the emcc case. 165 ;; 166 *) 167 echo "Unknown clang version: ${clang_version}" >&2 168 return 1 169 esac 170 } 171 172 # Temporary files cleanup hooks. 173 CLEANUP_FILES=() 174 cleanup() { 175 if [[ ${#CLEANUP_FILES[@]} -ne 0 ]]; then 176 rm -fr "${CLEANUP_FILES[@]}" 177 fi 178 } 179 180 # Executed on exit. 181 on_exit() { 182 local retcode="$1" 183 # Always cleanup the CLEANUP_FILES. 184 cleanup 185 } 186 187 trap 'retcode=$?; { set +x; } 2>/dev/null; on_exit ${retcode}' INT TERM EXIT 188 189 190 # These variables are populated when calling merge_request_commits(). 191 192 # The current hash at the top of the current branch or merge request branch (if 193 # running from a merge request pipeline). 194 MR_HEAD_SHA="" 195 # The common ancestor between the current commit and the tracked branch, such 196 # as main. This includes a list 197 MR_ANCESTOR_SHA="" 198 199 # Populate MR_HEAD_SHA and MR_ANCESTOR_SHA. 200 merge_request_commits() { 201 { set +x; } 2>/dev/null 202 # GITHUB_SHA is the current reference being build in GitHub Actions. 203 if [[ -n "${GITHUB_SHA:-}" ]]; then 204 # GitHub normally does a checkout of a merge commit on a shallow repository 205 # by default. We want to get a bit more of the history to be able to diff 206 # changes on the Pull Request if needed. This fetches 10 more commits which 207 # should be enough given that PR normally should have 1 commit. 208 git -C "${MYDIR}" fetch -q origin "${GITHUB_SHA}" --depth 10 209 if [ "${GITHUB_EVENT_NAME}" = "pull_request" ]; then 210 MR_HEAD_SHA="$(git rev-parse "FETCH_HEAD^2" 2>/dev/null || 211 echo "${GITHUB_SHA}")" 212 else 213 MR_HEAD_SHA="${GITHUB_SHA}" 214 fi 215 else 216 MR_HEAD_SHA=$(git -C "${MYDIR}" rev-parse -q "HEAD") 217 fi 218 219 if [[ -n "${GITHUB_BASE_REF:-}" ]]; then 220 # Pull request workflow in GitHub Actions. GitHub checkout action uses 221 # "origin" as the remote for the git checkout. 222 git -C "${MYDIR}" fetch -q origin "${GITHUB_BASE_REF}" 223 MR_ANCESTOR_SHA=$(git -C "${MYDIR}" rev-parse -q FETCH_HEAD) 224 else 225 # We are in a local branch, not a pull request workflow. 226 MR_ANCESTOR_SHA=$(git -C "${MYDIR}" rev-parse -q HEAD@{upstream} || true) 227 fi 228 229 if [[ -z "${MR_ANCESTOR_SHA}" ]]; then 230 echo "Warning, not tracking any branch, using the last commit in HEAD.">&2 231 # This prints the return value with just HEAD. 232 MR_ANCESTOR_SHA=$(git -C "${MYDIR}" rev-parse -q "${MR_HEAD_SHA}^") 233 else 234 # GitHub runs the pipeline on a merge commit, no need to look for the common 235 # ancestor in that case. 236 if [[ -z "${GITHUB_BASE_REF:-}" ]]; then 237 MR_ANCESTOR_SHA=$(git -C "${MYDIR}" merge-base \ 238 "${MR_ANCESTOR_SHA}" "${MR_HEAD_SHA}") 239 fi 240 fi 241 set -x 242 } 243 244 245 # Set up and export the environment variables needed by the child processes. 246 export_env() { 247 if [[ "${BUILD_TARGET}" == *mingw32 ]]; then 248 # Wine needs to know the paths to the mingw dlls. These should be 249 # separated by ';'. 250 WINEPATH=$("${CC:-clang}" -print-search-dirs --target="${BUILD_TARGET}" \ 251 | grep -F 'libraries: =' | cut -f 2- -d '=' | tr ':' ';') 252 # We also need our own libraries in the wine path. 253 local real_build_dir=$(realpath "${BUILD_DIR}") 254 # Some library .dll dependencies are installed in /bin: 255 export WINEPATH="${WINEPATH};${real_build_dir};${real_build_dir}/third_party/brotli;/usr/${BUILD_TARGET}/bin" 256 257 local prefix="${BUILD_DIR}/wineprefix" 258 mkdir -p "${prefix}" 259 export WINEPREFIX=$(realpath "${prefix}") 260 fi 261 # Sanitizers need these variables to print and properly format the stack 262 # traces: 263 LLVM_SYMBOLIZER=$("${CC:-clang}" -print-prog-name=llvm-symbolizer || true) 264 if [[ -n "${LLVM_SYMBOLIZER}" ]]; then 265 export ASAN_SYMBOLIZER_PATH="${LLVM_SYMBOLIZER}" 266 export MSAN_SYMBOLIZER_PATH="${LLVM_SYMBOLIZER}" 267 export UBSAN_SYMBOLIZER_PATH="${LLVM_SYMBOLIZER}" 268 fi 269 } 270 271 cmake_configure() { 272 export_env 273 274 if [[ "${STACK_SIZE:-0}" == 1 ]]; then 275 # Dump the stack size of each function in the .stack_sizes section for 276 # analysis. 277 CMAKE_C_FLAGS+=" -fstack-size-section" 278 CMAKE_CXX_FLAGS+=" -fstack-size-section" 279 fi 280 281 local args=( 282 -B"${BUILD_DIR}" -H"${MYDIR}" 283 -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" 284 -G Ninja 285 -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}" 286 -DCMAKE_C_FLAGS="${CMAKE_C_FLAGS}" 287 -DCMAKE_EXE_LINKER_FLAGS="${CMAKE_EXE_LINKER_FLAGS}" 288 -DCMAKE_MODULE_LINKER_FLAGS="${CMAKE_MODULE_LINKER_FLAGS}" 289 -DCMAKE_SHARED_LINKER_FLAGS="${CMAKE_SHARED_LINKER_FLAGS}" 290 -DJPEGXL_VERSION="${JPEGXL_VERSION}" 291 -DSANITIZER="${SANITIZER}" 292 # These are not enabled by default in cmake. 293 -DJPEGXL_ENABLE_VIEWERS=ON 294 -DJPEGXL_ENABLE_PLUGINS=ON 295 -DJPEGXL_ENABLE_DEVTOOLS=ON 296 # We always use libfuzzer in the ci.sh wrapper. 297 -DJPEGXL_FUZZER_LINK_FLAGS="-fsanitize=fuzzer" 298 ) 299 if [[ "${BUILD_TARGET}" != *mingw32 ]]; then 300 args+=( 301 -DJPEGXL_WARNINGS_AS_ERRORS=ON 302 ) 303 fi 304 if [[ -n "${BUILD_TARGET}" ]]; then 305 local system_name="Linux" 306 if [[ "${BUILD_TARGET}" == *mingw32 ]]; then 307 # When cross-compiling with mingw the target must be set to Windows and 308 # run programs with wine. 309 system_name="Windows" 310 args+=( 311 -DCMAKE_CROSSCOMPILING_EMULATOR="${WINE_BIN}" 312 # Normally CMake automatically defines MINGW=1 when building with the 313 # mingw compiler (x86_64-w64-mingw32-gcc) but we are normally compiling 314 # with clang. 315 -DMINGW=1 316 ) 317 fi 318 # EMSCRIPTEN toolchain sets the right values itself 319 if [[ "${BUILD_TARGET}" != wasm* ]]; then 320 # If set, BUILD_TARGET must be the target triplet such as 321 # x86_64-unknown-linux-gnu. 322 args+=( 323 -DCMAKE_C_COMPILER_TARGET="${BUILD_TARGET}" 324 -DCMAKE_CXX_COMPILER_TARGET="${BUILD_TARGET}" 325 # Only the first element of the target triplet. 326 -DCMAKE_SYSTEM_PROCESSOR="${BUILD_TARGET%%-*}" 327 -DCMAKE_SYSTEM_NAME="${system_name}" 328 -DCMAKE_TOOLCHAIN_FILE="${CMAKE_TOOLCHAIN_FILE}" 329 ) 330 else 331 args+=( 332 # sjpeg confuses WASM SIMD with SSE. 333 -DSJPEG_ENABLE_SIMD=OFF 334 # Building shared libs is not very useful for WASM. 335 -DBUILD_SHARED_LIBS=OFF 336 ) 337 fi 338 args+=( 339 # These are needed to make googletest work when cross-compiling. 340 -DCMAKE_CROSSCOMPILING=1 341 -DHAVE_STD_REGEX=0 342 -DHAVE_POSIX_REGEX=0 343 -DHAVE_GNU_POSIX_REGEX=0 344 -DHAVE_STEADY_CLOCK=0 345 -DHAVE_THREAD_SAFETY_ATTRIBUTES=0 346 ) 347 if [[ -z "${CMAKE_FIND_ROOT_PATH}" ]]; then 348 # find_package() will look in this prefix for libraries. 349 CMAKE_FIND_ROOT_PATH="/usr/${BUILD_TARGET}" 350 fi 351 if [[ -z "${CMAKE_PREFIX_PATH}" ]]; then 352 CMAKE_PREFIX_PATH="/usr/${BUILD_TARGET}" 353 fi 354 # Use pkg-config for the target. If there's no pkg-config available for the 355 # target we can set the PKG_CONFIG_PATH to the appropriate path in most 356 # linux distributions. 357 local pkg_config=$(which "${BUILD_TARGET}-pkg-config" || true) 358 if [[ -z "${pkg_config}" ]]; then 359 pkg_config=$(which pkg-config) 360 export PKG_CONFIG_LIBDIR="/usr/${BUILD_TARGET}/lib/pkgconfig" 361 fi 362 if [[ -n "${pkg_config}" ]]; then 363 args+=(-DPKG_CONFIG_EXECUTABLE="${pkg_config}") 364 fi 365 fi 366 if [[ -n "${CMAKE_CROSSCOMPILING_EMULATOR}" ]]; then 367 args+=( 368 -DCMAKE_CROSSCOMPILING_EMULATOR="${CMAKE_CROSSCOMPILING_EMULATOR}" 369 ) 370 fi 371 if [[ -n "${CMAKE_FIND_ROOT_PATH}" ]]; then 372 args+=( 373 -DCMAKE_FIND_ROOT_PATH="${CMAKE_FIND_ROOT_PATH}" 374 ) 375 fi 376 if [[ -n "${CMAKE_PREFIX_PATH}" ]]; then 377 args+=( 378 -DCMAKE_PREFIX_PATH="${CMAKE_PREFIX_PATH}" 379 ) 380 fi 381 if [[ -n "${CMAKE_C_COMPILER_LAUNCHER}" ]]; then 382 args+=( 383 -DCMAKE_C_COMPILER_LAUNCHER="${CMAKE_C_COMPILER_LAUNCHER}" 384 ) 385 fi 386 if [[ -n "${CMAKE_CXX_COMPILER_LAUNCHER}" ]]; then 387 args+=( 388 -DCMAKE_CXX_COMPILER_LAUNCHER="${CMAKE_CXX_COMPILER_LAUNCHER}" 389 ) 390 fi 391 if [[ -n "${CMAKE_MAKE_PROGRAM}" ]]; then 392 args+=( 393 -DCMAKE_MAKE_PROGRAM="${CMAKE_MAKE_PROGRAM}" 394 ) 395 fi 396 if [[ "${BUILD_TARGET}" == wasm* ]]; then 397 emcmake cmake "${args[@]}" "$@" 398 else 399 cmake "${args[@]}" "$@" 400 fi 401 } 402 403 cmake_build_and_test() { 404 if [[ "${SKIP_BUILD}" -eq "1" ]]; then 405 return 0 406 fi 407 # gtest_discover_tests() runs the test binaries to discover the list of tests 408 # at build time, which fails under qemu. 409 ASAN_OPTIONS=detect_leaks=0 cmake --build "${BUILD_DIR}" -- $TARGETS 410 # Pack test binaries if requested. 411 if [[ "${PACK_TEST:-}" == "1" ]]; then 412 (cd "${BUILD_DIR}" 413 ${FIND_BIN} -name '*.cmake' -a '!' -path '*CMakeFiles*' 414 # gtest / gmock / gtest_main shared libs 415 ${FIND_BIN} lib/ -name 'libg*.so*' 416 ${FIND_BIN} -type d -name tests -a '!' -path '*CMakeFiles*' 417 ) | tar -C "${BUILD_DIR}" -cf "${BUILD_DIR}/tests.tar.xz" -T - \ 418 --use-compress-program="xz --threads=$(nproc --all || echo 1) -6" 419 du -h "${BUILD_DIR}/tests.tar.xz" 420 # Pack coverage data if also available. 421 touch "${BUILD_DIR}/gcno.sentinel" 422 (cd "${BUILD_DIR}"; echo gcno.sentinel; ${FIND_BIN} -name '*gcno') | \ 423 tar -C "${BUILD_DIR}" -cvf "${BUILD_DIR}/gcno.tar.xz" -T - \ 424 --use-compress-program="xz --threads=$(nproc --all || echo 1) -6" 425 fi 426 427 if [[ "${SKIP_TEST}" -ne "1" ]]; then 428 (cd "${BUILD_DIR}" 429 export UBSAN_OPTIONS=print_stacktrace=1 430 [[ "${TEST_STACK_LIMIT}" == "none" ]] || ulimit -s "${TEST_STACK_LIMIT}" 431 ctest -j $(nproc --all || echo 1) ${TEST_SELECTOR} --output-on-failure) 432 fi 433 } 434 435 # Configure the build to strip unused functions. This considerably reduces the 436 # output size, specially for tests which only use a small part of the whole 437 # library. 438 strip_dead_code() { 439 # Emscripten does tree shaking without any extra flags. 440 if [[ "${BUILD_TARGET}" == wasm* ]]; then 441 return 0 442 fi 443 # -ffunction-sections, -fdata-sections and -Wl,--gc-sections effectively 444 # discard all unreachable code, reducing the code size. For this to work, we 445 # need to also pass --no-export-dynamic to prevent it from exporting all the 446 # internal symbols (like functions) making them all reachable and thus not a 447 # candidate for removal. 448 CMAKE_CXX_FLAGS+=" -ffunction-sections -fdata-sections" 449 CMAKE_C_FLAGS+=" -ffunction-sections -fdata-sections" 450 if [[ "${OS}" == "Darwin" ]]; then 451 CMAKE_EXE_LINKER_FLAGS+=" -dead_strip" 452 CMAKE_SHARED_LINKER_FLAGS+=" -dead_strip" 453 else 454 CMAKE_EXE_LINKER_FLAGS+=" -Wl,--gc-sections -Wl,--no-export-dynamic" 455 CMAKE_SHARED_LINKER_FLAGS+=" -Wl,--gc-sections -Wl,--no-export-dynamic" 456 fi 457 } 458 459 ### Externally visible commands 460 461 cmd_debug() { 462 CMAKE_BUILD_TYPE="Debug" 463 cmake_configure "$@" 464 cmake_build_and_test 465 } 466 467 cmd_release() { 468 CMAKE_BUILD_TYPE="Release" 469 strip_dead_code 470 cmake_configure "$@" 471 cmake_build_and_test 472 } 473 474 cmd_opt() { 475 CMAKE_BUILD_TYPE="RelWithDebInfo" 476 CMAKE_CXX_FLAGS+=" -DJXL_DEBUG_WARNING -DJXL_DEBUG_ON_ERROR" 477 cmake_configure "$@" 478 cmake_build_and_test 479 } 480 481 cmd_coverage() { 482 # -O0 prohibits stack space reuse -> causes stack-overflow on dozens of tests. 483 TEST_STACK_LIMIT="none" 484 485 cmd_release -DJPEGXL_ENABLE_COVERAGE=ON "$@" 486 487 if [[ "${SKIP_TEST}" -ne "1" ]]; then 488 # If we didn't run the test we also don't print a coverage report. 489 cmd_coverage_report 490 fi 491 } 492 493 cmd_coverage_report() { 494 LLVM_COV=$("${CC:-clang}" -print-prog-name=llvm-cov) 495 local real_build_dir=$(realpath "${BUILD_DIR}") 496 local gcovr_args=( 497 -r "${real_build_dir}" 498 --gcov-executable "${LLVM_COV} gcov" 499 # Only print coverage information for the libjxl directories. The rest 500 # is not part of the code under test. 501 --filter '.*jxl/.*' 502 --exclude '.*_gbench.cc' 503 --exclude '.*_test.cc' 504 --exclude '.*_testonly..*' 505 --exclude '.*_debug.*' 506 --exclude '.*test_utils..*' 507 --object-directory "${real_build_dir}" 508 ) 509 510 ( 511 cd "${real_build_dir}" 512 gcovr "${gcovr_args[@]}" --html --html-details \ 513 --output="${real_build_dir}/coverage.html" 514 gcovr "${gcovr_args[@]}" --print-summary | 515 tee "${real_build_dir}/coverage.txt" 516 gcovr "${gcovr_args[@]}" --xml --output="${real_build_dir}/coverage.xml" 517 ) 518 } 519 520 cmd_test() { 521 export_env 522 # Unpack tests if needed. 523 if [[ -e "${BUILD_DIR}/tests.tar.xz" && ! -d "${BUILD_DIR}/tests" ]]; then 524 tar -C "${BUILD_DIR}" -Jxvf "${BUILD_DIR}/tests.tar.xz" 525 fi 526 if [[ -e "${BUILD_DIR}/gcno.tar.xz" && ! -d "${BUILD_DIR}/gcno.sentinel" ]]; then 527 tar -C "${BUILD_DIR}" -Jxvf "${BUILD_DIR}/gcno.tar.xz" 528 fi 529 (cd "${BUILD_DIR}" 530 export UBSAN_OPTIONS=print_stacktrace=1 531 [[ "${TEST_STACK_LIMIT}" == "none" ]] || ulimit -s "${TEST_STACK_LIMIT}" 532 ctest -j $(nproc --all || echo 1) ${TEST_SELECTOR} --output-on-failure "$@") 533 } 534 535 cmd_gbench() { 536 export_env 537 (cd "${BUILD_DIR}" 538 export UBSAN_OPTIONS=print_stacktrace=1 539 lib/jxl_gbench \ 540 --benchmark_counters_tabular=true \ 541 --benchmark_out_format=json \ 542 --benchmark_out=gbench.json "$@" 543 ) 544 } 545 546 cmd_asanfuzz() { 547 CMAKE_CXX_FLAGS+=" -fsanitize=fuzzer-no-link -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION=1" 548 CMAKE_C_FLAGS+=" -fsanitize=fuzzer-no-link -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION=1" 549 cmd_asan -DJPEGXL_ENABLE_FUZZERS=ON "$@" 550 } 551 552 cmd_msanfuzz() { 553 # Install msan if needed before changing the flags. 554 detect_clang_version 555 local msan_prefix="${HOME}/.msan/${CLANG_VERSION}" 556 if [[ ! -d "${msan_prefix}" || -e "${msan_prefix}/lib/libc++abi.a" ]]; then 557 # Install msan libraries for this version if needed or if an older version 558 # with libc++abi was installed. 559 cmd_msan_install 560 fi 561 562 CMAKE_CXX_FLAGS+=" -fsanitize=fuzzer-no-link -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION=1" 563 CMAKE_C_FLAGS+=" -fsanitize=fuzzer-no-link -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION=1" 564 cmd_msan -DJPEGXL_ENABLE_FUZZERS=ON "$@" 565 } 566 567 cmd_asan() { 568 SANITIZER="asan" 569 CMAKE_C_FLAGS+=" -DJXL_ENABLE_ASSERT=1 -g -DADDRESS_SANITIZER \ 570 -fsanitize=address ${UBSAN_FLAGS[@]}" 571 CMAKE_CXX_FLAGS+=" -DJXL_ENABLE_ASSERT=1 -g -DADDRESS_SANITIZER \ 572 -fsanitize=address ${UBSAN_FLAGS[@]}" 573 strip_dead_code 574 cmake_configure "$@" -DJPEGXL_ENABLE_TCMALLOC=OFF 575 cmake_build_and_test 576 } 577 578 cmd_tsan() { 579 SANITIZER="tsan" 580 local tsan_args=( 581 -DJXL_ENABLE_ASSERT=1 582 -g 583 -DTHREAD_SANITIZER 584 ${UBSAN_FLAGS[@]} 585 -fsanitize=thread 586 ) 587 CMAKE_C_FLAGS+=" ${tsan_args[@]}" 588 CMAKE_CXX_FLAGS+=" ${tsan_args[@]}" 589 590 CMAKE_BUILD_TYPE="RelWithDebInfo" 591 cmake_configure "$@" -DJPEGXL_ENABLE_TCMALLOC=OFF 592 cmake_build_and_test 593 } 594 595 cmd_msan() { 596 SANITIZER="msan" 597 detect_clang_version 598 local msan_prefix="${HOME}/.msan/${CLANG_VERSION}" 599 if [[ ! -d "${msan_prefix}" || -e "${msan_prefix}/lib/libc++abi.a" ]]; then 600 # Install msan libraries for this version if needed or if an older version 601 # with libc++abi was installed. 602 cmd_msan_install 603 fi 604 605 local msan_c_flags=( 606 -fsanitize=memory 607 -fno-omit-frame-pointer 608 609 -DJXL_ENABLE_ASSERT=1 610 -g 611 -DMEMORY_SANITIZER 612 613 # Force gtest to not use the cxxbai. 614 -DGTEST_HAS_CXXABI_H_=0 615 ) 616 if [[ "${FASTER_MSAN_BUILD}" -ne "1" ]]; then 617 msan_c_flags=( 618 "${msan_c_flags[@]}" 619 -fsanitize-memory-track-origins 620 ) 621 fi 622 623 local msan_cxx_flags=( 624 "${msan_c_flags[@]}" 625 626 # Some C++ sources don't use the std at all, so the -stdlib=libc++ is unused 627 # in those cases. Ignore the warning. 628 -Wno-unused-command-line-argument 629 -stdlib=libc++ 630 631 # We include the libc++ from the msan directory instead, so we don't want 632 # the std includes. 633 -nostdinc++ 634 -cxx-isystem"${msan_prefix}/include/c++/v1" 635 ) 636 637 local msan_linker_flags=( 638 -L"${msan_prefix}"/lib 639 -Wl,-rpath -Wl,"${msan_prefix}"/lib/ 640 ) 641 642 CMAKE_C_FLAGS+=" ${msan_c_flags[@]} ${UBSAN_FLAGS[@]}" 643 CMAKE_CXX_FLAGS+=" ${msan_cxx_flags[@]} ${UBSAN_FLAGS[@]}" 644 CMAKE_EXE_LINKER_FLAGS+=" ${msan_linker_flags[@]}" 645 CMAKE_MODULE_LINKER_FLAGS+=" ${msan_linker_flags[@]}" 646 CMAKE_SHARED_LINKER_FLAGS+=" ${msan_linker_flags[@]}" 647 strip_dead_code 648 cmake_configure "$@" \ 649 -DCMAKE_CROSSCOMPILING=1 -DRUN_HAVE_STD_REGEX=0 -DRUN_HAVE_POSIX_REGEX=0 \ 650 -DJPEGXL_ENABLE_TCMALLOC=OFF -DJPEGXL_WARNINGS_AS_ERRORS=OFF \ 651 -DCMAKE_REQUIRED_LINK_OPTIONS="${msan_linker_flags[@]}" 652 cmake_build_and_test 653 } 654 655 # Install libc++ libraries compiled with msan in the msan_prefix for the current 656 # compiler version. 657 cmd_msan_install() { 658 local tmpdir=$(mktemp -d) 659 CLEANUP_FILES+=("${tmpdir}") 660 # Detect the llvm to install: 661 export CC="${CC:-clang}" 662 export CXX="${CXX:-clang++}" 663 detect_clang_version 664 # Allow overriding the LLVM checkout. 665 local llvm_root="${LLVM_ROOT:-}" 666 if [ -z "${llvm_root}" ]; then 667 local llvm_tag="llvmorg-${CLANG_VERSION}.0.0" 668 case "${CLANG_VERSION}" in 669 "6.0") 670 llvm_tag="llvmorg-6.0.1" 671 ;; 672 "7") 673 llvm_tag="llvmorg-7.0.1" 674 ;; 675 esac 676 local llvm_targz="${tmpdir}/${llvm_tag}.tar.gz" 677 curl -L --show-error -o "${llvm_targz}" \ 678 "https://github.com/llvm/llvm-project/archive/${llvm_tag}.tar.gz" 679 tar -C "${tmpdir}" -zxf "${llvm_targz}" 680 llvm_root="${tmpdir}/llvm-project-${llvm_tag}" 681 fi 682 683 local msan_prefix="${HOME}/.msan/${CLANG_VERSION}" 684 rm -rf "${msan_prefix}" 685 686 local TARGET_OPTS="" 687 if [[ -n "${BUILD_TARGET}" ]]; then 688 TARGET_OPTS=" \ 689 -DCMAKE_C_COMPILER_TARGET=\"${BUILD_TARGET}\" \ 690 -DCMAKE_CXX_COMPILER_TARGET=\"${BUILD_TARGET}\" \ 691 -DCMAKE_SYSTEM_PROCESSOR=\"${BUILD_TARGET%%-*}\" \ 692 " 693 fi 694 695 declare -A CMAKE_EXTRAS 696 CMAKE_EXTRAS[libcxx]="\ 697 -DLIBCXX_CXX_ABI=libstdc++ \ 698 -DLIBCXX_INSTALL_EXPERIMENTAL_LIBRARY=ON" 699 700 for project in libcxx; do 701 local proj_build="${tmpdir}/build-${project}" 702 local proj_dir="${llvm_root}/${project}" 703 mkdir -p "${proj_build}" 704 cmake -B"${proj_build}" -H"${proj_dir}" \ 705 -G Ninja \ 706 -DCMAKE_BUILD_TYPE=Release \ 707 -DLLVM_USE_SANITIZER=Memory \ 708 -DLLVM_PATH="${llvm_root}/llvm" \ 709 -DLLVM_CONFIG_PATH="$(which llvm-config llvm-config-7 llvm-config-6.0 | \ 710 head -n1)" \ 711 -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}" \ 712 -DCMAKE_C_FLAGS="${CMAKE_C_FLAGS}" \ 713 -DCMAKE_EXE_LINKER_FLAGS="${CMAKE_EXE_LINKER_FLAGS}" \ 714 -DCMAKE_SHARED_LINKER_FLAGS="${CMAKE_SHARED_LINKER_FLAGS}" \ 715 -DCMAKE_INSTALL_PREFIX="${msan_prefix}" \ 716 ${TARGET_OPTS} \ 717 ${CMAKE_EXTRAS[${project}]} 718 cmake --build "${proj_build}" 719 ninja -C "${proj_build}" install 720 done 721 } 722 723 # Internal build step shared between all cmd_ossfuzz_* commands. 724 _cmd_ossfuzz() { 725 local sanitizer="$1" 726 shift 727 mkdir -p "${BUILD_DIR}" 728 local real_build_dir=$(realpath "${BUILD_DIR}") 729 730 # oss-fuzz defines three directories: 731 # * /work, with the working directory to do re-builds 732 # * /src, with the source code to build 733 # * /out, with the output directory where to copy over the built files. 734 # We use $BUILD_DIR as the /work and the script directory as the /src. The 735 # /out directory is ignored as developers are used to look for the fuzzers in 736 # $BUILD_DIR/tools/ directly. 737 738 if [[ "${sanitizer}" = "memory" && ! -d "${BUILD_DIR}/msan" ]]; then 739 sudo docker run --rm -i \ 740 --user $(id -u):$(id -g) \ 741 -v "${real_build_dir}":/work \ 742 gcr.io/oss-fuzz-base/msan-libs-builder \ 743 bash -c "cp -r /msan /work" 744 fi 745 746 # Args passed to ninja. These will be evaluated as a string separated by 747 # spaces. 748 local jpegxl_extra_args="$@" 749 750 sudo docker run --rm -i \ 751 -e JPEGXL_UID=$(id -u) \ 752 -e JPEGXL_GID=$(id -g) \ 753 -e FUZZING_ENGINE="${FUZZING_ENGINE:-libfuzzer}" \ 754 -e SANITIZER="${sanitizer}" \ 755 -e ARCHITECTURE=x86_64 \ 756 -e FUZZING_LANGUAGE=c++ \ 757 -e MSAN_LIBS_PATH="/work/msan" \ 758 -e JPEGXL_EXTRA_ARGS="${jpegxl_extra_args}" \ 759 -v "${MYDIR}":/src/libjxl \ 760 -v "${MYDIR}/tools/scripts/ossfuzz-build.sh":/src/build.sh \ 761 -v "${real_build_dir}":/work \ 762 gcr.io/oss-fuzz/libjxl 763 } 764 765 cmd_ossfuzz_asan() { 766 _cmd_ossfuzz address "$@" 767 } 768 cmd_ossfuzz_msan() { 769 _cmd_ossfuzz memory "$@" 770 } 771 cmd_ossfuzz_ubsan() { 772 _cmd_ossfuzz undefined "$@" 773 } 774 775 cmd_ossfuzz_ninja() { 776 [[ -e "${BUILD_DIR}/build.ninja" ]] 777 local real_build_dir=$(realpath "${BUILD_DIR}") 778 779 if [[ -e "${BUILD_DIR}/msan" ]]; then 780 echo "ossfuzz_ninja doesn't work with msan builds. Use ossfuzz_msan." >&2 781 exit 1 782 fi 783 784 sudo docker run --rm -i \ 785 --user $(id -u):$(id -g) \ 786 -v "${MYDIR}":/src/libjxl \ 787 -v "${real_build_dir}":/work \ 788 gcr.io/oss-fuzz/libjxl \ 789 ninja -C /work "$@" 790 } 791 792 cmd_fast_benchmark() { 793 local small_corpus_tar="${BENCHMARK_CORPORA}/jyrki-full.tar" 794 mkdir -p "${BENCHMARK_CORPORA}" 795 curl --show-error -o "${small_corpus_tar}" -z "${small_corpus_tar}" \ 796 "https://storage.googleapis.com/artifacts.jpegxl.appspot.com/corpora/jyrki-full.tar" 797 798 local tmpdir=$(mktemp -d) 799 CLEANUP_FILES+=("${tmpdir}") 800 tar -xf "${small_corpus_tar}" -C "${tmpdir}" 801 802 run_benchmark "${tmpdir}" 1048576 803 } 804 805 cmd_benchmark() { 806 local nikon_corpus_tar="${BENCHMARK_CORPORA}/nikon-subset.tar" 807 mkdir -p "${BENCHMARK_CORPORA}" 808 curl --show-error -o "${nikon_corpus_tar}" -z "${nikon_corpus_tar}" \ 809 "https://storage.googleapis.com/artifacts.jpegxl.appspot.com/corpora/nikon-subset.tar" 810 811 local tmpdir=$(mktemp -d) 812 CLEANUP_FILES+=("${tmpdir}") 813 tar -xvf "${nikon_corpus_tar}" -C "${tmpdir}" 814 815 local sem_id="jpegxl_benchmark-$$" 816 local nprocs=$(nproc --all || echo 1) 817 images=() 818 local filename 819 while IFS= read -r filename; do 820 # This removes the './' 821 filename="${filename:2}" 822 local mode 823 if [[ "${filename:0:4}" == "srgb" ]]; then 824 mode="RGB_D65_SRG_Rel_SRG" 825 elif [[ "${filename:0:5}" == "adobe" ]]; then 826 mode="RGB_D65_Ado_Rel_Ado" 827 else 828 echo "Unknown image colorspace: ${filename}" >&2 829 exit 1 830 fi 831 png_filename="${filename%.ppm}.png" 832 png_filename=$(echo "${png_filename}" | tr '/' '_') 833 sem --bg --id "${sem_id}" -j"${nprocs}" -- \ 834 "${BUILD_DIR}/tools/decode_and_encode" \ 835 "${tmpdir}/${filename}" "${mode}" "${tmpdir}/${png_filename}" 836 images+=( "${png_filename}" ) 837 done < <(cd "${tmpdir}"; ${FIND_BIN} . -name '*.ppm' -type f) 838 sem --id "${sem_id}" --wait 839 840 # We need about 10 GiB per thread on these images. 841 run_benchmark "${tmpdir}" 10485760 842 } 843 844 get_mem_available() { 845 if [[ "${OS}" == "Darwin" ]]; then 846 echo $(vm_stat | grep -F 'Pages free:' | awk '{print $3 * 4}') 847 else 848 echo $(grep -F MemAvailable: /proc/meminfo | awk '{print $2}') 849 fi 850 } 851 852 run_benchmark() { 853 local src_img_dir="$1" 854 local mem_per_thread="${2:-10485760}" 855 856 local output_dir="${BUILD_DIR}/benchmark_results" 857 mkdir -p "${output_dir}" 858 859 # The memory available at the beginning of the benchmark run in kB. The number 860 # of threads depends on the available memory, and the passed memory per 861 # thread. We also add a 2 GiB of constant memory. 862 local mem_available="$(get_mem_available)" 863 # Check that we actually have a MemAvailable value. 864 [[ -n "${mem_available}" ]] 865 local num_threads=$(( (${mem_available} - 1048576) / ${mem_per_thread} )) 866 if [[ ${num_threads} -le 0 ]]; then 867 num_threads=1 868 fi 869 870 local benchmark_args=( 871 --input "${src_img_dir}/*.png" 872 --codec=jpeg:yuv420:q85,webp:q80,jxl:d1:6,jxl:d1:6:downsampling=8,jxl:d5:6,jxl:d5:6:downsampling=8,jxl:m:d0:2,jxl:m:d0:3,jxl:m:d2:2 873 --output_dir "${output_dir}" 874 --show_progress 875 --num_threads="${num_threads}" 876 ) 877 if [[ "${STORE_IMAGES}" == "1" ]]; then 878 benchmark_args+=(--save_decompressed --save_compressed) 879 fi 880 ( 881 [[ "${TEST_STACK_LIMIT}" == "none" ]] || ulimit -s "${TEST_STACK_LIMIT}" 882 "${BUILD_DIR}/tools/benchmark_xl" "${benchmark_args[@]}" | \ 883 tee "${output_dir}/results.txt" 884 885 # Check error code for benckmark_xl command. This will exit if not. 886 return ${PIPESTATUS[0]} 887 ) 888 889 890 } 891 892 # Helper function to wait for the CPU temperature to cool down on ARM. 893 wait_for_temp() { 894 { set +x; } 2>/dev/null 895 local temp_limit=${1:-38000} 896 if [[ -z "${THERMAL_FILE:-}" ]]; then 897 echo "Must define the THERMAL_FILE with the thermal_zoneX/temp file" \ 898 "to read the temperature from. This is normally set in the runner." >&2 899 exit 1 900 fi 901 local org_temp=$(cat "${THERMAL_FILE}") 902 if [[ "${org_temp}" -ge "${temp_limit}" ]]; then 903 echo -n "Waiting for temp to get down from ${org_temp}... " 904 fi 905 local temp="${org_temp}" 906 local secs=0 907 while [[ "${temp}" -ge "${temp_limit}" ]]; do 908 sleep 1 909 temp=$(cat "${THERMAL_FILE}") 910 echo -n "${temp} " 911 secs=$((secs + 1)) 912 if [[ ${secs} -ge 5 ]]; then 913 break 914 fi 915 done 916 if [[ "${org_temp}" -ge "${temp_limit}" ]]; then 917 echo "Done, temp=${temp}" 918 fi 919 set -x 920 } 921 922 # Helper function to set the cpuset restriction of the current process. 923 cmd_cpuset() { 924 [[ "${SKIP_CPUSET:-}" != "1" ]] || return 0 925 local newset="$1" 926 local mycpuset=$(cat /proc/self/cpuset) 927 mycpuset="/dev/cpuset${mycpuset}" 928 # Check that the directory exists: 929 [[ -d "${mycpuset}" ]] 930 if [[ -e "${mycpuset}/cpuset.cpus" ]]; then 931 echo "${newset}" >"${mycpuset}/cpuset.cpus" 932 else 933 echo "${newset}" >"${mycpuset}/cpus" 934 fi 935 } 936 937 # Return the encoding/decoding speed from the Stats output. 938 _speed_from_output() { 939 local speed="$1" 940 local unit="${2:-MP/s}" 941 if [[ "${speed}" == *"${unit}"* ]]; then 942 speed="${speed%% ${unit}*}" 943 speed="${speed##* }" 944 echo "${speed}" 945 fi 946 } 947 948 949 # Run benchmarks on ARM for the big and little CPUs. 950 cmd_arm_benchmark() { 951 # Flags used for cjxl encoder with .png inputs 952 local jxl_png_benchmarks=( 953 # Lossy options: 954 "--epf=0 --distance=1.0 --speed=cheetah" 955 "--epf=2 --distance=1.0 --speed=cheetah" 956 "--epf=0 --distance=8.0 --speed=cheetah" 957 "--epf=1 --distance=8.0 --speed=cheetah" 958 "--epf=2 --distance=8.0 --speed=cheetah" 959 "--epf=3 --distance=8.0 --speed=cheetah" 960 "--modular -Q 90" 961 "--modular -Q 50" 962 # Lossless options: 963 "--modular" 964 "--modular -E 0 -I 0" 965 "--modular -P 5" 966 "--modular --responsive=1" 967 # Near-lossless options: 968 "--epf=0 --distance=0.3 --speed=fast" 969 "--modular -Q 97" 970 ) 971 972 # Flags used for cjxl encoder with .jpg inputs. These should do lossless 973 # JPEG recompression (of pixels or full jpeg). 974 local jxl_jpeg_benchmarks=( 975 "--num_reps=3" 976 ) 977 978 local images=( 979 "testdata/jxl/flower/flower.png" 980 ) 981 982 local jpg_images=( 983 "testdata/jxl/flower/flower.png.im_q85_420.jpg" 984 ) 985 986 if [[ "${SKIP_CPUSET:-}" == "1" ]]; then 987 # Use a single cpu config in this case. 988 local cpu_confs=("?") 989 else 990 # Otherwise the CPU config comes from the environment: 991 local cpu_confs=( 992 "${RUNNER_CPU_LITTLE}" 993 "${RUNNER_CPU_BIG}" 994 # The CPU description is something like 3-7, so these configurations only 995 # take the first CPU of the group. 996 "${RUNNER_CPU_LITTLE%%-*}" 997 "${RUNNER_CPU_BIG%%-*}" 998 ) 999 # Check that RUNNER_CPU_ALL is defined. In the SKIP_CPUSET=1 case this will 1000 # be ignored but still evaluated when calling cmd_cpuset. 1001 [[ -n "${RUNNER_CPU_ALL}" ]] 1002 fi 1003 1004 local jpg_dirname="third_party/corpora/jpeg" 1005 mkdir -p "${jpg_dirname}" 1006 local jpg_qualities=( 50 80 95 ) 1007 for src_img in "${images[@]}"; do 1008 for q in "${jpg_qualities[@]}"; do 1009 local jpeg_name="${jpg_dirname}/"$(basename "${src_img}" .png)"-q${q}.jpg" 1010 convert -sampling-factor 1x1 -quality "${q}" \ 1011 "${src_img}" "${jpeg_name}" 1012 jpg_images+=("${jpeg_name}") 1013 done 1014 done 1015 1016 local output_dir="${BUILD_DIR}/benchmark_results" 1017 mkdir -p "${output_dir}" 1018 local runs_file="${output_dir}/runs.txt" 1019 1020 if [[ ! -e "${runs_file}" ]]; then 1021 echo -e "binary\tflags\tsrc_img\tsrc size\tsrc pixels\tcpuset\tenc size (B)\tenc speed (MP/s)\tdec speed (MP/s)\tJPG dec speed (MP/s)\tJPG dec speed (MB/s)" | 1022 tee -a "${runs_file}" 1023 fi 1024 1025 mkdir -p "${BUILD_DIR}/arm_benchmark" 1026 local flags 1027 local src_img 1028 for src_img in "${jpg_images[@]}" "${images[@]}"; do 1029 local src_img_hash=$(sha1sum "${src_img}" | cut -f 1 -d ' ') 1030 local enc_binaries=("${BUILD_DIR}/tools/cjxl") 1031 local src_ext="${src_img##*.}" 1032 for enc_binary in "${enc_binaries[@]}"; do 1033 local enc_binary_base=$(basename "${enc_binary}") 1034 1035 # Select the list of flags to use for the current encoder/image pair. 1036 local img_benchmarks 1037 if [[ "${src_ext}" == "jpg" ]]; then 1038 img_benchmarks=("${jxl_jpeg_benchmarks[@]}") 1039 else 1040 img_benchmarks=("${jxl_png_benchmarks[@]}") 1041 fi 1042 1043 for flags in "${img_benchmarks[@]}"; do 1044 # Encoding step. 1045 local enc_file_hash="${enc_binary_base} || $flags || ${src_img} || ${src_img_hash}" 1046 enc_file_hash=$(echo "${enc_file_hash}" | sha1sum | cut -f 1 -d ' ') 1047 local enc_file="${BUILD_DIR}/arm_benchmark/${enc_file_hash}.jxl" 1048 1049 for cpu_conf in "${cpu_confs[@]}"; do 1050 cmd_cpuset "${cpu_conf}" 1051 # nproc returns the number of active CPUs, which is given by the cpuset 1052 # mask. 1053 local num_threads="$(nproc)" 1054 1055 echo "Encoding with: ${enc_binary_base} img=${src_img} cpus=${cpu_conf} enc_flags=${flags}" 1056 local enc_output 1057 if [[ "${flags}" == *"modular"* ]]; then 1058 # We don't benchmark encoding speed in this case. 1059 if [[ ! -f "${enc_file}" ]]; then 1060 cmd_cpuset "${RUNNER_CPU_ALL:-}" 1061 "${enc_binary}" ${flags} "${src_img}" "${enc_file}.tmp" 1062 mv "${enc_file}.tmp" "${enc_file}" 1063 cmd_cpuset "${cpu_conf}" 1064 fi 1065 enc_output=" ?? MP/s" 1066 else 1067 wait_for_temp 1068 enc_output=$("${enc_binary}" ${flags} "${src_img}" "${enc_file}.tmp" \ 1069 2>&1 | tee /dev/stderr | grep -F "MP/s [") 1070 mv "${enc_file}.tmp" "${enc_file}" 1071 fi 1072 local enc_speed=$(_speed_from_output "${enc_output}") 1073 local enc_size=$(stat -c "%s" "${enc_file}") 1074 1075 echo "Decoding with: img=${src_img} cpus=${cpu_conf} enc_flags=${flags}" 1076 1077 local dec_output 1078 wait_for_temp 1079 dec_output=$("${BUILD_DIR}/tools/djxl" "${enc_file}" \ 1080 --num_reps=5 --num_threads="${num_threads}" 2>&1 | tee /dev/stderr | 1081 grep -E "M[BP]/s \[") 1082 local img_size=$(echo "${dec_output}" | cut -f 1 -d ',') 1083 local img_size_x=$(echo "${img_size}" | cut -f 1 -d ' ') 1084 local img_size_y=$(echo "${img_size}" | cut -f 3 -d ' ') 1085 local img_size_px=$(( ${img_size_x} * ${img_size_y} )) 1086 local dec_speed=$(_speed_from_output "${dec_output}") 1087 1088 # For JPEG lossless recompression modes (where the original is a JPEG) 1089 # decode to JPG as well. 1090 local jpeg_dec_mps_speed="" 1091 local jpeg_dec_mbs_speed="" 1092 if [[ "${src_ext}" == "jpg" ]]; then 1093 wait_for_temp 1094 local dec_file="${BUILD_DIR}/arm_benchmark/${enc_file_hash}.jpg" 1095 dec_output=$("${BUILD_DIR}/tools/djxl" "${enc_file}" \ 1096 "${dec_file}" --num_reps=5 --num_threads="${num_threads}" 2>&1 | \ 1097 tee /dev/stderr | grep -E "M[BP]/s \[") 1098 local jpeg_dec_mps_speed=$(_speed_from_output "${dec_output}") 1099 local jpeg_dec_mbs_speed=$(_speed_from_output "${dec_output}" MB/s) 1100 if ! cmp --quiet "${src_img}" "${dec_file}"; then 1101 # Add a start at the end to signal that the files are different. 1102 jpeg_dec_mbs_speed+="*" 1103 fi 1104 fi 1105 1106 # Record entry in a tab-separated file. 1107 local src_img_base=$(basename "${src_img}") 1108 echo -e "${enc_binary_base}\t${flags}\t${src_img_base}\t${img_size}\t${img_size_px}\t${cpu_conf}\t${enc_size}\t${enc_speed}\t${dec_speed}\t${jpeg_dec_mps_speed}\t${jpeg_dec_mbs_speed}" | 1109 tee -a "${runs_file}" 1110 done 1111 done 1112 done 1113 done 1114 cmd_cpuset "${RUNNER_CPU_ALL:-}" 1115 cat "${runs_file}" 1116 1117 } 1118 1119 # Generate a corpus and run the fuzzer on that corpus. 1120 cmd_fuzz() { 1121 local corpus_dir=$(realpath "${BUILD_DIR}/fuzzer_corpus") 1122 local fuzzer_crash_dir=$(realpath "${BUILD_DIR}/fuzzer_crash") 1123 mkdir -p "${corpus_dir}" "${fuzzer_crash_dir}" 1124 # Generate step. 1125 "${BUILD_DIR}/tools/fuzzer_corpus" "${corpus_dir}" 1126 # Run step: 1127 local nprocs=$(nproc --all || echo 1) 1128 ( 1129 cd "${BUILD_DIR}" 1130 "tools/djxl_fuzzer" "${fuzzer_crash_dir}" "${corpus_dir}" \ 1131 -max_total_time="${FUZZER_MAX_TIME}" -jobs=${nprocs} \ 1132 -artifact_prefix="${fuzzer_crash_dir}/" 1133 ) 1134 } 1135 1136 # Runs the linters (clang-format, build_cleaner, buildirier) on the pending CLs. 1137 cmd_lint() { 1138 merge_request_commits 1139 { set +x; } 2>/dev/null 1140 local versions=(${1:-16 15 14 13 12 11 10 9 8 7 6.0}) 1141 local clang_format_bins=("${versions[@]/#/clang-format-}" clang-format) 1142 local tmpdir=$(mktemp -d) 1143 CLEANUP_FILES+=("${tmpdir}") 1144 1145 local ret=0 1146 local build_patch="${tmpdir}/build_cleaner.patch" 1147 if ! "${MYDIR}/tools/scripts/build_cleaner.py" >"${build_patch}"; then 1148 ret=1 1149 echo "build_cleaner.py findings:" >&2 1150 "${COLORDIFF_BIN}" <"${build_patch}" 1151 echo "Run \`tools/scripts/build_cleaner.py --update\` to apply them" >&2 1152 fi 1153 1154 # It is ok, if buildifier is not installed. 1155 if which buildifier >/dev/null; then 1156 local buildifier_patch="${tmpdir}/buildifier.patch" 1157 local bazel_files=`git -C ${MYDIR} ls-files | grep -E "/BUILD$|WORKSPACE|.bzl$"` 1158 set -x 1159 buildifier -d ${bazel_files} >"${buildifier_patch}"|| true 1160 { set +x; } 2>/dev/null 1161 if [ -s "${buildifier_patch}" ]; then 1162 ret=1 1163 echo 'buildifier have found some problems in Bazel build files:' >&2 1164 "${COLORDIFF_BIN}" <"${buildifier_patch}" 1165 echo 'To fix them run (from the base directory):' >&2 1166 echo ' buildifier `git ls-files | grep -E "/BUILD$|WORKSPACE|.bzl$"`' >&2 1167 fi 1168 fi 1169 1170 local installed=() 1171 local clang_patch 1172 local clang_format 1173 for clang_format in "${clang_format_bins[@]}"; do 1174 if ! which "${clang_format}" >/dev/null; then 1175 continue 1176 fi 1177 installed+=("${clang_format}") 1178 local tmppatch="${tmpdir}/${clang_format}.patch" 1179 # We include in this linter all the changes including the uncommitted changes 1180 # to avoid printing changes already applied. 1181 set -x 1182 # Ignoring the error that git-clang-format outputs. 1183 git -C "${MYDIR}" "${clang_format}" --binary "${clang_format}" \ 1184 --style=file --diff "${MR_ANCESTOR_SHA}" -- >"${tmppatch}" || true 1185 { set +x; } 2>/dev/null 1186 if grep -E '^--- ' "${tmppatch}" | grep -v 'a/third_party' >/dev/null; then 1187 if [[ -n "${LINT_OUTPUT:-}" ]]; then 1188 cp "${tmppatch}" "${LINT_OUTPUT}" 1189 fi 1190 clang_patch="${tmppatch}" 1191 else 1192 echo "clang-format check OK" >&2 1193 return ${ret} 1194 fi 1195 done 1196 1197 if [[ ${#installed[@]} -eq 0 ]]; then 1198 echo "You must install clang-format for \"git clang-format\"" >&2 1199 exit 1 1200 fi 1201 1202 # clang-format is installed but found problems. 1203 echo "clang-format findings:" >&2 1204 "${COLORDIFF_BIN}" < "${clang_patch}" 1205 1206 echo "clang-format found issues in your patches from ${MR_ANCESTOR_SHA}" \ 1207 "to the current patch. Run \`./ci.sh lint | patch -p1\` from the base" \ 1208 "directory to apply them." >&2 1209 exit 1 1210 } 1211 1212 # Runs clang-tidy on the pending CLs. If the "all" argument is passed it runs 1213 # clang-tidy over all the source files instead. 1214 cmd_tidy() { 1215 local what="${1:-}" 1216 1217 if [[ -z "${CLANG_TIDY_BIN}" ]]; then 1218 echo "ERROR: You must install clang-tidy-7 or newer to use ci.sh tidy" >&2 1219 exit 1 1220 fi 1221 1222 local git_args=() 1223 if [[ "${what}" == "all" ]]; then 1224 git_args=(ls-files) 1225 shift 1226 else 1227 merge_request_commits 1228 git_args=( 1229 diff-tree --no-commit-id --name-only -r "${MR_ANCESTOR_SHA}" 1230 "${MR_HEAD_SHA}" 1231 ) 1232 fi 1233 1234 # Clang-tidy needs the compilation database generated by cmake. 1235 if [[ ! -e "${BUILD_DIR}/compile_commands.json" ]]; then 1236 # Generate the build options in debug mode, since we need the debug asserts 1237 # enabled for the clang-tidy analyzer to use them. 1238 CMAKE_BUILD_TYPE="Debug" 1239 cmake_configure 1240 # Build the autogen targets to generate the .h files from the .ui files. 1241 local autogen_targets=( 1242 $(ninja -C "${BUILD_DIR}" -t targets | grep -F _autogen: | 1243 cut -f 1 -d :) 1244 ) 1245 if [[ ${#autogen_targets[@]} != 0 ]]; then 1246 ninja -C "${BUILD_DIR}" "${autogen_targets[@]}" 1247 fi 1248 fi 1249 1250 cd "${MYDIR}" 1251 local nprocs=$(nproc --all || echo 1) 1252 local ret=0 1253 if ! parallel -j"${nprocs}" --keep-order -- \ 1254 "${CLANG_TIDY_BIN}" -p "${BUILD_DIR}" -format-style=file -quiet "$@" {} \ 1255 < <(git "${git_args[@]}" | grep -E '(\.cc|\.cpp)$') \ 1256 >"${BUILD_DIR}/clang-tidy.txt"; then 1257 ret=1 1258 fi 1259 { set +x; } 2>/dev/null 1260 echo "Findings statistics:" >&2 1261 grep -E ' \[[A-Za-z\.,\-]+\]' -o "${BUILD_DIR}/clang-tidy.txt" | sort \ 1262 | uniq -c >&2 1263 1264 if [[ $ret -ne 0 ]]; then 1265 cat >&2 <<EOF 1266 Errors found, see ${BUILD_DIR}/clang-tidy.txt for details. 1267 To automatically fix them, run: 1268 1269 SKIP_TEST=1 ./ci.sh debug 1270 ${CLANG_TIDY_BIN} -p ${BUILD_DIR} -fix -format-style=file -quiet $@ \$(git ${git_args[@]} | grep -E '(\.cc|\.cpp)\$') 1271 EOF 1272 fi 1273 1274 return ${ret} 1275 } 1276 1277 # Print stats about all the packages built in ${BUILD_DIR}/debs/. 1278 cmd_debian_stats() { 1279 { set +x; } 2>/dev/null 1280 local debsdir="${BUILD_DIR}/debs" 1281 local f 1282 while IFS='' read -r -d '' f; do 1283 echo "=====================================================================" 1284 echo "Package $f:" 1285 dpkg --info $f 1286 dpkg --contents $f 1287 done < <(find "${BUILD_DIR}/debs" -maxdepth 1 -mindepth 1 -type f \ 1288 -name '*.deb' -print0) 1289 } 1290 1291 build_debian_pkg() { 1292 local srcdir="$1" 1293 local srcpkg="$2" 1294 local options="${3:-}" 1295 1296 local debsdir="${BUILD_DIR}/debs" 1297 local builddir="${debsdir}/${srcpkg}" 1298 1299 # debuild doesn't have an easy way to build out of tree, so we make a copy 1300 # of with all symlinks on the first level. 1301 mkdir -p "${builddir}" 1302 for f in $(find "${srcdir}" -mindepth 1 -maxdepth 1 -printf '%P\n'); do 1303 if [[ ! -L "${builddir}/$f" ]]; then 1304 rm -f "${builddir}/$f" 1305 ln -s "${srcdir}/$f" "${builddir}/$f" 1306 fi 1307 done 1308 ( 1309 cd "${builddir}" 1310 debuild "${options}" -b -uc -us 1311 ) 1312 } 1313 1314 cmd_debian_build() { 1315 local srcpkg="${1:-}" 1316 1317 case "${srcpkg}" in 1318 jpeg-xl) 1319 build_debian_pkg "${MYDIR}" "jpeg-xl" 1320 ;; 1321 highway) 1322 build_debian_pkg "${MYDIR}/third_party/highway" "highway" "${HWY_PKG_OPTIONS}" 1323 ;; 1324 *) 1325 echo "ERROR: Must pass a valid source package name to build." >&2 1326 ;; 1327 esac 1328 } 1329 1330 get_version() { 1331 local varname=$1 1332 local line=$(grep -F "set(${varname} " lib/CMakeLists.txt | head -n 1) 1333 [[ -n "${line}" ]] 1334 line="${line#set(${varname} }" 1335 line="${line%)}" 1336 echo "${line}" 1337 } 1338 1339 cmd_bump_version() { 1340 local newver="${1:-}" 1341 1342 if ! which dch >/dev/null; then 1343 echo "Missing dch\nTo install it run:\n sudo apt install devscripts" 1344 exit 1 1345 fi 1346 1347 if [[ -z "${newver}" ]]; then 1348 local major=$(get_version JPEGXL_MAJOR_VERSION) 1349 local minor=$(get_version JPEGXL_MINOR_VERSION) 1350 local patch=0 1351 minor=$(( ${minor} + 1)) 1352 else 1353 local major="${newver%%.*}" 1354 newver="${newver#*.}" 1355 local minor="${newver%%.*}" 1356 newver="${newver#${minor}}" 1357 local patch="${newver#.}" 1358 if [[ -z "${patch}" ]]; then 1359 patch=0 1360 fi 1361 fi 1362 1363 newver="${major}.${minor}.${patch}" 1364 1365 echo "Bumping version to ${newver} (${major}.${minor}.${patch})" 1366 sed -E \ 1367 -e "s/(set\\(JPEGXL_MAJOR_VERSION) [0-9]+\\)/\\1 ${major})/" \ 1368 -e "s/(set\\(JPEGXL_MINOR_VERSION) [0-9]+\\)/\\1 ${minor})/" \ 1369 -e "s/(set\\(JPEGXL_PATCH_VERSION) [0-9]+\\)/\\1 ${patch})/" \ 1370 -i lib/CMakeLists.txt 1371 sed -E \ 1372 -e "s/(LIBJXL_VERSION: )\"[0-9.]+\"/\\1\"${major}.${minor}.${patch}\"/" \ 1373 -e "s/(LIBJXL_ABI_VERSION: )\"[0-9.]+\"/\\1\"${major}.${minor}\"/" \ 1374 -i .github/workflows/conformance.yml 1375 1376 # Update lib.gni 1377 tools/scripts/build_cleaner.py --update 1378 1379 # Mark the previous version as "unstable". 1380 DEBCHANGE_RELEASE_HEURISTIC=log dch -M --distribution unstable --release '' 1381 DEBCHANGE_RELEASE_HEURISTIC=log dch -M \ 1382 --newversion "${newver}" \ 1383 "Bump JPEG XL version to ${newver}." 1384 } 1385 1386 # Check that the AUTHORS file contains the email of the committer. 1387 cmd_authors() { 1388 merge_request_commits 1389 local emails 1390 local names 1391 readarray -t emails < <(git log --format='%ae' "${MR_ANCESTOR_SHA}..${MR_HEAD_SHA}") 1392 readarray -t names < <(git log --format='%an' "${MR_ANCESTOR_SHA}..${MR_HEAD_SHA}") 1393 for i in "${!names[@]}"; do 1394 echo "Checking name '${names[$i]}' with email '${emails[$i]}' ..." 1395 "${MYDIR}"/tools/scripts/check_author.py "${emails[$i]}" "${names[$i]}" 1396 done 1397 } 1398 1399 main() { 1400 local cmd="${1:-}" 1401 if [[ -z "${cmd}" ]]; then 1402 cat >&2 <<EOF 1403 Use: $0 CMD 1404 1405 Where cmd is one of: 1406 opt Build and test a Release with symbols build. 1407 debug Build and test a Debug build (NDEBUG is not defined). 1408 release Build and test a striped Release binary without debug information. 1409 asan Build and test an ASan (AddressSanitizer) build. 1410 msan Build and test an MSan (MemorySanitizer) build. Needs to have msan 1411 c++ libs installed with msan_install first. 1412 tsan Build and test a TSan (ThreadSanitizer) build. 1413 asanfuzz Build and test an ASan (AddressSanitizer) build for fuzzing. 1414 msanfuzz Build and test an MSan (MemorySanitizer) build for fuzzing. 1415 test Run the tests build by opt, debug, release, asan or msan. Useful when 1416 building with SKIP_TEST=1. 1417 gbench Run the Google benchmark tests. 1418 fuzz Generate the fuzzer corpus and run the fuzzer on it. Useful after 1419 building with asan or msan. 1420 benchmark Run the benchmark over the default corpus. 1421 fast_benchmark Run the benchmark over the small corpus. 1422 1423 coverage Build and run tests with coverage support. Runs coverage_report as 1424 well. 1425 coverage_report Generate HTML, XML and text coverage report after a coverage 1426 run. 1427 1428 lint Run the linter checks on the current commit or merge request. 1429 tidy Run clang-tidy on the current commit or merge request. 1430 authors Check that the last commit's author is listed in the AUTHORS file. 1431 1432 msan_install Install the libc++ libraries required to build in msan mode. This 1433 needs to be done once. 1434 1435 debian_build <srcpkg> Build the given source package. 1436 debian_stats Print stats about the built packages. 1437 1438 oss-fuzz commands: 1439 ossfuzz_asan Build the local source inside oss-fuzz docker with asan. 1440 ossfuzz_msan Build the local source inside oss-fuzz docker with msan. 1441 ossfuzz_ubsan Build the local source inside oss-fuzz docker with ubsan. 1442 ossfuzz_ninja Run ninja on the BUILD_DIR inside the oss-fuzz docker. Extra 1443 parameters are passed to ninja, for example "djxl_fuzzer" will 1444 only build that ninja target. Use for faster build iteration 1445 after one of the ossfuzz_*san commands. 1446 1447 You can pass some optional environment variables as well: 1448 - BUILD_DIR: The output build directory (by default "$$repo/build") 1449 - BUILD_TARGET: The target triplet used when cross-compiling. 1450 - CMAKE_FLAGS: Convenience flag to pass both CMAKE_C_FLAGS and CMAKE_CXX_FLAGS. 1451 - CMAKE_PREFIX_PATH: Installation prefixes to be searched by the find_package. 1452 - ENABLE_WASM_SIMD=1: enable experimental SIMD in WASM build (only). 1453 - FUZZER_MAX_TIME: "fuzz" command fuzzer running timeout in seconds. 1454 - LINT_OUTPUT: Path to the output patch from the "lint" command. 1455 - SKIP_CPUSET=1: Skip modifying the cpuset in the arm_benchmark. 1456 - SKIP_BUILD=1: Skip the build stage, cmake configure only. 1457 - SKIP_TEST=1: Skip the test stage. 1458 - STORE_IMAGES=0: Makes the benchmark discard the computed images. 1459 - TEST_STACK_LIMIT: Stack size limit (ulimit -s) during tests, in KiB. 1460 - TEST_SELECTOR: pass additional arguments to ctest, e.g. "-R .Resample.". 1461 - STACK_SIZE=1: Generate binaries with the .stack_sizes sections. 1462 1463 These optional environment variables are forwarded to the cmake call as 1464 parameters: 1465 - CMAKE_BUILD_TYPE 1466 - CMAKE_C_FLAGS 1467 - CMAKE_CXX_FLAGS 1468 - CMAKE_C_COMPILER_LAUNCHER 1469 - CMAKE_CXX_COMPILER_LAUNCHER 1470 - CMAKE_CROSSCOMPILING_EMULATOR 1471 - CMAKE_FIND_ROOT_PATH 1472 - CMAKE_EXE_LINKER_FLAGS 1473 - CMAKE_MAKE_PROGRAM 1474 - CMAKE_MODULE_LINKER_FLAGS 1475 - CMAKE_SHARED_LINKER_FLAGS 1476 - CMAKE_TOOLCHAIN_FILE 1477 1478 Example: 1479 BUILD_DIR=/tmp/build $0 opt 1480 EOF 1481 exit 1 1482 fi 1483 1484 cmd="cmd_${cmd}" 1485 shift 1486 set -x 1487 "${cmd}" "$@" 1488 } 1489 1490 main "$@"