build_test.yml (12756B)
1 # Copyright (c) the JPEG XL Project Authors. All rights reserved. 2 # 3 # Use of this source code is governed by a BSD-style 4 # license that can be found in the LICENSE file. 5 6 # Workflow for building and running tests. 7 8 name: Build/Test *nix 9 on: 10 merge_group: 11 push: 12 branches: 13 - main 14 - v*.*.x 15 pull_request: 16 types: [opened, reopened, labeled, synchronize] 17 18 permissions: 19 contents: read 20 21 concurrency: 22 group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} 23 cancel-in-progress: ${{ github.event_name == 'pull_request' }} 24 25 jobs: 26 build_test: 27 name: ${{ startsWith(matrix.os, 'macos-') && 'MacOS' || 'Ubuntu' }} Build ${{ matrix.name }} 28 runs-on: ${{ matrix.os || 'ubuntu-latest' }} 29 strategy: 30 fail-fast: false 31 matrix: 32 # We have one job per "name" in the matrix. Attributes are set on the 33 # specific job names. 34 name: [release, debug, asan, msan, scalar] 35 include: 36 - name: release 37 mode: release 38 run_bench: true 39 test_in_pr: true 40 cmake_args: >- 41 -DJPEGXL_TEST_TOOLS=ON 42 -DJPEGLI_LIBJPEG_LIBRARY_VERSION="8.2.2" 43 -DJPEGLI_LIBJPEG_LIBRARY_SOVERSION="8" 44 # Track static stack size on build and check it doesn't exceed 3 kB. 45 env_stack_size: 1 46 max_stack: 2400 47 # Conformance tooling test requires numpy. 48 apt_pkgs: doxygen graphviz python3-numpy 49 - name: lowprecision 50 mode: release 51 run_bench: true 52 test_in_pr: true 53 cmake_args: -DCMAKE_CXX_FLAGS=-DJXL_HIGH_PRECISION=0 54 - name: debug 55 # Runs on AVX3 CPUs require more stack than others. Make sure to 56 # test on AVX3-enabled CPUs when changing this value. 57 env_test_stack_size: 4000 58 # Build scalar-only hwy instructions. 59 - name: scalar 60 mode: release 61 cxxflags: -DHWY_COMPILE_ONLY_SCALAR -DFJXL_ENABLE_AVX2=0 -DFJXL_ENABLE_AVX512=0 62 # Disabling optional features to speed up MSAN build a little bit. 63 - name: msan 64 skip_install: true 65 cmake_args: >- 66 -DJPEGXL_ENABLE_DEVTOOLS=OFF -DJPEGXL_ENABLE_PLUGINS=OFF 67 -DJPEGXL_ENABLE_VIEWERS=OFF 68 apt_pkgs: clang-15 69 cc: clang-15 70 cxx: clang++-15 71 - name: asan 72 skip_install: true 73 - name: coverage 74 env_test_stack_size: 2048 75 skip_install: true 76 # Build with support for decoding to JPEG bytes disabled. Produces a 77 # smaller build if only decoding to pixels is needed. 78 - name: release-nojpeg 79 mode: release 80 cxxflags: -DJXL_DEBUG_ON_ABORT=0 81 cmake_args: >- 82 -DJPEGXL_ENABLE_TRANSCODE_JPEG=OFF 83 -DJPEGXL_ENABLE_PLUGINS=OFF 84 -DJPEGXL_ENABLE_VIEWERS=OFF 85 # Build with jxl_cms based on lcms2 library. 86 - name: release-lcms2 87 mode: release 88 cmake_args: >- 89 -DJPEGXL_ENABLE_SKCMS=OFF 90 - name: release-system-lcms2 91 mode: release 92 cmake_args: >- 93 -DJPEGXL_ENABLE_SKCMS=OFF 94 -DJPEGXL_FORCE_SYSTEM_LCMS2=ON 95 apt_pkgs: liblcms2-dev 96 # static build is impossible 97 skip_install: true 98 # Build optimized for binary size, all features not needed for 99 # reconstructing pixels is disabled. 100 - name: release:minimal 101 mode: release 102 cxxflags: -DJXL_DEBUG_ON_ABORT=0 103 cmake_args: >- 104 -DJPEGXL_ENABLE_TRANSCODE_JPEG=OFF 105 -DJPEGXL_ENABLE_BOXES=OFF 106 -DJPEGXL_ENABLE_PLUGINS=OFF 107 -DJPEGXL_ENABLE_VIEWERS=OFF 108 # Builds with gcc in release mode 109 - name: release:gcc8 110 os: ubuntu-20.04 111 mode: release 112 apt_pkgs: gcc-8 g++-8 113 cmake_args: >- 114 -DCMAKE_C_COMPILER=gcc-8 -DCMAKE_CXX_COMPILER=g++-8 115 # Builds with clang-7 in release mode 116 - name: release:clang-7 117 os: ubuntu-20.04 118 mode: release 119 skip_install: true 120 apt_pkgs: clang-7 121 cc: clang-7 122 cxx: clang++-7 123 - name: release:osx 124 os: macos-latest 125 mode: release 126 skip_install: true 127 cmake_args: >- 128 -DCMAKE_FIND_FRAMEWORK=NEVER 129 130 env: 131 CCACHE_DIR: ${{ github.workspace }}/.ccache 132 # Whether we track the stack size. 133 STACK_SIZE: ${{ matrix.env_stack_size }} 134 TEST_STACK_LIMIT: ${{ matrix.env_test_stack_size }} 135 WILL_TEST: ${{ github.event_name == 'push' || (github.event_name == 'pull_request' && matrix.name != 'coverage' && (matrix.test_in_pr || contains(github.event.pull_request.labels.*.name, 'CI:full'))) }} 136 WILL_BUILD: ${{ github.event_name == 'push' || (github.event_name == 'pull_request' && matrix.name != 'coverage') }} 137 WILL_BENCH: ${{ github.event_name != 'merge_group' && matrix.run_bench }} 138 WILL_DOC: ${{ github.event_name != 'merge_group' && matrix.name == 'release' }} 139 WILL_COV: ${{ github.event_name == 'push' && matrix.name == 'coverage' }} 140 JPEGXL_OPT_DBG: true 141 FASTER_MSAN_BUILD: 1 142 143 steps: 144 - name: Harden Runner 145 uses: step-security/harden-runner@63c24ba6bd7ba022e95695ff85de572c04a18142 # v2.7.0 146 with: 147 egress-policy: audit 148 149 - name: Install build deps Ubuntu 150 if: startsWith(matrix.os, 'macos-') == false 151 run: | 152 sudo rm -f /var/lib/man-db/auto-update 153 sudo apt update 154 sudo apt install -y \ 155 ccache \ 156 clang \ 157 cmake \ 158 graphviz \ 159 imagemagick \ 160 libbenchmark-dev \ 161 libbenchmark-tools \ 162 libbrotli-dev \ 163 libgdk-pixbuf2.0-dev \ 164 libgif-dev \ 165 libgtest-dev \ 166 libgtk2.0-dev \ 167 libjpeg-dev \ 168 libjpeg-turbo-progs \ 169 libopenexr-dev \ 170 libpng-dev \ 171 libwebp-dev \ 172 ninja-build \ 173 pkg-config \ 174 xvfb \ 175 ${{ matrix.apt_pkgs }} \ 176 # 177 echo "CC=${{ matrix.cc || 'clang' }}" >> $GITHUB_ENV 178 echo "CXX=${{ matrix.cxx || 'clang++' }}" >> $GITHUB_ENV 179 - name: Install build deps MacOS 180 if: startsWith(matrix.os, 'macos-') 181 run: | 182 # Should be already installed: 183 # brew install brotli giflib jpeg-turbo libpng zlib 184 # Not required, since we skip building documentation 185 # brew install doxygen 186 brew install binutils ccache coreutils google-benchmark googletest ninja sdl2 187 188 - name: Checkout the source 189 uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 190 with: 191 submodules: true 192 fetch-depth: 2 193 194 - name: Setup the Homebrew prefixes 195 if: startsWith(matrix.os, 'macos-') 196 run: | 197 CMAKE_PREFIX_PATH=`brew --prefix brotli`:`brew --prefix giflib`:`brew --prefix google-benchmark`:`brew --prefix jpeg-turbo`:`brew --prefix libpng`:`brew --prefix sdl2`:`brew --prefix zlib` 198 echo "CMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}" >> $GITHUB_ENV 199 200 - name: Suppress doxygen target 201 if: matrix.name != 'release' 202 run: | 203 echo "TARGETS=all" >> $GITHUB_ENV 204 205 - name: Setup the LLVM source path 206 if: matrix.name == 'msan' && env.WILL_BUILD == 'true' 207 run: | 208 LLVM_ROOT=${GITHUB_WORKSPACE}/llvm_root 209 mkdir -p ${LLVM_ROOT} 210 echo "LLVM_ROOT=${LLVM_ROOT}" >> $GITHUB_ENV 211 212 - name: Cache LLVM sources 213 if: matrix.name == 'msan' && env.WILL_BUILD == 'true' 214 uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 215 with: 216 path: ${{ env.LLVM_ROOT }} 217 key: llvm 218 219 - name: Checkout the LLVM source 220 if: matrix.name == 'msan' && env.WILL_BUILD == 'true' 221 uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 222 with: 223 submodules: false 224 repository: llvm/llvm-project 225 ref: llvmorg-14.0.0 # NB: 15.0.0 does not build ¯\_(ツ)_/¯ 226 path: llvm_root 227 228 - name: Sphinx dependencies 229 # Dependencies for sphinx HTML documentation 230 if: env.WILL_DOC == 'true' 231 run: | 232 pip3 install -r doc/sphinx/requirements.txt 233 234 - name: Install gcovr 235 if: env.WILL_COV == 'true' 236 run: pip install gcovr 237 238 - name: Git environment 239 id: git-env 240 run: | 241 echo "parent=$(git rev-parse ${{ github.sha }}^)" >> $GITHUB_OUTPUT 242 shell: bash 243 244 - name: ccache 245 uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 246 with: 247 path: ${{ env.CCACHE_DIR }} 248 # When the cache hits the key it is not updated, so if this is a rebuild 249 # of the same Pull Request it will reuse the cache if still around. For 250 # either Pull Requests or new pushes to main, this will use the parent 251 # hash as the starting point from the restore-keys entry. 252 key: build-${{ runner.os }}-${{ github.sha }}-${{ matrix.name }} 253 restore-keys: | 254 build-${{ runner.os }}-${{ steps.git-env.outputs.parent }}-${{ matrix.name }} 255 256 - name: Build 257 if: env.WILL_BUILD == 'true' 258 run: | 259 mkdir -p ${CCACHE_DIR} 260 echo "max_size = 200M" > ${CCACHE_DIR}/ccache.conf 261 mode="${{ matrix.mode }}" 262 build_tests=$([ "$WILL_TEST" == "true" ] && echo "ON" || echo "OFF") 263 [[ -n "${mode}" ]] || mode="${{ matrix.name }}" 264 ./ci.sh ${mode} -DJPEGXL_FORCE_SYSTEM_BROTLI=ON \ 265 -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ 266 -DCMAKE_C_COMPILER_LAUNCHER=ccache \ 267 -DBUILD_TESTING=${build_tests} \ 268 ${{ matrix.cmake_args }} 269 env: 270 SKIP_TEST: 1 271 CMAKE_CXX_FLAGS: ${{ matrix.cxxflags }} 272 273 - name: Build stats 274 if: env.WILL_BUILD == 'true' 275 run: | 276 awk '!/^#/ {total[$4]+=($2-$1);cntr[$4]+=1} END {for (key in total) print total[key]/cntr[key] " " key}' build/.ninja_log | sort -n | tail -n 25 277 278 - name: ccache stats 279 run: ccache --show-stats 280 281 - name: Build stats ${{ matrix.name }} 282 if: env.WILL_BUILD == 'true' && matrix.mode == 'release' 283 run: | 284 SHARED_LIB_EXT="${{ startsWith(matrix.os, 'macos-') && 'dylib' || 'so' }}" 285 SELECT_BINUTILS="${{ startsWith(matrix.os, 'macos-') && '--binutils `brew --prefix binutils`/bin/' || '' }}" 286 tools/scripts/build_stats.py --save build/stats.json \ 287 --max-stack ${{ matrix.max_stack || '0' }} ${SELECT_BINUTILS} \ 288 cjxl djxl libjxl.${SHARED_LIB_EXT} libjxl_dec.${SHARED_LIB_EXT} 289 290 # Check that we can build the example project against the installed libs. 291 - name: Install and build examples 292 if: env.WILL_BUILD == 'true' && matrix.mode == 'release' && !matrix.skip_install 293 run: | 294 set -x 295 sudo cmake --build build -- install 296 cmake -Bbuild-example -Hexamples -G Ninja 297 cmake --build build-example 298 # Test that the built binaries run. 299 echo -e -n "PF\n1 1\n-1.0\n\0\0\x80\x3f\0\0\x80\x3f\0\0\x80\x3f" > test.pfm 300 build-example/encode_oneshot test.pfm test.jxl 301 build-example/decode_oneshot test.jxl dec.pfm dec.icc 302 303 # Run the tests on push and when requested in pull_request. 304 - name: Test ${{ matrix.mode }} 305 if: env.WILL_TEST == 'true' 306 run: | 307 ./ci.sh test ${{ matrix.ctest_args }} 308 309 # Print the running time summary for the slowest tests. 310 - name: Test runtime stats 311 if: env.WILL_TEST == 'true' 312 run: | 313 sort build/Testing/Temporary/CTestCostData.txt -k 3 -n | tail -n 20 || true 314 315 - name: Build HTML documentation (sphinx/readthetdocs) 316 if: env.WILL_DOC == 'true' 317 run: | 318 cmake --build build -- rtd-html 319 320 - name: Coverage report 321 if: env.WILL_COV == 'true' 322 run: | 323 ./ci.sh coverage_report 324 325 - name: Coverage upload to Codecov 326 if: env.WILL_COV == 'true' 327 uses: codecov/codecov-action@e0b68c6749509c5f83f984dd99a76a1c1a231044 # v4.0.1 328 with: 329 flags: unittests 330 files: build/coverage.xml 331 332 - name: Fast benchmark ${{ matrix.mode }} 333 if: env.WILL_BENCH == 'true' 334 run: | 335 STORE_IMAGES=0 ./ci.sh fast_benchmark 336 337 # Run gbench once, just to make sure it runs, not for actual benchmarking. 338 # This doesn't work on MSAN because we use gbench library from the system 339 # which is not instrumented by MSAN. 340 - name: gbench check 341 if: env.WILL_BENCH == 'true' 342 run: | 343 ./ci.sh gbench --benchmark_min_time=0