run-coverity-scan (14490B)
1 #!/bin/sh -e 2 3 # Upload a created tarball to Coverity Scan, as per 4 # https://scan.coverity.com/projects/qemu/builds/new 5 6 # This work is licensed under the terms of the GNU GPL version 2, 7 # or (at your option) any later version. 8 # See the COPYING file in the top-level directory. 9 # 10 # Copyright (c) 2017-2020 Linaro Limited 11 # Written by Peter Maydell 12 13 # Note that this script will automatically download and 14 # run the (closed-source) coverity build tools, so don't 15 # use it if you don't trust them! 16 17 # This script assumes that you're running it from a QEMU source 18 # tree, and that tree is a fresh clean one, because we do an in-tree 19 # build. (This is necessary so that the filenames that the Coverity 20 # Scan server sees are relative paths that match up with the component 21 # regular expressions it uses; an out-of-tree build won't work for this.) 22 # The host machine should have as many of QEMU's dependencies 23 # installed as possible, for maximum coverity coverage. 24 25 # To do an upload you need to be a maintainer in the Coverity online 26 # service, and you will need to know the "Coverity token", which is a 27 # secret 8 digit hex string. You can find that from the web UI in the 28 # project settings, if you have maintainer access there. 29 30 # Command line options: 31 # --dry-run : run the tools, but don't actually do the upload 32 # --docker : create and work inside a container 33 # --docker-engine : specify the container engine to use (docker/podman/auto); 34 # implies --docker 35 # --update-tools-only : update the cached copy of the tools, but don't run them 36 # --no-update-tools : do not update the cached copy of the tools 37 # --tokenfile : file to read Coverity token from 38 # --version ver : specify version being analyzed (default: ask git) 39 # --description desc : specify description of this version (default: ask git) 40 # --srcdir : QEMU source tree to analyze (default: current working dir) 41 # --results-tarball : path to copy the results tarball to (default: don't 42 # copy it anywhere, just upload it) 43 # --src-tarball : tarball to untar into src dir (default: none); this 44 # is intended mainly for internal use by the Docker support 45 # 46 # User-specifiable environment variables: 47 # COVERITY_TOKEN -- Coverity token (default: looks at your 48 # coverity.token config) 49 # COVERITY_EMAIL -- the email address to use for uploads (default: 50 # looks at your git coverity.email or user.email config) 51 # COVERITY_BUILD_CMD -- make command (default: 'make -jN' where N is 52 # number of CPUs as determined by 'nproc') 53 # COVERITY_TOOL_BASE -- set to directory to put coverity tools 54 # (default: /tmp/coverity-tools) 55 # 56 # You must specify the token, either by environment variable or by 57 # putting it in a file and using --tokenfile. Everything else has 58 # a reasonable default if this is run from a git tree. 59 60 check_upload_permissions() { 61 # Check whether we can do an upload to the server; will exit the script 62 # with status 1 if the check failed (usually a bad token); 63 # will exit the script with status 0 if the check indicated that we 64 # can't upload yet (ie we are at quota) 65 # Assumes that COVERITY_TOKEN, PROJNAME and DRYRUN have been initialized. 66 67 echo "Checking upload permissions..." 68 69 if ! up_perm="$(wget https://scan.coverity.com/api/upload_permitted --post-data "token=$COVERITY_TOKEN&project=$PROJNAME" -q -O -)"; then 70 echo "Coverity Scan API access denied: bad token?" 71 exit 1 72 fi 73 74 # Really up_perm is a JSON response with either 75 # {upload_permitted:true} or {next_upload_permitted_at:<date>} 76 # We do some hacky string parsing instead of properly parsing it. 77 case "$up_perm" in 78 *upload_permitted*true*) 79 echo "Coverity Scan: upload permitted" 80 ;; 81 *next_upload_permitted_at*) 82 if [ "$DRYRUN" = yes ]; then 83 echo "Coverity Scan: upload quota reached, continuing dry run" 84 else 85 echo "Coverity Scan: upload quota reached; stopping here" 86 # Exit success as this isn't a build error. 87 exit 0 88 fi 89 ;; 90 *) 91 echo "Coverity Scan upload check: unexpected result $up_perm" 92 exit 1 93 ;; 94 esac 95 } 96 97 98 build_docker_image() { 99 # build docker container including the coverity-scan tools 100 echo "Building docker container..." 101 # TODO: This re-unpacks the tools every time, rather than caching 102 # and reusing the image produced by the COPY of the .tgz file. 103 # Not sure why. 104 tests/docker/docker.py --engine ${DOCKER_ENGINE} build \ 105 -t coverity-scanner -f scripts/coverity-scan/coverity-scan.docker \ 106 --extra-files scripts/coverity-scan/run-coverity-scan \ 107 "$COVERITY_TOOL_BASE"/coverity_tool.tgz 108 } 109 110 update_coverity_tools () { 111 # Check for whether we need to download the Coverity tools 112 # (either because we don't have a copy, or because it's out of date) 113 # Assumes that COVERITY_TOOL_BASE, COVERITY_TOKEN and PROJNAME are set. 114 115 mkdir -p "$COVERITY_TOOL_BASE" 116 cd "$COVERITY_TOOL_BASE" 117 118 echo "Checking for new version of coverity build tools..." 119 wget https://scan.coverity.com/download/linux64 --post-data "token=$COVERITY_TOKEN&project=$PROJNAME&md5=1" -O coverity_tool.md5.new 120 121 if ! cmp -s coverity_tool.md5 coverity_tool.md5.new; then 122 # out of date md5 or no md5: download new build tool 123 # blow away the old build tool 124 echo "Downloading coverity build tools..." 125 rm -rf coverity_tool coverity_tool.tgz 126 wget https://scan.coverity.com/download/linux64 --post-data "token=$COVERITY_TOKEN&project=$PROJNAME" -O coverity_tool.tgz 127 if ! (cat coverity_tool.md5.new; echo " coverity_tool.tgz") | md5sum -c --status; then 128 echo "Downloaded tarball didn't match md5sum!" 129 exit 1 130 fi 131 132 if [ "$DOCKER" != yes ]; then 133 # extract the new one, keeping it corralled in a 'coverity_tool' directory 134 echo "Unpacking coverity build tools..." 135 mkdir -p coverity_tool 136 cd coverity_tool 137 tar xf ../coverity_tool.tgz 138 cd .. 139 mv coverity_tool.md5.new coverity_tool.md5 140 fi 141 fi 142 rm -f coverity_tool.md5.new 143 cd "$SRCDIR" 144 145 if [ "$DOCKER" = yes ]; then 146 build_docker_image 147 fi 148 } 149 150 151 # Check user-provided environment variables and arguments 152 DRYRUN=no 153 UPDATE=yes 154 DOCKER=no 155 156 while [ "$#" -ge 1 ]; do 157 case "$1" in 158 --dry-run) 159 shift 160 DRYRUN=yes 161 ;; 162 --no-update-tools) 163 shift 164 UPDATE=no 165 ;; 166 --update-tools-only) 167 shift 168 UPDATE=only 169 ;; 170 --version) 171 shift 172 if [ $# -eq 0 ]; then 173 echo "--version needs an argument" 174 exit 1 175 fi 176 VERSION="$1" 177 shift 178 ;; 179 --description) 180 shift 181 if [ $# -eq 0 ]; then 182 echo "--description needs an argument" 183 exit 1 184 fi 185 DESCRIPTION="$1" 186 shift 187 ;; 188 --tokenfile) 189 shift 190 if [ $# -eq 0 ]; then 191 echo "--tokenfile needs an argument" 192 exit 1 193 fi 194 COVERITY_TOKEN="$(cat "$1")" 195 shift 196 ;; 197 --srcdir) 198 shift 199 if [ $# -eq 0 ]; then 200 echo "--srcdir needs an argument" 201 exit 1 202 fi 203 SRCDIR="$1" 204 shift 205 ;; 206 --results-tarball) 207 shift 208 if [ $# -eq 0 ]; then 209 echo "--results-tarball needs an argument" 210 exit 1 211 fi 212 RESULTSTARBALL="$1" 213 shift 214 ;; 215 --src-tarball) 216 shift 217 if [ $# -eq 0 ]; then 218 echo "--src-tarball needs an argument" 219 exit 1 220 fi 221 SRCTARBALL="$1" 222 shift 223 ;; 224 --docker) 225 DOCKER=yes 226 DOCKER_ENGINE=auto 227 shift 228 ;; 229 --docker-engine) 230 shift 231 if [ $# -eq 0 ]; then 232 echo "--docker-engine needs an argument" 233 exit 1 234 fi 235 DOCKER=yes 236 DOCKER_ENGINE="$1" 237 shift 238 ;; 239 *) 240 echo "Unexpected argument '$1'" 241 exit 1 242 ;; 243 esac 244 done 245 246 if [ -z "$COVERITY_TOKEN" ]; then 247 COVERITY_TOKEN="$(git config coverity.token)" 248 fi 249 if [ -z "$COVERITY_TOKEN" ]; then 250 echo "COVERITY_TOKEN environment variable not set" 251 exit 1 252 fi 253 254 if [ -z "$COVERITY_BUILD_CMD" ]; then 255 NPROC=$(nproc) 256 COVERITY_BUILD_CMD="make -j$NPROC" 257 echo "COVERITY_BUILD_CMD: using default '$COVERITY_BUILD_CMD'" 258 fi 259 260 if [ -z "$COVERITY_TOOL_BASE" ]; then 261 echo "COVERITY_TOOL_BASE: using default /tmp/coverity-tools" 262 COVERITY_TOOL_BASE=/tmp/coverity-tools 263 fi 264 265 if [ -z "$SRCDIR" ]; then 266 SRCDIR="$PWD" 267 fi 268 269 PROJNAME=QEMU 270 TARBALL=cov-int.tar.xz 271 272 if [ "$UPDATE" = only ]; then 273 # Just do the tools update; we don't need to check whether 274 # we are in a source tree or have upload rights for this, 275 # so do it before some of the command line and source tree checks. 276 277 if [ "$DOCKER" = yes ] && [ ! -z "$SRCTARBALL" ]; then 278 echo --update-tools-only --docker is incompatible with --src-tarball. 279 exit 1 280 fi 281 282 update_coverity_tools 283 exit 0 284 fi 285 286 if [ ! -e "$SRCDIR" ]; then 287 mkdir "$SRCDIR" 288 fi 289 290 cd "$SRCDIR" 291 292 if [ ! -z "$SRCTARBALL" ]; then 293 echo "Untarring source tarball into $SRCDIR..." 294 tar xvf "$SRCTARBALL" 295 fi 296 297 echo "Checking this is a QEMU source tree..." 298 if ! [ -e "$SRCDIR/VERSION" ]; then 299 echo "Not in a QEMU source tree?" 300 exit 1 301 fi 302 303 # Fill in defaults used by the non-update-only process 304 if [ -z "$VERSION" ]; then 305 VERSION="$(git describe --always HEAD)" 306 fi 307 308 if [ -z "$DESCRIPTION" ]; then 309 DESCRIPTION="$(git rev-parse HEAD)" 310 fi 311 312 if [ -z "$COVERITY_EMAIL" ]; then 313 COVERITY_EMAIL="$(git config coverity.email)" 314 fi 315 if [ -z "$COVERITY_EMAIL" ]; then 316 COVERITY_EMAIL="$(git config user.email)" 317 fi 318 319 # Otherwise, continue with the full build and upload process. 320 321 check_upload_permissions 322 323 if [ "$UPDATE" != no ]; then 324 update_coverity_tools 325 fi 326 327 # Run ourselves inside docker if that's what the user wants 328 if [ "$DOCKER" = yes ]; then 329 # Put the Coverity token into a temporary file that only 330 # we have read access to, and then pass it to docker build 331 # using a volume. A volume is enough for the token not to 332 # leak into the Docker image. 333 umask 077 334 SECRETDIR=$(mktemp -d) 335 if [ -z "$SECRETDIR" ]; then 336 echo "Failed to create temporary directory" 337 exit 1 338 fi 339 trap 'rm -rf "$SECRETDIR"' INT TERM EXIT 340 echo "Created temporary directory $SECRETDIR" 341 SECRET="$SECRETDIR/token" 342 echo "$COVERITY_TOKEN" > "$SECRET" 343 echo "Archiving sources to be analyzed..." 344 ./scripts/archive-source.sh "$SECRETDIR/qemu-sources.tgz" 345 ARGS="--no-update-tools" 346 if [ "$DRYRUN" = yes ]; then 347 ARGS="$ARGS --dry-run" 348 fi 349 echo "Running scanner..." 350 # If we need to capture the output tarball, get the inner run to 351 # save it to the secrets directory so we can copy it out before the 352 # directory is cleaned up. 353 if [ ! -z "$RESULTSTARBALL" ]; then 354 ARGS="$ARGS --results-tarball /work/cov-int.tar.xz" 355 fi 356 # Arrange for this docker run to get access to the sources with -v. 357 # We pass through all the configuration from the outer script to the inner. 358 export COVERITY_EMAIL COVERITY_BUILD_CMD 359 tests/docker/docker.py run -it --env COVERITY_EMAIL --env COVERITY_BUILD_CMD \ 360 -v "$SECRETDIR:/work" coverity-scanner \ 361 ./run-coverity-scan --version "$VERSION" \ 362 --description "$DESCRIPTION" $ARGS --tokenfile /work/token \ 363 --srcdir /qemu --src-tarball /work/qemu-sources.tgz 364 if [ ! -z "$RESULTSTARBALL" ]; then 365 echo "Copying results tarball to $RESULTSTARBALL..." 366 cp "$SECRETDIR/cov-int.tar.xz" "$RESULTSTARBALL" 367 fi 368 echo "Docker work complete." 369 exit 0 370 fi 371 372 TOOLBIN="$(cd "$COVERITY_TOOL_BASE" && echo $PWD/coverity_tool/cov-analysis-*/bin)" 373 374 if ! test -x "$TOOLBIN/cov-build"; then 375 echo "Couldn't find cov-build in the coverity build-tool directory??" 376 exit 1 377 fi 378 379 export PATH="$TOOLBIN:$PATH" 380 381 cd "$SRCDIR" 382 383 echo "Nuking build directory..." 384 rm -rf +build 385 mkdir +build 386 cd +build 387 388 echo "Configuring..." 389 # We configure with a fixed set of enables here to ensure that we don't 390 # accidentally reduce the scope of the analysis by doing the build on 391 # the system that's missing a dependency that we need to build part of 392 # the codebase. 393 ../configure --disable-modules --enable-sdl --enable-gtk \ 394 --enable-opengl --enable-vte --enable-gnutls \ 395 --enable-nettle --enable-curses --enable-curl \ 396 --audio-drv-list=oss,alsa,sdl,pa --enable-virtfs \ 397 --enable-vnc --enable-vnc-sasl --enable-vnc-jpeg --enable-png \ 398 --enable-xen --enable-brlapi \ 399 --enable-linux-aio --enable-attr \ 400 --enable-cap-ng --enable-trace-backends=log --enable-spice --enable-rbd \ 401 --enable-libusb --enable-usb-redir \ 402 --enable-libiscsi --enable-libnfs --enable-seccomp \ 403 --enable-tpm --enable-libssh --enable-lzo --enable-snappy --enable-bzip2 \ 404 --enable-numa --enable-rdma --enable-smartcard --enable-virglrenderer \ 405 --enable-mpath --enable-glusterfs \ 406 --enable-virtfs --enable-zstd 407 408 echo "Running cov-build..." 409 rm -rf cov-int 410 mkdir cov-int 411 cov-build --dir cov-int $COVERITY_BUILD_CMD 412 413 echo "Creating results tarball..." 414 tar cvf - cov-int | xz > "$TARBALL" 415 416 if [ ! -z "$RESULTSTARBALL" ]; then 417 echo "Copying results tarball to $RESULTSTARBALL..." 418 cp "$TARBALL" "$RESULTSTARBALL" 419 fi 420 421 echo "Uploading results tarball..." 422 423 if [ "$DRYRUN" = yes ]; then 424 echo "Dry run only, not uploading $TARBALL" 425 exit 0 426 fi 427 428 curl --form token="$COVERITY_TOKEN" --form email="$COVERITY_EMAIL" \ 429 --form file=@"$TARBALL" --form version="$VERSION" \ 430 --form description="$DESCRIPTION" \ 431 https://scan.coverity.com/builds?project="$PROJNAME" 432 433 echo "Done."