device-crash-test (25412B)
1 #!/usr/bin/env python3 2 # 3 # Copyright (c) 2017 Red Hat Inc 4 # 5 # Author: 6 # Eduardo Habkost <ehabkost@redhat.com> 7 # 8 # This program is free software; you can redistribute it and/or modify 9 # it under the terms of the GNU General Public License as published by 10 # the Free Software Foundation; either version 2 of the License, or 11 # (at your option) any later version. 12 # 13 # This program is distributed in the hope that it will be useful, 14 # but WITHOUT ANY WARRANTY; without even the implied warranty of 15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 # GNU General Public License for more details. 17 # 18 # You should have received a copy of the GNU General Public License along 19 # with this program; if not, write to the Free Software Foundation, Inc., 20 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 21 22 """ 23 Run QEMU with all combinations of -machine and -device types, 24 check for crashes and unexpected errors. 25 """ 26 27 import os 28 import sys 29 import glob 30 import logging 31 import traceback 32 import re 33 import random 34 import argparse 35 from itertools import chain 36 from pathlib import Path 37 38 try: 39 from qemu.machine import QEMUMachine 40 from qemu.qmp import ConnectError 41 except ModuleNotFoundError as exc: 42 path = Path(__file__).resolve() 43 print(f"Module '{exc.name}' not found.") 44 print(" Try 'make check-venv' from your build directory,") 45 print(" and then one way to run this script is like so:") 46 print(f' > $builddir/tests/venv/bin/python3 "{path}"') 47 sys.exit(1) 48 49 logger = logging.getLogger('device-crash-test') 50 dbg = logger.debug 51 52 53 # Purposes of the following rule list: 54 # * Avoiding verbose log messages when we find known non-fatal 55 # (exitcode=1) errors 56 # * Avoiding fatal errors when we find known crashes 57 # * Skipping machines/devices that are known not to work out of 58 # the box, when running in --quick mode 59 # 60 # Keeping the rule list updated is desirable, but not required, 61 # because unexpected cases where QEMU exits with exitcode=1 will 62 # just trigger a INFO message. 63 64 # Valid error rule keys: 65 # * accel: regexp, full match only 66 # * machine: regexp, full match only 67 # * device: regexp, full match only 68 # * log: regexp, partial match allowed 69 # * exitcode: if not present, defaults to 1. If None, matches any exitcode 70 # * warn: if True, matching failures will be logged as warnings 71 # * expected: if True, QEMU is expected to always fail every time 72 # when testing the corresponding test case 73 # * loglevel: log level of log output when there's a match. 74 ERROR_RULE_LIST = [ 75 # Machines that won't work out of the box: 76 # MACHINE | ERROR MESSAGE 77 {'machine':'niagara', 'expected':True}, # Unable to load a firmware for -M niagara 78 {'machine':'boston', 'expected':True}, # Please provide either a -kernel or -bios argument 79 {'machine':'leon3_generic', 'expected':True}, # Can't read bios image (null) 80 81 # devices that don't work out of the box because they require extra options to "-device DEV": 82 # DEVICE | ERROR MESSAGE 83 {'device':'.*-(i386|x86_64)-cpu', 'expected':True}, # CPU socket-id is not set 84 {'device':'icp', 'expected':True}, # icp_realize: required link 'xics' not found: Property '.xics' not found 85 {'device':'ics', 'expected':True}, # ics_base_realize: required link 'xics' not found: Property '.xics' not found 86 # "-device ide-cd" does work on more recent QEMU versions, so it doesn't have expected=True 87 {'device':'ide-cd'}, # No drive specified 88 {'device':'ide-hd', 'expected':True}, # No drive specified 89 {'device':'ipmi-bmc-extern', 'expected':True}, # IPMI external bmc requires chardev attribute 90 {'device':'isa-debugcon', 'expected':True}, # Can't create serial device, empty char device 91 {'device':'isa-ipmi-bt', 'expected':True}, # IPMI device requires a bmc attribute to be set 92 {'device':'isa-ipmi-kcs', 'expected':True}, # IPMI device requires a bmc attribute to be set 93 {'device':'isa-parallel', 'expected':True}, # Can't create serial device, empty char device 94 {'device':'ivshmem-doorbell', 'expected':True}, # You must specify a 'chardev' 95 {'device':'ivshmem-plain', 'expected':True}, # You must specify a 'memdev' 96 {'device':'loader', 'expected':True}, # please include valid arguments 97 {'device':'nand', 'expected':True}, # Unsupported NAND block size 0x1 98 {'device':'nvdimm', 'expected':True}, # 'memdev' property is not set 99 {'device':'nvme', 'expected':True}, # Device initialization failed 100 {'device':'pc-dimm', 'expected':True}, # 'memdev' property is not set 101 {'device':'pci-bridge', 'expected':True}, # Bridge chassis not specified. Each bridge is required to be assigned a unique chassis id > 0. 102 {'device':'pci-bridge-seat', 'expected':True}, # Bridge chassis not specified. Each bridge is required to be assigned a unique chassis id > 0. 103 {'device':'pxb', 'expected':True}, # Bridge chassis not specified. Each bridge is required to be assigned a unique chassis id > 0. 104 {'device':'pxb-cxl', 'expected':True}, # pxb-cxl devices cannot reside on a PCI bus. 105 {'device':'scsi-block', 'expected':True}, # drive property not set 106 {'device':'scsi-generic', 'expected':True}, # drive property not set 107 {'device':'scsi-hd', 'expected':True}, # drive property not set 108 {'device':'spapr-pci-host-bridge', 'expected':True}, # BUID not specified for PHB 109 {'device':'spapr-rng', 'expected':True}, # spapr-rng needs an RNG backend! 110 {'device':'spapr-vty', 'expected':True}, # chardev property not set 111 {'device':'tpm-tis', 'expected':True}, # tpm_tis: backend driver with id (null) could not be found 112 {'device':'unimplemented-device', 'expected':True}, # property 'size' not specified or zero 113 {'device':'usb-braille', 'expected':True}, # Property chardev is required 114 {'device':'usb-mtp', 'expected':True}, # rootdir property must be configured 115 {'device':'usb-redir', 'expected':True}, # Parameter 'chardev' is missing 116 {'device':'usb-serial', 'expected':True}, # Property chardev is required 117 {'device':'usb-storage', 'expected':True}, # drive property not set 118 {'device':'vfio-amd-xgbe', 'expected':True}, # -device vfio-amd-xgbe: vfio error: wrong host device name 119 {'device':'vfio-calxeda-xgmac', 'expected':True}, # -device vfio-calxeda-xgmac: vfio error: wrong host device name 120 {'device':'vfio-pci', 'expected':True}, # No provided host device 121 {'device':'vfio-pci-igd-lpc-bridge', 'expected':True}, # VFIO dummy ISA/LPC bridge must have address 1f.0 122 {'device':'vhost-scsi.*', 'expected':True}, # vhost-scsi: missing wwpn 123 {'device':'vhost-vsock-device', 'expected':True}, # guest-cid property must be greater than 2 124 {'device':'vhost-vsock-pci', 'expected':True}, # guest-cid property must be greater than 2 125 {'device':'virtio-9p-ccw', 'expected':True}, # 9pfs device couldn't find fsdev with the id = NULL 126 {'device':'virtio-9p-device', 'expected':True}, # 9pfs device couldn't find fsdev with the id = NULL 127 {'device':'virtio-9p-pci', 'expected':True}, # 9pfs device couldn't find fsdev with the id = NULL 128 {'device':'virtio-blk-ccw', 'expected':True}, # drive property not set 129 {'device':'virtio-blk-device', 'expected':True}, # drive property not set 130 {'device':'virtio-blk-device', 'expected':True}, # drive property not set 131 {'device':'virtio-blk-pci', 'expected':True}, # drive property not set 132 {'device':'virtio-crypto-ccw', 'expected':True}, # 'cryptodev' parameter expects a valid object 133 {'device':'virtio-crypto-device', 'expected':True}, # 'cryptodev' parameter expects a valid object 134 {'device':'virtio-crypto-pci', 'expected':True}, # 'cryptodev' parameter expects a valid object 135 {'device':'virtio-input-host-device', 'expected':True}, # evdev property is required 136 {'device':'virtio-input-host-pci', 'expected':True}, # evdev property is required 137 {'device':'xen-pvdevice', 'expected':True}, # Device ID invalid, it must always be supplied 138 {'device':'vhost-vsock-ccw', 'expected':True}, # guest-cid property must be greater than 2 139 {'device':'zpci', 'expected':True}, # target must be defined 140 {'device':'pnv-(occ|icp|lpc)', 'expected':True}, # required link 'xics' not found: Property '.xics' not found 141 {'device':'powernv-cpu-.*', 'expected':True}, # pnv_core_realize: required link 'xics' not found: Property '.xics' not found 142 143 # ioapic devices are already created by pc and will fail: 144 {'machine':'q35|pc.*', 'device':'kvm-ioapic', 'expected':True}, # Only 1 ioapics allowed 145 {'machine':'q35|pc.*', 'device':'ioapic', 'expected':True}, # Only 1 ioapics allowed 146 147 # "spapr-cpu-core needs a pseries machine" 148 {'machine':'(?!pseries).*', 'device':'.*-spapr-cpu-core', 'expected':True}, 149 150 # KVM-specific devices shouldn't be tried without accel=kvm: 151 {'accel':'(?!kvm).*', 'device':'kvmclock', 'expected':True}, 152 153 # xen-specific machines and devices: 154 {'accel':'(?!xen).*', 'machine':'xen.*', 'expected':True}, 155 {'accel':'(?!xen).*', 'device':'xen-.*', 'expected':True}, 156 157 # this fails on some machine-types, but not all, so they don't have expected=True: 158 {'device':'vmgenid'}, # vmgenid requires DMA write support in fw_cfg, which this machine type does not provide 159 160 # Silence INFO messages for errors that are common on multiple 161 # devices/machines: 162 {'log':r"No '[\w-]+' bus found for device '[\w-]+'"}, 163 {'log':r"images* must be given with the 'pflash' parameter"}, 164 {'log':r"(Guest|ROM|Flash|Kernel) image must be specified"}, 165 {'log':r"[cC]ould not load [\w ]+ (BIOS|bios) '[\w-]+\.bin'"}, 166 {'log':r"Couldn't find rom image '[\w-]+\.bin'"}, 167 {'log':r"speed mismatch trying to attach usb device"}, 168 {'log':r"Can't create a second ISA bus"}, 169 {'log':r"duplicate fw_cfg file name"}, 170 # sysbus-related error messages: most machines reject most dynamic sysbus devices: 171 {'log':r"Option '-device [\w.,-]+' cannot be handled by this machine"}, 172 {'log':r"Device [\w.,-]+ is not supported by this machine yet"}, 173 {'log':r"Device [\w.,-]+ can not be dynamically instantiated"}, 174 {'log':r"Platform Bus: Can not fit MMIO region of size "}, 175 # other more specific errors we will ignore: 176 {'device':'.*-spapr-cpu-core', 'log':r"CPU core type should be"}, 177 {'log':r"MSI(-X)? is not supported by interrupt controller"}, 178 {'log':r"pxb-pcie? devices cannot reside on a PCIe? bus"}, 179 {'log':r"Ignoring smp_cpus value"}, 180 {'log':r"sd_init failed: Drive 'sd0' is already in use because it has been automatically connected to another device"}, 181 {'log':r"This CPU requires a smaller page size than the system is using"}, 182 {'log':r"MSI-X support is mandatory in the S390 architecture"}, 183 {'log':r"rom check and register reset failed"}, 184 {'log':r"Unable to initialize GIC, CPUState for CPU#0 not valid"}, 185 {'log':r"Multiple VT220 operator consoles are not supported"}, 186 {'log':r"core 0 already populated"}, 187 {'log':r"could not find stage1 bootloader"}, 188 189 # other exitcode=1 failures not listed above will just generate INFO messages: 190 {'exitcode':1, 'loglevel':logging.INFO}, 191 192 # everything else (including SIGABRT and SIGSEGV) will be a fatal error: 193 {'exitcode':None, 'fatal':True, 'loglevel':logging.FATAL}, 194 ] 195 196 197 def errorRuleTestCaseMatch(rule, t): 198 """Check if a test case specification can match a error rule 199 200 This only checks if a error rule is a candidate match 201 for a given test case, it won't check if the test case 202 results/output match the rule. See ruleListResultMatch(). 203 """ 204 return (('machine' not in rule or 205 'machine' not in t or 206 re.match(rule['machine'] + '$', t['machine'])) and 207 ('accel' not in rule or 208 'accel' not in t or 209 re.match(rule['accel'] + '$', t['accel'])) and 210 ('device' not in rule or 211 'device' not in t or 212 re.match(rule['device'] + '$', t['device']))) 213 214 215 def ruleListCandidates(t): 216 """Generate the list of candidates that can match a test case""" 217 for i, rule in enumerate(ERROR_RULE_LIST): 218 if errorRuleTestCaseMatch(rule, t): 219 yield (i, rule) 220 221 222 def findExpectedResult(t): 223 """Check if there's an expected=True error rule for a test case 224 225 Returns (i, rule) tuple, where i is the index in 226 ERROR_RULE_LIST and rule is the error rule itself. 227 """ 228 for i, rule in ruleListCandidates(t): 229 if rule.get('expected'): 230 return (i, rule) 231 232 233 def ruleListResultMatch(rule, r): 234 """Check if test case results/output match a error rule 235 236 It is valid to call this function only if 237 errorRuleTestCaseMatch() is True for the rule (e.g. on 238 rules returned by ruleListCandidates()) 239 """ 240 assert errorRuleTestCaseMatch(rule, r['testcase']) 241 return ((rule.get('exitcode', 1) is None or 242 r['exitcode'] == rule.get('exitcode', 1)) and 243 ('log' not in rule or 244 re.search(rule['log'], r['log'], re.MULTILINE))) 245 246 247 def checkResultRuleList(r): 248 """Look up error rule for a given test case result 249 250 Returns (i, rule) tuple, where i is the index in 251 ERROR_RULE_LIST and rule is the error rule itself. 252 """ 253 for i, rule in ruleListCandidates(r['testcase']): 254 if ruleListResultMatch(rule, r): 255 return i, rule 256 257 raise Exception("this should never happen") 258 259 260 def qemuOptsEscape(s): 261 """Escape option value QemuOpts""" 262 return s.replace(",", ",,") 263 264 265 def formatTestCase(t): 266 """Format test case info as "key=value key=value" for prettier logging output""" 267 return ' '.join('%s=%s' % (k, v) for k, v in t.items()) 268 269 270 def qomListTypeNames(vm, **kwargs): 271 """Run qom-list-types QMP command, return type names""" 272 types = vm.command('qom-list-types', **kwargs) 273 return [t['name'] for t in types] 274 275 276 def infoQDM(vm): 277 """Parse 'info qdm' output""" 278 args = {'command-line': 'info qdm'} 279 devhelp = vm.command('human-monitor-command', **args) 280 for l in devhelp.split('\n'): 281 l = l.strip() 282 if l == '' or l.endswith(':'): 283 continue 284 d = {'name': re.search(r'name "([^"]+)"', l).group(1), 285 'no-user': (re.search(', no-user', l) is not None)} 286 yield d 287 288 289 class QemuBinaryInfo(object): 290 def __init__(self, binary, devtype): 291 if devtype is None: 292 devtype = 'device' 293 294 self.binary = binary 295 self._machine_info = {} 296 297 dbg("devtype: %r", devtype) 298 args = ['-S', '-machine', 'none,accel=kvm:tcg'] 299 dbg("querying info for QEMU binary: %s", binary) 300 vm = QEMUMachine(binary=binary, args=args) 301 vm.launch() 302 try: 303 self.alldevs = set(qomListTypeNames(vm, implements=devtype, abstract=False)) 304 # there's no way to query DeviceClass::user_creatable using QMP, 305 # so use 'info qdm': 306 self.no_user_devs = set([d['name'] for d in infoQDM(vm, ) if d['no-user']]) 307 self.machines = list(m['name'] for m in vm.command('query-machines')) 308 self.user_devs = self.alldevs.difference(self.no_user_devs) 309 self.kvm_available = vm.command('query-kvm')['enabled'] 310 finally: 311 vm.shutdown() 312 313 def machineInfo(self, machine): 314 """Query for information on a specific machine-type 315 316 Results are cached internally, in case the same machine- 317 type is queried multiple times. 318 """ 319 if machine in self._machine_info: 320 return self._machine_info[machine] 321 322 mi = {} 323 args = ['-S', '-machine', '%s' % (machine)] 324 dbg("querying machine info for binary=%s machine=%s", self.binary, machine) 325 vm = QEMUMachine(binary=self.binary, args=args) 326 try: 327 vm.launch() 328 mi['runnable'] = True 329 except Exception: 330 dbg("exception trying to run binary=%s machine=%s", self.binary, machine, exc_info=sys.exc_info()) 331 dbg("log: %r", vm.get_log()) 332 mi['runnable'] = False 333 334 vm.shutdown() 335 self._machine_info[machine] = mi 336 return mi 337 338 339 BINARY_INFO = {} 340 341 342 def getBinaryInfo(args, binary): 343 if binary not in BINARY_INFO: 344 BINARY_INFO[binary] = QemuBinaryInfo(binary, args.devtype) 345 return BINARY_INFO[binary] 346 347 348 def checkOneCase(args, testcase): 349 """Check one specific case 350 351 Returns a dictionary containing failure information on error, 352 or None on success 353 """ 354 binary = testcase['binary'] 355 accel = testcase['accel'] 356 machine = testcase['machine'] 357 device = testcase['device'] 358 359 dbg("will test: %r", testcase) 360 361 args = ['-S', '-machine', '%s,accel=%s' % (machine, accel), 362 '-device', qemuOptsEscape(device)] 363 cmdline = ' '.join([binary] + args) 364 dbg("will launch QEMU: %s", cmdline) 365 vm = QEMUMachine(binary=binary, args=args, qmp_timer=15) 366 367 exc = None 368 exc_traceback = None 369 try: 370 vm.launch() 371 except Exception as this_exc: 372 exc = this_exc 373 exc_traceback = traceback.format_exc() 374 dbg("Exception while running test case") 375 finally: 376 vm.shutdown() 377 ec = vm.exitcode() 378 log = vm.get_log() 379 380 if exc is not None or ec != 0: 381 return {'exc': exc, 382 'exc_traceback':exc_traceback, 383 'exitcode':ec, 384 'log':log, 385 'testcase':testcase, 386 'cmdline':cmdline} 387 388 389 def binariesToTest(args, testcase): 390 if args.qemu: 391 r = args.qemu 392 else: 393 r = [f.path for f in os.scandir('.') 394 if f.name.startswith('qemu-system-') and 395 f.is_file() and os.access(f, os.X_OK)] 396 return r 397 398 399 def accelsToTest(args, testcase): 400 if getBinaryInfo(args, testcase['binary']).kvm_available: 401 yield 'kvm' 402 yield 'tcg' 403 404 405 def machinesToTest(args, testcase): 406 return getBinaryInfo(args, testcase['binary']).machines 407 408 409 def devicesToTest(args, testcase): 410 return getBinaryInfo(args, testcase['binary']).user_devs 411 412 413 TESTCASE_VARIABLES = [ 414 ('binary', binariesToTest), 415 ('accel', accelsToTest), 416 ('machine', machinesToTest), 417 ('device', devicesToTest), 418 ] 419 420 421 def genCases1(args, testcases, var, fn): 422 """Generate new testcases for one variable 423 424 If an existing item already has a variable set, don't 425 generate new items and just return it directly. This 426 allows the "-t" command-line option to be used to choose 427 a specific test case. 428 """ 429 for testcase in testcases: 430 if var in testcase: 431 yield testcase.copy() 432 else: 433 for i in fn(args, testcase): 434 t = testcase.copy() 435 t[var] = i 436 yield t 437 438 439 def genCases(args, testcase): 440 """Generate test cases for all variables 441 """ 442 cases = [testcase.copy()] 443 for var, fn in TESTCASE_VARIABLES: 444 dbg("var: %r, fn: %r", var, fn) 445 cases = genCases1(args, cases, var, fn) 446 return cases 447 448 449 def casesToTest(args, testcase): 450 cases = genCases(args, testcase) 451 if args.random: 452 cases = list(cases) 453 cases = random.sample(cases, min(args.random, len(cases))) 454 if args.debug: 455 cases = list(cases) 456 dbg("%d test cases to test", len(cases)) 457 if args.shuffle: 458 cases = list(cases) 459 random.shuffle(cases) 460 return cases 461 462 463 def logFailure(f, level): 464 t = f['testcase'] 465 logger.log(level, "failed: %s", formatTestCase(t)) 466 logger.log(level, "cmdline: %s", f['cmdline']) 467 for l in f['log'].strip().split('\n'): 468 logger.log(level, "log: %s", l) 469 logger.log(level, "exit code: %r", f['exitcode']) 470 471 # If the Exception is merely a QMP connect error, 472 # reduce the logging level for its traceback to 473 # improve visual clarity. 474 if isinstance(f.get('exc'), ConnectError): 475 logger.log(level, "%s.%s: %s", 476 type(f['exc']).__module__, 477 type(f['exc']).__qualname__, 478 str(f['exc'])) 479 level = logging.DEBUG 480 481 if f['exc_traceback']: 482 logger.log(level, "exception:") 483 for l in f['exc_traceback'].split('\n'): 484 logger.log(level, " %s", l.rstrip('\n')) 485 486 487 def main(): 488 parser = argparse.ArgumentParser(description="QEMU -device crash test") 489 parser.add_argument('-t', metavar='KEY=VALUE', nargs='*', 490 help="Limit test cases to KEY=VALUE", 491 action='append', dest='testcases', default=[]) 492 parser.add_argument('-d', '--debug', action='store_true', 493 help='debug output') 494 parser.add_argument('-v', '--verbose', action='store_true', default=True, 495 help='verbose output') 496 parser.add_argument('-q', '--quiet', dest='verbose', action='store_false', 497 help='non-verbose output') 498 parser.add_argument('-r', '--random', type=int, metavar='COUNT', 499 help='run a random sample of COUNT test cases', 500 default=0) 501 parser.add_argument('--shuffle', action='store_true', 502 help='Run test cases in random order') 503 parser.add_argument('--dry-run', action='store_true', 504 help="Don't run any tests, just generate list") 505 parser.add_argument('-D', '--devtype', metavar='TYPE', 506 help="Test only device types that implement TYPE") 507 parser.add_argument('-Q', '--quick', action='store_true', default=True, 508 help="Quick mode: skip test cases that are expected to fail") 509 parser.add_argument('-F', '--full', action='store_false', dest='quick', 510 help="Full mode: test cases that are expected to fail") 511 parser.add_argument('--strict', action='store_true', dest='strict', 512 help="Treat all warnings as fatal") 513 parser.add_argument('qemu', nargs='*', metavar='QEMU', 514 help='QEMU binary to run') 515 args = parser.parse_args() 516 517 if args.debug: 518 lvl = logging.DEBUG 519 elif args.verbose: 520 lvl = logging.INFO 521 else: 522 lvl = logging.WARN 523 logging.basicConfig(stream=sys.stdout, level=lvl, format='%(levelname)s: %(message)s') 524 525 if not args.debug: 526 # Async QMP, when in use, is chatty about connection failures. 527 # This script knowingly generates a ton of connection errors. 528 # Silence this logger. 529 logging.getLogger('qemu.qmp.qmp_client').setLevel(logging.CRITICAL) 530 531 fatal_failures = [] 532 wl_stats = {} 533 skipped = 0 534 total = 0 535 536 tc = {} 537 dbg("testcases: %r", args.testcases) 538 if args.testcases: 539 for t in chain(*args.testcases): 540 for kv in t.split(): 541 k, v = kv.split('=', 1) 542 tc[k] = v 543 544 if len(binariesToTest(args, tc)) == 0: 545 print("No QEMU binary found", file=sys.stderr) 546 parser.print_usage(sys.stderr) 547 return 1 548 549 for t in casesToTest(args, tc): 550 logger.info("running test case: %s", formatTestCase(t)) 551 total += 1 552 553 expected_match = findExpectedResult(t) 554 if (args.quick and 555 (expected_match or 556 not getBinaryInfo(args, t['binary']).machineInfo(t['machine'])['runnable'])): 557 dbg("skipped: %s", formatTestCase(t)) 558 skipped += 1 559 continue 560 561 if args.dry_run: 562 continue 563 564 try: 565 f = checkOneCase(args, t) 566 except KeyboardInterrupt: 567 break 568 569 if f: 570 i, rule = checkResultRuleList(f) 571 dbg("testcase: %r, rule list match: %r", t, rule) 572 wl_stats.setdefault(i, []).append(f) 573 level = rule.get('loglevel', logging.DEBUG) 574 logFailure(f, level) 575 if rule.get('fatal') or (args.strict and level >= logging.WARN): 576 fatal_failures.append(f) 577 else: 578 dbg("success: %s", formatTestCase(t)) 579 if expected_match: 580 logger.warn("Didn't fail as expected: %s", formatTestCase(t)) 581 582 logger.info("Total: %d test cases", total) 583 if skipped: 584 logger.info("Skipped %d test cases", skipped) 585 586 if args.debug: 587 stats = sorted([(len(wl_stats.get(i, [])), rule) for i, rule in 588 enumerate(ERROR_RULE_LIST)], key=lambda x: x[0]) 589 for count, rule in stats: 590 dbg("error rule stats: %d: %r", count, rule) 591 592 if fatal_failures: 593 for f in fatal_failures: 594 t = f['testcase'] 595 logger.error("Fatal failure: %s", formatTestCase(t)) 596 logger.error("Fatal failures on some machine/device combinations") 597 return 1 598 599 if __name__ == '__main__': 600 sys.exit(main())