SDL_test_harness.c (24449B)
1 /* 2 Simple DirectMedia Layer 3 Copyright (C) 1997-2020 Sam Lantinga <slouken@libsdl.org> 4 5 This software is provided 'as-is', without any express or implied 6 warranty. In no event will the authors be held liable for any damages 7 arising from the use of this software. 8 9 Permission is granted to anyone to use this software for any purpose, 10 including commercial applications, and to alter it and redistribute it 11 freely, subject to the following restrictions: 12 13 1. The origin of this software must not be misrepresented; you must not 14 claim that you wrote the original software. If you use this software 15 in a product, an acknowledgment in the product documentation would be 16 appreciated but is not required. 17 2. Altered source versions must be plainly marked as such, and must not be 18 misrepresented as being the original software. 19 3. This notice may not be removed or altered from any source distribution. 20 */ 21 22 #include "SDL_config.h" 23 24 #include "SDL_test.h" 25 26 #include <stdio.h> 27 #include <stdlib.h> 28 #include <string.h> 29 #include <time.h> 30 31 /* Invalid test name/description message format */ 32 #define SDLTEST_INVALID_NAME_FORMAT "(Invalid)" 33 34 /* Log summary message format */ 35 #define SDLTEST_LOG_SUMMARY_FORMAT "%s Summary: Total=%d Passed=%d Failed=%d Skipped=%d" 36 37 /* Final result message format */ 38 #define SDLTEST_FINAL_RESULT_FORMAT ">>> %s '%s': %s\n" 39 40 /* ! \brief Timeout for single test case execution */ 41 static Uint32 SDLTest_TestCaseTimeout = 3600; 42 43 /** 44 * Generates a random run seed string for the harness. The generated seed 45 * will contain alphanumeric characters (0-9A-Z). 46 * 47 * Note: The returned string needs to be deallocated by the caller. 48 * 49 * \param length The length of the seed string to generate 50 * 51 * \returns The generated seed string 52 */ 53 char * 54 SDLTest_GenerateRunSeed(const int length) 55 { 56 char *seed = NULL; 57 SDLTest_RandomContext randomContext; 58 int counter; 59 60 /* Sanity check input */ 61 if (length <= 0) { 62 SDLTest_LogError("The length of the harness seed must be >0."); 63 return NULL; 64 } 65 66 /* Allocate output buffer */ 67 seed = (char *)SDL_malloc((length + 1) * sizeof(char)); 68 if (seed == NULL) { 69 SDLTest_LogError("SDL_malloc for run seed output buffer failed."); 70 SDL_Error(SDL_ENOMEM); 71 return NULL; 72 } 73 74 /* Generate a random string of alphanumeric characters */ 75 SDLTest_RandomInitTime(&randomContext); 76 for (counter = 0; counter < length; counter++) { 77 unsigned int number = SDLTest_Random(&randomContext); 78 char ch = (char) (number % (91 - 48)) + 48; 79 if (ch >= 58 && ch <= 64) { 80 ch = 65; 81 } 82 seed[counter] = ch; 83 } 84 seed[length] = '\0'; 85 86 return seed; 87 } 88 89 /** 90 * Generates an execution key for the fuzzer. 91 * 92 * \param runSeed The run seed to use 93 * \param suiteName The name of the test suite 94 * \param testName The name of the test 95 * \param iteration The iteration count 96 * 97 * \returns The generated execution key to initialize the fuzzer with. 98 * 99 */ 100 static Uint64 101 SDLTest_GenerateExecKey(const char *runSeed, char *suiteName, char *testName, int iteration) 102 { 103 SDLTest_Md5Context md5Context; 104 Uint64 *keys; 105 char iterationString[16]; 106 size_t runSeedLength; 107 size_t suiteNameLength; 108 size_t testNameLength; 109 size_t iterationStringLength; 110 size_t entireStringLength; 111 char *buffer; 112 113 if (runSeed == NULL || runSeed[0] == '\0') { 114 SDLTest_LogError("Invalid runSeed string."); 115 return -1; 116 } 117 118 if (suiteName == NULL || suiteName[0] == '\0') { 119 SDLTest_LogError("Invalid suiteName string."); 120 return -1; 121 } 122 123 if (testName == NULL || testName[0] == '\0') { 124 SDLTest_LogError("Invalid testName string."); 125 return -1; 126 } 127 128 if (iteration <= 0) { 129 SDLTest_LogError("Invalid iteration count."); 130 return -1; 131 } 132 133 /* Convert iteration number into a string */ 134 SDL_memset(iterationString, 0, sizeof(iterationString)); 135 SDL_snprintf(iterationString, sizeof(iterationString) - 1, "%d", iteration); 136 137 /* Combine the parameters into single string */ 138 runSeedLength = SDL_strlen(runSeed); 139 suiteNameLength = SDL_strlen(suiteName); 140 testNameLength = SDL_strlen(testName); 141 iterationStringLength = SDL_strlen(iterationString); 142 entireStringLength = runSeedLength + suiteNameLength + testNameLength + iterationStringLength + 1; 143 buffer = (char *)SDL_malloc(entireStringLength); 144 if (buffer == NULL) { 145 SDLTest_LogError("Failed to allocate buffer for execKey generation."); 146 SDL_Error(SDL_ENOMEM); 147 return 0; 148 } 149 SDL_snprintf(buffer, entireStringLength, "%s%s%s%d", runSeed, suiteName, testName, iteration); 150 151 /* Hash string and use half of the digest as 64bit exec key */ 152 SDLTest_Md5Init(&md5Context); 153 SDLTest_Md5Update(&md5Context, (unsigned char *)buffer, (unsigned int) entireStringLength); 154 SDLTest_Md5Final(&md5Context); 155 SDL_free(buffer); 156 keys = (Uint64 *)md5Context.digest; 157 158 return keys[0]; 159 } 160 161 /** 162 * \brief Set timeout handler for test. 163 * 164 * Note: SDL_Init(SDL_INIT_TIMER) will be called if it wasn't done so before. 165 * 166 * \param timeout Timeout interval in seconds. 167 * \param callback Function that will be called after timeout has elapsed. 168 * 169 * \return Timer id or -1 on failure. 170 */ 171 static SDL_TimerID 172 SDLTest_SetTestTimeout(int timeout, void (*callback)(void)) 173 { 174 Uint32 timeoutInMilliseconds; 175 SDL_TimerID timerID; 176 177 if (callback == NULL) { 178 SDLTest_LogError("Timeout callback can't be NULL"); 179 return -1; 180 } 181 182 if (timeout < 0) { 183 SDLTest_LogError("Timeout value must be bigger than zero."); 184 return -1; 185 } 186 187 /* Init SDL timer if not initialized before */ 188 if (SDL_WasInit(SDL_INIT_TIMER) == 0) { 189 if (SDL_InitSubSystem(SDL_INIT_TIMER)) { 190 SDLTest_LogError("Failed to init timer subsystem: %s", SDL_GetError()); 191 return -1; 192 } 193 } 194 195 /* Set timer */ 196 timeoutInMilliseconds = timeout * 1000; 197 timerID = SDL_AddTimer(timeoutInMilliseconds, (SDL_TimerCallback)callback, 0x0); 198 if (timerID == 0) { 199 SDLTest_LogError("Creation of SDL timer failed: %s", SDL_GetError()); 200 return -1; 201 } 202 203 return timerID; 204 } 205 206 /** 207 * \brief Timeout handler. Aborts test run and exits harness process. 208 */ 209 #if defined(__WATCOMC__) 210 #pragma aux SDLTest_BailOut aborts; 211 #endif 212 static SDL_NORETURN void 213 SDLTest_BailOut(void) 214 { 215 SDLTest_LogError("TestCaseTimeout timer expired. Aborting test run."); 216 exit(TEST_ABORTED); /* bail out from the test */ 217 } 218 219 /** 220 * \brief Execute a test using the given execution key. 221 * 222 * \param testSuite Suite containing the test case. 223 * \param testCase Case to execute. 224 * \param execKey Execution key for the fuzzer. 225 * \param forceTestRun Force test to run even if test was disabled in suite. 226 * 227 * \returns Test case result. 228 */ 229 static int 230 SDLTest_RunTest(SDLTest_TestSuiteReference *testSuite, const SDLTest_TestCaseReference *testCase, Uint64 execKey, SDL_bool forceTestRun) 231 { 232 SDL_TimerID timer = 0; 233 int testCaseResult = 0; 234 int testResult = 0; 235 int fuzzerCount; 236 237 if (testSuite==NULL || testCase==NULL || testSuite->name==NULL || testCase->name==NULL) 238 { 239 SDLTest_LogError("Setup failure: testSuite or testCase references NULL"); 240 return TEST_RESULT_SETUP_FAILURE; 241 } 242 243 if (!testCase->enabled && forceTestRun == SDL_FALSE) 244 { 245 SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Test", testCase->name, "Skipped (Disabled)"); 246 return TEST_RESULT_SKIPPED; 247 } 248 249 /* Initialize fuzzer */ 250 SDLTest_FuzzerInit(execKey); 251 252 /* Reset assert tracker */ 253 SDLTest_ResetAssertSummary(); 254 255 /* Set timeout timer */ 256 timer = SDLTest_SetTestTimeout(SDLTest_TestCaseTimeout, SDLTest_BailOut); 257 258 /* Maybe run suite initalizer function */ 259 if (testSuite->testSetUp) { 260 testSuite->testSetUp(0x0); 261 if (SDLTest_AssertSummaryToTestResult() == TEST_RESULT_FAILED) { 262 SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Suite Setup", testSuite->name, "Failed"); 263 return TEST_RESULT_SETUP_FAILURE; 264 } 265 } 266 267 /* Run test case function */ 268 testCaseResult = testCase->testCase(0x0); 269 270 /* Convert test execution result into harness result */ 271 if (testCaseResult == TEST_SKIPPED) { 272 /* Test was programatically skipped */ 273 testResult = TEST_RESULT_SKIPPED; 274 } else if (testCaseResult == TEST_STARTED) { 275 /* Test did not return a TEST_COMPLETED value; assume it failed */ 276 testResult = TEST_RESULT_FAILED; 277 } else if (testCaseResult == TEST_ABORTED) { 278 /* Test was aborted early; assume it failed */ 279 testResult = TEST_RESULT_FAILED; 280 } else { 281 /* Perform failure analysis based on asserts */ 282 testResult = SDLTest_AssertSummaryToTestResult(); 283 } 284 285 /* Maybe run suite cleanup function (ignore failed asserts) */ 286 if (testSuite->testTearDown) { 287 testSuite->testTearDown(0x0); 288 } 289 290 /* Cancel timeout timer */ 291 if (timer) { 292 SDL_RemoveTimer(timer); 293 } 294 295 /* Report on asserts and fuzzer usage */ 296 fuzzerCount = SDLTest_GetFuzzerInvocationCount(); 297 if (fuzzerCount > 0) { 298 SDLTest_Log("Fuzzer invocations: %d", fuzzerCount); 299 } 300 301 /* Final log based on test execution result */ 302 if (testCaseResult == TEST_SKIPPED) { 303 /* Test was programatically skipped */ 304 SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Test", testCase->name, "Skipped (Programmatically)"); 305 } else if (testCaseResult == TEST_STARTED) { 306 /* Test did not return a TEST_COMPLETED value; assume it failed */ 307 SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Test", testCase->name, "Failed (test started, but did not return TEST_COMPLETED)"); 308 } else if (testCaseResult == TEST_ABORTED) { 309 /* Test was aborted early; assume it failed */ 310 SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Test", testCase->name, "Failed (Aborted)"); 311 } else { 312 SDLTest_LogAssertSummary(); 313 } 314 315 return testResult; 316 } 317 318 /* Prints summary of all suites/tests contained in the given reference */ 319 #if 0 320 static void SDLTest_LogTestSuiteSummary(SDLTest_TestSuiteReference *testSuites) 321 { 322 int suiteCounter; 323 int testCounter; 324 SDLTest_TestSuiteReference *testSuite; 325 SDLTest_TestCaseReference *testCase; 326 327 /* Loop over all suites */ 328 suiteCounter = 0; 329 while(&testSuites[suiteCounter]) { 330 testSuite=&testSuites[suiteCounter]; 331 suiteCounter++; 332 SDLTest_Log("Test Suite %i - %s\n", suiteCounter, 333 (testSuite->name) ? testSuite->name : SDLTEST_INVALID_NAME_FORMAT); 334 335 /* Loop over all test cases */ 336 testCounter = 0; 337 while(testSuite->testCases[testCounter]) 338 { 339 testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter]; 340 testCounter++; 341 SDLTest_Log(" Test Case %i - %s: %s", testCounter, 342 (testCase->name) ? testCase->name : SDLTEST_INVALID_NAME_FORMAT, 343 (testCase->description) ? testCase->description : SDLTEST_INVALID_NAME_FORMAT); 344 } 345 } 346 } 347 #endif 348 349 /* Gets a timer value in seconds */ 350 static float GetClock() 351 { 352 float currentClock = clock() / (float) CLOCKS_PER_SEC; 353 return currentClock; 354 } 355 356 /** 357 * \brief Execute a test suite using the given run seed and execution key. 358 * 359 * The filter string is matched to the suite name (full comparison) to select a single suite, 360 * or if no suite matches, it is matched to the test names (full comparison) to select a single test. 361 * 362 * \param testSuites Suites containing the test case. 363 * \param userRunSeed Custom run seed provided by user, or NULL to autogenerate one. 364 * \param userExecKey Custom execution key provided by user, or 0 to autogenerate one. 365 * \param filter Filter specification. NULL disables. Case sensitive. 366 * \param testIterations Number of iterations to run each test case. 367 * 368 * \returns Test run result; 0 when all tests passed, 1 if any tests failed. 369 */ 370 int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *userRunSeed, Uint64 userExecKey, const char *filter, int testIterations) 371 { 372 int totalNumberOfTests = 0; 373 int failedNumberOfTests = 0; 374 int suiteCounter; 375 int testCounter; 376 int iterationCounter; 377 SDLTest_TestSuiteReference *testSuite; 378 const SDLTest_TestCaseReference *testCase; 379 const char *runSeed = NULL; 380 char *currentSuiteName; 381 char *currentTestName; 382 Uint64 execKey; 383 float runStartSeconds; 384 float suiteStartSeconds; 385 float testStartSeconds; 386 float runEndSeconds; 387 float suiteEndSeconds; 388 float testEndSeconds; 389 float runtime; 390 int suiteFilter = 0; 391 char *suiteFilterName = NULL; 392 int testFilter = 0; 393 char *testFilterName = NULL; 394 SDL_bool forceTestRun = SDL_FALSE; 395 int testResult = 0; 396 int runResult = 0; 397 int totalTestFailedCount = 0; 398 int totalTestPassedCount = 0; 399 int totalTestSkippedCount = 0; 400 int testFailedCount = 0; 401 int testPassedCount = 0; 402 int testSkippedCount = 0; 403 int countSum = 0; 404 const SDLTest_TestCaseReference **failedTests; 405 406 /* Sanitize test iterations */ 407 if (testIterations < 1) { 408 testIterations = 1; 409 } 410 411 /* Generate run see if we don't have one already */ 412 if (userRunSeed == NULL || userRunSeed[0] == '\0') { 413 runSeed = SDLTest_GenerateRunSeed(16); 414 if (runSeed == NULL) { 415 SDLTest_LogError("Generating a random seed failed"); 416 return 2; 417 } 418 } else { 419 runSeed = userRunSeed; 420 } 421 422 423 /* Reset per-run counters */ 424 totalTestFailedCount = 0; 425 totalTestPassedCount = 0; 426 totalTestSkippedCount = 0; 427 428 /* Take time - run start */ 429 runStartSeconds = GetClock(); 430 431 /* Log run with fuzzer parameters */ 432 SDLTest_Log("::::: Test Run /w seed '%s' started\n", runSeed); 433 434 /* Count the total number of tests */ 435 suiteCounter = 0; 436 while (testSuites[suiteCounter]) { 437 testSuite = testSuites[suiteCounter]; 438 suiteCounter++; 439 testCounter = 0; 440 while (testSuite->testCases[testCounter]) 441 { 442 testCounter++; 443 totalNumberOfTests++; 444 } 445 } 446 447 /* Pre-allocate an array for tracking failed tests (potentially all test cases) */ 448 failedTests = (const SDLTest_TestCaseReference **)SDL_malloc(totalNumberOfTests * sizeof(SDLTest_TestCaseReference *)); 449 if (failedTests == NULL) { 450 SDLTest_LogError("Unable to allocate cache for failed tests"); 451 SDL_Error(SDL_ENOMEM); 452 return -1; 453 } 454 455 /* Initialize filtering */ 456 if (filter != NULL && filter[0] != '\0') { 457 /* Loop over all suites to check if we have a filter match */ 458 suiteCounter = 0; 459 while (testSuites[suiteCounter] && suiteFilter == 0) { 460 testSuite = testSuites[suiteCounter]; 461 suiteCounter++; 462 if (testSuite->name != NULL && SDL_strcmp(filter, testSuite->name) == 0) { 463 /* Matched a suite name */ 464 suiteFilter = 1; 465 suiteFilterName = testSuite->name; 466 SDLTest_Log("Filtering: running only suite '%s'", suiteFilterName); 467 break; 468 } 469 470 /* Within each suite, loop over all test cases to check if we have a filter match */ 471 testCounter = 0; 472 while (testSuite->testCases[testCounter] && testFilter == 0) 473 { 474 testCase = testSuite->testCases[testCounter]; 475 testCounter++; 476 if (testCase->name != NULL && SDL_strcmp(filter, testCase->name) == 0) { 477 /* Matched a test name */ 478 suiteFilter = 1; 479 suiteFilterName = testSuite->name; 480 testFilter = 1; 481 testFilterName = testCase->name; 482 SDLTest_Log("Filtering: running only test '%s' in suite '%s'", testFilterName, suiteFilterName); 483 break; 484 } 485 } 486 } 487 488 if (suiteFilter == 0 && testFilter == 0) { 489 SDLTest_LogError("Filter '%s' did not match any test suite/case.", filter); 490 SDLTest_Log("Exit code: 2"); 491 SDL_free((void *) failedTests); 492 return 2; 493 } 494 } 495 496 /* Loop over all suites */ 497 suiteCounter = 0; 498 while(testSuites[suiteCounter]) { 499 testSuite = testSuites[suiteCounter]; 500 currentSuiteName = (testSuite->name ? testSuite->name : SDLTEST_INVALID_NAME_FORMAT); 501 suiteCounter++; 502 503 /* Filter suite if flag set and we have a name */ 504 if (suiteFilter == 1 && suiteFilterName != NULL && testSuite->name != NULL && 505 SDL_strcmp(suiteFilterName, testSuite->name) != 0) { 506 /* Skip suite */ 507 SDLTest_Log("===== Test Suite %i: '%s' skipped\n", 508 suiteCounter, 509 currentSuiteName); 510 } else { 511 512 /* Reset per-suite counters */ 513 testFailedCount = 0; 514 testPassedCount = 0; 515 testSkippedCount = 0; 516 517 /* Take time - suite start */ 518 suiteStartSeconds = GetClock(); 519 520 /* Log suite started */ 521 SDLTest_Log("===== Test Suite %i: '%s' started\n", 522 suiteCounter, 523 currentSuiteName); 524 525 /* Loop over all test cases */ 526 testCounter = 0; 527 while(testSuite->testCases[testCounter]) 528 { 529 testCase = testSuite->testCases[testCounter]; 530 currentTestName = (testCase->name ? testCase->name : SDLTEST_INVALID_NAME_FORMAT); 531 testCounter++; 532 533 /* Filter tests if flag set and we have a name */ 534 if (testFilter == 1 && testFilterName != NULL && testCase->name != NULL && 535 SDL_strcmp(testFilterName, testCase->name) != 0) { 536 /* Skip test */ 537 SDLTest_Log("===== Test Case %i.%i: '%s' skipped\n", 538 suiteCounter, 539 testCounter, 540 currentTestName); 541 } else { 542 /* Override 'disabled' flag if we specified a test filter (i.e. force run for debugging) */ 543 if (testFilter == 1 && !testCase->enabled) { 544 SDLTest_Log("Force run of disabled test since test filter was set"); 545 forceTestRun = SDL_TRUE; 546 } 547 548 /* Take time - test start */ 549 testStartSeconds = GetClock(); 550 551 /* Log test started */ 552 SDLTest_Log("----- Test Case %i.%i: '%s' started", 553 suiteCounter, 554 testCounter, 555 currentTestName); 556 if (testCase->description != NULL && testCase->description[0] != '\0') { 557 SDLTest_Log("Test Description: '%s'", 558 (testCase->description) ? testCase->description : SDLTEST_INVALID_NAME_FORMAT); 559 } 560 561 /* Loop over all iterations */ 562 iterationCounter = 0; 563 while(iterationCounter < testIterations) 564 { 565 iterationCounter++; 566 567 if (userExecKey != 0) { 568 execKey = userExecKey; 569 } else { 570 execKey = SDLTest_GenerateExecKey(runSeed, testSuite->name, testCase->name, iterationCounter); 571 } 572 573 SDLTest_Log("Test Iteration %i: execKey %" SDL_PRIu64, iterationCounter, execKey); 574 testResult = SDLTest_RunTest(testSuite, testCase, execKey, forceTestRun); 575 576 if (testResult == TEST_RESULT_PASSED) { 577 testPassedCount++; 578 totalTestPassedCount++; 579 } else if (testResult == TEST_RESULT_SKIPPED) { 580 testSkippedCount++; 581 totalTestSkippedCount++; 582 } else { 583 testFailedCount++; 584 totalTestFailedCount++; 585 } 586 } 587 588 /* Take time - test end */ 589 testEndSeconds = GetClock(); 590 runtime = testEndSeconds - testStartSeconds; 591 if (runtime < 0.0f) runtime = 0.0f; 592 593 if (testIterations > 1) { 594 /* Log test runtime */ 595 SDLTest_Log("Runtime of %i iterations: %.1f sec", testIterations, runtime); 596 SDLTest_Log("Average Test runtime: %.5f sec", runtime / (float)testIterations); 597 } else { 598 /* Log test runtime */ 599 SDLTest_Log("Total Test runtime: %.1f sec", runtime); 600 } 601 602 /* Log final test result */ 603 switch (testResult) { 604 case TEST_RESULT_PASSED: 605 SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Test", currentTestName, "Passed"); 606 break; 607 case TEST_RESULT_FAILED: 608 SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Test", currentTestName, "Failed"); 609 break; 610 case TEST_RESULT_NO_ASSERT: 611 SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT,"Test", currentTestName, "No Asserts"); 612 break; 613 } 614 615 /* Collect failed test case references for repro-step display */ 616 if (testResult == TEST_RESULT_FAILED) { 617 failedTests[failedNumberOfTests] = testCase; 618 failedNumberOfTests++; 619 } 620 } 621 } 622 623 /* Take time - suite end */ 624 suiteEndSeconds = GetClock(); 625 runtime = suiteEndSeconds - suiteStartSeconds; 626 if (runtime < 0.0f) runtime = 0.0f; 627 628 /* Log suite runtime */ 629 SDLTest_Log("Total Suite runtime: %.1f sec", runtime); 630 631 /* Log summary and final Suite result */ 632 countSum = testPassedCount + testFailedCount + testSkippedCount; 633 if (testFailedCount == 0) 634 { 635 SDLTest_Log(SDLTEST_LOG_SUMMARY_FORMAT, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount); 636 SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Suite", currentSuiteName, "Passed"); 637 } 638 else 639 { 640 SDLTest_LogError(SDLTEST_LOG_SUMMARY_FORMAT, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount); 641 SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Suite", currentSuiteName, "Failed"); 642 } 643 644 } 645 } 646 647 /* Take time - run end */ 648 runEndSeconds = GetClock(); 649 runtime = runEndSeconds - runStartSeconds; 650 if (runtime < 0.0f) runtime = 0.0f; 651 652 /* Log total runtime */ 653 SDLTest_Log("Total Run runtime: %.1f sec", runtime); 654 655 /* Log summary and final run result */ 656 countSum = totalTestPassedCount + totalTestFailedCount + totalTestSkippedCount; 657 if (totalTestFailedCount == 0) 658 { 659 runResult = 0; 660 SDLTest_Log(SDLTEST_LOG_SUMMARY_FORMAT, "Run", countSum, totalTestPassedCount, totalTestFailedCount, totalTestSkippedCount); 661 SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Run /w seed", runSeed, "Passed"); 662 } 663 else 664 { 665 runResult = 1; 666 SDLTest_LogError(SDLTEST_LOG_SUMMARY_FORMAT, "Run", countSum, totalTestPassedCount, totalTestFailedCount, totalTestSkippedCount); 667 SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Run /w seed", runSeed, "Failed"); 668 } 669 670 /* Print repro steps for failed tests */ 671 if (failedNumberOfTests > 0) { 672 SDLTest_Log("Harness input to repro failures:"); 673 for (testCounter = 0; testCounter < failedNumberOfTests; testCounter++) { 674 SDLTest_Log(" --seed %s --filter %s", runSeed, failedTests[testCounter]->name); 675 } 676 } 677 SDL_free((void *) failedTests); 678 679 SDLTest_Log("Exit code: %d", runResult); 680 return runResult; 681 } 682 683 /* vi: set ts=4 sw=4 expandtab: */