testharness.c (15599B)
1 /* See COPYING.txt for the full license governing this code. */ 2 /** 3 * \file testharness.c 4 * 5 * Source file for the test harness. 6 */ 7 8 #include <stdlib.h> 9 #include <SDL_test.h> 10 #include <SDL.h> 11 #include <SDL_assert.h> 12 #include "SDL_visualtest_harness_argparser.h" 13 #include "SDL_visualtest_process.h" 14 #include "SDL_visualtest_variators.h" 15 #include "SDL_visualtest_screenshot.h" 16 #include "SDL_visualtest_mischelper.h" 17 18 #if defined(__WIN32__) && !defined(__CYGWIN__) 19 #include <direct.h> 20 #elif defined(__WIN32__) && defined(__CYGWIN__) 21 #include <signal.h> 22 #elif defined(__LINUX__) 23 #include <sys/stat.h> 24 #include <sys/types.h> 25 #include <signal.h> 26 #else 27 #error "Unsupported platform" 28 #endif 29 30 /** Code for the user event triggered when a new action is to be executed */ 31 #define ACTION_TIMER_EVENT 0 32 /** Code for the user event triggered when the maximum timeout is reached */ 33 #define KILL_TIMER_EVENT 1 34 /** FPS value used for delays in the action loop */ 35 #define ACTION_LOOP_FPS 10 36 37 /** Value returned by RunSUTAndTest() when the test has passed */ 38 #define TEST_PASSED 1 39 /** Value returned by RunSUTAndTest() when the test has failed */ 40 #define TEST_FAILED 0 41 /** Value returned by RunSUTAndTest() on a fatal error */ 42 #define TEST_ERROR -1 43 44 static SDL_ProcessInfo pinfo; 45 static SDL_ProcessExitStatus sut_exitstatus; 46 static SDLVisualTest_HarnessState state; 47 static SDLVisualTest_Variator variator; 48 static SDLVisualTest_ActionNode* current; /* the current action being performed */ 49 static SDL_TimerID action_timer, kill_timer; 50 51 /* returns a char* to be passed as the format argument of a printf-style function. */ 52 static char* 53 usage() 54 { 55 return "Usage: \n%s --sutapp xyz" 56 " [--sutargs abc | --parameter-config xyz.parameters" 57 " [--variator exhaustive|random]" 58 " [--num-variations N] [--no-launch]] [--timeout hh:mm:ss]" 59 " [--action-config xyz.actions]" 60 " [--output-dir /path/to/output]" 61 " [--verify-dir /path/to/verify]" 62 " or --config app.config"; 63 } 64 65 /* register Ctrl+C handlers */ 66 #if defined(__LINUX__) || defined(__CYGWIN__) 67 static void 68 CtrlCHandlerCallback(int signum) 69 { 70 SDL_Event event; 71 SDLTest_Log("Ctrl+C received"); 72 event.type = SDL_QUIT; 73 SDL_PushEvent(&event); 74 } 75 #endif 76 77 static Uint32 78 ActionTimerCallback(Uint32 interval, void* param) 79 { 80 SDL_Event event; 81 SDL_UserEvent userevent; 82 Uint32 next_action_time; 83 84 /* push an event to handle the action */ 85 userevent.type = SDL_USEREVENT; 86 userevent.code = ACTION_TIMER_EVENT; 87 userevent.data1 = ¤t->action; 88 userevent.data2 = NULL; 89 90 event.type = SDL_USEREVENT; 91 event.user = userevent; 92 SDL_PushEvent(&event); 93 94 /* calculate the new interval and return it */ 95 if(current->next) 96 next_action_time = current->next->action.time - current->action.time; 97 else 98 { 99 next_action_time = 0; 100 action_timer = 0; 101 } 102 103 current = current->next; 104 return next_action_time; 105 } 106 107 static Uint32 108 KillTimerCallback(Uint32 interval, void* param) 109 { 110 SDL_Event event; 111 SDL_UserEvent userevent; 112 113 userevent.type = SDL_USEREVENT; 114 userevent.code = KILL_TIMER_EVENT; 115 userevent.data1 = NULL; 116 userevent.data2 = NULL; 117 118 event.type = SDL_USEREVENT; 119 event.user = userevent; 120 SDL_PushEvent(&event); 121 122 kill_timer = 0; 123 return 0; 124 } 125 126 static int 127 ProcessAction(SDLVisualTest_Action* action, int* sut_running, char* args) 128 { 129 if(!action || !sut_running) 130 return TEST_ERROR; 131 132 switch(action->type) 133 { 134 case SDL_ACTION_KILL: 135 SDLTest_Log("Action: Kill SUT"); 136 if(SDL_IsProcessRunning(&pinfo) == 1 && 137 !SDL_KillProcess(&pinfo, &sut_exitstatus)) 138 { 139 SDLTest_LogError("SDL_KillProcess() failed"); 140 return TEST_ERROR; 141 } 142 *sut_running = 0; 143 break; 144 145 case SDL_ACTION_QUIT: 146 SDLTest_Log("Action: Quit SUT"); 147 if(SDL_IsProcessRunning(&pinfo) == 1 && 148 !SDL_QuitProcess(&pinfo, &sut_exitstatus)) 149 { 150 SDLTest_LogError("SDL_QuitProcess() failed"); 151 return TEST_FAILED; 152 } 153 *sut_running = 0; 154 break; 155 156 case SDL_ACTION_LAUNCH: 157 { 158 char* path; 159 char* args; 160 SDL_ProcessInfo action_process; 161 SDL_ProcessExitStatus ps; 162 163 path = action->extra.process.path; 164 args = action->extra.process.args; 165 if(args) 166 { 167 SDLTest_Log("Action: Launch process: %s with arguments: %s", 168 path, args); 169 } 170 else 171 SDLTest_Log("Action: Launch process: %s", path); 172 if(!SDL_LaunchProcess(path, args, &action_process)) 173 { 174 SDLTest_LogError("SDL_LaunchProcess() failed"); 175 return TEST_ERROR; 176 } 177 178 /* small delay so that the process can do its job */ 179 SDL_Delay(1000); 180 181 if(SDL_IsProcessRunning(&action_process) > 0) 182 { 183 SDLTest_LogError("Process %s took too long too complete." 184 " Force killing...", action->extra); 185 if(!SDL_KillProcess(&action_process, &ps)) 186 { 187 SDLTest_LogError("SDL_KillProcess() failed"); 188 return TEST_ERROR; 189 } 190 } 191 } 192 break; 193 194 case SDL_ACTION_SCREENSHOT: 195 { 196 char path[MAX_PATH_LEN], hash[33]; 197 198 SDLTest_Log("Action: Take screenshot"); 199 /* can't take a screenshot if the SUT isn't running */ 200 if(SDL_IsProcessRunning(&pinfo) != 1) 201 { 202 SDLTest_LogError("SUT has quit."); 203 *sut_running = 0; 204 return TEST_FAILED; 205 } 206 207 /* file name for the screenshot image */ 208 SDLVisualTest_HashString(args, hash); 209 SDL_snprintf(path, MAX_PATH_LEN, "%s/%s", state.output_dir, hash); 210 if(!SDLVisualTest_ScreenshotProcess(&pinfo, path)) 211 { 212 SDLTest_LogError("SDLVisualTest_ScreenshotProcess() failed"); 213 return TEST_ERROR; 214 } 215 } 216 break; 217 218 case SDL_ACTION_VERIFY: 219 { 220 int ret; 221 222 SDLTest_Log("Action: Verify screenshot"); 223 ret = SDLVisualTest_VerifyScreenshots(args, state.output_dir, 224 state.verify_dir); 225 226 if(ret == -1) 227 { 228 SDLTest_LogError("SDLVisualTest_VerifyScreenshots() failed"); 229 return TEST_ERROR; 230 } 231 else if(ret == 0) 232 { 233 SDLTest_Log("Verification failed: Images were not equal."); 234 return TEST_FAILED; 235 } 236 else if(ret == 1) 237 SDLTest_Log("Verification successful."); 238 else 239 { 240 SDLTest_Log("Verfication skipped."); 241 return TEST_FAILED; 242 } 243 } 244 break; 245 246 default: 247 SDLTest_LogError("Invalid action type"); 248 return TEST_ERROR; 249 break; 250 } 251 252 return TEST_PASSED; 253 } 254 255 static int 256 RunSUTAndTest(char* sutargs, int variation_num) 257 { 258 int success, sut_running, return_code; 259 char hash[33]; 260 SDL_Event event; 261 262 return_code = TEST_PASSED; 263 264 if(!sutargs) 265 { 266 SDLTest_LogError("sutargs argument cannot be NULL"); 267 return_code = TEST_ERROR; 268 goto runsutandtest_cleanup_generic; 269 } 270 271 SDLVisualTest_HashString(sutargs, hash); 272 SDLTest_Log("Hash: %s", hash); 273 274 success = SDL_LaunchProcess(state.sutapp, sutargs, &pinfo); 275 if(!success) 276 { 277 SDLTest_Log("Could not launch SUT."); 278 return_code = TEST_ERROR; 279 goto runsutandtest_cleanup_generic; 280 } 281 SDLTest_Log("SUT launch successful."); 282 SDLTest_Log("Process will be killed in %d milliseconds", state.timeout); 283 sut_running = 1; 284 285 /* launch the timers */ 286 SDLTest_Log("Performing actions.."); 287 current = state.action_queue.front; 288 action_timer = 0; 289 kill_timer = 0; 290 if(current) 291 { 292 action_timer = SDL_AddTimer(current->action.time, ActionTimerCallback, NULL); 293 if(!action_timer) 294 { 295 SDLTest_LogError("SDL_AddTimer() failed"); 296 return_code = TEST_ERROR; 297 goto runsutandtest_cleanup_timer; 298 } 299 } 300 kill_timer = SDL_AddTimer(state.timeout, KillTimerCallback, NULL); 301 if(!kill_timer) 302 { 303 SDLTest_LogError("SDL_AddTimer() failed"); 304 return_code = TEST_ERROR; 305 goto runsutandtest_cleanup_timer; 306 } 307 308 /* the timer stops running if the actions queue is empty, and the 309 SUT stops running if it crashes or if we encounter a KILL/QUIT action */ 310 while(sut_running) 311 { 312 /* process the actions by using an event queue */ 313 while(SDL_PollEvent(&event)) 314 { 315 if(event.type == SDL_USEREVENT) 316 { 317 if(event.user.code == ACTION_TIMER_EVENT) 318 { 319 SDLVisualTest_Action* action; 320 321 action = (SDLVisualTest_Action*)event.user.data1; 322 323 switch(ProcessAction(action, &sut_running, sutargs)) 324 { 325 case TEST_PASSED: 326 break; 327 328 case TEST_FAILED: 329 return_code = TEST_FAILED; 330 goto runsutandtest_cleanup_timer; 331 break; 332 333 default: 334 SDLTest_LogError("ProcessAction() failed"); 335 return_code = TEST_ERROR; 336 goto runsutandtest_cleanup_timer; 337 } 338 } 339 else if(event.user.code == KILL_TIMER_EVENT) 340 { 341 SDLTest_LogError("Maximum timeout reached. Force killing.."); 342 return_code = TEST_FAILED; 343 goto runsutandtest_cleanup_timer; 344 } 345 } 346 else if(event.type == SDL_QUIT) 347 { 348 SDLTest_LogError("Received QUIT event. Testharness is quitting.."); 349 return_code = TEST_ERROR; 350 goto runsutandtest_cleanup_timer; 351 } 352 } 353 SDL_Delay(1000/ACTION_LOOP_FPS); 354 } 355 356 SDLTest_Log("SUT exit code was: %d", sut_exitstatus.exit_status); 357 if(sut_exitstatus.exit_status == 0) 358 { 359 return_code = TEST_PASSED; 360 goto runsutandtest_cleanup_timer; 361 } 362 else 363 { 364 return_code = TEST_FAILED; 365 goto runsutandtest_cleanup_timer; 366 } 367 368 return_code = TEST_ERROR; 369 goto runsutandtest_cleanup_generic; 370 371 runsutandtest_cleanup_timer: 372 if(action_timer && !SDL_RemoveTimer(action_timer)) 373 { 374 SDLTest_Log("SDL_RemoveTimer() failed"); 375 return_code = TEST_ERROR; 376 } 377 378 if(kill_timer && !SDL_RemoveTimer(kill_timer)) 379 { 380 SDLTest_Log("SDL_RemoveTimer() failed"); 381 return_code = TEST_ERROR; 382 } 383 /* runsutandtest_cleanup_process: */ 384 if(SDL_IsProcessRunning(&pinfo) && !SDL_KillProcess(&pinfo, &sut_exitstatus)) 385 { 386 SDLTest_Log("SDL_KillProcess() failed"); 387 return_code = TEST_ERROR; 388 } 389 runsutandtest_cleanup_generic: 390 return return_code; 391 } 392 393 /** Entry point for testharness */ 394 int 395 main(int argc, char* argv[]) 396 { 397 int i, passed, return_code, failed; 398 399 /* freeing resources, linux style! */ 400 return_code = 0; 401 402 if(argc < 2) 403 { 404 SDLTest_Log(usage(), argv[0]); 405 goto cleanup_generic; 406 } 407 408 #if defined(__LINUX__) || defined(__CYGWIN__) 409 signal(SIGINT, CtrlCHandlerCallback); 410 #endif 411 412 /* parse arguments */ 413 if(!SDLVisualTest_ParseHarnessArgs(argv + 1, &state)) 414 { 415 SDLTest_Log(usage(), argv[0]); 416 return_code = 1; 417 goto cleanup_generic; 418 } 419 SDLTest_Log("Parsed harness arguments successfully."); 420 421 /* initialize SDL */ 422 if(SDL_Init(SDL_INIT_TIMER) == -1) 423 { 424 SDLTest_LogError("SDL_Init() failed."); 425 SDLVisualTest_FreeHarnessState(&state); 426 return_code = 1; 427 goto cleanup_harness_state; 428 } 429 430 /* create an output directory if none exists */ 431 #if defined(__LINUX__) || defined(__CYGWIN__) 432 mkdir(state.output_dir, 0777); 433 #elif defined(__WIN32__) 434 _mkdir(state.output_dir); 435 #else 436 #error "Unsupported platform" 437 #endif 438 439 /* test with sutargs */ 440 if(SDL_strlen(state.sutargs)) 441 { 442 SDLTest_Log("Running: %s %s", state.sutapp, state.sutargs); 443 if(!state.no_launch) 444 { 445 switch(RunSUTAndTest(state.sutargs, 0)) 446 { 447 case TEST_PASSED: 448 SDLTest_Log("Status: PASSED"); 449 break; 450 451 case TEST_FAILED: 452 SDLTest_Log("Status: FAILED"); 453 break; 454 455 case TEST_ERROR: 456 SDLTest_LogError("Some error occurred while testing."); 457 return_code = 1; 458 goto cleanup_sdl; 459 break; 460 } 461 } 462 } 463 464 if(state.sut_config.num_options > 0) 465 { 466 char* variator_name = state.variator_type == SDL_VARIATOR_RANDOM ? 467 "RANDOM" : "EXHAUSTIVE"; 468 if(state.num_variations > 0) 469 SDLTest_Log("Testing SUT with variator: %s for %d variations", 470 variator_name, state.num_variations); 471 else 472 SDLTest_Log("Testing SUT with variator: %s and ALL variations", 473 variator_name); 474 /* initialize the variator */ 475 if(!SDLVisualTest_InitVariator(&variator, &state.sut_config, 476 state.variator_type, 0)) 477 { 478 SDLTest_LogError("Could not initialize variator"); 479 return_code = 1; 480 goto cleanup_sdl; 481 } 482 483 /* iterate through all the variations */ 484 passed = 0; 485 failed = 0; 486 for(i = 0; state.num_variations > 0 ? (i < state.num_variations) : 1; i++) 487 { 488 char* args = SDLVisualTest_GetNextVariation(&variator); 489 if(!args) 490 break; 491 SDLTest_Log("\nVariation number: %d\nArguments: %s", i + 1, args); 492 493 if(!state.no_launch) 494 { 495 switch(RunSUTAndTest(args, i + 1)) 496 { 497 case TEST_PASSED: 498 SDLTest_Log("Status: PASSED"); 499 passed++; 500 break; 501 502 case TEST_FAILED: 503 SDLTest_Log("Status: FAILED"); 504 failed++; 505 break; 506 507 case TEST_ERROR: 508 SDLTest_LogError("Some error occurred while testing."); 509 goto cleanup_variator; 510 break; 511 } 512 } 513 } 514 if(!state.no_launch) 515 { 516 /* report stats */ 517 SDLTest_Log("Testing complete."); 518 SDLTest_Log("%d/%d tests passed.", passed, passed + failed); 519 } 520 goto cleanup_variator; 521 } 522 523 goto cleanup_sdl; 524 525 cleanup_variator: 526 SDLVisualTest_FreeVariator(&variator); 527 cleanup_sdl: 528 SDL_Quit(); 529 cleanup_harness_state: 530 SDLVisualTest_FreeHarnessState(&state); 531 cleanup_generic: 532 return return_code; 533 }