lj_trace.c (25968B)
1 /* 2 ** Trace management. 3 ** Copyright (C) 2005-2016 Mike Pall. See Copyright Notice in luajit.h 4 */ 5 6 #define lj_trace_c 7 #define LUA_CORE 8 9 #include "lj_obj.h" 10 11 #if LJ_HASJIT 12 13 #include "lj_gc.h" 14 #include "lj_err.h" 15 #include "lj_debug.h" 16 #include "lj_str.h" 17 #include "lj_frame.h" 18 #include "lj_state.h" 19 #include "lj_bc.h" 20 #include "lj_ir.h" 21 #include "lj_jit.h" 22 #include "lj_iropt.h" 23 #include "lj_mcode.h" 24 #include "lj_trace.h" 25 #include "lj_snap.h" 26 #include "lj_gdbjit.h" 27 #include "lj_record.h" 28 #include "lj_asm.h" 29 #include "lj_dispatch.h" 30 #include "lj_vm.h" 31 #include "lj_vmevent.h" 32 #include "lj_target.h" 33 34 /* -- Error handling ------------------------------------------------------ */ 35 36 /* Synchronous abort with error message. */ 37 void lj_trace_err(jit_State *J, TraceError e) 38 { 39 setnilV(&J->errinfo); /* No error info. */ 40 setintV(J->L->top++, (int32_t)e); 41 lj_err_throw(J->L, LUA_ERRRUN); 42 } 43 44 /* Synchronous abort with error message and error info. */ 45 void lj_trace_err_info(jit_State *J, TraceError e) 46 { 47 setintV(J->L->top++, (int32_t)e); 48 lj_err_throw(J->L, LUA_ERRRUN); 49 } 50 51 /* -- Trace management ---------------------------------------------------- */ 52 53 /* The current trace is first assembled in J->cur. The variable length 54 ** arrays point to shared, growable buffers (J->irbuf etc.). When trace 55 ** recording ends successfully, the current trace and its data structures 56 ** are copied to a new (compact) GCtrace object. 57 */ 58 59 /* Find a free trace number. */ 60 static TraceNo trace_findfree(jit_State *J) 61 { 62 MSize osz, lim; 63 if (J->freetrace == 0) 64 J->freetrace = 1; 65 for (; J->freetrace < J->sizetrace; J->freetrace++) 66 if (traceref(J, J->freetrace) == NULL) 67 return J->freetrace++; 68 /* Need to grow trace array. */ 69 lim = (MSize)J->param[JIT_P_maxtrace] + 1; 70 if (lim < 2) lim = 2; else if (lim > 65535) lim = 65535; 71 osz = J->sizetrace; 72 if (osz >= lim) 73 return 0; /* Too many traces. */ 74 lj_mem_growvec(J->L, J->trace, J->sizetrace, lim, GCRef); 75 for (; osz < J->sizetrace; osz++) 76 setgcrefnull(J->trace[osz]); 77 return J->freetrace; 78 } 79 80 #define TRACE_APPENDVEC(field, szfield, tp) \ 81 T->field = (tp *)p; \ 82 memcpy(p, J->cur.field, J->cur.szfield*sizeof(tp)); \ 83 p += J->cur.szfield*sizeof(tp); 84 85 #ifdef LUAJIT_USE_PERFTOOLS 86 /* 87 ** Create symbol table of JIT-compiled code. For use with Linux perf tools. 88 ** Example usage: 89 ** perf record -f -e cycles luajit test.lua 90 ** perf report -s symbol 91 ** rm perf.data /tmp/perf-*.map 92 */ 93 #include <stdio.h> 94 #include <unistd.h> 95 96 static void perftools_addtrace(GCtrace *T) 97 { 98 static FILE *fp; 99 GCproto *pt = &gcref(T->startpt)->pt; 100 const BCIns *startpc = mref(T->startpc, const BCIns); 101 const char *name = proto_chunknamestr(pt); 102 BCLine lineno; 103 if (name[0] == '@' || name[0] == '=') 104 name++; 105 else 106 name = "(string)"; 107 lua_assert(startpc >= proto_bc(pt) && startpc < proto_bc(pt) + pt->sizebc); 108 lineno = lj_debug_line(pt, proto_bcpos(pt, startpc)); 109 if (!fp) { 110 char fname[40]; 111 sprintf(fname, "/tmp/perf-%d.map", getpid()); 112 if (!(fp = fopen(fname, "w"))) return; 113 setlinebuf(fp); 114 } 115 fprintf(fp, "%lx %x TRACE_%d::%s:%u\n", 116 (long)T->mcode, T->szmcode, T->traceno, name, lineno); 117 } 118 #endif 119 120 /* Allocate space for copy of T. */ 121 GCtrace * LJ_FASTCALL lj_trace_alloc(lua_State *L, GCtrace *T) 122 { 123 size_t sztr = ((sizeof(GCtrace)+7)&~7); 124 size_t szins = (T->nins-T->nk)*sizeof(IRIns); 125 size_t sz = sztr + szins + 126 T->nsnap*sizeof(SnapShot) + 127 T->nsnapmap*sizeof(SnapEntry); 128 GCtrace *T2 = lj_mem_newt(L, (MSize)sz, GCtrace); 129 char *p = (char *)T2 + sztr; 130 T2->gct = ~LJ_TTRACE; 131 T2->marked = 0; 132 T2->traceno = 0; 133 T2->ir = (IRIns *)p - T->nk; 134 T2->nins = T->nins; 135 T2->nk = T->nk; 136 T2->nsnap = T->nsnap; 137 T2->nsnapmap = T->nsnapmap; 138 memcpy(p, T->ir + T->nk, szins); 139 return T2; 140 } 141 142 /* Save current trace by copying and compacting it. */ 143 static void trace_save(jit_State *J, GCtrace *T) 144 { 145 size_t sztr = ((sizeof(GCtrace)+7)&~7); 146 size_t szins = (J->cur.nins-J->cur.nk)*sizeof(IRIns); 147 char *p = (char *)T + sztr; 148 memcpy(T, &J->cur, sizeof(GCtrace)); 149 setgcrefr(T->nextgc, J2G(J)->gc.root); 150 setgcrefp(J2G(J)->gc.root, T); 151 newwhite(J2G(J), T); 152 T->gct = ~LJ_TTRACE; 153 T->ir = (IRIns *)p - J->cur.nk; /* The IR has already been copied above. */ 154 p += szins; 155 TRACE_APPENDVEC(snap, nsnap, SnapShot) 156 TRACE_APPENDVEC(snapmap, nsnapmap, SnapEntry) 157 J->cur.traceno = 0; 158 J->curfinal = NULL; 159 setgcrefp(J->trace[T->traceno], T); 160 lj_gc_barriertrace(J2G(J), T->traceno); 161 lj_gdbjit_addtrace(J, T); 162 #ifdef LUAJIT_USE_PERFTOOLS 163 perftools_addtrace(T); 164 #endif 165 } 166 167 void LJ_FASTCALL lj_trace_free(global_State *g, GCtrace *T) 168 { 169 jit_State *J = G2J(g); 170 if (T->traceno) { 171 lj_gdbjit_deltrace(J, T); 172 if (T->traceno < J->freetrace) 173 J->freetrace = T->traceno; 174 setgcrefnull(J->trace[T->traceno]); 175 } 176 lj_mem_free(g, T, 177 ((sizeof(GCtrace)+7)&~7) + (T->nins-T->nk)*sizeof(IRIns) + 178 T->nsnap*sizeof(SnapShot) + T->nsnapmap*sizeof(SnapEntry)); 179 } 180 181 /* Re-enable compiling a prototype by unpatching any modified bytecode. */ 182 void lj_trace_reenableproto(GCproto *pt) 183 { 184 if ((pt->flags & PROTO_ILOOP)) { 185 BCIns *bc = proto_bc(pt); 186 BCPos i, sizebc = pt->sizebc;; 187 pt->flags &= ~PROTO_ILOOP; 188 if (bc_op(bc[0]) == BC_IFUNCF) 189 setbc_op(&bc[0], BC_FUNCF); 190 for (i = 1; i < sizebc; i++) { 191 BCOp op = bc_op(bc[i]); 192 if (op == BC_IFORL || op == BC_IITERL || op == BC_ILOOP) 193 setbc_op(&bc[i], (int)op+(int)BC_LOOP-(int)BC_ILOOP); 194 } 195 } 196 } 197 198 /* Unpatch the bytecode modified by a root trace. */ 199 static void trace_unpatch(jit_State *J, GCtrace *T) 200 { 201 BCOp op = bc_op(T->startins); 202 BCIns *pc = mref(T->startpc, BCIns); 203 UNUSED(J); 204 if (op == BC_JMP) 205 return; /* No need to unpatch branches in parent traces (yet). */ 206 switch (bc_op(*pc)) { 207 case BC_JFORL: 208 lua_assert(traceref(J, bc_d(*pc)) == T); 209 *pc = T->startins; 210 pc += bc_j(T->startins); 211 lua_assert(bc_op(*pc) == BC_JFORI); 212 setbc_op(pc, BC_FORI); 213 break; 214 case BC_JITERL: 215 case BC_JLOOP: 216 lua_assert(op == BC_ITERL || op == BC_LOOP || bc_isret(op)); 217 *pc = T->startins; 218 break; 219 case BC_JMP: 220 lua_assert(op == BC_ITERL); 221 pc += bc_j(*pc)+2; 222 if (bc_op(*pc) == BC_JITERL) { 223 lua_assert(traceref(J, bc_d(*pc)) == T); 224 *pc = T->startins; 225 } 226 break; 227 case BC_JFUNCF: 228 lua_assert(op == BC_FUNCF); 229 *pc = T->startins; 230 break; 231 default: /* Already unpatched. */ 232 break; 233 } 234 } 235 236 /* Flush a root trace. */ 237 static void trace_flushroot(jit_State *J, GCtrace *T) 238 { 239 GCproto *pt = &gcref(T->startpt)->pt; 240 lua_assert(T->root == 0 && pt != NULL); 241 /* First unpatch any modified bytecode. */ 242 trace_unpatch(J, T); 243 /* Unlink root trace from chain anchored in prototype. */ 244 if (pt->trace == T->traceno) { /* Trace is first in chain. Easy. */ 245 pt->trace = T->nextroot; 246 } else if (pt->trace) { /* Otherwise search in chain of root traces. */ 247 GCtrace *T2 = traceref(J, pt->trace); 248 if (T2) { 249 for (; T2->nextroot; T2 = traceref(J, T2->nextroot)) 250 if (T2->nextroot == T->traceno) { 251 T2->nextroot = T->nextroot; /* Unlink from chain. */ 252 break; 253 } 254 } 255 } 256 } 257 258 /* Flush a trace. Only root traces are considered. */ 259 void lj_trace_flush(jit_State *J, TraceNo traceno) 260 { 261 if (traceno > 0 && traceno < J->sizetrace) { 262 GCtrace *T = traceref(J, traceno); 263 if (T && T->root == 0) 264 trace_flushroot(J, T); 265 } 266 } 267 268 /* Flush all traces associated with a prototype. */ 269 void lj_trace_flushproto(global_State *g, GCproto *pt) 270 { 271 while (pt->trace != 0) 272 trace_flushroot(G2J(g), traceref(G2J(g), pt->trace)); 273 } 274 275 /* Flush all traces. */ 276 int lj_trace_flushall(lua_State *L) 277 { 278 jit_State *J = L2J(L); 279 ptrdiff_t i; 280 if ((J2G(J)->hookmask & HOOK_GC)) 281 return 1; 282 for (i = (ptrdiff_t)J->sizetrace-1; i > 0; i--) { 283 GCtrace *T = traceref(J, i); 284 if (T) { 285 if (T->root == 0) 286 trace_flushroot(J, T); 287 lj_gdbjit_deltrace(J, T); 288 T->traceno = T->link = 0; /* Blacklist the link for cont_stitch. */ 289 setgcrefnull(J->trace[i]); 290 } 291 } 292 J->cur.traceno = 0; 293 J->freetrace = 0; 294 /* Clear penalty cache. */ 295 memset(J->penalty, 0, sizeof(J->penalty)); 296 /* Free the whole machine code and invalidate all exit stub groups. */ 297 lj_mcode_free(J); 298 memset(J->exitstubgroup, 0, sizeof(J->exitstubgroup)); 299 lj_vmevent_send(L, TRACE, 300 setstrV(L, L->top++, lj_str_newlit(L, "flush")); 301 ); 302 return 0; 303 } 304 305 /* Initialize JIT compiler state. */ 306 void lj_trace_initstate(global_State *g) 307 { 308 jit_State *J = G2J(g); 309 TValue *tv; 310 311 /* Initialize aligned SIMD constants. */ 312 tv = LJ_KSIMD(J, LJ_KSIMD_ABS); 313 tv[0].u64 = U64x(7fffffff,ffffffff); 314 tv[1].u64 = U64x(7fffffff,ffffffff); 315 tv = LJ_KSIMD(J, LJ_KSIMD_NEG); 316 tv[0].u64 = U64x(80000000,00000000); 317 tv[1].u64 = U64x(80000000,00000000); 318 319 /* Initialize 32/64 bit constants. */ 320 #if LJ_TARGET_X86ORX64 321 J->k64[LJ_K64_TOBIT].u64 = U64x(43380000,00000000); 322 J->k64[LJ_K64_2P64].u64 = U64x(43f00000,00000000); 323 J->k64[LJ_K64_M2P64].u64 = U64x(c3f00000,00000000); 324 #if LJ_32 325 J->k64[LJ_K64_M2P64_31].u64 = U64x(c1e00000,00000000); 326 #endif 327 J->k32[LJ_K32_M2P64_31] = LJ_64 ? 0xdf800000 : 0xcf000000; 328 #endif 329 #if LJ_TARGET_PPC 330 J->k32[LJ_K32_2P52_2P31] = 0x59800004; 331 J->k32[LJ_K32_2P52] = 0x59800000; 332 #endif 333 #if LJ_TARGET_PPC || LJ_TARGET_MIPS 334 J->k32[LJ_K32_2P31] = 0x4f000000; 335 #endif 336 #if LJ_TARGET_MIPS 337 J->k64[LJ_K64_2P31].u64 = U64x(41e00000,00000000); 338 #endif 339 } 340 341 /* Free everything associated with the JIT compiler state. */ 342 void lj_trace_freestate(global_State *g) 343 { 344 jit_State *J = G2J(g); 345 #ifdef LUA_USE_ASSERT 346 { /* This assumes all traces have already been freed. */ 347 ptrdiff_t i; 348 for (i = 1; i < (ptrdiff_t)J->sizetrace; i++) 349 lua_assert(i == (ptrdiff_t)J->cur.traceno || traceref(J, i) == NULL); 350 } 351 #endif 352 lj_mcode_free(J); 353 lj_mem_freevec(g, J->snapmapbuf, J->sizesnapmap, SnapEntry); 354 lj_mem_freevec(g, J->snapbuf, J->sizesnap, SnapShot); 355 lj_mem_freevec(g, J->irbuf + J->irbotlim, J->irtoplim - J->irbotlim, IRIns); 356 lj_mem_freevec(g, J->trace, J->sizetrace, GCRef); 357 } 358 359 /* -- Penalties and blacklisting ------------------------------------------ */ 360 361 /* Blacklist a bytecode instruction. */ 362 static void blacklist_pc(GCproto *pt, BCIns *pc) 363 { 364 setbc_op(pc, (int)bc_op(*pc)+(int)BC_ILOOP-(int)BC_LOOP); 365 pt->flags |= PROTO_ILOOP; 366 } 367 368 /* Penalize a bytecode instruction. */ 369 static void penalty_pc(jit_State *J, GCproto *pt, BCIns *pc, TraceError e) 370 { 371 uint32_t i, val = PENALTY_MIN; 372 for (i = 0; i < PENALTY_SLOTS; i++) 373 if (mref(J->penalty[i].pc, const BCIns) == pc) { /* Cache slot found? */ 374 /* First try to bump its hotcount several times. */ 375 val = ((uint32_t)J->penalty[i].val << 1) + 376 LJ_PRNG_BITS(J, PENALTY_RNDBITS); 377 if (val > PENALTY_MAX) { 378 blacklist_pc(pt, pc); /* Blacklist it, if that didn't help. */ 379 return; 380 } 381 goto setpenalty; 382 } 383 /* Assign a new penalty cache slot. */ 384 i = J->penaltyslot; 385 J->penaltyslot = (J->penaltyslot + 1) & (PENALTY_SLOTS-1); 386 setmref(J->penalty[i].pc, pc); 387 setpenalty: 388 J->penalty[i].val = (uint16_t)val; 389 J->penalty[i].reason = e; 390 hotcount_set(J2GG(J), pc+1, val); 391 } 392 393 /* -- Trace compiler state machine ---------------------------------------- */ 394 395 /* Start tracing. */ 396 static void trace_start(jit_State *J) 397 { 398 lua_State *L; 399 TraceNo traceno; 400 401 if ((J->pt->flags & PROTO_NOJIT)) { /* JIT disabled for this proto? */ 402 if (J->parent == 0 && J->exitno == 0) { 403 /* Lazy bytecode patching to disable hotcount events. */ 404 lua_assert(bc_op(*J->pc) == BC_FORL || bc_op(*J->pc) == BC_ITERL || 405 bc_op(*J->pc) == BC_LOOP || bc_op(*J->pc) == BC_FUNCF); 406 setbc_op(J->pc, (int)bc_op(*J->pc)+(int)BC_ILOOP-(int)BC_LOOP); 407 J->pt->flags |= PROTO_ILOOP; 408 } 409 J->state = LJ_TRACE_IDLE; /* Silently ignored. */ 410 return; 411 } 412 413 /* Get a new trace number. */ 414 traceno = trace_findfree(J); 415 if (LJ_UNLIKELY(traceno == 0)) { /* No free trace? */ 416 lua_assert((J2G(J)->hookmask & HOOK_GC) == 0); 417 lj_trace_flushall(J->L); 418 J->state = LJ_TRACE_IDLE; /* Silently ignored. */ 419 return; 420 } 421 setgcrefp(J->trace[traceno], &J->cur); 422 423 /* Setup enough of the current trace to be able to send the vmevent. */ 424 memset(&J->cur, 0, sizeof(GCtrace)); 425 J->cur.traceno = traceno; 426 J->cur.nins = J->cur.nk = REF_BASE; 427 J->cur.ir = J->irbuf; 428 J->cur.snap = J->snapbuf; 429 J->cur.snapmap = J->snapmapbuf; 430 J->mergesnap = 0; 431 J->needsnap = 0; 432 J->bcskip = 0; 433 J->guardemit.irt = 0; 434 J->postproc = LJ_POST_NONE; 435 lj_resetsplit(J); 436 J->retryrec = 0; 437 J->ktrace = 0; 438 setgcref(J->cur.startpt, obj2gco(J->pt)); 439 440 L = J->L; 441 lj_vmevent_send(L, TRACE, 442 setstrV(L, L->top++, lj_str_newlit(L, "start")); 443 setintV(L->top++, traceno); 444 setfuncV(L, L->top++, J->fn); 445 setintV(L->top++, proto_bcpos(J->pt, J->pc)); 446 if (J->parent) { 447 setintV(L->top++, J->parent); 448 setintV(L->top++, J->exitno); 449 } 450 ); 451 lj_record_setup(J); 452 } 453 454 /* Stop tracing. */ 455 static void trace_stop(jit_State *J) 456 { 457 BCIns *pc = mref(J->cur.startpc, BCIns); 458 BCOp op = bc_op(J->cur.startins); 459 GCproto *pt = &gcref(J->cur.startpt)->pt; 460 TraceNo traceno = J->cur.traceno; 461 GCtrace *T = J->curfinal; 462 lua_State *L; 463 464 switch (op) { 465 case BC_FORL: 466 setbc_op(pc+bc_j(J->cur.startins), BC_JFORI); /* Patch FORI, too. */ 467 /* fallthrough */ 468 case BC_LOOP: 469 case BC_ITERL: 470 case BC_FUNCF: 471 /* Patch bytecode of starting instruction in root trace. */ 472 setbc_op(pc, (int)op+(int)BC_JLOOP-(int)BC_LOOP); 473 setbc_d(pc, traceno); 474 addroot: 475 /* Add to root trace chain in prototype. */ 476 J->cur.nextroot = pt->trace; 477 pt->trace = (TraceNo1)traceno; 478 break; 479 case BC_RET: 480 case BC_RET0: 481 case BC_RET1: 482 *pc = BCINS_AD(BC_JLOOP, J->cur.snap[0].nslots, traceno); 483 goto addroot; 484 case BC_JMP: 485 /* Patch exit branch in parent to side trace entry. */ 486 lua_assert(J->parent != 0 && J->cur.root != 0); 487 lj_asm_patchexit(J, traceref(J, J->parent), J->exitno, J->cur.mcode); 488 /* Avoid compiling a side trace twice (stack resizing uses parent exit). */ 489 traceref(J, J->parent)->snap[J->exitno].count = SNAPCOUNT_DONE; 490 /* Add to side trace chain in root trace. */ 491 { 492 GCtrace *root = traceref(J, J->cur.root); 493 root->nchild++; 494 J->cur.nextside = root->nextside; 495 root->nextside = (TraceNo1)traceno; 496 } 497 break; 498 case BC_CALLM: 499 case BC_CALL: 500 case BC_ITERC: 501 /* Trace stitching: patch link of previous trace. */ 502 traceref(J, J->exitno)->link = traceno; 503 break; 504 default: 505 lua_assert(0); 506 break; 507 } 508 509 /* Commit new mcode only after all patching is done. */ 510 lj_mcode_commit(J, J->cur.mcode); 511 J->postproc = LJ_POST_NONE; 512 trace_save(J, T); 513 514 L = J->L; 515 lj_vmevent_send(L, TRACE, 516 setstrV(L, L->top++, lj_str_newlit(L, "stop")); 517 setintV(L->top++, traceno); 518 setfuncV(L, L->top++, J->fn); 519 ); 520 } 521 522 /* Start a new root trace for down-recursion. */ 523 static int trace_downrec(jit_State *J) 524 { 525 /* Restart recording at the return instruction. */ 526 lua_assert(J->pt != NULL); 527 lua_assert(bc_isret(bc_op(*J->pc))); 528 if (bc_op(*J->pc) == BC_RETM) 529 return 0; /* NYI: down-recursion with RETM. */ 530 J->parent = 0; 531 J->exitno = 0; 532 J->state = LJ_TRACE_RECORD; 533 trace_start(J); 534 return 1; 535 } 536 537 /* Abort tracing. */ 538 static int trace_abort(jit_State *J) 539 { 540 lua_State *L = J->L; 541 TraceError e = LJ_TRERR_RECERR; 542 TraceNo traceno; 543 544 J->postproc = LJ_POST_NONE; 545 lj_mcode_abort(J); 546 if (J->curfinal) { 547 lj_trace_free(J2G(J), J->curfinal); 548 J->curfinal = NULL; 549 } 550 if (tvisnumber(L->top-1)) 551 e = (TraceError)numberVint(L->top-1); 552 if (e == LJ_TRERR_MCODELM) { 553 L->top--; /* Remove error object */ 554 J->state = LJ_TRACE_ASM; 555 return 1; /* Retry ASM with new MCode area. */ 556 } 557 /* Penalize or blacklist starting bytecode instruction. */ 558 if (J->parent == 0 && !bc_isret(bc_op(J->cur.startins))) { 559 if (J->exitno == 0) { 560 BCIns *startpc = mref(J->cur.startpc, BCIns); 561 if (e == LJ_TRERR_RETRY) 562 hotcount_set(J2GG(J), startpc+1, 1); /* Immediate retry. */ 563 else 564 penalty_pc(J, &gcref(J->cur.startpt)->pt, startpc, e); 565 } else { 566 traceref(J, J->exitno)->link = J->exitno; /* Self-link is blacklisted. */ 567 } 568 } 569 570 /* Is there anything to abort? */ 571 traceno = J->cur.traceno; 572 if (traceno) { 573 ptrdiff_t errobj = savestack(L, L->top-1); /* Stack may be resized. */ 574 J->cur.link = 0; 575 J->cur.linktype = LJ_TRLINK_NONE; 576 lj_vmevent_send(L, TRACE, 577 TValue *frame; 578 const BCIns *pc; 579 GCfunc *fn; 580 setstrV(L, L->top++, lj_str_newlit(L, "abort")); 581 setintV(L->top++, traceno); 582 /* Find original Lua function call to generate a better error message. */ 583 frame = J->L->base-1; 584 pc = J->pc; 585 while (!isluafunc(frame_func(frame))) { 586 pc = (frame_iscont(frame) ? frame_contpc(frame) : frame_pc(frame)) - 1; 587 frame = frame_prev(frame); 588 } 589 fn = frame_func(frame); 590 setfuncV(L, L->top++, fn); 591 setintV(L->top++, proto_bcpos(funcproto(fn), pc)); 592 copyTV(L, L->top++, restorestack(L, errobj)); 593 copyTV(L, L->top++, &J->errinfo); 594 ); 595 /* Drop aborted trace after the vmevent (which may still access it). */ 596 setgcrefnull(J->trace[traceno]); 597 if (traceno < J->freetrace) 598 J->freetrace = traceno; 599 J->cur.traceno = 0; 600 } 601 L->top--; /* Remove error object */ 602 if (e == LJ_TRERR_DOWNREC) 603 return trace_downrec(J); 604 else if (e == LJ_TRERR_MCODEAL) 605 lj_trace_flushall(L); 606 return 0; 607 } 608 609 /* Perform pending re-patch of a bytecode instruction. */ 610 static LJ_AINLINE void trace_pendpatch(jit_State *J, int force) 611 { 612 if (LJ_UNLIKELY(J->patchpc)) { 613 if (force || J->bcskip == 0) { 614 *J->patchpc = J->patchins; 615 J->patchpc = NULL; 616 } else { 617 J->bcskip = 0; 618 } 619 } 620 } 621 622 /* State machine for the trace compiler. Protected callback. */ 623 static TValue *trace_state(lua_State *L, lua_CFunction dummy, void *ud) 624 { 625 jit_State *J = (jit_State *)ud; 626 UNUSED(dummy); 627 do { 628 retry: 629 switch (J->state) { 630 case LJ_TRACE_START: 631 J->state = LJ_TRACE_RECORD; /* trace_start() may change state. */ 632 trace_start(J); 633 lj_dispatch_update(J2G(J)); 634 break; 635 636 case LJ_TRACE_RECORD: 637 trace_pendpatch(J, 0); 638 setvmstate(J2G(J), RECORD); 639 lj_vmevent_send_(L, RECORD, 640 /* Save/restore tmptv state for trace recorder. */ 641 TValue savetv = J2G(J)->tmptv; 642 TValue savetv2 = J2G(J)->tmptv2; 643 setintV(L->top++, J->cur.traceno); 644 setfuncV(L, L->top++, J->fn); 645 setintV(L->top++, J->pt ? (int32_t)proto_bcpos(J->pt, J->pc) : -1); 646 setintV(L->top++, J->framedepth); 647 , 648 J2G(J)->tmptv = savetv; 649 J2G(J)->tmptv2 = savetv2; 650 ); 651 lj_record_ins(J); 652 break; 653 654 case LJ_TRACE_END: 655 trace_pendpatch(J, 1); 656 J->loopref = 0; 657 if ((J->flags & JIT_F_OPT_LOOP) && 658 J->cur.link == J->cur.traceno && J->framedepth + J->retdepth == 0) { 659 setvmstate(J2G(J), OPT); 660 lj_opt_dce(J); 661 if (lj_opt_loop(J)) { /* Loop optimization failed? */ 662 J->cur.link = 0; 663 J->cur.linktype = LJ_TRLINK_NONE; 664 J->loopref = J->cur.nins; 665 J->state = LJ_TRACE_RECORD; /* Try to continue recording. */ 666 break; 667 } 668 J->loopref = J->chain[IR_LOOP]; /* Needed by assembler. */ 669 } 670 lj_opt_split(J); 671 lj_opt_sink(J); 672 if (!J->loopref) J->cur.snap[J->cur.nsnap-1].count = SNAPCOUNT_DONE; 673 J->state = LJ_TRACE_ASM; 674 break; 675 676 case LJ_TRACE_ASM: 677 setvmstate(J2G(J), ASM); 678 lj_asm_trace(J, &J->cur); 679 trace_stop(J); 680 setvmstate(J2G(J), INTERP); 681 J->state = LJ_TRACE_IDLE; 682 lj_dispatch_update(J2G(J)); 683 return NULL; 684 685 default: /* Trace aborted asynchronously. */ 686 setintV(L->top++, (int32_t)LJ_TRERR_RECERR); 687 /* fallthrough */ 688 case LJ_TRACE_ERR: 689 trace_pendpatch(J, 1); 690 if (trace_abort(J)) 691 goto retry; 692 setvmstate(J2G(J), INTERP); 693 J->state = LJ_TRACE_IDLE; 694 lj_dispatch_update(J2G(J)); 695 return NULL; 696 } 697 } while (J->state > LJ_TRACE_RECORD); 698 return NULL; 699 } 700 701 /* -- Event handling ------------------------------------------------------ */ 702 703 /* A bytecode instruction is about to be executed. Record it. */ 704 void lj_trace_ins(jit_State *J, const BCIns *pc) 705 { 706 /* Note: J->L must already be set. pc is the true bytecode PC here. */ 707 J->pc = pc; 708 J->fn = curr_func(J->L); 709 J->pt = isluafunc(J->fn) ? funcproto(J->fn) : NULL; 710 while (lj_vm_cpcall(J->L, NULL, (void *)J, trace_state) != 0) 711 J->state = LJ_TRACE_ERR; 712 } 713 714 /* A hotcount triggered. Start recording a root trace. */ 715 void LJ_FASTCALL lj_trace_hot(jit_State *J, const BCIns *pc) 716 { 717 /* Note: pc is the interpreter bytecode PC here. It's offset by 1. */ 718 ERRNO_SAVE 719 /* Reset hotcount. */ 720 hotcount_set(J2GG(J), pc, J->param[JIT_P_hotloop]*HOTCOUNT_LOOP); 721 /* Only start a new trace if not recording or inside __gc call or vmevent. */ 722 if (J->state == LJ_TRACE_IDLE && 723 !(J2G(J)->hookmask & (HOOK_GC|HOOK_VMEVENT))) { 724 J->parent = 0; /* Root trace. */ 725 J->exitno = 0; 726 J->state = LJ_TRACE_START; 727 lj_trace_ins(J, pc-1); 728 } 729 ERRNO_RESTORE 730 } 731 732 /* Check for a hot side exit. If yes, start recording a side trace. */ 733 static void trace_hotside(jit_State *J, const BCIns *pc) 734 { 735 SnapShot *snap = &traceref(J, J->parent)->snap[J->exitno]; 736 if (!(J2G(J)->hookmask & (HOOK_GC|HOOK_VMEVENT)) && 737 isluafunc(curr_func(J->L)) && 738 snap->count != SNAPCOUNT_DONE && 739 ++snap->count >= J->param[JIT_P_hotexit]) { 740 lua_assert(J->state == LJ_TRACE_IDLE); 741 /* J->parent is non-zero for a side trace. */ 742 J->state = LJ_TRACE_START; 743 lj_trace_ins(J, pc); 744 } 745 } 746 747 /* Stitch a new trace to the previous trace. */ 748 void LJ_FASTCALL lj_trace_stitch(jit_State *J, const BCIns *pc) 749 { 750 /* Only start a new trace if not recording or inside __gc call or vmevent. */ 751 if (J->state == LJ_TRACE_IDLE && 752 !(J2G(J)->hookmask & (HOOK_GC|HOOK_VMEVENT))) { 753 J->parent = 0; /* Have to treat it like a root trace. */ 754 /* J->exitno is set to the invoking trace. */ 755 J->state = LJ_TRACE_START; 756 lj_trace_ins(J, pc); 757 } 758 } 759 760 761 /* Tiny struct to pass data to protected call. */ 762 typedef struct ExitDataCP { 763 jit_State *J; 764 void *exptr; /* Pointer to exit state. */ 765 const BCIns *pc; /* Restart interpreter at this PC. */ 766 } ExitDataCP; 767 768 /* Need to protect lj_snap_restore because it may throw. */ 769 static TValue *trace_exit_cp(lua_State *L, lua_CFunction dummy, void *ud) 770 { 771 ExitDataCP *exd = (ExitDataCP *)ud; 772 cframe_errfunc(L->cframe) = -1; /* Inherit error function. */ 773 exd->pc = lj_snap_restore(exd->J, exd->exptr); 774 UNUSED(dummy); 775 return NULL; 776 } 777 778 #ifndef LUAJIT_DISABLE_VMEVENT 779 /* Push all registers from exit state. */ 780 static void trace_exit_regs(lua_State *L, ExitState *ex) 781 { 782 int32_t i; 783 setintV(L->top++, RID_NUM_GPR); 784 setintV(L->top++, RID_NUM_FPR); 785 for (i = 0; i < RID_NUM_GPR; i++) { 786 if (sizeof(ex->gpr[i]) == sizeof(int32_t)) 787 setintV(L->top++, (int32_t)ex->gpr[i]); 788 else 789 setnumV(L->top++, (lua_Number)ex->gpr[i]); 790 } 791 #if !LJ_SOFTFP 792 for (i = 0; i < RID_NUM_FPR; i++) { 793 setnumV(L->top, ex->fpr[i]); 794 if (LJ_UNLIKELY(tvisnan(L->top))) 795 setnanV(L->top); 796 L->top++; 797 } 798 #endif 799 } 800 #endif 801 802 #ifdef EXITSTATE_PCREG 803 /* Determine trace number from pc of exit instruction. */ 804 static TraceNo trace_exit_find(jit_State *J, MCode *pc) 805 { 806 TraceNo traceno; 807 for (traceno = 1; traceno < J->sizetrace; traceno++) { 808 GCtrace *T = traceref(J, traceno); 809 if (T && pc >= T->mcode && pc < (MCode *)((char *)T->mcode + T->szmcode)) 810 return traceno; 811 } 812 lua_assert(0); 813 return 0; 814 } 815 #endif 816 817 /* A trace exited. Restore interpreter state. */ 818 int LJ_FASTCALL lj_trace_exit(jit_State *J, void *exptr) 819 { 820 ERRNO_SAVE 821 lua_State *L = J->L; 822 ExitState *ex = (ExitState *)exptr; 823 ExitDataCP exd; 824 int errcode; 825 const BCIns *pc; 826 void *cf; 827 GCtrace *T; 828 #ifdef EXITSTATE_PCREG 829 J->parent = trace_exit_find(J, (MCode *)(intptr_t)ex->gpr[EXITSTATE_PCREG]); 830 #endif 831 T = traceref(J, J->parent); UNUSED(T); 832 #ifdef EXITSTATE_CHECKEXIT 833 if (J->exitno == T->nsnap) { /* Treat stack check like a parent exit. */ 834 lua_assert(T->root != 0); 835 J->exitno = T->ir[REF_BASE].op2; 836 J->parent = T->ir[REF_BASE].op1; 837 T = traceref(J, J->parent); 838 } 839 #endif 840 lua_assert(T != NULL && J->exitno < T->nsnap); 841 exd.J = J; 842 exd.exptr = exptr; 843 errcode = lj_vm_cpcall(L, NULL, &exd, trace_exit_cp); 844 if (errcode) 845 return -errcode; /* Return negated error code. */ 846 847 if (!(LJ_HASPROFILE && (G(L)->hookmask & HOOK_PROFILE))) 848 lj_vmevent_send(L, TEXIT, 849 lj_state_checkstack(L, 4+RID_NUM_GPR+RID_NUM_FPR+LUA_MINSTACK); 850 setintV(L->top++, J->parent); 851 setintV(L->top++, J->exitno); 852 trace_exit_regs(L, ex); 853 ); 854 855 pc = exd.pc; 856 cf = cframe_raw(L->cframe); 857 setcframe_pc(cf, pc); 858 if (LJ_HASPROFILE && (G(L)->hookmask & HOOK_PROFILE)) { 859 /* Just exit to interpreter. */ 860 } else if (G(L)->gc.state == GCSatomic || G(L)->gc.state == GCSfinalize) { 861 if (!(G(L)->hookmask & HOOK_GC)) 862 lj_gc_step(L); /* Exited because of GC: drive GC forward. */ 863 } else { 864 trace_hotside(J, pc); 865 } 866 if (bc_op(*pc) == BC_JLOOP) { 867 BCIns *retpc = &traceref(J, bc_d(*pc))->startins; 868 if (bc_isret(bc_op(*retpc))) { 869 if (J->state == LJ_TRACE_RECORD) { 870 J->patchins = *pc; 871 J->patchpc = (BCIns *)pc; 872 *J->patchpc = *retpc; 873 J->bcskip = 1; 874 } else { 875 pc = retpc; 876 setcframe_pc(cf, pc); 877 } 878 } 879 } 880 /* Return MULTRES or 0. */ 881 ERRNO_RESTORE 882 switch (bc_op(*pc)) { 883 case BC_CALLM: case BC_CALLMT: 884 return (int)((BCReg)(L->top - L->base) - bc_a(*pc) - bc_c(*pc) + LJ_FR2); 885 case BC_RETM: 886 return (int)((BCReg)(L->top - L->base) + 1 - bc_a(*pc) - bc_d(*pc)); 887 case BC_TSETM: 888 return (int)((BCReg)(L->top - L->base) + 1 - bc_a(*pc)); 889 default: 890 if (bc_op(*pc) >= BC_FUNCF) 891 return (int)((BCReg)(L->top - L->base) + 1); 892 return 0; 893 } 894 } 895 896 #endif