lj_ir.c (12806B)
1 /* 2 ** SSA IR (Intermediate Representation) emitter. 3 ** Copyright (C) 2005-2016 Mike Pall. See Copyright Notice in luajit.h 4 */ 5 6 #define lj_ir_c 7 #define LUA_CORE 8 9 /* For pointers to libc/libm functions. */ 10 #include <stdio.h> 11 #include <math.h> 12 13 #include "lj_obj.h" 14 15 #if LJ_HASJIT 16 17 #include "lj_gc.h" 18 #include "lj_buf.h" 19 #include "lj_str.h" 20 #include "lj_tab.h" 21 #include "lj_ir.h" 22 #include "lj_jit.h" 23 #include "lj_ircall.h" 24 #include "lj_iropt.h" 25 #include "lj_trace.h" 26 #if LJ_HASFFI 27 #include "lj_ctype.h" 28 #include "lj_cdata.h" 29 #include "lj_carith.h" 30 #endif 31 #include "lj_vm.h" 32 #include "lj_strscan.h" 33 #include "lj_strfmt.h" 34 #include "lj_lib.h" 35 36 /* Some local macros to save typing. Undef'd at the end. */ 37 #define IR(ref) (&J->cur.ir[(ref)]) 38 #define fins (&J->fold.ins) 39 40 /* Pass IR on to next optimization in chain (FOLD). */ 41 #define emitir(ot, a, b) (lj_ir_set(J, (ot), (a), (b)), lj_opt_fold(J)) 42 43 /* -- IR tables ----------------------------------------------------------- */ 44 45 /* IR instruction modes. */ 46 LJ_DATADEF const uint8_t lj_ir_mode[IR__MAX+1] = { 47 IRDEF(IRMODE) 48 0 49 }; 50 51 /* IR type sizes. */ 52 LJ_DATADEF const uint8_t lj_ir_type_size[IRT__MAX+1] = { 53 #define IRTSIZE(name, size) size, 54 IRTDEF(IRTSIZE) 55 #undef IRTSIZE 56 0 57 }; 58 59 /* C call info for CALL* instructions. */ 60 LJ_DATADEF const CCallInfo lj_ir_callinfo[] = { 61 #define IRCALLCI(cond, name, nargs, kind, type, flags) \ 62 { (ASMFunction)IRCALLCOND_##cond(name), \ 63 (nargs)|(CCI_CALL_##kind)|(IRT_##type<<CCI_OTSHIFT)|(flags) }, 64 IRCALLDEF(IRCALLCI) 65 #undef IRCALLCI 66 { NULL, 0 } 67 }; 68 69 /* -- IR emitter ---------------------------------------------------------- */ 70 71 /* Grow IR buffer at the top. */ 72 void LJ_FASTCALL lj_ir_growtop(jit_State *J) 73 { 74 IRIns *baseir = J->irbuf + J->irbotlim; 75 MSize szins = J->irtoplim - J->irbotlim; 76 if (szins) { 77 baseir = (IRIns *)lj_mem_realloc(J->L, baseir, szins*sizeof(IRIns), 78 2*szins*sizeof(IRIns)); 79 J->irtoplim = J->irbotlim + 2*szins; 80 } else { 81 baseir = (IRIns *)lj_mem_realloc(J->L, NULL, 0, LJ_MIN_IRSZ*sizeof(IRIns)); 82 J->irbotlim = REF_BASE - LJ_MIN_IRSZ/4; 83 J->irtoplim = J->irbotlim + LJ_MIN_IRSZ; 84 } 85 J->cur.ir = J->irbuf = baseir - J->irbotlim; 86 } 87 88 /* Grow IR buffer at the bottom or shift it up. */ 89 static void lj_ir_growbot(jit_State *J) 90 { 91 IRIns *baseir = J->irbuf + J->irbotlim; 92 MSize szins = J->irtoplim - J->irbotlim; 93 lua_assert(szins != 0); 94 lua_assert(J->cur.nk == J->irbotlim || J->cur.nk-1 == J->irbotlim); 95 if (J->cur.nins + (szins >> 1) < J->irtoplim) { 96 /* More than half of the buffer is free on top: shift up by a quarter. */ 97 MSize ofs = szins >> 2; 98 memmove(baseir + ofs, baseir, (J->cur.nins - J->irbotlim)*sizeof(IRIns)); 99 J->irbotlim -= ofs; 100 J->irtoplim -= ofs; 101 J->cur.ir = J->irbuf = baseir - J->irbotlim; 102 } else { 103 /* Double the buffer size, but split the growth amongst top/bottom. */ 104 IRIns *newbase = lj_mem_newt(J->L, 2*szins*sizeof(IRIns), IRIns); 105 MSize ofs = szins >= 256 ? 128 : (szins >> 1); /* Limit bottom growth. */ 106 memcpy(newbase + ofs, baseir, (J->cur.nins - J->irbotlim)*sizeof(IRIns)); 107 lj_mem_free(G(J->L), baseir, szins*sizeof(IRIns)); 108 J->irbotlim -= ofs; 109 J->irtoplim = J->irbotlim + 2*szins; 110 J->cur.ir = J->irbuf = newbase - J->irbotlim; 111 } 112 } 113 114 /* Emit IR without any optimizations. */ 115 TRef LJ_FASTCALL lj_ir_emit(jit_State *J) 116 { 117 IRRef ref = lj_ir_nextins(J); 118 IRIns *ir = IR(ref); 119 IROp op = fins->o; 120 ir->prev = J->chain[op]; 121 J->chain[op] = (IRRef1)ref; 122 ir->o = op; 123 ir->op1 = fins->op1; 124 ir->op2 = fins->op2; 125 J->guardemit.irt |= fins->t.irt; 126 return TREF(ref, irt_t((ir->t = fins->t))); 127 } 128 129 /* Emit call to a C function. */ 130 TRef lj_ir_call(jit_State *J, IRCallID id, ...) 131 { 132 const CCallInfo *ci = &lj_ir_callinfo[id]; 133 uint32_t n = CCI_NARGS(ci); 134 TRef tr = TREF_NIL; 135 va_list argp; 136 va_start(argp, id); 137 if ((ci->flags & CCI_L)) n--; 138 if (n > 0) 139 tr = va_arg(argp, IRRef); 140 while (n-- > 1) 141 tr = emitir(IRT(IR_CARG, IRT_NIL), tr, va_arg(argp, IRRef)); 142 va_end(argp); 143 if (CCI_OP(ci) == IR_CALLS) 144 J->needsnap = 1; /* Need snapshot after call with side effect. */ 145 return emitir(CCI_OPTYPE(ci), tr, id); 146 } 147 148 /* Load field of type t from GG_State + offset. */ 149 LJ_FUNC TRef lj_ir_ggfload(jit_State *J, IRType t, uintptr_t ofs) 150 { 151 lua_assert(ofs >= IRFL__MAX && ofs < REF_BIAS); 152 lj_ir_set(J, IRT(IR_FLOAD, t), REF_NIL, ofs); 153 return lj_opt_fold(J); 154 } 155 156 /* -- Interning of constants ---------------------------------------------- */ 157 158 /* 159 ** IR instructions for constants are kept between J->cur.nk >= ref < REF_BIAS. 160 ** They are chained like all other instructions, but grow downwards. 161 ** The are interned (like strings in the VM) to facilitate reference 162 ** comparisons. The same constant must get the same reference. 163 */ 164 165 /* Get ref of next IR constant and optionally grow IR. 166 ** Note: this may invalidate all IRIns *! 167 */ 168 static LJ_AINLINE IRRef ir_nextk(jit_State *J) 169 { 170 IRRef ref = J->cur.nk; 171 if (LJ_UNLIKELY(ref <= J->irbotlim)) lj_ir_growbot(J); 172 J->cur.nk = --ref; 173 return ref; 174 } 175 176 /* Get ref of next 64 bit IR constant and optionally grow IR. 177 ** Note: this may invalidate all IRIns *! 178 */ 179 static LJ_AINLINE IRRef ir_nextk64(jit_State *J) 180 { 181 IRRef ref = J->cur.nk - 2; 182 lua_assert(J->state != LJ_TRACE_ASM); 183 if (LJ_UNLIKELY(ref < J->irbotlim)) lj_ir_growbot(J); 184 J->cur.nk = ref; 185 return ref; 186 } 187 188 #if LJ_GC64 189 #define ir_nextkgc ir_nextk64 190 #else 191 #define ir_nextkgc ir_nextk 192 #endif 193 194 /* Intern int32_t constant. */ 195 TRef LJ_FASTCALL lj_ir_kint(jit_State *J, int32_t k) 196 { 197 IRIns *ir, *cir = J->cur.ir; 198 IRRef ref; 199 for (ref = J->chain[IR_KINT]; ref; ref = cir[ref].prev) 200 if (cir[ref].i == k) 201 goto found; 202 ref = ir_nextk(J); 203 ir = IR(ref); 204 ir->i = k; 205 ir->t.irt = IRT_INT; 206 ir->o = IR_KINT; 207 ir->prev = J->chain[IR_KINT]; 208 J->chain[IR_KINT] = (IRRef1)ref; 209 found: 210 return TREF(ref, IRT_INT); 211 } 212 213 /* Intern 64 bit constant, given by its 64 bit pattern. */ 214 TRef lj_ir_k64(jit_State *J, IROp op, uint64_t u64) 215 { 216 IRIns *ir, *cir = J->cur.ir; 217 IRRef ref; 218 IRType t = op == IR_KNUM ? IRT_NUM : IRT_I64; 219 for (ref = J->chain[op]; ref; ref = cir[ref].prev) 220 if (ir_k64(&cir[ref])->u64 == u64) 221 goto found; 222 ref = ir_nextk64(J); 223 ir = IR(ref); 224 ir[1].tv.u64 = u64; 225 ir->t.irt = t; 226 ir->o = op; 227 ir->op12 = 0; 228 ir->prev = J->chain[op]; 229 J->chain[op] = (IRRef1)ref; 230 found: 231 return TREF(ref, t); 232 } 233 234 /* Intern FP constant, given by its 64 bit pattern. */ 235 TRef lj_ir_knum_u64(jit_State *J, uint64_t u64) 236 { 237 return lj_ir_k64(J, IR_KNUM, u64); 238 } 239 240 /* Intern 64 bit integer constant. */ 241 TRef lj_ir_kint64(jit_State *J, uint64_t u64) 242 { 243 return lj_ir_k64(J, IR_KINT64, u64); 244 } 245 246 /* Check whether a number is int and return it. -0 is NOT considered an int. */ 247 static int numistrueint(lua_Number n, int32_t *kp) 248 { 249 int32_t k = lj_num2int(n); 250 if (n == (lua_Number)k) { 251 if (kp) *kp = k; 252 if (k == 0) { /* Special check for -0. */ 253 TValue tv; 254 setnumV(&tv, n); 255 if (tv.u32.hi != 0) 256 return 0; 257 } 258 return 1; 259 } 260 return 0; 261 } 262 263 /* Intern number as int32_t constant if possible, otherwise as FP constant. */ 264 TRef lj_ir_knumint(jit_State *J, lua_Number n) 265 { 266 int32_t k; 267 if (numistrueint(n, &k)) 268 return lj_ir_kint(J, k); 269 else 270 return lj_ir_knum(J, n); 271 } 272 273 /* Intern GC object "constant". */ 274 TRef lj_ir_kgc(jit_State *J, GCobj *o, IRType t) 275 { 276 IRIns *ir, *cir = J->cur.ir; 277 IRRef ref; 278 lua_assert(!isdead(J2G(J), o)); 279 for (ref = J->chain[IR_KGC]; ref; ref = cir[ref].prev) 280 if (ir_kgc(&cir[ref]) == o) 281 goto found; 282 ref = ir_nextkgc(J); 283 ir = IR(ref); 284 /* NOBARRIER: Current trace is a GC root. */ 285 ir->op12 = 0; 286 setgcref(ir[LJ_GC64].gcr, o); 287 ir->t.irt = (uint8_t)t; 288 ir->o = IR_KGC; 289 ir->prev = J->chain[IR_KGC]; 290 J->chain[IR_KGC] = (IRRef1)ref; 291 found: 292 return TREF(ref, t); 293 } 294 295 /* Allocate GCtrace constant placeholder (no interning). */ 296 TRef lj_ir_ktrace(jit_State *J) 297 { 298 IRRef ref = ir_nextkgc(J); 299 IRIns *ir = IR(ref); 300 lua_assert(irt_toitype_(IRT_P64) == LJ_TTRACE); 301 ir->t.irt = IRT_P64; 302 ir->o = LJ_GC64 ? IR_KNUM : IR_KNULL; /* Not IR_KGC yet, but same size. */ 303 ir->op12 = 0; 304 ir->prev = 0; 305 return TREF(ref, IRT_P64); 306 } 307 308 /* Intern pointer constant. */ 309 TRef lj_ir_kptr_(jit_State *J, IROp op, void *ptr) 310 { 311 IRIns *ir, *cir = J->cur.ir; 312 IRRef ref; 313 #if LJ_64 && !LJ_GC64 314 lua_assert((void *)(uintptr_t)u32ptr(ptr) == ptr); 315 #endif 316 for (ref = J->chain[op]; ref; ref = cir[ref].prev) 317 if (ir_kptr(&cir[ref]) == ptr) 318 goto found; 319 #if LJ_GC64 320 ref = ir_nextk64(J); 321 #else 322 ref = ir_nextk(J); 323 #endif 324 ir = IR(ref); 325 ir->op12 = 0; 326 setmref(ir[LJ_GC64].ptr, ptr); 327 ir->t.irt = IRT_PGC; 328 ir->o = op; 329 ir->prev = J->chain[op]; 330 J->chain[op] = (IRRef1)ref; 331 found: 332 return TREF(ref, IRT_PGC); 333 } 334 335 /* Intern typed NULL constant. */ 336 TRef lj_ir_knull(jit_State *J, IRType t) 337 { 338 IRIns *ir, *cir = J->cur.ir; 339 IRRef ref; 340 for (ref = J->chain[IR_KNULL]; ref; ref = cir[ref].prev) 341 if (irt_t(cir[ref].t) == t) 342 goto found; 343 ref = ir_nextk(J); 344 ir = IR(ref); 345 ir->i = 0; 346 ir->t.irt = (uint8_t)t; 347 ir->o = IR_KNULL; 348 ir->prev = J->chain[IR_KNULL]; 349 J->chain[IR_KNULL] = (IRRef1)ref; 350 found: 351 return TREF(ref, t); 352 } 353 354 /* Intern key slot. */ 355 TRef lj_ir_kslot(jit_State *J, TRef key, IRRef slot) 356 { 357 IRIns *ir, *cir = J->cur.ir; 358 IRRef2 op12 = IRREF2((IRRef1)key, (IRRef1)slot); 359 IRRef ref; 360 /* Const part is not touched by CSE/DCE, so 0-65535 is ok for IRMlit here. */ 361 lua_assert(tref_isk(key) && slot == (IRRef)(IRRef1)slot); 362 for (ref = J->chain[IR_KSLOT]; ref; ref = cir[ref].prev) 363 if (cir[ref].op12 == op12) 364 goto found; 365 ref = ir_nextk(J); 366 ir = IR(ref); 367 ir->op12 = op12; 368 ir->t.irt = IRT_P32; 369 ir->o = IR_KSLOT; 370 ir->prev = J->chain[IR_KSLOT]; 371 J->chain[IR_KSLOT] = (IRRef1)ref; 372 found: 373 return TREF(ref, IRT_P32); 374 } 375 376 /* -- Access to IR constants ---------------------------------------------- */ 377 378 /* Copy value of IR constant. */ 379 void lj_ir_kvalue(lua_State *L, TValue *tv, const IRIns *ir) 380 { 381 UNUSED(L); 382 lua_assert(ir->o != IR_KSLOT); /* Common mistake. */ 383 switch (ir->o) { 384 case IR_KPRI: setpriV(tv, irt_toitype(ir->t)); break; 385 case IR_KINT: setintV(tv, ir->i); break; 386 case IR_KGC: setgcV(L, tv, ir_kgc(ir), irt_toitype(ir->t)); break; 387 case IR_KPTR: case IR_KKPTR: setlightudV(tv, ir_kptr(ir)); break; 388 case IR_KNULL: setlightudV(tv, NULL); break; 389 case IR_KNUM: setnumV(tv, ir_knum(ir)->n); break; 390 #if LJ_HASFFI 391 case IR_KINT64: { 392 GCcdata *cd = lj_cdata_new_(L, CTID_INT64, 8); 393 *(uint64_t *)cdataptr(cd) = ir_kint64(ir)->u64; 394 setcdataV(L, tv, cd); 395 break; 396 } 397 #endif 398 default: lua_assert(0); break; 399 } 400 } 401 402 /* -- Convert IR operand types -------------------------------------------- */ 403 404 /* Convert from string to number. */ 405 TRef LJ_FASTCALL lj_ir_tonumber(jit_State *J, TRef tr) 406 { 407 if (!tref_isnumber(tr)) { 408 if (tref_isstr(tr)) 409 tr = emitir(IRTG(IR_STRTO, IRT_NUM), tr, 0); 410 else 411 lj_trace_err(J, LJ_TRERR_BADTYPE); 412 } 413 return tr; 414 } 415 416 /* Convert from integer or string to number. */ 417 TRef LJ_FASTCALL lj_ir_tonum(jit_State *J, TRef tr) 418 { 419 if (!tref_isnum(tr)) { 420 if (tref_isinteger(tr)) 421 tr = emitir(IRTN(IR_CONV), tr, IRCONV_NUM_INT); 422 else if (tref_isstr(tr)) 423 tr = emitir(IRTG(IR_STRTO, IRT_NUM), tr, 0); 424 else 425 lj_trace_err(J, LJ_TRERR_BADTYPE); 426 } 427 return tr; 428 } 429 430 /* Convert from integer or number to string. */ 431 TRef LJ_FASTCALL lj_ir_tostr(jit_State *J, TRef tr) 432 { 433 if (!tref_isstr(tr)) { 434 if (!tref_isnumber(tr)) 435 lj_trace_err(J, LJ_TRERR_BADTYPE); 436 tr = emitir(IRT(IR_TOSTR, IRT_STR), tr, 437 tref_isnum(tr) ? IRTOSTR_NUM : IRTOSTR_INT); 438 } 439 return tr; 440 } 441 442 /* -- Miscellaneous IR ops ------------------------------------------------ */ 443 444 /* Evaluate numeric comparison. */ 445 int lj_ir_numcmp(lua_Number a, lua_Number b, IROp op) 446 { 447 switch (op) { 448 case IR_EQ: return (a == b); 449 case IR_NE: return (a != b); 450 case IR_LT: return (a < b); 451 case IR_GE: return (a >= b); 452 case IR_LE: return (a <= b); 453 case IR_GT: return (a > b); 454 case IR_ULT: return !(a >= b); 455 case IR_UGE: return !(a < b); 456 case IR_ULE: return !(a > b); 457 case IR_UGT: return !(a <= b); 458 default: lua_assert(0); return 0; 459 } 460 } 461 462 /* Evaluate string comparison. */ 463 int lj_ir_strcmp(GCstr *a, GCstr *b, IROp op) 464 { 465 int res = lj_str_cmp(a, b); 466 switch (op) { 467 case IR_LT: return (res < 0); 468 case IR_GE: return (res >= 0); 469 case IR_LE: return (res <= 0); 470 case IR_GT: return (res > 0); 471 default: lua_assert(0); return 0; 472 } 473 } 474 475 /* Rollback IR to previous state. */ 476 void lj_ir_rollback(jit_State *J, IRRef ref) 477 { 478 IRRef nins = J->cur.nins; 479 while (nins > ref) { 480 IRIns *ir; 481 nins--; 482 ir = IR(nins); 483 J->chain[ir->o] = ir->prev; 484 } 485 J->cur.nins = nins; 486 } 487 488 #undef IR 489 #undef fins 490 #undef emitir 491 492 #endif