lj_ffrecord.c (41722B)
1 /* 2 ** Fast function call recorder. 3 ** Copyright (C) 2005-2014 Mike Pall. See Copyright Notice in luajit.h 4 ** 5 ** Lua 5.2 string semantics and string.match() compiling 6 ** Copyright (C) 2014 Karel Tuma. See Copyright Notice in luajit.h 7 */ 8 9 #define lj_ffrecord_c 10 #define LUA_CORE 11 12 #include "lj_obj.h" 13 14 #if LJ_HASJIT 15 16 #include "lj_err.h" 17 #include "lj_str.h" 18 #include "lj_tab.h" 19 #include "lj_frame.h" 20 #include "lj_bc.h" 21 #include "lj_ff.h" 22 #include "lj_ir.h" 23 #include "lj_jit.h" 24 #include "lj_ircall.h" 25 #include "lj_iropt.h" 26 #include "lj_trace.h" 27 #include "lj_record.h" 28 #include "lj_ffrecord.h" 29 #include "lj_crecord.h" 30 #include "lj_dispatch.h" 31 #include "lj_vm.h" 32 #include "lj_strscan.h" 33 #include "lj_strfmt.h" 34 35 /* Some local macros to save typing. Undef'd at the end. */ 36 #define IR(ref) (&J->cur.ir[(ref)]) 37 38 /* Pass IR on to next optimization in chain (FOLD). */ 39 #define emitir(ot, a, b) (lj_ir_set(J, (ot), (a), (b)), lj_opt_fold(J)) 40 41 /* -- Fast function recording handlers ------------------------------------ */ 42 43 /* Conventions for fast function call handlers: 44 ** 45 ** The argument slots start at J->base[0]. All of them are guaranteed to be 46 ** valid and type-specialized references. J->base[J->maxslot] is set to 0 47 ** as a sentinel. The runtime argument values start at rd->argv[0]. 48 ** 49 ** In general fast functions should check for presence of all of their 50 ** arguments and for the correct argument types. Some simplifications 51 ** are allowed if the interpreter throws instead. But even if recording 52 ** is aborted, the generated IR must be consistent (no zero-refs). 53 ** 54 ** The number of results in rd->nres is set to 1. Handlers that return 55 ** a different number of results need to override it. A negative value 56 ** prevents return processing (e.g. for pending calls). 57 ** 58 ** Results need to be stored starting at J->base[0]. Return processing 59 ** moves them to the right slots later. 60 ** 61 ** The per-ffid auxiliary data is the value of the 2nd part of the 62 ** LJLIB_REC() annotation. This allows handling similar functionality 63 ** in a common handler. 64 */ 65 66 /* Type of handler to record a fast function. */ 67 typedef void (LJ_FASTCALL *RecordFunc)(jit_State *J, RecordFFData *rd); 68 69 /* Get runtime value of int argument. */ 70 static int32_t argv2int(jit_State *J, TValue *o) 71 { 72 if (!lj_strscan_numberobj(o)) 73 lj_trace_err(J, LJ_TRERR_BADTYPE); 74 return tvisint(o) ? intV(o) : lj_num2int(numV(o)); 75 } 76 77 /* Get runtime value of string argument. */ 78 static GCstr *argv2str(jit_State *J, TValue *o) 79 { 80 if (LJ_LIKELY(tvisstr(o))) { 81 return strV(o); 82 } else { 83 GCstr *s; 84 if (!tvisnumber(o)) 85 lj_trace_err(J, LJ_TRERR_BADTYPE); 86 s = lj_strfmt_number(J->L, o); 87 setstrV(J->L, o, s); 88 return s; 89 } 90 } 91 92 /* Return number of results wanted by caller. */ 93 static ptrdiff_t results_wanted(jit_State *J) 94 { 95 TValue *frame = J->L->base-1; 96 if (frame_islua(frame)) 97 return (ptrdiff_t)bc_b(frame_pc(frame)[-1]) - 1; 98 else 99 return -1; 100 } 101 102 /* Trace stitching: add continuation below frame to start a new trace. */ 103 static void recff_stitch(jit_State *J) 104 { 105 ASMFunction cont = lj_cont_stitch; 106 lua_State *L = J->L; 107 TValue *base = L->base; 108 BCReg nslot = J->maxslot + 1 + LJ_FR2; 109 TValue *nframe = base + 1 + LJ_FR2; 110 const BCIns *pc = frame_pc(base-1); 111 TValue *pframe = frame_prevl(base-1); 112 113 /* Move func + args up in Lua stack and insert continuation. */ 114 memmove(&base[1], &base[-1-LJ_FR2], sizeof(TValue)*nslot); 115 setframe_ftsz(nframe, ((char *)nframe - (char *)pframe) + FRAME_CONT); 116 setcont(base-LJ_FR2, cont); 117 setframe_pc(base, pc); 118 setnilV(base-1-LJ_FR2); /* Incorrect, but rec_check_slots() won't run anymore. */ 119 L->base += 2 + LJ_FR2; 120 L->top += 2 + LJ_FR2; 121 122 /* Ditto for the IR. */ 123 memmove(&J->base[1], &J->base[-1-LJ_FR2], sizeof(TRef)*nslot); 124 #if LJ_FR2 125 J->base[2] = TREF_FRAME; 126 J->base[-1] = lj_ir_k64(J, IR_KNUM, u64ptr(contptr(cont))); 127 J->base[0] = lj_ir_k64(J, IR_KNUM, u64ptr(pc)) | TREF_CONT; 128 #else 129 J->base[0] = lj_ir_kptr(J, contptr(cont)) | TREF_CONT; 130 #endif 131 J->ktrace = tref_ref((J->base[-1-LJ_FR2] = lj_ir_ktrace(J))); 132 J->base += 2 + LJ_FR2; 133 J->baseslot += 2 + LJ_FR2; 134 J->framedepth++; 135 136 lj_record_stop(J, LJ_TRLINK_STITCH, 0); 137 138 /* Undo Lua stack changes. */ 139 memmove(&base[-1-LJ_FR2], &base[1], sizeof(TValue)*nslot); 140 setframe_pc(base-1, pc); 141 L->base -= 2 + LJ_FR2; 142 L->top -= 2 + LJ_FR2; 143 } 144 145 /* Fallback handler for fast functions that are not recorded (yet). */ 146 static void LJ_FASTCALL recff_nyi(jit_State *J, RecordFFData *rd) 147 { 148 if (J->cur.nins < (IRRef)J->param[JIT_P_minstitch] + REF_BASE) { 149 lj_trace_err_info(J, LJ_TRERR_TRACEUV); 150 } else { 151 /* Can only stitch from Lua call. */ 152 if (J->framedepth && frame_islua(J->L->base-1)) { 153 BCOp op = bc_op(*frame_pc(J->L->base-1)); 154 /* Stitched trace cannot start with *M op with variable # of args. */ 155 if (!(op == BC_CALLM || op == BC_CALLMT || 156 op == BC_RETM || op == BC_TSETM)) { 157 switch (J->fn->c.ffid) { 158 case FF_error: 159 case FF_debug_sethook: 160 case FF_jit_flush: 161 break; /* Don't stitch across special builtins. */ 162 default: 163 if (!(J->flags & JIT_F_OPT_STITCH)) 164 break; 165 recff_stitch(J); /* Use trace stitching. */ 166 rd->nres = -1; 167 return; 168 } 169 } 170 } 171 /* Otherwise stop trace and return to interpreter. */ 172 lj_record_stop(J, LJ_TRLINK_RETURN, 0); 173 rd->nres = -1; 174 } 175 } 176 177 /* Fallback handler for unsupported variants of fast functions. */ 178 #define recff_nyiu recff_nyi 179 180 /* Must stop the trace for classic C functions with arbitrary side-effects. */ 181 #define recff_c recff_nyi 182 183 /* Emit BUFHDR for the global temporary buffer. */ 184 static TRef recff_bufhdr(jit_State *J) 185 { 186 return emitir(IRT(IR_BUFHDR, IRT_PGC), 187 lj_ir_kptr(J, &J2G(J)->tmpbuf), IRBUFHDR_RESET); 188 } 189 190 /* -- Base library fast functions ----------------------------------------- */ 191 192 static void LJ_FASTCALL recff_assert(jit_State *J, RecordFFData *rd) 193 { 194 /* Arguments already specialized. The interpreter throws for nil/false. */ 195 rd->nres = J->maxslot; /* Pass through all arguments. */ 196 } 197 198 static void LJ_FASTCALL recff_type(jit_State *J, RecordFFData *rd) 199 { 200 /* Arguments already specialized. Result is a constant string. Neat, huh? */ 201 uint32_t t; 202 if (tvisnumber(&rd->argv[0])) 203 t = ~LJ_TNUMX; 204 else if (LJ_64 && !LJ_GC64 && tvislightud(&rd->argv[0])) 205 t = ~LJ_TLIGHTUD; 206 else 207 t = ~itype(&rd->argv[0]); 208 J->base[0] = lj_ir_kstr(J, strV(&J->fn->c.upvalue[t])); 209 UNUSED(rd); 210 } 211 212 static void LJ_FASTCALL recff_getmetatable(jit_State *J, RecordFFData *rd) 213 { 214 TRef tr = J->base[0]; 215 if (tr) { 216 RecordIndex ix; 217 ix.tab = tr; 218 copyTV(J->L, &ix.tabv, &rd->argv[0]); 219 if (lj_record_mm_lookup(J, &ix, MM_metatable)) 220 J->base[0] = ix.mobj; 221 else 222 J->base[0] = ix.mt; 223 } /* else: Interpreter will throw. */ 224 } 225 226 #if !LJ_51 227 static void setmetatable_gc_guard(jit_State *J, RecordFFData *rd) 228 { 229 RecordIndex mix; 230 GCstr *mmstr = mmname_str(J2G(J), MM_gc); 231 232 /* Look for __gc */ 233 copyTV(J->L, &mix.tabv, &rd->argv[1]); 234 setstrV(J->L, &mix.keyv, mmstr); 235 mix.tab = J->base[1]; 236 mix.key = lj_ir_kstr(J, mmstr); 237 mix.val = 0; /* Load */ 238 mix.idxchain = 0; 239 if (!tref_isnil(lj_record_idx(J, &mix))) 240 lj_ir_call(J, IRCALL_lj_gc_tab_finalized, J->base[0]); 241 } 242 #endif 243 244 static void LJ_FASTCALL recff_setmetatable(jit_State *J, RecordFFData *rd) 245 { 246 TRef tr = J->base[0]; 247 TRef mt = J->base[1]; 248 if (tref_istab(tr) && (tref_istab(mt) || (mt && tref_isnil(mt)))) { 249 TRef fref, mtref; 250 RecordIndex ix; 251 ix.tab = tr; 252 copyTV(J->L, &ix.tabv, &rd->argv[0]); 253 lj_record_mm_lookup(J, &ix, MM_metatable); /* Guard for no __metatable. */ 254 fref = emitir(IRT(IR_FREF, IRT_PGC), tr, IRFL_TAB_META); 255 mtref = tref_isnil(mt) ? lj_ir_knull(J, IRT_TAB) : mt; 256 emitir(IRT(IR_FSTORE, IRT_TAB), fref, mtref); 257 if (!tref_isnil(mt)) { 258 #if !LJ_51 259 setmetatable_gc_guard(J, rd); 260 #endif 261 emitir(IRT(IR_TBAR, IRT_TAB), tr, 0); 262 } 263 J->base[0] = tr; 264 J->needsnap = 1; 265 } /* else: Interpreter will throw. */ 266 } 267 268 static void LJ_FASTCALL recff_rawget(jit_State *J, RecordFFData *rd) 269 { 270 RecordIndex ix; 271 ix.tab = J->base[0]; ix.key = J->base[1]; 272 if (tref_istab(ix.tab) && ix.key) { 273 ix.val = 0; ix.idxchain = 0; 274 settabV(J->L, &ix.tabv, tabV(&rd->argv[0])); 275 copyTV(J->L, &ix.keyv, &rd->argv[1]); 276 J->base[0] = lj_record_idx(J, &ix); 277 } /* else: Interpreter will throw. */ 278 } 279 280 static void LJ_FASTCALL recff_rawset(jit_State *J, RecordFFData *rd) 281 { 282 RecordIndex ix; 283 ix.tab = J->base[0]; ix.key = J->base[1]; ix.val = J->base[2]; 284 if (tref_istab(ix.tab) && ix.key && ix.val) { 285 ix.idxchain = 0; 286 settabV(J->L, &ix.tabv, tabV(&rd->argv[0])); 287 copyTV(J->L, &ix.keyv, &rd->argv[1]); 288 copyTV(J->L, &ix.valv, &rd->argv[2]); 289 lj_record_idx(J, &ix); 290 /* Pass through table at J->base[0] as result. */ 291 } /* else: Interpreter will throw. */ 292 } 293 294 static void LJ_FASTCALL recff_rawequal(jit_State *J, RecordFFData *rd) 295 { 296 TRef tra = J->base[0]; 297 TRef trb = J->base[1]; 298 if (tra && trb) { 299 int diff = lj_record_objcmp(J, tra, trb, &rd->argv[0], &rd->argv[1]); 300 J->base[0] = diff ? TREF_FALSE : TREF_TRUE; 301 } /* else: Interpreter will throw. */ 302 } 303 304 #if !LJ_51 305 static void LJ_FASTCALL recff_rawlen(jit_State *J, RecordFFData *rd) 306 { 307 TRef tr = J->base[0]; 308 if (tref_isstr(tr)) 309 J->base[0] = emitir(IRTI(IR_FLOAD), tr, IRFL_STR_LEN); 310 else if (tref_istab(tr)) 311 J->base[0] = lj_ir_call(J, IRCALL_lj_tab_len, tr); 312 /* else: Interpreter will throw. */ 313 UNUSED(rd); 314 } 315 #endif 316 317 /* Determine mode of select() call. */ 318 int32_t lj_ffrecord_select_mode(jit_State *J, TRef tr, TValue *tv) 319 { 320 if (tref_isstr(tr) && *strVdata(tv) == '#') { /* select('#', ...) */ 321 if (strV(tv)->len == 1) { 322 emitir(IRTG(IR_EQ, IRT_STR), tr, lj_ir_kstr(J, strV(tv))); 323 } else { 324 TRef trptr = emitir(IRT(IR_STRREF, IRT_PGC), tr, lj_ir_kint(J, 0)); 325 TRef trchar = emitir(IRT(IR_XLOAD, IRT_U8), trptr, IRXLOAD_READONLY); 326 emitir(IRTG(IR_EQ, IRT_INT), trchar, lj_ir_kint(J, '#')); 327 } 328 return 0; 329 } else { /* select(n, ...) */ 330 int32_t start = argv2int(J, tv); 331 if (start == 0) lj_trace_err(J, LJ_TRERR_BADTYPE); /* A bit misleading. */ 332 return start; 333 } 334 } 335 336 static void LJ_FASTCALL recff_select(jit_State *J, RecordFFData *rd) 337 { 338 TRef tr = J->base[0]; 339 if (tr) { 340 ptrdiff_t start = lj_ffrecord_select_mode(J, tr, &rd->argv[0]); 341 if (start == 0) { /* select('#', ...) */ 342 J->base[0] = lj_ir_kint(J, J->maxslot - 1); 343 } else if (tref_isk(tr)) { /* select(k, ...) */ 344 ptrdiff_t n = (ptrdiff_t)J->maxslot; 345 if (start < 0) start += n; 346 else if (start > n) start = n; 347 rd->nres = n - start; 348 if (start >= 1) { 349 ptrdiff_t i; 350 for (i = 0; i < n - start; i++) 351 J->base[i] = J->base[start+i]; 352 } /* else: Interpreter will throw. */ 353 } else { 354 recff_nyiu(J, rd); 355 return; 356 } 357 } /* else: Interpreter will throw. */ 358 } 359 360 static void LJ_FASTCALL recff_tonumber(jit_State *J, RecordFFData *rd) 361 { 362 TRef tr = J->base[0]; 363 TRef base = J->base[1]; 364 if (tr && !tref_isnil(base)) { 365 base = lj_opt_narrow_toint(J, base); 366 if (!tref_isk(base) || IR(tref_ref(base))->i != 10) { 367 recff_nyiu(J, rd); 368 return; 369 } 370 } 371 if (tref_isnumber_str(tr)) { 372 if (tref_isstr(tr)) { 373 TValue tmp; 374 if (!lj_strscan_num(strV(&rd->argv[0]), &tmp)) { 375 recff_nyiu(J, rd); /* Would need an inverted STRTO for this case. */ 376 return; 377 } 378 tr = emitir(IRTG(IR_STRTO, IRT_NUM), tr, 0); 379 } 380 #if LJ_HASFFI 381 } else if (tref_iscdata(tr)) { 382 lj_crecord_tonumber(J, rd); 383 return; 384 #endif 385 } else { 386 tr = TREF_NIL; 387 } 388 J->base[0] = tr; 389 UNUSED(rd); 390 } 391 392 static TValue *recff_metacall_cp(lua_State *L, lua_CFunction dummy, void *ud) 393 { 394 jit_State *J = (jit_State *)ud; 395 lj_record_tailcall(J, 0, 1); 396 UNUSED(L); UNUSED(dummy); 397 return NULL; 398 } 399 400 static int recff_metacall(jit_State *J, RecordFFData *rd, MMS mm) 401 { 402 RecordIndex ix; 403 ix.tab = J->base[0]; 404 copyTV(J->L, &ix.tabv, &rd->argv[0]); 405 if (lj_record_mm_lookup(J, &ix, mm)) { /* Has metamethod? */ 406 int errcode; 407 TValue argv0; 408 /* Temporarily insert metamethod below object. */ 409 J->base[1+LJ_FR2] = J->base[0]; 410 J->base[0] = ix.mobj; 411 copyTV(J->L, &argv0, &rd->argv[0]); 412 copyTV(J->L, &rd->argv[1+LJ_FR2], &rd->argv[0]); 413 copyTV(J->L, &rd->argv[0], &ix.mobjv); 414 /* Need to protect lj_record_tailcall because it may throw. */ 415 errcode = lj_vm_cpcall(J->L, NULL, J, recff_metacall_cp); 416 /* Always undo Lua stack changes to avoid confusing the interpreter. */ 417 copyTV(J->L, &rd->argv[0], &argv0); 418 if (errcode) 419 lj_err_throw(J->L, errcode); /* Propagate errors. */ 420 rd->nres = -1; /* Pending call. */ 421 return 1; /* Tailcalled to metamethod. */ 422 } 423 return 0; 424 } 425 426 static void LJ_FASTCALL recff_tostring(jit_State *J, RecordFFData *rd) 427 { 428 TRef tr = J->base[0]; 429 if (tref_isstr(tr)) { 430 /* Ignore __tostring in the string base metatable. */ 431 /* Pass on result in J->base[0]. */ 432 } else if (tr && !recff_metacall(J, rd, MM_tostring)) { 433 if (tref_isnumber(tr)) { 434 J->base[0] = emitir(IRT(IR_TOSTR, IRT_STR), tr, 435 tref_isnum(tr) ? IRTOSTR_NUM : IRTOSTR_INT); 436 } else if (tref_ispri(tr)) { 437 J->base[0] = lj_ir_kstr(J, lj_strfmt_obj(J->L, &rd->argv[0])); 438 } else { 439 recff_nyiu(J, rd); 440 return; 441 } 442 } 443 } 444 445 static void LJ_FASTCALL recff_ipairs_aux(jit_State *J, RecordFFData *rd) 446 { 447 RecordIndex ix; 448 ix.tab = J->base[0]; 449 if (tref_istab(ix.tab)) { 450 if (!tvisnumber(&rd->argv[1])) /* No support for string coercion. */ 451 lj_trace_err(J, LJ_TRERR_BADTYPE); 452 setintV(&ix.keyv, numberVint(&rd->argv[1])+1); 453 settabV(J->L, &ix.tabv, tabV(&rd->argv[0])); 454 ix.val = 0; ix.idxchain = 0; 455 ix.key = lj_opt_narrow_toint(J, J->base[1]); 456 J->base[0] = ix.key = emitir(IRTI(IR_ADD), ix.key, lj_ir_kint(J, 1)); 457 J->base[1] = lj_record_idx(J, &ix); 458 rd->nres = tref_isnil(J->base[1]) ? 0 : 2; 459 } /* else: Interpreter will throw. */ 460 } 461 462 static void LJ_FASTCALL recff_xpairs(jit_State *J, RecordFFData *rd) 463 { 464 TRef tr = J->base[0]; 465 if (!(((!LJ_51) || (LJ_HASFFI && tref_iscdata(tr))) && 466 recff_metacall(J, rd, MM_pairs + rd->data))) { 467 if (tref_istab(tr)) { 468 J->base[0] = lj_ir_kfunc(J, funcV(&J->fn->c.upvalue[0])); 469 J->base[1] = tr; 470 J->base[2] = rd->data ? lj_ir_kint(J, 0) : TREF_NIL; 471 rd->nres = 3; 472 } /* else: Interpreter will throw. */ 473 } 474 } 475 476 static void LJ_FASTCALL recff_pcall(jit_State *J, RecordFFData *rd) 477 { 478 if (J->maxslot >= 1) { 479 #if LJ_FR2 480 /* Shift function arguments up. */ 481 memmove(J->base + 1, J->base, sizeof(TRef) * J->maxslot); 482 #endif 483 lj_record_call(J, 0, J->maxslot - 1); 484 rd->nres = -1; /* Pending call. */ 485 } /* else: Interpreter will throw. */ 486 } 487 488 static TValue *recff_xpcall_cp(lua_State *L, lua_CFunction dummy, void *ud) 489 { 490 jit_State *J = (jit_State *)ud; 491 lj_record_call(J, 1, J->maxslot - 2); 492 UNUSED(L); UNUSED(dummy); 493 return NULL; 494 } 495 496 static void LJ_FASTCALL recff_xpcall(jit_State *J, RecordFFData *rd) 497 { 498 if (J->maxslot >= 2) { 499 TValue argv0, argv1; 500 TRef tmp; 501 int errcode; 502 /* Swap function and traceback. */ 503 tmp = J->base[0]; J->base[0] = J->base[1]; J->base[1] = tmp; 504 copyTV(J->L, &argv0, &rd->argv[0]); 505 copyTV(J->L, &argv1, &rd->argv[1]); 506 copyTV(J->L, &rd->argv[0], &argv1); 507 copyTV(J->L, &rd->argv[1], &argv0); 508 #if LJ_FR2 509 /* Shift function arguments up. */ 510 memmove(J->base + 2, J->base + 1, sizeof(TRef) * (J->maxslot-1)); 511 #endif 512 /* Need to protect lj_record_call because it may throw. */ 513 errcode = lj_vm_cpcall(J->L, NULL, J, recff_xpcall_cp); 514 /* Always undo Lua stack swap to avoid confusing the interpreter. */ 515 copyTV(J->L, &rd->argv[0], &argv0); 516 copyTV(J->L, &rd->argv[1], &argv1); 517 if (errcode) 518 lj_err_throw(J->L, errcode); /* Propagate errors. */ 519 rd->nres = -1; /* Pending call. */ 520 } /* else: Interpreter will throw. */ 521 } 522 523 static void LJ_FASTCALL recff_getfenv(jit_State *J, RecordFFData *rd) 524 { 525 TRef tr = J->base[0]; 526 /* Only support getfenv(0) for now. */ 527 if (tref_isint(tr) && tref_isk(tr) && IR(tref_ref(tr))->i == 0) { 528 TRef trl = emitir(IRT(IR_LREF, IRT_THREAD), 0, 0); 529 J->base[0] = emitir(IRT(IR_FLOAD, IRT_TAB), trl, IRFL_THREAD_ENV); 530 return; 531 } 532 recff_nyiu(J, rd); 533 } 534 535 /* -- Math library fast functions ----------------------------------------- */ 536 537 static void LJ_FASTCALL recff_math_abs(jit_State *J, RecordFFData *rd) 538 { 539 TRef tr = lj_ir_tonum(J, J->base[0]); 540 J->base[0] = emitir(IRTN(IR_ABS), tr, lj_ir_ksimd(J, LJ_KSIMD_ABS)); 541 UNUSED(rd); 542 } 543 544 /* Record rounding functions math.floor and math.ceil. */ 545 static void LJ_FASTCALL recff_math_round(jit_State *J, RecordFFData *rd) 546 { 547 TRef tr = J->base[0]; 548 if (!tref_isinteger(tr)) { /* Pass through integers unmodified. */ 549 tr = emitir(IRTN(IR_FPMATH), lj_ir_tonum(J, tr), rd->data); 550 /* Result is integral (or NaN/Inf), but may not fit an int32_t. */ 551 if (LJ_DUALNUM) { /* Try to narrow using a guarded conversion to int. */ 552 lua_Number n = lj_vm_foldfpm(numberVnum(&rd->argv[0]), rd->data); 553 if (n == (lua_Number)lj_num2int(n)) 554 tr = emitir(IRTGI(IR_CONV), tr, IRCONV_INT_NUM|IRCONV_CHECK); 555 } 556 J->base[0] = tr; 557 } 558 } 559 560 /* Record unary math.* functions, mapped to IR_FPMATH opcode. */ 561 static void LJ_FASTCALL recff_math_unary(jit_State *J, RecordFFData *rd) 562 { 563 J->base[0] = emitir(IRTN(IR_FPMATH), lj_ir_tonum(J, J->base[0]), rd->data); 564 } 565 566 /* Record math.log. */ 567 static void LJ_FASTCALL recff_math_log(jit_State *J, RecordFFData *rd) 568 { 569 TRef tr = lj_ir_tonum(J, J->base[0]); 570 if (J->base[1]) { 571 #ifdef LUAJIT_NO_LOG2 572 uint32_t fpm = IRFPM_LOG; 573 #else 574 uint32_t fpm = IRFPM_LOG2; 575 #endif 576 TRef trb = lj_ir_tonum(J, J->base[1]); 577 tr = emitir(IRTN(IR_FPMATH), tr, fpm); 578 trb = emitir(IRTN(IR_FPMATH), trb, fpm); 579 trb = emitir(IRTN(IR_DIV), lj_ir_knum_one(J), trb); 580 tr = emitir(IRTN(IR_MUL), tr, trb); 581 } else { 582 tr = emitir(IRTN(IR_FPMATH), tr, IRFPM_LOG); 583 } 584 J->base[0] = tr; 585 UNUSED(rd); 586 } 587 588 /* Record math.atan2. */ 589 static void LJ_FASTCALL recff_math_atan2(jit_State *J, RecordFFData *rd) 590 { 591 TRef tr = lj_ir_tonum(J, J->base[0]); 592 TRef tr2 = lj_ir_tonum(J, J->base[1]); 593 J->base[0] = emitir(IRTN(IR_ATAN2), tr, tr2); 594 UNUSED(rd); 595 } 596 597 /* Record math.ldexp. */ 598 static void LJ_FASTCALL recff_math_ldexp(jit_State *J, RecordFFData *rd) 599 { 600 TRef tr = lj_ir_tonum(J, J->base[0]); 601 #if LJ_TARGET_X86ORX64 602 TRef tr2 = lj_ir_tonum(J, J->base[1]); 603 #else 604 TRef tr2 = lj_opt_narrow_toint(J, J->base[1]); 605 #endif 606 J->base[0] = emitir(IRTN(IR_LDEXP), tr, tr2); 607 UNUSED(rd); 608 } 609 610 /* Record math.asin, math.acos, math.atan. */ 611 static void LJ_FASTCALL recff_math_atrig(jit_State *J, RecordFFData *rd) 612 { 613 TRef y = lj_ir_tonum(J, J->base[0]); 614 TRef x = lj_ir_knum_one(J); 615 uint32_t ffid = rd->data; 616 if (ffid != FF_math_atan) { 617 TRef tmp = emitir(IRTN(IR_MUL), y, y); 618 tmp = emitir(IRTN(IR_SUB), x, tmp); 619 tmp = emitir(IRTN(IR_FPMATH), tmp, IRFPM_SQRT); 620 if (ffid == FF_math_asin) { x = tmp; } else { x = y; y = tmp; } 621 } 622 J->base[0] = emitir(IRTN(IR_ATAN2), y, x); 623 } 624 625 static void LJ_FASTCALL recff_math_htrig(jit_State *J, RecordFFData *rd) 626 { 627 TRef tr = lj_ir_tonum(J, J->base[0]); 628 J->base[0] = emitir(IRTN(IR_CALLN), tr, rd->data); 629 } 630 631 static void LJ_FASTCALL recff_math_modf(jit_State *J, RecordFFData *rd) 632 { 633 TRef tr = J->base[0]; 634 if (tref_isinteger(tr)) { 635 J->base[0] = tr; 636 J->base[1] = lj_ir_kint(J, 0); 637 } else { 638 TRef trt; 639 tr = lj_ir_tonum(J, tr); 640 trt = emitir(IRTN(IR_FPMATH), tr, IRFPM_TRUNC); 641 J->base[0] = trt; 642 J->base[1] = emitir(IRTN(IR_SUB), tr, trt); 643 } 644 rd->nres = 2; 645 } 646 647 static void LJ_FASTCALL recff_math_pow(jit_State *J, RecordFFData *rd) 648 { 649 J->base[0] = lj_opt_narrow_pow(J, J->base[0], J->base[1], 650 &rd->argv[0], &rd->argv[1]); 651 UNUSED(rd); 652 } 653 654 static void LJ_FASTCALL recff_math_minmax(jit_State *J, RecordFFData *rd) 655 { 656 TRef tr = lj_ir_tonumber(J, J->base[0]); 657 uint32_t op = rd->data; 658 BCReg i; 659 for (i = 1; J->base[i] != 0; i++) { 660 TRef tr2 = lj_ir_tonumber(J, J->base[i]); 661 IRType t = IRT_INT; 662 if (!(tref_isinteger(tr) && tref_isinteger(tr2))) { 663 if (tref_isinteger(tr)) tr = emitir(IRTN(IR_CONV), tr, IRCONV_NUM_INT); 664 if (tref_isinteger(tr2)) tr2 = emitir(IRTN(IR_CONV), tr2, IRCONV_NUM_INT); 665 t = IRT_NUM; 666 } 667 tr = emitir(IRT(op, t), tr, tr2); 668 } 669 J->base[0] = tr; 670 } 671 672 static void LJ_FASTCALL recff_math_random(jit_State *J, RecordFFData *rd) 673 { 674 GCudata *ud = udataV(&J->fn->c.upvalue[0]); 675 TRef tr, one; 676 lj_ir_kgc(J, obj2gco(ud), IRT_UDATA); /* Prevent collection. */ 677 tr = lj_ir_call(J, IRCALL_lj_math_random_step, lj_ir_kptr(J, uddata(ud))); 678 one = lj_ir_knum_one(J); 679 tr = emitir(IRTN(IR_SUB), tr, one); 680 if (J->base[0]) { 681 TRef tr1 = lj_ir_tonum(J, J->base[0]); 682 if (J->base[1]) { /* d = floor(d*(r2-r1+1.0)) + r1 */ 683 TRef tr2 = lj_ir_tonum(J, J->base[1]); 684 tr2 = emitir(IRTN(IR_SUB), tr2, tr1); 685 tr2 = emitir(IRTN(IR_ADD), tr2, one); 686 tr = emitir(IRTN(IR_MUL), tr, tr2); 687 tr = emitir(IRTN(IR_FPMATH), tr, IRFPM_FLOOR); 688 tr = emitir(IRTN(IR_ADD), tr, tr1); 689 } else { /* d = floor(d*r1) + 1.0 */ 690 tr = emitir(IRTN(IR_MUL), tr, tr1); 691 tr = emitir(IRTN(IR_FPMATH), tr, IRFPM_FLOOR); 692 tr = emitir(IRTN(IR_ADD), tr, one); 693 } 694 } 695 J->base[0] = tr; 696 UNUSED(rd); 697 } 698 699 /* -- Bit library fast functions ------------------------------------------ */ 700 701 /* Record bit.tobit. */ 702 static void LJ_FASTCALL recff_bit_tobit(jit_State *J, RecordFFData *rd) 703 { 704 TRef tr = J->base[0]; 705 #if LJ_HASFFI 706 if (tref_iscdata(tr)) { recff_bit64_tobit(J, rd); return; } 707 #endif 708 J->base[0] = lj_opt_narrow_tobit(J, tr); 709 UNUSED(rd); 710 } 711 712 /* Record unary bit.bnot, bit.bswap. */ 713 static void LJ_FASTCALL recff_bit_unary(jit_State *J, RecordFFData *rd) 714 { 715 #if LJ_HASFFI 716 if (recff_bit64_unary(J, rd)) 717 return; 718 #endif 719 J->base[0] = emitir(IRTI(rd->data), lj_opt_narrow_tobit(J, J->base[0]), 0); 720 } 721 722 /* Record N-ary bit.band, bit.bor, bit.bxor. */ 723 static void LJ_FASTCALL recff_bit_nary(jit_State *J, RecordFFData *rd) 724 { 725 #if LJ_HASFFI 726 if (recff_bit64_nary(J, rd)) 727 return; 728 #endif 729 { 730 TRef tr = lj_opt_narrow_tobit(J, J->base[0]); 731 uint32_t ot = IRTI(rd->data); 732 BCReg i; 733 for (i = 1; J->base[i] != 0; i++) 734 tr = emitir(ot, tr, lj_opt_narrow_tobit(J, J->base[i])); 735 J->base[0] = tr; 736 } 737 } 738 739 /* Record bit shifts. */ 740 static void LJ_FASTCALL recff_bit_shift(jit_State *J, RecordFFData *rd) 741 { 742 #if LJ_HASFFI 743 if (recff_bit64_shift(J, rd)) 744 return; 745 #endif 746 { 747 TRef tr = lj_opt_narrow_tobit(J, J->base[0]); 748 TRef tsh = lj_opt_narrow_tobit(J, J->base[1]); 749 IROp op = (IROp)rd->data; 750 if (!(op < IR_BROL ? LJ_TARGET_MASKSHIFT : LJ_TARGET_MASKROT) && 751 !tref_isk(tsh)) 752 tsh = emitir(IRTI(IR_BAND), tsh, lj_ir_kint(J, 31)); 753 #ifdef LJ_TARGET_UNIFYROT 754 if (op == (LJ_TARGET_UNIFYROT == 1 ? IR_BROR : IR_BROL)) { 755 op = LJ_TARGET_UNIFYROT == 1 ? IR_BROL : IR_BROR; 756 tsh = emitir(IRTI(IR_NEG), tsh, tsh); 757 } 758 #endif 759 J->base[0] = emitir(IRTI(op), tr, tsh); 760 } 761 } 762 763 static void LJ_FASTCALL recff_bit_tohex(jit_State *J, RecordFFData *rd) 764 { 765 #if LJ_HASFFI 766 TRef hdr = recff_bufhdr(J); 767 TRef tr = recff_bit64_tohex(J, rd, hdr); 768 J->base[0] = emitir(IRT(IR_BUFSTR, IRT_STR), tr, hdr); 769 #else 770 recff_nyiu(J, rd); /* Don't bother working around this NYI. */ 771 #endif 772 } 773 774 /* -- String library fast functions --------------------------------------- */ 775 776 /* Specialize to relative starting position for string. */ 777 static TRef recff_string_start(jit_State *J, GCstr *s, int32_t *st, TRef tr, 778 TRef trlen, TRef tr0) 779 { 780 int32_t start = *st; 781 if (start < 0) { 782 emitir(IRTGI(IR_LT), tr, tr0); 783 tr = emitir(IRTI(IR_ADD), trlen, tr); 784 start = start + (int32_t)s->len; 785 emitir(start < 0 ? IRTGI(IR_LT) : IRTGI(IR_GE), tr, tr0); 786 if (start < 0) { 787 tr = tr0; 788 start = 0; 789 } 790 } else if (start == 0) { 791 emitir(IRTGI(IR_EQ), tr, tr0); 792 tr = tr0; 793 } else { 794 tr = emitir(IRTI(IR_ADD), tr, lj_ir_kint(J, -1)); 795 emitir(IRTGI(IR_GE), tr, tr0); 796 start--; 797 } 798 *st = start; 799 return tr; 800 } 801 802 /* Handle string.byte (rd->data = 0) and string.sub (rd->data = 1). */ 803 static void LJ_FASTCALL recff_string_range(jit_State *J, RecordFFData *rd) 804 { 805 TRef trstr = lj_ir_tostr(J, J->base[0]); 806 TRef trlen = emitir(IRTI(IR_FLOAD), trstr, IRFL_STR_LEN); 807 TRef tr0 = lj_ir_kint(J, 0); 808 TRef trstart, trend; 809 GCstr *str = argv2str(J, &rd->argv[0]); 810 int32_t start, end; 811 if (rd->data) { /* string.sub(str, start [,end]) */ 812 start = argv2int(J, &rd->argv[1]); 813 trstart = lj_opt_narrow_toint(J, J->base[1]); 814 trend = J->base[2]; 815 if (tref_isnil(trend)) { 816 trend = lj_ir_kint(J, -1); 817 end = -1; 818 } else { 819 trend = lj_opt_narrow_toint(J, trend); 820 end = argv2int(J, &rd->argv[2]); 821 } 822 } else { /* string.byte(str, [,start [,end]]) */ 823 if (tref_isnil(J->base[1])) { 824 start = 1; 825 trstart = lj_ir_kint(J, 1); 826 } else { 827 start = argv2int(J, &rd->argv[1]); 828 trstart = lj_opt_narrow_toint(J, J->base[1]); 829 } 830 if (J->base[1] && !tref_isnil(J->base[2])) { 831 trend = lj_opt_narrow_toint(J, J->base[2]); 832 end = argv2int(J, &rd->argv[2]); 833 } else { 834 trend = trstart; 835 end = start; 836 } 837 } 838 if (end < 0) { 839 emitir(IRTGI(IR_LT), trend, tr0); 840 trend = emitir(IRTI(IR_ADD), emitir(IRTI(IR_ADD), trlen, trend), 841 lj_ir_kint(J, 1)); 842 end = end+(int32_t)str->len+1; 843 } else if ((MSize)end <= str->len) { 844 emitir(IRTGI(IR_ULE), trend, trlen); 845 } else { 846 emitir(IRTGI(IR_UGT), trend, trlen); 847 end = (int32_t)str->len; 848 trend = trlen; 849 } 850 trstart = recff_string_start(J, str, &start, trstart, trlen, tr0); 851 if (rd->data) { /* Return string.sub result. */ 852 if (end - start >= 0) { 853 /* Also handle empty range here, to avoid extra traces. */ 854 TRef trptr, trslen = emitir(IRTI(IR_SUB), trend, trstart); 855 emitir(IRTGI(IR_GE), trslen, tr0); 856 trptr = emitir(IRT(IR_STRREF, IRT_PGC), trstr, trstart); 857 J->base[0] = emitir(IRT(IR_SNEW, IRT_STR), trptr, trslen); 858 } else { /* Range underflow: return empty string. */ 859 emitir(IRTGI(IR_LT), trend, trstart); 860 J->base[0] = lj_ir_kstr(J, &J2G(J)->strempty); 861 } 862 } else { /* Return string.byte result(s). */ 863 ptrdiff_t i, len = end - start; 864 if (len > 0) { 865 TRef trslen = emitir(IRTI(IR_SUB), trend, trstart); 866 emitir(IRTGI(IR_EQ), trslen, lj_ir_kint(J, (int32_t)len)); 867 if (J->baseslot + len > LJ_MAX_JSLOTS) 868 lj_trace_err_info(J, LJ_TRERR_STACKOV); 869 rd->nres = len; 870 for (i = 0; i < len; i++) { 871 TRef tmp = emitir(IRTI(IR_ADD), trstart, lj_ir_kint(J, (int32_t)i)); 872 tmp = emitir(IRT(IR_STRREF, IRT_PGC), trstr, tmp); 873 J->base[i] = emitir(IRT(IR_XLOAD, IRT_U8), tmp, IRXLOAD_READONLY); 874 } 875 } else { /* Empty range or range underflow: return no results. */ 876 emitir(IRTGI(IR_LE), trend, trstart); 877 rd->nres = 0; 878 } 879 } 880 } 881 882 static void LJ_FASTCALL recff_string_char(jit_State *J, RecordFFData *rd) 883 { 884 TRef k255 = lj_ir_kint(J, 255); 885 BCReg i; 886 for (i = 0; J->base[i] != 0; i++) { /* Convert char values to strings. */ 887 TRef tr = lj_opt_narrow_toint(J, J->base[i]); 888 emitir(IRTGI(IR_ULE), tr, k255); 889 J->base[i] = emitir(IRT(IR_TOSTR, IRT_STR), tr, IRTOSTR_CHAR); 890 } 891 if (i > 1) { /* Concatenate the strings, if there's more than one. */ 892 TRef hdr = recff_bufhdr(J), tr = hdr; 893 for (i = 0; J->base[i] != 0; i++) 894 tr = emitir(IRT(IR_BUFPUT, IRT_PGC), tr, J->base[i]); 895 J->base[0] = emitir(IRT(IR_BUFSTR, IRT_STR), tr, hdr); 896 } 897 UNUSED(rd); 898 } 899 900 static void LJ_FASTCALL recff_string_rep(jit_State *J, RecordFFData *rd) 901 { 902 TRef str = lj_ir_tostr(J, J->base[0]); 903 TRef rep = lj_opt_narrow_toint(J, J->base[1]); 904 TRef hdr, tr, str2 = 0; 905 if (!tref_isnil(J->base[2])) { 906 TRef sep = lj_ir_tostr(J, J->base[2]); 907 int32_t vrep = argv2int(J, &rd->argv[1]); 908 emitir(IRTGI(vrep > 1 ? IR_GT : IR_LE), rep, lj_ir_kint(J, 1)); 909 if (vrep > 1) { 910 TRef hdr2 = recff_bufhdr(J); 911 TRef tr2 = emitir(IRT(IR_BUFPUT, IRT_PGC), hdr2, sep); 912 tr2 = emitir(IRT(IR_BUFPUT, IRT_PGC), tr2, str); 913 str2 = emitir(IRT(IR_BUFSTR, IRT_STR), tr2, hdr2); 914 } 915 } 916 tr = hdr = recff_bufhdr(J); 917 if (str2) { 918 tr = emitir(IRT(IR_BUFPUT, IRT_PGC), tr, str); 919 str = str2; 920 rep = emitir(IRTI(IR_ADD), rep, lj_ir_kint(J, -1)); 921 } 922 tr = lj_ir_call(J, IRCALL_lj_buf_putstr_rep, tr, str, rep); 923 J->base[0] = emitir(IRT(IR_BUFSTR, IRT_STR), tr, hdr); 924 } 925 926 static void LJ_FASTCALL recff_string_op(jit_State *J, RecordFFData *rd) 927 { 928 TRef str = lj_ir_tostr(J, J->base[0]); 929 TRef hdr = recff_bufhdr(J); 930 TRef tr = lj_ir_call(J, rd->data, hdr, str); 931 J->base[0] = emitir(IRT(IR_BUFSTR, IRT_STR), tr, hdr); 932 } 933 934 static int recff_emit_captures(jit_State *J, const MatchState *ms, 935 TRef tr, int off, TRef sptr) 936 { 937 TRef captures; 938 int i; 939 int capoff = offsetof(MatchState, capture[0]); 940 int capsize = sizeof(ms->capture[0]); 941 int initoff = offsetof(MatchState, capture[0].init) - capoff; 942 int lenoff = offsetof(MatchState, capture[0].len) - capoff; 943 944 if (!ms->level) 945 return 0; 946 947 captures = emitir(IRT(IR_ADD, IRT_P32), tr, /* IRFL_ not really applicable. */ 948 lj_ir_kint(J, capoff)); 949 for (i = 0; i < ms->level; i++) { 950 int init = i*capsize+initoff; 951 if (ms->capture[i].len == CAP_POSITION) { 952 J->base[off+i] = 953 emitir(IRT(IR_SUB, IRT_INT), 954 emitir(IRT(IR_XLOAD, IRT_P32), 955 emitir(IRT(IR_ADD, IRT_P32), captures, lj_ir_kint(J, init)), 0), 956 emitir(IRT(IR_SUB, IRT_P32), sptr, lj_ir_kint(J, 1)) 957 ); /* IR_SUB */ 958 } else { 959 int len = i*capsize+lenoff; 960 J->base[off+i] = 961 emitir(IRT(IR_SNEW, IRT_STR), 962 emitir(IRT(IR_XLOAD, IRT_P32), 963 emitir(IRT(IR_ADD, IRT_P32), captures, lj_ir_kint(J, init)), 0), 964 emitir(IRT(IR_XLOAD, IRT_INT), 965 emitir(IRT(IR_ADD, IRT_P32), captures, lj_ir_kint(J, len)), 0) 966 ); /* IR_SNEW */ 967 } 968 } 969 return ms->level; 970 } 971 972 static void LJ_FASTCALL recff_string_findmatch(jit_State *J, RecordFFData *rd) 973 { 974 int find = rd->data; 975 TRef tr0, trstr, trsptr, trslen, trplen, trpat, trpptr; 976 TRef trstart; 977 int32_t start; 978 GCstr *str = argv2str(J, &rd->argv[0]); 979 GCstr *pat = argv2str(J, &rd->argv[1]); 980 TRef kpat = 0; 981 int rawfind = 0; 982 983 J->needsnap = 1; 984 985 tr0 = lj_ir_kint(J, 0); 986 trstr = lj_ir_tostr(J, J->base[0]); 987 trpat = lj_ir_tostr(J, J->base[1]); 988 989 /* Optional start argument. */ 990 if (tref_isnil(J->base[2])) { 991 trstart = lj_ir_kint(J, 1); 992 start = 1; 993 } else { 994 trstart = lj_opt_narrow_toint(J, J->base[2]); 995 start = argv2int(J, &rd->argv[2]); 996 } 997 998 /* Specialize on pattern only if no raw flag is specified. */ 999 if (!find || !(rawfind = (J->base[2] && tref_istruecond(J->base[3])))) { 1000 kpat = lj_ir_kstr(J, pat); 1001 emitir(IRTG(IR_EQ, IRT_STR), trpat, kpat); 1002 trpat = kpat; 1003 } 1004 1005 trsptr = emitir(IRT(IR_STRREF, IRT_P32), trstr, tr0); 1006 trslen = emitir(IRTI(IR_FLOAD), trstr, IRFL_STR_LEN); 1007 trpptr = emitir(IRT(IR_STRREF, IRT_P32), trpat, tr0); 1008 trplen = emitir(IRTI(IR_FLOAD), trpat, IRFL_STR_LEN); 1009 1010 if (rawfind || !lj_str_haspattern(pat)) { 1011 TRef tr = lj_ir_call(J, IRCALL_lj_str_find, 1012 trsptr, trpptr, trslen, trplen, trstart); 1013 if (lj_str_find(strdata(str), strdata(pat), 1014 str->len, pat->len, start)) { 1015 emitir(IRTG(IR_NE, IRT_INT), tr, tr0); 1016 if (!find) 1017 J->base[0] = kpat; 1018 else { 1019 J->base[0] = tr; 1020 J->base[1] = emitir(IRTI(IR_ADD), tr, emitir(IRTI(IR_SUB), trplen, 1021 lj_ir_kint(J, 1))); 1022 rd->nres = 2; 1023 } 1024 } else { 1025 emitir(IRTG(IR_EQ, IRT_INT), tr, tr0); 1026 J->base[0] = TREF_NIL; 1027 } 1028 } else { 1029 TRef tr = lj_ir_call(J, IRCALL_ljx_str_match, 1030 trsptr, trpptr, trslen, trplen, trstart); 1031 TRef trp0 = lj_ir_kkptr(J, NULL); 1032 MatchState *ms = ljx_str_match(J->L, strdata(str), strdata(pat), 1033 str->len, pat->len, start); 1034 if (ms) { 1035 int rpos = 0; 1036 emitir(IRTG(IR_NE, IRT_P32), tr, trp0); 1037 if (find) { 1038 J->base[0] = emitir(IRTI(IR_FLOAD), tr, IRFL_MS_FINDRET1); 1039 J->base[1] = emitir(IRTI(IR_FLOAD), tr, IRFL_MS_FINDRET2); 1040 rpos = 2; 1041 } 1042 rd->nres = rpos + recff_emit_captures(J, ms, tr, rpos, trsptr); 1043 } else { 1044 emitir(IRTG(IR_EQ, IRT_PGC), tr, trp0); 1045 J->base[0] = TREF_NIL; 1046 } 1047 } 1048 } 1049 1050 static void LJ_FASTCALL recff_string_format(jit_State *J, RecordFFData *rd) 1051 { 1052 TRef trfmt = lj_ir_tostr(J, J->base[0]); 1053 GCstr *fmt = argv2str(J, &rd->argv[0]); 1054 int arg = 1; 1055 TRef hdr, tr; 1056 FormatState fs; 1057 SFormat sf; 1058 /* Specialize to the format string. */ 1059 emitir(IRTG(IR_EQ, IRT_STR), trfmt, lj_ir_kstr(J, fmt)); 1060 tr = hdr = recff_bufhdr(J); 1061 lj_strfmt_init(&fs, strdata(fmt), fmt->len); 1062 while ((sf = lj_strfmt_parse(&fs)) != STRFMT_EOF) { /* Parse format. */ 1063 TRef tra = sf == STRFMT_LIT ? 0 : J->base[arg++]; 1064 TRef trsf = lj_ir_kint(J, (int32_t)sf); 1065 IRCallID id; 1066 switch (STRFMT_TYPE(sf)) { 1067 case STRFMT_LIT: 1068 tr = emitir(IRT(IR_BUFPUT, IRT_PGC), tr, 1069 lj_ir_kstr(J, lj_str_new(J->L, fs.str, fs.len))); 1070 break; 1071 case STRFMT_INT: 1072 id = IRCALL_lj_strfmt_putfnum_int; 1073 handle_int: 1074 if (!tref_isinteger(tra)) 1075 goto handle_num; 1076 if (sf == STRFMT_INT) { /* Shortcut for plain %d. */ 1077 tr = emitir(IRT(IR_BUFPUT, IRT_PGC), tr, 1078 emitir(IRT(IR_TOSTR, IRT_STR), tra, IRTOSTR_INT)); 1079 } else { 1080 #if LJ_HASFFI 1081 tra = emitir(IRT(IR_CONV, IRT_U64), tra, 1082 (IRT_INT|(IRT_U64<<5)|IRCONV_SEXT)); 1083 tr = lj_ir_call(J, IRCALL_lj_strfmt_putfxint, tr, trsf, tra); 1084 lj_needsplit(J); 1085 #else 1086 recff_nyiu(J, rd); /* Don't bother working around this NYI. */ 1087 return; 1088 #endif 1089 } 1090 break; 1091 case STRFMT_UINT: 1092 id = IRCALL_lj_strfmt_putfnum_uint; 1093 goto handle_int; 1094 case STRFMT_NUM: 1095 id = IRCALL_lj_strfmt_putfnum; 1096 handle_num: 1097 tra = lj_ir_tonum(J, tra); 1098 tr = lj_ir_call(J, id, tr, trsf, tra); 1099 if (LJ_SOFTFP) lj_needsplit(J); 1100 break; 1101 case STRFMT_STR: 1102 if (!tref_isstr(tra)) { 1103 recff_nyiu(J, rd); /* NYI: __tostring and non-string types for %s. */ 1104 return; 1105 } 1106 if (sf == STRFMT_STR) /* Shortcut for plain %s. */ 1107 tr = emitir(IRT(IR_BUFPUT, IRT_PGC), tr, tra); 1108 else if ((sf & STRFMT_T_QUOTED)) 1109 tr = lj_ir_call(J, IRCALL_lj_strfmt_putquoted, tr, tra); 1110 else 1111 tr = lj_ir_call(J, IRCALL_lj_strfmt_putfstr, tr, trsf, tra); 1112 break; 1113 case STRFMT_CHAR: 1114 tra = lj_opt_narrow_toint(J, tra); 1115 if (sf == STRFMT_CHAR) /* Shortcut for plain %c. */ 1116 tr = emitir(IRT(IR_BUFPUT, IRT_PGC), tr, 1117 emitir(IRT(IR_TOSTR, IRT_STR), tra, IRTOSTR_CHAR)); 1118 else 1119 tr = lj_ir_call(J, IRCALL_lj_strfmt_putfchar, tr, trsf, tra); 1120 break; 1121 case STRFMT_PTR: /* NYI */ 1122 case STRFMT_ERR: 1123 default: 1124 recff_nyiu(J, rd); 1125 return; 1126 } 1127 } 1128 J->base[0] = emitir(IRT(IR_BUFSTR, IRT_STR), tr, hdr); 1129 } 1130 1131 /* -- Table library fast functions ---------------------------------------- */ 1132 1133 static void LJ_FASTCALL recff_table_insert(jit_State *J, RecordFFData *rd) 1134 { 1135 RecordIndex ix; 1136 ix.tab = J->base[0]; 1137 ix.val = J->base[1]; 1138 rd->nres = 0; 1139 if (tref_istab(ix.tab) && ix.val) { 1140 if (!J->base[2]) { /* Simple push: t[#t+1] = v */ 1141 TRef trlen = lj_ir_call(J, IRCALL_lj_tab_len, ix.tab); 1142 GCtab *t = tabV(&rd->argv[0]); 1143 ix.key = emitir(IRTI(IR_ADD), trlen, lj_ir_kint(J, 1)); 1144 settabV(J->L, &ix.tabv, t); 1145 setintV(&ix.keyv, lj_tab_len(t) + 1); 1146 ix.idxchain = 0; 1147 lj_record_idx(J, &ix); /* Set new value. */ 1148 } else { /* Complex case: insert in the middle. */ 1149 recff_nyiu(J, rd); 1150 return; 1151 } 1152 } /* else: Interpreter will throw. */ 1153 } 1154 1155 static void LJ_FASTCALL recff_table_concat(jit_State *J, RecordFFData *rd) 1156 { 1157 TRef tab = J->base[0]; 1158 if (tref_istab(tab)) { 1159 TRef sep = !tref_isnil(J->base[1]) ? 1160 lj_ir_tostr(J, J->base[1]) : lj_ir_knull(J, IRT_STR); 1161 TRef tri = (J->base[1] && !tref_isnil(J->base[2])) ? 1162 lj_opt_narrow_toint(J, J->base[2]) : lj_ir_kint(J, 1); 1163 TRef tre = (J->base[1] && J->base[2] && !tref_isnil(J->base[3])) ? 1164 lj_opt_narrow_toint(J, J->base[3]) : 1165 lj_ir_call(J, IRCALL_lj_tab_len, tab); 1166 TRef hdr = recff_bufhdr(J); 1167 TRef tr = lj_ir_call(J, IRCALL_lj_buf_puttab, hdr, tab, sep, tri, tre); 1168 emitir(IRTG(IR_NE, IRT_PTR), tr, lj_ir_kptr(J, NULL)); 1169 J->base[0] = emitir(IRT(IR_BUFSTR, IRT_STR), tr, hdr); 1170 } /* else: Interpreter will throw. */ 1171 UNUSED(rd); 1172 } 1173 1174 static void LJ_FASTCALL recff_table_new(jit_State *J, RecordFFData *rd) 1175 { 1176 TRef tra = lj_opt_narrow_toint(J, J->base[0]); 1177 TRef trh = lj_opt_narrow_toint(J, J->base[1]); 1178 J->base[0] = lj_ir_call(J, IRCALL_lj_tab_new_ah, tra, trh); 1179 UNUSED(rd); 1180 } 1181 1182 static void LJ_FASTCALL recff_table_clear(jit_State *J, RecordFFData *rd) 1183 { 1184 TRef tr = J->base[0]; 1185 if (tref_istab(tr)) { 1186 rd->nres = 0; 1187 lj_ir_call(J, IRCALL_lj_tab_clear, tr); 1188 J->needsnap = 1; 1189 } /* else: Interpreter will throw. */ 1190 } 1191 1192 /* -- I/O library fast functions ------------------------------------------ */ 1193 1194 /* Get FILE* for I/O function. Any I/O error aborts recording, so there's 1195 ** no need to encode the alternate cases for any of the guards. 1196 */ 1197 static TRef recff_io_fp(jit_State *J, TRef *udp, int32_t id) 1198 { 1199 TRef tr, ud, fp; 1200 if (id) { /* io.func() */ 1201 #if LJ_GC64 1202 /* TODO: fix ARM32 asm_fload(), so we can use this for all archs. */ 1203 ud = lj_ir_ggfload(J, IRT_UDATA, GG_OFS(g.gcroot[id])); 1204 #else 1205 tr = lj_ir_kptr(J, &J2G(J)->gcroot[id]); 1206 ud = emitir(IRT(IR_XLOAD, IRT_UDATA), tr, 0); 1207 #endif 1208 } else { /* fp:method() */ 1209 ud = J->base[0]; 1210 if (!tref_isudata(ud)) 1211 lj_trace_err(J, LJ_TRERR_BADTYPE); 1212 tr = emitir(IRT(IR_FLOAD, IRT_U8), ud, IRFL_UDATA_UDTYPE); 1213 emitir(IRTGI(IR_EQ), tr, lj_ir_kint(J, UDTYPE_IO_FILE)); 1214 } 1215 *udp = ud; 1216 fp = emitir(IRT(IR_FLOAD, IRT_PTR), ud, IRFL_UDATA_FILE); 1217 emitir(IRTG(IR_NE, IRT_PTR), fp, lj_ir_knull(J, IRT_PTR)); 1218 return fp; 1219 } 1220 1221 static void LJ_FASTCALL recff_io_write(jit_State *J, RecordFFData *rd) 1222 { 1223 TRef ud, fp = recff_io_fp(J, &ud, rd->data); 1224 TRef zero = lj_ir_kint(J, 0); 1225 TRef one = lj_ir_kint(J, 1); 1226 ptrdiff_t i = rd->data == 0 ? 1 : 0; 1227 for (; J->base[i]; i++) { 1228 TRef str = lj_ir_tostr(J, J->base[i]); 1229 TRef buf = emitir(IRT(IR_STRREF, IRT_PGC), str, zero); 1230 TRef len = emitir(IRTI(IR_FLOAD), str, IRFL_STR_LEN); 1231 if (tref_isk(len) && IR(tref_ref(len))->i == 1) { 1232 IRIns *irs = IR(tref_ref(str)); 1233 TRef tr = (irs->o == IR_TOSTR && irs->op2 == IRTOSTR_CHAR) ? 1234 irs->op1 : 1235 emitir(IRT(IR_XLOAD, IRT_U8), buf, IRXLOAD_READONLY); 1236 tr = lj_ir_call(J, IRCALL_fputc, tr, fp); 1237 if (results_wanted(J) != 0) /* Check result only if not ignored. */ 1238 emitir(IRTGI(IR_NE), tr, lj_ir_kint(J, -1)); 1239 } else { 1240 TRef tr = lj_ir_call(J, IRCALL_fwrite, buf, one, len, fp); 1241 if (results_wanted(J) != 0) /* Check result only if not ignored. */ 1242 emitir(IRTGI(IR_EQ), tr, len); 1243 } 1244 } 1245 J->base[0] = LJ_51 ? TREF_TRUE : ud; 1246 } 1247 1248 static void LJ_FASTCALL recff_io_flush(jit_State *J, RecordFFData *rd) 1249 { 1250 TRef ud, fp = recff_io_fp(J, &ud, rd->data); 1251 TRef tr = lj_ir_call(J, IRCALL_fflush, fp); 1252 if (results_wanted(J) != 0) /* Check result only if not ignored. */ 1253 emitir(IRTGI(IR_EQ), tr, lj_ir_kint(J, 0)); 1254 J->base[0] = TREF_TRUE; 1255 } 1256 1257 /* -- Debug library fast functions ---------------------------------------- */ 1258 1259 static void LJ_FASTCALL recff_debug_getmetatable(jit_State *J, RecordFFData *rd) 1260 { 1261 GCtab *mt; 1262 TRef mtref; 1263 TRef tr = J->base[0]; 1264 if (tref_istab(tr)) { 1265 mt = tabref(tabV(&rd->argv[0])->metatable); 1266 mtref = emitir(IRT(IR_FLOAD, IRT_TAB), tr, IRFL_TAB_META); 1267 } else if (tref_isudata(tr)) { 1268 mt = tabref(udataV(&rd->argv[0])->metatable); 1269 mtref = emitir(IRT(IR_FLOAD, IRT_TAB), tr, IRFL_UDATA_META); 1270 } else { 1271 mt = tabref(basemt_obj(J2G(J), &rd->argv[0])); 1272 J->base[0] = mt ? lj_ir_ktab(J, mt) : TREF_NIL; 1273 return; 1274 } 1275 emitir(IRTG(mt ? IR_NE : IR_EQ, IRT_TAB), mtref, lj_ir_knull(J, IRT_TAB)); 1276 J->base[0] = mt ? mtref : TREF_NIL; 1277 } 1278 1279 static void LJ_FASTCALL recff_debug_setmetatable(jit_State *J, RecordFFData *rd) 1280 { 1281 TRef tr = J->base[0]; 1282 TRef mt = J->base[1]; 1283 /* Note: lua_setmetatable for base_mt provides no benefit. */ 1284 if (tref_istab(tr) && (tref_istab(mt) || (mt && tref_isnil(mt)))) { 1285 TRef fref, mtref; 1286 fref = emitir(IRT(IR_FREF, IRT_P32), tr, IRFL_TAB_META); 1287 mtref = tref_isnil(mt) ? lj_ir_knull(J, IRT_TAB) : mt; 1288 emitir(IRT(IR_FSTORE, IRT_TAB), fref, mtref); 1289 if (!tref_isnil(mt)) { 1290 emitir(IRT(IR_TBAR, IRT_TAB), tr, 0); 1291 #if !LJ_51 1292 setmetatable_gc_guard(J, rd); 1293 #endif 1294 } 1295 J->base[0] = tr; 1296 J->needsnap = 1; 1297 } else { 1298 recff_nyiu(J, rd); 1299 } 1300 } 1301 1302 1303 /* -- Record calls to fast functions -------------------------------------- */ 1304 1305 #include "lj_recdef.h" 1306 1307 static uint32_t recdef_lookup(GCfunc *fn) 1308 { 1309 if (fn->c.ffid < sizeof(recff_idmap)/sizeof(recff_idmap[0])) 1310 return recff_idmap[fn->c.ffid]; 1311 else 1312 return 0; 1313 } 1314 1315 /* Record entry to a fast function or C function. */ 1316 void lj_ffrecord_func(jit_State *J) 1317 { 1318 RecordFFData rd; 1319 uint32_t m = recdef_lookup(J->fn); 1320 rd.data = m & 0xff; 1321 rd.nres = 1; /* Default is one result. */ 1322 rd.argv = J->L->base; 1323 J->base[J->maxslot] = 0; /* Mark end of arguments. */ 1324 (recff_func[m >> 8])(J, &rd); /* Call recff_* handler. */ 1325 if (rd.nres >= 0) { 1326 if (J->postproc == LJ_POST_NONE) J->postproc = LJ_POST_FFRETRY; 1327 lj_record_ret(J, 0, rd.nres); 1328 } 1329 } 1330 1331 #undef IR 1332 #undef emitir 1333 1334 #endif