lj_ccallback.c (21973B)
1 /* 2 ** FFI C callback handling. 3 ** Copyright (C) 2005-2016 Mike Pall. See Copyright Notice in luajit.h 4 */ 5 6 #include "lj_obj.h" 7 8 #if LJ_HASFFI 9 10 #include "lj_gc.h" 11 #include "lj_err.h" 12 #include "lj_tab.h" 13 #include "lj_state.h" 14 #include "lj_frame.h" 15 #include "lj_ctype.h" 16 #include "lj_cconv.h" 17 #include "lj_ccall.h" 18 #include "lj_ccallback.h" 19 #include "lj_target.h" 20 #include "lj_mcode.h" 21 #include "lj_trace.h" 22 #include "lj_vm.h" 23 24 /* -- Target-specific handling of callback slots -------------------------- */ 25 26 #define CALLBACK_MCODE_SIZE (LJ_PAGESIZE * LJ_NUM_CBPAGE) 27 28 #if LJ_OS_NOJIT 29 30 /* Callbacks disabled. */ 31 #define CALLBACK_SLOT2OFS(slot) (0*(slot)) 32 #define CALLBACK_OFS2SLOT(ofs) (0*(ofs)) 33 #define CALLBACK_MAX_SLOT 0 34 35 #elif LJ_TARGET_X86ORX64 36 37 #define CALLBACK_MCODE_HEAD (LJ_64 ? 8 : 0) 38 #define CALLBACK_MCODE_GROUP (-2+1+2+(LJ_GC64 ? 10 : 5)+(LJ_64 ? 6 : 5)) 39 40 #define CALLBACK_SLOT2OFS(slot) \ 41 (CALLBACK_MCODE_HEAD + CALLBACK_MCODE_GROUP*((slot)/32) + 4*(slot)) 42 43 static MSize CALLBACK_OFS2SLOT(MSize ofs) 44 { 45 MSize group; 46 ofs -= CALLBACK_MCODE_HEAD; 47 group = ofs / (32*4 + CALLBACK_MCODE_GROUP); 48 return (ofs % (32*4 + CALLBACK_MCODE_GROUP))/4 + group*32; 49 } 50 51 #define CALLBACK_MAX_SLOT \ 52 (((CALLBACK_MCODE_SIZE-CALLBACK_MCODE_HEAD)/(CALLBACK_MCODE_GROUP+4*32))*32) 53 54 #elif LJ_TARGET_ARM 55 56 #define CALLBACK_MCODE_HEAD 32 57 58 #elif LJ_TARGET_ARM64 59 60 #define CALLBACK_MCODE_HEAD 32 61 62 #elif LJ_TARGET_PPC 63 64 #define CALLBACK_MCODE_HEAD 24 65 66 #elif LJ_TARGET_MIPS32 67 68 #define CALLBACK_MCODE_HEAD 20 69 70 #elif LJ_TARGET_MIPS64 71 72 #define CALLBACK_MCODE_HEAD 52 73 74 #else 75 76 /* Missing support for this architecture. */ 77 #define CALLBACK_SLOT2OFS(slot) (0*(slot)) 78 #define CALLBACK_OFS2SLOT(ofs) (0*(ofs)) 79 #define CALLBACK_MAX_SLOT 0 80 81 #endif 82 83 #ifndef CALLBACK_SLOT2OFS 84 #define CALLBACK_SLOT2OFS(slot) (CALLBACK_MCODE_HEAD + 8*(slot)) 85 #define CALLBACK_OFS2SLOT(ofs) (((ofs)-CALLBACK_MCODE_HEAD)/8) 86 #define CALLBACK_MAX_SLOT (CALLBACK_OFS2SLOT(CALLBACK_MCODE_SIZE)) 87 #endif 88 89 /* Convert callback slot number to callback function pointer. */ 90 static void *callback_slot2ptr(CTState *cts, MSize slot) 91 { 92 return (uint8_t *)cts->cb.mcode + CALLBACK_SLOT2OFS(slot); 93 } 94 95 /* Convert callback function pointer to slot number. */ 96 MSize lj_ccallback_ptr2slot(CTState *cts, void *p) 97 { 98 uintptr_t ofs = (uintptr_t)((uint8_t *)p -(uint8_t *)cts->cb.mcode); 99 if (ofs < CALLBACK_MCODE_SIZE) { 100 MSize slot = CALLBACK_OFS2SLOT((MSize)ofs); 101 if (CALLBACK_SLOT2OFS(slot) == (MSize)ofs) 102 return slot; 103 } 104 return ~0u; /* Not a known callback function pointer. */ 105 } 106 107 /* Initialize machine code for callback function pointers. */ 108 #if LJ_OS_NOJIT 109 /* Disabled callback support. */ 110 #define callback_mcode_init(g, p) UNUSED(p) 111 #elif LJ_TARGET_X86ORX64 112 static void callback_mcode_init(global_State *g, uint8_t *page) 113 { 114 uint8_t *p = page; 115 uint8_t *target = (uint8_t *)(void *)lj_vm_ffi_callback; 116 MSize slot; 117 #if LJ_64 118 *(void **)p = target; p += 8; 119 #endif 120 for (slot = 0; slot < CALLBACK_MAX_SLOT; slot++) { 121 /* mov al, slot; jmp group */ 122 *p++ = XI_MOVrib | RID_EAX; *p++ = (uint8_t)slot; 123 if ((slot & 31) == 31 || slot == CALLBACK_MAX_SLOT-1) { 124 /* push ebp/rbp; mov ah, slot>>8; mov ebp, &g. */ 125 *p++ = XI_PUSH + RID_EBP; 126 *p++ = XI_MOVrib | (RID_EAX+4); *p++ = (uint8_t)(slot >> 8); 127 #if LJ_GC64 128 *p++ = 0x48; *p++ = XI_MOVri | RID_EBP; 129 *(uint64_t *)p = (uint64_t)(g); p += 8; 130 #else 131 *p++ = XI_MOVri | RID_EBP; 132 *(int32_t *)p = i32ptr(g); p += 4; 133 #endif 134 #if LJ_64 135 /* jmp [rip-pageofs] where lj_vm_ffi_callback is stored. */ 136 *p++ = XI_GROUP5; *p++ = XM_OFS0 + (XOg_JMP<<3) + RID_EBP; 137 *(int32_t *)p = (int32_t)(page-(p+4)); p += 4; 138 #else 139 /* jmp lj_vm_ffi_callback. */ 140 *p++ = XI_JMP; *(int32_t *)p = target-(p+4); p += 4; 141 #endif 142 } else { 143 *p++ = XI_JMPs; *p++ = (uint8_t)((2+2)*(31-(slot&31)) - 2); 144 } 145 } 146 lua_assert(p - page <= CALLBACK_MCODE_SIZE); 147 } 148 #elif LJ_TARGET_ARM 149 static void callback_mcode_init(global_State *g, uint32_t *page) 150 { 151 uint32_t *p = page; 152 void *target = (void *)lj_vm_ffi_callback; 153 MSize slot; 154 /* This must match with the saveregs macro in buildvm_arm.dasc. */ 155 *p++ = ARMI_SUB|ARMF_D(RID_R12)|ARMF_N(RID_R12)|ARMF_M(RID_PC); 156 *p++ = ARMI_PUSH|ARMF_N(RID_SP)|RSET_RANGE(RID_R4,RID_R11+1)|RID2RSET(RID_LR); 157 *p++ = ARMI_SUB|ARMI_K12|ARMF_D(RID_R12)|ARMF_N(RID_R12)|CALLBACK_MCODE_HEAD; 158 *p++ = ARMI_STR|ARMI_LS_P|ARMI_LS_W|ARMF_D(RID_R12)|ARMF_N(RID_SP)|(CFRAME_SIZE-4*9); 159 *p++ = ARMI_LDR|ARMI_LS_P|ARMI_LS_U|ARMF_D(RID_R12)|ARMF_N(RID_PC); 160 *p++ = ARMI_LDR|ARMI_LS_P|ARMI_LS_U|ARMF_D(RID_PC)|ARMF_N(RID_PC); 161 *p++ = u32ptr(g); 162 *p++ = u32ptr(target); 163 for (slot = 0; slot < CALLBACK_MAX_SLOT; slot++) { 164 *p++ = ARMI_MOV|ARMF_D(RID_R12)|ARMF_M(RID_PC); 165 *p = ARMI_B | ((page-p-2) & 0x00ffffffu); 166 p++; 167 } 168 lua_assert(p - page <= CALLBACK_MCODE_SIZE); 169 } 170 #elif LJ_TARGET_ARM64 171 static void callback_mcode_init(global_State *g, uint32_t *page) 172 { 173 uint32_t *p = page; 174 void *target = (void *)lj_vm_ffi_callback; 175 MSize slot; 176 *p++ = A64I_LDRLx | A64F_D(RID_X11) | A64F_S19(4); 177 *p++ = A64I_LDRLx | A64F_D(RID_X10) | A64F_S19(5); 178 *p++ = A64I_BR | A64F_N(RID_X11); 179 *p++ = A64I_NOP; 180 ((void **)p)[0] = target; 181 ((void **)p)[1] = g; 182 p += 4; 183 for (slot = 0; slot < CALLBACK_MAX_SLOT; slot++) { 184 *p++ = A64I_MOVZw | A64F_D(RID_X9) | A64F_U16(slot); 185 *p = A64I_B | A64F_S26((page-p) & 0x03ffffffu); 186 p++; 187 } 188 lua_assert(p - page <= CALLBACK_MCODE_SIZE); 189 } 190 #elif LJ_TARGET_PPC 191 static void callback_mcode_init(global_State *g, uint32_t *page) 192 { 193 uint32_t *p = page; 194 void *target = (void *)lj_vm_ffi_callback; 195 MSize slot; 196 *p++ = PPCI_LIS | PPCF_T(RID_TMP) | (u32ptr(target) >> 16); 197 *p++ = PPCI_LIS | PPCF_T(RID_R12) | (u32ptr(g) >> 16); 198 *p++ = PPCI_ORI | PPCF_A(RID_TMP)|PPCF_T(RID_TMP) | (u32ptr(target) & 0xffff); 199 *p++ = PPCI_ORI | PPCF_A(RID_R12)|PPCF_T(RID_R12) | (u32ptr(g) & 0xffff); 200 *p++ = PPCI_MTCTR | PPCF_T(RID_TMP); 201 *p++ = PPCI_BCTR; 202 for (slot = 0; slot < CALLBACK_MAX_SLOT; slot++) { 203 *p++ = PPCI_LI | PPCF_T(RID_R11) | slot; 204 *p = PPCI_B | (((page-p) & 0x00ffffffu) << 2); 205 p++; 206 } 207 lua_assert(p - page <= CALLBACK_MCODE_SIZE); 208 } 209 #elif LJ_TARGET_MIPS 210 static void callback_mcode_init(global_State *g, uint32_t *page) 211 { 212 uint32_t *p = page; 213 uintptr_t target = (uintptr_t)(void *)lj_vm_ffi_callback; 214 uintptr_t ug = (uintptr_t)(void *)g; 215 MSize slot; 216 #if LJ_TARGET_MIPS32 217 *p++ = MIPSI_LUI | MIPSF_T(RID_R3) | (target >> 16); 218 *p++ = MIPSI_LUI | MIPSF_T(RID_R2) | (ug >> 16); 219 #else 220 *p++ = MIPSI_LUI | MIPSF_T(RID_R3) | (target >> 48); 221 *p++ = MIPSI_LUI | MIPSF_T(RID_R2) | (ug >> 48); 222 *p++ = MIPSI_ORI | MIPSF_T(RID_R3)|MIPSF_S(RID_R3) | ((target >> 32) & 0xffff); 223 *p++ = MIPSI_ORI | MIPSF_T(RID_R2)|MIPSF_S(RID_R2) | ((ug >> 32) & 0xffff); 224 *p++ = MIPSI_DSLL | MIPSF_D(RID_R3)|MIPSF_T(RID_R3) | MIPSF_A(16); 225 *p++ = MIPSI_DSLL | MIPSF_D(RID_R2)|MIPSF_T(RID_R2) | MIPSF_A(16); 226 *p++ = MIPSI_ORI | MIPSF_T(RID_R3)|MIPSF_S(RID_R3) | ((target >> 16) & 0xffff); 227 *p++ = MIPSI_ORI | MIPSF_T(RID_R2)|MIPSF_S(RID_R2) | ((ug >> 16) & 0xffff); 228 *p++ = MIPSI_DSLL | MIPSF_D(RID_R3)|MIPSF_T(RID_R3) | MIPSF_A(16); 229 *p++ = MIPSI_DSLL | MIPSF_D(RID_R2)|MIPSF_T(RID_R2) | MIPSF_A(16); 230 #endif 231 *p++ = MIPSI_ORI | MIPSF_T(RID_R3)|MIPSF_S(RID_R3) | (target & 0xffff); 232 *p++ = MIPSI_JR | MIPSF_S(RID_R3); 233 *p++ = MIPSI_ORI | MIPSF_T(RID_R2)|MIPSF_S(RID_R2) | (ug & 0xffff); 234 for (slot = 0; slot < CALLBACK_MAX_SLOT; slot++) { 235 *p = MIPSI_B | ((page-p-1) & 0x0000ffffu); 236 p++; 237 *p++ = MIPSI_LI | MIPSF_T(RID_R1) | slot; 238 } 239 lua_assert(p - page <= CALLBACK_MCODE_SIZE); 240 } 241 #else 242 /* Missing support for this architecture. */ 243 #define callback_mcode_init(g, p) UNUSED(p) 244 #endif 245 246 /* -- Machine code management --------------------------------------------- */ 247 248 #if LJ_TARGET_WINDOWS 249 250 #define WIN32_LEAN_AND_MEAN 251 #include <windows.h> 252 253 #elif LJ_TARGET_POSIX 254 255 #include <sys/mman.h> 256 #ifndef MAP_ANONYMOUS 257 #define MAP_ANONYMOUS MAP_ANON 258 #endif 259 260 #endif 261 262 /* Allocate and initialize area for callback function pointers. */ 263 static void callback_mcode_new(CTState *cts) 264 { 265 size_t sz = (size_t)CALLBACK_MCODE_SIZE; 266 void *p; 267 if (CALLBACK_MAX_SLOT == 0) 268 lj_err_caller(cts->L, LJ_ERR_FFI_CBACKOV); 269 #if LJ_TARGET_WINDOWS 270 p = VirtualAlloc(NULL, sz, MEM_RESERVE|MEM_COMMIT, PAGE_READWRITE); 271 if (!p) 272 lj_err_caller(cts->L, LJ_ERR_FFI_CBACKOV); 273 #elif LJ_TARGET_POSIX 274 p = mmap(NULL, sz, (PROT_READ|PROT_WRITE), MAP_PRIVATE|MAP_ANONYMOUS, 275 -1, 0); 276 if (p == MAP_FAILED) 277 lj_err_caller(cts->L, LJ_ERR_FFI_CBACKOV); 278 #else 279 /* Fallback allocator. Fails if memory is not executable by default. */ 280 p = lj_mem_new(cts->L, sz); 281 #endif 282 cts->cb.mcode = p; 283 callback_mcode_init(cts->g, p); 284 lj_mcode_sync(p, (char *)p + sz); 285 #if LJ_TARGET_WINDOWS 286 { 287 DWORD oprot; 288 VirtualProtect(p, sz, PAGE_EXECUTE_READ, &oprot); 289 } 290 #elif LJ_TARGET_POSIX 291 mprotect(p, sz, (PROT_READ|PROT_EXEC)); 292 #endif 293 } 294 295 /* Free area for callback function pointers. */ 296 void lj_ccallback_mcode_free(CTState *cts) 297 { 298 size_t sz = (size_t)CALLBACK_MCODE_SIZE; 299 void *p = cts->cb.mcode; 300 if (p == NULL) return; 301 #if LJ_TARGET_WINDOWS 302 VirtualFree(p, 0, MEM_RELEASE); 303 UNUSED(sz); 304 #elif LJ_TARGET_POSIX 305 munmap(p, sz); 306 #else 307 lj_mem_free(cts->g, p, sz); 308 #endif 309 } 310 311 /* -- C callback entry ---------------------------------------------------- */ 312 313 /* Target-specific handling of register arguments. Similar to lj_ccall.c. */ 314 #if LJ_TARGET_X86 315 316 #define CALLBACK_HANDLE_REGARG \ 317 if (!isfp) { /* Only non-FP values may be passed in registers. */ \ 318 if (n > 1) { /* Anything > 32 bit is passed on the stack. */ \ 319 if (!LJ_ABI_WIN) ngpr = maxgpr; /* Prevent reordering. */ \ 320 } else if (ngpr + 1 <= maxgpr) { \ 321 sp = &cts->cb.gpr[ngpr]; \ 322 ngpr += n; \ 323 goto done; \ 324 } \ 325 } 326 327 #elif LJ_TARGET_X64 && LJ_ABI_WIN 328 329 /* Windows/x64 argument registers are strictly positional (use ngpr). */ 330 #define CALLBACK_HANDLE_REGARG \ 331 if (isfp) { \ 332 if (ngpr < maxgpr) { sp = &cts->cb.fpr[ngpr++]; UNUSED(nfpr); goto done; } \ 333 } else { \ 334 if (ngpr < maxgpr) { sp = &cts->cb.gpr[ngpr++]; goto done; } \ 335 } 336 337 #elif LJ_TARGET_X64 338 339 #define CALLBACK_HANDLE_REGARG \ 340 if (isfp) { \ 341 if (nfpr + n <= CCALL_NARG_FPR) { \ 342 sp = &cts->cb.fpr[nfpr]; \ 343 nfpr += n; \ 344 goto done; \ 345 } \ 346 } else { \ 347 if (ngpr + n <= maxgpr) { \ 348 sp = &cts->cb.gpr[ngpr]; \ 349 ngpr += n; \ 350 goto done; \ 351 } \ 352 } 353 354 #elif LJ_TARGET_ARM 355 356 #if LJ_ABI_SOFTFP 357 358 #define CALLBACK_HANDLE_REGARG_FP1 UNUSED(isfp); 359 #define CALLBACK_HANDLE_REGARG_FP2 360 361 #else 362 363 #define CALLBACK_HANDLE_REGARG_FP1 \ 364 if (isfp) { \ 365 if (n == 1) { \ 366 if (fprodd) { \ 367 sp = &cts->cb.fpr[fprodd-1]; \ 368 fprodd = 0; \ 369 goto done; \ 370 } else if (nfpr + 1 <= CCALL_NARG_FPR) { \ 371 sp = &cts->cb.fpr[nfpr++]; \ 372 fprodd = nfpr; \ 373 goto done; \ 374 } \ 375 } else { \ 376 if (nfpr + 1 <= CCALL_NARG_FPR) { \ 377 sp = &cts->cb.fpr[nfpr++]; \ 378 goto done; \ 379 } \ 380 } \ 381 fprodd = 0; /* No reordering after the first FP value is on stack. */ \ 382 } else { 383 384 #define CALLBACK_HANDLE_REGARG_FP2 } 385 386 #endif 387 388 #define CALLBACK_HANDLE_REGARG \ 389 CALLBACK_HANDLE_REGARG_FP1 \ 390 if (n > 1) ngpr = (ngpr + 1u) & ~1u; /* Align to regpair. */ \ 391 if (ngpr + n <= maxgpr) { \ 392 sp = &cts->cb.gpr[ngpr]; \ 393 ngpr += n; \ 394 goto done; \ 395 } CALLBACK_HANDLE_REGARG_FP2 396 397 #elif LJ_TARGET_ARM64 398 399 #define CALLBACK_HANDLE_REGARG \ 400 if (isfp) { \ 401 if (nfpr + n <= CCALL_NARG_FPR) { \ 402 sp = &cts->cb.fpr[nfpr]; \ 403 nfpr += n; \ 404 goto done; \ 405 } else { \ 406 nfpr = CCALL_NARG_FPR; /* Prevent reordering. */ \ 407 } \ 408 } else { \ 409 if (!LJ_TARGET_IOS && n > 1) \ 410 ngpr = (ngpr + 1u) & ~1u; /* Align to regpair. */ \ 411 if (ngpr + n <= maxgpr) { \ 412 sp = &cts->cb.gpr[ngpr]; \ 413 ngpr += n; \ 414 goto done; \ 415 } else { \ 416 ngpr = CCALL_NARG_GPR; /* Prevent reordering. */ \ 417 } \ 418 } 419 420 #elif LJ_TARGET_PPC 421 422 #define CALLBACK_HANDLE_REGARG \ 423 if (isfp) { \ 424 if (nfpr + 1 <= CCALL_NARG_FPR) { \ 425 sp = &cts->cb.fpr[nfpr++]; \ 426 cta = ctype_get(cts, CTID_DOUBLE); /* FPRs always hold doubles. */ \ 427 goto done; \ 428 } \ 429 } else { /* Try to pass argument in GPRs. */ \ 430 if (n > 1) { \ 431 lua_assert(ctype_isinteger(cta->info) && n == 2); /* int64_t. */ \ 432 ngpr = (ngpr + 1u) & ~1u; /* Align int64_t to regpair. */ \ 433 } \ 434 if (ngpr + n <= maxgpr) { \ 435 sp = &cts->cb.gpr[ngpr]; \ 436 ngpr += n; \ 437 goto done; \ 438 } \ 439 } 440 441 #define CALLBACK_HANDLE_RET \ 442 if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \ 443 *(double *)dp = *(float *)dp; /* FPRs always hold doubles. */ 444 445 #elif LJ_TARGET_MIPS32 446 447 #define CALLBACK_HANDLE_GPR \ 448 if (n > 1) ngpr = (ngpr + 1u) & ~1u; /* Align to regpair. */ \ 449 if (ngpr + n <= maxgpr) { \ 450 sp = &cts->cb.gpr[ngpr]; \ 451 ngpr += n; \ 452 goto done; \ 453 } 454 455 #if !LJ_ABI_SOFTFP /* MIPS32 hard-float */ 456 #define CALLBACK_HANDLE_REGARG \ 457 if (isfp && nfpr < CCALL_NARG_FPR) { /* Try to pass argument in FPRs. */ \ 458 sp = (void *)((uint8_t *)&cts->cb.fpr[nfpr] + ((LJ_BE && n==1) ? 4 : 0)); \ 459 nfpr++; ngpr += n; \ 460 goto done; \ 461 } else { /* Try to pass argument in GPRs. */ \ 462 nfpr = CCALL_NARG_FPR; \ 463 CALLBACK_HANDLE_GPR \ 464 } 465 #else /* MIPS32 soft-float */ 466 #define CALLBACK_HANDLE_REGARG \ 467 CALLBACK_HANDLE_GPR \ 468 UNUSED(isfp); 469 #endif 470 471 #define CALLBACK_HANDLE_RET \ 472 if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \ 473 ((float *)dp)[1] = *(float *)dp; 474 475 #elif LJ_TARGET_MIPS64 476 477 #if !LJ_ABI_SOFTFP /* MIPS64 hard-float */ 478 #define CALLBACK_HANDLE_REGARG \ 479 if (ngpr + n <= maxgpr) { \ 480 sp = isfp ? (void*) &cts->cb.fpr[ngpr] : (void*) &cts->cb.gpr[ngpr]; \ 481 ngpr += n; \ 482 goto done; \ 483 } 484 #else /* MIPS64 soft-float */ 485 #define CALLBACK_HANDLE_REGARG \ 486 if (ngpr + n <= maxgpr) { \ 487 UNUSED(isfp); \ 488 sp = (void*) &cts->cb.gpr[ngpr]; \ 489 ngpr += n; \ 490 goto done; \ 491 } 492 #endif 493 494 #define CALLBACK_HANDLE_RET \ 495 if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \ 496 ((float *)dp)[1] = *(float *)dp; 497 498 #else 499 #error "Missing calling convention definitions for this architecture" 500 #endif 501 502 /* Convert and push callback arguments to Lua stack. */ 503 static void callback_conv_args(CTState *cts, lua_State *L) 504 { 505 TValue *o = L->top; 506 intptr_t *stack = cts->cb.stack; 507 MSize slot = cts->cb.slot; 508 CTypeID id = 0, rid, fid; 509 int gcsteps = 0; 510 CType *ct; 511 GCfunc *fn; 512 int fntp; 513 MSize ngpr = 0, nsp = 0, maxgpr = CCALL_NARG_GPR; 514 #if CCALL_NARG_FPR 515 MSize nfpr = 0; 516 #if LJ_TARGET_ARM 517 MSize fprodd = 0; 518 #endif 519 #endif 520 521 if (slot < cts->cb.sizeid && (id = cts->cb.cbid[slot]) != 0) { 522 ct = ctype_get(cts, id); 523 rid = ctype_cid(ct->info); /* Return type. x86: +(spadj<<16). */ 524 fn = funcV(lj_tab_getint(cts->miscmap, (int32_t)slot)); 525 fntp = LJ_TFUNC; 526 } else { /* Must set up frame first, before throwing the error. */ 527 ct = NULL; 528 rid = 0; 529 fn = (GCfunc *)L; 530 fntp = LJ_TTHREAD; 531 } 532 /* Continuation returns from callback. */ 533 if (LJ_FR2) { 534 (o++)->u64 = LJ_CONT_FFI_CALLBACK; 535 (o++)->u64 = rid; 536 o++; 537 } else { 538 o->u32.lo = LJ_CONT_FFI_CALLBACK; 539 o->u32.hi = rid; 540 o++; 541 } 542 setframe_gc(o, obj2gco(fn), fntp); 543 setframe_ftsz(o, ((char *)(o+1) - (char *)L->base) + FRAME_CONT); 544 L->top = L->base = ++o; 545 if (!ct) 546 lj_err_caller(cts->L, LJ_ERR_FFI_BADCBACK); 547 if (isluafunc(fn)) 548 setcframe_pc(L->cframe, proto_bc(funcproto(fn))+1); 549 lj_state_checkstack(L, LUA_MINSTACK); /* May throw. */ 550 o = L->base; /* Might have been reallocated. */ 551 552 #if LJ_TARGET_X86 553 /* x86 has several different calling conventions. */ 554 switch (ctype_cconv(ct->info)) { 555 case CTCC_FASTCALL: maxgpr = 2; break; 556 case CTCC_THISCALL: maxgpr = 1; break; 557 default: maxgpr = 0; break; 558 } 559 #endif 560 561 fid = ct->sib; 562 while (fid) { 563 CType *ctf = ctype_get(cts, fid); 564 if (!ctype_isattrib(ctf->info)) { 565 CType *cta; 566 void *sp; 567 CTSize sz; 568 int isfp; 569 MSize n; 570 lua_assert(ctype_isfield(ctf->info)); 571 cta = ctype_rawchild(cts, ctf); 572 isfp = ctype_isfp(cta->info); 573 sz = (cta->size + CTSIZE_PTR-1) & ~(CTSIZE_PTR-1); 574 n = sz / CTSIZE_PTR; /* Number of GPRs or stack slots needed. */ 575 576 CALLBACK_HANDLE_REGARG /* Handle register arguments. */ 577 578 /* Otherwise pass argument on stack. */ 579 if (CCALL_ALIGN_STACKARG && LJ_32 && sz == 8) 580 nsp = (nsp + 1) & ~1u; /* Align 64 bit argument on stack. */ 581 sp = &stack[nsp]; 582 nsp += n; 583 584 done: 585 if (LJ_BE && cta->size < CTSIZE_PTR 586 #if LJ_TARGET_MIPS64 587 && !(isfp && nsp) 588 #endif 589 ) 590 sp = (void *)((uint8_t *)sp + CTSIZE_PTR-cta->size); 591 gcsteps += lj_cconv_tv_ct(cts, cta, 0, o++, sp); 592 } 593 fid = ctf->sib; 594 } 595 L->top = o; 596 #if LJ_TARGET_X86 597 /* Store stack adjustment for returns from non-cdecl callbacks. */ 598 if (ctype_cconv(ct->info) != CTCC_CDECL) { 599 #if LJ_FR2 600 (L->base-3)->u64 |= (nsp << (16+2)); 601 #else 602 (L->base-2)->u32.hi |= (nsp << (16+2)); 603 #endif 604 } 605 #endif 606 while (gcsteps-- > 0) 607 lj_gc_check(L); 608 } 609 610 /* Convert Lua object to callback result. */ 611 static void callback_conv_result(CTState *cts, lua_State *L, TValue *o) 612 { 613 #if LJ_FR2 614 CType *ctr = ctype_raw(cts, (uint16_t)(L->base-3)->u64); 615 #else 616 CType *ctr = ctype_raw(cts, (uint16_t)(L->base-2)->u32.hi); 617 #endif 618 #if LJ_TARGET_X86 619 cts->cb.gpr[2] = 0; 620 #endif 621 if (!ctype_isvoid(ctr->info)) { 622 uint8_t *dp = (uint8_t *)&cts->cb.gpr[0]; 623 #if CCALL_NUM_FPR 624 if (ctype_isfp(ctr->info)) 625 dp = (uint8_t *)&cts->cb.fpr[0]; 626 #endif 627 lj_cconv_ct_tv(cts, ctr, dp, o, 0); 628 #ifdef CALLBACK_HANDLE_RET 629 CALLBACK_HANDLE_RET 630 #endif 631 /* Extend returned integers to (at least) 32 bits. */ 632 if (ctype_isinteger_or_bool(ctr->info) && ctr->size < 4) { 633 if (ctr->info & CTF_UNSIGNED) 634 *(uint32_t *)dp = ctr->size == 1 ? (uint32_t)*(uint8_t *)dp : 635 (uint32_t)*(uint16_t *)dp; 636 else 637 *(int32_t *)dp = ctr->size == 1 ? (int32_t)*(int8_t *)dp : 638 (int32_t)*(int16_t *)dp; 639 } 640 #if LJ_TARGET_MIPS64 641 /* Always sign-extend results to 64 bits. Even a soft-fp 'float'. */ 642 if (ctr->size <= 4 && 643 (LJ_ABI_SOFTFP || ctype_isinteger_or_bool(ctr->info))) 644 *(int64_t *)dp = (int64_t)*(int32_t *)dp; 645 #endif 646 #if LJ_TARGET_X86 647 if (ctype_isfp(ctr->info)) 648 cts->cb.gpr[2] = ctr->size == sizeof(float) ? 1 : 2; 649 #endif 650 } 651 } 652 653 /* Enter callback. */ 654 lua_State * LJ_FASTCALL lj_ccallback_enter(CTState *cts, void *cf) 655 { 656 lua_State *L = cts->L; 657 global_State *g = cts->g; 658 lua_assert(L != NULL); 659 if (tvref(g->jit_base)) { 660 setstrV(L, L->top++, lj_err_str(L, LJ_ERR_FFI_BADCBACK)); 661 if (g->panic) g->panic(L); 662 exit(EXIT_FAILURE); 663 } 664 lj_trace_abort(g); /* Never record across callback. */ 665 /* Setup C frame. */ 666 cframe_prev(cf) = L->cframe; 667 setcframe_L(cf, L); 668 cframe_errfunc(cf) = -1; 669 cframe_nres(cf) = 0; 670 L->cframe = cf; 671 callback_conv_args(cts, L); 672 return L; /* Now call the function on this stack. */ 673 } 674 675 /* Leave callback. */ 676 void LJ_FASTCALL lj_ccallback_leave(CTState *cts, TValue *o) 677 { 678 lua_State *L = cts->L; 679 GCfunc *fn; 680 TValue *obase = L->base; 681 L->base = L->top; /* Keep continuation frame for throwing errors. */ 682 if (o >= L->base) { 683 /* PC of RET* is lost. Point to last line for result conv. errors. */ 684 fn = curr_func(L); 685 if (isluafunc(fn)) { 686 GCproto *pt = funcproto(fn); 687 setcframe_pc(L->cframe, proto_bc(pt)+pt->sizebc+1); 688 } 689 } 690 callback_conv_result(cts, L, o); 691 /* Finally drop C frame and continuation frame. */ 692 L->top -= 2+2*LJ_FR2; 693 L->base = obase; 694 L->cframe = cframe_prev(L->cframe); 695 cts->cb.slot = 0; /* Blacklist C function that called the callback. */ 696 } 697 698 /* -- C callback management ----------------------------------------------- */ 699 700 /* Get an unused slot in the callback slot table. */ 701 static MSize callback_slot_new(CTState *cts, CType *ct) 702 { 703 CTypeID id = ctype_typeid(cts, ct); 704 CTypeID1 *cbid = cts->cb.cbid; 705 MSize top; 706 for (top = cts->cb.topid; top < cts->cb.sizeid; top++) 707 if (LJ_LIKELY(cbid[top] == 0)) 708 goto found; 709 #if CALLBACK_MAX_SLOT 710 if (top >= CALLBACK_MAX_SLOT) 711 #endif 712 lj_err_caller(cts->L, LJ_ERR_FFI_CBACKOV); 713 if (!cts->cb.mcode) 714 callback_mcode_new(cts); 715 lj_mem_growvec(cts->L, cbid, cts->cb.sizeid, CALLBACK_MAX_SLOT, CTypeID1); 716 cts->cb.cbid = cbid; 717 memset(cbid+top, 0, (cts->cb.sizeid-top)*sizeof(CTypeID1)); 718 found: 719 cbid[top] = id; 720 cts->cb.topid = top+1; 721 return top; 722 } 723 724 /* Check for function pointer and supported argument/result types. */ 725 static CType *callback_checkfunc(CTState *cts, CType *ct) 726 { 727 int narg = 0; 728 if (!ctype_isptr(ct->info) || (LJ_64 && ct->size != CTSIZE_PTR)) 729 return NULL; 730 ct = ctype_rawchild(cts, ct); 731 if (ctype_isfunc(ct->info)) { 732 CType *ctr = ctype_rawchild(cts, ct); 733 CTypeID fid = ct->sib; 734 if (!(ctype_isvoid(ctr->info) || ctype_isenum(ctr->info) || 735 ctype_isptr(ctr->info) || (ctype_isnum(ctr->info) && ctr->size <= 8))) 736 return NULL; 737 if ((ct->info & CTF_VARARG)) 738 return NULL; 739 while (fid) { 740 CType *ctf = ctype_get(cts, fid); 741 if (!ctype_isattrib(ctf->info)) { 742 CType *cta; 743 lua_assert(ctype_isfield(ctf->info)); 744 cta = ctype_rawchild(cts, ctf); 745 if (!(ctype_isenum(cta->info) || ctype_isptr(cta->info) || 746 (ctype_isnum(cta->info) && cta->size <= 8)) || 747 ++narg >= LUA_MINSTACK-3) 748 return NULL; 749 } 750 fid = ctf->sib; 751 } 752 return ct; 753 } 754 return NULL; 755 } 756 757 /* Create a new callback and return the callback function pointer. */ 758 void *lj_ccallback_new(CTState *cts, CType *ct, GCfunc *fn) 759 { 760 ct = callback_checkfunc(cts, ct); 761 if (ct) { 762 MSize slot = callback_slot_new(cts, ct); 763 GCtab *t = cts->miscmap; 764 setfuncV(cts->L, lj_tab_setint(cts->L, t, (int32_t)slot), fn); 765 lj_gc_anybarriert(cts->L, t); 766 return callback_slot2ptr(cts, slot); 767 } 768 return NULL; /* Bad conversion. */ 769 } 770 771 #endif