ljx

lj_crecord.c (60289B)

---

 /*
 ** Trace recorder for C data operations.
 ** Copyright (C) 2005-2014 Mike Pall. See Copyright Notice in luajit.h
 **
 ** Minor FFI API extensions
 ** Copyright (C) 2014 Karel Tuma
 */
 
 #define lj_ffrecord_c
 #define LUA_CORE
 
 #include "lj_obj.h"
 
 #if LJ_HASJIT && LJ_HASFFI
 
 #include "lj_err.h"
 #include "lj_tab.h"
 #include "lj_frame.h"
 #include "lj_ctype.h"
 #include "lj_cdata.h"
 #include "lj_cparse.h"
 #include "lj_cconv.h"
 #include "lj_carith.h"
 #include "lj_clib.h"
 #include "lj_ccall.h"
 #include "lj_ff.h"
 #include "lj_ir.h"
 #include "lj_jit.h"
 #include "lj_ircall.h"
 #include "lj_iropt.h"
 #include "lj_trace.h"
 #include "lj_record.h"
 #include "lj_ffrecord.h"
 #include "lj_snap.h"
 #include "lj_crecord.h"
 #include "lj_dispatch.h"
 #include "lj_strfmt.h"
 
 /* Some local macros to save typing. Undef'd at the end. */
 #define IR(ref)			(&J->cur.ir[(ref)])
 
 /* Pass IR on to next optimization in chain (FOLD). */
 #define emitir(ot, a, b)	(lj_ir_set(J, (ot), (a), (b)), lj_opt_fold(J))
 
 #define emitconv(a, dt, st, flags) \
   emitir(IRT(IR_CONV, (dt)), (a), (st)|((dt) << 5)|(flags))
 
 /* -- C type checks ------------------------------------------------------- */
 
 static GCcdata *argv2cdata(jit_State *J, TRef tr, cTValue *o)
 {
   GCcdata *cd;
   TRef trtypeid;
   if (!tref_iscdata(tr))
     lj_trace_err(J, LJ_TRERR_BADTYPE);
   cd = cdataV(o);
   /* Specialize to the CTypeID. */
   trtypeid = emitir(IRT(IR_FLOAD, IRT_U16), tr, IRFL_CDATA_CTYPEID);
   emitir(IRTG(IR_EQ, IRT_INT), trtypeid, lj_ir_kint(J, (int32_t)cd->ctypeid));
   return cd;
 }
 
 /* Specialize to the CTypeID held by a cdata constructor. */
 static CTypeID crec_constructor(jit_State *J, GCcdata *cd, TRef tr)
 {
   CTypeID id;
   lua_assert(tref_iscdata(tr) && cd->ctypeid == CTID_CTYPEID);
   id = *(CTypeID *)cdataptr(cd);
   tr = emitir(IRT(IR_FLOAD, IRT_INT), tr, IRFL_CDATA_INT);
   emitir(IRTG(IR_EQ, IRT_INT), tr, lj_ir_kint(J, (int32_t)id));
   return id;
 }
 
 static CTypeID argv2ctype(jit_State *J, TRef tr, cTValue *o)
 {
   if (tref_isstr(tr)) {
     GCstr *s = strV(o);
     CPState cp;
     CTypeID oldtop;
     /* Specialize to the string containing the C type declaration. */
     emitir(IRTG(IR_EQ, IRT_STR), tr, lj_ir_kstr(J, s));
     cp.L = J->L;
     cp.cts = ctype_ctsG(J2G(J));
     oldtop = cp.cts->top;
     cp.srcname = strdata(s);
     cp.p = strdata(s);
     cp.param = NULL;
     cp.mode = CPARSE_MODE_ABSTRACT|CPARSE_MODE_NOIMPLICIT;
     if (lj_cparse(&cp) || cp.cts->top > oldtop)  /* Avoid new struct defs. */
       lj_trace_err(J, LJ_TRERR_BADTYPE);
     return cp.val.id;
   } else {
     GCcdata *cd = argv2cdata(J, tr, o);
     return cd->ctypeid == CTID_CTYPEID ? crec_constructor(J, cd, tr) :
 					cd->ctypeid;
   }
 }
 
 /* Convert CType to IRType (if possible). */
 static IRType crec_ct2irt(CTState *cts, CType *ct)
 {
   if (ctype_isenum(ct->info)) ct = ctype_child(cts, ct);
   if (LJ_LIKELY(ctype_isnum(ct->info))) {
     if ((ct->info & CTF_FP)) {
       if (ct->size == sizeof(double))
 	return IRT_NUM;
       else if (ct->size == sizeof(float))
 	return IRT_FLOAT;
     } else {
       uint32_t b = lj_fls(ct->size);
       if (b <= 3)
 	return IRT_I8 + 2*b + ((ct->info & CTF_UNSIGNED) ? 1 : 0);
     }
   } else if (ctype_isptr(ct->info)) {
     return (LJ_64 && ct->size == 8) ? IRT_P64 : IRT_P32;
   } else if (ctype_iscomplex(ct->info)) {
     if (ct->size == 2*sizeof(double))
       return IRT_NUM;
     else if (ct->size == 2*sizeof(float))
       return IRT_FLOAT;
   }
   return IRT_CDATA;
 }
 
 /* -- Optimized memory fill and copy -------------------------------------- */
 
 /* Maximum length and unroll of inlined copy/fill. */
 #define CREC_COPY_MAXUNROLL		16
 #define CREC_COPY_MAXLEN		128
 
 #define CREC_FILL_MAXUNROLL		16
 
 /* Number of windowed registers used for optimized memory copy. */
 #if LJ_TARGET_X86
 #define CREC_COPY_REGWIN		2
 #elif LJ_TARGET_PPC || LJ_TARGET_MIPS
 #define CREC_COPY_REGWIN		8
 #else
 #define CREC_COPY_REGWIN		4
 #endif
 
 /* List of memory offsets for copy/fill. */
 typedef struct CRecMemList {
   CTSize ofs;		/* Offset in bytes. */
   IRType tp;		/* Type of load/store. */
   TRef trofs;		/* TRef of interned offset. */
   TRef trval;		/* TRef of load value. */
 } CRecMemList;
 
 /* Generate copy list for element-wise struct copy. */
 static MSize crec_copy_struct(CRecMemList *ml, CTState *cts, CType *ct)
 {
   CTypeID fid = ct->sib;
   MSize mlp = 0;
   while (fid) {
     CType *df = ctype_get(cts, fid);
     fid = df->sib;
     if (ctype_isfield(df->info)) {
       CType *cct;
       IRType tp;
       if (!gcref(df->name)) continue;  /* Ignore unnamed fields. */
       cct = ctype_rawchild(cts, df);  /* Field type. */
       tp = crec_ct2irt(cts, cct);
       if (tp == IRT_CDATA) return 0;  /* NYI: aggregates. */
       if (mlp >= CREC_COPY_MAXUNROLL) return 0;
       ml[mlp].ofs = df->size;
       ml[mlp].tp = tp;
       mlp++;
       if (ctype_iscomplex(cct->info)) {
 	if (mlp >= CREC_COPY_MAXUNROLL) return 0;
 	ml[mlp].ofs = df->size + (cct->size >> 1);
 	ml[mlp].tp = tp;
 	mlp++;
       }
     } else if (!ctype_isconstval(df->info)) {
       /* NYI: bitfields and sub-structures. */
       return 0;
     }
   }
   return mlp;
 }
 
 /* Generate unrolled copy list, from highest to lowest step size/alignment. */
 static MSize crec_copy_unroll(CRecMemList *ml, CTSize len, CTSize step,
 			      IRType tp)
 {
   CTSize ofs = 0;
   MSize mlp = 0;
   if (tp == IRT_CDATA) tp = IRT_U8 + 2*lj_fls(step);
   do {
     while (ofs + step <= len) {
       if (mlp >= CREC_COPY_MAXUNROLL) return 0;
       ml[mlp].ofs = ofs;
       ml[mlp].tp = tp;
       mlp++;
       ofs += step;
     }
     step >>= 1;
     tp -= 2;
   } while (ofs < len);
   return mlp;
 }
 
 /*
 ** Emit copy list with windowed loads/stores.
 ** LJ_TARGET_UNALIGNED: may emit unaligned loads/stores (not marked as such).
 */
 static void crec_copy_emit(jit_State *J, CRecMemList *ml, MSize mlp,
 			   TRef trdst, TRef trsrc)
 {
   MSize i, j, rwin = 0;
   for (i = 0, j = 0; i < mlp; ) {
     TRef trofs = lj_ir_kintp(J, ml[i].ofs);
     TRef trsptr = emitir(IRT(IR_ADD, IRT_PTR), trsrc, trofs);
     ml[i].trval = emitir(IRT(IR_XLOAD, ml[i].tp), trsptr, 0);
     ml[i].trofs = trofs;
     i++;
     rwin += (LJ_SOFTFP && ml[i].tp == IRT_NUM) ? 2 : 1;
     if (rwin >= CREC_COPY_REGWIN || i >= mlp) {  /* Flush buffered stores. */
       rwin = 0;
       for ( ; j < i; j++) {
 	TRef trdptr = emitir(IRT(IR_ADD, IRT_PTR), trdst, ml[j].trofs);
 	emitir(IRT(IR_XSTORE, ml[j].tp), trdptr, ml[j].trval);
       }
     }
   }
 }
 
 /* Optimized memory copy. */
 static void crec_copy(jit_State *J, TRef trdst, TRef trsrc, TRef trlen,
 		      CType *ct)
 {
   if (tref_isk(trlen)) {  /* Length must be constant. */
     CRecMemList ml[CREC_COPY_MAXUNROLL];
     MSize mlp = 0;
     CTSize step = 1, len = (CTSize)IR(tref_ref(trlen))->i;
     IRType tp = IRT_CDATA;
     int needxbar = 0;
     if (len == 0) return;  /* Shortcut. */
     if (len > CREC_COPY_MAXLEN) goto fallback;
     if (ct) {
       CTState *cts = ctype_ctsG(J2G(J));
       lua_assert(ctype_isarray(ct->info) || ctype_isstruct(ct->info));
       if (ctype_isarray(ct->info)) {
 	CType *cct = ctype_rawchild(cts, ct);
 	tp = crec_ct2irt(cts, cct);
 	if (tp == IRT_CDATA) goto rawcopy;
 	step = lj_ir_type_size[tp];
 	lua_assert((len & (step-1)) == 0);
       } else if ((ct->info & CTF_UNION)) {
 	step = (1u << ctype_align(ct->info));
 	goto rawcopy;
       } else {
 	mlp = crec_copy_struct(ml, cts, ct);
 	goto emitcopy;
       }
     } else {
     rawcopy:
       needxbar = 1;
       if (LJ_TARGET_UNALIGNED || step >= CTSIZE_PTR)
 	step = CTSIZE_PTR;
     }
     mlp = crec_copy_unroll(ml, len, step, tp);
   emitcopy:
     if (mlp) {
       crec_copy_emit(J, ml, mlp, trdst, trsrc);
       if (needxbar)
 	emitir(IRT(IR_XBAR, IRT_NIL), 0, 0);
       return;
     }
   }
 fallback:
   /* Call memcpy. Always needs a barrier to disable alias analysis. */
   lj_ir_call(J, IRCALL_memcpy, trdst, trsrc, trlen);
   emitir(IRT(IR_XBAR, IRT_NIL), 0, 0);
 }
 
 /* Generate unrolled fill list, from highest to lowest step size/alignment. */
 static MSize crec_fill_unroll(CRecMemList *ml, CTSize len, CTSize step)
 {
   CTSize ofs = 0;
   MSize mlp = 0;
   IRType tp = IRT_U8 + 2*lj_fls(step);
   do {
     while (ofs + step <= len) {
       if (mlp >= CREC_COPY_MAXUNROLL) return 0;
       ml[mlp].ofs = ofs;
       ml[mlp].tp = tp;
       mlp++;
       ofs += step;
     }
     step >>= 1;
     tp -= 2;
   } while (ofs < len);
   return mlp;
 }
 
 /*
 ** Emit stores for fill list.
 ** LJ_TARGET_UNALIGNED: may emit unaligned stores (not marked as such).
 */
 static void crec_fill_emit(jit_State *J, CRecMemList *ml, MSize mlp,
 			   TRef trdst, TRef trfill)
 {
   MSize i;
   for (i = 0; i < mlp; i++) {
     TRef trofs = lj_ir_kintp(J, ml[i].ofs);
     TRef trdptr = emitir(IRT(IR_ADD, IRT_PTR), trdst, trofs);
     emitir(IRT(IR_XSTORE, ml[i].tp), trdptr, trfill);
   }
 }
 
 /* Optimized memory fill. */
 static void crec_fill(jit_State *J, TRef trdst, TRef trlen, TRef trfill,
 		      CTSize step)
 {
   if (tref_isk(trlen)) {  /* Length must be constant. */
     CRecMemList ml[CREC_FILL_MAXUNROLL];
     MSize mlp;
     CTSize len = (CTSize)IR(tref_ref(trlen))->i;
     if (len == 0) return;  /* Shortcut. */
     if (LJ_TARGET_UNALIGNED || step >= CTSIZE_PTR)
       step = CTSIZE_PTR;
     if (step * CREC_FILL_MAXUNROLL < len) goto fallback;
     mlp = crec_fill_unroll(ml, len, step);
     if (!mlp) goto fallback;
     if (tref_isk(trfill) || ml[0].tp != IRT_U8)
       trfill = emitconv(trfill, IRT_INT, IRT_U8, 0);
     if (ml[0].tp != IRT_U8) {  /* Scatter U8 to U16/U32/U64. */
       if (CTSIZE_PTR == 8 && ml[0].tp == IRT_U64) {
 	if (tref_isk(trfill))  /* Pointless on x64 with zero-extended regs. */
 	  trfill = emitconv(trfill, IRT_U64, IRT_U32, 0);
 	trfill = emitir(IRT(IR_MUL, IRT_U64), trfill,
 			lj_ir_kint64(J, U64x(01010101,01010101)));
       } else {
 	trfill = emitir(IRTI(IR_MUL), trfill,
 		   lj_ir_kint(J, ml[0].tp == IRT_U16 ? 0x0101 : 0x01010101));
       }
     }
     crec_fill_emit(J, ml, mlp, trdst, trfill);
   } else {
 fallback:
     /* Call memset. Always needs a barrier to disable alias analysis. */
     lj_ir_call(J, IRCALL_memset, trdst, trfill, trlen);  /* Note: arg order! */
   }
   emitir(IRT(IR_XBAR, IRT_NIL), 0, 0);
 }
 
 /* -- Convert C type to C type -------------------------------------------- */
 
 /*
 ** This code mirrors the code in lj_cconv.c. It performs the same steps
 ** for the trace recorder that lj_cconv.c does for the interpreter.
 **
 ** One major difference is that we can get away with much fewer checks
 ** here. E.g. checks for casts, constness or correct types can often be
 ** omitted, even if they might fail. The interpreter subsequently throws
 ** an error, which aborts the trace.
 **
 ** All operations are specialized to their C types, so the on-trace
 ** outcome must be the same as the outcome in the interpreter. If the
 ** interpreter doesn't throw an error, then the trace is correct, too.
 ** Care must be taken not to generate invalid (temporary) IR or to
 ** trigger asserts.
 */
 
 /* Determine whether a passed number or cdata number is non-zero. */
 static int crec_isnonzero(CType *s, void *p)
 {
   if (p == (void *)0)
     return 0;
   if (p == (void *)1)
     return 1;
   if ((s->info & CTF_FP)) {
     if (s->size == sizeof(float))
       return (*(float *)p != 0);
     else
       return (*(double *)p != 0);
   } else {
     if (s->size == 1)
       return (*(uint8_t *)p != 0);
     else if (s->size == 2)
       return (*(uint16_t *)p != 0);
     else if (s->size == 4)
       return (*(uint32_t *)p != 0);
     else
       return (*(uint64_t *)p != 0);
   }
 }
 
 static TRef crec_ct_ct(jit_State *J, CType *d, CType *s, TRef dp, TRef sp,
 		       void *svisnz)
 {
   IRType dt = crec_ct2irt(ctype_ctsG(J2G(J)), d);
   IRType st = crec_ct2irt(ctype_ctsG(J2G(J)), s);
   CTSize dsize = d->size, ssize = s->size;
   CTInfo dinfo = d->info, sinfo = s->info;
 
   if (ctype_type(dinfo) > CT_MAYCONVERT || ctype_type(sinfo) > CT_MAYCONVERT)
     goto err_conv;
 
   /*
   ** Note: Unlike lj_cconv_ct_ct(), sp holds the _value_ of pointers and
   ** numbers up to 8 bytes. Otherwise sp holds a pointer.
   */
 
   switch (cconv_idx2(dinfo, sinfo)) {
   /* Destination is a bool. */
   case CCX(B, B):
     goto xstore;  /* Source operand is already normalized. */
   case CCX(B, I):
   case CCX(B, F):
     if (st != IRT_CDATA) {
       /* Specialize to the result of a comparison against 0. */
       TRef zero = (st == IRT_NUM  || st == IRT_FLOAT) ? lj_ir_knum(J, 0) :
 		  (st == IRT_I64 || st == IRT_U64) ? lj_ir_kint64(J, 0) :
 		  lj_ir_kint(J, 0);
       int isnz = crec_isnonzero(s, svisnz);
       emitir(IRTG(isnz ? IR_NE : IR_EQ, st), sp, zero);
       sp = lj_ir_kint(J, isnz);
       goto xstore;
     }
     goto err_nyi;
 
   /* Destination is an integer. */
   case CCX(I, B):
   case CCX(I, I):
   conv_I_I:
     if (dt == IRT_CDATA || st == IRT_CDATA) goto err_nyi;
     /* Extend 32 to 64 bit integer. */
     if (dsize == 8 && ssize < 8 && !(LJ_64 && (sinfo & CTF_UNSIGNED)))
       sp = emitconv(sp, dt, ssize < 4 ? IRT_INT : st,
 		    (sinfo & CTF_UNSIGNED) ? 0 : IRCONV_SEXT);
     else if (dsize < 8 && ssize == 8)  /* Truncate from 64 bit integer. */
       sp = emitconv(sp, dsize < 4 ? IRT_INT : dt, st, 0);
     else if (st == IRT_INT)
       sp = lj_opt_narrow_toint(J, sp);
   xstore:
     if (dt == IRT_I64 || dt == IRT_U64) lj_needsplit(J);
     if (dp == 0) return sp;
     emitir(IRT(IR_XSTORE, dt), dp, sp);
     break;
   case CCX(I, C):
     sp = emitir(IRT(IR_XLOAD, st), sp, 0);  /* Load re. */
     /* fallthrough */
   case CCX(I, F):
     if (dt == IRT_CDATA || st == IRT_CDATA) goto err_nyi;
     sp = emitconv(sp, dsize < 4 ? IRT_INT : dt, st, IRCONV_ANY);
     goto xstore;
   case CCX(I, P):
   case CCX(I, A):
     sinfo = CTINFO(CT_NUM, CTF_UNSIGNED);
     ssize = CTSIZE_PTR;
     st = IRT_UINTP;
     if (((dsize ^ ssize) & 8) == 0) {  /* Must insert no-op type conversion. */
       sp = emitconv(sp, dsize < 4 ? IRT_INT : dt, IRT_PTR, 0);
       goto xstore;
     }
     goto conv_I_I;
 
   /* Destination is a floating-point number. */
   case CCX(F, B):
   case CCX(F, I):
   conv_F_I:
     if (dt == IRT_CDATA || st == IRT_CDATA) goto err_nyi;
     sp = emitconv(sp, dt, ssize < 4 ? IRT_INT : st, 0);
     goto xstore;
   case CCX(F, C):
     sp = emitir(IRT(IR_XLOAD, st), sp, 0);  /* Load re. */
     /* fallthrough */
   case CCX(F, F):
   conv_F_F:
     if (dt == IRT_CDATA || st == IRT_CDATA) goto err_nyi;
     if (dt != st) sp = emitconv(sp, dt, st, 0);
     goto xstore;
 
   /* Destination is a complex number. */
   case CCX(C, I):
   case CCX(C, F):
     {  /* Clear im. */
       TRef ptr = emitir(IRT(IR_ADD, IRT_PTR), dp, lj_ir_kintp(J, (dsize >> 1)));
       emitir(IRT(IR_XSTORE, dt), ptr, lj_ir_knum(J, 0));
     }
     /* Convert to re. */
     if ((sinfo & CTF_FP)) goto conv_F_F; else goto conv_F_I;
 
   case CCX(C, C):
     if (dt == IRT_CDATA || st == IRT_CDATA) goto err_nyi;
     {
       TRef re, im, ptr;
       re = emitir(IRT(IR_XLOAD, st), sp, 0);
       ptr = emitir(IRT(IR_ADD, IRT_PTR), sp, lj_ir_kintp(J, (ssize >> 1)));
       im = emitir(IRT(IR_XLOAD, st), ptr, 0);
       if (dt != st) {
 	re = emitconv(re, dt, st, 0);
 	im = emitconv(im, dt, st, 0);
       }
       emitir(IRT(IR_XSTORE, dt), dp, re);
       ptr = emitir(IRT(IR_ADD, IRT_PTR), dp, lj_ir_kintp(J, (dsize >> 1)));
       emitir(IRT(IR_XSTORE, dt), ptr, im);
     }
     break;
 
   /* Destination is a vector. */
   case CCX(V, I):
   case CCX(V, F):
   case CCX(V, C):
   case CCX(V, V):
     goto err_nyi;
 
   /* Destination is a pointer. */
   case CCX(P, P):
   case CCX(P, A):
   case CCX(P, S):
     /* There are only 32 bit pointers/addresses on 32 bit machines.
     ** Also ok on x64, since all 32 bit ops clear the upper part of the reg.
     */
     goto xstore;
   case CCX(P, I):
     if (st == IRT_CDATA) goto err_nyi;
     if (!LJ_64 && ssize == 8)  /* Truncate from 64 bit integer. */
       sp = emitconv(sp, IRT_U32, st, 0);
     goto xstore;
   case CCX(P, F):
     if (st == IRT_CDATA) goto err_nyi;
     /* The signed conversion is cheaper. x64 really has 47 bit pointers. */
     sp = emitconv(sp, (LJ_64 && dsize == 8) ? IRT_I64 : IRT_U32,
 		  st, IRCONV_ANY);
     goto xstore;
 
   /* Destination is an array. */
   case CCX(A, A):
   /* Destination is a struct/union. */
   case CCX(S, S):
     if (dp == 0) goto err_conv;
     crec_copy(J, dp, sp, lj_ir_kint(J, dsize), d);
     break;
 
   default:
   err_conv:
   err_nyi:
     lj_trace_err(J, LJ_TRERR_NYICONV);
     break;
   }
   return 0;
 }
 
 /* -- Convert C type to TValue (load) ------------------------------------- */
 
 static TRef crec_tv_ct(jit_State *J, CType *s, CTypeID sid, TRef sp)
 {
   CTState *cts = ctype_ctsG(J2G(J));
   IRType t = crec_ct2irt(cts, s);
   CTInfo sinfo = s->info;
   if (ctype_isnum(sinfo)) {
     TRef tr;
     if (t == IRT_CDATA)
       goto err_nyi;  /* NYI: copyval of >64 bit integers. */
     tr = emitir(IRT(IR_XLOAD, t), sp, 0);
     if (t == IRT_FLOAT || t == IRT_U32) {  /* Keep uint32_t/float as numbers. */
       return emitconv(tr, IRT_NUM, t, 0);
     } else if (t == IRT_I64 || t == IRT_U64) {  /* Box 64 bit integer. */
       sp = tr;
       lj_needsplit(J);
     } else if ((sinfo & CTF_BOOL)) {
       /* Assume not equal to zero. Fixup and emit pending guard later. */
       lj_ir_set(J, IRTGI(IR_NE), tr, lj_ir_kint(J, 0));
       J->postproc = LJ_POST_FIXGUARD;
       return TREF_TRUE;
     } else {
       return tr;
     }
   } else if (ctype_isptr(sinfo) || ctype_isenum(sinfo)) {
     sp = emitir(IRT(IR_XLOAD, t), sp, 0);  /* Box pointers and enums. */
   } else if (ctype_isrefarray(sinfo) || ctype_isstruct(sinfo)) {
     cts->L = J->L;
     sid = lj_ctype_intern(cts, CTINFO_REF(sid), CTSIZE_PTR);  /* Create ref. */
   } else if (ctype_iscomplex(sinfo)) {  /* Unbox/box complex. */
     ptrdiff_t esz = (ptrdiff_t)(s->size >> 1);
     TRef ptr, tr1, tr2, dp;
     dp = emitir(IRTG(IR_CNEW, IRT_CDATA), lj_ir_kint(J, sid), TREF_NIL);
     tr1 = emitir(IRT(IR_XLOAD, t), sp, 0);
     ptr = emitir(IRT(IR_ADD, IRT_PTR), sp, lj_ir_kintp(J, esz));
     tr2 = emitir(IRT(IR_XLOAD, t), ptr, 0);
     ptr = emitir(IRT(IR_ADD, IRT_PTR), dp, lj_ir_kintp(J, sizeof(GCcdata)));
     emitir(IRT(IR_XSTORE, t), ptr, tr1);
     ptr = emitir(IRT(IR_ADD, IRT_PTR), dp, lj_ir_kintp(J, sizeof(GCcdata)+esz));
     emitir(IRT(IR_XSTORE, t), ptr, tr2);
     return dp;
   } else {
     /* NYI: copyval of vectors. */
   err_nyi:
     lj_trace_err(J, LJ_TRERR_NYICONV);
   }
   /* Box pointer, ref, enum or 64 bit integer. */
   return emitir(IRTG(IR_CNEWI, IRT_CDATA), lj_ir_kint(J, sid), sp);
 }
 
 /* -- Convert TValue to C type (store) ------------------------------------ */
 
 TRef lj_crec_ct_tv(jit_State *J, CType *d, TRef dp, TRef sp, cTValue *sval)
 {
   CTState *cts = ctype_ctsG(J2G(J));
   CTypeID sid = CTID_P_VOID;
   void *svisnz = 0;
   CType *s;
   if (LJ_LIKELY(tref_isinteger(sp))) {
     sid = CTID_INT32;
     svisnz = (void *)(intptr_t)(tvisint(sval)?(intV(sval)!=0):!tviszero(sval));
   } else if (tref_isnum(sp)) {
     sid = CTID_DOUBLE;
     svisnz = (void *)(intptr_t)(tvisint(sval)?(intV(sval)!=0):!tviszero(sval));
   } else if (tref_isbool(sp)) {
     sp = lj_ir_kint(J, tref_istrue(sp) ? 1 : 0);
     sid = CTID_BOOL;
   } else if (tref_isnil(sp)) {
     sp = lj_ir_kptr(J, NULL);
   } else if (tref_isudata(sp)) {
     GCudata *ud = udataV(sval);
     if (ud->udtype == UDTYPE_IO_FILE) {
       TRef tr = emitir(IRT(IR_FLOAD, IRT_U8), sp, IRFL_UDATA_UDTYPE);
       emitir(IRTGI(IR_EQ), tr, lj_ir_kint(J, UDTYPE_IO_FILE));
       sp = emitir(IRT(IR_FLOAD, IRT_PTR), sp, IRFL_UDATA_FILE);
     } else {
       sp = emitir(IRT(IR_ADD, IRT_PTR), sp, lj_ir_kintp(J, sizeof(GCudata)));
     }
   } else if (tref_isstr(sp)) {
     if (ctype_isenum(d->info)) {  /* Match string against enum constant. */
       GCstr *str = strV(sval);
       CTSize ofs;
       CType *cct = lj_ctype_getfield(cts, d, str, &ofs);
       /* Specialize to the name of the enum constant. */
       emitir(IRTG(IR_EQ, IRT_STR), sp, lj_ir_kstr(J, str));
       if (cct && ctype_isconstval(cct->info)) {
 	lua_assert(ctype_child(cts, cct)->size == 4);
 	svisnz = (void *)(intptr_t)(ofs != 0);
 	sp = lj_ir_kint(J, (int32_t)ofs);
 	sid = ctype_cid(cct->info);
       }  /* else: interpreter will throw. */
     } else if (ctype_isrefarray(d->info)) {  /* Copy string to array. */
       lj_trace_err(J, LJ_TRERR_BADTYPE);  /* NYI */
     } else {  /* Otherwise pass the string data as a const char[]. */
       /* Don't use STRREF. It folds with SNEW, which loses the trailing NUL. */
       sp = emitir(IRT(IR_ADD, IRT_PTR), sp, lj_ir_kintp(J, sizeof(GCstr)));
       sid = CTID_A_CCHAR;
     }
   } else if (tref_islightud(sp)) {
 #if LJ_64
     sp = emitir(IRT(IR_BAND, IRT_P64), sp,
 		lj_ir_kint64(J, U64x(00007fff,ffffffff)));
 #endif
   } else {  /* NYI: tref_istab(sp). */
     IRType t;
     sid = argv2cdata(J, sp, sval)->ctypeid;
     s = ctype_raw(cts, sid);
     svisnz = cdataptr(cdataV(sval));
     if (ctype_isfunc(s->info)) {
       sid = lj_ctype_intern(cts, CTINFO(CT_PTR, CTALIGN_PTR|sid), CTSIZE_PTR);
       s = ctype_get(cts, sid);
       t = IRT_PTR;
     } else {
       t = crec_ct2irt(cts, s);
     }
     if (ctype_isptr(s->info)) {
       sp = emitir(IRT(IR_FLOAD, t), sp, IRFL_CDATA_PTR);
       if (ctype_isref(s->info)) {
 	svisnz = *(void **)svisnz;
 	s = ctype_rawchild(cts, s);
 	if (ctype_isenum(s->info)) s = ctype_child(cts, s);
 	t = crec_ct2irt(cts, s);
       } else {
 	goto doconv;
       }
     } else if (t == IRT_I64 || t == IRT_U64) {
       sp = emitir(IRT(IR_FLOAD, t), sp, IRFL_CDATA_INT64);
       lj_needsplit(J);
       goto doconv;
     } else if (t == IRT_INT || t == IRT_U32) {
       if (ctype_isenum(s->info)) s = ctype_child(cts, s);
       sp = emitir(IRT(IR_FLOAD, t), sp, IRFL_CDATA_INT);
       goto doconv;
     } else {
       sp = emitir(IRT(IR_ADD, IRT_PTR), sp, lj_ir_kintp(J, sizeof(GCcdata)));
     }
     if (ctype_isnum(s->info) && t != IRT_CDATA)
       sp = emitir(IRT(IR_XLOAD, t), sp, 0);  /* Load number value. */
     goto doconv;
   }
   s = ctype_get(cts, sid);
 doconv:
   if (ctype_isenum(d->info)) d = ctype_child(cts, d);
   return crec_ct_ct(J, d, s, dp, sp, svisnz);
 }
 
 /* -- C data metamethods -------------------------------------------------- */
 
 /* This would be rather difficult in FOLD, so do it here:
 ** (base+k)+(idx*sz)+ofs ==> (base+idx*sz)+(ofs+k)
 ** (base+(idx+k)*sz)+ofs ==> (base+idx*sz)+(ofs+k*sz)
 */
 static TRef crec_reassoc_ofs(jit_State *J, TRef tr, ptrdiff_t *ofsp, MSize sz)
 {
   IRIns *ir = IR(tref_ref(tr));
   if (LJ_LIKELY(J->flags & JIT_F_OPT_FOLD) && irref_isk(ir->op2) &&
       (ir->o == IR_ADD || ir->o == IR_ADDOV || ir->o == IR_SUBOV)) {
     IRIns *irk = IR(ir->op2);
     ptrdiff_t k;
     if (LJ_64 && irk->o == IR_KINT64)
       k = (ptrdiff_t)ir_kint64(irk)->u64 * sz;
     else
       k = (ptrdiff_t)irk->i * sz;
     if (ir->o == IR_SUBOV) *ofsp -= k; else *ofsp += k;
     tr = ir->op1;  /* Not a TRef, but the caller doesn't care. */
   }
   return tr;
 }
 
 /* Tailcall to function. */
 static void crec_tailcall(jit_State *J, RecordFFData *rd, cTValue *tv)
 {
   TRef kfunc = lj_ir_kfunc(J, funcV(tv));
 #if LJ_FR2
   J->base[-2] = kfunc;
   J->base[-1] = TREF_FRAME;
 #else
   J->base[-1] = kfunc | TREF_FRAME;
 #endif
   rd->nres = -1;  /* Pending tailcall. */
 }
 
 /* Record ctype __index/__newindex metamethods. */
 static void crec_index_meta(jit_State *J, CTState *cts, CType *ct,
 			    RecordFFData *rd)
 {
   CTypeID id = ctype_typeid(cts, ct);
   cTValue *tv = lj_ctype_meta(cts, id, rd->data ? MM_newindex : MM_index);
   if (!tv)
     lj_trace_err(J, LJ_TRERR_BADTYPE);
   if (tvisfunc(tv)) {
     crec_tailcall(J, rd, tv);
   } else if (rd->data == 0 && tvistab(tv) && tref_isstr(J->base[1])) {
     /* Specialize to result of __index lookup. */
     cTValue *o = lj_tab_get(J->L, tabV(tv), &rd->argv[1]);
     J->base[0] = lj_record_constify(J, o);
     if (!J->base[0])
       lj_trace_err(J, LJ_TRERR_BADTYPE);
     /* Always specialize to the key. */
     emitir(IRTG(IR_EQ, IRT_STR), J->base[1], lj_ir_kstr(J, strV(&rd->argv[1])));
   } else {
     /* NYI: resolving of non-function metamethods. */
     /* NYI: non-string keys for __index table. */
     /* NYI: stores to __newindex table. */
     lj_trace_err(J, LJ_TRERR_BADTYPE);
   }
 }
 
 void LJ_FASTCALL recff_cdata_index(jit_State *J, RecordFFData *rd)
 {
   TRef idx, ptr = J->base[0];
   ptrdiff_t ofs = sizeof(GCcdata);
   GCcdata *cd = argv2cdata(J, ptr, &rd->argv[0]);
   CTState *cts = ctype_ctsG(J2G(J));
   CType *ct = ctype_raw(cts, cd->ctypeid);
   CTypeID sid = 0;
 
   /* Resolve pointer or reference for cdata object. */
   if (ctype_isptr(ct->info)) {
     IRType t = (LJ_64 && ct->size == 8) ? IRT_P64 : IRT_P32;
     if (ctype_isref(ct->info)) ct = ctype_rawchild(cts, ct);
     ptr = emitir(IRT(IR_FLOAD, t), ptr, IRFL_CDATA_PTR);
     ofs = 0;
     ptr = crec_reassoc_ofs(J, ptr, &ofs, 1);
   }
 
 again:
   idx = J->base[1];
   if (tref_isnumber(idx)) {
     idx = lj_opt_narrow_cindex(J, idx);
     if (ctype_ispointer(ct->info)) {
       CTSize sz;
   integer_key:
       if ((ct->info & CTF_COMPLEX))
 	idx = emitir(IRT(IR_BAND, IRT_INTP), idx, lj_ir_kintp(J, 1));
       sz = lj_ctype_size(cts, (sid = ctype_cid(ct->info)));
       idx = crec_reassoc_ofs(J, idx, &ofs, sz);
 #if LJ_TARGET_ARM || LJ_TARGET_PPC
       /* Hoist base add to allow fusion of index/shift into operands. */
       if (LJ_LIKELY(J->flags & JIT_F_OPT_LOOP) && ofs
 #if LJ_TARGET_ARM
 	  && (sz == 1 || sz == 4)
 #endif
 	  ) {
 	ptr = emitir(IRT(IR_ADD, IRT_PTR), ptr, lj_ir_kintp(J, ofs));
 	ofs = 0;
       }
 #endif
       idx = emitir(IRT(IR_MUL, IRT_INTP), idx, lj_ir_kintp(J, sz));
       ptr = emitir(IRT(IR_ADD, IRT_PTR), idx, ptr);
     }
   } else if (tref_iscdata(idx)) {
     GCcdata *cdk = cdataV(&rd->argv[1]);
     CType *ctk = ctype_raw(cts, cdk->ctypeid);
     IRType t = crec_ct2irt(cts, ctk);
     if (ctype_ispointer(ct->info) && t >= IRT_I8 && t <= IRT_U64) {
       if (ctk->size == 8) {
 	idx = emitir(IRT(IR_FLOAD, t), idx, IRFL_CDATA_INT64);
       } else if (ctk->size == 4) {
 	idx = emitir(IRT(IR_FLOAD, t), idx, IRFL_CDATA_INT);
       } else {
 	idx = emitir(IRT(IR_ADD, IRT_PTR), idx,
 		     lj_ir_kintp(J, sizeof(GCcdata)));
 	idx = emitir(IRT(IR_XLOAD, t), idx, 0);
       }
       if (LJ_64 && ctk->size < sizeof(intptr_t) && !(ctk->info & CTF_UNSIGNED))
 	idx = emitconv(idx, IRT_INTP, IRT_INT, IRCONV_SEXT);
       if (!LJ_64 && ctk->size > sizeof(intptr_t)) {
 	idx = emitconv(idx, IRT_INTP, t, 0);
 	lj_needsplit(J);
       }
       goto integer_key;
     }
   } else if (tref_isstr(idx)) {
     GCstr *name = strV(&rd->argv[1]);
     if (cd && cd->ctypeid == CTID_CTYPEID)
       ct = ctype_raw(cts, crec_constructor(J, cd, ptr));
     if (ctype_isstruct(ct->info)) {
       CTSize fofs;
       CType *fct;
       fct = lj_ctype_getfield(cts, ct, name, &fofs);
       if (fct) {
 	/* Always specialize to the field name. */
 	emitir(IRTG(IR_EQ, IRT_STR), idx, lj_ir_kstr(J, name));
 	if (ctype_isconstval(fct->info)) {
 	  if (fct->size >= 0x80000000u &&
 	      (ctype_child(cts, fct)->info & CTF_UNSIGNED)) {
 	    J->base[0] = lj_ir_knum(J, (lua_Number)(uint32_t)fct->size);
 	    return;
 	  }
 	  J->base[0] = lj_ir_kint(J, (int32_t)fct->size);
 	  return;  /* Interpreter will throw for newindex. */
 	} else if (ctype_isbitfield(fct->info)) {
 	  lj_trace_err(J, LJ_TRERR_NYICONV);
 	} else {
 	  lua_assert(ctype_isfield(fct->info));
 	  sid = ctype_cid(fct->info);
 	}
 	ofs += (ptrdiff_t)fofs;
       }
     } else if (ctype_iscomplex(ct->info)) {
       if (name->len == 2 &&
 	  ((strdata(name)[0] == 'r' && strdata(name)[1] == 'e') ||
 	   (strdata(name)[0] == 'i' && strdata(name)[1] == 'm'))) {
 	/* Always specialize to the field name. */
 	emitir(IRTG(IR_EQ, IRT_STR), idx, lj_ir_kstr(J, name));
 	if (strdata(name)[0] == 'i') ofs += (ct->size >> 1);
 	sid = ctype_cid(ct->info);
       }
     }
   }
   if (!sid) {
     if (ctype_isptr(ct->info)) {  /* Automatically perform '->'. */
       CType *cct = ctype_rawchild(cts, ct);
       if (ctype_isstruct(cct->info)) {
 	ct = cct;
 	cd = NULL;
 	if (tref_isstr(idx)) goto again;
       }
     }
     crec_index_meta(J, cts, ct, rd);
     return;
   }
 
   if (ofs)
     ptr = emitir(IRT(IR_ADD, IRT_PTR), ptr, lj_ir_kintp(J, ofs));
 
   /* Resolve reference for field. */
   ct = ctype_get(cts, sid);
   if (ctype_isref(ct->info)) {
     ptr = emitir(IRT(IR_XLOAD, IRT_PTR), ptr, 0);
     sid = ctype_cid(ct->info);
     ct = ctype_get(cts, sid);
   }
 
   while (ctype_isattrib(ct->info))
     ct = ctype_child(cts, ct);  /* Skip attributes. */
 
   if (rd->data == 0) {  /* __index metamethod. */
     J->base[0] = crec_tv_ct(J, ct, sid, ptr);
   } else {  /* __newindex metamethod. */
     rd->nres = 0;
     J->needsnap = 1;
     lj_crec_ct_tv(J, ct, ptr, J->base[2], &rd->argv[2]);
   }
 }
 
 /* Record setting a finalizer. */
 static void crec_finalizer(jit_State *J, TRef trcd, TRef trfin, cTValue *fin)
 {
   if (tvisgcv(fin)) {
     if (!trfin) trfin = lj_ir_kptr(J, gcval(fin));
   } else if (tvisnil(fin)) {
     trfin = lj_ir_kptr(J, NULL);
   } else {
     lj_trace_err(J, LJ_TRERR_BADTYPE);
   }
   lj_ir_call(J, IRCALL_lj_cdata_setfin, trcd,
 	     trfin, lj_ir_kint(J, (int32_t)itype(fin)));
   J->needsnap = 1;
 }
 
 /* Record cdata allocation. */
 static void crec_alloc(jit_State *J, RecordFFData *rd, CTypeID id)
 {
   CTState *cts = ctype_ctsG(J2G(J));
   CTSize sz;
   CTInfo info = lj_ctype_info(cts, id, &sz);
   CType *d = ctype_raw(cts, id);
   TRef trcd, trid = lj_ir_kint(J, id);
   cTValue *fin;
   /* Use special instruction to box pointer or 32/64 bit integer. */
   if (ctype_isptr(info) || (ctype_isinteger(info) && (sz == 4 || sz == 8))) {
     TRef sp = J->base[1] ? lj_crec_ct_tv(J, d, 0, J->base[1], &rd->argv[1]) :
 	      ctype_isptr(info) ? lj_ir_kptr(J, NULL) :
 	      sz == 4 ? lj_ir_kint(J, 0) :
 	      (lj_needsplit(J), lj_ir_kint64(J, 0));
     J->base[0] = emitir(IRTG(IR_CNEWI, IRT_CDATA), trid, sp);
     return;
   } else {
     TRef trsz = TREF_NIL;
     if ((info & CTF_VLA)) {  /* Calculate VLA/VLS size at runtime. */
       CTSize sz0, sz1;
       if (!J->base[1] || J->base[2])
 	lj_trace_err(J, LJ_TRERR_NYICONV);  /* NYI: init VLA/VLS. */
       trsz = lj_crec_ct_tv(J, ctype_get(cts, CTID_INT32), 0,
 			J->base[1], &rd->argv[1]);
       sz0 = lj_ctype_vlsize(cts, d, 0);
       sz1 = lj_ctype_vlsize(cts, d, 1);
       trsz = emitir(IRTGI(IR_MULOV), trsz, lj_ir_kint(J, (int32_t)(sz1-sz0)));
       trsz = emitir(IRTGI(IR_ADDOV), trsz, lj_ir_kint(J, (int32_t)sz0));
       J->base[1] = 0;  /* Simplify logic below. */
     } else if (ctype_align(info) > CT_MEMALIGN) {
       trsz = lj_ir_kint(J, sz);
     }
     trcd = emitir(IRTG(IR_CNEW, IRT_CDATA), trid, trsz);
     if (sz > 128 || (info & CTF_VLA)) {
       TRef dp;
       CTSize align;
     special:  /* Only handle bulk zero-fill for large/VLA/VLS types. */
       if (J->base[1])
 	lj_trace_err(J, LJ_TRERR_NYICONV);  /* NYI: init large/VLA/VLS types. */
       dp = emitir(IRT(IR_ADD, IRT_PTR), trcd, lj_ir_kintp(J, sizeof(GCcdata)));
       if (trsz == TREF_NIL) trsz = lj_ir_kint(J, sz);
       align = ctype_align(info);
       if (align < CT_MEMALIGN) align = CT_MEMALIGN;
       crec_fill(J, dp, trsz, lj_ir_kint(J, 0), (1u << align));
     } else if (J->base[1] && !J->base[2] &&
 	!lj_cconv_multi_init(cts, d, &rd->argv[1])) {
       goto single_init;
     } else if (ctype_isarray(d->info)) {
       CType *dc = ctype_rawchild(cts, d);  /* Array element type. */
       CTSize ofs, esize = dc->size;
       TRef sp = 0;
       TValue tv;
       TValue *sval = &tv;
       MSize i;
       tv.u64 = 0;
       if (!(ctype_isnum(dc->info) || ctype_isptr(dc->info)) ||
 	  esize * CREC_FILL_MAXUNROLL < sz)
 	goto special;
       for (i = 1, ofs = 0; ofs < sz; ofs += esize) {
 	TRef dp = emitir(IRT(IR_ADD, IRT_PTR), trcd,
 			 lj_ir_kintp(J, ofs + sizeof(GCcdata)));
 	if (J->base[i]) {
 	  sp = J->base[i];
 	  sval = &rd->argv[i];
 	  i++;
 	} else if (i != 2) {
 	  sp = ctype_isnum(dc->info) ? lj_ir_kint(J, 0) : TREF_NIL;
 	}
 	lj_crec_ct_tv(J, dc, dp, sp, sval);
       }
     } else if (ctype_isstruct(d->info)) {
       CTypeID fid = d->sib;
       MSize i = 1;
       while (fid) {
 	CType *df = ctype_get(cts, fid);
 	fid = df->sib;
 	if (ctype_isfield(df->info)) {
 	  CType *dc;
 	  TRef sp, dp;
 	  TValue tv;
 	  TValue *sval = &tv;
 	  setintV(&tv, 0);
 	  if (!gcref(df->name)) continue;  /* Ignore unnamed fields. */
 	  dc = ctype_rawchild(cts, df);  /* Field type. */
 	  if (!(ctype_isnum(dc->info) || ctype_isptr(dc->info) ||
 		ctype_isenum(dc->info)))
 	    lj_trace_err(J, LJ_TRERR_NYICONV);  /* NYI: init aggregates. */
 	  if (J->base[i]) {
 	    sp = J->base[i];
 	    sval = &rd->argv[i];
 	    i++;
 	  } else {
 	    sp = ctype_isptr(dc->info) ? TREF_NIL : lj_ir_kint(J, 0);
 	  }
 	  dp = emitir(IRT(IR_ADD, IRT_PTR), trcd,
 		      lj_ir_kintp(J, df->size + sizeof(GCcdata)));
 	  lj_crec_ct_tv(J, dc, dp, sp, sval);
 	} else if (!ctype_isconstval(df->info)) {
 	  /* NYI: init bitfields and sub-structures. */
 	  lj_trace_err(J, LJ_TRERR_NYICONV);
 	}
       }
     } else {
       TRef dp;
     single_init:
       dp = emitir(IRT(IR_ADD, IRT_PTR), trcd, lj_ir_kintp(J, sizeof(GCcdata)));
       if (J->base[1]) {
 	lj_crec_ct_tv(J, d, dp, J->base[1], &rd->argv[1]);
       } else {
 	TValue tv;
 	tv.u64 = 0;
 	lj_crec_ct_tv(J, d, dp, lj_ir_kint(J, 0), &tv);
       }
     }
   }
   J->base[0] = trcd;
   /* Handle __gc metamethod. */
   fin = lj_ctype_meta(cts, id, MM_gc);
   if (fin)
     crec_finalizer(J, trcd, 0, fin);
 }
 
 /* Record argument conversions. */
 static TRef crec_call_args(jit_State *J, RecordFFData *rd,
 			   CTState *cts, CType *ct)
 {
   TRef args[CCI_NARGS_MAX];
   CTypeID fid;
   MSize i, n;
   TRef tr, *base;
   cTValue *o;
 #if LJ_TARGET_X86
 #if LJ_ABI_WIN
   TRef *arg0 = NULL, *arg1 = NULL;
 #endif
   int ngpr = 0;
   if (ctype_cconv(ct->info) == CTCC_THISCALL)
     ngpr = 1;
   else if (ctype_cconv(ct->info) == CTCC_FASTCALL)
     ngpr = 2;
 #endif
 
   /* Skip initial attributes. */
   fid = ct->sib;
   while (fid) {
     CType *ctf = ctype_get(cts, fid);
     if (!ctype_isattrib(ctf->info)) break;
     fid = ctf->sib;
   }
   args[0] = TREF_NIL;
   for (n = 0, base = J->base+1, o = rd->argv+1; *base; n++, base++, o++) {
     CTypeID did;
     CType *d;
 
     if (n >= CCI_NARGS_MAX)
       lj_trace_err(J, LJ_TRERR_NYICALL);
 
     if (fid) {  /* Get argument type from field. */
       CType *ctf = ctype_get(cts, fid);
       fid = ctf->sib;
       lua_assert(ctype_isfield(ctf->info));
       did = ctype_cid(ctf->info);
     } else {
       if (!(ct->info & CTF_VARARG))
 	lj_trace_err(J, LJ_TRERR_NYICALL);  /* Too many arguments. */
       did = lj_ccall_ctid_vararg(cts, o);  /* Infer vararg type. */
     }
     d = ctype_raw(cts, did);
     if (!(ctype_isnum(d->info) || ctype_isptr(d->info) ||
 	  ctype_isenum(d->info)))
       lj_trace_err(J, LJ_TRERR_NYICALL);
     tr = lj_crec_ct_tv(J, d, 0, *base, o);
     if (ctype_isinteger_or_bool(d->info)) {
       if (d->size < 4) {
 	if ((d->info & CTF_UNSIGNED))
 	  tr = emitconv(tr, IRT_INT, d->size==1 ? IRT_U8 : IRT_U16, 0);
 	else
 	  tr = emitconv(tr, IRT_INT, d->size==1 ? IRT_I8 : IRT_I16,IRCONV_SEXT);
       }
     } else if (LJ_SOFTFP && ctype_isfp(d->info) && d->size > 4) {
       lj_needsplit(J);
     }
 #if LJ_TARGET_X86
     /* 64 bit args must not end up in registers for fastcall/thiscall. */
 #if LJ_ABI_WIN
     if (!ctype_isfp(d->info)) {
       /* Sigh, the Windows/x86 ABI allows reordering across 64 bit args. */
       if (tref_typerange(tr, IRT_I64, IRT_U64)) {
 	if (ngpr) {
 	  arg0 = &args[n]; args[n++] = TREF_NIL; ngpr--;
 	  if (ngpr) {
 	    arg1 = &args[n]; args[n++] = TREF_NIL; ngpr--;
 	  }
 	}
       } else {
 	if (arg0) { *arg0 = tr; arg0 = NULL; n--; continue; }
 	if (arg1) { *arg1 = tr; arg1 = NULL; n--; continue; }
 	if (ngpr) ngpr--;
       }
     }
 #else
     if (!ctype_isfp(d->info) && ngpr) {
       if (tref_typerange(tr, IRT_I64, IRT_U64)) {
 	/* No reordering for other x86 ABIs. Simply add alignment args. */
 	do { args[n++] = TREF_NIL; } while (--ngpr);
       } else {
 	ngpr--;
       }
     }
 #endif
 #endif
     args[n] = tr;
   }
   tr = args[0];
   for (i = 1; i < n; i++)
     tr = emitir(IRT(IR_CARG, IRT_NIL), tr, args[i]);
   return tr;
 }
 
 /* Create a snapshot for the caller, simulating a 'false' return value. */
 static void crec_snap_caller(jit_State *J)
 {
   lua_State *L = J->L;
   TValue *base = L->base, *top = L->top;
   const BCIns *pc = J->pc;
   TRef ftr = J->base[-1-LJ_FR2];
   ptrdiff_t delta;
   if (!frame_islua(base-1) || J->framedepth <= 0)
     lj_trace_err(J, LJ_TRERR_NYICALL);
   J->pc = frame_pc(base-1); delta = 1+LJ_FR2+bc_a(J->pc[-1]);
   L->top = base; L->base = base - delta;
   J->base[-1-LJ_FR2] = TREF_FALSE;
   J->base -= delta; J->baseslot -= (BCReg)delta;
   J->maxslot = (BCReg)delta-LJ_FR2; J->framedepth--;
   lj_snap_add(J);
   L->base = base; L->top = top;
   J->framedepth++; J->maxslot = 1;
   J->base += delta; J->baseslot += (BCReg)delta;
   J->base[-1-LJ_FR2] = ftr; J->pc = pc;
 }
 
 /* Record function call. */
 static int crec_call(jit_State *J, RecordFFData *rd, GCcdata *cd)
 {
   CTState *cts = ctype_ctsG(J2G(J));
   CType *ct = ctype_raw(cts, cd->ctypeid);
   IRType tp = IRT_PTR;
   if (ctype_isptr(ct->info)) {
     tp = (LJ_64 && ct->size == 8) ? IRT_P64 : IRT_P32;
     ct = ctype_rawchild(cts, ct);
   }
   if (ctype_isfunc(ct->info)) {
     TRef func = emitir(IRT(IR_FLOAD, tp), J->base[0], IRFL_CDATA_PTR);
     CType *ctr = ctype_rawchild(cts, ct);
     IRType t = crec_ct2irt(cts, ctr);
     TRef tr;
     TValue tv;
     /* Check for blacklisted C functions that might call a callback. */
     setlightudV(&tv,
 		cdata_getptr(cdataptr(cd), (LJ_64 && tp == IRT_P64) ? 8 : 4));
     if (tvistrue(lj_tab_get(J->L, cts->miscmap, &tv)))
       lj_trace_err(J, LJ_TRERR_BLACKL);
     if (ctype_isvoid(ctr->info)) {
       t = IRT_NIL;
       rd->nres = 0;
     } else if (!(ctype_isnum(ctr->info) || ctype_isptr(ctr->info) ||
 		 ctype_isenum(ctr->info)) || t == IRT_CDATA) {
       lj_trace_err(J, LJ_TRERR_NYICALL);
     }
     if ((ct->info & CTF_VARARG)
 #if LJ_TARGET_X86
 	|| ctype_cconv(ct->info) != CTCC_CDECL
 #endif
 	)
       func = emitir(IRT(IR_CARG, IRT_NIL), func,
 		    lj_ir_kint(J, ctype_typeid(cts, ct)));
     tr = emitir(IRT(IR_CALLXS, t), crec_call_args(J, rd, cts, ct), func);
     if (ctype_isbool(ctr->info)) {
       if (frame_islua(J->L->base-1) && bc_b(frame_pc(J->L->base-1)[-1]) == 1) {
 	/* Don't check result if ignored. */
 	tr = TREF_NIL;
       } else {
 	crec_snap_caller(J);
 #if LJ_TARGET_X86ORX64
 	/* Note: only the x86/x64 backend supports U8 and only for EQ(tr, 0). */
 	lj_ir_set(J, IRTG(IR_NE, IRT_U8), tr, lj_ir_kint(J, 0));
 #else
 	lj_ir_set(J, IRTGI(IR_NE), tr, lj_ir_kint(J, 0));
 #endif
 	J->postproc = LJ_POST_FIXGUARDSNAP;
 	tr = TREF_TRUE;
       }
     } else if (t == IRT_PTR || (LJ_64 && t == IRT_P32) ||
 	       t == IRT_I64 || t == IRT_U64 || ctype_isenum(ctr->info)) {
       TRef trid = lj_ir_kint(J, ctype_cid(ct->info));
       tr = emitir(IRTG(IR_CNEWI, IRT_CDATA), trid, tr);
       if (t == IRT_I64 || t == IRT_U64) lj_needsplit(J);
     } else if (t == IRT_FLOAT || t == IRT_U32) {
       tr = emitconv(tr, IRT_NUM, t, 0);
     } else if (t == IRT_I8 || t == IRT_I16) {
       tr = emitconv(tr, IRT_INT, t, IRCONV_SEXT);
     } else if (t == IRT_U8 || t == IRT_U16) {
       tr = emitconv(tr, IRT_INT, t, 0);
     }
     J->base[0] = tr;
     J->needsnap = 1;
     return 1;
   }
   return 0;
 }
 
 void LJ_FASTCALL recff_cdata_call(jit_State *J, RecordFFData *rd)
 {
   CTState *cts = ctype_ctsG(J2G(J));
   GCcdata *cd = argv2cdata(J, J->base[0], &rd->argv[0]);
   CTypeID id = cd->ctypeid;
   CType *ct;
   cTValue *tv;
   MMS mm = MM_call;
   if (id == CTID_CTYPEID) {
     id = crec_constructor(J, cd, J->base[0]);
     mm = MM_new;
   } else if (crec_call(J, rd, cd)) {
     return;
   }
   /* Record ctype __call/__new metamethod. */
   ct = ctype_raw(cts, id);
   tv = lj_ctype_meta(cts, ctype_isptr(ct->info) ? ctype_cid(ct->info) : id, mm);
   if (tv) {
     if (tvisfunc(tv)) {
       crec_tailcall(J, rd, tv);
       return;
     }
   } else if (mm == MM_new) {
     crec_alloc(J, rd, id);
     return;
   }
   /* No metamethod or NYI: non-function metamethods. */
   lj_trace_err(J, LJ_TRERR_BADTYPE);
 }
 
 static TRef crec_arith_int64(jit_State *J, TRef *sp, CType **s, MMS mm)
 {
   if (ctype_isnum(s[0]->info) && ctype_isnum(s[1]->info)) {
     IRType dt;
     CTypeID id;
     TRef tr;
     MSize i;
     IROp op;
     lj_needsplit(J);
     if (((s[0]->info & CTF_UNSIGNED) && s[0]->size == 8) ||
 	((s[1]->info & CTF_UNSIGNED) && s[1]->size == 8)) {
       dt = IRT_U64; id = CTID_UINT64;
     } else {
       dt = IRT_I64; id = CTID_INT64;
       if (mm < MM_add &&
 	  !((s[0]->info | s[1]->info) & CTF_FP) &&
 	  s[0]->size == 4 && s[1]->size == 4) {  /* Try to narrow comparison. */
 	if (!((s[0]->info ^ s[1]->info) & CTF_UNSIGNED) ||
 	    (tref_isk(sp[1]) && IR(tref_ref(sp[1]))->i >= 0)) {
 	  dt = (s[0]->info & CTF_UNSIGNED) ? IRT_U32 : IRT_INT;
 	  goto comp;
 	} else if (tref_isk(sp[0]) && IR(tref_ref(sp[0]))->i >= 0) {
 	  dt = (s[1]->info & CTF_UNSIGNED) ? IRT_U32 : IRT_INT;
 	  goto comp;
 	}
       }
     }
     for (i = 0; i < 2; i++) {
       IRType st = tref_type(sp[i]);
       if (st == IRT_NUM || st == IRT_FLOAT)
 	sp[i] = emitconv(sp[i], dt, st, IRCONV_ANY);
       else if (!(st == IRT_I64 || st == IRT_U64))
 	sp[i] = emitconv(sp[i], dt, IRT_INT,
 			 (s[i]->info & CTF_UNSIGNED) ? 0 : IRCONV_SEXT);
     }
     if (mm < MM_add) {
     comp:
       /* Assume true comparison. Fixup and emit pending guard later. */
       if (mm == MM_eq) {
 	op = IR_EQ;
       } else {
 	op = mm == MM_lt ? IR_LT : IR_LE;
 	if (dt == IRT_U32 || dt == IRT_U64)
 	  op += (IR_ULT-IR_LT);
       }
       lj_ir_set(J, IRTG(op, dt), sp[0], sp[1]);
       J->postproc = LJ_POST_FIXGUARD;
       return TREF_TRUE;
     } else {
       tr = emitir(IRT(mm+(int)IR_ADD-(int)MM_add, dt), sp[0], sp[1]);
     }
     return emitir(IRTG(IR_CNEWI, IRT_CDATA), lj_ir_kint(J, id), tr);
   }
   return 0;
 }
 
 static TRef crec_arith_ptr(jit_State *J, TRef *sp, CType **s, MMS mm)
 {
   CTState *cts = ctype_ctsG(J2G(J));
   CType *ctp = s[0];
   if (ctype_isptr(ctp->info) || ctype_isrefarray(ctp->info)) {
     if ((mm == MM_sub || mm == MM_eq || mm == MM_lt || mm == MM_le) &&
 	(ctype_isptr(s[1]->info) || ctype_isrefarray(s[1]->info))) {
       if (mm == MM_sub) {  /* Pointer difference. */
 	TRef tr;
 	CTSize sz = lj_ctype_size(cts, ctype_cid(ctp->info));
 	if (sz == 0 || (sz & (sz-1)) != 0)
 	  return 0;  /* NYI: integer division. */
 	tr = emitir(IRT(IR_SUB, IRT_INTP), sp[0], sp[1]);
 	tr = emitir(IRT(IR_BSAR, IRT_INTP), tr, lj_ir_kint(J, lj_fls(sz)));
 #if LJ_64
 	tr = emitconv(tr, IRT_NUM, IRT_INTP, 0);
 #endif
 	return tr;
       } else {  /* Pointer comparison (unsigned). */
 	/* Assume true comparison. Fixup and emit pending guard later. */
 	IROp op = mm == MM_eq ? IR_EQ : mm == MM_lt ? IR_ULT : IR_ULE;
 	lj_ir_set(J, IRTG(op, IRT_PTR), sp[0], sp[1]);
 	J->postproc = LJ_POST_FIXGUARD;
 	return TREF_TRUE;
       }
     }
     if (!((mm == MM_add || mm == MM_sub) && ctype_isnum(s[1]->info)))
       return 0;
   } else if (mm == MM_add && ctype_isnum(ctp->info) &&
 	     (ctype_isptr(s[1]->info) || ctype_isrefarray(s[1]->info))) {
     TRef tr = sp[0]; sp[0] = sp[1]; sp[1] = tr;  /* Swap pointer and index. */
     ctp = s[1];
   } else {
     return 0;
   }
   {
     TRef tr = sp[1];
     IRType t = tref_type(tr);
     CTSize sz = lj_ctype_size(cts, ctype_cid(ctp->info));
     CTypeID id;
 #if LJ_64
     if (t == IRT_NUM || t == IRT_FLOAT)
       tr = emitconv(tr, IRT_INTP, t, IRCONV_ANY);
     else if (!(t == IRT_I64 || t == IRT_U64))
       tr = emitconv(tr, IRT_INTP, IRT_INT,
 		    ((t - IRT_I8) & 1) ? 0 : IRCONV_SEXT);
 #else
     if (!tref_typerange(sp[1], IRT_I8, IRT_U32)) {
       tr = emitconv(tr, IRT_INTP, t,
 		    (t == IRT_NUM || t == IRT_FLOAT) ? IRCONV_ANY : 0);
     }
 #endif
     tr = emitir(IRT(IR_MUL, IRT_INTP), tr, lj_ir_kintp(J, sz));
     tr = emitir(IRT(mm+(int)IR_ADD-(int)MM_add, IRT_PTR), sp[0], tr);
     id = lj_ctype_intern(cts, CTINFO(CT_PTR, CTALIGN_PTR|ctype_cid(ctp->info)),
 			 CTSIZE_PTR);
     return emitir(IRTG(IR_CNEWI, IRT_CDATA), lj_ir_kint(J, id), tr);
   }
 }
 
 /* Record ctype arithmetic metamethods. */
 static TRef crec_arith_meta(jit_State *J, TRef *sp, CType **s, CTState *cts,
 			    RecordFFData *rd)
 {
   cTValue *tv = NULL;
   if (J->base[0]) {
     if (tviscdata(&rd->argv[0])) {
       CTypeID id = argv2cdata(J, J->base[0], &rd->argv[0])->ctypeid;
       CType *ct = ctype_raw(cts, id);
       if (ctype_isptr(ct->info)) id = ctype_cid(ct->info);
       tv = lj_ctype_meta(cts, id, (MMS)rd->data);
     }
     if (!tv && J->base[1] && tviscdata(&rd->argv[1])) {
       CTypeID id = argv2cdata(J, J->base[1], &rd->argv[1])->ctypeid;
       CType *ct = ctype_raw(cts, id);
       if (ctype_isptr(ct->info)) id = ctype_cid(ct->info);
       tv = lj_ctype_meta(cts, id, (MMS)rd->data);
     }
   }
   if (tv) {
     if (tvisfunc(tv)) {
       crec_tailcall(J, rd, tv);
       return 0;
     }  /* NYI: non-function metamethods. */
   } else if ((MMS)rd->data == MM_eq) {  /* Fallback cdata pointer comparison. */
     if (sp[0] && sp[1] && ctype_isnum(s[0]->info) == ctype_isnum(s[1]->info)) {
       /* Assume true comparison. Fixup and emit pending guard later. */
       lj_ir_set(J, IRTG(IR_EQ, IRT_PTR), sp[0], sp[1]);
       J->postproc = LJ_POST_FIXGUARD;
       return TREF_TRUE;
     } else {
       return TREF_FALSE;
     }
   }
   lj_trace_err(J, LJ_TRERR_BADTYPE);
   return 0;
 }
 
 void LJ_FASTCALL recff_cdata_arith(jit_State *J, RecordFFData *rd)
 {
   CTState *cts = ctype_ctsG(J2G(J));
   TRef sp[2];
   CType *s[2];
   MSize i;
   for (i = 0; i < 2; i++) {
     TRef tr = J->base[i];
     CType *ct = ctype_get(cts, CTID_DOUBLE);
     if (!tr) {
       lj_trace_err(J, LJ_TRERR_BADTYPE);
     } else if (tref_iscdata(tr)) {
       CTypeID id = argv2cdata(J, tr, &rd->argv[i])->ctypeid;
       IRType t;
       ct = ctype_raw(cts, id);
       t = crec_ct2irt(cts, ct);
       if (ctype_isptr(ct->info)) {  /* Resolve pointer or reference. */
 	tr = emitir(IRT(IR_FLOAD, t), tr, IRFL_CDATA_PTR);
 	if (ctype_isref(ct->info)) {
 	  ct = ctype_rawchild(cts, ct);
 	  t = crec_ct2irt(cts, ct);
 	}
       } else if (t == IRT_I64 || t == IRT_U64) {
 	tr = emitir(IRT(IR_FLOAD, t), tr, IRFL_CDATA_INT64);
 	lj_needsplit(J);
 	goto ok;
       } else if (t == IRT_INT || t == IRT_U32) {
 	tr = emitir(IRT(IR_FLOAD, t), tr, IRFL_CDATA_INT);
 	if (ctype_isenum(ct->info)) ct = ctype_child(cts, ct);
 	goto ok;
       } else if (ctype_isfunc(ct->info)) {
 	tr = emitir(IRT(IR_FLOAD, IRT_PTR), tr, IRFL_CDATA_PTR);
 	ct = ctype_get(cts,
 	  lj_ctype_intern(cts, CTINFO(CT_PTR, CTALIGN_PTR|id), CTSIZE_PTR));
 	goto ok;
       } else {
 	tr = emitir(IRT(IR_ADD, IRT_PTR), tr, lj_ir_kintp(J, sizeof(GCcdata)));
       }
       if (ctype_isenum(ct->info)) ct = ctype_child(cts, ct);
       if (ctype_isnum(ct->info)) {
 	if (t == IRT_CDATA) {
 	  tr = 0;
 	} else {
 	  if (t == IRT_I64 || t == IRT_U64) lj_needsplit(J);
 	  tr = emitir(IRT(IR_XLOAD, t), tr, 0);
 	}
       }
     } else if (tref_isnil(tr)) {
       tr = lj_ir_kptr(J, NULL);
       ct = ctype_get(cts, CTID_P_VOID);
     } else if (tref_isinteger(tr)) {
       ct = ctype_get(cts, CTID_INT32);
     } else if (tref_isstr(tr)) {
       TRef tr2 = J->base[1-i];
       CTypeID id = argv2cdata(J, tr2, &rd->argv[1-i])->ctypeid;
       ct = ctype_raw(cts, id);
       if (ctype_isenum(ct->info)) {  /* Match string against enum constant. */
 	GCstr *str = strV(&rd->argv[i]);
 	CTSize ofs;
 	CType *cct = lj_ctype_getfield(cts, ct, str, &ofs);
 	if (cct && ctype_isconstval(cct->info)) {
 	  /* Specialize to the name of the enum constant. */
 	  emitir(IRTG(IR_EQ, IRT_STR), tr, lj_ir_kstr(J, str));
 	  ct = ctype_child(cts, cct);
 	  tr = lj_ir_kint(J, (int32_t)ofs);
 	} else {  /* Interpreter will throw or return false. */
 	  ct = ctype_get(cts, CTID_P_VOID);
 	}
       } else if (ctype_isptr(ct->info)) {
 	tr = emitir(IRT(IR_ADD, IRT_PTR), tr, lj_ir_kintp(J, sizeof(GCstr)));
       } else {
 	ct = ctype_get(cts, CTID_P_VOID);
       }
     } else if (!tref_isnum(tr)) {
       tr = 0;
       ct = ctype_get(cts, CTID_P_VOID);
     }
   ok:
     s[i] = ct;
     sp[i] = tr;
   }
   {
     TRef tr;
     if (!(tr = crec_arith_int64(J, sp, s, (MMS)rd->data)) &&
 	!(tr = crec_arith_ptr(J, sp, s, (MMS)rd->data)) &&
 	!(tr = crec_arith_meta(J, sp, s, cts, rd)))
       return;
     J->base[0] = tr;
     /* Fixup cdata comparisons, too. Avoids some cdata escapes. */
     if (J->postproc == LJ_POST_FIXGUARD && frame_iscont(J->L->base-1) &&
 	!irt_isguard(J->guardemit)) {
       const BCIns *pc = frame_contpc(J->L->base-1) - 1;
       if (bc_op(*pc) <= BC_ISNEP) {
 	J2G(J)->tmptv.u64 = (uint64_t)(uintptr_t)pc;
 	J->postproc = LJ_POST_FIXCOMP;
       }
     }
   }
 }
 
 /* -- C library namespace metamethods ------------------------------------- */
 
 void LJ_FASTCALL recff_clib_index(jit_State *J, RecordFFData *rd)
 {
   CTState *cts = ctype_ctsG(J2G(J));
   if (tref_isudata(J->base[0]) && tref_isstr(J->base[1]) &&
       udataV(&rd->argv[0])->udtype == UDTYPE_FFI_CLIB) {
     CLibrary *cl = (CLibrary *)uddata(udataV(&rd->argv[0]));
     GCstr *name = strV(&rd->argv[1]);
     CType *ct;
     CTypeID id = lj_ctype_getname(cts, &ct, name, CLNS_INDEX);
     cTValue *tv = lj_tab_getstr(cl->cache, name);
     rd->nres = rd->data;
     if (id && tv && !tvisnil(tv)) {
       /* Specialize to the symbol name and make the result a constant. */
       emitir(IRTG(IR_EQ, IRT_STR), J->base[1], lj_ir_kstr(J, name));
       if (ctype_isconstval(ct->info)) {
 	if (ct->size >= 0x80000000u &&
 	    (ctype_child(cts, ct)->info & CTF_UNSIGNED))
 	  J->base[0] = lj_ir_knum(J, (lua_Number)(uint32_t)ct->size);
 	else
 	  J->base[0] = lj_ir_kint(J, (int32_t)ct->size);
       } else if (ctype_isextern(ct->info)) {
 	CTypeID sid = ctype_cid(ct->info);
 	void *sp = *(void **)cdataptr(cdataV(tv));
 	TRef ptr;
 	ct = ctype_raw(cts, sid);
 	if (LJ_64 && !checkptr32(sp))
 	  ptr = lj_ir_kintp(J, (uintptr_t)sp);
 	else
 	  ptr = lj_ir_kptr(J, sp);
 	if (rd->data) {
 	  J->base[0] = crec_tv_ct(J, ct, sid, ptr);
 	} else {
 	  J->needsnap = 1;
 	  lj_crec_ct_tv(J, ct, ptr, J->base[2], &rd->argv[2]);
 	}
       } else {
 	J->base[0] = lj_ir_kgc(J, obj2gco(cdataV(tv)), IRT_CDATA);
       }
     } else {
       lj_trace_err(J, LJ_TRERR_NOCACHE);
     }
   }  /* else: interpreter will throw. */
 }
 
 /* -- FFI library functions ----------------------------------------------- */
 
 static TRef crec_toint(jit_State *J, CTState *cts, TRef sp, TValue *sval)
 {
   return lj_crec_ct_tv(J, ctype_get(cts, CTID_INT32), 0, sp, sval);
 }
 
 void LJ_FASTCALL recff_ffi_new(jit_State *J, RecordFFData *rd)
 {
   crec_alloc(J, rd, argv2ctype(J, J->base[0], &rd->argv[0]));
 }
 
 void LJ_FASTCALL recff_ffi_errno(jit_State *J, RecordFFData *rd)
 {
   UNUSED(rd);
   if (J->base[0])
     lj_trace_err(J, LJ_TRERR_NYICALL);
   J->base[0] = lj_ir_call(J, IRCALL_lj_vm_errno);
 }
 
 void LJ_FASTCALL recff_ffi_string(jit_State *J, RecordFFData *rd)
 {
   CTState *cts = ctype_ctsG(J2G(J));
   TRef tr = J->base[0];
   if (tr) {
     TRef trlen = J->base[1];
     if (!tref_isnil(trlen)) {
       trlen = crec_toint(J, cts, trlen, &rd->argv[1]);
       tr = lj_crec_ct_tv(J, ctype_get(cts, CTID_P_CVOID), 0, tr, &rd->argv[0]);
     } else {
       tr = lj_crec_ct_tv(J, ctype_get(cts, CTID_P_CCHAR), 0, tr, &rd->argv[0]);
       trlen = lj_ir_call(J, IRCALL_strlen, tr);
     }
     J->base[0] = emitir(IRT(IR_XSNEW, IRT_STR), tr, trlen);
   }  /* else: interpreter will throw. */
 }
 
 void LJ_FASTCALL recff_ffi_strbuf(jit_State *J, RecordFFData *rd)
 {
   CTState *cts = ctype_ctsG(J2G(J));
   TRef dst, tr = J->base[0];
   if (tr) {
     dst = lj_crec_ct_tv(J, ctype_get(cts, CTID_P_VOID), 0, tr, &rd->argv[0]);
     /* TBD: big endian */
     TRef hdr = lj_ir_kint64(J, ((0L|LJ_GC_SFIXED|LJ_GC_FIXED)<<32) +
         ((0L+(~LJ_TSTR))<<40));
     emitir(IRT(IR_XSTORE, IRT_U64), dst, hdr);
     J->base[0] = emitconv(dst, IRT_STR, IRT_PTR, 0);
     emitir(IRT(IR_XBAR, IRT_NIL), 0, 0);
   }
 }
 
 void LJ_FASTCALL recff_ffi_copy(jit_State *J, RecordFFData *rd)
 {
   CTState *cts = ctype_ctsG(J2G(J));
   TRef trdst = J->base[0], trsrc = J->base[1], trlen = J->base[2];
   if (trdst && trsrc && (trlen || tref_isstr(trsrc))) {
     trdst = lj_crec_ct_tv(J, ctype_get(cts, CTID_P_VOID), 0, trdst, &rd->argv[0]);
     trsrc = lj_crec_ct_tv(J, ctype_get(cts, CTID_P_CVOID), 0, trsrc, &rd->argv[1]);
     if (trlen) {
       trlen = crec_toint(J, cts, trlen, &rd->argv[2]);
     } else {
       trlen = emitir(IRTI(IR_FLOAD), J->base[1], IRFL_STR_LEN);
       trlen = emitir(IRTI(IR_ADD), trlen, lj_ir_kint(J, 1));
     }
     rd->nres = 0;
     crec_copy(J, trdst, trsrc, trlen, NULL);
   }  /* else: interpreter will throw. */
 }
 
 void LJ_FASTCALL recff_ffi_fill(jit_State *J, RecordFFData *rd)
 {
   CTState *cts = ctype_ctsG(J2G(J));
   TRef trdst = J->base[0], trlen = J->base[1], trfill = J->base[2];
   if (trdst && trlen) {
     CTSize step = 1;
     if (tviscdata(&rd->argv[0])) {  /* Get alignment of original destination. */
       CTSize sz;
       CType *ct = ctype_raw(cts, cdataV(&rd->argv[0])->ctypeid);
       if (ctype_isptr(ct->info))
 	ct = ctype_rawchild(cts, ct);
       step = (1u<<ctype_align(lj_ctype_info(cts, ctype_typeid(cts, ct), &sz)));
     }
     trdst = lj_crec_ct_tv(J, ctype_get(cts, CTID_P_VOID), 0, trdst, &rd->argv[0]);
     trlen = crec_toint(J, cts, trlen, &rd->argv[1]);
     if (trfill)
       trfill = crec_toint(J, cts, trfill, &rd->argv[2]);
     else
       trfill = lj_ir_kint(J, 0);
     rd->nres = 0;
     crec_fill(J, trdst, trlen, trfill, step);
   }  /* else: interpreter will throw. */
 }
 
 void LJ_FASTCALL recff_ffi_typeof(jit_State *J, RecordFFData *rd)
 {
   if (tref_iscdata(J->base[0])) {
     TRef trid = lj_ir_kint(J, argv2ctype(J, J->base[0], &rd->argv[0]));
     J->base[0] = emitir(IRTG(IR_CNEWI, IRT_CDATA),
 			lj_ir_kint(J, CTID_CTYPEID), trid);
   } else {
     setfuncV(J->L, &J->errinfo, J->fn);
     lj_trace_err_info(J, LJ_TRERR_NYIFFU);
   }
 }
 
 void LJ_FASTCALL recff_ffi_istype(jit_State *J, RecordFFData *rd)
 {
   argv2ctype(J, J->base[0], &rd->argv[0]);
   if (tref_iscdata(J->base[1])) {
     argv2ctype(J, J->base[1], &rd->argv[1]);
     J->postproc = LJ_POST_FIXBOOL;
     J->base[0] = TREF_TRUE;
   } else {
     J->base[0] = TREF_FALSE;
   }
 }
 
 void LJ_FASTCALL recff_ffi_abi(jit_State *J, RecordFFData *rd)
 {
   if (tref_isstr(J->base[0])) {
     /* Specialize to the ABI string to make the boolean result a constant. */
     emitir(IRTG(IR_EQ, IRT_STR), J->base[0], lj_ir_kstr(J, strV(&rd->argv[0])));
     J->postproc = LJ_POST_FIXBOOL;
     J->base[0] = TREF_TRUE;
   } else {
     lj_trace_err(J, LJ_TRERR_BADTYPE);
   }
 }
 
 /* Record ffi.sizeof(), ffi.alignof(), ffi.offsetof(). */
 void LJ_FASTCALL recff_ffi_xof(jit_State *J, RecordFFData *rd)
 {
   CTypeID id = argv2ctype(J, J->base[0], &rd->argv[0]);
   if (rd->data == FF_ffi_sizeof) {
     CType *ct = lj_ctype_rawref(ctype_ctsG(J2G(J)), id);
     if (ctype_isvltype(ct->info))
       lj_trace_err(J, LJ_TRERR_BADTYPE);
   } else if (rd->data == FF_ffi_offsetof) {  /* Specialize to the field name. */
     if (!tref_isstr(J->base[1]))
       lj_trace_err(J, LJ_TRERR_BADTYPE);
     emitir(IRTG(IR_EQ, IRT_STR), J->base[1], lj_ir_kstr(J, strV(&rd->argv[1])));
     rd->nres = 3;  /* Just in case. */
   }
   J->postproc = LJ_POST_FIXCONST;
   J->base[0] = J->base[1] = J->base[2] = TREF_NIL;
 }
 
 void LJ_FASTCALL recff_ffi_gc(jit_State *J, RecordFFData *rd)
 {
   argv2cdata(J, J->base[0], &rd->argv[0]);
   if (!J->base[1])
     lj_trace_err(J, LJ_TRERR_BADTYPE);
   crec_finalizer(J, J->base[0], J->base[1], &rd->argv[1]);
 }
 
 /* -- 64 bit bit.* library functions -------------------------------------- */
 
 /* Determine bit operation type from argument type. */
 CTypeID LJ_FASTCALL lj_crec_bit64_type(CTState *cts, cTValue *tv)
 {
   if (tviscdata(tv)) {
     CType *ct = lj_ctype_rawref(cts, cdataV(tv)->ctypeid);
     if (ctype_isenum(ct->info)) ct = ctype_child(cts, ct);
     if ((ct->info & (CTMASK_NUM|CTF_BOOL|CTF_FP|CTF_UNSIGNED)) ==
 	CTINFO(CT_NUM, CTF_UNSIGNED) && ct->size == 8)
       return CTID_UINT64;  /* Use uint64_t, since it has the highest rank. */
     return CTID_INT64;  /* Otherwise use int64_t. */
   }
   return 0;  /* Use regular 32 bit ops. */
 }
 
 void LJ_FASTCALL recff_bit64_tobit(jit_State *J, RecordFFData *rd)
 {
   CTState *cts = ctype_ctsG(J2G(J));
   TRef tr = lj_crec_ct_tv(J, ctype_get(cts, CTID_INT64), 0,
 		       J->base[0], &rd->argv[0]);
   if (!tref_isinteger(tr))
     tr = emitconv(tr, IRT_INT, tref_type(tr), 0);
   J->base[0] = tr;
 }
 
 int LJ_FASTCALL recff_bit64_unary(jit_State *J, RecordFFData *rd)
 {
   CTState *cts = ctype_ctsG(J2G(J));
   CTypeID id = lj_crec_bit64_type(cts, &rd->argv[0]);
   if (id) {
     TRef tr = lj_crec_ct_tv(J, ctype_get(cts, id), 0, J->base[0], &rd->argv[0]);
     tr = emitir(IRT(rd->data, id-CTID_INT64+IRT_I64), tr, 0);
     J->base[0] = emitir(IRTG(IR_CNEWI, IRT_CDATA), lj_ir_kint(J, id), tr);
     return 1;
   }
   return 0;
 }
 
 int LJ_FASTCALL recff_bit64_nary(jit_State *J, RecordFFData *rd)
 {
   CTState *cts = ctype_ctsG(J2G(J));
   CTypeID id = 0;
   MSize i;
   for (i = 0; J->base[i] != 0; i++) {
     CTypeID aid = lj_crec_bit64_type(cts, &rd->argv[i]);
     if (id < aid) id = aid;  /* Determine highest type rank of all arguments. */
   }
   if (id) {
     CType *ct = ctype_get(cts, id);
     uint32_t ot = IRT(rd->data, id-CTID_INT64+IRT_I64);
     TRef tr = lj_crec_ct_tv(J, ct, 0, J->base[0], &rd->argv[0]);
     for (i = 1; J->base[i] != 0; i++) {
       TRef tr2 = lj_crec_ct_tv(J, ct, 0, J->base[i], &rd->argv[i]);
       tr = emitir(ot, tr, tr2);
     }
     J->base[0] = emitir(IRTG(IR_CNEWI, IRT_CDATA), lj_ir_kint(J, id), tr);
     return 1;
   }
   return 0;
 }
 
 int LJ_FASTCALL recff_bit64_shift(jit_State *J, RecordFFData *rd)
 {
   CTState *cts = ctype_ctsG(J2G(J));
   CTypeID id;
   TRef tsh = 0;
   if (J->base[0] && tref_iscdata(J->base[1])) {
     tsh = lj_crec_ct_tv(J, ctype_get(cts, CTID_INT64), 0,
 		     J->base[1], &rd->argv[1]);
     if (!tref_isinteger(tsh))
       tsh = emitconv(tsh, IRT_INT, tref_type(tsh), 0);
     J->base[1] = tsh;
   }
   id = lj_crec_bit64_type(cts, &rd->argv[0]);
   if (id) {
     TRef tr = lj_crec_ct_tv(J, ctype_get(cts, id), 0, J->base[0], &rd->argv[0]);
     uint32_t op = rd->data;
     if (!tsh) tsh = lj_opt_narrow_tobit(J, J->base[1]);
     if (!(op < IR_BROL ? LJ_TARGET_MASKSHIFT : LJ_TARGET_MASKROT) &&
 	!tref_isk(tsh))
       tsh = emitir(IRTI(IR_BAND), tsh, lj_ir_kint(J, 63));
 #ifdef LJ_TARGET_UNIFYROT
       if (op == (LJ_TARGET_UNIFYROT == 1 ? IR_BROR : IR_BROL)) {
 	op = LJ_TARGET_UNIFYROT == 1 ? IR_BROL : IR_BROR;
 	tsh = emitir(IRTI(IR_NEG), tsh, tsh);
       }
 #endif
     tr = emitir(IRT(op, id-CTID_INT64+IRT_I64), tr, tsh);
     J->base[0] = emitir(IRTG(IR_CNEWI, IRT_CDATA), lj_ir_kint(J, id), tr);
     return 1;
   }
   return 0;
 }
 
 TRef recff_bit64_tohex(jit_State *J, RecordFFData *rd, TRef hdr)
 {
   CTState *cts = ctype_ctsG(J2G(J));
   CTypeID id = lj_crec_bit64_type(cts, &rd->argv[0]);
   TRef tr, trsf = J->base[1];
   SFormat sf = (STRFMT_UINT|STRFMT_T_HEX);
   int32_t n;
   if (trsf) {
     CTypeID id2 = 0;
     n = (int32_t)lj_carith_check64(J->L, 2, &id2);
     if (id2)
       trsf = lj_crec_ct_tv(J, ctype_get(cts, CTID_INT32), 0, trsf, &rd->argv[1]);
     else
       trsf = lj_opt_narrow_tobit(J, trsf);
     emitir(IRTGI(IR_EQ), trsf, lj_ir_kint(J, n));  /* Specialize to n. */
   } else {
     n = id ? 16 : 8;
   }
   if (n < 0) { n = -n; sf |= STRFMT_F_UPPER; }
   sf |= ((SFormat)((n+1)&255) << STRFMT_SH_PREC);
   if (id) {
     tr = lj_crec_ct_tv(J, ctype_get(cts, id), 0, J->base[0], &rd->argv[0]);
     if (n < 16)
       tr = emitir(IRT(IR_BAND, IRT_U64), tr,
 		  lj_ir_kint64(J, ((uint64_t)1 << 4*n)-1));
   } else {
     tr = lj_opt_narrow_tobit(J, J->base[0]);
     if (n < 8)
       tr = emitir(IRTI(IR_BAND), tr, lj_ir_kint(J, (int32_t)((1u << 4*n)-1)));
     tr = emitconv(tr, IRT_U64, IRT_INT, 0);  /* No sign-extension. */
     lj_needsplit(J);
   }
   return lj_ir_call(J, IRCALL_lj_strfmt_putfxint, hdr, lj_ir_kint(J, sf), tr);
 }
 
 /* -- Miscellaneous library functions ------------------------------------- */
 
 void LJ_FASTCALL lj_crecord_tonumber(jit_State *J, RecordFFData *rd)
 {
   CTState *cts = ctype_ctsG(J2G(J));
   CType *d, *ct = lj_ctype_rawref(cts, cdataV(&rd->argv[0])->ctypeid);
   if (ctype_isenum(ct->info)) ct = ctype_child(cts, ct);
   if (ctype_isnum(ct->info) || ctype_iscomplex(ct->info)) {
     if (ctype_isinteger_or_bool(ct->info) && ct->size <= 4 &&
 	!(ct->size == 4 && (ct->info & CTF_UNSIGNED)))
       d = ctype_get(cts, CTID_INT32);
     else
       d = ctype_get(cts, CTID_DOUBLE);
     J->base[0] = lj_crec_ct_tv(J, d, 0, J->base[0], &rd->argv[0]);
   } else {
     J->base[0] = TREF_NIL;
   }
 }
 
 #undef IR
 #undef emitir
 #undef emitconv
 
 #endif