tcg-op.c - qemu - FORK: QEMU emulator

tcg-op.c (105789B)
      1 /*
      2  * Tiny Code Generator for QEMU
      3  *
      4  * Copyright (c) 2008 Fabrice Bellard
      5  *
      6  * Permission is hereby granted, free of charge, to any person obtaining a copy
      7  * of this software and associated documentation files (the "Software"), to deal
      8  * in the Software without restriction, including without limitation the rights
      9  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
     10  * copies of the Software, and to permit persons to whom the Software is
     11  * furnished to do so, subject to the following conditions:
     12  *
     13  * The above copyright notice and this permission notice shall be included in
     14  * all copies or substantial portions of the Software.
     15  *
     16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
     17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
     18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
     19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
     20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
     21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
     22  * THE SOFTWARE.
     23  */
     24 
     25 #include "qemu/osdep.h"
     26 #include "exec/exec-all.h"
     27 #include "tcg/tcg.h"
     28 #include "tcg/tcg-op.h"
     29 #include "tcg/tcg-mo.h"
     30 #include "exec/plugin-gen.h"
     31 
     32 /* Reduce the number of ifdefs below.  This assumes that all uses of
     33    TCGV_HIGH and TCGV_LOW are properly protected by a conditional that
     34    the compiler can eliminate.  */
     35 #if TCG_TARGET_REG_BITS == 64
     36 extern TCGv_i32 TCGV_LOW_link_error(TCGv_i64);
     37 extern TCGv_i32 TCGV_HIGH_link_error(TCGv_i64);
     38 #define TCGV_LOW  TCGV_LOW_link_error
     39 #define TCGV_HIGH TCGV_HIGH_link_error
     40 #endif
     41 
     42 void tcg_gen_op1(TCGOpcode opc, TCGArg a1)
     43 {
     44     TCGOp *op = tcg_emit_op(opc);
     45     op->args[0] = a1;
     46 }
     47 
     48 void tcg_gen_op2(TCGOpcode opc, TCGArg a1, TCGArg a2)
     49 {
     50     TCGOp *op = tcg_emit_op(opc);
     51     op->args[0] = a1;
     52     op->args[1] = a2;
     53 }
     54 
     55 void tcg_gen_op3(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3)
     56 {
     57     TCGOp *op = tcg_emit_op(opc);
     58     op->args[0] = a1;
     59     op->args[1] = a2;
     60     op->args[2] = a3;
     61 }
     62 
     63 void tcg_gen_op4(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, TCGArg a4)
     64 {
     65     TCGOp *op = tcg_emit_op(opc);
     66     op->args[0] = a1;
     67     op->args[1] = a2;
     68     op->args[2] = a3;
     69     op->args[3] = a4;
     70 }
     71 
     72 void tcg_gen_op5(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3,
     73                  TCGArg a4, TCGArg a5)
     74 {
     75     TCGOp *op = tcg_emit_op(opc);
     76     op->args[0] = a1;
     77     op->args[1] = a2;
     78     op->args[2] = a3;
     79     op->args[3] = a4;
     80     op->args[4] = a5;
     81 }
     82 
     83 void tcg_gen_op6(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3,
     84                  TCGArg a4, TCGArg a5, TCGArg a6)
     85 {
     86     TCGOp *op = tcg_emit_op(opc);
     87     op->args[0] = a1;
     88     op->args[1] = a2;
     89     op->args[2] = a3;
     90     op->args[3] = a4;
     91     op->args[4] = a5;
     92     op->args[5] = a6;
     93 }
     94 
     95 void tcg_gen_mb(TCGBar mb_type)
     96 {
     97     if (tcg_ctx->tb_cflags & CF_PARALLEL) {
     98         tcg_gen_op1(INDEX_op_mb, mb_type);
     99     }
    100 }
    101 
    102 /* 32 bit ops */
    103 
    104 void tcg_gen_movi_i32(TCGv_i32 ret, int32_t arg)
    105 {
    106     tcg_gen_mov_i32(ret, tcg_constant_i32(arg));
    107 }
    108 
    109 void tcg_gen_addi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
    110 {
    111     /* some cases can be optimized here */
    112     if (arg2 == 0) {
    113         tcg_gen_mov_i32(ret, arg1);
    114     } else {
    115         tcg_gen_add_i32(ret, arg1, tcg_constant_i32(arg2));
    116     }
    117 }
    118 
    119 void tcg_gen_subfi_i32(TCGv_i32 ret, int32_t arg1, TCGv_i32 arg2)
    120 {
    121     if (arg1 == 0 && TCG_TARGET_HAS_neg_i32) {
    122         /* Don't recurse with tcg_gen_neg_i32.  */
    123         tcg_gen_op2_i32(INDEX_op_neg_i32, ret, arg2);
    124     } else {
    125         tcg_gen_sub_i32(ret, tcg_constant_i32(arg1), arg2);
    126     }
    127 }
    128 
    129 void tcg_gen_subi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
    130 {
    131     /* some cases can be optimized here */
    132     if (arg2 == 0) {
    133         tcg_gen_mov_i32(ret, arg1);
    134     } else {
    135         tcg_gen_sub_i32(ret, arg1, tcg_constant_i32(arg2));
    136     }
    137 }
    138 
    139 void tcg_gen_andi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
    140 {
    141     /* Some cases can be optimized here.  */
    142     switch (arg2) {
    143     case 0:
    144         tcg_gen_movi_i32(ret, 0);
    145         return;
    146     case -1:
    147         tcg_gen_mov_i32(ret, arg1);
    148         return;
    149     case 0xff:
    150         /* Don't recurse with tcg_gen_ext8u_i32.  */
    151         if (TCG_TARGET_HAS_ext8u_i32) {
    152             tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg1);
    153             return;
    154         }
    155         break;
    156     case 0xffff:
    157         if (TCG_TARGET_HAS_ext16u_i32) {
    158             tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg1);
    159             return;
    160         }
    161         break;
    162     }
    163 
    164     tcg_gen_and_i32(ret, arg1, tcg_constant_i32(arg2));
    165 }
    166 
    167 void tcg_gen_ori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
    168 {
    169     /* Some cases can be optimized here.  */
    170     if (arg2 == -1) {
    171         tcg_gen_movi_i32(ret, -1);
    172     } else if (arg2 == 0) {
    173         tcg_gen_mov_i32(ret, arg1);
    174     } else {
    175         tcg_gen_or_i32(ret, arg1, tcg_constant_i32(arg2));
    176     }
    177 }
    178 
    179 void tcg_gen_xori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
    180 {
    181     /* Some cases can be optimized here.  */
    182     if (arg2 == 0) {
    183         tcg_gen_mov_i32(ret, arg1);
    184     } else if (arg2 == -1 && TCG_TARGET_HAS_not_i32) {
    185         /* Don't recurse with tcg_gen_not_i32.  */
    186         tcg_gen_op2_i32(INDEX_op_not_i32, ret, arg1);
    187     } else {
    188         tcg_gen_xor_i32(ret, arg1, tcg_constant_i32(arg2));
    189     }
    190 }
    191 
    192 void tcg_gen_shli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
    193 {
    194     tcg_debug_assert(arg2 >= 0 && arg2 < 32);
    195     if (arg2 == 0) {
    196         tcg_gen_mov_i32(ret, arg1);
    197     } else {
    198         tcg_gen_shl_i32(ret, arg1, tcg_constant_i32(arg2));
    199     }
    200 }
    201 
    202 void tcg_gen_shri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
    203 {
    204     tcg_debug_assert(arg2 >= 0 && arg2 < 32);
    205     if (arg2 == 0) {
    206         tcg_gen_mov_i32(ret, arg1);
    207     } else {
    208         tcg_gen_shr_i32(ret, arg1, tcg_constant_i32(arg2));
    209     }
    210 }
    211 
    212 void tcg_gen_sari_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
    213 {
    214     tcg_debug_assert(arg2 >= 0 && arg2 < 32);
    215     if (arg2 == 0) {
    216         tcg_gen_mov_i32(ret, arg1);
    217     } else {
    218         tcg_gen_sar_i32(ret, arg1, tcg_constant_i32(arg2));
    219     }
    220 }
    221 
    222 void tcg_gen_brcond_i32(TCGCond cond, TCGv_i32 arg1, TCGv_i32 arg2, TCGLabel *l)
    223 {
    224     if (cond == TCG_COND_ALWAYS) {
    225         tcg_gen_br(l);
    226     } else if (cond != TCG_COND_NEVER) {
    227         l->refs++;
    228         tcg_gen_op4ii_i32(INDEX_op_brcond_i32, arg1, arg2, cond, label_arg(l));
    229     }
    230 }
    231 
    232 void tcg_gen_brcondi_i32(TCGCond cond, TCGv_i32 arg1, int32_t arg2, TCGLabel *l)
    233 {
    234     if (cond == TCG_COND_ALWAYS) {
    235         tcg_gen_br(l);
    236     } else if (cond != TCG_COND_NEVER) {
    237         tcg_gen_brcond_i32(cond, arg1, tcg_constant_i32(arg2), l);
    238     }
    239 }
    240 
    241 void tcg_gen_setcond_i32(TCGCond cond, TCGv_i32 ret,
    242                          TCGv_i32 arg1, TCGv_i32 arg2)
    243 {
    244     if (cond == TCG_COND_ALWAYS) {
    245         tcg_gen_movi_i32(ret, 1);
    246     } else if (cond == TCG_COND_NEVER) {
    247         tcg_gen_movi_i32(ret, 0);
    248     } else {
    249         tcg_gen_op4i_i32(INDEX_op_setcond_i32, ret, arg1, arg2, cond);
    250     }
    251 }
    252 
    253 void tcg_gen_setcondi_i32(TCGCond cond, TCGv_i32 ret,
    254                           TCGv_i32 arg1, int32_t arg2)
    255 {
    256     tcg_gen_setcond_i32(cond, ret, arg1, tcg_constant_i32(arg2));
    257 }
    258 
    259 void tcg_gen_muli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
    260 {
    261     if (arg2 == 0) {
    262         tcg_gen_movi_i32(ret, 0);
    263     } else if (is_power_of_2(arg2)) {
    264         tcg_gen_shli_i32(ret, arg1, ctz32(arg2));
    265     } else {
    266         tcg_gen_mul_i32(ret, arg1, tcg_constant_i32(arg2));
    267     }
    268 }
    269 
    270 void tcg_gen_div_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
    271 {
    272     if (TCG_TARGET_HAS_div_i32) {
    273         tcg_gen_op3_i32(INDEX_op_div_i32, ret, arg1, arg2);
    274     } else if (TCG_TARGET_HAS_div2_i32) {
    275         TCGv_i32 t0 = tcg_temp_new_i32();
    276         tcg_gen_sari_i32(t0, arg1, 31);
    277         tcg_gen_op5_i32(INDEX_op_div2_i32, ret, t0, arg1, t0, arg2);
    278         tcg_temp_free_i32(t0);
    279     } else {
    280         gen_helper_div_i32(ret, arg1, arg2);
    281     }
    282 }
    283 
    284 void tcg_gen_rem_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
    285 {
    286     if (TCG_TARGET_HAS_rem_i32) {
    287         tcg_gen_op3_i32(INDEX_op_rem_i32, ret, arg1, arg2);
    288     } else if (TCG_TARGET_HAS_div_i32) {
    289         TCGv_i32 t0 = tcg_temp_new_i32();
    290         tcg_gen_op3_i32(INDEX_op_div_i32, t0, arg1, arg2);
    291         tcg_gen_mul_i32(t0, t0, arg2);
    292         tcg_gen_sub_i32(ret, arg1, t0);
    293         tcg_temp_free_i32(t0);
    294     } else if (TCG_TARGET_HAS_div2_i32) {
    295         TCGv_i32 t0 = tcg_temp_new_i32();
    296         tcg_gen_sari_i32(t0, arg1, 31);
    297         tcg_gen_op5_i32(INDEX_op_div2_i32, t0, ret, arg1, t0, arg2);
    298         tcg_temp_free_i32(t0);
    299     } else {
    300         gen_helper_rem_i32(ret, arg1, arg2);
    301     }
    302 }
    303 
    304 void tcg_gen_divu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
    305 {
    306     if (TCG_TARGET_HAS_div_i32) {
    307         tcg_gen_op3_i32(INDEX_op_divu_i32, ret, arg1, arg2);
    308     } else if (TCG_TARGET_HAS_div2_i32) {
    309         TCGv_i32 t0 = tcg_temp_new_i32();
    310         tcg_gen_movi_i32(t0, 0);
    311         tcg_gen_op5_i32(INDEX_op_divu2_i32, ret, t0, arg1, t0, arg2);
    312         tcg_temp_free_i32(t0);
    313     } else {
    314         gen_helper_divu_i32(ret, arg1, arg2);
    315     }
    316 }
    317 
    318 void tcg_gen_remu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
    319 {
    320     if (TCG_TARGET_HAS_rem_i32) {
    321         tcg_gen_op3_i32(INDEX_op_remu_i32, ret, arg1, arg2);
    322     } else if (TCG_TARGET_HAS_div_i32) {
    323         TCGv_i32 t0 = tcg_temp_new_i32();
    324         tcg_gen_op3_i32(INDEX_op_divu_i32, t0, arg1, arg2);
    325         tcg_gen_mul_i32(t0, t0, arg2);
    326         tcg_gen_sub_i32(ret, arg1, t0);
    327         tcg_temp_free_i32(t0);
    328     } else if (TCG_TARGET_HAS_div2_i32) {
    329         TCGv_i32 t0 = tcg_temp_new_i32();
    330         tcg_gen_movi_i32(t0, 0);
    331         tcg_gen_op5_i32(INDEX_op_divu2_i32, t0, ret, arg1, t0, arg2);
    332         tcg_temp_free_i32(t0);
    333     } else {
    334         gen_helper_remu_i32(ret, arg1, arg2);
    335     }
    336 }
    337 
    338 void tcg_gen_andc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
    339 {
    340     if (TCG_TARGET_HAS_andc_i32) {
    341         tcg_gen_op3_i32(INDEX_op_andc_i32, ret, arg1, arg2);
    342     } else {
    343         TCGv_i32 t0 = tcg_temp_new_i32();
    344         tcg_gen_not_i32(t0, arg2);
    345         tcg_gen_and_i32(ret, arg1, t0);
    346         tcg_temp_free_i32(t0);
    347     }
    348 }
    349 
    350 void tcg_gen_eqv_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
    351 {
    352     if (TCG_TARGET_HAS_eqv_i32) {
    353         tcg_gen_op3_i32(INDEX_op_eqv_i32, ret, arg1, arg2);
    354     } else {
    355         tcg_gen_xor_i32(ret, arg1, arg2);
    356         tcg_gen_not_i32(ret, ret);
    357     }
    358 }
    359 
    360 void tcg_gen_nand_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
    361 {
    362     if (TCG_TARGET_HAS_nand_i32) {
    363         tcg_gen_op3_i32(INDEX_op_nand_i32, ret, arg1, arg2);
    364     } else {
    365         tcg_gen_and_i32(ret, arg1, arg2);
    366         tcg_gen_not_i32(ret, ret);
    367     }
    368 }
    369 
    370 void tcg_gen_nor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
    371 {
    372     if (TCG_TARGET_HAS_nor_i32) {
    373         tcg_gen_op3_i32(INDEX_op_nor_i32, ret, arg1, arg2);
    374     } else {
    375         tcg_gen_or_i32(ret, arg1, arg2);
    376         tcg_gen_not_i32(ret, ret);
    377     }
    378 }
    379 
    380 void tcg_gen_orc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
    381 {
    382     if (TCG_TARGET_HAS_orc_i32) {
    383         tcg_gen_op3_i32(INDEX_op_orc_i32, ret, arg1, arg2);
    384     } else {
    385         TCGv_i32 t0 = tcg_temp_new_i32();
    386         tcg_gen_not_i32(t0, arg2);
    387         tcg_gen_or_i32(ret, arg1, t0);
    388         tcg_temp_free_i32(t0);
    389     }
    390 }
    391 
    392 void tcg_gen_clz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
    393 {
    394     if (TCG_TARGET_HAS_clz_i32) {
    395         tcg_gen_op3_i32(INDEX_op_clz_i32, ret, arg1, arg2);
    396     } else if (TCG_TARGET_HAS_clz_i64) {
    397         TCGv_i64 t1 = tcg_temp_new_i64();
    398         TCGv_i64 t2 = tcg_temp_new_i64();
    399         tcg_gen_extu_i32_i64(t1, arg1);
    400         tcg_gen_extu_i32_i64(t2, arg2);
    401         tcg_gen_addi_i64(t2, t2, 32);
    402         tcg_gen_clz_i64(t1, t1, t2);
    403         tcg_gen_extrl_i64_i32(ret, t1);
    404         tcg_temp_free_i64(t1);
    405         tcg_temp_free_i64(t2);
    406         tcg_gen_subi_i32(ret, ret, 32);
    407     } else {
    408         gen_helper_clz_i32(ret, arg1, arg2);
    409     }
    410 }
    411 
    412 void tcg_gen_clzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2)
    413 {
    414     tcg_gen_clz_i32(ret, arg1, tcg_constant_i32(arg2));
    415 }
    416 
    417 void tcg_gen_ctz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
    418 {
    419     if (TCG_TARGET_HAS_ctz_i32) {
    420         tcg_gen_op3_i32(INDEX_op_ctz_i32, ret, arg1, arg2);
    421     } else if (TCG_TARGET_HAS_ctz_i64) {
    422         TCGv_i64 t1 = tcg_temp_new_i64();
    423         TCGv_i64 t2 = tcg_temp_new_i64();
    424         tcg_gen_extu_i32_i64(t1, arg1);
    425         tcg_gen_extu_i32_i64(t2, arg2);
    426         tcg_gen_ctz_i64(t1, t1, t2);
    427         tcg_gen_extrl_i64_i32(ret, t1);
    428         tcg_temp_free_i64(t1);
    429         tcg_temp_free_i64(t2);
    430     } else if (TCG_TARGET_HAS_ctpop_i32
    431                || TCG_TARGET_HAS_ctpop_i64
    432                || TCG_TARGET_HAS_clz_i32
    433                || TCG_TARGET_HAS_clz_i64) {
    434         TCGv_i32 z, t = tcg_temp_new_i32();
    435 
    436         if (TCG_TARGET_HAS_ctpop_i32 || TCG_TARGET_HAS_ctpop_i64) {
    437             tcg_gen_subi_i32(t, arg1, 1);
    438             tcg_gen_andc_i32(t, t, arg1);
    439             tcg_gen_ctpop_i32(t, t);
    440         } else {
    441             /* Since all non-x86 hosts have clz(0) == 32, don't fight it.  */
    442             tcg_gen_neg_i32(t, arg1);
    443             tcg_gen_and_i32(t, t, arg1);
    444             tcg_gen_clzi_i32(t, t, 32);
    445             tcg_gen_xori_i32(t, t, 31);
    446         }
    447         z = tcg_constant_i32(0);
    448         tcg_gen_movcond_i32(TCG_COND_EQ, ret, arg1, z, arg2, t);
    449         tcg_temp_free_i32(t);
    450     } else {
    451         gen_helper_ctz_i32(ret, arg1, arg2);
    452     }
    453 }
    454 
    455 void tcg_gen_ctzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2)
    456 {
    457     if (!TCG_TARGET_HAS_ctz_i32 && TCG_TARGET_HAS_ctpop_i32 && arg2 == 32) {
    458         /* This equivalence has the advantage of not requiring a fixup.  */
    459         TCGv_i32 t = tcg_temp_new_i32();
    460         tcg_gen_subi_i32(t, arg1, 1);
    461         tcg_gen_andc_i32(t, t, arg1);
    462         tcg_gen_ctpop_i32(ret, t);
    463         tcg_temp_free_i32(t);
    464     } else {
    465         tcg_gen_ctz_i32(ret, arg1, tcg_constant_i32(arg2));
    466     }
    467 }
    468 
    469 void tcg_gen_clrsb_i32(TCGv_i32 ret, TCGv_i32 arg)
    470 {
    471     if (TCG_TARGET_HAS_clz_i32) {
    472         TCGv_i32 t = tcg_temp_new_i32();
    473         tcg_gen_sari_i32(t, arg, 31);
    474         tcg_gen_xor_i32(t, t, arg);
    475         tcg_gen_clzi_i32(t, t, 32);
    476         tcg_gen_subi_i32(ret, t, 1);
    477         tcg_temp_free_i32(t);
    478     } else {
    479         gen_helper_clrsb_i32(ret, arg);
    480     }
    481 }
    482 
    483 void tcg_gen_ctpop_i32(TCGv_i32 ret, TCGv_i32 arg1)
    484 {
    485     if (TCG_TARGET_HAS_ctpop_i32) {
    486         tcg_gen_op2_i32(INDEX_op_ctpop_i32, ret, arg1);
    487     } else if (TCG_TARGET_HAS_ctpop_i64) {
    488         TCGv_i64 t = tcg_temp_new_i64();
    489         tcg_gen_extu_i32_i64(t, arg1);
    490         tcg_gen_ctpop_i64(t, t);
    491         tcg_gen_extrl_i64_i32(ret, t);
    492         tcg_temp_free_i64(t);
    493     } else {
    494         gen_helper_ctpop_i32(ret, arg1);
    495     }
    496 }
    497 
    498 void tcg_gen_rotl_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
    499 {
    500     if (TCG_TARGET_HAS_rot_i32) {
    501         tcg_gen_op3_i32(INDEX_op_rotl_i32, ret, arg1, arg2);
    502     } else {
    503         TCGv_i32 t0, t1;
    504 
    505         t0 = tcg_temp_new_i32();
    506         t1 = tcg_temp_new_i32();
    507         tcg_gen_shl_i32(t0, arg1, arg2);
    508         tcg_gen_subfi_i32(t1, 32, arg2);
    509         tcg_gen_shr_i32(t1, arg1, t1);
    510         tcg_gen_or_i32(ret, t0, t1);
    511         tcg_temp_free_i32(t0);
    512         tcg_temp_free_i32(t1);
    513     }
    514 }
    515 
    516 void tcg_gen_rotli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
    517 {
    518     tcg_debug_assert(arg2 >= 0 && arg2 < 32);
    519     /* some cases can be optimized here */
    520     if (arg2 == 0) {
    521         tcg_gen_mov_i32(ret, arg1);
    522     } else if (TCG_TARGET_HAS_rot_i32) {
    523         tcg_gen_rotl_i32(ret, arg1, tcg_constant_i32(arg2));
    524     } else {
    525         TCGv_i32 t0, t1;
    526         t0 = tcg_temp_new_i32();
    527         t1 = tcg_temp_new_i32();
    528         tcg_gen_shli_i32(t0, arg1, arg2);
    529         tcg_gen_shri_i32(t1, arg1, 32 - arg2);
    530         tcg_gen_or_i32(ret, t0, t1);
    531         tcg_temp_free_i32(t0);
    532         tcg_temp_free_i32(t1);
    533     }
    534 }
    535 
    536 void tcg_gen_rotr_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
    537 {
    538     if (TCG_TARGET_HAS_rot_i32) {
    539         tcg_gen_op3_i32(INDEX_op_rotr_i32, ret, arg1, arg2);
    540     } else {
    541         TCGv_i32 t0, t1;
    542 
    543         t0 = tcg_temp_new_i32();
    544         t1 = tcg_temp_new_i32();
    545         tcg_gen_shr_i32(t0, arg1, arg2);
    546         tcg_gen_subfi_i32(t1, 32, arg2);
    547         tcg_gen_shl_i32(t1, arg1, t1);
    548         tcg_gen_or_i32(ret, t0, t1);
    549         tcg_temp_free_i32(t0);
    550         tcg_temp_free_i32(t1);
    551     }
    552 }
    553 
    554 void tcg_gen_rotri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
    555 {
    556     tcg_debug_assert(arg2 >= 0 && arg2 < 32);
    557     /* some cases can be optimized here */
    558     if (arg2 == 0) {
    559         tcg_gen_mov_i32(ret, arg1);
    560     } else {
    561         tcg_gen_rotli_i32(ret, arg1, 32 - arg2);
    562     }
    563 }
    564 
    565 void tcg_gen_deposit_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2,
    566                          unsigned int ofs, unsigned int len)
    567 {
    568     uint32_t mask;
    569     TCGv_i32 t1;
    570 
    571     tcg_debug_assert(ofs < 32);
    572     tcg_debug_assert(len > 0);
    573     tcg_debug_assert(len <= 32);
    574     tcg_debug_assert(ofs + len <= 32);
    575 
    576     if (len == 32) {
    577         tcg_gen_mov_i32(ret, arg2);
    578         return;
    579     }
    580     if (TCG_TARGET_HAS_deposit_i32 && TCG_TARGET_deposit_i32_valid(ofs, len)) {
    581         tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, arg1, arg2, ofs, len);
    582         return;
    583     }
    584 
    585     t1 = tcg_temp_new_i32();
    586 
    587     if (TCG_TARGET_HAS_extract2_i32) {
    588         if (ofs + len == 32) {
    589             tcg_gen_shli_i32(t1, arg1, len);
    590             tcg_gen_extract2_i32(ret, t1, arg2, len);
    591             goto done;
    592         }
    593         if (ofs == 0) {
    594             tcg_gen_extract2_i32(ret, arg1, arg2, len);
    595             tcg_gen_rotli_i32(ret, ret, len);
    596             goto done;
    597         }
    598     }
    599 
    600     mask = (1u << len) - 1;
    601     if (ofs + len < 32) {
    602         tcg_gen_andi_i32(t1, arg2, mask);
    603         tcg_gen_shli_i32(t1, t1, ofs);
    604     } else {
    605         tcg_gen_shli_i32(t1, arg2, ofs);
    606     }
    607     tcg_gen_andi_i32(ret, arg1, ~(mask << ofs));
    608     tcg_gen_or_i32(ret, ret, t1);
    609  done:
    610     tcg_temp_free_i32(t1);
    611 }
    612 
    613 void tcg_gen_deposit_z_i32(TCGv_i32 ret, TCGv_i32 arg,
    614                            unsigned int ofs, unsigned int len)
    615 {
    616     tcg_debug_assert(ofs < 32);
    617     tcg_debug_assert(len > 0);
    618     tcg_debug_assert(len <= 32);
    619     tcg_debug_assert(ofs + len <= 32);
    620 
    621     if (ofs + len == 32) {
    622         tcg_gen_shli_i32(ret, arg, ofs);
    623     } else if (ofs == 0) {
    624         tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
    625     } else if (TCG_TARGET_HAS_deposit_i32
    626                && TCG_TARGET_deposit_i32_valid(ofs, len)) {
    627         TCGv_i32 zero = tcg_constant_i32(0);
    628         tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, zero, arg, ofs, len);
    629     } else {
    630         /* To help two-operand hosts we prefer to zero-extend first,
    631            which allows ARG to stay live.  */
    632         switch (len) {
    633         case 16:
    634             if (TCG_TARGET_HAS_ext16u_i32) {
    635                 tcg_gen_ext16u_i32(ret, arg);
    636                 tcg_gen_shli_i32(ret, ret, ofs);
    637                 return;
    638             }
    639             break;
    640         case 8:
    641             if (TCG_TARGET_HAS_ext8u_i32) {
    642                 tcg_gen_ext8u_i32(ret, arg);
    643                 tcg_gen_shli_i32(ret, ret, ofs);
    644                 return;
    645             }
    646             break;
    647         }
    648         /* Otherwise prefer zero-extension over AND for code size.  */
    649         switch (ofs + len) {
    650         case 16:
    651             if (TCG_TARGET_HAS_ext16u_i32) {
    652                 tcg_gen_shli_i32(ret, arg, ofs);
    653                 tcg_gen_ext16u_i32(ret, ret);
    654                 return;
    655             }
    656             break;
    657         case 8:
    658             if (TCG_TARGET_HAS_ext8u_i32) {
    659                 tcg_gen_shli_i32(ret, arg, ofs);
    660                 tcg_gen_ext8u_i32(ret, ret);
    661                 return;
    662             }
    663             break;
    664         }
    665         tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
    666         tcg_gen_shli_i32(ret, ret, ofs);
    667     }
    668 }
    669 
    670 void tcg_gen_extract_i32(TCGv_i32 ret, TCGv_i32 arg,
    671                          unsigned int ofs, unsigned int len)
    672 {
    673     tcg_debug_assert(ofs < 32);
    674     tcg_debug_assert(len > 0);
    675     tcg_debug_assert(len <= 32);
    676     tcg_debug_assert(ofs + len <= 32);
    677 
    678     /* Canonicalize certain special cases, even if extract is supported.  */
    679     if (ofs + len == 32) {
    680         tcg_gen_shri_i32(ret, arg, 32 - len);
    681         return;
    682     }
    683     if (ofs == 0) {
    684         tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
    685         return;
    686     }
    687 
    688     if (TCG_TARGET_HAS_extract_i32
    689         && TCG_TARGET_extract_i32_valid(ofs, len)) {
    690         tcg_gen_op4ii_i32(INDEX_op_extract_i32, ret, arg, ofs, len);
    691         return;
    692     }
    693 
    694     /* Assume that zero-extension, if available, is cheaper than a shift.  */
    695     switch (ofs + len) {
    696     case 16:
    697         if (TCG_TARGET_HAS_ext16u_i32) {
    698             tcg_gen_ext16u_i32(ret, arg);
    699             tcg_gen_shri_i32(ret, ret, ofs);
    700             return;
    701         }
    702         break;
    703     case 8:
    704         if (TCG_TARGET_HAS_ext8u_i32) {
    705             tcg_gen_ext8u_i32(ret, arg);
    706             tcg_gen_shri_i32(ret, ret, ofs);
    707             return;
    708         }
    709         break;
    710     }
    711 
    712     /* ??? Ideally we'd know what values are available for immediate AND.
    713        Assume that 8 bits are available, plus the special case of 16,
    714        so that we get ext8u, ext16u.  */
    715     switch (len) {
    716     case 1 ... 8: case 16:
    717         tcg_gen_shri_i32(ret, arg, ofs);
    718         tcg_gen_andi_i32(ret, ret, (1u << len) - 1);
    719         break;
    720     default:
    721         tcg_gen_shli_i32(ret, arg, 32 - len - ofs);
    722         tcg_gen_shri_i32(ret, ret, 32 - len);
    723         break;
    724     }
    725 }
    726 
    727 void tcg_gen_sextract_i32(TCGv_i32 ret, TCGv_i32 arg,
    728                           unsigned int ofs, unsigned int len)
    729 {
    730     tcg_debug_assert(ofs < 32);
    731     tcg_debug_assert(len > 0);
    732     tcg_debug_assert(len <= 32);
    733     tcg_debug_assert(ofs + len <= 32);
    734 
    735     /* Canonicalize certain special cases, even if extract is supported.  */
    736     if (ofs + len == 32) {
    737         tcg_gen_sari_i32(ret, arg, 32 - len);
    738         return;
    739     }
    740     if (ofs == 0) {
    741         switch (len) {
    742         case 16:
    743             tcg_gen_ext16s_i32(ret, arg);
    744             return;
    745         case 8:
    746             tcg_gen_ext8s_i32(ret, arg);
    747             return;
    748         }
    749     }
    750 
    751     if (TCG_TARGET_HAS_sextract_i32
    752         && TCG_TARGET_extract_i32_valid(ofs, len)) {
    753         tcg_gen_op4ii_i32(INDEX_op_sextract_i32, ret, arg, ofs, len);
    754         return;
    755     }
    756 
    757     /* Assume that sign-extension, if available, is cheaper than a shift.  */
    758     switch (ofs + len) {
    759     case 16:
    760         if (TCG_TARGET_HAS_ext16s_i32) {
    761             tcg_gen_ext16s_i32(ret, arg);
    762             tcg_gen_sari_i32(ret, ret, ofs);
    763             return;
    764         }
    765         break;
    766     case 8:
    767         if (TCG_TARGET_HAS_ext8s_i32) {
    768             tcg_gen_ext8s_i32(ret, arg);
    769             tcg_gen_sari_i32(ret, ret, ofs);
    770             return;
    771         }
    772         break;
    773     }
    774     switch (len) {
    775     case 16:
    776         if (TCG_TARGET_HAS_ext16s_i32) {
    777             tcg_gen_shri_i32(ret, arg, ofs);
    778             tcg_gen_ext16s_i32(ret, ret);
    779             return;
    780         }
    781         break;
    782     case 8:
    783         if (TCG_TARGET_HAS_ext8s_i32) {
    784             tcg_gen_shri_i32(ret, arg, ofs);
    785             tcg_gen_ext8s_i32(ret, ret);
    786             return;
    787         }
    788         break;
    789     }
    790 
    791     tcg_gen_shli_i32(ret, arg, 32 - len - ofs);
    792     tcg_gen_sari_i32(ret, ret, 32 - len);
    793 }
    794 
    795 /*
    796  * Extract 32-bits from a 64-bit input, ah:al, starting from ofs.
    797  * Unlike tcg_gen_extract_i32 above, len is fixed at 32.
    798  */
    799 void tcg_gen_extract2_i32(TCGv_i32 ret, TCGv_i32 al, TCGv_i32 ah,
    800                           unsigned int ofs)
    801 {
    802     tcg_debug_assert(ofs <= 32);
    803     if (ofs == 0) {
    804         tcg_gen_mov_i32(ret, al);
    805     } else if (ofs == 32) {
    806         tcg_gen_mov_i32(ret, ah);
    807     } else if (al == ah) {
    808         tcg_gen_rotri_i32(ret, al, ofs);
    809     } else if (TCG_TARGET_HAS_extract2_i32) {
    810         tcg_gen_op4i_i32(INDEX_op_extract2_i32, ret, al, ah, ofs);
    811     } else {
    812         TCGv_i32 t0 = tcg_temp_new_i32();
    813         tcg_gen_shri_i32(t0, al, ofs);
    814         tcg_gen_deposit_i32(ret, t0, ah, 32 - ofs, ofs);
    815         tcg_temp_free_i32(t0);
    816     }
    817 }
    818 
    819 void tcg_gen_movcond_i32(TCGCond cond, TCGv_i32 ret, TCGv_i32 c1,
    820                          TCGv_i32 c2, TCGv_i32 v1, TCGv_i32 v2)
    821 {
    822     if (cond == TCG_COND_ALWAYS) {
    823         tcg_gen_mov_i32(ret, v1);
    824     } else if (cond == TCG_COND_NEVER) {
    825         tcg_gen_mov_i32(ret, v2);
    826     } else if (TCG_TARGET_HAS_movcond_i32) {
    827         tcg_gen_op6i_i32(INDEX_op_movcond_i32, ret, c1, c2, v1, v2, cond);
    828     } else {
    829         TCGv_i32 t0 = tcg_temp_new_i32();
    830         TCGv_i32 t1 = tcg_temp_new_i32();
    831         tcg_gen_setcond_i32(cond, t0, c1, c2);
    832         tcg_gen_neg_i32(t0, t0);
    833         tcg_gen_and_i32(t1, v1, t0);
    834         tcg_gen_andc_i32(ret, v2, t0);
    835         tcg_gen_or_i32(ret, ret, t1);
    836         tcg_temp_free_i32(t0);
    837         tcg_temp_free_i32(t1);
    838     }
    839 }
    840 
    841 void tcg_gen_add2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al,
    842                       TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh)
    843 {
    844     if (TCG_TARGET_HAS_add2_i32) {
    845         tcg_gen_op6_i32(INDEX_op_add2_i32, rl, rh, al, ah, bl, bh);
    846     } else {
    847         TCGv_i64 t0 = tcg_temp_new_i64();
    848         TCGv_i64 t1 = tcg_temp_new_i64();
    849         tcg_gen_concat_i32_i64(t0, al, ah);
    850         tcg_gen_concat_i32_i64(t1, bl, bh);
    851         tcg_gen_add_i64(t0, t0, t1);
    852         tcg_gen_extr_i64_i32(rl, rh, t0);
    853         tcg_temp_free_i64(t0);
    854         tcg_temp_free_i64(t1);
    855     }
    856 }
    857 
    858 void tcg_gen_sub2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al,
    859                       TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh)
    860 {
    861     if (TCG_TARGET_HAS_sub2_i32) {
    862         tcg_gen_op6_i32(INDEX_op_sub2_i32, rl, rh, al, ah, bl, bh);
    863     } else {
    864         TCGv_i64 t0 = tcg_temp_new_i64();
    865         TCGv_i64 t1 = tcg_temp_new_i64();
    866         tcg_gen_concat_i32_i64(t0, al, ah);
    867         tcg_gen_concat_i32_i64(t1, bl, bh);
    868         tcg_gen_sub_i64(t0, t0, t1);
    869         tcg_gen_extr_i64_i32(rl, rh, t0);
    870         tcg_temp_free_i64(t0);
    871         tcg_temp_free_i64(t1);
    872     }
    873 }
    874 
    875 void tcg_gen_mulu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
    876 {
    877     if (TCG_TARGET_HAS_mulu2_i32) {
    878         tcg_gen_op4_i32(INDEX_op_mulu2_i32, rl, rh, arg1, arg2);
    879     } else if (TCG_TARGET_HAS_muluh_i32) {
    880         TCGv_i32 t = tcg_temp_new_i32();
    881         tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2);
    882         tcg_gen_op3_i32(INDEX_op_muluh_i32, rh, arg1, arg2);
    883         tcg_gen_mov_i32(rl, t);
    884         tcg_temp_free_i32(t);
    885     } else {
    886         TCGv_i64 t0 = tcg_temp_new_i64();
    887         TCGv_i64 t1 = tcg_temp_new_i64();
    888         tcg_gen_extu_i32_i64(t0, arg1);
    889         tcg_gen_extu_i32_i64(t1, arg2);
    890         tcg_gen_mul_i64(t0, t0, t1);
    891         tcg_gen_extr_i64_i32(rl, rh, t0);
    892         tcg_temp_free_i64(t0);
    893         tcg_temp_free_i64(t1);
    894     }
    895 }
    896 
    897 void tcg_gen_muls2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
    898 {
    899     if (TCG_TARGET_HAS_muls2_i32) {
    900         tcg_gen_op4_i32(INDEX_op_muls2_i32, rl, rh, arg1, arg2);
    901     } else if (TCG_TARGET_HAS_mulsh_i32) {
    902         TCGv_i32 t = tcg_temp_new_i32();
    903         tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2);
    904         tcg_gen_op3_i32(INDEX_op_mulsh_i32, rh, arg1, arg2);
    905         tcg_gen_mov_i32(rl, t);
    906         tcg_temp_free_i32(t);
    907     } else if (TCG_TARGET_REG_BITS == 32) {
    908         TCGv_i32 t0 = tcg_temp_new_i32();
    909         TCGv_i32 t1 = tcg_temp_new_i32();
    910         TCGv_i32 t2 = tcg_temp_new_i32();
    911         TCGv_i32 t3 = tcg_temp_new_i32();
    912         tcg_gen_mulu2_i32(t0, t1, arg1, arg2);
    913         /* Adjust for negative inputs.  */
    914         tcg_gen_sari_i32(t2, arg1, 31);
    915         tcg_gen_sari_i32(t3, arg2, 31);
    916         tcg_gen_and_i32(t2, t2, arg2);
    917         tcg_gen_and_i32(t3, t3, arg1);
    918         tcg_gen_sub_i32(rh, t1, t2);
    919         tcg_gen_sub_i32(rh, rh, t3);
    920         tcg_gen_mov_i32(rl, t0);
    921         tcg_temp_free_i32(t0);
    922         tcg_temp_free_i32(t1);
    923         tcg_temp_free_i32(t2);
    924         tcg_temp_free_i32(t3);
    925     } else {
    926         TCGv_i64 t0 = tcg_temp_new_i64();
    927         TCGv_i64 t1 = tcg_temp_new_i64();
    928         tcg_gen_ext_i32_i64(t0, arg1);
    929         tcg_gen_ext_i32_i64(t1, arg2);
    930         tcg_gen_mul_i64(t0, t0, t1);
    931         tcg_gen_extr_i64_i32(rl, rh, t0);
    932         tcg_temp_free_i64(t0);
    933         tcg_temp_free_i64(t1);
    934     }
    935 }
    936 
    937 void tcg_gen_mulsu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
    938 {
    939     if (TCG_TARGET_REG_BITS == 32) {
    940         TCGv_i32 t0 = tcg_temp_new_i32();
    941         TCGv_i32 t1 = tcg_temp_new_i32();
    942         TCGv_i32 t2 = tcg_temp_new_i32();
    943         tcg_gen_mulu2_i32(t0, t1, arg1, arg2);
    944         /* Adjust for negative input for the signed arg1.  */
    945         tcg_gen_sari_i32(t2, arg1, 31);
    946         tcg_gen_and_i32(t2, t2, arg2);
    947         tcg_gen_sub_i32(rh, t1, t2);
    948         tcg_gen_mov_i32(rl, t0);
    949         tcg_temp_free_i32(t0);
    950         tcg_temp_free_i32(t1);
    951         tcg_temp_free_i32(t2);
    952     } else {
    953         TCGv_i64 t0 = tcg_temp_new_i64();
    954         TCGv_i64 t1 = tcg_temp_new_i64();
    955         tcg_gen_ext_i32_i64(t0, arg1);
    956         tcg_gen_extu_i32_i64(t1, arg2);
    957         tcg_gen_mul_i64(t0, t0, t1);
    958         tcg_gen_extr_i64_i32(rl, rh, t0);
    959         tcg_temp_free_i64(t0);
    960         tcg_temp_free_i64(t1);
    961     }
    962 }
    963 
    964 void tcg_gen_ext8s_i32(TCGv_i32 ret, TCGv_i32 arg)
    965 {
    966     if (TCG_TARGET_HAS_ext8s_i32) {
    967         tcg_gen_op2_i32(INDEX_op_ext8s_i32, ret, arg);
    968     } else {
    969         tcg_gen_shli_i32(ret, arg, 24);
    970         tcg_gen_sari_i32(ret, ret, 24);
    971     }
    972 }
    973 
    974 void tcg_gen_ext16s_i32(TCGv_i32 ret, TCGv_i32 arg)
    975 {
    976     if (TCG_TARGET_HAS_ext16s_i32) {
    977         tcg_gen_op2_i32(INDEX_op_ext16s_i32, ret, arg);
    978     } else {
    979         tcg_gen_shli_i32(ret, arg, 16);
    980         tcg_gen_sari_i32(ret, ret, 16);
    981     }
    982 }
    983 
    984 void tcg_gen_ext8u_i32(TCGv_i32 ret, TCGv_i32 arg)
    985 {
    986     if (TCG_TARGET_HAS_ext8u_i32) {
    987         tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg);
    988     } else {
    989         tcg_gen_andi_i32(ret, arg, 0xffu);
    990     }
    991 }
    992 
    993 void tcg_gen_ext16u_i32(TCGv_i32 ret, TCGv_i32 arg)
    994 {
    995     if (TCG_TARGET_HAS_ext16u_i32) {
    996         tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg);
    997     } else {
    998         tcg_gen_andi_i32(ret, arg, 0xffffu);
    999     }
   1000 }
   1001 
   1002 void tcg_gen_bswap16_i32(TCGv_i32 ret, TCGv_i32 arg, int flags)
   1003 {
   1004     /* Only one extension flag may be present. */
   1005     tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ));
   1006 
   1007     if (TCG_TARGET_HAS_bswap16_i32) {
   1008         tcg_gen_op3i_i32(INDEX_op_bswap16_i32, ret, arg, flags);
   1009     } else {
   1010         TCGv_i32 t0 = tcg_temp_new_i32();
   1011         TCGv_i32 t1 = tcg_temp_new_i32();
   1012 
   1013         tcg_gen_shri_i32(t0, arg, 8);
   1014         if (!(flags & TCG_BSWAP_IZ)) {
   1015             tcg_gen_ext8u_i32(t0, t0);
   1016         }
   1017 
   1018         if (flags & TCG_BSWAP_OS) {
   1019             tcg_gen_shli_i32(t1, arg, 24);
   1020             tcg_gen_sari_i32(t1, t1, 16);
   1021         } else if (flags & TCG_BSWAP_OZ) {
   1022             tcg_gen_ext8u_i32(t1, arg);
   1023             tcg_gen_shli_i32(t1, t1, 8);
   1024         } else {
   1025             tcg_gen_shli_i32(t1, arg, 8);
   1026         }
   1027 
   1028         tcg_gen_or_i32(ret, t0, t1);
   1029         tcg_temp_free_i32(t0);
   1030         tcg_temp_free_i32(t1);
   1031     }
   1032 }
   1033 
   1034 void tcg_gen_bswap32_i32(TCGv_i32 ret, TCGv_i32 arg)
   1035 {
   1036     if (TCG_TARGET_HAS_bswap32_i32) {
   1037         tcg_gen_op3i_i32(INDEX_op_bswap32_i32, ret, arg, 0);
   1038     } else {
   1039         TCGv_i32 t0 = tcg_temp_new_i32();
   1040         TCGv_i32 t1 = tcg_temp_new_i32();
   1041         TCGv_i32 t2 = tcg_constant_i32(0x00ff00ff);
   1042 
   1043                                         /* arg = abcd */
   1044         tcg_gen_shri_i32(t0, arg, 8);   /*  t0 = .abc */
   1045         tcg_gen_and_i32(t1, arg, t2);   /*  t1 = .b.d */
   1046         tcg_gen_and_i32(t0, t0, t2);    /*  t0 = .a.c */
   1047         tcg_gen_shli_i32(t1, t1, 8);    /*  t1 = b.d. */
   1048         tcg_gen_or_i32(ret, t0, t1);    /* ret = badc */
   1049 
   1050         tcg_gen_shri_i32(t0, ret, 16);  /*  t0 = ..ba */
   1051         tcg_gen_shli_i32(t1, ret, 16);  /*  t1 = dc.. */
   1052         tcg_gen_or_i32(ret, t0, t1);    /* ret = dcba */
   1053 
   1054         tcg_temp_free_i32(t0);
   1055         tcg_temp_free_i32(t1);
   1056     }
   1057 }
   1058 
   1059 void tcg_gen_hswap_i32(TCGv_i32 ret, TCGv_i32 arg)
   1060 {
   1061     /* Swapping 2 16-bit elements is a rotate. */
   1062     tcg_gen_rotli_i32(ret, arg, 16);
   1063 }
   1064 
   1065 void tcg_gen_smin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
   1066 {
   1067     tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, a, b);
   1068 }
   1069 
   1070 void tcg_gen_umin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
   1071 {
   1072     tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, a, b);
   1073 }
   1074 
   1075 void tcg_gen_smax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
   1076 {
   1077     tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, b, a);
   1078 }
   1079 
   1080 void tcg_gen_umax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
   1081 {
   1082     tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, b, a);
   1083 }
   1084 
   1085 void tcg_gen_abs_i32(TCGv_i32 ret, TCGv_i32 a)
   1086 {
   1087     TCGv_i32 t = tcg_temp_new_i32();
   1088 
   1089     tcg_gen_sari_i32(t, a, 31);
   1090     tcg_gen_xor_i32(ret, a, t);
   1091     tcg_gen_sub_i32(ret, ret, t);
   1092     tcg_temp_free_i32(t);
   1093 }
   1094 
   1095 /* 64-bit ops */
   1096 
   1097 #if TCG_TARGET_REG_BITS == 32
   1098 /* These are all inline for TCG_TARGET_REG_BITS == 64.  */
   1099 
   1100 void tcg_gen_discard_i64(TCGv_i64 arg)
   1101 {
   1102     tcg_gen_discard_i32(TCGV_LOW(arg));
   1103     tcg_gen_discard_i32(TCGV_HIGH(arg));
   1104 }
   1105 
   1106 void tcg_gen_mov_i64(TCGv_i64 ret, TCGv_i64 arg)
   1107 {
   1108     TCGTemp *ts = tcgv_i64_temp(arg);
   1109 
   1110     /* Canonicalize TCGv_i64 TEMP_CONST into TCGv_i32 TEMP_CONST. */
   1111     if (ts->kind == TEMP_CONST) {
   1112         tcg_gen_movi_i64(ret, ts->val);
   1113     } else {
   1114         tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
   1115         tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg));
   1116     }
   1117 }
   1118 
   1119 void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg)
   1120 {
   1121     tcg_gen_movi_i32(TCGV_LOW(ret), arg);
   1122     tcg_gen_movi_i32(TCGV_HIGH(ret), arg >> 32);
   1123 }
   1124 
   1125 void tcg_gen_ld8u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
   1126 {
   1127     tcg_gen_ld8u_i32(TCGV_LOW(ret), arg2, offset);
   1128     tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
   1129 }
   1130 
   1131 void tcg_gen_ld8s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
   1132 {
   1133     tcg_gen_ld8s_i32(TCGV_LOW(ret), arg2, offset);
   1134     tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
   1135 }
   1136 
   1137 void tcg_gen_ld16u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
   1138 {
   1139     tcg_gen_ld16u_i32(TCGV_LOW(ret), arg2, offset);
   1140     tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
   1141 }
   1142 
   1143 void tcg_gen_ld16s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
   1144 {
   1145     tcg_gen_ld16s_i32(TCGV_LOW(ret), arg2, offset);
   1146     tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
   1147 }
   1148 
   1149 void tcg_gen_ld32u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
   1150 {
   1151     tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
   1152     tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
   1153 }
   1154 
   1155 void tcg_gen_ld32s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
   1156 {
   1157     tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
   1158     tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
   1159 }
   1160 
   1161 void tcg_gen_ld_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
   1162 {
   1163     /* Since arg2 and ret have different types,
   1164        they cannot be the same temporary */
   1165 #if HOST_BIG_ENDIAN
   1166     tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset);
   1167     tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset + 4);
   1168 #else
   1169     tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
   1170     tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset + 4);
   1171 #endif
   1172 }
   1173 
   1174 void tcg_gen_st_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset)
   1175 {
   1176 #if HOST_BIG_ENDIAN
   1177     tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset);
   1178     tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset + 4);
   1179 #else
   1180     tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset);
   1181     tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset + 4);
   1182 #endif
   1183 }
   1184 
   1185 void tcg_gen_and_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
   1186 {
   1187     tcg_gen_and_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
   1188     tcg_gen_and_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
   1189 }
   1190 
   1191 void tcg_gen_or_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
   1192 {
   1193     tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
   1194     tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
   1195 }
   1196 
   1197 void tcg_gen_xor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
   1198 {
   1199     tcg_gen_xor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
   1200     tcg_gen_xor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
   1201 }
   1202 
   1203 void tcg_gen_shl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
   1204 {
   1205     gen_helper_shl_i64(ret, arg1, arg2);
   1206 }
   1207 
   1208 void tcg_gen_shr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
   1209 {
   1210     gen_helper_shr_i64(ret, arg1, arg2);
   1211 }
   1212 
   1213 void tcg_gen_sar_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
   1214 {
   1215     gen_helper_sar_i64(ret, arg1, arg2);
   1216 }
   1217 
   1218 void tcg_gen_mul_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
   1219 {
   1220     TCGv_i64 t0;
   1221     TCGv_i32 t1;
   1222 
   1223     t0 = tcg_temp_new_i64();
   1224     t1 = tcg_temp_new_i32();
   1225 
   1226     tcg_gen_mulu2_i32(TCGV_LOW(t0), TCGV_HIGH(t0),
   1227                       TCGV_LOW(arg1), TCGV_LOW(arg2));
   1228 
   1229     tcg_gen_mul_i32(t1, TCGV_LOW(arg1), TCGV_HIGH(arg2));
   1230     tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1);
   1231     tcg_gen_mul_i32(t1, TCGV_HIGH(arg1), TCGV_LOW(arg2));
   1232     tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1);
   1233 
   1234     tcg_gen_mov_i64(ret, t0);
   1235     tcg_temp_free_i64(t0);
   1236     tcg_temp_free_i32(t1);
   1237 }
   1238 
   1239 #else
   1240 
   1241 void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg)
   1242 {
   1243     tcg_gen_mov_i64(ret, tcg_constant_i64(arg));
   1244 }
   1245 
   1246 #endif /* TCG_TARGET_REG_SIZE == 32 */
   1247 
   1248 void tcg_gen_addi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
   1249 {
   1250     /* some cases can be optimized here */
   1251     if (arg2 == 0) {
   1252         tcg_gen_mov_i64(ret, arg1);
   1253     } else if (TCG_TARGET_REG_BITS == 64) {
   1254         tcg_gen_add_i64(ret, arg1, tcg_constant_i64(arg2));
   1255     } else {
   1256         tcg_gen_add2_i32(TCGV_LOW(ret), TCGV_HIGH(ret),
   1257                          TCGV_LOW(arg1), TCGV_HIGH(arg1),
   1258                          tcg_constant_i32(arg2), tcg_constant_i32(arg2 >> 32));
   1259     }
   1260 }
   1261 
   1262 void tcg_gen_subfi_i64(TCGv_i64 ret, int64_t arg1, TCGv_i64 arg2)
   1263 {
   1264     if (arg1 == 0 && TCG_TARGET_HAS_neg_i64) {
   1265         /* Don't recurse with tcg_gen_neg_i64.  */
   1266         tcg_gen_op2_i64(INDEX_op_neg_i64, ret, arg2);
   1267     } else if (TCG_TARGET_REG_BITS == 64) {
   1268         tcg_gen_sub_i64(ret, tcg_constant_i64(arg1), arg2);
   1269     } else {
   1270         tcg_gen_sub2_i32(TCGV_LOW(ret), TCGV_HIGH(ret),
   1271                          tcg_constant_i32(arg1), tcg_constant_i32(arg1 >> 32),
   1272                          TCGV_LOW(arg2), TCGV_HIGH(arg2));
   1273     }
   1274 }
   1275 
   1276 void tcg_gen_subi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
   1277 {
   1278     /* some cases can be optimized here */
   1279     if (arg2 == 0) {
   1280         tcg_gen_mov_i64(ret, arg1);
   1281     } else if (TCG_TARGET_REG_BITS == 64) {
   1282         tcg_gen_sub_i64(ret, arg1, tcg_constant_i64(arg2));
   1283     } else {
   1284         tcg_gen_sub2_i32(TCGV_LOW(ret), TCGV_HIGH(ret),
   1285                          TCGV_LOW(arg1), TCGV_HIGH(arg1),
   1286                          tcg_constant_i32(arg2), tcg_constant_i32(arg2 >> 32));
   1287     }
   1288 }
   1289 
   1290 void tcg_gen_andi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
   1291 {
   1292     if (TCG_TARGET_REG_BITS == 32) {
   1293         tcg_gen_andi_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
   1294         tcg_gen_andi_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
   1295         return;
   1296     }
   1297 
   1298     /* Some cases can be optimized here.  */
   1299     switch (arg2) {
   1300     case 0:
   1301         tcg_gen_movi_i64(ret, 0);
   1302         return;
   1303     case -1:
   1304         tcg_gen_mov_i64(ret, arg1);
   1305         return;
   1306     case 0xff:
   1307         /* Don't recurse with tcg_gen_ext8u_i64.  */
   1308         if (TCG_TARGET_HAS_ext8u_i64) {
   1309             tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg1);
   1310             return;
   1311         }
   1312         break;
   1313     case 0xffff:
   1314         if (TCG_TARGET_HAS_ext16u_i64) {
   1315             tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg1);
   1316             return;
   1317         }
   1318         break;
   1319     case 0xffffffffu:
   1320         if (TCG_TARGET_HAS_ext32u_i64) {
   1321             tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg1);
   1322             return;
   1323         }
   1324         break;
   1325     }
   1326 
   1327     tcg_gen_and_i64(ret, arg1, tcg_constant_i64(arg2));
   1328 }
   1329 
   1330 void tcg_gen_ori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
   1331 {
   1332     if (TCG_TARGET_REG_BITS == 32) {
   1333         tcg_gen_ori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
   1334         tcg_gen_ori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
   1335         return;
   1336     }
   1337     /* Some cases can be optimized here.  */
   1338     if (arg2 == -1) {
   1339         tcg_gen_movi_i64(ret, -1);
   1340     } else if (arg2 == 0) {
   1341         tcg_gen_mov_i64(ret, arg1);
   1342     } else {
   1343         tcg_gen_or_i64(ret, arg1, tcg_constant_i64(arg2));
   1344     }
   1345 }
   1346 
   1347 void tcg_gen_xori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
   1348 {
   1349     if (TCG_TARGET_REG_BITS == 32) {
   1350         tcg_gen_xori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
   1351         tcg_gen_xori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
   1352         return;
   1353     }
   1354     /* Some cases can be optimized here.  */
   1355     if (arg2 == 0) {
   1356         tcg_gen_mov_i64(ret, arg1);
   1357     } else if (arg2 == -1 && TCG_TARGET_HAS_not_i64) {
   1358         /* Don't recurse with tcg_gen_not_i64.  */
   1359         tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg1);
   1360     } else {
   1361         tcg_gen_xor_i64(ret, arg1, tcg_constant_i64(arg2));
   1362     }
   1363 }
   1364 
   1365 static inline void tcg_gen_shifti_i64(TCGv_i64 ret, TCGv_i64 arg1,
   1366                                       unsigned c, bool right, bool arith)
   1367 {
   1368     tcg_debug_assert(c < 64);
   1369     if (c == 0) {
   1370         tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
   1371         tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
   1372     } else if (c >= 32) {
   1373         c -= 32;
   1374         if (right) {
   1375             if (arith) {
   1376                 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c);
   1377                 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 31);
   1378             } else {
   1379                 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c);
   1380                 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
   1381             }
   1382         } else {
   1383             tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_LOW(arg1), c);
   1384             tcg_gen_movi_i32(TCGV_LOW(ret), 0);
   1385         }
   1386     } else if (right) {
   1387         if (TCG_TARGET_HAS_extract2_i32) {
   1388             tcg_gen_extract2_i32(TCGV_LOW(ret),
   1389                                  TCGV_LOW(arg1), TCGV_HIGH(arg1), c);
   1390         } else {
   1391             tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c);
   1392             tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(ret),
   1393                                 TCGV_HIGH(arg1), 32 - c, c);
   1394         }
   1395         if (arith) {
   1396             tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c);
   1397         } else {
   1398             tcg_gen_shri_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c);
   1399         }
   1400     } else {
   1401         if (TCG_TARGET_HAS_extract2_i32) {
   1402             tcg_gen_extract2_i32(TCGV_HIGH(ret),
   1403                                  TCGV_LOW(arg1), TCGV_HIGH(arg1), 32 - c);
   1404         } else {
   1405             TCGv_i32 t0 = tcg_temp_new_i32();
   1406             tcg_gen_shri_i32(t0, TCGV_LOW(arg1), 32 - c);
   1407             tcg_gen_deposit_i32(TCGV_HIGH(ret), t0,
   1408                                 TCGV_HIGH(arg1), c, 32 - c);
   1409             tcg_temp_free_i32(t0);
   1410         }
   1411         tcg_gen_shli_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c);
   1412     }
   1413 }
   1414 
   1415 void tcg_gen_shli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
   1416 {
   1417     tcg_debug_assert(arg2 >= 0 && arg2 < 64);
   1418     if (TCG_TARGET_REG_BITS == 32) {
   1419         tcg_gen_shifti_i64(ret, arg1, arg2, 0, 0);
   1420     } else if (arg2 == 0) {
   1421         tcg_gen_mov_i64(ret, arg1);
   1422     } else {
   1423         tcg_gen_shl_i64(ret, arg1, tcg_constant_i64(arg2));
   1424     }
   1425 }
   1426 
   1427 void tcg_gen_shri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
   1428 {
   1429     tcg_debug_assert(arg2 >= 0 && arg2 < 64);
   1430     if (TCG_TARGET_REG_BITS == 32) {
   1431         tcg_gen_shifti_i64(ret, arg1, arg2, 1, 0);
   1432     } else if (arg2 == 0) {
   1433         tcg_gen_mov_i64(ret, arg1);
   1434     } else {
   1435         tcg_gen_shr_i64(ret, arg1, tcg_constant_i64(arg2));
   1436     }
   1437 }
   1438 
   1439 void tcg_gen_sari_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
   1440 {
   1441     tcg_debug_assert(arg2 >= 0 && arg2 < 64);
   1442     if (TCG_TARGET_REG_BITS == 32) {
   1443         tcg_gen_shifti_i64(ret, arg1, arg2, 1, 1);
   1444     } else if (arg2 == 0) {
   1445         tcg_gen_mov_i64(ret, arg1);
   1446     } else {
   1447         tcg_gen_sar_i64(ret, arg1, tcg_constant_i64(arg2));
   1448     }
   1449 }
   1450 
   1451 void tcg_gen_brcond_i64(TCGCond cond, TCGv_i64 arg1, TCGv_i64 arg2, TCGLabel *l)
   1452 {
   1453     if (cond == TCG_COND_ALWAYS) {
   1454         tcg_gen_br(l);
   1455     } else if (cond != TCG_COND_NEVER) {
   1456         l->refs++;
   1457         if (TCG_TARGET_REG_BITS == 32) {
   1458             tcg_gen_op6ii_i32(INDEX_op_brcond2_i32, TCGV_LOW(arg1),
   1459                               TCGV_HIGH(arg1), TCGV_LOW(arg2),
   1460                               TCGV_HIGH(arg2), cond, label_arg(l));
   1461         } else {
   1462             tcg_gen_op4ii_i64(INDEX_op_brcond_i64, arg1, arg2, cond,
   1463                               label_arg(l));
   1464         }
   1465     }
   1466 }
   1467 
   1468 void tcg_gen_brcondi_i64(TCGCond cond, TCGv_i64 arg1, int64_t arg2, TCGLabel *l)
   1469 {
   1470     if (TCG_TARGET_REG_BITS == 64) {
   1471         tcg_gen_brcond_i64(cond, arg1, tcg_constant_i64(arg2), l);
   1472     } else if (cond == TCG_COND_ALWAYS) {
   1473         tcg_gen_br(l);
   1474     } else if (cond != TCG_COND_NEVER) {
   1475         l->refs++;
   1476         tcg_gen_op6ii_i32(INDEX_op_brcond2_i32,
   1477                           TCGV_LOW(arg1), TCGV_HIGH(arg1),
   1478                           tcg_constant_i32(arg2),
   1479                           tcg_constant_i32(arg2 >> 32),
   1480                           cond, label_arg(l));
   1481     }
   1482 }
   1483 
   1484 void tcg_gen_setcond_i64(TCGCond cond, TCGv_i64 ret,
   1485                          TCGv_i64 arg1, TCGv_i64 arg2)
   1486 {
   1487     if (cond == TCG_COND_ALWAYS) {
   1488         tcg_gen_movi_i64(ret, 1);
   1489     } else if (cond == TCG_COND_NEVER) {
   1490         tcg_gen_movi_i64(ret, 0);
   1491     } else {
   1492         if (TCG_TARGET_REG_BITS == 32) {
   1493             tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret),
   1494                              TCGV_LOW(arg1), TCGV_HIGH(arg1),
   1495                              TCGV_LOW(arg2), TCGV_HIGH(arg2), cond);
   1496             tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
   1497         } else {
   1498             tcg_gen_op4i_i64(INDEX_op_setcond_i64, ret, arg1, arg2, cond);
   1499         }
   1500     }
   1501 }
   1502 
   1503 void tcg_gen_setcondi_i64(TCGCond cond, TCGv_i64 ret,
   1504                           TCGv_i64 arg1, int64_t arg2)
   1505 {
   1506     if (TCG_TARGET_REG_BITS == 64) {
   1507         tcg_gen_setcond_i64(cond, ret, arg1, tcg_constant_i64(arg2));
   1508     } else if (cond == TCG_COND_ALWAYS) {
   1509         tcg_gen_movi_i64(ret, 1);
   1510     } else if (cond == TCG_COND_NEVER) {
   1511         tcg_gen_movi_i64(ret, 0);
   1512     } else {
   1513         tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret),
   1514                          TCGV_LOW(arg1), TCGV_HIGH(arg1),
   1515                          tcg_constant_i32(arg2),
   1516                          tcg_constant_i32(arg2 >> 32), cond);
   1517         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
   1518     }
   1519 }
   1520 
   1521 void tcg_gen_muli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
   1522 {
   1523     if (arg2 == 0) {
   1524         tcg_gen_movi_i64(ret, 0);
   1525     } else if (is_power_of_2(arg2)) {
   1526         tcg_gen_shli_i64(ret, arg1, ctz64(arg2));
   1527     } else {
   1528         TCGv_i64 t0 = tcg_const_i64(arg2);
   1529         tcg_gen_mul_i64(ret, arg1, t0);
   1530         tcg_temp_free_i64(t0);
   1531     }
   1532 }
   1533 
   1534 void tcg_gen_div_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
   1535 {
   1536     if (TCG_TARGET_HAS_div_i64) {
   1537         tcg_gen_op3_i64(INDEX_op_div_i64, ret, arg1, arg2);
   1538     } else if (TCG_TARGET_HAS_div2_i64) {
   1539         TCGv_i64 t0 = tcg_temp_new_i64();
   1540         tcg_gen_sari_i64(t0, arg1, 63);
   1541         tcg_gen_op5_i64(INDEX_op_div2_i64, ret, t0, arg1, t0, arg2);
   1542         tcg_temp_free_i64(t0);
   1543     } else {
   1544         gen_helper_div_i64(ret, arg1, arg2);
   1545     }
   1546 }
   1547 
   1548 void tcg_gen_rem_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
   1549 {
   1550     if (TCG_TARGET_HAS_rem_i64) {
   1551         tcg_gen_op3_i64(INDEX_op_rem_i64, ret, arg1, arg2);
   1552     } else if (TCG_TARGET_HAS_div_i64) {
   1553         TCGv_i64 t0 = tcg_temp_new_i64();
   1554         tcg_gen_op3_i64(INDEX_op_div_i64, t0, arg1, arg2);
   1555         tcg_gen_mul_i64(t0, t0, arg2);
   1556         tcg_gen_sub_i64(ret, arg1, t0);
   1557         tcg_temp_free_i64(t0);
   1558     } else if (TCG_TARGET_HAS_div2_i64) {
   1559         TCGv_i64 t0 = tcg_temp_new_i64();
   1560         tcg_gen_sari_i64(t0, arg1, 63);
   1561         tcg_gen_op5_i64(INDEX_op_div2_i64, t0, ret, arg1, t0, arg2);
   1562         tcg_temp_free_i64(t0);
   1563     } else {
   1564         gen_helper_rem_i64(ret, arg1, arg2);
   1565     }
   1566 }
   1567 
   1568 void tcg_gen_divu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
   1569 {
   1570     if (TCG_TARGET_HAS_div_i64) {
   1571         tcg_gen_op3_i64(INDEX_op_divu_i64, ret, arg1, arg2);
   1572     } else if (TCG_TARGET_HAS_div2_i64) {
   1573         TCGv_i64 t0 = tcg_temp_new_i64();
   1574         tcg_gen_movi_i64(t0, 0);
   1575         tcg_gen_op5_i64(INDEX_op_divu2_i64, ret, t0, arg1, t0, arg2);
   1576         tcg_temp_free_i64(t0);
   1577     } else {
   1578         gen_helper_divu_i64(ret, arg1, arg2);
   1579     }
   1580 }
   1581 
   1582 void tcg_gen_remu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
   1583 {
   1584     if (TCG_TARGET_HAS_rem_i64) {
   1585         tcg_gen_op3_i64(INDEX_op_remu_i64, ret, arg1, arg2);
   1586     } else if (TCG_TARGET_HAS_div_i64) {
   1587         TCGv_i64 t0 = tcg_temp_new_i64();
   1588         tcg_gen_op3_i64(INDEX_op_divu_i64, t0, arg1, arg2);
   1589         tcg_gen_mul_i64(t0, t0, arg2);
   1590         tcg_gen_sub_i64(ret, arg1, t0);
   1591         tcg_temp_free_i64(t0);
   1592     } else if (TCG_TARGET_HAS_div2_i64) {
   1593         TCGv_i64 t0 = tcg_temp_new_i64();
   1594         tcg_gen_movi_i64(t0, 0);
   1595         tcg_gen_op5_i64(INDEX_op_divu2_i64, t0, ret, arg1, t0, arg2);
   1596         tcg_temp_free_i64(t0);
   1597     } else {
   1598         gen_helper_remu_i64(ret, arg1, arg2);
   1599     }
   1600 }
   1601 
   1602 void tcg_gen_ext8s_i64(TCGv_i64 ret, TCGv_i64 arg)
   1603 {
   1604     if (TCG_TARGET_REG_BITS == 32) {
   1605         tcg_gen_ext8s_i32(TCGV_LOW(ret), TCGV_LOW(arg));
   1606         tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
   1607     } else if (TCG_TARGET_HAS_ext8s_i64) {
   1608         tcg_gen_op2_i64(INDEX_op_ext8s_i64, ret, arg);
   1609     } else {
   1610         tcg_gen_shli_i64(ret, arg, 56);
   1611         tcg_gen_sari_i64(ret, ret, 56);
   1612     }
   1613 }
   1614 
   1615 void tcg_gen_ext16s_i64(TCGv_i64 ret, TCGv_i64 arg)
   1616 {
   1617     if (TCG_TARGET_REG_BITS == 32) {
   1618         tcg_gen_ext16s_i32(TCGV_LOW(ret), TCGV_LOW(arg));
   1619         tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
   1620     } else if (TCG_TARGET_HAS_ext16s_i64) {
   1621         tcg_gen_op2_i64(INDEX_op_ext16s_i64, ret, arg);
   1622     } else {
   1623         tcg_gen_shli_i64(ret, arg, 48);
   1624         tcg_gen_sari_i64(ret, ret, 48);
   1625     }
   1626 }
   1627 
   1628 void tcg_gen_ext32s_i64(TCGv_i64 ret, TCGv_i64 arg)
   1629 {
   1630     if (TCG_TARGET_REG_BITS == 32) {
   1631         tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
   1632         tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
   1633     } else if (TCG_TARGET_HAS_ext32s_i64) {
   1634         tcg_gen_op2_i64(INDEX_op_ext32s_i64, ret, arg);
   1635     } else {
   1636         tcg_gen_shli_i64(ret, arg, 32);
   1637         tcg_gen_sari_i64(ret, ret, 32);
   1638     }
   1639 }
   1640 
   1641 void tcg_gen_ext8u_i64(TCGv_i64 ret, TCGv_i64 arg)
   1642 {
   1643     if (TCG_TARGET_REG_BITS == 32) {
   1644         tcg_gen_ext8u_i32(TCGV_LOW(ret), TCGV_LOW(arg));
   1645         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
   1646     } else if (TCG_TARGET_HAS_ext8u_i64) {
   1647         tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg);
   1648     } else {
   1649         tcg_gen_andi_i64(ret, arg, 0xffu);
   1650     }
   1651 }
   1652 
   1653 void tcg_gen_ext16u_i64(TCGv_i64 ret, TCGv_i64 arg)
   1654 {
   1655     if (TCG_TARGET_REG_BITS == 32) {
   1656         tcg_gen_ext16u_i32(TCGV_LOW(ret), TCGV_LOW(arg));
   1657         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
   1658     } else if (TCG_TARGET_HAS_ext16u_i64) {
   1659         tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg);
   1660     } else {
   1661         tcg_gen_andi_i64(ret, arg, 0xffffu);
   1662     }
   1663 }
   1664 
   1665 void tcg_gen_ext32u_i64(TCGv_i64 ret, TCGv_i64 arg)
   1666 {
   1667     if (TCG_TARGET_REG_BITS == 32) {
   1668         tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
   1669         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
   1670     } else if (TCG_TARGET_HAS_ext32u_i64) {
   1671         tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg);
   1672     } else {
   1673         tcg_gen_andi_i64(ret, arg, 0xffffffffu);
   1674     }
   1675 }
   1676 
   1677 void tcg_gen_bswap16_i64(TCGv_i64 ret, TCGv_i64 arg, int flags)
   1678 {
   1679     /* Only one extension flag may be present. */
   1680     tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ));
   1681 
   1682     if (TCG_TARGET_REG_BITS == 32) {
   1683         tcg_gen_bswap16_i32(TCGV_LOW(ret), TCGV_LOW(arg), flags);
   1684         if (flags & TCG_BSWAP_OS) {
   1685             tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
   1686         } else {
   1687             tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
   1688         }
   1689     } else if (TCG_TARGET_HAS_bswap16_i64) {
   1690         tcg_gen_op3i_i64(INDEX_op_bswap16_i64, ret, arg, flags);
   1691     } else {
   1692         TCGv_i64 t0 = tcg_temp_new_i64();
   1693         TCGv_i64 t1 = tcg_temp_new_i64();
   1694 
   1695         tcg_gen_shri_i64(t0, arg, 8);
   1696         if (!(flags & TCG_BSWAP_IZ)) {
   1697             tcg_gen_ext8u_i64(t0, t0);
   1698         }
   1699 
   1700         if (flags & TCG_BSWAP_OS) {
   1701             tcg_gen_shli_i64(t1, arg, 56);
   1702             tcg_gen_sari_i64(t1, t1, 48);
   1703         } else if (flags & TCG_BSWAP_OZ) {
   1704             tcg_gen_ext8u_i64(t1, arg);
   1705             tcg_gen_shli_i64(t1, t1, 8);
   1706         } else {
   1707             tcg_gen_shli_i64(t1, arg, 8);
   1708         }
   1709 
   1710         tcg_gen_or_i64(ret, t0, t1);
   1711         tcg_temp_free_i64(t0);
   1712         tcg_temp_free_i64(t1);
   1713     }
   1714 }
   1715 
   1716 void tcg_gen_bswap32_i64(TCGv_i64 ret, TCGv_i64 arg, int flags)
   1717 {
   1718     /* Only one extension flag may be present. */
   1719     tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ));
   1720 
   1721     if (TCG_TARGET_REG_BITS == 32) {
   1722         tcg_gen_bswap32_i32(TCGV_LOW(ret), TCGV_LOW(arg));
   1723         if (flags & TCG_BSWAP_OS) {
   1724             tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
   1725         } else {
   1726             tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
   1727         }
   1728     } else if (TCG_TARGET_HAS_bswap32_i64) {
   1729         tcg_gen_op3i_i64(INDEX_op_bswap32_i64, ret, arg, flags);
   1730     } else {
   1731         TCGv_i64 t0 = tcg_temp_new_i64();
   1732         TCGv_i64 t1 = tcg_temp_new_i64();
   1733         TCGv_i64 t2 = tcg_constant_i64(0x00ff00ff);
   1734 
   1735                                             /* arg = xxxxabcd */
   1736         tcg_gen_shri_i64(t0, arg, 8);       /*  t0 = .xxxxabc */
   1737         tcg_gen_and_i64(t1, arg, t2);       /*  t1 = .....b.d */
   1738         tcg_gen_and_i64(t0, t0, t2);        /*  t0 = .....a.c */
   1739         tcg_gen_shli_i64(t1, t1, 8);        /*  t1 = ....b.d. */
   1740         tcg_gen_or_i64(ret, t0, t1);        /* ret = ....badc */
   1741 
   1742         tcg_gen_shli_i64(t1, ret, 48);      /*  t1 = dc...... */
   1743         tcg_gen_shri_i64(t0, ret, 16);      /*  t0 = ......ba */
   1744         if (flags & TCG_BSWAP_OS) {
   1745             tcg_gen_sari_i64(t1, t1, 32);   /*  t1 = ssssdc.. */
   1746         } else {
   1747             tcg_gen_shri_i64(t1, t1, 32);   /*  t1 = ....dc.. */
   1748         }
   1749         tcg_gen_or_i64(ret, t0, t1);        /* ret = ssssdcba */
   1750 
   1751         tcg_temp_free_i64(t0);
   1752         tcg_temp_free_i64(t1);
   1753     }
   1754 }
   1755 
   1756 void tcg_gen_bswap64_i64(TCGv_i64 ret, TCGv_i64 arg)
   1757 {
   1758     if (TCG_TARGET_REG_BITS == 32) {
   1759         TCGv_i32 t0, t1;
   1760         t0 = tcg_temp_new_i32();
   1761         t1 = tcg_temp_new_i32();
   1762 
   1763         tcg_gen_bswap32_i32(t0, TCGV_LOW(arg));
   1764         tcg_gen_bswap32_i32(t1, TCGV_HIGH(arg));
   1765         tcg_gen_mov_i32(TCGV_LOW(ret), t1);
   1766         tcg_gen_mov_i32(TCGV_HIGH(ret), t0);
   1767         tcg_temp_free_i32(t0);
   1768         tcg_temp_free_i32(t1);
   1769     } else if (TCG_TARGET_HAS_bswap64_i64) {
   1770         tcg_gen_op3i_i64(INDEX_op_bswap64_i64, ret, arg, 0);
   1771     } else {
   1772         TCGv_i64 t0 = tcg_temp_new_i64();
   1773         TCGv_i64 t1 = tcg_temp_new_i64();
   1774         TCGv_i64 t2 = tcg_temp_new_i64();
   1775 
   1776                                         /* arg = abcdefgh */
   1777         tcg_gen_movi_i64(t2, 0x00ff00ff00ff00ffull);
   1778         tcg_gen_shri_i64(t0, arg, 8);   /*  t0 = .abcdefg */
   1779         tcg_gen_and_i64(t1, arg, t2);   /*  t1 = .b.d.f.h */
   1780         tcg_gen_and_i64(t0, t0, t2);    /*  t0 = .a.c.e.g */
   1781         tcg_gen_shli_i64(t1, t1, 8);    /*  t1 = b.d.f.h. */
   1782         tcg_gen_or_i64(ret, t0, t1);    /* ret = badcfehg */
   1783 
   1784         tcg_gen_movi_i64(t2, 0x0000ffff0000ffffull);
   1785         tcg_gen_shri_i64(t0, ret, 16);  /*  t0 = ..badcfe */
   1786         tcg_gen_and_i64(t1, ret, t2);   /*  t1 = ..dc..hg */
   1787         tcg_gen_and_i64(t0, t0, t2);    /*  t0 = ..ba..fe */
   1788         tcg_gen_shli_i64(t1, t1, 16);   /*  t1 = dc..hg.. */
   1789         tcg_gen_or_i64(ret, t0, t1);    /* ret = dcbahgfe */
   1790 
   1791         tcg_gen_shri_i64(t0, ret, 32);  /*  t0 = ....dcba */
   1792         tcg_gen_shli_i64(t1, ret, 32);  /*  t1 = hgfe.... */
   1793         tcg_gen_or_i64(ret, t0, t1);    /* ret = hgfedcba */
   1794 
   1795         tcg_temp_free_i64(t0);
   1796         tcg_temp_free_i64(t1);
   1797         tcg_temp_free_i64(t2);
   1798     }
   1799 }
   1800 
   1801 void tcg_gen_hswap_i64(TCGv_i64 ret, TCGv_i64 arg)
   1802 {
   1803     uint64_t m = 0x0000ffff0000ffffull;
   1804     TCGv_i64 t0 = tcg_temp_new_i64();
   1805     TCGv_i64 t1 = tcg_temp_new_i64();
   1806 
   1807     /* See include/qemu/bitops.h, hswap64. */
   1808     tcg_gen_rotli_i64(t1, arg, 32);
   1809     tcg_gen_andi_i64(t0, t1, m);
   1810     tcg_gen_shli_i64(t0, t0, 16);
   1811     tcg_gen_shri_i64(t1, t1, 16);
   1812     tcg_gen_andi_i64(t1, t1, m);
   1813     tcg_gen_or_i64(ret, t0, t1);
   1814 
   1815     tcg_temp_free_i64(t0);
   1816     tcg_temp_free_i64(t1);
   1817 }
   1818 
   1819 void tcg_gen_wswap_i64(TCGv_i64 ret, TCGv_i64 arg)
   1820 {
   1821     /* Swapping 2 32-bit elements is a rotate. */
   1822     tcg_gen_rotli_i64(ret, arg, 32);
   1823 }
   1824 
   1825 void tcg_gen_not_i64(TCGv_i64 ret, TCGv_i64 arg)
   1826 {
   1827     if (TCG_TARGET_REG_BITS == 32) {
   1828         tcg_gen_not_i32(TCGV_LOW(ret), TCGV_LOW(arg));
   1829         tcg_gen_not_i32(TCGV_HIGH(ret), TCGV_HIGH(arg));
   1830     } else if (TCG_TARGET_HAS_not_i64) {
   1831         tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg);
   1832     } else {
   1833         tcg_gen_xori_i64(ret, arg, -1);
   1834     }
   1835 }
   1836 
   1837 void tcg_gen_andc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
   1838 {
   1839     if (TCG_TARGET_REG_BITS == 32) {
   1840         tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
   1841         tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
   1842     } else if (TCG_TARGET_HAS_andc_i64) {
   1843         tcg_gen_op3_i64(INDEX_op_andc_i64, ret, arg1, arg2);
   1844     } else {
   1845         TCGv_i64 t0 = tcg_temp_new_i64();
   1846         tcg_gen_not_i64(t0, arg2);
   1847         tcg_gen_and_i64(ret, arg1, t0);
   1848         tcg_temp_free_i64(t0);
   1849     }
   1850 }
   1851 
   1852 void tcg_gen_eqv_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
   1853 {
   1854     if (TCG_TARGET_REG_BITS == 32) {
   1855         tcg_gen_eqv_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
   1856         tcg_gen_eqv_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
   1857     } else if (TCG_TARGET_HAS_eqv_i64) {
   1858         tcg_gen_op3_i64(INDEX_op_eqv_i64, ret, arg1, arg2);
   1859     } else {
   1860         tcg_gen_xor_i64(ret, arg1, arg2);
   1861         tcg_gen_not_i64(ret, ret);
   1862     }
   1863 }
   1864 
   1865 void tcg_gen_nand_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
   1866 {
   1867     if (TCG_TARGET_REG_BITS == 32) {
   1868         tcg_gen_nand_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
   1869         tcg_gen_nand_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
   1870     } else if (TCG_TARGET_HAS_nand_i64) {
   1871         tcg_gen_op3_i64(INDEX_op_nand_i64, ret, arg1, arg2);
   1872     } else {
   1873         tcg_gen_and_i64(ret, arg1, arg2);
   1874         tcg_gen_not_i64(ret, ret);
   1875     }
   1876 }
   1877 
   1878 void tcg_gen_nor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
   1879 {
   1880     if (TCG_TARGET_REG_BITS == 32) {
   1881         tcg_gen_nor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
   1882         tcg_gen_nor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
   1883     } else if (TCG_TARGET_HAS_nor_i64) {
   1884         tcg_gen_op3_i64(INDEX_op_nor_i64, ret, arg1, arg2);
   1885     } else {
   1886         tcg_gen_or_i64(ret, arg1, arg2);
   1887         tcg_gen_not_i64(ret, ret);
   1888     }
   1889 }
   1890 
   1891 void tcg_gen_orc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
   1892 {
   1893     if (TCG_TARGET_REG_BITS == 32) {
   1894         tcg_gen_orc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
   1895         tcg_gen_orc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
   1896     } else if (TCG_TARGET_HAS_orc_i64) {
   1897         tcg_gen_op3_i64(INDEX_op_orc_i64, ret, arg1, arg2);
   1898     } else {
   1899         TCGv_i64 t0 = tcg_temp_new_i64();
   1900         tcg_gen_not_i64(t0, arg2);
   1901         tcg_gen_or_i64(ret, arg1, t0);
   1902         tcg_temp_free_i64(t0);
   1903     }
   1904 }
   1905 
   1906 void tcg_gen_clz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
   1907 {
   1908     if (TCG_TARGET_HAS_clz_i64) {
   1909         tcg_gen_op3_i64(INDEX_op_clz_i64, ret, arg1, arg2);
   1910     } else {
   1911         gen_helper_clz_i64(ret, arg1, arg2);
   1912     }
   1913 }
   1914 
   1915 void tcg_gen_clzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2)
   1916 {
   1917     if (TCG_TARGET_REG_BITS == 32
   1918         && TCG_TARGET_HAS_clz_i32
   1919         && arg2 <= 0xffffffffu) {
   1920         TCGv_i32 t = tcg_temp_new_i32();
   1921         tcg_gen_clzi_i32(t, TCGV_LOW(arg1), arg2 - 32);
   1922         tcg_gen_addi_i32(t, t, 32);
   1923         tcg_gen_clz_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), t);
   1924         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
   1925         tcg_temp_free_i32(t);
   1926     } else {
   1927         TCGv_i64 t0 = tcg_const_i64(arg2);
   1928         tcg_gen_clz_i64(ret, arg1, t0);
   1929         tcg_temp_free_i64(t0);
   1930     }
   1931 }
   1932 
   1933 void tcg_gen_ctz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
   1934 {
   1935     if (TCG_TARGET_HAS_ctz_i64) {
   1936         tcg_gen_op3_i64(INDEX_op_ctz_i64, ret, arg1, arg2);
   1937     } else if (TCG_TARGET_HAS_ctpop_i64 || TCG_TARGET_HAS_clz_i64) {
   1938         TCGv_i64 z, t = tcg_temp_new_i64();
   1939 
   1940         if (TCG_TARGET_HAS_ctpop_i64) {
   1941             tcg_gen_subi_i64(t, arg1, 1);
   1942             tcg_gen_andc_i64(t, t, arg1);
   1943             tcg_gen_ctpop_i64(t, t);
   1944         } else {
   1945             /* Since all non-x86 hosts have clz(0) == 64, don't fight it.  */
   1946             tcg_gen_neg_i64(t, arg1);
   1947             tcg_gen_and_i64(t, t, arg1);
   1948             tcg_gen_clzi_i64(t, t, 64);
   1949             tcg_gen_xori_i64(t, t, 63);
   1950         }
   1951         z = tcg_constant_i64(0);
   1952         tcg_gen_movcond_i64(TCG_COND_EQ, ret, arg1, z, arg2, t);
   1953         tcg_temp_free_i64(t);
   1954         tcg_temp_free_i64(z);
   1955     } else {
   1956         gen_helper_ctz_i64(ret, arg1, arg2);
   1957     }
   1958 }
   1959 
   1960 void tcg_gen_ctzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2)
   1961 {
   1962     if (TCG_TARGET_REG_BITS == 32
   1963         && TCG_TARGET_HAS_ctz_i32
   1964         && arg2 <= 0xffffffffu) {
   1965         TCGv_i32 t32 = tcg_temp_new_i32();
   1966         tcg_gen_ctzi_i32(t32, TCGV_HIGH(arg1), arg2 - 32);
   1967         tcg_gen_addi_i32(t32, t32, 32);
   1968         tcg_gen_ctz_i32(TCGV_LOW(ret), TCGV_LOW(arg1), t32);
   1969         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
   1970         tcg_temp_free_i32(t32);
   1971     } else if (!TCG_TARGET_HAS_ctz_i64
   1972                && TCG_TARGET_HAS_ctpop_i64
   1973                && arg2 == 64) {
   1974         /* This equivalence has the advantage of not requiring a fixup.  */
   1975         TCGv_i64 t = tcg_temp_new_i64();
   1976         tcg_gen_subi_i64(t, arg1, 1);
   1977         tcg_gen_andc_i64(t, t, arg1);
   1978         tcg_gen_ctpop_i64(ret, t);
   1979         tcg_temp_free_i64(t);
   1980     } else {
   1981         TCGv_i64 t0 = tcg_const_i64(arg2);
   1982         tcg_gen_ctz_i64(ret, arg1, t0);
   1983         tcg_temp_free_i64(t0);
   1984     }
   1985 }
   1986 
   1987 void tcg_gen_clrsb_i64(TCGv_i64 ret, TCGv_i64 arg)
   1988 {
   1989     if (TCG_TARGET_HAS_clz_i64 || TCG_TARGET_HAS_clz_i32) {
   1990         TCGv_i64 t = tcg_temp_new_i64();
   1991         tcg_gen_sari_i64(t, arg, 63);
   1992         tcg_gen_xor_i64(t, t, arg);
   1993         tcg_gen_clzi_i64(t, t, 64);
   1994         tcg_gen_subi_i64(ret, t, 1);
   1995         tcg_temp_free_i64(t);
   1996     } else {
   1997         gen_helper_clrsb_i64(ret, arg);
   1998     }
   1999 }
   2000 
   2001 void tcg_gen_ctpop_i64(TCGv_i64 ret, TCGv_i64 arg1)
   2002 {
   2003     if (TCG_TARGET_HAS_ctpop_i64) {
   2004         tcg_gen_op2_i64(INDEX_op_ctpop_i64, ret, arg1);
   2005     } else if (TCG_TARGET_REG_BITS == 32 && TCG_TARGET_HAS_ctpop_i32) {
   2006         tcg_gen_ctpop_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
   2007         tcg_gen_ctpop_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
   2008         tcg_gen_add_i32(TCGV_LOW(ret), TCGV_LOW(ret), TCGV_HIGH(ret));
   2009         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
   2010     } else {
   2011         gen_helper_ctpop_i64(ret, arg1);
   2012     }
   2013 }
   2014 
   2015 void tcg_gen_rotl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
   2016 {
   2017     if (TCG_TARGET_HAS_rot_i64) {
   2018         tcg_gen_op3_i64(INDEX_op_rotl_i64, ret, arg1, arg2);
   2019     } else {
   2020         TCGv_i64 t0, t1;
   2021         t0 = tcg_temp_new_i64();
   2022         t1 = tcg_temp_new_i64();
   2023         tcg_gen_shl_i64(t0, arg1, arg2);
   2024         tcg_gen_subfi_i64(t1, 64, arg2);
   2025         tcg_gen_shr_i64(t1, arg1, t1);
   2026         tcg_gen_or_i64(ret, t0, t1);
   2027         tcg_temp_free_i64(t0);
   2028         tcg_temp_free_i64(t1);
   2029     }
   2030 }
   2031 
   2032 void tcg_gen_rotli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
   2033 {
   2034     tcg_debug_assert(arg2 >= 0 && arg2 < 64);
   2035     /* some cases can be optimized here */
   2036     if (arg2 == 0) {
   2037         tcg_gen_mov_i64(ret, arg1);
   2038     } else if (TCG_TARGET_HAS_rot_i64) {
   2039         tcg_gen_rotl_i64(ret, arg1, tcg_constant_i64(arg2));
   2040     } else {
   2041         TCGv_i64 t0, t1;
   2042         t0 = tcg_temp_new_i64();
   2043         t1 = tcg_temp_new_i64();
   2044         tcg_gen_shli_i64(t0, arg1, arg2);
   2045         tcg_gen_shri_i64(t1, arg1, 64 - arg2);
   2046         tcg_gen_or_i64(ret, t0, t1);
   2047         tcg_temp_free_i64(t0);
   2048         tcg_temp_free_i64(t1);
   2049     }
   2050 }
   2051 
   2052 void tcg_gen_rotr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
   2053 {
   2054     if (TCG_TARGET_HAS_rot_i64) {
   2055         tcg_gen_op3_i64(INDEX_op_rotr_i64, ret, arg1, arg2);
   2056     } else {
   2057         TCGv_i64 t0, t1;
   2058         t0 = tcg_temp_new_i64();
   2059         t1 = tcg_temp_new_i64();
   2060         tcg_gen_shr_i64(t0, arg1, arg2);
   2061         tcg_gen_subfi_i64(t1, 64, arg2);
   2062         tcg_gen_shl_i64(t1, arg1, t1);
   2063         tcg_gen_or_i64(ret, t0, t1);
   2064         tcg_temp_free_i64(t0);
   2065         tcg_temp_free_i64(t1);
   2066     }
   2067 }
   2068 
   2069 void tcg_gen_rotri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
   2070 {
   2071     tcg_debug_assert(arg2 >= 0 && arg2 < 64);
   2072     /* some cases can be optimized here */
   2073     if (arg2 == 0) {
   2074         tcg_gen_mov_i64(ret, arg1);
   2075     } else {
   2076         tcg_gen_rotli_i64(ret, arg1, 64 - arg2);
   2077     }
   2078 }
   2079 
   2080 void tcg_gen_deposit_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2,
   2081                          unsigned int ofs, unsigned int len)
   2082 {
   2083     uint64_t mask;
   2084     TCGv_i64 t1;
   2085 
   2086     tcg_debug_assert(ofs < 64);
   2087     tcg_debug_assert(len > 0);
   2088     tcg_debug_assert(len <= 64);
   2089     tcg_debug_assert(ofs + len <= 64);
   2090 
   2091     if (len == 64) {
   2092         tcg_gen_mov_i64(ret, arg2);
   2093         return;
   2094     }
   2095     if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(ofs, len)) {
   2096         tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, arg1, arg2, ofs, len);
   2097         return;
   2098     }
   2099 
   2100     if (TCG_TARGET_REG_BITS == 32) {
   2101         if (ofs >= 32) {
   2102             tcg_gen_deposit_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1),
   2103                                 TCGV_LOW(arg2), ofs - 32, len);
   2104             tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
   2105             return;
   2106         }
   2107         if (ofs + len <= 32) {
   2108             tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(arg1),
   2109                                 TCGV_LOW(arg2), ofs, len);
   2110             tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
   2111             return;
   2112         }
   2113     }
   2114 
   2115     t1 = tcg_temp_new_i64();
   2116 
   2117     if (TCG_TARGET_HAS_extract2_i64) {
   2118         if (ofs + len == 64) {
   2119             tcg_gen_shli_i64(t1, arg1, len);
   2120             tcg_gen_extract2_i64(ret, t1, arg2, len);
   2121             goto done;
   2122         }
   2123         if (ofs == 0) {
   2124             tcg_gen_extract2_i64(ret, arg1, arg2, len);
   2125             tcg_gen_rotli_i64(ret, ret, len);
   2126             goto done;
   2127         }
   2128     }
   2129 
   2130     mask = (1ull << len) - 1;
   2131     if (ofs + len < 64) {
   2132         tcg_gen_andi_i64(t1, arg2, mask);
   2133         tcg_gen_shli_i64(t1, t1, ofs);
   2134     } else {
   2135         tcg_gen_shli_i64(t1, arg2, ofs);
   2136     }
   2137     tcg_gen_andi_i64(ret, arg1, ~(mask << ofs));
   2138     tcg_gen_or_i64(ret, ret, t1);
   2139  done:
   2140     tcg_temp_free_i64(t1);
   2141 }
   2142 
   2143 void tcg_gen_deposit_z_i64(TCGv_i64 ret, TCGv_i64 arg,
   2144                            unsigned int ofs, unsigned int len)
   2145 {
   2146     tcg_debug_assert(ofs < 64);
   2147     tcg_debug_assert(len > 0);
   2148     tcg_debug_assert(len <= 64);
   2149     tcg_debug_assert(ofs + len <= 64);
   2150 
   2151     if (ofs + len == 64) {
   2152         tcg_gen_shli_i64(ret, arg, ofs);
   2153     } else if (ofs == 0) {
   2154         tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
   2155     } else if (TCG_TARGET_HAS_deposit_i64
   2156                && TCG_TARGET_deposit_i64_valid(ofs, len)) {
   2157         TCGv_i64 zero = tcg_constant_i64(0);
   2158         tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, zero, arg, ofs, len);
   2159     } else {
   2160         if (TCG_TARGET_REG_BITS == 32) {
   2161             if (ofs >= 32) {
   2162                 tcg_gen_deposit_z_i32(TCGV_HIGH(ret), TCGV_LOW(arg),
   2163                                       ofs - 32, len);
   2164                 tcg_gen_movi_i32(TCGV_LOW(ret), 0);
   2165                 return;
   2166             }
   2167             if (ofs + len <= 32) {
   2168                 tcg_gen_deposit_z_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
   2169                 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
   2170                 return;
   2171             }
   2172         }
   2173         /* To help two-operand hosts we prefer to zero-extend first,
   2174            which allows ARG to stay live.  */
   2175         switch (len) {
   2176         case 32:
   2177             if (TCG_TARGET_HAS_ext32u_i64) {
   2178                 tcg_gen_ext32u_i64(ret, arg);
   2179                 tcg_gen_shli_i64(ret, ret, ofs);
   2180                 return;
   2181             }
   2182             break;
   2183         case 16:
   2184             if (TCG_TARGET_HAS_ext16u_i64) {
   2185                 tcg_gen_ext16u_i64(ret, arg);
   2186                 tcg_gen_shli_i64(ret, ret, ofs);
   2187                 return;
   2188             }
   2189             break;
   2190         case 8:
   2191             if (TCG_TARGET_HAS_ext8u_i64) {
   2192                 tcg_gen_ext8u_i64(ret, arg);
   2193                 tcg_gen_shli_i64(ret, ret, ofs);
   2194                 return;
   2195             }
   2196             break;
   2197         }
   2198         /* Otherwise prefer zero-extension over AND for code size.  */
   2199         switch (ofs + len) {
   2200         case 32:
   2201             if (TCG_TARGET_HAS_ext32u_i64) {
   2202                 tcg_gen_shli_i64(ret, arg, ofs);
   2203                 tcg_gen_ext32u_i64(ret, ret);
   2204                 return;
   2205             }
   2206             break;
   2207         case 16:
   2208             if (TCG_TARGET_HAS_ext16u_i64) {
   2209                 tcg_gen_shli_i64(ret, arg, ofs);
   2210                 tcg_gen_ext16u_i64(ret, ret);
   2211                 return;
   2212             }
   2213             break;
   2214         case 8:
   2215             if (TCG_TARGET_HAS_ext8u_i64) {
   2216                 tcg_gen_shli_i64(ret, arg, ofs);
   2217                 tcg_gen_ext8u_i64(ret, ret);
   2218                 return;
   2219             }
   2220             break;
   2221         }
   2222         tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
   2223         tcg_gen_shli_i64(ret, ret, ofs);
   2224     }
   2225 }
   2226 
   2227 void tcg_gen_extract_i64(TCGv_i64 ret, TCGv_i64 arg,
   2228                          unsigned int ofs, unsigned int len)
   2229 {
   2230     tcg_debug_assert(ofs < 64);
   2231     tcg_debug_assert(len > 0);
   2232     tcg_debug_assert(len <= 64);
   2233     tcg_debug_assert(ofs + len <= 64);
   2234 
   2235     /* Canonicalize certain special cases, even if extract is supported.  */
   2236     if (ofs + len == 64) {
   2237         tcg_gen_shri_i64(ret, arg, 64 - len);
   2238         return;
   2239     }
   2240     if (ofs == 0) {
   2241         tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
   2242         return;
   2243     }
   2244 
   2245     if (TCG_TARGET_REG_BITS == 32) {
   2246         /* Look for a 32-bit extract within one of the two words.  */
   2247         if (ofs >= 32) {
   2248             tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len);
   2249             tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
   2250             return;
   2251         }
   2252         if (ofs + len <= 32) {
   2253             tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
   2254             tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
   2255             return;
   2256         }
   2257         /* The field is split across two words.  One double-word
   2258            shift is better than two double-word shifts.  */
   2259         goto do_shift_and;
   2260     }
   2261 
   2262     if (TCG_TARGET_HAS_extract_i64
   2263         && TCG_TARGET_extract_i64_valid(ofs, len)) {
   2264         tcg_gen_op4ii_i64(INDEX_op_extract_i64, ret, arg, ofs, len);
   2265         return;
   2266     }
   2267 
   2268     /* Assume that zero-extension, if available, is cheaper than a shift.  */
   2269     switch (ofs + len) {
   2270     case 32:
   2271         if (TCG_TARGET_HAS_ext32u_i64) {
   2272             tcg_gen_ext32u_i64(ret, arg);
   2273             tcg_gen_shri_i64(ret, ret, ofs);
   2274             return;
   2275         }
   2276         break;
   2277     case 16:
   2278         if (TCG_TARGET_HAS_ext16u_i64) {
   2279             tcg_gen_ext16u_i64(ret, arg);
   2280             tcg_gen_shri_i64(ret, ret, ofs);
   2281             return;
   2282         }
   2283         break;
   2284     case 8:
   2285         if (TCG_TARGET_HAS_ext8u_i64) {
   2286             tcg_gen_ext8u_i64(ret, arg);
   2287             tcg_gen_shri_i64(ret, ret, ofs);
   2288             return;
   2289         }
   2290         break;
   2291     }
   2292 
   2293     /* ??? Ideally we'd know what values are available for immediate AND.
   2294        Assume that 8 bits are available, plus the special cases of 16 and 32,
   2295        so that we get ext8u, ext16u, and ext32u.  */
   2296     switch (len) {
   2297     case 1 ... 8: case 16: case 32:
   2298     do_shift_and:
   2299         tcg_gen_shri_i64(ret, arg, ofs);
   2300         tcg_gen_andi_i64(ret, ret, (1ull << len) - 1);
   2301         break;
   2302     default:
   2303         tcg_gen_shli_i64(ret, arg, 64 - len - ofs);
   2304         tcg_gen_shri_i64(ret, ret, 64 - len);
   2305         break;
   2306     }
   2307 }
   2308 
   2309 void tcg_gen_sextract_i64(TCGv_i64 ret, TCGv_i64 arg,
   2310                           unsigned int ofs, unsigned int len)
   2311 {
   2312     tcg_debug_assert(ofs < 64);
   2313     tcg_debug_assert(len > 0);
   2314     tcg_debug_assert(len <= 64);
   2315     tcg_debug_assert(ofs + len <= 64);
   2316 
   2317     /* Canonicalize certain special cases, even if sextract is supported.  */
   2318     if (ofs + len == 64) {
   2319         tcg_gen_sari_i64(ret, arg, 64 - len);
   2320         return;
   2321     }
   2322     if (ofs == 0) {
   2323         switch (len) {
   2324         case 32:
   2325             tcg_gen_ext32s_i64(ret, arg);
   2326             return;
   2327         case 16:
   2328             tcg_gen_ext16s_i64(ret, arg);
   2329             return;
   2330         case 8:
   2331             tcg_gen_ext8s_i64(ret, arg);
   2332             return;
   2333         }
   2334     }
   2335 
   2336     if (TCG_TARGET_REG_BITS == 32) {
   2337         /* Look for a 32-bit extract within one of the two words.  */
   2338         if (ofs >= 32) {
   2339             tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len);
   2340         } else if (ofs + len <= 32) {
   2341             tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
   2342         } else if (ofs == 0) {
   2343             tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
   2344             tcg_gen_sextract_i32(TCGV_HIGH(ret), TCGV_HIGH(arg), 0, len - 32);
   2345             return;
   2346         } else if (len > 32) {
   2347             TCGv_i32 t = tcg_temp_new_i32();
   2348             /* Extract the bits for the high word normally.  */
   2349             tcg_gen_sextract_i32(t, TCGV_HIGH(arg), ofs + 32, len - 32);
   2350             /* Shift the field down for the low part.  */
   2351             tcg_gen_shri_i64(ret, arg, ofs);
   2352             /* Overwrite the shift into the high part.  */
   2353             tcg_gen_mov_i32(TCGV_HIGH(ret), t);
   2354             tcg_temp_free_i32(t);
   2355             return;
   2356         } else {
   2357             /* Shift the field down for the low part, such that the
   2358                field sits at the MSB.  */
   2359             tcg_gen_shri_i64(ret, arg, ofs + len - 32);
   2360             /* Shift the field down from the MSB, sign extending.  */
   2361             tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_LOW(ret), 32 - len);
   2362         }
   2363         /* Sign-extend the field from 32 bits.  */
   2364         tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
   2365         return;
   2366     }
   2367 
   2368     if (TCG_TARGET_HAS_sextract_i64
   2369         && TCG_TARGET_extract_i64_valid(ofs, len)) {
   2370         tcg_gen_op4ii_i64(INDEX_op_sextract_i64, ret, arg, ofs, len);
   2371         return;
   2372     }
   2373 
   2374     /* Assume that sign-extension, if available, is cheaper than a shift.  */
   2375     switch (ofs + len) {
   2376     case 32:
   2377         if (TCG_TARGET_HAS_ext32s_i64) {
   2378             tcg_gen_ext32s_i64(ret, arg);
   2379             tcg_gen_sari_i64(ret, ret, ofs);
   2380             return;
   2381         }
   2382         break;
   2383     case 16:
   2384         if (TCG_TARGET_HAS_ext16s_i64) {
   2385             tcg_gen_ext16s_i64(ret, arg);
   2386             tcg_gen_sari_i64(ret, ret, ofs);
   2387             return;
   2388         }
   2389         break;
   2390     case 8:
   2391         if (TCG_TARGET_HAS_ext8s_i64) {
   2392             tcg_gen_ext8s_i64(ret, arg);
   2393             tcg_gen_sari_i64(ret, ret, ofs);
   2394             return;
   2395         }
   2396         break;
   2397     }
   2398     switch (len) {
   2399     case 32:
   2400         if (TCG_TARGET_HAS_ext32s_i64) {
   2401             tcg_gen_shri_i64(ret, arg, ofs);
   2402             tcg_gen_ext32s_i64(ret, ret);
   2403             return;
   2404         }
   2405         break;
   2406     case 16:
   2407         if (TCG_TARGET_HAS_ext16s_i64) {
   2408             tcg_gen_shri_i64(ret, arg, ofs);
   2409             tcg_gen_ext16s_i64(ret, ret);
   2410             return;
   2411         }
   2412         break;
   2413     case 8:
   2414         if (TCG_TARGET_HAS_ext8s_i64) {
   2415             tcg_gen_shri_i64(ret, arg, ofs);
   2416             tcg_gen_ext8s_i64(ret, ret);
   2417             return;
   2418         }
   2419         break;
   2420     }
   2421     tcg_gen_shli_i64(ret, arg, 64 - len - ofs);
   2422     tcg_gen_sari_i64(ret, ret, 64 - len);
   2423 }
   2424 
   2425 /*
   2426  * Extract 64 bits from a 128-bit input, ah:al, starting from ofs.
   2427  * Unlike tcg_gen_extract_i64 above, len is fixed at 64.
   2428  */
   2429 void tcg_gen_extract2_i64(TCGv_i64 ret, TCGv_i64 al, TCGv_i64 ah,
   2430                           unsigned int ofs)
   2431 {
   2432     tcg_debug_assert(ofs <= 64);
   2433     if (ofs == 0) {
   2434         tcg_gen_mov_i64(ret, al);
   2435     } else if (ofs == 64) {
   2436         tcg_gen_mov_i64(ret, ah);
   2437     } else if (al == ah) {
   2438         tcg_gen_rotri_i64(ret, al, ofs);
   2439     } else if (TCG_TARGET_HAS_extract2_i64) {
   2440         tcg_gen_op4i_i64(INDEX_op_extract2_i64, ret, al, ah, ofs);
   2441     } else {
   2442         TCGv_i64 t0 = tcg_temp_new_i64();
   2443         tcg_gen_shri_i64(t0, al, ofs);
   2444         tcg_gen_deposit_i64(ret, t0, ah, 64 - ofs, ofs);
   2445         tcg_temp_free_i64(t0);
   2446     }
   2447 }
   2448 
   2449 void tcg_gen_movcond_i64(TCGCond cond, TCGv_i64 ret, TCGv_i64 c1,
   2450                          TCGv_i64 c2, TCGv_i64 v1, TCGv_i64 v2)
   2451 {
   2452     if (cond == TCG_COND_ALWAYS) {
   2453         tcg_gen_mov_i64(ret, v1);
   2454     } else if (cond == TCG_COND_NEVER) {
   2455         tcg_gen_mov_i64(ret, v2);
   2456     } else if (TCG_TARGET_REG_BITS == 32) {
   2457         TCGv_i32 t0 = tcg_temp_new_i32();
   2458         TCGv_i32 t1 = tcg_temp_new_i32();
   2459         tcg_gen_op6i_i32(INDEX_op_setcond2_i32, t0,
   2460                          TCGV_LOW(c1), TCGV_HIGH(c1),
   2461                          TCGV_LOW(c2), TCGV_HIGH(c2), cond);
   2462 
   2463         if (TCG_TARGET_HAS_movcond_i32) {
   2464             tcg_gen_movi_i32(t1, 0);
   2465             tcg_gen_movcond_i32(TCG_COND_NE, TCGV_LOW(ret), t0, t1,
   2466                                 TCGV_LOW(v1), TCGV_LOW(v2));
   2467             tcg_gen_movcond_i32(TCG_COND_NE, TCGV_HIGH(ret), t0, t1,
   2468                                 TCGV_HIGH(v1), TCGV_HIGH(v2));
   2469         } else {
   2470             tcg_gen_neg_i32(t0, t0);
   2471 
   2472             tcg_gen_and_i32(t1, TCGV_LOW(v1), t0);
   2473             tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(v2), t0);
   2474             tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(ret), t1);
   2475 
   2476             tcg_gen_and_i32(t1, TCGV_HIGH(v1), t0);
   2477             tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(v2), t0);
   2478             tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), t1);
   2479         }
   2480         tcg_temp_free_i32(t0);
   2481         tcg_temp_free_i32(t1);
   2482     } else if (TCG_TARGET_HAS_movcond_i64) {
   2483         tcg_gen_op6i_i64(INDEX_op_movcond_i64, ret, c1, c2, v1, v2, cond);
   2484     } else {
   2485         TCGv_i64 t0 = tcg_temp_new_i64();
   2486         TCGv_i64 t1 = tcg_temp_new_i64();
   2487         tcg_gen_setcond_i64(cond, t0, c1, c2);
   2488         tcg_gen_neg_i64(t0, t0);
   2489         tcg_gen_and_i64(t1, v1, t0);
   2490         tcg_gen_andc_i64(ret, v2, t0);
   2491         tcg_gen_or_i64(ret, ret, t1);
   2492         tcg_temp_free_i64(t0);
   2493         tcg_temp_free_i64(t1);
   2494     }
   2495 }
   2496 
   2497 void tcg_gen_add2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al,
   2498                       TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh)
   2499 {
   2500     if (TCG_TARGET_HAS_add2_i64) {
   2501         tcg_gen_op6_i64(INDEX_op_add2_i64, rl, rh, al, ah, bl, bh);
   2502     } else {
   2503         TCGv_i64 t0 = tcg_temp_new_i64();
   2504         TCGv_i64 t1 = tcg_temp_new_i64();
   2505         tcg_gen_add_i64(t0, al, bl);
   2506         tcg_gen_setcond_i64(TCG_COND_LTU, t1, t0, al);
   2507         tcg_gen_add_i64(rh, ah, bh);
   2508         tcg_gen_add_i64(rh, rh, t1);
   2509         tcg_gen_mov_i64(rl, t0);
   2510         tcg_temp_free_i64(t0);
   2511         tcg_temp_free_i64(t1);
   2512     }
   2513 }
   2514 
   2515 void tcg_gen_sub2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al,
   2516                       TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh)
   2517 {
   2518     if (TCG_TARGET_HAS_sub2_i64) {
   2519         tcg_gen_op6_i64(INDEX_op_sub2_i64, rl, rh, al, ah, bl, bh);
   2520     } else {
   2521         TCGv_i64 t0 = tcg_temp_new_i64();
   2522         TCGv_i64 t1 = tcg_temp_new_i64();
   2523         tcg_gen_sub_i64(t0, al, bl);
   2524         tcg_gen_setcond_i64(TCG_COND_LTU, t1, al, bl);
   2525         tcg_gen_sub_i64(rh, ah, bh);
   2526         tcg_gen_sub_i64(rh, rh, t1);
   2527         tcg_gen_mov_i64(rl, t0);
   2528         tcg_temp_free_i64(t0);
   2529         tcg_temp_free_i64(t1);
   2530     }
   2531 }
   2532 
   2533 void tcg_gen_mulu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
   2534 {
   2535     if (TCG_TARGET_HAS_mulu2_i64) {
   2536         tcg_gen_op4_i64(INDEX_op_mulu2_i64, rl, rh, arg1, arg2);
   2537     } else if (TCG_TARGET_HAS_muluh_i64) {
   2538         TCGv_i64 t = tcg_temp_new_i64();
   2539         tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2);
   2540         tcg_gen_op3_i64(INDEX_op_muluh_i64, rh, arg1, arg2);
   2541         tcg_gen_mov_i64(rl, t);
   2542         tcg_temp_free_i64(t);
   2543     } else {
   2544         TCGv_i64 t0 = tcg_temp_new_i64();
   2545         tcg_gen_mul_i64(t0, arg1, arg2);
   2546         gen_helper_muluh_i64(rh, arg1, arg2);
   2547         tcg_gen_mov_i64(rl, t0);
   2548         tcg_temp_free_i64(t0);
   2549     }
   2550 }
   2551 
   2552 void tcg_gen_muls2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
   2553 {
   2554     if (TCG_TARGET_HAS_muls2_i64) {
   2555         tcg_gen_op4_i64(INDEX_op_muls2_i64, rl, rh, arg1, arg2);
   2556     } else if (TCG_TARGET_HAS_mulsh_i64) {
   2557         TCGv_i64 t = tcg_temp_new_i64();
   2558         tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2);
   2559         tcg_gen_op3_i64(INDEX_op_mulsh_i64, rh, arg1, arg2);
   2560         tcg_gen_mov_i64(rl, t);
   2561         tcg_temp_free_i64(t);
   2562     } else if (TCG_TARGET_HAS_mulu2_i64 || TCG_TARGET_HAS_muluh_i64) {
   2563         TCGv_i64 t0 = tcg_temp_new_i64();
   2564         TCGv_i64 t1 = tcg_temp_new_i64();
   2565         TCGv_i64 t2 = tcg_temp_new_i64();
   2566         TCGv_i64 t3 = tcg_temp_new_i64();
   2567         tcg_gen_mulu2_i64(t0, t1, arg1, arg2);
   2568         /* Adjust for negative inputs.  */
   2569         tcg_gen_sari_i64(t2, arg1, 63);
   2570         tcg_gen_sari_i64(t3, arg2, 63);
   2571         tcg_gen_and_i64(t2, t2, arg2);
   2572         tcg_gen_and_i64(t3, t3, arg1);
   2573         tcg_gen_sub_i64(rh, t1, t2);
   2574         tcg_gen_sub_i64(rh, rh, t3);
   2575         tcg_gen_mov_i64(rl, t0);
   2576         tcg_temp_free_i64(t0);
   2577         tcg_temp_free_i64(t1);
   2578         tcg_temp_free_i64(t2);
   2579         tcg_temp_free_i64(t3);
   2580     } else {
   2581         TCGv_i64 t0 = tcg_temp_new_i64();
   2582         tcg_gen_mul_i64(t0, arg1, arg2);
   2583         gen_helper_mulsh_i64(rh, arg1, arg2);
   2584         tcg_gen_mov_i64(rl, t0);
   2585         tcg_temp_free_i64(t0);
   2586     }
   2587 }
   2588 
   2589 void tcg_gen_mulsu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
   2590 {
   2591     TCGv_i64 t0 = tcg_temp_new_i64();
   2592     TCGv_i64 t1 = tcg_temp_new_i64();
   2593     TCGv_i64 t2 = tcg_temp_new_i64();
   2594     tcg_gen_mulu2_i64(t0, t1, arg1, arg2);
   2595     /* Adjust for negative input for the signed arg1.  */
   2596     tcg_gen_sari_i64(t2, arg1, 63);
   2597     tcg_gen_and_i64(t2, t2, arg2);
   2598     tcg_gen_sub_i64(rh, t1, t2);
   2599     tcg_gen_mov_i64(rl, t0);
   2600     tcg_temp_free_i64(t0);
   2601     tcg_temp_free_i64(t1);
   2602     tcg_temp_free_i64(t2);
   2603 }
   2604 
   2605 void tcg_gen_smin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
   2606 {
   2607     tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, a, b);
   2608 }
   2609 
   2610 void tcg_gen_umin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
   2611 {
   2612     tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, a, b);
   2613 }
   2614 
   2615 void tcg_gen_smax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
   2616 {
   2617     tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, b, a);
   2618 }
   2619 
   2620 void tcg_gen_umax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
   2621 {
   2622     tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, b, a);
   2623 }
   2624 
   2625 void tcg_gen_abs_i64(TCGv_i64 ret, TCGv_i64 a)
   2626 {
   2627     TCGv_i64 t = tcg_temp_new_i64();
   2628 
   2629     tcg_gen_sari_i64(t, a, 63);
   2630     tcg_gen_xor_i64(ret, a, t);
   2631     tcg_gen_sub_i64(ret, ret, t);
   2632     tcg_temp_free_i64(t);
   2633 }
   2634 
   2635 /* Size changing operations.  */
   2636 
   2637 void tcg_gen_extrl_i64_i32(TCGv_i32 ret, TCGv_i64 arg)
   2638 {
   2639     if (TCG_TARGET_REG_BITS == 32) {
   2640         tcg_gen_mov_i32(ret, TCGV_LOW(arg));
   2641     } else if (TCG_TARGET_HAS_extrl_i64_i32) {
   2642         tcg_gen_op2(INDEX_op_extrl_i64_i32,
   2643                     tcgv_i32_arg(ret), tcgv_i64_arg(arg));
   2644     } else {
   2645         tcg_gen_mov_i32(ret, (TCGv_i32)arg);
   2646     }
   2647 }
   2648 
   2649 void tcg_gen_extrh_i64_i32(TCGv_i32 ret, TCGv_i64 arg)
   2650 {
   2651     if (TCG_TARGET_REG_BITS == 32) {
   2652         tcg_gen_mov_i32(ret, TCGV_HIGH(arg));
   2653     } else if (TCG_TARGET_HAS_extrh_i64_i32) {
   2654         tcg_gen_op2(INDEX_op_extrh_i64_i32,
   2655                     tcgv_i32_arg(ret), tcgv_i64_arg(arg));
   2656     } else {
   2657         TCGv_i64 t = tcg_temp_new_i64();
   2658         tcg_gen_shri_i64(t, arg, 32);
   2659         tcg_gen_mov_i32(ret, (TCGv_i32)t);
   2660         tcg_temp_free_i64(t);
   2661     }
   2662 }
   2663 
   2664 void tcg_gen_extu_i32_i64(TCGv_i64 ret, TCGv_i32 arg)
   2665 {
   2666     if (TCG_TARGET_REG_BITS == 32) {
   2667         tcg_gen_mov_i32(TCGV_LOW(ret), arg);
   2668         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
   2669     } else {
   2670         tcg_gen_op2(INDEX_op_extu_i32_i64,
   2671                     tcgv_i64_arg(ret), tcgv_i32_arg(arg));
   2672     }
   2673 }
   2674 
   2675 void tcg_gen_ext_i32_i64(TCGv_i64 ret, TCGv_i32 arg)
   2676 {
   2677     if (TCG_TARGET_REG_BITS == 32) {
   2678         tcg_gen_mov_i32(TCGV_LOW(ret), arg);
   2679         tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
   2680     } else {
   2681         tcg_gen_op2(INDEX_op_ext_i32_i64,
   2682                     tcgv_i64_arg(ret), tcgv_i32_arg(arg));
   2683     }
   2684 }
   2685 
   2686 void tcg_gen_concat_i32_i64(TCGv_i64 dest, TCGv_i32 low, TCGv_i32 high)
   2687 {
   2688     TCGv_i64 tmp;
   2689 
   2690     if (TCG_TARGET_REG_BITS == 32) {
   2691         tcg_gen_mov_i32(TCGV_LOW(dest), low);
   2692         tcg_gen_mov_i32(TCGV_HIGH(dest), high);
   2693         return;
   2694     }
   2695 
   2696     tmp = tcg_temp_new_i64();
   2697     /* These extensions are only needed for type correctness.
   2698        We may be able to do better given target specific information.  */
   2699     tcg_gen_extu_i32_i64(tmp, high);
   2700     tcg_gen_extu_i32_i64(dest, low);
   2701     /* If deposit is available, use it.  Otherwise use the extra
   2702        knowledge that we have of the zero-extensions above.  */
   2703     if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(32, 32)) {
   2704         tcg_gen_deposit_i64(dest, dest, tmp, 32, 32);
   2705     } else {
   2706         tcg_gen_shli_i64(tmp, tmp, 32);
   2707         tcg_gen_or_i64(dest, dest, tmp);
   2708     }
   2709     tcg_temp_free_i64(tmp);
   2710 }
   2711 
   2712 void tcg_gen_extr_i64_i32(TCGv_i32 lo, TCGv_i32 hi, TCGv_i64 arg)
   2713 {
   2714     if (TCG_TARGET_REG_BITS == 32) {
   2715         tcg_gen_mov_i32(lo, TCGV_LOW(arg));
   2716         tcg_gen_mov_i32(hi, TCGV_HIGH(arg));
   2717     } else {
   2718         tcg_gen_extrl_i64_i32(lo, arg);
   2719         tcg_gen_extrh_i64_i32(hi, arg);
   2720     }
   2721 }
   2722 
   2723 void tcg_gen_extr32_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i64 arg)
   2724 {
   2725     tcg_gen_ext32u_i64(lo, arg);
   2726     tcg_gen_shri_i64(hi, arg, 32);
   2727 }
   2728 
   2729 /* QEMU specific operations.  */
   2730 
   2731 void tcg_gen_exit_tb(const TranslationBlock *tb, unsigned idx)
   2732 {
   2733     /*
   2734      * Let the jit code return the read-only version of the
   2735      * TranslationBlock, so that we minimize the pc-relative
   2736      * distance of the address of the exit_tb code to TB.
   2737      * This will improve utilization of pc-relative address loads.
   2738      *
   2739      * TODO: Move this to translator_loop, so that all const
   2740      * TranslationBlock pointers refer to read-only memory.
   2741      * This requires coordination with targets that do not use
   2742      * the translator_loop.
   2743      */
   2744     uintptr_t val = (uintptr_t)tcg_splitwx_to_rx((void *)tb) + idx;
   2745 
   2746     if (tb == NULL) {
   2747         tcg_debug_assert(idx == 0);
   2748     } else if (idx <= TB_EXIT_IDXMAX) {
   2749 #ifdef CONFIG_DEBUG_TCG
   2750         /* This is an exit following a goto_tb.  Verify that we have
   2751            seen this numbered exit before, via tcg_gen_goto_tb.  */
   2752         tcg_debug_assert(tcg_ctx->goto_tb_issue_mask & (1 << idx));
   2753 #endif
   2754     } else {
   2755         /* This is an exit via the exitreq label.  */
   2756         tcg_debug_assert(idx == TB_EXIT_REQUESTED);
   2757     }
   2758 
   2759     plugin_gen_disable_mem_helpers();
   2760     tcg_gen_op1i(INDEX_op_exit_tb, val);
   2761 }
   2762 
   2763 void tcg_gen_goto_tb(unsigned idx)
   2764 {
   2765     /* We tested CF_NO_GOTO_TB in translator_use_goto_tb. */
   2766     tcg_debug_assert(!(tcg_ctx->tb_cflags & CF_NO_GOTO_TB));
   2767     /* We only support two chained exits.  */
   2768     tcg_debug_assert(idx <= TB_EXIT_IDXMAX);
   2769 #ifdef CONFIG_DEBUG_TCG
   2770     /* Verify that we haven't seen this numbered exit before.  */
   2771     tcg_debug_assert((tcg_ctx->goto_tb_issue_mask & (1 << idx)) == 0);
   2772     tcg_ctx->goto_tb_issue_mask |= 1 << idx;
   2773 #endif
   2774     plugin_gen_disable_mem_helpers();
   2775     tcg_gen_op1i(INDEX_op_goto_tb, idx);
   2776 }
   2777 
   2778 void tcg_gen_lookup_and_goto_ptr(void)
   2779 {
   2780     TCGv_ptr ptr;
   2781 
   2782     if (tcg_ctx->tb_cflags & CF_NO_GOTO_PTR) {
   2783         tcg_gen_exit_tb(NULL, 0);
   2784         return;
   2785     }
   2786 
   2787     plugin_gen_disable_mem_helpers();
   2788     ptr = tcg_temp_new_ptr();
   2789     gen_helper_lookup_tb_ptr(ptr, cpu_env);
   2790     tcg_gen_op1i(INDEX_op_goto_ptr, tcgv_ptr_arg(ptr));
   2791     tcg_temp_free_ptr(ptr);
   2792 }
   2793 
   2794 static inline MemOp tcg_canonicalize_memop(MemOp op, bool is64, bool st)
   2795 {
   2796     /* Trigger the asserts within as early as possible.  */
   2797     unsigned a_bits = get_alignment_bits(op);
   2798 
   2799     /* Prefer MO_ALIGN+MO_XX over MO_ALIGN_XX+MO_XX */
   2800     if (a_bits == (op & MO_SIZE)) {
   2801         op = (op & ~MO_AMASK) | MO_ALIGN;
   2802     }
   2803 
   2804     switch (op & MO_SIZE) {
   2805     case MO_8:
   2806         op &= ~MO_BSWAP;
   2807         break;
   2808     case MO_16:
   2809         break;
   2810     case MO_32:
   2811         if (!is64) {
   2812             op &= ~MO_SIGN;
   2813         }
   2814         break;
   2815     case MO_64:
   2816         if (is64) {
   2817             op &= ~MO_SIGN;
   2818             break;
   2819         }
   2820         /* fall through */
   2821     default:
   2822         g_assert_not_reached();
   2823     }
   2824     if (st) {
   2825         op &= ~MO_SIGN;
   2826     }
   2827     return op;
   2828 }
   2829 
   2830 static void gen_ldst_i32(TCGOpcode opc, TCGv_i32 val, TCGv addr,
   2831                          MemOp memop, TCGArg idx)
   2832 {
   2833     MemOpIdx oi = make_memop_idx(memop, idx);
   2834 #if TARGET_LONG_BITS == 32
   2835     tcg_gen_op3i_i32(opc, val, addr, oi);
   2836 #else
   2837     if (TCG_TARGET_REG_BITS == 32) {
   2838         tcg_gen_op4i_i32(opc, val, TCGV_LOW(addr), TCGV_HIGH(addr), oi);
   2839     } else {
   2840         tcg_gen_op3(opc, tcgv_i32_arg(val), tcgv_i64_arg(addr), oi);
   2841     }
   2842 #endif
   2843 }
   2844 
   2845 static void gen_ldst_i64(TCGOpcode opc, TCGv_i64 val, TCGv addr,
   2846                          MemOp memop, TCGArg idx)
   2847 {
   2848     MemOpIdx oi = make_memop_idx(memop, idx);
   2849 #if TARGET_LONG_BITS == 32
   2850     if (TCG_TARGET_REG_BITS == 32) {
   2851         tcg_gen_op4i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val), addr, oi);
   2852     } else {
   2853         tcg_gen_op3(opc, tcgv_i64_arg(val), tcgv_i32_arg(addr), oi);
   2854     }
   2855 #else
   2856     if (TCG_TARGET_REG_BITS == 32) {
   2857         tcg_gen_op5i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val),
   2858                          TCGV_LOW(addr), TCGV_HIGH(addr), oi);
   2859     } else {
   2860         tcg_gen_op3i_i64(opc, val, addr, oi);
   2861     }
   2862 #endif
   2863 }
   2864 
   2865 static void tcg_gen_req_mo(TCGBar type)
   2866 {
   2867 #ifdef TCG_GUEST_DEFAULT_MO
   2868     type &= TCG_GUEST_DEFAULT_MO;
   2869 #endif
   2870     type &= ~TCG_TARGET_DEFAULT_MO;
   2871     if (type) {
   2872         tcg_gen_mb(type | TCG_BAR_SC);
   2873     }
   2874 }
   2875 
   2876 static inline TCGv plugin_prep_mem_callbacks(TCGv vaddr)
   2877 {
   2878 #ifdef CONFIG_PLUGIN
   2879     if (tcg_ctx->plugin_insn != NULL) {
   2880         /* Save a copy of the vaddr for use after a load.  */
   2881         TCGv temp = tcg_temp_new();
   2882         tcg_gen_mov_tl(temp, vaddr);
   2883         return temp;
   2884     }
   2885 #endif
   2886     return vaddr;
   2887 }
   2888 
   2889 static void plugin_gen_mem_callbacks(TCGv vaddr, MemOpIdx oi,
   2890                                      enum qemu_plugin_mem_rw rw)
   2891 {
   2892 #ifdef CONFIG_PLUGIN
   2893     if (tcg_ctx->plugin_insn != NULL) {
   2894         qemu_plugin_meminfo_t info = make_plugin_meminfo(oi, rw);
   2895         plugin_gen_empty_mem_callback(vaddr, info);
   2896         tcg_temp_free(vaddr);
   2897     }
   2898 #endif
   2899 }
   2900 
   2901 void tcg_gen_qemu_ld_i32(TCGv_i32 val, TCGv addr, TCGArg idx, MemOp memop)
   2902 {
   2903     MemOp orig_memop;
   2904     MemOpIdx oi;
   2905 
   2906     tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
   2907     memop = tcg_canonicalize_memop(memop, 0, 0);
   2908     oi = make_memop_idx(memop, idx);
   2909 
   2910     orig_memop = memop;
   2911     if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) {
   2912         memop &= ~MO_BSWAP;
   2913         /* The bswap primitive benefits from zero-extended input.  */
   2914         if ((memop & MO_SSIZE) == MO_SW) {
   2915             memop &= ~MO_SIGN;
   2916         }
   2917     }
   2918 
   2919     addr = plugin_prep_mem_callbacks(addr);
   2920     gen_ldst_i32(INDEX_op_qemu_ld_i32, val, addr, memop, idx);
   2921     plugin_gen_mem_callbacks(addr, oi, QEMU_PLUGIN_MEM_R);
   2922 
   2923     if ((orig_memop ^ memop) & MO_BSWAP) {
   2924         switch (orig_memop & MO_SIZE) {
   2925         case MO_16:
   2926             tcg_gen_bswap16_i32(val, val, (orig_memop & MO_SIGN
   2927                                            ? TCG_BSWAP_IZ | TCG_BSWAP_OS
   2928                                            : TCG_BSWAP_IZ | TCG_BSWAP_OZ));
   2929             break;
   2930         case MO_32:
   2931             tcg_gen_bswap32_i32(val, val);
   2932             break;
   2933         default:
   2934             g_assert_not_reached();
   2935         }
   2936     }
   2937 }
   2938 
   2939 void tcg_gen_qemu_st_i32(TCGv_i32 val, TCGv addr, TCGArg idx, MemOp memop)
   2940 {
   2941     TCGv_i32 swap = NULL;
   2942     MemOpIdx oi;
   2943 
   2944     tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
   2945     memop = tcg_canonicalize_memop(memop, 0, 1);
   2946     oi = make_memop_idx(memop, idx);
   2947 
   2948     if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) {
   2949         swap = tcg_temp_new_i32();
   2950         switch (memop & MO_SIZE) {
   2951         case MO_16:
   2952             tcg_gen_bswap16_i32(swap, val, 0);
   2953             break;
   2954         case MO_32:
   2955             tcg_gen_bswap32_i32(swap, val);
   2956             break;
   2957         default:
   2958             g_assert_not_reached();
   2959         }
   2960         val = swap;
   2961         memop &= ~MO_BSWAP;
   2962     }
   2963 
   2964     addr = plugin_prep_mem_callbacks(addr);
   2965     if (TCG_TARGET_HAS_qemu_st8_i32 && (memop & MO_SIZE) == MO_8) {
   2966         gen_ldst_i32(INDEX_op_qemu_st8_i32, val, addr, memop, idx);
   2967     } else {
   2968         gen_ldst_i32(INDEX_op_qemu_st_i32, val, addr, memop, idx);
   2969     }
   2970     plugin_gen_mem_callbacks(addr, oi, QEMU_PLUGIN_MEM_W);
   2971 
   2972     if (swap) {
   2973         tcg_temp_free_i32(swap);
   2974     }
   2975 }
   2976 
   2977 void tcg_gen_qemu_ld_i64(TCGv_i64 val, TCGv addr, TCGArg idx, MemOp memop)
   2978 {
   2979     MemOp orig_memop;
   2980     MemOpIdx oi;
   2981 
   2982     if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) {
   2983         tcg_gen_qemu_ld_i32(TCGV_LOW(val), addr, idx, memop);
   2984         if (memop & MO_SIGN) {
   2985             tcg_gen_sari_i32(TCGV_HIGH(val), TCGV_LOW(val), 31);
   2986         } else {
   2987             tcg_gen_movi_i32(TCGV_HIGH(val), 0);
   2988         }
   2989         return;
   2990     }
   2991 
   2992     tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
   2993     memop = tcg_canonicalize_memop(memop, 1, 0);
   2994     oi = make_memop_idx(memop, idx);
   2995 
   2996     orig_memop = memop;
   2997     if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) {
   2998         memop &= ~MO_BSWAP;
   2999         /* The bswap primitive benefits from zero-extended input.  */
   3000         if ((memop & MO_SIGN) && (memop & MO_SIZE) < MO_64) {
   3001             memop &= ~MO_SIGN;
   3002         }
   3003     }
   3004 
   3005     addr = plugin_prep_mem_callbacks(addr);
   3006     gen_ldst_i64(INDEX_op_qemu_ld_i64, val, addr, memop, idx);
   3007     plugin_gen_mem_callbacks(addr, oi, QEMU_PLUGIN_MEM_R);
   3008 
   3009     if ((orig_memop ^ memop) & MO_BSWAP) {
   3010         int flags = (orig_memop & MO_SIGN
   3011                      ? TCG_BSWAP_IZ | TCG_BSWAP_OS
   3012                      : TCG_BSWAP_IZ | TCG_BSWAP_OZ);
   3013         switch (orig_memop & MO_SIZE) {
   3014         case MO_16:
   3015             tcg_gen_bswap16_i64(val, val, flags);
   3016             break;
   3017         case MO_32:
   3018             tcg_gen_bswap32_i64(val, val, flags);
   3019             break;
   3020         case MO_64:
   3021             tcg_gen_bswap64_i64(val, val);
   3022             break;
   3023         default:
   3024             g_assert_not_reached();
   3025         }
   3026     }
   3027 }
   3028 
   3029 void tcg_gen_qemu_st_i64(TCGv_i64 val, TCGv addr, TCGArg idx, MemOp memop)
   3030 {
   3031     TCGv_i64 swap = NULL;
   3032     MemOpIdx oi;
   3033 
   3034     if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) {
   3035         tcg_gen_qemu_st_i32(TCGV_LOW(val), addr, idx, memop);
   3036         return;
   3037     }
   3038 
   3039     tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
   3040     memop = tcg_canonicalize_memop(memop, 1, 1);
   3041     oi = make_memop_idx(memop, idx);
   3042 
   3043     if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) {
   3044         swap = tcg_temp_new_i64();
   3045         switch (memop & MO_SIZE) {
   3046         case MO_16:
   3047             tcg_gen_bswap16_i64(swap, val, 0);
   3048             break;
   3049         case MO_32:
   3050             tcg_gen_bswap32_i64(swap, val, 0);
   3051             break;
   3052         case MO_64:
   3053             tcg_gen_bswap64_i64(swap, val);
   3054             break;
   3055         default:
   3056             g_assert_not_reached();
   3057         }
   3058         val = swap;
   3059         memop &= ~MO_BSWAP;
   3060     }
   3061 
   3062     addr = plugin_prep_mem_callbacks(addr);
   3063     gen_ldst_i64(INDEX_op_qemu_st_i64, val, addr, memop, idx);
   3064     plugin_gen_mem_callbacks(addr, oi, QEMU_PLUGIN_MEM_W);
   3065 
   3066     if (swap) {
   3067         tcg_temp_free_i64(swap);
   3068     }
   3069 }
   3070 
   3071 static void tcg_gen_ext_i32(TCGv_i32 ret, TCGv_i32 val, MemOp opc)
   3072 {
   3073     switch (opc & MO_SSIZE) {
   3074     case MO_SB:
   3075         tcg_gen_ext8s_i32(ret, val);
   3076         break;
   3077     case MO_UB:
   3078         tcg_gen_ext8u_i32(ret, val);
   3079         break;
   3080     case MO_SW:
   3081         tcg_gen_ext16s_i32(ret, val);
   3082         break;
   3083     case MO_UW:
   3084         tcg_gen_ext16u_i32(ret, val);
   3085         break;
   3086     default:
   3087         tcg_gen_mov_i32(ret, val);
   3088         break;
   3089     }
   3090 }
   3091 
   3092 static void tcg_gen_ext_i64(TCGv_i64 ret, TCGv_i64 val, MemOp opc)
   3093 {
   3094     switch (opc & MO_SSIZE) {
   3095     case MO_SB:
   3096         tcg_gen_ext8s_i64(ret, val);
   3097         break;
   3098     case MO_UB:
   3099         tcg_gen_ext8u_i64(ret, val);
   3100         break;
   3101     case MO_SW:
   3102         tcg_gen_ext16s_i64(ret, val);
   3103         break;
   3104     case MO_UW:
   3105         tcg_gen_ext16u_i64(ret, val);
   3106         break;
   3107     case MO_SL:
   3108         tcg_gen_ext32s_i64(ret, val);
   3109         break;
   3110     case MO_UL:
   3111         tcg_gen_ext32u_i64(ret, val);
   3112         break;
   3113     default:
   3114         tcg_gen_mov_i64(ret, val);
   3115         break;
   3116     }
   3117 }
   3118 
   3119 typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv,
   3120                                   TCGv_i32, TCGv_i32, TCGv_i32);
   3121 typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv,
   3122                                   TCGv_i64, TCGv_i64, TCGv_i32);
   3123 typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv,
   3124                                   TCGv_i32, TCGv_i32);
   3125 typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv,
   3126                                   TCGv_i64, TCGv_i32);
   3127 
   3128 #ifdef CONFIG_ATOMIC64
   3129 # define WITH_ATOMIC64(X) X,
   3130 #else
   3131 # define WITH_ATOMIC64(X)
   3132 #endif
   3133 
   3134 static void * const table_cmpxchg[(MO_SIZE | MO_BSWAP) + 1] = {
   3135     [MO_8] = gen_helper_atomic_cmpxchgb,
   3136     [MO_16 | MO_LE] = gen_helper_atomic_cmpxchgw_le,
   3137     [MO_16 | MO_BE] = gen_helper_atomic_cmpxchgw_be,
   3138     [MO_32 | MO_LE] = gen_helper_atomic_cmpxchgl_le,
   3139     [MO_32 | MO_BE] = gen_helper_atomic_cmpxchgl_be,
   3140     WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_cmpxchgq_le)
   3141     WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_cmpxchgq_be)
   3142 };
   3143 
   3144 void tcg_gen_atomic_cmpxchg_i32(TCGv_i32 retv, TCGv addr, TCGv_i32 cmpv,
   3145                                 TCGv_i32 newv, TCGArg idx, MemOp memop)
   3146 {
   3147     memop = tcg_canonicalize_memop(memop, 0, 0);
   3148 
   3149     if (!(tcg_ctx->tb_cflags & CF_PARALLEL)) {
   3150         TCGv_i32 t1 = tcg_temp_new_i32();
   3151         TCGv_i32 t2 = tcg_temp_new_i32();
   3152 
   3153         tcg_gen_ext_i32(t2, cmpv, memop & MO_SIZE);
   3154 
   3155         tcg_gen_qemu_ld_i32(t1, addr, idx, memop & ~MO_SIGN);
   3156         tcg_gen_movcond_i32(TCG_COND_EQ, t2, t1, t2, newv, t1);
   3157         tcg_gen_qemu_st_i32(t2, addr, idx, memop);
   3158         tcg_temp_free_i32(t2);
   3159 
   3160         if (memop & MO_SIGN) {
   3161             tcg_gen_ext_i32(retv, t1, memop);
   3162         } else {
   3163             tcg_gen_mov_i32(retv, t1);
   3164         }
   3165         tcg_temp_free_i32(t1);
   3166     } else {
   3167         gen_atomic_cx_i32 gen;
   3168         MemOpIdx oi;
   3169 
   3170         gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)];
   3171         tcg_debug_assert(gen != NULL);
   3172 
   3173         oi = make_memop_idx(memop & ~MO_SIGN, idx);
   3174         gen(retv, cpu_env, addr, cmpv, newv, tcg_constant_i32(oi));
   3175 
   3176         if (memop & MO_SIGN) {
   3177             tcg_gen_ext_i32(retv, retv, memop);
   3178         }
   3179     }
   3180 }
   3181 
   3182 void tcg_gen_atomic_cmpxchg_i64(TCGv_i64 retv, TCGv addr, TCGv_i64 cmpv,
   3183                                 TCGv_i64 newv, TCGArg idx, MemOp memop)
   3184 {
   3185     memop = tcg_canonicalize_memop(memop, 1, 0);
   3186 
   3187     if (!(tcg_ctx->tb_cflags & CF_PARALLEL)) {
   3188         TCGv_i64 t1 = tcg_temp_new_i64();
   3189         TCGv_i64 t2 = tcg_temp_new_i64();
   3190 
   3191         tcg_gen_ext_i64(t2, cmpv, memop & MO_SIZE);
   3192 
   3193         tcg_gen_qemu_ld_i64(t1, addr, idx, memop & ~MO_SIGN);
   3194         tcg_gen_movcond_i64(TCG_COND_EQ, t2, t1, t2, newv, t1);
   3195         tcg_gen_qemu_st_i64(t2, addr, idx, memop);
   3196         tcg_temp_free_i64(t2);
   3197 
   3198         if (memop & MO_SIGN) {
   3199             tcg_gen_ext_i64(retv, t1, memop);
   3200         } else {
   3201             tcg_gen_mov_i64(retv, t1);
   3202         }
   3203         tcg_temp_free_i64(t1);
   3204     } else if ((memop & MO_SIZE) == MO_64) {
   3205 #ifdef CONFIG_ATOMIC64
   3206         gen_atomic_cx_i64 gen;
   3207         MemOpIdx oi;
   3208 
   3209         gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)];
   3210         tcg_debug_assert(gen != NULL);
   3211 
   3212         oi = make_memop_idx(memop, idx);
   3213         gen(retv, cpu_env, addr, cmpv, newv, tcg_constant_i32(oi));
   3214 #else
   3215         gen_helper_exit_atomic(cpu_env);
   3216         /* Produce a result, so that we have a well-formed opcode stream
   3217            with respect to uses of the result in the (dead) code following.  */
   3218         tcg_gen_movi_i64(retv, 0);
   3219 #endif /* CONFIG_ATOMIC64 */
   3220     } else {
   3221         TCGv_i32 c32 = tcg_temp_new_i32();
   3222         TCGv_i32 n32 = tcg_temp_new_i32();
   3223         TCGv_i32 r32 = tcg_temp_new_i32();
   3224 
   3225         tcg_gen_extrl_i64_i32(c32, cmpv);
   3226         tcg_gen_extrl_i64_i32(n32, newv);
   3227         tcg_gen_atomic_cmpxchg_i32(r32, addr, c32, n32, idx, memop & ~MO_SIGN);
   3228         tcg_temp_free_i32(c32);
   3229         tcg_temp_free_i32(n32);
   3230 
   3231         tcg_gen_extu_i32_i64(retv, r32);
   3232         tcg_temp_free_i32(r32);
   3233 
   3234         if (memop & MO_SIGN) {
   3235             tcg_gen_ext_i64(retv, retv, memop);
   3236         }
   3237     }
   3238 }
   3239 
   3240 static void do_nonatomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val,
   3241                                 TCGArg idx, MemOp memop, bool new_val,
   3242                                 void (*gen)(TCGv_i32, TCGv_i32, TCGv_i32))
   3243 {
   3244     TCGv_i32 t1 = tcg_temp_new_i32();
   3245     TCGv_i32 t2 = tcg_temp_new_i32();
   3246 
   3247     memop = tcg_canonicalize_memop(memop, 0, 0);
   3248 
   3249     tcg_gen_qemu_ld_i32(t1, addr, idx, memop);
   3250     tcg_gen_ext_i32(t2, val, memop);
   3251     gen(t2, t1, t2);
   3252     tcg_gen_qemu_st_i32(t2, addr, idx, memop);
   3253 
   3254     tcg_gen_ext_i32(ret, (new_val ? t2 : t1), memop);
   3255     tcg_temp_free_i32(t1);
   3256     tcg_temp_free_i32(t2);
   3257 }
   3258 
   3259 static void do_atomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val,
   3260                              TCGArg idx, MemOp memop, void * const table[])
   3261 {
   3262     gen_atomic_op_i32 gen;
   3263     MemOpIdx oi;
   3264 
   3265     memop = tcg_canonicalize_memop(memop, 0, 0);
   3266 
   3267     gen = table[memop & (MO_SIZE | MO_BSWAP)];
   3268     tcg_debug_assert(gen != NULL);
   3269 
   3270     oi = make_memop_idx(memop & ~MO_SIGN, idx);
   3271     gen(ret, cpu_env, addr, val, tcg_constant_i32(oi));
   3272 
   3273     if (memop & MO_SIGN) {
   3274         tcg_gen_ext_i32(ret, ret, memop);
   3275     }
   3276 }
   3277 
   3278 static void do_nonatomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val,
   3279                                 TCGArg idx, MemOp memop, bool new_val,
   3280                                 void (*gen)(TCGv_i64, TCGv_i64, TCGv_i64))
   3281 {
   3282     TCGv_i64 t1 = tcg_temp_new_i64();
   3283     TCGv_i64 t2 = tcg_temp_new_i64();
   3284 
   3285     memop = tcg_canonicalize_memop(memop, 1, 0);
   3286 
   3287     tcg_gen_qemu_ld_i64(t1, addr, idx, memop);
   3288     tcg_gen_ext_i64(t2, val, memop);
   3289     gen(t2, t1, t2);
   3290     tcg_gen_qemu_st_i64(t2, addr, idx, memop);
   3291 
   3292     tcg_gen_ext_i64(ret, (new_val ? t2 : t1), memop);
   3293     tcg_temp_free_i64(t1);
   3294     tcg_temp_free_i64(t2);
   3295 }
   3296 
   3297 static void do_atomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val,
   3298                              TCGArg idx, MemOp memop, void * const table[])
   3299 {
   3300     memop = tcg_canonicalize_memop(memop, 1, 0);
   3301 
   3302     if ((memop & MO_SIZE) == MO_64) {
   3303 #ifdef CONFIG_ATOMIC64
   3304         gen_atomic_op_i64 gen;
   3305         MemOpIdx oi;
   3306 
   3307         gen = table[memop & (MO_SIZE | MO_BSWAP)];
   3308         tcg_debug_assert(gen != NULL);
   3309 
   3310         oi = make_memop_idx(memop & ~MO_SIGN, idx);
   3311         gen(ret, cpu_env, addr, val, tcg_constant_i32(oi));
   3312 #else
   3313         gen_helper_exit_atomic(cpu_env);
   3314         /* Produce a result, so that we have a well-formed opcode stream
   3315            with respect to uses of the result in the (dead) code following.  */
   3316         tcg_gen_movi_i64(ret, 0);
   3317 #endif /* CONFIG_ATOMIC64 */
   3318     } else {
   3319         TCGv_i32 v32 = tcg_temp_new_i32();
   3320         TCGv_i32 r32 = tcg_temp_new_i32();
   3321 
   3322         tcg_gen_extrl_i64_i32(v32, val);
   3323         do_atomic_op_i32(r32, addr, v32, idx, memop & ~MO_SIGN, table);
   3324         tcg_temp_free_i32(v32);
   3325 
   3326         tcg_gen_extu_i32_i64(ret, r32);
   3327         tcg_temp_free_i32(r32);
   3328 
   3329         if (memop & MO_SIGN) {
   3330             tcg_gen_ext_i64(ret, ret, memop);
   3331         }
   3332     }
   3333 }
   3334 
   3335 #define GEN_ATOMIC_HELPER(NAME, OP, NEW)                                \
   3336 static void * const table_##NAME[(MO_SIZE | MO_BSWAP) + 1] = {          \
   3337     [MO_8] = gen_helper_atomic_##NAME##b,                               \
   3338     [MO_16 | MO_LE] = gen_helper_atomic_##NAME##w_le,                   \
   3339     [MO_16 | MO_BE] = gen_helper_atomic_##NAME##w_be,                   \
   3340     [MO_32 | MO_LE] = gen_helper_atomic_##NAME##l_le,                   \
   3341     [MO_32 | MO_BE] = gen_helper_atomic_##NAME##l_be,                   \
   3342     WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_##NAME##q_le)     \
   3343     WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_##NAME##q_be)     \
   3344 };                                                                      \
   3345 void tcg_gen_atomic_##NAME##_i32                                        \
   3346     (TCGv_i32 ret, TCGv addr, TCGv_i32 val, TCGArg idx, MemOp memop)    \
   3347 {                                                                       \
   3348     if (tcg_ctx->tb_cflags & CF_PARALLEL) {                             \
   3349         do_atomic_op_i32(ret, addr, val, idx, memop, table_##NAME);     \
   3350     } else {                                                            \
   3351         do_nonatomic_op_i32(ret, addr, val, idx, memop, NEW,            \
   3352                             tcg_gen_##OP##_i32);                        \
   3353     }                                                                   \
   3354 }                                                                       \
   3355 void tcg_gen_atomic_##NAME##_i64                                        \
   3356     (TCGv_i64 ret, TCGv addr, TCGv_i64 val, TCGArg idx, MemOp memop)    \
   3357 {                                                                       \
   3358     if (tcg_ctx->tb_cflags & CF_PARALLEL) {                             \
   3359         do_atomic_op_i64(ret, addr, val, idx, memop, table_##NAME);     \
   3360     } else {                                                            \
   3361         do_nonatomic_op_i64(ret, addr, val, idx, memop, NEW,            \
   3362                             tcg_gen_##OP##_i64);                        \
   3363     }                                                                   \
   3364 }
   3365 
   3366 GEN_ATOMIC_HELPER(fetch_add, add, 0)
   3367 GEN_ATOMIC_HELPER(fetch_and, and, 0)
   3368 GEN_ATOMIC_HELPER(fetch_or, or, 0)
   3369 GEN_ATOMIC_HELPER(fetch_xor, xor, 0)
   3370 GEN_ATOMIC_HELPER(fetch_smin, smin, 0)
   3371 GEN_ATOMIC_HELPER(fetch_umin, umin, 0)
   3372 GEN_ATOMIC_HELPER(fetch_smax, smax, 0)
   3373 GEN_ATOMIC_HELPER(fetch_umax, umax, 0)
   3374 
   3375 GEN_ATOMIC_HELPER(add_fetch, add, 1)
   3376 GEN_ATOMIC_HELPER(and_fetch, and, 1)
   3377 GEN_ATOMIC_HELPER(or_fetch, or, 1)
   3378 GEN_ATOMIC_HELPER(xor_fetch, xor, 1)
   3379 GEN_ATOMIC_HELPER(smin_fetch, smin, 1)
   3380 GEN_ATOMIC_HELPER(umin_fetch, umin, 1)
   3381 GEN_ATOMIC_HELPER(smax_fetch, smax, 1)
   3382 GEN_ATOMIC_HELPER(umax_fetch, umax, 1)
   3383 
   3384 static void tcg_gen_mov2_i32(TCGv_i32 r, TCGv_i32 a, TCGv_i32 b)
   3385 {
   3386     tcg_gen_mov_i32(r, b);
   3387 }
   3388 
   3389 static void tcg_gen_mov2_i64(TCGv_i64 r, TCGv_i64 a, TCGv_i64 b)
   3390 {
   3391     tcg_gen_mov_i64(r, b);
   3392 }
   3393 
   3394 GEN_ATOMIC_HELPER(xchg, mov2, 0)
   3395 
   3396 #undef GEN_ATOMIC_HELPER
	qemu FORK: QEMU emulator
	git clone https://git.neptards.moe/neptards/qemu.git
	Log \| Files \| Refs \| Submodules \| LICENSE