emit.c.inc (80977B)
1 /* 2 * New-style TCG opcode generator for i386 instructions 3 * 4 * Copyright (c) 2022 Red Hat, Inc. 5 * 6 * Author: Paolo Bonzini <pbonzini@redhat.com> 7 * 8 * This library is free software; you can redistribute it and/or 9 * modify it under the terms of the GNU Lesser General Public 10 * License as published by the Free Software Foundation; either 11 * version 2.1 of the License, or (at your option) any later version. 12 * 13 * This library is distributed in the hope that it will be useful, 14 * but WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 16 * Lesser General Public License for more details. 17 * 18 * You should have received a copy of the GNU Lesser General Public 19 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 20 */ 21 22 #define ZMM_OFFSET(reg) offsetof(CPUX86State, xmm_regs[reg]) 23 24 typedef void (*SSEFunc_i_ep)(TCGv_i32 val, TCGv_ptr env, TCGv_ptr reg); 25 typedef void (*SSEFunc_l_ep)(TCGv_i64 val, TCGv_ptr env, TCGv_ptr reg); 26 typedef void (*SSEFunc_0_epp)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b); 27 typedef void (*SSEFunc_0_eppp)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 28 TCGv_ptr reg_c); 29 typedef void (*SSEFunc_0_epppp)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 30 TCGv_ptr reg_c, TCGv_ptr reg_d); 31 typedef void (*SSEFunc_0_eppi)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 32 TCGv_i32 val); 33 typedef void (*SSEFunc_0_epppi)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 34 TCGv_ptr reg_c, TCGv_i32 val); 35 typedef void (*SSEFunc_0_ppi)(TCGv_ptr reg_a, TCGv_ptr reg_b, TCGv_i32 val); 36 typedef void (*SSEFunc_0_pppi)(TCGv_ptr reg_a, TCGv_ptr reg_b, TCGv_ptr reg_c, 37 TCGv_i32 val); 38 typedef void (*SSEFunc_0_eppt)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 39 TCGv val); 40 typedef void (*SSEFunc_0_epppti)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 41 TCGv_ptr reg_c, TCGv a0, TCGv_i32 scale); 42 typedef void (*SSEFunc_0_eppppi)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 43 TCGv_ptr reg_c, TCGv_ptr reg_d, TCGv_i32 flags); 44 typedef void (*SSEFunc_0_eppppii)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 45 TCGv_ptr reg_c, TCGv_ptr reg_d, TCGv_i32 even, 46 TCGv_i32 odd); 47 48 static inline TCGv_i32 tcg_constant8u_i32(uint8_t val) 49 { 50 return tcg_constant_i32(val); 51 } 52 53 static void gen_NM_exception(DisasContext *s) 54 { 55 gen_exception(s, EXCP07_PREX); 56 } 57 58 static void gen_illegal(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 59 { 60 gen_illegal_opcode(s); 61 } 62 63 static void gen_load_ea(DisasContext *s, AddressParts *mem, bool is_vsib) 64 { 65 TCGv ea = gen_lea_modrm_1(s, *mem, is_vsib); 66 gen_lea_v_seg(s, s->aflag, ea, mem->def_seg, s->override); 67 } 68 69 static inline int mmx_offset(MemOp ot) 70 { 71 switch (ot) { 72 case MO_8: 73 return offsetof(MMXReg, MMX_B(0)); 74 case MO_16: 75 return offsetof(MMXReg, MMX_W(0)); 76 case MO_32: 77 return offsetof(MMXReg, MMX_L(0)); 78 case MO_64: 79 return offsetof(MMXReg, MMX_Q(0)); 80 default: 81 g_assert_not_reached(); 82 } 83 } 84 85 static inline int xmm_offset(MemOp ot) 86 { 87 switch (ot) { 88 case MO_8: 89 return offsetof(ZMMReg, ZMM_B(0)); 90 case MO_16: 91 return offsetof(ZMMReg, ZMM_W(0)); 92 case MO_32: 93 return offsetof(ZMMReg, ZMM_L(0)); 94 case MO_64: 95 return offsetof(ZMMReg, ZMM_Q(0)); 96 case MO_128: 97 return offsetof(ZMMReg, ZMM_X(0)); 98 case MO_256: 99 return offsetof(ZMMReg, ZMM_Y(0)); 100 default: 101 g_assert_not_reached(); 102 } 103 } 104 105 static int vector_reg_offset(X86DecodedOp *op) 106 { 107 assert(op->unit == X86_OP_MMX || op->unit == X86_OP_SSE); 108 109 if (op->unit == X86_OP_MMX) { 110 return op->offset - mmx_offset(op->ot); 111 } else { 112 return op->offset - xmm_offset(op->ot); 113 } 114 } 115 116 static int vector_elem_offset(X86DecodedOp *op, MemOp ot, int n) 117 { 118 int base_ofs = vector_reg_offset(op); 119 switch(ot) { 120 case MO_8: 121 if (op->unit == X86_OP_MMX) { 122 return base_ofs + offsetof(MMXReg, MMX_B(n)); 123 } else { 124 return base_ofs + offsetof(ZMMReg, ZMM_B(n)); 125 } 126 case MO_16: 127 if (op->unit == X86_OP_MMX) { 128 return base_ofs + offsetof(MMXReg, MMX_W(n)); 129 } else { 130 return base_ofs + offsetof(ZMMReg, ZMM_W(n)); 131 } 132 case MO_32: 133 if (op->unit == X86_OP_MMX) { 134 return base_ofs + offsetof(MMXReg, MMX_L(n)); 135 } else { 136 return base_ofs + offsetof(ZMMReg, ZMM_L(n)); 137 } 138 case MO_64: 139 if (op->unit == X86_OP_MMX) { 140 return base_ofs; 141 } else { 142 return base_ofs + offsetof(ZMMReg, ZMM_Q(n)); 143 } 144 case MO_128: 145 assert(op->unit == X86_OP_SSE); 146 return base_ofs + offsetof(ZMMReg, ZMM_X(n)); 147 case MO_256: 148 assert(op->unit == X86_OP_SSE); 149 return base_ofs + offsetof(ZMMReg, ZMM_Y(n)); 150 default: 151 g_assert_not_reached(); 152 } 153 } 154 155 static void compute_mmx_offset(X86DecodedOp *op) 156 { 157 if (!op->has_ea) { 158 op->offset = offsetof(CPUX86State, fpregs[op->n].mmx) + mmx_offset(op->ot); 159 } else { 160 op->offset = offsetof(CPUX86State, mmx_t0) + mmx_offset(op->ot); 161 } 162 } 163 164 static void compute_xmm_offset(X86DecodedOp *op) 165 { 166 if (!op->has_ea) { 167 op->offset = ZMM_OFFSET(op->n) + xmm_offset(op->ot); 168 } else { 169 op->offset = offsetof(CPUX86State, xmm_t0) + xmm_offset(op->ot); 170 } 171 } 172 173 static void gen_load_sse(DisasContext *s, TCGv temp, MemOp ot, int dest_ofs, bool aligned) 174 { 175 switch(ot) { 176 case MO_8: 177 gen_op_ld_v(s, MO_8, temp, s->A0); 178 tcg_gen_st8_tl(temp, cpu_env, dest_ofs); 179 break; 180 case MO_16: 181 gen_op_ld_v(s, MO_16, temp, s->A0); 182 tcg_gen_st16_tl(temp, cpu_env, dest_ofs); 183 break; 184 case MO_32: 185 gen_op_ld_v(s, MO_32, temp, s->A0); 186 tcg_gen_st32_tl(temp, cpu_env, dest_ofs); 187 break; 188 case MO_64: 189 gen_ldq_env_A0(s, dest_ofs); 190 break; 191 case MO_128: 192 gen_ldo_env_A0(s, dest_ofs, aligned); 193 break; 194 case MO_256: 195 gen_ldy_env_A0(s, dest_ofs, aligned); 196 break; 197 default: 198 g_assert_not_reached(); 199 } 200 } 201 202 static bool sse_needs_alignment(DisasContext *s, X86DecodedInsn *decode, MemOp ot) 203 { 204 switch (decode->e.vex_class) { 205 case 2: 206 case 4: 207 if ((s->prefix & PREFIX_VEX) || 208 decode->e.vex_special == X86_VEX_SSEUnaligned) { 209 /* MOST legacy SSE instructions require aligned memory operands, but not all. */ 210 return false; 211 } 212 /* fall through */ 213 case 1: 214 return ot >= MO_128; 215 216 default: 217 return false; 218 } 219 } 220 221 static void gen_load(DisasContext *s, X86DecodedInsn *decode, int opn, TCGv v) 222 { 223 X86DecodedOp *op = &decode->op[opn]; 224 225 switch (op->unit) { 226 case X86_OP_SKIP: 227 return; 228 case X86_OP_SEG: 229 tcg_gen_ld32u_tl(v, cpu_env, 230 offsetof(CPUX86State,segs[op->n].selector)); 231 break; 232 case X86_OP_CR: 233 tcg_gen_ld_tl(v, cpu_env, offsetof(CPUX86State, cr[op->n])); 234 break; 235 case X86_OP_DR: 236 tcg_gen_ld_tl(v, cpu_env, offsetof(CPUX86State, dr[op->n])); 237 break; 238 case X86_OP_INT: 239 if (op->has_ea) { 240 gen_op_ld_v(s, op->ot, v, s->A0); 241 } else { 242 gen_op_mov_v_reg(s, op->ot, v, op->n); 243 } 244 break; 245 case X86_OP_IMM: 246 tcg_gen_movi_tl(v, decode->immediate); 247 break; 248 249 case X86_OP_MMX: 250 compute_mmx_offset(op); 251 goto load_vector; 252 253 case X86_OP_SSE: 254 compute_xmm_offset(op); 255 load_vector: 256 if (op->has_ea) { 257 bool aligned = sse_needs_alignment(s, decode, op->ot); 258 gen_load_sse(s, v, op->ot, op->offset, aligned); 259 } 260 break; 261 262 default: 263 g_assert_not_reached(); 264 } 265 } 266 267 static TCGv_ptr op_ptr(X86DecodedInsn *decode, int opn) 268 { 269 X86DecodedOp *op = &decode->op[opn]; 270 if (op->v_ptr) { 271 return op->v_ptr; 272 } 273 op->v_ptr = tcg_temp_new_ptr(); 274 275 /* The temporary points to the MMXReg or ZMMReg. */ 276 tcg_gen_addi_ptr(op->v_ptr, cpu_env, vector_reg_offset(op)); 277 return op->v_ptr; 278 } 279 280 #define OP_PTR0 op_ptr(decode, 0) 281 #define OP_PTR1 op_ptr(decode, 1) 282 #define OP_PTR2 op_ptr(decode, 2) 283 284 static void gen_writeback(DisasContext *s, X86DecodedInsn *decode, int opn, TCGv v) 285 { 286 X86DecodedOp *op = &decode->op[opn]; 287 switch (op->unit) { 288 case X86_OP_SKIP: 289 break; 290 case X86_OP_SEG: 291 /* Note that gen_movl_seg_T0 takes care of interrupt shadow and TF. */ 292 gen_movl_seg_T0(s, op->n); 293 break; 294 case X86_OP_INT: 295 if (op->has_ea) { 296 gen_op_st_v(s, op->ot, v, s->A0); 297 } else { 298 gen_op_mov_reg_v(s, op->ot, op->n, v); 299 } 300 break; 301 case X86_OP_MMX: 302 break; 303 case X86_OP_SSE: 304 if (!op->has_ea && (s->prefix & PREFIX_VEX) && op->ot <= MO_128) { 305 tcg_gen_gvec_dup_imm(MO_64, 306 offsetof(CPUX86State, xmm_regs[op->n].ZMM_X(1)), 307 16, 16, 0); 308 } 309 break; 310 case X86_OP_CR: 311 case X86_OP_DR: 312 default: 313 g_assert_not_reached(); 314 } 315 } 316 317 static inline int vector_len(DisasContext *s, X86DecodedInsn *decode) 318 { 319 if (decode->e.special == X86_SPECIAL_MMX && 320 !(s->prefix & (PREFIX_DATA | PREFIX_REPZ | PREFIX_REPNZ))) { 321 return 8; 322 } 323 return s->vex_l ? 32 : 16; 324 } 325 326 static void gen_store_sse(DisasContext *s, X86DecodedInsn *decode, int src_ofs) 327 { 328 MemOp ot = decode->op[0].ot; 329 int vec_len = vector_len(s, decode); 330 bool aligned = sse_needs_alignment(s, decode, ot); 331 332 if (!decode->op[0].has_ea) { 333 tcg_gen_gvec_mov(MO_64, decode->op[0].offset, src_ofs, vec_len, vec_len); 334 return; 335 } 336 337 switch (ot) { 338 case MO_64: 339 gen_stq_env_A0(s, src_ofs); 340 break; 341 case MO_128: 342 gen_sto_env_A0(s, src_ofs, aligned); 343 break; 344 case MO_256: 345 gen_sty_env_A0(s, src_ofs, aligned); 346 break; 347 default: 348 g_assert_not_reached(); 349 } 350 } 351 352 static void gen_helper_pavgusb(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b) 353 { 354 gen_helper_pavgb_mmx(env, reg_a, reg_a, reg_b); 355 } 356 357 #define FN_3DNOW_MOVE ((SSEFunc_0_epp) (uintptr_t) 1) 358 static const SSEFunc_0_epp fns_3dnow[] = { 359 [0x0c] = gen_helper_pi2fw, 360 [0x0d] = gen_helper_pi2fd, 361 [0x1c] = gen_helper_pf2iw, 362 [0x1d] = gen_helper_pf2id, 363 [0x8a] = gen_helper_pfnacc, 364 [0x8e] = gen_helper_pfpnacc, 365 [0x90] = gen_helper_pfcmpge, 366 [0x94] = gen_helper_pfmin, 367 [0x96] = gen_helper_pfrcp, 368 [0x97] = gen_helper_pfrsqrt, 369 [0x9a] = gen_helper_pfsub, 370 [0x9e] = gen_helper_pfadd, 371 [0xa0] = gen_helper_pfcmpgt, 372 [0xa4] = gen_helper_pfmax, 373 [0xa6] = FN_3DNOW_MOVE, /* PFRCPIT1; no need to actually increase precision */ 374 [0xa7] = FN_3DNOW_MOVE, /* PFRSQIT1 */ 375 [0xb6] = FN_3DNOW_MOVE, /* PFRCPIT2 */ 376 [0xaa] = gen_helper_pfsubr, 377 [0xae] = gen_helper_pfacc, 378 [0xb0] = gen_helper_pfcmpeq, 379 [0xb4] = gen_helper_pfmul, 380 [0xb7] = gen_helper_pmulhrw_mmx, 381 [0xbb] = gen_helper_pswapd, 382 [0xbf] = gen_helper_pavgusb, 383 }; 384 385 static void gen_3dnow(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 386 { 387 uint8_t b = decode->immediate; 388 SSEFunc_0_epp fn = b < ARRAY_SIZE(fns_3dnow) ? fns_3dnow[b] : NULL; 389 390 if (!fn) { 391 gen_illegal_opcode(s); 392 return; 393 } 394 if (s->flags & HF_TS_MASK) { 395 gen_NM_exception(s); 396 return; 397 } 398 if (s->flags & HF_EM_MASK) { 399 gen_illegal_opcode(s); 400 return; 401 } 402 403 gen_helper_enter_mmx(cpu_env); 404 if (fn == FN_3DNOW_MOVE) { 405 tcg_gen_ld_i64(s->tmp1_i64, cpu_env, decode->op[1].offset); 406 tcg_gen_st_i64(s->tmp1_i64, cpu_env, decode->op[0].offset); 407 } else { 408 fn(cpu_env, OP_PTR0, OP_PTR1); 409 } 410 } 411 412 /* 413 * 00 = v*ps Vps, Hps, Wpd 414 * 66 = v*pd Vpd, Hpd, Wps 415 * f3 = v*ss Vss, Hss, Wps 416 * f2 = v*sd Vsd, Hsd, Wps 417 */ 418 static inline void gen_unary_fp_sse(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode, 419 SSEFunc_0_epp pd_xmm, SSEFunc_0_epp ps_xmm, 420 SSEFunc_0_epp pd_ymm, SSEFunc_0_epp ps_ymm, 421 SSEFunc_0_eppp sd, SSEFunc_0_eppp ss) 422 { 423 if ((s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) != 0) { 424 SSEFunc_0_eppp fn = s->prefix & PREFIX_REPZ ? ss : sd; 425 if (!fn) { 426 gen_illegal_opcode(s); 427 return; 428 } 429 fn(cpu_env, OP_PTR0, OP_PTR1, OP_PTR2); 430 } else { 431 SSEFunc_0_epp ps, pd, fn; 432 ps = s->vex_l ? ps_ymm : ps_xmm; 433 pd = s->vex_l ? pd_ymm : pd_xmm; 434 fn = s->prefix & PREFIX_DATA ? pd : ps; 435 if (!fn) { 436 gen_illegal_opcode(s); 437 return; 438 } 439 fn(cpu_env, OP_PTR0, OP_PTR2); 440 } 441 } 442 #define UNARY_FP_SSE(uname, lname) \ 443 static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 444 { \ 445 gen_unary_fp_sse(s, env, decode, \ 446 gen_helper_##lname##pd_xmm, \ 447 gen_helper_##lname##ps_xmm, \ 448 gen_helper_##lname##pd_ymm, \ 449 gen_helper_##lname##ps_ymm, \ 450 gen_helper_##lname##sd, \ 451 gen_helper_##lname##ss); \ 452 } 453 UNARY_FP_SSE(VSQRT, sqrt) 454 455 /* 456 * 00 = v*ps Vps, Hps, Wpd 457 * 66 = v*pd Vpd, Hpd, Wps 458 * f3 = v*ss Vss, Hss, Wps 459 * f2 = v*sd Vsd, Hsd, Wps 460 */ 461 static inline void gen_fp_sse(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode, 462 SSEFunc_0_eppp pd_xmm, SSEFunc_0_eppp ps_xmm, 463 SSEFunc_0_eppp pd_ymm, SSEFunc_0_eppp ps_ymm, 464 SSEFunc_0_eppp sd, SSEFunc_0_eppp ss) 465 { 466 SSEFunc_0_eppp ps, pd, fn; 467 if ((s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) != 0) { 468 fn = s->prefix & PREFIX_REPZ ? ss : sd; 469 } else { 470 ps = s->vex_l ? ps_ymm : ps_xmm; 471 pd = s->vex_l ? pd_ymm : pd_xmm; 472 fn = s->prefix & PREFIX_DATA ? pd : ps; 473 } 474 if (fn) { 475 fn(cpu_env, OP_PTR0, OP_PTR1, OP_PTR2); 476 } else { 477 gen_illegal_opcode(s); 478 } 479 } 480 481 #define FP_SSE(uname, lname) \ 482 static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 483 { \ 484 gen_fp_sse(s, env, decode, \ 485 gen_helper_##lname##pd_xmm, \ 486 gen_helper_##lname##ps_xmm, \ 487 gen_helper_##lname##pd_ymm, \ 488 gen_helper_##lname##ps_ymm, \ 489 gen_helper_##lname##sd, \ 490 gen_helper_##lname##ss); \ 491 } 492 FP_SSE(VADD, add) 493 FP_SSE(VMUL, mul) 494 FP_SSE(VSUB, sub) 495 FP_SSE(VMIN, min) 496 FP_SSE(VDIV, div) 497 FP_SSE(VMAX, max) 498 499 #define FMA_SSE_PACKED(uname, ptr0, ptr1, ptr2, even, odd) \ 500 static void gen_##uname##Px(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 501 { \ 502 SSEFunc_0_eppppii xmm = s->vex_w ? gen_helper_fma4pd_xmm : gen_helper_fma4ps_xmm; \ 503 SSEFunc_0_eppppii ymm = s->vex_w ? gen_helper_fma4pd_ymm : gen_helper_fma4ps_ymm; \ 504 SSEFunc_0_eppppii fn = s->vex_l ? ymm : xmm; \ 505 \ 506 fn(cpu_env, OP_PTR0, ptr0, ptr1, ptr2, \ 507 tcg_constant_i32(even), \ 508 tcg_constant_i32((even) ^ (odd))); \ 509 } 510 511 #define FMA_SSE(uname, ptr0, ptr1, ptr2, flags) \ 512 FMA_SSE_PACKED(uname, ptr0, ptr1, ptr2, flags, flags) \ 513 static void gen_##uname##Sx(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 514 { \ 515 SSEFunc_0_eppppi fn = s->vex_w ? gen_helper_fma4sd : gen_helper_fma4ss; \ 516 \ 517 fn(cpu_env, OP_PTR0, ptr0, ptr1, ptr2, \ 518 tcg_constant_i32(flags)); \ 519 } \ 520 521 FMA_SSE(VFMADD231, OP_PTR1, OP_PTR2, OP_PTR0, 0) 522 FMA_SSE(VFMADD213, OP_PTR1, OP_PTR0, OP_PTR2, 0) 523 FMA_SSE(VFMADD132, OP_PTR0, OP_PTR2, OP_PTR1, 0) 524 525 FMA_SSE(VFNMADD231, OP_PTR1, OP_PTR2, OP_PTR0, float_muladd_negate_product) 526 FMA_SSE(VFNMADD213, OP_PTR1, OP_PTR0, OP_PTR2, float_muladd_negate_product) 527 FMA_SSE(VFNMADD132, OP_PTR0, OP_PTR2, OP_PTR1, float_muladd_negate_product) 528 529 FMA_SSE(VFMSUB231, OP_PTR1, OP_PTR2, OP_PTR0, float_muladd_negate_c) 530 FMA_SSE(VFMSUB213, OP_PTR1, OP_PTR0, OP_PTR2, float_muladd_negate_c) 531 FMA_SSE(VFMSUB132, OP_PTR0, OP_PTR2, OP_PTR1, float_muladd_negate_c) 532 533 FMA_SSE(VFNMSUB231, OP_PTR1, OP_PTR2, OP_PTR0, float_muladd_negate_c|float_muladd_negate_product) 534 FMA_SSE(VFNMSUB213, OP_PTR1, OP_PTR0, OP_PTR2, float_muladd_negate_c|float_muladd_negate_product) 535 FMA_SSE(VFNMSUB132, OP_PTR0, OP_PTR2, OP_PTR1, float_muladd_negate_c|float_muladd_negate_product) 536 537 FMA_SSE_PACKED(VFMADDSUB231, OP_PTR1, OP_PTR2, OP_PTR0, float_muladd_negate_c, 0) 538 FMA_SSE_PACKED(VFMADDSUB213, OP_PTR1, OP_PTR0, OP_PTR2, float_muladd_negate_c, 0) 539 FMA_SSE_PACKED(VFMADDSUB132, OP_PTR0, OP_PTR2, OP_PTR1, float_muladd_negate_c, 0) 540 541 FMA_SSE_PACKED(VFMSUBADD231, OP_PTR1, OP_PTR2, OP_PTR0, 0, float_muladd_negate_c) 542 FMA_SSE_PACKED(VFMSUBADD213, OP_PTR1, OP_PTR0, OP_PTR2, 0, float_muladd_negate_c) 543 FMA_SSE_PACKED(VFMSUBADD132, OP_PTR0, OP_PTR2, OP_PTR1, 0, float_muladd_negate_c) 544 545 #define FP_UNPACK_SSE(uname, lname) \ 546 static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 547 { \ 548 /* PS maps to the DQ integer instruction, PD maps to QDQ. */ \ 549 gen_fp_sse(s, env, decode, \ 550 gen_helper_##lname##qdq_xmm, \ 551 gen_helper_##lname##dq_xmm, \ 552 gen_helper_##lname##qdq_ymm, \ 553 gen_helper_##lname##dq_ymm, \ 554 NULL, NULL); \ 555 } 556 FP_UNPACK_SSE(VUNPCKLPx, punpckl) 557 FP_UNPACK_SSE(VUNPCKHPx, punpckh) 558 559 /* 560 * 00 = v*ps Vps, Wpd 561 * f3 = v*ss Vss, Wps 562 */ 563 static inline void gen_unary_fp32_sse(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode, 564 SSEFunc_0_epp ps_xmm, 565 SSEFunc_0_epp ps_ymm, 566 SSEFunc_0_eppp ss) 567 { 568 if ((s->prefix & (PREFIX_DATA | PREFIX_REPNZ)) != 0) { 569 goto illegal_op; 570 } else if (s->prefix & PREFIX_REPZ) { 571 if (!ss) { 572 goto illegal_op; 573 } 574 ss(cpu_env, OP_PTR0, OP_PTR1, OP_PTR2); 575 } else { 576 SSEFunc_0_epp fn = s->vex_l ? ps_ymm : ps_xmm; 577 if (!fn) { 578 goto illegal_op; 579 } 580 fn(cpu_env, OP_PTR0, OP_PTR2); 581 } 582 return; 583 584 illegal_op: 585 gen_illegal_opcode(s); 586 } 587 #define UNARY_FP32_SSE(uname, lname) \ 588 static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 589 { \ 590 gen_unary_fp32_sse(s, env, decode, \ 591 gen_helper_##lname##ps_xmm, \ 592 gen_helper_##lname##ps_ymm, \ 593 gen_helper_##lname##ss); \ 594 } 595 UNARY_FP32_SSE(VRSQRT, rsqrt) 596 UNARY_FP32_SSE(VRCP, rcp) 597 598 /* 599 * 66 = v*pd Vpd, Hpd, Wpd 600 * f2 = v*ps Vps, Hps, Wps 601 */ 602 static inline void gen_horizontal_fp_sse(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode, 603 SSEFunc_0_eppp pd_xmm, SSEFunc_0_eppp ps_xmm, 604 SSEFunc_0_eppp pd_ymm, SSEFunc_0_eppp ps_ymm) 605 { 606 SSEFunc_0_eppp ps, pd, fn; 607 ps = s->vex_l ? ps_ymm : ps_xmm; 608 pd = s->vex_l ? pd_ymm : pd_xmm; 609 fn = s->prefix & PREFIX_DATA ? pd : ps; 610 fn(cpu_env, OP_PTR0, OP_PTR1, OP_PTR2); 611 } 612 #define HORIZONTAL_FP_SSE(uname, lname) \ 613 static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 614 { \ 615 gen_horizontal_fp_sse(s, env, decode, \ 616 gen_helper_##lname##pd_xmm, gen_helper_##lname##ps_xmm, \ 617 gen_helper_##lname##pd_ymm, gen_helper_##lname##ps_ymm); \ 618 } 619 HORIZONTAL_FP_SSE(VHADD, hadd) 620 HORIZONTAL_FP_SSE(VHSUB, hsub) 621 HORIZONTAL_FP_SSE(VADDSUB, addsub) 622 623 static inline void gen_ternary_sse(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode, 624 int op3, SSEFunc_0_epppp xmm, SSEFunc_0_epppp ymm) 625 { 626 SSEFunc_0_epppp fn = s->vex_l ? ymm : xmm; 627 TCGv_ptr ptr3 = tcg_temp_new_ptr(); 628 629 /* The format of the fourth input is Lx */ 630 tcg_gen_addi_ptr(ptr3, cpu_env, ZMM_OFFSET(op3)); 631 fn(cpu_env, OP_PTR0, OP_PTR1, OP_PTR2, ptr3); 632 tcg_temp_free_ptr(ptr3); 633 } 634 #define TERNARY_SSE(uname, uvname, lname) \ 635 static void gen_##uvname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 636 { \ 637 gen_ternary_sse(s, env, decode, (uint8_t)decode->immediate >> 4, \ 638 gen_helper_##lname##_xmm, gen_helper_##lname##_ymm); \ 639 } \ 640 static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 641 { \ 642 gen_ternary_sse(s, env, decode, 0, \ 643 gen_helper_##lname##_xmm, gen_helper_##lname##_ymm); \ 644 } 645 TERNARY_SSE(BLENDVPS, VBLENDVPS, blendvps) 646 TERNARY_SSE(BLENDVPD, VBLENDVPD, blendvpd) 647 TERNARY_SSE(PBLENDVB, VPBLENDVB, pblendvb) 648 649 static inline void gen_binary_imm_sse(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode, 650 SSEFunc_0_epppi xmm, SSEFunc_0_epppi ymm) 651 { 652 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 653 if (!s->vex_l) { 654 xmm(cpu_env, OP_PTR0, OP_PTR1, OP_PTR2, imm); 655 } else { 656 ymm(cpu_env, OP_PTR0, OP_PTR1, OP_PTR2, imm); 657 } 658 } 659 660 #define BINARY_IMM_SSE(uname, lname) \ 661 static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 662 { \ 663 gen_binary_imm_sse(s, env, decode, \ 664 gen_helper_##lname##_xmm, \ 665 gen_helper_##lname##_ymm); \ 666 } 667 668 BINARY_IMM_SSE(VBLENDPD, blendpd) 669 BINARY_IMM_SSE(VBLENDPS, blendps) 670 BINARY_IMM_SSE(VPBLENDW, pblendw) 671 BINARY_IMM_SSE(VDDPS, dpps) 672 #define gen_helper_dppd_ymm NULL 673 BINARY_IMM_SSE(VDDPD, dppd) 674 BINARY_IMM_SSE(VMPSADBW, mpsadbw) 675 BINARY_IMM_SSE(PCLMULQDQ, pclmulqdq) 676 677 678 #define UNARY_INT_GVEC(uname, func, ...) \ 679 static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 680 { \ 681 int vec_len = vector_len(s, decode); \ 682 \ 683 func(__VA_ARGS__, decode->op[0].offset, \ 684 decode->op[2].offset, vec_len, vec_len); \ 685 } 686 UNARY_INT_GVEC(PABSB, tcg_gen_gvec_abs, MO_8) 687 UNARY_INT_GVEC(PABSW, tcg_gen_gvec_abs, MO_16) 688 UNARY_INT_GVEC(PABSD, tcg_gen_gvec_abs, MO_32) 689 UNARY_INT_GVEC(VBROADCASTx128, tcg_gen_gvec_dup_mem, MO_128) 690 UNARY_INT_GVEC(VPBROADCASTB, tcg_gen_gvec_dup_mem, MO_8) 691 UNARY_INT_GVEC(VPBROADCASTW, tcg_gen_gvec_dup_mem, MO_16) 692 UNARY_INT_GVEC(VPBROADCASTD, tcg_gen_gvec_dup_mem, MO_32) 693 UNARY_INT_GVEC(VPBROADCASTQ, tcg_gen_gvec_dup_mem, MO_64) 694 695 696 #define BINARY_INT_GVEC(uname, func, ...) \ 697 static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 698 { \ 699 int vec_len = vector_len(s, decode); \ 700 \ 701 func(__VA_ARGS__, \ 702 decode->op[0].offset, decode->op[1].offset, \ 703 decode->op[2].offset, vec_len, vec_len); \ 704 } 705 706 BINARY_INT_GVEC(PADDB, tcg_gen_gvec_add, MO_8) 707 BINARY_INT_GVEC(PADDW, tcg_gen_gvec_add, MO_16) 708 BINARY_INT_GVEC(PADDD, tcg_gen_gvec_add, MO_32) 709 BINARY_INT_GVEC(PADDQ, tcg_gen_gvec_add, MO_64) 710 BINARY_INT_GVEC(PADDSB, tcg_gen_gvec_ssadd, MO_8) 711 BINARY_INT_GVEC(PADDSW, tcg_gen_gvec_ssadd, MO_16) 712 BINARY_INT_GVEC(PADDUSB, tcg_gen_gvec_usadd, MO_8) 713 BINARY_INT_GVEC(PADDUSW, tcg_gen_gvec_usadd, MO_16) 714 BINARY_INT_GVEC(PAND, tcg_gen_gvec_and, MO_64) 715 BINARY_INT_GVEC(PCMPEQB, tcg_gen_gvec_cmp, TCG_COND_EQ, MO_8) 716 BINARY_INT_GVEC(PCMPEQD, tcg_gen_gvec_cmp, TCG_COND_EQ, MO_32) 717 BINARY_INT_GVEC(PCMPEQW, tcg_gen_gvec_cmp, TCG_COND_EQ, MO_16) 718 BINARY_INT_GVEC(PCMPEQQ, tcg_gen_gvec_cmp, TCG_COND_EQ, MO_64) 719 BINARY_INT_GVEC(PCMPGTB, tcg_gen_gvec_cmp, TCG_COND_GT, MO_8) 720 BINARY_INT_GVEC(PCMPGTW, tcg_gen_gvec_cmp, TCG_COND_GT, MO_16) 721 BINARY_INT_GVEC(PCMPGTD, tcg_gen_gvec_cmp, TCG_COND_GT, MO_32) 722 BINARY_INT_GVEC(PCMPGTQ, tcg_gen_gvec_cmp, TCG_COND_GT, MO_64) 723 BINARY_INT_GVEC(PMAXSB, tcg_gen_gvec_smax, MO_8) 724 BINARY_INT_GVEC(PMAXSW, tcg_gen_gvec_smax, MO_16) 725 BINARY_INT_GVEC(PMAXSD, tcg_gen_gvec_smax, MO_32) 726 BINARY_INT_GVEC(PMAXUB, tcg_gen_gvec_umax, MO_8) 727 BINARY_INT_GVEC(PMAXUW, tcg_gen_gvec_umax, MO_16) 728 BINARY_INT_GVEC(PMAXUD, tcg_gen_gvec_umax, MO_32) 729 BINARY_INT_GVEC(PMINSB, tcg_gen_gvec_smin, MO_8) 730 BINARY_INT_GVEC(PMINSW, tcg_gen_gvec_smin, MO_16) 731 BINARY_INT_GVEC(PMINSD, tcg_gen_gvec_smin, MO_32) 732 BINARY_INT_GVEC(PMINUB, tcg_gen_gvec_umin, MO_8) 733 BINARY_INT_GVEC(PMINUW, tcg_gen_gvec_umin, MO_16) 734 BINARY_INT_GVEC(PMINUD, tcg_gen_gvec_umin, MO_32) 735 BINARY_INT_GVEC(PMULLW, tcg_gen_gvec_mul, MO_16) 736 BINARY_INT_GVEC(PMULLD, tcg_gen_gvec_mul, MO_32) 737 BINARY_INT_GVEC(POR, tcg_gen_gvec_or, MO_64) 738 BINARY_INT_GVEC(PSUBB, tcg_gen_gvec_sub, MO_8) 739 BINARY_INT_GVEC(PSUBW, tcg_gen_gvec_sub, MO_16) 740 BINARY_INT_GVEC(PSUBD, tcg_gen_gvec_sub, MO_32) 741 BINARY_INT_GVEC(PSUBQ, tcg_gen_gvec_sub, MO_64) 742 BINARY_INT_GVEC(PSUBSB, tcg_gen_gvec_sssub, MO_8) 743 BINARY_INT_GVEC(PSUBSW, tcg_gen_gvec_sssub, MO_16) 744 BINARY_INT_GVEC(PSUBUSB, tcg_gen_gvec_ussub, MO_8) 745 BINARY_INT_GVEC(PSUBUSW, tcg_gen_gvec_ussub, MO_16) 746 BINARY_INT_GVEC(PXOR, tcg_gen_gvec_xor, MO_64) 747 748 749 /* 750 * 00 = p* Pq, Qq (if mmx not NULL; no VEX) 751 * 66 = vp* Vx, Hx, Wx 752 * 753 * These are really the same encoding, because 1) V is the same as P when VEX.V 754 * is not present 2) P and Q are the same as H and W apart from MM/XMM 755 */ 756 static inline void gen_binary_int_sse(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode, 757 SSEFunc_0_eppp mmx, SSEFunc_0_eppp xmm, SSEFunc_0_eppp ymm) 758 { 759 assert(!!mmx == !!(decode->e.special == X86_SPECIAL_MMX)); 760 761 if (mmx && (s->prefix & PREFIX_VEX) && !(s->prefix & PREFIX_DATA)) { 762 /* VEX encoding is not applicable to MMX instructions. */ 763 gen_illegal_opcode(s); 764 return; 765 } 766 if (!(s->prefix & PREFIX_DATA)) { 767 mmx(cpu_env, OP_PTR0, OP_PTR1, OP_PTR2); 768 } else if (!s->vex_l) { 769 xmm(cpu_env, OP_PTR0, OP_PTR1, OP_PTR2); 770 } else { 771 ymm(cpu_env, OP_PTR0, OP_PTR1, OP_PTR2); 772 } 773 } 774 775 776 #define BINARY_INT_MMX(uname, lname) \ 777 static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 778 { \ 779 gen_binary_int_sse(s, env, decode, \ 780 gen_helper_##lname##_mmx, \ 781 gen_helper_##lname##_xmm, \ 782 gen_helper_##lname##_ymm); \ 783 } 784 BINARY_INT_MMX(PUNPCKLBW, punpcklbw) 785 BINARY_INT_MMX(PUNPCKLWD, punpcklwd) 786 BINARY_INT_MMX(PUNPCKLDQ, punpckldq) 787 BINARY_INT_MMX(PACKSSWB, packsswb) 788 BINARY_INT_MMX(PACKUSWB, packuswb) 789 BINARY_INT_MMX(PUNPCKHBW, punpckhbw) 790 BINARY_INT_MMX(PUNPCKHWD, punpckhwd) 791 BINARY_INT_MMX(PUNPCKHDQ, punpckhdq) 792 BINARY_INT_MMX(PACKSSDW, packssdw) 793 794 BINARY_INT_MMX(PAVGB, pavgb) 795 BINARY_INT_MMX(PAVGW, pavgw) 796 BINARY_INT_MMX(PMADDWD, pmaddwd) 797 BINARY_INT_MMX(PMULHUW, pmulhuw) 798 BINARY_INT_MMX(PMULHW, pmulhw) 799 BINARY_INT_MMX(PMULUDQ, pmuludq) 800 BINARY_INT_MMX(PSADBW, psadbw) 801 802 BINARY_INT_MMX(PSLLW_r, psllw) 803 BINARY_INT_MMX(PSLLD_r, pslld) 804 BINARY_INT_MMX(PSLLQ_r, psllq) 805 BINARY_INT_MMX(PSRLW_r, psrlw) 806 BINARY_INT_MMX(PSRLD_r, psrld) 807 BINARY_INT_MMX(PSRLQ_r, psrlq) 808 BINARY_INT_MMX(PSRAW_r, psraw) 809 BINARY_INT_MMX(PSRAD_r, psrad) 810 811 BINARY_INT_MMX(PHADDW, phaddw) 812 BINARY_INT_MMX(PHADDSW, phaddsw) 813 BINARY_INT_MMX(PHADDD, phaddd) 814 BINARY_INT_MMX(PHSUBW, phsubw) 815 BINARY_INT_MMX(PHSUBSW, phsubsw) 816 BINARY_INT_MMX(PHSUBD, phsubd) 817 BINARY_INT_MMX(PMADDUBSW, pmaddubsw) 818 BINARY_INT_MMX(PSHUFB, pshufb) 819 BINARY_INT_MMX(PSIGNB, psignb) 820 BINARY_INT_MMX(PSIGNW, psignw) 821 BINARY_INT_MMX(PSIGND, psignd) 822 BINARY_INT_MMX(PMULHRSW, pmulhrsw) 823 824 /* Instructions with no MMX equivalent. */ 825 #define BINARY_INT_SSE(uname, lname) \ 826 static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 827 { \ 828 gen_binary_int_sse(s, env, decode, \ 829 NULL, \ 830 gen_helper_##lname##_xmm, \ 831 gen_helper_##lname##_ymm); \ 832 } 833 834 /* Instructions with no MMX equivalent. */ 835 BINARY_INT_SSE(PUNPCKLQDQ, punpcklqdq) 836 BINARY_INT_SSE(PUNPCKHQDQ, punpckhqdq) 837 BINARY_INT_SSE(VPACKUSDW, packusdw) 838 BINARY_INT_SSE(VPERMILPS, vpermilps) 839 BINARY_INT_SSE(VPERMILPD, vpermilpd) 840 BINARY_INT_SSE(VMASKMOVPS, vpmaskmovd) 841 BINARY_INT_SSE(VMASKMOVPD, vpmaskmovq) 842 843 BINARY_INT_SSE(PMULDQ, pmuldq) 844 845 BINARY_INT_SSE(VAESDEC, aesdec) 846 BINARY_INT_SSE(VAESDECLAST, aesdeclast) 847 BINARY_INT_SSE(VAESENC, aesenc) 848 BINARY_INT_SSE(VAESENCLAST, aesenclast) 849 850 #define UNARY_CMP_SSE(uname, lname) \ 851 static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 852 { \ 853 if (!s->vex_l) { \ 854 gen_helper_##lname##_xmm(cpu_env, OP_PTR1, OP_PTR2); \ 855 } else { \ 856 gen_helper_##lname##_ymm(cpu_env, OP_PTR1, OP_PTR2); \ 857 } \ 858 set_cc_op(s, CC_OP_EFLAGS); \ 859 } 860 UNARY_CMP_SSE(VPTEST, ptest) 861 UNARY_CMP_SSE(VTESTPS, vtestps) 862 UNARY_CMP_SSE(VTESTPD, vtestpd) 863 864 static inline void gen_unary_int_sse(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode, 865 SSEFunc_0_epp xmm, SSEFunc_0_epp ymm) 866 { 867 if (!s->vex_l) { 868 xmm(cpu_env, OP_PTR0, OP_PTR2); 869 } else { 870 ymm(cpu_env, OP_PTR0, OP_PTR2); 871 } 872 } 873 874 #define UNARY_INT_SSE(uname, lname) \ 875 static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 876 { \ 877 gen_unary_int_sse(s, env, decode, \ 878 gen_helper_##lname##_xmm, \ 879 gen_helper_##lname##_ymm); \ 880 } 881 882 UNARY_INT_SSE(VPMOVSXBW, pmovsxbw) 883 UNARY_INT_SSE(VPMOVSXBD, pmovsxbd) 884 UNARY_INT_SSE(VPMOVSXBQ, pmovsxbq) 885 UNARY_INT_SSE(VPMOVSXWD, pmovsxwd) 886 UNARY_INT_SSE(VPMOVSXWQ, pmovsxwq) 887 UNARY_INT_SSE(VPMOVSXDQ, pmovsxdq) 888 889 UNARY_INT_SSE(VPMOVZXBW, pmovzxbw) 890 UNARY_INT_SSE(VPMOVZXBD, pmovzxbd) 891 UNARY_INT_SSE(VPMOVZXBQ, pmovzxbq) 892 UNARY_INT_SSE(VPMOVZXWD, pmovzxwd) 893 UNARY_INT_SSE(VPMOVZXWQ, pmovzxwq) 894 UNARY_INT_SSE(VPMOVZXDQ, pmovzxdq) 895 896 UNARY_INT_SSE(VMOVSLDUP, pmovsldup) 897 UNARY_INT_SSE(VMOVSHDUP, pmovshdup) 898 UNARY_INT_SSE(VMOVDDUP, pmovdldup) 899 900 UNARY_INT_SSE(VCVTDQ2PD, cvtdq2pd) 901 UNARY_INT_SSE(VCVTPD2DQ, cvtpd2dq) 902 UNARY_INT_SSE(VCVTTPD2DQ, cvttpd2dq) 903 UNARY_INT_SSE(VCVTDQ2PS, cvtdq2ps) 904 UNARY_INT_SSE(VCVTPS2DQ, cvtps2dq) 905 UNARY_INT_SSE(VCVTTPS2DQ, cvttps2dq) 906 UNARY_INT_SSE(VCVTPH2PS, cvtph2ps) 907 908 909 static inline void gen_unary_imm_sse(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode, 910 SSEFunc_0_ppi xmm, SSEFunc_0_ppi ymm) 911 { 912 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 913 if (!s->vex_l) { 914 xmm(OP_PTR0, OP_PTR1, imm); 915 } else { 916 ymm(OP_PTR0, OP_PTR1, imm); 917 } 918 } 919 920 #define UNARY_IMM_SSE(uname, lname) \ 921 static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 922 { \ 923 gen_unary_imm_sse(s, env, decode, \ 924 gen_helper_##lname##_xmm, \ 925 gen_helper_##lname##_ymm); \ 926 } 927 928 UNARY_IMM_SSE(PSHUFD, pshufd) 929 UNARY_IMM_SSE(PSHUFHW, pshufhw) 930 UNARY_IMM_SSE(PSHUFLW, pshuflw) 931 #define gen_helper_vpermq_xmm NULL 932 UNARY_IMM_SSE(VPERMQ, vpermq) 933 UNARY_IMM_SSE(VPERMILPS_i, vpermilps_imm) 934 UNARY_IMM_SSE(VPERMILPD_i, vpermilpd_imm) 935 936 static inline void gen_unary_imm_fp_sse(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode, 937 SSEFunc_0_eppi xmm, SSEFunc_0_eppi ymm) 938 { 939 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 940 if (!s->vex_l) { 941 xmm(cpu_env, OP_PTR0, OP_PTR1, imm); 942 } else { 943 ymm(cpu_env, OP_PTR0, OP_PTR1, imm); 944 } 945 } 946 947 #define UNARY_IMM_FP_SSE(uname, lname) \ 948 static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 949 { \ 950 gen_unary_imm_fp_sse(s, env, decode, \ 951 gen_helper_##lname##_xmm, \ 952 gen_helper_##lname##_ymm); \ 953 } 954 955 UNARY_IMM_FP_SSE(VROUNDPS, roundps) 956 UNARY_IMM_FP_SSE(VROUNDPD, roundpd) 957 958 static inline void gen_vexw_avx(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode, 959 SSEFunc_0_eppp d_xmm, SSEFunc_0_eppp q_xmm, 960 SSEFunc_0_eppp d_ymm, SSEFunc_0_eppp q_ymm) 961 { 962 SSEFunc_0_eppp d = s->vex_l ? d_ymm : d_xmm; 963 SSEFunc_0_eppp q = s->vex_l ? q_ymm : q_xmm; 964 SSEFunc_0_eppp fn = s->vex_w ? q : d; 965 fn(cpu_env, OP_PTR0, OP_PTR1, OP_PTR2); 966 } 967 968 /* VEX.W affects whether to operate on 32- or 64-bit elements. */ 969 #define VEXW_AVX(uname, lname) \ 970 static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 971 { \ 972 gen_vexw_avx(s, env, decode, \ 973 gen_helper_##lname##d_xmm, gen_helper_##lname##q_xmm, \ 974 gen_helper_##lname##d_ymm, gen_helper_##lname##q_ymm); \ 975 } 976 VEXW_AVX(VPSLLV, vpsllv) 977 VEXW_AVX(VPSRLV, vpsrlv) 978 VEXW_AVX(VPSRAV, vpsrav) 979 VEXW_AVX(VPMASKMOV, vpmaskmov) 980 981 /* Same as above, but with extra arguments to the helper. */ 982 static inline void gen_vsib_avx(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode, 983 SSEFunc_0_epppti d_xmm, SSEFunc_0_epppti q_xmm, 984 SSEFunc_0_epppti d_ymm, SSEFunc_0_epppti q_ymm) 985 { 986 SSEFunc_0_epppti d = s->vex_l ? d_ymm : d_xmm; 987 SSEFunc_0_epppti q = s->vex_l ? q_ymm : q_xmm; 988 SSEFunc_0_epppti fn = s->vex_w ? q : d; 989 TCGv_i32 scale = tcg_constant_i32(decode->mem.scale); 990 TCGv_ptr index = tcg_temp_new_ptr(); 991 992 /* Pass third input as (index, base, scale) */ 993 tcg_gen_addi_ptr(index, cpu_env, ZMM_OFFSET(decode->mem.index)); 994 fn(cpu_env, OP_PTR0, OP_PTR1, index, s->A0, scale); 995 996 /* 997 * There are two output operands, so zero OP1's high 128 bits 998 * in the VEX.128 case. 999 */ 1000 if (!s->vex_l) { 1001 int ymmh_ofs = vector_elem_offset(&decode->op[1], MO_128, 1); 1002 tcg_gen_gvec_dup_imm(MO_64, ymmh_ofs, 16, 16, 0); 1003 } 1004 tcg_temp_free_ptr(index); 1005 } 1006 #define VSIB_AVX(uname, lname) \ 1007 static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 1008 { \ 1009 gen_vsib_avx(s, env, decode, \ 1010 gen_helper_##lname##d_xmm, gen_helper_##lname##q_xmm, \ 1011 gen_helper_##lname##d_ymm, gen_helper_##lname##q_ymm); \ 1012 } 1013 VSIB_AVX(VPGATHERD, vpgatherd) 1014 VSIB_AVX(VPGATHERQ, vpgatherq) 1015 1016 static void gen_ADCOX(DisasContext *s, CPUX86State *env, MemOp ot, int cc_op) 1017 { 1018 TCGv carry_in = NULL; 1019 TCGv carry_out = (cc_op == CC_OP_ADCX ? cpu_cc_dst : cpu_cc_src2); 1020 TCGv zero; 1021 1022 if (cc_op == s->cc_op || s->cc_op == CC_OP_ADCOX) { 1023 /* Re-use the carry-out from a previous round. */ 1024 carry_in = carry_out; 1025 cc_op = s->cc_op; 1026 } else if (s->cc_op == CC_OP_ADCX || s->cc_op == CC_OP_ADOX) { 1027 /* Merge with the carry-out from the opposite instruction. */ 1028 cc_op = CC_OP_ADCOX; 1029 } 1030 1031 /* If we don't have a carry-in, get it out of EFLAGS. */ 1032 if (!carry_in) { 1033 if (s->cc_op != CC_OP_ADCX && s->cc_op != CC_OP_ADOX) { 1034 gen_compute_eflags(s); 1035 } 1036 carry_in = s->tmp0; 1037 tcg_gen_extract_tl(carry_in, cpu_cc_src, 1038 ctz32(cc_op == CC_OP_ADCX ? CC_C : CC_O), 1); 1039 } 1040 1041 switch (ot) { 1042 #ifdef TARGET_X86_64 1043 case MO_32: 1044 /* If TL is 64-bit just do everything in 64-bit arithmetic. */ 1045 tcg_gen_add_i64(s->T0, s->T0, s->T1); 1046 tcg_gen_add_i64(s->T0, s->T0, carry_in); 1047 tcg_gen_shri_i64(carry_out, s->T0, 32); 1048 break; 1049 #endif 1050 default: 1051 zero = tcg_constant_tl(0); 1052 tcg_gen_add2_tl(s->T0, carry_out, s->T0, zero, carry_in, zero); 1053 tcg_gen_add2_tl(s->T0, carry_out, s->T0, carry_out, s->T1, zero); 1054 break; 1055 } 1056 set_cc_op(s, cc_op); 1057 } 1058 1059 static void gen_ADCX(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1060 { 1061 gen_ADCOX(s, env, decode->op[0].ot, CC_OP_ADCX); 1062 } 1063 1064 static void gen_ADOX(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1065 { 1066 gen_ADCOX(s, env, decode->op[0].ot, CC_OP_ADOX); 1067 } 1068 1069 static void gen_ANDN(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1070 { 1071 MemOp ot = decode->op[0].ot; 1072 1073 tcg_gen_andc_tl(s->T0, s->T1, s->T0); 1074 gen_op_update1_cc(s); 1075 set_cc_op(s, CC_OP_LOGICB + ot); 1076 } 1077 1078 static void gen_BEXTR(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1079 { 1080 MemOp ot = decode->op[0].ot; 1081 TCGv bound, zero; 1082 1083 /* 1084 * Extract START, and shift the operand. 1085 * Shifts larger than operand size get zeros. 1086 */ 1087 tcg_gen_ext8u_tl(s->A0, s->T1); 1088 tcg_gen_shr_tl(s->T0, s->T0, s->A0); 1089 1090 bound = tcg_constant_tl(ot == MO_64 ? 63 : 31); 1091 zero = tcg_constant_tl(0); 1092 tcg_gen_movcond_tl(TCG_COND_LEU, s->T0, s->A0, bound, s->T0, zero); 1093 1094 /* 1095 * Extract the LEN into a mask. Lengths larger than 1096 * operand size get all ones. 1097 */ 1098 tcg_gen_extract_tl(s->A0, s->T1, 8, 8); 1099 tcg_gen_movcond_tl(TCG_COND_LEU, s->A0, s->A0, bound, s->A0, bound); 1100 1101 tcg_gen_movi_tl(s->T1, 1); 1102 tcg_gen_shl_tl(s->T1, s->T1, s->A0); 1103 tcg_gen_subi_tl(s->T1, s->T1, 1); 1104 tcg_gen_and_tl(s->T0, s->T0, s->T1); 1105 1106 gen_op_update1_cc(s); 1107 set_cc_op(s, CC_OP_LOGICB + ot); 1108 } 1109 1110 static void gen_BLSI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1111 { 1112 MemOp ot = decode->op[0].ot; 1113 1114 tcg_gen_neg_tl(s->T1, s->T0); 1115 tcg_gen_and_tl(s->T0, s->T0, s->T1); 1116 tcg_gen_mov_tl(cpu_cc_dst, s->T0); 1117 set_cc_op(s, CC_OP_BMILGB + ot); 1118 } 1119 1120 static void gen_BLSMSK(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1121 { 1122 MemOp ot = decode->op[0].ot; 1123 1124 tcg_gen_subi_tl(s->T1, s->T0, 1); 1125 tcg_gen_xor_tl(s->T0, s->T0, s->T1); 1126 tcg_gen_mov_tl(cpu_cc_dst, s->T0); 1127 set_cc_op(s, CC_OP_BMILGB + ot); 1128 } 1129 1130 static void gen_BLSR(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1131 { 1132 MemOp ot = decode->op[0].ot; 1133 1134 tcg_gen_subi_tl(s->T1, s->T0, 1); 1135 tcg_gen_and_tl(s->T0, s->T0, s->T1); 1136 tcg_gen_mov_tl(cpu_cc_dst, s->T0); 1137 set_cc_op(s, CC_OP_BMILGB + ot); 1138 } 1139 1140 static void gen_BZHI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1141 { 1142 MemOp ot = decode->op[0].ot; 1143 TCGv bound; 1144 1145 tcg_gen_ext8u_tl(s->T1, cpu_regs[s->vex_v]); 1146 bound = tcg_constant_tl(ot == MO_64 ? 63 : 31); 1147 1148 /* 1149 * Note that since we're using BMILG (in order to get O 1150 * cleared) we need to store the inverse into C. 1151 */ 1152 tcg_gen_setcond_tl(TCG_COND_LT, cpu_cc_src, s->T1, bound); 1153 tcg_gen_movcond_tl(TCG_COND_GT, s->T1, s->T1, bound, bound, s->T1); 1154 1155 tcg_gen_movi_tl(s->A0, -1); 1156 tcg_gen_shl_tl(s->A0, s->A0, s->T1); 1157 tcg_gen_andc_tl(s->T0, s->T0, s->A0); 1158 1159 gen_op_update1_cc(s); 1160 set_cc_op(s, CC_OP_BMILGB + ot); 1161 } 1162 1163 static void gen_CRC32(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1164 { 1165 MemOp ot = decode->op[2].ot; 1166 1167 tcg_gen_trunc_tl_i32(s->tmp2_i32, s->T0); 1168 gen_helper_crc32(s->T0, s->tmp2_i32, s->T1, tcg_constant_i32(8 << ot)); 1169 } 1170 1171 static void gen_CVTPI2Px(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1172 { 1173 gen_helper_enter_mmx(cpu_env); 1174 if (s->prefix & PREFIX_DATA) { 1175 gen_helper_cvtpi2pd(cpu_env, OP_PTR0, OP_PTR2); 1176 } else { 1177 gen_helper_cvtpi2ps(cpu_env, OP_PTR0, OP_PTR2); 1178 } 1179 } 1180 1181 static void gen_CVTPx2PI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1182 { 1183 gen_helper_enter_mmx(cpu_env); 1184 if (s->prefix & PREFIX_DATA) { 1185 gen_helper_cvtpd2pi(cpu_env, OP_PTR0, OP_PTR2); 1186 } else { 1187 gen_helper_cvtps2pi(cpu_env, OP_PTR0, OP_PTR2); 1188 } 1189 } 1190 1191 static void gen_CVTTPx2PI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1192 { 1193 gen_helper_enter_mmx(cpu_env); 1194 if (s->prefix & PREFIX_DATA) { 1195 gen_helper_cvttpd2pi(cpu_env, OP_PTR0, OP_PTR2); 1196 } else { 1197 gen_helper_cvttps2pi(cpu_env, OP_PTR0, OP_PTR2); 1198 } 1199 } 1200 1201 static void gen_EMMS(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1202 { 1203 gen_helper_emms(cpu_env); 1204 } 1205 1206 static void gen_EXTRQ_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1207 { 1208 TCGv_i32 length = tcg_constant_i32(decode->immediate & 63); 1209 TCGv_i32 index = tcg_constant_i32((decode->immediate >> 8) & 63); 1210 1211 gen_helper_extrq_i(cpu_env, OP_PTR0, index, length); 1212 } 1213 1214 static void gen_EXTRQ_r(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1215 { 1216 gen_helper_extrq_r(cpu_env, OP_PTR0, OP_PTR2); 1217 } 1218 1219 static void gen_INSERTQ_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1220 { 1221 TCGv_i32 length = tcg_constant_i32(decode->immediate & 63); 1222 TCGv_i32 index = tcg_constant_i32((decode->immediate >> 8) & 63); 1223 1224 gen_helper_insertq_i(cpu_env, OP_PTR0, OP_PTR1, index, length); 1225 } 1226 1227 static void gen_INSERTQ_r(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1228 { 1229 gen_helper_insertq_r(cpu_env, OP_PTR0, OP_PTR2); 1230 } 1231 1232 static void gen_LDMXCSR(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1233 { 1234 if (s->vex_l) { 1235 gen_illegal_opcode(s); 1236 return; 1237 } 1238 tcg_gen_trunc_tl_i32(s->tmp2_i32, s->T1); 1239 gen_helper_ldmxcsr(cpu_env, s->tmp2_i32); 1240 } 1241 1242 static void gen_MASKMOV(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1243 { 1244 tcg_gen_mov_tl(s->A0, cpu_regs[R_EDI]); 1245 gen_extu(s->aflag, s->A0); 1246 gen_add_A0_ds_seg(s); 1247 1248 if (s->prefix & PREFIX_DATA) { 1249 gen_helper_maskmov_xmm(cpu_env, OP_PTR1, OP_PTR2, s->A0); 1250 } else { 1251 gen_helper_maskmov_mmx(cpu_env, OP_PTR1, OP_PTR2, s->A0); 1252 } 1253 } 1254 1255 static void gen_MOVBE(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1256 { 1257 MemOp ot = decode->op[0].ot; 1258 1259 /* M operand type does not load/store */ 1260 if (decode->e.op0 == X86_TYPE_M) { 1261 tcg_gen_qemu_st_tl(s->T0, s->A0, s->mem_index, ot | MO_BE); 1262 } else { 1263 tcg_gen_qemu_ld_tl(s->T0, s->A0, s->mem_index, ot | MO_BE); 1264 } 1265 } 1266 1267 static void gen_MOVD_from(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1268 { 1269 MemOp ot = decode->op[2].ot; 1270 1271 switch (ot) { 1272 case MO_32: 1273 #ifdef TARGET_X86_64 1274 tcg_gen_ld32u_tl(s->T0, cpu_env, decode->op[2].offset); 1275 break; 1276 case MO_64: 1277 #endif 1278 tcg_gen_ld_tl(s->T0, cpu_env, decode->op[2].offset); 1279 break; 1280 default: 1281 abort(); 1282 } 1283 } 1284 1285 static void gen_MOVD_to(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1286 { 1287 MemOp ot = decode->op[2].ot; 1288 int vec_len = vector_len(s, decode); 1289 int lo_ofs = vector_elem_offset(&decode->op[0], ot, 0); 1290 1291 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 1292 1293 switch (ot) { 1294 case MO_32: 1295 #ifdef TARGET_X86_64 1296 tcg_gen_st32_tl(s->T1, cpu_env, lo_ofs); 1297 break; 1298 case MO_64: 1299 #endif 1300 tcg_gen_st_tl(s->T1, cpu_env, lo_ofs); 1301 break; 1302 default: 1303 g_assert_not_reached(); 1304 } 1305 } 1306 1307 static void gen_MOVDQ(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1308 { 1309 gen_store_sse(s, decode, decode->op[2].offset); 1310 } 1311 1312 static void gen_MOVMSK(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1313 { 1314 typeof(gen_helper_movmskps_ymm) *ps, *pd, *fn; 1315 ps = s->vex_l ? gen_helper_movmskps_ymm : gen_helper_movmskps_xmm; 1316 pd = s->vex_l ? gen_helper_movmskpd_ymm : gen_helper_movmskpd_xmm; 1317 fn = s->prefix & PREFIX_DATA ? pd : ps; 1318 fn(s->tmp2_i32, cpu_env, OP_PTR2); 1319 tcg_gen_extu_i32_tl(s->T0, s->tmp2_i32); 1320 } 1321 1322 static void gen_MOVQ(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1323 { 1324 int vec_len = vector_len(s, decode); 1325 int lo_ofs = vector_elem_offset(&decode->op[0], MO_64, 0); 1326 1327 tcg_gen_ld_i64(s->tmp1_i64, cpu_env, decode->op[2].offset); 1328 if (decode->op[0].has_ea) { 1329 tcg_gen_qemu_st_i64(s->tmp1_i64, s->A0, s->mem_index, MO_LEUQ); 1330 } else { 1331 /* 1332 * tcg_gen_gvec_dup_i64(MO_64, op0.offset, 8, vec_len, s->tmp1_64) would 1333 * seem to work, but it does not on big-endian platforms; the cleared parts 1334 * are always at higher addresses, but cross-endian emulation inverts the 1335 * byte order so that the cleared parts need to be at *lower* addresses. 1336 * Because oprsz is 8, we see this here even for SSE; but more in general, 1337 * it disqualifies using oprsz < maxsz to emulate VEX128. 1338 */ 1339 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 1340 tcg_gen_st_i64(s->tmp1_i64, cpu_env, lo_ofs); 1341 } 1342 } 1343 1344 static void gen_MOVq_dq(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1345 { 1346 gen_helper_enter_mmx(cpu_env); 1347 /* Otherwise the same as any other movq. */ 1348 return gen_MOVQ(s, env, decode); 1349 } 1350 1351 static void gen_MULX(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1352 { 1353 MemOp ot = decode->op[0].ot; 1354 1355 /* low part of result in VEX.vvvv, high in MODRM */ 1356 switch (ot) { 1357 default: 1358 tcg_gen_trunc_tl_i32(s->tmp2_i32, s->T0); 1359 tcg_gen_trunc_tl_i32(s->tmp3_i32, s->T1); 1360 tcg_gen_mulu2_i32(s->tmp2_i32, s->tmp3_i32, 1361 s->tmp2_i32, s->tmp3_i32); 1362 tcg_gen_extu_i32_tl(cpu_regs[s->vex_v], s->tmp2_i32); 1363 tcg_gen_extu_i32_tl(s->T0, s->tmp3_i32); 1364 break; 1365 #ifdef TARGET_X86_64 1366 case MO_64: 1367 tcg_gen_mulu2_i64(cpu_regs[s->vex_v], s->T0, s->T0, s->T1); 1368 break; 1369 #endif 1370 } 1371 1372 } 1373 1374 static void gen_PALIGNR(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1375 { 1376 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 1377 if (!(s->prefix & PREFIX_DATA)) { 1378 gen_helper_palignr_mmx(cpu_env, OP_PTR0, OP_PTR1, OP_PTR2, imm); 1379 } else if (!s->vex_l) { 1380 gen_helper_palignr_xmm(cpu_env, OP_PTR0, OP_PTR1, OP_PTR2, imm); 1381 } else { 1382 gen_helper_palignr_ymm(cpu_env, OP_PTR0, OP_PTR1, OP_PTR2, imm); 1383 } 1384 } 1385 1386 static void gen_PANDN(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1387 { 1388 int vec_len = vector_len(s, decode); 1389 1390 /* Careful, operand order is reversed! */ 1391 tcg_gen_gvec_andc(MO_64, 1392 decode->op[0].offset, decode->op[2].offset, 1393 decode->op[1].offset, vec_len, vec_len); 1394 } 1395 1396 static void gen_PCMPESTRI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1397 { 1398 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 1399 gen_helper_pcmpestri_xmm(cpu_env, OP_PTR1, OP_PTR2, imm); 1400 set_cc_op(s, CC_OP_EFLAGS); 1401 } 1402 1403 static void gen_PCMPESTRM(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1404 { 1405 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 1406 gen_helper_pcmpestrm_xmm(cpu_env, OP_PTR1, OP_PTR2, imm); 1407 set_cc_op(s, CC_OP_EFLAGS); 1408 if ((s->prefix & PREFIX_VEX) && !s->vex_l) { 1409 tcg_gen_gvec_dup_imm(MO_64, offsetof(CPUX86State, xmm_regs[0].ZMM_X(1)), 1410 16, 16, 0); 1411 } 1412 } 1413 1414 static void gen_PCMPISTRI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1415 { 1416 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 1417 gen_helper_pcmpistri_xmm(cpu_env, OP_PTR1, OP_PTR2, imm); 1418 set_cc_op(s, CC_OP_EFLAGS); 1419 } 1420 1421 static void gen_PCMPISTRM(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1422 { 1423 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 1424 gen_helper_pcmpistrm_xmm(cpu_env, OP_PTR1, OP_PTR2, imm); 1425 set_cc_op(s, CC_OP_EFLAGS); 1426 if ((s->prefix & PREFIX_VEX) && !s->vex_l) { 1427 tcg_gen_gvec_dup_imm(MO_64, offsetof(CPUX86State, xmm_regs[0].ZMM_X(1)), 1428 16, 16, 0); 1429 } 1430 } 1431 1432 static void gen_PDEP(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1433 { 1434 MemOp ot = decode->op[1].ot; 1435 if (ot < MO_64) { 1436 tcg_gen_ext32u_tl(s->T0, s->T0); 1437 } 1438 gen_helper_pdep(s->T0, s->T0, s->T1); 1439 } 1440 1441 static void gen_PEXT(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1442 { 1443 MemOp ot = decode->op[1].ot; 1444 if (ot < MO_64) { 1445 tcg_gen_ext32u_tl(s->T0, s->T0); 1446 } 1447 gen_helper_pext(s->T0, s->T0, s->T1); 1448 } 1449 1450 static inline void gen_pextr(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode, MemOp ot) 1451 { 1452 int vec_len = vector_len(s, decode); 1453 int mask = (vec_len >> ot) - 1; 1454 int val = decode->immediate & mask; 1455 1456 switch (ot) { 1457 case MO_8: 1458 tcg_gen_ld8u_tl(s->T0, cpu_env, vector_elem_offset(&decode->op[1], ot, val)); 1459 break; 1460 case MO_16: 1461 tcg_gen_ld16u_tl(s->T0, cpu_env, vector_elem_offset(&decode->op[1], ot, val)); 1462 break; 1463 case MO_32: 1464 #ifdef TARGET_X86_64 1465 tcg_gen_ld32u_tl(s->T0, cpu_env, vector_elem_offset(&decode->op[1], ot, val)); 1466 break; 1467 case MO_64: 1468 #endif 1469 tcg_gen_ld_tl(s->T0, cpu_env, vector_elem_offset(&decode->op[1], ot, val)); 1470 break; 1471 default: 1472 abort(); 1473 } 1474 } 1475 1476 static void gen_PEXTRB(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1477 { 1478 gen_pextr(s, env, decode, MO_8); 1479 } 1480 1481 static void gen_PEXTRW(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1482 { 1483 gen_pextr(s, env, decode, MO_16); 1484 } 1485 1486 static void gen_PEXTR(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1487 { 1488 MemOp ot = decode->op[0].ot; 1489 gen_pextr(s, env, decode, ot); 1490 } 1491 1492 static inline void gen_pinsr(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode, MemOp ot) 1493 { 1494 int vec_len = vector_len(s, decode); 1495 int mask = (vec_len >> ot) - 1; 1496 int val = decode->immediate & mask; 1497 1498 if (decode->op[1].offset != decode->op[0].offset) { 1499 assert(vec_len == 16); 1500 gen_store_sse(s, decode, decode->op[1].offset); 1501 } 1502 1503 switch (ot) { 1504 case MO_8: 1505 tcg_gen_st8_tl(s->T1, cpu_env, vector_elem_offset(&decode->op[0], ot, val)); 1506 break; 1507 case MO_16: 1508 tcg_gen_st16_tl(s->T1, cpu_env, vector_elem_offset(&decode->op[0], ot, val)); 1509 break; 1510 case MO_32: 1511 #ifdef TARGET_X86_64 1512 tcg_gen_st32_tl(s->T1, cpu_env, vector_elem_offset(&decode->op[0], ot, val)); 1513 break; 1514 case MO_64: 1515 #endif 1516 tcg_gen_st_tl(s->T1, cpu_env, vector_elem_offset(&decode->op[0], ot, val)); 1517 break; 1518 default: 1519 abort(); 1520 } 1521 } 1522 1523 static void gen_PINSRB(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1524 { 1525 gen_pinsr(s, env, decode, MO_8); 1526 } 1527 1528 static void gen_PINSRW(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1529 { 1530 gen_pinsr(s, env, decode, MO_16); 1531 } 1532 1533 static void gen_PINSR(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1534 { 1535 gen_pinsr(s, env, decode, decode->op[2].ot); 1536 } 1537 1538 static void gen_pmovmskb_i64(TCGv_i64 d, TCGv_i64 s) 1539 { 1540 TCGv_i64 t = tcg_temp_new_i64(); 1541 1542 tcg_gen_andi_i64(d, s, 0x8080808080808080ull); 1543 1544 /* 1545 * After each shift+or pair: 1546 * 0: a.......b.......c.......d.......e.......f.......g.......h....... 1547 * 7: ab......bc......cd......de......ef......fg......gh......h....... 1548 * 14: abcd....bcde....cdef....defg....efgh....fgh.....gh......h....... 1549 * 28: abcdefghbcdefgh.cdefgh..defgh...efgh....fgh.....gh......h....... 1550 * The result is left in the high bits of the word. 1551 */ 1552 tcg_gen_shli_i64(t, d, 7); 1553 tcg_gen_or_i64(d, d, t); 1554 tcg_gen_shli_i64(t, d, 14); 1555 tcg_gen_or_i64(d, d, t); 1556 tcg_gen_shli_i64(t, d, 28); 1557 tcg_gen_or_i64(d, d, t); 1558 } 1559 1560 static void gen_pmovmskb_vec(unsigned vece, TCGv_vec d, TCGv_vec s) 1561 { 1562 TCGv_vec t = tcg_temp_new_vec_matching(d); 1563 TCGv_vec m = tcg_constant_vec_matching(d, MO_8, 0x80); 1564 1565 /* See above */ 1566 tcg_gen_and_vec(vece, d, s, m); 1567 tcg_gen_shli_vec(vece, t, d, 7); 1568 tcg_gen_or_vec(vece, d, d, t); 1569 tcg_gen_shli_vec(vece, t, d, 14); 1570 tcg_gen_or_vec(vece, d, d, t); 1571 tcg_gen_shli_vec(vece, t, d, 28); 1572 tcg_gen_or_vec(vece, d, d, t); 1573 } 1574 1575 #ifdef TARGET_X86_64 1576 #define TCG_TARGET_HAS_extract2_tl TCG_TARGET_HAS_extract2_i64 1577 #else 1578 #define TCG_TARGET_HAS_extract2_tl TCG_TARGET_HAS_extract2_i32 1579 #endif 1580 1581 static void gen_PMOVMSKB(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1582 { 1583 static const TCGOpcode vecop_list[] = { INDEX_op_shli_vec, 0 }; 1584 static const GVecGen2 g = { 1585 .fni8 = gen_pmovmskb_i64, 1586 .fniv = gen_pmovmskb_vec, 1587 .opt_opc = vecop_list, 1588 .vece = MO_64, 1589 .prefer_i64 = TCG_TARGET_REG_BITS == 64 1590 }; 1591 MemOp ot = decode->op[2].ot; 1592 int vec_len = vector_len(s, decode); 1593 TCGv t = tcg_temp_new(); 1594 1595 tcg_gen_gvec_2(offsetof(CPUX86State, xmm_t0) + xmm_offset(ot), decode->op[2].offset, 1596 vec_len, vec_len, &g); 1597 tcg_gen_ld8u_tl(s->T0, cpu_env, offsetof(CPUX86State, xmm_t0.ZMM_B(vec_len - 1))); 1598 while (vec_len > 8) { 1599 vec_len -= 8; 1600 if (TCG_TARGET_HAS_extract2_tl) { 1601 /* 1602 * Load the next byte of the result into the high byte of T. 1603 * TCG does a similar expansion of deposit to shl+extract2; by 1604 * loading the whole word, the shift left is avoided. 1605 */ 1606 #ifdef TARGET_X86_64 1607 tcg_gen_ld_tl(t, cpu_env, offsetof(CPUX86State, xmm_t0.ZMM_Q((vec_len - 1) / 8))); 1608 #else 1609 tcg_gen_ld_tl(t, cpu_env, offsetof(CPUX86State, xmm_t0.ZMM_L((vec_len - 1) / 4))); 1610 #endif 1611 1612 tcg_gen_extract2_tl(s->T0, t, s->T0, TARGET_LONG_BITS - 8); 1613 } else { 1614 /* 1615 * The _previous_ value is deposited into bits 8 and higher of t. Because 1616 * those bits are known to be zero after ld8u, this becomes a shift+or 1617 * if deposit is not available. 1618 */ 1619 tcg_gen_ld8u_tl(t, cpu_env, offsetof(CPUX86State, xmm_t0.ZMM_B(vec_len - 1))); 1620 tcg_gen_deposit_tl(s->T0, t, s->T0, 8, TARGET_LONG_BITS - 8); 1621 } 1622 } 1623 tcg_temp_free(t); 1624 } 1625 1626 static void gen_PSHUFW(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1627 { 1628 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 1629 gen_helper_pshufw_mmx(OP_PTR0, OP_PTR1, imm); 1630 } 1631 1632 static void gen_PSRLW_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1633 { 1634 int vec_len = vector_len(s, decode); 1635 1636 if (decode->immediate >= 16) { 1637 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 1638 } else { 1639 tcg_gen_gvec_shri(MO_16, 1640 decode->op[0].offset, decode->op[1].offset, 1641 decode->immediate, vec_len, vec_len); 1642 } 1643 } 1644 1645 static void gen_PSLLW_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1646 { 1647 int vec_len = vector_len(s, decode); 1648 1649 if (decode->immediate >= 16) { 1650 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 1651 } else { 1652 tcg_gen_gvec_shli(MO_16, 1653 decode->op[0].offset, decode->op[1].offset, 1654 decode->immediate, vec_len, vec_len); 1655 } 1656 } 1657 1658 static void gen_PSRAW_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1659 { 1660 int vec_len = vector_len(s, decode); 1661 1662 if (decode->immediate >= 16) { 1663 decode->immediate = 15; 1664 } 1665 tcg_gen_gvec_sari(MO_16, 1666 decode->op[0].offset, decode->op[1].offset, 1667 decode->immediate, vec_len, vec_len); 1668 } 1669 1670 static void gen_PSRLD_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1671 { 1672 int vec_len = vector_len(s, decode); 1673 1674 if (decode->immediate >= 32) { 1675 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 1676 } else { 1677 tcg_gen_gvec_shri(MO_32, 1678 decode->op[0].offset, decode->op[1].offset, 1679 decode->immediate, vec_len, vec_len); 1680 } 1681 } 1682 1683 static void gen_PSLLD_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1684 { 1685 int vec_len = vector_len(s, decode); 1686 1687 if (decode->immediate >= 32) { 1688 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 1689 } else { 1690 tcg_gen_gvec_shli(MO_32, 1691 decode->op[0].offset, decode->op[1].offset, 1692 decode->immediate, vec_len, vec_len); 1693 } 1694 } 1695 1696 static void gen_PSRAD_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1697 { 1698 int vec_len = vector_len(s, decode); 1699 1700 if (decode->immediate >= 32) { 1701 decode->immediate = 31; 1702 } 1703 tcg_gen_gvec_sari(MO_32, 1704 decode->op[0].offset, decode->op[1].offset, 1705 decode->immediate, vec_len, vec_len); 1706 } 1707 1708 static void gen_PSRLQ_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1709 { 1710 int vec_len = vector_len(s, decode); 1711 1712 if (decode->immediate >= 64) { 1713 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 1714 } else { 1715 tcg_gen_gvec_shri(MO_64, 1716 decode->op[0].offset, decode->op[1].offset, 1717 decode->immediate, vec_len, vec_len); 1718 } 1719 } 1720 1721 static void gen_PSLLQ_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1722 { 1723 int vec_len = vector_len(s, decode); 1724 1725 if (decode->immediate >= 64) { 1726 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 1727 } else { 1728 tcg_gen_gvec_shli(MO_64, 1729 decode->op[0].offset, decode->op[1].offset, 1730 decode->immediate, vec_len, vec_len); 1731 } 1732 } 1733 1734 static TCGv_ptr make_imm8u_xmm_vec(uint8_t imm, int vec_len) 1735 { 1736 MemOp ot = vec_len == 16 ? MO_128 : MO_256; 1737 TCGv_i32 imm_v = tcg_constant8u_i32(imm); 1738 TCGv_ptr ptr = tcg_temp_new_ptr(); 1739 1740 tcg_gen_gvec_dup_imm(MO_64, offsetof(CPUX86State, xmm_t0) + xmm_offset(ot), 1741 vec_len, vec_len, 0); 1742 1743 tcg_gen_addi_ptr(ptr, cpu_env, offsetof(CPUX86State, xmm_t0)); 1744 tcg_gen_st_i32(imm_v, cpu_env, offsetof(CPUX86State, xmm_t0.ZMM_L(0))); 1745 return ptr; 1746 } 1747 1748 static void gen_PSRLDQ_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1749 { 1750 int vec_len = vector_len(s, decode); 1751 TCGv_ptr imm_vec = make_imm8u_xmm_vec(decode->immediate, vec_len); 1752 1753 if (s->vex_l) { 1754 gen_helper_psrldq_ymm(cpu_env, OP_PTR0, OP_PTR1, imm_vec); 1755 } else { 1756 gen_helper_psrldq_xmm(cpu_env, OP_PTR0, OP_PTR1, imm_vec); 1757 } 1758 tcg_temp_free_ptr(imm_vec); 1759 } 1760 1761 static void gen_PSLLDQ_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1762 { 1763 int vec_len = vector_len(s, decode); 1764 TCGv_ptr imm_vec = make_imm8u_xmm_vec(decode->immediate, vec_len); 1765 1766 if (s->vex_l) { 1767 gen_helper_pslldq_ymm(cpu_env, OP_PTR0, OP_PTR1, imm_vec); 1768 } else { 1769 gen_helper_pslldq_xmm(cpu_env, OP_PTR0, OP_PTR1, imm_vec); 1770 } 1771 tcg_temp_free_ptr(imm_vec); 1772 } 1773 1774 static void gen_RORX(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1775 { 1776 MemOp ot = decode->op[0].ot; 1777 int b = decode->immediate; 1778 1779 if (ot == MO_64) { 1780 tcg_gen_rotri_tl(s->T0, s->T0, b & 63); 1781 } else { 1782 tcg_gen_trunc_tl_i32(s->tmp2_i32, s->T0); 1783 tcg_gen_rotri_i32(s->tmp2_i32, s->tmp2_i32, b & 31); 1784 tcg_gen_extu_i32_tl(s->T0, s->tmp2_i32); 1785 } 1786 } 1787 1788 static void gen_SARX(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1789 { 1790 MemOp ot = decode->op[0].ot; 1791 int mask; 1792 1793 mask = ot == MO_64 ? 63 : 31; 1794 tcg_gen_andi_tl(s->T1, s->T1, mask); 1795 if (ot != MO_64) { 1796 tcg_gen_ext32s_tl(s->T0, s->T0); 1797 } 1798 tcg_gen_sar_tl(s->T0, s->T0, s->T1); 1799 } 1800 1801 static void gen_SHLX(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1802 { 1803 MemOp ot = decode->op[0].ot; 1804 int mask; 1805 1806 mask = ot == MO_64 ? 63 : 31; 1807 tcg_gen_andi_tl(s->T1, s->T1, mask); 1808 tcg_gen_shl_tl(s->T0, s->T0, s->T1); 1809 } 1810 1811 static void gen_SHRX(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1812 { 1813 MemOp ot = decode->op[0].ot; 1814 int mask; 1815 1816 mask = ot == MO_64 ? 63 : 31; 1817 tcg_gen_andi_tl(s->T1, s->T1, mask); 1818 if (ot != MO_64) { 1819 tcg_gen_ext32u_tl(s->T0, s->T0); 1820 } 1821 tcg_gen_shr_tl(s->T0, s->T0, s->T1); 1822 } 1823 1824 static void gen_VAESKEYGEN(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1825 { 1826 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 1827 assert(!s->vex_l); 1828 gen_helper_aeskeygenassist_xmm(cpu_env, OP_PTR0, OP_PTR1, imm); 1829 } 1830 1831 static void gen_STMXCSR(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1832 { 1833 if (s->vex_l) { 1834 gen_illegal_opcode(s); 1835 return; 1836 } 1837 gen_helper_update_mxcsr(cpu_env); 1838 tcg_gen_ld32u_tl(s->T0, cpu_env, offsetof(CPUX86State, mxcsr)); 1839 } 1840 1841 static void gen_VAESIMC(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1842 { 1843 assert(!s->vex_l); 1844 gen_helper_aesimc_xmm(cpu_env, OP_PTR0, OP_PTR2); 1845 } 1846 1847 /* 1848 * 00 = v*ps Vps, Hps, Wpd 1849 * 66 = v*pd Vpd, Hpd, Wps 1850 * f3 = v*ss Vss, Hss, Wps 1851 * f2 = v*sd Vsd, Hsd, Wps 1852 */ 1853 #define SSE_CMP(x) { \ 1854 gen_helper_ ## x ## ps ## _xmm, gen_helper_ ## x ## pd ## _xmm, \ 1855 gen_helper_ ## x ## ss, gen_helper_ ## x ## sd, \ 1856 gen_helper_ ## x ## ps ## _ymm, gen_helper_ ## x ## pd ## _ymm} 1857 static const SSEFunc_0_eppp gen_helper_cmp_funcs[32][6] = { 1858 SSE_CMP(cmpeq), 1859 SSE_CMP(cmplt), 1860 SSE_CMP(cmple), 1861 SSE_CMP(cmpunord), 1862 SSE_CMP(cmpneq), 1863 SSE_CMP(cmpnlt), 1864 SSE_CMP(cmpnle), 1865 SSE_CMP(cmpord), 1866 1867 SSE_CMP(cmpequ), 1868 SSE_CMP(cmpnge), 1869 SSE_CMP(cmpngt), 1870 SSE_CMP(cmpfalse), 1871 SSE_CMP(cmpnequ), 1872 SSE_CMP(cmpge), 1873 SSE_CMP(cmpgt), 1874 SSE_CMP(cmptrue), 1875 1876 SSE_CMP(cmpeqs), 1877 SSE_CMP(cmpltq), 1878 SSE_CMP(cmpleq), 1879 SSE_CMP(cmpunords), 1880 SSE_CMP(cmpneqq), 1881 SSE_CMP(cmpnltq), 1882 SSE_CMP(cmpnleq), 1883 SSE_CMP(cmpords), 1884 1885 SSE_CMP(cmpequs), 1886 SSE_CMP(cmpngeq), 1887 SSE_CMP(cmpngtq), 1888 SSE_CMP(cmpfalses), 1889 SSE_CMP(cmpnequs), 1890 SSE_CMP(cmpgeq), 1891 SSE_CMP(cmpgtq), 1892 SSE_CMP(cmptrues), 1893 }; 1894 #undef SSE_CMP 1895 1896 static void gen_VCMP(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1897 { 1898 int index = decode->immediate & (s->prefix & PREFIX_VEX ? 31 : 7); 1899 int b = 1900 s->prefix & PREFIX_REPZ ? 2 /* ss */ : 1901 s->prefix & PREFIX_REPNZ ? 3 /* sd */ : 1902 !!(s->prefix & PREFIX_DATA) /* pd */ + (s->vex_l << 2); 1903 1904 gen_helper_cmp_funcs[index][b](cpu_env, OP_PTR0, OP_PTR1, OP_PTR2); 1905 } 1906 1907 static void gen_VCOMI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1908 { 1909 SSEFunc_0_epp fn; 1910 fn = s->prefix & PREFIX_DATA ? gen_helper_comisd : gen_helper_comiss; 1911 fn(cpu_env, OP_PTR1, OP_PTR2); 1912 set_cc_op(s, CC_OP_EFLAGS); 1913 } 1914 1915 static void gen_VCVTfp2fp(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1916 { 1917 gen_unary_fp_sse(s, env, decode, 1918 gen_helper_cvtpd2ps_xmm, gen_helper_cvtps2pd_xmm, 1919 gen_helper_cvtpd2ps_ymm, gen_helper_cvtps2pd_ymm, 1920 gen_helper_cvtsd2ss, gen_helper_cvtss2sd); 1921 } 1922 1923 static void gen_VCVTPS2PH(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1924 { 1925 gen_unary_imm_fp_sse(s, env, decode, 1926 gen_helper_cvtps2ph_xmm, 1927 gen_helper_cvtps2ph_ymm); 1928 /* 1929 * VCVTPS2PH is the only instruction that performs an operation on a 1930 * register source and then *stores* into memory. 1931 */ 1932 if (decode->op[0].has_ea) { 1933 gen_store_sse(s, decode, decode->op[0].offset); 1934 } 1935 } 1936 1937 static void gen_VCVTSI2Sx(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1938 { 1939 int vec_len = vector_len(s, decode); 1940 TCGv_i32 in; 1941 1942 tcg_gen_gvec_mov(MO_64, decode->op[0].offset, decode->op[1].offset, vec_len, vec_len); 1943 1944 #ifdef TARGET_X86_64 1945 MemOp ot = decode->op[2].ot; 1946 if (ot == MO_64) { 1947 if (s->prefix & PREFIX_REPNZ) { 1948 gen_helper_cvtsq2sd(cpu_env, OP_PTR0, s->T1); 1949 } else { 1950 gen_helper_cvtsq2ss(cpu_env, OP_PTR0, s->T1); 1951 } 1952 return; 1953 } 1954 in = s->tmp2_i32; 1955 tcg_gen_trunc_tl_i32(in, s->T1); 1956 #else 1957 in = s->T1; 1958 #endif 1959 1960 if (s->prefix & PREFIX_REPNZ) { 1961 gen_helper_cvtsi2sd(cpu_env, OP_PTR0, in); 1962 } else { 1963 gen_helper_cvtsi2ss(cpu_env, OP_PTR0, in); 1964 } 1965 } 1966 1967 static inline void gen_VCVTtSx2SI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode, 1968 SSEFunc_i_ep ss2si, SSEFunc_l_ep ss2sq, 1969 SSEFunc_i_ep sd2si, SSEFunc_l_ep sd2sq) 1970 { 1971 TCGv_i32 out; 1972 1973 #ifdef TARGET_X86_64 1974 MemOp ot = decode->op[0].ot; 1975 if (ot == MO_64) { 1976 if (s->prefix & PREFIX_REPNZ) { 1977 sd2sq(s->T0, cpu_env, OP_PTR2); 1978 } else { 1979 ss2sq(s->T0, cpu_env, OP_PTR2); 1980 } 1981 return; 1982 } 1983 1984 out = s->tmp2_i32; 1985 #else 1986 out = s->T0; 1987 #endif 1988 if (s->prefix & PREFIX_REPNZ) { 1989 sd2si(out, cpu_env, OP_PTR2); 1990 } else { 1991 ss2si(out, cpu_env, OP_PTR2); 1992 } 1993 #ifdef TARGET_X86_64 1994 tcg_gen_extu_i32_tl(s->T0, out); 1995 #endif 1996 } 1997 1998 #ifndef TARGET_X86_64 1999 #define gen_helper_cvtss2sq NULL 2000 #define gen_helper_cvtsd2sq NULL 2001 #define gen_helper_cvttss2sq NULL 2002 #define gen_helper_cvttsd2sq NULL 2003 #endif 2004 2005 static void gen_VCVTSx2SI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2006 { 2007 gen_VCVTtSx2SI(s, env, decode, 2008 gen_helper_cvtss2si, gen_helper_cvtss2sq, 2009 gen_helper_cvtsd2si, gen_helper_cvtsd2sq); 2010 } 2011 2012 static void gen_VCVTTSx2SI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2013 { 2014 gen_VCVTtSx2SI(s, env, decode, 2015 gen_helper_cvttss2si, gen_helper_cvttss2sq, 2016 gen_helper_cvttsd2si, gen_helper_cvttsd2sq); 2017 } 2018 2019 static void gen_VEXTRACTx128(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2020 { 2021 int mask = decode->immediate & 1; 2022 int src_ofs = vector_elem_offset(&decode->op[1], MO_128, mask); 2023 if (decode->op[0].has_ea) { 2024 /* VEX-only instruction, no alignment requirements. */ 2025 gen_sto_env_A0(s, src_ofs, false); 2026 } else { 2027 tcg_gen_gvec_mov(MO_64, decode->op[0].offset, src_ofs, 16, 16); 2028 } 2029 } 2030 2031 static void gen_VEXTRACTPS(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2032 { 2033 gen_pextr(s, env, decode, MO_32); 2034 } 2035 2036 static void gen_vinsertps(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2037 { 2038 int val = decode->immediate; 2039 int dest_word = (val >> 4) & 3; 2040 int new_mask = (val & 15) | (1 << dest_word); 2041 int vec_len = 16; 2042 2043 assert(!s->vex_l); 2044 2045 if (new_mask == 15) { 2046 /* All zeroes except possibly for the inserted element */ 2047 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 2048 } else if (decode->op[1].offset != decode->op[0].offset) { 2049 gen_store_sse(s, decode, decode->op[1].offset); 2050 } 2051 2052 if (new_mask != (val & 15)) { 2053 tcg_gen_st_i32(s->tmp2_i32, cpu_env, 2054 vector_elem_offset(&decode->op[0], MO_32, dest_word)); 2055 } 2056 2057 if (new_mask != 15) { 2058 TCGv_i32 zero = tcg_constant_i32(0); /* float32_zero */ 2059 int i; 2060 for (i = 0; i < 4; i++) { 2061 if ((val >> i) & 1) { 2062 tcg_gen_st_i32(zero, cpu_env, 2063 vector_elem_offset(&decode->op[0], MO_32, i)); 2064 } 2065 } 2066 } 2067 } 2068 2069 static void gen_VINSERTPS_r(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2070 { 2071 int val = decode->immediate; 2072 tcg_gen_ld_i32(s->tmp2_i32, cpu_env, 2073 vector_elem_offset(&decode->op[2], MO_32, (val >> 6) & 3)); 2074 gen_vinsertps(s, env, decode); 2075 } 2076 2077 static void gen_VINSERTPS_m(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2078 { 2079 tcg_gen_qemu_ld_i32(s->tmp2_i32, s->A0, s->mem_index, MO_LEUL); 2080 gen_vinsertps(s, env, decode); 2081 } 2082 2083 static void gen_VINSERTx128(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2084 { 2085 int mask = decode->immediate & 1; 2086 tcg_gen_gvec_mov(MO_64, 2087 decode->op[0].offset + offsetof(YMMReg, YMM_X(mask)), 2088 decode->op[2].offset + offsetof(YMMReg, YMM_X(0)), 16, 16); 2089 tcg_gen_gvec_mov(MO_64, 2090 decode->op[0].offset + offsetof(YMMReg, YMM_X(!mask)), 2091 decode->op[1].offset + offsetof(YMMReg, YMM_X(!mask)), 16, 16); 2092 } 2093 2094 static inline void gen_maskmov(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode, 2095 SSEFunc_0_eppt xmm, SSEFunc_0_eppt ymm) 2096 { 2097 if (!s->vex_l) { 2098 xmm(cpu_env, OP_PTR2, OP_PTR1, s->A0); 2099 } else { 2100 ymm(cpu_env, OP_PTR2, OP_PTR1, s->A0); 2101 } 2102 } 2103 2104 static void gen_VMASKMOVPD_st(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2105 { 2106 gen_maskmov(s, env, decode, gen_helper_vpmaskmovq_st_xmm, gen_helper_vpmaskmovq_st_ymm); 2107 } 2108 2109 static void gen_VMASKMOVPS_st(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2110 { 2111 gen_maskmov(s, env, decode, gen_helper_vpmaskmovd_st_xmm, gen_helper_vpmaskmovd_st_ymm); 2112 } 2113 2114 static void gen_VMOVHPx_ld(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2115 { 2116 gen_ldq_env_A0(s, decode->op[0].offset + offsetof(XMMReg, XMM_Q(1))); 2117 if (decode->op[0].offset != decode->op[1].offset) { 2118 tcg_gen_ld_i64(s->tmp1_i64, cpu_env, decode->op[1].offset + offsetof(XMMReg, XMM_Q(0))); 2119 tcg_gen_st_i64(s->tmp1_i64, cpu_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(0))); 2120 } 2121 } 2122 2123 static void gen_VMOVHPx_st(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2124 { 2125 gen_stq_env_A0(s, decode->op[2].offset + offsetof(XMMReg, XMM_Q(1))); 2126 } 2127 2128 static void gen_VMOVHPx(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2129 { 2130 if (decode->op[0].offset != decode->op[2].offset) { 2131 tcg_gen_ld_i64(s->tmp1_i64, cpu_env, decode->op[2].offset + offsetof(XMMReg, XMM_Q(1))); 2132 tcg_gen_st_i64(s->tmp1_i64, cpu_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(1))); 2133 } 2134 if (decode->op[0].offset != decode->op[1].offset) { 2135 tcg_gen_ld_i64(s->tmp1_i64, cpu_env, decode->op[1].offset + offsetof(XMMReg, XMM_Q(0))); 2136 tcg_gen_st_i64(s->tmp1_i64, cpu_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(0))); 2137 } 2138 } 2139 2140 static void gen_VMOVHLPS(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2141 { 2142 tcg_gen_ld_i64(s->tmp1_i64, cpu_env, decode->op[2].offset + offsetof(XMMReg, XMM_Q(1))); 2143 tcg_gen_st_i64(s->tmp1_i64, cpu_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(0))); 2144 if (decode->op[0].offset != decode->op[1].offset) { 2145 tcg_gen_ld_i64(s->tmp1_i64, cpu_env, decode->op[1].offset + offsetof(XMMReg, XMM_Q(1))); 2146 tcg_gen_st_i64(s->tmp1_i64, cpu_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(1))); 2147 } 2148 } 2149 2150 static void gen_VMOVLHPS(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2151 { 2152 tcg_gen_ld_i64(s->tmp1_i64, cpu_env, decode->op[2].offset); 2153 tcg_gen_st_i64(s->tmp1_i64, cpu_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(1))); 2154 if (decode->op[0].offset != decode->op[1].offset) { 2155 tcg_gen_ld_i64(s->tmp1_i64, cpu_env, decode->op[1].offset + offsetof(XMMReg, XMM_Q(0))); 2156 tcg_gen_st_i64(s->tmp1_i64, cpu_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(0))); 2157 } 2158 } 2159 2160 /* 2161 * Note that MOVLPx supports 256-bit operation unlike MOVHLPx, MOVLHPx, MOXHPx. 2162 * Use a gvec move to move everything above the bottom 64 bits. 2163 */ 2164 2165 static void gen_VMOVLPx(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2166 { 2167 int vec_len = vector_len(s, decode); 2168 2169 tcg_gen_ld_i64(s->tmp1_i64, cpu_env, decode->op[2].offset + offsetof(XMMReg, XMM_Q(0))); 2170 tcg_gen_gvec_mov(MO_64, decode->op[0].offset, decode->op[1].offset, vec_len, vec_len); 2171 tcg_gen_st_i64(s->tmp1_i64, cpu_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(0))); 2172 } 2173 2174 static void gen_VMOVLPx_ld(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2175 { 2176 int vec_len = vector_len(s, decode); 2177 2178 tcg_gen_qemu_ld_i64(s->tmp1_i64, s->A0, s->mem_index, MO_LEUQ); 2179 tcg_gen_gvec_mov(MO_64, decode->op[0].offset, decode->op[1].offset, vec_len, vec_len); 2180 tcg_gen_st_i64(s->tmp1_i64, OP_PTR0, offsetof(ZMMReg, ZMM_Q(0))); 2181 } 2182 2183 static void gen_VMOVLPx_st(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2184 { 2185 tcg_gen_ld_i64(s->tmp1_i64, OP_PTR2, offsetof(ZMMReg, ZMM_Q(0))); 2186 tcg_gen_qemu_st_i64(s->tmp1_i64, s->A0, s->mem_index, MO_LEUQ); 2187 } 2188 2189 static void gen_VMOVSD_ld(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2190 { 2191 TCGv_i64 zero = tcg_constant_i64(0); 2192 2193 tcg_gen_qemu_ld_i64(s->tmp1_i64, s->A0, s->mem_index, MO_LEUQ); 2194 tcg_gen_st_i64(zero, OP_PTR0, offsetof(ZMMReg, ZMM_Q(1))); 2195 tcg_gen_st_i64(s->tmp1_i64, OP_PTR0, offsetof(ZMMReg, ZMM_Q(0))); 2196 } 2197 2198 static void gen_VMOVSS(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2199 { 2200 int vec_len = vector_len(s, decode); 2201 2202 tcg_gen_ld_i32(s->tmp2_i32, OP_PTR2, offsetof(ZMMReg, ZMM_L(0))); 2203 tcg_gen_gvec_mov(MO_64, decode->op[0].offset, decode->op[1].offset, vec_len, vec_len); 2204 tcg_gen_st_i32(s->tmp2_i32, OP_PTR0, offsetof(ZMMReg, ZMM_L(0))); 2205 } 2206 2207 static void gen_VMOVSS_ld(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2208 { 2209 int vec_len = vector_len(s, decode); 2210 2211 tcg_gen_qemu_ld_i32(s->tmp2_i32, s->A0, s->mem_index, MO_LEUL); 2212 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 2213 tcg_gen_st_i32(s->tmp2_i32, OP_PTR0, offsetof(ZMMReg, ZMM_L(0))); 2214 } 2215 2216 static void gen_VMOVSS_st(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2217 { 2218 tcg_gen_ld_i32(s->tmp2_i32, OP_PTR2, offsetof(ZMMReg, ZMM_L(0))); 2219 tcg_gen_qemu_st_i32(s->tmp2_i32, s->A0, s->mem_index, MO_LEUL); 2220 } 2221 2222 static void gen_VPMASKMOV_st(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2223 { 2224 if (s->vex_w) { 2225 gen_VMASKMOVPD_st(s, env, decode); 2226 } else { 2227 gen_VMASKMOVPS_st(s, env, decode); 2228 } 2229 } 2230 2231 static void gen_VPERMD(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2232 { 2233 assert(s->vex_l); 2234 gen_helper_vpermd_ymm(OP_PTR0, OP_PTR1, OP_PTR2); 2235 } 2236 2237 static void gen_VPERM2x128(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2238 { 2239 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 2240 assert(s->vex_l); 2241 gen_helper_vpermdq_ymm(OP_PTR0, OP_PTR1, OP_PTR2, imm); 2242 } 2243 2244 static void gen_VPHMINPOSUW(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2245 { 2246 assert(!s->vex_l); 2247 gen_helper_phminposuw_xmm(cpu_env, OP_PTR0, OP_PTR2); 2248 } 2249 2250 static void gen_VROUNDSD(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2251 { 2252 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 2253 assert(!s->vex_l); 2254 gen_helper_roundsd_xmm(cpu_env, OP_PTR0, OP_PTR1, OP_PTR2, imm); 2255 } 2256 2257 static void gen_VROUNDSS(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2258 { 2259 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 2260 assert(!s->vex_l); 2261 gen_helper_roundss_xmm(cpu_env, OP_PTR0, OP_PTR1, OP_PTR2, imm); 2262 } 2263 2264 static void gen_VSHUF(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2265 { 2266 TCGv_i32 imm = tcg_constant_i32(decode->immediate); 2267 SSEFunc_0_pppi ps, pd, fn; 2268 ps = s->vex_l ? gen_helper_shufps_ymm : gen_helper_shufps_xmm; 2269 pd = s->vex_l ? gen_helper_shufpd_ymm : gen_helper_shufpd_xmm; 2270 fn = s->prefix & PREFIX_DATA ? pd : ps; 2271 fn(OP_PTR0, OP_PTR1, OP_PTR2, imm); 2272 } 2273 2274 static void gen_VUCOMI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2275 { 2276 SSEFunc_0_epp fn; 2277 fn = s->prefix & PREFIX_DATA ? gen_helper_ucomisd : gen_helper_ucomiss; 2278 fn(cpu_env, OP_PTR1, OP_PTR2); 2279 set_cc_op(s, CC_OP_EFLAGS); 2280 } 2281 2282 static void gen_VZEROALL(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2283 { 2284 TCGv_ptr ptr = tcg_temp_new_ptr(); 2285 2286 tcg_gen_addi_ptr(ptr, cpu_env, offsetof(CPUX86State, xmm_t0)); 2287 gen_helper_memset(ptr, ptr, tcg_constant_i32(0), 2288 tcg_constant_ptr(CPU_NB_REGS * sizeof(ZMMReg))); 2289 tcg_temp_free_ptr(ptr); 2290 } 2291 2292 static void gen_VZEROUPPER(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2293 { 2294 int i; 2295 2296 for (i = 0; i < CPU_NB_REGS; i++) { 2297 int offset = offsetof(CPUX86State, xmm_regs[i].ZMM_X(1)); 2298 tcg_gen_gvec_dup_imm(MO_64, offset, 16, 16, 0); 2299 } 2300 }