diff options
| -rw-r--r-- | compiler/utils/x86_64/assembler_x86_64.cc | 891 | ||||
| -rw-r--r-- | compiler/utils/x86_64/assembler_x86_64.h | 7 |
2 files changed, 86 insertions, 812 deletions
diff --git a/compiler/utils/x86_64/assembler_x86_64.cc b/compiler/utils/x86_64/assembler_x86_64.cc index 959e291dc9..6330cc5d62 100644 --- a/compiler/utils/x86_64/assembler_x86_64.cc +++ b/compiler/utils/x86_64/assembler_x86_64.cc @@ -736,59 +736,12 @@ void X86_64Assembler::subps(XmmRegister dst, XmmRegister src) { } void X86_64Assembler::vaddps(XmmRegister dst, XmmRegister add_left, XmmRegister add_right) { - DCHECK(CpuHasAVXorAVX2FeatureFlag()); - bool is_twobyte_form = false; - uint8_t ByteZero = 0x00, ByteOne = 0x00, ByteTwo = 0x00; - if (!add_right.NeedsRex()) { - is_twobyte_form = true; - } else if (!add_left.NeedsRex()) { - return vaddps(dst, add_right, add_left); - } - AssemblerBuffer::EnsureCapacity ensured(&buffer_); - X86_64ManagedRegister vvvv_reg = - X86_64ManagedRegister::FromXmmRegister(add_left.AsFloatRegister()); - ByteZero = EmitVexPrefixByteZero(is_twobyte_form); - if (is_twobyte_form) { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), vvvv_reg, SET_VEX_L_128, SET_VEX_PP_NONE); - } else { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), - /*X=*/ false, - add_right.NeedsRex(), - SET_VEX_M_0F); - ByteTwo = EmitVexPrefixByteTwo(/*W=*/ false, vvvv_reg, SET_VEX_L_128, SET_VEX_PP_NONE); - } - EmitUint8(ByteZero); - EmitUint8(ByteOne); - if (!is_twobyte_form) { - EmitUint8(ByteTwo); - } - EmitUint8(0x58); - EmitXmmRegisterOperand(dst.LowBits(), add_right); + EmitVecArithAndLogicalOperation( + dst, add_left, add_right, /*opcode=*/ 0x58, SET_VEX_PP_NONE, /*is_commutative=*/ true); } void X86_64Assembler::vsubps(XmmRegister dst, XmmRegister src1, XmmRegister src2) { - DCHECK(CpuHasAVXorAVX2FeatureFlag()); - AssemblerBuffer::EnsureCapacity ensured(&buffer_); - bool is_twobyte_form = false; - uint8_t byte_zero = 0x00, byte_one = 0x00, byte_two = 0x00; - if (!src2.NeedsRex()) { - is_twobyte_form = true; - } - byte_zero = EmitVexPrefixByteZero(is_twobyte_form); - X86_64ManagedRegister vvvv_reg = X86_64ManagedRegister::FromXmmRegister(src1.AsFloatRegister()); - if (is_twobyte_form) { - byte_one = EmitVexPrefixByteOne(dst.NeedsRex(), vvvv_reg, SET_VEX_L_128, SET_VEX_PP_NONE); - } else { - byte_one = EmitVexPrefixByteOne(dst.NeedsRex(), /*X=*/ false, src2.NeedsRex(), SET_VEX_M_0F); - byte_two = EmitVexPrefixByteTwo(/*W=*/ false, vvvv_reg, SET_VEX_L_128, SET_VEX_PP_NONE); - } - EmitUint8(byte_zero); - EmitUint8(byte_one); - if (!is_twobyte_form) { - EmitUint8(byte_two); - } - EmitUint8(0x5C); - EmitXmmRegisterOperand(dst.LowBits(), src2); + EmitVecArithAndLogicalOperation(dst, src1, src2, /*opcode=*/ 0x5C, SET_VEX_PP_NONE); } @@ -801,34 +754,8 @@ void X86_64Assembler::mulps(XmmRegister dst, XmmRegister src) { } void X86_64Assembler::vmulps(XmmRegister dst, XmmRegister src1, XmmRegister src2) { - DCHECK(CpuHasAVXorAVX2FeatureFlag()); - bool is_twobyte_form = false; - uint8_t ByteZero = 0x00, ByteOne = 0x00, ByteTwo = 0x00; - if (!src2.NeedsRex()) { - is_twobyte_form = true; - } else if (!src1.NeedsRex()) { - return vmulps(dst, src2, src1); - } - AssemblerBuffer::EnsureCapacity ensured(&buffer_); - ByteZero = EmitVexPrefixByteZero(is_twobyte_form); - X86_64ManagedRegister vvvv_reg = - X86_64ManagedRegister::FromXmmRegister(src1.AsFloatRegister()); - if (is_twobyte_form) { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), vvvv_reg, SET_VEX_L_128, SET_VEX_PP_NONE); - } else { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), - /*X=*/ false, - src2.NeedsRex(), - SET_VEX_M_0F); - ByteTwo = EmitVexPrefixByteTwo(/*W=*/ false, vvvv_reg, SET_VEX_L_128, SET_VEX_PP_NONE); - } - EmitUint8(ByteZero); - EmitUint8(ByteOne); - if (!is_twobyte_form) { - EmitUint8(ByteTwo); - } - EmitUint8(0x59); - EmitXmmRegisterOperand(dst.LowBits(), src2); + EmitVecArithAndLogicalOperation( + dst, src1, src2, /*opcode=*/ 0x59, SET_VEX_PP_NONE, /*is_commutative=*/ true); } void X86_64Assembler::divps(XmmRegister dst, XmmRegister src) { @@ -840,32 +767,7 @@ void X86_64Assembler::divps(XmmRegister dst, XmmRegister src) { } void X86_64Assembler::vdivps(XmmRegister dst, XmmRegister src1, XmmRegister src2) { - DCHECK(CpuHasAVXorAVX2FeatureFlag()); - AssemblerBuffer::EnsureCapacity ensured(&buffer_); - bool is_twobyte_form = false; - uint8_t ByteZero = 0x00, ByteOne = 0x00, ByteTwo = 0x00; - if (!src2.NeedsRex()) { - is_twobyte_form = true; - } - ByteZero = EmitVexPrefixByteZero(is_twobyte_form); - X86_64ManagedRegister vvvv_reg = - X86_64ManagedRegister::FromXmmRegister(src1.AsFloatRegister()); - if (is_twobyte_form) { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), vvvv_reg, SET_VEX_L_128, SET_VEX_PP_NONE); - } else { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), - /*X=*/ false, - src2.NeedsRex(), - SET_VEX_M_0F); - ByteTwo = EmitVexPrefixByteTwo(/*W=*/ false, vvvv_reg, SET_VEX_L_128, SET_VEX_PP_NONE); - } - EmitUint8(ByteZero); - EmitUint8(ByteOne); - if (!is_twobyte_form) { - EmitUint8(ByteTwo); - } - EmitUint8(0x5E); - EmitXmmRegisterOperand(dst.LowBits(), src2); + EmitVecArithAndLogicalOperation(dst, src1, src2, /*opcode=*/ 0x5E, SET_VEX_PP_NONE); } void X86_64Assembler::vfmadd213ss(XmmRegister acc, XmmRegister left, XmmRegister right) { @@ -1209,33 +1111,8 @@ void X86_64Assembler::addpd(XmmRegister dst, XmmRegister src) { void X86_64Assembler::vaddpd(XmmRegister dst, XmmRegister add_left, XmmRegister add_right) { - bool is_twobyte_form = false; - uint8_t ByteZero = 0x00, ByteOne = 0x00, ByteTwo = 0x00; - if (!add_right.NeedsRex()) { - is_twobyte_form = true; - } else if (!add_left.NeedsRex()) { - return vaddpd(dst, add_right, add_left); - } - AssemblerBuffer::EnsureCapacity ensured(&buffer_); - ByteZero = EmitVexPrefixByteZero(is_twobyte_form); - X86_64ManagedRegister vvvv_reg = - X86_64ManagedRegister::FromXmmRegister(add_left.AsFloatRegister()); - if (is_twobyte_form) { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), vvvv_reg, SET_VEX_L_128, SET_VEX_PP_66); - } else { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), - /*X=*/ false, - add_right.NeedsRex(), - SET_VEX_M_0F); - ByteTwo = EmitVexPrefixByteTwo(/*W=*/ false, vvvv_reg, SET_VEX_L_128, SET_VEX_PP_66); - } - EmitUint8(ByteZero); - EmitUint8(ByteOne); - if (!is_twobyte_form) { - EmitUint8(ByteTwo); - } - EmitUint8(0x58); - EmitXmmRegisterOperand(dst.LowBits(), add_right); + EmitVecArithAndLogicalOperation( + dst, add_left, add_right, /*opcode=*/ 0x58, SET_VEX_PP_66, /*is_commutative=*/ true); } @@ -1250,31 +1127,7 @@ void X86_64Assembler::subpd(XmmRegister dst, XmmRegister src) { void X86_64Assembler::vsubpd(XmmRegister dst, XmmRegister src1, XmmRegister src2) { - AssemblerBuffer::EnsureCapacity ensured(&buffer_); - bool is_twobyte_form = false; - uint8_t ByteZero = 0x00, ByteOne = 0x00, ByteTwo = 0x00; - if (!src2.NeedsRex()) { - is_twobyte_form = true; - } - ByteZero = EmitVexPrefixByteZero(is_twobyte_form); - X86_64ManagedRegister vvvv_reg = - X86_64ManagedRegister::FromXmmRegister(src1.AsFloatRegister()); - if (is_twobyte_form) { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), vvvv_reg, SET_VEX_L_128, SET_VEX_PP_66); - } else { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), - /*X=*/ false, - src2.NeedsRex(), - SET_VEX_M_0F); - ByteTwo = EmitVexPrefixByteTwo(/*W=*/ false, vvvv_reg, SET_VEX_L_128, SET_VEX_PP_66); - } - EmitUint8(ByteZero); - EmitUint8(ByteOne); - if (!is_twobyte_form) { - EmitUint8(ByteTwo); - } - EmitUint8(0x5C); - EmitXmmRegisterOperand(dst.LowBits(), src2); + EmitVecArithAndLogicalOperation(dst, src1, src2, /*opcode=*/ 0x5C, SET_VEX_PP_66); } @@ -1288,34 +1141,8 @@ void X86_64Assembler::mulpd(XmmRegister dst, XmmRegister src) { } void X86_64Assembler::vmulpd(XmmRegister dst, XmmRegister src1, XmmRegister src2) { - DCHECK(CpuHasAVXorAVX2FeatureFlag()); - bool is_twobyte_form = false; - uint8_t ByteZero = 0x00, ByteOne = 0x00, ByteTwo = 0x00; - if (!src2.NeedsRex()) { - is_twobyte_form = true; - } else if (!src1.NeedsRex()) { - return vmulpd(dst, src2, src1); - } - AssemblerBuffer::EnsureCapacity ensured(&buffer_); - ByteZero = EmitVexPrefixByteZero(is_twobyte_form); - X86_64ManagedRegister vvvv_reg = - X86_64ManagedRegister::FromXmmRegister(src1.AsFloatRegister()); - if (is_twobyte_form) { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), vvvv_reg, SET_VEX_L_128, SET_VEX_PP_66); - } else { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), - /*X=*/ false, - src2.NeedsRex(), - SET_VEX_M_0F); - ByteTwo = EmitVexPrefixByteTwo(/*W=*/ false, vvvv_reg, SET_VEX_L_128, SET_VEX_PP_66); - } - EmitUint8(ByteZero); - EmitUint8(ByteOne); - if (!is_twobyte_form) { - EmitUint8(ByteTwo); - } - EmitUint8(0x59); - EmitXmmRegisterOperand(dst.LowBits(), src2); + EmitVecArithAndLogicalOperation( + dst, src1, src2, /*opcode=*/ 0x59, SET_VEX_PP_66, /*is_commutative=*/ true); } void X86_64Assembler::divpd(XmmRegister dst, XmmRegister src) { @@ -1329,32 +1156,7 @@ void X86_64Assembler::divpd(XmmRegister dst, XmmRegister src) { void X86_64Assembler::vdivpd(XmmRegister dst, XmmRegister src1, XmmRegister src2) { - DCHECK(CpuHasAVXorAVX2FeatureFlag()); - AssemblerBuffer::EnsureCapacity ensured(&buffer_); - bool is_twobyte_form = false; - uint8_t ByteZero = 0x00, ByteOne = 0x00, ByteTwo = 0x00; - if (!src2.NeedsRex()) { - is_twobyte_form = true; - } - ByteZero = EmitVexPrefixByteZero(is_twobyte_form); - X86_64ManagedRegister vvvv_reg = - X86_64ManagedRegister::FromXmmRegister(src1.AsFloatRegister()); - if (is_twobyte_form) { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), vvvv_reg, SET_VEX_L_128, SET_VEX_PP_66); - } else { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), - /*X=*/ false, - src2.NeedsRex(), - SET_VEX_M_0F); - ByteTwo = EmitVexPrefixByteTwo(/*W=*/ false, vvvv_reg, SET_VEX_L_128, SET_VEX_PP_66); - } - EmitUint8(ByteZero); - EmitUint8(ByteOne); - if (!is_twobyte_form) { - EmitUint8(ByteTwo); - } - EmitUint8(0x5E); - EmitXmmRegisterOperand(dst.LowBits(), src2); + EmitVecArithAndLogicalOperation(dst, src1, src2, /*opcode=*/ 0x5E, SET_VEX_PP_66); } @@ -1531,34 +1333,8 @@ void X86_64Assembler::paddb(XmmRegister dst, XmmRegister src) { void X86_64Assembler::vpaddb(XmmRegister dst, XmmRegister add_left, XmmRegister add_right) { - DCHECK(CpuHasAVXorAVX2FeatureFlag()); - uint8_t ByteOne = 0x00, ByteZero = 0x00, ByteTwo = 0x00; - bool is_twobyte_form = false; - if (!add_right.NeedsRex()) { - is_twobyte_form = true; - } else if (!add_left.NeedsRex()) { - return vpaddb(dst, add_right, add_left); - } - AssemblerBuffer::EnsureCapacity ensured(&buffer_); - ByteZero = EmitVexPrefixByteZero(is_twobyte_form); - X86_64ManagedRegister vvvv_reg = - X86_64ManagedRegister::FromXmmRegister(add_left.AsFloatRegister()); - if (is_twobyte_form) { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), vvvv_reg, SET_VEX_L_128, SET_VEX_PP_66); - } else { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), - /*X=*/ false, - add_right.NeedsRex(), - SET_VEX_M_0F); - ByteTwo = EmitVexPrefixByteTwo(/*W=*/ false, vvvv_reg, SET_VEX_L_128, SET_VEX_PP_66); - } - EmitUint8(ByteZero); - EmitUint8(ByteOne); - if (!is_twobyte_form) { - EmitUint8(ByteTwo); - } - EmitUint8(0xFC); - EmitXmmRegisterOperand(dst.LowBits(), add_right); + EmitVecArithAndLogicalOperation( + dst, add_left, add_right, /*opcode=*/ 0xFC, SET_VEX_PP_66, /*is_commutative=*/ true); } @@ -1573,32 +1349,7 @@ void X86_64Assembler::psubb(XmmRegister dst, XmmRegister src) { void X86_64Assembler::vpsubb(XmmRegister dst, XmmRegister add_left, XmmRegister add_right) { - DCHECK(CpuHasAVXorAVX2FeatureFlag()); - AssemblerBuffer::EnsureCapacity ensured(&buffer_); - bool is_twobyte_form = false; - uint8_t ByteZero = 0x00, ByteOne = 0x00, ByteTwo = 0x00; - if (!add_right.NeedsRex()) { - is_twobyte_form = true; - } - ByteZero = EmitVexPrefixByteZero(is_twobyte_form); - X86_64ManagedRegister vvvv_reg = - X86_64ManagedRegister::FromXmmRegister(add_left.AsFloatRegister()); - if (is_twobyte_form) { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), vvvv_reg, SET_VEX_L_128, SET_VEX_PP_66); - } else { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), - /*X=*/ false, - add_right.NeedsRex(), - SET_VEX_M_0F); - ByteTwo = EmitVexPrefixByteTwo(/*W=*/ false, vvvv_reg, SET_VEX_L_128, SET_VEX_PP_66); - } - EmitUint8(ByteZero); - EmitUint8(ByteOne); - if (!is_twobyte_form) { - EmitUint8(ByteTwo); - } - EmitUint8(0xF8); - EmitXmmRegisterOperand(dst.LowBits(), add_right); + EmitVecArithAndLogicalOperation(dst, add_left, add_right, /*opcode=*/ 0xF8, SET_VEX_PP_66); } @@ -1612,34 +1363,8 @@ void X86_64Assembler::paddw(XmmRegister dst, XmmRegister src) { } void X86_64Assembler::vpaddw(XmmRegister dst, XmmRegister add_left, XmmRegister add_right) { - DCHECK(CpuHasAVXorAVX2FeatureFlag()); - bool is_twobyte_form = false; - uint8_t ByteZero = 0x00, ByteOne = 0x00, ByteTwo = 0x00; - if (!add_right.NeedsRex()) { - is_twobyte_form = true; - } else if (!add_left.NeedsRex()) { - return vpaddw(dst, add_right, add_left); - } - AssemblerBuffer::EnsureCapacity ensured(&buffer_); - ByteZero = EmitVexPrefixByteZero(is_twobyte_form); - X86_64ManagedRegister vvvv_reg = - X86_64ManagedRegister::FromXmmRegister(add_left.AsFloatRegister()); - if (is_twobyte_form) { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), vvvv_reg, SET_VEX_L_128, SET_VEX_PP_66); - } else { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), - /*X=*/ false, - add_right.NeedsRex(), - SET_VEX_M_0F); - ByteTwo = EmitVexPrefixByteTwo(/*W=*/ false, vvvv_reg, SET_VEX_L_128, SET_VEX_PP_66); - } - EmitUint8(ByteZero); - EmitUint8(ByteOne); - if (!is_twobyte_form) { - EmitUint8(ByteTwo); - } - EmitUint8(0xFD); - EmitXmmRegisterOperand(dst.LowBits(), add_right); + EmitVecArithAndLogicalOperation( + dst, add_left, add_right, /*opcode=*/ 0xFD, SET_VEX_PP_66, /*is_commutative=*/ true); } @@ -1653,32 +1378,7 @@ void X86_64Assembler::psubw(XmmRegister dst, XmmRegister src) { } void X86_64Assembler::vpsubw(XmmRegister dst, XmmRegister add_left, XmmRegister add_right) { - DCHECK(CpuHasAVXorAVX2FeatureFlag()); - AssemblerBuffer::EnsureCapacity ensured(&buffer_); - bool is_twobyte_form = false; - uint8_t ByteZero = 0x00, ByteOne = 0x00, ByteTwo = 0x00; - if (!add_right.NeedsRex()) { - is_twobyte_form = true; - } - ByteZero = EmitVexPrefixByteZero(is_twobyte_form); - X86_64ManagedRegister vvvv_reg = - X86_64ManagedRegister::FromXmmRegister(add_left.AsFloatRegister()); - if (is_twobyte_form) { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), vvvv_reg, SET_VEX_L_128, SET_VEX_PP_66); - } else { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), - /*X=*/ false, - add_right.NeedsRex(), - SET_VEX_M_0F); - ByteTwo = EmitVexPrefixByteTwo(/*W=*/ false, vvvv_reg, SET_VEX_L_128, SET_VEX_PP_66); - } - EmitUint8(ByteZero); - EmitUint8(ByteOne); - if (!is_twobyte_form) { - EmitUint8(ByteTwo); - } - EmitUint8(0xF9); - EmitXmmRegisterOperand(dst.LowBits(), add_right); + EmitVecArithAndLogicalOperation(dst, add_left, add_right, /*opcode=*/ 0xF9, SET_VEX_PP_66); } @@ -1692,34 +1392,8 @@ void X86_64Assembler::pmullw(XmmRegister dst, XmmRegister src) { } void X86_64Assembler::vpmullw(XmmRegister dst, XmmRegister src1, XmmRegister src2) { - DCHECK(CpuHasAVXorAVX2FeatureFlag()); - bool is_twobyte_form = false; - uint8_t ByteZero = 0x00, ByteOne = 0x00, ByteTwo = 0x00; - if (!src2.NeedsRex()) { - is_twobyte_form = true; - } else if (!src1.NeedsRex()) { - return vpmullw(dst, src2, src1); - } - AssemblerBuffer::EnsureCapacity ensured(&buffer_); - ByteZero = EmitVexPrefixByteZero(is_twobyte_form); - X86_64ManagedRegister vvvv_reg = - X86_64ManagedRegister::FromXmmRegister(src1.AsFloatRegister()); - if (is_twobyte_form) { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), vvvv_reg, SET_VEX_L_128, SET_VEX_PP_66); - } else { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), - /*X=*/ false, - src2.NeedsRex(), - SET_VEX_M_0F); - ByteTwo = EmitVexPrefixByteTwo(/*W=*/ false, vvvv_reg, SET_VEX_L_128, SET_VEX_PP_66); - } - EmitUint8(ByteZero); - EmitUint8(ByteOne); - if (!is_twobyte_form) { - EmitUint8(ByteTwo); - } - EmitUint8(0xD5); - EmitXmmRegisterOperand(dst.LowBits(), src2); + EmitVecArithAndLogicalOperation( + dst, src1, src2, /*opcode=*/ 0xD5, SET_VEX_PP_66, /*is_commutative=*/ true); } void X86_64Assembler::paddd(XmmRegister dst, XmmRegister src) { @@ -1732,34 +1406,8 @@ void X86_64Assembler::paddd(XmmRegister dst, XmmRegister src) { } void X86_64Assembler::vpaddd(XmmRegister dst, XmmRegister add_left, XmmRegister add_right) { - DCHECK(CpuHasAVXorAVX2FeatureFlag()); - bool is_twobyte_form = false; - uint8_t ByteZero = 0x00, ByteOne = 0x00, ByteTwo = 0x00; - if (!add_right.NeedsRex()) { - is_twobyte_form = true; - } else if (!add_left.NeedsRex()) { - return vpaddd(dst, add_right, add_left); - } - AssemblerBuffer::EnsureCapacity ensured(&buffer_); - ByteZero = EmitVexPrefixByteZero(is_twobyte_form); - X86_64ManagedRegister vvvv_reg = - X86_64ManagedRegister::FromXmmRegister(add_left.AsFloatRegister()); - if (is_twobyte_form) { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), vvvv_reg, SET_VEX_L_128, SET_VEX_PP_66); - } else { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), - /*X=*/ false, - add_right.NeedsRex(), - SET_VEX_M_0F); - ByteTwo = EmitVexPrefixByteTwo(/*W=*/ false, vvvv_reg, SET_VEX_L_128, SET_VEX_PP_66); - } - EmitUint8(ByteZero); - EmitUint8(ByteOne); - if (!is_twobyte_form) { - EmitUint8(ByteTwo); - } - EmitUint8(0xFE); - EmitXmmRegisterOperand(dst.LowBits(), add_right); + EmitVecArithAndLogicalOperation( + dst, add_left, add_right, /*opcode=*/ 0xFE, SET_VEX_PP_66, /*is_commutative=*/ true); } void X86_64Assembler::psubd(XmmRegister dst, XmmRegister src) { @@ -1812,34 +1460,8 @@ void X86_64Assembler::paddq(XmmRegister dst, XmmRegister src) { void X86_64Assembler::vpaddq(XmmRegister dst, XmmRegister add_left, XmmRegister add_right) { - DCHECK(CpuHasAVXorAVX2FeatureFlag()); - bool is_twobyte_form = false; - uint8_t ByteZero = 0x00, ByteOne = 0x00, ByteTwo = 0x00; - if (!add_right.NeedsRex()) { - is_twobyte_form = true; - } else if (!add_left.NeedsRex()) { - return vpaddq(dst, add_right, add_left); - } - AssemblerBuffer::EnsureCapacity ensured(&buffer_); - ByteZero = EmitVexPrefixByteZero(is_twobyte_form); - X86_64ManagedRegister vvvv_reg = - X86_64ManagedRegister::FromXmmRegister(add_left.AsFloatRegister()); - if (is_twobyte_form) { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), vvvv_reg, SET_VEX_L_128, SET_VEX_PP_66); - } else { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), - /*X=*/ false, - add_right.NeedsRex(), - SET_VEX_M_0F); - ByteTwo = EmitVexPrefixByteTwo(/*W=*/ false, vvvv_reg, SET_VEX_L_128, SET_VEX_PP_66); - } - EmitUint8(ByteZero); - EmitUint8(ByteOne); - if (!is_twobyte_form) { - EmitUint8(ByteTwo); - } - EmitUint8(0xD4); - EmitXmmRegisterOperand(dst.LowBits(), add_right); + EmitVecArithAndLogicalOperation( + dst, add_left, add_right, /*opcode=*/ 0xD4, SET_VEX_PP_66, /*is_commutative=*/ true); } @@ -1853,32 +1475,7 @@ void X86_64Assembler::psubq(XmmRegister dst, XmmRegister src) { } void X86_64Assembler::vpsubq(XmmRegister dst, XmmRegister add_left, XmmRegister add_right) { - DCHECK(CpuHasAVXorAVX2FeatureFlag()); - AssemblerBuffer::EnsureCapacity ensured(&buffer_); - bool is_twobyte_form = false; - uint8_t ByteZero = 0x00, ByteOne = 0x00, ByteTwo = 0x00; - if (!add_right.NeedsRex()) { - is_twobyte_form = true; - } - ByteZero = EmitVexPrefixByteZero(is_twobyte_form); - X86_64ManagedRegister vvvv_reg = - X86_64ManagedRegister::FromXmmRegister(add_left.AsFloatRegister()); - if (is_twobyte_form) { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), vvvv_reg, SET_VEX_L_128, SET_VEX_PP_66); - } else { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), - /*X=*/ false, - add_right.NeedsRex(), - SET_VEX_M_0F); - ByteTwo = EmitVexPrefixByteTwo(/*W=*/ false, vvvv_reg, SET_VEX_L_128, SET_VEX_PP_66); - } - EmitUint8(ByteZero); - EmitUint8(ByteOne); - if (!is_twobyte_form) { - EmitUint8(ByteTwo); - } - EmitUint8(0xFB); - EmitXmmRegisterOperand(dst.LowBits(), add_right); + EmitVecArithAndLogicalOperation(dst, add_left, add_right, /*opcode=*/ 0xFB, SET_VEX_PP_66); } @@ -1943,32 +1540,7 @@ void X86_64Assembler::psubsb(XmmRegister dst, XmmRegister src) { void X86_64Assembler::vpsubd(XmmRegister dst, XmmRegister add_left, XmmRegister add_right) { - DCHECK(CpuHasAVXorAVX2FeatureFlag()); - AssemblerBuffer::EnsureCapacity ensured(&buffer_); - bool is_twobyte_form = false; - uint8_t ByteZero = 0x00, ByteOne = 0x00, ByteTwo = 0x00; - if (!add_right.NeedsRex()) { - is_twobyte_form = true; - } - ByteZero = EmitVexPrefixByteZero(is_twobyte_form); - X86_64ManagedRegister vvvv_reg = - X86_64ManagedRegister::FromXmmRegister(add_left.AsFloatRegister()); - if (is_twobyte_form) { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), vvvv_reg, SET_VEX_L_128, SET_VEX_PP_66); - } else { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), - /*X=*/ false, - add_right.NeedsRex(), - SET_VEX_M_0F); - ByteTwo = EmitVexPrefixByteTwo(/*W=*/ false, vvvv_reg, SET_VEX_L_128, SET_VEX_PP_66); - } - EmitUint8(ByteZero); - EmitUint8(ByteOne); - if (!is_twobyte_form) { - EmitUint8(ByteTwo); - } - EmitUint8(0xFA); - EmitXmmRegisterOperand(dst.LowBits(), add_right); + EmitVecArithAndLogicalOperation(dst, add_left, add_right, /*opcode=*/ 0xFA, SET_VEX_PP_66); } @@ -2349,98 +1921,20 @@ void X86_64Assembler::pxor(XmmRegister dst, XmmRegister src) { /* VEX.128.66.0F.WIG EF /r VPXOR xmm1, xmm2, xmm3/m128 */ void X86_64Assembler::vpxor(XmmRegister dst, XmmRegister src1, XmmRegister src2) { - DCHECK(CpuHasAVXorAVX2FeatureFlag()); - bool is_twobyte_form = false; - uint8_t ByteZero = 0x00, ByteOne = 0x00, ByteTwo = 0x00; - if (!src2.NeedsRex()) { - is_twobyte_form = true; - } else if (!src1.NeedsRex()) { - return vpxor(dst, src2, src1); - } - AssemblerBuffer::EnsureCapacity ensured(&buffer_); - X86_64ManagedRegister vvvv_reg = - X86_64ManagedRegister::FromXmmRegister(src1.AsFloatRegister()); - ByteZero = EmitVexPrefixByteZero(is_twobyte_form); - if (is_twobyte_form) { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), vvvv_reg, SET_VEX_L_128, SET_VEX_PP_66); - } else { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), - /*X=*/ false, - src2.NeedsRex(), - SET_VEX_M_0F); - ByteTwo = EmitVexPrefixByteTwo(/*W=*/ false, vvvv_reg, SET_VEX_L_128, SET_VEX_PP_66); - } - EmitUint8(ByteZero); - EmitUint8(ByteOne); - if (!is_twobyte_form) { - EmitUint8(ByteTwo); - } - EmitUint8(0xEF); - EmitXmmRegisterOperand(dst.LowBits(), src2); + EmitVecArithAndLogicalOperation( + dst, src1, src2, /*opcode=*/ 0xEF, SET_VEX_PP_66, /*is_commutative=*/ true); } /* VEX.128.0F.WIG 57 /r VXORPS xmm1,xmm2, xmm3/m128 */ void X86_64Assembler::vxorps(XmmRegister dst, XmmRegister src1, XmmRegister src2) { - DCHECK(CpuHasAVXorAVX2FeatureFlag()); - bool is_twobyte_form = false; - uint8_t ByteZero = 0x00, ByteOne = 0x00, ByteTwo = 0x00; - if (!src2.NeedsRex()) { - is_twobyte_form = true; - } else if (!src1.NeedsRex()) { - return vxorps(dst, src2, src1); - } - AssemblerBuffer::EnsureCapacity ensured(&buffer_); - X86_64ManagedRegister vvvv_reg = - X86_64ManagedRegister::FromXmmRegister(src1.AsFloatRegister()); - ByteZero = EmitVexPrefixByteZero(is_twobyte_form); - if (is_twobyte_form) { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), vvvv_reg, SET_VEX_L_128, SET_VEX_PP_NONE); - } else { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), - /*X=*/ false, - src2.NeedsRex(), - SET_VEX_M_0F); - ByteTwo = EmitVexPrefixByteTwo(/*W=*/ false, vvvv_reg, SET_VEX_L_128, SET_VEX_PP_NONE); - } - EmitUint8(ByteZero); - EmitUint8(ByteOne); - if (!is_twobyte_form) { - EmitUint8(ByteTwo); - } - EmitUint8(0x57); - EmitXmmRegisterOperand(dst.LowBits(), src2); + EmitVecArithAndLogicalOperation( + dst, src1, src2, /*opcode=*/ 0x57, SET_VEX_PP_NONE, /*is_commutative=*/ true); } /* VEX.128.66.0F.WIG 57 /r VXORPD xmm1,xmm2, xmm3/m128 */ void X86_64Assembler::vxorpd(XmmRegister dst, XmmRegister src1, XmmRegister src2) { - DCHECK(CpuHasAVXorAVX2FeatureFlag()); - bool is_twobyte_form = false; - uint8_t ByteZero = 0x00, ByteOne = 0x00, ByteTwo = 0x00; - if (!src2.NeedsRex()) { - is_twobyte_form = true; - } else if (!src1.NeedsRex()) { - return vxorpd(dst, src2, src1); - } - AssemblerBuffer::EnsureCapacity ensured(&buffer_); - X86_64ManagedRegister vvvv_reg = - X86_64ManagedRegister::FromXmmRegister(src1.AsFloatRegister()); - ByteZero = EmitVexPrefixByteZero(is_twobyte_form); - if (is_twobyte_form) { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), vvvv_reg, SET_VEX_L_128, SET_VEX_PP_66); - } else { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), - /*X=*/ false, - src2.NeedsRex(), - SET_VEX_M_0F); - ByteTwo = EmitVexPrefixByteTwo(/*W=*/ false, vvvv_reg, SET_VEX_L_128, SET_VEX_PP_66); - } - EmitUint8(ByteZero); - EmitUint8(ByteOne); - if (!is_twobyte_form) { - EmitUint8(ByteTwo); - } - EmitUint8(0x57); - EmitXmmRegisterOperand(dst.LowBits(), src2); + EmitVecArithAndLogicalOperation( + dst, src1, src2, /*opcode=*/ 0x57, SET_VEX_PP_66, /*is_commutative=*/ true); } void X86_64Assembler::andpd(XmmRegister dst, const Address& src) { @@ -2480,98 +1974,20 @@ void X86_64Assembler::pand(XmmRegister dst, XmmRegister src) { /* VEX.128.66.0F.WIG DB /r VPAND xmm1, xmm2, xmm3/m128 */ void X86_64Assembler::vpand(XmmRegister dst, XmmRegister src1, XmmRegister src2) { - DCHECK(CpuHasAVXorAVX2FeatureFlag()); - bool is_twobyte_form = false; - uint8_t ByteZero = 0x00, ByteOne = 0x00, ByteTwo = 0x00; - if (!src2.NeedsRex()) { - is_twobyte_form = true; - } else if (!src1.NeedsRex()) { - return vpand(dst, src2, src1); - } - AssemblerBuffer::EnsureCapacity ensured(&buffer_); - X86_64ManagedRegister vvvv_reg = - X86_64ManagedRegister::FromXmmRegister(src1.AsFloatRegister()); - ByteZero = EmitVexPrefixByteZero(is_twobyte_form); - if (is_twobyte_form) { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), vvvv_reg, SET_VEX_L_128, SET_VEX_PP_66); - } else { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), - /*X=*/ false, - src2.NeedsRex(), - SET_VEX_M_0F); - ByteTwo = EmitVexPrefixByteTwo(/*W=*/ false, vvvv_reg, SET_VEX_L_128, SET_VEX_PP_66); - } - EmitUint8(ByteZero); - EmitUint8(ByteOne); - if (!is_twobyte_form) { - EmitUint8(ByteTwo); - } - EmitUint8(0xDB); - EmitXmmRegisterOperand(dst.LowBits(), src2); + EmitVecArithAndLogicalOperation( + dst, src1, src2, /*opcode=*/ 0xDB, SET_VEX_PP_66, /*is_commutative=*/ true); } /* VEX.128.0F 54 /r VANDPS xmm1,xmm2, xmm3/m128 */ void X86_64Assembler::vandps(XmmRegister dst, XmmRegister src1, XmmRegister src2) { - DCHECK(CpuHasAVXorAVX2FeatureFlag()); - bool is_twobyte_form = false; - uint8_t ByteZero = 0x00, ByteOne = 0x00, ByteTwo = 0x00; - if (!src2.NeedsRex()) { - is_twobyte_form = true; - } else if (!src1.NeedsRex()) { - return vandps(dst, src2, src1); - } - AssemblerBuffer::EnsureCapacity ensured(&buffer_); - X86_64ManagedRegister vvvv_reg = - X86_64ManagedRegister::FromXmmRegister(src1.AsFloatRegister()); - ByteZero = EmitVexPrefixByteZero(is_twobyte_form); - if (is_twobyte_form) { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), vvvv_reg, SET_VEX_L_128, SET_VEX_PP_NONE); - } else { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), - /*X=*/ false, - src2.NeedsRex(), - SET_VEX_M_0F); - ByteTwo = EmitVexPrefixByteTwo(/*W=*/ false, vvvv_reg, SET_VEX_L_128, SET_VEX_PP_NONE); - } - EmitUint8(ByteZero); - EmitUint8(ByteOne); - if (!is_twobyte_form) { - EmitUint8(ByteTwo); - } - EmitUint8(0x54); - EmitXmmRegisterOperand(dst.LowBits(), src2); + EmitVecArithAndLogicalOperation( + dst, src1, src2, /*opcode=*/ 0x54, SET_VEX_PP_NONE, /*is_commutative=*/ true); } /* VEX.128.66.0F 54 /r VANDPD xmm1, xmm2, xmm3/m128 */ void X86_64Assembler::vandpd(XmmRegister dst, XmmRegister src1, XmmRegister src2) { - DCHECK(CpuHasAVXorAVX2FeatureFlag()); - bool is_twobyte_form = false; - uint8_t ByteZero = 0x00, ByteOne = 0x00, ByteTwo = 0x00; - if (!src2.NeedsRex()) { - is_twobyte_form = true; - } else if (!src1.NeedsRex()) { - return vandpd(dst, src2, src1); - } - AssemblerBuffer::EnsureCapacity ensured(&buffer_); - X86_64ManagedRegister vvvv_reg = - X86_64ManagedRegister::FromXmmRegister(src1.AsFloatRegister()); - ByteZero = EmitVexPrefixByteZero(is_twobyte_form); - if (is_twobyte_form) { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), vvvv_reg, SET_VEX_L_128, SET_VEX_PP_66); - } else { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), - /*X=*/ false, - src2.NeedsRex(), - SET_VEX_M_0F); - ByteTwo = EmitVexPrefixByteTwo(/*W=*/ false, vvvv_reg, SET_VEX_L_128, SET_VEX_PP_66); - } - EmitUint8(ByteZero); - EmitUint8(ByteOne); - if (!is_twobyte_form) { - EmitUint8(ByteTwo); - } - EmitUint8(0x54); - EmitXmmRegisterOperand(dst.LowBits(), src2); + EmitVecArithAndLogicalOperation( + dst, src1, src2, /*opcode=*/ 0x54, SET_VEX_PP_66, /*is_commutative=*/ true); } void X86_64Assembler::andn(CpuRegister dst, CpuRegister src1, CpuRegister src2) { @@ -2621,92 +2037,17 @@ void X86_64Assembler::pandn(XmmRegister dst, XmmRegister src) { /* VEX.128.66.0F.WIG DF /r VPANDN xmm1, xmm2, xmm3/m128 */ void X86_64Assembler::vpandn(XmmRegister dst, XmmRegister src1, XmmRegister src2) { - DCHECK(CpuHasAVXorAVX2FeatureFlag()); - AssemblerBuffer::EnsureCapacity ensured(&buffer_); - bool is_twobyte_form = false; - uint8_t ByteZero = 0x00, ByteOne = 0x00, ByteTwo = 0x00; - if (!src2.NeedsRex()) { - is_twobyte_form = true; - } - X86_64ManagedRegister vvvv_reg = - X86_64ManagedRegister::FromXmmRegister(src1.AsFloatRegister()); - ByteZero = EmitVexPrefixByteZero(is_twobyte_form); - if (is_twobyte_form) { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), vvvv_reg, SET_VEX_L_128, SET_VEX_PP_66); - } else { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), - /*X=*/ false, - src2.NeedsRex(), - SET_VEX_M_0F); - ByteTwo = EmitVexPrefixByteTwo(/*W=*/ false, vvvv_reg, SET_VEX_L_128, SET_VEX_PP_66); - } - EmitUint8(ByteZero); - EmitUint8(ByteOne); - if (!is_twobyte_form) { - EmitUint8(ByteTwo); - } - EmitUint8(0xDF); - EmitXmmRegisterOperand(dst.LowBits(), src2); + EmitVecArithAndLogicalOperation(dst, src1, src2, /*opcode=*/ 0xDF, SET_VEX_PP_66); } /* VEX.128.0F 55 /r VANDNPS xmm1, xmm2, xmm3/m128 */ void X86_64Assembler::vandnps(XmmRegister dst, XmmRegister src1, XmmRegister src2) { - DCHECK(CpuHasAVXorAVX2FeatureFlag()); - AssemblerBuffer::EnsureCapacity ensured(&buffer_); - bool is_twobyte_form = false; - uint8_t ByteZero = 0x00, ByteOne = 0x00, ByteTwo = 0x00; - if (!src2.NeedsRex()) { - is_twobyte_form = true; - } - X86_64ManagedRegister vvvv_reg = - X86_64ManagedRegister::FromXmmRegister(src1.AsFloatRegister()); - ByteZero = EmitVexPrefixByteZero(is_twobyte_form); - if (is_twobyte_form) { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), vvvv_reg, SET_VEX_L_128, SET_VEX_PP_NONE); - } else { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), - /*X=*/ false, - src2.NeedsRex(), - SET_VEX_M_0F); - ByteTwo = EmitVexPrefixByteTwo(/*W=*/ false, vvvv_reg, SET_VEX_L_128, SET_VEX_PP_NONE); - } - EmitUint8(ByteZero); - EmitUint8(ByteOne); - if (!is_twobyte_form) { - EmitUint8(ByteTwo); - } - EmitUint8(0x55); - EmitXmmRegisterOperand(dst.LowBits(), src2); + EmitVecArithAndLogicalOperation(dst, src1, src2, /*opcode=*/ 0x55, SET_VEX_PP_NONE); } /* VEX.128.66.0F 55 /r VANDNPD xmm1, xmm2, xmm3/m128 */ void X86_64Assembler::vandnpd(XmmRegister dst, XmmRegister src1, XmmRegister src2) { - DCHECK(CpuHasAVXorAVX2FeatureFlag()); - AssemblerBuffer::EnsureCapacity ensured(&buffer_); - bool is_twobyte_form = false; - uint8_t ByteZero = 0x00, ByteOne = 0x00, ByteTwo = 0x00; - if (!src2.NeedsRex()) { - is_twobyte_form = true; - } - X86_64ManagedRegister vvvv_reg = - X86_64ManagedRegister::FromXmmRegister(src1.AsFloatRegister()); - ByteZero = EmitVexPrefixByteZero(is_twobyte_form); - if (is_twobyte_form) { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), vvvv_reg, SET_VEX_L_128, SET_VEX_PP_66); - } else { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), - /*X=*/ false, - src2.NeedsRex(), - SET_VEX_M_0F); - ByteTwo = EmitVexPrefixByteTwo(/*W=*/ false, vvvv_reg, SET_VEX_L_128, SET_VEX_PP_66); - } - EmitUint8(ByteZero); - EmitUint8(ByteOne); - if (!is_twobyte_form) { - EmitUint8(ByteTwo); - } - EmitUint8(0x55); - EmitXmmRegisterOperand(dst.LowBits(), src2); + EmitVecArithAndLogicalOperation(dst, src1, src2, /*opcode=*/ 0x55, SET_VEX_PP_66); } void X86_64Assembler::orpd(XmmRegister dst, XmmRegister src) { @@ -2737,98 +2078,20 @@ void X86_64Assembler::por(XmmRegister dst, XmmRegister src) { /* VEX.128.66.0F.WIG EB /r VPOR xmm1, xmm2, xmm3/m128 */ void X86_64Assembler::vpor(XmmRegister dst, XmmRegister src1, XmmRegister src2) { - DCHECK(CpuHasAVXorAVX2FeatureFlag()); - bool is_twobyte_form = false; - uint8_t ByteZero = 0x00, ByteOne = 0x00, ByteTwo = 0x00; - if (!src2.NeedsRex()) { - is_twobyte_form = true; - } else if (!src1.NeedsRex()) { - return vpor(dst, src2, src1); - } - AssemblerBuffer::EnsureCapacity ensured(&buffer_); - X86_64ManagedRegister vvvv_reg = - X86_64ManagedRegister::FromXmmRegister(src1.AsFloatRegister()); - ByteZero = EmitVexPrefixByteZero(is_twobyte_form); - if (is_twobyte_form) { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), vvvv_reg, SET_VEX_L_128, SET_VEX_PP_66); - } else { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), - /*X=*/ false, - src2.NeedsRex(), - SET_VEX_M_0F); - ByteTwo = EmitVexPrefixByteTwo(/*W=*/ false, vvvv_reg, SET_VEX_L_128, SET_VEX_PP_66); - } - EmitUint8(ByteZero); - EmitUint8(ByteOne); - if (!is_twobyte_form) { - EmitUint8(ByteTwo); - } - EmitUint8(0xEB); - EmitXmmRegisterOperand(dst.LowBits(), src2); + EmitVecArithAndLogicalOperation( + dst, src1, src2, /*opcode=*/ 0xEB, SET_VEX_PP_66, /*is_commutative=*/ true); } /* VEX.128.0F 56 /r VORPS xmm1,xmm2, xmm3/m128 */ void X86_64Assembler::vorps(XmmRegister dst, XmmRegister src1, XmmRegister src2) { - DCHECK(CpuHasAVXorAVX2FeatureFlag()); - bool is_twobyte_form = false; - uint8_t ByteZero = 0x00, ByteOne = 0x00, ByteTwo = 0x00; - if (!src2.NeedsRex()) { - is_twobyte_form = true; - } else if (!src1.NeedsRex()) { - return vorps(dst, src2, src1); - } - AssemblerBuffer::EnsureCapacity ensured(&buffer_); - X86_64ManagedRegister vvvv_reg = - X86_64ManagedRegister::FromXmmRegister(src1.AsFloatRegister()); - ByteZero = EmitVexPrefixByteZero(is_twobyte_form); - if (is_twobyte_form) { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), vvvv_reg, SET_VEX_L_128, SET_VEX_PP_NONE); - } else { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), - /*X=*/ false, - src2.NeedsRex(), - SET_VEX_M_0F); - ByteTwo = EmitVexPrefixByteTwo(/*W=*/ false, vvvv_reg, SET_VEX_L_128, SET_VEX_PP_NONE); - } - EmitUint8(ByteZero); - EmitUint8(ByteOne); - if (!is_twobyte_form) { - EmitUint8(ByteTwo); - } - EmitUint8(0x56); - EmitXmmRegisterOperand(dst.LowBits(), src2); + EmitVecArithAndLogicalOperation( + dst, src1, src2, /*opcode=*/ 0x56, SET_VEX_PP_NONE, /*is_commutative=*/ true); } /* VEX.128.66.0F 56 /r VORPD xmm1,xmm2, xmm3/m128 */ void X86_64Assembler::vorpd(XmmRegister dst, XmmRegister src1, XmmRegister src2) { - DCHECK(CpuHasAVXorAVX2FeatureFlag()); - bool is_twobyte_form = false; - uint8_t ByteZero = 0x00, ByteOne = 0x00, ByteTwo = 0x00; - if (!src2.NeedsRex()) { - is_twobyte_form = true; - } else if (!src1.NeedsRex()) { - return vorpd(dst, src2, src1); - } - AssemblerBuffer::EnsureCapacity ensured(&buffer_); - X86_64ManagedRegister vvvv_reg = - X86_64ManagedRegister::FromXmmRegister(src1.AsFloatRegister()); - ByteZero = EmitVexPrefixByteZero(is_twobyte_form); - if (is_twobyte_form) { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), vvvv_reg, SET_VEX_L_128, SET_VEX_PP_66); - } else { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), - /*X=*/ false, - src2.NeedsRex(), - SET_VEX_M_0F); - ByteTwo = EmitVexPrefixByteTwo(/*W=*/ false, vvvv_reg, SET_VEX_L_128, SET_VEX_PP_66); - } - EmitUint8(ByteZero); - EmitUint8(ByteOne); - if (!is_twobyte_form) { - EmitUint8(ByteTwo); - } - EmitUint8(0x56); - EmitXmmRegisterOperand(dst.LowBits(), src2); + EmitVecArithAndLogicalOperation( + dst, src1, src2, /*opcode=*/ 0x56, SET_VEX_PP_66, /*is_commutative=*/ true); } void X86_64Assembler::pavgb(XmmRegister dst, XmmRegister src) { @@ -2868,34 +2131,8 @@ void X86_64Assembler::pmaddwd(XmmRegister dst, XmmRegister src) { } void X86_64Assembler::vpmaddwd(XmmRegister dst, XmmRegister src1, XmmRegister src2) { - DCHECK(CpuHasAVXorAVX2FeatureFlag()); - bool is_twobyte_form = false; - uint8_t ByteZero = 0x00, ByteOne = 0x00, ByteTwo = 0x00; - if (!src2.NeedsRex()) { - is_twobyte_form = true; - } else if (!src1.NeedsRex()) { - return vpmaddwd(dst, src2, src1); - } - AssemblerBuffer::EnsureCapacity ensured(&buffer_); - ByteZero = EmitVexPrefixByteZero(is_twobyte_form); - X86_64ManagedRegister vvvv_reg = - X86_64ManagedRegister::FromXmmRegister(src1.AsFloatRegister()); - if (is_twobyte_form) { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), vvvv_reg, SET_VEX_L_128, SET_VEX_PP_66); - } else { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), - /*X=*/ false, - src2.NeedsRex(), - SET_VEX_M_0F); - ByteTwo = EmitVexPrefixByteTwo(/*W=*/ false, vvvv_reg, SET_VEX_L_128, SET_VEX_PP_66); - } - EmitUint8(ByteZero); - EmitUint8(ByteOne); - if (!is_twobyte_form) { - EmitUint8(ByteTwo); - } - EmitUint8(0xF5); - EmitXmmRegisterOperand(dst.LowBits(), src2); + EmitVecArithAndLogicalOperation( + dst, src1, src2, /*opcode=*/ 0xF5, SET_VEX_PP_66, /*is_commutative=*/ true); } void X86_64Assembler::phaddw(XmmRegister dst, XmmRegister src) { @@ -5529,5 +4766,35 @@ uint8_t X86_64Assembler::EmitVexPrefixByteTwo(bool W, return vex_prefix; } +void X86_64Assembler::EmitVecArithAndLogicalOperation(XmmRegister dst, + XmmRegister src1, + XmmRegister src2, + uint8_t opcode, + int vex_pp, + bool is_commutative) { + if (is_commutative && src2.NeedsRex() && !src1.NeedsRex()) { + return EmitVecArithAndLogicalOperation(dst, src2, src1, opcode, vex_pp, is_commutative); + } + DCHECK(CpuHasAVXorAVX2FeatureFlag()); + AssemblerBuffer::EnsureCapacity ensured(&buffer_); + X86_64ManagedRegister vvvv_reg = X86_64ManagedRegister::FromXmmRegister(src1.AsFloatRegister()); + bool is_twobyte_form = !src2.NeedsRex(); + uint8_t byte_zero = EmitVexPrefixByteZero(is_twobyte_form); + uint8_t byte_one, byte_two; + if (is_twobyte_form) { + byte_one = EmitVexPrefixByteOne(dst.NeedsRex(), vvvv_reg, SET_VEX_L_128, vex_pp); + } else { + byte_one = EmitVexPrefixByteOne(dst.NeedsRex(), /*X=*/ false, src2.NeedsRex(), SET_VEX_M_0F); + byte_two = EmitVexPrefixByteTwo(/*W=*/ false, vvvv_reg, SET_VEX_L_128, vex_pp); + } + EmitUint8(byte_zero); + EmitUint8(byte_one); + if (!is_twobyte_form) { + EmitUint8(byte_two); + } + EmitUint8(opcode); + EmitXmmRegisterOperand(dst.LowBits(), src2); +} + } // namespace x86_64 } // namespace art diff --git a/compiler/utils/x86_64/assembler_x86_64.h b/compiler/utils/x86_64/assembler_x86_64.h index 5f6d754f8d..2c3b3c44ee 100644 --- a/compiler/utils/x86_64/assembler_x86_64.h +++ b/compiler/utils/x86_64/assembler_x86_64.h @@ -1185,6 +1185,13 @@ class X86_64Assembler final : public Assembler { int SET_VEX_L, int SET_VEX_PP); + void EmitVecArithAndLogicalOperation(XmmRegister dst, + XmmRegister src1, + XmmRegister src2, + uint8_t opcode, + int vex_pp, + bool is_commutative = false); + // Helper function to emit a shorter variant of XCHG if at least one operand is RAX/EAX/AX. bool try_xchg_rax(CpuRegister dst, CpuRegister src, |