diff options
| author | 2025-01-02 08:56:15 -0800 | |
|---|---|---|
| committer | 2025-01-02 08:56:15 -0800 | |
| commit | ec0cba1afff0915f10cf87b35080fd45359b97be (patch) | |
| tree | ec3171c384fd5d9ac0a7a5e3ec6f858a274f9dfd | |
| parent | 0e22915958a51e4bf4298a185afa31c406af9527 (diff) | |
| parent | 277c818fb59106458976a3bee58a9165f0db1d5f (diff) | |
x86_64: Clean up VEX prefix generation for load/store. am: 0994519289 am: 277c818fb5
Original change: https://android-review.googlesource.com/c/platform/art/+/3432363
Change-Id: Iba299e668f4b4fd85a54673ee7b9091665fbc8e9
Signed-off-by: Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>
| -rw-r--r-- | compiler/utils/x86_64/assembler_x86_64.cc | 388 | ||||
| -rw-r--r-- | compiler/utils/x86_64/assembler_x86_64.h | 4 |
2 files changed, 37 insertions, 355 deletions
diff --git a/compiler/utils/x86_64/assembler_x86_64.cc b/compiler/utils/x86_64/assembler_x86_64.cc index aae182406d..959e291dc9 100644 --- a/compiler/utils/x86_64/assembler_x86_64.cc +++ b/compiler/utils/x86_64/assembler_x86_64.cc @@ -491,36 +491,8 @@ void X86_64Assembler::movaps(XmmRegister dst, const Address& src) { void X86_64Assembler::vmovaps(XmmRegister dst, const Address& src) { DCHECK(CpuHasAVXorAVX2FeatureFlag()); AssemblerBuffer::EnsureCapacity ensured(&buffer_); - uint8_t ByteZero, ByteOne, ByteTwo; - bool is_twobyte_form = false; // Instruction VEX Prefix - uint8_t rex = src.rex(); - bool Rex_x = rex & GET_REX_X; - bool Rex_b = rex & GET_REX_B; - if (!Rex_b && !Rex_x) { - is_twobyte_form = true; - } - ByteZero = EmitVexPrefixByteZero(is_twobyte_form); - if (is_twobyte_form) { - X86_64ManagedRegister vvvv_reg = ManagedRegister::NoRegister().AsX86_64(); - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), - vvvv_reg, - SET_VEX_L_128, - SET_VEX_PP_NONE); - } else { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), - Rex_x, - Rex_b, - SET_VEX_M_0F); - ByteTwo = EmitVexPrefixByteTwo(/*W=*/ false, - SET_VEX_L_128, - SET_VEX_PP_NONE); - } - EmitUint8(ByteZero); - EmitUint8(ByteOne); - if (!is_twobyte_form) { - EmitUint8(ByteTwo); - } + EmitVexPrefixForAddress(src, dst.NeedsRex(), SET_VEX_L_128, SET_VEX_PP_NONE); // Instruction Opcode EmitUint8(0x28); // Instruction Operands @@ -543,36 +515,8 @@ void X86_64Assembler::movups(XmmRegister dst, const Address& src) { void X86_64Assembler::vmovups(XmmRegister dst, const Address& src) { DCHECK(CpuHasAVXorAVX2FeatureFlag()); AssemblerBuffer::EnsureCapacity ensured(&buffer_); - uint8_t ByteZero, ByteOne, ByteTwo; - bool is_twobyte_form = false; // Instruction VEX Prefix - uint8_t rex = src.rex(); - bool Rex_x = rex & GET_REX_X; - bool Rex_b = rex & GET_REX_B; - if (!Rex_x && !Rex_b) { - is_twobyte_form = true; - } - ByteZero = EmitVexPrefixByteZero(is_twobyte_form); - if (is_twobyte_form) { - X86_64ManagedRegister vvvv_reg = ManagedRegister::NoRegister().AsX86_64(); - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), - vvvv_reg, - SET_VEX_L_128, - SET_VEX_PP_NONE); - } else { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), - Rex_x, - Rex_b, - SET_VEX_M_0F); - ByteTwo = EmitVexPrefixByteTwo(/*W=*/ false, - SET_VEX_L_128, - SET_VEX_PP_NONE); - } - EmitUint8(ByteZero); - EmitUint8(ByteOne); - if (!is_twobyte_form) { - EmitUint8(ByteTwo); - } + EmitVexPrefixForAddress(src, dst.NeedsRex(), SET_VEX_L_128, SET_VEX_PP_NONE); // Instruction Opcode EmitUint8(0x10); // Instruction Operands @@ -596,37 +540,8 @@ void X86_64Assembler::movaps(const Address& dst, XmmRegister src) { void X86_64Assembler::vmovaps(const Address& dst, XmmRegister src) { DCHECK(CpuHasAVXorAVX2FeatureFlag()); AssemblerBuffer::EnsureCapacity ensured(&buffer_); - uint8_t ByteZero, ByteOne, ByteTwo; - bool is_twobyte_form = false; - // Instruction VEX Prefix - uint8_t rex = dst.rex(); - bool Rex_x = rex & GET_REX_X; - bool Rex_b = rex & GET_REX_B; - if (!Rex_b && !Rex_x) { - is_twobyte_form = true; - } - ByteZero = EmitVexPrefixByteZero(is_twobyte_form); - if (is_twobyte_form) { - X86_64ManagedRegister vvvv_reg = ManagedRegister::NoRegister().AsX86_64(); - ByteOne = EmitVexPrefixByteOne(src.NeedsRex(), - vvvv_reg, - SET_VEX_L_128, - SET_VEX_PP_NONE); - } else { - ByteOne = EmitVexPrefixByteOne(src.NeedsRex(), - Rex_x, - Rex_b, - SET_VEX_M_0F); - ByteTwo = EmitVexPrefixByteTwo(/*W=*/ false, - SET_VEX_L_128, - SET_VEX_PP_NONE); - } - EmitUint8(ByteZero); - EmitUint8(ByteOne); - if (!is_twobyte_form) { - EmitUint8(ByteTwo); - } + EmitVexPrefixForAddress(dst, src.NeedsRex(), SET_VEX_L_128, SET_VEX_PP_NONE); // Instruction Opcode EmitUint8(0x29); // Instruction Operands @@ -649,37 +564,8 @@ void X86_64Assembler::movups(const Address& dst, XmmRegister src) { void X86_64Assembler::vmovups(const Address& dst, XmmRegister src) { DCHECK(CpuHasAVXorAVX2FeatureFlag()); AssemblerBuffer::EnsureCapacity ensured(&buffer_); - uint8_t ByteZero, ByteOne, ByteTwo; - bool is_twobyte_form = false; - // Instruction VEX Prefix - uint8_t rex = dst.rex(); - bool Rex_x = rex & GET_REX_X; - bool Rex_b = rex & GET_REX_B; - if (!Rex_b && !Rex_x) { - is_twobyte_form = true; - } - ByteZero = EmitVexPrefixByteZero(is_twobyte_form); - if (is_twobyte_form) { - X86_64ManagedRegister vvvv_reg = ManagedRegister::NoRegister().AsX86_64(); - ByteOne = EmitVexPrefixByteOne(src.NeedsRex(), - vvvv_reg, - SET_VEX_L_128, - SET_VEX_PP_NONE); - } else { - ByteOne = EmitVexPrefixByteOne(src.NeedsRex(), - Rex_x, - Rex_b, - SET_VEX_M_0F); - ByteTwo = EmitVexPrefixByteTwo(/*W=*/ false, - SET_VEX_L_128, - SET_VEX_PP_NONE); - } - EmitUint8(ByteZero); - EmitUint8(ByteOne); - if (!is_twobyte_form) { - EmitUint8(ByteTwo); - } + EmitVexPrefixForAddress(dst, src.NeedsRex(), SET_VEX_L_128, SET_VEX_PP_NONE); // Instruction Opcode EmitUint8(0x11); // Instruction Operands @@ -1118,37 +1004,8 @@ void X86_64Assembler::movapd(XmmRegister dst, const Address& src) { void X86_64Assembler::vmovapd(XmmRegister dst, const Address& src) { DCHECK(CpuHasAVXorAVX2FeatureFlag()); AssemblerBuffer::EnsureCapacity ensured(&buffer_); - uint8_t ByteZero, ByteOne, ByteTwo; - bool is_twobyte_form = false; - // Instruction VEX Prefix - uint8_t rex = src.rex(); - bool Rex_x = rex & GET_REX_X; - bool Rex_b = rex & GET_REX_B; - if (!Rex_b && !Rex_x) { - is_twobyte_form = true; - } - ByteZero = EmitVexPrefixByteZero(is_twobyte_form); - if (is_twobyte_form) { - X86_64ManagedRegister vvvv_reg = ManagedRegister::NoRegister().AsX86_64(); - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), - vvvv_reg, - SET_VEX_L_128, - SET_VEX_PP_66); - } else { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), - Rex_x, - Rex_b, - SET_VEX_M_0F); - ByteTwo = EmitVexPrefixByteTwo(/*W=*/ false, - SET_VEX_L_128, - SET_VEX_PP_66); - } - EmitUint8(ByteZero); - EmitUint8(ByteOne); - if (!is_twobyte_form) { - EmitUint8(ByteTwo); - } + EmitVexPrefixForAddress(src, dst.NeedsRex(), SET_VEX_L_128, SET_VEX_PP_66); // Instruction Opcode EmitUint8(0x28); // Instruction Operands @@ -1172,36 +1029,8 @@ void X86_64Assembler::movupd(XmmRegister dst, const Address& src) { void X86_64Assembler::vmovupd(XmmRegister dst, const Address& src) { DCHECK(CpuHasAVXorAVX2FeatureFlag()); AssemblerBuffer::EnsureCapacity ensured(&buffer_); - bool is_twobyte_form = false; - uint8_t ByteZero, ByteOne, ByteTwo; - // Instruction VEX Prefix - uint8_t rex = src.rex(); - bool Rex_x = rex & GET_REX_X; - bool Rex_b = rex & GET_REX_B; - if (!Rex_b && !Rex_x) { - is_twobyte_form = true; - } - ByteZero = EmitVexPrefixByteZero(is_twobyte_form); - if (is_twobyte_form) { - X86_64ManagedRegister vvvv_reg = ManagedRegister::NoRegister().AsX86_64(); - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), - vvvv_reg, - SET_VEX_L_128, - SET_VEX_PP_66); - } else { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), - Rex_x, - Rex_b, - SET_VEX_M_0F); - ByteTwo = EmitVexPrefixByteTwo(/*W=*/ false, - SET_VEX_L_128, - SET_VEX_PP_66); - } - EmitUint8(ByteZero); - EmitUint8(ByteOne); - if (!is_twobyte_form) - EmitUint8(ByteTwo); + EmitVexPrefixForAddress(src, dst.NeedsRex(), SET_VEX_L_128, SET_VEX_PP_66); // Instruction Opcode EmitUint8(0x10); // Instruction Operands @@ -1225,36 +1054,8 @@ void X86_64Assembler::movapd(const Address& dst, XmmRegister src) { void X86_64Assembler::vmovapd(const Address& dst, XmmRegister src) { DCHECK(CpuHasAVXorAVX2FeatureFlag()); AssemblerBuffer::EnsureCapacity ensured(&buffer_); - bool is_twobyte_form = false; - uint8_t ByteZero, ByteOne, ByteTwo; // Instruction VEX Prefix - uint8_t rex = dst.rex(); - bool Rex_x = rex & GET_REX_X; - bool Rex_b = rex & GET_REX_B; - if (!Rex_x && !Rex_b) { - is_twobyte_form = true; - } - ByteZero = EmitVexPrefixByteZero(is_twobyte_form); - if (is_twobyte_form) { - X86_64ManagedRegister vvvv_reg = ManagedRegister::NoRegister().AsX86_64(); - ByteOne = EmitVexPrefixByteOne(src.NeedsRex(), - vvvv_reg, - SET_VEX_L_128, - SET_VEX_PP_66); - } else { - ByteOne = EmitVexPrefixByteOne(src.NeedsRex(), - Rex_x, - Rex_b, - SET_VEX_M_0F); - ByteTwo = EmitVexPrefixByteTwo(/*W=*/ false, - SET_VEX_L_128, - SET_VEX_PP_66); - } - EmitUint8(ByteZero); - EmitUint8(ByteOne); - if (!is_twobyte_form) { - EmitUint8(ByteTwo); - } + EmitVexPrefixForAddress(dst, src.NeedsRex(), SET_VEX_L_128, SET_VEX_PP_66); // Instruction Opcode EmitUint8(0x29); // Instruction Operands @@ -1278,37 +1079,8 @@ void X86_64Assembler::movupd(const Address& dst, XmmRegister src) { void X86_64Assembler::vmovupd(const Address& dst, XmmRegister src) { DCHECK(CpuHasAVXorAVX2FeatureFlag()); AssemblerBuffer::EnsureCapacity ensured(&buffer_); - bool is_twobyte_form = false; - uint8_t ByteZero, ByteOne, ByteTwo; - // Instruction VEX Prefix - uint8_t rex = dst.rex(); - bool Rex_x = rex & GET_REX_X; - bool Rex_b = rex & GET_REX_B; - if (!Rex_x && !Rex_b) { - is_twobyte_form = true; - } - ByteZero = EmitVexPrefixByteZero(is_twobyte_form); - if (is_twobyte_form) { - X86_64ManagedRegister vvvv_reg = ManagedRegister::NoRegister().AsX86_64(); - ByteOne = EmitVexPrefixByteOne(src.NeedsRex(), - vvvv_reg, - SET_VEX_L_128, - SET_VEX_PP_66); - } else { - ByteOne = EmitVexPrefixByteOne(src.NeedsRex(), - Rex_x, - Rex_b, - SET_VEX_M_0F); - ByteTwo = EmitVexPrefixByteTwo(/*W=*/ false, - SET_VEX_L_128, - SET_VEX_PP_66); - } - EmitUint8(ByteZero); - EmitUint8(ByteOne); - if (!is_twobyte_form) { - EmitUint8(ByteTwo); - } + EmitVexPrefixForAddress(dst, src.NeedsRex(), SET_VEX_L_128, SET_VEX_PP_66); // Instruction Opcode EmitUint8(0x11); // Instruction Operands @@ -1664,37 +1436,8 @@ void X86_64Assembler::movdqa(XmmRegister dst, const Address& src) { void X86_64Assembler::vmovdqa(XmmRegister dst, const Address& src) { DCHECK(CpuHasAVXorAVX2FeatureFlag()); AssemblerBuffer::EnsureCapacity ensured(&buffer_); - uint8_t ByteZero, ByteOne, ByteTwo; - bool is_twobyte_form = false; - // Instruction VEX Prefix - uint8_t rex = src.rex(); - bool Rex_x = rex & GET_REX_X; - bool Rex_b = rex & GET_REX_B; - if (!Rex_x && !Rex_b) { - is_twobyte_form = true; - } - ByteZero = EmitVexPrefixByteZero(is_twobyte_form); - if (is_twobyte_form) { - X86_64ManagedRegister vvvv_reg = ManagedRegister::NoRegister().AsX86_64(); - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), - vvvv_reg, - SET_VEX_L_128, - SET_VEX_PP_66); - } else { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), - Rex_x, - Rex_b, - SET_VEX_M_0F); - ByteTwo = EmitVexPrefixByteTwo(/*W=*/ false, - SET_VEX_L_128, - SET_VEX_PP_66); - } - EmitUint8(ByteZero); - EmitUint8(ByteOne); - if (!is_twobyte_form) { - EmitUint8(ByteTwo); - } + EmitVexPrefixForAddress(src, dst.NeedsRex(), SET_VEX_L_128, SET_VEX_PP_66); // Instruction Opcode EmitUint8(0x6F); // Instruction Operands @@ -1719,37 +1462,8 @@ Load Unaligned */ void X86_64Assembler::vmovdqu(XmmRegister dst, const Address& src) { DCHECK(CpuHasAVXorAVX2FeatureFlag()); AssemblerBuffer::EnsureCapacity ensured(&buffer_); - uint8_t ByteZero, ByteOne, ByteTwo; - bool is_twobyte_form = false; - // Instruction VEX Prefix - uint8_t rex = src.rex(); - bool Rex_x = rex & GET_REX_X; - bool Rex_b = rex & GET_REX_B; - if (!Rex_x && !Rex_b) { - is_twobyte_form = true; - } - ByteZero = EmitVexPrefixByteZero(is_twobyte_form); - if (is_twobyte_form) { - X86_64ManagedRegister vvvv_reg = ManagedRegister::NoRegister().AsX86_64(); - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), - vvvv_reg, - SET_VEX_L_128, - SET_VEX_PP_F3); - } else { - ByteOne = EmitVexPrefixByteOne(dst.NeedsRex(), - Rex_x, - Rex_b, - SET_VEX_M_0F); - ByteTwo = EmitVexPrefixByteTwo(/*W=*/ false, - SET_VEX_L_128, - SET_VEX_PP_F3); - } - EmitUint8(ByteZero); - EmitUint8(ByteOne); - if (!is_twobyte_form) { - EmitUint8(ByteTwo); - } + EmitVexPrefixForAddress(src, dst.NeedsRex(), SET_VEX_L_128, SET_VEX_PP_F3); // Instruction Opcode EmitUint8(0x6F); // Instruction Operands @@ -1773,36 +1487,8 @@ void X86_64Assembler::movdqa(const Address& dst, XmmRegister src) { void X86_64Assembler::vmovdqa(const Address& dst, XmmRegister src) { DCHECK(CpuHasAVXorAVX2FeatureFlag()); AssemblerBuffer::EnsureCapacity ensured(&buffer_); - bool is_twobyte_form = false; - uint8_t ByteZero, ByteOne, ByteTwo; // Instruction VEX Prefix - uint8_t rex = dst.rex(); - bool Rex_x = rex & GET_REX_X; - bool Rex_b = rex & GET_REX_B; - if (!Rex_x && !Rex_b) { - is_twobyte_form = true; - } - ByteZero = EmitVexPrefixByteZero(is_twobyte_form); - if (is_twobyte_form) { - X86_64ManagedRegister vvvv_reg = ManagedRegister::NoRegister().AsX86_64(); - ByteOne = EmitVexPrefixByteOne(src.NeedsRex(), - vvvv_reg, - SET_VEX_L_128, - SET_VEX_PP_66); - } else { - ByteOne = EmitVexPrefixByteOne(src.NeedsRex(), - Rex_x, - Rex_b, - SET_VEX_M_0F); - ByteTwo = EmitVexPrefixByteTwo(/*W=*/ false, - SET_VEX_L_128, - SET_VEX_PP_66); - } - EmitUint8(ByteZero); - EmitUint8(ByteOne); - if (!is_twobyte_form) { - EmitUint8(ByteTwo); - } + EmitVexPrefixForAddress(dst, src.NeedsRex(), SET_VEX_L_128, SET_VEX_PP_66); // Instruction Opcode EmitUint8(0x7F); // Instruction Operands @@ -1826,37 +1512,8 @@ void X86_64Assembler::movdqu(const Address& dst, XmmRegister src) { void X86_64Assembler::vmovdqu(const Address& dst, XmmRegister src) { DCHECK(CpuHasAVXorAVX2FeatureFlag()); AssemblerBuffer::EnsureCapacity ensured(&buffer_); - uint8_t ByteZero, ByteOne, ByteTwo; - bool is_twobyte_form = false; - // Instruction VEX Prefix - uint8_t rex = dst.rex(); - bool Rex_x = rex & GET_REX_X; - bool Rex_b = rex & GET_REX_B; - if (!Rex_b && !Rex_x) { - is_twobyte_form = true; - } - ByteZero = EmitVexPrefixByteZero(is_twobyte_form); - if (is_twobyte_form) { - X86_64ManagedRegister vvvv_reg = ManagedRegister::NoRegister().AsX86_64(); - ByteOne = EmitVexPrefixByteOne(src.NeedsRex(), - vvvv_reg, - SET_VEX_L_128, - SET_VEX_PP_F3); - } else { - ByteOne = EmitVexPrefixByteOne(src.NeedsRex(), - Rex_x, - Rex_b, - SET_VEX_M_0F); - ByteTwo = EmitVexPrefixByteTwo(/*W=*/ false, - SET_VEX_L_128, - SET_VEX_PP_F3); - } - EmitUint8(ByteZero); - EmitUint8(ByteOne); - if (!is_twobyte_form) { - EmitUint8(ByteTwo); - } + EmitVexPrefixForAddress(dst, src.NeedsRex(), SET_VEX_L_128, SET_VEX_PP_F3); // Instruction Opcode EmitUint8(0x7F); // Instruction Operands @@ -5736,6 +5393,27 @@ uint8_t X86_64Assembler::EmitVexPrefixByteZero(bool is_twobyte_form) { return vex_prefix; } +void X86_64Assembler::EmitVexPrefixForAddress(const Address& addr, bool r, int vex_l, int vex_pp) { + uint8_t rex = addr.rex(); + bool rex_x = (rex & GET_REX_X) != 0u; + bool rex_b = (rex & GET_REX_B) != 0u; + bool is_twobyte_form = (!rex_b && !rex_x); + uint8_t byte_zero = EmitVexPrefixByteZero(is_twobyte_form); + uint8_t byte_one, byte_two; + if (is_twobyte_form) { + X86_64ManagedRegister vvvv_reg = ManagedRegister::NoRegister().AsX86_64(); + byte_one = EmitVexPrefixByteOne(r, vvvv_reg, vex_l, vex_pp); + } else { + byte_one = EmitVexPrefixByteOne(r, rex_x, rex_b, SET_VEX_M_0F); + byte_two = EmitVexPrefixByteTwo(/*W=*/ false, vex_l, vex_pp); + } + EmitUint8(byte_zero); + EmitUint8(byte_one); + if (!is_twobyte_form) { + EmitUint8(byte_two); + } +} + uint8_t X86_64Assembler::EmitVexPrefixByteOne(bool R, bool X, bool B, int SET_VEX_M) { // Vex Byte 1, uint8_t vex_prefix = VEX_INIT; diff --git a/compiler/utils/x86_64/assembler_x86_64.h b/compiler/utils/x86_64/assembler_x86_64.h index b6076c141b..5f6d754f8d 100644 --- a/compiler/utils/x86_64/assembler_x86_64.h +++ b/compiler/utils/x86_64/assembler_x86_64.h @@ -1167,6 +1167,10 @@ class X86_64Assembler final : public Assembler { bool normalize_both = false); void EmitOptionalByteRegNormalizingRex32(CpuRegister dst, const Operand& operand); + void EmitVexPrefixForAddress(const Address& addr, bool r, int vex_l, int vex_pp); + + // TODO: Rename these functions. They calculate the byte but they do not "emit" that + // byte to the code buffer. uint8_t EmitVexPrefixByteZero(bool is_twobyte_form); uint8_t EmitVexPrefixByteOne(bool R, bool X, bool B, int SET_VEX_M); uint8_t EmitVexPrefixByteOne(bool R, |