diff options
| -rw-r--r-- | compiler/optimizing/instruction_builder.cc | 145 | ||||
| -rw-r--r-- | compiler/optimizing/instruction_simplifier.cc | 60 | ||||
| -rw-r--r-- | compiler/optimizing/intrinsics.h | 8 | ||||
| -rw-r--r-- | compiler/optimizing/intrinsics_arm64.cc | 107 | ||||
| -rw-r--r-- | compiler/optimizing/intrinsics_arm_vixl.cc | 111 | ||||
| -rw-r--r-- | compiler/optimizing/intrinsics_x86.cc | 112 | ||||
| -rw-r--r-- | compiler/optimizing/intrinsics_x86_64.cc | 74 |
7 files changed, 283 insertions, 334 deletions
diff --git a/compiler/optimizing/instruction_builder.cc b/compiler/optimizing/instruction_builder.cc index 4364d39263..8ec28e0ab1 100644 --- a/compiler/optimizing/instruction_builder.cc +++ b/compiler/optimizing/instruction_builder.cc @@ -1168,6 +1168,149 @@ static bool VarHandleAccessorNeedsReturnTypeCheck(HInvoke* invoke, DataType::Typ } } +// This function initializes `VarHandleOptimizations`, does a number of static checks and disables +// the intrinsic if some of the checks fail. This is necessary for the code generator to work (for +// both the baseline and the optimizing compiler). +static void DecideVarHandleIntrinsic(HInvoke* invoke) { + switch (invoke->GetIntrinsic()) { + case Intrinsics::kVarHandleCompareAndExchange: + case Intrinsics::kVarHandleCompareAndExchangeAcquire: + case Intrinsics::kVarHandleCompareAndExchangeRelease: + case Intrinsics::kVarHandleCompareAndSet: + case Intrinsics::kVarHandleGet: + case Intrinsics::kVarHandleGetAcquire: + case Intrinsics::kVarHandleGetAndAdd: + case Intrinsics::kVarHandleGetAndAddAcquire: + case Intrinsics::kVarHandleGetAndAddRelease: + case Intrinsics::kVarHandleGetAndBitwiseAnd: + case Intrinsics::kVarHandleGetAndBitwiseAndAcquire: + case Intrinsics::kVarHandleGetAndBitwiseAndRelease: + case Intrinsics::kVarHandleGetAndBitwiseOr: + case Intrinsics::kVarHandleGetAndBitwiseOrAcquire: + case Intrinsics::kVarHandleGetAndBitwiseOrRelease: + case Intrinsics::kVarHandleGetAndBitwiseXor: + case Intrinsics::kVarHandleGetAndBitwiseXorAcquire: + case Intrinsics::kVarHandleGetAndBitwiseXorRelease: + case Intrinsics::kVarHandleGetAndSet: + case Intrinsics::kVarHandleGetAndSetAcquire: + case Intrinsics::kVarHandleGetAndSetRelease: + case Intrinsics::kVarHandleGetOpaque: + case Intrinsics::kVarHandleGetVolatile: + case Intrinsics::kVarHandleSet: + case Intrinsics::kVarHandleSetOpaque: + case Intrinsics::kVarHandleSetRelease: + case Intrinsics::kVarHandleSetVolatile: + case Intrinsics::kVarHandleWeakCompareAndSet: + case Intrinsics::kVarHandleWeakCompareAndSetAcquire: + case Intrinsics::kVarHandleWeakCompareAndSetPlain: + case Intrinsics::kVarHandleWeakCompareAndSetRelease: + break; + default: + return; // Not a VarHandle intrinsic, skip. + } + + DCHECK(invoke->IsInvokePolymorphic()); + VarHandleOptimizations optimizations(invoke); + + // Do only simple static checks here (those for which we have enough information). More complex + // checks should be done in instruction simplifier, which runs after other optimization passes + // that may provide useful information. + + size_t expected_coordinates_count = GetExpectedVarHandleCoordinatesCount(invoke); + if (expected_coordinates_count > 2u) { + optimizations.SetDoNotIntrinsify(); + return; + } + if (expected_coordinates_count != 0u) { + // Except for static fields (no coordinates), the first coordinate must be a reference. + // Do not intrinsify if the reference is null as we would always go to slow path anyway. + HInstruction* object = invoke->InputAt(1); + if (object->GetType() != DataType::Type::kReference || object->IsNullConstant()) { + optimizations.SetDoNotIntrinsify(); + return; + } + } + if (expected_coordinates_count == 2u) { + // For arrays and views, the second coordinate must be convertible to `int`. + // In this context, `boolean` is not convertible but we have to look at the shorty + // as compiler transformations can give the invoke a valid boolean input. + DataType::Type index_type = GetDataTypeFromShorty(invoke, 2); + if (index_type == DataType::Type::kBool || + DataType::Kind(index_type) != DataType::Type::kInt32) { + optimizations.SetDoNotIntrinsify(); + return; + } + } + + uint32_t number_of_arguments = invoke->GetNumberOfArguments(); + DataType::Type return_type = invoke->GetType(); + mirror::VarHandle::AccessModeTemplate access_mode_template = + mirror::VarHandle::GetAccessModeTemplateByIntrinsic(invoke->GetIntrinsic()); + switch (access_mode_template) { + case mirror::VarHandle::AccessModeTemplate::kGet: + // The return type should be the same as varType, so it shouldn't be void. + if (return_type == DataType::Type::kVoid) { + optimizations.SetDoNotIntrinsify(); + return; + } + break; + case mirror::VarHandle::AccessModeTemplate::kSet: + if (return_type != DataType::Type::kVoid) { + optimizations.SetDoNotIntrinsify(); + return; + } + break; + case mirror::VarHandle::AccessModeTemplate::kCompareAndSet: { + if (return_type != DataType::Type::kBool) { + optimizations.SetDoNotIntrinsify(); + return; + } + uint32_t expected_value_index = number_of_arguments - 2; + uint32_t new_value_index = number_of_arguments - 1; + DataType::Type expected_value_type = GetDataTypeFromShorty(invoke, expected_value_index); + DataType::Type new_value_type = GetDataTypeFromShorty(invoke, new_value_index); + if (expected_value_type != new_value_type) { + optimizations.SetDoNotIntrinsify(); + return; + } + break; + } + case mirror::VarHandle::AccessModeTemplate::kCompareAndExchange: { + uint32_t expected_value_index = number_of_arguments - 2; + uint32_t new_value_index = number_of_arguments - 1; + DataType::Type expected_value_type = GetDataTypeFromShorty(invoke, expected_value_index); + DataType::Type new_value_type = GetDataTypeFromShorty(invoke, new_value_index); + if (expected_value_type != new_value_type || return_type != expected_value_type) { + optimizations.SetDoNotIntrinsify(); + return; + } + break; + } + case mirror::VarHandle::AccessModeTemplate::kGetAndUpdate: { + DataType::Type value_type = GetDataTypeFromShorty(invoke, number_of_arguments - 1); + if (IsVarHandleGetAndAdd(invoke) && + (value_type == DataType::Type::kReference || value_type == DataType::Type::kBool)) { + // We should only add numerical types. + optimizations.SetDoNotIntrinsify(); + return; + } else if (IsVarHandleGetAndBitwiseOp(invoke) && !DataType::IsIntegralType(value_type)) { + // We can only apply operators to bitwise integral types. + // Note that bitwise VarHandle operations accept a non-integral boolean type and + // perform the appropriate logical operation. However, the result is the same as + // using the bitwise operation on our boolean representation and this fits well + // with DataType::IsIntegralType() treating the compiler type kBool as integral. + optimizations.SetDoNotIntrinsify(); + return; + } + if (value_type != return_type) { + optimizations.SetDoNotIntrinsify(); + return; + } + break; + } + } +} + bool HInstructionBuilder::BuildInvokePolymorphic(uint32_t dex_pc, uint32_t method_idx, dex::ProtoIndex proto_idx, @@ -1217,6 +1360,8 @@ bool HInstructionBuilder::BuildInvokePolymorphic(uint32_t dex_pc, latest_result_ = current_block_->GetLastInstruction(); } + DecideVarHandleIntrinsic(invoke); + return true; } diff --git a/compiler/optimizing/instruction_simplifier.cc b/compiler/optimizing/instruction_simplifier.cc index a6104effd7..1a7544da55 100644 --- a/compiler/optimizing/instruction_simplifier.cc +++ b/compiler/optimizing/instruction_simplifier.cc @@ -22,6 +22,7 @@ #include "data_type-inl.h" #include "escape.h" #include "intrinsics.h" +#include "intrinsics_utils.h" #include "mirror/class-inl.h" #include "optimizing/nodes.h" #include "scoped_thread_state_change-inl.h" @@ -112,8 +113,6 @@ class InstructionSimplifierVisitor : public HGraphDelegateVisitor { void VisitVecMul(HVecMul* instruction) override; void VisitPredicatedInstanceFieldGet(HPredicatedInstanceFieldGet* instruction) override; - bool CanEnsureNotNullAt(HInstruction* instr, HInstruction* at) const; - void SimplifySystemArrayCopy(HInvoke* invoke); void SimplifyStringEquals(HInvoke* invoke); void SimplifyFP2Int(HInvoke* invoke); @@ -123,6 +122,7 @@ class InstructionSimplifierVisitor : public HGraphDelegateVisitor { void SimplifyNPEOnArgN(HInvoke* invoke, size_t); void SimplifyReturnThis(HInvoke* invoke); void SimplifyAllocationIntrinsic(HInvoke* invoke); + void SimplifyVarHandleIntrinsic(HInvoke* invoke); CodeGenerator* codegen_; OptimizingCompilerStats* stats_; @@ -580,7 +580,7 @@ void InstructionSimplifierVisitor::VisitNullCheck(HNullCheck* null_check) { } } -bool InstructionSimplifierVisitor::CanEnsureNotNullAt(HInstruction* input, HInstruction* at) const { +bool CanEnsureNotNullAt(HInstruction* input, HInstruction* at) { if (!input->CanBeNull()) { return true; } @@ -2762,6 +2762,27 @@ void InstructionSimplifierVisitor::SimplifyAllocationIntrinsic(HInvoke* invoke) } } +void InstructionSimplifierVisitor::SimplifyVarHandleIntrinsic(HInvoke* invoke) { + DCHECK(invoke->IsInvokePolymorphic()); + VarHandleOptimizations optimizations(invoke); + + if (optimizations.GetDoNotIntrinsify()) { + // Preceding static checks disabled intrinsic, so no need to analyze further. + return; + } + + size_t expected_coordinates_count = GetExpectedVarHandleCoordinatesCount(invoke); + if (expected_coordinates_count == 1u) { + HInstruction* object = invoke->InputAt(1); + // The following has been ensured by static checks in done in the instruction builder. + DCHECK(object->GetType() == DataType::Type::kReference && !object->IsNullConstant()); + // Test whether we can avoid the null check on the object. + if (CanEnsureNotNullAt(object, invoke)) { + optimizations.SetSkipObjectNullCheck(); + } + } +} + void InstructionSimplifierVisitor::VisitInvoke(HInvoke* instruction) { switch (instruction->GetIntrinsic()) { case Intrinsics::kStringEquals: @@ -2809,6 +2830,39 @@ void InstructionSimplifierVisitor::VisitInvoke(HInvoke* instruction) { case Intrinsics::kStringBuilderToString: SimplifyAllocationIntrinsic(instruction); break; + case Intrinsics::kVarHandleCompareAndExchange: + case Intrinsics::kVarHandleCompareAndExchangeAcquire: + case Intrinsics::kVarHandleCompareAndExchangeRelease: + case Intrinsics::kVarHandleCompareAndSet: + case Intrinsics::kVarHandleGet: + case Intrinsics::kVarHandleGetAcquire: + case Intrinsics::kVarHandleGetAndAdd: + case Intrinsics::kVarHandleGetAndAddAcquire: + case Intrinsics::kVarHandleGetAndAddRelease: + case Intrinsics::kVarHandleGetAndBitwiseAnd: + case Intrinsics::kVarHandleGetAndBitwiseAndAcquire: + case Intrinsics::kVarHandleGetAndBitwiseAndRelease: + case Intrinsics::kVarHandleGetAndBitwiseOr: + case Intrinsics::kVarHandleGetAndBitwiseOrAcquire: + case Intrinsics::kVarHandleGetAndBitwiseOrRelease: + case Intrinsics::kVarHandleGetAndBitwiseXor: + case Intrinsics::kVarHandleGetAndBitwiseXorAcquire: + case Intrinsics::kVarHandleGetAndBitwiseXorRelease: + case Intrinsics::kVarHandleGetAndSet: + case Intrinsics::kVarHandleGetAndSetAcquire: + case Intrinsics::kVarHandleGetAndSetRelease: + case Intrinsics::kVarHandleGetOpaque: + case Intrinsics::kVarHandleGetVolatile: + case Intrinsics::kVarHandleSet: + case Intrinsics::kVarHandleSetOpaque: + case Intrinsics::kVarHandleSetRelease: + case Intrinsics::kVarHandleSetVolatile: + case Intrinsics::kVarHandleWeakCompareAndSet: + case Intrinsics::kVarHandleWeakCompareAndSetAcquire: + case Intrinsics::kVarHandleWeakCompareAndSetPlain: + case Intrinsics::kVarHandleWeakCompareAndSetRelease: + SimplifyVarHandleIntrinsic(instruction); + break; case Intrinsics::kIntegerRotateRight: case Intrinsics::kLongRotateRight: case Intrinsics::kIntegerRotateLeft: diff --git a/compiler/optimizing/intrinsics.h b/compiler/optimizing/intrinsics.h index 62b5faa7d0..9bc8c5126b 100644 --- a/compiler/optimizing/intrinsics.h +++ b/compiler/optimizing/intrinsics.h @@ -224,6 +224,14 @@ class SystemArrayCopyOptimizations : public IntrinsicOptimizations { DISALLOW_COPY_AND_ASSIGN(SystemArrayCopyOptimizations); }; +class VarHandleOptimizations : public IntrinsicOptimizations { + public: + explicit VarHandleOptimizations(HInvoke* invoke) : IntrinsicOptimizations(invoke) {} + + INTRINSIC_OPTIMIZATION(DoNotIntrinsify, 0); // One of the checks is statically known to fail. + INTRINSIC_OPTIMIZATION(SkipObjectNullCheck, 1); // Not applicable for static fields. +}; + #undef INTRISIC_OPTIMIZATION // diff --git a/compiler/optimizing/intrinsics_arm64.cc b/compiler/optimizing/intrinsics_arm64.cc index d0c64c2230..800e5bdd52 100644 --- a/compiler/optimizing/intrinsics_arm64.cc +++ b/compiler/optimizing/intrinsics_arm64.cc @@ -4045,6 +4045,7 @@ static void GenerateVarHandleStaticFieldCheck(HInvoke* invoke, static void GenerateVarHandleInstanceFieldChecks(HInvoke* invoke, CodeGeneratorARM64* codegen, SlowPathCodeARM64* slow_path) { + VarHandleOptimizations optimizations(invoke); MacroAssembler* masm = codegen->GetVIXLAssembler(); Register varhandle = InputRegisterAt(invoke, 0); Register object = InputRegisterAt(invoke, 1); @@ -4053,7 +4054,9 @@ static void GenerateVarHandleInstanceFieldChecks(HInvoke* invoke, const MemberOffset coordinate_type1_offset = mirror::VarHandle::CoordinateType1Offset(); // Null-check the object. - __ Cbz(object, slow_path->GetEntryLabel()); + if (!optimizations.GetSkipObjectNullCheck()) { + __ Cbz(object, slow_path->GetEntryLabel()); + } UseScratchRegisterScope temps(masm); Register temp = temps.AcquireW(); @@ -4090,6 +4093,7 @@ static DataType::Type GetVarHandleExpectedValueType(HInvoke* invoke, static void GenerateVarHandleArrayChecks(HInvoke* invoke, CodeGeneratorARM64* codegen, VarHandleSlowPathARM64* slow_path) { + VarHandleOptimizations optimizations(invoke); MacroAssembler* masm = codegen->GetVIXLAssembler(); Register varhandle = InputRegisterAt(invoke, 0); Register object = InputRegisterAt(invoke, 1); @@ -4106,7 +4110,9 @@ static void GenerateVarHandleArrayChecks(HInvoke* invoke, const MemberOffset array_length_offset = mirror::Array::LengthOffset(); // Null-check the object. - __ Cbz(object, slow_path->GetEntryLabel()); + if (!optimizations.GetSkipObjectNullCheck()) { + __ Cbz(object, slow_path->GetEntryLabel()); + } UseScratchRegisterScope temps(masm); Register temp = temps.AcquireW(); @@ -4262,91 +4268,6 @@ static void GenerateVarHandleTarget(HInvoke* invoke, } } -static bool HasVarHandleIntrinsicImplementation(HInvoke* invoke) { - size_t expected_coordinates_count = GetExpectedVarHandleCoordinatesCount(invoke); - if (expected_coordinates_count > 2u) { - // Invalid coordinate count. This invoke shall throw at runtime. - return false; - } - if (expected_coordinates_count != 0u && - invoke->InputAt(1)->GetType() != DataType::Type::kReference) { - // Except for static fields (no coordinates), the first coordinate must be a reference. - return false; - } - if (expected_coordinates_count == 2u) { - // For arrays and views, the second coordinate must be convertible to `int`. - // In this context, `boolean` is not convertible but we have to look at the shorty - // as compiler transformations can give the invoke a valid boolean input. - DataType::Type index_type = GetDataTypeFromShorty(invoke, 2); - if (index_type == DataType::Type::kBool || - DataType::Kind(index_type) != DataType::Type::kInt32) { - return false; - } - } - - uint32_t number_of_arguments = invoke->GetNumberOfArguments(); - DataType::Type return_type = invoke->GetType(); - mirror::VarHandle::AccessModeTemplate access_mode_template = - mirror::VarHandle::GetAccessModeTemplateByIntrinsic(invoke->GetIntrinsic()); - switch (access_mode_template) { - case mirror::VarHandle::AccessModeTemplate::kGet: - // The return type should be the same as varType, so it shouldn't be void. - if (return_type == DataType::Type::kVoid) { - return false; - } - break; - case mirror::VarHandle::AccessModeTemplate::kSet: - if (return_type != DataType::Type::kVoid) { - return false; - } - break; - case mirror::VarHandle::AccessModeTemplate::kCompareAndSet: { - if (return_type != DataType::Type::kBool) { - return false; - } - uint32_t expected_value_index = number_of_arguments - 2; - uint32_t new_value_index = number_of_arguments - 1; - DataType::Type expected_value_type = GetDataTypeFromShorty(invoke, expected_value_index); - DataType::Type new_value_type = GetDataTypeFromShorty(invoke, new_value_index); - if (expected_value_type != new_value_type) { - return false; - } - break; - } - case mirror::VarHandle::AccessModeTemplate::kCompareAndExchange: { - uint32_t expected_value_index = number_of_arguments - 2; - uint32_t new_value_index = number_of_arguments - 1; - DataType::Type expected_value_type = GetDataTypeFromShorty(invoke, expected_value_index); - DataType::Type new_value_type = GetDataTypeFromShorty(invoke, new_value_index); - if (expected_value_type != new_value_type || return_type != expected_value_type) { - return false; - } - break; - } - case mirror::VarHandle::AccessModeTemplate::kGetAndUpdate: { - DataType::Type value_type = GetDataTypeFromShorty(invoke, number_of_arguments - 1); - if (IsVarHandleGetAndAdd(invoke) && - (value_type == DataType::Type::kReference || value_type == DataType::Type::kBool)) { - // We should only add numerical types. - return false; - } else if (IsVarHandleGetAndBitwiseOp(invoke) && !DataType::IsIntegralType(value_type)) { - // We can only apply operators to bitwise integral types. - // Note that bitwise VarHandle operations accept a non-integral boolean type and - // perform the appropriate logical operation. However, the result is the same as - // using the bitwise operation on our boolean representation and this fits well - // with DataType::IsIntegralType() treating the compiler type kBool as integral. - return false; - } - if (value_type != return_type) { - return false; - } - break; - } - } - - return true; -} - static LocationSummary* CreateVarHandleCommonLocations(HInvoke* invoke) { size_t expected_coordinates_count = GetExpectedVarHandleCoordinatesCount(invoke); DataType::Type return_type = invoke->GetType(); @@ -4399,7 +4320,8 @@ static LocationSummary* CreateVarHandleCommonLocations(HInvoke* invoke) { } static void CreateVarHandleGetLocations(HInvoke* invoke) { - if (!HasVarHandleIntrinsicImplementation(invoke)) { + VarHandleOptimizations optimizations(invoke); + if (optimizations.GetDoNotIntrinsify()) { return; } @@ -4525,7 +4447,8 @@ void IntrinsicCodeGeneratorARM64::VisitVarHandleGetVolatile(HInvoke* invoke) { } static void CreateVarHandleSetLocations(HInvoke* invoke) { - if (!HasVarHandleIntrinsicImplementation(invoke)) { + VarHandleOptimizations optimizations(invoke); + if (optimizations.GetDoNotIntrinsify()) { return; } @@ -4630,7 +4553,8 @@ void IntrinsicCodeGeneratorARM64::VisitVarHandleSetVolatile(HInvoke* invoke) { } static void CreateVarHandleCompareAndSetOrExchangeLocations(HInvoke* invoke, bool return_success) { - if (!HasVarHandleIntrinsicImplementation(invoke)) { + VarHandleOptimizations optimizations(invoke); + if (optimizations.GetDoNotIntrinsify()) { return; } @@ -4977,7 +4901,8 @@ void IntrinsicCodeGeneratorARM64::VisitVarHandleWeakCompareAndSetRelease(HInvoke static void CreateVarHandleGetAndUpdateLocations(HInvoke* invoke, GetAndUpdateOp get_and_update_op) { - if (!HasVarHandleIntrinsicImplementation(invoke)) { + VarHandleOptimizations optimizations(invoke); + if (optimizations.GetDoNotIntrinsify()) { return; } diff --git a/compiler/optimizing/intrinsics_arm_vixl.cc b/compiler/optimizing/intrinsics_arm_vixl.cc index 2a2f245025..f1dfa9e008 100644 --- a/compiler/optimizing/intrinsics_arm_vixl.cc +++ b/compiler/optimizing/intrinsics_arm_vixl.cc @@ -3891,6 +3891,7 @@ static void GenerateVarHandleStaticFieldCheck(HInvoke* invoke, static void GenerateVarHandleInstanceFieldChecks(HInvoke* invoke, CodeGeneratorARMVIXL* codegen, SlowPathCodeARMVIXL* slow_path) { + VarHandleOptimizations optimizations(invoke); ArmVIXLAssembler* assembler = codegen->GetAssembler(); vixl32::Register varhandle = InputRegisterAt(invoke, 0); vixl32::Register object = InputRegisterAt(invoke, 1); @@ -3899,8 +3900,10 @@ static void GenerateVarHandleInstanceFieldChecks(HInvoke* invoke, const MemberOffset coordinate_type1_offset = mirror::VarHandle::CoordinateType1Offset(); // Null-check the object. - __ Cmp(object, 0); - __ B(eq, slow_path->GetEntryLabel()); + if (!optimizations.GetSkipObjectNullCheck()) { + __ Cmp(object, 0); + __ B(eq, slow_path->GetEntryLabel()); + } // Use the first temporary register, whether it's for the declaring class or the offset. // It is not used yet at this point. @@ -3941,6 +3944,7 @@ static DataType::Type GetVarHandleExpectedValueType(HInvoke* invoke, static void GenerateVarHandleArrayChecks(HInvoke* invoke, CodeGeneratorARMVIXL* codegen, VarHandleSlowPathARMVIXL* slow_path) { + VarHandleOptimizations optimizations(invoke); ArmVIXLAssembler* assembler = codegen->GetAssembler(); vixl32::Register varhandle = InputRegisterAt(invoke, 0); vixl32::Register object = InputRegisterAt(invoke, 1); @@ -3957,8 +3961,10 @@ static void GenerateVarHandleArrayChecks(HInvoke* invoke, const MemberOffset array_length_offset = mirror::Array::LengthOffset(); // Null-check the object. - __ Cmp(object, 0); - __ B(eq, slow_path->GetEntryLabel()); + if (!optimizations.GetSkipObjectNullCheck()) { + __ Cmp(object, 0); + __ B(eq, slow_path->GetEntryLabel()); + } // Use the offset temporary register. It is not used yet at this point. vixl32::Register temp = RegisterFrom(invoke->GetLocations()->GetTemp(0u)); @@ -4114,91 +4120,6 @@ static void GenerateVarHandleTarget(HInvoke* invoke, } } -static bool HasVarHandleIntrinsicImplementation(HInvoke* invoke) { - size_t expected_coordinates_count = GetExpectedVarHandleCoordinatesCount(invoke); - if (expected_coordinates_count > 2u) { - // Invalid coordinate count. This invoke shall throw at runtime. - return false; - } - if (expected_coordinates_count != 0u && - invoke->InputAt(1)->GetType() != DataType::Type::kReference) { - // Except for static fields (no coordinates), the first coordinate must be a reference. - return false; - } - if (expected_coordinates_count == 2u) { - // For arrays and views, the second coordinate must be convertible to `int`. - // In this context, `boolean` is not convertible but we have to look at the shorty - // as compiler transformations can give the invoke a valid boolean input. - DataType::Type index_type = GetDataTypeFromShorty(invoke, 2); - if (index_type == DataType::Type::kBool || - DataType::Kind(index_type) != DataType::Type::kInt32) { - return false; - } - } - - uint32_t number_of_arguments = invoke->GetNumberOfArguments(); - DataType::Type return_type = invoke->GetType(); - mirror::VarHandle::AccessModeTemplate access_mode_template = - mirror::VarHandle::GetAccessModeTemplateByIntrinsic(invoke->GetIntrinsic()); - switch (access_mode_template) { - case mirror::VarHandle::AccessModeTemplate::kGet: - // The return type should be the same as varType, so it shouldn't be void. - if (return_type == DataType::Type::kVoid) { - return false; - } - break; - case mirror::VarHandle::AccessModeTemplate::kSet: - if (return_type != DataType::Type::kVoid) { - return false; - } - break; - case mirror::VarHandle::AccessModeTemplate::kCompareAndSet: { - if (return_type != DataType::Type::kBool) { - return false; - } - uint32_t expected_value_index = number_of_arguments - 2; - uint32_t new_value_index = number_of_arguments - 1; - DataType::Type expected_value_type = GetDataTypeFromShorty(invoke, expected_value_index); - DataType::Type new_value_type = GetDataTypeFromShorty(invoke, new_value_index); - if (expected_value_type != new_value_type) { - return false; - } - break; - } - case mirror::VarHandle::AccessModeTemplate::kCompareAndExchange: { - uint32_t expected_value_index = number_of_arguments - 2; - uint32_t new_value_index = number_of_arguments - 1; - DataType::Type expected_value_type = GetDataTypeFromShorty(invoke, expected_value_index); - DataType::Type new_value_type = GetDataTypeFromShorty(invoke, new_value_index); - if (expected_value_type != new_value_type || return_type != expected_value_type) { - return false; - } - break; - } - case mirror::VarHandle::AccessModeTemplate::kGetAndUpdate: { - DataType::Type value_type = GetDataTypeFromShorty(invoke, number_of_arguments - 1); - if (IsVarHandleGetAndAdd(invoke) && - (value_type == DataType::Type::kReference || value_type == DataType::Type::kBool)) { - // We should only add numerical types. - return false; - } else if (IsVarHandleGetAndBitwiseOp(invoke) && !DataType::IsIntegralType(value_type)) { - // We can only apply operators to bitwise integral types. - // Note that bitwise VarHandle operations accept a non-integral boolean type and - // perform the appropriate logical operation. However, the result is the same as - // using the bitwise operation on our boolean representation and this fits well - // with DataType::IsIntegralType() treating the compiler type kBool as integral. - return false; - } - if (value_type != return_type) { - return false; - } - break; - } - } - - return true; -} - static LocationSummary* CreateVarHandleCommonLocations(HInvoke* invoke) { size_t expected_coordinates_count = GetExpectedVarHandleCoordinatesCount(invoke); DataType::Type return_type = invoke->GetType(); @@ -4251,7 +4172,8 @@ static LocationSummary* CreateVarHandleCommonLocations(HInvoke* invoke) { static void CreateVarHandleGetLocations(HInvoke* invoke, CodeGeneratorARMVIXL* codegen, bool atomic) { - if (!HasVarHandleIntrinsicImplementation(invoke)) { + VarHandleOptimizations optimizations(invoke); + if (optimizations.GetDoNotIntrinsify()) { return; } @@ -4399,7 +4321,8 @@ void IntrinsicCodeGeneratorARMVIXL::VisitVarHandleGetVolatile(HInvoke* invoke) { static void CreateVarHandleSetLocations(HInvoke* invoke, CodeGeneratorARMVIXL* codegen, bool atomic) { - if (!HasVarHandleIntrinsicImplementation(invoke)) { + VarHandleOptimizations optimizations(invoke); + if (optimizations.GetDoNotIntrinsify()) { return; } @@ -4561,7 +4484,8 @@ void IntrinsicCodeGeneratorARMVIXL::VisitVarHandleSetVolatile(HInvoke* invoke) { } static void CreateVarHandleCompareAndSetOrExchangeLocations(HInvoke* invoke, bool return_success) { - if (!HasVarHandleIntrinsicImplementation(invoke)) { + VarHandleOptimizations optimizations(invoke); + if (optimizations.GetDoNotIntrinsify()) { return; } @@ -4892,7 +4816,8 @@ void IntrinsicCodeGeneratorARMVIXL::VisitVarHandleWeakCompareAndSetRelease(HInvo static void CreateVarHandleGetAndUpdateLocations(HInvoke* invoke, GetAndUpdateOp get_and_update_op) { - if (!HasVarHandleIntrinsicImplementation(invoke)) { + VarHandleOptimizations optimizations(invoke); + if (optimizations.GetDoNotIntrinsify()) { return; } diff --git a/compiler/optimizing/intrinsics_x86.cc b/compiler/optimizing/intrinsics_x86.cc index d11bfa27d3..bed9ebe4b4 100644 --- a/compiler/optimizing/intrinsics_x86.cc +++ b/compiler/optimizing/intrinsics_x86.cc @@ -3291,81 +3291,20 @@ void IntrinsicCodeGeneratorX86::VisitIntegerDivideUnsigned(HInvoke* invoke) { __ Bind(slow_path->GetExitLabel()); } -static bool IsValidFieldVarHandleExpected(HInvoke* invoke) { - size_t expected_coordinates_count = GetExpectedVarHandleCoordinatesCount(invoke); - if (expected_coordinates_count > 1u) { - // Only static and instance fields VarHandle are supported now. +static bool HasVarHandleIntrinsicImplementation(HInvoke* invoke) { + VarHandleOptimizations optimizations(invoke); + if (optimizations.GetDoNotIntrinsify()) { return false; } - if (expected_coordinates_count == 1u && - invoke->InputAt(1)->GetType() != DataType::Type::kReference) { - // For instance fields, the source object must be a reference + size_t expected_coordinates_count = GetExpectedVarHandleCoordinatesCount(invoke); + DCHECK_LE(expected_coordinates_count, 2u); // Filtered by the `DoNotIntrinsify` flag above. + if (expected_coordinates_count > 1u) { + // Only static and instance fields VarHandle are supported now. + // TODO: add support for arrays and views. return false; } - uint32_t number_of_arguments = invoke->GetNumberOfArguments(); - DataType::Type return_type = invoke->GetType(); - mirror::VarHandle::AccessModeTemplate access_mode_template = - mirror::VarHandle::GetAccessModeTemplateByIntrinsic(invoke->GetIntrinsic()); - switch (access_mode_template) { - case mirror::VarHandle::AccessModeTemplate::kGet: - // The return type should be the same as varType, so it shouldn't be void. - if (return_type == DataType::Type::kVoid) { - return false; - } - break; - case mirror::VarHandle::AccessModeTemplate::kSet: - if (return_type != DataType::Type::kVoid) { - return false; - } - break; - case mirror::VarHandle::AccessModeTemplate::kCompareAndSet: { - if (return_type != DataType::Type::kBool) { - return false; - } - uint32_t expected_value_index = number_of_arguments - 2; - uint32_t new_value_index = number_of_arguments - 1; - DataType::Type expected_value_type = GetDataTypeFromShorty(invoke, expected_value_index); - DataType::Type new_value_type = GetDataTypeFromShorty(invoke, new_value_index); - - if (expected_value_type != new_value_type) { - return false; - } - break; - } - case mirror::VarHandle::AccessModeTemplate::kGetAndUpdate: { - DataType::Type value_type = GetDataTypeFromShorty(invoke, number_of_arguments - 1); - if (IsVarHandleGetAndAdd(invoke) && - (value_type == DataType::Type::kReference || value_type == DataType::Type::kBool)) { - // We should only add numerical types. - return false; - } else if (IsVarHandleGetAndBitwiseOp(invoke) && !DataType::IsIntegralType(value_type)) { - // We can only apply operators to bitwise integral types. - // Note that bitwise VarHandle operations accept a non-integral boolean type and - // perform the appropriate logical operation. However, the result is the same as - // using the bitwise operation on our boolean representation and this fits well - // with DataType::IsIntegralType() treating the compiler type kBool as integral. - return false; - } - if (value_type != return_type) { - return false; - } - break; - } - case mirror::VarHandle::AccessModeTemplate::kCompareAndExchange: { - uint32_t expected_value_index = number_of_arguments - 2; - uint32_t new_value_index = number_of_arguments - 1; - DataType::Type expected_value_type = GetDataTypeFromShorty(invoke, expected_value_index); - DataType::Type new_value_type = GetDataTypeFromShorty(invoke, new_value_index); - - if (expected_value_type != new_value_type || return_type != expected_value_type) { - return false; - } - break; - } - } - return true; } @@ -3426,11 +3365,15 @@ static void GenerateSubTypeObjectCheck(Register object, __ Bind(&type_matched); } -static void GenerateVarHandleInstanceFieldObjectCheck(Register varhandle_object, - Register object, - Register temp, - SlowPathCode* slow_path, - X86Assembler* assembler) { +static void GenerateVarHandleInstanceFieldChecks(HInvoke* invoke, + Register temp, + SlowPathCode* slow_path, + X86Assembler* assembler) { + VarHandleOptimizations optimizations(invoke); + LocationSummary* locations = invoke->GetLocations(); + Register varhandle_object = locations->InAt(0).AsRegister<Register>(); + Register object = locations->InAt(1).AsRegister<Register>(); + const uint32_t coordtype0_offset = mirror::VarHandle::CoordinateType0Offset().Uint32Value(); const uint32_t coordtype1_offset = mirror::VarHandle::CoordinateType1Offset().Uint32Value(); @@ -3441,8 +3384,10 @@ static void GenerateVarHandleInstanceFieldObjectCheck(Register varhandle_object, __ j(kNotEqual, slow_path->GetEntryLabel()); // Check if the object is null - __ testl(object, object); - __ j(kZero, slow_path->GetEntryLabel()); + if (!optimizations.GetSkipObjectNullCheck()) { + __ testl(object, object); + __ j(kZero, slow_path->GetEntryLabel()); + } // Check the object's class against coordinateType0. GenerateSubTypeObjectCheck(object, @@ -3490,8 +3435,7 @@ static void GenerateVarHandleCommonChecks(HInvoke *invoke, GenerateVarHandleStaticFieldCheck(vh_object, slow_path, assembler); break; case 1u: { - Register object = locations->InAt(1).AsRegister<Register>(); - GenerateVarHandleInstanceFieldObjectCheck(vh_object, object, temp, slow_path, assembler); + GenerateVarHandleInstanceFieldChecks(invoke, temp, slow_path, assembler); break; } default: @@ -3603,7 +3547,7 @@ static void CreateVarHandleGetLocations(HInvoke* invoke) { return; } - if (!IsValidFieldVarHandleExpected(invoke)) { + if (!HasVarHandleIntrinsicImplementation(invoke)) { return; } @@ -3726,7 +3670,7 @@ static void CreateVarHandleSetLocations(HInvoke* invoke) { return; } - if (!IsValidFieldVarHandleExpected(invoke)) { + if (!HasVarHandleIntrinsicImplementation(invoke)) { return; } @@ -3896,7 +3840,7 @@ static void CreateVarHandleGetAndSetLocations(HInvoke* invoke) { return; } - if (!IsValidFieldVarHandleExpected(invoke)) { + if (!HasVarHandleIntrinsicImplementation(invoke)) { return; } @@ -4067,7 +4011,7 @@ static void CreateVarHandleCompareAndSetOrExchangeLocations(HInvoke* invoke) { return; } - if (!IsValidFieldVarHandleExpected(invoke)) { + if (!HasVarHandleIntrinsicImplementation(invoke)) { return; } @@ -4250,7 +4194,7 @@ static void CreateVarHandleGetAndAddLocations(HInvoke* invoke) { return; } - if (!IsValidFieldVarHandleExpected(invoke)) { + if (!HasVarHandleIntrinsicImplementation(invoke)) { return; } @@ -4400,7 +4344,7 @@ static void CreateVarHandleGetAndBitwiseOpLocations(HInvoke* invoke) { return; } - if (!IsValidFieldVarHandleExpected(invoke)) { + if (!HasVarHandleIntrinsicImplementation(invoke)) { return; } diff --git a/compiler/optimizing/intrinsics_x86_64.cc b/compiler/optimizing/intrinsics_x86_64.cc index 33f46485e7..55ae14b373 100644 --- a/compiler/optimizing/intrinsics_x86_64.cc +++ b/compiler/optimizing/intrinsics_x86_64.cc @@ -3129,6 +3129,7 @@ static void GenerateVarHandleStaticFieldCheck(HInvoke* invoke, static void GenerateVarHandleInstanceFieldChecks(HInvoke* invoke, CodeGeneratorX86_64* codegen, SlowPathCode* slow_path) { + VarHandleOptimizations optimizations(invoke); X86_64Assembler* assembler = codegen->GetAssembler(); LocationSummary* locations = invoke->GetLocations(); @@ -3140,8 +3141,10 @@ static void GenerateVarHandleInstanceFieldChecks(HInvoke* invoke, const MemberOffset coordinate_type1_offset = mirror::VarHandle::CoordinateType1Offset(); // Null-check the object. - __ testl(object, object); - __ j(kZero, slow_path->GetEntryLabel()); + if (!optimizations.GetSkipObjectNullCheck()) { + __ testl(object, object); + __ j(kZero, slow_path->GetEntryLabel()); + } // Check that the VarHandle references an instance field by checking that // coordinateType1 == null. coordinateType0 should be not null, but this is handled by the @@ -3242,73 +3245,18 @@ static bool HasVarHandleIntrinsicImplementation(HInvoke* invoke) { return false; } + VarHandleOptimizations optimizations(invoke); + if (optimizations.GetDoNotIntrinsify()) { + return false; + } + size_t expected_coordinates_count = GetExpectedVarHandleCoordinatesCount(invoke); + DCHECK_LE(expected_coordinates_count, 2u); // Filtered by the `DoNotIntrinsify` flag above. if (expected_coordinates_count > 1u) { // Only static and instance fields VarHandle are supported now. // TODO: add support for arrays and views. return false; } - if (expected_coordinates_count != 0u && - invoke->InputAt(1)->GetType() != DataType::Type::kReference) { - // Except for static fields (no coordinates), the first coordinate must be a reference. - return false; - } - - uint32_t number_of_arguments = invoke->GetNumberOfArguments(); - DataType::Type return_type = invoke->GetType(); - mirror::VarHandle::AccessModeTemplate access_mode_template = - mirror::VarHandle::GetAccessModeTemplateByIntrinsic(invoke->GetIntrinsic()); - switch (access_mode_template) { - case mirror::VarHandle::AccessModeTemplate::kGet: - // The return type should be the same as varType, so it shouldn't be void. - if (return_type == DataType::Type::kVoid) { - return false; - } - break; - case mirror::VarHandle::AccessModeTemplate::kSet: - if (return_type != DataType::Type::kVoid) { - return false; - } - break; - case mirror::VarHandle::AccessModeTemplate::kCompareAndSet: { - if (return_type != DataType::Type::kBool) { - return false; - } - DataType::Type expected_type = GetDataTypeFromShorty(invoke, number_of_arguments - 2); - DataType::Type new_value_type = GetDataTypeFromShorty(invoke, number_of_arguments - 1); - if (expected_type != new_value_type) { - return false; - } - break; - } - case mirror::VarHandle::AccessModeTemplate::kCompareAndExchange: { - DataType::Type expected_type = GetDataTypeFromShorty(invoke, number_of_arguments - 2); - DataType::Type new_value_type = GetDataTypeFromShorty(invoke, number_of_arguments - 1); - if (expected_type != new_value_type || return_type != expected_type) { - return false; - } - break; - } - case mirror::VarHandle::AccessModeTemplate::kGetAndUpdate: { - DataType::Type value_type = GetDataTypeFromShorty(invoke, number_of_arguments - 1); - if (IsVarHandleGetAndAdd(invoke) && - (value_type == DataType::Type::kReference || value_type == DataType::Type::kBool)) { - // We should only add numerical types. - return false; - } else if (IsVarHandleGetAndBitwiseOp(invoke) && !DataType::IsIntegralType(value_type)) { - // We can only apply operators to bitwise integral types. - // Note that bitwise VarHandle operations accept a non-integral boolean type and - // perform the appropriate logical operation. However, the result is the same as - // using the bitwise operation on our boolean representation and this fits well - // with DataType::IsIntegralType() treating the compiler type kBool as integral. - return false; - } - if (return_type != value_type) { - return false; - } - break; - } - } return true; } |