diff options
Diffstat (limited to 'compiler/optimizing')
46 files changed, 2668 insertions, 1616 deletions
diff --git a/compiler/optimizing/cha_guard_optimization.cc b/compiler/optimizing/cha_guard_optimization.cc new file mode 100644 index 0000000000..fe423012ca --- /dev/null +++ b/compiler/optimizing/cha_guard_optimization.cc @@ -0,0 +1,253 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "cha_guard_optimization.h" + +namespace art { + +// Note we can only do CHA guard elimination/motion in a single pass, since +// if a guard is not removed, another guard might be removed due to +// the existence of the first guard. The first guard should not be further +// removed in another pass. For example, due to further optimizations, +// a receiver of a guard might turn out to be a parameter value, or defined at +// a different site, which makes the guard removable as a result. However +// it's not safe to remove the guard in another pass since another guard might +// have been removed due to the existence of this guard. +// +// As a consequence, we decided not to rely on other passes to remove them +// (such as GVN or instruction simplifier). + +class CHAGuardVisitor : HGraphVisitor { + public: + explicit CHAGuardVisitor(HGraph* graph) + : HGraphVisitor(graph), + block_has_cha_guard_(GetGraph()->GetBlocks().size(), + 0, + graph->GetArena()->Adapter(kArenaAllocCHA)) { + number_of_guards_to_visit_ = GetGraph()->GetNumberOfCHAGuards(); + DCHECK_NE(number_of_guards_to_visit_, 0u); + // Will recount number of guards during guard optimization. + GetGraph()->SetNumberOfCHAGuards(0); + } + + void VisitShouldDeoptimizeFlag(HShouldDeoptimizeFlag* flag) OVERRIDE; + + void VisitBasicBlock(HBasicBlock* block) OVERRIDE; + + private: + void RemoveGuard(HShouldDeoptimizeFlag* flag); + // Return true if `flag` is removed. + bool OptimizeForParameter(HShouldDeoptimizeFlag* flag, HInstruction* receiver); + // Return true if `flag` is removed. + bool OptimizeWithDominatingGuard(HShouldDeoptimizeFlag* flag, HInstruction* receiver); + // Return true if `flag` is hoisted. + bool HoistGuard(HShouldDeoptimizeFlag* flag, HInstruction* receiver); + + // Record if each block has any CHA guard. It's updated during the + // reverse post order visit. Use int instead of bool since ArenaVector + // does not support bool. + ArenaVector<int> block_has_cha_guard_; + + // The iterator that's being used for this visitor. Need it to manually + // advance the iterator due to removing/moving more than one instruction. + HInstructionIterator* instruction_iterator_; + + // Used to short-circuit the pass when there is no more guards left to visit. + uint32_t number_of_guards_to_visit_; + + DISALLOW_COPY_AND_ASSIGN(CHAGuardVisitor); +}; + +void CHAGuardVisitor::VisitBasicBlock(HBasicBlock* block) { + if (number_of_guards_to_visit_ == 0) { + return; + } + // Skip phis, just iterate through instructions. + HInstructionIterator it(block->GetInstructions()); + instruction_iterator_ = ⁢ + for (; !it.Done(); it.Advance()) { + DCHECK(it.Current()->IsInBlock()); + it.Current()->Accept(this); + } +} + +void CHAGuardVisitor::RemoveGuard(HShouldDeoptimizeFlag* flag) { + HBasicBlock* block = flag->GetBlock(); + HInstruction* compare = flag->GetNext(); + DCHECK(compare->IsNotEqual()); + HInstruction* deopt = compare->GetNext(); + DCHECK(deopt->IsDeoptimize()); + + // Advance instruction iterator first before we remove the guard. + // We need to do it twice since we remove three instructions and the + // visitor is responsible for advancing it once. + instruction_iterator_->Advance(); + instruction_iterator_->Advance(); + block->RemoveInstruction(deopt); + block->RemoveInstruction(compare); + block->RemoveInstruction(flag); +} + +bool CHAGuardVisitor::OptimizeForParameter(HShouldDeoptimizeFlag* flag, + HInstruction* receiver) { + // If some compiled code is invalidated by CHA due to class loading, the + // compiled code will not be entered anymore. So the very fact that the + // compiled code is invoked guarantees that a parameter receiver conforms + // to all the CHA devirtualization assumptions made by the compiled code, + // since all parameter receivers pre-exist any (potential) invalidation of + // the compiled code. + // + // TODO: allow more cases such as a phi whose inputs are all parameters. + if (receiver->IsParameterValue()) { + RemoveGuard(flag); + return true; + } + return false; +} + +bool CHAGuardVisitor::OptimizeWithDominatingGuard(HShouldDeoptimizeFlag* flag, + HInstruction* receiver) { + // If there is another guard that dominates the current guard, and + // that guard is dominated by receiver's definition, then the current + // guard can be eliminated, since receiver must pre-exist that other + // guard, and passing that guard guarantees that receiver conforms to + // all the CHA devirtualization assumptions. + HBasicBlock* dominator = flag->GetBlock(); + HBasicBlock* receiver_def_block = receiver->GetBlock(); + + // Complexity of the following algorithm: + // We potentially need to traverse the full dominator chain to receiver_def_block, + // plus a (partial) linear search within one block for each guard. + // So the worst case for each guard is bounded by the size of the + // biggest block plus the depth of the dominating tree. + + while (dominator != receiver_def_block) { + if (block_has_cha_guard_[dominator->GetBlockId()] == 1) { + RemoveGuard(flag); + return true; + } + dominator = dominator->GetDominator(); + } + + // At this point dominator is the block where receiver is defined. + // We do a linear search within dominator to see if there is a guard after + // receiver's definition. + HInstruction* instruction; + if (dominator == flag->GetBlock()) { + // Flag and receiver are defined in the same block. Search backward from + // the current guard. + instruction = flag->GetPrevious(); + } else { + // Search backward from the last instruction of that dominator. + instruction = dominator->GetLastInstruction(); + } + while (instruction != receiver) { + if (instruction == nullptr) { + // receiver must be defined in this block, we didn't find it + // in the instruction list, so it must be a Phi. + DCHECK(receiver->IsPhi()); + break; + } + if (instruction->IsShouldDeoptimizeFlag()) { + RemoveGuard(flag); + return true; + } + instruction = instruction->GetPrevious(); + } + return false; +} + +bool CHAGuardVisitor::HoistGuard(HShouldDeoptimizeFlag* flag, + HInstruction* receiver) { + // If receiver is loop invariant, we can hoist the guard out of the + // loop since passing a guard before entering the loop guarantees that + // receiver conforms to all the CHA devirtualization assumptions. + // We only hoist guards out of the inner loop since that offers most of the + // benefit and it might help remove other guards in the inner loop. + HBasicBlock* block = flag->GetBlock(); + HLoopInformation* loop_info = block->GetLoopInformation(); + if (loop_info != nullptr && + !loop_info->IsIrreducible() && + loop_info->IsDefinedOutOfTheLoop(receiver)) { + HInstruction* compare = flag->GetNext(); + DCHECK(compare->IsNotEqual()); + HInstruction* deopt = compare->GetNext(); + DCHECK(deopt->IsDeoptimize()); + + // Advance instruction iterator first before we move the guard. + // We need to do it twice since we move three instructions and the + // visitor is responsible for advancing it once. + instruction_iterator_->Advance(); + instruction_iterator_->Advance(); + + HBasicBlock* pre_header = loop_info->GetPreHeader(); + flag->MoveBefore(pre_header->GetLastInstruction()); + compare->MoveBefore(pre_header->GetLastInstruction()); + + block->RemoveInstruction(deopt); + HInstruction* suspend = loop_info->GetSuspendCheck(); + // Need a new deoptimize instruction that copies the environment + // of the suspend instruction for the loop. + HDeoptimize* deoptimize = + new (GetGraph()->GetArena()) HDeoptimize(compare, suspend->GetDexPc()); + pre_header->InsertInstructionBefore(deoptimize, pre_header->GetLastInstruction()); + deoptimize->CopyEnvironmentFromWithLoopPhiAdjustment( + suspend->GetEnvironment(), loop_info->GetHeader()); + block_has_cha_guard_[pre_header->GetBlockId()] = 1; + GetGraph()->IncrementNumberOfCHAGuards(); + return true; + } + return false; +} + +void CHAGuardVisitor::VisitShouldDeoptimizeFlag(HShouldDeoptimizeFlag* flag) { + number_of_guards_to_visit_--; + HInstruction* receiver = flag->InputAt(0); + // Don't need the receiver anymore. + flag->RemoveInputAt(0); + if (receiver->IsNullCheck()) { + receiver = receiver->InputAt(0); + } + + if (OptimizeForParameter(flag, receiver)) { + DCHECK(!flag->IsInBlock()); + return; + } + if (OptimizeWithDominatingGuard(flag, receiver)) { + DCHECK(!flag->IsInBlock()); + return; + } + if (HoistGuard(flag, receiver)) { + DCHECK(flag->IsInBlock()); + return; + } + + // Need to keep the CHA guard in place. + block_has_cha_guard_[flag->GetBlock()->GetBlockId()] = 1; + GetGraph()->IncrementNumberOfCHAGuards(); +} + +void CHAGuardOptimization::Run() { + if (graph_->GetNumberOfCHAGuards() == 0) { + return; + } + CHAGuardVisitor visitor(graph_); + for (HBasicBlock* block : graph_->GetReversePostOrder()) { + visitor.VisitBasicBlock(block); + } +} + +} // namespace art diff --git a/compiler/optimizing/cha_guard_optimization.h b/compiler/optimizing/cha_guard_optimization.h new file mode 100644 index 0000000000..ba0cdb81fd --- /dev/null +++ b/compiler/optimizing/cha_guard_optimization.h @@ -0,0 +1,42 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_COMPILER_OPTIMIZING_CHA_GUARD_OPTIMIZATION_H_ +#define ART_COMPILER_OPTIMIZING_CHA_GUARD_OPTIMIZATION_H_ + +#include "optimization.h" + +namespace art { + +/** + * Optimize CHA guards by removing/moving them. + */ +class CHAGuardOptimization : public HOptimization { + public: + explicit CHAGuardOptimization(HGraph* graph) + : HOptimization(graph, kCHAGuardOptimizationPassName) {} + + void Run() OVERRIDE; + + static constexpr const char* kCHAGuardOptimizationPassName = "cha_guard_optimization"; + + private: + DISALLOW_COPY_AND_ASSIGN(CHAGuardOptimization); +}; + +} // namespace art + +#endif // ART_COMPILER_OPTIMIZING_CHA_GUARD_OPTIMIZATION_H_ diff --git a/compiler/optimizing/code_generator.cc b/compiler/optimizing/code_generator.cc index fa6a5225e7..402eeee65f 100644 --- a/compiler/optimizing/code_generator.cc +++ b/compiler/optimizing/code_generator.cc @@ -1402,6 +1402,14 @@ void CodeGenerator::EmitJitRoots(uint8_t* code, entry.second = index; ++index; } + for (auto& entry : jit_class_roots_) { + // Update the `roots` with the class, and replace the address temporarily + // stored to the index in the table. + uint64_t address = entry.second; + roots->Set(index, reinterpret_cast<StackReference<mirror::Class>*>(address)->AsMirrorPtr()); + entry.second = index; + ++index; + } EmitJitRootPatches(code, roots_data); } diff --git a/compiler/optimizing/code_generator.h b/compiler/optimizing/code_generator.h index 4b11e7c699..2e2c3c00af 100644 --- a/compiler/optimizing/code_generator.h +++ b/compiler/optimizing/code_generator.h @@ -34,6 +34,7 @@ #include "stack_map_stream.h" #include "string_reference.h" #include "utils/label.h" +#include "utils/type_reference.h" namespace art { @@ -343,7 +344,7 @@ class CodeGenerator : public DeletableArenaObject<kArenaAllocCodeGenerator> { void BuildStackMaps(MemoryRegion region, const DexFile::CodeItem& code_item); size_t ComputeStackMapsSize(); size_t GetNumberOfJitRoots() const { - return jit_string_roots_.size(); + return jit_string_roots_.size() + jit_class_roots_.size(); } // Fills the `literals` array with literals collected during code generation. @@ -611,6 +612,8 @@ class CodeGenerator : public DeletableArenaObject<kArenaAllocCodeGenerator> { block_order_(nullptr), jit_string_roots_(StringReferenceValueComparator(), graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + jit_class_roots_(TypeReferenceValueComparator(), + graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), disasm_info_(nullptr), stats_(stats), graph_(graph), @@ -681,6 +684,7 @@ class CodeGenerator : public DeletableArenaObject<kArenaAllocCodeGenerator> { virtual void EmitJitRootPatches(uint8_t* code ATTRIBUTE_UNUSED, const uint8_t* roots_data ATTRIBUTE_UNUSED) { DCHECK_EQ(jit_string_roots_.size(), 0u); + DCHECK_EQ(jit_class_roots_.size(), 0u); } // Frame size required for this method. @@ -711,7 +715,12 @@ class CodeGenerator : public DeletableArenaObject<kArenaAllocCodeGenerator> { // Maps a StringReference (dex_file, string_index) to the index in the literal table. // Entries are intially added with a 0 index, and `EmitJitRoots` will compute all the // indices. - ArenaSafeMap<StringReference, size_t, StringReferenceValueComparator> jit_string_roots_; + ArenaSafeMap<StringReference, uint32_t, StringReferenceValueComparator> jit_string_roots_; + + // Maps a ClassReference (dex_file, type_index) to the index in the literal table. + // Entries are intially added with a pointer in the handle zone, and `EmitJitRoots` + // will compute all the indices. + ArenaSafeMap<TypeReference, uint64_t, TypeReferenceValueComparator> jit_class_roots_; DisassemblyInformation* disasm_info_; diff --git a/compiler/optimizing/code_generator_arm.cc b/compiler/optimizing/code_generator_arm.cc index ed6eef1b55..8a7f6d3a33 100644 --- a/compiler/optimizing/code_generator_arm.cc +++ b/compiler/optimizing/code_generator_arm.cc @@ -1201,11 +1201,6 @@ CodeGeneratorARM::CodeGeneratorARM(HGraph* graph, isa_features_(isa_features), uint32_literals_(std::less<uint32_t>(), graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), - method_patches_(MethodReferenceComparator(), - graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), - call_patches_(MethodReferenceComparator(), - graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), - relative_call_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), boot_image_string_patches_(StringReferenceValueComparator(), graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), @@ -1216,7 +1211,9 @@ CodeGeneratorARM::CodeGeneratorARM(HGraph* graph, boot_image_address_patches_(std::less<uint32_t>(), graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), jit_string_patches_(StringReferenceValueComparator(), - graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) { + graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + jit_class_patches_(TypeReferenceValueComparator(), + graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) { // Always save the LR register to mimic Quick. AddAllocatedRegister(Location::RegisterLocation(LR)); } @@ -5712,8 +5709,7 @@ HLoadClass::LoadKind CodeGeneratorARM::GetSupportedLoadClassKind( break; case HLoadClass::LoadKind::kBootImageAddress: break; - case HLoadClass::LoadKind::kDexCacheAddress: - DCHECK(Runtime::Current()->UseJitCompilation()); + case HLoadClass::LoadKind::kJitTableAddress: break; case HLoadClass::LoadKind::kDexCachePcRelative: DCHECK(!Runtime::Current()->UseJitCompilation()); @@ -5814,22 +5810,12 @@ void InstructionCodeGeneratorARM::VisitLoadClass(HLoadClass* cls) { __ LoadLiteral(out, codegen_->DeduplicateBootImageAddressLiteral(address)); break; } - case HLoadClass::LoadKind::kDexCacheAddress: { - DCHECK_NE(cls->GetAddress(), 0u); - uint32_t address = dchecked_integral_cast<uint32_t>(cls->GetAddress()); - // 16-bit LDR immediate has a 5-bit offset multiplied by the size and that gives - // a 128B range. To try and reduce the number of literals if we load multiple types, - // simply split the dex cache address to a 128B aligned base loaded from a literal - // and the remaining offset embedded in the load. - static_assert(sizeof(GcRoot<mirror::Class>) == 4u, "Expected GC root to be 4 bytes."); - DCHECK_ALIGNED(cls->GetAddress(), 4u); - constexpr size_t offset_bits = /* encoded bits */ 5 + /* scale */ 2; - uint32_t base_address = address & ~MaxInt<uint32_t>(offset_bits); - uint32_t offset = address & MaxInt<uint32_t>(offset_bits); - __ LoadLiteral(out, codegen_->DeduplicateDexCacheAddressLiteral(base_address)); - // /* GcRoot<mirror::Class> */ out = *(base_address + offset) - GenerateGcRootFieldLoad(cls, out_loc, out, offset, read_barrier_option); - generate_null_check = !cls->IsInDexCache(); + case HLoadClass::LoadKind::kJitTableAddress: { + __ LoadLiteral(out, codegen_->DeduplicateJitClassLiteral(cls->GetDexFile(), + cls->GetTypeIndex(), + cls->GetAddress())); + // /* GcRoot<mirror::Class> */ out = *out + GenerateGcRootFieldLoad(cls, out_loc, out, /* offset */ 0, kCompilerReadBarrierOption); break; } case HLoadClass::LoadKind::kDexCachePcRelative: { @@ -7148,7 +7134,7 @@ void CodeGeneratorARM::GenerateReadBarrierForRootSlow(HInstruction* instruction, HInvokeStaticOrDirect::DispatchInfo CodeGeneratorARM::GetSupportedInvokeStaticOrDirectDispatch( const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info, - HInvokeStaticOrDirect* invoke) { + HInvokeStaticOrDirect* invoke ATTRIBUTE_UNUSED) { HInvokeStaticOrDirect::DispatchInfo dispatch_info = desired_dispatch_info; // We disable pc-relative load when there is an irreducible loop, as the optimization // is incompatible with it. @@ -7160,24 +7146,6 @@ HInvokeStaticOrDirect::DispatchInfo CodeGeneratorARM::GetSupportedInvokeStaticOr dispatch_info.method_load_kind = HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod; } - if (dispatch_info.code_ptr_location == HInvokeStaticOrDirect::CodePtrLocation::kCallPCRelative) { - const DexFile& outer_dex_file = GetGraph()->GetDexFile(); - if (&outer_dex_file != invoke->GetTargetMethod().dex_file) { - // Calls across dex files are more likely to exceed the available BL range, - // so use absolute patch with fixup if available and kCallArtMethod otherwise. - HInvokeStaticOrDirect::CodePtrLocation code_ptr_location = - (desired_dispatch_info.method_load_kind == - HInvokeStaticOrDirect::MethodLoadKind::kDirectAddressWithFixup) - ? HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup - : HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod; - return HInvokeStaticOrDirect::DispatchInfo { - dispatch_info.method_load_kind, - code_ptr_location, - dispatch_info.method_load_data, - 0u - }; - } - } return dispatch_info; } @@ -7208,20 +7176,6 @@ Register CodeGeneratorARM::GetInvokeStaticOrDirectExtraParameter(HInvokeStaticOr } void CodeGeneratorARM::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) { - // For better instruction scheduling we load the direct code pointer before the method pointer. - switch (invoke->GetCodePtrLocation()) { - case HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup: - // LR = code address from literal pool with link-time patch. - __ LoadLiteral(LR, DeduplicateMethodCodeLiteral(invoke->GetTargetMethod())); - break; - case HInvokeStaticOrDirect::CodePtrLocation::kCallDirect: - // LR = invoke->GetDirectCodePtr(); - __ LoadImmediate(LR, invoke->GetDirectCodePtr()); - break; - default: - break; - } - Location callee_method = temp; // For all kinds except kRecursive, callee will be in temp. switch (invoke->GetMethodLoadKind()) { case HInvokeStaticOrDirect::MethodLoadKind::kStringInit: { @@ -7237,10 +7191,6 @@ void CodeGeneratorARM::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress: __ LoadImmediate(temp.AsRegister<Register>(), invoke->GetMethodAddress()); break; - case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddressWithFixup: - __ LoadLiteral(temp.AsRegister<Register>(), - DeduplicateMethodAddressLiteral(invoke->GetTargetMethod())); - break; case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: { HArmDexCacheArraysBase* base = invoke->InputAt(invoke->GetSpecialInputIndex())->AsArmDexCacheArraysBase(); @@ -7279,19 +7229,6 @@ void CodeGeneratorARM::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, case HInvokeStaticOrDirect::CodePtrLocation::kCallSelf: __ bl(GetFrameEntryLabel()); break; - case HInvokeStaticOrDirect::CodePtrLocation::kCallPCRelative: - relative_call_patches_.emplace_back(*invoke->GetTargetMethod().dex_file, - invoke->GetTargetMethod().dex_method_index); - __ BindTrackedLabel(&relative_call_patches_.back().label); - // Arbitrarily branch to the BL itself, override at link time. - __ bl(&relative_call_patches_.back().label); - break; - case HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup: - case HInvokeStaticOrDirect::CodePtrLocation::kCallDirect: - // LR prepared above for better instruction scheduling. - // LR() - __ blx(LR); - break; case HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod: // LR = callee_method->entry_point_from_quick_compiled_code_ __ LoadFromOffset( @@ -7379,10 +7316,6 @@ Literal* CodeGeneratorARM::DeduplicateBootImageAddressLiteral(uint32_t address) return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address), map); } -Literal* CodeGeneratorARM::DeduplicateDexCacheAddressLiteral(uint32_t address) { - return DeduplicateUint32Literal(address, &uint32_literals_); -} - Literal* CodeGeneratorARM::DeduplicateJitStringLiteral(const DexFile& dex_file, dex::StringIndex string_index) { jit_string_roots_.Overwrite(StringReference(&dex_file, string_index), /* placeholder */ 0u); @@ -7391,6 +7324,15 @@ Literal* CodeGeneratorARM::DeduplicateJitStringLiteral(const DexFile& dex_file, [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); } +Literal* CodeGeneratorARM::DeduplicateJitClassLiteral(const DexFile& dex_file, + dex::TypeIndex type_index, + uint64_t address) { + jit_class_roots_.Overwrite(TypeReference(&dex_file, type_index), address); + return jit_class_patches_.GetOrCreate( + TypeReference(&dex_file, type_index), + [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); +} + template <LinkerPatch (*Factory)(size_t, const DexFile*, uint32_t, uint32_t)> inline void CodeGeneratorARM::EmitPcRelativeLinkerPatches( const ArenaDeque<PcRelativePatchInfo>& infos, @@ -7414,9 +7356,6 @@ inline void CodeGeneratorARM::EmitPcRelativeLinkerPatches( void CodeGeneratorARM::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) { DCHECK(linker_patches->empty()); size_t size = - method_patches_.size() + - call_patches_.size() + - relative_call_patches_.size() + /* MOVW+MOVT for each entry */ 2u * pc_relative_dex_cache_patches_.size() + boot_image_string_patches_.size() + /* MOVW+MOVT for each entry */ 2u * pc_relative_string_patches_.size() + @@ -7424,29 +7363,6 @@ void CodeGeneratorARM::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patche /* MOVW+MOVT for each entry */ 2u * pc_relative_type_patches_.size() + boot_image_address_patches_.size(); linker_patches->reserve(size); - for (const auto& entry : method_patches_) { - const MethodReference& target_method = entry.first; - Literal* literal = entry.second; - DCHECK(literal->GetLabel()->IsBound()); - uint32_t literal_offset = literal->GetLabel()->Position(); - linker_patches->push_back(LinkerPatch::MethodPatch(literal_offset, - target_method.dex_file, - target_method.dex_method_index)); - } - for (const auto& entry : call_patches_) { - const MethodReference& target_method = entry.first; - Literal* literal = entry.second; - DCHECK(literal->GetLabel()->IsBound()); - uint32_t literal_offset = literal->GetLabel()->Position(); - linker_patches->push_back(LinkerPatch::CodePatch(literal_offset, - target_method.dex_file, - target_method.dex_method_index)); - } - for (const PatchInfo<Label>& info : relative_call_patches_) { - uint32_t literal_offset = info.label.Position(); - linker_patches->push_back( - LinkerPatch::RelativeCodePatch(literal_offset, &info.dex_file, info.index)); - } EmitPcRelativeLinkerPatches<LinkerPatch::DexCacheArrayPatch>(pc_relative_dex_cache_patches_, linker_patches); for (const auto& entry : boot_image_string_patches_) { @@ -7498,14 +7414,6 @@ Literal* CodeGeneratorARM::DeduplicateMethodLiteral(MethodReference target_metho [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); } -Literal* CodeGeneratorARM::DeduplicateMethodAddressLiteral(MethodReference target_method) { - return DeduplicateMethodLiteral(target_method, &method_patches_); -} - -Literal* CodeGeneratorARM::DeduplicateMethodCodeLiteral(MethodReference target_method) { - return DeduplicateMethodLiteral(target_method, &call_patches_); -} - void LocationsBuilderARM::VisitMultiplyAccumulate(HMultiplyAccumulate* instr) { LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instr, LocationSummary::kNoCall); @@ -7707,18 +7615,28 @@ void InstructionCodeGeneratorARM::VisitClassTableGet(HClassTableGet* instruction } } +static void PatchJitRootUse(uint8_t* code, + const uint8_t* roots_data, + Literal* literal, + uint64_t index_in_table) { + DCHECK(literal->GetLabel()->IsBound()); + uint32_t literal_offset = literal->GetLabel()->Position(); + uintptr_t address = + reinterpret_cast<uintptr_t>(roots_data) + index_in_table * sizeof(GcRoot<mirror::Object>); + uint8_t* data = code + literal_offset; + reinterpret_cast<uint32_t*>(data)[0] = dchecked_integral_cast<uint32_t>(address); +} + void CodeGeneratorARM::EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) { for (const auto& entry : jit_string_patches_) { const auto& it = jit_string_roots_.find(entry.first); DCHECK(it != jit_string_roots_.end()); - size_t index_in_table = it->second; - Literal* literal = entry.second; - DCHECK(literal->GetLabel()->IsBound()); - uint32_t literal_offset = literal->GetLabel()->Position(); - uintptr_t address = - reinterpret_cast<uintptr_t>(roots_data) + index_in_table * sizeof(GcRoot<mirror::Object>); - uint8_t* data = code + literal_offset; - reinterpret_cast<uint32_t*>(data)[0] = dchecked_integral_cast<uint32_t>(address); + PatchJitRootUse(code, roots_data, entry.second, it->second); + } + for (const auto& entry : jit_class_patches_) { + const auto& it = jit_class_roots_.find(entry.first); + DCHECK(it != jit_class_roots_.end()); + PatchJitRootUse(code, roots_data, entry.second, it->second); } } diff --git a/compiler/optimizing/code_generator_arm.h b/compiler/optimizing/code_generator_arm.h index 8230512825..6435851320 100644 --- a/compiler/optimizing/code_generator_arm.h +++ b/compiler/optimizing/code_generator_arm.h @@ -489,8 +489,10 @@ class CodeGeneratorARM : public CodeGenerator { dex::StringIndex string_index); Literal* DeduplicateBootImageTypeLiteral(const DexFile& dex_file, dex::TypeIndex type_index); Literal* DeduplicateBootImageAddressLiteral(uint32_t address); - Literal* DeduplicateDexCacheAddressLiteral(uint32_t address); Literal* DeduplicateJitStringLiteral(const DexFile& dex_file, dex::StringIndex string_index); + Literal* DeduplicateJitClassLiteral(const DexFile& dex_file, + dex::TypeIndex type_index, + uint64_t address); void EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) OVERRIDE; @@ -599,14 +601,12 @@ class CodeGeneratorARM : public CodeGenerator { using StringToLiteralMap = ArenaSafeMap<StringReference, Literal*, StringReferenceValueComparator>; - using BootTypeToLiteralMap = ArenaSafeMap<TypeReference, - Literal*, - TypeReferenceValueComparator>; + using TypeToLiteralMap = ArenaSafeMap<TypeReference, + Literal*, + TypeReferenceValueComparator>; Literal* DeduplicateUint32Literal(uint32_t value, Uint32ToLiteralMap* map); Literal* DeduplicateMethodLiteral(MethodReference target_method, MethodToLiteralMap* map); - Literal* DeduplicateMethodAddressLiteral(MethodReference target_method); - Literal* DeduplicateMethodCodeLiteral(MethodReference target_method); PcRelativePatchInfo* NewPcRelativePatch(const DexFile& dex_file, uint32_t offset_or_index, ArenaDeque<PcRelativePatchInfo>* patches); @@ -625,12 +625,6 @@ class CodeGeneratorARM : public CodeGenerator { // Deduplication map for 32-bit literals, used for non-patchable boot image addresses. Uint32ToLiteralMap uint32_literals_; - // Method patch info, map MethodReference to a literal for method address and method code. - MethodToLiteralMap method_patches_; - MethodToLiteralMap call_patches_; - // Relative call patch info. - // Using ArenaDeque<> which retains element addresses on push/emplace_back(). - ArenaDeque<PatchInfo<Label>> relative_call_patches_; // PC-relative patch info for each HArmDexCacheArraysBase. ArenaDeque<PcRelativePatchInfo> pc_relative_dex_cache_patches_; // Deduplication map for boot string literals for kBootImageLinkTimeAddress. @@ -638,7 +632,7 @@ class CodeGeneratorARM : public CodeGenerator { // PC-relative String patch info; type depends on configuration (app .bss or boot image PIC). ArenaDeque<PcRelativePatchInfo> pc_relative_string_patches_; // Deduplication map for boot type literals for kBootImageLinkTimeAddress. - BootTypeToLiteralMap boot_image_type_patches_; + TypeToLiteralMap boot_image_type_patches_; // PC-relative type patch info. ArenaDeque<PcRelativePatchInfo> pc_relative_type_patches_; // Deduplication map for patchable boot image addresses. @@ -646,6 +640,8 @@ class CodeGeneratorARM : public CodeGenerator { // Patches for string literals in JIT compiled code. StringToLiteralMap jit_string_patches_; + // Patches for class literals in JIT compiled code. + TypeToLiteralMap jit_class_patches_; DISALLOW_COPY_AND_ASSIGN(CodeGeneratorARM); }; diff --git a/compiler/optimizing/code_generator_arm64.cc b/compiler/optimizing/code_generator_arm64.cc index 6eebd69a04..5c33fe1a7d 100644 --- a/compiler/optimizing/code_generator_arm64.cc +++ b/compiler/optimizing/code_generator_arm64.cc @@ -1147,11 +1147,6 @@ CodeGeneratorARM64::CodeGeneratorARM64(HGraph* graph, graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), uint64_literals_(std::less<uint64_t>(), graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), - method_patches_(MethodReferenceComparator(), - graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), - call_patches_(MethodReferenceComparator(), - graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), - relative_call_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), boot_image_string_patches_(StringReferenceValueComparator(), graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), @@ -1162,7 +1157,9 @@ CodeGeneratorARM64::CodeGeneratorARM64(HGraph* graph, boot_image_address_patches_(std::less<uint32_t>(), graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), jit_string_patches_(StringReferenceValueComparator(), - graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) { + graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + jit_class_patches_(TypeReferenceValueComparator(), + graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) { // Save the link register (containing the return address) to mimic Quick. AddAllocatedRegister(LocationFrom(lr)); } @@ -1536,8 +1533,17 @@ void CodeGeneratorARM64::MoveLocation(Location destination, DCHECK(source.IsStackSlot() || source.IsDoubleStackSlot()); DCHECK(source.IsDoubleStackSlot() == destination.IsDoubleStackSlot()); UseScratchRegisterScope temps(GetVIXLAssembler()); - // There is generally less pressure on FP registers. - FPRegister temp = destination.IsDoubleStackSlot() ? temps.AcquireD() : temps.AcquireS(); + // Use any scratch register (a core or a floating-point one) + // from VIXL scratch register pools as a temporary. + // + // We used to only use the FP scratch register pool, but in some + // rare cases the only register from this pool (D31) would + // already be used (e.g. within a ParallelMove instruction, when + // a move is blocked by a another move requiring a scratch FP + // register, which would reserve D31). To prevent this issue, we + // ask for a scratch register of any type (core or FP). + CPURegister temp = + temps.AcquireCPURegisterOfSize(destination.IsDoubleStackSlot() ? kXRegSize : kWRegSize); __ Ldr(temp, StackOperandFrom(source)); __ Str(temp, StackOperandFrom(destination)); } @@ -3969,23 +3975,6 @@ HInvokeStaticOrDirect::DispatchInfo CodeGeneratorARM64::GetSupportedInvokeStatic } void CodeGeneratorARM64::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) { - // For better instruction scheduling we load the direct code pointer before the method pointer. - bool direct_code_loaded = false; - switch (invoke->GetCodePtrLocation()) { - case HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup: - // LR = code address from literal pool with link-time patch. - __ Ldr(lr, DeduplicateMethodCodeLiteral(invoke->GetTargetMethod())); - direct_code_loaded = true; - break; - case HInvokeStaticOrDirect::CodePtrLocation::kCallDirect: - // LR = invoke->GetDirectCodePtr(); - __ Ldr(lr, DeduplicateUint64Literal(invoke->GetDirectCodePtr())); - direct_code_loaded = true; - break; - default: - break; - } - // Make sure that ArtMethod* is passed in kArtMethodRegister as per the calling convention. Location callee_method = temp; // For all kinds except kRecursive, callee will be in temp. switch (invoke->GetMethodLoadKind()) { @@ -4003,11 +3992,6 @@ void CodeGeneratorARM64::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invok // Load method address from literal pool. __ Ldr(XRegisterFrom(temp), DeduplicateUint64Literal(invoke->GetMethodAddress())); break; - case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddressWithFixup: - // Load method address from literal pool with a link-time patch. - __ Ldr(XRegisterFrom(temp), - DeduplicateMethodAddressLiteral(invoke->GetTargetMethod())); - break; case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: { // Add ADRP with its PC-relative DexCache access patch. const DexFile& dex_file = invoke->GetDexFile(); @@ -4049,23 +4033,6 @@ void CodeGeneratorARM64::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invok case HInvokeStaticOrDirect::CodePtrLocation::kCallSelf: __ Bl(&frame_entry_label_); break; - case HInvokeStaticOrDirect::CodePtrLocation::kCallPCRelative: { - relative_call_patches_.emplace_back(*invoke->GetTargetMethod().dex_file, - invoke->GetTargetMethod().dex_method_index); - vixl::aarch64::Label* label = &relative_call_patches_.back().label; - SingleEmissionCheckScope guard(GetVIXLAssembler()); - __ Bind(label); - // Branch and link to itself. This will be overriden at link time. - __ bl(static_cast<int64_t>(0)); - break; - } - case HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup: - case HInvokeStaticOrDirect::CodePtrLocation::kCallDirect: - // LR prepared above for better instruction scheduling. - DCHECK(direct_code_loaded); - // lr() - __ Blr(lr); - break; case HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod: // LR = callee_method->entry_point_from_quick_compiled_code_; __ Ldr(lr, MemOperand( @@ -4169,11 +4136,6 @@ vixl::aarch64::Literal<uint32_t>* CodeGeneratorARM64::DeduplicateBootImageAddres return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address), map); } -vixl::aarch64::Literal<uint64_t>* CodeGeneratorARM64::DeduplicateDexCacheAddressLiteral( - uint64_t address) { - return DeduplicateUint64Literal(address); -} - vixl::aarch64::Literal<uint32_t>* CodeGeneratorARM64::DeduplicateJitStringLiteral( const DexFile& dex_file, dex::StringIndex string_index) { jit_string_roots_.Overwrite(StringReference(&dex_file, string_index), /* placeholder */ 0u); @@ -4182,6 +4144,14 @@ vixl::aarch64::Literal<uint32_t>* CodeGeneratorARM64::DeduplicateJitStringLitera [this]() { return __ CreateLiteralDestroyedWithPool<uint32_t>(/* placeholder */ 0u); }); } +vixl::aarch64::Literal<uint32_t>* CodeGeneratorARM64::DeduplicateJitClassLiteral( + const DexFile& dex_file, dex::TypeIndex type_index, uint64_t address) { + jit_class_roots_.Overwrite(TypeReference(&dex_file, type_index), address); + return jit_class_patches_.GetOrCreate( + TypeReference(&dex_file, type_index), + [this]() { return __ CreateLiteralDestroyedWithPool<uint32_t>(/* placeholder */ 0u); }); +} + void CodeGeneratorARM64::EmitAdrpPlaceholder(vixl::aarch64::Label* fixup_label, vixl::aarch64::Register reg) { DCHECK(reg.IsX()); @@ -4224,9 +4194,6 @@ inline void CodeGeneratorARM64::EmitPcRelativeLinkerPatches( void CodeGeneratorARM64::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) { DCHECK(linker_patches->empty()); size_t size = - method_patches_.size() + - call_patches_.size() + - relative_call_patches_.size() + pc_relative_dex_cache_patches_.size() + boot_image_string_patches_.size() + pc_relative_string_patches_.size() + @@ -4234,24 +4201,6 @@ void CodeGeneratorARM64::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patc pc_relative_type_patches_.size() + boot_image_address_patches_.size(); linker_patches->reserve(size); - for (const auto& entry : method_patches_) { - const MethodReference& target_method = entry.first; - vixl::aarch64::Literal<uint64_t>* literal = entry.second; - linker_patches->push_back(LinkerPatch::MethodPatch(literal->GetOffset(), - target_method.dex_file, - target_method.dex_method_index)); - } - for (const auto& entry : call_patches_) { - const MethodReference& target_method = entry.first; - vixl::aarch64::Literal<uint64_t>* literal = entry.second; - linker_patches->push_back(LinkerPatch::CodePatch(literal->GetOffset(), - target_method.dex_file, - target_method.dex_method_index)); - } - for (const PatchInfo<vixl::aarch64::Label>& info : relative_call_patches_) { - linker_patches->push_back( - LinkerPatch::RelativeCodePatch(info.label.GetLocation(), &info.dex_file, info.index)); - } for (const PcRelativePatchInfo& info : pc_relative_dex_cache_patches_) { linker_patches->push_back(LinkerPatch::DexCacheArrayPatch(info.label.GetLocation(), &info.target_dex_file, @@ -4309,17 +4258,6 @@ vixl::aarch64::Literal<uint64_t>* CodeGeneratorARM64::DeduplicateMethodLiteral( [this]() { return __ CreateLiteralDestroyedWithPool<uint64_t>(/* placeholder */ 0u); }); } -vixl::aarch64::Literal<uint64_t>* CodeGeneratorARM64::DeduplicateMethodAddressLiteral( - MethodReference target_method) { - return DeduplicateMethodLiteral(target_method, &method_patches_); -} - -vixl::aarch64::Literal<uint64_t>* CodeGeneratorARM64::DeduplicateMethodCodeLiteral( - MethodReference target_method) { - return DeduplicateMethodLiteral(target_method, &call_patches_); -} - - void InstructionCodeGeneratorARM64::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) { // Explicit clinit checks triggered by static invokes must have been pruned by // art::PrepareForRegisterAllocation. @@ -4359,7 +4297,7 @@ HLoadClass::LoadKind CodeGeneratorARM64::GetSupportedLoadClassKind( break; case HLoadClass::LoadKind::kBootImageAddress: break; - case HLoadClass::LoadKind::kDexCacheAddress: + case HLoadClass::LoadKind::kJitTableAddress: DCHECK(Runtime::Current()->UseJitCompilation()); break; case HLoadClass::LoadKind::kDexCachePcRelative: @@ -4452,26 +4390,16 @@ void InstructionCodeGeneratorARM64::VisitLoadClass(HLoadClass* cls) { __ Ldr(out.W(), codegen_->DeduplicateBootImageAddressLiteral(cls->GetAddress())); break; } - case HLoadClass::LoadKind::kDexCacheAddress: { - DCHECK_NE(cls->GetAddress(), 0u); - // LDR immediate has a 12-bit offset multiplied by the size and for 32-bit loads - // that gives a 16KiB range. To try and reduce the number of literals if we load - // multiple types, simply split the dex cache address to a 16KiB aligned base - // loaded from a literal and the remaining offset embedded in the load. - static_assert(sizeof(GcRoot<mirror::Class>) == 4u, "Expected GC root to be 4 bytes."); - DCHECK_ALIGNED(cls->GetAddress(), 4u); - constexpr size_t offset_bits = /* encoded bits */ 12 + /* scale */ 2; - uint64_t base_address = cls->GetAddress() & ~MaxInt<uint64_t>(offset_bits); - uint32_t offset = cls->GetAddress() & MaxInt<uint64_t>(offset_bits); - __ Ldr(out.X(), codegen_->DeduplicateDexCacheAddressLiteral(base_address)); - // /* GcRoot<mirror::Class> */ out = *(base_address + offset) + case HLoadClass::LoadKind::kJitTableAddress: { + __ Ldr(out, codegen_->DeduplicateJitClassLiteral(cls->GetDexFile(), + cls->GetTypeIndex(), + cls->GetAddress())); GenerateGcRootFieldLoad(cls, out_loc, out.X(), - offset, + /* offset */ 0, /* fixup_label */ nullptr, - read_barrier_option); - generate_null_check = !cls->IsInDexCache(); + kCompilerReadBarrierOption); break; } case HLoadClass::LoadKind::kDexCachePcRelative: { @@ -5782,17 +5710,27 @@ void InstructionCodeGeneratorARM64::VisitClassTableGet(HClassTableGet* instructi } } +static void PatchJitRootUse(uint8_t* code, + const uint8_t* roots_data, + vixl::aarch64::Literal<uint32_t>* literal, + uint64_t index_in_table) { + uint32_t literal_offset = literal->GetOffset(); + uintptr_t address = + reinterpret_cast<uintptr_t>(roots_data) + index_in_table * sizeof(GcRoot<mirror::Object>); + uint8_t* data = code + literal_offset; + reinterpret_cast<uint32_t*>(data)[0] = dchecked_integral_cast<uint32_t>(address); +} + void CodeGeneratorARM64::EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) { for (const auto& entry : jit_string_patches_) { const auto& it = jit_string_roots_.find(entry.first); DCHECK(it != jit_string_roots_.end()); - size_t index_in_table = it->second; - vixl::aarch64::Literal<uint32_t>* literal = entry.second; - uint32_t literal_offset = literal->GetOffset(); - uintptr_t address = - reinterpret_cast<uintptr_t>(roots_data) + index_in_table * sizeof(GcRoot<mirror::Object>); - uint8_t* data = code + literal_offset; - reinterpret_cast<uint32_t*>(data)[0] = dchecked_integral_cast<uint32_t>(address); + PatchJitRootUse(code, roots_data, entry.second, it->second); + } + for (const auto& entry : jit_class_patches_) { + const auto& it = jit_class_roots_.find(entry.first); + DCHECK(it != jit_class_roots_.end()); + PatchJitRootUse(code, roots_data, entry.second, it->second); } } diff --git a/compiler/optimizing/code_generator_arm64.h b/compiler/optimizing/code_generator_arm64.h index 868c8b07ed..8f33b6becf 100644 --- a/compiler/optimizing/code_generator_arm64.h +++ b/compiler/optimizing/code_generator_arm64.h @@ -566,9 +566,11 @@ class CodeGeneratorARM64 : public CodeGenerator { vixl::aarch64::Literal<uint32_t>* DeduplicateBootImageTypeLiteral(const DexFile& dex_file, dex::TypeIndex type_index); vixl::aarch64::Literal<uint32_t>* DeduplicateBootImageAddressLiteral(uint64_t address); - vixl::aarch64::Literal<uint64_t>* DeduplicateDexCacheAddressLiteral(uint64_t address); vixl::aarch64::Literal<uint32_t>* DeduplicateJitStringLiteral(const DexFile& dex_file, dex::StringIndex string_index); + vixl::aarch64::Literal<uint32_t>* DeduplicateJitClassLiteral(const DexFile& dex_file, + dex::TypeIndex string_index, + uint64_t address); void EmitAdrpPlaceholder(vixl::aarch64::Label* fixup_label, vixl::aarch64::Register reg); void EmitAddPlaceholder(vixl::aarch64::Label* fixup_label, @@ -682,17 +684,15 @@ class CodeGeneratorARM64 : public CodeGenerator { using StringToLiteralMap = ArenaSafeMap<StringReference, vixl::aarch64::Literal<uint32_t>*, StringReferenceValueComparator>; - using BootTypeToLiteralMap = ArenaSafeMap<TypeReference, - vixl::aarch64::Literal<uint32_t>*, - TypeReferenceValueComparator>; + using TypeToLiteralMap = ArenaSafeMap<TypeReference, + vixl::aarch64::Literal<uint32_t>*, + TypeReferenceValueComparator>; vixl::aarch64::Literal<uint32_t>* DeduplicateUint32Literal(uint32_t value, Uint32ToLiteralMap* map); vixl::aarch64::Literal<uint64_t>* DeduplicateUint64Literal(uint64_t value); vixl::aarch64::Literal<uint64_t>* DeduplicateMethodLiteral(MethodReference target_method, MethodToLiteralMap* map); - vixl::aarch64::Literal<uint64_t>* DeduplicateMethodAddressLiteral(MethodReference target_method); - vixl::aarch64::Literal<uint64_t>* DeduplicateMethodCodeLiteral(MethodReference target_method); // The PcRelativePatchInfo is used for PC-relative addressing of dex cache arrays // and boot image strings/types. The only difference is the interpretation of the @@ -733,15 +733,8 @@ class CodeGeneratorARM64 : public CodeGenerator { // Deduplication map for 32-bit literals, used for non-patchable boot image addresses. Uint32ToLiteralMap uint32_literals_; - // Deduplication map for 64-bit literals, used for non-patchable method address, method code - // or string dex cache address. + // Deduplication map for 64-bit literals, used for non-patchable method address or method code. Uint64ToLiteralMap uint64_literals_; - // Method patch info, map MethodReference to a literal for method address and method code. - MethodToLiteralMap method_patches_; - MethodToLiteralMap call_patches_; - // Relative call patch info. - // Using ArenaDeque<> which retains element addresses on push/emplace_back(). - ArenaDeque<PatchInfo<vixl::aarch64::Label>> relative_call_patches_; // PC-relative DexCache access info. ArenaDeque<PcRelativePatchInfo> pc_relative_dex_cache_patches_; // Deduplication map for boot string literals for kBootImageLinkTimeAddress. @@ -749,7 +742,7 @@ class CodeGeneratorARM64 : public CodeGenerator { // PC-relative String patch info; type depends on configuration (app .bss or boot image PIC). ArenaDeque<PcRelativePatchInfo> pc_relative_string_patches_; // Deduplication map for boot type literals for kBootImageLinkTimeAddress. - BootTypeToLiteralMap boot_image_type_patches_; + TypeToLiteralMap boot_image_type_patches_; // PC-relative type patch info. ArenaDeque<PcRelativePatchInfo> pc_relative_type_patches_; // Deduplication map for patchable boot image addresses. @@ -757,6 +750,8 @@ class CodeGeneratorARM64 : public CodeGenerator { // Patches for string literals in JIT compiled code. StringToLiteralMap jit_string_patches_; + // Patches for class literals in JIT compiled code. + TypeToLiteralMap jit_class_patches_; DISALLOW_COPY_AND_ASSIGN(CodeGeneratorARM64); }; diff --git a/compiler/optimizing/code_generator_arm_vixl.cc b/compiler/optimizing/code_generator_arm_vixl.cc index 3a3d2a9db1..f108595a00 100644 --- a/compiler/optimizing/code_generator_arm_vixl.cc +++ b/compiler/optimizing/code_generator_arm_vixl.cc @@ -46,8 +46,10 @@ using helpers::InputOperandAt; using helpers::InputRegister; using helpers::InputRegisterAt; using helpers::InputSRegisterAt; +using helpers::InputVRegister; using helpers::InputVRegisterAt; using helpers::Int32ConstantFrom; +using helpers::Int64ConstantFrom; using helpers::LocationFrom; using helpers::LowRegisterFrom; using helpers::LowSRegisterFrom; @@ -56,6 +58,10 @@ using helpers::OutputSRegister; using helpers::OutputVRegister; using helpers::RegisterFrom; using helpers::SRegisterFrom; +using helpers::Uint64ConstantFrom; + +using vixl::ExactAssemblyScope; +using vixl::CodeBufferCheckScope; using RegisterList = vixl32::RegisterList; @@ -799,7 +805,7 @@ class ReadBarrierMarkAndUpdateFieldSlowPathARMVIXL : public SlowPathCodeARMVIXL // as-is. vixl32::Label done; __ Cmp(temp1_, ref_reg); - __ B(eq, &done); + __ B(eq, &done, /* far_target */ false); // Update the the holder's field atomically. This may fail if // mutator updates before us, but it's OK. This is achieved @@ -843,19 +849,19 @@ class ReadBarrierMarkAndUpdateFieldSlowPathARMVIXL : public SlowPathCodeARMVIXL __ Subs(tmp, tmp, expected); { - AssemblerAccurateScope aas(arm_codegen->GetVIXLAssembler(), - 2 * kMaxInstructionSizeInBytes, - CodeBufferCheckScope::kMaximumSize); + ExactAssemblyScope aas(arm_codegen->GetVIXLAssembler(), + 2 * kMaxInstructionSizeInBytes, + CodeBufferCheckScope::kMaximumSize); __ it(ne); __ clrex(ne); } - __ B(ne, &exit_loop); + __ B(ne, &exit_loop, /* far_target */ false); __ Strex(tmp, value, MemOperand(tmp_ptr)); __ Cmp(tmp, 1); - __ B(eq, &loop_head); + __ B(eq, &loop_head, /* far_target */ false); __ Bind(&exit_loop); @@ -1237,10 +1243,21 @@ CodeGeneratorARMVIXL::CodeGeneratorARMVIXL(HGraph* graph, move_resolver_(graph->GetArena(), this), assembler_(graph->GetArena()), isa_features_(isa_features), - relative_call_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + uint32_literals_(std::less<uint32_t>(), + graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + boot_image_string_patches_(StringReferenceValueComparator(), + graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), pc_relative_string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), - pc_relative_type_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) { + boot_image_type_patches_(TypeReferenceValueComparator(), + graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + pc_relative_type_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + boot_image_address_patches_(std::less<uint32_t>(), + graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + jit_string_patches_(StringReferenceValueComparator(), + graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + jit_class_patches_(TypeReferenceValueComparator(), + graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) { // Always save the LR register to mimic Quick. AddAllocatedRegister(Location::RegisterLocation(LR)); // Give d14 and d15 as scratch registers to VIXL. @@ -1261,9 +1278,9 @@ void JumpTableARMVIXL::EmitTable(CodeGeneratorARMVIXL* codegen) { // We are about to use the assembler to place literals directly. Make sure we have enough // underlying code buffer and we have generated a jump table of the right size, using // codegen->GetVIXLAssembler()->GetBuffer().Align(); - AssemblerAccurateScope aas(codegen->GetVIXLAssembler(), - num_entries * sizeof(int32_t), - CodeBufferCheckScope::kMaximumSize); + ExactAssemblyScope aas(codegen->GetVIXLAssembler(), + num_entries * sizeof(int32_t), + CodeBufferCheckScope::kMaximumSize); // TODO(VIXL): Check that using lower case bind is fine here. codegen->GetVIXLAssembler()->bind(&table_start_); for (uint32_t i = 0; i < num_entries; i++) { @@ -1375,11 +1392,11 @@ void CodeGeneratorARMVIXL::GenerateFrameEntry() { if (!skip_overflow_check) { UseScratchRegisterScope temps(GetVIXLAssembler()); vixl32::Register temp = temps.Acquire(); - __ Sub(temp, sp, static_cast<int32_t>(GetStackOverflowReservedBytes(kArm))); + __ Sub(temp, sp, Operand::From(GetStackOverflowReservedBytes(kArm))); // The load must immediately precede RecordPcInfo. - AssemblerAccurateScope aas(GetVIXLAssembler(), - vixl32::kMaxInstructionSizeInBytes, - CodeBufferCheckScope::kMaximumSize); + ExactAssemblyScope aas(GetVIXLAssembler(), + vixl32::kMaxInstructionSizeInBytes, + CodeBufferCheckScope::kMaximumSize); __ ldr(temp, MemOperand(temp)); RecordPcInfo(nullptr, 0); } @@ -1637,9 +1654,9 @@ void CodeGeneratorARMVIXL::InvokeRuntime(QuickEntrypointEnum entrypoint, __ Ldr(lr, MemOperand(tr, GetThreadOffset<kArmPointerSize>(entrypoint).Int32Value())); // Ensure the pc position is recorded immediately after the `blx` instruction. // blx in T32 has only 16bit encoding that's why a stricter check for the scope is used. - AssemblerAccurateScope aas(GetVIXLAssembler(), - vixl32::k16BitT32InstructionSizeInBytes, - CodeBufferCheckScope::kExactSize); + ExactAssemblyScope aas(GetVIXLAssembler(), + vixl32::k16BitT32InstructionSizeInBytes, + CodeBufferCheckScope::kExactSize); __ blx(lr); if (EntrypointRequiresStackMap(entrypoint)) { RecordPcInfo(instruction, dex_pc, slow_path); @@ -1792,7 +1809,7 @@ void InstructionCodeGeneratorARMVIXL::GenerateLongComparesAndJumps(HCondition* c break; } if (right.IsConstant()) { - int64_t value = right.GetConstant()->AsLongConstant()->GetValue(); + int64_t value = Int64ConstantFrom(right); int32_t val_low = Low32Bits(value); int32_t val_high = High32Bits(value); @@ -1877,7 +1894,7 @@ void InstructionCodeGeneratorARMVIXL::GenerateTestAndBranch(HInstruction* instru __ B(true_target); } } else { - DCHECK(cond->AsIntConstant()->IsFalse()) << cond->AsIntConstant()->GetValue(); + DCHECK(cond->AsIntConstant()->IsFalse()) << Int32ConstantFrom(cond); if (false_target != nullptr) { __ B(false_target); } @@ -2082,9 +2099,9 @@ void InstructionCodeGeneratorARMVIXL::HandleCondition(HCondition* cond) { __ Cmp(InputRegisterAt(cond, 0), CodeGenerator::GetInt32ValueOf(right.GetConstant())); } - AssemblerAccurateScope aas(GetVIXLAssembler(), - 3 * vixl32::kMaxInstructionSizeInBytes, - CodeBufferCheckScope::kMaximumSize); + ExactAssemblyScope aas(GetVIXLAssembler(), + 3 * vixl32::kMaxInstructionSizeInBytes, + CodeBufferCheckScope::kMaximumSize); __ ite(ARMCondition(cond->GetCondition())); __ mov(ARMCondition(cond->GetCondition()), OutputRegister(cond), 1); __ mov(ARMCondition(cond->GetOppositeCondition()), OutputRegister(cond), 0); @@ -2370,9 +2387,9 @@ void InstructionCodeGeneratorARMVIXL::VisitInvokeInterface(HInvokeInterface* inv // Ensure the pc position is recorded immediately after the `ldr` instruction. { - AssemblerAccurateScope aas(GetVIXLAssembler(), - vixl32::kMaxInstructionSizeInBytes, - CodeBufferCheckScope::kMaximumSize); + ExactAssemblyScope aas(GetVIXLAssembler(), + vixl32::kMaxInstructionSizeInBytes, + CodeBufferCheckScope::kMaximumSize); // /* HeapReference<Class> */ temp = receiver->klass_ __ ldr(temp, MemOperand(RegisterFrom(receiver), class_offset)); codegen_->MaybeRecordImplicitNullCheck(invoke); @@ -2418,7 +2435,7 @@ void InstructionCodeGeneratorARMVIXL::VisitInvokeInterface(HInvokeInterface* inv { // Ensure the pc position is recorded immediately after the `blx` instruction. // blx in T32 has only 16bit encoding that's why a stricter check for the scope is used. - AssemblerAccurateScope aas(GetVIXLAssembler(), + ExactAssemblyScope aas(GetVIXLAssembler(), vixl32::k16BitT32InstructionSizeInBytes, CodeBufferCheckScope::kExactSize); // LR(); @@ -2479,9 +2496,7 @@ void InstructionCodeGeneratorARMVIXL::VisitNeg(HNeg* neg) { case Primitive::kPrimFloat: case Primitive::kPrimDouble: - // TODO(VIXL): Consider introducing an InputVRegister() - // helper function (equivalent to InputRegister()). - __ Vneg(OutputVRegister(neg), InputVRegisterAt(neg, 0)); + __ Vneg(OutputVRegister(neg), InputVRegister(neg)); break; default: @@ -2771,8 +2786,8 @@ void InstructionCodeGeneratorARMVIXL::VisitTypeConversion(HTypeConversion* conve } else { DCHECK(in.IsConstant()); DCHECK(in.GetConstant()->IsLongConstant()); - int64_t value = in.GetConstant()->AsLongConstant()->GetValue(); - __ Mov(OutputRegister(conversion), static_cast<int32_t>(value)); + int32_t value = Int32ConstantFrom(in); + __ Mov(OutputRegister(conversion), value); } break; @@ -3111,8 +3126,8 @@ void InstructionCodeGeneratorARMVIXL::VisitMul(HMul* mul) { // Extra checks to protect caused by the existence of R1_R2. // The algorithm is wrong if out.hi is either in1.lo or in2.lo: // (e.g. in1=r0_r1, in2=r2_r3 and out=r1_r2); - DCHECK_NE(out_hi.GetCode(), in1_lo.GetCode()); - DCHECK_NE(out_hi.GetCode(), in2_lo.GetCode()); + DCHECK(!out_hi.Is(in1_lo)); + DCHECK(!out_hi.Is(in2_lo)); // input: in1 - 64 bits, in2 - 64 bits // output: out @@ -3152,7 +3167,7 @@ void InstructionCodeGeneratorARMVIXL::DivRemOneOrMinusOne(HBinaryOperation* inst vixl32::Register out = OutputRegister(instruction); vixl32::Register dividend = InputRegisterAt(instruction, 0); - int32_t imm = second.GetConstant()->AsIntConstant()->GetValue(); + int32_t imm = Int32ConstantFrom(second); DCHECK(imm == 1 || imm == -1); if (instruction->IsRem()) { @@ -3177,7 +3192,7 @@ void InstructionCodeGeneratorARMVIXL::DivRemByPowerOfTwo(HBinaryOperation* instr vixl32::Register out = OutputRegister(instruction); vixl32::Register dividend = InputRegisterAt(instruction, 0); vixl32::Register temp = RegisterFrom(locations->GetTemp(0)); - int32_t imm = second.GetConstant()->AsIntConstant()->GetValue(); + int32_t imm = Int32ConstantFrom(second); uint32_t abs_imm = static_cast<uint32_t>(AbsOrMin(imm)); int ctz_imm = CTZ(abs_imm); @@ -3250,7 +3265,7 @@ void InstructionCodeGeneratorARMVIXL::GenerateDivRemConstantIntegral( Location second = instruction->GetLocations()->InAt(1); DCHECK(second.IsConstant()); - int32_t imm = second.GetConstant()->AsIntConstant()->GetValue(); + int32_t imm = Int32ConstantFrom(second); if (imm == 0) { // Do not generate anything. DivZeroCheck would prevent any code to be executed. } else if (imm == 1 || imm == -1) { @@ -3284,7 +3299,7 @@ void LocationsBuilderARMVIXL::VisitDiv(HDiv* div) { locations->SetInAt(0, Location::RequiresRegister()); locations->SetInAt(1, Location::ConstantLocation(div->InputAt(1)->AsConstant())); locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); - int32_t value = div->InputAt(1)->AsIntConstant()->GetValue(); + int32_t value = Int32ConstantFrom(div->InputAt(1)); if (value == 1 || value == 0 || value == -1) { // No temp register required. } else { @@ -3397,7 +3412,7 @@ void LocationsBuilderARMVIXL::VisitRem(HRem* rem) { locations->SetInAt(0, Location::RequiresRegister()); locations->SetInAt(1, Location::ConstantLocation(rem->InputAt(1)->AsConstant())); locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); - int32_t value = rem->InputAt(1)->AsIntConstant()->GetValue(); + int32_t value = Int32ConstantFrom(rem->InputAt(1)); if (value == 1 || value == 0 || value == -1) { // No temp register required. } else { @@ -3532,7 +3547,7 @@ void InstructionCodeGeneratorARMVIXL::VisitDivZeroCheck(HDivZeroCheck* instructi __ CompareAndBranchIfZero(InputRegisterAt(instruction, 0), slow_path->GetEntryLabel()); } else { DCHECK(value.IsConstant()) << value; - if (value.GetConstant()->AsIntConstant()->GetValue() == 0) { + if (Int32ConstantFrom(value) == 0) { __ B(slow_path->GetEntryLabel()); } } @@ -3546,7 +3561,7 @@ void InstructionCodeGeneratorARMVIXL::VisitDivZeroCheck(HDivZeroCheck* instructi __ B(eq, slow_path->GetEntryLabel()); } else { DCHECK(value.IsConstant()) << value; - if (value.GetConstant()->AsLongConstant()->GetValue() == 0) { + if (Int64ConstantFrom(value) == 0) { __ B(slow_path->GetEntryLabel()); } } @@ -3622,7 +3637,7 @@ void InstructionCodeGeneratorARMVIXL::HandleLongRotate(HRor* ror) { __ And(shift_right, RegisterFrom(rhs), 0x1F); __ Lsrs(shift_left, RegisterFrom(rhs), 6); __ Rsb(LeaveFlags, shift_left, shift_right, Operand::From(kArmBitsPerWord)); - __ B(cc, &shift_by_32_plus_shift_right); + __ B(cc, &shift_by_32_plus_shift_right, /* far_target */ false); // out_reg_hi = (reg_hi << shift_left) | (reg_lo >> shift_right). // out_reg_lo = (reg_lo << shift_left) | (reg_hi >> shift_right). @@ -3756,7 +3771,7 @@ void InstructionCodeGeneratorARMVIXL::HandleShift(HBinaryOperation* op) { __ Lsr(out_reg, first_reg, out_reg); } } else { - int32_t cst = second.GetConstant()->AsIntConstant()->GetValue(); + int32_t cst = Int32ConstantFrom(second); uint32_t shift_value = cst & kMaxIntShiftDistance; if (shift_value == 0) { // ARM does not support shifting with 0 immediate. __ Mov(out_reg, first_reg); @@ -3793,9 +3808,9 @@ void InstructionCodeGeneratorARMVIXL::HandleShift(HBinaryOperation* op) { // If the shift is > 32 bits, override the high part __ Subs(temp, o_l, Operand::From(kArmBitsPerWord)); { - AssemblerAccurateScope guard(GetVIXLAssembler(), - 2 * vixl32::kMaxInstructionSizeInBytes, - CodeBufferCheckScope::kMaximumSize); + ExactAssemblyScope guard(GetVIXLAssembler(), + 2 * vixl32::kMaxInstructionSizeInBytes, + CodeBufferCheckScope::kMaximumSize); __ it(pl); __ lsl(pl, o_h, low, temp); } @@ -3812,9 +3827,9 @@ void InstructionCodeGeneratorARMVIXL::HandleShift(HBinaryOperation* op) { // If the shift is > 32 bits, override the low part __ Subs(temp, o_h, Operand::From(kArmBitsPerWord)); { - AssemblerAccurateScope guard(GetVIXLAssembler(), - 2 * vixl32::kMaxInstructionSizeInBytes, - CodeBufferCheckScope::kMaximumSize); + ExactAssemblyScope guard(GetVIXLAssembler(), + 2 * vixl32::kMaxInstructionSizeInBytes, + CodeBufferCheckScope::kMaximumSize); __ it(pl); __ asr(pl, o_l, high, temp); } @@ -3829,9 +3844,9 @@ void InstructionCodeGeneratorARMVIXL::HandleShift(HBinaryOperation* op) { __ Orr(o_l, o_l, temp); __ Subs(temp, o_h, Operand::From(kArmBitsPerWord)); { - AssemblerAccurateScope guard(GetVIXLAssembler(), - 2 * vixl32::kMaxInstructionSizeInBytes, - CodeBufferCheckScope::kMaximumSize); + ExactAssemblyScope guard(GetVIXLAssembler(), + 2 * vixl32::kMaxInstructionSizeInBytes, + CodeBufferCheckScope::kMaximumSize); __ it(pl); __ lsr(pl, o_l, high, temp); } @@ -3841,7 +3856,7 @@ void InstructionCodeGeneratorARMVIXL::HandleShift(HBinaryOperation* op) { // Register allocator doesn't create partial overlap. DCHECK(!o_l.Is(high)); DCHECK(!o_h.Is(low)); - int32_t cst = second.GetConstant()->AsIntConstant()->GetValue(); + int32_t cst = Int32ConstantFrom(second); uint32_t shift_value = cst & kMaxLongShiftDistance; if (shift_value > 32) { if (op->IsShl()) { @@ -3948,9 +3963,9 @@ void InstructionCodeGeneratorARMVIXL::VisitNewInstance(HNewInstance* instruction GetAssembler()->LoadFromOffset(kLoadWord, temp, tr, QUICK_ENTRY_POINT(pNewEmptyString)); GetAssembler()->LoadFromOffset(kLoadWord, lr, temp, code_offset.Int32Value()); // blx in T32 has only 16bit encoding that's why a stricter check for the scope is used. - AssemblerAccurateScope aas(GetVIXLAssembler(), - vixl32::k16BitT32InstructionSizeInBytes, - CodeBufferCheckScope::kExactSize); + ExactAssemblyScope aas(GetVIXLAssembler(), + vixl32::k16BitT32InstructionSizeInBytes, + CodeBufferCheckScope::kExactSize); __ blx(lr); codegen_->RecordPcInfo(instruction, instruction->GetDexPc()); } else { @@ -4094,8 +4109,8 @@ void InstructionCodeGeneratorARMVIXL::VisitCompare(HCompare* compare) { } case Primitive::kPrimLong: { __ Cmp(HighRegisterFrom(left), HighRegisterFrom(right)); // Signed compare. - __ B(lt, &less); - __ B(gt, &greater); + __ B(lt, &less, /* far_target */ false); + __ B(gt, &greater, /* far_target */ false); // Emit move to `out` before the last `Cmp`, as `Mov` might affect the status flags. __ Mov(out, 0); __ Cmp(LowRegisterFrom(left), LowRegisterFrom(right)); // Unsigned compare. @@ -4116,8 +4131,8 @@ void InstructionCodeGeneratorARMVIXL::VisitCompare(HCompare* compare) { UNREACHABLE(); } - __ B(eq, &done); - __ B(less_cond, &less); + __ B(eq, &done, /* far_target */ false); + __ B(less_cond, &less, /* far_target */ false); __ Bind(&greater); __ Mov(out, 1); @@ -4192,9 +4207,9 @@ void InstructionCodeGeneratorARMVIXL::GenerateWideAtomicStore(vixl32::Register a __ Bind(&fail); { // Ensure the pc position is recorded immediately after the `ldrexd` instruction. - AssemblerAccurateScope aas(GetVIXLAssembler(), - vixl32::kMaxInstructionSizeInBytes, - CodeBufferCheckScope::kMaximumSize); + ExactAssemblyScope aas(GetVIXLAssembler(), + vixl32::kMaxInstructionSizeInBytes, + CodeBufferCheckScope::kMaximumSize); // We need a load followed by store. (The address used in a STREX instruction must // be the same as the address in the most recently executed LDREX instruction.) __ ldrexd(temp1, temp2, MemOperand(addr)); @@ -4411,7 +4426,7 @@ void LocationsBuilderARMVIXL::HandleFieldGet(HInstruction* instruction, locations->AddTemp(Location::RequiresRegister()); } else if (object_field_get_with_read_barrier && kUseBakerReadBarrier) { // We need a temporary register for the read barrier marking slow - // path in CodeGeneratorARM::GenerateFieldLoadWithBakerReadBarrier. + // path in CodeGeneratorARMVIXL::GenerateFieldLoadWithBakerReadBarrier. locations->AddTemp(Location::RequiresRegister()); } } @@ -4715,9 +4730,9 @@ void CodeGeneratorARMVIXL::GenerateImplicitNullCheck(HNullCheck* instruction) { UseScratchRegisterScope temps(GetVIXLAssembler()); // Ensure the pc position is recorded immediately after the `ldr` instruction. - AssemblerAccurateScope aas(GetVIXLAssembler(), - vixl32::kMaxInstructionSizeInBytes, - CodeBufferCheckScope::kMaximumSize); + ExactAssemblyScope aas(GetVIXLAssembler(), + vixl32::kMaxInstructionSizeInBytes, + CodeBufferCheckScope::kMaximumSize); __ ldr(temps.Acquire(), MemOperand(InputRegisterAt(instruction, 0))); RecordPcInfo(instruction, instruction->GetDexPc()); } @@ -4873,7 +4888,7 @@ void LocationsBuilderARMVIXL::VisitArrayGet(HArrayGet* instruction) { object_array_get_with_read_barrier ? Location::kOutputOverlap : Location::kNoOutputOverlap); } // We need a temporary register for the read barrier marking slow - // path in CodeGeneratorARM::GenerateArrayLoadWithBakerReadBarrier. + // path in CodeGeneratorARMVIXL::GenerateArrayLoadWithBakerReadBarrier. // Also need for String compression feature. if ((object_array_get_with_read_barrier && kUseBakerReadBarrier) || (mirror::kUseStringCompression && instruction->IsStringCharAt())) { @@ -4908,13 +4923,13 @@ void InstructionCodeGeneratorARMVIXL::VisitArrayGet(HArrayGet* instruction) { codegen_->MaybeRecordImplicitNullCheck(instruction); } if (index.IsConstant()) { - int32_t const_index = index.GetConstant()->AsIntConstant()->GetValue(); + int32_t const_index = Int32ConstantFrom(index); if (maybe_compressed_char_at) { vixl32::Label uncompressed_load, done; __ Lsrs(length, length, 1u); // LSRS has a 16-bit encoding, TST (immediate) does not. static_assert(static_cast<uint32_t>(mirror::StringCompressionFlag::kCompressed) == 0u, "Expecting 0=compressed, 1=uncompressed"); - __ B(cs, &uncompressed_load); + __ B(cs, &uncompressed_load, /* far_target */ false); GetAssembler()->LoadFromOffset(kLoadUnsignedByte, RegisterFrom(out_loc), obj, @@ -4942,7 +4957,7 @@ void InstructionCodeGeneratorARMVIXL::VisitArrayGet(HArrayGet* instruction) { // `TryExtractArrayAccessAddress()`. if (kIsDebugBuild) { HIntermediateAddress* tmp = array_instr->AsIntermediateAddress(); - DCHECK_EQ(tmp->GetOffset()->AsIntConstant()->GetValueAsUint64(), data_offset); + DCHECK_EQ(Uint64ConstantFrom(tmp->GetOffset()), data_offset); } temp = obj; } else { @@ -4953,7 +4968,7 @@ void InstructionCodeGeneratorARMVIXL::VisitArrayGet(HArrayGet* instruction) { __ Lsrs(length, length, 1u); // LSRS has a 16-bit encoding, TST (immediate) does not. static_assert(static_cast<uint32_t>(mirror::StringCompressionFlag::kCompressed) == 0u, "Expecting 0=compressed, 1=uncompressed"); - __ B(cs, &uncompressed_load); + __ B(cs, &uncompressed_load, /* far_target */ false); __ Ldrb(RegisterFrom(out_loc), MemOperand(temp, RegisterFrom(index), vixl32::LSL, 0)); __ B(&done); __ Bind(&uncompressed_load); @@ -4987,7 +5002,7 @@ void InstructionCodeGeneratorARMVIXL::VisitArrayGet(HArrayGet* instruction) { vixl32::Register out = OutputRegister(instruction); if (index.IsConstant()) { size_t offset = - (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; + (Int32ConstantFrom(index) << TIMES_4) + data_offset; GetAssembler()->LoadFromOffset(kLoadWord, out, obj, offset); // TODO(VIXL): Here and for other calls to `MaybeRecordImplicitNullCheck` in this method, // we should use a scope and the assembler to emit the load instruction to guarantee that @@ -5009,7 +5024,7 @@ void InstructionCodeGeneratorARMVIXL::VisitArrayGet(HArrayGet* instruction) { // `TryExtractArrayAccessAddress()`. if (kIsDebugBuild) { HIntermediateAddress* tmp = array_instr->AsIntermediateAddress(); - DCHECK_EQ(tmp->GetOffset()->AsIntConstant()->GetValueAsUint64(), data_offset); + DCHECK_EQ(Uint64ConstantFrom(tmp->GetOffset()), data_offset); } temp = obj; } else { @@ -5034,7 +5049,7 @@ void InstructionCodeGeneratorARMVIXL::VisitArrayGet(HArrayGet* instruction) { case Primitive::kPrimLong: { if (index.IsConstant()) { size_t offset = - (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; + (Int32ConstantFrom(index) << TIMES_8) + data_offset; GetAssembler()->LoadFromOffset(kLoadWordPair, LowRegisterFrom(out_loc), obj, offset); } else { UseScratchRegisterScope temps(GetVIXLAssembler()); @@ -5048,7 +5063,7 @@ void InstructionCodeGeneratorARMVIXL::VisitArrayGet(HArrayGet* instruction) { case Primitive::kPrimFloat: { vixl32::SRegister out = SRegisterFrom(out_loc); if (index.IsConstant()) { - size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; + size_t offset = (Int32ConstantFrom(index) << TIMES_4) + data_offset; GetAssembler()->LoadSFromOffset(out, obj, offset); } else { UseScratchRegisterScope temps(GetVIXLAssembler()); @@ -5061,7 +5076,7 @@ void InstructionCodeGeneratorARMVIXL::VisitArrayGet(HArrayGet* instruction) { case Primitive::kPrimDouble: { if (index.IsConstant()) { - size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; + size_t offset = (Int32ConstantFrom(index) << TIMES_8) + data_offset; GetAssembler()->LoadDFromOffset(DRegisterFrom(out_loc), obj, offset); } else { UseScratchRegisterScope temps(GetVIXLAssembler()); @@ -5135,7 +5150,7 @@ void InstructionCodeGeneratorARMVIXL::VisitArraySet(HArraySet* instruction) { case Primitive::kPrimChar: case Primitive::kPrimInt: { if (index.IsConstant()) { - int32_t const_index = index.GetConstant()->AsIntConstant()->GetValue(); + int32_t const_index = Int32ConstantFrom(index); uint32_t full_offset = data_offset + (const_index << Primitive::ComponentSizeShift(value_type)); StoreOperandType store_type = GetStoreOperandType(value_type); @@ -5150,7 +5165,7 @@ void InstructionCodeGeneratorARMVIXL::VisitArraySet(HArraySet* instruction) { // `TryExtractArrayAccessAddress()`. if (kIsDebugBuild) { HIntermediateAddress* tmp = array_instr->AsIntermediateAddress(); - DCHECK(tmp->GetOffset()->AsIntConstant()->GetValueAsUint64() == data_offset); + DCHECK_EQ(Uint64ConstantFrom(tmp->GetOffset()), data_offset); } temp = array; } else { @@ -5171,7 +5186,7 @@ void InstructionCodeGeneratorARMVIXL::VisitArraySet(HArraySet* instruction) { // Just setting null. if (index.IsConstant()) { size_t offset = - (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; + (Int32ConstantFrom(index) << TIMES_4) + data_offset; GetAssembler()->StoreToOffset(kStoreWord, value, array, offset); } else { DCHECK(index.IsRegister()) << index; @@ -5207,7 +5222,7 @@ void InstructionCodeGeneratorARMVIXL::VisitArraySet(HArraySet* instruction) { __ CompareAndBranchIfNonZero(value, &non_zero); if (index.IsConstant()) { size_t offset = - (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; + (Int32ConstantFrom(index) << TIMES_4) + data_offset; GetAssembler()->StoreToOffset(kStoreWord, value, array, offset); } else { DCHECK(index.IsRegister()) << index; @@ -5233,9 +5248,9 @@ void InstructionCodeGeneratorARMVIXL::VisitArraySet(HArraySet* instruction) { { // Ensure we record the pc position immediately after the `ldr` instruction. - AssemblerAccurateScope aas(GetVIXLAssembler(), - vixl32::kMaxInstructionSizeInBytes, - CodeBufferCheckScope::kMaximumSize); + ExactAssemblyScope aas(GetVIXLAssembler(), + vixl32::kMaxInstructionSizeInBytes, + CodeBufferCheckScope::kMaximumSize); // /* HeapReference<Class> */ temp1 = array->klass_ __ ldr(temp1, MemOperand(array, class_offset)); codegen_->MaybeRecordImplicitNullCheck(instruction); @@ -5252,7 +5267,7 @@ void InstructionCodeGeneratorARMVIXL::VisitArraySet(HArraySet* instruction) { if (instruction->StaticTypeOfArrayIsObjectArray()) { vixl32::Label do_put; - __ B(eq, &do_put); + __ B(eq, &do_put, /* far_target */ false); // If heap poisoning is enabled, the `temp1` reference has // not been unpoisoned yet; unpoison it now. GetAssembler()->MaybeUnpoisonHeapReference(temp1); @@ -5281,7 +5296,7 @@ void InstructionCodeGeneratorARMVIXL::VisitArraySet(HArraySet* instruction) { if (index.IsConstant()) { size_t offset = - (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; + (Int32ConstantFrom(index) << TIMES_4) + data_offset; GetAssembler()->StoreToOffset(kStoreWord, source, array, offset); } else { DCHECK(index.IsRegister()) << index; @@ -5318,7 +5333,7 @@ void InstructionCodeGeneratorARMVIXL::VisitArraySet(HArraySet* instruction) { Location value = locations->InAt(2); if (index.IsConstant()) { size_t offset = - (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; + (Int32ConstantFrom(index) << TIMES_8) + data_offset; GetAssembler()->StoreToOffset(kStoreWordPair, LowRegisterFrom(value), array, offset); } else { UseScratchRegisterScope temps(GetVIXLAssembler()); @@ -5333,7 +5348,7 @@ void InstructionCodeGeneratorARMVIXL::VisitArraySet(HArraySet* instruction) { Location value = locations->InAt(2); DCHECK(value.IsFpuRegister()); if (index.IsConstant()) { - size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; + size_t offset = (Int32ConstantFrom(index) << TIMES_4) + data_offset; GetAssembler()->StoreSToOffset(SRegisterFrom(value), array, offset); } else { UseScratchRegisterScope temps(GetVIXLAssembler()); @@ -5348,7 +5363,7 @@ void InstructionCodeGeneratorARMVIXL::VisitArraySet(HArraySet* instruction) { Location value = locations->InAt(2); DCHECK(value.IsFpuRegisterPair()); if (index.IsConstant()) { - size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; + size_t offset = (Int32ConstantFrom(index) << TIMES_8) + data_offset; GetAssembler()->StoreDToOffset(DRegisterFrom(value), array, offset); } else { UseScratchRegisterScope temps(GetVIXLAssembler()); @@ -5384,9 +5399,9 @@ void InstructionCodeGeneratorARMVIXL::VisitArrayLength(HArrayLength* instruction vixl32::Register obj = InputRegisterAt(instruction, 0); vixl32::Register out = OutputRegister(instruction); { - AssemblerAccurateScope aas(GetVIXLAssembler(), - vixl32::kMaxInstructionSizeInBytes, - CodeBufferCheckScope::kMaximumSize); + ExactAssemblyScope aas(GetVIXLAssembler(), + vixl32::kMaxInstructionSizeInBytes, + CodeBufferCheckScope::kMaximumSize); __ ldr(out, MemOperand(obj, offset)); codegen_->MaybeRecordImplicitNullCheck(instruction); } @@ -5413,7 +5428,7 @@ void InstructionCodeGeneratorARMVIXL::VisitIntermediateAddress(HIntermediateAddr if (second.IsRegister()) { __ Add(out, first, RegisterFrom(second)); } else { - __ Add(out, first, second.GetConstant()->AsIntConstant()->GetValue()); + __ Add(out, first, Int32ConstantFrom(second)); } } @@ -5609,7 +5624,7 @@ void ParallelMoveResolverARMVIXL::EmitMove(size_t index) { GetAssembler()->StoreToOffset(kStoreWord, temp, sp, destination.GetStackIndex()); } } else if (constant->IsLongConstant()) { - int64_t value = constant->AsLongConstant()->GetValue(); + int64_t value = Int64ConstantFrom(source); if (destination.IsRegisterPair()) { __ Mov(LowRegisterFrom(destination), Low32Bits(value)); __ Mov(HighRegisterFrom(destination), High32Bits(value)); @@ -5768,17 +5783,15 @@ HLoadClass::LoadKind CodeGeneratorARMVIXL::GetSupportedLoadClassKind( case HLoadClass::LoadKind::kReferrersClass: break; case HLoadClass::LoadKind::kBootImageLinkTimeAddress: - // TODO(VIXL): Enable it back when literal pools are fixed in VIXL. - return HLoadClass::LoadKind::kDexCacheViaMethod; + DCHECK(!GetCompilerOptions().GetCompilePic()); + break; case HLoadClass::LoadKind::kBootImageLinkTimePcRelative: DCHECK(GetCompilerOptions().GetCompilePic()); break; case HLoadClass::LoadKind::kBootImageAddress: - // TODO(VIXL): Enable it back when literal pools are fixed in VIXL. - return HLoadClass::LoadKind::kDexCacheViaMethod; - case HLoadClass::LoadKind::kDexCacheAddress: - // TODO(VIXL): Enable it back when literal pools are fixed in VIXL. - return HLoadClass::LoadKind::kDexCacheViaMethod; + break; + case HLoadClass::LoadKind::kJitTableAddress: + break; case HLoadClass::LoadKind::kDexCachePcRelative: DCHECK(!Runtime::Current()->UseJitCompilation()); // We disable pc-relative load when there is an irreducible loop, as the optimization @@ -5854,7 +5867,9 @@ void InstructionCodeGeneratorARMVIXL::VisitLoadClass(HLoadClass* cls) { break; } case HLoadClass::LoadKind::kBootImageLinkTimeAddress: { - TODO_VIXL32(FATAL); + DCHECK_EQ(read_barrier_option, kWithoutReadBarrier); + __ Ldr(out, codegen_->DeduplicateBootImageTypeLiteral(cls->GetDexFile(), + cls->GetTypeIndex())); break; } case HLoadClass::LoadKind::kBootImageLinkTimePcRelative: { @@ -5865,11 +5880,18 @@ void InstructionCodeGeneratorARMVIXL::VisitLoadClass(HLoadClass* cls) { break; } case HLoadClass::LoadKind::kBootImageAddress: { - TODO_VIXL32(FATAL); + DCHECK_EQ(read_barrier_option, kWithoutReadBarrier); + DCHECK_NE(cls->GetAddress(), 0u); + uint32_t address = dchecked_integral_cast<uint32_t>(cls->GetAddress()); + __ Ldr(out, codegen_->DeduplicateBootImageAddressLiteral(address)); break; } - case HLoadClass::LoadKind::kDexCacheAddress: { - TODO_VIXL32(FATAL); + case HLoadClass::LoadKind::kJitTableAddress: { + __ Ldr(out, codegen_->DeduplicateJitClassLiteral(cls->GetDexFile(), + cls->GetTypeIndex(), + cls->GetAddress())); + // /* GcRoot<mirror::Class> */ out = *out + GenerateGcRootFieldLoad(cls, out_loc, out, /* offset */ 0, kCompilerReadBarrierOption); break; } case HLoadClass::LoadKind::kDexCachePcRelative: { @@ -5954,21 +5976,19 @@ HLoadString::LoadKind CodeGeneratorARMVIXL::GetSupportedLoadStringKind( HLoadString::LoadKind desired_string_load_kind) { switch (desired_string_load_kind) { case HLoadString::LoadKind::kBootImageLinkTimeAddress: - // TODO(VIXL): Implement missing optimization. - return HLoadString::LoadKind::kDexCacheViaMethod; + DCHECK(!GetCompilerOptions().GetCompilePic()); + break; case HLoadString::LoadKind::kBootImageLinkTimePcRelative: DCHECK(GetCompilerOptions().GetCompilePic()); break; case HLoadString::LoadKind::kBootImageAddress: - // TODO(VIXL): Implement missing optimization. - return HLoadString::LoadKind::kDexCacheViaMethod; + break; case HLoadString::LoadKind::kBssEntry: DCHECK(!Runtime::Current()->UseJitCompilation()); break; case HLoadString::LoadKind::kJitTableAddress: DCHECK(Runtime::Current()->UseJitCompilation()); - // TODO(VIXL): Implement missing optimization. - return HLoadString::LoadKind::kDexCacheViaMethod; + break; case HLoadString::LoadKind::kDexCacheViaMethod: break; } @@ -6010,8 +6030,9 @@ void InstructionCodeGeneratorARMVIXL::VisitLoadString(HLoadString* load) { switch (load_kind) { case HLoadString::LoadKind::kBootImageLinkTimeAddress: { - TODO_VIXL32(FATAL); - break; + __ Ldr(out, codegen_->DeduplicateBootImageStringLiteral(load->GetDexFile(), + load->GetStringIndex())); + return; // No dex cache slow path. } case HLoadString::LoadKind::kBootImageLinkTimePcRelative: { DCHECK(codegen_->GetCompilerOptions().IsBootImage()); @@ -6021,8 +6042,10 @@ void InstructionCodeGeneratorARMVIXL::VisitLoadString(HLoadString* load) { return; // No dex cache slow path. } case HLoadString::LoadKind::kBootImageAddress: { - TODO_VIXL32(FATAL); - break; + DCHECK_NE(load->GetAddress(), 0u); + uint32_t address = dchecked_integral_cast<uint32_t>(load->GetAddress()); + __ Ldr(out, codegen_->DeduplicateBootImageAddressLiteral(address)); + return; // No dex cache slow path. } case HLoadString::LoadKind::kBssEntry: { DCHECK(!codegen_->GetCompilerOptions().IsBootImage()); @@ -6039,8 +6062,11 @@ void InstructionCodeGeneratorARMVIXL::VisitLoadString(HLoadString* load) { return; } case HLoadString::LoadKind::kJitTableAddress: { - TODO_VIXL32(FATAL); - break; + __ Ldr(out, codegen_->DeduplicateJitStringLiteral(load->GetDexFile(), + load->GetStringIndex())); + // /* GcRoot<mirror::String> */ out = *out + GenerateGcRootFieldLoad(load, out_loc, out, /* offset */ 0, kCompilerReadBarrierOption); + return; } default: break; @@ -6182,7 +6208,7 @@ void InstructionCodeGeneratorARMVIXL::VisitInstanceOf(HInstanceOf* instruction) kCompilerReadBarrierOption); __ Cmp(out, cls); // Classes must be equal for the instanceof to succeed. - __ B(ne, &zero); + __ B(ne, &zero, /* far_target */ false); __ Mov(out, 1); __ B(&done); break; @@ -6209,7 +6235,7 @@ void InstructionCodeGeneratorARMVIXL::VisitInstanceOf(HInstanceOf* instruction) // If `out` is null, we use it for the result, and jump to `done`. __ CompareAndBranchIfZero(out, &done, /* far_target */ false); __ Cmp(out, cls); - __ B(ne, &loop); + __ B(ne, &loop, /* far_target */ false); __ Mov(out, 1); if (zero.IsReferenced()) { __ B(&done); @@ -6229,7 +6255,7 @@ void InstructionCodeGeneratorARMVIXL::VisitInstanceOf(HInstanceOf* instruction) vixl32::Label loop, success; __ Bind(&loop); __ Cmp(out, cls); - __ B(eq, &success); + __ B(eq, &success, /* far_target */ false); // /* HeapReference<Class> */ out = out->super_class_ GenerateReferenceLoadOneRegister(instruction, out_loc, @@ -6258,7 +6284,7 @@ void InstructionCodeGeneratorARMVIXL::VisitInstanceOf(HInstanceOf* instruction) // Do an exact check. vixl32::Label exact_check; __ Cmp(out, cls); - __ B(eq, &exact_check); + __ B(eq, &exact_check, /* far_target */ false); // Otherwise, we need to check that the object's class is a non-primitive array. // /* HeapReference<Class> */ out = out->component_type_ GenerateReferenceLoadOneRegister(instruction, @@ -6460,7 +6486,7 @@ void InstructionCodeGeneratorARMVIXL::VisitCheckCast(HCheckCast* instruction) { // Otherwise, compare the classes. __ Cmp(temp, cls); - __ B(ne, &loop); + __ B(ne, &loop, /* far_target */ false); break; } @@ -6477,7 +6503,7 @@ void InstructionCodeGeneratorARMVIXL::VisitCheckCast(HCheckCast* instruction) { vixl32::Label loop; __ Bind(&loop); __ Cmp(temp, cls); - __ B(eq, &done); + __ B(eq, &done, /* far_target */ false); // /* HeapReference<Class> */ temp = temp->super_class_ GenerateReferenceLoadOneRegister(instruction, @@ -6505,7 +6531,7 @@ void InstructionCodeGeneratorARMVIXL::VisitCheckCast(HCheckCast* instruction) { // Do an exact check. __ Cmp(temp, cls); - __ B(eq, &done); + __ B(eq, &done, /* far_target */ false); // Otherwise, we need to check that the object's class is a non-primitive array. // /* HeapReference<Class> */ temp = temp->component_type_ @@ -6569,7 +6595,7 @@ void InstructionCodeGeneratorARMVIXL::VisitCheckCast(HCheckCast* instruction) { __ Sub(RegisterFrom(maybe_temp2_loc), RegisterFrom(maybe_temp2_loc), 2); // Compare the classes and continue the loop if they do not match. __ Cmp(cls, RegisterFrom(maybe_temp3_loc)); - __ B(ne, &start_loop); + __ B(ne, &start_loop, /* far_target */ false); break; } } @@ -7202,20 +7228,7 @@ void CodeGeneratorARMVIXL::GenerateReadBarrierForRootSlow(HInstruction* instruct // otherwise return a fall-back info that should be used instead. HInvokeStaticOrDirect::DispatchInfo CodeGeneratorARMVIXL::GetSupportedInvokeStaticOrDirectDispatch( const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info, - HInvokeStaticOrDirect* invoke) { - // TODO(VIXL): Implement optimized code paths. - if (desired_dispatch_info.method_load_kind == - HInvokeStaticOrDirect::MethodLoadKind::kDirectAddressWithFixup || - desired_dispatch_info.code_ptr_location == - HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup) { - return { - HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod, - HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod, - 0u, - 0u - }; - } - + HInvokeStaticOrDirect* invoke ATTRIBUTE_UNUSED) { HInvokeStaticOrDirect::DispatchInfo dispatch_info = desired_dispatch_info; // We disable pc-relative load when there is an irreducible loop, as the optimization // is incompatible with it. @@ -7227,24 +7240,6 @@ HInvokeStaticOrDirect::DispatchInfo CodeGeneratorARMVIXL::GetSupportedInvokeStat dispatch_info.method_load_kind = HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod; } - if (dispatch_info.code_ptr_location == HInvokeStaticOrDirect::CodePtrLocation::kCallPCRelative) { - const DexFile& outer_dex_file = GetGraph()->GetDexFile(); - if (&outer_dex_file != invoke->GetTargetMethod().dex_file) { - // Calls across dex files are more likely to exceed the available BL range, - // so use absolute patch with fixup if available and kCallArtMethod otherwise. - HInvokeStaticOrDirect::CodePtrLocation code_ptr_location = - (desired_dispatch_info.method_load_kind == - HInvokeStaticOrDirect::MethodLoadKind::kDirectAddressWithFixup) - ? HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup - : HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod; - return HInvokeStaticOrDirect::DispatchInfo { - dispatch_info.method_load_kind, - code_ptr_location, - dispatch_info.method_load_data, - 0u - }; - } - } return dispatch_info; } @@ -7276,20 +7271,6 @@ vixl32::Register CodeGeneratorARMVIXL::GetInvokeStaticOrDirectExtraParameter( void CodeGeneratorARMVIXL::GenerateStaticOrDirectCall( HInvokeStaticOrDirect* invoke, Location temp) { - // For better instruction scheduling we load the direct code pointer before the method pointer. - switch (invoke->GetCodePtrLocation()) { - case HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup: - // LR = code address from literal pool with link-time patch. - TODO_VIXL32(FATAL); - break; - case HInvokeStaticOrDirect::CodePtrLocation::kCallDirect: - // LR = invoke->GetDirectCodePtr(); - __ Mov(lr, Operand::From(invoke->GetDirectCodePtr())); - break; - default: - break; - } - Location callee_method = temp; // For all kinds except kRecursive, callee will be in temp. switch (invoke->GetMethodLoadKind()) { case HInvokeStaticOrDirect::MethodLoadKind::kStringInit: { @@ -7305,9 +7286,6 @@ void CodeGeneratorARMVIXL::GenerateStaticOrDirectCall( case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress: __ Mov(RegisterFrom(temp), Operand::From(invoke->GetMethodAddress())); break; - case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddressWithFixup: - TODO_VIXL32(FATAL); - break; case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: { HArmDexCacheArraysBase* base = invoke->InputAt(invoke->GetSpecialInputIndex())->AsArmDexCacheArraysBase(); @@ -7347,30 +7325,6 @@ void CodeGeneratorARMVIXL::GenerateStaticOrDirectCall( case HInvokeStaticOrDirect::CodePtrLocation::kCallSelf: __ Bl(GetFrameEntryLabel()); break; - case HInvokeStaticOrDirect::CodePtrLocation::kCallPCRelative: - relative_call_patches_.emplace_back(*invoke->GetTargetMethod().dex_file, - invoke->GetTargetMethod().dex_method_index); - { - AssemblerAccurateScope aas(GetVIXLAssembler(), - vixl32::kMaxInstructionSizeInBytes, - CodeBufferCheckScope::kMaximumSize); - __ bind(&relative_call_patches_.back().label); - // Arbitrarily branch to the BL itself, override at link time. - __ bl(&relative_call_patches_.back().label); - } - break; - case HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup: - case HInvokeStaticOrDirect::CodePtrLocation::kCallDirect: - // LR prepared above for better instruction scheduling. - // LR() - { - // blx in T32 has only 16bit encoding that's why a stricter check for the scope is used. - AssemblerAccurateScope aas(GetVIXLAssembler(), - vixl32::k16BitT32InstructionSizeInBytes, - CodeBufferCheckScope::kExactSize); - __ blx(lr); - } - break; case HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod: // LR = callee_method->entry_point_from_quick_compiled_code_ GetAssembler()->LoadFromOffset( @@ -7380,9 +7334,9 @@ void CodeGeneratorARMVIXL::GenerateStaticOrDirectCall( ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmPointerSize).Int32Value()); { // blx in T32 has only 16bit encoding that's why a stricter check for the scope is used. - AssemblerAccurateScope aas(GetVIXLAssembler(), - vixl32::k16BitT32InstructionSizeInBytes, - CodeBufferCheckScope::kExactSize); + ExactAssemblyScope aas(GetVIXLAssembler(), + vixl32::k16BitT32InstructionSizeInBytes, + CodeBufferCheckScope::kExactSize); // LR() __ blx(lr); } @@ -7406,9 +7360,9 @@ void CodeGeneratorARMVIXL::GenerateVirtualCall(HInvokeVirtual* invoke, Location uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); { // Make sure the pc is recorded immediately after the `ldr` instruction. - AssemblerAccurateScope aas(GetVIXLAssembler(), - vixl32::kMaxInstructionSizeInBytes, - CodeBufferCheckScope::kMaximumSize); + ExactAssemblyScope aas(GetVIXLAssembler(), + vixl32::kMaxInstructionSizeInBytes, + CodeBufferCheckScope::kMaximumSize); // /* HeapReference<Class> */ temp = receiver->klass_ __ ldr(temp, MemOperand(receiver, class_offset)); MaybeRecordImplicitNullCheck(invoke); @@ -7433,9 +7387,9 @@ void CodeGeneratorARMVIXL::GenerateVirtualCall(HInvokeVirtual* invoke, Location // `RecordPcInfo()` immediately following record the correct pc. Use a scope to help guarantee // that. // blx in T32 has only 16bit encoding that's why a stricter check for the scope is used. - AssemblerAccurateScope aas(GetVIXLAssembler(), - vixl32::k16BitT32InstructionSizeInBytes, - CodeBufferCheckScope::kExactSize); + ExactAssemblyScope aas(GetVIXLAssembler(), + vixl32::k16BitT32InstructionSizeInBytes, + CodeBufferCheckScope::kExactSize); __ blx(lr); } @@ -7460,6 +7414,57 @@ CodeGeneratorARMVIXL::PcRelativePatchInfo* CodeGeneratorARMVIXL::NewPcRelativePa return &patches->back(); } +VIXLUInt32Literal* CodeGeneratorARMVIXL::DeduplicateBootImageStringLiteral( + const DexFile& dex_file, + dex::StringIndex string_index) { + return boot_image_string_patches_.GetOrCreate( + StringReference(&dex_file, string_index), + [this]() { + return GetAssembler()->CreateLiteralDestroyedWithPool<uint32_t>(/* placeholder */ 0u); + }); +} + +VIXLUInt32Literal* CodeGeneratorARMVIXL::DeduplicateBootImageTypeLiteral( + const DexFile& dex_file, + dex::TypeIndex type_index) { + return boot_image_type_patches_.GetOrCreate( + TypeReference(&dex_file, type_index), + [this]() { + return GetAssembler()->CreateLiteralDestroyedWithPool<uint32_t>(/* placeholder */ 0u); + }); +} + +VIXLUInt32Literal* CodeGeneratorARMVIXL::DeduplicateBootImageAddressLiteral(uint32_t address) { + bool needs_patch = GetCompilerOptions().GetIncludePatchInformation(); + Uint32ToLiteralMap* map = needs_patch ? &boot_image_address_patches_ : &uint32_literals_; + return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address), map); +} + +VIXLUInt32Literal* CodeGeneratorARMVIXL::DeduplicateDexCacheAddressLiteral(uint32_t address) { + return DeduplicateUint32Literal(address, &uint32_literals_); +} + +VIXLUInt32Literal* CodeGeneratorARMVIXL::DeduplicateJitStringLiteral(const DexFile& dex_file, + dex::StringIndex string_index) { + jit_string_roots_.Overwrite(StringReference(&dex_file, string_index), /* placeholder */ 0u); + return jit_string_patches_.GetOrCreate( + StringReference(&dex_file, string_index), + [this]() { + return GetAssembler()->CreateLiteralDestroyedWithPool<uint32_t>(/* placeholder */ 0u); + }); +} + +VIXLUInt32Literal* CodeGeneratorARMVIXL::DeduplicateJitClassLiteral(const DexFile& dex_file, + dex::TypeIndex type_index, + uint64_t address) { + jit_class_roots_.Overwrite(TypeReference(&dex_file, type_index), address); + return jit_class_patches_.GetOrCreate( + TypeReference(&dex_file, type_index), + [this]() { + return GetAssembler()->CreateLiteralDestroyedWithPool<uint32_t>(/* placeholder */ 0u); + }); +} + template <LinkerPatch (*Factory)(size_t, const DexFile*, uint32_t, uint32_t)> inline void CodeGeneratorARMVIXL::EmitPcRelativeLinkerPatches( const ArenaDeque<PcRelativePatchInfo>& infos, @@ -7483,18 +7488,24 @@ inline void CodeGeneratorARMVIXL::EmitPcRelativeLinkerPatches( void CodeGeneratorARMVIXL::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) { DCHECK(linker_patches->empty()); size_t size = - relative_call_patches_.size() + /* MOVW+MOVT for each entry */ 2u * pc_relative_dex_cache_patches_.size() + + boot_image_string_patches_.size() + /* MOVW+MOVT for each entry */ 2u * pc_relative_string_patches_.size() + - /* MOVW+MOVT for each entry */ 2u * pc_relative_type_patches_.size(); + boot_image_type_patches_.size() + + /* MOVW+MOVT for each entry */ 2u * pc_relative_type_patches_.size() + + boot_image_address_patches_.size(); linker_patches->reserve(size); - for (const PatchInfo<vixl32::Label>& info : relative_call_patches_) { - uint32_t literal_offset = info.label.GetLocation(); - linker_patches->push_back( - LinkerPatch::RelativeCodePatch(literal_offset, &info.dex_file, info.index)); - } EmitPcRelativeLinkerPatches<LinkerPatch::DexCacheArrayPatch>(pc_relative_dex_cache_patches_, linker_patches); + for (const auto& entry : boot_image_string_patches_) { + const StringReference& target_string = entry.first; + VIXLUInt32Literal* literal = entry.second; + DCHECK(literal->IsBound()); + uint32_t literal_offset = literal->GetLocation(); + linker_patches->push_back(LinkerPatch::StringPatch(literal_offset, + target_string.dex_file, + target_string.string_index.index_)); + } if (!GetCompilerOptions().IsBootImage()) { EmitPcRelativeLinkerPatches<LinkerPatch::StringBssEntryPatch>(pc_relative_string_patches_, linker_patches); @@ -7502,8 +7513,44 @@ void CodeGeneratorARMVIXL::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_pa EmitPcRelativeLinkerPatches<LinkerPatch::RelativeStringPatch>(pc_relative_string_patches_, linker_patches); } + for (const auto& entry : boot_image_type_patches_) { + const TypeReference& target_type = entry.first; + VIXLUInt32Literal* literal = entry.second; + DCHECK(literal->IsBound()); + uint32_t literal_offset = literal->GetLocation(); + linker_patches->push_back(LinkerPatch::TypePatch(literal_offset, + target_type.dex_file, + target_type.type_index.index_)); + } EmitPcRelativeLinkerPatches<LinkerPatch::RelativeTypePatch>(pc_relative_type_patches_, linker_patches); + for (const auto& entry : boot_image_address_patches_) { + DCHECK(GetCompilerOptions().GetIncludePatchInformation()); + VIXLUInt32Literal* literal = entry.second; + DCHECK(literal->IsBound()); + uint32_t literal_offset = literal->GetLocation(); + linker_patches->push_back(LinkerPatch::RecordPosition(literal_offset)); + } +} + +VIXLUInt32Literal* CodeGeneratorARMVIXL::DeduplicateUint32Literal( + uint32_t value, + Uint32ToLiteralMap* map) { + return map->GetOrCreate( + value, + [this, value]() { + return GetAssembler()->CreateLiteralDestroyedWithPool<uint32_t>(/* placeholder */ value); + }); +} + +VIXLUInt32Literal* CodeGeneratorARMVIXL::DeduplicateMethodLiteral( + MethodReference target_method, + MethodToLiteralMap* map) { + return map->GetOrCreate( + target_method, + [this]() { + return GetAssembler()->CreateLiteralDestroyedWithPool<uint32_t>(/* placeholder */ 0u); + }); } void LocationsBuilderARMVIXL::VisitMultiplyAccumulate(HMultiplyAccumulate* instr) { @@ -7699,12 +7746,37 @@ void InstructionCodeGeneratorARMVIXL::VisitClassTableGet(HClassTableGet* instruc } } +static void PatchJitRootUse(uint8_t* code, + const uint8_t* roots_data, + VIXLUInt32Literal* literal, + uint64_t index_in_table) { + DCHECK(literal->IsBound()); + uint32_t literal_offset = literal->GetLocation(); + uintptr_t address = + reinterpret_cast<uintptr_t>(roots_data) + index_in_table * sizeof(GcRoot<mirror::Object>); + uint8_t* data = code + literal_offset; + reinterpret_cast<uint32_t*>(data)[0] = dchecked_integral_cast<uint32_t>(address); +} + +void CodeGeneratorARMVIXL::EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) { + for (const auto& entry : jit_string_patches_) { + const auto& it = jit_string_roots_.find(entry.first); + DCHECK(it != jit_string_roots_.end()); + PatchJitRootUse(code, roots_data, entry.second, it->second); + } + for (const auto& entry : jit_class_patches_) { + const auto& it = jit_class_roots_.find(entry.first); + DCHECK(it != jit_class_roots_.end()); + PatchJitRootUse(code, roots_data, entry.second, it->second); + } +} + void CodeGeneratorARMVIXL::EmitMovwMovtPlaceholder( CodeGeneratorARMVIXL::PcRelativePatchInfo* labels, vixl32::Register out) { - AssemblerAccurateScope aas(GetVIXLAssembler(), - 3 * vixl32::kMaxInstructionSizeInBytes, - CodeBufferCheckScope::kMaximumSize); + ExactAssemblyScope aas(GetVIXLAssembler(), + 3 * vixl32::kMaxInstructionSizeInBytes, + CodeBufferCheckScope::kMaximumSize); // TODO(VIXL): Think about using mov instead of movw. __ bind(&labels->movw_label); __ movw(out, /* placeholder */ 0u); diff --git a/compiler/optimizing/code_generator_arm_vixl.h b/compiler/optimizing/code_generator_arm_vixl.h index 5ec3da4652..297d63cefd 100644 --- a/compiler/optimizing/code_generator_arm_vixl.h +++ b/compiler/optimizing/code_generator_arm_vixl.h @@ -107,20 +107,20 @@ static const size_t kRuntimeParameterFpuRegistersLengthVIXL = arraysize(kRuntimeParameterFpuRegistersVIXL); class LoadClassSlowPathARMVIXL; - class CodeGeneratorARMVIXL; +using VIXLInt32Literal = vixl::aarch32::Literal<int32_t>; +using VIXLUInt32Literal = vixl::aarch32::Literal<uint32_t>; + class JumpTableARMVIXL : public DeletableArenaObject<kArenaAllocSwitchTable> { public: - typedef vixl::aarch32::Literal<int32_t> IntLiteral; - explicit JumpTableARMVIXL(HPackedSwitch* switch_instr) : switch_instr_(switch_instr), table_start_(), bb_addresses_(switch_instr->GetArena()->Adapter(kArenaAllocCodeGenerator)) { uint32_t num_entries = switch_instr_->GetNumEntries(); for (uint32_t i = 0; i < num_entries; i++) { - IntLiteral *lit = new IntLiteral(0); + VIXLInt32Literal *lit = new VIXLInt32Literal(0, vixl32::RawLiteral::kManuallyPlaced); bb_addresses_.emplace_back(lit); } } @@ -133,7 +133,7 @@ class JumpTableARMVIXL : public DeletableArenaObject<kArenaAllocSwitchTable> { private: HPackedSwitch* const switch_instr_; vixl::aarch32::Label table_start_; - ArenaVector<std::unique_ptr<IntLiteral>> bb_addresses_; + ArenaVector<std::unique_ptr<VIXLInt32Literal>> bb_addresses_; DISALLOW_COPY_AND_ASSIGN(JumpTableARMVIXL); }; @@ -566,8 +566,22 @@ class CodeGeneratorARMVIXL : public CodeGenerator { PcRelativePatchInfo* NewPcRelativeTypePatch(const DexFile& dex_file, dex::TypeIndex type_index); PcRelativePatchInfo* NewPcRelativeDexCacheArrayPatch(const DexFile& dex_file, uint32_t element_offset); + VIXLUInt32Literal* DeduplicateBootImageStringLiteral(const DexFile& dex_file, + dex::StringIndex string_index); + VIXLUInt32Literal* DeduplicateBootImageTypeLiteral(const DexFile& dex_file, + dex::TypeIndex type_index); + VIXLUInt32Literal* DeduplicateBootImageAddressLiteral(uint32_t address); + VIXLUInt32Literal* DeduplicateDexCacheAddressLiteral(uint32_t address); + VIXLUInt32Literal* DeduplicateJitStringLiteral(const DexFile& dex_file, + dex::StringIndex string_index); + VIXLUInt32Literal* DeduplicateJitClassLiteral(const DexFile& dex_file, + dex::TypeIndex type_index, + uint64_t address); + void EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) OVERRIDE; + void EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) OVERRIDE; + // Fast path implementation of ReadBarrier::Barrier for a heap // reference field load when Baker's read barriers are used. void GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction, @@ -673,10 +687,19 @@ class CodeGeneratorARMVIXL : public CodeGenerator { vixl::aarch32::Register GetInvokeStaticOrDirectExtraParameter(HInvokeStaticOrDirect* invoke, vixl::aarch32::Register temp); - using Uint32ToLiteralMap = ArenaSafeMap<uint32_t, vixl::aarch32::Literal<uint32_t>*>; + using Uint32ToLiteralMap = ArenaSafeMap<uint32_t, VIXLUInt32Literal*>; using MethodToLiteralMap = - ArenaSafeMap<MethodReference, vixl::aarch32::Literal<uint32_t>*, MethodReferenceComparator>; - + ArenaSafeMap<MethodReference, VIXLUInt32Literal*, MethodReferenceComparator>; + using StringToLiteralMap = ArenaSafeMap<StringReference, + VIXLUInt32Literal*, + StringReferenceValueComparator>; + using TypeToLiteralMap = ArenaSafeMap<TypeReference, + VIXLUInt32Literal*, + TypeReferenceValueComparator>; + + VIXLUInt32Literal* DeduplicateUint32Literal(uint32_t value, Uint32ToLiteralMap* map); + VIXLUInt32Literal* DeduplicateMethodLiteral(MethodReference target_method, + MethodToLiteralMap* map); PcRelativePatchInfo* NewPcRelativePatch(const DexFile& dex_file, uint32_t offset_or_index, ArenaDeque<PcRelativePatchInfo>* patches); @@ -697,15 +720,25 @@ class CodeGeneratorARMVIXL : public CodeGenerator { ArmVIXLAssembler assembler_; const ArmInstructionSetFeatures& isa_features_; - // Relative call patch info. - // Using ArenaDeque<> which retains element addresses on push/emplace_back(). - ArenaDeque<PatchInfo<vixl::aarch32::Label>> relative_call_patches_; + // Deduplication map for 32-bit literals, used for non-patchable boot image addresses. + Uint32ToLiteralMap uint32_literals_; // PC-relative patch info for each HArmDexCacheArraysBase. ArenaDeque<PcRelativePatchInfo> pc_relative_dex_cache_patches_; + // Deduplication map for boot string literals for kBootImageLinkTimeAddress. + StringToLiteralMap boot_image_string_patches_; // PC-relative String patch info; type depends on configuration (app .bss or boot image PIC). ArenaDeque<PcRelativePatchInfo> pc_relative_string_patches_; + // Deduplication map for boot type literals for kBootImageLinkTimeAddress. + TypeToLiteralMap boot_image_type_patches_; // PC-relative type patch info. ArenaDeque<PcRelativePatchInfo> pc_relative_type_patches_; + // Deduplication map for patchable boot image addresses. + Uint32ToLiteralMap boot_image_address_patches_; + + // Patches for string literals in JIT compiled code. + StringToLiteralMap jit_string_patches_; + // Patches for class literals in JIT compiled code. + TypeToLiteralMap jit_class_patches_; DISALLOW_COPY_AND_ASSIGN(CodeGeneratorARMVIXL); }; diff --git a/compiler/optimizing/code_generator_mips.cc b/compiler/optimizing/code_generator_mips.cc index ff48f6642d..01e0dac33e 100644 --- a/compiler/optimizing/code_generator_mips.cc +++ b/compiler/optimizing/code_generator_mips.cc @@ -458,10 +458,6 @@ CodeGeneratorMIPS::CodeGeneratorMIPS(HGraph* graph, isa_features_(isa_features), uint32_literals_(std::less<uint32_t>(), graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), - method_patches_(MethodReferenceComparator(), - graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), - call_patches_(MethodReferenceComparator(), - graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), boot_image_string_patches_(StringReferenceValueComparator(), graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), @@ -757,6 +753,11 @@ void CodeGeneratorMIPS::GenerateFrameEntry() { if (RequiresCurrentMethod()) { __ StoreToOffset(kStoreWord, kMethodRegisterArgument, SP, kCurrentMethodStackOffset); } + + if (GetGraph()->HasShouldDeoptimizeFlag()) { + // Initialize should deoptimize flag to 0. + __ StoreToOffset(kStoreWord, ZERO, SP, GetStackOffsetOfShouldDeoptimizeFlag()); + } } void CodeGeneratorMIPS::GenerateFrameExit() { @@ -1003,8 +1004,6 @@ inline void CodeGeneratorMIPS::EmitPcRelativeLinkerPatches( void CodeGeneratorMIPS::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) { DCHECK(linker_patches->empty()); size_t size = - method_patches_.size() + - call_patches_.size() + pc_relative_dex_cache_patches_.size() + pc_relative_string_patches_.size() + pc_relative_type_patches_.size() + @@ -1012,24 +1011,6 @@ void CodeGeneratorMIPS::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patch boot_image_type_patches_.size() + boot_image_address_patches_.size(); linker_patches->reserve(size); - for (const auto& entry : method_patches_) { - const MethodReference& target_method = entry.first; - Literal* literal = entry.second; - DCHECK(literal->GetLabel()->IsBound()); - uint32_t literal_offset = __ GetLabelLocation(literal->GetLabel()); - linker_patches->push_back(LinkerPatch::MethodPatch(literal_offset, - target_method.dex_file, - target_method.dex_method_index)); - } - for (const auto& entry : call_patches_) { - const MethodReference& target_method = entry.first; - Literal* literal = entry.second; - DCHECK(literal->GetLabel()->IsBound()); - uint32_t literal_offset = __ GetLabelLocation(literal->GetLabel()); - linker_patches->push_back(LinkerPatch::CodePatch(literal_offset, - target_method.dex_file, - target_method.dex_method_index)); - } EmitPcRelativeLinkerPatches<LinkerPatch::DexCacheArrayPatch>(pc_relative_dex_cache_patches_, linker_patches); if (!GetCompilerOptions().IsBootImage()) { @@ -1102,14 +1083,6 @@ Literal* CodeGeneratorMIPS::DeduplicateMethodLiteral(MethodReference target_meth [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); } -Literal* CodeGeneratorMIPS::DeduplicateMethodAddressLiteral(MethodReference target_method) { - return DeduplicateMethodLiteral(target_method, &method_patches_); -} - -Literal* CodeGeneratorMIPS::DeduplicateMethodCodeLiteral(MethodReference target_method) { - return DeduplicateMethodLiteral(target_method, &call_patches_); -} - Literal* CodeGeneratorMIPS::DeduplicateBootImageStringLiteral(const DexFile& dex_file, dex::StringIndex string_index) { return boot_image_string_patches_.GetOrCreate( @@ -1160,11 +1133,15 @@ void CodeGeneratorMIPS::EmitPcRelativeAddressPlaceholder( __ SetReorder(reordering); } -void CodeGeneratorMIPS::MarkGCCard(Register object, Register value) { +void CodeGeneratorMIPS::MarkGCCard(Register object, + Register value, + bool value_can_be_null) { MipsLabel done; Register card = AT; Register temp = TMP; - __ Beqz(value, &done); + if (value_can_be_null) { + __ Beqz(value, &done); + } __ LoadFromOffset(kLoadWord, card, TR, @@ -1172,7 +1149,9 @@ void CodeGeneratorMIPS::MarkGCCard(Register object, Register value) { __ Srl(temp, object, gc::accounting::CardTable::kCardShift); __ Addu(temp, card, temp); __ Sb(card, temp, 0); - __ Bind(&done); + if (value_can_be_null) { + __ Bind(&done); + } } void CodeGeneratorMIPS::SetupBlockedRegisters() const { @@ -2091,7 +2070,7 @@ void InstructionCodeGeneratorMIPS::VisitArraySet(HArraySet* instruction) { __ StoreToOffset(kStoreWord, value, base_reg, data_offset, null_checker); if (needs_write_barrier) { DCHECK_EQ(value_type, Primitive::kPrimNot); - codegen_->MarkGCCard(obj, value); + codegen_->MarkGCCard(obj, value, instruction->GetValueCanBeNull()); } } } else { @@ -4689,14 +4668,17 @@ void InstructionCodeGeneratorMIPS::GenConditionalMoveR6(HSelect* select) { } } -void LocationsBuilderMIPS::VisitShouldDeoptimizeFlag( - HShouldDeoptimizeFlag* flag ATTRIBUTE_UNUSED) { - // TODO: to be implemented. +void LocationsBuilderMIPS::VisitShouldDeoptimizeFlag(HShouldDeoptimizeFlag* flag) { + LocationSummary* locations = new (GetGraph()->GetArena()) + LocationSummary(flag, LocationSummary::kNoCall); + locations->SetOut(Location::RequiresRegister()); } -void InstructionCodeGeneratorMIPS::VisitShouldDeoptimizeFlag( - HShouldDeoptimizeFlag* flag ATTRIBUTE_UNUSED) { - // TODO: to be implemented. +void InstructionCodeGeneratorMIPS::VisitShouldDeoptimizeFlag(HShouldDeoptimizeFlag* flag) { + __ LoadFromOffset(kLoadWord, + flag->GetLocations()->Out().AsRegister<Register>(), + SP, + codegen_->GetStackOffsetOfShouldDeoptimizeFlag()); } void LocationsBuilderMIPS::VisitSelect(HSelect* select) { @@ -4892,7 +4874,8 @@ void LocationsBuilderMIPS::HandleFieldSet(HInstruction* instruction, const Field void InstructionCodeGeneratorMIPS::HandleFieldSet(HInstruction* instruction, const FieldInfo& field_info, - uint32_t dex_pc) { + uint32_t dex_pc, + bool value_can_be_null) { Primitive::Type type = field_info.GetFieldType(); LocationSummary* locations = instruction->GetLocations(); Register obj = locations->InAt(0).AsRegister<Register>(); @@ -4987,7 +4970,7 @@ void InstructionCodeGeneratorMIPS::HandleFieldSet(HInstruction* instruction, // TODO: memory barriers? if (CodeGenerator::StoreNeedsWriteBarrier(type, instruction->InputAt(1))) { Register src = value_location.AsRegister<Register>(); - codegen_->MarkGCCard(obj, src); + codegen_->MarkGCCard(obj, src, value_can_be_null); } if (is_volatile) { @@ -5008,7 +4991,10 @@ void LocationsBuilderMIPS::VisitInstanceFieldSet(HInstanceFieldSet* instruction) } void InstructionCodeGeneratorMIPS::VisitInstanceFieldSet(HInstanceFieldSet* instruction) { - HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetDexPc()); + HandleFieldSet(instruction, + instruction->GetFieldInfo(), + instruction->GetDexPc(), + instruction->GetValueCanBeNull()); } void InstructionCodeGeneratorMIPS::GenerateGcRootFieldLoad( @@ -5149,22 +5135,7 @@ void LocationsBuilderMIPS::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invo // art::PrepareForRegisterAllocation. DCHECK(!invoke->IsStaticWithExplicitClinitCheck()); - HInvokeStaticOrDirect::MethodLoadKind method_load_kind = invoke->GetMethodLoadKind(); - HInvokeStaticOrDirect::CodePtrLocation code_ptr_location = invoke->GetCodePtrLocation(); - bool isR6 = codegen_->GetInstructionSetFeatures().IsR6(); - - // kDirectAddressWithFixup and kCallDirectWithFixup need no extra input on R6 because - // R6 has PC-relative addressing. - bool has_extra_input = !isR6 && - ((method_load_kind == HInvokeStaticOrDirect::MethodLoadKind::kDirectAddressWithFixup) || - (code_ptr_location == HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup)); - - if (invoke->HasPcRelativeDexCache()) { - // kDexCachePcRelative is mutually exclusive with - // kDirectAddressWithFixup/kCallDirectWithFixup. - CHECK(!has_extra_input); - has_extra_input = true; - } + bool has_extra_input = invoke->HasPcRelativeDexCache(); IntrinsicLocationsBuilderMIPS intrinsic(codegen_); if (intrinsic.TryDispatch(invoke)) { @@ -5251,9 +5222,9 @@ HLoadClass::LoadKind CodeGeneratorMIPS::GetSupportedLoadClassKind( break; case HLoadClass::LoadKind::kBootImageAddress: break; - case HLoadClass::LoadKind::kDexCacheAddress: + case HLoadClass::LoadKind::kJitTableAddress: DCHECK(Runtime::Current()->UseJitCompilation()); - fallback_load = false; + fallback_load = true; break; case HLoadClass::LoadKind::kDexCachePcRelative: DCHECK(!Runtime::Current()->UseJitCompilation()); @@ -5304,9 +5275,7 @@ HInvokeStaticOrDirect::DispatchInfo CodeGeneratorMIPS::GetSupportedInvokeStaticO // is incompatible with it. bool has_irreducible_loops = GetGraph()->HasIrreducibleLoops(); bool fallback_load = true; - bool fallback_call = true; switch (dispatch_info.method_load_kind) { - case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddressWithFixup: case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: fallback_load = has_irreducible_loops; break; @@ -5314,25 +5283,10 @@ HInvokeStaticOrDirect::DispatchInfo CodeGeneratorMIPS::GetSupportedInvokeStaticO fallback_load = false; break; } - switch (dispatch_info.code_ptr_location) { - case HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup: - fallback_call = has_irreducible_loops; - break; - case HInvokeStaticOrDirect::CodePtrLocation::kCallPCRelative: - // TODO: Implement this type. - break; - default: - fallback_call = false; - break; - } if (fallback_load) { dispatch_info.method_load_kind = HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod; dispatch_info.method_load_data = 0; } - if (fallback_call) { - dispatch_info.code_ptr_location = HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod; - dispatch_info.direct_code_ptr = 0; - } return dispatch_info; } @@ -5341,31 +5295,10 @@ void CodeGeneratorMIPS::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke Location callee_method = temp; // For all kinds except kRecursive, callee will be in temp. HInvokeStaticOrDirect::MethodLoadKind method_load_kind = invoke->GetMethodLoadKind(); HInvokeStaticOrDirect::CodePtrLocation code_ptr_location = invoke->GetCodePtrLocation(); - bool isR6 = isa_features_.IsR6(); - // kDirectAddressWithFixup and kCallDirectWithFixup have no extra input on R6 because - // R6 has PC-relative addressing. - bool has_extra_input = invoke->HasPcRelativeDexCache() || - (!isR6 && - ((method_load_kind == HInvokeStaticOrDirect::MethodLoadKind::kDirectAddressWithFixup) || - (code_ptr_location == HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup))); - Register base_reg = has_extra_input + Register base_reg = invoke->HasPcRelativeDexCache() ? GetInvokeStaticOrDirectExtraParameter(invoke, temp.AsRegister<Register>()) : ZERO; - // For better instruction scheduling we load the direct code pointer before the method pointer. - switch (code_ptr_location) { - case HInvokeStaticOrDirect::CodePtrLocation::kCallDirect: - // T9 = invoke->GetDirectCodePtr(); - __ LoadConst32(T9, invoke->GetDirectCodePtr()); - break; - case HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup: - // T9 = code address from literal pool with link-time patch. - __ LoadLiteral(T9, base_reg, DeduplicateMethodCodeLiteral(invoke->GetTargetMethod())); - break; - default: - break; - } - switch (method_load_kind) { case HInvokeStaticOrDirect::MethodLoadKind::kStringInit: { // temp = thread->string_init_entrypoint @@ -5383,11 +5316,6 @@ void CodeGeneratorMIPS::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress: __ LoadConst32(temp.AsRegister<Register>(), invoke->GetMethodAddress()); break; - case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddressWithFixup: - __ LoadLiteral(temp.AsRegister<Register>(), - base_reg, - DeduplicateMethodAddressLiteral(invoke->GetTargetMethod())); - break; case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: { HMipsDexCacheArraysBase* base = invoke->InputAt(invoke->GetSpecialInputIndex())->AsMipsDexCacheArraysBase(); @@ -5430,18 +5358,6 @@ void CodeGeneratorMIPS::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke case HInvokeStaticOrDirect::CodePtrLocation::kCallSelf: __ Bal(&frame_entry_label_); break; - case HInvokeStaticOrDirect::CodePtrLocation::kCallDirect: - case HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup: - // T9 prepared above for better instruction scheduling. - // T9() - __ Jalr(T9); - __ NopIfNoReordering(); - break; - case HInvokeStaticOrDirect::CodePtrLocation::kCallPCRelative: - // TODO: Implement this type. - // Currently filtered out by GetSupportedInvokeStaticOrDirectDispatch(). - LOG(FATAL) << "Unsupported"; - UNREACHABLE(); case HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod: // T9 = callee_method->entry_point_from_quick_compiled_code_; __ LoadFromOffset(kLoadWord, @@ -5614,17 +5530,8 @@ void InstructionCodeGeneratorMIPS::VisitLoadClass(HLoadClass* cls) { codegen_->DeduplicateBootImageAddressLiteral(address)); break; } - case HLoadClass::LoadKind::kDexCacheAddress: { - DCHECK_NE(cls->GetAddress(), 0u); - uint32_t address = dchecked_integral_cast<uint32_t>(cls->GetAddress()); - static_assert(sizeof(GcRoot<mirror::Class>) == 4u, "Expected GC root to be 4 bytes."); - DCHECK_ALIGNED(cls->GetAddress(), 4u); - int16_t offset = Low16Bits(address); - uint32_t base_address = address - offset; // This accounts for offset sign extension. - __ Lui(out, High16Bits(base_address)); - // /* GcRoot<mirror::Class> */ out = *(base_address + offset) - GenerateGcRootFieldLoad(cls, out_loc, out, offset); - generate_null_check = !cls->IsInDexCache(); + case HLoadClass::LoadKind::kJitTableAddress: { + LOG(FATAL) << "Unimplemented"; break; } case HLoadClass::LoadKind::kDexCachePcRelative: { @@ -5690,11 +5597,7 @@ void InstructionCodeGeneratorMIPS::VisitClearException(HClearException* clear AT } void LocationsBuilderMIPS::VisitLoadString(HLoadString* load) { - LocationSummary::CallKind call_kind = (load->NeedsEnvironment() || kEmitCompilerReadBarrier) - ? ((load->GetLoadKind() == HLoadString::LoadKind::kDexCacheViaMethod) - ? LocationSummary::kCallOnMainOnly - : LocationSummary::kCallOnSlowPath) - : LocationSummary::kNoCall; + LocationSummary::CallKind call_kind = CodeGenerator::GetLoadStringCallKind(load); LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(load, call_kind); HLoadString::LoadKind load_kind = load->GetLoadKind(); switch (load_kind) { @@ -5744,14 +5647,12 @@ void InstructionCodeGeneratorMIPS::VisitLoadString(HLoadString* load) { switch (load_kind) { case HLoadString::LoadKind::kBootImageLinkTimeAddress: - DCHECK(!kEmitCompilerReadBarrier); __ LoadLiteral(out, base_or_current_method_reg, codegen_->DeduplicateBootImageStringLiteral(load->GetDexFile(), load->GetStringIndex())); return; // No dex cache slow path. case HLoadString::LoadKind::kBootImageLinkTimePcRelative: { - DCHECK(!kEmitCompilerReadBarrier); DCHECK(codegen_->GetCompilerOptions().IsBootImage()); CodeGeneratorMIPS::PcRelativePatchInfo* info = codegen_->NewPcRelativeStringPatch(load->GetDexFile(), load->GetStringIndex().index_); @@ -5759,7 +5660,6 @@ void InstructionCodeGeneratorMIPS::VisitLoadString(HLoadString* load) { return; // No dex cache slow path. } case HLoadString::LoadKind::kBootImageAddress: { - DCHECK(!kEmitCompilerReadBarrier); DCHECK_NE(load->GetAddress(), 0u); uint32_t address = dchecked_integral_cast<uint32_t>(load->GetAddress()); __ LoadLiteral(out, @@ -6285,7 +6185,10 @@ void LocationsBuilderMIPS::VisitStaticFieldSet(HStaticFieldSet* instruction) { } void InstructionCodeGeneratorMIPS::VisitStaticFieldSet(HStaticFieldSet* instruction) { - HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetDexPc()); + HandleFieldSet(instruction, + instruction->GetFieldInfo(), + instruction->GetDexPc(), + instruction->GetValueCanBeNull()); } void LocationsBuilderMIPS::VisitUnresolvedInstanceFieldGet( diff --git a/compiler/optimizing/code_generator_mips.h b/compiler/optimizing/code_generator_mips.h index f03f29c5d4..7b0812cb7b 100644 --- a/compiler/optimizing/code_generator_mips.h +++ b/compiler/optimizing/code_generator_mips.h @@ -236,7 +236,10 @@ class InstructionCodeGeneratorMIPS : public InstructionCodeGenerator { void HandleBinaryOp(HBinaryOperation* operation); void HandleCondition(HCondition* instruction); void HandleShift(HBinaryOperation* operation); - void HandleFieldSet(HInstruction* instruction, const FieldInfo& field_info, uint32_t dex_pc); + void HandleFieldSet(HInstruction* instruction, + const FieldInfo& field_info, + uint32_t dex_pc, + bool value_can_be_null); void HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info, uint32_t dex_pc); // Generate a GC root reference load: // @@ -350,7 +353,7 @@ class CodeGeneratorMIPS : public CodeGenerator { // Emit linker patches. void EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) OVERRIDE; - void MarkGCCard(Register object, Register value); + void MarkGCCard(Register object, Register value, bool value_can_be_null); // Register allocation. @@ -474,8 +477,6 @@ class CodeGeneratorMIPS : public CodeGenerator { Literal* DeduplicateUint32Literal(uint32_t value, Uint32ToLiteralMap* map); Literal* DeduplicateMethodLiteral(MethodReference target_method, MethodToLiteralMap* map); - Literal* DeduplicateMethodAddressLiteral(MethodReference target_method); - Literal* DeduplicateMethodCodeLiteral(MethodReference target_method); PcRelativePatchInfo* NewPcRelativePatch(const DexFile& dex_file, uint32_t offset_or_index, ArenaDeque<PcRelativePatchInfo>* patches); @@ -495,9 +496,6 @@ class CodeGeneratorMIPS : public CodeGenerator { // Deduplication map for 32-bit literals, used for non-patchable boot image addresses. Uint32ToLiteralMap uint32_literals_; - // Method patch info, map MethodReference to a literal for method address and method code. - MethodToLiteralMap method_patches_; - MethodToLiteralMap call_patches_; // PC-relative patch info for each HMipsDexCacheArraysBase. ArenaDeque<PcRelativePatchInfo> pc_relative_dex_cache_patches_; // Deduplication map for boot string literals for kBootImageLinkTimeAddress. diff --git a/compiler/optimizing/code_generator_mips64.cc b/compiler/optimizing/code_generator_mips64.cc index b1f9b1db53..5cf3c246cf 100644 --- a/compiler/optimizing/code_generator_mips64.cc +++ b/compiler/optimizing/code_generator_mips64.cc @@ -18,6 +18,7 @@ #include "art_method.h" #include "code_generator_utils.h" +#include "compiled_method.h" #include "entrypoints/quick/quick_entrypoints.h" #include "entrypoints/quick/quick_entrypoints_enum.h" #include "gc/accounting/card_table.h" @@ -90,8 +91,6 @@ Location InvokeDexCallingConventionVisitorMIPS64::GetNextLocation(Primitive::Typ // Space on the stack is reserved for all arguments. stack_index_ += Primitive::Is64BitType(type) ? 2 : 1; - // TODO: review - // TODO: shouldn't we use a whole machine word per argument on the stack? // Implicit 4-byte method pointer (and such) will cause misalignment. @@ -234,6 +233,7 @@ class LoadStringSlowPathMIPS64 : public SlowPathCodeMIPS64 { SaveLiveRegisters(codegen, locations); InvokeRuntimeCallingConvention calling_convention; + HLoadString* load = instruction_->AsLoadString(); const uint32_t string_index = instruction_->AsLoadString()->GetStringIndex().index_; __ LoadConst32(calling_convention.GetRegisterAt(0), string_index); mips64_codegen->InvokeRuntime(kQuickResolveString, @@ -247,6 +247,17 @@ class LoadStringSlowPathMIPS64 : public SlowPathCodeMIPS64 { type); RestoreLiveRegisters(codegen, locations); + + // Store the resolved String to the BSS entry. + // TODO: Change art_quick_resolve_string to kSaveEverything and use a temporary for the + // .bss entry address in the fast path, so that we can avoid another calculation here. + GpuRegister out = locations->Out().AsRegister<GpuRegister>(); + DCHECK_NE(out, AT); + CodeGeneratorMIPS64::PcRelativePatchInfo* info = + mips64_codegen->NewPcRelativeStringPatch(load->GetDexFile(), string_index); + mips64_codegen->EmitPcRelativeAddressPlaceholderHigh(info, AT); + __ Sw(out, AT, /* placeholder */ 0x5678); + __ Bc(GetExitLabel()); } @@ -399,7 +410,20 @@ CodeGeneratorMIPS64::CodeGeneratorMIPS64(HGraph* graph, instruction_visitor_(graph, this), move_resolver_(graph->GetArena(), this), assembler_(graph->GetArena()), - isa_features_(isa_features) { + isa_features_(isa_features), + uint32_literals_(std::less<uint32_t>(), + graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + uint64_literals_(std::less<uint64_t>(), + graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + boot_image_string_patches_(StringReferenceValueComparator(), + graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + pc_relative_string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + boot_image_type_patches_(TypeReferenceValueComparator(), + graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + pc_relative_type_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + boot_image_address_patches_(std::less<uint32_t>(), + graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) { // Save RA (containing the return address) to mimic Quick. AddAllocatedRegister(Location::RegisterLocation(RA)); } @@ -510,8 +534,6 @@ void CodeGeneratorMIPS64::GenerateFrameEntry() { RecordPcInfo(nullptr, 0); } - // TODO: anything related to T9/GP/GOT/PIC/.so's? - if (HasEmptyFrame()) { return; } @@ -562,13 +584,16 @@ void CodeGeneratorMIPS64::GenerateFrameEntry() { "kCurrentMethodStackOffset must fit into int16_t"); __ Sd(kMethodRegisterArgument, SP, kCurrentMethodStackOffset); } + + if (GetGraph()->HasShouldDeoptimizeFlag()) { + // Initialize should_deoptimize flag to 0. + __ StoreToOffset(kStoreWord, ZERO, SP, GetStackOffsetOfShouldDeoptimizeFlag()); + } } void CodeGeneratorMIPS64::GenerateFrameExit() { __ cfi().RememberState(); - // TODO: anything related to T9/GP/GOT/PIC/.so's? - if (!HasEmptyFrame()) { // Deallocate the rest of the frame. @@ -878,6 +903,136 @@ void CodeGeneratorMIPS64::MarkGCCard(GpuRegister object, } } +template <LinkerPatch (*Factory)(size_t, const DexFile*, uint32_t, uint32_t)> +inline void CodeGeneratorMIPS64::EmitPcRelativeLinkerPatches( + const ArenaDeque<PcRelativePatchInfo>& infos, + ArenaVector<LinkerPatch>* linker_patches) { + for (const PcRelativePatchInfo& info : infos) { + const DexFile& dex_file = info.target_dex_file; + size_t offset_or_index = info.offset_or_index; + DCHECK(info.pc_rel_label.IsBound()); + uint32_t pc_rel_offset = __ GetLabelLocation(&info.pc_rel_label); + linker_patches->push_back(Factory(pc_rel_offset, &dex_file, pc_rel_offset, offset_or_index)); + } +} + +void CodeGeneratorMIPS64::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) { + DCHECK(linker_patches->empty()); + size_t size = + pc_relative_dex_cache_patches_.size() + + pc_relative_string_patches_.size() + + pc_relative_type_patches_.size() + + boot_image_string_patches_.size() + + boot_image_type_patches_.size() + + boot_image_address_patches_.size(); + linker_patches->reserve(size); + EmitPcRelativeLinkerPatches<LinkerPatch::DexCacheArrayPatch>(pc_relative_dex_cache_patches_, + linker_patches); + if (!GetCompilerOptions().IsBootImage()) { + EmitPcRelativeLinkerPatches<LinkerPatch::StringBssEntryPatch>(pc_relative_string_patches_, + linker_patches); + } else { + EmitPcRelativeLinkerPatches<LinkerPatch::RelativeStringPatch>(pc_relative_string_patches_, + linker_patches); + } + EmitPcRelativeLinkerPatches<LinkerPatch::RelativeTypePatch>(pc_relative_type_patches_, + linker_patches); + for (const auto& entry : boot_image_string_patches_) { + const StringReference& target_string = entry.first; + Literal* literal = entry.second; + DCHECK(literal->GetLabel()->IsBound()); + uint32_t literal_offset = __ GetLabelLocation(literal->GetLabel()); + linker_patches->push_back(LinkerPatch::StringPatch(literal_offset, + target_string.dex_file, + target_string.string_index.index_)); + } + for (const auto& entry : boot_image_type_patches_) { + const TypeReference& target_type = entry.first; + Literal* literal = entry.second; + DCHECK(literal->GetLabel()->IsBound()); + uint32_t literal_offset = __ GetLabelLocation(literal->GetLabel()); + linker_patches->push_back(LinkerPatch::TypePatch(literal_offset, + target_type.dex_file, + target_type.type_index.index_)); + } + for (const auto& entry : boot_image_address_patches_) { + DCHECK(GetCompilerOptions().GetIncludePatchInformation()); + Literal* literal = entry.second; + DCHECK(literal->GetLabel()->IsBound()); + uint32_t literal_offset = __ GetLabelLocation(literal->GetLabel()); + linker_patches->push_back(LinkerPatch::RecordPosition(literal_offset)); + } +} + +CodeGeneratorMIPS64::PcRelativePatchInfo* CodeGeneratorMIPS64::NewPcRelativeStringPatch( + const DexFile& dex_file, uint32_t string_index) { + return NewPcRelativePatch(dex_file, string_index, &pc_relative_string_patches_); +} + +CodeGeneratorMIPS64::PcRelativePatchInfo* CodeGeneratorMIPS64::NewPcRelativeTypePatch( + const DexFile& dex_file, dex::TypeIndex type_index) { + return NewPcRelativePatch(dex_file, type_index.index_, &pc_relative_type_patches_); +} + +CodeGeneratorMIPS64::PcRelativePatchInfo* CodeGeneratorMIPS64::NewPcRelativeDexCacheArrayPatch( + const DexFile& dex_file, uint32_t element_offset) { + return NewPcRelativePatch(dex_file, element_offset, &pc_relative_dex_cache_patches_); +} + +CodeGeneratorMIPS64::PcRelativePatchInfo* CodeGeneratorMIPS64::NewPcRelativePatch( + const DexFile& dex_file, uint32_t offset_or_index, ArenaDeque<PcRelativePatchInfo>* patches) { + patches->emplace_back(dex_file, offset_or_index); + return &patches->back(); +} + +Literal* CodeGeneratorMIPS64::DeduplicateUint32Literal(uint32_t value, Uint32ToLiteralMap* map) { + return map->GetOrCreate( + value, + [this, value]() { return __ NewLiteral<uint32_t>(value); }); +} + +Literal* CodeGeneratorMIPS64::DeduplicateUint64Literal(uint64_t value) { + return uint64_literals_.GetOrCreate( + value, + [this, value]() { return __ NewLiteral<uint64_t>(value); }); +} + +Literal* CodeGeneratorMIPS64::DeduplicateMethodLiteral(MethodReference target_method, + MethodToLiteralMap* map) { + return map->GetOrCreate( + target_method, + [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); +} + +Literal* CodeGeneratorMIPS64::DeduplicateBootImageStringLiteral(const DexFile& dex_file, + dex::StringIndex string_index) { + return boot_image_string_patches_.GetOrCreate( + StringReference(&dex_file, string_index), + [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); +} + +Literal* CodeGeneratorMIPS64::DeduplicateBootImageTypeLiteral(const DexFile& dex_file, + dex::TypeIndex type_index) { + return boot_image_type_patches_.GetOrCreate( + TypeReference(&dex_file, type_index), + [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); +} + +Literal* CodeGeneratorMIPS64::DeduplicateBootImageAddressLiteral(uint64_t address) { + bool needs_patch = GetCompilerOptions().GetIncludePatchInformation(); + Uint32ToLiteralMap* map = needs_patch ? &boot_image_address_patches_ : &uint32_literals_; + return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address), map); +} + +void CodeGeneratorMIPS64::EmitPcRelativeAddressPlaceholderHigh(PcRelativePatchInfo* info, + GpuRegister out) { + __ Bind(&info->pc_rel_label); + // Add the high half of a 32-bit offset to PC. + __ Auipc(out, /* placeholder */ 0x1234); + // The immediately following instruction will add the sign-extended low half of the 32-bit + // offset to `out` (e.g. ld, jialc, daddiu). +} + void CodeGeneratorMIPS64::SetupBlockedRegisters() const { // ZERO, K0, K1, GP, SP, RA are always reserved and can't be allocated. blocked_core_registers_[ZERO] = true; @@ -901,8 +1056,6 @@ void CodeGeneratorMIPS64::SetupBlockedRegisters() const { // Reserve T9 for function calls blocked_core_registers_[T9] = true; - // TODO: review; anything else? - if (GetGraph()->IsDebuggable()) { // Stubs do not save callee-save floating point registers. If the graph // is debuggable, we need to deal with these registers differently. For @@ -946,7 +1099,6 @@ void CodeGeneratorMIPS64::InvokeRuntime(QuickEntrypointEnum entrypoint, uint32_t dex_pc, SlowPathCode* slow_path) { ValidateInvokeRuntime(entrypoint, instruction, slow_path); - // TODO: anything related to T9/GP/GOT/PIC/.so's? __ LoadFromOffset(kLoadDoubleword, T9, TR, @@ -1777,9 +1929,6 @@ void InstructionCodeGeneratorMIPS64::HandleCondition(HCondition* instruction) { Primitive::Type type = instruction->InputAt(0)->GetType(); LocationSummary* locations = instruction->GetLocations(); - GpuRegister dst = locations->Out().AsRegister<GpuRegister>(); - Mips64Label true_label; - switch (type) { default: // Integer case. @@ -1788,29 +1937,11 @@ void InstructionCodeGeneratorMIPS64::HandleCondition(HCondition* instruction) { case Primitive::kPrimLong: GenerateIntLongCompare(instruction->GetCondition(), /* is64bit */ true, locations); return; - case Primitive::kPrimFloat: case Primitive::kPrimDouble: - // TODO: don't use branches. - GenerateFpCompareAndBranch(instruction->GetCondition(), - instruction->IsGtBias(), - type, - locations, - &true_label); - break; + GenerateFpCompare(instruction->GetCondition(), instruction->IsGtBias(), type, locations); + return; } - - // Convert the branches into the result. - Mips64Label done; - - // False case: result = 0. - __ LoadConst32(dst, 0); - __ Bc(&done); - - // True case: result = 1. - __ Bind(&true_label); - __ LoadConst32(dst, 1); - __ Bind(&done); } void InstructionCodeGeneratorMIPS64::DivRemOneOrMinusOne(HBinaryOperation* instruction) { @@ -2225,19 +2356,40 @@ void InstructionCodeGeneratorMIPS64::GenerateIntLongCompare(IfCondition cond, switch (cond) { case kCondEQ: case kCondNE: - if (use_imm && IsUint<16>(rhs_imm)) { - __ Xori(dst, lhs, rhs_imm); - } else { - if (use_imm) { - rhs_reg = TMP; - __ LoadConst64(rhs_reg, rhs_imm); + if (use_imm && IsInt<16>(-rhs_imm)) { + if (rhs_imm == 0) { + if (cond == kCondEQ) { + __ Sltiu(dst, lhs, 1); + } else { + __ Sltu(dst, ZERO, lhs); + } + } else { + if (is64bit) { + __ Daddiu(dst, lhs, -rhs_imm); + } else { + __ Addiu(dst, lhs, -rhs_imm); + } + if (cond == kCondEQ) { + __ Sltiu(dst, dst, 1); + } else { + __ Sltu(dst, ZERO, dst); + } } - __ Xor(dst, lhs, rhs_reg); - } - if (cond == kCondEQ) { - __ Sltiu(dst, dst, 1); } else { - __ Sltu(dst, ZERO, dst); + if (use_imm && IsUint<16>(rhs_imm)) { + __ Xori(dst, lhs, rhs_imm); + } else { + if (use_imm) { + rhs_reg = TMP; + __ LoadConst64(rhs_reg, rhs_imm); + } + __ Xor(dst, lhs, rhs_reg); + } + if (cond == kCondEQ) { + __ Sltiu(dst, dst, 1); + } else { + __ Sltu(dst, ZERO, dst); + } } break; @@ -2424,6 +2576,121 @@ void InstructionCodeGeneratorMIPS64::GenerateIntLongCompareAndBranch(IfCondition } } +void InstructionCodeGeneratorMIPS64::GenerateFpCompare(IfCondition cond, + bool gt_bias, + Primitive::Type type, + LocationSummary* locations) { + GpuRegister dst = locations->Out().AsRegister<GpuRegister>(); + FpuRegister lhs = locations->InAt(0).AsFpuRegister<FpuRegister>(); + FpuRegister rhs = locations->InAt(1).AsFpuRegister<FpuRegister>(); + if (type == Primitive::kPrimFloat) { + switch (cond) { + case kCondEQ: + __ CmpEqS(FTMP, lhs, rhs); + __ Mfc1(dst, FTMP); + __ Andi(dst, dst, 1); + break; + case kCondNE: + __ CmpEqS(FTMP, lhs, rhs); + __ Mfc1(dst, FTMP); + __ Addiu(dst, dst, 1); + break; + case kCondLT: + if (gt_bias) { + __ CmpLtS(FTMP, lhs, rhs); + } else { + __ CmpUltS(FTMP, lhs, rhs); + } + __ Mfc1(dst, FTMP); + __ Andi(dst, dst, 1); + break; + case kCondLE: + if (gt_bias) { + __ CmpLeS(FTMP, lhs, rhs); + } else { + __ CmpUleS(FTMP, lhs, rhs); + } + __ Mfc1(dst, FTMP); + __ Andi(dst, dst, 1); + break; + case kCondGT: + if (gt_bias) { + __ CmpUltS(FTMP, rhs, lhs); + } else { + __ CmpLtS(FTMP, rhs, lhs); + } + __ Mfc1(dst, FTMP); + __ Andi(dst, dst, 1); + break; + case kCondGE: + if (gt_bias) { + __ CmpUleS(FTMP, rhs, lhs); + } else { + __ CmpLeS(FTMP, rhs, lhs); + } + __ Mfc1(dst, FTMP); + __ Andi(dst, dst, 1); + break; + default: + LOG(FATAL) << "Unexpected non-floating-point condition " << cond; + UNREACHABLE(); + } + } else { + DCHECK_EQ(type, Primitive::kPrimDouble); + switch (cond) { + case kCondEQ: + __ CmpEqD(FTMP, lhs, rhs); + __ Mfc1(dst, FTMP); + __ Andi(dst, dst, 1); + break; + case kCondNE: + __ CmpEqD(FTMP, lhs, rhs); + __ Mfc1(dst, FTMP); + __ Addiu(dst, dst, 1); + break; + case kCondLT: + if (gt_bias) { + __ CmpLtD(FTMP, lhs, rhs); + } else { + __ CmpUltD(FTMP, lhs, rhs); + } + __ Mfc1(dst, FTMP); + __ Andi(dst, dst, 1); + break; + case kCondLE: + if (gt_bias) { + __ CmpLeD(FTMP, lhs, rhs); + } else { + __ CmpUleD(FTMP, lhs, rhs); + } + __ Mfc1(dst, FTMP); + __ Andi(dst, dst, 1); + break; + case kCondGT: + if (gt_bias) { + __ CmpUltD(FTMP, rhs, lhs); + } else { + __ CmpLtD(FTMP, rhs, lhs); + } + __ Mfc1(dst, FTMP); + __ Andi(dst, dst, 1); + break; + case kCondGE: + if (gt_bias) { + __ CmpUleD(FTMP, rhs, lhs); + } else { + __ CmpLeD(FTMP, rhs, lhs); + } + __ Mfc1(dst, FTMP); + __ Andi(dst, dst, 1); + break; + default: + LOG(FATAL) << "Unexpected non-floating-point condition " << cond; + UNREACHABLE(); + } + } +} + void InstructionCodeGeneratorMIPS64::GenerateFpCompareAndBranch(IfCondition cond, bool gt_bias, Primitive::Type type, @@ -2636,14 +2903,17 @@ void InstructionCodeGeneratorMIPS64::VisitDeoptimize(HDeoptimize* deoptimize) { /* false_target */ nullptr); } -void LocationsBuilderMIPS64::VisitShouldDeoptimizeFlag( - HShouldDeoptimizeFlag* flag ATTRIBUTE_UNUSED) { - // TODO: to be implemented. +void LocationsBuilderMIPS64::VisitShouldDeoptimizeFlag(HShouldDeoptimizeFlag* flag) { + LocationSummary* locations = new (GetGraph()->GetArena()) + LocationSummary(flag, LocationSummary::kNoCall); + locations->SetOut(Location::RequiresRegister()); } -void InstructionCodeGeneratorMIPS64::VisitShouldDeoptimizeFlag( - HShouldDeoptimizeFlag* flag ATTRIBUTE_UNUSED) { - // TODO: to be implemented. +void InstructionCodeGeneratorMIPS64::VisitShouldDeoptimizeFlag(HShouldDeoptimizeFlag* flag) { + __ LoadFromOffset(kLoadWord, + flag->GetLocations()->Out().AsRegister<GpuRegister>(), + SP, + codegen_->GetStackOffsetOfShouldDeoptimizeFlag()); } void LocationsBuilderMIPS64::VisitSelect(HSelect* select) { @@ -2820,6 +3090,31 @@ void InstructionCodeGeneratorMIPS64::VisitInstanceFieldSet(HInstanceFieldSet* in HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull()); } +void InstructionCodeGeneratorMIPS64::GenerateGcRootFieldLoad( + HInstruction* instruction ATTRIBUTE_UNUSED, + Location root, + GpuRegister obj, + uint32_t offset) { + // When handling HLoadClass::LoadKind::kDexCachePcRelative, the caller calls + // EmitPcRelativeAddressPlaceholderHigh() and then GenerateGcRootFieldLoad(). + // The relative patcher expects the two methods to emit the following patchable + // sequence of instructions in this case: + // auipc reg1, 0x1234 // 0x1234 is a placeholder for offset_high. + // lwu reg2, 0x5678(reg1) // 0x5678 is a placeholder for offset_low. + // TODO: Adjust GenerateGcRootFieldLoad() and its caller when this method is + // extended (e.g. for read barriers) so as not to break the relative patcher. + GpuRegister root_reg = root.AsRegister<GpuRegister>(); + if (kEmitCompilerReadBarrier) { + UNIMPLEMENTED(FATAL) << "for read barrier"; + } else { + // Plain GC root load with no read barrier. + // /* GcRoot<mirror::Object> */ root = *(obj + offset) + __ LoadFromOffset(kLoadUnsignedWord, root_reg, obj, offset); + // Note that GC roots are not affected by heap poisoning, thus we + // do not have to unpoison `root_reg` here. + } +} + void LocationsBuilderMIPS64::VisitInstanceOf(HInstanceOf* instruction) { LocationSummary::CallKind call_kind = instruction->IsExactCheck() ? LocationSummary::kNoCall : LocationSummary::kCallOnSlowPath; @@ -2971,54 +3266,85 @@ static bool TryGenerateIntrinsicCode(HInvoke* invoke, CodeGeneratorMIPS64* codeg } HLoadString::LoadKind CodeGeneratorMIPS64::GetSupportedLoadStringKind( - HLoadString::LoadKind desired_string_load_kind ATTRIBUTE_UNUSED) { - // TODO: Implement other kinds. - return HLoadString::LoadKind::kDexCacheViaMethod; + HLoadString::LoadKind desired_string_load_kind) { + if (kEmitCompilerReadBarrier) { + UNIMPLEMENTED(FATAL) << "for read barrier"; + } + bool fallback_load = false; + switch (desired_string_load_kind) { + case HLoadString::LoadKind::kBootImageLinkTimeAddress: + DCHECK(!GetCompilerOptions().GetCompilePic()); + break; + case HLoadString::LoadKind::kBootImageLinkTimePcRelative: + DCHECK(GetCompilerOptions().GetCompilePic()); + break; + case HLoadString::LoadKind::kBootImageAddress: + break; + case HLoadString::LoadKind::kBssEntry: + DCHECK(!Runtime::Current()->UseJitCompilation()); + break; + case HLoadString::LoadKind::kDexCacheViaMethod: + break; + case HLoadString::LoadKind::kJitTableAddress: + DCHECK(Runtime::Current()->UseJitCompilation()); + // TODO: implement. + fallback_load = true; + break; + } + if (fallback_load) { + desired_string_load_kind = HLoadString::LoadKind::kDexCacheViaMethod; + } + return desired_string_load_kind; } HLoadClass::LoadKind CodeGeneratorMIPS64::GetSupportedLoadClassKind( HLoadClass::LoadKind desired_class_load_kind) { - DCHECK_NE(desired_class_load_kind, HLoadClass::LoadKind::kReferrersClass); - // TODO: Implement other kinds. - return HLoadClass::LoadKind::kDexCacheViaMethod; + if (kEmitCompilerReadBarrier) { + UNIMPLEMENTED(FATAL) << "for read barrier"; + } + bool fallback_load = false; + switch (desired_class_load_kind) { + case HLoadClass::LoadKind::kReferrersClass: + break; + case HLoadClass::LoadKind::kBootImageLinkTimeAddress: + DCHECK(!GetCompilerOptions().GetCompilePic()); + break; + case HLoadClass::LoadKind::kBootImageLinkTimePcRelative: + DCHECK(GetCompilerOptions().GetCompilePic()); + break; + case HLoadClass::LoadKind::kBootImageAddress: + break; + case HLoadClass::LoadKind::kJitTableAddress: + DCHECK(Runtime::Current()->UseJitCompilation()); + // TODO: implement. + fallback_load = true; + break; + case HLoadClass::LoadKind::kDexCachePcRelative: + DCHECK(!Runtime::Current()->UseJitCompilation()); + break; + case HLoadClass::LoadKind::kDexCacheViaMethod: + break; + } + if (fallback_load) { + desired_class_load_kind = HLoadClass::LoadKind::kDexCacheViaMethod; + } + return desired_class_load_kind; } HInvokeStaticOrDirect::DispatchInfo CodeGeneratorMIPS64::GetSupportedInvokeStaticOrDirectDispatch( const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info, HInvokeStaticOrDirect* invoke ATTRIBUTE_UNUSED) { - switch (desired_dispatch_info.method_load_kind) { - case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddressWithFixup: - case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: - // TODO: Implement these types. For the moment, we fall back to kDexCacheViaMethod. - return HInvokeStaticOrDirect::DispatchInfo { - HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod, - HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod, - 0u, - 0u - }; - default: - break; - } - switch (desired_dispatch_info.code_ptr_location) { - case HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup: - case HInvokeStaticOrDirect::CodePtrLocation::kCallPCRelative: - // TODO: Implement these types. For the moment, we fall back to kCallArtMethod. - return HInvokeStaticOrDirect::DispatchInfo { - desired_dispatch_info.method_load_kind, - HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod, - desired_dispatch_info.method_load_data, - 0u - }; - default: - return desired_dispatch_info; - } + // On MIPS64 we support all dispatch types. + return desired_dispatch_info; } void CodeGeneratorMIPS64::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) { // All registers are assumed to be correctly set up per the calling convention. - Location callee_method = temp; // For all kinds except kRecursive, callee will be in temp. - switch (invoke->GetMethodLoadKind()) { + HInvokeStaticOrDirect::MethodLoadKind method_load_kind = invoke->GetMethodLoadKind(); + HInvokeStaticOrDirect::CodePtrLocation code_ptr_location = invoke->GetCodePtrLocation(); + + switch (method_load_kind) { case HInvokeStaticOrDirect::MethodLoadKind::kStringInit: { // temp = thread->string_init_entrypoint uint32_t offset = @@ -3033,14 +3359,18 @@ void CodeGeneratorMIPS64::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invo callee_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); break; case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress: - __ LoadConst64(temp.AsRegister<GpuRegister>(), invoke->GetMethodAddress()); + __ LoadLiteral(temp.AsRegister<GpuRegister>(), + kLoadDoubleword, + DeduplicateUint64Literal(invoke->GetMethodAddress())); break; - case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddressWithFixup: - case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: - // TODO: Implement these types. - // Currently filtered out by GetSupportedInvokeStaticOrDirectDispatch(). - LOG(FATAL) << "Unsupported"; - UNREACHABLE(); + case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: { + uint32_t offset = invoke->GetDexCacheArrayOffset(); + CodeGeneratorMIPS64::PcRelativePatchInfo* info = + NewPcRelativeDexCacheArrayPatch(invoke->GetDexFile(), offset); + EmitPcRelativeAddressPlaceholderHigh(info, AT); + __ Ld(temp.AsRegister<GpuRegister>(), AT, /* placeholder */ 0x5678); + break; + } case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: { Location current_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); GpuRegister reg = temp.AsRegister<GpuRegister>(); @@ -3071,23 +3401,10 @@ void CodeGeneratorMIPS64::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invo } } - switch (invoke->GetCodePtrLocation()) { + switch (code_ptr_location) { case HInvokeStaticOrDirect::CodePtrLocation::kCallSelf: - __ Jialc(&frame_entry_label_, T9); - break; - case HInvokeStaticOrDirect::CodePtrLocation::kCallDirect: - // LR = invoke->GetDirectCodePtr(); - __ LoadConst64(T9, invoke->GetDirectCodePtr()); - // LR() - __ Jalr(T9); - __ Nop(); + __ Balc(&frame_entry_label_); break; - case HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup: - case HInvokeStaticOrDirect::CodePtrLocation::kCallPCRelative: - // TODO: Implement these types. - // Currently filtered out by GetSupportedInvokeStaticOrDirectDispatch(). - LOG(FATAL) << "Unsupported"; - UNREACHABLE(); case HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod: // T9 = callee_method->entry_point_from_quick_compiled_code_; __ LoadFromOffset(kLoadDoubleword, @@ -3157,11 +3474,26 @@ void InstructionCodeGeneratorMIPS64::VisitInvokeVirtual(HInvokeVirtual* invoke) } void LocationsBuilderMIPS64::VisitLoadClass(HLoadClass* cls) { - InvokeRuntimeCallingConvention calling_convention; - CodeGenerator::CreateLoadClassLocationSummary( - cls, - Location::RegisterLocation(calling_convention.GetRegisterAt(0)), - calling_convention.GetReturnLocation(cls->GetType())); + if (cls->NeedsAccessCheck()) { + InvokeRuntimeCallingConvention calling_convention; + CodeGenerator::CreateLoadClassLocationSummary( + cls, + Location::RegisterLocation(calling_convention.GetRegisterAt(0)), + calling_convention.GetReturnLocation(Primitive::kPrimNot), + /* code_generator_supports_read_barrier */ false); + return; + } + + LocationSummary::CallKind call_kind = (cls->NeedsEnvironment() || kEmitCompilerReadBarrier) + ? LocationSummary::kCallOnSlowPath + : LocationSummary::kNoCall; + LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(cls, call_kind); + HLoadClass::LoadKind load_kind = cls->GetLoadKind(); + if (load_kind == HLoadClass::LoadKind::kReferrersClass || + load_kind == HLoadClass::LoadKind::kDexCacheViaMethod) { + locations->SetInAt(0, Location::RequiresRegister()); + } + locations->SetOut(Location::RequiresRegister()); } void InstructionCodeGeneratorMIPS64::VisitLoadClass(HLoadClass* cls) { @@ -3173,35 +3505,90 @@ void InstructionCodeGeneratorMIPS64::VisitLoadClass(HLoadClass* cls) { return; } - GpuRegister out = locations->Out().AsRegister<GpuRegister>(); - GpuRegister current_method = locations->InAt(0).AsRegister<GpuRegister>(); - if (cls->IsReferrersClass()) { - DCHECK(!cls->CanCallRuntime()); - DCHECK(!cls->MustGenerateClinitCheck()); - __ LoadFromOffset(kLoadUnsignedWord, out, current_method, - ArtMethod::DeclaringClassOffset().Int32Value()); - } else { - __ LoadFromOffset(kLoadDoubleword, out, current_method, - ArtMethod::DexCacheResolvedTypesOffset(kMips64PointerSize).Int32Value()); - __ LoadFromOffset( - kLoadUnsignedWord, out, out, CodeGenerator::GetCacheOffset(cls->GetTypeIndex().index_)); - // TODO: We will need a read barrier here. - if (!cls->IsInDexCache() || cls->MustGenerateClinitCheck()) { - DCHECK(cls->CanCallRuntime()); - SlowPathCodeMIPS64* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathMIPS64( - cls, - cls, - cls->GetDexPc(), - cls->MustGenerateClinitCheck()); - codegen_->AddSlowPath(slow_path); - if (!cls->IsInDexCache()) { - __ Beqzc(out, slow_path->GetEntryLabel()); - } - if (cls->MustGenerateClinitCheck()) { - GenerateClassInitializationCheck(slow_path, out); - } else { - __ Bind(slow_path->GetExitLabel()); - } + HLoadClass::LoadKind load_kind = cls->GetLoadKind(); + Location out_loc = locations->Out(); + GpuRegister out = out_loc.AsRegister<GpuRegister>(); + GpuRegister current_method_reg = ZERO; + if (load_kind == HLoadClass::LoadKind::kReferrersClass || + load_kind == HLoadClass::LoadKind::kDexCacheViaMethod) { + current_method_reg = locations->InAt(0).AsRegister<GpuRegister>(); + } + + bool generate_null_check = false; + switch (load_kind) { + case HLoadClass::LoadKind::kReferrersClass: + DCHECK(!cls->CanCallRuntime()); + DCHECK(!cls->MustGenerateClinitCheck()); + // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_ + GenerateGcRootFieldLoad(cls, + out_loc, + current_method_reg, + ArtMethod::DeclaringClassOffset().Int32Value()); + break; + case HLoadClass::LoadKind::kBootImageLinkTimeAddress: + DCHECK(!kEmitCompilerReadBarrier); + __ LoadLiteral(out, + kLoadUnsignedWord, + codegen_->DeduplicateBootImageTypeLiteral(cls->GetDexFile(), + cls->GetTypeIndex())); + break; + case HLoadClass::LoadKind::kBootImageLinkTimePcRelative: { + DCHECK(!kEmitCompilerReadBarrier); + CodeGeneratorMIPS64::PcRelativePatchInfo* info = + codegen_->NewPcRelativeTypePatch(cls->GetDexFile(), cls->GetTypeIndex()); + codegen_->EmitPcRelativeAddressPlaceholderHigh(info, AT); + __ Daddiu(out, AT, /* placeholder */ 0x5678); + break; + } + case HLoadClass::LoadKind::kBootImageAddress: { + DCHECK(!kEmitCompilerReadBarrier); + DCHECK_NE(cls->GetAddress(), 0u); + uint32_t address = dchecked_integral_cast<uint32_t>(cls->GetAddress()); + __ LoadLiteral(out, + kLoadUnsignedWord, + codegen_->DeduplicateBootImageAddressLiteral(address)); + break; + } + case HLoadClass::LoadKind::kJitTableAddress: { + LOG(FATAL) << "Unimplemented"; + break; + } + case HLoadClass::LoadKind::kDexCachePcRelative: { + uint32_t element_offset = cls->GetDexCacheElementOffset(); + CodeGeneratorMIPS64::PcRelativePatchInfo* info = + codegen_->NewPcRelativeDexCacheArrayPatch(cls->GetDexFile(), element_offset); + codegen_->EmitPcRelativeAddressPlaceholderHigh(info, AT); + // /* GcRoot<mirror::Class> */ out = *address /* PC-relative */ + GenerateGcRootFieldLoad(cls, out_loc, AT, /* placeholder */ 0x5678); + generate_null_check = !cls->IsInDexCache(); + break; + } + case HLoadClass::LoadKind::kDexCacheViaMethod: { + // /* GcRoot<mirror::Class>[] */ out = + // current_method.ptr_sized_fields_->dex_cache_resolved_types_ + __ LoadFromOffset(kLoadDoubleword, + out, + current_method_reg, + ArtMethod::DexCacheResolvedTypesOffset(kMips64PointerSize).Int32Value()); + // /* GcRoot<mirror::Class> */ out = out[type_index] + size_t offset = CodeGenerator::GetCacheOffset(cls->GetTypeIndex().index_); + GenerateGcRootFieldLoad(cls, out_loc, out, offset); + generate_null_check = !cls->IsInDexCache(); + } + } + + if (generate_null_check || cls->MustGenerateClinitCheck()) { + DCHECK(cls->CanCallRuntime()); + SlowPathCodeMIPS64* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathMIPS64( + cls, cls, cls->GetDexPc(), cls->MustGenerateClinitCheck()); + codegen_->AddSlowPath(slow_path); + if (generate_null_check) { + __ Beqzc(out, slow_path->GetEntryLabel()); + } + if (cls->MustGenerateClinitCheck()) { + GenerateClassInitializationCheck(slow_path, out); + } else { + __ Bind(slow_path->GetExitLabel()); } } } @@ -3230,20 +3617,68 @@ void InstructionCodeGeneratorMIPS64::VisitClearException(HClearException* clear } void LocationsBuilderMIPS64::VisitLoadString(HLoadString* load) { - LocationSummary::CallKind call_kind = load->NeedsEnvironment() - ? LocationSummary::kCallOnSlowPath - : LocationSummary::kNoCall; + HLoadString::LoadKind load_kind = load->GetLoadKind(); + LocationSummary::CallKind call_kind = CodeGenerator::GetLoadStringCallKind(load); LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(load, call_kind); - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister()); + if (load_kind == HLoadString::LoadKind::kDexCacheViaMethod) { + InvokeRuntimeCallingConvention calling_convention; + locations->SetOut(calling_convention.GetReturnLocation(load->GetType())); + } else { + locations->SetOut(Location::RequiresRegister()); + } } void InstructionCodeGeneratorMIPS64::VisitLoadString(HLoadString* load) { + HLoadString::LoadKind load_kind = load->GetLoadKind(); + LocationSummary* locations = load->GetLocations(); + Location out_loc = locations->Out(); + GpuRegister out = out_loc.AsRegister<GpuRegister>(); + + switch (load_kind) { + case HLoadString::LoadKind::kBootImageLinkTimeAddress: + __ LoadLiteral(out, + kLoadUnsignedWord, + codegen_->DeduplicateBootImageStringLiteral(load->GetDexFile(), + load->GetStringIndex())); + return; // No dex cache slow path. + case HLoadString::LoadKind::kBootImageLinkTimePcRelative: { + DCHECK(codegen_->GetCompilerOptions().IsBootImage()); + CodeGeneratorMIPS64::PcRelativePatchInfo* info = + codegen_->NewPcRelativeStringPatch(load->GetDexFile(), load->GetStringIndex().index_); + codegen_->EmitPcRelativeAddressPlaceholderHigh(info, AT); + __ Daddiu(out, AT, /* placeholder */ 0x5678); + return; // No dex cache slow path. + } + case HLoadString::LoadKind::kBootImageAddress: { + DCHECK_NE(load->GetAddress(), 0u); + uint32_t address = dchecked_integral_cast<uint32_t>(load->GetAddress()); + __ LoadLiteral(out, + kLoadUnsignedWord, + codegen_->DeduplicateBootImageAddressLiteral(address)); + return; // No dex cache slow path. + } + case HLoadString::LoadKind::kBssEntry: { + DCHECK(!codegen_->GetCompilerOptions().IsBootImage()); + CodeGeneratorMIPS64::PcRelativePatchInfo* info = + codegen_->NewPcRelativeStringPatch(load->GetDexFile(), load->GetStringIndex().index_); + codegen_->EmitPcRelativeAddressPlaceholderHigh(info, AT); + __ Lwu(out, AT, /* placeholder */ 0x5678); + SlowPathCodeMIPS64* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathMIPS64(load); + codegen_->AddSlowPath(slow_path); + __ Beqzc(out, slow_path->GetEntryLabel()); + __ Bind(slow_path->GetExitLabel()); + return; + } + default: + break; + } + // TODO: Re-add the compiler code to do string dex cache lookup again. - SlowPathCodeMIPS64* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathMIPS64(load); - codegen_->AddSlowPath(slow_path); - __ Bc(slow_path->GetEntryLabel()); - __ Bind(slow_path->GetExitLabel()); + DCHECK(load_kind == HLoadString::LoadKind::kDexCacheViaMethod); + InvokeRuntimeCallingConvention calling_convention; + __ LoadConst32(calling_convention.GetRegisterAt(0), load->GetStringIndex().index_); + codegen_->InvokeRuntime(kQuickResolveString, load, load->GetDexPc()); + CheckEntrypointTypes<kQuickResolveString, void*, uint32_t>(); } void LocationsBuilderMIPS64::VisitLongConstant(HLongConstant* constant) { @@ -3840,9 +4275,12 @@ void InstructionCodeGeneratorMIPS64::VisitTypeConversion(HTypeConversion* conver break; case Primitive::kPrimInt: case Primitive::kPrimLong: - // Sign-extend 32-bit int into bits 32 through 63 for - // int-to-long and long-to-int conversions - __ Sll(dst, src, 0); + // Sign-extend 32-bit int into bits 32 through 63 for int-to-long and long-to-int + // conversions, except when the input and output registers are the same and we are not + // converting longs to shorter types. In these cases, do nothing. + if ((input_type == Primitive::kPrimLong) || (dst != src)) { + __ Sll(dst, src, 0); + } break; default: diff --git a/compiler/optimizing/code_generator_mips64.h b/compiler/optimizing/code_generator_mips64.h index 690eccb7d8..d5811c20e3 100644 --- a/compiler/optimizing/code_generator_mips64.h +++ b/compiler/optimizing/code_generator_mips64.h @@ -22,6 +22,7 @@ #include "nodes.h" #include "parallel_move_resolver.h" #include "utils/mips64/assembler_mips64.h" +#include "utils/type_reference.h" namespace art { namespace mips64 { @@ -227,6 +228,15 @@ class InstructionCodeGeneratorMIPS64 : public InstructionCodeGenerator { const FieldInfo& field_info, bool value_can_be_null); void HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info); + // Generate a GC root reference load: + // + // root <- *(obj + offset) + // + // while honoring read barriers (if any). + void GenerateGcRootFieldLoad(HInstruction* instruction, + Location root, + GpuRegister obj, + uint32_t offset); void GenerateTestAndBranch(HInstruction* instruction, size_t condition_input_index, Mips64Label* true_target, @@ -240,6 +250,10 @@ class InstructionCodeGeneratorMIPS64 : public InstructionCodeGenerator { bool is64bit, LocationSummary* locations, Mips64Label* label); + void GenerateFpCompare(IfCondition cond, + bool gt_bias, + Primitive::Type type, + LocationSummary* locations); void GenerateFpCompareAndBranch(IfCondition cond, bool gt_bias, Primitive::Type type, @@ -279,6 +293,9 @@ class CodeGeneratorMIPS64 : public CodeGenerator { Mips64Assembler* GetAssembler() OVERRIDE { return &assembler_; } const Mips64Assembler& GetAssembler() const OVERRIDE { return assembler_; } + // Emit linker patches. + void EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) OVERRIDE; + void MarkGCCard(GpuRegister object, GpuRegister value, bool value_can_be_null); // Register allocation. @@ -307,6 +324,10 @@ class CodeGeneratorMIPS64 : public CodeGenerator { block_labels_ = CommonInitializeLabels<Mips64Label>(); } + // We prefer aligned loads and stores (less code), so spill and restore registers in slow paths + // at aligned locations. + uint32_t GetPreferredSlotsAlignment() const OVERRIDE { return kMips64DoublewordSize; } + void Finalize(CodeAllocator* allocator) OVERRIDE; // Code generation helpers. @@ -357,7 +378,57 @@ class CodeGeneratorMIPS64 : public CodeGenerator { void GenerateImplicitNullCheck(HNullCheck* instruction) OVERRIDE; void GenerateExplicitNullCheck(HNullCheck* instruction) OVERRIDE; + // The PcRelativePatchInfo is used for PC-relative addressing of dex cache arrays, + // boot image strings and method calls. The only difference is the interpretation of + // the offset_or_index. + struct PcRelativePatchInfo { + PcRelativePatchInfo(const DexFile& dex_file, uint32_t off_or_idx) + : target_dex_file(dex_file), offset_or_index(off_or_idx) { } + PcRelativePatchInfo(PcRelativePatchInfo&& other) = default; + + const DexFile& target_dex_file; + // Either the dex cache array element offset or the string/type/method index. + uint32_t offset_or_index; + // Label for the auipc instruction. + Mips64Label pc_rel_label; + }; + + PcRelativePatchInfo* NewPcRelativeStringPatch(const DexFile& dex_file, uint32_t string_index); + PcRelativePatchInfo* NewPcRelativeTypePatch(const DexFile& dex_file, dex::TypeIndex type_index); + PcRelativePatchInfo* NewPcRelativeDexCacheArrayPatch(const DexFile& dex_file, + uint32_t element_offset); + PcRelativePatchInfo* NewPcRelativeCallPatch(const DexFile& dex_file, + uint32_t method_index); + Literal* DeduplicateBootImageStringLiteral(const DexFile& dex_file, + dex::StringIndex string_index); + Literal* DeduplicateBootImageTypeLiteral(const DexFile& dex_file, dex::TypeIndex type_index); + Literal* DeduplicateBootImageAddressLiteral(uint64_t address); + + void EmitPcRelativeAddressPlaceholderHigh(PcRelativePatchInfo* info, GpuRegister out); + private: + using Uint32ToLiteralMap = ArenaSafeMap<uint32_t, Literal*>; + using Uint64ToLiteralMap = ArenaSafeMap<uint64_t, Literal*>; + using MethodToLiteralMap = ArenaSafeMap<MethodReference, Literal*, MethodReferenceComparator>; + using BootStringToLiteralMap = ArenaSafeMap<StringReference, + Literal*, + StringReferenceValueComparator>; + using BootTypeToLiteralMap = ArenaSafeMap<TypeReference, + Literal*, + TypeReferenceValueComparator>; + + Literal* DeduplicateUint32Literal(uint32_t value, Uint32ToLiteralMap* map); + Literal* DeduplicateUint64Literal(uint64_t value); + Literal* DeduplicateMethodLiteral(MethodReference target_method, MethodToLiteralMap* map); + + PcRelativePatchInfo* NewPcRelativePatch(const DexFile& dex_file, + uint32_t offset_or_index, + ArenaDeque<PcRelativePatchInfo>* patches); + + template <LinkerPatch (*Factory)(size_t, const DexFile*, uint32_t, uint32_t)> + void EmitPcRelativeLinkerPatches(const ArenaDeque<PcRelativePatchInfo>& infos, + ArenaVector<LinkerPatch>* linker_patches); + // Labels for each block that will be compiled. Mips64Label* block_labels_; // Indexed by block id. Mips64Label frame_entry_label_; @@ -367,6 +438,24 @@ class CodeGeneratorMIPS64 : public CodeGenerator { Mips64Assembler assembler_; const Mips64InstructionSetFeatures& isa_features_; + // Deduplication map for 32-bit literals, used for non-patchable boot image addresses. + Uint32ToLiteralMap uint32_literals_; + // Deduplication map for 64-bit literals, used for non-patchable method address or method code + // address. + Uint64ToLiteralMap uint64_literals_; + // PC-relative patch info. + ArenaDeque<PcRelativePatchInfo> pc_relative_dex_cache_patches_; + // Deduplication map for boot string literals for kBootImageLinkTimeAddress. + BootStringToLiteralMap boot_image_string_patches_; + // PC-relative String patch info; type depends on configuration (app .bss or boot image PIC). + ArenaDeque<PcRelativePatchInfo> pc_relative_string_patches_; + // Deduplication map for boot type literals for kBootImageLinkTimeAddress. + BootTypeToLiteralMap boot_image_type_patches_; + // PC-relative type patch info. + ArenaDeque<PcRelativePatchInfo> pc_relative_type_patches_; + // Deduplication map for patchable boot image addresses. + Uint32ToLiteralMap boot_image_address_patches_; + DISALLOW_COPY_AND_ASSIGN(CodeGeneratorMIPS64); }; diff --git a/compiler/optimizing/code_generator_utils.h b/compiler/optimizing/code_generator_utils.h index 7efed8c9ec..a6b41c0588 100644 --- a/compiler/optimizing/code_generator_utils.h +++ b/compiler/optimizing/code_generator_utils.h @@ -18,6 +18,8 @@ #define ART_COMPILER_OPTIMIZING_CODE_GENERATOR_UTILS_H_ #include <cstdint> +#include <cstdlib> +#include <limits> namespace art { @@ -32,6 +34,12 @@ void CalculateMagicAndShiftForDivRem(int64_t divisor, bool is_long, int64_t* mag // that it has been previously visited by the InstructionCodeGenerator. bool IsBooleanValueOrMaterializedCondition(HInstruction* cond_input); +template <typename T> T AbsOrMin(T value) { + return (value == std::numeric_limits<T>::min()) + ? value + : std::abs(value); +} + } // namespace art #endif // ART_COMPILER_OPTIMIZING_CODE_GENERATOR_UTILS_H_ diff --git a/compiler/optimizing/code_generator_x86.cc b/compiler/optimizing/code_generator_x86.cc index d6e92ccb81..0abe85540c 100644 --- a/compiler/optimizing/code_generator_x86.cc +++ b/compiler/optimizing/code_generator_x86.cc @@ -1006,13 +1006,12 @@ CodeGeneratorX86::CodeGeneratorX86(HGraph* graph, move_resolver_(graph->GetArena(), this), assembler_(graph->GetArena()), isa_features_(isa_features), - method_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), - relative_call_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), simple_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), type_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), jit_string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + jit_class_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), constant_area_start_(-1), fixups_to_jump_tables_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), method_address_offset_(-1) { @@ -4453,20 +4452,7 @@ HInvokeStaticOrDirect::DispatchInfo CodeGeneratorX86::GetSupportedInvokeStaticOr HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative)) { dispatch_info.method_load_kind = HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod; } - switch (dispatch_info.code_ptr_location) { - case HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup: - case HInvokeStaticOrDirect::CodePtrLocation::kCallDirect: - // For direct code, we actually prefer to call via the code pointer from ArtMethod*. - // (Though the direct CALL ptr16:32 is available for consideration). - return HInvokeStaticOrDirect::DispatchInfo { - dispatch_info.method_load_kind, - HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod, - dispatch_info.method_load_data, - 0u - }; - default: - return dispatch_info; - } + return dispatch_info; } Register CodeGeneratorX86::GetInvokeStaticOrDirectExtraParameter(HInvokeStaticOrDirect* invoke, @@ -4513,12 +4499,6 @@ Location CodeGeneratorX86::GenerateCalleeMethodStaticOrDirectCall(HInvokeStaticO case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress: __ movl(temp.AsRegister<Register>(), Immediate(invoke->GetMethodAddress())); break; - case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddressWithFixup: - __ movl(temp.AsRegister<Register>(), Immediate(/* placeholder */ 0)); - method_patches_.emplace_back(*invoke->GetTargetMethod().dex_file, - invoke->GetTargetMethod().dex_method_index); - __ Bind(&method_patches_.back().label); // Bind the label at the end of the "movl" insn. - break; case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: { Register base_reg = GetInvokeStaticOrDirectExtraParameter(invoke, temp.AsRegister<Register>()); @@ -4560,19 +4540,6 @@ void CodeGeneratorX86::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, case HInvokeStaticOrDirect::CodePtrLocation::kCallSelf: __ call(GetFrameEntryLabel()); break; - case HInvokeStaticOrDirect::CodePtrLocation::kCallPCRelative: { - relative_call_patches_.emplace_back(*invoke->GetTargetMethod().dex_file, - invoke->GetTargetMethod().dex_method_index); - Label* label = &relative_call_patches_.back().label; - __ call(label); // Bind to the patch label, override at link time. - __ Bind(label); // Bind the label at the end of the "call" insn. - break; - } - case HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup: - case HInvokeStaticOrDirect::CodePtrLocation::kCallDirect: - // Filtered out by GetSupportedInvokeStaticOrDirectDispatch(). - LOG(FATAL) << "Unsupported"; - UNREACHABLE(); case HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod: // (callee_method + offset_of_quick_compiled_code)() __ call(Address(callee_method.AsRegister<Register>(), @@ -4663,22 +4630,11 @@ inline void CodeGeneratorX86::EmitPcRelativeLinkerPatches( void CodeGeneratorX86::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) { DCHECK(linker_patches->empty()); size_t size = - method_patches_.size() + - relative_call_patches_.size() + pc_relative_dex_cache_patches_.size() + simple_patches_.size() + string_patches_.size() + type_patches_.size(); linker_patches->reserve(size); - for (const PatchInfo<Label>& info : method_patches_) { - uint32_t literal_offset = info.label.Position() - kLabelPositionToLiteralOffsetAdjustment; - linker_patches->push_back(LinkerPatch::MethodPatch(literal_offset, &info.dex_file, info.index)); - } - for (const PatchInfo<Label>& info : relative_call_patches_) { - uint32_t literal_offset = info.label.Position() - kLabelPositionToLiteralOffsetAdjustment; - linker_patches->push_back( - LinkerPatch::RelativeCodePatch(literal_offset, &info.dex_file, info.index)); - } EmitPcRelativeLinkerPatches<LinkerPatch::DexCacheArrayPatch>(pc_relative_dex_cache_patches_, linker_patches); for (const Label& label : simple_patches_) { @@ -6034,7 +5990,7 @@ HLoadClass::LoadKind CodeGeneratorX86::GetSupportedLoadClassKind( break; case HLoadClass::LoadKind::kBootImageAddress: break; - case HLoadClass::LoadKind::kDexCacheAddress: + case HLoadClass::LoadKind::kJitTableAddress: DCHECK(Runtime::Current()->UseJitCompilation()); break; case HLoadClass::LoadKind::kDexCacheViaMethod: @@ -6073,6 +6029,16 @@ void LocationsBuilderX86::VisitLoadClass(HLoadClass* cls) { locations->SetOut(Location::RequiresRegister()); } +Label* CodeGeneratorX86::NewJitRootClassPatch(const DexFile& dex_file, + dex::TypeIndex dex_index, + uint64_t address) { + jit_class_roots_.Overwrite(TypeReference(&dex_file, dex_index), address); + // Add a patch entry and return the label. + jit_class_patches_.emplace_back(dex_file, dex_index.index_); + PatchInfo<Label>* info = &jit_class_patches_.back(); + return &info->label; +} + void InstructionCodeGeneratorX86::VisitLoadClass(HLoadClass* cls) { LocationSummary* locations = cls->GetLocations(); if (cls->NeedsAccessCheck()) { @@ -6124,16 +6090,12 @@ void InstructionCodeGeneratorX86::VisitLoadClass(HLoadClass* cls) { codegen_->RecordSimplePatch(); break; } - case HLoadClass::LoadKind::kDexCacheAddress: { - DCHECK_NE(cls->GetAddress(), 0u); - uint32_t address = dchecked_integral_cast<uint32_t>(cls->GetAddress()); + case HLoadClass::LoadKind::kJitTableAddress: { + Address address = Address::Absolute(CodeGeneratorX86::kDummy32BitOffset); + Label* fixup_label = codegen_->NewJitRootClassPatch( + cls->GetDexFile(), cls->GetTypeIndex(), cls->GetAddress()); // /* GcRoot<mirror::Class> */ out = *address - GenerateGcRootFieldLoad(cls, - out_loc, - Address::Absolute(address), - /* fixup_label */ nullptr, - read_barrier_option); - generate_null_check = !cls->IsInDexCache(); + GenerateGcRootFieldLoad(cls, out_loc, address, fixup_label, kCompilerReadBarrierOption); break; } case HLoadClass::LoadKind::kDexCachePcRelative: { @@ -7770,18 +7732,31 @@ void CodeGeneratorX86::MoveFromReturnRegister(Location target, Primitive::Type t } } +void CodeGeneratorX86::PatchJitRootUse(uint8_t* code, + const uint8_t* roots_data, + const PatchInfo<Label>& info, + uint64_t index_in_table) const { + uint32_t code_offset = info.label.Position() - kLabelPositionToLiteralOffsetAdjustment; + uintptr_t address = + reinterpret_cast<uintptr_t>(roots_data) + index_in_table * sizeof(GcRoot<mirror::Object>); + typedef __attribute__((__aligned__(1))) uint32_t unaligned_uint32_t; + reinterpret_cast<unaligned_uint32_t*>(code + code_offset)[0] = + dchecked_integral_cast<uint32_t>(address); +} + void CodeGeneratorX86::EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) { for (const PatchInfo<Label>& info : jit_string_patches_) { - const auto& it = jit_string_roots_.find(StringReference(&info.dex_file, - dex::StringIndex(info.index))); + const auto& it = jit_string_roots_.find( + StringReference(&info.dex_file, dex::StringIndex(info.index))); DCHECK(it != jit_string_roots_.end()); - size_t index_in_table = it->second; - uint32_t code_offset = info.label.Position() - kLabelPositionToLiteralOffsetAdjustment; - uintptr_t address = - reinterpret_cast<uintptr_t>(roots_data) + index_in_table * sizeof(GcRoot<mirror::Object>); - typedef __attribute__((__aligned__(1))) uint32_t unaligned_uint32_t; - reinterpret_cast<unaligned_uint32_t*>(code + code_offset)[0] = - dchecked_integral_cast<uint32_t>(address); + PatchJitRootUse(code, roots_data, info, it->second); + } + + for (const PatchInfo<Label>& info : jit_class_patches_) { + const auto& it = jit_class_roots_.find( + TypeReference(&info.dex_file, dex::TypeIndex(info.index))); + DCHECK(it != jit_class_roots_.end()); + PatchJitRootUse(code, roots_data, info, it->second); } } diff --git a/compiler/optimizing/code_generator_x86.h b/compiler/optimizing/code_generator_x86.h index 2ae3670bed..1af685087c 100644 --- a/compiler/optimizing/code_generator_x86.h +++ b/compiler/optimizing/code_generator_x86.h @@ -416,12 +416,17 @@ class CodeGeneratorX86 : public CodeGenerator { Label* NewStringBssEntryPatch(HLoadString* load_string); Label* NewPcRelativeDexCacheArrayPatch(const DexFile& dex_file, uint32_t element_offset); Label* NewJitRootStringPatch(const DexFile& dex_file, dex::StringIndex dex_index); + Label* NewJitRootClassPatch(const DexFile& dex_file, dex::TypeIndex dex_index, uint64_t address); void MoveFromReturnRegister(Location trg, Primitive::Type type) OVERRIDE; // Emit linker patches. void EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) OVERRIDE; + void PatchJitRootUse(uint8_t* code, + const uint8_t* roots_data, + const PatchInfo<Label>& info, + uint64_t index_in_table) const; void EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) OVERRIDE; // Emit a write barrier. @@ -608,9 +613,6 @@ class CodeGeneratorX86 : public CodeGenerator { X86Assembler assembler_; const X86InstructionSetFeatures& isa_features_; - // Method patch info. Using ArenaDeque<> which retains element addresses on push/emplace_back(). - ArenaDeque<PatchInfo<Label>> method_patches_; - ArenaDeque<PatchInfo<Label>> relative_call_patches_; // PC-relative DexCache access info. ArenaDeque<PatchInfo<Label>> pc_relative_dex_cache_patches_; // Patch locations for patchoat where the linker doesn't do any other work. @@ -623,6 +625,9 @@ class CodeGeneratorX86 : public CodeGenerator { // Patches for string root accesses in JIT compiled code. ArenaDeque<PatchInfo<Label>> jit_string_patches_; + // Patches for class root accesses in JIT compiled code. + ArenaDeque<PatchInfo<Label>> jit_class_patches_; + // Offset to the start of the constant area in the assembled code. // Used for fixups to the constant area. int32_t constant_area_start_; diff --git a/compiler/optimizing/code_generator_x86_64.cc b/compiler/optimizing/code_generator_x86_64.cc index 4474decf59..903844fcdb 100644 --- a/compiler/optimizing/code_generator_x86_64.cc +++ b/compiler/optimizing/code_generator_x86_64.cc @@ -960,19 +960,7 @@ inline Condition X86_64FPCondition(IfCondition cond) { HInvokeStaticOrDirect::DispatchInfo CodeGeneratorX86_64::GetSupportedInvokeStaticOrDirectDispatch( const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info, HInvokeStaticOrDirect* invoke ATTRIBUTE_UNUSED) { - switch (desired_dispatch_info.code_ptr_location) { - case HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup: - case HInvokeStaticOrDirect::CodePtrLocation::kCallDirect: - // For direct code, we actually prefer to call via the code pointer from ArtMethod*. - return HInvokeStaticOrDirect::DispatchInfo { - desired_dispatch_info.method_load_kind, - HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod, - desired_dispatch_info.method_load_data, - 0u - }; - default: - return desired_dispatch_info; - } + return desired_dispatch_info; } Location CodeGeneratorX86_64::GenerateCalleeMethodStaticOrDirectCall(HInvokeStaticOrDirect* invoke, @@ -993,12 +981,6 @@ Location CodeGeneratorX86_64::GenerateCalleeMethodStaticOrDirectCall(HInvokeStat case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress: __ movq(temp.AsRegister<CpuRegister>(), Immediate(invoke->GetMethodAddress())); break; - case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddressWithFixup: - __ movl(temp.AsRegister<CpuRegister>(), Immediate(0)); // Placeholder. - method_patches_.emplace_back(*invoke->GetTargetMethod().dex_file, - invoke->GetTargetMethod().dex_method_index); - __ Bind(&method_patches_.back().label); // Bind the label at the end of the "movl" insn. - break; case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: { __ movq(temp.AsRegister<CpuRegister>(), Address::Absolute(kDummy32BitOffset, /* no_rip */ false)); @@ -1042,19 +1024,6 @@ void CodeGeneratorX86_64::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invo case HInvokeStaticOrDirect::CodePtrLocation::kCallSelf: __ call(&frame_entry_label_); break; - case HInvokeStaticOrDirect::CodePtrLocation::kCallPCRelative: { - relative_call_patches_.emplace_back(*invoke->GetTargetMethod().dex_file, - invoke->GetTargetMethod().dex_method_index); - Label* label = &relative_call_patches_.back().label; - __ call(label); // Bind to the patch label, override at link time. - __ Bind(label); // Bind the label at the end of the "call" insn. - break; - } - case HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup: - case HInvokeStaticOrDirect::CodePtrLocation::kCallDirect: - // Filtered out by GetSupportedInvokeStaticOrDirectDispatch(). - LOG(FATAL) << "Unsupported"; - UNREACHABLE(); case HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod: // (callee_method + offset_of_quick_compiled_code)() __ call(Address(callee_method.AsRegister<CpuRegister>(), @@ -1146,22 +1115,11 @@ inline void CodeGeneratorX86_64::EmitPcRelativeLinkerPatches( void CodeGeneratorX86_64::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) { DCHECK(linker_patches->empty()); size_t size = - method_patches_.size() + - relative_call_patches_.size() + pc_relative_dex_cache_patches_.size() + simple_patches_.size() + string_patches_.size() + type_patches_.size(); linker_patches->reserve(size); - for (const PatchInfo<Label>& info : method_patches_) { - uint32_t literal_offset = info.label.Position() - kLabelPositionToLiteralOffsetAdjustment; - linker_patches->push_back(LinkerPatch::MethodPatch(literal_offset, &info.dex_file, info.index)); - } - for (const PatchInfo<Label>& info : relative_call_patches_) { - uint32_t literal_offset = info.label.Position() - kLabelPositionToLiteralOffsetAdjustment; - linker_patches->push_back( - LinkerPatch::RelativeCodePatch(literal_offset, &info.dex_file, info.index)); - } EmitPcRelativeLinkerPatches<LinkerPatch::DexCacheArrayPatch>(pc_relative_dex_cache_patches_, linker_patches); for (const Label& label : simple_patches_) { @@ -1253,14 +1211,13 @@ CodeGeneratorX86_64::CodeGeneratorX86_64(HGraph* graph, assembler_(graph->GetArena()), isa_features_(isa_features), constant_area_start_(0), - method_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), - relative_call_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), simple_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), type_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), fixups_to_jump_tables_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), - jit_string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) { + jit_string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + jit_class_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) { AddAllocatedRegister(Location::RegisterLocation(kFakeReturnRegister)); } @@ -5460,8 +5417,7 @@ HLoadClass::LoadKind CodeGeneratorX86_64::GetSupportedLoadClassKind( break; case HLoadClass::LoadKind::kBootImageAddress: break; - case HLoadClass::LoadKind::kDexCacheAddress: - DCHECK(Runtime::Current()->UseJitCompilation()); + case HLoadClass::LoadKind::kJitTableAddress: break; case HLoadClass::LoadKind::kDexCachePcRelative: DCHECK(!Runtime::Current()->UseJitCompilation()); @@ -5500,6 +5456,16 @@ void LocationsBuilderX86_64::VisitLoadClass(HLoadClass* cls) { locations->SetOut(Location::RequiresRegister()); } +Label* CodeGeneratorX86_64::NewJitRootClassPatch(const DexFile& dex_file, + dex::TypeIndex dex_index, + uint64_t address) { + jit_class_roots_.Overwrite(TypeReference(&dex_file, dex_index), address); + // Add a patch entry and return the label. + jit_class_patches_.emplace_back(dex_file, dex_index.index_); + PatchInfo<Label>* info = &jit_class_patches_.back(); + return &info->label; +} + void InstructionCodeGeneratorX86_64::VisitLoadClass(HLoadClass* cls) { LocationSummary* locations = cls->GetLocations(); if (cls->NeedsAccessCheck()) { @@ -5543,26 +5509,13 @@ void InstructionCodeGeneratorX86_64::VisitLoadClass(HLoadClass* cls) { codegen_->RecordSimplePatch(); break; } - case HLoadClass::LoadKind::kDexCacheAddress: { - DCHECK_NE(cls->GetAddress(), 0u); + case HLoadClass::LoadKind::kJitTableAddress: { + Address address = Address::Absolute(CodeGeneratorX86_64::kDummy32BitOffset, + /* no_rip */ true); + Label* fixup_label = + codegen_->NewJitRootClassPatch(cls->GetDexFile(), cls->GetTypeIndex(), cls->GetAddress()); // /* GcRoot<mirror::Class> */ out = *address - if (IsUint<32>(cls->GetAddress())) { - Address address = Address::Absolute(cls->GetAddress(), /* no_rip */ true); - GenerateGcRootFieldLoad(cls, - out_loc, - address, - /* fixup_label */ nullptr, - read_barrier_option); - } else { - // TODO: Consider using opcode A1, i.e. movl eax, moff32 (with 64-bit address). - __ movq(out, Immediate(cls->GetAddress())); - GenerateGcRootFieldLoad(cls, - out_loc, - Address(out, 0), - /* fixup_label */ nullptr, - read_barrier_option); - } - generate_null_check = !cls->IsInDexCache(); + GenerateGcRootFieldLoad(cls, out_loc, address, fixup_label, kCompilerReadBarrierOption); break; } case HLoadClass::LoadKind::kDexCachePcRelative: { @@ -7127,18 +7080,31 @@ void CodeGeneratorX86_64::MoveInt64ToAddress(const Address& addr_low, } } +void CodeGeneratorX86_64::PatchJitRootUse(uint8_t* code, + const uint8_t* roots_data, + const PatchInfo<Label>& info, + uint64_t index_in_table) const { + uint32_t code_offset = info.label.Position() - kLabelPositionToLiteralOffsetAdjustment; + uintptr_t address = + reinterpret_cast<uintptr_t>(roots_data) + index_in_table * sizeof(GcRoot<mirror::Object>); + typedef __attribute__((__aligned__(1))) uint32_t unaligned_uint32_t; + reinterpret_cast<unaligned_uint32_t*>(code + code_offset)[0] = + dchecked_integral_cast<uint32_t>(address); +} + void CodeGeneratorX86_64::EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) { for (const PatchInfo<Label>& info : jit_string_patches_) { - const auto& it = jit_string_roots_.find(StringReference(&info.dex_file, - dex::StringIndex(info.index))); + const auto& it = jit_string_roots_.find( + StringReference(&info.dex_file, dex::StringIndex(info.index))); DCHECK(it != jit_string_roots_.end()); - size_t index_in_table = it->second; - uint32_t code_offset = info.label.Position() - kLabelPositionToLiteralOffsetAdjustment; - uintptr_t address = - reinterpret_cast<uintptr_t>(roots_data) + index_in_table * sizeof(GcRoot<mirror::Object>); - typedef __attribute__((__aligned__(1))) uint32_t unaligned_uint32_t; - reinterpret_cast<unaligned_uint32_t*>(code + code_offset)[0] = - dchecked_integral_cast<uint32_t>(address); + PatchJitRootUse(code, roots_data, info, it->second); + } + + for (const PatchInfo<Label>& info : jit_class_patches_) { + const auto& it = jit_class_roots_.find( + TypeReference(&info.dex_file, dex::TypeIndex(info.index))); + DCHECK(it != jit_class_roots_.end()); + PatchJitRootUse(code, roots_data, info, it->second); } } diff --git a/compiler/optimizing/code_generator_x86_64.h b/compiler/optimizing/code_generator_x86_64.h index 2f41f73da6..f827e79a94 100644 --- a/compiler/optimizing/code_generator_x86_64.h +++ b/compiler/optimizing/code_generator_x86_64.h @@ -413,11 +413,17 @@ class CodeGeneratorX86_64 : public CodeGenerator { Label* NewStringBssEntryPatch(HLoadString* load_string); Label* NewPcRelativeDexCacheArrayPatch(const DexFile& dex_file, uint32_t element_offset); Label* NewJitRootStringPatch(const DexFile& dex_file, dex::StringIndex dex_index); + Label* NewJitRootClassPatch(const DexFile& dex_file, dex::TypeIndex dex_index, uint64_t address); void MoveFromReturnRegister(Location trg, Primitive::Type type) OVERRIDE; void EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) OVERRIDE; + void PatchJitRootUse(uint8_t* code, + const uint8_t* roots_data, + const PatchInfo<Label>& info, + uint64_t index_in_table) const; + void EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) OVERRIDE; const X86_64InstructionSetFeatures& GetInstructionSetFeatures() const { @@ -590,9 +596,6 @@ class CodeGeneratorX86_64 : public CodeGenerator { // Used for fixups to the constant area. int constant_area_start_; - // Method patch info. Using ArenaDeque<> which retains element addresses on push/emplace_back(). - ArenaDeque<PatchInfo<Label>> method_patches_; - ArenaDeque<PatchInfo<Label>> relative_call_patches_; // PC-relative DexCache access info. ArenaDeque<PatchInfo<Label>> pc_relative_dex_cache_patches_; // Patch locations for patchoat where the linker doesn't do any other work. @@ -608,6 +611,9 @@ class CodeGeneratorX86_64 : public CodeGenerator { // Patches for string literals in JIT compiled code. ArenaDeque<PatchInfo<Label>> jit_string_patches_; + // Patches for class literals in JIT compiled code. + ArenaDeque<PatchInfo<Label>> jit_class_patches_; + DISALLOW_COPY_AND_ASSIGN(CodeGeneratorX86_64); }; diff --git a/compiler/optimizing/common_arm.h b/compiler/optimizing/common_arm.h index eabdbad13c..21c3ae628a 100644 --- a/compiler/optimizing/common_arm.h +++ b/compiler/optimizing/common_arm.h @@ -122,10 +122,16 @@ inline vixl::aarch32::VRegister InputVRegisterAt(HInstruction* instr, int input_ if (type == Primitive::kPrimFloat) { return InputSRegisterAt(instr, input_index); } else { + DCHECK_EQ(type, Primitive::kPrimDouble); return InputDRegisterAt(instr, input_index); } } +inline vixl::aarch32::VRegister InputVRegister(HInstruction* instr) { + DCHECK_EQ(instr->InputCount(), 1u); + return InputVRegisterAt(instr, 0); +} + inline vixl::aarch32::Register OutputRegister(HInstruction* instr) { return RegisterFrom(instr->GetLocations()->Out(), instr->GetType()); } @@ -140,8 +146,7 @@ inline vixl::aarch32::Register InputRegister(HInstruction* instr) { return InputRegisterAt(instr, 0); } -inline int32_t Int32ConstantFrom(Location location) { - HConstant* instr = location.GetConstant(); +inline int32_t Int32ConstantFrom(HInstruction* instr) { if (instr->IsIntConstant()) { return instr->AsIntConstant()->GetValue(); } else if (instr->IsNullConstant()) { @@ -155,6 +160,10 @@ inline int32_t Int32ConstantFrom(Location location) { } } +inline int32_t Int32ConstantFrom(Location location) { + return Int32ConstantFrom(location.GetConstant()); +} + inline int64_t Int64ConstantFrom(Location location) { HConstant* instr = location.GetConstant(); if (instr->IsIntConstant()) { @@ -167,6 +176,11 @@ inline int64_t Int64ConstantFrom(Location location) { } } +inline uint64_t Uint64ConstantFrom(HInstruction* instr) { + DCHECK(instr->IsConstant()) << instr->DebugName(); + return instr->AsConstant()->GetValueAsUint64(); +} + inline vixl::aarch32::Operand OperandFrom(Location location, Primitive::Type type) { if (location.IsRegister()) { return vixl::aarch32::Operand(RegisterFrom(location, type)); diff --git a/compiler/optimizing/graph_checker.cc b/compiler/optimizing/graph_checker.cc index 188ee3a8d1..34b52a87b5 100644 --- a/compiler/optimizing/graph_checker.cc +++ b/compiler/optimizing/graph_checker.cc @@ -20,12 +20,15 @@ #include <string> #include <sstream> +#include "android-base/stringprintf.h" + #include "base/arena_containers.h" #include "base/bit_vector-inl.h" -#include "base/stringprintf.h" namespace art { +using android::base::StringPrintf; + static bool IsAllowedToJumpToExitBlock(HInstruction* instruction) { return instruction->IsThrow() || instruction->IsReturn() || instruction->IsReturnVoid(); } diff --git a/compiler/optimizing/graph_test.cc b/compiler/optimizing/graph_test.cc index d5305646a8..28ee3a5e8b 100644 --- a/compiler/optimizing/graph_test.cc +++ b/compiler/optimizing/graph_test.cc @@ -15,7 +15,6 @@ */ #include "base/arena_allocator.h" -#include "base/stringprintf.h" #include "builder.h" #include "nodes.h" #include "optimizing_unit_test.h" diff --git a/compiler/optimizing/induction_var_analysis.cc b/compiler/optimizing/induction_var_analysis.cc index c240c67e79..88473f02e5 100644 --- a/compiler/optimizing/induction_var_analysis.cc +++ b/compiler/optimizing/induction_var_analysis.cc @@ -73,10 +73,18 @@ static bool IsNarrowingIntegralConversion(Primitive::Type from, Primitive::Type } /** - * Returns narrowest data type. + * Returns result of implicit widening type conversion done in HIR. */ -static Primitive::Type Narrowest(Primitive::Type type1, Primitive::Type type2) { - return Primitive::ComponentSize(type1) <= Primitive::ComponentSize(type2) ? type1 : type2; +static Primitive::Type ImplicitConversion(Primitive::Type type) { + switch (type) { + case Primitive::kPrimShort: + case Primitive::kPrimChar: + case Primitive::kPrimByte: + case Primitive::kPrimBoolean: + return Primitive::kPrimInt; + default: + return type; + } } // @@ -211,7 +219,7 @@ uint32_t HInductionVarAnalysis::VisitDescendant(HLoopInformation* loop, HInstruc void HInductionVarAnalysis::ClassifyTrivial(HLoopInformation* loop, HInstruction* instruction) { InductionInfo* info = nullptr; if (instruction->IsPhi()) { - info = TransferPhi(loop, instruction, /* input_index */ 0); + info = TransferPhi(loop, instruction, /*input_index*/ 0, /*adjust_input_size*/ 0); } else if (instruction->IsAdd()) { info = TransferAddSub(LookupInfo(loop, instruction->InputAt(0)), LookupInfo(loop, instruction->InputAt(1)), kAdd); @@ -224,15 +232,17 @@ void HInductionVarAnalysis::ClassifyTrivial(HLoopInformation* loop, HInstruction info = TransferMul(LookupInfo(loop, instruction->InputAt(0)), LookupInfo(loop, instruction->InputAt(1))); } else if (instruction->IsShl()) { - HInstruction* mulc = GetMultConstantForShift(loop, instruction); + HInstruction* mulc = GetShiftConstant(loop, instruction, /*initial*/ nullptr); if (mulc != nullptr) { info = TransferMul(LookupInfo(loop, instruction->InputAt(0)), LookupInfo(loop, mulc)); } + } else if (instruction->IsSelect()) { + info = TransferPhi(loop, instruction, /*input_index*/ 0, /*adjust_input_size*/ 1); } else if (instruction->IsTypeConversion()) { - info = TransferCnv(LookupInfo(loop, instruction->InputAt(0)), - instruction->AsTypeConversion()->GetInputType(), - instruction->AsTypeConversion()->GetResultType()); + info = TransferConversion(LookupInfo(loop, instruction->InputAt(0)), + instruction->AsTypeConversion()->GetInputType(), + instruction->AsTypeConversion()->GetResultType()); } else if (instruction->IsBoundsCheck()) { info = LookupInfo(loop, instruction->InputAt(0)); // Pass-through. } @@ -265,12 +275,16 @@ void HInductionVarAnalysis::ClassifyNonTrivial(HLoopInformation* loop) { return; } - // Store interesting cycle. - AssignCycle(phi->AsPhi()); + // Store interesting cycle in each loop phi. + for (size_t i = 0; i < size; i++) { + if (scc_[i]->IsLoopHeaderPhi()) { + AssignCycle(scc_[i]->AsPhi()); + } + } // Singleton is wrap-around induction if all internal links have the same meaning. if (size == 1) { - InductionInfo* update = TransferPhi(loop, phi, /* input_index */ 1); + InductionInfo* update = TransferPhi(loop, phi, /*input_index*/ 1, /*adjust_input_size*/ 0); if (update != nullptr) { AssignInfo(loop, phi, CreateInduction(kWrapAround, kNop, @@ -305,10 +319,15 @@ void HInductionVarAnalysis::ClassifyNonTrivial(HLoopInformation* loop) { update = SolveOp( loop, phi, instruction, instruction->InputAt(0), instruction->InputAt(1), kRem); } else if (instruction->IsShl()) { - HInstruction* mulc = GetMultConstantForShift(loop, instruction); + HInstruction* mulc = GetShiftConstant(loop, instruction, /*initial*/ nullptr); if (mulc != nullptr) { update = SolveOp(loop, phi, instruction, instruction->InputAt(0), mulc, kMul); } + } else if (instruction->IsShr() || instruction->IsUShr()) { + HInstruction* divc = GetShiftConstant(loop, instruction, initial); + if (divc != nullptr) { + update = SolveOp(loop, phi, instruction, instruction->InputAt(0), divc, kDiv); + } } else if (instruction->IsXor()) { update = SolveOp( loop, phi, instruction, instruction->InputAt(0), instruction->InputAt(1), kXor); @@ -316,8 +335,10 @@ void HInductionVarAnalysis::ClassifyNonTrivial(HLoopInformation* loop) { update = SolveTest(loop, phi, instruction, 0); } else if (instruction->IsNotEqual()) { update = SolveTest(loop, phi, instruction, 1); + } else if (instruction->IsSelect()) { + update = SolvePhi(instruction, /*input_index*/ 0, /*adjust_input_size*/ 1); // acts like Phi } else if (instruction->IsTypeConversion()) { - update = SolveCnv(instruction->AsTypeConversion()); + update = SolveConversion(loop, phi, instruction->AsTypeConversion()); } if (update == nullptr) { return; @@ -326,7 +347,7 @@ void HInductionVarAnalysis::ClassifyNonTrivial(HLoopInformation* loop) { } // Success if all internal links received the same temporary meaning. - InductionInfo* induction = SolvePhi(phi, /* input_index */ 1); + InductionInfo* induction = SolvePhi(phi, /*input_index*/ 1, /*adjust_input_size*/ 0); if (induction != nullptr) { switch (induction->induction_class) { case kInvariant: @@ -385,12 +406,13 @@ HInductionVarAnalysis::InductionInfo* HInductionVarAnalysis::RotatePeriodicInduc HInductionVarAnalysis::InductionInfo* HInductionVarAnalysis::TransferPhi(HLoopInformation* loop, HInstruction* phi, - size_t input_index) { + size_t input_index, + size_t adjust_input_size) { // Match all phi inputs from input_index onwards exactly. HInputsRef inputs = phi->GetInputs(); DCHECK_LT(input_index, inputs.size()); InductionInfo* a = LookupInfo(loop, inputs[input_index]); - for (size_t i = input_index + 1; i < inputs.size(); i++) { + for (size_t i = input_index + 1, n = inputs.size() - adjust_input_size; i < n; i++) { InductionInfo* b = LookupInfo(loop, inputs[i]); if (!InductionEqual(a, b)) { return nullptr; @@ -406,18 +428,20 @@ HInductionVarAnalysis::InductionInfo* HInductionVarAnalysis::TransferAddSub(Indu // wrap-around, or periodic can be combined with an invariant to yield a similar result. // Two linear or two polynomial inputs can be combined too. Other combinations fail. if (a != nullptr && b != nullptr) { - type_ = Narrowest(type_, Narrowest(a->type, b->type)); - if (a->induction_class == kInvariant && b->induction_class == kInvariant) { - return CreateInvariantOp(op, a, b); + if (IsNarrowingLinear(a) || IsNarrowingLinear(b)) { + return nullptr; // no transfer + } else if (a->induction_class == kInvariant && b->induction_class == kInvariant) { + return CreateInvariantOp(op, a, b); // direct invariant } else if ((a->induction_class == kLinear && b->induction_class == kLinear) || (a->induction_class == kPolynomial && b->induction_class == kPolynomial)) { - return CreateInduction(a->induction_class, - a->operation, - TransferAddSub(a->op_a, b->op_a, op), - TransferAddSub(a->op_b, b->op_b, op), - /*fetch*/ nullptr, - type_); + // Rule induc(a, b) + induc(a', b') -> induc(a + a', b + b'). + InductionInfo* new_a = TransferAddSub(a->op_a, b->op_a, op); + InductionInfo* new_b = TransferAddSub(a->op_b, b->op_b, op); + if (new_a != nullptr && new_b != nullptr) { + return CreateInduction(a->induction_class, a->operation, new_a, new_b, a->fetch, type_); + } } else if (a->induction_class == kInvariant) { + // Rule a + induc(a', b') -> induc(a', a + b') or induc(a + a', a + b'). InductionInfo* new_a = b->op_a; InductionInfo* new_b = TransferAddSub(a, b->op_b, op); if (b->induction_class == kWrapAround || b->induction_class == kPeriodic) { @@ -425,14 +449,19 @@ HInductionVarAnalysis::InductionInfo* HInductionVarAnalysis::TransferAddSub(Indu } else if (op == kSub) { // Negation required. new_a = TransferNeg(new_a); } - return CreateInduction(b->induction_class, b->operation, new_a, new_b, b->fetch, type_); + if (new_a != nullptr && new_b != nullptr) { + return CreateInduction(b->induction_class, b->operation, new_a, new_b, b->fetch, type_); + } } else if (b->induction_class == kInvariant) { + // Rule induc(a, b) + b' -> induc(a, b + b') or induc(a + b', b + b'). InductionInfo* new_a = a->op_a; InductionInfo* new_b = TransferAddSub(a->op_b, b, op); if (a->induction_class == kWrapAround || a->induction_class == kPeriodic) { new_a = TransferAddSub(new_a, b, op); } - return CreateInduction(a->induction_class, a->operation, new_a, new_b, a->fetch, type_); + if (new_a != nullptr && new_b != nullptr) { + return CreateInduction(a->induction_class, a->operation, new_a, new_b, a->fetch, type_); + } } } return nullptr; @@ -442,16 +471,17 @@ HInductionVarAnalysis::InductionInfo* HInductionVarAnalysis::TransferNeg(Inducti // Transfer over a unary negation: an invariant, linear, polynomial, geometric (mul), // wrap-around, or periodic input yields a similar but negated induction as result. if (a != nullptr) { - type_ = Narrowest(type_, a->type); - if (a->induction_class == kInvariant) { - return CreateInvariantOp(kNeg, nullptr, a); + if (IsNarrowingLinear(a)) { + return nullptr; // no transfer + } else if (a->induction_class == kInvariant) { + return CreateInvariantOp(kNeg, nullptr, a); // direct invariant } else if (a->induction_class != kGeometric || a->operation == kMul) { - return CreateInduction(a->induction_class, - a->operation, - TransferNeg(a->op_a), - TransferNeg(a->op_b), - a->fetch, - type_); + // Rule - induc(a, b) -> induc(-a, -b). + InductionInfo* new_a = TransferNeg(a->op_a); + InductionInfo* new_b = TransferNeg(a->op_b); + if (new_a != nullptr && new_b != nullptr) { + return CreateInduction(a->induction_class, a->operation, new_a, new_b, a->fetch, type_); + } } } return nullptr; @@ -463,54 +493,56 @@ HInductionVarAnalysis::InductionInfo* HInductionVarAnalysis::TransferMul(Inducti // wrap-around, or periodic can be multiplied with an invariant to yield a similar // but multiplied result. Two non-invariant inputs cannot be multiplied, however. if (a != nullptr && b != nullptr) { - type_ = Narrowest(type_, Narrowest(a->type, b->type)); - if (a->induction_class == kInvariant && b->induction_class == kInvariant) { - return CreateInvariantOp(kMul, a, b); + if (IsNarrowingLinear(a) || IsNarrowingLinear(b)) { + return nullptr; // no transfer + } else if (a->induction_class == kInvariant && b->induction_class == kInvariant) { + return CreateInvariantOp(kMul, a, b); // direct invariant } else if (a->induction_class == kInvariant && (b->induction_class != kGeometric || b->operation == kMul)) { - return CreateInduction(b->induction_class, - b->operation, - TransferMul(a, b->op_a), - TransferMul(a, b->op_b), - b->fetch, - type_); + // Rule a * induc(a', b') -> induc(a * a', b * b'). + InductionInfo* new_a = TransferMul(a, b->op_a); + InductionInfo* new_b = TransferMul(a, b->op_b); + if (new_a != nullptr && new_b != nullptr) { + return CreateInduction(b->induction_class, b->operation, new_a, new_b, b->fetch, type_); + } } else if (b->induction_class == kInvariant && (a->induction_class != kGeometric || a->operation == kMul)) { - return CreateInduction(a->induction_class, - a->operation, - TransferMul(a->op_a, b), - TransferMul(a->op_b, b), - a->fetch, - type_); + // Rule induc(a, b) * b' -> induc(a * b', b * b'). + InductionInfo* new_a = TransferMul(a->op_a, b); + InductionInfo* new_b = TransferMul(a->op_b, b); + if (new_a != nullptr && new_b != nullptr) { + return CreateInduction(a->induction_class, a->operation, new_a, new_b, a->fetch, type_); + } } } return nullptr; } -HInductionVarAnalysis::InductionInfo* HInductionVarAnalysis::TransferCnv(InductionInfo* a, - Primitive::Type from, - Primitive::Type to) { +HInductionVarAnalysis::InductionInfo* HInductionVarAnalysis::TransferConversion( + InductionInfo* a, + Primitive::Type from, + Primitive::Type to) { if (a != nullptr) { - // Allow narrowing conversion on linear induction in certain cases. - if (IsNarrowingIntegralConversion(from, to)) { - if (a->induction_class == kLinear) { - if (a->type == to || (a->type == from && IsNarrowingIntegralConversion(from, to))) { - return CreateInduction(kLinear, kNop, a->op_a, a->op_b, /*fetch*/ nullptr, to); - } - } + // Allow narrowing conversion on linear induction in certain cases: + // induction is already at narrow type, or can be made narrower. + if (IsNarrowingIntegralConversion(from, to) && + a->induction_class == kLinear && + (a->type == to || IsNarrowingIntegralConversion(a->type, to))) { + return CreateInduction(kLinear, kNop, a->op_a, a->op_b, a->fetch, to); } } return nullptr; } HInductionVarAnalysis::InductionInfo* HInductionVarAnalysis::SolvePhi(HInstruction* phi, - size_t input_index) { + size_t input_index, + size_t adjust_input_size) { // Match all phi inputs from input_index onwards exactly. HInputsRef inputs = phi->GetInputs(); DCHECK_LT(input_index, inputs.size()); auto ita = cycle_.find(inputs[input_index]); if (ita != cycle_.end()) { - for (size_t i = input_index + 1; i < inputs.size(); i++) { + for (size_t i = input_index + 1, n = inputs.size() - adjust_input_size; i < n; i++) { auto itb = cycle_.find(inputs[i]); if (itb == cycle_.end() || !HInductionVarAnalysis::InductionEqual(ita->second, itb->second)) { @@ -527,7 +559,7 @@ HInductionVarAnalysis::InductionInfo* HInductionVarAnalysis::SolvePhiAllInputs( HInstruction* entry_phi, HInstruction* phi) { // Match all phi inputs. - InductionInfo* match = SolvePhi(phi, /* input_index */ 0); + InductionInfo* match = SolvePhi(phi, /*input_index*/ 0, /*adjust_input_size*/ 0); if (match != nullptr) { return match; } @@ -542,7 +574,7 @@ HInductionVarAnalysis::InductionInfo* HInductionVarAnalysis::SolvePhiAllInputs( InductionInfo* initial = LookupInfo(loop, entry_phi->InputAt(0)); return CreateInduction(kPeriodic, kNop, a, initial, /*fetch*/ nullptr, type_); } - InductionInfo* b = SolvePhi(phi, /* input_index */ 1); + InductionInfo* b = SolvePhi(phi, /*input_index*/ 1, /*adjust_input_size*/ 0); if (b != nullptr && b->induction_class == kPeriodic) { return CreateInduction(kPeriodic, kNop, a, b, /*fetch*/ nullptr, type_); } @@ -574,17 +606,15 @@ HInductionVarAnalysis::InductionInfo* HInductionVarAnalysis::SolveAddSub(HLoopIn return CreateInvariantOp(op, a, b); } } - } else if (op == kAdd && b->induction_class == kLinear) { + } else if (b->induction_class == kLinear && b->type == type_) { // Solve within a tight cycle that adds a term that is already classified as a linear // induction for a polynomial induction k = k + i (represented as sum over linear terms). if (x == entry_phi && entry_phi->InputCount() == 2 && instruction == entry_phi->InputAt(1)) { InductionInfo* initial = LookupInfo(loop, entry_phi->InputAt(0)); - return CreateInduction(kPolynomial, - kNop, - b, - initial, - /*fetch*/ nullptr, - type_); + InductionInfo* new_a = op == kAdd ? b : TransferNeg(b); + if (new_a != nullptr) { + return CreateInduction(kPolynomial, kNop, new_a, initial, /*fetch*/ nullptr, type_); + } } } } @@ -689,16 +719,29 @@ HInductionVarAnalysis::InductionInfo* HInductionVarAnalysis::SolveTest(HLoopInfo return nullptr; } -HInductionVarAnalysis::InductionInfo* HInductionVarAnalysis::SolveCnv(HTypeConversion* conversion) { +HInductionVarAnalysis::InductionInfo* HInductionVarAnalysis::SolveConversion( + HLoopInformation* loop, + HInstruction* entry_phi, + HTypeConversion* conversion) { Primitive::Type from = conversion->GetInputType(); Primitive::Type to = conversion->GetResultType(); - // A narrowing conversion is allowed within the cycle of a linear induction, provided that the - // narrowest encountered type is recorded with the induction to account for the precision loss. - if (IsNarrowingIntegralConversion(from, to)) { - auto it = cycle_.find(conversion->GetInput()); - if (it != cycle_.end() && it->second->induction_class == kInvariant) { - type_ = Narrowest(type_, to); - return it->second; + // A narrowing conversion is allowed as *last* operation of the cycle of a linear induction + // with an initial value that fits the type, provided that the narrowest encountered type is + // recorded with the induction to account for the precision loss. The narrower induction does + // *not* transfer to any wider operations, however, since these may yield out-of-type values + if (entry_phi->InputCount() == 2 && conversion == entry_phi->InputAt(1)) { + int64_t min = Primitive::MinValueOfIntegralType(to); + int64_t max = Primitive::MaxValueOfIntegralType(to); + int64_t value = 0; + InductionInfo* initial = LookupInfo(loop, entry_phi->InputAt(0)); + if (IsNarrowingIntegralConversion(from, to) && + IsAtLeast(initial, &value) && value >= min && + IsAtMost(initial, &value) && value <= max) { + auto it = cycle_.find(conversion->GetInput()); + if (it != cycle_.end() && it->second->induction_class == kInvariant) { + type_ = to; + return it->second; + } } } return nullptr; @@ -718,7 +761,7 @@ void HInductionVarAnalysis::VisitControl(HLoopInformation* loop) { HCondition* condition = if_expr->AsCondition(); InductionInfo* a = LookupInfo(loop, condition->InputAt(0)); InductionInfo* b = LookupInfo(loop, condition->InputAt(1)); - Primitive::Type type = condition->InputAt(0)->GetType(); + Primitive::Type type = ImplicitConversion(condition->InputAt(0)->GetType()); // Determine if the loop control uses a known sequence on an if-exit (X outside) or on // an if-iterate (X inside), expressed as if-iterate when passed into VisitCondition(). if (a == nullptr || b == nullptr) { @@ -890,8 +933,8 @@ bool HInductionVarAnalysis::IsFinite(InductionInfo* upper_expr, int64_t stride_value, Primitive::Type type, IfCondition cmp) { - const int64_t min = Primitive::MinValueOfIntegralType(type); - const int64_t max = Primitive::MaxValueOfIntegralType(type); + int64_t min = Primitive::MinValueOfIntegralType(type); + int64_t max = Primitive::MaxValueOfIntegralType(type); // Some rules under which it is certain at compile-time that the loop is finite. int64_t value; switch (cmp) { @@ -927,8 +970,6 @@ bool HInductionVarAnalysis::FitsNarrowerControl(InductionInfo* lower_expr, min++; } // Do both bounds fit the range? - // Note: The `value` is initialized to please valgrind - the compiler can reorder - // the return value check with the `value` check, b/27651442 . int64_t value = 0; return IsAtLeast(lower_expr, &value) && value >= min && IsAtMost(lower_expr, &value) && value <= max && @@ -1035,16 +1076,27 @@ HInductionVarAnalysis::InductionInfo* HInductionVarAnalysis::CreateSimplifiedInv return CreateSimplifiedInvariant(kSub, b->op_b, b->op_a); } } - return new (graph_->GetArena()) InductionInfo(kInvariant, op, a, b, nullptr, b->type); + return new (graph_->GetArena()) InductionInfo( + kInvariant, op, a, b, nullptr, ImplicitConversion(b->type)); } -HInstruction* HInductionVarAnalysis::GetMultConstantForShift(HLoopInformation* loop, - HInstruction* instruction) { - // Obtain the constant needed to treat shift as equivalent multiplication. This yields an - // existing instruction if the constant is already there. Otherwise, this has a side effect - // on the HIR. The restriction on the shift factor avoids generating a negative constant - // (viz. 1 << 31 and 1L << 63 set the sign bit). The code assumes that generalization for - // shift factors outside [0,32) and [0,64) ranges is done by earlier simplification. +HInstruction* HInductionVarAnalysis::GetShiftConstant(HLoopInformation* loop, + HInstruction* instruction, + InductionInfo* initial) { + DCHECK(instruction->IsShl() || instruction->IsShr() || instruction->IsUShr()); + // Shift-rights are only the same as division for non-negative initial inputs. + // Otherwise we would round incorrectly. + if (initial != nullptr) { + int64_t value = -1; + if (!IsAtLeast(initial, &value) || value < 0) { + return nullptr; + } + } + // Obtain the constant needed to treat shift as equivalent multiplication or division. + // This yields an existing instruction if the constant is already there. Otherwise, this + // has a side effect on the HIR. The restriction on the shift factor avoids generating a + // negative constant (viz. 1 << 31 and 1L << 63 set the sign bit). The code assumes that + // generalization for shift factors outside [0,32) and [0,64) ranges is done earlier. InductionInfo* b = LookupInfo(loop, instruction->InputAt(1)); int64_t value = -1; if (IsExact(b, &value)) { @@ -1087,6 +1139,16 @@ bool HInductionVarAnalysis::IsAtLeast(InductionInfo* info, int64_t* value) { return InductionVarRange(this).IsConstant(info, InductionVarRange::kAtLeast, value); } +bool HInductionVarAnalysis::IsNarrowingLinear(InductionInfo* info) { + return info != nullptr && + info->induction_class == kLinear && + (info->type == Primitive::kPrimByte || + info->type == Primitive::kPrimShort || + info->type == Primitive::kPrimChar || + (info->type == Primitive::kPrimInt && (info->op_a->type == Primitive::kPrimLong || + info->op_b->type == Primitive::kPrimLong))); +} + bool HInductionVarAnalysis::InductionEqual(InductionInfo* info1, InductionInfo* info2) { // Test structural equality only, without accounting for simplifications. diff --git a/compiler/optimizing/induction_var_analysis.h b/compiler/optimizing/induction_var_analysis.h index 4720f2d61c..39b39cdf55 100644 --- a/compiler/optimizing/induction_var_analysis.h +++ b/compiler/optimizing/induction_var_analysis.h @@ -115,7 +115,7 @@ class HInductionVarAnalysis : public HOptimization { InductionInfo* op_a; InductionInfo* op_b; HInstruction* fetch; - Primitive::Type type; // precision of induction + Primitive::Type type; // precision of operation }; bool IsVisitedNode(HInstruction* instruction) const { @@ -160,14 +160,17 @@ class HInductionVarAnalysis : public HOptimization { InductionInfo* RotatePeriodicInduction(InductionInfo* induction, InductionInfo* last); // Transfer operations. - InductionInfo* TransferPhi(HLoopInformation* loop, HInstruction* phi, size_t input_index); + InductionInfo* TransferPhi(HLoopInformation* loop, + HInstruction* phi, + size_t input_index, + size_t adjust_input_size); InductionInfo* TransferAddSub(InductionInfo* a, InductionInfo* b, InductionOp op); InductionInfo* TransferNeg(InductionInfo* a); InductionInfo* TransferMul(InductionInfo* a, InductionInfo* b); - InductionInfo* TransferCnv(InductionInfo* a, Primitive::Type from, Primitive::Type to); + InductionInfo* TransferConversion(InductionInfo* a, Primitive::Type from, Primitive::Type to); // Solvers. - InductionInfo* SolvePhi(HInstruction* phi, size_t input_index); + InductionInfo* SolvePhi(HInstruction* phi, size_t input_index, size_t adjust_input_size); InductionInfo* SolvePhiAllInputs(HLoopInformation* loop, HInstruction* entry_phi, HInstruction* phi); @@ -188,7 +191,9 @@ class HInductionVarAnalysis : public HOptimization { HInstruction* entry_phi, HInstruction* instruction, int64_t oppositive_value); - InductionInfo* SolveCnv(HTypeConversion* conversion); + InductionInfo* SolveConversion(HLoopInformation* loop, + HInstruction* entry_phi, + HTypeConversion* conversion); // Trip count information. void VisitControl(HLoopInformation* loop); @@ -220,7 +225,9 @@ class HInductionVarAnalysis : public HOptimization { InductionInfo* LookupInfo(HLoopInformation* loop, HInstruction* instruction); InductionInfo* CreateConstant(int64_t value, Primitive::Type type); InductionInfo* CreateSimplifiedInvariant(InductionOp op, InductionInfo* a, InductionInfo* b); - HInstruction* GetMultConstantForShift(HLoopInformation* loop, HInstruction* instruction); + HInstruction* GetShiftConstant(HLoopInformation* loop, + HInstruction* instruction, + InductionInfo* initial); void AssignCycle(HPhi* phi); ArenaSet<HInstruction*>* LookupCycle(HPhi* phi); @@ -230,6 +237,7 @@ class HInductionVarAnalysis : public HOptimization { bool IsAtLeast(InductionInfo* info, /*out*/ int64_t* value); // Helpers. + static bool IsNarrowingLinear(InductionInfo* info); static bool InductionEqual(InductionInfo* info1, InductionInfo* info2); static std::string FetchToString(HInstruction* fetch); static std::string InductionToString(InductionInfo* info); diff --git a/compiler/optimizing/induction_var_analysis_test.cc b/compiler/optimizing/induction_var_analysis_test.cc index 2d182f6483..82ee93d5c2 100644 --- a/compiler/optimizing/induction_var_analysis_test.cc +++ b/compiler/optimizing/induction_var_analysis_test.cc @@ -87,6 +87,7 @@ class InductionVarAnalysisTest : public CommonCompilerTest { constant2_ = graph_->GetIntConstant(2); constant7_ = graph_->GetIntConstant(7); constant100_ = graph_->GetIntConstant(100); + constantm1_ = graph_->GetIntConstant(-1); float_constant0_ = graph_->GetFloatConstant(0.0f); return_->AddInstruction(new (&allocator_) HReturnVoid()); exit_->AddInstruction(new (&allocator_) HExit()); @@ -173,6 +174,12 @@ class InductionVarAnalysisTest : public CommonCompilerTest { iva_->LookupInfo(loop_body_[0]->GetLoopInformation(), instruction2)); } + // Returns true for narrowing linear induction. + bool IsNarrowingLinear(HInstruction* instruction) { + return HInductionVarAnalysis::IsNarrowingLinear( + iva_->LookupInfo(loop_body_[0]->GetLoopInformation(), instruction)); + } + // Performs InductionVarAnalysis (after proper set up). void PerformInductionVarAnalysis() { graph_->BuildDominatorTree(); @@ -196,6 +203,7 @@ class InductionVarAnalysisTest : public CommonCompilerTest { HInstruction* constant2_; HInstruction* constant7_; HInstruction* constant100_; + HInstruction* constantm1_; HInstruction* float_constant0_; // Loop specifics. @@ -612,6 +620,45 @@ TEST_F(InductionVarAnalysisTest, FindGeometricDivInductionAndDerived) { EXPECT_STREQ("", GetInductionInfo(div, 0).c_str()); } +TEST_F(InductionVarAnalysisTest, FindGeometricShrInduction) { + // Setup: + // k = 100; + // for (int i = 0; i < 100; i++) { + // k = k >> 1; // geometric (/ 2) + // } + BuildLoopNest(1); + HPhi* k_header = InsertLoopPhi(0, 0); + k_header->AddInput(constant100_); + + HInstruction* shr = InsertInstruction( + new (&allocator_) HShr(Primitive::kPrimInt, k_header, constant1_), 0); + k_header->AddInput(shr); + PerformInductionVarAnalysis(); + + // Note, only the phi in the cycle is classified. + EXPECT_STREQ("geo((100) * 2 ^ -i + (0)):PrimInt", GetInductionInfo(k_header, 0).c_str()); + EXPECT_STREQ("", GetInductionInfo(shr, 0).c_str()); +} + +TEST_F(InductionVarAnalysisTest, FindNotGeometricShrInduction) { + // Setup: + // k = -1; + // for (int i = 0; i < 100; i++) { + // k = k >> 1; // initial value is negative + // } + BuildLoopNest(1); + HPhi* k_header = InsertLoopPhi(0, 0); + k_header->AddInput(constantm1_); + + HInstruction* shr = InsertInstruction( + new (&allocator_) HShr(Primitive::kPrimInt, k_header, constant1_), 0); + k_header->AddInput(shr); + PerformInductionVarAnalysis(); + + EXPECT_STREQ("", GetInductionInfo(k_header, 0).c_str()); + EXPECT_STREQ("", GetInductionInfo(shr, 0).c_str()); +} + TEST_F(InductionVarAnalysisTest, FindRemWrapAroundInductionAndDerived) { // Setup: // k = 100; @@ -1025,16 +1072,20 @@ TEST_F(InductionVarAnalysisTest, ByteInductionIntLoopControl) { // } BuildLoopNest(1); HInstruction* conv = InsertInstruction( - new (&allocator_) HTypeConversion(Primitive::kPrimByte, basic_[0], -1), 0); + new (&allocator_) HTypeConversion(Primitive::kPrimByte, basic_[0], kNoDexPc), 0); HInstruction* store1 = InsertArrayStore(conv, 0); HInstruction* store2 = InsertArrayStore(basic_[0], 0); PerformInductionVarAnalysis(); - // Regular int induction (i) is "transferred" over conversion into byte induction (k). + // Regular int induction (i) is transferred over conversion into byte induction (k). EXPECT_STREQ("((1) * i + (0)):PrimByte", GetInductionInfo(store1->InputAt(1), 0).c_str()); EXPECT_STREQ("((1) * i + (0)):PrimInt", GetInductionInfo(store2->InputAt(1), 0).c_str()); EXPECT_STREQ("((1) * i + (1)):PrimInt", GetInductionInfo(increment_[0], 0).c_str()); + // Narrowing detected. + EXPECT_TRUE(IsNarrowingLinear(store1->InputAt(1))); + EXPECT_FALSE(IsNarrowingLinear(store2->InputAt(1))); + // Type matters! EXPECT_FALSE(HaveSameInduction(store1->InputAt(1), store2->InputAt(1))); @@ -1052,7 +1103,7 @@ TEST_F(InductionVarAnalysisTest, ByteInductionDerivedIntLoopControl) { // } BuildLoopNest(1); HInstruction* conv = InsertInstruction( - new (&allocator_) HTypeConversion(Primitive::kPrimByte, basic_[0], -1), 0); + new (&allocator_) HTypeConversion(Primitive::kPrimByte, basic_[0], kNoDexPc), 0); HInstruction* store1 = InsertArrayStore(conv, 0); HInstruction* add = InsertInstruction( new (&allocator_) HAdd(Primitive::kPrimInt, conv, constant1_), 0); @@ -1060,11 +1111,86 @@ TEST_F(InductionVarAnalysisTest, ByteInductionDerivedIntLoopControl) { PerformInductionVarAnalysis(); - // Byte induction (k) is "transferred" over conversion into addition (k + 1). - // This means only values within byte range can be trusted (even though - // addition can jump out of the range of course). + // Byte induction (k) is detected, but it does not transfer over the addition, + // since this may yield out-of-type values. EXPECT_STREQ("((1) * i + (0)):PrimByte", GetInductionInfo(store1->InputAt(1), 0).c_str()); - EXPECT_STREQ("((1) * i + (1)):PrimByte", GetInductionInfo(store2->InputAt(1), 0).c_str()); + EXPECT_STREQ("", GetInductionInfo(store2->InputAt(1), 0).c_str()); + + // Narrowing detected. + EXPECT_TRUE(IsNarrowingLinear(store1->InputAt(1))); + EXPECT_FALSE(IsNarrowingLinear(store2->InputAt(1))); // works for null +} + +TEST_F(InductionVarAnalysisTest, ByteInduction) { + // Setup: + // k = -128; + // for (int i = 0; i < 100; i++) { + // k = k + 1; + // k = (byte) k; + // } + BuildLoopNest(1); + HPhi* k_header = InsertLoopPhi(0, 0); + k_header->AddInput(graph_->GetIntConstant(-128)); + + HInstruction* add = InsertInstruction( + new (&allocator_) HAdd(Primitive::kPrimInt, k_header, constant1_), 0); + HInstruction* conv = InsertInstruction( + new (&allocator_) HTypeConversion(Primitive::kPrimByte, add, kNoDexPc), 0); + k_header->AddInput(conv); + PerformInductionVarAnalysis(); + + // Byte induction (k) is detected, but it does not transfer over the addition, + // since this may yield out-of-type values. + EXPECT_STREQ("((1) * i + (-128)):PrimByte", GetInductionInfo(k_header, 0).c_str()); + EXPECT_STREQ("", GetInductionInfo(add, 0).c_str()); + + // Narrowing detected. + EXPECT_TRUE(IsNarrowingLinear(k_header)); + EXPECT_FALSE(IsNarrowingLinear(add)); // works for null +} + +TEST_F(InductionVarAnalysisTest, NoByteInduction1) { + // Setup: + // k = -129; / does not fit! + // for (int i = 0; i < 100; i++) { + // k = k + 1; + // k = (byte) k; + // } + BuildLoopNest(1); + HPhi* k_header = InsertLoopPhi(0, 0); + k_header->AddInput(graph_->GetIntConstant(-129)); + + HInstruction* add = InsertInstruction( + new (&allocator_) HAdd(Primitive::kPrimInt, k_header, constant1_), 0); + HInstruction* conv = InsertInstruction( + new (&allocator_) HTypeConversion(Primitive::kPrimByte, add, kNoDexPc), 0); + k_header->AddInput(conv); + PerformInductionVarAnalysis(); + + EXPECT_STREQ("", GetInductionInfo(k_header, 0).c_str()); + EXPECT_STREQ("", GetInductionInfo(add, 0).c_str()); +} + +TEST_F(InductionVarAnalysisTest, NoByteInduction2) { + // Setup: + // k = 0; + // for (int i = 0; i < 100; i++) { + // k = (byte) k; // conversion not done last! + // k = k + 1; + // } + BuildLoopNest(1); + HPhi* k_header = InsertLoopPhi(0, 0); + k_header->AddInput(constant0_); + + HInstruction* conv = InsertInstruction( + new (&allocator_) HTypeConversion(Primitive::kPrimByte, k_header, kNoDexPc), 0); + HInstruction* add = InsertInstruction( + new (&allocator_) HAdd(Primitive::kPrimInt, conv, constant1_), 0); + k_header->AddInput(add); + PerformInductionVarAnalysis(); + + EXPECT_STREQ("", GetInductionInfo(k_header, 0).c_str()); + EXPECT_STREQ("", GetInductionInfo(add, 0).c_str()); } TEST_F(InductionVarAnalysisTest, ByteLoopControl1) { @@ -1075,12 +1201,20 @@ TEST_F(InductionVarAnalysisTest, ByteLoopControl1) { basic_[0]->ReplaceInput(graph_->GetIntConstant(-128), 0); HInstruction* ifs = loop_header_[0]->GetLastInstruction()->GetPrevious(); ifs->ReplaceInput(graph_->GetIntConstant(127), 1); - HInstruction* conv = new(&allocator_) HTypeConversion(Primitive::kPrimByte, increment_[0], -1); + HInstruction* conv = + new (&allocator_) HTypeConversion(Primitive::kPrimByte, increment_[0], kNoDexPc); loop_body_[0]->InsertInstructionBefore(conv, increment_[0]->GetNext()); basic_[0]->ReplaceInput(conv, 1); PerformInductionVarAnalysis(); - EXPECT_STREQ("((1) * i + ((-128) + (1))):PrimByte", GetInductionInfo(increment_[0], 0).c_str()); + // Recorded at the phi, but not transferred to increment. + EXPECT_STREQ("((1) * i + (-128)):PrimByte", GetInductionInfo(basic_[0], 0).c_str()); + EXPECT_STREQ("", GetInductionInfo(increment_[0], 0).c_str()); + + // Narrowing detected. + EXPECT_TRUE(IsNarrowingLinear(basic_[0])); + EXPECT_FALSE(IsNarrowingLinear(increment_[0])); // works for null + // Trip-count. EXPECT_STREQ("(((127) - (-128)) (TC-loop) ((-128) < (127)))", GetTripCount(0).c_str()); } @@ -1093,12 +1227,20 @@ TEST_F(InductionVarAnalysisTest, ByteLoopControl2) { basic_[0]->ReplaceInput(graph_->GetIntConstant(-128), 0); HInstruction* ifs = loop_header_[0]->GetLastInstruction()->GetPrevious(); ifs->ReplaceInput(graph_->GetIntConstant(128), 1); - HInstruction* conv = new(&allocator_) HTypeConversion(Primitive::kPrimByte, increment_[0], -1); + HInstruction* conv = + new (&allocator_) HTypeConversion(Primitive::kPrimByte, increment_[0], kNoDexPc); loop_body_[0]->InsertInstructionBefore(conv, increment_[0]->GetNext()); basic_[0]->ReplaceInput(conv, 1); PerformInductionVarAnalysis(); - EXPECT_STREQ("((1) * i + ((-128) + (1))):PrimByte", GetInductionInfo(increment_[0], 0).c_str()); + // Recorded at the phi, but not transferred to increment. + EXPECT_STREQ("((1) * i + (-128)):PrimByte", GetInductionInfo(basic_[0], 0).c_str()); + EXPECT_STREQ("", GetInductionInfo(increment_[0], 0).c_str()); + + // Narrowing detected. + EXPECT_TRUE(IsNarrowingLinear(basic_[0])); + EXPECT_FALSE(IsNarrowingLinear(increment_[0])); // works for null + // Trip-count undefined. EXPECT_STREQ("", GetTripCount(0).c_str()); } @@ -1111,13 +1253,20 @@ TEST_F(InductionVarAnalysisTest, ShortLoopControl1) { basic_[0]->ReplaceInput(graph_->GetIntConstant(-32768), 0); HInstruction* ifs = loop_header_[0]->GetLastInstruction()->GetPrevious(); ifs->ReplaceInput(graph_->GetIntConstant(32767), 1); - HInstruction* conv = new(&allocator_) HTypeConversion(Primitive::kPrimShort, increment_[0], -1); + HInstruction* conv = + new (&allocator_) HTypeConversion(Primitive::kPrimShort, increment_[0], kNoDexPc); loop_body_[0]->InsertInstructionBefore(conv, increment_[0]->GetNext()); basic_[0]->ReplaceInput(conv, 1); PerformInductionVarAnalysis(); - EXPECT_STREQ("((1) * i + ((-32768) + (1))):PrimShort", - GetInductionInfo(increment_[0], 0).c_str()); + // Recorded at the phi, but not transferred to increment. + EXPECT_STREQ("((1) * i + (-32768)):PrimShort", GetInductionInfo(basic_[0], 0).c_str()); + EXPECT_STREQ("", GetInductionInfo(increment_[0], 0).c_str()); + + // Narrowing detected. + EXPECT_TRUE(IsNarrowingLinear(basic_[0])); + EXPECT_FALSE(IsNarrowingLinear(increment_[0])); // works for null + // Trip-count. EXPECT_STREQ("(((32767) - (-32768)) (TC-loop) ((-32768) < (32767)))", GetTripCount(0).c_str()); } @@ -1130,13 +1279,20 @@ TEST_F(InductionVarAnalysisTest, ShortLoopControl2) { basic_[0]->ReplaceInput(graph_->GetIntConstant(-32768), 0); HInstruction* ifs = loop_header_[0]->GetLastInstruction()->GetPrevious(); ifs->ReplaceInput(graph_->GetIntConstant(32768), 1); - HInstruction* conv = new(&allocator_) HTypeConversion(Primitive::kPrimShort, increment_[0], -1); + HInstruction* conv = + new (&allocator_) HTypeConversion(Primitive::kPrimShort, increment_[0], kNoDexPc); loop_body_[0]->InsertInstructionBefore(conv, increment_[0]->GetNext()); basic_[0]->ReplaceInput(conv, 1); PerformInductionVarAnalysis(); - EXPECT_STREQ("((1) * i + ((-32768) + (1))):PrimShort", - GetInductionInfo(increment_[0], 0).c_str()); + // Recorded at the phi, but not transferred to increment. + EXPECT_STREQ("((1) * i + (-32768)):PrimShort", GetInductionInfo(basic_[0], 0).c_str()); + EXPECT_STREQ("", GetInductionInfo(increment_[0], 0).c_str()); + + // Narrowing detected. + EXPECT_TRUE(IsNarrowingLinear(basic_[0])); + EXPECT_FALSE(IsNarrowingLinear(increment_[0])); // works for null + // Trip-count undefined. EXPECT_STREQ("", GetTripCount(0).c_str()); } @@ -1148,12 +1304,20 @@ TEST_F(InductionVarAnalysisTest, CharLoopControl1) { BuildLoopNest(1); HInstruction* ifs = loop_header_[0]->GetLastInstruction()->GetPrevious(); ifs->ReplaceInput(graph_->GetIntConstant(65535), 1); - HInstruction* conv = new(&allocator_) HTypeConversion(Primitive::kPrimChar, increment_[0], -1); + HInstruction* conv = + new (&allocator_) HTypeConversion(Primitive::kPrimChar, increment_[0], kNoDexPc); loop_body_[0]->InsertInstructionBefore(conv, increment_[0]->GetNext()); basic_[0]->ReplaceInput(conv, 1); PerformInductionVarAnalysis(); - EXPECT_STREQ("((1) * i + (1)):PrimChar", GetInductionInfo(increment_[0], 0).c_str()); + // Recorded at the phi, but not transferred to increment. + EXPECT_STREQ("((1) * i + (0)):PrimChar", GetInductionInfo(basic_[0], 0).c_str()); + EXPECT_STREQ("", GetInductionInfo(increment_[0], 0).c_str()); + + // Narrowing detected. + EXPECT_TRUE(IsNarrowingLinear(basic_[0])); + EXPECT_FALSE(IsNarrowingLinear(increment_[0])); // works for null + // Trip-count. EXPECT_STREQ("((65535) (TC-loop) ((0) < (65535)))", GetTripCount(0).c_str()); } @@ -1165,12 +1329,20 @@ TEST_F(InductionVarAnalysisTest, CharLoopControl2) { BuildLoopNest(1); HInstruction* ifs = loop_header_[0]->GetLastInstruction()->GetPrevious(); ifs->ReplaceInput(graph_->GetIntConstant(65536), 1); - HInstruction* conv = new(&allocator_) HTypeConversion(Primitive::kPrimChar, increment_[0], -1); + HInstruction* conv = + new (&allocator_) HTypeConversion(Primitive::kPrimChar, increment_[0], kNoDexPc); loop_body_[0]->InsertInstructionBefore(conv, increment_[0]->GetNext()); basic_[0]->ReplaceInput(conv, 1); PerformInductionVarAnalysis(); - EXPECT_STREQ("((1) * i + (1)):PrimChar", GetInductionInfo(increment_[0], 0).c_str()); + // Recorded at the phi, but not transferred to increment. + EXPECT_STREQ("((1) * i + (0)):PrimChar", GetInductionInfo(basic_[0], 0).c_str()); + EXPECT_STREQ("", GetInductionInfo(increment_[0], 0).c_str()); + + // Narrowing detected. + EXPECT_TRUE(IsNarrowingLinear(basic_[0])); + EXPECT_FALSE(IsNarrowingLinear(increment_[0])); // works for null + // Trip-count undefined. EXPECT_STREQ("", GetTripCount(0).c_str()); } diff --git a/compiler/optimizing/induction_var_range.cc b/compiler/optimizing/induction_var_range.cc index e665551012..d5c4c2fa69 100644 --- a/compiler/optimizing/induction_var_range.cc +++ b/compiler/optimizing/induction_var_range.cc @@ -169,8 +169,8 @@ static InductionVarRange::Value CorrectForType(InductionVarRange::Value v, Primi case Primitive::kPrimByte: { // Constants within range only. // TODO: maybe some room for improvement, like allowing widening conversions - const int32_t min = Primitive::MinValueOfIntegralType(type); - const int32_t max = Primitive::MaxValueOfIntegralType(type); + int32_t min = Primitive::MinValueOfIntegralType(type); + int32_t max = Primitive::MaxValueOfIntegralType(type); return (IsConstantValue(v) && min <= v.b_constant && v.b_constant <= max) ? v : InductionVarRange::Value(); @@ -551,7 +551,7 @@ InductionVarRange::Value InductionVarRange::GetPolynomial(HInductionVarAnalysis: int64_t b = 0; if (IsConstant(info->op_a->op_a, kExact, &a) && CanLongValueFitIntoInt(a) && a >= 0 && IsConstant(info->op_a->op_b, kExact, &b) && CanLongValueFitIntoInt(b) && b >= 0) { - // Evaluate bounds on sum_i=0^m-1(a * i + b) + c with a,b >= 0 for known + // Evaluate bounds on sum_i=0^m-1(a * i + b) + c with a,b >= 0 for // maximum index value m as a * (m * (m-1)) / 2 + b * m + c. Value c = GetVal(info->op_b, trip, in_body, is_min); if (is_min) { @@ -629,6 +629,7 @@ InductionVarRange::Value InductionVarRange::GetFetch(HInstruction* instruction, } } else if (instruction->IsTypeConversion()) { // Since analysis is 32-bit (or narrower), chase beyond widening along the path. + // For example, this discovers the length in: for (long i = 0; i < a.length; i++); if (instruction->AsTypeConversion()->GetInputType() == Primitive::kPrimInt && instruction->AsTypeConversion()->GetResultType() == Primitive::kPrimLong) { return GetFetch(instruction->InputAt(0), trip, in_body, is_min); @@ -843,7 +844,7 @@ InductionVarRange::Value InductionVarRange::DivRangeAndConstant( InductionVarRange::Value InductionVarRange::AddValue(Value v1, Value v2) const { if (v1.is_known && v2.is_known && IsSafeAdd(v1.b_constant, v2.b_constant)) { - const int32_t b = v1.b_constant + v2.b_constant; + int32_t b = v1.b_constant + v2.b_constant; if (v1.a_constant == 0) { return Value(v2.instruction, v2.a_constant, b); } else if (v2.a_constant == 0) { @@ -857,7 +858,7 @@ InductionVarRange::Value InductionVarRange::AddValue(Value v1, Value v2) const { InductionVarRange::Value InductionVarRange::SubValue(Value v1, Value v2) const { if (v1.is_known && v2.is_known && IsSafeSub(v1.b_constant, v2.b_constant)) { - const int32_t b = v1.b_constant - v2.b_constant; + int32_t b = v1.b_constant - v2.b_constant; if (v1.a_constant == 0 && IsSafeSub(0, v2.a_constant)) { return Value(v2.instruction, -v2.a_constant, b); } else if (v2.a_constant == 0) { @@ -983,18 +984,21 @@ bool InductionVarRange::GenerateLastValuePolynomial(HInductionVarAnalysis::Induc int64_t a = 0; int64_t b = 0; int64_t m = 0; - if (IsConstant(info->op_a->op_a, kExact, &a) && a >= 0 && - IsConstant(info->op_a->op_b, kExact, &b) && b >= 0 && + if (IsConstant(info->op_a->op_a, kExact, &a) && + IsConstant(info->op_a->op_b, kExact, &b) && IsConstant(trip->op_a, kExact, &m) && m >= 1) { - // Evaluate bounds on sum_i=0^m-1(a * i + b) + c with a,b >= 0 for known + // Evaluate bounds on sum_i=0^m-1(a * i + b) + c for known // maximum index value m as a * (m * (m-1)) / 2 + b * m + c. - // TODO: generalize - HInstruction* c_instr = nullptr; - if (GenerateCode(info->op_b, nullptr, graph, block, graph ? &c_instr : nullptr, false, false)) { + HInstruction* c = nullptr; + if (GenerateCode(info->op_b, nullptr, graph, block, graph ? &c : nullptr, false, false)) { if (graph != nullptr) { + Primitive::Type type = info->type; int64_t sum = a * ((m * (m - 1)) / 2) + b * m; - *result = Insert(block, new (graph->GetArena()) HAdd(info->type, - graph->GetIntConstant(sum), c_instr)); + if (type != Primitive::kPrimLong) { + sum = static_cast<int32_t>(sum); // okay to truncate + } + *result = + Insert(block, new (graph->GetArena()) HAdd(type, graph->GetConstant(type, sum), c)); } return true; } @@ -1011,35 +1015,33 @@ bool InductionVarRange::GenerateLastValueGeometric(HInductionVarAnalysis::Induct DCHECK_EQ(info->induction_class, HInductionVarAnalysis::kGeometric); // Detect known base and trip count (always taken). int64_t f = 0; - int64_t t = 0; - if (IsIntAndGet(info->fetch, &f) && f >= 1 && IsConstant(trip->op_a, kExact, &t) && t >= 1) { + int64_t m = 0; + if (IsIntAndGet(info->fetch, &f) && f >= 1 && IsConstant(trip->op_a, kExact, &m) && m >= 1) { HInstruction* opa = nullptr; HInstruction* opb = nullptr; if (GenerateCode(info->op_a, nullptr, graph, block, &opa, false, false) && GenerateCode(info->op_b, nullptr, graph, block, &opb, false, false)) { - // Compute f ^ t. - int64_t fpowt = IntPow(f, t); + // Compute f ^ m for known maximum index value m. + int64_t fpow = IntPow(f, m); if (graph != nullptr) { - DCHECK(info->type == Primitive::kPrimInt); // due to codegen, generalize? - if (fpowt == 0) { + DCHECK(info->operation == HInductionVarAnalysis::kMul || + info->operation == HInductionVarAnalysis::kDiv); + Primitive::Type type = info->type; + if (fpow == 0) { // Special case: repeated mul/div always yields zero. - *result = graph->GetIntConstant(0); - } else if (info->operation == HInductionVarAnalysis::kMul) { - // Last value multiplication: a * f ^ t + b. - HInstruction* mul = Insert(block, - new (graph->GetArena()) HMul(info->type, - opa, - graph->GetIntConstant(fpowt))); - *result = Insert(block, new (graph->GetArena()) HAdd(info->type, mul, opb)); + *result = graph->GetConstant(type, 0); } else { - // Last value multiplication: a * f ^ -t + b. - DCHECK_EQ(info->operation, HInductionVarAnalysis::kDiv); - HInstruction* div = Insert(block, - new (graph->GetArena()) HDiv(info->type, - opa, - graph->GetIntConstant(fpowt), - kNoDexPc)); - *result = Insert(block, new (graph->GetArena()) HAdd(info->type, div, opb)); + // Last value: a * f ^ m + b or a * f ^ -m + b. + if (type != Primitive::kPrimLong) { + fpow = static_cast<int32_t>(fpow); // okay to truncate + } + HInstruction* e = nullptr; + if (info->operation == HInductionVarAnalysis::kMul) { + e = new (graph->GetArena()) HMul(type, opa, graph->GetConstant(type, fpow)); + } else { + e = new (graph->GetArena()) HDiv(type, opa, graph->GetConstant(type, fpow), kNoDexPc); + } + *result = Insert(block, new (graph->GetArena()) HAdd(type, Insert(block, e), opb)); } } return true; @@ -1060,12 +1062,11 @@ bool InductionVarRange::GenerateLastValueWrapAround(HInductionVarAnalysis::Induc for (; info->induction_class == HInductionVarAnalysis::kWrapAround; info = info->op_b, ++depth) {} // Handle wrap(x, wrap(.., y)) if trip count reaches an invariant at end. - // TODO: generalize - int64_t t = 0; + // TODO: generalize, but be careful to adjust the terminal. + int64_t m = 0; if (info->induction_class == HInductionVarAnalysis::kInvariant && - IsConstant(trip->op_a, kExact, &t) && t >= depth && - GenerateCode(info, nullptr, graph, block, result, false, false)) { - return true; + IsConstant(trip->op_a, kExact, &m) && m >= depth) { + return GenerateCode(info, nullptr, graph, block, result, false, false); } return false; } @@ -1079,43 +1080,49 @@ bool InductionVarRange::GenerateLastValuePeriodic(HInductionVarAnalysis::Inducti DCHECK(info != nullptr); DCHECK_EQ(info->induction_class, HInductionVarAnalysis::kPeriodic); // Count period. - int32_t period = 1; + int64_t period = 1; for (HInductionVarAnalysis::InductionInfo* p = info; p->induction_class == HInductionVarAnalysis::kPeriodic; p = p->op_b, ++period) {} - // Handle periodic(x, y) case for restricted types. - // TODO: generalize - if (period != 2 || - trip->op_a->type != Primitive::kPrimInt || - (info->type != Primitive::kPrimInt && info->type != Primitive::kPrimBoolean)) { - return false; + // Handle any periodic(x, periodic(.., y)) for known maximum index value m. + int64_t m = 0; + if (IsConstant(trip->op_a, kExact, &m) && m >= 1) { + int64_t li = m % period; + for (int64_t i = 0; i < li; info = info->op_b, i++) {} + if (info->induction_class == HInductionVarAnalysis::kPeriodic) { + info = info->op_a; + } + return GenerateCode(info, nullptr, graph, block, result, false, false); } - HInstruction* x_instr = nullptr; - HInstruction* y_instr = nullptr; - HInstruction* trip_expr = nullptr; - if (GenerateCode(info->op_a, nullptr, graph, block, graph ? &x_instr : nullptr, false, false) && - GenerateCode(info->op_b, nullptr, graph, block, graph ? &y_instr : nullptr, false, false) && - GenerateCode(trip->op_a, nullptr, graph, block, graph ? &trip_expr : nullptr, false, false)) { - // During actual code generation (graph != nullptr), - // generate is_even ? x : y select instruction. + // Handle periodic(x, y) using even/odd-select on trip count. Enter trip count expression + // directly to obtain the maximum index value t even if taken test is needed. + HInstruction* x = nullptr; + HInstruction* y = nullptr; + HInstruction* t = nullptr; + if (period == 2 && + GenerateCode(info->op_a, nullptr, graph, block, graph ? &x : nullptr, false, false) && + GenerateCode(info->op_b, nullptr, graph, block, graph ? &y : nullptr, false, false) && + GenerateCode(trip->op_a, nullptr, graph, block, graph ? &t : nullptr, false, false)) { + // During actual code generation (graph != nullptr), generate is_even ? x : y. if (graph != nullptr) { - HInstruction* is_even = Insert(block, new (graph->GetArena()) HEqual( - Insert(block, new (graph->GetArena()) HAnd( - Primitive::kPrimInt, trip_expr, graph->GetIntConstant(1))), - graph->GetIntConstant(0), kNoDexPc)); - *result = Insert(block, new (graph->GetArena()) HSelect(is_even, x_instr, y_instr, kNoDexPc)); + Primitive::Type type = trip->type; + HInstruction* msk = + Insert(block, new (graph->GetArena()) HAnd(type, t, graph->GetConstant(type, 1))); + HInstruction* is_even = + Insert(block, new (graph->GetArena()) HEqual(msk, graph->GetConstant(type, 0), kNoDexPc)); + *result = Insert(block, new (graph->GetArena()) HSelect(is_even, x, y, kNoDexPc)); } // Guard select with taken test if needed. if (*needs_taken_test) { - HInstruction* taken_test = nullptr; - if (!GenerateCode( - trip->op_b, nullptr, graph, block, graph ? &taken_test : nullptr, false, false)) { + HInstruction* is_taken = nullptr; + if (GenerateCode(trip->op_b, nullptr, graph, block, graph ? &is_taken : nullptr, false, false)) { + if (graph != nullptr) { + *result = Insert(block, new (graph->GetArena()) HSelect(is_taken, *result, x, kNoDexPc)); + } + *needs_taken_test = false; // taken care of + } else { return false; - } else if (graph != nullptr) { - *result = Insert(block, - new (graph->GetArena()) HSelect(taken_test, *result, x_instr, kNoDexPc)); } - *needs_taken_test = false; // taken care of } return true; } @@ -1134,13 +1141,8 @@ bool InductionVarRange::GenerateCode(HInductionVarAnalysis::InductionInfo* info, if (graph != nullptr && result == nullptr) { return true; } - // Verify type safety. - // TODO: generalize - Primitive::Type type = Primitive::kPrimInt; - if (info->type != Primitive::kPrimInt && info->type != Primitive::kPrimBoolean) { - return false; - } // Handle current operation. + Primitive::Type type = info->type; HInstruction* opa = nullptr; HInstruction* opb = nullptr; switch (info->induction_class) { @@ -1214,15 +1216,15 @@ bool InductionVarRange::GenerateCode(HInductionVarAnalysis::InductionInfo* info, case HInductionVarAnalysis::kTripCountInBodyUnsafe: if (is_min) { if (graph != nullptr) { - *result = graph->GetIntConstant(0); + *result = graph->GetConstant(type, 0); } return true; } else if (in_body) { if (GenerateCode(info->op_a, trip, graph, block, &opb, in_body, is_min)) { if (graph != nullptr) { - *result = Insert(block, - new (graph->GetArena()) - HSub(type, opb, graph->GetIntConstant(1))); + *result = + Insert(block, + new (graph->GetArena()) HSub(type, opb, graph->GetConstant(type, 1))); } return true; } @@ -1236,26 +1238,31 @@ bool InductionVarRange::GenerateCode(HInductionVarAnalysis::InductionInfo* info, // Linear induction a * i + b, for normalized 0 <= i < TC. For ranges, this should // be restricted to a unit stride to avoid arithmetic wrap-around situations that // are harder to guard against. For a last value, requesting min/max based on any - // stride yields right value. - int64_t stride_value = 0; - if (IsConstant(info->op_a, kExact, &stride_value)) { - const bool is_min_a = stride_value >= 0 ? is_min : !is_min; - if (GenerateCode(trip, trip, graph, block, &opa, in_body, is_min_a) && - GenerateCode(info->op_b, trip, graph, block, &opb, in_body, is_min)) { - if (graph != nullptr) { - HInstruction* oper; - if (stride_value == 1) { - oper = new (graph->GetArena()) HAdd(type, opa, opb); - } else if (stride_value == -1) { - oper = new (graph->GetArena()) HSub(type, opb, opa); - } else { - HInstruction* mul = new (graph->GetArena()) HMul( - type, graph->GetIntConstant(stride_value), opa); - oper = new (graph->GetArena()) HAdd(type, Insert(block, mul), opb); + // known stride yields right value. Always avoid any narrowing linear induction or + // any type mismatch between the linear induction and the trip count expression. + // TODO: careful runtime type conversions could generalize this latter restriction. + if (!HInductionVarAnalysis::IsNarrowingLinear(info) && trip->type == type) { + int64_t stride_value = 0; + if (IsConstant(info->op_a, kExact, &stride_value) && + CanLongValueFitIntoInt(stride_value)) { + const bool is_min_a = stride_value >= 0 ? is_min : !is_min; + if (GenerateCode(trip, trip, graph, block, &opa, in_body, is_min_a) && + GenerateCode(info->op_b, trip, graph, block, &opb, in_body, is_min)) { + if (graph != nullptr) { + HInstruction* oper; + if (stride_value == 1) { + oper = new (graph->GetArena()) HAdd(type, opa, opb); + } else if (stride_value == -1) { + oper = new (graph->GetArena()) HSub(type, opb, opa); + } else { + HInstruction* mul = + new (graph->GetArena()) HMul(type, graph->GetConstant(type, stride_value), opa); + oper = new (graph->GetArena()) HAdd(type, Insert(block, mul), opb); + } + *result = Insert(block, oper); } - *result = Insert(block, oper); + return true; } - return true; } } break; @@ -1270,7 +1277,7 @@ bool InductionVarRange::GenerateCode(HInductionVarAnalysis::InductionInfo* info, Value extreme = GetVal(info, trip, in_body, is_min); if (IsConstantValue(extreme)) { if (graph != nullptr) { - *result = graph->GetIntConstant(extreme.b_constant); + *result = graph->GetConstant(type, extreme.b_constant); } return true; } diff --git a/compiler/optimizing/inliner.cc b/compiler/optimizing/inliner.cc index 8d93867230..d84787984d 100644 --- a/compiler/optimizing/inliner.cc +++ b/compiler/optimizing/inliner.cc @@ -480,13 +480,11 @@ bool HInliner::TryInlineMonomorphicCall(HInvoke* invoke_instruction, } // We successfully inlined, now add a guard. - bool is_referrer = - (GetMonomorphicType(classes) == outermost_graph_->GetArtMethod()->GetDeclaringClass()); AddTypeGuard(receiver, cursor, bb_cursor, class_index, - is_referrer, + GetMonomorphicType(classes), invoke_instruction, /* with_deoptimization */ true); @@ -506,52 +504,62 @@ void HInliner::AddCHAGuard(HInstruction* invoke_instruction, uint32_t dex_pc, HInstruction* cursor, HBasicBlock* bb_cursor) { - HInstruction* deopt_flag = new (graph_->GetArena()) HShouldDeoptimizeFlag(dex_pc); - HInstruction* should_deopt = new (graph_->GetArena()) HNotEqual( + HShouldDeoptimizeFlag* deopt_flag = new (graph_->GetArena()) + HShouldDeoptimizeFlag(graph_->GetArena(), dex_pc); + HInstruction* compare = new (graph_->GetArena()) HNotEqual( deopt_flag, graph_->GetIntConstant(0, dex_pc)); - HInstruction* deopt = new (graph_->GetArena()) HDeoptimize(should_deopt, dex_pc); + HInstruction* deopt = new (graph_->GetArena()) HDeoptimize(compare, dex_pc); if (cursor != nullptr) { bb_cursor->InsertInstructionAfter(deopt_flag, cursor); } else { bb_cursor->InsertInstructionBefore(deopt_flag, bb_cursor->GetFirstInstruction()); } - bb_cursor->InsertInstructionAfter(should_deopt, deopt_flag); - bb_cursor->InsertInstructionAfter(deopt, should_deopt); + bb_cursor->InsertInstructionAfter(compare, deopt_flag); + bb_cursor->InsertInstructionAfter(deopt, compare); + + // Add receiver as input to aid CHA guard optimization later. + deopt_flag->AddInput(invoke_instruction->InputAt(0)); + DCHECK_EQ(deopt_flag->InputCount(), 1u); deopt->CopyEnvironmentFrom(invoke_instruction->GetEnvironment()); + outermost_graph_->IncrementNumberOfCHAGuards(); } HInstruction* HInliner::AddTypeGuard(HInstruction* receiver, HInstruction* cursor, HBasicBlock* bb_cursor, dex::TypeIndex class_index, - bool is_referrer, + mirror::Class* klass, HInstruction* invoke_instruction, bool with_deoptimization) { + ScopedAssertNoThreadSuspension sants("Adding compiler type guard"); + ClassLinker* class_linker = caller_compilation_unit_.GetClassLinker(); HInstanceFieldGet* receiver_class = BuildGetReceiverClass( class_linker, receiver, invoke_instruction->GetDexPc()); + if (cursor != nullptr) { + bb_cursor->InsertInstructionAfter(receiver_class, cursor); + } else { + bb_cursor->InsertInstructionBefore(receiver_class, bb_cursor->GetFirstInstruction()); + } const DexFile& caller_dex_file = *caller_compilation_unit_.GetDexFile(); + bool is_referrer = (klass == outermost_graph_->GetArtMethod()->GetDeclaringClass()); // Note that we will just compare the classes, so we don't need Java semantics access checks. - // Also, the caller of `AddTypeGuard` must have guaranteed that the class is in the dex cache. + // Note that the type index and the dex file are relative to the method this type guard is + // inlined into. HLoadClass* load_class = new (graph_->GetArena()) HLoadClass(graph_->GetCurrentMethod(), class_index, caller_dex_file, is_referrer, invoke_instruction->GetDexPc(), - /* needs_access_check */ false, - /* is_in_dex_cache */ true, - /* is_in_boot_image */ false); + /* needs_access_check */ false); + bb_cursor->InsertInstructionAfter(load_class, receiver_class); + // Sharpen after adding the instruction, as the sharpening may remove inputs. + HSharpening::SharpenClass(load_class, klass, handles_, codegen_, compiler_driver_); - HNotEqual* compare = new (graph_->GetArena()) HNotEqual(load_class, receiver_class); // TODO: Extend reference type propagation to understand the guard. - if (cursor != nullptr) { - bb_cursor->InsertInstructionAfter(receiver_class, cursor); - } else { - bb_cursor->InsertInstructionBefore(receiver_class, bb_cursor->GetFirstInstruction()); - } - bb_cursor->InsertInstructionAfter(load_class, receiver_class); + HNotEqual* compare = new (graph_->GetArena()) HNotEqual(load_class, receiver_class); bb_cursor->InsertInstructionAfter(compare, load_class); if (with_deoptimization) { HDeoptimize* deoptimize = new (graph_->GetArena()) HDeoptimize( @@ -604,7 +612,6 @@ bool HInliner::TryInlinePolymorphicCall(HInvoke* invoke_instruction, all_targets_inlined = false; } else { one_target_inlined = true; - bool is_referrer = (classes->Get(i) == outermost_graph_->GetArtMethod()->GetDeclaringClass()); // If we have inlined all targets before, and this receiver is the last seen, // we deoptimize instead of keeping the original invoke instruction. @@ -616,8 +623,13 @@ bool HInliner::TryInlinePolymorphicCall(HInvoke* invoke_instruction, // We do not support HDeoptimize in OSR methods. deoptimize = false; } - HInstruction* compare = AddTypeGuard( - receiver, cursor, bb_cursor, class_index, is_referrer, invoke_instruction, deoptimize); + HInstruction* compare = AddTypeGuard(receiver, + cursor, + bb_cursor, + class_index, + classes->Get(i), + invoke_instruction, + deoptimize); if (deoptimize) { if (return_replacement != nullptr) { invoke_instruction->ReplaceWith(return_replacement); @@ -1444,7 +1456,7 @@ size_t HInliner::RunOptimizations(HGraph* callee_graph, // optimization that could lead to a HDeoptimize. The following optimizations do not. HDeadCodeElimination dce(callee_graph, stats_, "dead_code_elimination$inliner"); HConstantFolding fold(callee_graph, "constant_folding$inliner"); - HSharpening sharpening(callee_graph, codegen_, dex_compilation_unit, compiler_driver_); + HSharpening sharpening(callee_graph, codegen_, dex_compilation_unit, compiler_driver_, handles_); InstructionSimplifier simplify(callee_graph, stats_); IntrinsicsRecognizer intrinsics(callee_graph, stats_); diff --git a/compiler/optimizing/inliner.h b/compiler/optimizing/inliner.h index ffebd97cb8..0c6436235f 100644 --- a/compiler/optimizing/inliner.h +++ b/compiler/optimizing/inliner.h @@ -167,7 +167,7 @@ class HInliner : public HOptimization { HInstruction* cursor, HBasicBlock* bb_cursor, dex::TypeIndex class_index, - bool is_referrer, + mirror::Class* klass, HInstruction* invoke_instruction, bool with_deoptimization) REQUIRES_SHARED(Locks::mutator_lock_); diff --git a/compiler/optimizing/instruction_builder.cc b/compiler/optimizing/instruction_builder.cc index b97581beb3..af8e2c8a7c 100644 --- a/compiler/optimizing/instruction_builder.cc +++ b/compiler/optimizing/instruction_builder.cc @@ -816,8 +816,7 @@ bool HInstructionBuilder::BuildInvoke(const Instruction& instruction, HInvokeStaticOrDirect::DispatchInfo dispatch_info = { HInvokeStaticOrDirect::MethodLoadKind::kStringInit, HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod, - dchecked_integral_cast<uint64_t>(string_init_entry_point), - 0U + dchecked_integral_cast<uint64_t>(string_init_entry_point) }; MethodReference target_method(dex_file_, method_idx); HInvoke* invoke = new (arena_) HInvokeStaticOrDirect( @@ -862,8 +861,7 @@ bool HInstructionBuilder::BuildInvoke(const Instruction& instruction, HInvokeStaticOrDirect::DispatchInfo dispatch_info = { HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod, HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod, - 0u, - 0U + 0u }; MethodReference target_method(resolved_method->GetDexFile(), resolved_method->GetDexMethodIndex()); @@ -937,9 +935,7 @@ bool HInstructionBuilder::BuildNewInstance(dex::TypeIndex type_index, uint32_t d outer_dex_file, IsOutermostCompilingClass(type_index), dex_pc, - needs_access_check, - /* is_in_dex_cache */ false, - /* is_in_boot_image */ false); + needs_access_check); AppendInstruction(load_class); HInstruction* cls = load_class; @@ -1029,9 +1025,7 @@ HClinitCheck* HInstructionBuilder::ProcessClinitCheckForInvoke( outer_dex_file, is_outer_class, dex_pc, - /*needs_access_check*/ false, - /* is_in_dex_cache */ false, - /* is_in_boot_image */ false); + /*needs_access_check*/ false); AppendInstruction(load_class); clinit_check = new (arena_) HClinitCheck(load_class, dex_pc); AppendInstruction(clinit_check); @@ -1388,9 +1382,7 @@ bool HInstructionBuilder::BuildStaticFieldAccess(const Instruction& instruction, outer_dex_file, is_outer_class, dex_pc, - /*needs_access_check*/ false, - /* is_in_dex_cache */ false, - /* is_in_boot_image */ false); + /*needs_access_check*/ false); AppendInstruction(constant); HInstruction* cls = constant; @@ -1664,9 +1656,7 @@ void HInstructionBuilder::BuildTypeCheck(const Instruction& instruction, dex_file, IsOutermostCompilingClass(type_index), dex_pc, - !can_access, - /* is_in_dex_cache */ false, - /* is_in_boot_image */ false); + !can_access); AppendInstruction(cls); TypeCheckKind check_kind = ComputeTypeCheckKind(resolved_class); @@ -2656,9 +2646,7 @@ bool HInstructionBuilder::ProcessDexInstruction(const Instruction& instruction, *dex_file_, IsOutermostCompilingClass(type_index), dex_pc, - !can_access, - /* is_in_dex_cache */ false, - /* is_in_boot_image */ false)); + !can_access)); UpdateLocal(instruction.VRegA_21c(), current_block_->GetLastInstruction()); break; } diff --git a/compiler/optimizing/instruction_simplifier.cc b/compiler/optimizing/instruction_simplifier.cc index 658b80468e..439e3b66db 100644 --- a/compiler/optimizing/instruction_simplifier.cc +++ b/compiler/optimizing/instruction_simplifier.cc @@ -1185,6 +1185,18 @@ void InstructionSimplifierVisitor::VisitCondition(HCondition* condition) { RecordSimplification(); } +// Return whether x / divisor == x * (1.0f / divisor), for every float x. +static constexpr bool CanDivideByReciprocalMultiplyFloat(int32_t divisor) { + // True, if the most significant bits of divisor are 0. + return ((divisor & 0x7fffff) == 0); +} + +// Return whether x / divisor == x * (1.0 / divisor), for every double x. +static constexpr bool CanDivideByReciprocalMultiplyDouble(int64_t divisor) { + // True, if the most significant bits of divisor are 0. + return ((divisor & ((UINT64_C(1) << 52) - 1)) == 0); +} + void InstructionSimplifierVisitor::VisitDiv(HDiv* instruction) { HConstant* input_cst = instruction->GetConstantRight(); HInstruction* input_other = instruction->GetLeastConstantLeft(); @@ -1885,7 +1897,8 @@ void InstructionSimplifierVisitor::SimplifyReturnThis(HInvoke* invoke) { static bool NoEscapeForStringBufferReference(HInstruction* reference, HInstruction* user) { if (user->IsInvokeStaticOrDirect()) { // Any constructor on StringBuffer is okay. - return user->AsInvokeStaticOrDirect()->GetResolvedMethod()->IsConstructor() && + return user->AsInvokeStaticOrDirect()->GetResolvedMethod() != nullptr && + user->AsInvokeStaticOrDirect()->GetResolvedMethod()->IsConstructor() && user->InputAt(0) == reference; } else if (user->IsInvokeVirtual()) { switch (user->AsInvokeVirtual()->GetIntrinsic()) { diff --git a/compiler/optimizing/intrinsics_arm_vixl.cc b/compiler/optimizing/intrinsics_arm_vixl.cc index 433dced9d7..85e84d8d2c 100644 --- a/compiler/optimizing/intrinsics_arm_vixl.cc +++ b/compiler/optimizing/intrinsics_arm_vixl.cc @@ -47,6 +47,9 @@ using helpers::SRegisterFrom; using namespace vixl::aarch32; // NOLINT(build/namespaces) +using vixl::ExactAssemblyScope; +using vixl::CodeBufferCheckScope; + ArmVIXLAssembler* IntrinsicCodeGeneratorARMVIXL::GetAssembler() { return codegen_->GetAssembler(); } @@ -184,7 +187,7 @@ class ReadBarrierSystemArrayCopySlowPathARMVIXL : public SlowPathCodeARMVIXL { assembler->MaybePoisonHeapReference(tmp); __ Str(tmp, MemOperand(dst_curr_addr, element_size, PostIndex)); __ Cmp(src_curr_addr, src_stop_addr); - __ B(ne, &loop); + __ B(ne, &loop, /* far_target */ false); __ B(GetExitLabel()); } @@ -467,9 +470,9 @@ static void GenMinMax(HInvoke* invoke, bool is_min, ArmVIXLAssembler* assembler) __ Cmp(op1, op2); { - AssemblerAccurateScope aas(assembler->GetVIXLAssembler(), - 3 * kMaxInstructionSizeInBytes, - CodeBufferCheckScope::kMaximumSize); + ExactAssemblyScope aas(assembler->GetVIXLAssembler(), + 3 * kMaxInstructionSizeInBytes, + CodeBufferCheckScope::kMaximumSize); __ ite(is_min ? lt : gt); __ mov(is_min ? lt : gt, out, op1); @@ -848,7 +851,7 @@ static void GenUnsafePut(LocationSummary* locations, __ Ldrexd(temp_lo, temp_hi, MemOperand(temp_reg)); __ Strexd(temp_lo, value_lo, value_hi, MemOperand(temp_reg)); __ Cmp(temp_lo, 0); - __ B(ne, &loop_head); + __ B(ne, &loop_head, /* far_target */ false); } else { __ Strd(value_lo, value_hi, MemOperand(base, offset)); } @@ -1050,25 +1053,25 @@ static void GenCas(HInvoke* invoke, Primitive::Type type, CodeGeneratorARMVIXL* __ Subs(tmp, tmp, expected); { - AssemblerAccurateScope aas(assembler->GetVIXLAssembler(), - 3 * kMaxInstructionSizeInBytes, - CodeBufferCheckScope::kMaximumSize); + ExactAssemblyScope aas(assembler->GetVIXLAssembler(), + 3 * kMaxInstructionSizeInBytes, + CodeBufferCheckScope::kMaximumSize); __ itt(eq); __ strex(eq, tmp, value, MemOperand(tmp_ptr)); __ cmp(eq, tmp, 1); } - __ B(eq, &loop_head); + __ B(eq, &loop_head, /* far_target */ false); __ Dmb(vixl32::ISH); __ Rsbs(out, tmp, 1); { - AssemblerAccurateScope aas(assembler->GetVIXLAssembler(), - 2 * kMaxInstructionSizeInBytes, - CodeBufferCheckScope::kMaximumSize); + ExactAssemblyScope aas(assembler->GetVIXLAssembler(), + 2 * kMaxInstructionSizeInBytes, + CodeBufferCheckScope::kMaximumSize); __ it(cc); __ mov(cc, out, 0); @@ -1185,9 +1188,9 @@ void IntrinsicCodeGeneratorARMVIXL::VisitStringCompareTo(HInvoke* invoke) { // temp0 = min(len(str), len(arg)). { - AssemblerAccurateScope aas(assembler->GetVIXLAssembler(), - 2 * kMaxInstructionSizeInBytes, - CodeBufferCheckScope::kMaximumSize); + ExactAssemblyScope aas(assembler->GetVIXLAssembler(), + 2 * kMaxInstructionSizeInBytes, + CodeBufferCheckScope::kMaximumSize); __ it(gt); __ mov(gt, temp0, temp1); @@ -1207,9 +1210,9 @@ void IntrinsicCodeGeneratorARMVIXL::VisitStringCompareTo(HInvoke* invoke) { // This could in theory exceed INT32_MAX, so treat temp0 as unsigned. __ Lsls(temp3, temp3, 31u); // Extract purely the compression flag. - AssemblerAccurateScope aas(assembler->GetVIXLAssembler(), - 2 * kMaxInstructionSizeInBytes, - CodeBufferCheckScope::kMaximumSize); + ExactAssemblyScope aas(assembler->GetVIXLAssembler(), + 2 * kMaxInstructionSizeInBytes, + CodeBufferCheckScope::kMaximumSize); __ it(ne); __ add(ne, temp0, temp0, temp0); @@ -1235,23 +1238,23 @@ void IntrinsicCodeGeneratorARMVIXL::VisitStringCompareTo(HInvoke* invoke) { __ Ldr(temp_reg, MemOperand(str, temp1)); __ Ldr(temp2, MemOperand(arg, temp1)); __ Cmp(temp_reg, temp2); - __ B(ne, &find_char_diff); + __ B(ne, &find_char_diff, /* far_target */ false); __ Add(temp1, temp1, char_size * 2); __ Ldr(temp_reg, MemOperand(str, temp1)); __ Ldr(temp2, MemOperand(arg, temp1)); __ Cmp(temp_reg, temp2); - __ B(ne, &find_char_diff_2nd_cmp); + __ B(ne, &find_char_diff_2nd_cmp, /* far_target */ false); __ Add(temp1, temp1, char_size * 2); // With string compression, we have compared 8 bytes, otherwise 4 chars. __ Subs(temp0, temp0, (mirror::kUseStringCompression ? 8 : 4)); - __ B(hi, &loop); + __ B(hi, &loop, /* far_target */ false); __ B(&end); __ Bind(&find_char_diff_2nd_cmp); if (mirror::kUseStringCompression) { __ Subs(temp0, temp0, 4); // 4 bytes previously compared. - __ B(ls, &end); // Was the second comparison fully beyond the end? + __ B(ls, &end, /* far_target */ false); // Was the second comparison fully beyond the end? } else { // Without string compression, we can start treating temp0 as signed // and rely on the signed comparison below. @@ -1279,7 +1282,7 @@ void IntrinsicCodeGeneratorARMVIXL::VisitStringCompareTo(HInvoke* invoke) { // the remaining string data, so just return length diff (out). // The comparison is unsigned for string compression, otherwise signed. __ Cmp(temp0, Operand(temp1, vixl32::LSR, (mirror::kUseStringCompression ? 3 : 4))); - __ B((mirror::kUseStringCompression ? ls : le), &end); + __ B((mirror::kUseStringCompression ? ls : le), &end, /* far_target */ false); // Extract the characters and calculate the difference. if (mirror::kUseStringCompression) { @@ -1324,9 +1327,9 @@ void IntrinsicCodeGeneratorARMVIXL::VisitStringCompareTo(HInvoke* invoke) { __ Mov(temp2, arg); __ Lsrs(temp3, temp3, 1u); // Continue the move of the compression flag. { - AssemblerAccurateScope aas(assembler->GetVIXLAssembler(), - 3 * kMaxInstructionSizeInBytes, - CodeBufferCheckScope::kMaximumSize); + ExactAssemblyScope aas(assembler->GetVIXLAssembler(), + 3 * kMaxInstructionSizeInBytes, + CodeBufferCheckScope::kMaximumSize); __ itt(cs); // Interleave with selection of temp1 and temp2. __ mov(cs, temp1, arg); // Preserves flags. __ mov(cs, temp2, str); // Preserves flags. @@ -1346,9 +1349,9 @@ void IntrinsicCodeGeneratorARMVIXL::VisitStringCompareTo(HInvoke* invoke) { __ Ldrb(temp_reg, MemOperand(temp1, c_char_size, PostIndex)); __ Ldrh(temp3, MemOperand(temp2, char_size, PostIndex)); __ Cmp(temp_reg, temp3); - __ B(ne, &different_compression_diff); + __ B(ne, &different_compression_diff, /* far_target */ false); __ Subs(temp0, temp0, 2); - __ B(hi, &different_compression_loop); + __ B(hi, &different_compression_loop, /* far_target */ false); __ B(&end); // Calculate the difference. @@ -1361,9 +1364,9 @@ void IntrinsicCodeGeneratorARMVIXL::VisitStringCompareTo(HInvoke* invoke) { static_assert(static_cast<uint32_t>(mirror::StringCompressionFlag::kCompressed) == 0u, "Expecting 0=compressed, 1=uncompressed"); - AssemblerAccurateScope aas(assembler->GetVIXLAssembler(), - 2 * kMaxInstructionSizeInBytes, - CodeBufferCheckScope::kMaximumSize); + ExactAssemblyScope aas(assembler->GetVIXLAssembler(), + 2 * kMaxInstructionSizeInBytes, + CodeBufferCheckScope::kMaximumSize); __ it(cc); __ rsb(cc, out, out, 0); } @@ -1424,7 +1427,7 @@ void IntrinsicCodeGeneratorARMVIXL::VisitStringEquals(HInvoke* invoke) { // Reference equality check, return true if same reference. __ Cmp(str, arg); - __ B(eq, &return_true); + __ B(eq, &return_true, /* far_target */ false); if (!optimizations.GetArgumentIsString()) { // Instanceof check for the argument by comparing class fields. @@ -1434,7 +1437,7 @@ void IntrinsicCodeGeneratorARMVIXL::VisitStringEquals(HInvoke* invoke) { __ Ldr(temp, MemOperand(str, class_offset)); __ Ldr(temp1, MemOperand(arg, class_offset)); __ Cmp(temp, temp1); - __ B(ne, &return_false); + __ B(ne, &return_false, /* far_target */ false); } // Load `count` fields of this and argument strings. @@ -1443,7 +1446,7 @@ void IntrinsicCodeGeneratorARMVIXL::VisitStringEquals(HInvoke* invoke) { // Check if `count` fields are equal, return false if they're not. // Also compares the compression style, if differs return false. __ Cmp(temp, temp1); - __ B(ne, &return_false); + __ B(ne, &return_false, /* far_target */ false); // Return true if both strings are empty. Even with string compression `count == 0` means empty. static_assert(static_cast<uint32_t>(mirror::StringCompressionFlag::kCompressed) == 0u, "Expecting 0=compressed, 1=uncompressed"); @@ -1457,9 +1460,9 @@ void IntrinsicCodeGeneratorARMVIXL::VisitStringEquals(HInvoke* invoke) { // For string compression, calculate the number of bytes to compare (not chars). // This could in theory exceed INT32_MAX, so treat temp as unsigned. __ Lsrs(temp, temp, 1u); // Extract length and check compression flag. - AssemblerAccurateScope aas(assembler->GetVIXLAssembler(), - 2 * kMaxInstructionSizeInBytes, - CodeBufferCheckScope::kMaximumSize); + ExactAssemblyScope aas(assembler->GetVIXLAssembler(), + 2 * kMaxInstructionSizeInBytes, + CodeBufferCheckScope::kMaximumSize); __ it(cs); // If uncompressed, __ add(cs, temp, temp, temp); // double the byte count. } @@ -1474,10 +1477,10 @@ void IntrinsicCodeGeneratorARMVIXL::VisitStringEquals(HInvoke* invoke) { __ Ldr(temp2, MemOperand(arg, temp1)); __ Add(temp1, temp1, Operand::From(sizeof(uint32_t))); __ Cmp(out, temp2); - __ B(ne, &return_false); + __ B(ne, &return_false, /* far_target */ false); // With string compression, we have compared 4 bytes, otherwise 2 chars. __ Subs(temp, temp, mirror::kUseStringCompression ? 4 : 2); - __ B(hi, &loop); + __ B(hi, &loop, /* far_target */ false); // Return true and exit the function. // If loop does not result in returning false, we return true. @@ -1506,7 +1509,7 @@ static void GenerateVisitStringIndexOf(HInvoke* invoke, SlowPathCodeARMVIXL* slow_path = nullptr; HInstruction* code_point = invoke->InputAt(1); if (code_point->IsIntConstant()) { - if (static_cast<uint32_t>(code_point->AsIntConstant()->GetValue()) > + if (static_cast<uint32_t>(Int32ConstantFrom(code_point)) > std::numeric_limits<uint16_t>::max()) { // Always needs the slow-path. We could directly dispatch to it, but this case should be // rare, so for simplicity just put the full slow-path down and branch unconditionally. @@ -1797,7 +1800,7 @@ void IntrinsicCodeGeneratorARMVIXL::VisitSystemArrayCopy(HInvoke* invoke) { } else { if (!optimizations.GetDestinationIsSource()) { __ Cmp(src, dest); - __ B(ne, &conditions_on_positions_validated); + __ B(ne, &conditions_on_positions_validated, /* far_target */ false); } __ Cmp(RegisterFrom(dest_pos), src_pos_constant); __ B(gt, intrinsic_slow_path->GetEntryLabel()); @@ -1805,7 +1808,7 @@ void IntrinsicCodeGeneratorARMVIXL::VisitSystemArrayCopy(HInvoke* invoke) { } else { if (!optimizations.GetDestinationIsSource()) { __ Cmp(src, dest); - __ B(ne, &conditions_on_positions_validated); + __ B(ne, &conditions_on_positions_validated, /* far_target */ false); } if (dest_pos.IsConstant()) { int32_t dest_pos_constant = Int32ConstantFrom(dest_pos); @@ -1913,7 +1916,7 @@ void IntrinsicCodeGeneratorARMVIXL::VisitSystemArrayCopy(HInvoke* invoke) { if (optimizations.GetDestinationIsTypedObjectArray()) { vixl32::Label do_copy; - __ B(eq, &do_copy); + __ B(eq, &do_copy, /* far_target */ false); // /* HeapReference<Class> */ temp1 = temp1->component_type_ codegen_->GenerateFieldLoadWithBakerReadBarrier( invoke, temp1_loc, temp1, component_offset, temp2_loc, /* needs_null_check */ false); @@ -1973,7 +1976,7 @@ void IntrinsicCodeGeneratorARMVIXL::VisitSystemArrayCopy(HInvoke* invoke) { if (optimizations.GetDestinationIsTypedObjectArray()) { vixl32::Label do_copy; - __ B(eq, &do_copy); + __ B(eq, &do_copy, /* far_target */ false); if (!did_unpoison) { assembler->MaybeUnpoisonHeapReference(temp1); } @@ -2066,7 +2069,7 @@ void IntrinsicCodeGeneratorARMVIXL::VisitSystemArrayCopy(HInvoke* invoke) { // Don't enter copy loop if `length == 0`. __ Cmp(temp1, temp3); - __ B(eq, &done); + __ B(eq, &done, /* far_target */ false); // /* int32_t */ monitor = src->monitor_ __ Ldr(temp2, MemOperand(src, monitor_offset)); @@ -2119,7 +2122,7 @@ void IntrinsicCodeGeneratorARMVIXL::VisitSystemArrayCopy(HInvoke* invoke) { } __ Cmp(temp1, temp3); - __ B(ne, &loop); + __ B(ne, &loop, /* far_target */ false); __ Bind(read_barrier_slow_path->GetExitLabel()); __ Bind(&done); @@ -2139,7 +2142,7 @@ void IntrinsicCodeGeneratorARMVIXL::VisitSystemArrayCopy(HInvoke* invoke) { // poison/unpoison. vixl32::Label loop, done; __ Cmp(temp1, temp3); - __ B(eq, &done); + __ B(eq, &done, /* far_target */ false); __ Bind(&loop); { @@ -2151,7 +2154,7 @@ void IntrinsicCodeGeneratorARMVIXL::VisitSystemArrayCopy(HInvoke* invoke) { } __ Cmp(temp1, temp3); - __ B(ne, &loop); + __ B(ne, &loop, /* far_target */ false); __ Bind(&done); } @@ -2557,7 +2560,7 @@ void IntrinsicCodeGeneratorARMVIXL::VisitStringGetCharsNoCheck(HInvoke* invoke) __ Subs(num_chr, srcEnd, srcBegin); // Early out for valid zero-length retrievals. - __ B(eq, &done); + __ B(eq, &done, /* far_target */ false); // src range to copy. __ Add(src_ptr, srcObj, value_offset); @@ -2573,7 +2576,7 @@ void IntrinsicCodeGeneratorARMVIXL::VisitStringGetCharsNoCheck(HInvoke* invoke) __ Ldr(temp, MemOperand(srcObj, count_offset)); __ Tst(temp, 1); temps.Release(temp); - __ B(eq, &compressed_string_preloop); + __ B(eq, &compressed_string_preloop, /* far_target */ false); } __ Add(src_ptr, src_ptr, Operand(srcBegin, vixl32::LSL, 1)); @@ -2583,7 +2586,7 @@ void IntrinsicCodeGeneratorARMVIXL::VisitStringGetCharsNoCheck(HInvoke* invoke) temp = temps.Acquire(); // Save repairing the value of num_chr on the < 4 character path. __ Subs(temp, num_chr, 4); - __ B(lt, &remainder); + __ B(lt, &remainder, /* far_target */ false); // Keep the result of the earlier subs, we are going to fetch at least 4 characters. __ Mov(num_chr, temp); @@ -2598,10 +2601,10 @@ void IntrinsicCodeGeneratorARMVIXL::VisitStringGetCharsNoCheck(HInvoke* invoke) __ Ldr(temp, MemOperand(src_ptr, char_size * 4, PostIndex)); __ Str(temp, MemOperand(dst_ptr, char_size * 4, PostIndex)); temps.Release(temp); - __ B(ge, &loop); + __ B(ge, &loop, /* far_target */ false); __ Adds(num_chr, num_chr, 4); - __ B(eq, &done); + __ B(eq, &done, /* far_target */ false); // Main loop for < 4 character case and remainder handling. Loads and stores one // 16-bit Java character at a time. @@ -2611,7 +2614,7 @@ void IntrinsicCodeGeneratorARMVIXL::VisitStringGetCharsNoCheck(HInvoke* invoke) __ Subs(num_chr, num_chr, 1); __ Strh(temp, MemOperand(dst_ptr, char_size, PostIndex)); temps.Release(temp); - __ B(gt, &remainder); + __ B(gt, &remainder, /* far_target */ false); if (mirror::kUseStringCompression) { __ B(&done); @@ -2627,7 +2630,7 @@ void IntrinsicCodeGeneratorARMVIXL::VisitStringGetCharsNoCheck(HInvoke* invoke) __ Strh(temp, MemOperand(dst_ptr, char_size, PostIndex)); temps.Release(temp); __ Subs(num_chr, num_chr, 1); - __ B(gt, &compressed_string_loop); + __ B(gt, &compressed_string_loop, /* far_target */ false); } __ Bind(&done); diff --git a/compiler/optimizing/intrinsics_mips.cc b/compiler/optimizing/intrinsics_mips.cc index 9b5d7a02dd..1fb90e5113 100644 --- a/compiler/optimizing/intrinsics_mips.cc +++ b/compiler/optimizing/intrinsics_mips.cc @@ -1648,7 +1648,8 @@ static void GenUnsafePut(LocationSummary* locations, } if (type == Primitive::kPrimNot) { - codegen->MarkGCCard(base, locations->InAt(3).AsRegister<Register>()); + bool value_can_be_null = true; // TODO: Worth finding out this information? + codegen->MarkGCCard(base, locations->InAt(3).AsRegister<Register>(), value_can_be_null); } } @@ -1806,7 +1807,8 @@ static void GenCas(LocationSummary* locations, Primitive::Type type, CodeGenerat if (type == Primitive::kPrimNot) { // Mark card for object assuming new value is stored. - codegen->MarkGCCard(base, value); + bool value_can_be_null = true; // TODO: Worth finding out this information? + codegen->MarkGCCard(base, value, value_can_be_null); } // do { diff --git a/compiler/optimizing/intrinsics_mips64.cc b/compiler/optimizing/intrinsics_mips64.cc index 5a998861eb..3022e975e8 100644 --- a/compiler/optimizing/intrinsics_mips64.cc +++ b/compiler/optimizing/intrinsics_mips64.cc @@ -1846,6 +1846,84 @@ void IntrinsicCodeGeneratorMIPS64::VisitDoubleIsInfinite(HInvoke* invoke) { GenIsInfinite(invoke->GetLocations(), /* is64bit */ true, GetAssembler()); } +// void java.lang.String.getChars(int srcBegin, int srcEnd, char[] dst, int dstBegin) +void IntrinsicLocationsBuilderMIPS64::VisitStringGetCharsNoCheck(HInvoke* invoke) { + LocationSummary* locations = new (arena_) LocationSummary(invoke, + LocationSummary::kCallOnMainOnly, + kIntrinsified); + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetInAt(1, Location::RequiresRegister()); + locations->SetInAt(2, Location::RequiresRegister()); + locations->SetInAt(3, Location::RequiresRegister()); + locations->SetInAt(4, Location::RequiresRegister()); + + // We will call memcpy() to do the actual work. Allocate the temporary + // registers to use the correct input registers, and output register. + // memcpy() uses the normal MIPS calling conventions. + InvokeRuntimeCallingConvention calling_convention; + + locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); + locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(1))); + locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(2))); + + Location outLocation = calling_convention.GetReturnLocation(Primitive::kPrimLong); + locations->AddTemp(Location::RegisterLocation(outLocation.AsRegister<GpuRegister>())); +} + +void IntrinsicCodeGeneratorMIPS64::VisitStringGetCharsNoCheck(HInvoke* invoke) { + Mips64Assembler* assembler = GetAssembler(); + LocationSummary* locations = invoke->GetLocations(); + + // Check assumption that sizeof(Char) is 2 (used in scaling below). + const size_t char_size = Primitive::ComponentSize(Primitive::kPrimChar); + DCHECK_EQ(char_size, 2u); + const size_t char_shift = Primitive::ComponentSizeShift(Primitive::kPrimChar); + + GpuRegister srcObj = locations->InAt(0).AsRegister<GpuRegister>(); + GpuRegister srcBegin = locations->InAt(1).AsRegister<GpuRegister>(); + GpuRegister srcEnd = locations->InAt(2).AsRegister<GpuRegister>(); + GpuRegister dstObj = locations->InAt(3).AsRegister<GpuRegister>(); + GpuRegister dstBegin = locations->InAt(4).AsRegister<GpuRegister>(); + + GpuRegister dstPtr = locations->GetTemp(0).AsRegister<GpuRegister>(); + DCHECK_EQ(dstPtr, A0); + GpuRegister srcPtr = locations->GetTemp(1).AsRegister<GpuRegister>(); + DCHECK_EQ(srcPtr, A1); + GpuRegister numChrs = locations->GetTemp(2).AsRegister<GpuRegister>(); + DCHECK_EQ(numChrs, A2); + + GpuRegister dstReturn = locations->GetTemp(3).AsRegister<GpuRegister>(); + DCHECK_EQ(dstReturn, V0); + + Mips64Label done; + + // Location of data in char array buffer. + const uint32_t data_offset = mirror::Array::DataOffset(char_size).Uint32Value(); + + // Get offset of value field within a string object. + const int32_t value_offset = mirror::String::ValueOffset().Int32Value(); + + __ Beqc(srcEnd, srcBegin, &done); // No characters to move. + + // Calculate number of characters to be copied. + __ Dsubu(numChrs, srcEnd, srcBegin); + + // Calculate destination address. + __ Daddiu(dstPtr, dstObj, data_offset); + __ Dlsa(dstPtr, dstBegin, dstPtr, char_shift); + + // Calculate source address. + __ Daddiu(srcPtr, srcObj, value_offset); + __ Dlsa(srcPtr, srcBegin, srcPtr, char_shift); + + // Calculate number of bytes to copy from number of characters. + __ Dsll(numChrs, numChrs, char_shift); + + codegen_->InvokeRuntime(kQuickMemcpy, invoke, invoke->GetDexPc(), nullptr); + + __ Bind(&done); +} + static void GenHighestOneBit(LocationSummary* locations, Primitive::Type type, Mips64Assembler* assembler) { @@ -1925,7 +2003,6 @@ void IntrinsicCodeGeneratorMIPS64::VisitLongLowestOneBit(HInvoke* invoke) { } UNIMPLEMENTED_INTRINSIC(MIPS64, ReferenceGetReferent) -UNIMPLEMENTED_INTRINSIC(MIPS64, StringGetCharsNoCheck) UNIMPLEMENTED_INTRINSIC(MIPS64, SystemArrayCopyChar) UNIMPLEMENTED_INTRINSIC(MIPS64, SystemArrayCopy) diff --git a/compiler/optimizing/linearize_test.cc b/compiler/optimizing/linearize_test.cc index 13e14c53b5..3831aa6c91 100644 --- a/compiler/optimizing/linearize_test.cc +++ b/compiler/optimizing/linearize_test.cc @@ -18,7 +18,6 @@ #include "arch/x86/instruction_set_features_x86.h" #include "base/arena_allocator.h" -#include "base/stringprintf.h" #include "builder.h" #include "code_generator.h" #include "code_generator_x86.h" diff --git a/compiler/optimizing/nodes.cc b/compiler/optimizing/nodes.cc index 594255c625..a599c2aa84 100644 --- a/compiler/optimizing/nodes.cc +++ b/compiler/optimizing/nodes.cc @@ -1108,13 +1108,23 @@ size_t HInstruction::EnvironmentSize() const { return HasEnvironment() ? environment_->Size() : 0; } -void HPhi::AddInput(HInstruction* input) { +void HVariableInputSizeInstruction::AddInput(HInstruction* input) { DCHECK(input->GetBlock() != nullptr); inputs_.push_back(HUserRecord<HInstruction*>(input)); input->AddUseAt(this, inputs_.size() - 1); } -void HPhi::RemoveInputAt(size_t index) { +void HVariableInputSizeInstruction::InsertInputAt(size_t index, HInstruction* input) { + inputs_.insert(inputs_.begin() + index, HUserRecord<HInstruction*>(input)); + input->AddUseAt(this, index); + // Update indexes in use nodes of inputs that have been pushed further back by the insert(). + for (size_t i = index + 1u, e = inputs_.size(); i < e; ++i) { + DCHECK_EQ(inputs_[i].GetUseNode()->GetIndex(), i - 1u); + inputs_[i].GetUseNode()->SetIndex(i); + } +} + +void HVariableInputSizeInstruction::RemoveInputAt(size_t index) { RemoveAsUserOfInput(index); inputs_.erase(inputs_.begin() + index); // Update indexes in use nodes of inputs that have been pulled forward by the erase(). @@ -1347,7 +1357,9 @@ std::ostream& operator<<(std::ostream& os, const HInstruction::InstructionKind& void HInstruction::MoveBefore(HInstruction* cursor) { DCHECK(!IsPhi()); DCHECK(!IsControlFlow()); - DCHECK(CanBeMoved()); + DCHECK(CanBeMoved() || + // HShouldDeoptimizeFlag can only be moved by CHAGuardOptimization. + IsShouldDeoptimizeFlag()); DCHECK(!cursor->IsPhi()); next_->previous_ = previous_; @@ -2386,26 +2398,6 @@ bool HInvokeStaticOrDirect::NeedsDexCacheOfDeclaringClass() const { return !opt.GetDoesNotNeedDexCache(); } -void HInvokeStaticOrDirect::InsertInputAt(size_t index, HInstruction* input) { - inputs_.insert(inputs_.begin() + index, HUserRecord<HInstruction*>(input)); - input->AddUseAt(this, index); - // Update indexes in use nodes of inputs that have been pushed further back by the insert(). - for (size_t i = index + 1u, e = inputs_.size(); i < e; ++i) { - DCHECK_EQ(inputs_[i].GetUseNode()->GetIndex(), i - 1u); - inputs_[i].GetUseNode()->SetIndex(i); - } -} - -void HInvokeStaticOrDirect::RemoveInputAt(size_t index) { - RemoveAsUserOfInput(index); - inputs_.erase(inputs_.begin() + index); - // Update indexes in use nodes of inputs that have been pulled forward by the erase(). - for (size_t i = index, e = inputs_.size(); i < e; ++i) { - DCHECK_EQ(inputs_[i].GetUseNode()->GetIndex(), i + 1u); - inputs_[i].GetUseNode()->SetIndex(i); - } -} - std::ostream& operator<<(std::ostream& os, HInvokeStaticOrDirect::MethodLoadKind rhs) { switch (rhs) { case HInvokeStaticOrDirect::MethodLoadKind::kStringInit: @@ -2414,8 +2406,6 @@ std::ostream& operator<<(std::ostream& os, HInvokeStaticOrDirect::MethodLoadKind return os << "recursive"; case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress: return os << "direct"; - case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddressWithFixup: - return os << "direct_fixup"; case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: return os << "dex_cache_pc_relative"; case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: @@ -2440,6 +2430,17 @@ std::ostream& operator<<(std::ostream& os, HInvokeStaticOrDirect::ClinitCheckReq } } +// Helper for InstructionDataEquals to fetch the mirror Class out +// from a kJitTableAddress LoadClass kind. +// NO_THREAD_SAFETY_ANALYSIS because even though we're accessing +// mirrors, they are stored in a variable size handle scope which is always +// visited during a pause. Also, the only caller of this helper +// only uses the mirror for pointer comparison. +static inline mirror::Class* AsMirrorInternal(uint64_t address) + NO_THREAD_SAFETY_ANALYSIS { + return reinterpret_cast<StackReference<mirror::Class>*>(address)->AsMirrorPtr(); +} + bool HLoadClass::InstructionDataEquals(const HInstruction* other) const { const HLoadClass* other_load_class = other->AsLoadClass(); // TODO: To allow GVN for HLoadClass from different dex files, we should compare the type @@ -2448,16 +2449,14 @@ bool HLoadClass::InstructionDataEquals(const HInstruction* other) const { GetPackedFields() != other_load_class->GetPackedFields()) { return false; } - LoadKind load_kind = GetLoadKind(); - if (HasAddress(load_kind)) { - return GetAddress() == other_load_class->GetAddress(); - } else if (HasTypeReference(load_kind)) { - return IsSameDexFile(GetDexFile(), other_load_class->GetDexFile()); - } else { - DCHECK(HasDexCacheReference(load_kind)) << load_kind; - // If the type indexes and dex files are the same, dex cache element offsets - // must also be the same, so we don't need to compare them. - return IsSameDexFile(GetDexFile(), other_load_class->GetDexFile()); + switch (GetLoadKind()) { + case LoadKind::kBootImageAddress: + return GetAddress() == other_load_class->GetAddress(); + case LoadKind::kJitTableAddress: + return AsMirrorInternal(GetAddress()) == AsMirrorInternal(other_load_class->GetAddress()); + default: + DCHECK(HasTypeReference(GetLoadKind()) || HasDexCacheReference(GetLoadKind())); + return IsSameDexFile(GetDexFile(), other_load_class->GetDexFile()); } } @@ -2487,8 +2486,8 @@ std::ostream& operator<<(std::ostream& os, HLoadClass::LoadKind rhs) { return os << "BootImageLinkTimePcRelative"; case HLoadClass::LoadKind::kBootImageAddress: return os << "BootImageAddress"; - case HLoadClass::LoadKind::kDexCacheAddress: - return os << "DexCacheAddress"; + case HLoadClass::LoadKind::kJitTableAddress: + return os << "JitTableAddress"; case HLoadClass::LoadKind::kDexCachePcRelative: return os << "DexCachePcRelative"; case HLoadClass::LoadKind::kDexCacheViaMethod: diff --git a/compiler/optimizing/nodes.h b/compiler/optimizing/nodes.h index e3f4d8f035..afa17cefa2 100644 --- a/compiler/optimizing/nodes.h +++ b/compiler/optimizing/nodes.h @@ -125,6 +125,11 @@ enum GraphAnalysisResult { kAnalysisSuccess, }; +template <typename T> +static inline typename std::make_unsigned<T>::type MakeUnsigned(T x) { + return static_cast<typename std::make_unsigned<T>::type>(x); +} + class HInstructionList : public ValueObject { public: HInstructionList() : first_instruction_(nullptr), last_instruction_(nullptr) {} @@ -325,6 +330,7 @@ class HGraph : public ArenaObject<kArenaAllocGraph> { invoke_type_(invoke_type), in_ssa_form_(false), should_generate_constructor_barrier_(should_generate_constructor_barrier), + number_of_cha_guards_(0), instruction_set_(instruction_set), cached_null_constant_(nullptr), cached_int_constants_(std::less<int32_t>(), arena->Adapter(kArenaAllocConstantsMap)), @@ -546,9 +552,7 @@ class HGraph : public ArenaObject<kArenaAllocGraph> { } bool HasShouldDeoptimizeFlag() const { - // TODO: if all CHA guards can be eliminated, there is no need for the flag - // even if cha_single_implementation_list_ is not empty. - return !cha_single_implementation_list_.empty(); + return number_of_cha_guards_ != 0; } bool HasTryCatch() const { return has_try_catch_; } @@ -567,6 +571,10 @@ class HGraph : public ArenaObject<kArenaAllocGraph> { ReferenceTypeInfo GetInexactObjectRti() const { return inexact_object_rti_; } + uint32_t GetNumberOfCHAGuards() { return number_of_cha_guards_; } + void SetNumberOfCHAGuards(uint32_t num) { number_of_cha_guards_ = num; } + void IncrementNumberOfCHAGuards() { number_of_cha_guards_++; } + private: void RemoveInstructionsAsUsersFromDeadBlocks(const ArenaBitVector& visited) const; void RemoveDeadBlocks(const ArenaBitVector& visited); @@ -662,6 +670,10 @@ class HGraph : public ArenaObject<kArenaAllocGraph> { const bool should_generate_constructor_barrier_; + // Number of CHA guards in the graph. Used to short-circuit the + // CHA guard optimization pass when there is no CHA guard left. + uint32_t number_of_cha_guards_; + const InstructionSet instruction_set_; // Cached constants. @@ -2342,6 +2354,32 @@ class HBackwardInstructionIterator : public ValueObject { DISALLOW_COPY_AND_ASSIGN(HBackwardInstructionIterator); }; +class HVariableInputSizeInstruction : public HInstruction { + public: + using HInstruction::GetInputRecords; // Keep the const version visible. + ArrayRef<HUserRecord<HInstruction*>> GetInputRecords() OVERRIDE { + return ArrayRef<HUserRecord<HInstruction*>>(inputs_); + } + + void AddInput(HInstruction* input); + void InsertInputAt(size_t index, HInstruction* input); + void RemoveInputAt(size_t index); + + protected: + HVariableInputSizeInstruction(SideEffects side_effects, + uint32_t dex_pc, + ArenaAllocator* arena, + size_t number_of_inputs, + ArenaAllocKind kind) + : HInstruction(side_effects, dex_pc), + inputs_(number_of_inputs, arena->Adapter(kind)) {} + + ArenaVector<HUserRecord<HInstruction*>> inputs_; + + private: + DISALLOW_COPY_AND_ASSIGN(HVariableInputSizeInstruction); +}; + template<size_t N> class HTemplateInstruction: public HInstruction { public: @@ -2433,15 +2471,19 @@ class HReturn FINAL : public HTemplateInstruction<1> { DISALLOW_COPY_AND_ASSIGN(HReturn); }; -class HPhi FINAL : public HInstruction { +class HPhi FINAL : public HVariableInputSizeInstruction { public: HPhi(ArenaAllocator* arena, uint32_t reg_number, size_t number_of_inputs, Primitive::Type type, uint32_t dex_pc = kNoDexPc) - : HInstruction(SideEffects::None(), dex_pc), - inputs_(number_of_inputs, arena->Adapter(kArenaAllocPhiInputs)), + : HVariableInputSizeInstruction( + SideEffects::None(), + dex_pc, + arena, + number_of_inputs, + kArenaAllocPhiInputs), reg_number_(reg_number) { SetPackedField<TypeField>(ToPhiType(type)); DCHECK_NE(GetType(), Primitive::kPrimVoid); @@ -2459,14 +2501,6 @@ class HPhi FINAL : public HInstruction { bool IsCatchPhi() const { return GetBlock()->IsCatchBlock(); } - using HInstruction::GetInputRecords; // Keep the const version visible. - ArrayRef<HUserRecord<HInstruction*>> GetInputRecords() OVERRIDE FINAL { - return ArrayRef<HUserRecord<HInstruction*>>(inputs_); - } - - void AddInput(HInstruction* input); - void RemoveInputAt(size_t index); - Primitive::Type GetType() const OVERRIDE { return GetPackedField<TypeField>(); } void SetType(Primitive::Type new_type) { // Make sure that only valid type changes occur. The following are allowed: @@ -2522,7 +2556,6 @@ class HPhi FINAL : public HInstruction { static_assert(kNumberOfPhiPackedBits <= kMaxNumberOfPackedBits, "Too many packed fields."); using TypeField = BitField<Primitive::Type, kFieldType, kFieldTypeSize>; - ArenaVector<HUserRecord<HInstruction*>> inputs_; const uint32_t reg_number_; DISALLOW_COPY_AND_ASSIGN(HPhi); @@ -2899,14 +2932,20 @@ class HDeoptimize FINAL : public HTemplateInstruction<1> { // if it's true, starts to do deoptimization. // It has a 4-byte slot on stack. // TODO: allocate a register for this flag. -class HShouldDeoptimizeFlag FINAL : public HExpression<0> { +class HShouldDeoptimizeFlag FINAL : public HVariableInputSizeInstruction { public: - // TODO: use SideEffects to aid eliminating some CHA guards. - explicit HShouldDeoptimizeFlag(uint32_t dex_pc) - : HExpression(Primitive::kPrimInt, SideEffects::None(), dex_pc) { + // CHA guards are only optimized in a separate pass and it has no side effects + // with regard to other passes. + HShouldDeoptimizeFlag(ArenaAllocator* arena, uint32_t dex_pc) + : HVariableInputSizeInstruction(SideEffects::None(), dex_pc, arena, 0, kArenaAllocCHA) { } - // We don't eliminate CHA guards yet. + Primitive::Type GetType() const OVERRIDE { return Primitive::kPrimInt; } + + // We do all CHA guard elimination/motion in a single pass, after which there is no + // further guard elimination/motion since a guard might have been used for justification + // of the elimination of another guard. Therefore, we pretend this guard cannot be moved + // to avoid other optimizations trying to move it. bool CanBeMoved() const OVERRIDE { return false; } DECLARE_INSTRUCTION(ShouldDeoptimizeFlag); @@ -3786,15 +3825,10 @@ enum IntrinsicExceptions { kCanThrow // Intrinsic may throw exceptions. }; -class HInvoke : public HInstruction { +class HInvoke : public HVariableInputSizeInstruction { public: bool NeedsEnvironment() const OVERRIDE; - using HInstruction::GetInputRecords; // Keep the const version visible. - ArrayRef<HUserRecord<HInstruction*>> GetInputRecords() OVERRIDE { - return ArrayRef<HUserRecord<HInstruction*>>(inputs_); - } - void SetArgumentAt(size_t index, HInstruction* argument) { SetRawInputAt(index, argument); } @@ -3873,12 +3907,14 @@ class HInvoke : public HInstruction { uint32_t dex_method_index, ArtMethod* resolved_method, InvokeType invoke_type) - : HInstruction( - SideEffects::AllExceptGCDependency(), dex_pc), // Assume write/read on all fields/arrays. + : HVariableInputSizeInstruction( + SideEffects::AllExceptGCDependency(), // Assume write/read on all fields/arrays. + dex_pc, + arena, + number_of_arguments + number_of_other_inputs, + kArenaAllocInvokeInputs), number_of_arguments_(number_of_arguments), resolved_method_(resolved_method), - inputs_(number_of_arguments + number_of_other_inputs, - arena->Adapter(kArenaAllocInvokeInputs)), dex_method_index_(dex_method_index), intrinsic_(Intrinsics::kNone), intrinsic_optimizations_(0) { @@ -3889,7 +3925,6 @@ class HInvoke : public HInstruction { uint32_t number_of_arguments_; ArtMethod* const resolved_method_; - ArenaVector<HUserRecord<HInstruction*>> inputs_; const uint32_t dex_method_index_; Intrinsics intrinsic_; @@ -3947,12 +3982,6 @@ class HInvokeStaticOrDirect FINAL : public HInvoke { // Used for app->boot calls with non-relocatable image and for JIT-compiled calls. kDirectAddress, - // Use ArtMethod* at an address that will be known at link time, embed the direct - // address in the code. If the image is relocatable, emit .patch_oat entry. - // Used for app->boot calls with relocatable image and boot->boot calls, whether - // the image relocatable or not. - kDirectAddressWithFixup, - // Load from resolved methods array in the dex cache using a PC-relative load. // Used when we need to use the dex cache, for example for invoke-static that // may cause class initialization (the entry may point to a resolution method), @@ -3971,20 +4000,6 @@ class HInvokeStaticOrDirect FINAL : public HInvoke { // Recursive call, use local PC-relative call instruction. kCallSelf, - // Use PC-relative call instruction patched at link time. - // Used for calls within an oat file, boot->boot or app->app. - kCallPCRelative, - - // Call to a known target address, embed the direct address in code. - // Used for app->boot call with non-relocatable image and for JIT-compiled calls. - kCallDirect, - - // Call to a target address that will be known at link time, embed the direct - // address in code. If the image is relocatable, emit .patch_oat entry. - // Used for app->boot calls with relocatable image and boot->boot calls, whether - // the image relocatable or not. - kCallDirectWithFixup, - // Use code pointer from the ArtMethod*. // Used when we don't know the target code. This is also the last-resort-kind used when // other kinds are unimplemented or impractical (i.e. slow) on a particular architecture. @@ -4000,7 +4015,6 @@ class HInvokeStaticOrDirect FINAL : public HInvoke { // - the method address for kDirectAddress // - the dex cache arrays offset for kDexCachePcRel. uint64_t method_load_data; - uint64_t direct_code_ptr; }; HInvokeStaticOrDirect(ArenaAllocator* arena, @@ -4110,7 +4124,6 @@ class HInvokeStaticOrDirect FINAL : public HInvoke { return false; } } - bool HasDirectCodePtr() const { return GetCodePtrLocation() == CodePtrLocation::kCallDirect; } QuickEntrypointEnum GetStringInitEntryPoint() const { DCHECK(IsStringInit()); @@ -4127,11 +4140,6 @@ class HInvokeStaticOrDirect FINAL : public HInvoke { return dispatch_info_.method_load_data; } - uint64_t GetDirectCodePtr() const { - DCHECK(HasDirectCodePtr()); - return dispatch_info_.direct_code_ptr; - } - ClinitCheckRequirement GetClinitCheckRequirement() const { return GetPackedField<ClinitCheckRequirementField>(); } @@ -4179,10 +4187,6 @@ class HInvokeStaticOrDirect FINAL : public HInvoke { DECLARE_INSTRUCTION(InvokeStaticOrDirect); - protected: - void InsertInputAt(size_t index, HInstruction* input); - void RemoveInputAt(size_t index); - private: static constexpr size_t kFieldClinitCheckRequirement = kNumberOfInvokePackedBits; static constexpr size_t kFieldClinitCheckRequirementSize = @@ -5493,9 +5497,8 @@ class HLoadClass FINAL : public HInstruction { // GetIncludePatchInformation(). kBootImageAddress, - // Load from the resolved types array at an absolute address. - // Used for classes outside the boot image referenced by JIT-compiled code. - kDexCacheAddress, + // Load from the root table associated with the JIT compiled method. + kJitTableAddress, // Load from resolved types array in the dex cache using a PC-relative load. // Used for classes outside boot image when we know that we can access @@ -5515,9 +5518,7 @@ class HLoadClass FINAL : public HInstruction { const DexFile& dex_file, bool is_referrers_class, uint32_t dex_pc, - bool needs_access_check, - bool is_in_dex_cache, - bool is_in_boot_image) + bool needs_access_check) : HInstruction(SideEffectsForArchRuntimeCalls(), dex_pc), special_input_(HUserRecord<HInstruction*>(current_method)), type_index_(type_index), @@ -5530,8 +5531,8 @@ class HLoadClass FINAL : public HInstruction { SetPackedField<LoadKindField>( is_referrers_class ? LoadKind::kReferrersClass : LoadKind::kDexCacheViaMethod); SetPackedFlag<kFlagNeedsAccessCheck>(needs_access_check); - SetPackedFlag<kFlagIsInDexCache>(is_in_dex_cache); - SetPackedFlag<kFlagIsInBootImage>(is_in_boot_image); + SetPackedFlag<kFlagIsInDexCache>(false); + SetPackedFlag<kFlagIsInBootImage>(false); SetPackedFlag<kFlagGenerateClInitCheck>(false); } @@ -5588,7 +5589,6 @@ class HLoadClass FINAL : public HInstruction { NeedsAccessCheck(); } - bool CanThrow() const OVERRIDE { return CanCallRuntime(); } @@ -5613,7 +5613,9 @@ class HLoadClass FINAL : public HInstruction { return load_data_.address; } - bool NeedsDexCacheOfDeclaringClass() const OVERRIDE { return !IsReferrersClass(); } + bool NeedsDexCacheOfDeclaringClass() const OVERRIDE { + return !IsReferrersClass(); + } static SideEffects SideEffectsForArchRuntimeCalls() { return SideEffects::CanTriggerGC(); @@ -5672,7 +5674,8 @@ class HLoadClass FINAL : public HInstruction { } static bool HasAddress(LoadKind load_kind) { - return load_kind == LoadKind::kBootImageAddress || load_kind == LoadKind::kDexCacheAddress; + return load_kind == LoadKind::kBootImageAddress || + load_kind == LoadKind::kJitTableAddress; } static bool HasDexCacheReference(LoadKind load_kind) { @@ -5691,7 +5694,7 @@ class HLoadClass FINAL : public HInstruction { union { uint32_t dex_cache_element_index; // Only for dex cache reference. - uint64_t address; // Up to 64-bit, needed for kDexCacheAddress on 64-bit targets. + uint64_t address; // Up to 64-bit, needed for kJitTableAddress on 64-bit targets. } load_data_; ReferenceTypeInfo loaded_class_rti_; diff --git a/compiler/optimizing/optimizing_cfi_test.cc b/compiler/optimizing/optimizing_cfi_test.cc index 013e110b87..0e02311672 100644 --- a/compiler/optimizing/optimizing_cfi_test.cc +++ b/compiler/optimizing/optimizing_cfi_test.cc @@ -24,12 +24,22 @@ #include "optimizing/code_generator.h" #include "optimizing/optimizing_unit_test.h" #include "utils/assembler.h" +#ifdef ART_USE_VIXL_ARM_BACKEND +#include "utils/arm/assembler_arm_vixl.h" +#else #include "utils/arm/assembler_thumb2.h" +#endif #include "utils/mips/assembler_mips.h" #include "utils/mips64/assembler_mips64.h" #include "optimizing/optimizing_cfi_test_expected.inc" +#ifdef ART_USE_VIXL_ARM_BACKEND +namespace vixl32 = vixl::aarch32; + +using vixl32::r0; +#endif + namespace art { // Run the tests only on host. @@ -158,8 +168,7 @@ class OptimizingCFITest : public CFITest { TestImpl(isa, #isa, expected_asm, expected_cfi); \ } -// TODO(VIXL): Support this test for the VIXL backend. -#if defined(ART_ENABLE_CODEGEN_arm) && !defined(ART_USE_VIXL_ARM_BACKEND) +#ifdef ART_ENABLE_CODEGEN_arm TEST_ISA(kThumb2) #endif #ifdef ART_ENABLE_CODEGEN_arm64 @@ -178,8 +187,7 @@ TEST_ISA(kMips) TEST_ISA(kMips64) #endif -// TODO(VIXL): Support this test for the VIXL backend. -#if defined(ART_ENABLE_CODEGEN_arm) && !defined(ART_USE_VIXL_ARM_BACKEND) +#ifdef ART_ENABLE_CODEGEN_arm TEST_F(OptimizingCFITest, kThumb2Adjust) { std::vector<uint8_t> expected_asm( expected_asm_kThumb2_adjust, @@ -188,6 +196,16 @@ TEST_F(OptimizingCFITest, kThumb2Adjust) { expected_cfi_kThumb2_adjust, expected_cfi_kThumb2_adjust + arraysize(expected_cfi_kThumb2_adjust)); SetUpFrame(kThumb2); +#ifdef ART_USE_VIXL_ARM_BACKEND +#define __ down_cast<arm::ArmVIXLAssembler*>(GetCodeGenerator() \ + ->GetAssembler())->GetVIXLAssembler()-> + vixl32::Label target; + __ CompareAndBranchIfZero(r0, &target); + // Push the target out of range of CBZ. + for (size_t i = 0; i != 65; ++i) { + __ Ldr(r0, vixl32::MemOperand(r0)); + } +#else #define __ down_cast<arm::Thumb2Assembler*>(GetCodeGenerator()->GetAssembler())-> Label target; __ CompareAndBranchIfZero(arm::R0, &target); @@ -195,6 +213,7 @@ TEST_F(OptimizingCFITest, kThumb2Adjust) { for (size_t i = 0; i != 65; ++i) { __ ldr(arm::R0, arm::Address(arm::R0)); } +#endif __ Bind(&target); #undef __ Finish(); diff --git a/compiler/optimizing/optimizing_cfi_test_expected.inc b/compiler/optimizing/optimizing_cfi_test_expected.inc index f735dc8cb3..82670c38fe 100644 --- a/compiler/optimizing/optimizing_cfi_test_expected.inc +++ b/compiler/optimizing/optimizing_cfi_test_expected.inc @@ -223,8 +223,16 @@ static constexpr uint8_t expected_cfi_kMips64[] = { // 0x00000040: .cfi_def_cfa_offset: 64 static constexpr uint8_t expected_asm_kThumb2_adjust[] = { +#ifdef ART_USE_VIXL_ARM_BACKEND + // VIXL emits an extra 2 bytes here for a 32-bit beq as there is no + // optimistic 16-bit emit and subsequent fixup for out of reach targets + // as with the current assembler. + 0x60, 0xB5, 0x2D, 0xED, 0x02, 0x8A, 0x8B, 0xB0, 0x00, 0x28, 0x00, 0xF0, + 0x41, 0x80, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, +#else 0x60, 0xB5, 0x2D, 0xED, 0x02, 0x8A, 0x8B, 0xB0, 0x00, 0x28, 0x40, 0xD0, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, +#endif 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, @@ -239,7 +247,11 @@ static constexpr uint8_t expected_asm_kThumb2_adjust[] = { }; static constexpr uint8_t expected_cfi_kThumb2_adjust[] = { 0x42, 0x0E, 0x0C, 0x85, 0x03, 0x86, 0x02, 0x8E, 0x01, 0x44, 0x0E, 0x14, +#ifdef ART_USE_VIXL_ARM_BACKEND + 0x05, 0x50, 0x05, 0x05, 0x51, 0x04, 0x42, 0x0E, 0x40, 0x02, 0x88, 0x0A, +#else 0x05, 0x50, 0x05, 0x05, 0x51, 0x04, 0x42, 0x0E, 0x40, 0x02, 0x86, 0x0A, +#endif 0x42, 0x0E, 0x14, 0x44, 0x0E, 0x0C, 0x06, 0x50, 0x06, 0x51, 0x42, 0x0B, 0x0E, 0x40, }; diff --git a/compiler/optimizing/optimizing_compiler.cc b/compiler/optimizing/optimizing_compiler.cc index 8ea2b06530..4bf5b080a7 100644 --- a/compiler/optimizing/optimizing_compiler.cc +++ b/compiler/optimizing/optimizing_compiler.cc @@ -22,6 +22,8 @@ #include <stdint.h> +#include "android-base/strings.h" + #ifdef ART_ENABLE_CODEGEN_arm #include "dex_cache_array_fixups_arm.h" #endif @@ -52,6 +54,7 @@ #include "base/timing_logger.h" #include "bounds_check_elimination.h" #include "builder.h" +#include "cha_guard_optimization.h" #include "code_generator.h" #include "compiled_method.h" #include "compiler.h" @@ -375,7 +378,8 @@ class OptimizingCompiler FINAL : public Compiler { const DexFile& dex_file, Handle<mirror::DexCache> dex_cache, ArtMethod* method, - bool osr) const; + bool osr, + VariableSizedHandleScope* handles) const; void MaybeRunInliner(HGraph* graph, CodeGenerator* codegen, @@ -495,7 +499,7 @@ static HOptimization* BuildOptimization( number_of_dex_registers, /* depth */ 0); } else if (opt_name == HSharpening::kSharpeningPassName) { - return new (arena) HSharpening(graph, codegen, dex_compilation_unit, driver); + return new (arena) HSharpening(graph, codegen, dex_compilation_unit, driver, handles); } else if (opt_name == HSelectGenerator::kSelectGeneratorPassName) { return new (arena) HSelectGenerator(graph, stats); } else if (opt_name == HInductionVarAnalysis::kInductionPassName) { @@ -514,6 +518,8 @@ static HOptimization* BuildOptimization( return new (arena) SideEffectsAnalysis(graph); } else if (opt_name == HLoopOptimization::kLoopOptimizationPassName) { return new (arena) HLoopOptimization(graph, most_recent_induction); + } else if (opt_name == CHAGuardOptimization::kCHAGuardOptimizationPassName) { + return new (arena) CHAGuardOptimization(graph); #ifdef ART_ENABLE_CODEGEN_arm } else if (opt_name == arm::DexCacheArrayFixups::kDexCacheArrayFixupsArmPassName) { return new (arena) arm::DexCacheArrayFixups(graph, codegen, stats); @@ -767,7 +773,8 @@ void OptimizingCompiler::RunOptimizations(HGraph* graph, HInductionVarAnalysis* induction = new (arena) HInductionVarAnalysis(graph); BoundsCheckElimination* bce = new (arena) BoundsCheckElimination(graph, *side_effects, induction); HLoopOptimization* loop = new (arena) HLoopOptimization(graph, induction); - HSharpening* sharpening = new (arena) HSharpening(graph, codegen, dex_compilation_unit, driver); + HSharpening* sharpening = new (arena) HSharpening( + graph, codegen, dex_compilation_unit, driver, handles); InstructionSimplifier* simplify2 = new (arena) InstructionSimplifier( graph, stats, "instruction_simplifier$after_inlining"); InstructionSimplifier* simplify3 = new (arena) InstructionSimplifier( @@ -775,6 +782,7 @@ void OptimizingCompiler::RunOptimizations(HGraph* graph, InstructionSimplifier* simplify4 = new (arena) InstructionSimplifier( graph, stats, "instruction_simplifier$before_codegen"); IntrinsicsRecognizer* intrinsics = new (arena) IntrinsicsRecognizer(graph, stats); + CHAGuardOptimization* cha_guard = new (arena) CHAGuardOptimization(graph); HOptimization* optimizations1[] = { intrinsics, @@ -803,6 +811,7 @@ void OptimizingCompiler::RunOptimizations(HGraph* graph, fold3, // evaluates code generated by dynamic bce simplify3, lse, + cha_guard, dce3, // The codegen has a few assumptions that only the instruction simplifier // can satisfy. For example, the code generator does not expect to see a @@ -866,7 +875,8 @@ CodeGenerator* OptimizingCompiler::TryCompile(ArenaAllocator* arena, const DexFile& dex_file, Handle<mirror::DexCache> dex_cache, ArtMethod* method, - bool osr) const { + bool osr, + VariableSizedHandleScope* handles) const { MaybeRecordStat(MethodCompilationStat::kAttemptCompilation); CompilerDriver* compiler_driver = GetCompilerDriver(); InstructionSet instruction_set = compiler_driver->GetInstructionSet(); @@ -976,63 +986,55 @@ CodeGenerator* OptimizingCompiler::TryCompile(ArenaAllocator* arena, compiler_driver, dump_mutex_); - VLOG(compiler) << "Building " << pass_observer.GetMethodName(); - { - ScopedObjectAccess soa(Thread::Current()); - VariableSizedHandleScope handles(soa.Self()); - // Do not hold `mutator_lock_` between optimizations. - ScopedThreadSuspension sts(soa.Self(), kNative); - - { - PassScope scope(HGraphBuilder::kBuilderPassName, &pass_observer); - HGraphBuilder builder(graph, - &dex_compilation_unit, - &dex_compilation_unit, - &dex_file, - *code_item, - compiler_driver, - compilation_stats_.get(), - interpreter_metadata, - dex_cache, - &handles); - GraphAnalysisResult result = builder.BuildGraph(); - if (result != kAnalysisSuccess) { - switch (result) { - case kAnalysisSkipped: - MaybeRecordStat(MethodCompilationStat::kNotCompiledSkipped); - break; - case kAnalysisInvalidBytecode: - MaybeRecordStat(MethodCompilationStat::kNotCompiledInvalidBytecode); - break; - case kAnalysisFailThrowCatchLoop: - MaybeRecordStat(MethodCompilationStat::kNotCompiledThrowCatchLoop); - break; - case kAnalysisFailAmbiguousArrayOp: - MaybeRecordStat(MethodCompilationStat::kNotCompiledAmbiguousArrayOp); - break; - case kAnalysisSuccess: - UNREACHABLE(); - } - pass_observer.SetGraphInBadState(); - return nullptr; + VLOG(compiler) << "Building " << pass_observer.GetMethodName(); + PassScope scope(HGraphBuilder::kBuilderPassName, &pass_observer); + HGraphBuilder builder(graph, + &dex_compilation_unit, + &dex_compilation_unit, + &dex_file, + *code_item, + compiler_driver, + compilation_stats_.get(), + interpreter_metadata, + dex_cache, + handles); + GraphAnalysisResult result = builder.BuildGraph(); + if (result != kAnalysisSuccess) { + switch (result) { + case kAnalysisSkipped: + MaybeRecordStat(MethodCompilationStat::kNotCompiledSkipped); + break; + case kAnalysisInvalidBytecode: + MaybeRecordStat(MethodCompilationStat::kNotCompiledInvalidBytecode); + break; + case kAnalysisFailThrowCatchLoop: + MaybeRecordStat(MethodCompilationStat::kNotCompiledThrowCatchLoop); + break; + case kAnalysisFailAmbiguousArrayOp: + MaybeRecordStat(MethodCompilationStat::kNotCompiledAmbiguousArrayOp); + break; + case kAnalysisSuccess: + UNREACHABLE(); } + pass_observer.SetGraphInBadState(); + return nullptr; } + } - RunOptimizations(graph, - codegen.get(), - compiler_driver, - dex_compilation_unit, - &pass_observer, - &handles); + RunOptimizations(graph, + codegen.get(), + compiler_driver, + dex_compilation_unit, + &pass_observer, + handles); - RegisterAllocator::Strategy regalloc_strategy = - compiler_options.GetRegisterAllocationStrategy(); - AllocateRegisters(graph, codegen.get(), &pass_observer, regalloc_strategy); + RegisterAllocator::Strategy regalloc_strategy = + compiler_options.GetRegisterAllocationStrategy(); + AllocateRegisters(graph, codegen.get(), &pass_observer, regalloc_strategy); - codegen->Compile(code_allocator); - pass_observer.DumpDisassembly(); - } + codegen->Compile(code_allocator); + pass_observer.DumpDisassembly(); return codegen.release(); } @@ -1055,19 +1057,27 @@ CompiledMethod* OptimizingCompiler::Compile(const DexFile::CodeItem* code_item, verified_method->GetEncounteredVerificationFailures())) { ArenaAllocator arena(Runtime::Current()->GetArenaPool()); CodeVectorAllocator code_allocator(&arena); - std::unique_ptr<CodeGenerator> codegen( - TryCompile(&arena, - &code_allocator, - code_item, - access_flags, - invoke_type, - class_def_idx, - method_idx, - jclass_loader, - dex_file, - dex_cache, - nullptr, - /* osr */ false)); + std::unique_ptr<CodeGenerator> codegen; + { + ScopedObjectAccess soa(Thread::Current()); + VariableSizedHandleScope handles(soa.Self()); + // Go to native so that we don't block GC during compilation. + ScopedThreadSuspension sts(soa.Self(), kNative); + codegen.reset( + TryCompile(&arena, + &code_allocator, + code_item, + access_flags, + invoke_type, + class_def_idx, + method_idx, + jclass_loader, + dex_file, + dex_cache, + nullptr, + /* osr */ false, + &handles)); + } if (codegen.get() != nullptr) { MaybeRecordStat(MethodCompilationStat::kCompiled); method = Emit(&arena, &code_allocator, codegen.get(), compiler_driver, code_item); @@ -1112,7 +1122,8 @@ Compiler* CreateOptimizingCompiler(CompilerDriver* driver) { bool IsCompilingWithCoreImage() { const std::string& image = Runtime::Current()->GetImageLocation(); // TODO: This is under-approximating... - if (EndsWith(image, "core.art") || EndsWith(image, "core-optimizing.art")) { + if (android::base::EndsWith(image, "core.art") || + android::base::EndsWith(image, "core-optimizing.art")) { return true; } return false; @@ -1138,6 +1149,8 @@ bool OptimizingCompiler::JitCompile(Thread* self, ArenaAllocator arena(Runtime::Current()->GetJitArenaPool()); CodeVectorAllocator code_allocator(&arena); + VariableSizedHandleScope handles(self); + std::unique_ptr<CodeGenerator> codegen; { // Go to native so that we don't block GC during compilation. @@ -1154,7 +1167,8 @@ bool OptimizingCompiler::JitCompile(Thread* self, *dex_file, dex_cache, method, - osr)); + osr, + &handles)); if (codegen.get() == nullptr) { return false; } @@ -1184,7 +1198,7 @@ bool OptimizingCompiler::JitCompile(Thread* self, } uint8_t* stack_map_data = nullptr; uint8_t* roots_data = nullptr; - code_cache->ReserveData( + uint32_t data_size = code_cache->ReserveData( self, stack_map_size, number_of_roots, method, &stack_map_data, &roots_data); if (stack_map_data == nullptr || roots_data == nullptr) { return false; @@ -1203,6 +1217,7 @@ bool OptimizingCompiler::JitCompile(Thread* self, codegen->GetFpuSpillMask(), code_allocator.GetMemory().data(), code_allocator.GetSize(), + data_size, osr, roots, codegen->GetGraph()->HasShouldDeoptimizeFlag(), diff --git a/compiler/optimizing/pc_relative_fixups_mips.cc b/compiler/optimizing/pc_relative_fixups_mips.cc index 82feb95a2f..e321b9e3aa 100644 --- a/compiler/optimizing/pc_relative_fixups_mips.cc +++ b/compiler/optimizing/pc_relative_fixups_mips.cc @@ -45,10 +45,6 @@ class PCRelativeHandlerVisitor : public HGraphVisitor { } private: - void VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) OVERRIDE { - HandleInvoke(invoke); - } - void InitializePCRelativeBasePointer() { // Ensure we only initialize the pointer once. if (base_ != nullptr) { @@ -112,38 +108,6 @@ class PCRelativeHandlerVisitor : public HGraphVisitor { block->ReplaceAndRemoveInstructionWith(switch_insn, mips_switch); } - void HandleInvoke(HInvoke* invoke) { - // If this is an invoke-static/-direct with PC-relative dex cache array - // addressing, we need the PC-relative address base. - HInvokeStaticOrDirect* invoke_static_or_direct = invoke->AsInvokeStaticOrDirect(); - if (invoke_static_or_direct != nullptr) { - HInvokeStaticOrDirect::MethodLoadKind method_load_kind = - invoke_static_or_direct->GetMethodLoadKind(); - HInvokeStaticOrDirect::CodePtrLocation code_ptr_location = - invoke_static_or_direct->GetCodePtrLocation(); - - bool has_extra_input = - (method_load_kind == HInvokeStaticOrDirect::MethodLoadKind::kDirectAddressWithFixup) || - (code_ptr_location == HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup); - - // We can't add a pointer to the constant area if we already have a current - // method pointer. This may arise when sharpening doesn't remove the current - // method pointer from the invoke. - if (invoke_static_or_direct->HasCurrentMethodInput()) { - DCHECK(!invoke_static_or_direct->HasPcRelativeDexCache()); - CHECK(!has_extra_input); - return; - } - - if (has_extra_input && - !IsCallFreeIntrinsic<IntrinsicLocationsBuilderMIPS>(invoke, codegen_)) { - InitializePCRelativeBasePointer(); - // Add the extra parameter base_. - invoke_static_or_direct->AddSpecialInput(base_); - } - } - } - CodeGeneratorMIPS* codegen_; // The generated HMipsComputeBaseMethodAddress in the entry block needed as an diff --git a/compiler/optimizing/pretty_printer.h b/compiler/optimizing/pretty_printer.h index 5891350894..c6579dc5e0 100644 --- a/compiler/optimizing/pretty_printer.h +++ b/compiler/optimizing/pretty_printer.h @@ -17,7 +17,8 @@ #ifndef ART_COMPILER_OPTIMIZING_PRETTY_PRINTER_H_ #define ART_COMPILER_OPTIMIZING_PRETTY_PRINTER_H_ -#include "base/stringprintf.h" +#include "android-base/stringprintf.h" + #include "nodes.h" namespace art { @@ -108,7 +109,7 @@ class StringPrettyPrinter : public HPrettyPrinter { : HPrettyPrinter(graph), str_(""), current_block_(nullptr) { } void PrintInt(int value) OVERRIDE { - str_ += StringPrintf("%d", value); + str_ += android::base::StringPrintf("%d", value); } void PrintString(const char* value) OVERRIDE { diff --git a/compiler/optimizing/pretty_printer_test.cc b/compiler/optimizing/pretty_printer_test.cc index 951cdfbd8b..1af94f3445 100644 --- a/compiler/optimizing/pretty_printer_test.cc +++ b/compiler/optimizing/pretty_printer_test.cc @@ -15,7 +15,6 @@ */ #include "base/arena_allocator.h" -#include "base/stringprintf.h" #include "builder.h" #include "dex_file.h" #include "dex_instruction.h" diff --git a/compiler/optimizing/reference_type_propagation.cc b/compiler/optimizing/reference_type_propagation.cc index c191c6651f..33b3875e3b 100644 --- a/compiler/optimizing/reference_type_propagation.cc +++ b/compiler/optimizing/reference_type_propagation.cc @@ -270,7 +270,7 @@ void ReferenceTypePropagation::BoundTypeForIfNotNull(HBasicBlock* block) { ScopedObjectAccess soa(Thread::Current()); HInstruction* insert_point = notNullBlock->GetFirstInstruction(); ReferenceTypeInfo object_rti = ReferenceTypeInfo::Create( - handle_cache_.GetObjectClassHandle(), /* is_exact */ true); + handle_cache_.GetObjectClassHandle(), /* is_exact */ false); if (ShouldCreateBoundType(insert_point, obj, object_rti, nullptr, notNullBlock)) { bound_type = new (graph_->GetArena()) HBoundType(obj); bound_type->SetUpperBound(object_rti, /* bound_can_be_null */ false); @@ -411,7 +411,9 @@ void ReferenceTypePropagation::BoundTypeForIfInstanceOf(HBasicBlock* block) { HInstruction* insert_point = instanceOfTrueBlock->GetFirstInstruction(); if (ShouldCreateBoundType(insert_point, obj, class_rti, nullptr, instanceOfTrueBlock)) { bound_type = new (graph_->GetArena()) HBoundType(obj); - bound_type->SetUpperBound(class_rti, /* InstanceOf fails for null. */ false); + bool is_exact = class_rti.GetTypeHandle()->CannotBeAssignedFromOtherTypes(); + bound_type->SetUpperBound(ReferenceTypeInfo::Create(class_rti.GetTypeHandle(), is_exact), + /* InstanceOf fails for null. */ false); instanceOfTrueBlock->InsertInstructionBefore(bound_type, insert_point); } else { // We already have a bound type on the position we would need to insert @@ -605,15 +607,17 @@ void ReferenceTypePropagation::RTPVisitor::VisitBoundType(HBoundType* instr) { // Narrow the type as much as possible. HInstruction* obj = instr->InputAt(0); ReferenceTypeInfo obj_rti = obj->GetReferenceTypeInfo(); - if (class_rti.GetTypeHandle()->CannotBeAssignedFromOtherTypes()) { - instr->SetReferenceTypeInfo( - ReferenceTypeInfo::Create(class_rti.GetTypeHandle(), /* is_exact */ true)); + if (class_rti.IsExact()) { + instr->SetReferenceTypeInfo(class_rti); } else if (obj_rti.IsValid()) { if (class_rti.IsSupertypeOf(obj_rti)) { // Object type is more specific. instr->SetReferenceTypeInfo(obj_rti); } else { - // Upper bound is more specific. + // Upper bound is more specific, or unrelated to the object's type. + // Note that the object might then be exact, and we know the code dominated by this + // bound type is dead. To not confuse potential other optimizations, we mark + // the bound as non-exact. instr->SetReferenceTypeInfo( ReferenceTypeInfo::Create(class_rti.GetTypeHandle(), /* is_exact */ false)); } @@ -644,8 +648,11 @@ void ReferenceTypePropagation::RTPVisitor::VisitCheckCast(HCheckCast* check_cast if (class_rti.IsValid()) { DCHECK(is_first_run_); + ScopedObjectAccess soa(Thread::Current()); // This is the first run of RTP and class is resolved. - bound_type->SetUpperBound(class_rti, /* CheckCast succeeds for nulls. */ true); + bool is_exact = class_rti.GetTypeHandle()->CannotBeAssignedFromOtherTypes(); + bound_type->SetUpperBound(ReferenceTypeInfo::Create(class_rti.GetTypeHandle(), is_exact), + /* CheckCast succeeds for nulls. */ true); } else { // This is the first run of RTP and class is unresolved. Remove the binding. // The instruction itself is removed in VisitBoundType so as to not @@ -795,21 +802,25 @@ void ReferenceTypePropagation::RTPVisitor::VisitArrayGet(HArrayGet* instr) { } void ReferenceTypePropagation::UpdateBoundType(HBoundType* instr) { - ReferenceTypeInfo new_rti = instr->InputAt(0)->GetReferenceTypeInfo(); - if (!new_rti.IsValid()) { + ReferenceTypeInfo input_rti = instr->InputAt(0)->GetReferenceTypeInfo(); + if (!input_rti.IsValid()) { return; // No new info yet. } - // Make sure that we don't go over the bounded type. ReferenceTypeInfo upper_bound_rti = instr->GetUpperBound(); - if (!upper_bound_rti.IsSupertypeOf(new_rti)) { - // Note that the input might be exact, in which case we know the branch leading - // to the bound type is dead. We play it safe by not marking the bound type as - // exact. - bool is_exact = upper_bound_rti.GetTypeHandle()->CannotBeAssignedFromOtherTypes(); - new_rti = ReferenceTypeInfo::Create(upper_bound_rti.GetTypeHandle(), is_exact); - } - instr->SetReferenceTypeInfo(new_rti); + if (upper_bound_rti.IsExact()) { + instr->SetReferenceTypeInfo(upper_bound_rti); + } else if (upper_bound_rti.IsSupertypeOf(input_rti)) { + // input is more specific. + instr->SetReferenceTypeInfo(input_rti); + } else { + // upper_bound is more specific or unrelated. + // Note that the object might then be exact, and we know the code dominated by this + // bound type is dead. To not confuse potential other optimizations, we mark + // the bound as non-exact. + instr->SetReferenceTypeInfo( + ReferenceTypeInfo::Create(upper_bound_rti.GetTypeHandle(), /* is_exact */ false)); + } } // NullConstant inputs are ignored during merging as they do not provide any useful information. diff --git a/compiler/optimizing/sharpening.cc b/compiler/optimizing/sharpening.cc index daf160a483..9fdeccfa1a 100644 --- a/compiler/optimizing/sharpening.cc +++ b/compiler/optimizing/sharpening.cc @@ -54,6 +54,24 @@ void HSharpening::Run() { } } +static bool IsInBootImage(ArtMethod* method) { + const std::vector<gc::space::ImageSpace*>& image_spaces = + Runtime::Current()->GetHeap()->GetBootImageSpaces(); + for (gc::space::ImageSpace* image_space : image_spaces) { + const auto& method_section = image_space->GetImageHeader().GetMethodsSection(); + if (method_section.Contains(reinterpret_cast<uint8_t*>(method) - image_space->Begin())) { + return true; + } + } + return false; +} + +static bool AOTCanEmbedMethod(ArtMethod* method, const CompilerOptions& options) { + // Including patch information means the AOT code will be patched, which we don't + // support in the compiler, and is anyways moving away b/33192586. + return IsInBootImage(method) && !options.GetCompilePic() && !options.GetIncludePatchInformation(); +} + void HSharpening::ProcessInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) { if (invoke->IsStringInit()) { // Not using the dex cache arrays. But we could still try to use a better dispatch... @@ -61,68 +79,42 @@ void HSharpening::ProcessInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) { return; } - HGraph* outer_graph = codegen_->GetGraph(); - ArtMethod* compiling_method = graph_->GetArtMethod(); + ArtMethod* callee = invoke->GetResolvedMethod(); + DCHECK(callee != nullptr); HInvokeStaticOrDirect::MethodLoadKind method_load_kind; HInvokeStaticOrDirect::CodePtrLocation code_ptr_location; uint64_t method_load_data = 0u; - uint64_t direct_code_ptr = 0u; - if (invoke->GetResolvedMethod() == outer_graph->GetArtMethod()) { - DCHECK(outer_graph->GetArtMethod() != nullptr); + // Note: we never call an ArtMethod through a known code pointer, as + // we do not want to keep on invoking it if it gets deoptimized. This + // applies to both AOT and JIT. + // This also avoids having to find out if the code pointer of an ArtMethod + // is the resolution trampoline (for ensuring the class is initialized), or + // the interpreter entrypoint. Such code pointers we do not want to call + // directly. + // Only in the case of a recursive call can we call directly, as we know the + // class is initialized already or being initialized, and the call will not + // be invoked once the method is deoptimized. + + if (callee == codegen_->GetGraph()->GetArtMethod()) { + // Recursive call. method_load_kind = HInvokeStaticOrDirect::MethodLoadKind::kRecursive; code_ptr_location = HInvokeStaticOrDirect::CodePtrLocation::kCallSelf; + } else if (Runtime::Current()->UseJitCompilation() || + AOTCanEmbedMethod(callee, codegen_->GetCompilerOptions())) { + // JIT or on-device AOT compilation referencing a boot image method. + // Use the method address directly. + method_load_kind = HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress; + method_load_data = reinterpret_cast<uintptr_t>(callee); + code_ptr_location = HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod; } else { - uintptr_t direct_code, direct_method; - { - ScopedObjectAccess soa(Thread::Current()); - compiler_driver_->GetCodeAndMethodForDirectCall( - (compiling_method == nullptr) ? nullptr : compiling_method->GetDeclaringClass(), - invoke->GetResolvedMethod(), - &direct_code, - &direct_method); - } - if (direct_method != 0u) { // Should we use a direct pointer to the method? - // Note: For JIT, kDirectAddressWithFixup doesn't make sense at all and while - // kDirectAddress would be fine for image methods, we don't support it at the moment. - DCHECK(!Runtime::Current()->UseJitCompilation()); - if (direct_method != static_cast<uintptr_t>(-1)) { // Is the method pointer known now? - method_load_kind = HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress; - method_load_data = direct_method; - } else { // The direct pointer will be known at link time. - method_load_kind = HInvokeStaticOrDirect::MethodLoadKind::kDirectAddressWithFixup; - } - } else { // Use dex cache. - if (!Runtime::Current()->UseJitCompilation()) { - // Use PC-relative access to the dex cache arrays. - method_load_kind = HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative; - DexCacheArraysLayout layout(GetInstructionSetPointerSize(codegen_->GetInstructionSet()), - &graph_->GetDexFile()); - method_load_data = layout.MethodOffset(invoke->GetDexMethodIndex()); - } else { // We must go through the ArtMethod's pointer to resolved methods. - method_load_kind = HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod; - } - } - if (direct_code != 0u) { // Should we use a direct pointer to the code? - // Note: For JIT, kCallPCRelative and kCallDirectWithFixup don't make sense at all and - // while kCallDirect would be fine for image methods, we don't support it at the moment. - DCHECK(!Runtime::Current()->UseJitCompilation()); - const DexFile* dex_file_of_callee = invoke->GetTargetMethod().dex_file; - if (direct_code != static_cast<uintptr_t>(-1)) { // Is the code pointer known now? - code_ptr_location = HInvokeStaticOrDirect::CodePtrLocation::kCallDirect; - direct_code_ptr = direct_code; - } else if (ContainsElement(compiler_driver_->GetDexFilesForOatFile(), dex_file_of_callee)) { - // Use PC-relative calls for invokes within a multi-dex oat file. - code_ptr_location = HInvokeStaticOrDirect::CodePtrLocation::kCallPCRelative; - } else { // The direct pointer will be known at link time. - // NOTE: This is used for app->boot calls when compiling an app against - // a relocatable but not yet relocated image. - code_ptr_location = HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup; - } - } else { // We must use the code pointer from the ArtMethod. - code_ptr_location = HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod; - } + // Use PC-relative access to the dex cache arrays. + method_load_kind = HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative; + DexCacheArraysLayout layout(GetInstructionSetPointerSize(codegen_->GetInstructionSet()), + &graph_->GetDexFile()); + method_load_data = layout.MethodOffset(invoke->GetDexMethodIndex()); + code_ptr_location = HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod; } if (graph_->IsDebuggable()) { @@ -132,7 +124,7 @@ void HSharpening::ProcessInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) { } HInvokeStaticOrDirect::DispatchInfo desired_dispatch_info = { - method_load_kind, code_ptr_location, method_load_data, direct_code_ptr + method_load_kind, code_ptr_location, method_load_data }; HInvokeStaticOrDirect::DispatchInfo dispatch_info = codegen_->GetSupportedInvokeStaticOrDirectDispatch(desired_dispatch_info, invoke); @@ -140,6 +132,25 @@ void HSharpening::ProcessInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) { } void HSharpening::ProcessLoadClass(HLoadClass* load_class) { + ScopedObjectAccess soa(Thread::Current()); + StackHandleScope<1> hs(soa.Self()); + Runtime* runtime = Runtime::Current(); + ClassLinker* class_linker = runtime->GetClassLinker(); + const DexFile& dex_file = load_class->GetDexFile(); + dex::TypeIndex type_index = load_class->GetTypeIndex(); + Handle<mirror::DexCache> dex_cache = IsSameDexFile(dex_file, *compilation_unit_.GetDexFile()) + ? compilation_unit_.GetDexCache() + : hs.NewHandle(class_linker->FindDexCache(soa.Self(), dex_file)); + mirror::Class* cls = dex_cache->GetResolvedType(type_index); + SharpenClass(load_class, cls, handles_, codegen_, compiler_driver_); +} + +void HSharpening::SharpenClass(HLoadClass* load_class, + mirror::Class* klass, + VariableSizedHandleScope* handles, + CodeGenerator* codegen, + CompilerDriver* compiler_driver) { + ScopedAssertNoThreadSuspension sants("Sharpening class in compiler"); DCHECK(load_class->GetLoadKind() == HLoadClass::LoadKind::kDexCacheViaMethod || load_class->GetLoadKind() == HLoadClass::LoadKind::kReferrersClass) << load_class->GetLoadKind(); @@ -151,70 +162,65 @@ void HSharpening::ProcessLoadClass(HLoadClass* load_class) { bool is_in_dex_cache = false; bool is_in_boot_image = false; - HLoadClass::LoadKind desired_load_kind; + HLoadClass::LoadKind desired_load_kind = static_cast<HLoadClass::LoadKind>(-1); uint64_t address = 0u; // Class or dex cache element address. - { - ScopedObjectAccess soa(Thread::Current()); - StackHandleScope<1> hs(soa.Self()); - Runtime* runtime = Runtime::Current(); - ClassLinker* class_linker = runtime->GetClassLinker(); - Handle<mirror::DexCache> dex_cache = IsSameDexFile(dex_file, *compilation_unit_.GetDexFile()) - ? compilation_unit_.GetDexCache() - : hs.NewHandle(class_linker->FindDexCache(soa.Self(), dex_file)); - mirror::Class* klass = dex_cache->GetResolvedType(type_index); - if (codegen_->GetCompilerOptions().IsBootImage()) { - // Compiling boot image. Check if the class is a boot image class. - DCHECK(!runtime->UseJitCompilation()); - if (!compiler_driver_->GetSupportBootImageFixup()) { - // MIPS64 or compiler_driver_test. Do not sharpen. - desired_load_kind = HLoadClass::LoadKind::kDexCacheViaMethod; - } else if ((klass != nullptr) && compiler_driver_->IsImageClass( - dex_file.StringDataByIdx(dex_file.GetTypeId(type_index).descriptor_idx_))) { - is_in_boot_image = true; - is_in_dex_cache = true; - desired_load_kind = codegen_->GetCompilerOptions().GetCompilePic() - ? HLoadClass::LoadKind::kBootImageLinkTimePcRelative - : HLoadClass::LoadKind::kBootImageLinkTimeAddress; - } else { - // Not a boot image class. We must go through the dex cache. - DCHECK(ContainsElement(compiler_driver_->GetDexFilesForOatFile(), &dex_file)); - desired_load_kind = HLoadClass::LoadKind::kDexCachePcRelative; - } + Runtime* runtime = Runtime::Current(); + if (codegen->GetCompilerOptions().IsBootImage()) { + // Compiling boot image. Check if the class is a boot image class. + DCHECK(!runtime->UseJitCompilation()); + if (!compiler_driver->GetSupportBootImageFixup()) { + // MIPS64 or compiler_driver_test. Do not sharpen. + desired_load_kind = HLoadClass::LoadKind::kDexCacheViaMethod; + } else if ((klass != nullptr) && compiler_driver->IsImageClass( + dex_file.StringDataByIdx(dex_file.GetTypeId(type_index).descriptor_idx_))) { + is_in_boot_image = true; + is_in_dex_cache = true; + desired_load_kind = codegen->GetCompilerOptions().GetCompilePic() + ? HLoadClass::LoadKind::kBootImageLinkTimePcRelative + : HLoadClass::LoadKind::kBootImageLinkTimeAddress; } else { - is_in_boot_image = (klass != nullptr) && runtime->GetHeap()->ObjectIsInBootImageSpace(klass); - if (runtime->UseJitCompilation()) { - // TODO: Make sure we don't set the "compile PIC" flag for JIT as that's bogus. - // DCHECK(!codegen_->GetCompilerOptions().GetCompilePic()); - is_in_dex_cache = (klass != nullptr); - if (is_in_boot_image) { - // TODO: Use direct pointers for all non-moving spaces, not just boot image. Bug: 29530787 - desired_load_kind = HLoadClass::LoadKind::kBootImageAddress; - address = reinterpret_cast64<uint64_t>(klass); - } else { - // Note: If the class is not in the dex cache or isn't initialized, the - // instruction needs environment and will not be inlined across dex files. - // Within a dex file, the slow-path helper loads the correct class and - // inlined frames are used correctly for OOM stack trace. - // TODO: Write a test for this. Bug: 29416588 - desired_load_kind = HLoadClass::LoadKind::kDexCacheAddress; - void* dex_cache_element_address = &dex_cache->GetResolvedTypes()[type_index.index_]; - address = reinterpret_cast64<uint64_t>(dex_cache_element_address); - } - // AOT app compilation. Check if the class is in the boot image. - } else if (is_in_boot_image && !codegen_->GetCompilerOptions().GetCompilePic()) { + // Not a boot image class. We must go through the dex cache. + DCHECK(ContainsElement(compiler_driver->GetDexFilesForOatFile(), &dex_file)); + desired_load_kind = HLoadClass::LoadKind::kDexCachePcRelative; + } + } else { + is_in_boot_image = (klass != nullptr) && runtime->GetHeap()->ObjectIsInBootImageSpace(klass); + if (runtime->UseJitCompilation()) { + // TODO: Make sure we don't set the "compile PIC" flag for JIT as that's bogus. + // DCHECK(!codegen_->GetCompilerOptions().GetCompilePic()); + is_in_dex_cache = (klass != nullptr); + if (is_in_boot_image) { + // TODO: Use direct pointers for all non-moving spaces, not just boot image. Bug: 29530787 desired_load_kind = HLoadClass::LoadKind::kBootImageAddress; address = reinterpret_cast64<uint64_t>(klass); + } else if (is_in_dex_cache) { + desired_load_kind = HLoadClass::LoadKind::kJitTableAddress; + // We store in the address field the location of the stack reference maintained + // by the handle. We do this now so that the code generation does not need to figure + // out which class loader to use. + address = reinterpret_cast<uint64_t>(handles->NewHandle(klass).GetReference()); } else { - // Not JIT and either the klass is not in boot image or we are compiling in PIC mode. - // Use PC-relative load from the dex cache if the dex file belongs - // to the oat file that we're currently compiling. - desired_load_kind = - ContainsElement(compiler_driver_->GetDexFilesForOatFile(), &load_class->GetDexFile()) - ? HLoadClass::LoadKind::kDexCachePcRelative - : HLoadClass::LoadKind::kDexCacheViaMethod; + // Class not loaded yet. This happens when the dex code requesting + // this `HLoadClass` hasn't been executed in the interpreter. + // Fallback to the dex cache. + // TODO(ngeoffray): Generate HDeoptimize instead. + desired_load_kind = HLoadClass::LoadKind::kDexCacheViaMethod; } + } else if (is_in_boot_image && !codegen->GetCompilerOptions().GetCompilePic()) { + // AOT app compilation. Check if the class is in the boot image. + desired_load_kind = HLoadClass::LoadKind::kBootImageAddress; + address = reinterpret_cast64<uint64_t>(klass); + } else { + // Not JIT and either the klass is not in boot image or we are compiling in PIC mode. + // Use PC-relative load from the dex cache if the dex file belongs + // to the oat file that we're currently compiling. + desired_load_kind = + ContainsElement(compiler_driver->GetDexFilesForOatFile(), &load_class->GetDexFile()) + ? HLoadClass::LoadKind::kDexCachePcRelative + : HLoadClass::LoadKind::kDexCacheViaMethod; } } + DCHECK_NE(desired_load_kind, static_cast<HLoadClass::LoadKind>(-1)); if (is_in_boot_image) { load_class->MarkInBootImage(); @@ -237,7 +243,7 @@ void HSharpening::ProcessLoadClass(HLoadClass* load_class) { load_class->MarkInDexCache(); } - HLoadClass::LoadKind load_kind = codegen_->GetSupportedLoadClassKind(desired_load_kind); + HLoadClass::LoadKind load_kind = codegen->GetSupportedLoadClassKind(desired_load_kind); switch (load_kind) { case HLoadClass::LoadKind::kBootImageLinkTimeAddress: case HLoadClass::LoadKind::kBootImageLinkTimePcRelative: @@ -245,12 +251,12 @@ void HSharpening::ProcessLoadClass(HLoadClass* load_class) { load_class->SetLoadKindWithTypeReference(load_kind, dex_file, type_index); break; case HLoadClass::LoadKind::kBootImageAddress: - case HLoadClass::LoadKind::kDexCacheAddress: + case HLoadClass::LoadKind::kJitTableAddress: DCHECK_NE(address, 0u); load_class->SetLoadKindWithAddress(load_kind, address); break; case HLoadClass::LoadKind::kDexCachePcRelative: { - PointerSize pointer_size = InstructionSetPointerSize(codegen_->GetInstructionSet()); + PointerSize pointer_size = InstructionSetPointerSize(codegen->GetInstructionSet()); DexCacheArraysLayout layout(pointer_size, &dex_file); size_t element_index = layout.TypeOffset(type_index); load_class->SetLoadKindWithDexCacheReference(load_kind, dex_file, element_index); diff --git a/compiler/optimizing/sharpening.h b/compiler/optimizing/sharpening.h index d35ae66e05..ae5ccb33ab 100644 --- a/compiler/optimizing/sharpening.h +++ b/compiler/optimizing/sharpening.h @@ -35,16 +35,26 @@ class HSharpening : public HOptimization { HSharpening(HGraph* graph, CodeGenerator* codegen, const DexCompilationUnit& compilation_unit, - CompilerDriver* compiler_driver) + CompilerDriver* compiler_driver, + VariableSizedHandleScope* handles) : HOptimization(graph, kSharpeningPassName), codegen_(codegen), compilation_unit_(compilation_unit), - compiler_driver_(compiler_driver) { } + compiler_driver_(compiler_driver), + handles_(handles) { } void Run() OVERRIDE; static constexpr const char* kSharpeningPassName = "sharpening"; + // Used internally but also by the inliner. + static void SharpenClass(HLoadClass* load_class, + mirror::Class* klass, + VariableSizedHandleScope* handles, + CodeGenerator* codegen, + CompilerDriver* compiler_driver) + REQUIRES_SHARED(Locks::mutator_lock_); + private: void ProcessInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke); void ProcessLoadClass(HLoadClass* load_class); @@ -53,6 +63,7 @@ class HSharpening : public HOptimization { CodeGenerator* codegen_; const DexCompilationUnit& compilation_unit_; CompilerDriver* compiler_driver_; + VariableSizedHandleScope* handles_; }; } // namespace art diff --git a/compiler/optimizing/ssa_test.cc b/compiler/optimizing/ssa_test.cc index 429763423c..f69f417efc 100644 --- a/compiler/optimizing/ssa_test.cc +++ b/compiler/optimizing/ssa_test.cc @@ -14,8 +14,9 @@ * limitations under the License. */ +#include "android-base/stringprintf.h" + #include "base/arena_allocator.h" -#include "base/stringprintf.h" #include "builder.h" #include "dex_file.h" #include "dex_instruction.h" @@ -35,7 +36,7 @@ class SsaPrettyPrinter : public HPrettyPrinter { explicit SsaPrettyPrinter(HGraph* graph) : HPrettyPrinter(graph), str_("") {} void PrintInt(int value) OVERRIDE { - str_ += StringPrintf("%d", value); + str_ += android::base::StringPrintf("%d", value); } void PrintString(const char* value) OVERRIDE { |