diff options
Diffstat (limited to 'compiler/optimizing')
21 files changed, 1342 insertions, 217 deletions
diff --git a/compiler/optimizing/builder.cc b/compiler/optimizing/builder.cc index 1b62531cb1..b6b8322f03 100644 --- a/compiler/optimizing/builder.cc +++ b/compiler/optimizing/builder.cc @@ -2745,20 +2745,16 @@ bool HGraphBuilder::AnalyzeDexInstruction(const Instruction& instruction, uint32 case Instruction::CONST_STRING: { uint32_t string_index = instruction.VRegB_21c(); - bool in_dex_cache = compiler_driver_->CanAssumeStringIsPresentInDexCache( - *dex_file_, string_index); current_block_->AddInstruction( - new (arena_) HLoadString(graph_->GetCurrentMethod(), string_index, dex_pc, in_dex_cache)); + new (arena_) HLoadString(graph_->GetCurrentMethod(), string_index, *dex_file_, dex_pc)); UpdateLocal(instruction.VRegA_21c(), current_block_->GetLastInstruction(), dex_pc); break; } case Instruction::CONST_STRING_JUMBO: { uint32_t string_index = instruction.VRegB_31c(); - bool in_dex_cache = compiler_driver_->CanAssumeStringIsPresentInDexCache( - *dex_file_, string_index); current_block_->AddInstruction( - new (arena_) HLoadString(graph_->GetCurrentMethod(), string_index, dex_pc, in_dex_cache)); + new (arena_) HLoadString(graph_->GetCurrentMethod(), string_index, *dex_file_, dex_pc)); UpdateLocal(instruction.VRegA_31c(), current_block_->GetLastInstruction(), dex_pc); break; } diff --git a/compiler/optimizing/code_generator.h b/compiler/optimizing/code_generator.h index e56323ff0f..cad55296bc 100644 --- a/compiler/optimizing/code_generator.h +++ b/compiler/optimizing/code_generator.h @@ -443,6 +443,11 @@ class CodeGenerator { uint32_t dex_pc, SlowPathCode* slow_path) = 0; + // Check if the desired_string_load_kind is supported. If it is, return it, + // otherwise return a fall-back info that should be used instead. + virtual HLoadString::LoadKind GetSupportedLoadStringKind( + HLoadString::LoadKind desired_string_load_kind) = 0; + // Check if the desired_dispatch_info is supported. If it is, return it, // otherwise return a fall-back info that should be used instead. virtual HInvokeStaticOrDirect::DispatchInfo GetSupportedInvokeStaticOrDirectDispatch( @@ -471,6 +476,18 @@ class CodeGenerator { LabelType label; }; + // String patch info used for recording locations of required linker patches and + // target strings. The actual string address can be absolute or PC-relative. + template <typename LabelType> + struct StringPatchInfo { + StringPatchInfo(const DexFile& df, uint32_t index) + : dex_file(df), string_index(index), label() { } + + const DexFile& dex_file; + uint32_t string_index; + LabelType label; + }; + CodeGenerator(HGraph* graph, size_t number_of_core_registers, size_t number_of_fpu_registers, diff --git a/compiler/optimizing/code_generator_arm.cc b/compiler/optimizing/code_generator_arm.cc index 3a18a0d7e7..98577d67ea 100644 --- a/compiler/optimizing/code_generator_arm.cc +++ b/compiler/optimizing/code_generator_arm.cc @@ -779,13 +779,19 @@ CodeGeneratorARM::CodeGeneratorARM(HGraph* graph, move_resolver_(graph->GetArena(), this), assembler_(), isa_features_(isa_features), + uint32_literals_(std::less<uint32_t>(), + graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), method_patches_(MethodReferenceComparator(), graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), call_patches_(MethodReferenceComparator(), graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), relative_call_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), - dex_cache_arrays_base_labels_(std::less<HArmDexCacheArraysBase*>(), - graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) { + pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + boot_image_string_patches_(StringReferenceValueComparator(), + graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + pc_relative_string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + boot_image_address_patches_(std::less<uint32_t>(), + graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) { // Always save the LR register to mimic Quick. AddAllocatedRegister(Location::RegisterLocation(LR)); } @@ -5221,12 +5227,57 @@ void InstructionCodeGeneratorARM::GenerateClassInitializationCheck( __ Bind(slow_path->GetExitLabel()); } +HLoadString::LoadKind CodeGeneratorARM::GetSupportedLoadStringKind( + HLoadString::LoadKind desired_string_load_kind) { + if (kEmitCompilerReadBarrier) { + switch (desired_string_load_kind) { + case HLoadString::LoadKind::kBootImageLinkTimeAddress: + case HLoadString::LoadKind::kBootImageLinkTimePcRelative: + case HLoadString::LoadKind::kBootImageAddress: + // TODO: Implement for read barrier. + return HLoadString::LoadKind::kDexCacheViaMethod; + default: + break; + } + } + switch (desired_string_load_kind) { + case HLoadString::LoadKind::kBootImageLinkTimeAddress: + DCHECK(!GetCompilerOptions().GetCompilePic()); + break; + case HLoadString::LoadKind::kBootImageLinkTimePcRelative: + DCHECK(GetCompilerOptions().GetCompilePic()); + break; + case HLoadString::LoadKind::kBootImageAddress: + break; + case HLoadString::LoadKind::kDexCacheAddress: + DCHECK(Runtime::Current()->UseJit()); + break; + case HLoadString::LoadKind::kDexCachePcRelative: + DCHECK(!Runtime::Current()->UseJit()); + // We disable pc-relative load when there is an irreducible loop, as the optimization + // is incompatible with it. + // TODO: Create as many ArmDexCacheArraysBase instructions as needed for methods + // with irreducible loops. + if (GetGraph()->HasIrreducibleLoops()) { + return HLoadString::LoadKind::kDexCacheViaMethod; + } + break; + case HLoadString::LoadKind::kDexCacheViaMethod: + break; + } + return desired_string_load_kind; +} + void LocationsBuilderARM::VisitLoadString(HLoadString* load) { - LocationSummary::CallKind call_kind = (!load->IsInDexCache() || kEmitCompilerReadBarrier) + LocationSummary::CallKind call_kind = (load->NeedsEnvironment() || kEmitCompilerReadBarrier) ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall; LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(load, call_kind); - locations->SetInAt(0, Location::RequiresRegister()); + HLoadString::LoadKind load_kind = load->GetLoadKind(); + if (load_kind == HLoadString::LoadKind::kDexCacheViaMethod || + load_kind == HLoadString::LoadKind::kDexCachePcRelative) { + locations->SetInAt(0, Location::RequiresRegister()); + } locations->SetOut(Location::RequiresRegister()); } @@ -5234,16 +5285,73 @@ void InstructionCodeGeneratorARM::VisitLoadString(HLoadString* load) { LocationSummary* locations = load->GetLocations(); Location out_loc = locations->Out(); Register out = out_loc.AsRegister<Register>(); - Register current_method = locations->InAt(0).AsRegister<Register>(); - // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_ - GenerateGcRootFieldLoad( - load, out_loc, current_method, ArtMethod::DeclaringClassOffset().Int32Value()); - // /* GcRoot<mirror::String>[] */ out = out->dex_cache_strings_ - __ LoadFromOffset(kLoadWord, out, out, mirror::Class::DexCacheStringsOffset().Int32Value()); - // /* GcRoot<mirror::String> */ out = out[string_index] - GenerateGcRootFieldLoad( - load, out_loc, out, CodeGenerator::GetCacheOffset(load->GetStringIndex())); + switch (load->GetLoadKind()) { + case HLoadString::LoadKind::kBootImageLinkTimeAddress: { + DCHECK(!kEmitCompilerReadBarrier); + __ LoadLiteral(out, codegen_->DeduplicateBootImageStringLiteral(load->GetDexFile(), + load->GetStringIndex())); + return; // No dex cache slow path. + } + case HLoadString::LoadKind::kBootImageLinkTimePcRelative: { + DCHECK(!kEmitCompilerReadBarrier); + CodeGeneratorARM::PcRelativePatchInfo* labels = + codegen_->NewPcRelativeStringPatch(load->GetDexFile(), load->GetStringIndex()); + __ BindTrackedLabel(&labels->movw_label); + __ movw(out, /* placeholder */ 0u); + __ BindTrackedLabel(&labels->movt_label); + __ movt(out, /* placeholder */ 0u); + __ BindTrackedLabel(&labels->add_pc_label); + __ add(out, out, ShifterOperand(PC)); + return; // No dex cache slow path. + } + case HLoadString::LoadKind::kBootImageAddress: { + DCHECK(!kEmitCompilerReadBarrier); + DCHECK_NE(load->GetAddress(), 0u); + uint32_t address = dchecked_integral_cast<uint32_t>(load->GetAddress()); + __ LoadLiteral(out, codegen_->DeduplicateBootImageAddressLiteral(address)); + return; // No dex cache slow path. + } + case HLoadString::LoadKind::kDexCacheAddress: { + DCHECK_NE(load->GetAddress(), 0u); + uint32_t address = dchecked_integral_cast<uint32_t>(load->GetAddress()); + // 16-bit LDR immediate has a 5-bit offset multiplied by the size and that gives + // a 128B range. To try and reduce the number of literals if we load multiple strings, + // simply split the dex cache address to a 128B aligned base loaded from a literal + // and the remaining offset embedded in the load. + static_assert(sizeof(GcRoot<mirror::String>) == 4u, "Expected GC root to be 4 bytes."); + DCHECK_ALIGNED(load->GetAddress(), 4u); + constexpr size_t offset_bits = /* encoded bits */ 5 + /* scale */ 2; + uint32_t base_address = address & ~MaxInt<uint32_t>(offset_bits); + uint32_t offset = address & MaxInt<uint32_t>(offset_bits); + __ LoadLiteral(out, codegen_->DeduplicateDexCacheAddressLiteral(base_address)); + GenerateGcRootFieldLoad(load, out_loc, out, offset); + break; + } + case HLoadString::LoadKind::kDexCachePcRelative: { + Register base_reg = locations->InAt(0).AsRegister<Register>(); + HArmDexCacheArraysBase* base = load->InputAt(0)->AsArmDexCacheArraysBase(); + int32_t offset = load->GetDexCacheElementOffset() - base->GetElementOffset(); + GenerateGcRootFieldLoad(load, out_loc, base_reg, offset); + break; + } + case HLoadString::LoadKind::kDexCacheViaMethod: { + Register current_method = locations->InAt(0).AsRegister<Register>(); + + // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_ + GenerateGcRootFieldLoad( + load, out_loc, current_method, ArtMethod::DeclaringClassOffset().Int32Value()); + // /* GcRoot<mirror::String>[] */ out = out->dex_cache_strings_ + __ LoadFromOffset(kLoadWord, out, out, mirror::Class::DexCacheStringsOffset().Int32Value()); + // /* GcRoot<mirror::String> */ out = out[string_index] + GenerateGcRootFieldLoad( + load, out_loc, out, CodeGenerator::GetCacheOffset(load->GetStringIndex())); + break; + } + default: + LOG(FATAL) << "Unexpected load kind: " << load->GetLoadKind(); + UNREACHABLE(); + } if (!load->IsInDexCache()) { SlowPathCode* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathARM(load); @@ -6220,6 +6328,8 @@ HInvokeStaticOrDirect::DispatchInfo CodeGeneratorARM::GetSupportedInvokeStaticOr HInvokeStaticOrDirect::DispatchInfo dispatch_info = desired_dispatch_info; // We disable pc-relative load when there is an irreducible loop, as the optimization // is incompatible with it. + // TODO: Create as many ArmDexCacheArraysBase instructions as needed for methods + // with irreducible loops. if (GetGraph()->HasIrreducibleLoops() && (dispatch_info.method_load_kind == HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative)) { @@ -6399,13 +6509,49 @@ void CodeGeneratorARM::GenerateVirtualCall(HInvokeVirtual* invoke, Location temp __ blx(LR); } +CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativeStringPatch( + const DexFile& dex_file, uint32_t string_index) { + return NewPcRelativePatch(dex_file, string_index, &pc_relative_string_patches_); +} + +CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativeDexCacheArrayPatch( + const DexFile& dex_file, uint32_t element_offset) { + return NewPcRelativePatch(dex_file, element_offset, &pc_relative_dex_cache_patches_); +} + +CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativePatch( + const DexFile& dex_file, uint32_t offset_or_index, ArenaDeque<PcRelativePatchInfo>* patches) { + patches->emplace_back(dex_file, offset_or_index); + return &patches->back(); +} + +Literal* CodeGeneratorARM::DeduplicateBootImageStringLiteral(const DexFile& dex_file, + uint32_t string_index) { + return boot_image_string_patches_.GetOrCreate( + StringReference(&dex_file, string_index), + [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); +} + +Literal* CodeGeneratorARM::DeduplicateBootImageAddressLiteral(uint32_t address) { + bool needs_patch = GetCompilerOptions().GetIncludePatchInformation(); + Uint32ToLiteralMap* map = needs_patch ? &boot_image_address_patches_ : &uint32_literals_; + return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address), map); +} + +Literal* CodeGeneratorARM::DeduplicateDexCacheAddressLiteral(uint32_t address) { + return DeduplicateUint32Literal(address, &uint32_literals_); +} + void CodeGeneratorARM::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) { DCHECK(linker_patches->empty()); size_t size = method_patches_.size() + call_patches_.size() + relative_call_patches_.size() + - /* MOVW+MOVT for each base */ 2u * dex_cache_arrays_base_labels_.size(); + /* MOVW+MOVT for each base */ 2u * pc_relative_dex_cache_patches_.size() + + boot_image_string_patches_.size() + + /* MOVW+MOVT for each base */ 2u * pc_relative_string_patches_.size() + + boot_image_address_patches_.size(); linker_patches->reserve(size); for (const auto& entry : method_patches_) { const MethodReference& target_method = entry.first; @@ -6431,41 +6577,75 @@ void CodeGeneratorARM::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patche info.target_method.dex_file, info.target_method.dex_method_index)); } - for (const auto& pair : dex_cache_arrays_base_labels_) { - HArmDexCacheArraysBase* base = pair.first; - const DexCacheArraysBaseLabels* labels = &pair.second; - const DexFile& dex_file = base->GetDexFile(); - size_t base_element_offset = base->GetElementOffset(); - DCHECK(labels->add_pc_label.IsBound()); - uint32_t add_pc_offset = dchecked_integral_cast<uint32_t>(labels->add_pc_label.Position()); + for (const PcRelativePatchInfo& info : pc_relative_dex_cache_patches_) { + const DexFile& dex_file = info.target_dex_file; + size_t base_element_offset = info.offset_or_index; + DCHECK(info.add_pc_label.IsBound()); + uint32_t add_pc_offset = dchecked_integral_cast<uint32_t>(info.add_pc_label.Position()); // Add MOVW patch. - DCHECK(labels->movw_label.IsBound()); - uint32_t movw_offset = dchecked_integral_cast<uint32_t>(labels->movw_label.Position()); + DCHECK(info.movw_label.IsBound()); + uint32_t movw_offset = dchecked_integral_cast<uint32_t>(info.movw_label.Position()); linker_patches->push_back(LinkerPatch::DexCacheArrayPatch(movw_offset, &dex_file, add_pc_offset, base_element_offset)); // Add MOVT patch. - DCHECK(labels->movt_label.IsBound()); - uint32_t movt_offset = dchecked_integral_cast<uint32_t>(labels->movt_label.Position()); + DCHECK(info.movt_label.IsBound()); + uint32_t movt_offset = dchecked_integral_cast<uint32_t>(info.movt_label.Position()); linker_patches->push_back(LinkerPatch::DexCacheArrayPatch(movt_offset, &dex_file, add_pc_offset, base_element_offset)); } + for (const auto& entry : boot_image_string_patches_) { + const StringReference& target_string = entry.first; + Literal* literal = entry.second; + DCHECK(literal->GetLabel()->IsBound()); + uint32_t literal_offset = literal->GetLabel()->Position(); + linker_patches->push_back(LinkerPatch::StringPatch(literal_offset, + target_string.dex_file, + target_string.string_index)); + } + for (const PcRelativePatchInfo& info : pc_relative_string_patches_) { + const DexFile& dex_file = info.target_dex_file; + uint32_t string_index = info.offset_or_index; + DCHECK(info.add_pc_label.IsBound()); + uint32_t add_pc_offset = dchecked_integral_cast<uint32_t>(info.add_pc_label.Position()); + // Add MOVW patch. + DCHECK(info.movw_label.IsBound()); + uint32_t movw_offset = dchecked_integral_cast<uint32_t>(info.movw_label.Position()); + linker_patches->push_back(LinkerPatch::RelativeStringPatch(movw_offset, + &dex_file, + add_pc_offset, + string_index)); + // Add MOVT patch. + DCHECK(info.movt_label.IsBound()); + uint32_t movt_offset = dchecked_integral_cast<uint32_t>(info.movt_label.Position()); + linker_patches->push_back(LinkerPatch::RelativeStringPatch(movt_offset, + &dex_file, + add_pc_offset, + string_index)); + } + for (const auto& entry : boot_image_address_patches_) { + DCHECK(GetCompilerOptions().GetIncludePatchInformation()); + Literal* literal = entry.second; + DCHECK(literal->GetLabel()->IsBound()); + uint32_t literal_offset = literal->GetLabel()->Position(); + linker_patches->push_back(LinkerPatch::RecordPosition(literal_offset)); + } +} + +Literal* CodeGeneratorARM::DeduplicateUint32Literal(uint32_t value, Uint32ToLiteralMap* map) { + return map->GetOrCreate( + value, + [this, value]() { return __ NewLiteral<uint32_t>(value); }); } Literal* CodeGeneratorARM::DeduplicateMethodLiteral(MethodReference target_method, MethodToLiteralMap* map) { - // Look up the literal for target_method. - auto lb = map->lower_bound(target_method); - if (lb != map->end() && !map->key_comp()(target_method, lb->first)) { - return lb->second; - } - // We don't have a literal for this method yet, insert a new one. - Literal* literal = __ NewLiteral<uint32_t>(0u); - map->PutBefore(lb, target_method, literal); - return literal; + return map->GetOrCreate( + target_method, + [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); } Literal* CodeGeneratorARM::DeduplicateMethodAddressLiteral(MethodReference target_method) { @@ -6600,16 +6780,16 @@ void InstructionCodeGeneratorARM::VisitPackedSwitch(HPackedSwitch* switch_instr) void LocationsBuilderARM::VisitArmDexCacheArraysBase(HArmDexCacheArraysBase* base) { LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(base); locations->SetOut(Location::RequiresRegister()); - codegen_->AddDexCacheArraysBase(base); } void InstructionCodeGeneratorARM::VisitArmDexCacheArraysBase(HArmDexCacheArraysBase* base) { Register base_reg = base->GetLocations()->Out().AsRegister<Register>(); - CodeGeneratorARM::DexCacheArraysBaseLabels* labels = codegen_->GetDexCacheArraysBaseLabels(base); + CodeGeneratorARM::PcRelativePatchInfo* labels = + codegen_->NewPcRelativeDexCacheArrayPatch(base->GetDexFile(), base->GetElementOffset()); __ BindTrackedLabel(&labels->movw_label); - __ movw(base_reg, 0u); + __ movw(base_reg, /* placeholder */ 0u); __ BindTrackedLabel(&labels->movt_label); - __ movt(base_reg, 0u); + __ movt(base_reg, /* placeholder */ 0u); __ BindTrackedLabel(&labels->add_pc_label); __ add(base_reg, base_reg, ShifterOperand(PC)); } diff --git a/compiler/optimizing/code_generator_arm.h b/compiler/optimizing/code_generator_arm.h index cc4aa144c0..84341284a3 100644 --- a/compiler/optimizing/code_generator_arm.h +++ b/compiler/optimizing/code_generator_arm.h @@ -23,6 +23,7 @@ #include "nodes.h" #include "parallel_move_resolver.h" #include "utils/arm/assembler_thumb2.h" +#include "utils/string_reference.h" namespace art { namespace arm { @@ -403,6 +404,11 @@ class CodeGeneratorARM : public CodeGenerator { Label* GetFrameEntryLabel() { return &frame_entry_label_; } + // Check if the desired_string_load_kind is supported. If it is, return it, + // otherwise return a fall-back kind that should be used instead. + HLoadString::LoadKind GetSupportedLoadStringKind( + HLoadString::LoadKind desired_string_load_kind) OVERRIDE; + // Check if the desired_dispatch_info is supported. If it is, return it, // otherwise return a fall-back info that should be used instead. HInvokeStaticOrDirect::DispatchInfo GetSupportedInvokeStaticOrDirectDispatch( @@ -414,32 +420,34 @@ class CodeGeneratorARM : public CodeGenerator { void MoveFromReturnRegister(Location trg, Primitive::Type type) OVERRIDE; - void EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) OVERRIDE; - - // The PC-relative base address is loaded with three instructions, MOVW+MOVT + // The PcRelativePatchInfo is used for PC-relative addressing of dex cache arrays + // and boot image strings. The only difference is the interpretation of the offset_or_index. + // The PC-relative address is loaded with three instructions, MOVW+MOVT // to load the offset to base_reg and then ADD base_reg, PC. The offset is // calculated from the ADD's effective PC, i.e. PC+4 on Thumb2. Though we // currently emit these 3 instructions together, instruction scheduling could // split this sequence apart, so we keep separate labels for each of them. - struct DexCacheArraysBaseLabels { - DexCacheArraysBaseLabels() = default; - DexCacheArraysBaseLabels(DexCacheArraysBaseLabels&& other) = default; - + struct PcRelativePatchInfo { + PcRelativePatchInfo(const DexFile& dex_file, uint32_t off_or_idx) + : target_dex_file(dex_file), offset_or_index(off_or_idx) { } + PcRelativePatchInfo(PcRelativePatchInfo&& other) = default; + + const DexFile& target_dex_file; + // Either the dex cache array element offset or the string index. + uint32_t offset_or_index; Label movw_label; Label movt_label; Label add_pc_label; }; - void AddDexCacheArraysBase(HArmDexCacheArraysBase* base) { - DexCacheArraysBaseLabels labels; - dex_cache_arrays_base_labels_.Put(base, std::move(labels)); - } + PcRelativePatchInfo* NewPcRelativeStringPatch(const DexFile& dex_file, uint32_t string_index); + PcRelativePatchInfo* NewPcRelativeDexCacheArrayPatch(const DexFile& dex_file, + uint32_t element_offset); + Literal* DeduplicateBootImageStringLiteral(const DexFile& dex_file, uint32_t string_index); + Literal* DeduplicateBootImageAddressLiteral(uint32_t address); + Literal* DeduplicateDexCacheAddressLiteral(uint32_t address); - DexCacheArraysBaseLabels* GetDexCacheArraysBaseLabels(HArmDexCacheArraysBase* base) { - auto it = dex_cache_arrays_base_labels_.find(base); - DCHECK(it != dex_cache_arrays_base_labels_.end()); - return &it->second; - } + void EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) OVERRIDE; // Fast path implementation of ReadBarrier::Barrier for a heap // reference field load when Baker's read barriers are used. @@ -525,14 +533,19 @@ class CodeGeneratorARM : public CodeGenerator { Register GetInvokeStaticOrDirectExtraParameter(HInvokeStaticOrDirect* invoke, Register temp); + using Uint32ToLiteralMap = ArenaSafeMap<uint32_t, Literal*>; using MethodToLiteralMap = ArenaSafeMap<MethodReference, Literal*, MethodReferenceComparator>; - using DexCacheArraysBaseToLabelsMap = ArenaSafeMap<HArmDexCacheArraysBase*, - DexCacheArraysBaseLabels, - std::less<HArmDexCacheArraysBase*>>; + using BootStringToLiteralMap = ArenaSafeMap<StringReference, + Literal*, + StringReferenceValueComparator>; + Literal* DeduplicateUint32Literal(uint32_t value, Uint32ToLiteralMap* map); Literal* DeduplicateMethodLiteral(MethodReference target_method, MethodToLiteralMap* map); Literal* DeduplicateMethodAddressLiteral(MethodReference target_method); Literal* DeduplicateMethodCodeLiteral(MethodReference target_method); + PcRelativePatchInfo* NewPcRelativePatch(const DexFile& dex_file, + uint32_t offset_or_index, + ArenaDeque<PcRelativePatchInfo>* patches); // Labels for each block that will be compiled. Label* block_labels_; // Indexed by block id. @@ -543,14 +556,22 @@ class CodeGeneratorARM : public CodeGenerator { Thumb2Assembler assembler_; const ArmInstructionSetFeatures& isa_features_; + // Deduplication map for 32-bit literals, used for non-patchable boot image addresses. + Uint32ToLiteralMap uint32_literals_; // Method patch info, map MethodReference to a literal for method address and method code. MethodToLiteralMap method_patches_; MethodToLiteralMap call_patches_; // Relative call patch info. // Using ArenaDeque<> which retains element addresses on push/emplace_back(). ArenaDeque<MethodPatchInfo<Label>> relative_call_patches_; - - DexCacheArraysBaseToLabelsMap dex_cache_arrays_base_labels_; + // PC-relative patch info for each HArmDexCacheArraysBase. + ArenaDeque<PcRelativePatchInfo> pc_relative_dex_cache_patches_; + // Deduplication map for boot string literals for kBootImageLinkTimeAddress. + BootStringToLiteralMap boot_image_string_patches_; + // PC-relative String patch info. + ArenaDeque<PcRelativePatchInfo> pc_relative_string_patches_; + // Deduplication map for patchable boot image addresses. + Uint32ToLiteralMap boot_image_address_patches_; DISALLOW_COPY_AND_ASSIGN(CodeGeneratorARM); }; diff --git a/compiler/optimizing/code_generator_arm64.cc b/compiler/optimizing/code_generator_arm64.cc index 1f577b38b7..491014d3e5 100644 --- a/compiler/optimizing/code_generator_arm64.cc +++ b/compiler/optimizing/code_generator_arm64.cc @@ -905,6 +905,8 @@ CodeGeneratorARM64::CodeGeneratorARM64(HGraph* graph, instruction_visitor_(graph, this), move_resolver_(graph->GetArena(), this), isa_features_(isa_features), + uint32_literals_(std::less<uint32_t>(), + graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), uint64_literals_(std::less<uint64_t>(), graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), method_patches_(MethodReferenceComparator(), @@ -912,7 +914,12 @@ CodeGeneratorARM64::CodeGeneratorARM64(HGraph* graph, call_patches_(MethodReferenceComparator(), graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), relative_call_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), - pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) { + pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + boot_image_string_patches_(StringReferenceValueComparator(), + graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + pc_relative_string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + boot_image_address_patches_(std::less<uint32_t>(), + graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) { // Save the link register (containing the return address) to mimic Quick. AddAllocatedRegister(LocationFrom(lr)); } @@ -3662,23 +3669,21 @@ void CodeGeneratorARM64::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invok break; case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: { // Add ADRP with its PC-relative DexCache access patch. - pc_relative_dex_cache_patches_.emplace_back(*invoke->GetTargetMethod().dex_file, - invoke->GetDexCacheArrayOffset()); - vixl::Label* pc_insn_label = &pc_relative_dex_cache_patches_.back().label; + const DexFile& dex_file = *invoke->GetTargetMethod().dex_file; + uint32_t element_offset = invoke->GetDexCacheArrayOffset(); + vixl::Label* adrp_label = NewPcRelativeDexCacheArrayPatch(dex_file, element_offset); { vixl::SingleEmissionCheckScope guard(GetVIXLAssembler()); - __ Bind(pc_insn_label); - __ adrp(XRegisterFrom(temp), 0); + __ Bind(adrp_label); + __ adrp(XRegisterFrom(temp), /* offset placeholder */ 0); } - pc_relative_dex_cache_patches_.back().pc_insn_label = pc_insn_label; // Add LDR with its PC-relative DexCache access patch. - pc_relative_dex_cache_patches_.emplace_back(*invoke->GetTargetMethod().dex_file, - invoke->GetDexCacheArrayOffset()); + vixl::Label* ldr_label = + NewPcRelativeDexCacheArrayPatch(dex_file, element_offset, adrp_label); { vixl::SingleEmissionCheckScope guard(GetVIXLAssembler()); - __ Bind(&pc_relative_dex_cache_patches_.back().label); - __ ldr(XRegisterFrom(temp), MemOperand(XRegisterFrom(temp), 0)); - pc_relative_dex_cache_patches_.back().pc_insn_label = pc_insn_label; + __ Bind(ldr_label); + __ ldr(XRegisterFrom(temp), MemOperand(XRegisterFrom(temp), /* offset placeholder */ 0)); } break; } @@ -3772,13 +3777,58 @@ void CodeGeneratorARM64::GenerateVirtualCall(HInvokeVirtual* invoke, Location te __ Blr(lr); } +vixl::Label* CodeGeneratorARM64::NewPcRelativeStringPatch(const DexFile& dex_file, + uint32_t string_index, + vixl::Label* adrp_label) { + return NewPcRelativePatch(dex_file, string_index, adrp_label, &pc_relative_string_patches_); +} + +vixl::Label* CodeGeneratorARM64::NewPcRelativeDexCacheArrayPatch(const DexFile& dex_file, + uint32_t element_offset, + vixl::Label* adrp_label) { + return NewPcRelativePatch(dex_file, element_offset, adrp_label, &pc_relative_dex_cache_patches_); +} + +vixl::Label* CodeGeneratorARM64::NewPcRelativePatch(const DexFile& dex_file, + uint32_t offset_or_index, + vixl::Label* adrp_label, + ArenaDeque<PcRelativePatchInfo>* patches) { + // Add a patch entry and return the label. + patches->emplace_back(dex_file, offset_or_index); + PcRelativePatchInfo* info = &patches->back(); + vixl::Label* label = &info->label; + // If adrp_label is null, this is the ADRP patch and needs to point to its own label. + info->pc_insn_label = (adrp_label != nullptr) ? adrp_label : label; + return label; +} + +vixl::Literal<uint32_t>* CodeGeneratorARM64::DeduplicateBootImageStringLiteral( + const DexFile& dex_file, uint32_t string_index) { + return boot_image_string_patches_.GetOrCreate( + StringReference(&dex_file, string_index), + [this]() { return __ CreateLiteralDestroyedWithPool<uint32_t>(/* placeholder */ 0u); }); +} + +vixl::Literal<uint32_t>* CodeGeneratorARM64::DeduplicateBootImageAddressLiteral(uint64_t address) { + bool needs_patch = GetCompilerOptions().GetIncludePatchInformation(); + Uint32ToLiteralMap* map = needs_patch ? &boot_image_address_patches_ : &uint32_literals_; + return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address), map); +} + +vixl::Literal<uint64_t>* CodeGeneratorARM64::DeduplicateDexCacheAddressLiteral(uint64_t address) { + return DeduplicateUint64Literal(address); +} + void CodeGeneratorARM64::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) { DCHECK(linker_patches->empty()); size_t size = method_patches_.size() + call_patches_.size() + relative_call_patches_.size() + - pc_relative_dex_cache_patches_.size(); + pc_relative_dex_cache_patches_.size() + + boot_image_string_patches_.size() + + pc_relative_string_patches_.size() + + boot_image_address_patches_.size(); linker_patches->reserve(size); for (const auto& entry : method_patches_) { const MethodReference& target_method = entry.first; @@ -3799,38 +3849,51 @@ void CodeGeneratorARM64::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patc info.target_method.dex_file, info.target_method.dex_method_index)); } - for (const PcRelativeDexCacheAccessInfo& info : pc_relative_dex_cache_patches_) { + for (const PcRelativePatchInfo& info : pc_relative_dex_cache_patches_) { linker_patches->push_back(LinkerPatch::DexCacheArrayPatch(info.label.location(), &info.target_dex_file, info.pc_insn_label->location(), - info.element_offset)); + info.offset_or_index)); } + for (const auto& entry : boot_image_string_patches_) { + const StringReference& target_string = entry.first; + vixl::Literal<uint32_t>* literal = entry.second; + linker_patches->push_back(LinkerPatch::StringPatch(literal->offset(), + target_string.dex_file, + target_string.string_index)); + } + for (const PcRelativePatchInfo& info : pc_relative_string_patches_) { + linker_patches->push_back(LinkerPatch::RelativeStringPatch(info.label.location(), + &info.target_dex_file, + info.pc_insn_label->location(), + info.offset_or_index)); + } + for (const auto& entry : boot_image_address_patches_) { + DCHECK(GetCompilerOptions().GetIncludePatchInformation()); + vixl::Literal<uint32_t>* literal = entry.second; + linker_patches->push_back(LinkerPatch::RecordPosition(literal->offset())); + } +} + +vixl::Literal<uint32_t>* CodeGeneratorARM64::DeduplicateUint32Literal(uint32_t value, + Uint32ToLiteralMap* map) { + return map->GetOrCreate( + value, + [this, value]() { return __ CreateLiteralDestroyedWithPool<uint32_t>(value); }); } vixl::Literal<uint64_t>* CodeGeneratorARM64::DeduplicateUint64Literal(uint64_t value) { - // Look up the literal for value. - auto lb = uint64_literals_.lower_bound(value); - if (lb != uint64_literals_.end() && !uint64_literals_.key_comp()(value, lb->first)) { - return lb->second; - } - // We don't have a literal for this value, insert a new one. - vixl::Literal<uint64_t>* literal = __ CreateLiteralDestroyedWithPool<uint64_t>(value); - uint64_literals_.PutBefore(lb, value, literal); - return literal; + return uint64_literals_.GetOrCreate( + value, + [this, value]() { return __ CreateLiteralDestroyedWithPool<uint64_t>(value); }); } vixl::Literal<uint64_t>* CodeGeneratorARM64::DeduplicateMethodLiteral( MethodReference target_method, MethodToLiteralMap* map) { - // Look up the literal for target_method. - auto lb = map->lower_bound(target_method); - if (lb != map->end() && !map->key_comp()(target_method, lb->first)) { - return lb->second; - } - // We don't have a literal for this method yet, insert a new one. - vixl::Literal<uint64_t>* literal = __ CreateLiteralDestroyedWithPool<uint64_t>(0u); - map->PutBefore(lb, target_method, literal); - return literal; + return map->GetOrCreate( + target_method, + [this]() { return __ CreateLiteralDestroyedWithPool<uint64_t>(/* placeholder */ 0u); }); } vixl::Literal<uint64_t>* CodeGeneratorARM64::DeduplicateMethodAddressLiteral( @@ -3955,28 +4018,135 @@ void InstructionCodeGeneratorARM64::VisitLoadLocal(HLoadLocal* load ATTRIBUTE_UN // Nothing to do, this is driven by the code generator. } +HLoadString::LoadKind CodeGeneratorARM64::GetSupportedLoadStringKind( + HLoadString::LoadKind desired_string_load_kind) { + if (kEmitCompilerReadBarrier) { + switch (desired_string_load_kind) { + case HLoadString::LoadKind::kBootImageLinkTimeAddress: + case HLoadString::LoadKind::kBootImageLinkTimePcRelative: + case HLoadString::LoadKind::kBootImageAddress: + // TODO: Implement for read barrier. + return HLoadString::LoadKind::kDexCacheViaMethod; + default: + break; + } + } + switch (desired_string_load_kind) { + case HLoadString::LoadKind::kBootImageLinkTimeAddress: + DCHECK(!GetCompilerOptions().GetCompilePic()); + break; + case HLoadString::LoadKind::kBootImageLinkTimePcRelative: + DCHECK(GetCompilerOptions().GetCompilePic()); + break; + case HLoadString::LoadKind::kBootImageAddress: + break; + case HLoadString::LoadKind::kDexCacheAddress: + DCHECK(Runtime::Current()->UseJit()); + break; + case HLoadString::LoadKind::kDexCachePcRelative: + DCHECK(!Runtime::Current()->UseJit()); + break; + case HLoadString::LoadKind::kDexCacheViaMethod: + break; + } + return desired_string_load_kind; +} + void LocationsBuilderARM64::VisitLoadString(HLoadString* load) { - LocationSummary::CallKind call_kind = (!load->IsInDexCache() || kEmitCompilerReadBarrier) + LocationSummary::CallKind call_kind = (load->NeedsEnvironment() || kEmitCompilerReadBarrier) ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall; LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(load, call_kind); - locations->SetInAt(0, Location::RequiresRegister()); + if (load->GetLoadKind() == HLoadString::LoadKind::kDexCacheViaMethod) { + locations->SetInAt(0, Location::RequiresRegister()); + } locations->SetOut(Location::RequiresRegister()); } void InstructionCodeGeneratorARM64::VisitLoadString(HLoadString* load) { Location out_loc = load->GetLocations()->Out(); Register out = OutputRegister(load); - Register current_method = InputRegisterAt(load, 0); - // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_ - GenerateGcRootFieldLoad( - load, out_loc, current_method, ArtMethod::DeclaringClassOffset().Int32Value()); - // /* GcRoot<mirror::String>[] */ out = out->dex_cache_strings_ - __ Ldr(out.X(), HeapOperand(out, mirror::Class::DexCacheStringsOffset().Uint32Value())); - // /* GcRoot<mirror::String> */ out = out[string_index] - GenerateGcRootFieldLoad( - load, out_loc, out.X(), CodeGenerator::GetCacheOffset(load->GetStringIndex())); + switch (load->GetLoadKind()) { + case HLoadString::LoadKind::kBootImageLinkTimeAddress: + DCHECK(!kEmitCompilerReadBarrier); + __ Ldr(out, codegen_->DeduplicateBootImageStringLiteral(load->GetDexFile(), + load->GetStringIndex())); + return; // No dex cache slow path. + case HLoadString::LoadKind::kBootImageLinkTimePcRelative: { + DCHECK(!kEmitCompilerReadBarrier); + // Add ADRP with its PC-relative String patch. + const DexFile& dex_file = load->GetDexFile(); + uint32_t string_index = load->GetStringIndex(); + vixl::Label* adrp_label = codegen_->NewPcRelativeStringPatch(dex_file, string_index); + { + vixl::SingleEmissionCheckScope guard(GetVIXLAssembler()); + __ Bind(adrp_label); + __ adrp(out.X(), /* offset placeholder */ 0); + } + // Add ADD with its PC-relative String patch. + vixl::Label* add_label = + codegen_->NewPcRelativeStringPatch(dex_file, string_index, adrp_label); + { + vixl::SingleEmissionCheckScope guard(GetVIXLAssembler()); + __ Bind(add_label); + __ add(out.X(), out.X(), Operand(/* offset placeholder */ 0)); + } + return; // No dex cache slow path. + } + case HLoadString::LoadKind::kBootImageAddress: { + DCHECK(!kEmitCompilerReadBarrier); + DCHECK(load->GetAddress() != 0u && IsUint<32>(load->GetAddress())); + __ Ldr(out.W(), codegen_->DeduplicateBootImageAddressLiteral(load->GetAddress())); + return; // No dex cache slow path. + } + case HLoadString::LoadKind::kDexCacheAddress: { + DCHECK_NE(load->GetAddress(), 0u); + // LDR immediate has a 12-bit offset multiplied by the size and for 32-bit loads + // that gives a 16KiB range. To try and reduce the number of literals if we load + // multiple strings, simply split the dex cache address to a 16KiB aligned base + // loaded from a literal and the remaining offset embedded in the load. + static_assert(sizeof(GcRoot<mirror::String>) == 4u, "Expected GC root to be 4 bytes."); + DCHECK_ALIGNED(load->GetAddress(), 4u); + constexpr size_t offset_bits = /* encoded bits */ 12 + /* scale */ 2; + uint64_t base_address = load->GetAddress() & ~MaxInt<uint64_t>(offset_bits); + uint32_t offset = load->GetAddress() & MaxInt<uint64_t>(offset_bits); + __ Ldr(out.X(), codegen_->DeduplicateDexCacheAddressLiteral(base_address)); + GenerateGcRootFieldLoad(load, out_loc, out.X(), offset); + break; + } + case HLoadString::LoadKind::kDexCachePcRelative: { + // Add ADRP with its PC-relative DexCache access patch. + const DexFile& dex_file = load->GetDexFile(); + uint32_t element_offset = load->GetDexCacheElementOffset(); + vixl::Label* adrp_label = codegen_->NewPcRelativeDexCacheArrayPatch(dex_file, element_offset); + { + vixl::SingleEmissionCheckScope guard(GetVIXLAssembler()); + __ Bind(adrp_label); + __ adrp(out.X(), /* offset placeholder */ 0); + } + // Add LDR with its PC-relative DexCache access patch. + vixl::Label* ldr_label = + codegen_->NewPcRelativeDexCacheArrayPatch(dex_file, element_offset, adrp_label); + GenerateGcRootFieldLoad(load, out_loc, out.X(), /* offset placeholder */ 0, ldr_label); + break; + } + case HLoadString::LoadKind::kDexCacheViaMethod: { + Register current_method = InputRegisterAt(load, 0); + // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_ + GenerateGcRootFieldLoad( + load, out_loc, current_method, ArtMethod::DeclaringClassOffset().Int32Value()); + // /* GcRoot<mirror::String>[] */ out = out->dex_cache_strings_ + __ Ldr(out.X(), HeapOperand(out, mirror::Class::DexCacheStringsOffset().Uint32Value())); + // /* GcRoot<mirror::String> */ out = out[string_index] + GenerateGcRootFieldLoad( + load, out_loc, out.X(), CodeGenerator::GetCacheOffset(load->GetStringIndex())); + break; + } + default: + LOG(FATAL) << "Unexpected load kind: " << load->GetLoadKind(); + UNREACHABLE(); + } if (!load->IsInDexCache()) { SlowPathCodeARM64* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathARM64(load); @@ -4791,7 +4961,8 @@ void InstructionCodeGeneratorARM64::GenerateReferenceLoadTwoRegisters(HInstructi void InstructionCodeGeneratorARM64::GenerateGcRootFieldLoad(HInstruction* instruction, Location root, vixl::Register obj, - uint32_t offset) { + uint32_t offset, + vixl::Label* fixup_label) { Register root_reg = RegisterFrom(root, Primitive::kPrimNot); if (kEmitCompilerReadBarrier) { if (kUseBakerReadBarrier) { @@ -4804,7 +4975,13 @@ void InstructionCodeGeneratorARM64::GenerateGcRootFieldLoad(HInstruction* instru // } // /* GcRoot<mirror::Object> */ root = *(obj + offset) - __ Ldr(root_reg, MemOperand(obj, offset)); + if (fixup_label == nullptr) { + __ Ldr(root_reg, MemOperand(obj, offset)); + } else { + vixl::SingleEmissionCheckScope guard(GetVIXLAssembler()); + __ Bind(fixup_label); + __ ldr(root_reg, MemOperand(obj, offset)); + } static_assert( sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(GcRoot<mirror::Object>), "art::mirror::CompressedReference<mirror::Object> and art::GcRoot<mirror::Object> " @@ -4829,14 +5006,26 @@ void InstructionCodeGeneratorARM64::GenerateGcRootFieldLoad(HInstruction* instru // GC root loaded through a slow path for read barriers other // than Baker's. // /* GcRoot<mirror::Object>* */ root = obj + offset - __ Add(root_reg.X(), obj.X(), offset); + if (fixup_label == nullptr) { + __ Add(root_reg.X(), obj.X(), offset); + } else { + vixl::SingleEmissionCheckScope guard(GetVIXLAssembler()); + __ Bind(fixup_label); + __ add(root_reg.X(), obj.X(), offset); + } // /* mirror::Object* */ root = root->Read() codegen_->GenerateReadBarrierForRootSlow(instruction, root, root); } } else { // Plain GC root load with no read barrier. // /* GcRoot<mirror::Object> */ root = *(obj + offset) - __ Ldr(root_reg, MemOperand(obj, offset)); + if (fixup_label == nullptr) { + __ Ldr(root_reg, MemOperand(obj, offset)); + } else { + vixl::SingleEmissionCheckScope guard(GetVIXLAssembler()); + __ Bind(fixup_label); + __ ldr(root_reg, MemOperand(obj, offset)); + } // Note that GC roots are not affected by heap poisoning, thus we // do not have to unpoison `root_reg` here. } diff --git a/compiler/optimizing/code_generator_arm64.h b/compiler/optimizing/code_generator_arm64.h index cf9dc1bdc1..8ec753159a 100644 --- a/compiler/optimizing/code_generator_arm64.h +++ b/compiler/optimizing/code_generator_arm64.h @@ -17,6 +17,7 @@ #ifndef ART_COMPILER_OPTIMIZING_CODE_GENERATOR_ARM64_H_ #define ART_COMPILER_OPTIMIZING_CODE_GENERATOR_ARM64_H_ +#include "arch/arm64/quick_method_frame_info_arm64.h" #include "code_generator.h" #include "common_arm64.h" #include "dex/compiler_enums.h" @@ -24,9 +25,9 @@ #include "nodes.h" #include "parallel_move_resolver.h" #include "utils/arm64/assembler_arm64.h" +#include "utils/string_reference.h" #include "vixl/a64/disasm-a64.h" #include "vixl/a64/macro-assembler-a64.h" -#include "arch/arm64/quick_method_frame_info_arm64.h" namespace art { namespace arm64 { @@ -255,7 +256,8 @@ class InstructionCodeGeneratorARM64 : public InstructionCodeGenerator { void GenerateGcRootFieldLoad(HInstruction* instruction, Location root, vixl::Register obj, - uint32_t offset); + uint32_t offset, + vixl::Label* fixup_label = nullptr); // Generate a floating-point comparison. void GenerateFcmp(HInstruction* instruction); @@ -453,6 +455,11 @@ class CodeGeneratorARM64 : public CodeGenerator { return false; } + // Check if the desired_string_load_kind is supported. If it is, return it, + // otherwise return a fall-back kind that should be used instead. + HLoadString::LoadKind GetSupportedLoadStringKind( + HLoadString::LoadKind desired_string_load_kind) OVERRIDE; + // Check if the desired_dispatch_info is supported. If it is, return it, // otherwise return a fall-back info that should be used instead. HInvokeStaticOrDirect::DispatchInfo GetSupportedInvokeStaticOrDirectDispatch( @@ -467,6 +474,27 @@ class CodeGeneratorARM64 : public CodeGenerator { UNIMPLEMENTED(FATAL); } + // Add a new PC-relative string patch for an instruction and return the label + // to be bound before the instruction. The instruction will be either the + // ADRP (pass `adrp_label = null`) or the ADD (pass `adrp_label` pointing + // to the associated ADRP patch label). + vixl::Label* NewPcRelativeStringPatch(const DexFile& dex_file, + uint32_t string_index, + vixl::Label* adrp_label = nullptr); + + // Add a new PC-relative dex cache array patch for an instruction and return + // the label to be bound before the instruction. The instruction will be + // either the ADRP (pass `adrp_label = null`) or the LDR (pass `adrp_label` + // pointing to the associated ADRP patch label). + vixl::Label* NewPcRelativeDexCacheArrayPatch(const DexFile& dex_file, + uint32_t element_offset, + vixl::Label* adrp_label = nullptr); + + vixl::Literal<uint32_t>* DeduplicateBootImageStringLiteral(const DexFile& dex_file, + uint32_t string_index); + vixl::Literal<uint32_t>* DeduplicateBootImageAddressLiteral(uint64_t address); + vixl::Literal<uint64_t>* DeduplicateDexCacheAddressLiteral(uint64_t address); + void EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) OVERRIDE; // Fast path implementation of ReadBarrier::Barrier for a heap @@ -554,26 +582,39 @@ class CodeGeneratorARM64 : public CodeGenerator { bool use_load_acquire); using Uint64ToLiteralMap = ArenaSafeMap<uint64_t, vixl::Literal<uint64_t>*>; + using Uint32ToLiteralMap = ArenaSafeMap<uint32_t, vixl::Literal<uint32_t>*>; using MethodToLiteralMap = ArenaSafeMap<MethodReference, vixl::Literal<uint64_t>*, MethodReferenceComparator>; + using BootStringToLiteralMap = ArenaSafeMap<StringReference, + vixl::Literal<uint32_t>*, + StringReferenceValueComparator>; + vixl::Literal<uint32_t>* DeduplicateUint32Literal(uint32_t value, Uint32ToLiteralMap* map); vixl::Literal<uint64_t>* DeduplicateUint64Literal(uint64_t value); vixl::Literal<uint64_t>* DeduplicateMethodLiteral(MethodReference target_method, MethodToLiteralMap* map); vixl::Literal<uint64_t>* DeduplicateMethodAddressLiteral(MethodReference target_method); vixl::Literal<uint64_t>* DeduplicateMethodCodeLiteral(MethodReference target_method); - struct PcRelativeDexCacheAccessInfo { - PcRelativeDexCacheAccessInfo(const DexFile& dex_file, uint32_t element_off) - : target_dex_file(dex_file), element_offset(element_off), label(), pc_insn_label() { } + // The PcRelativePatchInfo is used for PC-relative addressing of dex cache arrays + // and boot image strings. The only difference is the interpretation of the offset_or_index. + struct PcRelativePatchInfo { + PcRelativePatchInfo(const DexFile& dex_file, uint32_t off_or_idx) + : target_dex_file(dex_file), offset_or_index(off_or_idx), label(), pc_insn_label() { } const DexFile& target_dex_file; - uint32_t element_offset; + // Either the dex cache array element offset or the string index. + uint32_t offset_or_index; vixl::Label label; vixl::Label* pc_insn_label; }; + vixl::Label* NewPcRelativePatch(const DexFile& dex_file, + uint32_t offset_or_index, + vixl::Label* adrp_label, + ArenaDeque<PcRelativePatchInfo>* patches); + void EmitJumpTables(); // Labels for each block that will be compiled. @@ -587,7 +628,10 @@ class CodeGeneratorARM64 : public CodeGenerator { Arm64Assembler assembler_; const Arm64InstructionSetFeatures& isa_features_; - // Deduplication map for 64-bit literals, used for non-patchable method address and method code. + // Deduplication map for 32-bit literals, used for non-patchable boot image addresses. + Uint32ToLiteralMap uint32_literals_; + // Deduplication map for 64-bit literals, used for non-patchable method address, method code + // or string dex cache address. Uint64ToLiteralMap uint64_literals_; // Method patch info, map MethodReference to a literal for method address and method code. MethodToLiteralMap method_patches_; @@ -596,7 +640,13 @@ class CodeGeneratorARM64 : public CodeGenerator { // Using ArenaDeque<> which retains element addresses on push/emplace_back(). ArenaDeque<MethodPatchInfo<vixl::Label>> relative_call_patches_; // PC-relative DexCache access info. - ArenaDeque<PcRelativeDexCacheAccessInfo> pc_relative_dex_cache_patches_; + ArenaDeque<PcRelativePatchInfo> pc_relative_dex_cache_patches_; + // Deduplication map for boot string literals for kBootImageLinkTimeAddress. + BootStringToLiteralMap boot_image_string_patches_; + // PC-relative String patch info. + ArenaDeque<PcRelativePatchInfo> pc_relative_string_patches_; + // Deduplication map for patchable boot image addresses. + Uint32ToLiteralMap boot_image_address_patches_; DISALLOW_COPY_AND_ASSIGN(CodeGeneratorARM64); }; diff --git a/compiler/optimizing/code_generator_mips.cc b/compiler/optimizing/code_generator_mips.cc index a29d8394db..8b19f84e1c 100644 --- a/compiler/optimizing/code_generator_mips.cc +++ b/compiler/optimizing/code_generator_mips.cc @@ -3816,6 +3816,12 @@ static bool TryGenerateIntrinsicCode(HInvoke* invoke, CodeGeneratorMIPS* codegen return false; } +HLoadString::LoadKind CodeGeneratorMIPS::GetSupportedLoadStringKind( + HLoadString::LoadKind desired_string_load_kind ATTRIBUTE_UNUSED) { + // TODO: Implement other kinds. + return HLoadString::LoadKind::kDexCacheViaMethod; +} + HInvokeStaticOrDirect::DispatchInfo CodeGeneratorMIPS::GetSupportedInvokeStaticOrDirectDispatch( const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info, MethodReference target_method ATTRIBUTE_UNUSED) { @@ -4066,9 +4072,9 @@ void InstructionCodeGeneratorMIPS::VisitLoadLocal(HLoadLocal* load ATTRIBUTE_UNU } void LocationsBuilderMIPS::VisitLoadString(HLoadString* load) { - LocationSummary::CallKind call_kind = load->IsInDexCache() - ? LocationSummary::kNoCall - : LocationSummary::kCallOnSlowPath; + LocationSummary::CallKind call_kind = load->NeedsEnvironment() + ? LocationSummary::kCallOnSlowPath + : LocationSummary::kNoCall; LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(load, call_kind); locations->SetInAt(0, Location::RequiresRegister()); locations->SetOut(Location::RequiresRegister()); diff --git a/compiler/optimizing/code_generator_mips.h b/compiler/optimizing/code_generator_mips.h index b720573897..afe7917cc4 100644 --- a/compiler/optimizing/code_generator_mips.h +++ b/compiler/optimizing/code_generator_mips.h @@ -345,6 +345,11 @@ class CodeGeneratorMIPS : public CodeGenerator { return type == Primitive::kPrimLong; } + // Check if the desired_string_load_kind is supported. If it is, return it, + // otherwise return a fall-back kind that should be used instead. + HLoadString::LoadKind GetSupportedLoadStringKind( + HLoadString::LoadKind desired_string_load_kind) OVERRIDE; + // Check if the desired_dispatch_info is supported. If it is, return it, // otherwise return a fall-back info that should be used instead. HInvokeStaticOrDirect::DispatchInfo GetSupportedInvokeStaticOrDirectDispatch( diff --git a/compiler/optimizing/code_generator_mips64.cc b/compiler/optimizing/code_generator_mips64.cc index 72ef49966e..2f9eca6ac3 100644 --- a/compiler/optimizing/code_generator_mips64.cc +++ b/compiler/optimizing/code_generator_mips64.cc @@ -3030,6 +3030,12 @@ static bool TryGenerateIntrinsicCode(HInvoke* invoke, CodeGeneratorMIPS64* codeg return false; } +HLoadString::LoadKind CodeGeneratorMIPS64::GetSupportedLoadStringKind( + HLoadString::LoadKind desired_string_load_kind ATTRIBUTE_UNUSED) { + // TODO: Implement other kinds. + return HLoadString::LoadKind::kDexCacheViaMethod; +} + HInvokeStaticOrDirect::DispatchInfo CodeGeneratorMIPS64::GetSupportedInvokeStaticOrDirectDispatch( const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info, MethodReference target_method ATTRIBUTE_UNUSED) { @@ -3284,9 +3290,9 @@ void InstructionCodeGeneratorMIPS64::VisitLoadLocal(HLoadLocal* load ATTRIBUTE_U } void LocationsBuilderMIPS64::VisitLoadString(HLoadString* load) { - LocationSummary::CallKind call_kind = load->IsInDexCache() - ? LocationSummary::kNoCall - : LocationSummary::kCallOnSlowPath; + LocationSummary::CallKind call_kind = load->NeedsEnvironment() + ? LocationSummary::kCallOnSlowPath + : LocationSummary::kNoCall; LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(load, call_kind); locations->SetInAt(0, Location::RequiresRegister()); locations->SetOut(Location::RequiresRegister()); diff --git a/compiler/optimizing/code_generator_mips64.h b/compiler/optimizing/code_generator_mips64.h index 9464a140ad..94767cba4b 100644 --- a/compiler/optimizing/code_generator_mips64.h +++ b/compiler/optimizing/code_generator_mips64.h @@ -337,6 +337,11 @@ class CodeGeneratorMIPS64 : public CodeGenerator { bool NeedsTwoRegisters(Primitive::Type type ATTRIBUTE_UNUSED) const { return false; } + // Check if the desired_string_load_kind is supported. If it is, return it, + // otherwise return a fall-back kind that should be used instead. + HLoadString::LoadKind GetSupportedLoadStringKind( + HLoadString::LoadKind desired_string_load_kind) OVERRIDE; + // Check if the desired_dispatch_info is supported. If it is, return it, // otherwise return a fall-back info that should be used instead. HInvokeStaticOrDirect::DispatchInfo GetSupportedInvokeStaticOrDirectDispatch( diff --git a/compiler/optimizing/code_generator_x86.cc b/compiler/optimizing/code_generator_x86.cc index 394f4eef68..715b5be2c8 100644 --- a/compiler/optimizing/code_generator_x86.cc +++ b/compiler/optimizing/code_generator_x86.cc @@ -799,6 +799,8 @@ CodeGeneratorX86::CodeGeneratorX86(HGraph* graph, method_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), relative_call_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + simple_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), fixups_to_jump_tables_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) { // Use a fake return address register to mimic Quick. AddAllocatedRegister(Location::RegisterLocation(kFakeReturnRegister)); @@ -4340,6 +4342,8 @@ HInvokeStaticOrDirect::DispatchInfo CodeGeneratorX86::GetSupportedInvokeStaticOr // We disable pc-relative load when there is an irreducible loop, as the optimization // is incompatible with it. + // TODO: Create as many X86ComputeBaseMethodAddress instructions + // as needed for methods with irreducible loops. if (GetGraph()->HasIrreducibleLoops() && (dispatch_info.method_load_kind == HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative)) { @@ -4401,18 +4405,17 @@ void CodeGeneratorX86::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, __ movl(temp.AsRegister<Register>(), Immediate(invoke->GetMethodAddress())); break; case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddressWithFixup: - __ movl(temp.AsRegister<Register>(), Immediate(0)); // Placeholder. + __ movl(temp.AsRegister<Register>(), Immediate(/* placeholder */ 0)); method_patches_.emplace_back(invoke->GetTargetMethod()); __ Bind(&method_patches_.back().label); // Bind the label at the end of the "movl" insn. break; case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: { Register base_reg = GetInvokeStaticOrDirectExtraParameter(invoke, temp.AsRegister<Register>()); - uint32_t offset = invoke->GetDexCacheArrayOffset(); __ movl(temp.AsRegister<Register>(), Address(base_reg, kDummy32BitOffset)); - // Add the patch entry and bind its label at the end of the instruction. - pc_relative_dex_cache_patches_.emplace_back(*invoke->GetTargetMethod().dex_file, offset); - __ Bind(&pc_relative_dex_cache_patches_.back().label); + // Bind a new fixup label at the end of the "movl" insn. + uint32_t offset = invoke->GetDexCacheArrayOffset(); + __ Bind(NewPcRelativeDexCacheArrayPatch(*invoke->GetTargetMethod().dex_file, offset)); break; } case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: { @@ -4494,12 +4497,33 @@ void CodeGeneratorX86::GenerateVirtualCall(HInvokeVirtual* invoke, Location temp temp, ArtMethod::EntryPointFromQuickCompiledCodeOffset(kX86WordSize).Int32Value())); } +void CodeGeneratorX86::RecordSimplePatch() { + if (GetCompilerOptions().GetIncludePatchInformation()) { + simple_patches_.emplace_back(); + __ Bind(&simple_patches_.back()); + } +} + +void CodeGeneratorX86::RecordStringPatch(HLoadString* load_string) { + string_patches_.emplace_back(load_string->GetDexFile(), load_string->GetStringIndex()); + __ Bind(&string_patches_.back().label); +} + +Label* CodeGeneratorX86::NewPcRelativeDexCacheArrayPatch(const DexFile& dex_file, + uint32_t element_offset) { + // Add the patch entry and bind its label at the end of the instruction. + pc_relative_dex_cache_patches_.emplace_back(dex_file, element_offset); + return &pc_relative_dex_cache_patches_.back().label; +} + void CodeGeneratorX86::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) { DCHECK(linker_patches->empty()); size_t size = method_patches_.size() + relative_call_patches_.size() + - pc_relative_dex_cache_patches_.size(); + pc_relative_dex_cache_patches_.size() + + simple_patches_.size() + + string_patches_.size(); linker_patches->reserve(size); // The label points to the end of the "movl" insn but the literal offset for method // patch needs to point to the embedded constant which occupies the last 4 bytes. @@ -4523,6 +4547,26 @@ void CodeGeneratorX86::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patche GetMethodAddressOffset(), info.element_offset)); } + for (const Label& label : simple_patches_) { + uint32_t literal_offset = label.Position() - kLabelPositionToLiteralOffsetAdjustment; + linker_patches->push_back(LinkerPatch::RecordPosition(literal_offset)); + } + if (GetCompilerOptions().GetCompilePic()) { + for (const StringPatchInfo<Label>& info : string_patches_) { + uint32_t literal_offset = info.label.Position() - kLabelPositionToLiteralOffsetAdjustment; + linker_patches->push_back(LinkerPatch::RelativeStringPatch(literal_offset, + &info.dex_file, + GetMethodAddressOffset(), + info.string_index)); + } + } else { + for (const StringPatchInfo<Label>& info : string_patches_) { + uint32_t literal_offset = info.label.Position() - kLabelPositionToLiteralOffsetAdjustment; + linker_patches->push_back(LinkerPatch::StringPatch(literal_offset, + &info.dex_file, + info.string_index)); + } + } } void CodeGeneratorX86::MarkGCCard(Register temp, @@ -5916,14 +5960,15 @@ void InstructionCodeGeneratorX86::VisitLoadClass(HLoadClass* cls) { DCHECK(!cls->MustGenerateClinitCheck()); // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_ GenerateGcRootFieldLoad( - cls, out_loc, current_method, ArtMethod::DeclaringClassOffset().Int32Value()); + cls, out_loc, Address(current_method, ArtMethod::DeclaringClassOffset().Int32Value())); } else { // /* GcRoot<mirror::Class>[] */ out = // current_method.ptr_sized_fields_->dex_cache_resolved_types_ __ movl(out, Address(current_method, ArtMethod::DexCacheResolvedTypesOffset(kX86PointerSize).Int32Value())); // /* GcRoot<mirror::Class> */ out = out[type_index] - GenerateGcRootFieldLoad(cls, out_loc, out, CodeGenerator::GetCacheOffset(cls->GetTypeIndex())); + GenerateGcRootFieldLoad( + cls, out_loc, Address(out, CodeGenerator::GetCacheOffset(cls->GetTypeIndex()))); if (!cls->IsInDexCache() || cls->MustGenerateClinitCheck()) { DCHECK(cls->CanCallRuntime()); @@ -5972,12 +6017,58 @@ void InstructionCodeGeneratorX86::GenerateClassInitializationCheck( // No need for memory fence, thanks to the X86 memory model. } +HLoadString::LoadKind CodeGeneratorX86::GetSupportedLoadStringKind( + HLoadString::LoadKind desired_string_load_kind) { + if (kEmitCompilerReadBarrier) { + switch (desired_string_load_kind) { + case HLoadString::LoadKind::kBootImageLinkTimeAddress: + case HLoadString::LoadKind::kBootImageLinkTimePcRelative: + case HLoadString::LoadKind::kBootImageAddress: + // TODO: Implement for read barrier. + return HLoadString::LoadKind::kDexCacheViaMethod; + default: + break; + } + } + switch (desired_string_load_kind) { + case HLoadString::LoadKind::kBootImageLinkTimeAddress: + DCHECK(!GetCompilerOptions().GetCompilePic()); + break; + case HLoadString::LoadKind::kBootImageLinkTimePcRelative: + DCHECK(GetCompilerOptions().GetCompilePic()); + FALLTHROUGH_INTENDED; + case HLoadString::LoadKind::kDexCachePcRelative: + DCHECK(!Runtime::Current()->UseJit()); // Note: boot image is also non-JIT. + // We disable pc-relative load when there is an irreducible loop, as the optimization + // is incompatible with it. + // TODO: Create as many X86ComputeBaseMethodAddress instructions as needed for methods + // with irreducible loops. + if (GetGraph()->HasIrreducibleLoops()) { + return HLoadString::LoadKind::kDexCacheViaMethod; + } + break; + case HLoadString::LoadKind::kBootImageAddress: + break; + case HLoadString::LoadKind::kDexCacheAddress: + DCHECK(Runtime::Current()->UseJit()); + break; + case HLoadString::LoadKind::kDexCacheViaMethod: + break; + } + return desired_string_load_kind; +} + void LocationsBuilderX86::VisitLoadString(HLoadString* load) { - LocationSummary::CallKind call_kind = (!load->IsInDexCache() || kEmitCompilerReadBarrier) + LocationSummary::CallKind call_kind = (load->NeedsEnvironment() || kEmitCompilerReadBarrier) ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall; LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(load, call_kind); - locations->SetInAt(0, Location::RequiresRegister()); + HLoadString::LoadKind load_kind = load->GetLoadKind(); + if (load_kind == HLoadString::LoadKind::kDexCacheViaMethod || + load_kind == HLoadString::LoadKind::kBootImageLinkTimePcRelative || + load_kind == HLoadString::LoadKind::kDexCachePcRelative) { + locations->SetInAt(0, Location::RequiresRegister()); + } locations->SetOut(Location::RequiresRegister()); } @@ -5985,16 +6076,61 @@ void InstructionCodeGeneratorX86::VisitLoadString(HLoadString* load) { LocationSummary* locations = load->GetLocations(); Location out_loc = locations->Out(); Register out = out_loc.AsRegister<Register>(); - Register current_method = locations->InAt(0).AsRegister<Register>(); - // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_ - GenerateGcRootFieldLoad( - load, out_loc, current_method, ArtMethod::DeclaringClassOffset().Int32Value()); - // /* GcRoot<mirror::String>[] */ out = out->dex_cache_strings_ - __ movl(out, Address(out, mirror::Class::DexCacheStringsOffset().Int32Value())); - // /* GcRoot<mirror::String> */ out = out[string_index] - GenerateGcRootFieldLoad( - load, out_loc, out, CodeGenerator::GetCacheOffset(load->GetStringIndex())); + switch (load->GetLoadKind()) { + case HLoadString::LoadKind::kBootImageLinkTimeAddress: { + DCHECK(!kEmitCompilerReadBarrier); + __ movl(out, Immediate(/* placeholder */ 0)); + codegen_->RecordStringPatch(load); + return; // No dex cache slow path. + } + case HLoadString::LoadKind::kBootImageLinkTimePcRelative: { + DCHECK(!kEmitCompilerReadBarrier); + Register method_address = locations->InAt(0).AsRegister<Register>(); + __ leal(out, Address(method_address, CodeGeneratorX86::kDummy32BitOffset)); + codegen_->RecordStringPatch(load); + return; // No dex cache slow path. + } + case HLoadString::LoadKind::kBootImageAddress: { + DCHECK(!kEmitCompilerReadBarrier); + DCHECK_NE(load->GetAddress(), 0u); + uint32_t address = dchecked_integral_cast<uint32_t>(load->GetAddress()); + __ movl(out, Immediate(address)); + codegen_->RecordSimplePatch(); + return; // No dex cache slow path. + } + case HLoadString::LoadKind::kDexCacheAddress: { + DCHECK_NE(load->GetAddress(), 0u); + uint32_t address = dchecked_integral_cast<uint32_t>(load->GetAddress()); + GenerateGcRootFieldLoad(load, out_loc, Address::Absolute(address)); + break; + } + case HLoadString::LoadKind::kDexCachePcRelative: { + Register base_reg = locations->InAt(0).AsRegister<Register>(); + uint32_t offset = load->GetDexCacheElementOffset(); + Label* fixup_label = codegen_->NewPcRelativeDexCacheArrayPatch(load->GetDexFile(), offset); + GenerateGcRootFieldLoad( + load, out_loc, Address(base_reg, CodeGeneratorX86::kDummy32BitOffset), fixup_label); + break; + } + case HLoadString::LoadKind::kDexCacheViaMethod: { + Register current_method = locations->InAt(0).AsRegister<Register>(); + + // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_ + GenerateGcRootFieldLoad( + load, out_loc, Address(current_method, ArtMethod::DeclaringClassOffset().Int32Value())); + + // /* GcRoot<mirror::String>[] */ out = out->dex_cache_strings_ + __ movl(out, Address(out, mirror::Class::DexCacheStringsOffset().Int32Value())); + // /* GcRoot<mirror::String> */ out = out[string_index] + GenerateGcRootFieldLoad( + load, out_loc, Address(out, CodeGenerator::GetCacheOffset(load->GetStringIndex()))); + break; + } + default: + LOG(FATAL) << "Unexpected load kind: " << load->GetLoadKind(); + UNREACHABLE(); + } if (!load->IsInDexCache()) { SlowPathCode* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathX86(load); @@ -6692,21 +6828,24 @@ void InstructionCodeGeneratorX86::GenerateReferenceLoadTwoRegisters(HInstruction void InstructionCodeGeneratorX86::GenerateGcRootFieldLoad(HInstruction* instruction, Location root, - Register obj, - uint32_t offset) { + const Address& address, + Label* fixup_label) { Register root_reg = root.AsRegister<Register>(); if (kEmitCompilerReadBarrier) { if (kUseBakerReadBarrier) { // Fast path implementation of art::ReadBarrier::BarrierForRoot when // Baker's read barrier are used: // - // root = obj.field; + // root = *address; // if (Thread::Current()->GetIsGcMarking()) { // root = ReadBarrier::Mark(root) // } - // /* GcRoot<mirror::Object> */ root = *(obj + offset) - __ movl(root_reg, Address(obj, offset)); + // /* GcRoot<mirror::Object> */ root = *address + __ movl(root_reg, address); + if (fixup_label != nullptr) { + __ Bind(fixup_label); + } static_assert( sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(GcRoot<mirror::Object>), "art::mirror::CompressedReference<mirror::Object> and art::GcRoot<mirror::Object> " @@ -6727,15 +6866,21 @@ void InstructionCodeGeneratorX86::GenerateGcRootFieldLoad(HInstruction* instruct } else { // GC root loaded through a slow path for read barriers other // than Baker's. - // /* GcRoot<mirror::Object>* */ root = obj + offset - __ leal(root_reg, Address(obj, offset)); + // /* GcRoot<mirror::Object>* */ root = address + __ leal(root_reg, address); + if (fixup_label != nullptr) { + __ Bind(fixup_label); + } // /* mirror::Object* */ root = root->Read() codegen_->GenerateReadBarrierForRootSlow(instruction, root, root); } } else { // Plain GC root load with no read barrier. - // /* GcRoot<mirror::Object> */ root = *(obj + offset) - __ movl(root_reg, Address(obj, offset)); + // /* GcRoot<mirror::Object> */ root = *address + __ movl(root_reg, address); + if (fixup_label != nullptr) { + __ Bind(fixup_label); + } // Note that GC roots are not affected by heap poisoning, thus we // do not have to unpoison `root_reg` here. } diff --git a/compiler/optimizing/code_generator_x86.h b/compiler/optimizing/code_generator_x86.h index c397899892..1fa22fcfbe 100644 --- a/compiler/optimizing/code_generator_x86.h +++ b/compiler/optimizing/code_generator_x86.h @@ -258,13 +258,13 @@ class InstructionCodeGeneratorX86 : public InstructionCodeGenerator { Location maybe_temp); // Generate a GC root reference load: // - // root <- *(obj + offset) + // root <- *address // // while honoring read barriers (if any). void GenerateGcRootFieldLoad(HInstruction* instruction, Location root, - Register obj, - uint32_t offset); + const Address& address, + Label* fixup_label = nullptr); // Push value to FPU stack. `is_fp` specifies whether the value is floating point or not. // `is_wide` specifies whether it is long/double or not. @@ -388,6 +388,11 @@ class CodeGeneratorX86 : public CodeGenerator { // Helper method to move a 64bits value between two locations. void Move64(Location destination, Location source); + // Check if the desired_string_load_kind is supported. If it is, return it, + // otherwise return a fall-back kind that should be used instead. + HLoadString::LoadKind GetSupportedLoadStringKind( + HLoadString::LoadKind desired_string_load_kind) OVERRIDE; + // Check if the desired_dispatch_info is supported. If it is, return it, // otherwise return a fall-back info that should be used instead. HInvokeStaticOrDirect::DispatchInfo GetSupportedInvokeStaticOrDirectDispatch( @@ -399,6 +404,10 @@ class CodeGeneratorX86 : public CodeGenerator { // Generate a call to a virtual method. void GenerateVirtualCall(HInvokeVirtual* invoke, Location temp) OVERRIDE; + void RecordSimplePatch(); + void RecordStringPatch(HLoadString* load_string); + Label* NewPcRelativeDexCacheArrayPatch(const DexFile& dex_file, uint32_t element_offset); + void MoveFromReturnRegister(Location trg, Primitive::Type type) OVERRIDE; // Emit linker patches. @@ -542,6 +551,10 @@ class CodeGeneratorX86 : public CodeGenerator { void GenerateImplicitNullCheck(HNullCheck* instruction); void GenerateExplicitNullCheck(HNullCheck* instruction); + // When we don't know the proper offset for the value, we use kDummy32BitOffset. + // The correct value will be inserted when processing Assembler fixups. + static constexpr int32_t kDummy32BitOffset = 256; + private: // Factored implementation of GenerateFieldLoadWithBakerReadBarrier // and GenerateArrayLoadWithBakerReadBarrier. @@ -578,6 +591,10 @@ class CodeGeneratorX86 : public CodeGenerator { ArenaDeque<MethodPatchInfo<Label>> relative_call_patches_; // PC-relative DexCache access info. ArenaDeque<PcRelativeDexCacheAccessInfo> pc_relative_dex_cache_patches_; + // Patch locations for patchoat where the linker doesn't do any other work. + ArenaDeque<Label> simple_patches_; + // String patch locations. + ArenaDeque<StringPatchInfo<Label>> string_patches_; // Offset to the start of the constant area in the assembled code. // Used for fixups to the constant area. @@ -592,10 +609,6 @@ class CodeGeneratorX86 : public CodeGenerator { // instruction gives the address of the start of this method. int32_t method_address_offset_; - // When we don't know the proper offset for the value, we use kDummy32BitOffset. - // The correct value will be inserted when processing Assembler fixups. - static constexpr int32_t kDummy32BitOffset = 256; - DISALLOW_COPY_AND_ASSIGN(CodeGeneratorX86); }; diff --git a/compiler/optimizing/code_generator_x86_64.cc b/compiler/optimizing/code_generator_x86_64.cc index b20043c545..fe2abe4793 100644 --- a/compiler/optimizing/code_generator_x86_64.cc +++ b/compiler/optimizing/code_generator_x86_64.cc @@ -784,14 +784,14 @@ void CodeGeneratorX86_64::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invo method_patches_.emplace_back(invoke->GetTargetMethod()); __ Bind(&method_patches_.back().label); // Bind the label at the end of the "movl" insn. break; - case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: - pc_relative_dex_cache_patches_.emplace_back(*invoke->GetTargetMethod().dex_file, - invoke->GetDexCacheArrayOffset()); + case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: { __ movq(temp.AsRegister<CpuRegister>(), Address::Absolute(kDummy32BitOffset, /* no_rip */ false)); - // Bind the label at the end of the "movl" insn. - __ Bind(&pc_relative_dex_cache_patches_.back().label); + // Bind a new fixup label at the end of the "movl" insn. + uint32_t offset = invoke->GetDexCacheArrayOffset(); + __ Bind(NewPcRelativeDexCacheArrayPatch(*invoke->GetTargetMethod().dex_file, offset)); break; + } case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: { Location current_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); Register method_reg; @@ -873,12 +873,33 @@ void CodeGeneratorX86_64::GenerateVirtualCall(HInvokeVirtual* invoke, Location t kX86_64WordSize).SizeValue())); } +void CodeGeneratorX86_64::RecordSimplePatch() { + if (GetCompilerOptions().GetIncludePatchInformation()) { + simple_patches_.emplace_back(); + __ Bind(&simple_patches_.back()); + } +} + +void CodeGeneratorX86_64::RecordStringPatch(HLoadString* load_string) { + string_patches_.emplace_back(load_string->GetDexFile(), load_string->GetStringIndex()); + __ Bind(&string_patches_.back().label); +} + +Label* CodeGeneratorX86_64::NewPcRelativeDexCacheArrayPatch(const DexFile& dex_file, + uint32_t element_offset) { + // Add a patch entry and return the label. + pc_relative_dex_cache_patches_.emplace_back(dex_file, element_offset); + return &pc_relative_dex_cache_patches_.back().label; +} + void CodeGeneratorX86_64::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) { DCHECK(linker_patches->empty()); size_t size = method_patches_.size() + relative_call_patches_.size() + - pc_relative_dex_cache_patches_.size(); + pc_relative_dex_cache_patches_.size() + + simple_patches_.size() + + string_patches_.size(); linker_patches->reserve(size); // The label points to the end of the "movl" insn but the literal offset for method // patch needs to point to the embedded constant which occupies the last 4 bytes. @@ -902,6 +923,18 @@ void CodeGeneratorX86_64::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_pat info.label.Position(), info.element_offset)); } + for (const Label& label : simple_patches_) { + uint32_t literal_offset = label.Position() - kLabelPositionToLiteralOffsetAdjustment; + linker_patches->push_back(LinkerPatch::RecordPosition(literal_offset)); + } + for (const StringPatchInfo<Label>& info : string_patches_) { + // These are always PC-relative, see GetSupportedLoadStringKind(). + uint32_t literal_offset = info.label.Position() - kLabelPositionToLiteralOffsetAdjustment; + linker_patches->push_back(LinkerPatch::RelativeStringPatch(literal_offset, + &info.dex_file, + info.label.Position(), + info.string_index)); + } } void CodeGeneratorX86_64::DumpCoreRegister(std::ostream& stream, int reg) const { @@ -978,6 +1011,8 @@ CodeGeneratorX86_64::CodeGeneratorX86_64(HGraph* graph, method_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), relative_call_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + simple_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), fixups_to_jump_tables_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) { AddAllocatedRegister(Location::RegisterLocation(kFakeReturnRegister)); } @@ -5371,14 +5406,15 @@ void InstructionCodeGeneratorX86_64::VisitLoadClass(HLoadClass* cls) { DCHECK(!cls->MustGenerateClinitCheck()); // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_ GenerateGcRootFieldLoad( - cls, out_loc, current_method, ArtMethod::DeclaringClassOffset().Int32Value()); + cls, out_loc, Address(current_method, ArtMethod::DeclaringClassOffset().Int32Value())); } else { // /* GcRoot<mirror::Class>[] */ out = // current_method.ptr_sized_fields_->dex_cache_resolved_types_ __ movq(out, Address(current_method, ArtMethod::DexCacheResolvedTypesOffset(kX86_64PointerSize).Int32Value())); // /* GcRoot<mirror::Class> */ out = out[type_index] - GenerateGcRootFieldLoad(cls, out_loc, out, CodeGenerator::GetCacheOffset(cls->GetTypeIndex())); + GenerateGcRootFieldLoad( + cls, out_loc, Address(out, CodeGenerator::GetCacheOffset(cls->GetTypeIndex()))); if (!cls->IsInDexCache() || cls->MustGenerateClinitCheck()) { DCHECK(cls->CanCallRuntime()); @@ -5416,12 +5452,49 @@ void InstructionCodeGeneratorX86_64::VisitClinitCheck(HClinitCheck* check) { check->GetLocations()->InAt(0).AsRegister<CpuRegister>()); } +HLoadString::LoadKind CodeGeneratorX86_64::GetSupportedLoadStringKind( + HLoadString::LoadKind desired_string_load_kind) { + if (kEmitCompilerReadBarrier) { + switch (desired_string_load_kind) { + case HLoadString::LoadKind::kBootImageLinkTimeAddress: + case HLoadString::LoadKind::kBootImageLinkTimePcRelative: + case HLoadString::LoadKind::kBootImageAddress: + // TODO: Implement for read barrier. + return HLoadString::LoadKind::kDexCacheViaMethod; + default: + break; + } + } + switch (desired_string_load_kind) { + case HLoadString::LoadKind::kBootImageLinkTimeAddress: + DCHECK(!GetCompilerOptions().GetCompilePic()); + // We prefer the always-available RIP-relative address for the x86-64 boot image. + return HLoadString::LoadKind::kBootImageLinkTimePcRelative; + case HLoadString::LoadKind::kBootImageLinkTimePcRelative: + DCHECK(GetCompilerOptions().GetCompilePic()); + break; + case HLoadString::LoadKind::kBootImageAddress: + break; + case HLoadString::LoadKind::kDexCacheAddress: + DCHECK(Runtime::Current()->UseJit()); + break; + case HLoadString::LoadKind::kDexCachePcRelative: + DCHECK(!Runtime::Current()->UseJit()); + break; + case HLoadString::LoadKind::kDexCacheViaMethod: + break; + } + return desired_string_load_kind; +} + void LocationsBuilderX86_64::VisitLoadString(HLoadString* load) { - LocationSummary::CallKind call_kind = (!load->IsInDexCache() || kEmitCompilerReadBarrier) + LocationSummary::CallKind call_kind = (load->NeedsEnvironment() || kEmitCompilerReadBarrier) ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall; LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(load, call_kind); - locations->SetInAt(0, Location::RequiresRegister()); + if (load->GetLoadKind() == HLoadString::LoadKind::kDexCacheViaMethod) { + locations->SetInAt(0, Location::RequiresRegister()); + } locations->SetOut(Location::RequiresRegister()); } @@ -5429,16 +5502,59 @@ void InstructionCodeGeneratorX86_64::VisitLoadString(HLoadString* load) { LocationSummary* locations = load->GetLocations(); Location out_loc = locations->Out(); CpuRegister out = out_loc.AsRegister<CpuRegister>(); - CpuRegister current_method = locations->InAt(0).AsRegister<CpuRegister>(); - // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_ - GenerateGcRootFieldLoad( - load, out_loc, current_method, ArtMethod::DeclaringClassOffset().Int32Value()); - // /* GcRoot<mirror::String>[] */ out = out->dex_cache_strings_ - __ movq(out, Address(out, mirror::Class::DexCacheStringsOffset().Uint32Value())); - // /* GcRoot<mirror::String> */ out = out[string_index] - GenerateGcRootFieldLoad( - load, out_loc, out, CodeGenerator::GetCacheOffset(load->GetStringIndex())); + switch (load->GetLoadKind()) { + case HLoadString::LoadKind::kBootImageLinkTimePcRelative: { + DCHECK(!kEmitCompilerReadBarrier); + __ leal(out, Address::Absolute(CodeGeneratorX86_64::kDummy32BitOffset, /* no_rip */ false)); + codegen_->RecordStringPatch(load); + return; // No dex cache slow path. + } + case HLoadString::LoadKind::kBootImageAddress: { + DCHECK(!kEmitCompilerReadBarrier); + DCHECK_NE(load->GetAddress(), 0u); + uint32_t address = dchecked_integral_cast<uint32_t>(load->GetAddress()); + __ movl(out, Immediate(address)); // Zero-extended. + codegen_->RecordSimplePatch(); + return; // No dex cache slow path. + } + case HLoadString::LoadKind::kDexCacheAddress: { + DCHECK_NE(load->GetAddress(), 0u); + if (IsUint<32>(load->GetAddress())) { + Address address = Address::Absolute(load->GetAddress(), /* no_rip */ true); + GenerateGcRootFieldLoad(load, out_loc, address); + } else { + // TODO: Consider using opcode A1, i.e. movl eax, moff32 (with 64-bit address). + __ movq(out, Immediate(load->GetAddress())); + GenerateGcRootFieldLoad(load, out_loc, Address(out, 0)); + } + break; + } + case HLoadString::LoadKind::kDexCachePcRelative: { + uint32_t offset = load->GetDexCacheElementOffset(); + Label* fixup_label = codegen_->NewPcRelativeDexCacheArrayPatch(load->GetDexFile(), offset); + Address address = Address::Absolute(CodeGeneratorX86_64::kDummy32BitOffset, + /* no_rip */ false); + GenerateGcRootFieldLoad(load, out_loc, address, fixup_label); + break; + } + case HLoadString::LoadKind::kDexCacheViaMethod: { + CpuRegister current_method = locations->InAt(0).AsRegister<CpuRegister>(); + + // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_ + GenerateGcRootFieldLoad( + load, out_loc, Address(current_method, ArtMethod::DeclaringClassOffset().Int32Value())); + // /* GcRoot<mirror::String>[] */ out = out->dex_cache_strings_ + __ movq(out, Address(out, mirror::Class::DexCacheStringsOffset().Uint32Value())); + // /* GcRoot<mirror::String> */ out = out[string_index] + GenerateGcRootFieldLoad( + load, out_loc, Address(out, CodeGenerator::GetCacheOffset(load->GetStringIndex()))); + break; + } + default: + LOG(FATAL) << "Unexpected load kind: " << load->GetLoadKind(); + UNREACHABLE(); + } if (!load->IsInDexCache()) { SlowPathCode* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathX86_64(load); @@ -6177,21 +6293,24 @@ void InstructionCodeGeneratorX86_64::GenerateReferenceLoadTwoRegisters(HInstruct void InstructionCodeGeneratorX86_64::GenerateGcRootFieldLoad(HInstruction* instruction, Location root, - CpuRegister obj, - uint32_t offset) { + const Address& address, + Label* fixup_label) { CpuRegister root_reg = root.AsRegister<CpuRegister>(); if (kEmitCompilerReadBarrier) { if (kUseBakerReadBarrier) { // Fast path implementation of art::ReadBarrier::BarrierForRoot when // Baker's read barrier are used: // - // root = obj.field; + // root = *address; // if (Thread::Current()->GetIsGcMarking()) { // root = ReadBarrier::Mark(root) // } - // /* GcRoot<mirror::Object> */ root = *(obj + offset) - __ movl(root_reg, Address(obj, offset)); + // /* GcRoot<mirror::Object> */ root = *address + __ movl(root_reg, address); + if (fixup_label != nullptr) { + __ Bind(fixup_label); + } static_assert( sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(GcRoot<mirror::Object>), "art::mirror::CompressedReference<mirror::Object> and art::GcRoot<mirror::Object> " @@ -6213,15 +6332,21 @@ void InstructionCodeGeneratorX86_64::GenerateGcRootFieldLoad(HInstruction* instr } else { // GC root loaded through a slow path for read barriers other // than Baker's. - // /* GcRoot<mirror::Object>* */ root = obj + offset - __ leaq(root_reg, Address(obj, offset)); + // /* GcRoot<mirror::Object>* */ root = address + __ leaq(root_reg, address); + if (fixup_label != nullptr) { + __ Bind(fixup_label); + } // /* mirror::Object* */ root = root->Read() codegen_->GenerateReadBarrierForRootSlow(instruction, root, root); } } else { // Plain GC root load with no read barrier. - // /* GcRoot<mirror::Object> */ root = *(obj + offset) - __ movl(root_reg, Address(obj, offset)); + // /* GcRoot<mirror::Object> */ root = *address + __ movl(root_reg, address); + if (fixup_label != nullptr) { + __ Bind(fixup_label); + } // Note that GC roots are not affected by heap poisoning, thus we // do not have to unpoison `root_reg` here. } diff --git a/compiler/optimizing/code_generator_x86_64.h b/compiler/optimizing/code_generator_x86_64.h index 396ddf2889..7ebce58f6c 100644 --- a/compiler/optimizing/code_generator_x86_64.h +++ b/compiler/optimizing/code_generator_x86_64.h @@ -253,13 +253,13 @@ class InstructionCodeGeneratorX86_64 : public InstructionCodeGenerator { Location maybe_temp); // Generate a GC root reference load: // - // root <- *(obj + offset) + // root <- *address // // while honoring read barriers (if any). void GenerateGcRootFieldLoad(HInstruction* instruction, Location root, - CpuRegister obj, - uint32_t offset); + const Address& address, + Label* fixup_label = nullptr); void PushOntoFPStack(Location source, uint32_t temp_offset, uint32_t stack_adjustment, bool is_float); @@ -385,6 +385,11 @@ class CodeGeneratorX86_64 : public CodeGenerator { return false; } + // Check if the desired_string_load_kind is supported. If it is, return it, + // otherwise return a fall-back kind that should be used instead. + HLoadString::LoadKind GetSupportedLoadStringKind( + HLoadString::LoadKind desired_string_load_kind) OVERRIDE; + // Check if the desired_dispatch_info is supported. If it is, return it, // otherwise return a fall-back info that should be used instead. HInvokeStaticOrDirect::DispatchInfo GetSupportedInvokeStaticOrDirectDispatch( @@ -394,6 +399,10 @@ class CodeGeneratorX86_64 : public CodeGenerator { void GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) OVERRIDE; void GenerateVirtualCall(HInvokeVirtual* invoke, Location temp) OVERRIDE; + void RecordSimplePatch(); + void RecordStringPatch(HLoadString* load_string); + Label* NewPcRelativeDexCacheArrayPatch(const DexFile& dex_file, uint32_t element_offset); + void MoveFromReturnRegister(Location trg, Primitive::Type type) OVERRIDE; void EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) OVERRIDE; @@ -516,6 +525,10 @@ class CodeGeneratorX86_64 : public CodeGenerator { void GenerateImplicitNullCheck(HNullCheck* instruction); void GenerateExplicitNullCheck(HNullCheck* instruction); + // When we don't know the proper offset for the value, we use kDummy32BitOffset. + // We will fix this up in the linker later to have the right value. + static constexpr int32_t kDummy32BitOffset = 256; + private: // Factored implementation of GenerateFieldLoadWithBakerReadBarrier // and GenerateArrayLoadWithBakerReadBarrier. @@ -553,10 +566,10 @@ class CodeGeneratorX86_64 : public CodeGenerator { ArenaDeque<MethodPatchInfo<Label>> relative_call_patches_; // PC-relative DexCache access info. ArenaDeque<PcRelativeDexCacheAccessInfo> pc_relative_dex_cache_patches_; - - // When we don't know the proper offset for the value, we use kDummy32BitOffset. - // We will fix this up in the linker later to have the right value. - static constexpr int32_t kDummy32BitOffset = 256; + // Patch locations for patchoat where the linker doesn't do any other work. + ArenaDeque<Label> simple_patches_; + // String patch locations. + ArenaDeque<StringPatchInfo<Label>> string_patches_; // Fixups for jump tables need to be handled specially. ArenaVector<JumpTableRIPFixup*> fixups_to_jump_tables_; diff --git a/compiler/optimizing/dex_cache_array_fixups_arm.cc b/compiler/optimizing/dex_cache_array_fixups_arm.cc index 3db254a004..e9072b9c77 100644 --- a/compiler/optimizing/dex_cache_array_fixups_arm.cc +++ b/compiler/optimizing/dex_cache_array_fixups_arm.cc @@ -44,6 +44,21 @@ class DexCacheArrayFixupsVisitor : public HGraphVisitor { } private: + void VisitLoadString(HLoadString* load_string) OVERRIDE { + // If this is a load with PC-relative access to the dex cache methods array, + // we need to add the dex cache arrays base as the special input. + if (load_string->GetLoadKind() == HLoadString::LoadKind::kDexCachePcRelative) { + // Initialize base for target dex file if needed. + const DexFile& dex_file = load_string->GetDexFile(); + HArmDexCacheArraysBase* base = GetOrCreateDexCacheArrayBase(dex_file); + // Update the element offset in base. + DexCacheArraysLayout layout(kArmPointerSize, &dex_file); + base->UpdateElementOffset(layout.StringOffset(load_string->GetStringIndex())); + // Add the special argument base to the load. + load_string->AddSpecialInput(base); + } + } + void VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) OVERRIDE { // If this is an invoke with PC-relative access to the dex cache methods array, // we need to add the dex cache arrays base as the special input. diff --git a/compiler/optimizing/graph_visualizer.cc b/compiler/optimizing/graph_visualizer.cc index b3eff1460d..fe47f7db7d 100644 --- a/compiler/optimizing/graph_visualizer.cc +++ b/compiler/optimizing/graph_visualizer.cc @@ -373,6 +373,10 @@ class HGraphVisualizerPrinter : public HGraphDelegateVisitor { << load_class->NeedsAccessCheck() << std::noboolalpha; } + void VisitLoadString(HLoadString* load_string) OVERRIDE { + StartAttributeStream("load_kind") << load_string->GetLoadKind(); + } + void VisitCheckCast(HCheckCast* check_cast) OVERRIDE { StartAttributeStream("check_kind") << check_cast->GetTypeCheckKind(); StartAttributeStream("must_do_null_check") << std::boolalpha diff --git a/compiler/optimizing/nodes.cc b/compiler/optimizing/nodes.cc index 1a426d5930..950448136d 100644 --- a/compiler/optimizing/nodes.cc +++ b/compiler/optimizing/nodes.cc @@ -86,11 +86,7 @@ void HGraph::FindBackEdges(ArenaBitVector* visited) { } } -static void RemoveAsUser(HInstruction* instruction) { - for (size_t i = 0; i < instruction->InputCount(); i++) { - instruction->RemoveAsUserOfInput(i); - } - +static void RemoveEnvironmentUses(HInstruction* instruction) { for (HEnvironment* environment = instruction->GetEnvironment(); environment != nullptr; environment = environment->GetParent()) { @@ -102,6 +98,14 @@ static void RemoveAsUser(HInstruction* instruction) { } } +static void RemoveAsUser(HInstruction* instruction) { + for (size_t i = 0; i < instruction->InputCount(); i++) { + instruction->RemoveAsUserOfInput(i); + } + + RemoveEnvironmentUses(instruction); +} + void HGraph::RemoveInstructionsAsUsersFromDeadBlocks(const ArenaBitVector& visited) const { for (size_t i = 0; i < blocks_.size(); ++i) { if (!visited.IsBitSet(i)) { @@ -1007,6 +1011,11 @@ bool HInstruction::StrictlyDominates(HInstruction* other_instruction) const { } } +void HInstruction::RemoveEnvironment() { + RemoveEnvironmentUses(this); + environment_ = nullptr; +} + void HInstruction::ReplaceWith(HInstruction* other) { DCHECK(other != nullptr); for (HUseIterator<HInstruction*> it(GetUses()); !it.Done(); it.Advance()) { @@ -2387,6 +2396,59 @@ std::ostream& operator<<(std::ostream& os, HInvokeStaticOrDirect::ClinitCheckReq } } +bool HLoadString::InstructionDataEquals(HInstruction* other) const { + HLoadString* other_load_string = other->AsLoadString(); + if (string_index_ != other_load_string->string_index_ || + GetPackedFields() != other_load_string->GetPackedFields()) { + return false; + } + LoadKind load_kind = GetLoadKind(); + if (HasAddress(load_kind)) { + return GetAddress() == other_load_string->GetAddress(); + } else if (HasStringReference(load_kind)) { + return IsSameDexFile(GetDexFile(), other_load_string->GetDexFile()); + } else { + DCHECK(HasDexCacheReference(load_kind)) << load_kind; + // If the string indexes and dex files are the same, dex cache element offsets + // must also be the same, so we don't need to compare them. + return IsSameDexFile(GetDexFile(), other_load_string->GetDexFile()); + } +} + +void HLoadString::SetLoadKindInternal(LoadKind load_kind) { + // Once sharpened, the load kind should not be changed again. + DCHECK_EQ(GetLoadKind(), LoadKind::kDexCacheViaMethod); + SetPackedField<LoadKindField>(load_kind); + + if (load_kind != LoadKind::kDexCacheViaMethod) { + RemoveAsUserOfInput(0u); + SetRawInputAt(0u, nullptr); + } + if (!NeedsEnvironment()) { + RemoveEnvironment(); + } +} + +std::ostream& operator<<(std::ostream& os, HLoadString::LoadKind rhs) { + switch (rhs) { + case HLoadString::LoadKind::kBootImageLinkTimeAddress: + return os << "BootImageLinkTimeAddress"; + case HLoadString::LoadKind::kBootImageLinkTimePcRelative: + return os << "BootImageLinkTimePcRelative"; + case HLoadString::LoadKind::kBootImageAddress: + return os << "BootImageAddress"; + case HLoadString::LoadKind::kDexCacheAddress: + return os << "DexCacheAddress"; + case HLoadString::LoadKind::kDexCachePcRelative: + return os << "DexCachePcRelative"; + case HLoadString::LoadKind::kDexCacheViaMethod: + return os << "DexCacheViaMethod"; + default: + LOG(FATAL) << "Unknown HLoadString::LoadKind: " << static_cast<int>(rhs); + UNREACHABLE(); + } +} + void HInstruction::RemoveEnvironmentUsers() { for (HUseIterator<HEnvironment*> use_it(GetEnvUses()); !use_it.Done(); use_it.Advance()) { HUseListNode<HEnvironment*>* user_node = use_it.Current(); diff --git a/compiler/optimizing/nodes.h b/compiler/optimizing/nodes.h index e9a42cb0ce..ba4242112b 100644 --- a/compiler/optimizing/nodes.h +++ b/compiler/optimizing/nodes.h @@ -1995,6 +1995,8 @@ class HInstruction : public ArenaObject<kArenaAllocInstruction> { environment_ = environment; } + void RemoveEnvironment(); + // Set the environment of this instruction, copying it from `environment`. While // copying, the uses lists are being updated. void CopyEnvironmentFrom(HEnvironment* environment) { @@ -5557,32 +5559,117 @@ class HLoadClass : public HExpression<1> { class HLoadString : public HExpression<1> { public: + // Determines how to load the String. + enum class LoadKind { + // Use boot image String* address that will be known at link time. + // Used for boot image strings referenced by boot image code in non-PIC mode. + kBootImageLinkTimeAddress, + + // Use PC-relative boot image String* address that will be known at link time. + // Used for boot image strings referenced by boot image code in PIC mode. + kBootImageLinkTimePcRelative, + + // Use a known boot image String* address, embedded in the code by the codegen. + // Used for boot image strings referenced by apps in AOT- and JIT-compiled code. + // Note: codegen needs to emit a linker patch if indicated by compiler options' + // GetIncludePatchInformation(). + kBootImageAddress, + + // Load from the resolved strings array at an absolute address. + // Used for strings outside the boot image referenced by JIT-compiled code. + kDexCacheAddress, + + // Load from resolved strings array in the dex cache using a PC-relative load. + // Used for strings outside boot image when we know that we can access + // the dex cache arrays using a PC-relative load. + kDexCachePcRelative, + + // Load from resolved strings array accessed through the class loaded from + // the compiled method's own ArtMethod*. This is the default access type when + // all other types are unavailable. + kDexCacheViaMethod, + + kLast = kDexCacheViaMethod + }; + HLoadString(HCurrentMethod* current_method, uint32_t string_index, - uint32_t dex_pc, - bool is_in_dex_cache) + const DexFile& dex_file, + uint32_t dex_pc) : HExpression(Primitive::kPrimNot, SideEffectsForArchRuntimeCalls(), dex_pc), string_index_(string_index) { - SetPackedFlag<kFlagIsInDexCache>(is_in_dex_cache); + SetPackedFlag<kFlagIsInDexCache>(false); + SetPackedField<LoadKindField>(LoadKind::kDexCacheViaMethod); + load_data_.ref.dex_file = &dex_file; SetRawInputAt(0, current_method); } - bool CanBeMoved() const OVERRIDE { return true; } + void SetLoadKindWithAddress(LoadKind load_kind, uint64_t address) { + DCHECK(HasAddress(load_kind)); + load_data_.address = address; + SetLoadKindInternal(load_kind); + } - bool InstructionDataEquals(HInstruction* other) const OVERRIDE { - return other->AsLoadString()->string_index_ == string_index_; + void SetLoadKindWithStringReference(LoadKind load_kind, + const DexFile& dex_file, + uint32_t string_index) { + DCHECK(HasStringReference(load_kind)); + load_data_.ref.dex_file = &dex_file; + string_index_ = string_index; + SetLoadKindInternal(load_kind); } + void SetLoadKindWithDexCacheReference(LoadKind load_kind, + const DexFile& dex_file, + uint32_t element_index) { + DCHECK(HasDexCacheReference(load_kind)); + load_data_.ref.dex_file = &dex_file; + load_data_.ref.dex_cache_element_index = element_index; + SetLoadKindInternal(load_kind); + } + + LoadKind GetLoadKind() const { + return GetPackedField<LoadKindField>(); + } + + const DexFile& GetDexFile() const; + + uint32_t GetStringIndex() const { + DCHECK(HasStringReference(GetLoadKind()) || /* For slow paths. */ !IsInDexCache()); + return string_index_; + } + + uint32_t GetDexCacheElementOffset() const; + + uint64_t GetAddress() const { + DCHECK(HasAddress(GetLoadKind())); + return load_data_.address; + } + + bool CanBeMoved() const OVERRIDE { return true; } + + bool InstructionDataEquals(HInstruction* other) const OVERRIDE; + size_t ComputeHashCode() const OVERRIDE { return string_index_; } - uint32_t GetStringIndex() const { return string_index_; } + // Will call the runtime if we need to load the string through + // the dex cache and the string is not guaranteed to be there yet. + bool NeedsEnvironment() const OVERRIDE { + LoadKind load_kind = GetLoadKind(); + if (load_kind == LoadKind::kBootImageLinkTimeAddress || + load_kind == LoadKind::kBootImageLinkTimePcRelative || + load_kind == LoadKind::kBootImageAddress) { + return false; + } + return !IsInDexCache(); + } - // Will call the runtime if the string is not already in the dex cache. - bool NeedsEnvironment() const OVERRIDE { return !IsInDexCache(); } + bool NeedsDexCacheOfDeclaringClass() const OVERRIDE { + return GetLoadKind() == LoadKind::kDexCacheViaMethod; + } - bool NeedsDexCacheOfDeclaringClass() const OVERRIDE { return true; } bool CanBeNull() const OVERRIDE { return false; } - bool CanThrow() const OVERRIDE { return !IsInDexCache(); } + bool CanThrow() const OVERRIDE { return NeedsEnvironment(); } static SideEffects SideEffectsForArchRuntimeCalls() { return SideEffects::CanTriggerGC(); @@ -5590,17 +5677,83 @@ class HLoadString : public HExpression<1> { bool IsInDexCache() const { return GetPackedFlag<kFlagIsInDexCache>(); } + void MarkInDexCache() { + SetPackedFlag<kFlagIsInDexCache>(true); + DCHECK(!NeedsEnvironment()); + RemoveEnvironment(); + } + + size_t InputCount() const OVERRIDE { + return (InputAt(0) != nullptr) ? 1u : 0u; + } + + void AddSpecialInput(HInstruction* special_input); + DECLARE_INSTRUCTION(LoadString); private: static constexpr size_t kFlagIsInDexCache = kNumberOfExpressionPackedBits; - static constexpr size_t kNumberOfLoadStringPackedBits = kFlagIsInDexCache + 1; + static constexpr size_t kFieldLoadKind = kFlagIsInDexCache + 1; + static constexpr size_t kFieldLoadKindSize = + MinimumBitsToStore(static_cast<size_t>(LoadKind::kLast)); + static constexpr size_t kNumberOfLoadStringPackedBits = kFieldLoadKind + kFieldLoadKindSize; static_assert(kNumberOfLoadStringPackedBits <= kMaxNumberOfPackedBits, "Too many packed fields."); + using LoadKindField = BitField<LoadKind, kFieldLoadKind, kFieldLoadKindSize>; + + static bool HasStringReference(LoadKind load_kind) { + return load_kind == LoadKind::kBootImageLinkTimeAddress || + load_kind == LoadKind::kBootImageLinkTimePcRelative || + load_kind == LoadKind::kDexCacheViaMethod; + } - const uint32_t string_index_; + static bool HasAddress(LoadKind load_kind) { + return load_kind == LoadKind::kBootImageAddress || load_kind == LoadKind::kDexCacheAddress; + } + + static bool HasDexCacheReference(LoadKind load_kind) { + return load_kind == LoadKind::kDexCachePcRelative; + } + + void SetLoadKindInternal(LoadKind load_kind); + + // String index serves also as the hash code and it's also needed for slow-paths, + // so it must not be overwritten with other load data. + uint32_t string_index_; + + union { + struct { + const DexFile* dex_file; // For string reference and dex cache reference. + uint32_t dex_cache_element_index; // Only for dex cache reference. + } ref; + uint64_t address; // Up to 64-bit, needed for kDexCacheAddress on 64-bit targets. + } load_data_; DISALLOW_COPY_AND_ASSIGN(HLoadString); }; +std::ostream& operator<<(std::ostream& os, HLoadString::LoadKind rhs); + +// Note: defined outside class to see operator<<(., HLoadString::LoadKind). +inline const DexFile& HLoadString::GetDexFile() const { + DCHECK(HasStringReference(GetLoadKind()) || HasDexCacheReference(GetLoadKind())) + << GetLoadKind(); + return *load_data_.ref.dex_file; +} + +// Note: defined outside class to see operator<<(., HLoadString::LoadKind). +inline uint32_t HLoadString::GetDexCacheElementOffset() const { + DCHECK(HasDexCacheReference(GetLoadKind())) << GetLoadKind(); + return load_data_.ref.dex_cache_element_index; +} + +// Note: defined outside class to see operator<<(., HLoadString::LoadKind). +inline void HLoadString::AddSpecialInput(HInstruction* special_input) { + // The special input is used for PC-relative loads on some architectures. + DCHECK(GetLoadKind() == LoadKind::kBootImageLinkTimePcRelative || + GetLoadKind() == LoadKind::kDexCachePcRelative) << GetLoadKind(); + DCHECK(InputAt(0) == nullptr); + SetRawInputAt(0u, special_input); + special_input->AddUseAt(this, 0); +} /** * Performs an initialization check on its Class object input. diff --git a/compiler/optimizing/pc_relative_fixups_x86.cc b/compiler/optimizing/pc_relative_fixups_x86.cc index d281a9fc6c..dafbd3d7d1 100644 --- a/compiler/optimizing/pc_relative_fixups_x86.cc +++ b/compiler/optimizing/pc_relative_fixups_x86.cc @@ -80,6 +80,15 @@ class PCRelativeHandlerVisitor : public HGraphVisitor { HandleInvoke(invoke); } + void VisitLoadString(HLoadString* load_string) OVERRIDE { + HLoadString::LoadKind load_kind = load_string->GetLoadKind(); + if (load_kind == HLoadString::LoadKind::kBootImageLinkTimePcRelative || + load_kind == HLoadString::LoadKind::kDexCachePcRelative) { + InitializePCRelativeBasePointer(); + load_string->AddSpecialInput(base_); + } + } + void BinaryFP(HBinaryOperation* bin) { HConstant* rhs = bin->InputAt(1)->AsConstant(); if (rhs != nullptr && Primitive::IsFloatingPointType(rhs->GetType())) { diff --git a/compiler/optimizing/sharpening.cc b/compiler/optimizing/sharpening.cc index 5e1d1d9954..45ae3366c1 100644 --- a/compiler/optimizing/sharpening.cc +++ b/compiler/optimizing/sharpening.cc @@ -16,11 +16,19 @@ #include "sharpening.h" +#include "class_linker.h" #include "code_generator.h" +#include "driver/dex_compilation_unit.h" #include "utils/dex_cache_arrays_layout-inl.h" #include "driver/compiler_driver.h" +#include "gc/heap.h" +#include "gc/space/image_space.h" +#include "handle_scope-inl.h" +#include "mirror/dex_cache.h" +#include "mirror/string.h" #include "nodes.h" #include "runtime.h" +#include "scoped_thread_state_change.h" namespace art { @@ -31,12 +39,13 @@ void HSharpening::Run() { HInstruction* instruction = it.Current(); if (instruction->IsInvokeStaticOrDirect()) { ProcessInvokeStaticOrDirect(instruction->AsInvokeStaticOrDirect()); + } else if (instruction->IsLoadString()) { + ProcessLoadString(instruction->AsLoadString()); } // TODO: Move the sharpening of invoke-virtual/-interface/-super from HGraphBuilder // here. Rewrite it to avoid the CompilerDriver's reliance on verifier data // because we know the type better when inlining. - // TODO: HLoadClass, HLoadString - select PC relative dex cache array access if - // available. + // TODO: HLoadClass - select better load kind if available. } } } @@ -143,4 +152,105 @@ void HSharpening::ProcessInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) { invoke->SetDispatchInfo(dispatch_info); } +void HSharpening::ProcessLoadString(HLoadString* load_string) { + DCHECK_EQ(load_string->GetLoadKind(), HLoadString::LoadKind::kDexCacheViaMethod); + DCHECK(!load_string->IsInDexCache()); + + const DexFile& dex_file = load_string->GetDexFile(); + uint32_t string_index = load_string->GetStringIndex(); + + bool is_in_dex_cache = false; + HLoadString::LoadKind desired_load_kind; + uint64_t address = 0u; // String or dex cache element address. + { + Runtime* runtime = Runtime::Current(); + ClassLinker* class_linker = runtime->GetClassLinker(); + ScopedObjectAccess soa(Thread::Current()); + StackHandleScope<1> hs(soa.Self()); + Handle<mirror::DexCache> dex_cache = IsSameDexFile(dex_file, *compilation_unit_.GetDexFile()) + ? compilation_unit_.GetDexCache() + : hs.NewHandle(class_linker->FindDexCache(soa.Self(), dex_file)); + + if (compiler_driver_->IsBootImage()) { + // Compiling boot image. Resolve the string and allocate it if needed. + DCHECK(!runtime->UseJit()); + mirror::String* string = class_linker->ResolveString(dex_file, string_index, dex_cache); + CHECK(string != nullptr); + if (!compiler_driver_->GetSupportBootImageFixup()) { + // MIPS/MIPS64 or compiler_driver_test. Do not sharpen. + desired_load_kind = HLoadString::LoadKind::kDexCacheViaMethod; + } else { + DCHECK(ContainsElement(compiler_driver_->GetDexFilesForOatFile(), + &load_string->GetDexFile())); + is_in_dex_cache = true; + desired_load_kind = codegen_->GetCompilerOptions().GetCompilePic() + ? HLoadString::LoadKind::kBootImageLinkTimePcRelative + : HLoadString::LoadKind::kBootImageLinkTimeAddress; + } + } else { + // Not compiling boot image. Try to lookup the string without allocating if not found. + mirror::String* string = class_linker->LookupString(dex_file, string_index, dex_cache); + if (runtime->UseJit()) { + // TODO: Make sure we don't set the "compile PIC" flag for JIT as that's bogus. + // DCHECK(!codegen_->GetCompilerOptions().GetCompilePic()); + is_in_dex_cache = (string != nullptr); + if (string != nullptr && runtime->GetHeap()->ObjectIsInBootImageSpace(string)) { + desired_load_kind = HLoadString::LoadKind::kBootImageAddress; + // Convert to uintptr_t first to avoid sign-extension if a 32-bit pointer is "signed." + address = static_cast<uint64_t>(reinterpret_cast<uintptr_t>(string)); + } else { + // Note: If the string is not in the dex cache, the instruction needs environment + // and will not be inlined across dex files. Within a dex file, the slow-path helper + // loads the correct string and inlined frames are used correctly for OOM stack trace. + // TODO: Write a test for this. + desired_load_kind = HLoadString::LoadKind::kDexCacheAddress; + void* dex_cache_element_address = &dex_cache->GetStrings()[string_index]; + // Convert to uintptr_t first to avoid sign-extension if a 32-bit pointer is "signed." + address = static_cast<uint64_t>(reinterpret_cast<uintptr_t>(dex_cache_element_address)); + } + } else if (string != nullptr && runtime->GetHeap()->ObjectIsInBootImageSpace(string)) { + if (codegen_->GetCompilerOptions().GetCompilePic()) { + // Use PC-relative load from the dex cache if the dex file belongs + // to the oat file that we're currently compiling. + desired_load_kind = + ContainsElement(compiler_driver_->GetDexFilesForOatFile(), &load_string->GetDexFile()) + ? HLoadString::LoadKind::kDexCachePcRelative + : HLoadString::LoadKind::kDexCacheViaMethod; + } else { + desired_load_kind = HLoadString::LoadKind::kBootImageAddress; + // Convert to uintptr_t first to avoid sign-extension if a 32-bit pointer is "signed." + address = static_cast<uint64_t>(reinterpret_cast<uintptr_t>(string)); + } + } else { + // Not JIT and the string is not in boot image. + desired_load_kind = HLoadString::LoadKind::kDexCachePcRelative; + } + } + } + if (is_in_dex_cache) { + load_string->MarkInDexCache(); + } + + HLoadString::LoadKind load_kind = codegen_->GetSupportedLoadStringKind(desired_load_kind); + switch (load_kind) { + case HLoadString::LoadKind::kBootImageLinkTimeAddress: + case HLoadString::LoadKind::kBootImageLinkTimePcRelative: + case HLoadString::LoadKind::kDexCacheViaMethod: + load_string->SetLoadKindWithStringReference(load_kind, dex_file, string_index); + break; + case HLoadString::LoadKind::kBootImageAddress: + case HLoadString::LoadKind::kDexCacheAddress: + DCHECK_NE(address, 0u); + load_string->SetLoadKindWithAddress(load_kind, address); + break; + case HLoadString::LoadKind::kDexCachePcRelative: { + size_t pointer_size = InstructionSetPointerSize(codegen_->GetInstructionSet()); + DexCacheArraysLayout layout(pointer_size, &dex_file); + size_t element_index = layout.StringOffset(string_index); + load_string->SetLoadKindWithDexCacheReference(load_kind, dex_file, element_index); + break; + } + } +} + } // namespace art diff --git a/compiler/optimizing/sharpening.h b/compiler/optimizing/sharpening.h index adae7007dd..24152f6b71 100644 --- a/compiler/optimizing/sharpening.h +++ b/compiler/optimizing/sharpening.h @@ -47,6 +47,7 @@ class HSharpening : public HOptimization { private: void ProcessInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke); + void ProcessLoadString(HLoadString* load_string); CodeGenerator* codegen_; const DexCompilationUnit& compilation_unit_; |