diff options
Diffstat (limited to 'compiler/optimizing')
31 files changed, 1578 insertions, 832 deletions
diff --git a/compiler/optimizing/builder.h b/compiler/optimizing/builder.h index 8cf4089eba..3a4c9dbd16 100644 --- a/compiler/optimizing/builder.h +++ b/compiler/optimizing/builder.h @@ -32,6 +32,8 @@ namespace art { +class CodeGenerator; + class HGraphBuilder : public ValueObject { public: HGraphBuilder(HGraph* graph, @@ -40,6 +42,7 @@ class HGraphBuilder : public ValueObject { const DexFile* dex_file, const DexFile::CodeItem& code_item, CompilerDriver* driver, + CodeGenerator* code_generator, OptimizingCompilerStats* compiler_stats, const uint8_t* interpreter_metadata, Handle<mirror::DexCache> dex_cache, @@ -51,7 +54,10 @@ class HGraphBuilder : public ValueObject { compiler_driver_(driver), compilation_stats_(compiler_stats), block_builder_(graph, dex_file, code_item), - ssa_builder_(graph, dex_compilation_unit->GetDexCache(), handles), + ssa_builder_(graph, + dex_compilation_unit->GetClassLoader(), + dex_compilation_unit->GetDexCache(), + handles), instruction_builder_(graph, &block_builder_, &ssa_builder_, @@ -61,6 +67,7 @@ class HGraphBuilder : public ValueObject { dex_compilation_unit, outer_compilation_unit, driver, + code_generator, interpreter_metadata, compiler_stats, dex_cache, @@ -76,10 +83,12 @@ class HGraphBuilder : public ValueObject { code_item_(code_item), dex_compilation_unit_(nullptr), compiler_driver_(nullptr), - null_dex_cache_(), compilation_stats_(nullptr), block_builder_(graph, nullptr, code_item), - ssa_builder_(graph, null_dex_cache_, handles), + ssa_builder_(graph, + handles->NewHandle<mirror::ClassLoader>(nullptr), + handles->NewHandle<mirror::DexCache>(nullptr), + handles), instruction_builder_(graph, &block_builder_, &ssa_builder_, @@ -89,9 +98,10 @@ class HGraphBuilder : public ValueObject { /* dex_compilation_unit */ nullptr, /* outer_compilation_unit */ nullptr, /* compiler_driver */ nullptr, + /* code_generator */ nullptr, /* interpreter_metadata */ nullptr, /* compiler_stats */ nullptr, - null_dex_cache_, + handles->NewHandle<mirror::DexCache>(nullptr), handles) {} GraphAnalysisResult BuildGraph(); @@ -112,8 +122,6 @@ class HGraphBuilder : public ValueObject { CompilerDriver* const compiler_driver_; - ScopedNullHandle<mirror::DexCache> null_dex_cache_; - OptimizingCompilerStats* compilation_stats_; HBasicBlockBuilder block_builder_; diff --git a/compiler/optimizing/code_generator_arm.cc b/compiler/optimizing/code_generator_arm.cc index f5b6ebef9c..759a951d6b 100644 --- a/compiler/optimizing/code_generator_arm.cc +++ b/compiler/optimizing/code_generator_arm.cc @@ -367,22 +367,37 @@ class BoundsCheckSlowPathARM : public SlowPathCodeARM { class LoadClassSlowPathARM : public SlowPathCodeARM { public: - LoadClassSlowPathARM(HLoadClass* cls, - HInstruction* at, - uint32_t dex_pc, - bool do_clinit) + LoadClassSlowPathARM(HLoadClass* cls, HInstruction* at, uint32_t dex_pc, bool do_clinit) : SlowPathCodeARM(at), cls_(cls), dex_pc_(dex_pc), do_clinit_(do_clinit) { DCHECK(at->IsLoadClass() || at->IsClinitCheck()); } void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { LocationSummary* locations = instruction_->GetLocations(); + Location out = locations->Out(); + constexpr bool call_saves_everything_except_r0 = (!kUseReadBarrier || kUseBakerReadBarrier); CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); __ Bind(GetEntryLabel()); SaveLiveRegisters(codegen, locations); InvokeRuntimeCallingConvention calling_convention; + // For HLoadClass/kBssEntry/kSaveEverything, make sure we preserve the address of the entry. + DCHECK_EQ(instruction_->IsLoadClass(), cls_ == instruction_); + bool is_load_class_bss_entry = + (cls_ == instruction_) && (cls_->GetLoadKind() == HLoadClass::LoadKind::kBssEntry); + Register entry_address = kNoRegister; + if (is_load_class_bss_entry && call_saves_everything_except_r0) { + Register temp = locations->GetTemp(0).AsRegister<Register>(); + // In the unlucky case that the `temp` is R0, we preserve the address in `out` across + // the kSaveEverything call. + bool temp_is_r0 = (temp == calling_convention.GetRegisterAt(0)); + entry_address = temp_is_r0 ? out.AsRegister<Register>() : temp; + DCHECK_NE(entry_address, calling_convention.GetRegisterAt(0)); + if (temp_is_r0) { + __ mov(entry_address, ShifterOperand(temp)); + } + } dex::TypeIndex type_index = cls_->GetTypeIndex(); __ LoadImmediate(calling_convention.GetRegisterAt(0), type_index.index_); QuickEntrypointEnum entrypoint = do_clinit_ ? kQuickInitializeStaticStorage @@ -394,30 +409,31 @@ class LoadClassSlowPathARM : public SlowPathCodeARM { CheckEntrypointTypes<kQuickInitializeType, void*, uint32_t>(); } + // For HLoadClass/kBssEntry, store the resolved Class to the BSS entry. + if (is_load_class_bss_entry) { + if (call_saves_everything_except_r0) { + // The class entry address was preserved in `entry_address` thanks to kSaveEverything. + __ str(R0, Address(entry_address)); + } else { + // For non-Baker read barrier, we need to re-calculate the address of the string entry. + Register temp = IP; + CodeGeneratorARM::PcRelativePatchInfo* labels = + arm_codegen->NewTypeBssEntryPatch(cls_->GetDexFile(), type_index); + __ BindTrackedLabel(&labels->movw_label); + __ movw(temp, /* placeholder */ 0u); + __ BindTrackedLabel(&labels->movt_label); + __ movt(temp, /* placeholder */ 0u); + __ BindTrackedLabel(&labels->add_pc_label); + __ add(temp, temp, ShifterOperand(PC)); + __ str(R0, Address(temp)); + } + } // Move the class to the desired location. - Location out = locations->Out(); if (out.IsValid()) { DCHECK(out.IsRegister() && !locations->GetLiveRegisters()->ContainsCoreRegister(out.reg())); arm_codegen->Move32(locations->Out(), Location::RegisterLocation(R0)); } RestoreLiveRegisters(codegen, locations); - // For HLoadClass/kBssEntry, store the resolved Class to the BSS entry. - DCHECK_EQ(instruction_->IsLoadClass(), cls_ == instruction_); - if (cls_ == instruction_ && cls_->GetLoadKind() == HLoadClass::LoadKind::kBssEntry) { - DCHECK(out.IsValid()); - // TODO: Change art_quick_initialize_type/art_quick_initialize_static_storage to - // kSaveEverything and use a temporary for the .bss entry address in the fast path, - // so that we can avoid another calculation here. - CodeGeneratorARM::PcRelativePatchInfo* labels = - arm_codegen->NewTypeBssEntryPatch(cls_->GetDexFile(), type_index); - __ BindTrackedLabel(&labels->movw_label); - __ movw(IP, /* placeholder */ 0u); - __ BindTrackedLabel(&labels->movt_label); - __ movt(IP, /* placeholder */ 0u); - __ BindTrackedLabel(&labels->add_pc_label); - __ add(IP, IP, ShifterOperand(PC)); - __ str(locations->Out().AsRegister<Register>(), Address(IP)); - } __ b(GetExitLabel()); } @@ -441,12 +457,13 @@ class LoadStringSlowPathARM : public SlowPathCodeARM { explicit LoadStringSlowPathARM(HLoadString* instruction) : SlowPathCodeARM(instruction) {} void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { + DCHECK(instruction_->IsLoadString()); + DCHECK_EQ(instruction_->AsLoadString()->GetLoadKind(), HLoadString::LoadKind::kBssEntry); LocationSummary* locations = instruction_->GetLocations(); DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg())); HLoadString* load = instruction_->AsLoadString(); const dex::StringIndex string_index = load->GetStringIndex(); Register out = locations->Out().AsRegister<Register>(); - Register temp = locations->GetTemp(0).AsRegister<Register>(); constexpr bool call_saves_everything_except_r0 = (!kUseReadBarrier || kUseBakerReadBarrier); CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); @@ -455,12 +472,16 @@ class LoadStringSlowPathARM : public SlowPathCodeARM { InvokeRuntimeCallingConvention calling_convention; // In the unlucky case that the `temp` is R0, we preserve the address in `out` across - // the kSaveEverything call (or use `out` for the address after non-kSaveEverything call). - bool temp_is_r0 = (temp == calling_convention.GetRegisterAt(0)); - Register entry_address = temp_is_r0 ? out : temp; - DCHECK_NE(entry_address, calling_convention.GetRegisterAt(0)); - if (call_saves_everything_except_r0 && temp_is_r0) { - __ mov(entry_address, ShifterOperand(temp)); + // the kSaveEverything call. + Register entry_address = kNoRegister; + if (call_saves_everything_except_r0) { + Register temp = locations->GetTemp(0).AsRegister<Register>(); + bool temp_is_r0 = (temp == calling_convention.GetRegisterAt(0)); + entry_address = temp_is_r0 ? out : temp; + DCHECK_NE(entry_address, calling_convention.GetRegisterAt(0)); + if (temp_is_r0) { + __ mov(entry_address, ShifterOperand(temp)); + } } __ LoadImmediate(calling_convention.GetRegisterAt(0), string_index.index_); @@ -473,15 +494,16 @@ class LoadStringSlowPathARM : public SlowPathCodeARM { __ str(R0, Address(entry_address)); } else { // For non-Baker read barrier, we need to re-calculate the address of the string entry. + Register temp = IP; CodeGeneratorARM::PcRelativePatchInfo* labels = arm_codegen->NewPcRelativeStringPatch(load->GetDexFile(), string_index); __ BindTrackedLabel(&labels->movw_label); - __ movw(entry_address, /* placeholder */ 0u); + __ movw(temp, /* placeholder */ 0u); __ BindTrackedLabel(&labels->movt_label); - __ movt(entry_address, /* placeholder */ 0u); + __ movt(temp, /* placeholder */ 0u); __ BindTrackedLabel(&labels->add_pc_label); - __ add(entry_address, entry_address, ShifterOperand(PC)); - __ str(R0, Address(entry_address)); + __ add(temp, temp, ShifterOperand(PC)); + __ str(R0, Address(temp)); } arm_codegen->Move32(locations->Out(), Location::RegisterLocation(R0)); @@ -624,6 +646,10 @@ class ArraySetSlowPathARM : public SlowPathCodeARM { // probably still be a from-space reference (unless it gets updated by // another thread, or if another thread installed another object // reference (different from `ref`) in `obj.field`). +// +// If `entrypoint` is a valid location it is assumed to already be +// holding the entrypoint. The case where the entrypoint is passed in +// is for the GcRoot read barrier. class ReadBarrierMarkSlowPathARM : public SlowPathCodeARM { public: ReadBarrierMarkSlowPathARM(HInstruction* instruction, @@ -3993,8 +4019,11 @@ void LocationsBuilderARM::VisitNewArray(HNewArray* instruction) { void InstructionCodeGeneratorARM::VisitNewArray(HNewArray* instruction) { // Note: if heap poisoning is enabled, the entry point takes cares // of poisoning the reference. - codegen_->InvokeRuntime(kQuickAllocArrayResolved, instruction, instruction->GetDexPc()); + QuickEntrypointEnum entrypoint = + CodeGenerator::GetArrayAllocationEntrypoint(instruction->GetLoadClass()->GetClass()); + codegen_->InvokeRuntime(entrypoint, instruction, instruction->GetDexPc()); CheckEntrypointTypes<kQuickAllocArrayResolved, void*, mirror::Class*, int32_t>(); + DCHECK(!codegen_->IsLeafMethod()); } void LocationsBuilderARM::VisitParameterValue(HParameterValue* instruction) { @@ -5719,6 +5748,9 @@ void ParallelMoveResolverARM::RestoreScratch(int reg) { HLoadClass::LoadKind CodeGeneratorARM::GetSupportedLoadClassKind( HLoadClass::LoadKind desired_class_load_kind) { switch (desired_class_load_kind) { + case HLoadClass::LoadKind::kInvalid: + LOG(FATAL) << "UNREACHABLE"; + UNREACHABLE(); case HLoadClass::LoadKind::kReferrersClass: break; case HLoadClass::LoadKind::kBootImageLinkTimeAddress: @@ -5749,6 +5781,7 @@ void LocationsBuilderARM::VisitLoadClass(HLoadClass* cls) { cls, Location::RegisterLocation(calling_convention.GetRegisterAt(0)), Location::RegisterLocation(R0)); + DCHECK_EQ(calling_convention.GetRegisterAt(0), R0); return; } DCHECK(!cls->NeedsAccessCheck()); @@ -5766,6 +5799,22 @@ void LocationsBuilderARM::VisitLoadClass(HLoadClass* cls) { locations->SetInAt(0, Location::RequiresRegister()); } locations->SetOut(Location::RequiresRegister()); + if (load_kind == HLoadClass::LoadKind::kBssEntry) { + if (!kUseReadBarrier || kUseBakerReadBarrier) { + // Rely on the type resolution or initialization and marking to save everything we need. + // Note that IP may be clobbered by saving/restoring the live register (only one thanks + // to the custom calling convention) or by marking, so we request a different temp. + locations->AddTemp(Location::RequiresRegister()); + RegisterSet caller_saves = RegisterSet::Empty(); + InvokeRuntimeCallingConvention calling_convention; + caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); + // TODO: Add GetReturnLocation() to the calling convention so that we can DCHECK() + // that the the kPrimNot result register is the same as the first argument register. + locations->SetCustomSlowPathCallerSaves(caller_saves); + } else { + // For non-Baker read barrier we have a temp-clobbering call. + } + } } // NO_THREAD_SAFETY_ANALYSIS as we manipulate handles whose internal object we know does not @@ -5828,15 +5877,18 @@ void InstructionCodeGeneratorARM::VisitLoadClass(HLoadClass* cls) NO_THREAD_SAFE break; } case HLoadClass::LoadKind::kBssEntry: { + Register temp = (!kUseReadBarrier || kUseBakerReadBarrier) + ? locations->GetTemp(0).AsRegister<Register>() + : out; CodeGeneratorARM::PcRelativePatchInfo* labels = codegen_->NewTypeBssEntryPatch(cls->GetDexFile(), cls->GetTypeIndex()); __ BindTrackedLabel(&labels->movw_label); - __ movw(out, /* placeholder */ 0u); + __ movw(temp, /* placeholder */ 0u); __ BindTrackedLabel(&labels->movt_label); - __ movt(out, /* placeholder */ 0u); + __ movt(temp, /* placeholder */ 0u); __ BindTrackedLabel(&labels->add_pc_label); - __ add(out, out, ShifterOperand(PC)); - GenerateGcRootFieldLoad(cls, out_loc, out, 0, kCompilerReadBarrierOption); + __ add(temp, temp, ShifterOperand(PC)); + GenerateGcRootFieldLoad(cls, out_loc, temp, /* offset */ 0, read_barrier_option); generate_null_check = true; break; } @@ -5845,10 +5897,11 @@ void InstructionCodeGeneratorARM::VisitLoadClass(HLoadClass* cls) NO_THREAD_SAFE cls->GetTypeIndex(), cls->GetClass())); // /* GcRoot<mirror::Class> */ out = *out - GenerateGcRootFieldLoad(cls, out_loc, out, /* offset */ 0, kCompilerReadBarrierOption); + GenerateGcRootFieldLoad(cls, out_loc, out, /* offset */ 0, read_barrier_option); break; } case HLoadClass::LoadKind::kDexCacheViaMethod: + case HLoadClass::LoadKind::kInvalid: LOG(FATAL) << "UNREACHABLE"; UNREACHABLE(); } @@ -5931,9 +5984,9 @@ void LocationsBuilderARM::VisitLoadString(HLoadString* load) { locations->SetOut(Location::RequiresRegister()); if (load_kind == HLoadString::LoadKind::kBssEntry) { if (!kUseReadBarrier || kUseBakerReadBarrier) { - // Rely on the pResolveString and/or marking to save everything, including temps. - // Note that IP may theoretically be clobbered by saving/restoring the live register - // (only one thanks to the custom calling convention), so we request a different temp. + // Rely on the pResolveString and marking to save everything we need, including temps. + // Note that IP may be clobbered by saving/restoring the live register (only one thanks + // to the custom calling convention) or by marking, so we request a different temp. locations->AddTemp(Location::RequiresRegister()); RegisterSet caller_saves = RegisterSet::Empty(); InvokeRuntimeCallingConvention calling_convention; @@ -5984,7 +6037,9 @@ void InstructionCodeGeneratorARM::VisitLoadString(HLoadString* load) NO_THREAD_S } case HLoadString::LoadKind::kBssEntry: { DCHECK(!codegen_->GetCompilerOptions().IsBootImage()); - Register temp = locations->GetTemp(0).AsRegister<Register>(); + Register temp = (!kUseReadBarrier || kUseBakerReadBarrier) + ? locations->GetTemp(0).AsRegister<Register>() + : out; CodeGeneratorARM::PcRelativePatchInfo* labels = codegen_->NewPcRelativeStringPatch(load->GetDexFile(), load->GetStringIndex()); __ BindTrackedLabel(&labels->movw_label); diff --git a/compiler/optimizing/code_generator_arm64.cc b/compiler/optimizing/code_generator_arm64.cc index 26c8254c76..e6032d2381 100644 --- a/compiler/optimizing/code_generator_arm64.cc +++ b/compiler/optimizing/code_generator_arm64.cc @@ -34,6 +34,9 @@ #include "utils/stack_checks.h" using namespace vixl::aarch64; // NOLINT(build/namespaces) +using vixl::ExactAssemblyScope; +using vixl::CodeBufferCheckScope; +using vixl::EmissionCheckScope; #ifdef __ #error "ARM64 Codegen VIXL macro-assembler macro already defined." @@ -275,15 +278,37 @@ class LoadClassSlowPathARM64 : public SlowPathCodeARM64 { LoadClassSlowPathARM64(HLoadClass* cls, HInstruction* at, uint32_t dex_pc, - bool do_clinit) - : SlowPathCodeARM64(at), cls_(cls), dex_pc_(dex_pc), do_clinit_(do_clinit) { + bool do_clinit, + vixl::aarch64::Register bss_entry_temp = vixl::aarch64::Register(), + vixl::aarch64::Label* bss_entry_adrp_label = nullptr) + : SlowPathCodeARM64(at), + cls_(cls), + dex_pc_(dex_pc), + do_clinit_(do_clinit), + bss_entry_temp_(bss_entry_temp), + bss_entry_adrp_label_(bss_entry_adrp_label) { DCHECK(at->IsLoadClass() || at->IsClinitCheck()); } void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { LocationSummary* locations = instruction_->GetLocations(); + Location out = locations->Out(); + constexpr bool call_saves_everything_except_r0_ip0 = (!kUseReadBarrier || kUseBakerReadBarrier); CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen); + // For HLoadClass/kBssEntry/kSaveEverything, make sure we preserve the page address of + // the entry which is in a scratch register. Make sure it's not used for saving/restoring + // registers. Exclude the scratch register also for non-Baker read barrier for simplicity. + DCHECK_EQ(instruction_->IsLoadClass(), cls_ == instruction_); + bool is_load_class_bss_entry = + (cls_ == instruction_) && (cls_->GetLoadKind() == HLoadClass::LoadKind::kBssEntry); + UseScratchRegisterScope temps(arm64_codegen->GetVIXLAssembler()); + if (is_load_class_bss_entry) { + // This temp is a scratch register. + DCHECK(bss_entry_temp_.IsValid()); + temps.Exclude(bss_entry_temp_); + } + __ Bind(GetEntryLabel()); SaveLiveRegisters(codegen, locations); @@ -300,7 +325,6 @@ class LoadClassSlowPathARM64 : public SlowPathCodeARM64 { } // Move the class to the desired location. - Location out = locations->Out(); if (out.IsValid()) { DCHECK(out.IsRegister() && !locations->GetLiveRegisters()->ContainsCoreRegister(out.reg())); Primitive::Type type = instruction_->GetType(); @@ -308,25 +332,23 @@ class LoadClassSlowPathARM64 : public SlowPathCodeARM64 { } RestoreLiveRegisters(codegen, locations); // For HLoadClass/kBssEntry, store the resolved Class to the BSS entry. - DCHECK_EQ(instruction_->IsLoadClass(), cls_ == instruction_); - if (cls_ == instruction_ && cls_->GetLoadKind() == HLoadClass::LoadKind::kBssEntry) { + if (is_load_class_bss_entry) { DCHECK(out.IsValid()); - UseScratchRegisterScope temps(arm64_codegen->GetVIXLAssembler()); - Register temp = temps.AcquireX(); const DexFile& dex_file = cls_->GetDexFile(); - // TODO: Change art_quick_initialize_type/art_quick_initialize_static_storage to - // kSaveEverything and use a temporary for the ADRP in the fast path, so that we - // can avoid the ADRP here. - vixl::aarch64::Label* adrp_label = - arm64_codegen->NewBssEntryTypePatch(dex_file, type_index); - arm64_codegen->EmitAdrpPlaceholder(adrp_label, temp); + if (call_saves_everything_except_r0_ip0) { + // The class entry page address was preserved in bss_entry_temp_ thanks to kSaveEverything. + } else { + // For non-Baker read barrier, we need to re-calculate the address of the class entry page. + bss_entry_adrp_label_ = arm64_codegen->NewBssEntryTypePatch(dex_file, type_index); + arm64_codegen->EmitAdrpPlaceholder(bss_entry_adrp_label_, bss_entry_temp_); + } vixl::aarch64::Label* strp_label = - arm64_codegen->NewBssEntryTypePatch(dex_file, type_index, adrp_label); + arm64_codegen->NewBssEntryTypePatch(dex_file, type_index, bss_entry_adrp_label_); { SingleEmissionCheckScope guard(arm64_codegen->GetVIXLAssembler()); __ Bind(strp_label); __ str(RegisterFrom(locations->Out(), Primitive::kPrimNot), - MemOperand(temp, /* offset placeholder */ 0)); + MemOperand(bss_entry_temp_, /* offset placeholder */ 0)); } } __ B(GetExitLabel()); @@ -344,6 +366,10 @@ class LoadClassSlowPathARM64 : public SlowPathCodeARM64 { // Whether to initialize the class. const bool do_clinit_; + // For HLoadClass/kBssEntry, the temp register and the label of the ADRP where it was loaded. + vixl::aarch64::Register bss_entry_temp_; + vixl::aarch64::Label* bss_entry_adrp_label_; + DISALLOW_COPY_AND_ASSIGN(LoadClassSlowPathARM64); }; @@ -590,10 +616,9 @@ void JumpTableARM64::EmitTable(CodeGeneratorARM64* codegen) { // We are about to use the assembler to place literals directly. Make sure we have enough // underlying code buffer and we have generated the jump table with right size. - vixl::CodeBufferCheckScope scope(codegen->GetVIXLAssembler(), - num_entries * sizeof(int32_t), - vixl::CodeBufferCheckScope::kReserveBufferSpace, - vixl::CodeBufferCheckScope::kExactSize); + EmissionCheckScope scope(codegen->GetVIXLAssembler(), + num_entries * sizeof(int32_t), + CodeBufferCheckScope::kExactSize); __ Bind(&table_start_); const ArenaVector<HBasicBlock*>& successors = switch_instr_->GetBlock()->GetSuccessors(); @@ -619,8 +644,10 @@ void JumpTableARM64::EmitTable(CodeGeneratorARM64* codegen) { // probably still be a from-space reference (unless it gets updated by // another thread, or if another thread installed another object // reference (different from `ref`) in `obj.field`). -// If entrypoint is a valid location it is assumed to already be holding the entrypoint. The case -// where the entrypoint is passed in is for the GcRoot read barrier. +// +// If `entrypoint` is a valid location it is assumed to already be +// holding the entrypoint. The case where the entrypoint is passed in +// is for the GcRoot read barrier. class ReadBarrierMarkSlowPathARM64 : public SlowPathCodeARM64 { public: ReadBarrierMarkSlowPathARM64(HInstruction* instruction, @@ -1254,7 +1281,6 @@ void ParallelMoveResolverARM64::EmitMove(size_t index) { void CodeGeneratorARM64::GenerateFrameEntry() { MacroAssembler* masm = GetVIXLAssembler(); - BlockPoolsScope block_pools(masm); __ Bind(&frame_entry_label_); bool do_overflow_check = FrameNeedsStackCheck(GetFrameSize(), kArm64) || !IsLeafMethod(); @@ -1263,8 +1289,14 @@ void CodeGeneratorARM64::GenerateFrameEntry() { Register temp = temps.AcquireX(); DCHECK(GetCompilerOptions().GetImplicitStackOverflowChecks()); __ Sub(temp, sp, static_cast<int32_t>(GetStackOverflowReservedBytes(kArm64))); - __ Ldr(wzr, MemOperand(temp, 0)); - RecordPcInfo(nullptr, 0); + { + // Ensure that between load and RecordPcInfo there are no pools emitted. + ExactAssemblyScope eas(GetVIXLAssembler(), + kInstructionSize, + CodeBufferCheckScope::kExactSize); + __ ldr(wzr, MemOperand(temp, 0)); + RecordPcInfo(nullptr, 0); + } } if (!HasEmptyFrame()) { @@ -1299,7 +1331,6 @@ void CodeGeneratorARM64::GenerateFrameEntry() { } void CodeGeneratorARM64::GenerateFrameExit() { - BlockPoolsScope block_pools(GetVIXLAssembler()); GetAssembler()->cfi().RememberState(); if (!HasEmptyFrame()) { int frame_size = GetFrameSize(); @@ -1626,7 +1657,6 @@ void CodeGeneratorARM64::LoadAcquire(HInstruction* instruction, const MemOperand& src, bool needs_null_check) { MacroAssembler* masm = GetVIXLAssembler(); - BlockPoolsScope block_pools(masm); UseScratchRegisterScope temps(masm); Register temp_base = temps.AcquireX(); Primitive::Type type = instruction->GetType(); @@ -1636,58 +1666,79 @@ void CodeGeneratorARM64::LoadAcquire(HInstruction* instruction, // TODO(vixl): Let the MacroAssembler handle MemOperand. __ Add(temp_base, src.GetBaseRegister(), OperandFromMemOperand(src)); - MemOperand base = MemOperand(temp_base); - switch (type) { - case Primitive::kPrimBoolean: - __ Ldarb(Register(dst), base); - if (needs_null_check) { - MaybeRecordImplicitNullCheck(instruction); - } - break; - case Primitive::kPrimByte: - __ Ldarb(Register(dst), base); - if (needs_null_check) { - MaybeRecordImplicitNullCheck(instruction); - } - __ Sbfx(Register(dst), Register(dst), 0, Primitive::ComponentSize(type) * kBitsPerByte); - break; - case Primitive::kPrimChar: - __ Ldarh(Register(dst), base); - if (needs_null_check) { - MaybeRecordImplicitNullCheck(instruction); - } - break; - case Primitive::kPrimShort: - __ Ldarh(Register(dst), base); - if (needs_null_check) { - MaybeRecordImplicitNullCheck(instruction); - } - __ Sbfx(Register(dst), Register(dst), 0, Primitive::ComponentSize(type) * kBitsPerByte); - break; - case Primitive::kPrimInt: - case Primitive::kPrimNot: - case Primitive::kPrimLong: - DCHECK_EQ(dst.Is64Bits(), Primitive::Is64BitType(type)); - __ Ldar(Register(dst), base); - if (needs_null_check) { - MaybeRecordImplicitNullCheck(instruction); - } - break; - case Primitive::kPrimFloat: - case Primitive::kPrimDouble: { - DCHECK(dst.IsFPRegister()); - DCHECK_EQ(dst.Is64Bits(), Primitive::Is64BitType(type)); - - Register temp = dst.Is64Bits() ? temps.AcquireX() : temps.AcquireW(); - __ Ldar(temp, base); - if (needs_null_check) { - MaybeRecordImplicitNullCheck(instruction); + { + // Ensure that between load and MaybeRecordImplicitNullCheck there are no pools emitted. + MemOperand base = MemOperand(temp_base); + switch (type) { + case Primitive::kPrimBoolean: + { + ExactAssemblyScope eas(masm, kInstructionSize, CodeBufferCheckScope::kExactSize); + __ ldarb(Register(dst), base); + if (needs_null_check) { + MaybeRecordImplicitNullCheck(instruction); + } + } + break; + case Primitive::kPrimByte: + { + ExactAssemblyScope eas(masm, kInstructionSize, CodeBufferCheckScope::kExactSize); + __ ldarb(Register(dst), base); + if (needs_null_check) { + MaybeRecordImplicitNullCheck(instruction); + } + } + __ Sbfx(Register(dst), Register(dst), 0, Primitive::ComponentSize(type) * kBitsPerByte); + break; + case Primitive::kPrimChar: + { + ExactAssemblyScope eas(masm, kInstructionSize, CodeBufferCheckScope::kExactSize); + __ ldarh(Register(dst), base); + if (needs_null_check) { + MaybeRecordImplicitNullCheck(instruction); + } + } + break; + case Primitive::kPrimShort: + { + ExactAssemblyScope eas(masm, kInstructionSize, CodeBufferCheckScope::kExactSize); + __ ldarh(Register(dst), base); + if (needs_null_check) { + MaybeRecordImplicitNullCheck(instruction); + } + } + __ Sbfx(Register(dst), Register(dst), 0, Primitive::ComponentSize(type) * kBitsPerByte); + break; + case Primitive::kPrimInt: + case Primitive::kPrimNot: + case Primitive::kPrimLong: + DCHECK_EQ(dst.Is64Bits(), Primitive::Is64BitType(type)); + { + ExactAssemblyScope eas(masm, kInstructionSize, CodeBufferCheckScope::kExactSize); + __ ldar(Register(dst), base); + if (needs_null_check) { + MaybeRecordImplicitNullCheck(instruction); + } + } + break; + case Primitive::kPrimFloat: + case Primitive::kPrimDouble: { + DCHECK(dst.IsFPRegister()); + DCHECK_EQ(dst.Is64Bits(), Primitive::Is64BitType(type)); + + Register temp = dst.Is64Bits() ? temps.AcquireX() : temps.AcquireW(); + { + ExactAssemblyScope eas(masm, kInstructionSize, CodeBufferCheckScope::kExactSize); + __ ldar(temp, base); + if (needs_null_check) { + MaybeRecordImplicitNullCheck(instruction); + } + } + __ Fmov(FPRegister(dst), temp); + break; } - __ Fmov(FPRegister(dst), temp); - break; + case Primitive::kPrimVoid: + LOG(FATAL) << "Unreachable type " << type; } - case Primitive::kPrimVoid: - LOG(FATAL) << "Unreachable type " << type; } } @@ -1716,9 +1767,12 @@ void CodeGeneratorARM64::Store(Primitive::Type type, } } -void CodeGeneratorARM64::StoreRelease(Primitive::Type type, +void CodeGeneratorARM64::StoreRelease(HInstruction* instruction, + Primitive::Type type, CPURegister src, - const MemOperand& dst) { + const MemOperand& dst, + bool needs_null_check) { + MacroAssembler* masm = GetVIXLAssembler(); UseScratchRegisterScope temps(GetVIXLAssembler()); Register temp_base = temps.AcquireX(); @@ -1729,20 +1783,39 @@ void CodeGeneratorARM64::StoreRelease(Primitive::Type type, Operand op = OperandFromMemOperand(dst); __ Add(temp_base, dst.GetBaseRegister(), op); MemOperand base = MemOperand(temp_base); + // Ensure that between store and MaybeRecordImplicitNullCheck there are no pools emitted. switch (type) { case Primitive::kPrimBoolean: case Primitive::kPrimByte: - __ Stlrb(Register(src), base); + { + ExactAssemblyScope eas(masm, kInstructionSize, CodeBufferCheckScope::kExactSize); + __ stlrb(Register(src), base); + if (needs_null_check) { + MaybeRecordImplicitNullCheck(instruction); + } + } break; case Primitive::kPrimChar: case Primitive::kPrimShort: - __ Stlrh(Register(src), base); + { + ExactAssemblyScope eas(masm, kInstructionSize, CodeBufferCheckScope::kExactSize); + __ stlrh(Register(src), base); + if (needs_null_check) { + MaybeRecordImplicitNullCheck(instruction); + } + } break; case Primitive::kPrimInt: case Primitive::kPrimNot: case Primitive::kPrimLong: DCHECK_EQ(src.Is64Bits(), Primitive::Is64BitType(type)); - __ Stlr(Register(src), base); + { + ExactAssemblyScope eas(masm, kInstructionSize, CodeBufferCheckScope::kExactSize); + __ stlr(Register(src), base); + if (needs_null_check) { + MaybeRecordImplicitNullCheck(instruction); + } + } break; case Primitive::kPrimFloat: case Primitive::kPrimDouble: { @@ -1756,8 +1829,13 @@ void CodeGeneratorARM64::StoreRelease(Primitive::Type type, temp_src = src.Is64Bits() ? temps.AcquireX() : temps.AcquireW(); __ Fmov(temp_src, FPRegister(src)); } - - __ Stlr(temp_src, base); + { + ExactAssemblyScope eas(masm, kInstructionSize, CodeBufferCheckScope::kExactSize); + __ stlr(temp_src, base); + if (needs_null_check) { + MaybeRecordImplicitNullCheck(instruction); + } + } break; } case Primitive::kPrimVoid: @@ -1770,9 +1848,15 @@ void CodeGeneratorARM64::InvokeRuntime(QuickEntrypointEnum entrypoint, uint32_t dex_pc, SlowPathCode* slow_path) { ValidateInvokeRuntime(entrypoint, instruction, slow_path); - GenerateInvokeRuntime(GetThreadOffset<kArm64PointerSize>(entrypoint).Int32Value()); - if (EntrypointRequiresStackMap(entrypoint)) { - RecordPcInfo(instruction, dex_pc, slow_path); + + __ Ldr(lr, MemOperand(tr, GetThreadOffset<kArm64PointerSize>(entrypoint).Int32Value())); + { + // Ensure the pc position is recorded immediately after the `blr` instruction. + ExactAssemblyScope eas(GetVIXLAssembler(), kInstructionSize, CodeBufferCheckScope::kExactSize); + __ blr(lr); + if (EntrypointRequiresStackMap(entrypoint)) { + RecordPcInfo(instruction, dex_pc, slow_path); + } } } @@ -1780,11 +1864,6 @@ void CodeGeneratorARM64::InvokeRuntimeWithoutRecordingPcInfo(int32_t entry_point HInstruction* instruction, SlowPathCode* slow_path) { ValidateInvokeRuntimeWithoutRecordingPcInfo(instruction, slow_path); - GenerateInvokeRuntime(entry_point_offset); -} - -void CodeGeneratorARM64::GenerateInvokeRuntime(int32_t entry_point_offset) { - BlockPoolsScope block_pools(GetVIXLAssembler()); __ Ldr(lr, MemOperand(tr, entry_point_offset)); __ Blr(lr); } @@ -1951,7 +2030,6 @@ void InstructionCodeGeneratorARM64::HandleFieldGet(HInstruction* instruction, Location out = locations->Out(); uint32_t offset = field_info.GetFieldOffset().Uint32Value(); Primitive::Type field_type = field_info.GetFieldType(); - BlockPoolsScope block_pools(GetVIXLAssembler()); MemOperand field = HeapOperand(InputRegisterAt(instruction, 0), field_info.GetFieldOffset()); if (field_type == Primitive::kPrimNot && kEmitCompilerReadBarrier && kUseBakerReadBarrier) { @@ -1978,6 +2056,8 @@ void InstructionCodeGeneratorARM64::HandleFieldGet(HInstruction* instruction, codegen_->LoadAcquire( instruction, OutputCPURegister(instruction), field, /* needs_null_check */ true); } else { + // Ensure that between load and MaybeRecordImplicitNullCheck there are no pools emitted. + EmissionCheckScope guard(GetVIXLAssembler(), kMaxMacroInstructionSizeInBytes); codegen_->Load(field_type, OutputCPURegister(instruction), field); codegen_->MaybeRecordImplicitNullCheck(instruction); } @@ -2007,7 +2087,6 @@ void InstructionCodeGeneratorARM64::HandleFieldSet(HInstruction* instruction, const FieldInfo& field_info, bool value_can_be_null) { DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet()); - BlockPoolsScope block_pools(GetVIXLAssembler()); Register obj = InputRegisterAt(instruction, 0); CPURegister value = InputCPURegisterOrZeroRegAt(instruction, 1); @@ -2029,9 +2108,11 @@ void InstructionCodeGeneratorARM64::HandleFieldSet(HInstruction* instruction, } if (field_info.IsVolatile()) { - codegen_->StoreRelease(field_type, source, HeapOperand(obj, offset)); - codegen_->MaybeRecordImplicitNullCheck(instruction); + codegen_->StoreRelease( + instruction, field_type, source, HeapOperand(obj, offset), /* needs_null_check */ true); } else { + // Ensure that between store and MaybeRecordImplicitNullCheck there are no pools emitted. + EmissionCheckScope guard(GetVIXLAssembler(), kMaxMacroInstructionSizeInBytes); codegen_->Store(field_type, source, HeapOperand(obj, offset)); codegen_->MaybeRecordImplicitNullCheck(instruction); } @@ -2317,10 +2398,7 @@ void InstructionCodeGeneratorARM64::VisitMultiplyAccumulate(HMultiplyAccumulate* masm->GetCursorAddress<vixl::aarch64::Instruction*>() - kInstructionSize; if (prev->IsLoadOrStore()) { // Make sure we emit only exactly one nop. - vixl::CodeBufferCheckScope scope(masm, - kInstructionSize, - vixl::CodeBufferCheckScope::kReserveBufferSpace, - vixl::CodeBufferCheckScope::kExactSize); + ExactAssemblyScope scope(masm, kInstructionSize, CodeBufferCheckScope::kExactSize); __ nop(); } } @@ -2376,8 +2454,6 @@ void InstructionCodeGeneratorARM64::VisitArrayGet(HArrayGet* instruction) { instruction->IsStringCharAt(); MacroAssembler* masm = GetVIXLAssembler(); UseScratchRegisterScope temps(masm); - // Block pools between `Load` and `MaybeRecordImplicitNullCheck`. - BlockPoolsScope block_pools(masm); // The read barrier instrumentation of object ArrayGet instructions // does not support the HIntermediateAddress instruction. @@ -2399,15 +2475,21 @@ void InstructionCodeGeneratorARM64::VisitArrayGet(HArrayGet* instruction) { if (maybe_compressed_char_at) { uint32_t count_offset = mirror::String::CountOffset().Uint32Value(); length = temps.AcquireW(); - if (instruction->GetArray()->IsIntermediateAddress()) { - DCHECK_LT(count_offset, offset); - int64_t adjusted_offset = static_cast<int64_t>(count_offset) - static_cast<int64_t>(offset); - // Note that `adjusted_offset` is negative, so this will be a LDUR. - __ Ldr(length, MemOperand(obj.X(), adjusted_offset)); - } else { - __ Ldr(length, HeapOperand(obj, count_offset)); + { + // Ensure that between load and MaybeRecordImplicitNullCheck there are no pools emitted. + EmissionCheckScope guard(GetVIXLAssembler(), kMaxMacroInstructionSizeInBytes); + + if (instruction->GetArray()->IsIntermediateAddress()) { + DCHECK_LT(count_offset, offset); + int64_t adjusted_offset = + static_cast<int64_t>(count_offset) - static_cast<int64_t>(offset); + // Note that `adjusted_offset` is negative, so this will be a LDUR. + __ Ldr(length, MemOperand(obj.X(), adjusted_offset)); + } else { + __ Ldr(length, HeapOperand(obj, count_offset)); + } + codegen_->MaybeRecordImplicitNullCheck(instruction); } - codegen_->MaybeRecordImplicitNullCheck(instruction); } if (index.IsConstant()) { if (maybe_compressed_char_at) { @@ -2457,6 +2539,8 @@ void InstructionCodeGeneratorARM64::VisitArrayGet(HArrayGet* instruction) { } } if (!maybe_compressed_char_at) { + // Ensure that between load and MaybeRecordImplicitNullCheck there are no pools emitted. + EmissionCheckScope guard(GetVIXLAssembler(), kMaxMacroInstructionSizeInBytes); codegen_->Load(type, OutputCPURegister(instruction), source); codegen_->MaybeRecordImplicitNullCheck(instruction); } @@ -2484,9 +2568,12 @@ void LocationsBuilderARM64::VisitArrayLength(HArrayLength* instruction) { void InstructionCodeGeneratorARM64::VisitArrayLength(HArrayLength* instruction) { uint32_t offset = CodeGenerator::GetArrayLengthOffset(instruction); vixl::aarch64::Register out = OutputRegister(instruction); - BlockPoolsScope block_pools(GetVIXLAssembler()); - __ Ldr(out, HeapOperand(InputRegisterAt(instruction, 0), offset)); - codegen_->MaybeRecordImplicitNullCheck(instruction); + { + // Ensure that between load and MaybeRecordImplicitNullCheck there are no pools emitted. + EmissionCheckScope guard(GetVIXLAssembler(), kMaxMacroInstructionSizeInBytes); + __ Ldr(out, HeapOperand(InputRegisterAt(instruction, 0), offset)); + codegen_->MaybeRecordImplicitNullCheck(instruction); + } // Mask out compression flag from String's array length. if (mirror::kUseStringCompression && instruction->IsStringLength()) { __ Lsr(out.W(), out.W(), 1u); @@ -2527,7 +2614,6 @@ void InstructionCodeGeneratorARM64::VisitArraySet(HArraySet* instruction) { size_t offset = mirror::Array::DataOffset(Primitive::ComponentSize(value_type)).Uint32Value(); MemOperand destination = HeapOperand(array); MacroAssembler* masm = GetVIXLAssembler(); - BlockPoolsScope block_pools(masm); if (!needs_write_barrier) { DCHECK(!may_need_runtime_call_for_type_check); @@ -2554,8 +2640,12 @@ void InstructionCodeGeneratorARM64::VisitArraySet(HArraySet* instruction) { LSL, Primitive::ComponentSizeShift(value_type)); } - codegen_->Store(value_type, value, destination); - codegen_->MaybeRecordImplicitNullCheck(instruction); + { + // Ensure that between store and MaybeRecordImplicitNullCheck there are no pools emitted. + EmissionCheckScope guard(GetVIXLAssembler(), kMaxMacroInstructionSizeInBytes); + codegen_->Store(value_type, value, destination); + codegen_->MaybeRecordImplicitNullCheck(instruction); + } } else { DCHECK(!instruction->GetArray()->IsIntermediateAddress()); vixl::aarch64::Label done; @@ -2588,8 +2678,13 @@ void InstructionCodeGeneratorARM64::VisitArraySet(HArraySet* instruction) { if (!index.IsConstant()) { __ Add(temp, array, offset); } - __ Str(wzr, destination); - codegen_->MaybeRecordImplicitNullCheck(instruction); + { + // Ensure that between store and MaybeRecordImplicitNullCheck there are no pools + // emitted. + EmissionCheckScope guard(GetVIXLAssembler(), kMaxMacroInstructionSizeInBytes); + __ Str(wzr, destination); + codegen_->MaybeRecordImplicitNullCheck(instruction); + } __ B(&done); __ Bind(&non_zero); } @@ -2604,8 +2699,12 @@ void InstructionCodeGeneratorARM64::VisitArraySet(HArraySet* instruction) { Register temp2 = temps.AcquireSameSizeAs(array); // /* HeapReference<Class> */ temp = array->klass_ - __ Ldr(temp, HeapOperand(array, class_offset)); - codegen_->MaybeRecordImplicitNullCheck(instruction); + { + // Ensure that between load and MaybeRecordImplicitNullCheck there are no pools emitted. + EmissionCheckScope guard(GetVIXLAssembler(), kMaxMacroInstructionSizeInBytes); + __ Ldr(temp, HeapOperand(array, class_offset)); + codegen_->MaybeRecordImplicitNullCheck(instruction); + } GetAssembler()->MaybeUnpoisonHeapReference(temp); // /* HeapReference<Class> */ temp = temp->component_type_ @@ -2646,10 +2745,14 @@ void InstructionCodeGeneratorARM64::VisitArraySet(HArraySet* instruction) { if (!index.IsConstant()) { __ Add(temp, array, offset); } - __ Str(source, destination); + { + // Ensure that between store and MaybeRecordImplicitNullCheck there are no pools emitted. + EmissionCheckScope guard(GetVIXLAssembler(), kMaxMacroInstructionSizeInBytes); + __ Str(source, destination); - if (!may_need_runtime_call_for_type_check) { - codegen_->MaybeRecordImplicitNullCheck(instruction); + if (!may_need_runtime_call_for_type_check) { + codegen_->MaybeRecordImplicitNullCheck(instruction); + } } } @@ -3944,19 +4047,25 @@ void InstructionCodeGeneratorARM64::VisitInvokeInterface(HInvokeInterface* invok // art_quick_imt_conflict_trampoline, so prevent VIXL from using it. MacroAssembler* masm = GetVIXLAssembler(); UseScratchRegisterScope scratch_scope(masm); - BlockPoolsScope block_pools(masm); scratch_scope.Exclude(ip1); __ Mov(ip1, invoke->GetDexMethodIndex()); + // Ensure that between load and MaybeRecordImplicitNullCheck there are no pools emitted. if (receiver.IsStackSlot()) { __ Ldr(temp.W(), StackOperandFrom(receiver)); - // /* HeapReference<Class> */ temp = temp->klass_ - __ Ldr(temp.W(), HeapOperand(temp.W(), class_offset)); + { + EmissionCheckScope guard(GetVIXLAssembler(), kMaxMacroInstructionSizeInBytes); + // /* HeapReference<Class> */ temp = temp->klass_ + __ Ldr(temp.W(), HeapOperand(temp.W(), class_offset)); + codegen_->MaybeRecordImplicitNullCheck(invoke); + } } else { + EmissionCheckScope guard(GetVIXLAssembler(), kMaxMacroInstructionSizeInBytes); // /* HeapReference<Class> */ temp = receiver->klass_ __ Ldr(temp.W(), HeapOperandFrom(receiver, class_offset)); + codegen_->MaybeRecordImplicitNullCheck(invoke); } - codegen_->MaybeRecordImplicitNullCheck(invoke); + // Instead of simply (possibly) unpoisoning `temp` here, we should // emit a read barrier for the previous class reference load. // However this is not required in practice, as this is an @@ -3973,10 +4082,16 @@ void InstructionCodeGeneratorARM64::VisitInvokeInterface(HInvokeInterface* invok __ Ldr(temp, MemOperand(temp, method_offset)); // lr = temp->GetEntryPoint(); __ Ldr(lr, MemOperand(temp, entry_point.Int32Value())); - // lr(); - __ Blr(lr); - DCHECK(!codegen_->IsLeafMethod()); - codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); + + { + // Ensure the pc position is recorded immediately after the `blr` instruction. + ExactAssemblyScope eas(GetVIXLAssembler(), kInstructionSize, CodeBufferCheckScope::kExactSize); + + // lr(); + __ blr(lr); + DCHECK(!codegen_->IsLeafMethod()); + codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); + } } void LocationsBuilderARM64::VisitInvokeVirtual(HInvokeVirtual* invoke) { @@ -4088,8 +4203,16 @@ void CodeGeneratorARM64::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invok __ Ldr(lr, MemOperand( XRegisterFrom(callee_method), ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArm64PointerSize).Int32Value())); - // lr() - __ Blr(lr); + { + // To ensure that the pc position is recorded immediately after the `blr` instruction + // BLR must be the last instruction emitted in this function. + // Recording the pc will occur right after returning from this function. + ExactAssemblyScope eas(GetVIXLAssembler(), + kInstructionSize, + CodeBufferCheckScope::kExactSize); + // lr() + __ blr(lr); + } break; } @@ -4109,12 +4232,15 @@ void CodeGeneratorARM64::GenerateVirtualCall(HInvokeVirtual* invoke, Location te Offset class_offset = mirror::Object::ClassOffset(); Offset entry_point = ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArm64PointerSize); - BlockPoolsScope block_pools(GetVIXLAssembler()); - DCHECK(receiver.IsRegister()); - // /* HeapReference<Class> */ temp = receiver->klass_ - __ Ldr(temp.W(), HeapOperandFrom(LocationFrom(receiver), class_offset)); - MaybeRecordImplicitNullCheck(invoke); + + { + // Ensure that between load and MaybeRecordImplicitNullCheck there are no pools emitted. + EmissionCheckScope guard(GetVIXLAssembler(), kMaxMacroInstructionSizeInBytes); + // /* HeapReference<Class> */ temp = receiver->klass_ + __ Ldr(temp.W(), HeapOperandFrom(LocationFrom(receiver), class_offset)); + MaybeRecordImplicitNullCheck(invoke); + } // Instead of simply (possibly) unpoisoning `temp` here, we should // emit a read barrier for the previous class reference load. // intermediate/temporary reference and because the current @@ -4126,8 +4252,14 @@ void CodeGeneratorARM64::GenerateVirtualCall(HInvokeVirtual* invoke, Location te __ Ldr(temp, MemOperand(temp, method_offset)); // lr = temp->GetEntryPoint(); __ Ldr(lr, MemOperand(temp, entry_point.SizeValue())); - // lr(); - __ Blr(lr); + { + // To ensure that the pc position is recorded immediately after the `blr` instruction + // BLR should be the last instruction emitted in this function. + // Recording the pc will occur right after returning from this function. + ExactAssemblyScope eas(GetVIXLAssembler(), kInstructionSize, CodeBufferCheckScope::kExactSize); + // lr(); + __ blr(lr); + } } void LocationsBuilderARM64::VisitInvokePolymorphic(HInvokePolymorphic* invoke) { @@ -4340,7 +4472,9 @@ void InstructionCodeGeneratorARM64::VisitInvokeStaticOrDirect(HInvokeStaticOrDir return; } - BlockPoolsScope block_pools(GetVIXLAssembler()); + // Ensure that between the BLR (emitted by GenerateStaticOrDirectCall) and RecordPcInfo there + // are no pools emitted. + EmissionCheckScope guard(GetVIXLAssembler(), kInvokeCodeMarginSizeInBytes); LocationSummary* locations = invoke->GetLocations(); codegen_->GenerateStaticOrDirectCall( invoke, locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation()); @@ -4352,6 +4486,9 @@ void InstructionCodeGeneratorARM64::VisitInvokeVirtual(HInvokeVirtual* invoke) { return; } + // Ensure that between the BLR (emitted by GenerateVirtualCall) and RecordPcInfo there + // are no pools emitted. + EmissionCheckScope guard(GetVIXLAssembler(), kInvokeCodeMarginSizeInBytes); codegen_->GenerateVirtualCall(invoke, invoke->GetLocations()->GetTemp(0)); DCHECK(!codegen_->IsLeafMethod()); codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); @@ -4360,6 +4497,9 @@ void InstructionCodeGeneratorARM64::VisitInvokeVirtual(HInvokeVirtual* invoke) { HLoadClass::LoadKind CodeGeneratorARM64::GetSupportedLoadClassKind( HLoadClass::LoadKind desired_class_load_kind) { switch (desired_class_load_kind) { + case HLoadClass::LoadKind::kInvalid: + LOG(FATAL) << "UNREACHABLE"; + UNREACHABLE(); case HLoadClass::LoadKind::kReferrersClass: break; case HLoadClass::LoadKind::kBootImageLinkTimeAddress: @@ -4390,6 +4530,7 @@ void LocationsBuilderARM64::VisitLoadClass(HLoadClass* cls) { cls, LocationFrom(calling_convention.GetRegisterAt(0)), LocationFrom(vixl::aarch64::x0)); + DCHECK(calling_convention.GetRegisterAt(0).Is(vixl::aarch64::x0)); return; } DCHECK(!cls->NeedsAccessCheck()); @@ -4407,6 +4548,22 @@ void LocationsBuilderARM64::VisitLoadClass(HLoadClass* cls) { locations->SetInAt(0, Location::RequiresRegister()); } locations->SetOut(Location::RequiresRegister()); + if (cls->GetLoadKind() == HLoadClass::LoadKind::kBssEntry) { + if (!kUseReadBarrier || kUseBakerReadBarrier) { + // Rely on the type resolution or initialization and marking to save everything we need. + // Note that IP0 may be clobbered by saving/restoring the live register (only one thanks + // to the custom calling convention) or by marking, so we shall use IP1. + RegisterSet caller_saves = RegisterSet::Empty(); + InvokeRuntimeCallingConvention calling_convention; + caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0).GetCode())); + DCHECK_EQ(calling_convention.GetRegisterAt(0).GetCode(), + RegisterFrom(calling_convention.GetReturnLocation(Primitive::kPrimNot), + Primitive::kPrimNot).GetCode()); + locations->SetCustomSlowPathCallerSaves(caller_saves); + } else { + // For non-Baker read barrier we have a temp-clobbering call. + } + } } // NO_THREAD_SAFETY_ANALYSIS as we manipulate handles whose internal object we know does not @@ -4421,6 +4578,8 @@ void InstructionCodeGeneratorARM64::VisitLoadClass(HLoadClass* cls) NO_THREAD_SA Location out_loc = cls->GetLocations()->Out(); Register out = OutputRegister(cls); + Register bss_entry_temp; + vixl::aarch64::Label* bss_entry_adrp_label = nullptr; const ReadBarrierOption read_barrier_option = cls->IsInBootImage() ? kWithoutReadBarrier @@ -4470,18 +4629,23 @@ void InstructionCodeGeneratorARM64::VisitLoadClass(HLoadClass* cls) NO_THREAD_SA // Add ADRP with its PC-relative Class .bss entry patch. const DexFile& dex_file = cls->GetDexFile(); dex::TypeIndex type_index = cls->GetTypeIndex(); - vixl::aarch64::Label* adrp_label = codegen_->NewBssEntryTypePatch(dex_file, type_index); - codegen_->EmitAdrpPlaceholder(adrp_label, out.X()); + // We can go to slow path even with non-zero reference and in that case marking + // can clobber IP0, so we need to use IP1 which shall be preserved. + bss_entry_temp = ip1; + UseScratchRegisterScope temps(codegen_->GetVIXLAssembler()); + temps.Exclude(bss_entry_temp); + bss_entry_adrp_label = codegen_->NewBssEntryTypePatch(dex_file, type_index); + codegen_->EmitAdrpPlaceholder(bss_entry_adrp_label, bss_entry_temp); // Add LDR with its PC-relative Class patch. vixl::aarch64::Label* ldr_label = - codegen_->NewBssEntryTypePatch(dex_file, type_index, adrp_label); + codegen_->NewBssEntryTypePatch(dex_file, type_index, bss_entry_adrp_label); // /* GcRoot<mirror::Class> */ out = *(base_address + offset) /* PC-relative */ GenerateGcRootFieldLoad(cls, - cls->GetLocations()->Out(), - out.X(), - /* placeholder */ 0u, + out_loc, + bss_entry_temp, + /* offset placeholder */ 0u, ldr_label, - kCompilerReadBarrierOption); + read_barrier_option); generate_null_check = true; break; } @@ -4494,18 +4658,20 @@ void InstructionCodeGeneratorARM64::VisitLoadClass(HLoadClass* cls) NO_THREAD_SA out.X(), /* offset */ 0, /* fixup_label */ nullptr, - kCompilerReadBarrierOption); + read_barrier_option); break; } case HLoadClass::LoadKind::kDexCacheViaMethod: + case HLoadClass::LoadKind::kInvalid: LOG(FATAL) << "UNREACHABLE"; UNREACHABLE(); } - if (generate_null_check || cls->MustGenerateClinitCheck()) { + bool do_clinit = cls->MustGenerateClinitCheck(); + if (generate_null_check || do_clinit) { DCHECK(cls->CanCallRuntime()); SlowPathCodeARM64* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathARM64( - cls, cls, cls->GetDexPc(), cls->MustGenerateClinitCheck()); + cls, cls, cls->GetDexPc(), do_clinit, bss_entry_temp, bss_entry_adrp_label); codegen_->AddSlowPath(slow_path); if (generate_null_check) { __ Cbz(out, slow_path->GetEntryLabel()); @@ -4573,7 +4739,9 @@ void LocationsBuilderARM64::VisitLoadString(HLoadString* load) { locations->SetOut(Location::RequiresRegister()); if (load->GetLoadKind() == HLoadString::LoadKind::kBssEntry) { if (!kUseReadBarrier || kUseBakerReadBarrier) { - // Rely on the pResolveString and/or marking to save everything, including temps. + // Rely on the pResolveString and marking to save everything we need. + // Note that IP0 may be clobbered by saving/restoring the live register (only one thanks + // to the custom calling convention) or by marking, so we shall use IP1. RegisterSet caller_saves = RegisterSet::Empty(); InvokeRuntimeCallingConvention calling_convention; caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0).GetCode())); @@ -4624,8 +4792,11 @@ void InstructionCodeGeneratorARM64::VisitLoadString(HLoadString* load) NO_THREAD const DexFile& dex_file = load->GetDexFile(); const dex::StringIndex string_index = load->GetStringIndex(); DCHECK(!codegen_->GetCompilerOptions().IsBootImage()); + // We could use IP0 as the marking shall not clobber IP0 if the reference is null and + // that's when we need the slow path. But let's not rely on such details and use IP1. + Register temp = ip1; UseScratchRegisterScope temps(codegen_->GetVIXLAssembler()); - Register temp = temps.AcquireX(); + temps.Exclude(temp); vixl::aarch64::Label* adrp_label = codegen_->NewPcRelativeStringPatch(dex_file, string_index); codegen_->EmitAdrpPlaceholder(adrp_label, temp); // Add LDR with its PC-relative String patch. @@ -4813,8 +4984,15 @@ void InstructionCodeGeneratorARM64::VisitNewInstance(HNewInstance* instruction) MemberOffset code_offset = ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArm64PointerSize); __ Ldr(XRegisterFrom(temp), MemOperand(tr, QUICK_ENTRY_POINT(pNewEmptyString))); __ Ldr(lr, MemOperand(XRegisterFrom(temp), code_offset.Int32Value())); - __ Blr(lr); - codegen_->RecordPcInfo(instruction, instruction->GetDexPc()); + + { + // Ensure the pc position is recorded immediately after the `blr` instruction. + ExactAssemblyScope eas(GetVIXLAssembler(), + kInstructionSize, + CodeBufferCheckScope::kExactSize); + __ blr(lr); + codegen_->RecordPcInfo(instruction, instruction->GetDexPc()); + } } else { codegen_->InvokeRuntime(instruction->GetEntrypoint(), instruction, instruction->GetDexPc()); CheckEntrypointTypes<kQuickAllocObjectWithChecks, void*, mirror::Class*>(); @@ -4858,11 +5036,13 @@ void CodeGeneratorARM64::GenerateImplicitNullCheck(HNullCheck* instruction) { if (CanMoveNullCheckToUser(instruction)) { return; } - - BlockPoolsScope block_pools(GetVIXLAssembler()); - Location obj = instruction->GetLocations()->InAt(0); - __ Ldr(wzr, HeapOperandFrom(obj, Offset(0))); - RecordPcInfo(instruction, instruction->GetDexPc()); + { + // Ensure that between load and MaybeRecordImplicitNullCheck there are no pools emitted. + EmissionCheckScope guard(GetVIXLAssembler(), kMaxMacroInstructionSizeInBytes); + Location obj = instruction->GetLocations()->InAt(0); + __ Ldr(wzr, HeapOperandFrom(obj, Offset(0))); + RecordPcInfo(instruction, instruction->GetDexPc()); + } } void CodeGeneratorARM64::GenerateExplicitNullCheck(HNullCheck* instruction) { @@ -5599,10 +5779,14 @@ void CodeGeneratorARM64::GenerateReferenceLoadWithBakerReadBarrier(HInstruction* DCHECK(obj.IsW()); uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value(); - // /* int32_t */ monitor = obj->monitor_ - __ Ldr(temp, HeapOperand(obj, monitor_offset)); - if (needs_null_check) { - MaybeRecordImplicitNullCheck(instruction); + { + // Ensure that between load and MaybeRecordImplicitNullCheck there are no pools emitted. + EmissionCheckScope guard(GetVIXLAssembler(), kMaxMacroInstructionSizeInBytes); + // /* int32_t */ monitor = obj->monitor_ + __ Ldr(temp, HeapOperand(obj, monitor_offset)); + if (needs_null_check) { + MaybeRecordImplicitNullCheck(instruction); + } } // /* LockWord */ lock_word = LockWord(monitor) static_assert(sizeof(LockWord) == sizeof(int32_t), diff --git a/compiler/optimizing/code_generator_arm64.h b/compiler/optimizing/code_generator_arm64.h index f6cb90a63a..5faf29a90f 100644 --- a/compiler/optimizing/code_generator_arm64.h +++ b/compiler/optimizing/code_generator_arm64.h @@ -43,6 +43,11 @@ class CodeGeneratorARM64; // Use a local definition to prevent copying mistakes. static constexpr size_t kArm64WordSize = static_cast<size_t>(kArm64PointerSize); +// These constants are used as an approximate margin when emission of veneer and literal pools +// must be blocked. +static constexpr int kMaxMacroInstructionSizeInBytes = 15 * vixl::aarch64::kInstructionSize; +static constexpr int kInvokeCodeMarginSizeInBytes = 6 * kMaxMacroInstructionSizeInBytes; + static const vixl::aarch64::Register kParameterCoreRegisters[] = { vixl::aarch64::x1, vixl::aarch64::x2, @@ -486,9 +491,11 @@ class CodeGeneratorARM64 : public CodeGenerator { vixl::aarch64::CPURegister dst, const vixl::aarch64::MemOperand& src, bool needs_null_check); - void StoreRelease(Primitive::Type type, + void StoreRelease(HInstruction* instruction, + Primitive::Type type, vixl::aarch64::CPURegister src, - const vixl::aarch64::MemOperand& dst); + const vixl::aarch64::MemOperand& dst, + bool needs_null_check); // Generate code to invoke a runtime entry point. void InvokeRuntime(QuickEntrypointEnum entrypoint, @@ -502,8 +509,6 @@ class CodeGeneratorARM64 : public CodeGenerator { HInstruction* instruction, SlowPathCode* slow_path); - void GenerateInvokeRuntime(int32_t entry_point_offset); - ParallelMoveResolverARM64* GetMoveResolver() OVERRIDE { return &move_resolver_; } bool NeedsTwoRegisters(Primitive::Type type ATTRIBUTE_UNUSED) const OVERRIDE { diff --git a/compiler/optimizing/code_generator_arm_vixl.cc b/compiler/optimizing/code_generator_arm_vixl.cc index f4d3ec54ee..5c4ca5bc17 100644 --- a/compiler/optimizing/code_generator_arm_vixl.cc +++ b/compiler/optimizing/code_generator_arm_vixl.cc @@ -400,12 +400,30 @@ class LoadClassSlowPathARMVIXL : public SlowPathCodeARMVIXL { void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { LocationSummary* locations = instruction_->GetLocations(); + Location out = locations->Out(); + constexpr bool call_saves_everything_except_r0 = (!kUseReadBarrier || kUseBakerReadBarrier); CodeGeneratorARMVIXL* arm_codegen = down_cast<CodeGeneratorARMVIXL*>(codegen); __ Bind(GetEntryLabel()); SaveLiveRegisters(codegen, locations); InvokeRuntimeCallingConventionARMVIXL calling_convention; + // For HLoadClass/kBssEntry/kSaveEverything, make sure we preserve the address of the entry. + DCHECK_EQ(instruction_->IsLoadClass(), cls_ == instruction_); + bool is_load_class_bss_entry = + (cls_ == instruction_) && (cls_->GetLoadKind() == HLoadClass::LoadKind::kBssEntry); + vixl32::Register entry_address; + if (is_load_class_bss_entry && call_saves_everything_except_r0) { + vixl32::Register temp = RegisterFrom(locations->GetTemp(0)); + // In the unlucky case that the `temp` is R0, we preserve the address in `out` across + // the kSaveEverything call. + bool temp_is_r0 = temp.Is(calling_convention.GetRegisterAt(0)); + entry_address = temp_is_r0 ? RegisterFrom(out) : temp; + DCHECK(!entry_address.Is(calling_convention.GetRegisterAt(0))); + if (temp_is_r0) { + __ Mov(entry_address, temp); + } + } dex::TypeIndex type_index = cls_->GetTypeIndex(); __ Mov(calling_convention.GetRegisterAt(0), type_index.index_); QuickEntrypointEnum entrypoint = do_clinit_ ? kQuickInitializeStaticStorage @@ -417,27 +435,28 @@ class LoadClassSlowPathARMVIXL : public SlowPathCodeARMVIXL { CheckEntrypointTypes<kQuickInitializeType, void*, uint32_t>(); } + // For HLoadClass/kBssEntry, store the resolved Class to the BSS entry. + if (is_load_class_bss_entry) { + if (call_saves_everything_except_r0) { + // The class entry address was preserved in `entry_address` thanks to kSaveEverything. + __ Str(r0, MemOperand(entry_address)); + } else { + // For non-Baker read barrier, we need to re-calculate the address of the string entry. + UseScratchRegisterScope temps( + down_cast<CodeGeneratorARMVIXL*>(codegen)->GetVIXLAssembler()); + vixl32::Register temp = temps.Acquire(); + CodeGeneratorARMVIXL::PcRelativePatchInfo* labels = + arm_codegen->NewTypeBssEntryPatch(cls_->GetDexFile(), type_index); + arm_codegen->EmitMovwMovtPlaceholder(labels, temp); + __ Str(r0, MemOperand(temp)); + } + } // Move the class to the desired location. - Location out = locations->Out(); if (out.IsValid()) { DCHECK(out.IsRegister() && !locations->GetLiveRegisters()->ContainsCoreRegister(out.reg())); arm_codegen->Move32(locations->Out(), LocationFrom(r0)); } RestoreLiveRegisters(codegen, locations); - // For HLoadClass/kBssEntry, store the resolved Class to the BSS entry. - DCHECK_EQ(instruction_->IsLoadClass(), cls_ == instruction_); - if (cls_ == instruction_ && cls_->GetLoadKind() == HLoadClass::LoadKind::kBssEntry) { - DCHECK(out.IsValid()); - // TODO: Change art_quick_initialize_type/art_quick_initialize_static_storage to - // kSaveEverything and use a temporary for the .bss entry address in the fast path, - // so that we can avoid another calculation here. - UseScratchRegisterScope temps(down_cast<CodeGeneratorARMVIXL*>(codegen)->GetVIXLAssembler()); - vixl32::Register temp = temps.Acquire(); - CodeGeneratorARMVIXL::PcRelativePatchInfo* labels = - arm_codegen->NewTypeBssEntryPatch(cls_->GetDexFile(), type_index); - arm_codegen->EmitMovwMovtPlaceholder(labels, temp); - __ Str(OutputRegister(cls_), MemOperand(temp)); - } __ B(GetExitLabel()); } @@ -462,12 +481,13 @@ class LoadStringSlowPathARMVIXL : public SlowPathCodeARMVIXL { : SlowPathCodeARMVIXL(instruction) {} void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { + DCHECK(instruction_->IsLoadString()); + DCHECK_EQ(instruction_->AsLoadString()->GetLoadKind(), HLoadString::LoadKind::kBssEntry); LocationSummary* locations = instruction_->GetLocations(); DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg())); HLoadString* load = instruction_->AsLoadString(); const dex::StringIndex string_index = load->GetStringIndex(); vixl32::Register out = OutputRegister(load); - vixl32::Register temp = RegisterFrom(locations->GetTemp(0)); constexpr bool call_saves_everything_except_r0 = (!kUseReadBarrier || kUseBakerReadBarrier); CodeGeneratorARMVIXL* arm_codegen = down_cast<CodeGeneratorARMVIXL*>(codegen); @@ -476,12 +496,16 @@ class LoadStringSlowPathARMVIXL : public SlowPathCodeARMVIXL { InvokeRuntimeCallingConventionARMVIXL calling_convention; // In the unlucky case that the `temp` is R0, we preserve the address in `out` across - // the kSaveEverything call (or use `out` for the address after non-kSaveEverything call). - bool temp_is_r0 = (temp.Is(calling_convention.GetRegisterAt(0))); - vixl32::Register entry_address = temp_is_r0 ? out : temp; - DCHECK(!entry_address.Is(calling_convention.GetRegisterAt(0))); - if (call_saves_everything_except_r0 && temp_is_r0) { - __ Mov(entry_address, temp); + // the kSaveEverything call. + vixl32::Register entry_address; + if (call_saves_everything_except_r0) { + vixl32::Register temp = RegisterFrom(locations->GetTemp(0)); + bool temp_is_r0 = (temp.Is(calling_convention.GetRegisterAt(0))); + entry_address = temp_is_r0 ? out : temp; + DCHECK(!entry_address.Is(calling_convention.GetRegisterAt(0))); + if (temp_is_r0) { + __ Mov(entry_address, temp); + } } __ Mov(calling_convention.GetRegisterAt(0), string_index.index_); @@ -494,10 +518,13 @@ class LoadStringSlowPathARMVIXL : public SlowPathCodeARMVIXL { __ Str(r0, MemOperand(entry_address)); } else { // For non-Baker read barrier, we need to re-calculate the address of the string entry. + UseScratchRegisterScope temps( + down_cast<CodeGeneratorARMVIXL*>(codegen)->GetVIXLAssembler()); + vixl32::Register temp = temps.Acquire(); CodeGeneratorARMVIXL::PcRelativePatchInfo* labels = arm_codegen->NewPcRelativeStringPatch(load->GetDexFile(), string_index); - arm_codegen->EmitMovwMovtPlaceholder(labels, out); - __ Str(r0, MemOperand(entry_address)); + arm_codegen->EmitMovwMovtPlaceholder(labels, temp); + __ Str(r0, MemOperand(temp)); } arm_codegen->Move32(locations->Out(), LocationFrom(r0)); @@ -4005,8 +4032,11 @@ void LocationsBuilderARMVIXL::VisitNewArray(HNewArray* instruction) { void InstructionCodeGeneratorARMVIXL::VisitNewArray(HNewArray* instruction) { // Note: if heap poisoning is enabled, the entry point takes cares // of poisoning the reference. - codegen_->InvokeRuntime(kQuickAllocArrayResolved, instruction, instruction->GetDexPc()); + QuickEntrypointEnum entrypoint = + CodeGenerator::GetArrayAllocationEntrypoint(instruction->GetLoadClass()->GetClass()); + codegen_->InvokeRuntime(entrypoint, instruction, instruction->GetDexPc()); CheckEntrypointTypes<kQuickAllocArrayResolved, void*, mirror::Class*, int32_t>(); + DCHECK(!codegen_->IsLeafMethod()); } void LocationsBuilderARMVIXL::VisitParameterValue(HParameterValue* instruction) { @@ -5796,6 +5826,9 @@ void ParallelMoveResolverARMVIXL::RestoreScratch(int reg ATTRIBUTE_UNUSED) { HLoadClass::LoadKind CodeGeneratorARMVIXL::GetSupportedLoadClassKind( HLoadClass::LoadKind desired_class_load_kind) { switch (desired_class_load_kind) { + case HLoadClass::LoadKind::kInvalid: + LOG(FATAL) << "UNREACHABLE"; + UNREACHABLE(); case HLoadClass::LoadKind::kReferrersClass: break; case HLoadClass::LoadKind::kBootImageLinkTimeAddress: @@ -5826,6 +5859,7 @@ void LocationsBuilderARMVIXL::VisitLoadClass(HLoadClass* cls) { cls, LocationFrom(calling_convention.GetRegisterAt(0)), LocationFrom(r0)); + DCHECK(calling_convention.GetRegisterAt(0).Is(r0)); return; } DCHECK(!cls->NeedsAccessCheck()); @@ -5843,6 +5877,22 @@ void LocationsBuilderARMVIXL::VisitLoadClass(HLoadClass* cls) { locations->SetInAt(0, Location::RequiresRegister()); } locations->SetOut(Location::RequiresRegister()); + if (load_kind == HLoadClass::LoadKind::kBssEntry) { + if (!kUseReadBarrier || kUseBakerReadBarrier) { + // Rely on the type resolution or initialization and marking to save everything we need. + // Note that IP may be clobbered by saving/restoring the live register (only one thanks + // to the custom calling convention) or by marking, so we request a different temp. + locations->AddTemp(Location::RequiresRegister()); + RegisterSet caller_saves = RegisterSet::Empty(); + InvokeRuntimeCallingConventionARMVIXL calling_convention; + caller_saves.Add(LocationFrom(calling_convention.GetRegisterAt(0))); + // TODO: Add GetReturnLocation() to the calling convention so that we can DCHECK() + // that the the kPrimNot result register is the same as the first argument register. + locations->SetCustomSlowPathCallerSaves(caller_saves); + } else { + // For non-Baker read barrier we have a temp-clobbering call. + } + } } // NO_THREAD_SAFETY_ANALYSIS as we manipulate handles whose internal object we know does not @@ -5900,10 +5950,13 @@ void InstructionCodeGeneratorARMVIXL::VisitLoadClass(HLoadClass* cls) NO_THREAD_ break; } case HLoadClass::LoadKind::kBssEntry: { + vixl32::Register temp = (!kUseReadBarrier || kUseBakerReadBarrier) + ? RegisterFrom(locations->GetTemp(0)) + : out; CodeGeneratorARMVIXL::PcRelativePatchInfo* labels = codegen_->NewTypeBssEntryPatch(cls->GetDexFile(), cls->GetTypeIndex()); - codegen_->EmitMovwMovtPlaceholder(labels, out); - GenerateGcRootFieldLoad(cls, out_loc, out, 0, kCompilerReadBarrierOption); + codegen_->EmitMovwMovtPlaceholder(labels, temp); + GenerateGcRootFieldLoad(cls, out_loc, temp, /* offset */ 0, read_barrier_option); generate_null_check = true; break; } @@ -5912,10 +5965,11 @@ void InstructionCodeGeneratorARMVIXL::VisitLoadClass(HLoadClass* cls) NO_THREAD_ cls->GetTypeIndex(), cls->GetClass())); // /* GcRoot<mirror::Class> */ out = *out - GenerateGcRootFieldLoad(cls, out_loc, out, /* offset */ 0, kCompilerReadBarrierOption); + GenerateGcRootFieldLoad(cls, out_loc, out, /* offset */ 0, read_barrier_option); break; } case HLoadClass::LoadKind::kDexCacheViaMethod: + case HLoadClass::LoadKind::kInvalid: LOG(FATAL) << "UNREACHABLE"; UNREACHABLE(); } @@ -6005,9 +6059,9 @@ void LocationsBuilderARMVIXL::VisitLoadString(HLoadString* load) { locations->SetOut(Location::RequiresRegister()); if (load_kind == HLoadString::LoadKind::kBssEntry) { if (!kUseReadBarrier || kUseBakerReadBarrier) { - // Rely on the pResolveString and/or marking to save everything, including temps. - // Note that IP may theoretically be clobbered by saving/restoring the live register - // (only one thanks to the custom calling convention), so we request a different temp. + // Rely on the pResolveString and marking to save everything we need, including temps. + // Note that IP may be clobbered by saving/restoring the live register (only one thanks + // to the custom calling convention) or by marking, so we request a different temp. locations->AddTemp(Location::RequiresRegister()); RegisterSet caller_saves = RegisterSet::Empty(); InvokeRuntimeCallingConventionARMVIXL calling_convention; @@ -6052,7 +6106,9 @@ void InstructionCodeGeneratorARMVIXL::VisitLoadString(HLoadString* load) NO_THRE } case HLoadString::LoadKind::kBssEntry: { DCHECK(!codegen_->GetCompilerOptions().IsBootImage()); - vixl32::Register temp = RegisterFrom(locations->GetTemp(0)); + vixl32::Register temp = (!kUseReadBarrier || kUseBakerReadBarrier) + ? RegisterFrom(locations->GetTemp(0)) + : out; CodeGeneratorARMVIXL::PcRelativePatchInfo* labels = codegen_->NewPcRelativeStringPatch(load->GetDexFile(), load->GetStringIndex()); codegen_->EmitMovwMovtPlaceholder(labels, temp); @@ -7253,8 +7309,7 @@ vixl32::Register CodeGeneratorARMVIXL::GetInvokeStaticOrDirectExtraParameter( // save one load. However, since this is just an intrinsic slow path we prefer this // simple and more robust approach rather that trying to determine if that's the case. SlowPathCode* slow_path = GetCurrentSlowPath(); - DCHECK(slow_path != nullptr); // For intrinsified invokes the call is emitted on the slow path. - if (slow_path->IsCoreRegisterSaved(RegisterFrom(location).GetCode())) { + if (slow_path != nullptr && slow_path->IsCoreRegisterSaved(RegisterFrom(location).GetCode())) { int stack_offset = slow_path->GetStackOffsetOfCoreRegister(RegisterFrom(location).GetCode()); GetAssembler()->LoadFromOffset(kLoadWord, temp, sp, stack_offset); return temp; diff --git a/compiler/optimizing/code_generator_mips.cc b/compiler/optimizing/code_generator_mips.cc index a095970a1e..c9dde7cc55 100644 --- a/compiler/optimizing/code_generator_mips.cc +++ b/compiler/optimizing/code_generator_mips.cc @@ -484,6 +484,8 @@ CodeGeneratorMIPS::CodeGeneratorMIPS(HGraph* graph, type_bss_entry_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), boot_image_address_patches_(std::less<uint32_t>(), graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + jit_string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + jit_class_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), clobbered_ra_(false) { // Save RA (containing the return address) to mimic Quick. AddAllocatedRegister(Location::RegisterLocation(RA)); @@ -704,9 +706,6 @@ bool CodeGeneratorMIPS::HasAllocatedCalleeSaveRegisters() const { // (this can happen in leaf methods), force CodeGenerator::InitializeCodeGeneration() // into the path that creates a stack frame so that RA can be explicitly saved and restored. // RA can't otherwise be saved/restored when it's the only spilled register. - // TODO: Can this be improved? It causes creation of a stack frame (while RA might be - // saved in an unused temporary register) and saving of RA and the current method pointer - // in the frame. return CodeGenerator::HasAllocatedCalleeSaveRegisters() || clobbered_ra_; } @@ -1160,6 +1159,67 @@ void CodeGeneratorMIPS::EmitPcRelativeAddressPlaceholderHigh(PcRelativePatchInfo // offset to `out` (e.g. lw, jialc, addiu). } +CodeGeneratorMIPS::JitPatchInfo* CodeGeneratorMIPS::NewJitRootStringPatch( + const DexFile& dex_file, + dex::StringIndex dex_index, + Handle<mirror::String> handle) { + jit_string_roots_.Overwrite(StringReference(&dex_file, dex_index), + reinterpret_cast64<uint64_t>(handle.GetReference())); + jit_string_patches_.emplace_back(dex_file, dex_index.index_); + return &jit_string_patches_.back(); +} + +CodeGeneratorMIPS::JitPatchInfo* CodeGeneratorMIPS::NewJitRootClassPatch( + const DexFile& dex_file, + dex::TypeIndex dex_index, + Handle<mirror::Class> handle) { + jit_class_roots_.Overwrite(TypeReference(&dex_file, dex_index), + reinterpret_cast64<uint64_t>(handle.GetReference())); + jit_class_patches_.emplace_back(dex_file, dex_index.index_); + return &jit_class_patches_.back(); +} + +void CodeGeneratorMIPS::PatchJitRootUse(uint8_t* code, + const uint8_t* roots_data, + const CodeGeneratorMIPS::JitPatchInfo& info, + uint64_t index_in_table) const { + uint32_t literal_offset = GetAssembler().GetLabelLocation(&info.high_label); + uintptr_t address = + reinterpret_cast<uintptr_t>(roots_data) + index_in_table * sizeof(GcRoot<mirror::Object>); + uint32_t addr32 = dchecked_integral_cast<uint32_t>(address); + // lui reg, addr32_high + DCHECK_EQ(code[literal_offset + 0], 0x34); + DCHECK_EQ(code[literal_offset + 1], 0x12); + DCHECK_EQ((code[literal_offset + 2] & 0xE0), 0x00); + DCHECK_EQ(code[literal_offset + 3], 0x3C); + // lw reg, reg, addr32_low + DCHECK_EQ(code[literal_offset + 4], 0x78); + DCHECK_EQ(code[literal_offset + 5], 0x56); + DCHECK_EQ((code[literal_offset + 7] & 0xFC), 0x8C); + addr32 += (addr32 & 0x8000) << 1; // Account for sign extension in "lw reg, reg, addr32_low". + // lui reg, addr32_high + code[literal_offset + 0] = static_cast<uint8_t>(addr32 >> 16); + code[literal_offset + 1] = static_cast<uint8_t>(addr32 >> 24); + // lw reg, reg, addr32_low + code[literal_offset + 4] = static_cast<uint8_t>(addr32 >> 0); + code[literal_offset + 5] = static_cast<uint8_t>(addr32 >> 8); +} + +void CodeGeneratorMIPS::EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) { + for (const JitPatchInfo& info : jit_string_patches_) { + const auto& it = jit_string_roots_.find(StringReference(&info.target_dex_file, + dex::StringIndex(info.index))); + DCHECK(it != jit_string_roots_.end()); + PatchJitRootUse(code, roots_data, info, it->second); + } + for (const JitPatchInfo& info : jit_class_patches_) { + const auto& it = jit_class_roots_.find(TypeReference(&info.target_dex_file, + dex::TypeIndex(info.index))); + DCHECK(it != jit_class_roots_.end()); + PatchJitRootUse(code, roots_data, info, it->second); + } +} + void CodeGeneratorMIPS::MarkGCCard(Register object, Register value, bool value_can_be_null) { @@ -1854,6 +1914,8 @@ void InstructionCodeGeneratorMIPS::VisitArrayGet(HArrayGet* instruction) { auto null_checker = GetImplicitNullChecker(instruction); Primitive::Type type = instruction->GetType(); + const bool maybe_compressed_char_at = mirror::kUseStringCompression && + instruction->IsStringCharAt(); switch (type) { case Primitive::kPrimBoolean: { Register out = locations->Out().AsRegister<Register>(); @@ -1897,14 +1959,54 @@ void InstructionCodeGeneratorMIPS::VisitArrayGet(HArrayGet* instruction) { case Primitive::kPrimChar: { Register out = locations->Out().AsRegister<Register>(); + if (maybe_compressed_char_at) { + uint32_t count_offset = mirror::String::CountOffset().Uint32Value(); + __ LoadFromOffset(kLoadWord, TMP, obj, count_offset, null_checker); + __ Sll(TMP, TMP, 31); // Extract compression flag into the most significant bit of TMP. + static_assert(static_cast<uint32_t>(mirror::StringCompressionFlag::kCompressed) == 0u, + "Expecting 0=compressed, 1=uncompressed"); + } if (index.IsConstant()) { - size_t offset = - (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset; - __ LoadFromOffset(kLoadUnsignedHalfword, out, obj, offset, null_checker); + int32_t const_index = index.GetConstant()->AsIntConstant()->GetValue(); + if (maybe_compressed_char_at) { + MipsLabel uncompressed_load, done; + __ Bnez(TMP, &uncompressed_load); + __ LoadFromOffset(kLoadUnsignedByte, + out, + obj, + data_offset + (const_index << TIMES_1)); + __ B(&done); + __ Bind(&uncompressed_load); + __ LoadFromOffset(kLoadUnsignedHalfword, + out, + obj, + data_offset + (const_index << TIMES_2)); + __ Bind(&done); + } else { + __ LoadFromOffset(kLoadUnsignedHalfword, + out, + obj, + data_offset + (const_index << TIMES_2), + null_checker); + } } else { - __ Sll(TMP, index.AsRegister<Register>(), TIMES_2); - __ Addu(TMP, obj, TMP); - __ LoadFromOffset(kLoadUnsignedHalfword, out, TMP, data_offset, null_checker); + Register index_reg = index.AsRegister<Register>(); + if (maybe_compressed_char_at) { + MipsLabel uncompressed_load, done; + __ Bnez(TMP, &uncompressed_load); + __ Addu(TMP, obj, index_reg); + __ LoadFromOffset(kLoadUnsignedByte, out, TMP, data_offset); + __ B(&done); + __ Bind(&uncompressed_load); + __ Sll(TMP, index_reg, TIMES_2); + __ Addu(TMP, obj, TMP); + __ LoadFromOffset(kLoadUnsignedHalfword, out, TMP, data_offset); + __ Bind(&done); + } else { + __ Sll(TMP, index_reg, TIMES_2); + __ Addu(TMP, obj, TMP); + __ LoadFromOffset(kLoadUnsignedHalfword, out, TMP, data_offset, null_checker); + } } break; } @@ -1986,6 +2088,10 @@ void InstructionCodeGeneratorMIPS::VisitArrayLength(HArrayLength* instruction) { Register out = locations->Out().AsRegister<Register>(); __ LoadFromOffset(kLoadWord, out, obj, offset); codegen_->MaybeRecordImplicitNullCheck(instruction); + // Mask out compression flag from String's array length. + if (mirror::kUseStringCompression && instruction->IsStringLength()) { + __ Srl(out, out, 1u); + } } Location LocationsBuilderMIPS::RegisterOrZeroConstant(HInstruction* instruction) { @@ -5225,8 +5331,7 @@ HLoadString::LoadKind CodeGeneratorMIPS::GetSupportedLoadStringKind( break; case HLoadString::LoadKind::kJitTableAddress: DCHECK(Runtime::Current()->UseJitCompilation()); - // TODO: implement. - fallback_load = true; + fallback_load = false; break; case HLoadString::LoadKind::kDexCacheViaMethod: fallback_load = false; @@ -5249,6 +5354,9 @@ HLoadClass::LoadKind CodeGeneratorMIPS::GetSupportedLoadClassKind( bool is_r6 = GetInstructionSetFeatures().IsR6(); bool fallback_load = has_irreducible_loops && !is_r6; switch (desired_class_load_kind) { + case HLoadClass::LoadKind::kInvalid: + LOG(FATAL) << "UNREACHABLE"; + UNREACHABLE(); case HLoadClass::LoadKind::kReferrersClass: fallback_load = false; break; @@ -5265,8 +5373,7 @@ HLoadClass::LoadKind CodeGeneratorMIPS::GetSupportedLoadClassKind( break; case HLoadClass::LoadKind::kJitTableAddress: DCHECK(Runtime::Current()->UseJitCompilation()); - // TODO: implement. - fallback_load = true; + fallback_load = false; break; case HLoadClass::LoadKind::kDexCacheViaMethod: fallback_load = false; @@ -5591,10 +5698,18 @@ void InstructionCodeGeneratorMIPS::VisitLoadClass(HLoadClass* cls) NO_THREAD_SAF break; } case HLoadClass::LoadKind::kJitTableAddress: { - LOG(FATAL) << "Unimplemented"; + CodeGeneratorMIPS::JitPatchInfo* info = codegen_->NewJitRootClassPatch(cls->GetDexFile(), + cls->GetTypeIndex(), + cls->GetClass()); + bool reordering = __ SetReorder(false); + __ Bind(&info->high_label); + __ Lui(out, /* placeholder */ 0x1234); + GenerateGcRootFieldLoad(cls, out_loc, out, /* placeholder */ 0x5678); + __ SetReorder(reordering); break; } case HLoadClass::LoadKind::kDexCacheViaMethod: + case HLoadClass::LoadKind::kInvalid: LOG(FATAL) << "UNREACHABLE"; UNREACHABLE(); } @@ -5730,6 +5845,18 @@ void InstructionCodeGeneratorMIPS::VisitLoadString(HLoadString* load) NO_THREAD_ __ Bind(slow_path->GetExitLabel()); return; } + case HLoadString::LoadKind::kJitTableAddress: { + CodeGeneratorMIPS::JitPatchInfo* info = + codegen_->NewJitRootStringPatch(load->GetDexFile(), + load->GetStringIndex(), + load->GetString()); + bool reordering = __ SetReorder(false); + __ Bind(&info->high_label); + __ Lui(out, /* placeholder */ 0x1234); + GenerateGcRootFieldLoad(load, out_loc, out, /* placeholder */ 0x5678); + __ SetReorder(reordering); + return; + } default: break; } diff --git a/compiler/optimizing/code_generator_mips.h b/compiler/optimizing/code_generator_mips.h index e92eeef88f..47eba50248 100644 --- a/compiler/optimizing/code_generator_mips.h +++ b/compiler/optimizing/code_generator_mips.h @@ -352,6 +352,7 @@ class CodeGeneratorMIPS : public CodeGenerator { // Emit linker patches. void EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) OVERRIDE; + void EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) OVERRIDE; void MarkGCCard(Register object, Register value, bool value_can_be_null); @@ -465,6 +466,31 @@ class CodeGeneratorMIPS : public CodeGenerator { void EmitPcRelativeAddressPlaceholderHigh(PcRelativePatchInfo* info, Register out, Register base); + // The JitPatchInfo is used for JIT string and class loads. + struct JitPatchInfo { + JitPatchInfo(const DexFile& dex_file, uint64_t idx) + : target_dex_file(dex_file), index(idx) { } + JitPatchInfo(JitPatchInfo&& other) = default; + + const DexFile& target_dex_file; + // String/type index. + uint64_t index; + // Label for the instruction loading the most significant half of the address. + // The least significant half is loaded with the instruction that follows immediately. + MipsLabel high_label; + }; + + void PatchJitRootUse(uint8_t* code, + const uint8_t* roots_data, + const JitPatchInfo& info, + uint64_t index_in_table) const; + JitPatchInfo* NewJitRootStringPatch(const DexFile& dex_file, + dex::StringIndex dex_index, + Handle<mirror::String> handle); + JitPatchInfo* NewJitRootClassPatch(const DexFile& dex_file, + dex::TypeIndex dex_index, + Handle<mirror::Class> handle); + private: Register GetInvokeStaticOrDirectExtraParameter(HInvokeStaticOrDirect* invoke, Register temp); @@ -512,6 +538,10 @@ class CodeGeneratorMIPS : public CodeGenerator { ArenaDeque<PcRelativePatchInfo> type_bss_entry_patches_; // Deduplication map for patchable boot image addresses. Uint32ToLiteralMap boot_image_address_patches_; + // Patches for string root accesses in JIT compiled code. + ArenaDeque<JitPatchInfo> jit_string_patches_; + // Patches for class root accesses in JIT compiled code. + ArenaDeque<JitPatchInfo> jit_class_patches_; // PC-relative loads on R2 clobber RA, which may need to be preserved explicitly in leaf methods. // This is a flag set by pc_relative_fixups_mips and dex_cache_array_fixups_mips optimizations. diff --git a/compiler/optimizing/code_generator_mips64.cc b/compiler/optimizing/code_generator_mips64.cc index e96e3d75e1..5be0da4011 100644 --- a/compiler/optimizing/code_generator_mips64.cc +++ b/compiler/optimizing/code_generator_mips64.cc @@ -91,9 +91,6 @@ Location InvokeDexCallingConventionVisitorMIPS64::GetNextLocation(Primitive::Typ // Space on the stack is reserved for all arguments. stack_index_ += Primitive::Is64BitType(type) ? 2 : 1; - // TODO: shouldn't we use a whole machine word per argument on the stack? - // Implicit 4-byte method pointer (and such) will cause misalignment. - return next_location; } @@ -434,7 +431,11 @@ CodeGeneratorMIPS64::CodeGeneratorMIPS64(HGraph* graph, pc_relative_type_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), type_bss_entry_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), boot_image_address_patches_(std::less<uint32_t>(), - graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) { + graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + jit_string_patches_(StringReferenceValueComparator(), + graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + jit_class_patches_(TypeReferenceValueComparator(), + graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) { // Save RA (containing the return address) to mimic Quick. AddAllocatedRegister(Location::RegisterLocation(RA)); } @@ -1055,6 +1056,49 @@ void CodeGeneratorMIPS64::EmitPcRelativeAddressPlaceholderHigh(PcRelativePatchIn // offset to `out` (e.g. ld, jialc, daddiu). } +Literal* CodeGeneratorMIPS64::DeduplicateJitStringLiteral(const DexFile& dex_file, + dex::StringIndex string_index, + Handle<mirror::String> handle) { + jit_string_roots_.Overwrite(StringReference(&dex_file, string_index), + reinterpret_cast64<uint64_t>(handle.GetReference())); + return jit_string_patches_.GetOrCreate( + StringReference(&dex_file, string_index), + [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); +} + +Literal* CodeGeneratorMIPS64::DeduplicateJitClassLiteral(const DexFile& dex_file, + dex::TypeIndex type_index, + Handle<mirror::Class> handle) { + jit_class_roots_.Overwrite(TypeReference(&dex_file, type_index), + reinterpret_cast64<uint64_t>(handle.GetReference())); + return jit_class_patches_.GetOrCreate( + TypeReference(&dex_file, type_index), + [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); +} + +void CodeGeneratorMIPS64::PatchJitRootUse(uint8_t* code, + const uint8_t* roots_data, + const Literal* literal, + uint64_t index_in_table) const { + uint32_t literal_offset = GetAssembler().GetLabelLocation(literal->GetLabel()); + uintptr_t address = + reinterpret_cast<uintptr_t>(roots_data) + index_in_table * sizeof(GcRoot<mirror::Object>); + reinterpret_cast<uint32_t*>(code + literal_offset)[0] = dchecked_integral_cast<uint32_t>(address); +} + +void CodeGeneratorMIPS64::EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) { + for (const auto& entry : jit_string_patches_) { + const auto& it = jit_string_roots_.find(entry.first); + DCHECK(it != jit_string_roots_.end()); + PatchJitRootUse(code, roots_data, entry.second, it->second); + } + for (const auto& entry : jit_class_patches_) { + const auto& it = jit_class_roots_.find(entry.first); + DCHECK(it != jit_class_roots_.end()); + PatchJitRootUse(code, roots_data, entry.second, it->second); + } +} + void CodeGeneratorMIPS64::SetupBlockedRegisters() const { // ZERO, K0, K1, GP, SP, RA are always reserved and can't be allocated. blocked_core_registers_[ZERO] = true; @@ -1446,6 +1490,8 @@ void InstructionCodeGeneratorMIPS64::VisitArrayGet(HArrayGet* instruction) { uint32_t data_offset = CodeGenerator::GetArrayDataOffset(instruction); Primitive::Type type = instruction->GetType(); + const bool maybe_compressed_char_at = mirror::kUseStringCompression && + instruction->IsStringCharAt(); switch (type) { case Primitive::kPrimBoolean: { GpuRegister out = locations->Out().AsRegister<GpuRegister>(); @@ -1489,14 +1535,54 @@ void InstructionCodeGeneratorMIPS64::VisitArrayGet(HArrayGet* instruction) { case Primitive::kPrimChar: { GpuRegister out = locations->Out().AsRegister<GpuRegister>(); + if (maybe_compressed_char_at) { + uint32_t count_offset = mirror::String::CountOffset().Uint32Value(); + __ LoadFromOffset(kLoadWord, TMP, obj, count_offset); + codegen_->MaybeRecordImplicitNullCheck(instruction); + __ Dext(TMP, TMP, 0, 1); + static_assert(static_cast<uint32_t>(mirror::StringCompressionFlag::kCompressed) == 0u, + "Expecting 0=compressed, 1=uncompressed"); + } if (index.IsConstant()) { - size_t offset = - (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset; - __ LoadFromOffset(kLoadUnsignedHalfword, out, obj, offset); + int32_t const_index = index.GetConstant()->AsIntConstant()->GetValue(); + if (maybe_compressed_char_at) { + Mips64Label uncompressed_load, done; + __ Bnezc(TMP, &uncompressed_load); + __ LoadFromOffset(kLoadUnsignedByte, + out, + obj, + data_offset + (const_index << TIMES_1)); + __ Bc(&done); + __ Bind(&uncompressed_load); + __ LoadFromOffset(kLoadUnsignedHalfword, + out, + obj, + data_offset + (const_index << TIMES_2)); + __ Bind(&done); + } else { + __ LoadFromOffset(kLoadUnsignedHalfword, + out, + obj, + data_offset + (const_index << TIMES_2)); + } } else { - __ Dsll(TMP, index.AsRegister<GpuRegister>(), TIMES_2); - __ Daddu(TMP, obj, TMP); - __ LoadFromOffset(kLoadUnsignedHalfword, out, TMP, data_offset); + GpuRegister index_reg = index.AsRegister<GpuRegister>(); + if (maybe_compressed_char_at) { + Mips64Label uncompressed_load, done; + __ Bnezc(TMP, &uncompressed_load); + __ Daddu(TMP, obj, index_reg); + __ LoadFromOffset(kLoadUnsignedByte, out, TMP, data_offset); + __ Bc(&done); + __ Bind(&uncompressed_load); + __ Dsll(TMP, index_reg, TIMES_2); + __ Daddu(TMP, obj, TMP); + __ LoadFromOffset(kLoadUnsignedHalfword, out, TMP, data_offset); + __ Bind(&done); + } else { + __ Dsll(TMP, index_reg, TIMES_2); + __ Daddu(TMP, obj, TMP); + __ LoadFromOffset(kLoadUnsignedHalfword, out, TMP, data_offset); + } } break; } @@ -1564,7 +1650,9 @@ void InstructionCodeGeneratorMIPS64::VisitArrayGet(HArrayGet* instruction) { LOG(FATAL) << "Unreachable type " << instruction->GetType(); UNREACHABLE(); } - codegen_->MaybeRecordImplicitNullCheck(instruction); + if (!maybe_compressed_char_at) { + codegen_->MaybeRecordImplicitNullCheck(instruction); + } } void LocationsBuilderMIPS64::VisitArrayLength(HArrayLength* instruction) { @@ -1580,6 +1668,10 @@ void InstructionCodeGeneratorMIPS64::VisitArrayLength(HArrayLength* instruction) GpuRegister out = locations->Out().AsRegister<GpuRegister>(); __ LoadFromOffset(kLoadWord, out, obj, offset); codegen_->MaybeRecordImplicitNullCheck(instruction); + // Mask out compression flag from String's array length. + if (mirror::kUseStringCompression && instruction->IsStringLength()) { + __ Srl(out, out, 1u); + } } void LocationsBuilderMIPS64::VisitArraySet(HArraySet* instruction) { @@ -3309,8 +3401,6 @@ HLoadString::LoadKind CodeGeneratorMIPS64::GetSupportedLoadStringKind( break; case HLoadString::LoadKind::kJitTableAddress: DCHECK(Runtime::Current()->UseJitCompilation()); - // TODO: implement. - fallback_load = true; break; } if (fallback_load) { @@ -3326,6 +3416,9 @@ HLoadClass::LoadKind CodeGeneratorMIPS64::GetSupportedLoadClassKind( } bool fallback_load = false; switch (desired_class_load_kind) { + case HLoadClass::LoadKind::kInvalid: + LOG(FATAL) << "UNREACHABLE"; + UNREACHABLE(); case HLoadClass::LoadKind::kReferrersClass: break; case HLoadClass::LoadKind::kBootImageLinkTimeAddress: @@ -3341,8 +3434,6 @@ HLoadClass::LoadKind CodeGeneratorMIPS64::GetSupportedLoadClassKind( break; case HLoadClass::LoadKind::kJitTableAddress: DCHECK(Runtime::Current()->UseJitCompilation()); - // TODO: implement. - fallback_load = true; break; case HLoadClass::LoadKind::kDexCacheViaMethod: break; @@ -3580,11 +3671,16 @@ void InstructionCodeGeneratorMIPS64::VisitLoadClass(HLoadClass* cls) NO_THREAD_S generate_null_check = true; break; } - case HLoadClass::LoadKind::kJitTableAddress: { - LOG(FATAL) << "Unimplemented"; + case HLoadClass::LoadKind::kJitTableAddress: + __ LoadLiteral(out, + kLoadUnsignedWord, + codegen_->DeduplicateJitClassLiteral(cls->GetDexFile(), + cls->GetTypeIndex(), + cls->GetClass())); + GenerateGcRootFieldLoad(cls, out_loc, out, 0); break; - } case HLoadClass::LoadKind::kDexCacheViaMethod: + case HLoadClass::LoadKind::kInvalid: LOG(FATAL) << "UNREACHABLE"; UNREACHABLE(); } @@ -3685,6 +3781,14 @@ void InstructionCodeGeneratorMIPS64::VisitLoadString(HLoadString* load) NO_THREA __ Bind(slow_path->GetExitLabel()); return; } + case HLoadString::LoadKind::kJitTableAddress: + __ LoadLiteral(out, + kLoadUnsignedWord, + codegen_->DeduplicateJitStringLiteral(load->GetDexFile(), + load->GetStringIndex(), + load->GetString())); + GenerateGcRootFieldLoad(load, out_loc, out, 0); + return; default: break; } diff --git a/compiler/optimizing/code_generator_mips64.h b/compiler/optimizing/code_generator_mips64.h index 5ba8912134..26cc7dc788 100644 --- a/compiler/optimizing/code_generator_mips64.h +++ b/compiler/optimizing/code_generator_mips64.h @@ -52,7 +52,7 @@ static constexpr size_t kRuntimeParameterFpuRegistersLength = static constexpr GpuRegister kCoreCalleeSaves[] = - { S0, S1, S2, S3, S4, S5, S6, S7, GP, S8, RA }; // TODO: review + { S0, S1, S2, S3, S4, S5, S6, S7, GP, S8, RA }; static constexpr FpuRegister kFpuCalleeSaves[] = { F24, F25, F26, F27, F28, F29, F30, F31 }; @@ -312,6 +312,7 @@ class CodeGeneratorMIPS64 : public CodeGenerator { // Emit linker patches. void EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) OVERRIDE; + void EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) OVERRIDE; void MarkGCCard(GpuRegister object, GpuRegister value, bool value_can_be_null); @@ -425,10 +426,27 @@ class CodeGeneratorMIPS64 : public CodeGenerator { void EmitPcRelativeAddressPlaceholderHigh(PcRelativePatchInfo* info, GpuRegister out); + void PatchJitRootUse(uint8_t* code, + const uint8_t* roots_data, + const Literal* literal, + uint64_t index_in_table) const; + Literal* DeduplicateJitStringLiteral(const DexFile& dex_file, + dex::StringIndex string_index, + Handle<mirror::String> handle); + Literal* DeduplicateJitClassLiteral(const DexFile& dex_file, + dex::TypeIndex type_index, + Handle<mirror::Class> handle); + private: using Uint32ToLiteralMap = ArenaSafeMap<uint32_t, Literal*>; using Uint64ToLiteralMap = ArenaSafeMap<uint64_t, Literal*>; using MethodToLiteralMap = ArenaSafeMap<MethodReference, Literal*, MethodReferenceComparator>; + using StringToLiteralMap = ArenaSafeMap<StringReference, + Literal*, + StringReferenceValueComparator>; + using TypeToLiteralMap = ArenaSafeMap<TypeReference, + Literal*, + TypeReferenceValueComparator>; using BootStringToLiteralMap = ArenaSafeMap<StringReference, Literal*, StringReferenceValueComparator>; @@ -476,6 +494,10 @@ class CodeGeneratorMIPS64 : public CodeGenerator { ArenaDeque<PcRelativePatchInfo> type_bss_entry_patches_; // Deduplication map for patchable boot image addresses. Uint32ToLiteralMap boot_image_address_patches_; + // Patches for string root accesses in JIT compiled code. + StringToLiteralMap jit_string_patches_; + // Patches for class root accesses in JIT compiled code. + TypeToLiteralMap jit_class_patches_; DISALLOW_COPY_AND_ASSIGN(CodeGeneratorMIPS64); }; diff --git a/compiler/optimizing/code_generator_x86.cc b/compiler/optimizing/code_generator_x86.cc index 1b7431612d..09612c8dbf 100644 --- a/compiler/optimizing/code_generator_x86.cc +++ b/compiler/optimizing/code_generator_x86.cc @@ -4214,7 +4214,9 @@ void LocationsBuilderX86::VisitNewArray(HNewArray* instruction) { void InstructionCodeGeneratorX86::VisitNewArray(HNewArray* instruction) { // Note: if heap poisoning is enabled, the entry point takes cares // of poisoning the reference. - codegen_->InvokeRuntime(kQuickAllocArrayResolved, instruction, instruction->GetDexPc()); + QuickEntrypointEnum entrypoint = + CodeGenerator::GetArrayAllocationEntrypoint(instruction->GetLoadClass()->GetClass()); + codegen_->InvokeRuntime(entrypoint, instruction, instruction->GetDexPc()); CheckEntrypointTypes<kQuickAllocArrayResolved, void*, mirror::Class*, int32_t>(); DCHECK(!codegen_->IsLeafMethod()); } @@ -6022,6 +6024,9 @@ void ParallelMoveResolverX86::RestoreScratch(int reg) { HLoadClass::LoadKind CodeGeneratorX86::GetSupportedLoadClassKind( HLoadClass::LoadKind desired_class_load_kind) { switch (desired_class_load_kind) { + case HLoadClass::LoadKind::kInvalid: + LOG(FATAL) << "UNREACHABLE"; + UNREACHABLE(); case HLoadClass::LoadKind::kReferrersClass: break; case HLoadClass::LoadKind::kBootImageLinkTimeAddress: @@ -6052,6 +6057,7 @@ void LocationsBuilderX86::VisitLoadClass(HLoadClass* cls) { cls, Location::RegisterLocation(calling_convention.GetRegisterAt(0)), Location::RegisterLocation(EAX)); + DCHECK_EQ(calling_convention.GetRegisterAt(0), EAX); return; } DCHECK(!cls->NeedsAccessCheck()); @@ -6071,6 +6077,17 @@ void LocationsBuilderX86::VisitLoadClass(HLoadClass* cls) { locations->SetInAt(0, Location::RequiresRegister()); } locations->SetOut(Location::RequiresRegister()); + if (load_kind == HLoadClass::LoadKind::kBssEntry) { + if (!kUseReadBarrier || kUseBakerReadBarrier) { + // Rely on the type resolution and/or initialization to save everything. + RegisterSet caller_saves = RegisterSet::Empty(); + InvokeRuntimeCallingConvention calling_convention; + caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); + locations->SetCustomSlowPathCallerSaves(caller_saves); + } else { + // For non-Baker read barrier we have a temp-clobbering call. + } + } } Label* CodeGeneratorX86::NewJitRootClassPatch(const DexFile& dex_file, @@ -6153,10 +6170,11 @@ void InstructionCodeGeneratorX86::VisitLoadClass(HLoadClass* cls) NO_THREAD_SAFE Label* fixup_label = codegen_->NewJitRootClassPatch( cls->GetDexFile(), cls->GetTypeIndex(), cls->GetClass()); // /* GcRoot<mirror::Class> */ out = *address - GenerateGcRootFieldLoad(cls, out_loc, address, fixup_label, kCompilerReadBarrierOption); + GenerateGcRootFieldLoad(cls, out_loc, address, fixup_label, read_barrier_option); break; } case HLoadClass::LoadKind::kDexCacheViaMethod: + case HLoadClass::LoadKind::kInvalid: LOG(FATAL) << "UNREACHABLE"; UNREACHABLE(); } @@ -6244,7 +6262,7 @@ void LocationsBuilderX86::VisitLoadString(HLoadString* load) { locations->SetOut(Location::RequiresRegister()); if (load_kind == HLoadString::LoadKind::kBssEntry) { if (!kUseReadBarrier || kUseBakerReadBarrier) { - // Rely on the pResolveString and/or marking to save everything. + // Rely on the pResolveString to save everything. RegisterSet caller_saves = RegisterSet::Empty(); InvokeRuntimeCallingConvention calling_convention; caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); @@ -7130,9 +7148,10 @@ void InstructionCodeGeneratorX86::GenerateGcRootFieldLoad( // Fast path implementation of art::ReadBarrier::BarrierForRoot when // Baker's read barrier are used: // - // root = *address; - // if (Thread::Current()->GetIsGcMarking()) { - // root = ReadBarrier::Mark(root) + // root = obj.field; + // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg() + // if (temp != null) { + // root = temp(root) // } // /* GcRoot<mirror::Object> */ root = *address @@ -7153,8 +7172,11 @@ void InstructionCodeGeneratorX86::GenerateGcRootFieldLoad( instruction, root, /* unpoison_ref_before_marking */ false); codegen_->AddSlowPath(slow_path); - __ fs()->cmpl(Address::Absolute(Thread::IsGcMarkingOffset<kX86PointerSize>().Int32Value()), - Immediate(0)); + // Test the entrypoint (`Thread::Current()->pReadBarrierMarkReg ## root.reg()`). + const int32_t entry_point_offset = + CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kX86PointerSize>(root.reg()); + __ fs()->cmpl(Address::Absolute(entry_point_offset), Immediate(0)); + // The entrypoint is null when the GC is not marking. __ j(kNotEqual, slow_path->GetEntryLabel()); __ Bind(slow_path->GetExitLabel()); } else { diff --git a/compiler/optimizing/code_generator_x86_64.cc b/compiler/optimizing/code_generator_x86_64.cc index abd8246325..0879992e32 100644 --- a/compiler/optimizing/code_generator_x86_64.cc +++ b/compiler/optimizing/code_generator_x86_64.cc @@ -245,9 +245,8 @@ class LoadClassSlowPathX86_64 : public SlowPathCode { SaveLiveRegisters(codegen, locations); - InvokeRuntimeCallingConvention calling_convention; - __ movl(CpuRegister(calling_convention.GetRegisterAt(0)), - Immediate(cls_->GetTypeIndex().index_)); + // Custom calling convention: RAX serves as both input and output. + __ movl(CpuRegister(RAX), Immediate(cls_->GetTypeIndex().index_)); x86_64_codegen->InvokeRuntime(do_clinit_ ? kQuickInitializeStaticStorage : kQuickInitializeType, instruction_, dex_pc_, @@ -5427,6 +5426,9 @@ void InstructionCodeGeneratorX86_64::GenerateClassInitializationCheck( HLoadClass::LoadKind CodeGeneratorX86_64::GetSupportedLoadClassKind( HLoadClass::LoadKind desired_class_load_kind) { switch (desired_class_load_kind) { + case HLoadClass::LoadKind::kInvalid: + LOG(FATAL) << "UNREACHABLE"; + UNREACHABLE(); case HLoadClass::LoadKind::kReferrersClass: break; case HLoadClass::LoadKind::kBootImageLinkTimeAddress: @@ -5453,10 +5455,10 @@ HLoadClass::LoadKind CodeGeneratorX86_64::GetSupportedLoadClassKind( void LocationsBuilderX86_64::VisitLoadClass(HLoadClass* cls) { HLoadClass::LoadKind load_kind = cls->GetLoadKind(); if (load_kind == HLoadClass::LoadKind::kDexCacheViaMethod) { - InvokeRuntimeCallingConvention calling_convention; + // Custom calling convention: RAX serves as both input and output. CodeGenerator::CreateLoadClassRuntimeCallLocationSummary( cls, - Location::RegisterLocation(calling_convention.GetRegisterAt(0)), + Location::RegisterLocation(RAX), Location::RegisterLocation(RAX)); return; } @@ -5475,6 +5477,17 @@ void LocationsBuilderX86_64::VisitLoadClass(HLoadClass* cls) { locations->SetInAt(0, Location::RequiresRegister()); } locations->SetOut(Location::RequiresRegister()); + if (load_kind == HLoadClass::LoadKind::kBssEntry) { + if (!kUseReadBarrier || kUseBakerReadBarrier) { + // Rely on the type resolution and/or initialization to save everything. + // Custom calling convention: RAX serves as both input and output. + RegisterSet caller_saves = RegisterSet::Empty(); + caller_saves.Add(Location::RegisterLocation(RAX)); + locations->SetCustomSlowPathCallerSaves(caller_saves); + } else { + // For non-Baker read barrier we have a temp-clobbering call. + } + } } Label* CodeGeneratorX86_64::NewJitRootClassPatch(const DexFile& dex_file, @@ -5550,7 +5563,7 @@ void InstructionCodeGeneratorX86_64::VisitLoadClass(HLoadClass* cls) NO_THREAD_S Label* fixup_label = codegen_->NewJitRootClassPatch(cls->GetDexFile(), cls->GetTypeIndex(), cls->GetClass()); // /* GcRoot<mirror::Class> */ out = *address - GenerateGcRootFieldLoad(cls, out_loc, address, fixup_label, kCompilerReadBarrierOption); + GenerateGcRootFieldLoad(cls, out_loc, address, fixup_label, read_barrier_option); break; } default: @@ -5626,7 +5639,7 @@ void LocationsBuilderX86_64::VisitLoadString(HLoadString* load) { locations->SetOut(Location::RequiresRegister()); if (load->GetLoadKind() == HLoadString::LoadKind::kBssEntry) { if (!kUseReadBarrier || kUseBakerReadBarrier) { - // Rely on the pResolveString and/or marking to save everything. + // Rely on the pResolveString to save everything. // Custom calling convention: RAX serves as both input and output. RegisterSet caller_saves = RegisterSet::Empty(); caller_saves.Add(Location::RegisterLocation(RAX)); @@ -6498,9 +6511,10 @@ void InstructionCodeGeneratorX86_64::GenerateGcRootFieldLoad( // Fast path implementation of art::ReadBarrier::BarrierForRoot when // Baker's read barrier are used: // - // root = *address; - // if (Thread::Current()->GetIsGcMarking()) { - // root = ReadBarrier::Mark(root) + // root = obj.field; + // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg() + // if (temp != null) { + // root = temp(root) // } // /* GcRoot<mirror::Object> */ root = *address @@ -6521,9 +6535,11 @@ void InstructionCodeGeneratorX86_64::GenerateGcRootFieldLoad( instruction, root, /* unpoison_ref_before_marking */ false); codegen_->AddSlowPath(slow_path); - __ gs()->cmpl(Address::Absolute(Thread::IsGcMarkingOffset<kX86_64PointerSize>().Int32Value(), - /* no_rip */ true), - Immediate(0)); + // Test the `Thread::Current()->pReadBarrierMarkReg ## root.reg()` entrypoint. + const int32_t entry_point_offset = + CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kX86_64PointerSize>(root.reg()); + __ gs()->cmpl(Address::Absolute(entry_point_offset, /* no_rip */ true), Immediate(0)); + // The entrypoint is null when the GC is not marking. __ j(kNotEqual, slow_path->GetEntryLabel()); __ Bind(slow_path->GetExitLabel()); } else { diff --git a/compiler/optimizing/inliner.cc b/compiler/optimizing/inliner.cc index 7772e8f973..b56ef0f866 100644 --- a/compiler/optimizing/inliner.cc +++ b/compiler/optimizing/inliner.cc @@ -198,9 +198,9 @@ static uint32_t FindMethodIndexIn(ArtMethod* method, } static dex::TypeIndex FindClassIndexIn(mirror::Class* cls, - const DexFile& dex_file, - Handle<mirror::DexCache> dex_cache) + const DexCompilationUnit& compilation_unit) REQUIRES_SHARED(Locks::mutator_lock_) { + const DexFile& dex_file = *compilation_unit.GetDexFile(); dex::TypeIndex index; if (cls->GetDexCache() == nullptr) { DCHECK(cls->IsArrayClass()) << cls->PrettyClass(); @@ -209,22 +209,19 @@ static dex::TypeIndex FindClassIndexIn(mirror::Class* cls, DCHECK(cls->IsProxyClass()) << cls->PrettyClass(); // TODO: deal with proxy classes. } else if (IsSameDexFile(cls->GetDexFile(), dex_file)) { - DCHECK_EQ(cls->GetDexCache(), dex_cache.Get()); + DCHECK_EQ(cls->GetDexCache(), compilation_unit.GetDexCache().Get()); index = cls->GetDexTypeIndex(); - // Update the dex cache to ensure the class is in. The generated code will - // consider it is. We make it safe by updating the dex cache, as other - // dex files might also load the class, and there is no guarantee the dex - // cache of the dex file of the class will be updated. - if (dex_cache->GetResolvedType(index) == nullptr) { - dex_cache->SetResolvedType(index, cls); - } } else { index = cls->FindTypeIndexInOtherDexFile(dex_file); - // We cannot guarantee the entry in the dex cache will resolve to the same class, + // We cannot guarantee the entry will resolve to the same class, // as there may be different class loaders. So only return the index if it's - // the right class in the dex cache already. - if (index.IsValid() && dex_cache->GetResolvedType(index) != cls) { - index = dex::TypeIndex::Invalid(); + // the right class already resolved with the class loader. + if (index.IsValid()) { + ObjPtr<mirror::Class> resolved = ClassLinker::LookupResolvedType( + index, compilation_unit.GetDexCache().Get(), compilation_unit.GetClassLoader().Get()); + if (resolved != cls) { + index = dex::TypeIndex::Invalid(); + } } } @@ -451,9 +448,8 @@ bool HInliner::TryInlineMonomorphicCall(HInvoke* invoke_instruction, DCHECK(invoke_instruction->IsInvokeVirtual() || invoke_instruction->IsInvokeInterface()) << invoke_instruction->DebugName(); - const DexFile& caller_dex_file = *caller_compilation_unit_.GetDexFile(); dex::TypeIndex class_index = FindClassIndexIn( - GetMonomorphicType(classes), caller_dex_file, caller_compilation_unit_.GetDexCache()); + GetMonomorphicType(classes), caller_compilation_unit_); if (!class_index.IsValid()) { VLOG(compiler) << "Call to " << ArtMethod::PrettyMethod(resolved_method) << " from inline cache is not inlined because its class is not" @@ -496,6 +492,7 @@ bool HInliner::TryInlineMonomorphicCall(HInvoke* invoke_instruction, // Run type propagation to get the guard typed, and eventually propagate the // type of the receiver. ReferenceTypePropagation rtp_fixup(graph_, + outer_compilation_unit_.GetClassLoader(), outer_compilation_unit_.GetDexCache(), handles_, /* is_first_run */ false); @@ -558,9 +555,13 @@ HInstruction* HInliner::AddTypeGuard(HInstruction* receiver, is_referrer, invoke_instruction->GetDexPc(), /* needs_access_check */ false); + HLoadClass::LoadKind kind = HSharpening::SharpenClass( + load_class, codegen_, compiler_driver_, caller_compilation_unit_); + DCHECK(kind != HLoadClass::LoadKind::kInvalid) + << "We should always be able to reference a class for inline caches"; + // Insert before setting the kind, as setting the kind affects the inputs. bb_cursor->InsertInstructionAfter(load_class, receiver_class); - // Sharpen after adding the instruction, as the sharpening may remove inputs. - HSharpening::SharpenClass(load_class, codegen_, compiler_driver_); + load_class->SetLoadKind(kind); // TODO: Extend reference type propagation to understand the guard. HNotEqual* compare = new (graph_->GetArena()) HNotEqual(load_class, receiver_class); @@ -586,7 +587,6 @@ bool HInliner::TryInlinePolymorphicCall(HInvoke* invoke_instruction, ClassLinker* class_linker = caller_compilation_unit_.GetClassLinker(); PointerSize pointer_size = class_linker->GetImagePointerSize(); - const DexFile& caller_dex_file = *caller_compilation_unit_.GetDexFile(); bool all_targets_inlined = true; bool one_target_inlined = false; @@ -608,8 +608,7 @@ bool HInliner::TryInlinePolymorphicCall(HInvoke* invoke_instruction, HInstruction* cursor = invoke_instruction->GetPrevious(); HBasicBlock* bb_cursor = invoke_instruction->GetBlock(); - dex::TypeIndex class_index = FindClassIndexIn( - handle.Get(), caller_dex_file, caller_compilation_unit_.GetDexCache()); + dex::TypeIndex class_index = FindClassIndexIn(handle.Get(), caller_compilation_unit_); HInstruction* return_replacement = nullptr; if (!class_index.IsValid() || !TryBuildAndInline(invoke_instruction, @@ -665,6 +664,7 @@ bool HInliner::TryInlinePolymorphicCall(HInvoke* invoke_instruction, // Run type propagation to get the guards typed. ReferenceTypePropagation rtp_fixup(graph_, + outer_compilation_unit_.GetClassLoader(), outer_compilation_unit_.GetDexCache(), handles_, /* is_first_run */ false); @@ -859,6 +859,7 @@ bool HInliner::TryInlinePolymorphicCallToSameTarget( // Run type propagation to get the guard typed. ReferenceTypePropagation rtp_fixup(graph_, + outer_compilation_unit_.GetClassLoader(), outer_compilation_unit_.GetDexCache(), handles_, /* is_first_run */ false); @@ -927,6 +928,7 @@ bool HInliner::TryInlineAndReplace(HInvoke* invoke_instruction, // Actual return value has a more specific type than the method's declared // return type. Run RTP again on the outer graph to propagate it. ReferenceTypePropagation(graph_, + outer_compilation_unit_.GetClassLoader(), outer_compilation_unit_.GetDexCache(), handles_, /* is_first_run */ false).Run(); @@ -1179,7 +1181,11 @@ HInstanceFieldGet* HInliner::CreateInstanceFieldGet(Handle<mirror::DexCache> dex /* dex_pc */ 0); if (iget->GetType() == Primitive::kPrimNot) { // Use the same dex_cache that we used for field lookup as the hint_dex_cache. - ReferenceTypePropagation rtp(graph_, dex_cache, handles_, /* is_first_run */ false); + ReferenceTypePropagation rtp(graph_, + outer_compilation_unit_.GetClassLoader(), + dex_cache, + handles_, + /* is_first_run */ false); rtp.Visit(iget); } return iget; @@ -1225,7 +1231,7 @@ bool HInliner::TryBuildAndInlineHelper(HInvoke* invoke_instruction, resolved_method->GetDeclaringClass()->GetClassLoader())); DexCompilationUnit dex_compilation_unit( - class_loader.ToJObject(), + class_loader, class_linker, callee_dex_file, code_item, @@ -1286,6 +1292,7 @@ bool HInliner::TryBuildAndInlineHelper(HInvoke* invoke_instruction, resolved_method->GetDexFile(), *code_item, compiler_driver_, + codegen_, inline_stats.get(), resolved_method->GetQuickenedInfo(class_linker->GetImagePointerSize()), dex_cache, @@ -1341,6 +1348,7 @@ bool HInliner::TryBuildAndInlineHelper(HInvoke* invoke_instruction, // are more specific than the declared ones, run RTP again on the inner graph. if (run_rtp || ArgumentTypesMoreSpecific(invoke_instruction, resolved_method)) { ReferenceTypePropagation(callee_graph, + outer_compilation_unit_.GetClassLoader(), dex_compilation_unit.GetDexCache(), handles_, /* is_first_run */ false).Run(); @@ -1416,10 +1424,13 @@ bool HInliner::TryBuildAndInlineHelper(HInvoke* invoke_instruction, return false; } - if (!same_dex_file && current->NeedsEnvironment()) { + if (current->NeedsEnvironment() && + !CanEncodeInlinedMethodInStackMap(*caller_compilation_unit_.GetDexFile(), + resolved_method)) { VLOG(compiler) << "Method " << callee_dex_file.PrettyMethod(method_index) << " could not be inlined because " << current->DebugName() - << " needs an environment and is in a different dex file"; + << " needs an environment, is in a different dex file" + << ", and cannot be encoded in the stack maps."; return false; } diff --git a/compiler/optimizing/instruction_builder.cc b/compiler/optimizing/instruction_builder.cc index cac385ce3c..3aaf2ca102 100644 --- a/compiler/optimizing/instruction_builder.cc +++ b/compiler/optimizing/instruction_builder.cc @@ -22,6 +22,7 @@ #include "dex_instruction-inl.h" #include "driver/compiler_options.h" #include "imtable-inl.h" +#include "sharpening.h" #include "scoped_thread_state_change-inl.h" namespace art { @@ -668,11 +669,10 @@ static InvokeType GetInvokeTypeFromOpCode(Instruction::Code opcode) { ArtMethod* HInstructionBuilder::ResolveMethod(uint16_t method_idx, InvokeType invoke_type) { ScopedObjectAccess soa(Thread::Current()); - StackHandleScope<3> hs(soa.Self()); + StackHandleScope<2> hs(soa.Self()); ClassLinker* class_linker = dex_compilation_unit_->GetClassLinker(); - Handle<mirror::ClassLoader> class_loader(hs.NewHandle( - soa.Decode<mirror::ClassLoader>(dex_compilation_unit_->GetClassLoader()))); + Handle<mirror::ClassLoader> class_loader = dex_compilation_unit_->GetClassLoader(); Handle<mirror::Class> compiling_class(hs.NewHandle(GetCompilingClass())); // We fetch the referenced class eagerly (that is, the class pointed by in the MethodId // at method_idx), as `CanAccessResolvedMethod` expects it be be in the dex cache. @@ -847,7 +847,7 @@ bool HInstructionBuilder::BuildInvoke(const Instruction& instruction, ScopedObjectAccess soa(Thread::Current()); if (invoke_type == kStatic) { clinit_check = ProcessClinitCheckForInvoke( - dex_pc, resolved_method, method_idx, &clinit_check_requirement); + dex_pc, resolved_method, &clinit_check_requirement); } else if (invoke_type == kSuper) { if (IsSameDexFile(*resolved_method->GetDexFile(), *dex_compilation_unit_->GetDexFile())) { // Update the method index to the one resolved. Note that this may be a no-op if @@ -933,15 +933,8 @@ bool HInstructionBuilder::BuildInvokePolymorphic(const Instruction& instruction bool HInstructionBuilder::BuildNewInstance(dex::TypeIndex type_index, uint32_t dex_pc) { ScopedObjectAccess soa(Thread::Current()); - Handle<mirror::DexCache> dex_cache = dex_compilation_unit_->GetDexCache(); - Handle<mirror::DexCache> outer_dex_cache = outer_compilation_unit_->GetDexCache(); - if (outer_dex_cache.Get() != dex_cache.Get()) { - // We currently do not support inlining allocations across dex files. - return false; - } - - HLoadClass* load_class = BuildLoadClass(type_index, dex_pc, /* check_access */ true); + HLoadClass* load_class = BuildLoadClass(type_index, dex_pc); HInstruction* cls = load_class; Handle<mirror::Class> klass = load_class->GetClass(); @@ -1005,39 +998,23 @@ bool HInstructionBuilder::IsInitialized(Handle<mirror::Class> cls) const { HClinitCheck* HInstructionBuilder::ProcessClinitCheckForInvoke( uint32_t dex_pc, ArtMethod* resolved_method, - uint32_t method_idx, HInvokeStaticOrDirect::ClinitCheckRequirement* clinit_check_requirement) { - Thread* self = Thread::Current(); - StackHandleScope<2> hs(self); - Handle<mirror::DexCache> dex_cache = dex_compilation_unit_->GetDexCache(); - Handle<mirror::DexCache> outer_dex_cache = outer_compilation_unit_->GetDexCache(); - Handle<mirror::Class> outer_class(hs.NewHandle(GetOutermostCompilingClass())); - Handle<mirror::Class> resolved_method_class(hs.NewHandle(resolved_method->GetDeclaringClass())); - - // The index at which the method's class is stored in the DexCache's type array. - dex::TypeIndex storage_index; - bool is_outer_class = (resolved_method->GetDeclaringClass() == outer_class.Get()); - if (is_outer_class) { - storage_index = outer_class->GetDexTypeIndex(); - } else if (outer_dex_cache.Get() == dex_cache.Get()) { - // Get `storage_index` from IsClassOfStaticMethodAvailableToReferrer. - compiler_driver_->IsClassOfStaticMethodAvailableToReferrer(outer_dex_cache.Get(), - GetCompilingClass(), - resolved_method, - method_idx, - &storage_index); - } + Handle<mirror::Class> klass = handles_->NewHandle(resolved_method->GetDeclaringClass()); HClinitCheck* clinit_check = nullptr; - - if (IsInitialized(resolved_method_class)) { + if (IsInitialized(klass)) { *clinit_check_requirement = HInvokeStaticOrDirect::ClinitCheckRequirement::kNone; - } else if (storage_index.IsValid()) { - *clinit_check_requirement = HInvokeStaticOrDirect::ClinitCheckRequirement::kExplicit; - HLoadClass* cls = BuildLoadClass( - storage_index, dex_pc, /* check_access */ false, /* outer */ true); - clinit_check = new (arena_) HClinitCheck(cls, dex_pc); - AppendInstruction(clinit_check); + } else { + HLoadClass* cls = BuildLoadClass(klass->GetDexTypeIndex(), + klass->GetDexFile(), + klass, + dex_pc, + /* needs_access_check */ false); + if (cls != nullptr) { + *clinit_check_requirement = HInvokeStaticOrDirect::ClinitCheckRequirement::kExplicit; + clinit_check = new (arena_) HClinitCheck(cls, dex_pc); + AppendInstruction(clinit_check); + } } return clinit_check; } @@ -1216,9 +1193,7 @@ bool HInstructionBuilder::BuildInstanceFieldAccess(const Instruction& instructio } ScopedObjectAccess soa(Thread::Current()); - ArtField* resolved_field = - compiler_driver_->ComputeInstanceFieldInfo(field_index, dex_compilation_unit_, is_put, soa); - + ArtField* resolved_field = ResolveField(field_index, /* is_static */ false, is_put); // Generate an explicit null check on the reference, unless the field access // is unresolved. In that case, we rely on the runtime to perform various @@ -1284,9 +1259,7 @@ bool HInstructionBuilder::BuildInstanceFieldAccess(const Instruction& instructio static mirror::Class* GetClassFrom(CompilerDriver* driver, const DexCompilationUnit& compilation_unit) { ScopedObjectAccess soa(Thread::Current()); - StackHandleScope<1> hs(soa.Self()); - Handle<mirror::ClassLoader> class_loader(hs.NewHandle( - soa.Decode<mirror::ClassLoader>(compilation_unit.GetClassLoader()))); + Handle<mirror::ClassLoader> class_loader = compilation_unit.GetClassLoader(); Handle<mirror::DexCache> dex_cache = compilation_unit.GetDexCache(); return driver->ResolveCompilingMethodsClass(soa, dex_cache, class_loader, &compilation_unit); @@ -1302,10 +1275,9 @@ mirror::Class* HInstructionBuilder::GetCompilingClass() const { bool HInstructionBuilder::IsOutermostCompilingClass(dex::TypeIndex type_index) const { ScopedObjectAccess soa(Thread::Current()); - StackHandleScope<3> hs(soa.Self()); + StackHandleScope<2> hs(soa.Self()); Handle<mirror::DexCache> dex_cache = dex_compilation_unit_->GetDexCache(); - Handle<mirror::ClassLoader> class_loader(hs.NewHandle( - soa.Decode<mirror::ClassLoader>(dex_compilation_unit_->GetClassLoader()))); + Handle<mirror::ClassLoader> class_loader = dex_compilation_unit_->GetClassLoader(); Handle<mirror::Class> cls(hs.NewHandle(compiler_driver_->ResolveClass( soa, dex_cache, class_loader, type_index, dex_compilation_unit_))); Handle<mirror::Class> outer_class(hs.NewHandle(GetOutermostCompilingClass())); @@ -1336,6 +1308,55 @@ void HInstructionBuilder::BuildUnresolvedStaticFieldAccess(const Instruction& in } } +ArtField* HInstructionBuilder::ResolveField(uint16_t field_idx, bool is_static, bool is_put) { + ScopedObjectAccess soa(Thread::Current()); + StackHandleScope<2> hs(soa.Self()); + + ClassLinker* class_linker = dex_compilation_unit_->GetClassLinker(); + Handle<mirror::ClassLoader> class_loader = dex_compilation_unit_->GetClassLoader(); + Handle<mirror::Class> compiling_class(hs.NewHandle(GetCompilingClass())); + + ArtField* resolved_field = class_linker->ResolveField(*dex_compilation_unit_->GetDexFile(), + field_idx, + dex_compilation_unit_->GetDexCache(), + class_loader, + is_static); + + if (UNLIKELY(resolved_field == nullptr)) { + // Clean up any exception left by type resolution. + soa.Self()->ClearException(); + return nullptr; + } + + // Check static/instance. The class linker has a fast path for looking into the dex cache + // and does not check static/instance if it hits it. + if (UNLIKELY(resolved_field->IsStatic() != is_static)) { + return nullptr; + } + + // Check access. + if (compiling_class.Get() == nullptr) { + if (!resolved_field->IsPublic()) { + return nullptr; + } + } else if (!compiling_class->CanAccessResolvedField(resolved_field->GetDeclaringClass(), + resolved_field, + dex_compilation_unit_->GetDexCache().Get(), + field_idx)) { + return nullptr; + } + + if (is_put && + resolved_field->IsFinal() && + (compiling_class.Get() != resolved_field->GetDeclaringClass())) { + // Final fields can only be updated within their own class. + // TODO: Only allow it in constructors. b/34966607. + return nullptr; + } + + return resolved_field; +} + bool HInstructionBuilder::BuildStaticFieldAccess(const Instruction& instruction, uint32_t dex_pc, bool is_put) { @@ -1343,12 +1364,7 @@ bool HInstructionBuilder::BuildStaticFieldAccess(const Instruction& instruction, uint16_t field_index = instruction.VRegB_21c(); ScopedObjectAccess soa(Thread::Current()); - StackHandleScope<3> hs(soa.Self()); - Handle<mirror::DexCache> dex_cache = dex_compilation_unit_->GetDexCache(); - Handle<mirror::ClassLoader> class_loader(hs.NewHandle( - soa.Decode<mirror::ClassLoader>(dex_compilation_unit_->GetClassLoader()))); - ArtField* resolved_field = compiler_driver_->ResolveField( - soa, dex_cache, class_loader, dex_compilation_unit_, field_index, true); + ArtField* resolved_field = ResolveField(field_index, /* is_static */ true, is_put); if (resolved_field == nullptr) { MaybeRecordStat(MethodCompilationStat::kUnresolvedField); @@ -1358,38 +1374,23 @@ bool HInstructionBuilder::BuildStaticFieldAccess(const Instruction& instruction, } Primitive::Type field_type = resolved_field->GetTypeAsPrimitiveType(); - Handle<mirror::DexCache> outer_dex_cache = outer_compilation_unit_->GetDexCache(); - Handle<mirror::Class> outer_class(hs.NewHandle(GetOutermostCompilingClass())); - // The index at which the field's class is stored in the DexCache's type array. - dex::TypeIndex storage_index; - bool is_outer_class = (outer_class.Get() == resolved_field->GetDeclaringClass()); - if (is_outer_class) { - storage_index = outer_class->GetDexTypeIndex(); - } else if (outer_dex_cache.Get() != dex_cache.Get()) { - // The compiler driver cannot currently understand multiple dex caches involved. Just bailout. - return false; - } else { - // TODO: This is rather expensive. Perf it and cache the results if needed. - std::pair<bool, bool> pair = compiler_driver_->IsFastStaticField( - outer_dex_cache.Get(), - GetCompilingClass(), - resolved_field, - field_index, - &storage_index); - bool can_easily_access = is_put ? pair.second : pair.first; - if (!can_easily_access) { - MaybeRecordStat(MethodCompilationStat::kUnresolvedFieldNotAFastAccess); - BuildUnresolvedStaticFieldAccess(instruction, dex_pc, is_put, field_type); - return true; - } + Handle<mirror::Class> klass = handles_->NewHandle(resolved_field->GetDeclaringClass()); + HLoadClass* constant = BuildLoadClass(klass->GetDexTypeIndex(), + klass->GetDexFile(), + klass, + dex_pc, + /* needs_access_check */ false); + + if (constant == nullptr) { + // The class cannot be referenced from this compiled code. Generate + // an unresolved access. + MaybeRecordStat(MethodCompilationStat::kUnresolvedFieldNotAFastAccess); + BuildUnresolvedStaticFieldAccess(instruction, dex_pc, is_put, field_type); + return true; } - HLoadClass* constant = BuildLoadClass( - storage_index, dex_pc, /* check_access */ false, /* outer */ true); - HInstruction* cls = constant; - Handle<mirror::Class> klass(hs.NewHandle(resolved_field->GetDeclaringClass())); if (!IsInitialized(klass)) { cls = new (arena_) HClinitCheck(constant, dex_pc); AppendInstruction(cls); @@ -1497,7 +1498,7 @@ void HInstructionBuilder::BuildFilledNewArray(uint32_t dex_pc, uint32_t* args, uint32_t register_index) { HInstruction* length = graph_->GetIntConstant(number_of_vreg_arguments, dex_pc); - HLoadClass* cls = BuildLoadClass(type_index, dex_pc, /* check_access */ true); + HLoadClass* cls = BuildLoadClass(type_index, dex_pc); HInstruction* object = new (arena_) HNewArray(cls, length, dex_pc); AppendInstruction(object); @@ -1627,44 +1628,66 @@ static TypeCheckKind ComputeTypeCheckKind(Handle<mirror::Class> cls) } } -HLoadClass* HInstructionBuilder::BuildLoadClass(dex::TypeIndex type_index, - uint32_t dex_pc, - bool check_access, - bool outer) { +HLoadClass* HInstructionBuilder::BuildLoadClass(dex::TypeIndex type_index, uint32_t dex_pc) { ScopedObjectAccess soa(Thread::Current()); - const DexCompilationUnit* compilation_unit = - outer ? outer_compilation_unit_ : dex_compilation_unit_; - const DexFile& dex_file = *compilation_unit->GetDexFile(); - StackHandleScope<1> hs(soa.Self()); - Handle<mirror::ClassLoader> class_loader(hs.NewHandle( - soa.Decode<mirror::ClassLoader>(dex_compilation_unit_->GetClassLoader()))); + const DexFile& dex_file = *dex_compilation_unit_->GetDexFile(); + Handle<mirror::ClassLoader> class_loader = dex_compilation_unit_->GetClassLoader(); Handle<mirror::Class> klass = handles_->NewHandle(compiler_driver_->ResolveClass( - soa, compilation_unit->GetDexCache(), class_loader, type_index, compilation_unit)); + soa, dex_compilation_unit_->GetDexCache(), class_loader, type_index, dex_compilation_unit_)); - bool is_accessible = false; - if (!check_access) { - is_accessible = true; - } else if (klass.Get() != nullptr) { + bool needs_access_check = true; + if (klass.Get() != nullptr) { if (klass->IsPublic()) { - is_accessible = true; + needs_access_check = false; } else { mirror::Class* compiling_class = GetCompilingClass(); if (compiling_class != nullptr && compiling_class->CanAccess(klass.Get())) { - is_accessible = true; + needs_access_check = false; } } } + return BuildLoadClass(type_index, dex_file, klass, dex_pc, needs_access_check); +} + +HLoadClass* HInstructionBuilder::BuildLoadClass(dex::TypeIndex type_index, + const DexFile& dex_file, + Handle<mirror::Class> klass, + uint32_t dex_pc, + bool needs_access_check) { + // Try to find a reference in the compiling dex file. + const DexFile* actual_dex_file = &dex_file; + if (!IsSameDexFile(dex_file, *dex_compilation_unit_->GetDexFile())) { + dex::TypeIndex local_type_index = + klass->FindTypeIndexInOtherDexFile(*dex_compilation_unit_->GetDexFile()); + if (local_type_index.IsValid()) { + type_index = local_type_index; + actual_dex_file = dex_compilation_unit_->GetDexFile(); + } + } + + // Note: `klass` must be from `handles_`. HLoadClass* load_class = new (arena_) HLoadClass( graph_->GetCurrentMethod(), type_index, - dex_file, + *actual_dex_file, klass, klass.Get() != nullptr && (klass.Get() == GetOutermostCompilingClass()), dex_pc, - !is_accessible); + needs_access_check); + + HLoadClass::LoadKind load_kind = HSharpening::SharpenClass(load_class, + code_generator_, + compiler_driver_, + *dex_compilation_unit_); + if (load_kind == HLoadClass::LoadKind::kInvalid) { + // We actually cannot reference this class, we're forced to bail. + return nullptr; + } + // Append the instruction first, as setting the load kind affects the inputs. AppendInstruction(load_class); + load_class->SetLoadKind(load_kind); return load_class; } @@ -1674,7 +1697,7 @@ void HInstructionBuilder::BuildTypeCheck(const Instruction& instruction, dex::TypeIndex type_index, uint32_t dex_pc) { HInstruction* object = LoadLocal(reference, Primitive::kPrimNot); - HLoadClass* cls = BuildLoadClass(type_index, dex_pc, /* check_access */ true); + HLoadClass* cls = BuildLoadClass(type_index, dex_pc); ScopedObjectAccess soa(Thread::Current()); TypeCheckKind check_kind = ComputeTypeCheckKind(cls->GetClass()); @@ -1692,17 +1715,9 @@ void HInstructionBuilder::BuildTypeCheck(const Instruction& instruction, } } -bool HInstructionBuilder::NeedsAccessCheck(dex::TypeIndex type_index, - Handle<mirror::DexCache> dex_cache, - bool* finalizable) const { - return !compiler_driver_->CanAccessInstantiableTypeWithoutChecks( - dex_compilation_unit_->GetDexMethodIndex(), dex_cache, type_index, finalizable); -} - bool HInstructionBuilder::NeedsAccessCheck(dex::TypeIndex type_index, bool* finalizable) const { - ScopedObjectAccess soa(Thread::Current()); - Handle<mirror::DexCache> dex_cache = dex_compilation_unit_->GetDexCache(); - return NeedsAccessCheck(type_index, dex_cache, finalizable); + return !compiler_driver_->CanAccessInstantiableTypeWithoutChecks( + LookupReferrerClass(), LookupResolvedType(type_index, *dex_compilation_unit_), finalizable); } bool HInstructionBuilder::CanDecodeQuickenedInfo() const { @@ -2498,7 +2513,7 @@ bool HInstructionBuilder::ProcessDexInstruction(const Instruction& instruction, case Instruction::NEW_ARRAY: { dex::TypeIndex type_index(instruction.VRegC_22c()); HInstruction* length = LoadLocal(instruction.VRegB_22c(), Primitive::kPrimInt); - HLoadClass* cls = BuildLoadClass(type_index, dex_pc, /* check_access */ true); + HLoadClass* cls = BuildLoadClass(type_index, dex_pc); AppendInstruction(new (arena_) HNewArray(cls, length, dex_pc)); UpdateLocal(instruction.VRegA_22c(), current_block_->GetLastInstruction()); break; @@ -2673,7 +2688,7 @@ bool HInstructionBuilder::ProcessDexInstruction(const Instruction& instruction, case Instruction::CONST_CLASS: { dex::TypeIndex type_index(instruction.VRegB_21c()); - BuildLoadClass(type_index, dex_pc, /* check_access */ true); + BuildLoadClass(type_index, dex_pc); UpdateLocal(instruction.VRegA_21c(), current_block_->GetLastInstruction()); break; } @@ -2742,4 +2757,18 @@ bool HInstructionBuilder::ProcessDexInstruction(const Instruction& instruction, return true; } // NOLINT(readability/fn_size) +ObjPtr<mirror::Class> HInstructionBuilder::LookupResolvedType( + dex::TypeIndex type_index, + const DexCompilationUnit& compilation_unit) const { + return ClassLinker::LookupResolvedType( + type_index, compilation_unit.GetDexCache().Get(), compilation_unit.GetClassLoader().Get()); +} + +ObjPtr<mirror::Class> HInstructionBuilder::LookupReferrerClass() const { + // TODO: Cache the result in a Handle<mirror::Class>. + const DexFile::MethodId& method_id = + dex_compilation_unit_->GetDexFile()->GetMethodId(dex_compilation_unit_->GetDexMethodIndex()); + return LookupResolvedType(method_id.class_idx_, *dex_compilation_unit_); +} + } // namespace art diff --git a/compiler/optimizing/instruction_builder.h b/compiler/optimizing/instruction_builder.h index 5efe95094c..e735a0c46d 100644 --- a/compiler/optimizing/instruction_builder.h +++ b/compiler/optimizing/instruction_builder.h @@ -31,6 +31,7 @@ namespace art { +class CodeGenerator; class Instruction; class HInstructionBuilder : public ValueObject { @@ -44,6 +45,7 @@ class HInstructionBuilder : public ValueObject { DexCompilationUnit* dex_compilation_unit, const DexCompilationUnit* const outer_compilation_unit, CompilerDriver* driver, + CodeGenerator* code_generator, const uint8_t* interpreter_metadata, OptimizingCompilerStats* compiler_stats, Handle<mirror::DexCache> dex_cache, @@ -61,6 +63,7 @@ class HInstructionBuilder : public ValueObject { current_locals_(nullptr), latest_result_(nullptr), compiler_driver_(driver), + code_generator_(code_generator), dex_compilation_unit_(dex_compilation_unit), outer_compilation_unit_(outer_compilation_unit), interpreter_metadata_(interpreter_metadata), @@ -103,11 +106,8 @@ class HInstructionBuilder : public ValueObject { // Returns whether the current method needs access check for the type. // Output parameter finalizable is set to whether the type is finalizable. - bool NeedsAccessCheck(dex::TypeIndex type_index, - Handle<mirror::DexCache> dex_cache, - /*out*/bool* finalizable) const + bool NeedsAccessCheck(dex::TypeIndex type_index, /*out*/bool* finalizable) const REQUIRES_SHARED(Locks::mutator_lock_); - bool NeedsAccessCheck(dex::TypeIndex type_index, /*out*/bool* finalizable) const; template<typename T> void Unop_12x(const Instruction& instruction, Primitive::Type type, uint32_t dex_pc); @@ -228,10 +228,14 @@ class HInstructionBuilder : public ValueObject { // Builds a `HLoadClass` loading the given `type_index`. If `outer` is true, // this method will use the outer class's dex file to lookup the type at // `type_index`. + HLoadClass* BuildLoadClass(dex::TypeIndex type_index, uint32_t dex_pc); + HLoadClass* BuildLoadClass(dex::TypeIndex type_index, + const DexFile& dex_file, + Handle<mirror::Class> klass, uint32_t dex_pc, - bool check_access, - bool outer = false); + bool needs_access_check) + REQUIRES_SHARED(Locks::mutator_lock_); // Returns the outer-most compiling method's class. mirror::Class* GetOutermostCompilingClass() const; @@ -275,7 +279,6 @@ class HInstructionBuilder : public ValueObject { HClinitCheck* ProcessClinitCheckForInvoke( uint32_t dex_pc, ArtMethod* method, - uint32_t method_idx, HInvokeStaticOrDirect::ClinitCheckRequirement* clinit_check_requirement) REQUIRES_SHARED(Locks::mutator_lock_); @@ -290,6 +293,16 @@ class HInstructionBuilder : public ValueObject { // not be resolved. ArtMethod* ResolveMethod(uint16_t method_idx, InvokeType invoke_type); + // Try to resolve a field using the class linker. Return null if it could not + // be found. + ArtField* ResolveField(uint16_t field_idx, bool is_static, bool is_put); + + ObjPtr<mirror::Class> LookupResolvedType(dex::TypeIndex type_index, + const DexCompilationUnit& compilation_unit) const + REQUIRES_SHARED(Locks::mutator_lock_); + + ObjPtr<mirror::Class> LookupReferrerClass() const REQUIRES_SHARED(Locks::mutator_lock_); + ArenaAllocator* const arena_; HGraph* const graph_; VariableSizedHandleScope* handles_; @@ -311,6 +324,8 @@ class HInstructionBuilder : public ValueObject { CompilerDriver* const compiler_driver_; + CodeGenerator* const code_generator_; + // The compilation unit of the current method being compiled. Note that // it can be an inlined method. DexCompilationUnit* const dex_compilation_unit_; diff --git a/compiler/optimizing/intrinsics_arm64.cc b/compiler/optimizing/intrinsics_arm64.cc index bbf826ce7e..1047d3beb6 100644 --- a/compiler/optimizing/intrinsics_arm64.cc +++ b/compiler/optimizing/intrinsics_arm64.cc @@ -115,13 +115,18 @@ class IntrinsicSlowPathARM64 : public SlowPathCodeARM64 { MoveArguments(invoke_, codegen); - if (invoke_->IsInvokeStaticOrDirect()) { - codegen->GenerateStaticOrDirectCall(invoke_->AsInvokeStaticOrDirect(), - LocationFrom(kArtMethodRegister)); - } else { - codegen->GenerateVirtualCall(invoke_->AsInvokeVirtual(), LocationFrom(kArtMethodRegister)); + { + // Ensure that between the BLR (emitted by Generate*Call) and RecordPcInfo there + // are no pools emitted. + vixl::EmissionCheckScope guard(codegen->GetVIXLAssembler(), kInvokeCodeMarginSizeInBytes); + if (invoke_->IsInvokeStaticOrDirect()) { + codegen->GenerateStaticOrDirectCall(invoke_->AsInvokeStaticOrDirect(), + LocationFrom(kArtMethodRegister)); + } else { + codegen->GenerateVirtualCall(invoke_->AsInvokeVirtual(), LocationFrom(kArtMethodRegister)); + } + codegen->RecordPcInfo(invoke_, invoke_->GetDexPc(), this); } - codegen->RecordPcInfo(invoke_, invoke_->GetDexPc(), this); // Copy the result back to the expected output. Location out = invoke_->GetLocations()->Out(); @@ -980,11 +985,12 @@ void IntrinsicLocationsBuilderARM64::VisitUnsafePutLongVolatile(HInvoke* invoke) CreateIntIntIntIntToVoid(arena_, invoke); } -static void GenUnsafePut(LocationSummary* locations, +static void GenUnsafePut(HInvoke* invoke, Primitive::Type type, bool is_volatile, bool is_ordered, CodeGeneratorARM64* codegen) { + LocationSummary* locations = invoke->GetLocations(); MacroAssembler* masm = codegen->GetVIXLAssembler(); Register base = WRegisterFrom(locations->InAt(1)); // Object pointer. @@ -1007,7 +1013,7 @@ static void GenUnsafePut(LocationSummary* locations, } if (is_volatile || is_ordered) { - codegen->StoreRelease(type, source, mem_op); + codegen->StoreRelease(invoke, type, source, mem_op, /* needs_null_check */ false); } else { codegen->Store(type, source, mem_op); } @@ -1020,63 +1026,63 @@ static void GenUnsafePut(LocationSummary* locations, } void IntrinsicCodeGeneratorARM64::VisitUnsafePut(HInvoke* invoke) { - GenUnsafePut(invoke->GetLocations(), + GenUnsafePut(invoke, Primitive::kPrimInt, /* is_volatile */ false, /* is_ordered */ false, codegen_); } void IntrinsicCodeGeneratorARM64::VisitUnsafePutOrdered(HInvoke* invoke) { - GenUnsafePut(invoke->GetLocations(), + GenUnsafePut(invoke, Primitive::kPrimInt, /* is_volatile */ false, /* is_ordered */ true, codegen_); } void IntrinsicCodeGeneratorARM64::VisitUnsafePutVolatile(HInvoke* invoke) { - GenUnsafePut(invoke->GetLocations(), + GenUnsafePut(invoke, Primitive::kPrimInt, /* is_volatile */ true, /* is_ordered */ false, codegen_); } void IntrinsicCodeGeneratorARM64::VisitUnsafePutObject(HInvoke* invoke) { - GenUnsafePut(invoke->GetLocations(), + GenUnsafePut(invoke, Primitive::kPrimNot, /* is_volatile */ false, /* is_ordered */ false, codegen_); } void IntrinsicCodeGeneratorARM64::VisitUnsafePutObjectOrdered(HInvoke* invoke) { - GenUnsafePut(invoke->GetLocations(), + GenUnsafePut(invoke, Primitive::kPrimNot, /* is_volatile */ false, /* is_ordered */ true, codegen_); } void IntrinsicCodeGeneratorARM64::VisitUnsafePutObjectVolatile(HInvoke* invoke) { - GenUnsafePut(invoke->GetLocations(), + GenUnsafePut(invoke, Primitive::kPrimNot, /* is_volatile */ true, /* is_ordered */ false, codegen_); } void IntrinsicCodeGeneratorARM64::VisitUnsafePutLong(HInvoke* invoke) { - GenUnsafePut(invoke->GetLocations(), + GenUnsafePut(invoke, Primitive::kPrimLong, /* is_volatile */ false, /* is_ordered */ false, codegen_); } void IntrinsicCodeGeneratorARM64::VisitUnsafePutLongOrdered(HInvoke* invoke) { - GenUnsafePut(invoke->GetLocations(), + GenUnsafePut(invoke, Primitive::kPrimLong, /* is_volatile */ false, /* is_ordered */ true, codegen_); } void IntrinsicCodeGeneratorARM64::VisitUnsafePutLongVolatile(HInvoke* invoke) { - GenUnsafePut(invoke->GetLocations(), + GenUnsafePut(invoke, Primitive::kPrimLong, /* is_volatile */ true, /* is_ordered */ false, @@ -2825,9 +2831,13 @@ void IntrinsicCodeGeneratorARM64::VisitReferenceGetReferent(HInvoke* invoke) { } __ Cbnz(temp0, slow_path->GetEntryLabel()); - // Fast path. - __ Ldr(out, HeapOperand(obj, mirror::Reference::ReferentOffset().Int32Value())); - codegen_->MaybeRecordImplicitNullCheck(invoke); + { + // Ensure that between load and MaybeRecordImplicitNullCheck there are no pools emitted. + vixl::EmissionCheckScope guard(codegen_->GetVIXLAssembler(), kMaxMacroInstructionSizeInBytes); + // Fast path. + __ Ldr(out, HeapOperand(obj, mirror::Reference::ReferentOffset().Int32Value())); + codegen_->MaybeRecordImplicitNullCheck(invoke); + } codegen_->GetAssembler()->MaybeUnpoisonHeapReference(out); __ Bind(slow_path->GetExitLabel()); } diff --git a/compiler/optimizing/intrinsics_mips.cc b/compiler/optimizing/intrinsics_mips.cc index 6cf9b83d44..64a68403e9 100644 --- a/compiler/optimizing/intrinsics_mips.cc +++ b/compiler/optimizing/intrinsics_mips.cc @@ -2004,31 +2004,48 @@ void IntrinsicCodeGeneratorMIPS::VisitStringEquals(HInvoke* invoke) { __ Lw(temp2, arg, class_offset); __ Bne(temp1, temp2, &return_false); - // Load lengths of this and argument strings. + // Load `count` fields of this and argument strings. __ Lw(temp1, str, count_offset); __ Lw(temp2, arg, count_offset); - // Check if lengths are equal, return false if they're not. + // Check if `count` fields are equal, return false if they're not. + // Also compares the compression style, if differs return false. __ Bne(temp1, temp2, &return_false); - // Return true if both strings are empty. + // Return true if both strings are empty. Even with string compression `count == 0` means empty. + static_assert(static_cast<uint32_t>(mirror::StringCompressionFlag::kCompressed) == 0u, + "Expecting 0=compressed, 1=uncompressed"); __ Beqz(temp1, &return_true); // Don't overwrite input registers __ Move(TMP, str); __ Move(temp3, arg); - // Assertions that must hold in order to compare strings 2 characters at a time. + // Assertions that must hold in order to compare strings 4 bytes at a time. DCHECK_ALIGNED(value_offset, 4); static_assert(IsAligned<4>(kObjectAlignment), "String of odd length is not zero padded"); - // Loop to compare strings 2 characters at a time starting at the beginning of the string. - // Ok to do this because strings are zero-padded. + // For string compression, calculate the number of bytes to compare (not chars). + if (mirror::kUseStringCompression) { + // Extract compression flag. + if (IsR2OrNewer()) { + __ Ext(temp2, temp1, 0, 1); + } else { + __ Sll(temp2, temp1, 31); + __ Srl(temp2, temp2, 31); + } + __ Srl(temp1, temp1, 1); // Extract length. + __ Sllv(temp1, temp1, temp2); // Double the byte count if uncompressed. + } + + // Loop to compare strings 4 bytes at a time starting at the beginning of the string. + // Ok to do this because strings are zero-padded to kObjectAlignment. __ Bind(&loop); __ Lw(out, TMP, value_offset); __ Lw(temp2, temp3, value_offset); __ Bne(out, temp2, &return_false); __ Addiu(TMP, TMP, 4); __ Addiu(temp3, temp3, 4); - __ Addiu(temp1, temp1, -2); + // With string compression, we have compared 4 bytes, otherwise 2 chars. + __ Addiu(temp1, temp1, mirror::kUseStringCompression ? -4 : -2); __ Bgtz(temp1, &loop); // Return true and exit the function. @@ -2578,6 +2595,30 @@ void IntrinsicCodeGeneratorMIPS::VisitStringGetCharsNoCheck(HInvoke* invoke) { __ Addu(dstPtr, dstPtr, AT); } + if (mirror::kUseStringCompression) { + MipsLabel uncompressed_copy, compressed_loop; + const uint32_t count_offset = mirror::String::CountOffset().Uint32Value(); + // Load count field and extract compression flag. + __ LoadFromOffset(kLoadWord, TMP, srcObj, count_offset); + __ Sll(TMP, TMP, 31); + + // If string is uncompressed, use memcpy() path. + __ Bnez(TMP, &uncompressed_copy); + + // Copy loop for compressed src, copying 1 character (8-bit) to (16-bit) at a time. + __ Addu(srcPtr, srcObj, srcBegin); + __ Bind(&compressed_loop); + __ LoadFromOffset(kLoadUnsignedByte, TMP, srcPtr, value_offset); + __ StoreToOffset(kStoreHalfword, TMP, dstPtr, 0); + __ Addiu(numChrs, numChrs, -1); + __ Addiu(srcPtr, srcPtr, 1); + __ Addiu(dstPtr, dstPtr, 2); + __ Bnez(numChrs, &compressed_loop); + + __ B(&done); + __ Bind(&uncompressed_copy); + } + // Calculate source address. __ Addiu(srcPtr, srcObj, value_offset); if (IsR6()) { diff --git a/compiler/optimizing/intrinsics_mips64.cc b/compiler/optimizing/intrinsics_mips64.cc index 00a1fa11bb..3888828722 100644 --- a/compiler/optimizing/intrinsics_mips64.cc +++ b/compiler/optimizing/intrinsics_mips64.cc @@ -1607,31 +1607,42 @@ void IntrinsicCodeGeneratorMIPS64::VisitStringEquals(HInvoke* invoke) { __ Lw(temp2, arg, class_offset); __ Bnec(temp1, temp2, &return_false); - // Load lengths of this and argument strings. + // Load `count` fields of this and argument strings. __ Lw(temp1, str, count_offset); __ Lw(temp2, arg, count_offset); - // Check if lengths are equal, return false if they're not. + // Check if `count` fields are equal, return false if they're not. + // Also compares the compression style, if differs return false. __ Bnec(temp1, temp2, &return_false); - // Return true if both strings are empty. + // Return true if both strings are empty. Even with string compression `count == 0` means empty. + static_assert(static_cast<uint32_t>(mirror::StringCompressionFlag::kCompressed) == 0u, + "Expecting 0=compressed, 1=uncompressed"); __ Beqzc(temp1, &return_true); // Don't overwrite input registers __ Move(TMP, str); __ Move(temp3, arg); - // Assertions that must hold in order to compare strings 4 characters at a time. + // Assertions that must hold in order to compare strings 8 bytes at a time. DCHECK_ALIGNED(value_offset, 8); static_assert(IsAligned<8>(kObjectAlignment), "String of odd length is not zero padded"); - // Loop to compare strings 4 characters at a time starting at the beginning of the string. - // Ok to do this because strings are zero-padded to be 8-byte aligned. + if (mirror::kUseStringCompression) { + // For string compression, calculate the number of bytes to compare (not chars). + __ Dext(temp2, temp1, 0, 1); // Extract compression flag. + __ Srl(temp1, temp1, 1); // Extract length. + __ Sllv(temp1, temp1, temp2); // Double the byte count if uncompressed. + } + + // Loop to compare strings 8 bytes at a time starting at the beginning of the string. + // Ok to do this because strings are zero-padded to kObjectAlignment. __ Bind(&loop); __ Ld(out, TMP, value_offset); __ Ld(temp2, temp3, value_offset); __ Bnec(out, temp2, &return_false); __ Daddiu(TMP, TMP, 8); __ Daddiu(temp3, temp3, 8); - __ Addiu(temp1, temp1, -4); + // With string compression, we have compared 8 bytes, otherwise 4 chars. + __ Addiu(temp1, temp1, mirror::kUseStringCompression ? -8 : -4); __ Bgtzc(temp1, &loop); // Return true and exit the function. @@ -1912,6 +1923,30 @@ void IntrinsicCodeGeneratorMIPS64::VisitStringGetCharsNoCheck(HInvoke* invoke) { __ Daddiu(dstPtr, dstObj, data_offset); __ Dlsa(dstPtr, dstBegin, dstPtr, char_shift); + if (mirror::kUseStringCompression) { + Mips64Label uncompressed_copy, compressed_loop; + const uint32_t count_offset = mirror::String::CountOffset().Uint32Value(); + // Load count field and extract compression flag. + __ LoadFromOffset(kLoadWord, TMP, srcObj, count_offset); + __ Dext(TMP, TMP, 0, 1); + + // If string is uncompressed, use memcpy() path. + __ Bnezc(TMP, &uncompressed_copy); + + // Copy loop for compressed src, copying 1 character (8-bit) to (16-bit) at a time. + __ Daddu(srcPtr, srcObj, srcBegin); + __ Bind(&compressed_loop); + __ LoadFromOffset(kLoadUnsignedByte, TMP, srcPtr, value_offset); + __ StoreToOffset(kStoreHalfword, TMP, dstPtr, 0); + __ Daddiu(numChrs, numChrs, -1); + __ Daddiu(srcPtr, srcPtr, 1); + __ Daddiu(dstPtr, dstPtr, 2); + __ Bnezc(numChrs, &compressed_loop); + + __ Bc(&done); + __ Bind(&uncompressed_copy); + } + // Calculate source address. __ Daddiu(srcPtr, srcObj, value_offset); __ Dlsa(srcPtr, srcBegin, srcPtr, char_shift); diff --git a/compiler/optimizing/nodes.cc b/compiler/optimizing/nodes.cc index 76900f23a9..abbb91a1a9 100644 --- a/compiler/optimizing/nodes.cc +++ b/compiler/optimizing/nodes.cc @@ -2464,16 +2464,15 @@ bool HLoadClass::InstructionDataEquals(const HInstruction* other) const { } } -void HLoadClass::SetLoadKindInternal(LoadKind load_kind) { - // Once sharpened, the load kind should not be changed again. - // Also, kReferrersClass should never be overwritten. - DCHECK_EQ(GetLoadKind(), LoadKind::kDexCacheViaMethod); +void HLoadClass::SetLoadKind(LoadKind load_kind) { SetPackedField<LoadKindField>(load_kind); - if (load_kind != LoadKind::kDexCacheViaMethod) { + if (load_kind != LoadKind::kDexCacheViaMethod && + load_kind != LoadKind::kReferrersClass) { RemoveAsUserOfInput(0u); SetRawInputAt(0u, nullptr); } + if (!NeedsEnvironment()) { RemoveEnvironment(); SetSideEffects(SideEffects::None()); diff --git a/compiler/optimizing/nodes.h b/compiler/optimizing/nodes.h index acf14aa726..96f9abafbf 100644 --- a/compiler/optimizing/nodes.h +++ b/compiler/optimizing/nodes.h @@ -4322,6 +4322,11 @@ class HInvokeInterface FINAL : public HInvoke { return (obj == InputAt(0)) && !GetLocations()->Intrinsified(); } + bool NeedsDexCacheOfDeclaringClass() const OVERRIDE { + // The assembly stub currently needs it. + return true; + } + uint32_t GetImtIndex() const { return imt_index_; } uint32_t GetDexMethodIndex() const { return dex_method_index_; } @@ -5508,6 +5513,9 @@ class HLoadClass FINAL : public HInstruction { public: // Determines how to load the Class. enum class LoadKind { + // We cannot load this class. See HSharpening::SharpenLoadClass. + kInvalid = -1, + // Use the Class* from the method's own ArtMethod*. kReferrersClass, @@ -5564,18 +5572,7 @@ class HLoadClass FINAL : public HInstruction { SetPackedFlag<kFlagGenerateClInitCheck>(false); } - void SetLoadKind(LoadKind load_kind) { - SetLoadKindInternal(load_kind); - } - - void SetLoadKindWithTypeReference(LoadKind load_kind, - const DexFile& dex_file, - dex::TypeIndex type_index) { - DCHECK(HasTypeReference(load_kind)); - DCHECK(IsSameDexFile(dex_file_, dex_file)); - DCHECK_EQ(type_index_, type_index); - SetLoadKindInternal(load_kind); - } + void SetLoadKind(LoadKind load_kind); LoadKind GetLoadKind() const { return GetPackedField<LoadKindField>(); @@ -5694,6 +5691,11 @@ class HLoadClass FINAL : public HInstruction { // for PC-relative loads, i.e. kBssEntry or kBootImageLinkTimePcRelative. HUserRecord<HInstruction*> special_input_; + // A type index and dex file where the class can be accessed. The dex file can be: + // - The compiling method's dex file if the class is defined there too. + // - The compiling method's dex file if the class is referenced there. + // - The dex file where the class is defined. When the load kind can only be + // kBssEntry or kDexCacheViaMethod, we cannot emit code for this `HLoadClass`. const dex::TypeIndex type_index_; const DexFile& dex_file_; diff --git a/compiler/optimizing/optimizing_compiler.cc b/compiler/optimizing/optimizing_compiler.cc index 1ab671022b..0375c66e42 100644 --- a/compiler/optimizing/optimizing_compiler.cc +++ b/compiler/optimizing/optimizing_compiler.cc @@ -306,7 +306,7 @@ class OptimizingCompiler FINAL : public Compiler { InvokeType invoke_type, uint16_t class_def_idx, uint32_t method_idx, - jobject class_loader, + Handle<mirror::ClassLoader> class_loader, const DexFile& dex_file, Handle<mirror::DexCache> dex_cache) const OVERRIDE; @@ -375,7 +375,7 @@ class OptimizingCompiler FINAL : public Compiler { InvokeType invoke_type, uint16_t class_def_idx, uint32_t method_idx, - jobject class_loader, + Handle<mirror::ClassLoader> class_loader, const DexFile& dex_file, Handle<mirror::DexCache> dex_cache, ArtMethod* method, @@ -875,7 +875,7 @@ CodeGenerator* OptimizingCompiler::TryCompile(ArenaAllocator* arena, InvokeType invoke_type, uint16_t class_def_idx, uint32_t method_idx, - jobject class_loader, + Handle<mirror::ClassLoader> class_loader, const DexFile& dex_file, Handle<mirror::DexCache> dex_cache, ArtMethod* method, @@ -946,11 +946,8 @@ CodeGenerator* OptimizingCompiler::TryCompile(ArenaAllocator* arena, const uint8_t* interpreter_metadata = nullptr; if (method == nullptr) { ScopedObjectAccess soa(Thread::Current()); - StackHandleScope<1> hs(soa.Self()); - Handle<mirror::ClassLoader> loader(hs.NewHandle( - soa.Decode<mirror::ClassLoader>(class_loader))); method = compiler_driver->ResolveMethod( - soa, dex_cache, loader, &dex_compilation_unit, method_idx, invoke_type); + soa, dex_cache, class_loader, &dex_compilation_unit, method_idx, invoke_type); } // For AOT compilation, we may not get a method, for example if its class is erroneous. // JIT should always have a method. @@ -959,16 +956,6 @@ CodeGenerator* OptimizingCompiler::TryCompile(ArenaAllocator* arena, graph->SetArtMethod(method); ScopedObjectAccess soa(Thread::Current()); interpreter_metadata = method->GetQuickenedInfo(class_linker->GetImagePointerSize()); - dex::TypeIndex type_index = method->GetDeclaringClass()->GetDexTypeIndex(); - - // Update the dex cache if the type is not in it yet. Note that under AOT, - // the verifier must have set it, but under JIT, there's no guarantee, as we - // don't necessarily run the verifier. - // The compiler and the compiler driver assume the compiling class is - // in the dex cache. - if (dex_cache->GetResolvedType(type_index) == nullptr) { - dex_cache->SetResolvedType(type_index, method->GetDeclaringClass()); - } } std::unique_ptr<CodeGenerator> codegen( @@ -999,6 +986,7 @@ CodeGenerator* OptimizingCompiler::TryCompile(ArenaAllocator* arena, &dex_file, *code_item, compiler_driver, + codegen.get(), compilation_stats_.get(), interpreter_metadata, dex_cache, @@ -1048,7 +1036,7 @@ CompiledMethod* OptimizingCompiler::Compile(const DexFile::CodeItem* code_item, InvokeType invoke_type, uint16_t class_def_idx, uint32_t method_idx, - jobject jclass_loader, + Handle<mirror::ClassLoader> jclass_loader, const DexFile& dex_file, Handle<mirror::DexCache> dex_cache) const { CompilerDriver* compiler_driver = GetCompilerDriver(); @@ -1133,6 +1121,25 @@ bool IsCompilingWithCoreImage() { return false; } +bool EncodeArtMethodInInlineInfo(ArtMethod* method ATTRIBUTE_UNUSED) { + // Note: the runtime is null only for unit testing. + return Runtime::Current() == nullptr || !Runtime::Current()->IsAotCompiler(); +} + +bool CanEncodeInlinedMethodInStackMap(const DexFile& caller_dex_file, ArtMethod* callee) { + if (!Runtime::Current()->IsAotCompiler()) { + // JIT can always encode methods in stack maps. + return true; + } + if (IsSameDexFile(caller_dex_file, *callee->GetDexFile())) { + return true; + } + // TODO(ngeoffray): Support more AOT cases for inlining: + // - methods in multidex + // - methods in boot image for on-device non-PIC compilation. + return false; +} + bool OptimizingCompiler::JitCompile(Thread* self, jit::JitCodeCache* code_cache, ArtMethod* method, @@ -1143,7 +1150,6 @@ bool OptimizingCompiler::JitCompile(Thread* self, Handle<mirror::DexCache> dex_cache(hs.NewHandle(method->GetDexCache())); DCHECK(method->IsCompilable()); - jobject jclass_loader = class_loader.ToJObject(); const DexFile* dex_file = method->GetDexFile(); const uint16_t class_def_idx = method->GetClassDefIndex(); const DexFile::CodeItem* code_item = dex_file->GetCodeItem(method->GetCodeItemOffset()); @@ -1167,7 +1173,7 @@ bool OptimizingCompiler::JitCompile(Thread* self, invoke_type, class_def_idx, method_idx, - jclass_loader, + class_loader, *dex_file, dex_cache, method, diff --git a/compiler/optimizing/optimizing_compiler.h b/compiler/optimizing/optimizing_compiler.h index 0c89da12e8..d8cea30a6b 100644 --- a/compiler/optimizing/optimizing_compiler.h +++ b/compiler/optimizing/optimizing_compiler.h @@ -17,10 +17,15 @@ #ifndef ART_COMPILER_OPTIMIZING_OPTIMIZING_COMPILER_H_ #define ART_COMPILER_OPTIMIZING_OPTIMIZING_COMPILER_H_ +#include "base/mutex.h" +#include "globals.h" + namespace art { +class ArtMethod; class Compiler; class CompilerDriver; +class DexFile; Compiler* CreateOptimizingCompiler(CompilerDriver* driver); @@ -29,6 +34,10 @@ Compiler* CreateOptimizingCompiler(CompilerDriver* driver); // information for checking invariants. bool IsCompilingWithCoreImage(); +bool EncodeArtMethodInInlineInfo(ArtMethod* method); +bool CanEncodeInlinedMethodInStackMap(const DexFile& caller_dex_file, ArtMethod* callee) + REQUIRES_SHARED(Locks::mutator_lock_); + } // namespace art #endif // ART_COMPILER_OPTIMIZING_OPTIMIZING_COMPILER_H_ diff --git a/compiler/optimizing/reference_type_propagation.cc b/compiler/optimizing/reference_type_propagation.cc index b02f2509ab..6e332ca59b 100644 --- a/compiler/optimizing/reference_type_propagation.cc +++ b/compiler/optimizing/reference_type_propagation.cc @@ -24,9 +24,8 @@ namespace art { -static inline mirror::DexCache* FindDexCacheWithHint(Thread* self, - const DexFile& dex_file, - Handle<mirror::DexCache> hint_dex_cache) +static inline ObjPtr<mirror::DexCache> FindDexCacheWithHint( + Thread* self, const DexFile& dex_file, Handle<mirror::DexCache> hint_dex_cache) REQUIRES_SHARED(Locks::mutator_lock_) { if (LIKELY(hint_dex_cache->GetDexFile() == &dex_file)) { return hint_dex_cache.Get(); @@ -66,11 +65,13 @@ ReferenceTypeInfo::TypeHandle ReferenceTypePropagation::HandleCache::GetThrowabl class ReferenceTypePropagation::RTPVisitor : public HGraphDelegateVisitor { public: RTPVisitor(HGraph* graph, + Handle<mirror::ClassLoader> class_loader, Handle<mirror::DexCache> hint_dex_cache, HandleCache* handle_cache, ArenaVector<HInstruction*>* worklist, bool is_first_run) : HGraphDelegateVisitor(graph), + class_loader_(class_loader), hint_dex_cache_(hint_dex_cache), handle_cache_(handle_cache), worklist_(worklist), @@ -102,6 +103,7 @@ class ReferenceTypePropagation::RTPVisitor : public HGraphDelegateVisitor { bool is_exact); private: + Handle<mirror::ClassLoader> class_loader_; Handle<mirror::DexCache> hint_dex_cache_; HandleCache* handle_cache_; ArenaVector<HInstruction*>* worklist_; @@ -109,11 +111,13 @@ class ReferenceTypePropagation::RTPVisitor : public HGraphDelegateVisitor { }; ReferenceTypePropagation::ReferenceTypePropagation(HGraph* graph, + Handle<mirror::ClassLoader> class_loader, Handle<mirror::DexCache> hint_dex_cache, VariableSizedHandleScope* handles, bool is_first_run, const char* name) : HOptimization(graph, name), + class_loader_(class_loader), hint_dex_cache_(hint_dex_cache), handle_cache_(handles), worklist_(graph->GetArena()->Adapter(kArenaAllocReferenceTypePropagation)), @@ -148,7 +152,12 @@ void ReferenceTypePropagation::ValidateTypes() { } void ReferenceTypePropagation::Visit(HInstruction* instruction) { - RTPVisitor visitor(graph_, hint_dex_cache_, &handle_cache_, &worklist_, is_first_run_); + RTPVisitor visitor(graph_, + class_loader_, + hint_dex_cache_, + &handle_cache_, + &worklist_, + is_first_run_); instruction->Accept(&visitor); } @@ -322,7 +331,12 @@ void ReferenceTypePropagation::Run() { } void ReferenceTypePropagation::VisitBasicBlock(HBasicBlock* block) { - RTPVisitor visitor(graph_, hint_dex_cache_, &handle_cache_, &worklist_, is_first_run_); + RTPVisitor visitor(graph_, + class_loader_, + hint_dex_cache_, + &handle_cache_, + &worklist_, + is_first_run_); // Handle Phis first as there might be instructions in the same block who depend on them. for (HInstructionIterator it(block->GetPhis()); !it.Done(); it.Advance()) { VisitPhi(it.Current()->AsPhi()); @@ -542,9 +556,10 @@ void ReferenceTypePropagation::RTPVisitor::UpdateReferenceTypeInfo(HInstruction* DCHECK_EQ(instr->GetType(), Primitive::kPrimNot); ScopedObjectAccess soa(Thread::Current()); - mirror::DexCache* dex_cache = FindDexCacheWithHint(soa.Self(), dex_file, hint_dex_cache_); - // Get type from dex cache assuming it was populated by the verifier. - SetClassAsTypeInfo(instr, dex_cache->GetResolvedType(type_idx), is_exact); + ObjPtr<mirror::DexCache> dex_cache = FindDexCacheWithHint(soa.Self(), dex_file, hint_dex_cache_); + ObjPtr<mirror::Class> klass = + ClassLinker::LookupResolvedType(type_idx, dex_cache, class_loader_.Get()); + SetClassAsTypeInfo(instr, klass, is_exact); } void ReferenceTypePropagation::RTPVisitor::VisitNewInstance(HNewInstance* instr) { @@ -557,25 +572,13 @@ void ReferenceTypePropagation::RTPVisitor::VisitNewArray(HNewArray* instr) { SetClassAsTypeInfo(instr, instr->GetLoadClass()->GetClass().Get(), /* is_exact */ true); } -static mirror::Class* GetClassFromDexCache(Thread* self, - const DexFile& dex_file, - dex::TypeIndex type_idx, - Handle<mirror::DexCache> hint_dex_cache) - REQUIRES_SHARED(Locks::mutator_lock_) { - mirror::DexCache* dex_cache = FindDexCacheWithHint(self, dex_file, hint_dex_cache); - // Get type from dex cache assuming it was populated by the verifier. - return dex_cache->GetResolvedType(type_idx); -} - void ReferenceTypePropagation::RTPVisitor::VisitParameterValue(HParameterValue* instr) { // We check if the existing type is valid: the inliner may have set it. if (instr->GetType() == Primitive::kPrimNot && !instr->GetReferenceTypeInfo().IsValid()) { - ScopedObjectAccess soa(Thread::Current()); - mirror::Class* resolved_class = GetClassFromDexCache(soa.Self(), - instr->GetDexFile(), - instr->GetTypeIndex(), - hint_dex_cache_); - SetClassAsTypeInfo(instr, resolved_class, /* is_exact */ false); + UpdateReferenceTypeInfo(instr, + instr->GetTypeIndex(), + instr->GetDexFile(), + /* is_exact */ false); } } diff --git a/compiler/optimizing/reference_type_propagation.h b/compiler/optimizing/reference_type_propagation.h index 4663471729..215e96786b 100644 --- a/compiler/optimizing/reference_type_propagation.h +++ b/compiler/optimizing/reference_type_propagation.h @@ -33,6 +33,7 @@ namespace art { class ReferenceTypePropagation : public HOptimization { public: ReferenceTypePropagation(HGraph* graph, + Handle<mirror::ClassLoader> class_loader, Handle<mirror::DexCache> hint_dex_cache, VariableSizedHandleScope* handles, bool is_first_run, @@ -105,6 +106,8 @@ class ReferenceTypePropagation : public HOptimization { void ValidateTypes(); + Handle<mirror::ClassLoader> class_loader_; + // Note: hint_dex_cache_ is usually, but not necessarily, the dex cache associated with // graph_->GetDexFile(). Since we may look up also in other dex files, it's used only // as a hint, to reduce the number of calls to the costly ClassLinker::FindDexCache(). diff --git a/compiler/optimizing/reference_type_propagation_test.cc b/compiler/optimizing/reference_type_propagation_test.cc index b061c871b0..84a4bab1a9 100644 --- a/compiler/optimizing/reference_type_propagation_test.cc +++ b/compiler/optimizing/reference_type_propagation_test.cc @@ -38,6 +38,7 @@ class ReferenceTypePropagationTest : public CommonCompilerTest { void SetupPropagation(VariableSizedHandleScope* handles) { graph_->InitializeInexactObjectRTI(handles); propagation_ = new (&allocator_) ReferenceTypePropagation(graph_, + Handle<mirror::ClassLoader>(), Handle<mirror::DexCache>(), handles, true, diff --git a/compiler/optimizing/sharpening.cc b/compiler/optimizing/sharpening.cc index e745c73091..f07f02a719 100644 --- a/compiler/optimizing/sharpening.cc +++ b/compiler/optimizing/sharpening.cc @@ -42,8 +42,6 @@ void HSharpening::Run() { HInstruction* instruction = it.Current(); if (instruction->IsInvokeStaticOrDirect()) { ProcessInvokeStaticOrDirect(instruction->AsInvokeStaticOrDirect()); - } else if (instruction->IsLoadClass()) { - ProcessLoadClass(instruction->AsLoadClass()); } else if (instruction->IsLoadString()) { ProcessLoadString(instruction->AsLoadString()); } @@ -133,104 +131,93 @@ void HSharpening::ProcessInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) { invoke->SetDispatchInfo(dispatch_info); } -void HSharpening::ProcessLoadClass(HLoadClass* load_class) { - ScopedObjectAccess soa(Thread::Current()); - SharpenClass(load_class, codegen_, compiler_driver_); -} - -void HSharpening::SharpenClass(HLoadClass* load_class, - CodeGenerator* codegen, - CompilerDriver* compiler_driver) { +HLoadClass::LoadKind HSharpening::SharpenClass(HLoadClass* load_class, + CodeGenerator* codegen, + CompilerDriver* compiler_driver, + const DexCompilationUnit& dex_compilation_unit) { Handle<mirror::Class> klass = load_class->GetClass(); DCHECK(load_class->GetLoadKind() == HLoadClass::LoadKind::kDexCacheViaMethod || load_class->GetLoadKind() == HLoadClass::LoadKind::kReferrersClass) << load_class->GetLoadKind(); DCHECK(!load_class->IsInBootImage()) << "HLoadClass should not be optimized before sharpening."; + HLoadClass::LoadKind load_kind = load_class->GetLoadKind(); + if (load_class->NeedsAccessCheck()) { // We need to call the runtime anyway, so we simply get the class as that call's return value. - return; - } - - if (load_class->GetLoadKind() == HLoadClass::LoadKind::kReferrersClass) { + } else if (load_kind == HLoadClass::LoadKind::kReferrersClass) { // Loading from the ArtMethod* is the most efficient retrieval in code size. // TODO: This may not actually be true for all architectures and // locations of target classes. The additional register pressure // for using the ArtMethod* should be considered. - return; - } - - const DexFile& dex_file = load_class->GetDexFile(); - dex::TypeIndex type_index = load_class->GetTypeIndex(); + } else { + const DexFile& dex_file = load_class->GetDexFile(); + dex::TypeIndex type_index = load_class->GetTypeIndex(); - bool is_in_boot_image = false; - HLoadClass::LoadKind desired_load_kind = static_cast<HLoadClass::LoadKind>(-1); - Runtime* runtime = Runtime::Current(); - if (codegen->GetCompilerOptions().IsBootImage()) { - // Compiling boot image. Check if the class is a boot image class. - DCHECK(!runtime->UseJitCompilation()); - if (!compiler_driver->GetSupportBootImageFixup()) { - // compiler_driver_test. Do not sharpen. - desired_load_kind = HLoadClass::LoadKind::kDexCacheViaMethod; - } else if ((klass.Get() != nullptr) && compiler_driver->IsImageClass( - dex_file.StringDataByIdx(dex_file.GetTypeId(type_index).descriptor_idx_))) { - is_in_boot_image = true; - desired_load_kind = codegen->GetCompilerOptions().GetCompilePic() - ? HLoadClass::LoadKind::kBootImageLinkTimePcRelative - : HLoadClass::LoadKind::kBootImageLinkTimeAddress; + bool is_in_boot_image = false; + HLoadClass::LoadKind desired_load_kind = HLoadClass::LoadKind::kInvalid; + Runtime* runtime = Runtime::Current(); + if (codegen->GetCompilerOptions().IsBootImage()) { + // Compiling boot image. Check if the class is a boot image class. + DCHECK(!runtime->UseJitCompilation()); + if (!compiler_driver->GetSupportBootImageFixup()) { + // compiler_driver_test. Do not sharpen. + desired_load_kind = HLoadClass::LoadKind::kDexCacheViaMethod; + } else if ((klass.Get() != nullptr) && compiler_driver->IsImageClass( + dex_file.StringDataByIdx(dex_file.GetTypeId(type_index).descriptor_idx_))) { + is_in_boot_image = true; + desired_load_kind = codegen->GetCompilerOptions().GetCompilePic() + ? HLoadClass::LoadKind::kBootImageLinkTimePcRelative + : HLoadClass::LoadKind::kBootImageLinkTimeAddress; + } else { + // Not a boot image class. + DCHECK(ContainsElement(compiler_driver->GetDexFilesForOatFile(), &dex_file)); + desired_load_kind = HLoadClass::LoadKind::kBssEntry; + } } else { - // Not a boot image class. - DCHECK(ContainsElement(compiler_driver->GetDexFilesForOatFile(), &dex_file)); - desired_load_kind = HLoadClass::LoadKind::kBssEntry; - } - } else { - is_in_boot_image = (klass.Get() != nullptr) && - runtime->GetHeap()->ObjectIsInBootImageSpace(klass.Get()); - if (runtime->UseJitCompilation()) { - // TODO: Make sure we don't set the "compile PIC" flag for JIT as that's bogus. - // DCHECK(!codegen_->GetCompilerOptions().GetCompilePic()); - if (is_in_boot_image) { - // TODO: Use direct pointers for all non-moving spaces, not just boot image. Bug: 29530787 + is_in_boot_image = (klass.Get() != nullptr) && + runtime->GetHeap()->ObjectIsInBootImageSpace(klass.Get()); + if (runtime->UseJitCompilation()) { + // TODO: Make sure we don't set the "compile PIC" flag for JIT as that's bogus. + // DCHECK(!codegen_->GetCompilerOptions().GetCompilePic()); + if (is_in_boot_image) { + // TODO: Use direct pointers for all non-moving spaces, not just boot image. Bug: 29530787 + desired_load_kind = HLoadClass::LoadKind::kBootImageAddress; + } else if (klass.Get() != nullptr) { + desired_load_kind = HLoadClass::LoadKind::kJitTableAddress; + } else { + // Class not loaded yet. This happens when the dex code requesting + // this `HLoadClass` hasn't been executed in the interpreter. + // Fallback to the dex cache. + // TODO(ngeoffray): Generate HDeoptimize instead. + desired_load_kind = HLoadClass::LoadKind::kDexCacheViaMethod; + } + } else if (is_in_boot_image && !codegen->GetCompilerOptions().GetCompilePic()) { + // AOT app compilation. Check if the class is in the boot image. desired_load_kind = HLoadClass::LoadKind::kBootImageAddress; - } else if (klass.Get() != nullptr) { - desired_load_kind = HLoadClass::LoadKind::kJitTableAddress; } else { - // Class not loaded yet. This happens when the dex code requesting - // this `HLoadClass` hasn't been executed in the interpreter. - // Fallback to the dex cache. - // TODO(ngeoffray): Generate HDeoptimize instead. - desired_load_kind = HLoadClass::LoadKind::kDexCacheViaMethod; + // Not JIT and either the klass is not in boot image or we are compiling in PIC mode. + desired_load_kind = HLoadClass::LoadKind::kBssEntry; } - } else if (is_in_boot_image && !codegen->GetCompilerOptions().GetCompilePic()) { - // AOT app compilation. Check if the class is in the boot image. - desired_load_kind = HLoadClass::LoadKind::kBootImageAddress; - } else { - // Not JIT and either the klass is not in boot image or we are compiling in PIC mode. - desired_load_kind = HLoadClass::LoadKind::kBssEntry; } - } - DCHECK_NE(desired_load_kind, static_cast<HLoadClass::LoadKind>(-1)); + DCHECK_NE(desired_load_kind, HLoadClass::LoadKind::kInvalid); - if (is_in_boot_image) { - load_class->MarkInBootImage(); + if (is_in_boot_image) { + load_class->MarkInBootImage(); + } + load_kind = codegen->GetSupportedLoadClassKind(desired_load_kind); } - HLoadClass::LoadKind load_kind = codegen->GetSupportedLoadClassKind(desired_load_kind); - switch (load_kind) { - case HLoadClass::LoadKind::kBootImageLinkTimeAddress: - case HLoadClass::LoadKind::kBootImageLinkTimePcRelative: - case HLoadClass::LoadKind::kBssEntry: - case HLoadClass::LoadKind::kDexCacheViaMethod: - load_class->SetLoadKindWithTypeReference(load_kind, dex_file, type_index); - break; - case HLoadClass::LoadKind::kBootImageAddress: - case HLoadClass::LoadKind::kJitTableAddress: - load_class->SetLoadKind(load_kind); - break; - default: - LOG(FATAL) << "Unexpected load kind: " << load_kind; - UNREACHABLE(); + if (!IsSameDexFile(load_class->GetDexFile(), *dex_compilation_unit.GetDexFile())) { + if ((load_kind == HLoadClass::LoadKind::kDexCacheViaMethod) || + (load_kind == HLoadClass::LoadKind::kBssEntry)) { + // We actually cannot reference this class, we're forced to bail. + // We cannot reference this class with Bss, as the entrypoint will lookup the class + // in the caller's dex file, but that dex file does not reference the class. + return HLoadClass::LoadKind::kInvalid; + } } + return load_kind; } void HSharpening::ProcessLoadString(HLoadString* load_string) { diff --git a/compiler/optimizing/sharpening.h b/compiler/optimizing/sharpening.h index ae3d83ef2c..4240b2f339 100644 --- a/compiler/optimizing/sharpening.h +++ b/compiler/optimizing/sharpening.h @@ -17,6 +17,7 @@ #ifndef ART_COMPILER_OPTIMIZING_SHARPENING_H_ #define ART_COMPILER_OPTIMIZING_SHARPENING_H_ +#include "nodes.h" #include "optimization.h" namespace art { @@ -24,7 +25,6 @@ namespace art { class CodeGenerator; class CompilerDriver; class DexCompilationUnit; -class HInvokeStaticOrDirect; // Optimization that tries to improve the way we dispatch methods and access types, // fields, etc. Besides actual method sharpening based on receiver type (for example @@ -47,15 +47,15 @@ class HSharpening : public HOptimization { static constexpr const char* kSharpeningPassName = "sharpening"; - // Used internally but also by the inliner. - static void SharpenClass(HLoadClass* load_class, - CodeGenerator* codegen, - CompilerDriver* compiler_driver) + // Used by the builder and the inliner. + static HLoadClass::LoadKind SharpenClass(HLoadClass* load_class, + CodeGenerator* codegen, + CompilerDriver* compiler_driver, + const DexCompilationUnit& dex_compilation_unit) REQUIRES_SHARED(Locks::mutator_lock_); private: void ProcessInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke); - void ProcessLoadClass(HLoadClass* load_class); void ProcessLoadString(HLoadString* load_string); CodeGenerator* codegen_; diff --git a/compiler/optimizing/ssa_builder.cc b/compiler/optimizing/ssa_builder.cc index ae1e369999..50ab11bc23 100644 --- a/compiler/optimizing/ssa_builder.cc +++ b/compiler/optimizing/ssa_builder.cc @@ -17,8 +17,10 @@ #include "ssa_builder.h" #include "bytecode_utils.h" +#include "mirror/class-inl.h" #include "nodes.h" #include "reference_type_propagation.h" +#include "scoped_thread_state_change-inl.h" #include "ssa_phi_elimination.h" namespace art { @@ -497,7 +499,11 @@ GraphAnalysisResult SsaBuilder::BuildSsa() { // 4) Compute type of reference type instructions. The pass assumes that // NullConstant has been fixed up. - ReferenceTypePropagation(graph_, dex_cache_, handles_, /* is_first_run */ true).Run(); + ReferenceTypePropagation(graph_, + class_loader_, + dex_cache_, + handles_, + /* is_first_run */ true).Run(); // 5) HInstructionBuilder duplicated ArrayGet instructions with ambiguous type // (int/float or long/double) and marked ArraySets with ambiguous input type. diff --git a/compiler/optimizing/ssa_builder.h b/compiler/optimizing/ssa_builder.h index 45dac54115..978f113ec4 100644 --- a/compiler/optimizing/ssa_builder.h +++ b/compiler/optimizing/ssa_builder.h @@ -48,9 +48,11 @@ namespace art { class SsaBuilder : public ValueObject { public: SsaBuilder(HGraph* graph, + Handle<mirror::ClassLoader> class_loader, Handle<mirror::DexCache> dex_cache, VariableSizedHandleScope* handles) : graph_(graph), + class_loader_(class_loader), dex_cache_(dex_cache), handles_(handles), agets_fixed_(false), @@ -115,6 +117,7 @@ class SsaBuilder : public ValueObject { void RemoveRedundantUninitializedStrings(); HGraph* graph_; + Handle<mirror::ClassLoader> class_loader_; Handle<mirror::DexCache> dex_cache_; VariableSizedHandleScope* const handles_; diff --git a/compiler/optimizing/stack_map_stream.cc b/compiler/optimizing/stack_map_stream.cc index 668108daa4..f8e01b7537 100644 --- a/compiler/optimizing/stack_map_stream.cc +++ b/compiler/optimizing/stack_map_stream.cc @@ -18,8 +18,9 @@ #include <unordered_map> +#include "art_method-inl.h" #include "base/stl_util.h" -#include "art_method.h" +#include "optimizing/optimizing_compiler.h" #include "runtime.h" #include "scoped_thread_state_change-inl.h" @@ -107,11 +108,6 @@ void StackMapStream::AddDexRegisterEntry(DexRegisterLocation::Kind kind, int32_t current_dex_register_++; } -static bool EncodeArtMethodInInlineInfo(ArtMethod* method ATTRIBUTE_UNUSED) { - // Note: the runtime is null only for unit testing. - return Runtime::Current() == nullptr || !Runtime::Current()->IsAotCompiler(); -} - void StackMapStream::BeginInlineInfoEntry(ArtMethod* method, uint32_t dex_pc, uint32_t num_dex_registers, @@ -157,56 +153,35 @@ CodeOffset StackMapStream::ComputeMaxNativePcCodeOffset() const { } size_t StackMapStream::PrepareForFillIn() { - const size_t stack_mask_size_in_bits = stack_mask_max_ + 1; // Need room for max element too. - const size_t number_of_stack_masks = PrepareStackMasks(stack_mask_size_in_bits); - const size_t register_mask_size_in_bits = MinimumBitsToStore(register_mask_max_); - const size_t number_of_register_masks = PrepareRegisterMasks(); - dex_register_maps_size_ = ComputeDexRegisterMapsSize(); - ComputeInlineInfoEncoding(); // needs dex_register_maps_size_. - inline_info_size_ = inline_infos_.size() * inline_info_encoding_.GetEntrySize(); + CodeInfoEncoding encoding; + encoding.dex_register_map.num_entries = 0; // TODO: Remove this field. + encoding.dex_register_map.num_bytes = ComputeDexRegisterMapsSize(); + encoding.location_catalog.num_entries = location_catalog_entries_.size(); + encoding.location_catalog.num_bytes = ComputeDexRegisterLocationCatalogSize(); + encoding.inline_info.num_entries = inline_infos_.size(); + ComputeInlineInfoEncoding(&encoding.inline_info.encoding, + encoding.dex_register_map.num_bytes); CodeOffset max_native_pc_offset = ComputeMaxNativePcCodeOffset(); - // The stack map contains compressed native PC offsets. - const size_t stack_map_size = stack_map_encoding_.SetFromSizes( + // Prepare the CodeInfo variable-sized encoding. + encoding.stack_mask.encoding.num_bits = stack_mask_max_ + 1; // Need room for max element too. + encoding.stack_mask.num_entries = PrepareStackMasks(encoding.stack_mask.encoding.num_bits); + encoding.register_mask.encoding.num_bits = MinimumBitsToStore(register_mask_max_); + encoding.register_mask.num_entries = PrepareRegisterMasks(); + encoding.stack_map.num_entries = stack_maps_.size(); + encoding.stack_map.encoding.SetFromSizes( + // The stack map contains compressed native PC offsets. max_native_pc_offset.CompressedValue(), dex_pc_max_, - dex_register_maps_size_, - inline_info_size_, - number_of_register_masks, - number_of_stack_masks); - stack_maps_size_ = RoundUp(stack_maps_.size() * stack_map_size, kBitsPerByte) / kBitsPerByte; - dex_register_location_catalog_size_ = ComputeDexRegisterLocationCatalogSize(); - const size_t stack_masks_bits = number_of_stack_masks * stack_mask_size_in_bits; - const size_t register_masks_bits = number_of_register_masks * register_mask_size_in_bits; - // Register masks are last, stack masks are right before that last. - // They are both bit packed / aligned. - const size_t non_header_size = - stack_maps_size_ + - dex_register_location_catalog_size_ + - dex_register_maps_size_ + - inline_info_size_ + - RoundUp(stack_masks_bits + register_masks_bits, kBitsPerByte) / kBitsPerByte; - - // Prepare the CodeInfo variable-sized encoding. - CodeInfoEncoding code_info_encoding; - code_info_encoding.non_header_size = non_header_size; - code_info_encoding.number_of_stack_maps = stack_maps_.size(); - code_info_encoding.number_of_stack_masks = number_of_stack_masks; - code_info_encoding.number_of_register_masks = number_of_register_masks; - code_info_encoding.stack_mask_size_in_bits = stack_mask_size_in_bits; - code_info_encoding.register_mask_size_in_bits = register_mask_size_in_bits; - code_info_encoding.stack_map_encoding = stack_map_encoding_; - code_info_encoding.inline_info_encoding = inline_info_encoding_; - code_info_encoding.number_of_location_catalog_entries = location_catalog_entries_.size(); - code_info_encoding.Compress(&code_info_encoding_); - - // TODO: Move the catalog at the end. It is currently too expensive at runtime - // to compute its size (note that we do not encode that size in the CodeInfo). - dex_register_location_catalog_start_ = code_info_encoding_.size() + stack_maps_size_; - dex_register_maps_start_ = - dex_register_location_catalog_start_ + dex_register_location_catalog_size_; - inline_infos_start_ = dex_register_maps_start_ + dex_register_maps_size_; - - needed_size_ = code_info_encoding_.size() + non_header_size; + encoding.dex_register_map.num_bytes, + encoding.inline_info.num_entries, + encoding.register_mask.num_entries, + encoding.stack_mask.num_entries); + DCHECK_EQ(code_info_encoding_.size(), 0u); + encoding.Compress(&code_info_encoding_); + encoding.ComputeTableOffsets(); + // Compute table offsets so we can get the non header size. + DCHECK_EQ(encoding.HeaderSize(), code_info_encoding_.size()); + needed_size_ = code_info_encoding_.size() + encoding.NonHeaderSize(); return needed_size_; } @@ -259,7 +234,8 @@ size_t StackMapStream::ComputeDexRegisterMapsSize() const { return size; } -void StackMapStream::ComputeInlineInfoEncoding() { +void StackMapStream::ComputeInlineInfoEncoding(InlineInfoEncoding* encoding, + size_t dex_register_maps_bytes) { uint32_t method_index_max = 0; uint32_t dex_pc_max = DexFile::kDexNoIndex; uint32_t extra_data_max = 0; @@ -285,10 +261,7 @@ void StackMapStream::ComputeInlineInfoEncoding() { } DCHECK_EQ(inline_info_index, inline_infos_.size()); - inline_info_encoding_.SetFromSizes(method_index_max, - dex_pc_max, - extra_data_max, - dex_register_maps_size_); + encoding->SetFromSizes(method_index_max, dex_pc_max, extra_data_max, dex_register_maps_bytes); } void StackMapStream::FillIn(MemoryRegion region) { @@ -303,19 +276,18 @@ void StackMapStream::FillIn(MemoryRegion region) { // Write the CodeInfo header. region.CopyFrom(0, MemoryRegion(code_info_encoding_.data(), code_info_encoding_.size())); - MemoryRegion dex_register_locations_region = region.Subregion( - dex_register_maps_start_, dex_register_maps_size_); - - MemoryRegion inline_infos_region = region.Subregion( - inline_infos_start_, inline_info_size_); - CodeInfo code_info(region); CodeInfoEncoding encoding = code_info.ExtractEncoding(); - DCHECK_EQ(code_info.GetStackMapsSize(encoding), stack_maps_size_); + DCHECK_EQ(encoding.stack_map.num_entries, stack_maps_.size()); + + MemoryRegion dex_register_locations_region = region.Subregion( + encoding.dex_register_map.byte_offset, + encoding.dex_register_map.num_bytes); // Set the Dex register location catalog. MemoryRegion dex_register_location_catalog_region = region.Subregion( - dex_register_location_catalog_start_, dex_register_location_catalog_size_); + encoding.location_catalog.byte_offset, + encoding.location_catalog.num_bytes); DexRegisterLocationCatalog dex_register_location_catalog(dex_register_location_catalog_region); // Offset in `dex_register_location_catalog` where to store the next // register location. @@ -329,27 +301,27 @@ void StackMapStream::FillIn(MemoryRegion region) { ArenaBitVector empty_bitmask(allocator_, 0, /* expandable */ false, kArenaAllocStackMapStream); uintptr_t next_dex_register_map_offset = 0; - uintptr_t next_inline_info_offset = 0; + uintptr_t next_inline_info_index = 0; for (size_t i = 0, e = stack_maps_.size(); i < e; ++i) { StackMap stack_map = code_info.GetStackMapAt(i, encoding); StackMapEntry entry = stack_maps_[i]; - stack_map.SetDexPc(stack_map_encoding_, entry.dex_pc); - stack_map.SetNativePcCodeOffset(stack_map_encoding_, entry.native_pc_code_offset); - stack_map.SetRegisterMaskIndex(stack_map_encoding_, entry.register_mask_index); - stack_map.SetStackMaskIndex(stack_map_encoding_, entry.stack_mask_index); + stack_map.SetDexPc(encoding.stack_map.encoding, entry.dex_pc); + stack_map.SetNativePcCodeOffset(encoding.stack_map.encoding, entry.native_pc_code_offset); + stack_map.SetRegisterMaskIndex(encoding.stack_map.encoding, entry.register_mask_index); + stack_map.SetStackMaskIndex(encoding.stack_map.encoding, entry.stack_mask_index); if (entry.num_dex_registers == 0 || (entry.live_dex_registers_mask->NumSetBits() == 0)) { // No dex map available. - stack_map.SetDexRegisterMapOffset(stack_map_encoding_, StackMap::kNoDexRegisterMap); + stack_map.SetDexRegisterMapOffset(encoding.stack_map.encoding, StackMap::kNoDexRegisterMap); } else { // Search for an entry with the same dex map. if (entry.same_dex_register_map_as_ != kNoSameDexMapFound) { // If we have a hit reuse the offset. stack_map.SetDexRegisterMapOffset( - stack_map_encoding_, + encoding.stack_map.encoding, code_info.GetStackMapAt(entry.same_dex_register_map_as_, encoding) - .GetDexRegisterMapOffset(stack_map_encoding_)); + .GetDexRegisterMapOffset(encoding.stack_map.encoding)); } else { // New dex registers maps should be added to the stack map. MemoryRegion register_region = dex_register_locations_region.Subregion( @@ -358,7 +330,8 @@ void StackMapStream::FillIn(MemoryRegion region) { next_dex_register_map_offset += register_region.size(); DexRegisterMap dex_register_map(register_region); stack_map.SetDexRegisterMapOffset( - stack_map_encoding_, register_region.begin() - dex_register_locations_region.begin()); + encoding.stack_map.encoding, + register_region.begin() - dex_register_locations_region.begin()); // Set the dex register location. FillInDexRegisterMap(dex_register_map, @@ -370,37 +343,37 @@ void StackMapStream::FillIn(MemoryRegion region) { // Set the inlining info. if (entry.inlining_depth != 0) { - MemoryRegion inline_region = inline_infos_region.Subregion( - next_inline_info_offset, - entry.inlining_depth * inline_info_encoding_.GetEntrySize()); - next_inline_info_offset += inline_region.size(); - InlineInfo inline_info(inline_region); + InlineInfo inline_info = code_info.GetInlineInfo(next_inline_info_index, encoding); - // Currently relative to the dex register map. - stack_map.SetInlineDescriptorOffset( - stack_map_encoding_, inline_region.begin() - dex_register_locations_region.begin()); + // Fill in the index. + stack_map.SetInlineInfoIndex(encoding.stack_map.encoding, next_inline_info_index); + DCHECK_EQ(next_inline_info_index, entry.inline_infos_start_index); + next_inline_info_index += entry.inlining_depth; - inline_info.SetDepth(inline_info_encoding_, entry.inlining_depth); + inline_info.SetDepth(encoding.inline_info.encoding, entry.inlining_depth); DCHECK_LE(entry.inline_infos_start_index + entry.inlining_depth, inline_infos_.size()); + for (size_t depth = 0; depth < entry.inlining_depth; ++depth) { InlineInfoEntry inline_entry = inline_infos_[depth + entry.inline_infos_start_index]; if (inline_entry.method != nullptr) { inline_info.SetMethodIndexAtDepth( - inline_info_encoding_, + encoding.inline_info.encoding, depth, High32Bits(reinterpret_cast<uintptr_t>(inline_entry.method))); inline_info.SetExtraDataAtDepth( - inline_info_encoding_, + encoding.inline_info.encoding, depth, Low32Bits(reinterpret_cast<uintptr_t>(inline_entry.method))); } else { - inline_info.SetMethodIndexAtDepth(inline_info_encoding_, depth, inline_entry.method_index); - inline_info.SetExtraDataAtDepth(inline_info_encoding_, depth, 1); + inline_info.SetMethodIndexAtDepth(encoding.inline_info.encoding, + depth, + inline_entry.method_index); + inline_info.SetExtraDataAtDepth(encoding.inline_info.encoding, depth, 1); } - inline_info.SetDexPcAtDepth(inline_info_encoding_, depth, inline_entry.dex_pc); + inline_info.SetDexPcAtDepth(encoding.inline_info.encoding, depth, inline_entry.dex_pc); if (inline_entry.num_dex_registers == 0) { // No dex map available. - inline_info.SetDexRegisterMapOffsetAtDepth(inline_info_encoding_, + inline_info.SetDexRegisterMapOffsetAtDepth(encoding.inline_info.encoding, depth, StackMap::kNoDexRegisterMap); DCHECK(inline_entry.live_dex_registers_mask == nullptr); @@ -412,8 +385,9 @@ void StackMapStream::FillIn(MemoryRegion region) { next_dex_register_map_offset += register_region.size(); DexRegisterMap dex_register_map(register_region); inline_info.SetDexRegisterMapOffsetAtDepth( - inline_info_encoding_, - depth, register_region.begin() - dex_register_locations_region.begin()); + encoding.inline_info.encoding, + depth, + register_region.begin() - dex_register_locations_region.begin()); FillInDexRegisterMap(dex_register_map, inline_entry.num_dex_registers, @@ -421,30 +395,28 @@ void StackMapStream::FillIn(MemoryRegion region) { inline_entry.dex_register_locations_start_index); } } - } else { - if (inline_info_size_ != 0) { - stack_map.SetInlineDescriptorOffset(stack_map_encoding_, StackMap::kNoInlineInfo); - } + } else if (encoding.stack_map.encoding.GetInlineInfoEncoding().BitSize() > 0) { + stack_map.SetInlineInfoIndex(encoding.stack_map.encoding, StackMap::kNoInlineInfo); } } // Write stack masks table. - size_t stack_mask_bits = encoding.stack_mask_size_in_bits; + const size_t stack_mask_bits = encoding.stack_mask.encoding.BitSize(); if (stack_mask_bits > 0) { size_t stack_mask_bytes = RoundUp(stack_mask_bits, kBitsPerByte) / kBitsPerByte; - for (size_t i = 0; i < encoding.number_of_stack_masks; ++i) { + for (size_t i = 0; i < encoding.stack_mask.num_entries; ++i) { MemoryRegion source(&stack_masks_[i * stack_mask_bytes], stack_mask_bytes); - BitMemoryRegion stack_mask = code_info.GetStackMask(encoding, i); - for (size_t bit_index = 0; bit_index < encoding.stack_mask_size_in_bits; ++bit_index) { + BitMemoryRegion stack_mask = code_info.GetStackMask(i, encoding); + for (size_t bit_index = 0; bit_index < stack_mask_bits; ++bit_index) { stack_mask.StoreBit(bit_index, source.LoadBit(bit_index)); } } } // Write register masks table. - for (size_t i = 0; i < encoding.number_of_register_masks; ++i) { - BitMemoryRegion register_mask = code_info.GetRegisterMask(encoding, i); - register_mask.StoreBits(0, register_masks_[i], encoding.register_mask_size_in_bits); + for (size_t i = 0; i < encoding.register_mask.num_entries; ++i) { + BitMemoryRegion register_mask = code_info.GetRegisterMask(i, encoding); + register_mask.StoreBits(0, register_masks_[i], encoding.register_mask.encoding.BitSize()); } // Verify all written data in debug build. @@ -546,7 +518,8 @@ void StackMapStream::CheckDexRegisterMap(const CodeInfo& code_info, } // Compare to the seen location. if (expected.GetKind() == DexRegisterLocation::Kind::kNone) { - DCHECK(!dex_register_map.IsValid() || !dex_register_map.IsDexRegisterLive(reg)); + DCHECK(!dex_register_map.IsValid() || !dex_register_map.IsDexRegisterLive(reg)) + << dex_register_map.IsValid() << " " << dex_register_map.IsDexRegisterLive(reg); } else { DCHECK(dex_register_map.IsDexRegisterLive(reg)); DexRegisterLocation seen = dex_register_map.GetDexRegisterLocation( @@ -599,7 +572,7 @@ void StackMapStream::CheckCodeInfo(MemoryRegion region) const { DCHECK_EQ(code_info.GetNumberOfStackMaps(encoding), stack_maps_.size()); for (size_t s = 0; s < stack_maps_.size(); ++s) { const StackMap stack_map = code_info.GetStackMapAt(s, encoding); - const StackMapEncoding& stack_map_encoding = encoding.stack_map_encoding; + const StackMapEncoding& stack_map_encoding = encoding.stack_map.encoding; StackMapEntry entry = stack_maps_[s]; // Check main stack map fields. @@ -633,18 +606,18 @@ void StackMapStream::CheckCodeInfo(MemoryRegion region) const { DCHECK_EQ(stack_map.HasInlineInfo(stack_map_encoding), (entry.inlining_depth != 0)); if (entry.inlining_depth != 0) { InlineInfo inline_info = code_info.GetInlineInfoOf(stack_map, encoding); - DCHECK_EQ(inline_info.GetDepth(encoding.inline_info_encoding), entry.inlining_depth); + DCHECK_EQ(inline_info.GetDepth(encoding.inline_info.encoding), entry.inlining_depth); for (size_t d = 0; d < entry.inlining_depth; ++d) { size_t inline_info_index = entry.inline_infos_start_index + d; DCHECK_LT(inline_info_index, inline_infos_.size()); InlineInfoEntry inline_entry = inline_infos_[inline_info_index]; - DCHECK_EQ(inline_info.GetDexPcAtDepth(encoding.inline_info_encoding, d), + DCHECK_EQ(inline_info.GetDexPcAtDepth(encoding.inline_info.encoding, d), inline_entry.dex_pc); - if (inline_info.EncodesArtMethodAtDepth(encoding.inline_info_encoding, d)) { - DCHECK_EQ(inline_info.GetArtMethodAtDepth(encoding.inline_info_encoding, d), + if (inline_info.EncodesArtMethodAtDepth(encoding.inline_info.encoding, d)) { + DCHECK_EQ(inline_info.GetArtMethodAtDepth(encoding.inline_info.encoding, d), inline_entry.method); } else { - DCHECK_EQ(inline_info.GetMethodIndexAtDepth(encoding.inline_info_encoding, d), + DCHECK_EQ(inline_info.GetMethodIndexAtDepth(encoding.inline_info.encoding, d), inline_entry.method_index); } diff --git a/compiler/optimizing/stack_map_stream.h b/compiler/optimizing/stack_map_stream.h index b1069a17be..08c1d3e3c0 100644 --- a/compiler/optimizing/stack_map_stream.h +++ b/compiler/optimizing/stack_map_stream.h @@ -79,13 +79,6 @@ class StackMapStream : public ValueObject { current_entry_(), current_inline_info_(), code_info_encoding_(allocator->Adapter(kArenaAllocStackMapStream)), - inline_info_size_(0), - dex_register_maps_size_(0), - stack_maps_size_(0), - dex_register_location_catalog_size_(0), - dex_register_location_catalog_start_(0), - dex_register_maps_start_(0), - inline_infos_start_(0), needed_size_(0), current_dex_register_(0), in_inline_frame_(false) { @@ -160,7 +153,8 @@ class StackMapStream : public ValueObject { size_t ComputeDexRegisterMapSize(uint32_t num_dex_registers, const BitVector* live_dex_registers_mask) const; size_t ComputeDexRegisterMapsSize() const; - void ComputeInlineInfoEncoding(); + void ComputeInlineInfoEncoding(InlineInfoEncoding* encoding, + size_t dex_register_maps_bytes); CodeOffset ComputeMaxNativePcCodeOffset() const; @@ -214,16 +208,7 @@ class StackMapStream : public ValueObject { StackMapEntry current_entry_; InlineInfoEntry current_inline_info_; - StackMapEncoding stack_map_encoding_; - InlineInfoEncoding inline_info_encoding_; ArenaVector<uint8_t> code_info_encoding_; - size_t inline_info_size_; - size_t dex_register_maps_size_; - size_t stack_maps_size_; - size_t dex_register_location_catalog_size_; - size_t dex_register_location_catalog_start_; - size_t dex_register_maps_start_; - size_t inline_infos_start_; size_t needed_size_; uint32_t current_dex_register_; bool in_inline_frame_; diff --git a/compiler/optimizing/stack_map_test.cc b/compiler/optimizing/stack_map_test.cc index ce6d5c2b22..bd0aa6dea7 100644 --- a/compiler/optimizing/stack_map_test.cc +++ b/compiler/optimizing/stack_map_test.cc @@ -32,10 +32,10 @@ static bool CheckStackMask( const StackMap& stack_map, const BitVector& bit_vector) { BitMemoryRegion stack_mask = code_info.GetStackMaskOf(encoding, stack_map); - if (bit_vector.GetNumberOfBits() > encoding.stack_mask_size_in_bits) { + if (bit_vector.GetNumberOfBits() > encoding.stack_mask.encoding.BitSize()) { return false; } - for (size_t i = 0; i < encoding.stack_mask_size_in_bits; ++i) { + for (size_t i = 0; i < encoding.stack_mask.encoding.BitSize(); ++i) { if (stack_mask.LoadBit(i) != bit_vector.IsBitSet(i)) { return false; } @@ -78,13 +78,13 @@ TEST(StackMapTest, Test1) { StackMap stack_map = code_info.GetStackMapAt(0, encoding); ASSERT_TRUE(stack_map.Equals(code_info.GetStackMapForDexPc(0, encoding))); ASSERT_TRUE(stack_map.Equals(code_info.GetStackMapForNativePcOffset(64, encoding))); - ASSERT_EQ(0u, stack_map.GetDexPc(encoding.stack_map_encoding)); - ASSERT_EQ(64u, stack_map.GetNativePcOffset(encoding.stack_map_encoding, kRuntimeISA)); + ASSERT_EQ(0u, stack_map.GetDexPc(encoding.stack_map.encoding)); + ASSERT_EQ(64u, stack_map.GetNativePcOffset(encoding.stack_map.encoding, kRuntimeISA)); ASSERT_EQ(0x3u, code_info.GetRegisterMaskOf(encoding, stack_map)); ASSERT_TRUE(CheckStackMask(code_info, encoding, stack_map, sp_mask)); - ASSERT_TRUE(stack_map.HasDexRegisterMap(encoding.stack_map_encoding)); + ASSERT_TRUE(stack_map.HasDexRegisterMap(encoding.stack_map.encoding)); DexRegisterMap dex_register_map = code_info.GetDexRegisterMapOf(stack_map, encoding, number_of_dex_registers); ASSERT_TRUE(dex_register_map.IsDexRegisterLive(0)); @@ -123,7 +123,7 @@ TEST(StackMapTest, Test1) { ASSERT_EQ(0, location0.GetValue()); ASSERT_EQ(-2, location1.GetValue()); - ASSERT_FALSE(stack_map.HasInlineInfo(encoding.stack_map_encoding)); + ASSERT_FALSE(stack_map.HasInlineInfo(encoding.stack_map.encoding)); } TEST(StackMapTest, Test2) { @@ -193,13 +193,13 @@ TEST(StackMapTest, Test2) { StackMap stack_map = code_info.GetStackMapAt(0, encoding); ASSERT_TRUE(stack_map.Equals(code_info.GetStackMapForDexPc(0, encoding))); ASSERT_TRUE(stack_map.Equals(code_info.GetStackMapForNativePcOffset(64, encoding))); - ASSERT_EQ(0u, stack_map.GetDexPc(encoding.stack_map_encoding)); - ASSERT_EQ(64u, stack_map.GetNativePcOffset(encoding.stack_map_encoding, kRuntimeISA)); + ASSERT_EQ(0u, stack_map.GetDexPc(encoding.stack_map.encoding)); + ASSERT_EQ(64u, stack_map.GetNativePcOffset(encoding.stack_map.encoding, kRuntimeISA)); ASSERT_EQ(0x3u, code_info.GetRegisterMaskOf(encoding, stack_map)); ASSERT_TRUE(CheckStackMask(code_info, encoding, stack_map, sp_mask1)); - ASSERT_TRUE(stack_map.HasDexRegisterMap(encoding.stack_map_encoding)); + ASSERT_TRUE(stack_map.HasDexRegisterMap(encoding.stack_map.encoding)); DexRegisterMap dex_register_map = code_info.GetDexRegisterMapOf(stack_map, encoding, number_of_dex_registers); ASSERT_TRUE(dex_register_map.IsDexRegisterLive(0)); @@ -238,13 +238,13 @@ TEST(StackMapTest, Test2) { ASSERT_EQ(0, location0.GetValue()); ASSERT_EQ(-2, location1.GetValue()); - ASSERT_TRUE(stack_map.HasInlineInfo(encoding.stack_map_encoding)); + ASSERT_TRUE(stack_map.HasInlineInfo(encoding.stack_map.encoding)); InlineInfo inline_info = code_info.GetInlineInfoOf(stack_map, encoding); - ASSERT_EQ(2u, inline_info.GetDepth(encoding.inline_info_encoding)); - ASSERT_EQ(3u, inline_info.GetDexPcAtDepth(encoding.inline_info_encoding, 0)); - ASSERT_EQ(2u, inline_info.GetDexPcAtDepth(encoding.inline_info_encoding, 1)); - ASSERT_TRUE(inline_info.EncodesArtMethodAtDepth(encoding.inline_info_encoding, 0)); - ASSERT_TRUE(inline_info.EncodesArtMethodAtDepth(encoding.inline_info_encoding, 1)); + ASSERT_EQ(2u, inline_info.GetDepth(encoding.inline_info.encoding)); + ASSERT_EQ(3u, inline_info.GetDexPcAtDepth(encoding.inline_info.encoding, 0)); + ASSERT_EQ(2u, inline_info.GetDexPcAtDepth(encoding.inline_info.encoding, 1)); + ASSERT_TRUE(inline_info.EncodesArtMethodAtDepth(encoding.inline_info.encoding, 0)); + ASSERT_TRUE(inline_info.EncodesArtMethodAtDepth(encoding.inline_info.encoding, 1)); } // Second stack map. @@ -252,13 +252,13 @@ TEST(StackMapTest, Test2) { StackMap stack_map = code_info.GetStackMapAt(1, encoding); ASSERT_TRUE(stack_map.Equals(code_info.GetStackMapForDexPc(1u, encoding))); ASSERT_TRUE(stack_map.Equals(code_info.GetStackMapForNativePcOffset(128u, encoding))); - ASSERT_EQ(1u, stack_map.GetDexPc(encoding.stack_map_encoding)); - ASSERT_EQ(128u, stack_map.GetNativePcOffset(encoding.stack_map_encoding, kRuntimeISA)); + ASSERT_EQ(1u, stack_map.GetDexPc(encoding.stack_map.encoding)); + ASSERT_EQ(128u, stack_map.GetNativePcOffset(encoding.stack_map.encoding, kRuntimeISA)); ASSERT_EQ(0xFFu, code_info.GetRegisterMaskOf(encoding, stack_map)); ASSERT_TRUE(CheckStackMask(code_info, encoding, stack_map, sp_mask2)); - ASSERT_TRUE(stack_map.HasDexRegisterMap(encoding.stack_map_encoding)); + ASSERT_TRUE(stack_map.HasDexRegisterMap(encoding.stack_map.encoding)); DexRegisterMap dex_register_map = code_info.GetDexRegisterMapOf(stack_map, encoding, number_of_dex_registers); ASSERT_TRUE(dex_register_map.IsDexRegisterLive(0)); @@ -298,7 +298,7 @@ TEST(StackMapTest, Test2) { ASSERT_EQ(18, location0.GetValue()); ASSERT_EQ(3, location1.GetValue()); - ASSERT_FALSE(stack_map.HasInlineInfo(encoding.stack_map_encoding)); + ASSERT_FALSE(stack_map.HasInlineInfo(encoding.stack_map.encoding)); } // Third stack map. @@ -306,13 +306,13 @@ TEST(StackMapTest, Test2) { StackMap stack_map = code_info.GetStackMapAt(2, encoding); ASSERT_TRUE(stack_map.Equals(code_info.GetStackMapForDexPc(2u, encoding))); ASSERT_TRUE(stack_map.Equals(code_info.GetStackMapForNativePcOffset(192u, encoding))); - ASSERT_EQ(2u, stack_map.GetDexPc(encoding.stack_map_encoding)); - ASSERT_EQ(192u, stack_map.GetNativePcOffset(encoding.stack_map_encoding, kRuntimeISA)); + ASSERT_EQ(2u, stack_map.GetDexPc(encoding.stack_map.encoding)); + ASSERT_EQ(192u, stack_map.GetNativePcOffset(encoding.stack_map.encoding, kRuntimeISA)); ASSERT_EQ(0xABu, code_info.GetRegisterMaskOf(encoding, stack_map)); ASSERT_TRUE(CheckStackMask(code_info, encoding, stack_map, sp_mask3)); - ASSERT_TRUE(stack_map.HasDexRegisterMap(encoding.stack_map_encoding)); + ASSERT_TRUE(stack_map.HasDexRegisterMap(encoding.stack_map.encoding)); DexRegisterMap dex_register_map = code_info.GetDexRegisterMapOf(stack_map, encoding, number_of_dex_registers); ASSERT_TRUE(dex_register_map.IsDexRegisterLive(0)); @@ -352,7 +352,7 @@ TEST(StackMapTest, Test2) { ASSERT_EQ(6, location0.GetValue()); ASSERT_EQ(8, location1.GetValue()); - ASSERT_FALSE(stack_map.HasInlineInfo(encoding.stack_map_encoding)); + ASSERT_FALSE(stack_map.HasInlineInfo(encoding.stack_map.encoding)); } // Fourth stack map. @@ -360,13 +360,13 @@ TEST(StackMapTest, Test2) { StackMap stack_map = code_info.GetStackMapAt(3, encoding); ASSERT_TRUE(stack_map.Equals(code_info.GetStackMapForDexPc(3u, encoding))); ASSERT_TRUE(stack_map.Equals(code_info.GetStackMapForNativePcOffset(256u, encoding))); - ASSERT_EQ(3u, stack_map.GetDexPc(encoding.stack_map_encoding)); - ASSERT_EQ(256u, stack_map.GetNativePcOffset(encoding.stack_map_encoding, kRuntimeISA)); + ASSERT_EQ(3u, stack_map.GetDexPc(encoding.stack_map.encoding)); + ASSERT_EQ(256u, stack_map.GetNativePcOffset(encoding.stack_map.encoding, kRuntimeISA)); ASSERT_EQ(0xCDu, code_info.GetRegisterMaskOf(encoding, stack_map)); ASSERT_TRUE(CheckStackMask(code_info, encoding, stack_map, sp_mask4)); - ASSERT_TRUE(stack_map.HasDexRegisterMap(encoding.stack_map_encoding)); + ASSERT_TRUE(stack_map.HasDexRegisterMap(encoding.stack_map.encoding)); DexRegisterMap dex_register_map = code_info.GetDexRegisterMapOf(stack_map, encoding, number_of_dex_registers); ASSERT_TRUE(dex_register_map.IsDexRegisterLive(0)); @@ -406,7 +406,7 @@ TEST(StackMapTest, Test2) { ASSERT_EQ(3, location0.GetValue()); ASSERT_EQ(1, location1.GetValue()); - ASSERT_FALSE(stack_map.HasInlineInfo(encoding.stack_map_encoding)); + ASSERT_FALSE(stack_map.HasInlineInfo(encoding.stack_map.encoding)); } } @@ -442,11 +442,11 @@ TEST(StackMapTest, TestNonLiveDexRegisters) { StackMap stack_map = code_info.GetStackMapAt(0, encoding); ASSERT_TRUE(stack_map.Equals(code_info.GetStackMapForDexPc(0, encoding))); ASSERT_TRUE(stack_map.Equals(code_info.GetStackMapForNativePcOffset(64, encoding))); - ASSERT_EQ(0u, stack_map.GetDexPc(encoding.stack_map_encoding)); - ASSERT_EQ(64u, stack_map.GetNativePcOffset(encoding.stack_map_encoding, kRuntimeISA)); + ASSERT_EQ(0u, stack_map.GetDexPc(encoding.stack_map.encoding)); + ASSERT_EQ(64u, stack_map.GetNativePcOffset(encoding.stack_map.encoding, kRuntimeISA)); ASSERT_EQ(0x3u, code_info.GetRegisterMaskOf(encoding, stack_map)); - ASSERT_TRUE(stack_map.HasDexRegisterMap(encoding.stack_map_encoding)); + ASSERT_TRUE(stack_map.HasDexRegisterMap(encoding.stack_map.encoding)); DexRegisterMap dex_register_map = code_info.GetDexRegisterMapOf(stack_map, encoding, number_of_dex_registers); ASSERT_FALSE(dex_register_map.IsDexRegisterLive(0)); @@ -483,7 +483,7 @@ TEST(StackMapTest, TestNonLiveDexRegisters) { ASSERT_EQ(0, location0.GetValue()); ASSERT_EQ(-2, location1.GetValue()); - ASSERT_FALSE(stack_map.HasInlineInfo(encoding.stack_map_encoding)); + ASSERT_FALSE(stack_map.HasInlineInfo(encoding.stack_map.encoding)); } // Generate a stack map whose dex register offset is @@ -543,13 +543,13 @@ TEST(StackMapTest, DexRegisterMapOffsetOverflow) { ASSERT_EQ(255u, dex_register_map0.Size()); StackMap stack_map1 = code_info.GetStackMapAt(1, encoding); - ASSERT_TRUE(stack_map1.HasDexRegisterMap(encoding.stack_map_encoding)); + ASSERT_TRUE(stack_map1.HasDexRegisterMap(encoding.stack_map.encoding)); // ...the offset of the second Dex register map (relative to the // beginning of the Dex register maps region) is 255 (i.e., // kNoDexRegisterMapSmallEncoding). - ASSERT_NE(stack_map1.GetDexRegisterMapOffset(encoding.stack_map_encoding), + ASSERT_NE(stack_map1.GetDexRegisterMapOffset(encoding.stack_map.encoding), StackMap::kNoDexRegisterMap); - ASSERT_EQ(stack_map1.GetDexRegisterMapOffset(encoding.stack_map_encoding), 0xFFu); + ASSERT_EQ(stack_map1.GetDexRegisterMapOffset(encoding.stack_map.encoding), 0xFFu); } TEST(StackMapTest, TestShareDexRegisterMap) { @@ -602,12 +602,12 @@ TEST(StackMapTest, TestShareDexRegisterMap) { ASSERT_EQ(-2, dex_registers2.GetConstant(1, number_of_dex_registers, ci, encoding)); // Verify dex register map offsets. - ASSERT_EQ(sm0.GetDexRegisterMapOffset(encoding.stack_map_encoding), - sm1.GetDexRegisterMapOffset(encoding.stack_map_encoding)); - ASSERT_NE(sm0.GetDexRegisterMapOffset(encoding.stack_map_encoding), - sm2.GetDexRegisterMapOffset(encoding.stack_map_encoding)); - ASSERT_NE(sm1.GetDexRegisterMapOffset(encoding.stack_map_encoding), - sm2.GetDexRegisterMapOffset(encoding.stack_map_encoding)); + ASSERT_EQ(sm0.GetDexRegisterMapOffset(encoding.stack_map.encoding), + sm1.GetDexRegisterMapOffset(encoding.stack_map.encoding)); + ASSERT_NE(sm0.GetDexRegisterMapOffset(encoding.stack_map.encoding), + sm2.GetDexRegisterMapOffset(encoding.stack_map.encoding)); + ASSERT_NE(sm1.GetDexRegisterMapOffset(encoding.stack_map.encoding), + sm2.GetDexRegisterMapOffset(encoding.stack_map.encoding)); } TEST(StackMapTest, TestNoDexRegisterMap) { @@ -641,22 +641,22 @@ TEST(StackMapTest, TestNoDexRegisterMap) { StackMap stack_map = code_info.GetStackMapAt(0, encoding); ASSERT_TRUE(stack_map.Equals(code_info.GetStackMapForDexPc(0, encoding))); ASSERT_TRUE(stack_map.Equals(code_info.GetStackMapForNativePcOffset(64, encoding))); - ASSERT_EQ(0u, stack_map.GetDexPc(encoding.stack_map_encoding)); - ASSERT_EQ(64u, stack_map.GetNativePcOffset(encoding.stack_map_encoding, kRuntimeISA)); + ASSERT_EQ(0u, stack_map.GetDexPc(encoding.stack_map.encoding)); + ASSERT_EQ(64u, stack_map.GetNativePcOffset(encoding.stack_map.encoding, kRuntimeISA)); ASSERT_EQ(0x3u, code_info.GetRegisterMaskOf(encoding, stack_map)); - ASSERT_FALSE(stack_map.HasDexRegisterMap(encoding.stack_map_encoding)); - ASSERT_FALSE(stack_map.HasInlineInfo(encoding.stack_map_encoding)); + ASSERT_FALSE(stack_map.HasDexRegisterMap(encoding.stack_map.encoding)); + ASSERT_FALSE(stack_map.HasInlineInfo(encoding.stack_map.encoding)); stack_map = code_info.GetStackMapAt(1, encoding); ASSERT_TRUE(stack_map.Equals(code_info.GetStackMapForDexPc(1, encoding))); ASSERT_TRUE(stack_map.Equals(code_info.GetStackMapForNativePcOffset(68, encoding))); - ASSERT_EQ(1u, stack_map.GetDexPc(encoding.stack_map_encoding)); - ASSERT_EQ(68u, stack_map.GetNativePcOffset(encoding.stack_map_encoding, kRuntimeISA)); + ASSERT_EQ(1u, stack_map.GetDexPc(encoding.stack_map.encoding)); + ASSERT_EQ(68u, stack_map.GetNativePcOffset(encoding.stack_map.encoding, kRuntimeISA)); ASSERT_EQ(0x4u, code_info.GetRegisterMaskOf(encoding, stack_map)); - ASSERT_FALSE(stack_map.HasDexRegisterMap(encoding.stack_map_encoding)); - ASSERT_FALSE(stack_map.HasInlineInfo(encoding.stack_map_encoding)); + ASSERT_FALSE(stack_map.HasDexRegisterMap(encoding.stack_map.encoding)); + ASSERT_FALSE(stack_map.HasInlineInfo(encoding.stack_map.encoding)); } TEST(StackMapTest, InlineTest) { @@ -743,11 +743,11 @@ TEST(StackMapTest, InlineTest) { ASSERT_EQ(4, dex_registers0.GetConstant(1, 2, ci, encoding)); InlineInfo if0 = ci.GetInlineInfoOf(sm0, encoding); - ASSERT_EQ(2u, if0.GetDepth(encoding.inline_info_encoding)); - ASSERT_EQ(2u, if0.GetDexPcAtDepth(encoding.inline_info_encoding, 0)); - ASSERT_TRUE(if0.EncodesArtMethodAtDepth(encoding.inline_info_encoding, 0)); - ASSERT_EQ(3u, if0.GetDexPcAtDepth(encoding.inline_info_encoding, 1)); - ASSERT_TRUE(if0.EncodesArtMethodAtDepth(encoding.inline_info_encoding, 1)); + ASSERT_EQ(2u, if0.GetDepth(encoding.inline_info.encoding)); + ASSERT_EQ(2u, if0.GetDexPcAtDepth(encoding.inline_info.encoding, 0)); + ASSERT_TRUE(if0.EncodesArtMethodAtDepth(encoding.inline_info.encoding, 0)); + ASSERT_EQ(3u, if0.GetDexPcAtDepth(encoding.inline_info.encoding, 1)); + ASSERT_TRUE(if0.EncodesArtMethodAtDepth(encoding.inline_info.encoding, 1)); DexRegisterMap dex_registers1 = ci.GetDexRegisterMapAtDepth(0, if0, encoding, 1); ASSERT_EQ(8, dex_registers1.GetStackOffsetInBytes(0, 1, ci, encoding)); @@ -767,13 +767,13 @@ TEST(StackMapTest, InlineTest) { ASSERT_EQ(0, dex_registers0.GetConstant(1, 2, ci, encoding)); InlineInfo if1 = ci.GetInlineInfoOf(sm1, encoding); - ASSERT_EQ(3u, if1.GetDepth(encoding.inline_info_encoding)); - ASSERT_EQ(2u, if1.GetDexPcAtDepth(encoding.inline_info_encoding, 0)); - ASSERT_TRUE(if1.EncodesArtMethodAtDepth(encoding.inline_info_encoding, 0)); - ASSERT_EQ(3u, if1.GetDexPcAtDepth(encoding.inline_info_encoding, 1)); - ASSERT_TRUE(if1.EncodesArtMethodAtDepth(encoding.inline_info_encoding, 1)); - ASSERT_EQ(5u, if1.GetDexPcAtDepth(encoding.inline_info_encoding, 2)); - ASSERT_TRUE(if1.EncodesArtMethodAtDepth(encoding.inline_info_encoding, 2)); + ASSERT_EQ(3u, if1.GetDepth(encoding.inline_info.encoding)); + ASSERT_EQ(2u, if1.GetDexPcAtDepth(encoding.inline_info.encoding, 0)); + ASSERT_TRUE(if1.EncodesArtMethodAtDepth(encoding.inline_info.encoding, 0)); + ASSERT_EQ(3u, if1.GetDexPcAtDepth(encoding.inline_info.encoding, 1)); + ASSERT_TRUE(if1.EncodesArtMethodAtDepth(encoding.inline_info.encoding, 1)); + ASSERT_EQ(5u, if1.GetDexPcAtDepth(encoding.inline_info.encoding, 2)); + ASSERT_TRUE(if1.EncodesArtMethodAtDepth(encoding.inline_info.encoding, 2)); DexRegisterMap dex_registers1 = ci.GetDexRegisterMapAtDepth(0, if1, encoding, 1); ASSERT_EQ(12, dex_registers1.GetStackOffsetInBytes(0, 1, ci, encoding)); @@ -783,7 +783,7 @@ TEST(StackMapTest, InlineTest) { ASSERT_EQ(10, dex_registers2.GetConstant(1, 3, ci, encoding)); ASSERT_EQ(5, dex_registers2.GetMachineRegister(2, 3, ci, encoding)); - ASSERT_FALSE(if1.HasDexRegisterMapAtDepth(encoding.inline_info_encoding, 2)); + ASSERT_FALSE(if1.HasDexRegisterMapAtDepth(encoding.inline_info.encoding, 2)); } { @@ -793,7 +793,7 @@ TEST(StackMapTest, InlineTest) { DexRegisterMap dex_registers0 = ci.GetDexRegisterMapOf(sm2, encoding, 2); ASSERT_FALSE(dex_registers0.IsDexRegisterLive(0)); ASSERT_EQ(4, dex_registers0.GetConstant(1, 2, ci, encoding)); - ASSERT_FALSE(sm2.HasInlineInfo(encoding.stack_map_encoding)); + ASSERT_FALSE(sm2.HasInlineInfo(encoding.stack_map.encoding)); } { @@ -805,15 +805,15 @@ TEST(StackMapTest, InlineTest) { ASSERT_EQ(0, dex_registers0.GetConstant(1, 2, ci, encoding)); InlineInfo if2 = ci.GetInlineInfoOf(sm3, encoding); - ASSERT_EQ(3u, if2.GetDepth(encoding.inline_info_encoding)); - ASSERT_EQ(2u, if2.GetDexPcAtDepth(encoding.inline_info_encoding, 0)); - ASSERT_TRUE(if2.EncodesArtMethodAtDepth(encoding.inline_info_encoding, 0)); - ASSERT_EQ(5u, if2.GetDexPcAtDepth(encoding.inline_info_encoding, 1)); - ASSERT_TRUE(if2.EncodesArtMethodAtDepth(encoding.inline_info_encoding, 1)); - ASSERT_EQ(10u, if2.GetDexPcAtDepth(encoding.inline_info_encoding, 2)); - ASSERT_TRUE(if2.EncodesArtMethodAtDepth(encoding.inline_info_encoding, 2)); + ASSERT_EQ(3u, if2.GetDepth(encoding.inline_info.encoding)); + ASSERT_EQ(2u, if2.GetDexPcAtDepth(encoding.inline_info.encoding, 0)); + ASSERT_TRUE(if2.EncodesArtMethodAtDepth(encoding.inline_info.encoding, 0)); + ASSERT_EQ(5u, if2.GetDexPcAtDepth(encoding.inline_info.encoding, 1)); + ASSERT_TRUE(if2.EncodesArtMethodAtDepth(encoding.inline_info.encoding, 1)); + ASSERT_EQ(10u, if2.GetDexPcAtDepth(encoding.inline_info.encoding, 2)); + ASSERT_TRUE(if2.EncodesArtMethodAtDepth(encoding.inline_info.encoding, 2)); - ASSERT_FALSE(if2.HasDexRegisterMapAtDepth(encoding.inline_info_encoding, 0)); + ASSERT_FALSE(if2.HasDexRegisterMapAtDepth(encoding.inline_info.encoding, 0)); DexRegisterMap dex_registers1 = ci.GetDexRegisterMapAtDepth(1, if2, encoding, 1); ASSERT_EQ(2, dex_registers1.GetMachineRegister(0, 1, ci, encoding)); @@ -865,8 +865,8 @@ TEST(StackMapTest, TestDeduplicateStackMask) { StackMap stack_map1 = code_info.GetStackMapForNativePcOffset(4, encoding); StackMap stack_map2 = code_info.GetStackMapForNativePcOffset(8, encoding); - EXPECT_EQ(stack_map1.GetStackMaskIndex(encoding.stack_map_encoding), - stack_map2.GetStackMaskIndex(encoding.stack_map_encoding)); + EXPECT_EQ(stack_map1.GetStackMaskIndex(encoding.stack_map.encoding), + stack_map2.GetStackMaskIndex(encoding.stack_map.encoding)); } } // namespace art |