diff options
Diffstat (limited to 'compiler')
35 files changed, 1249 insertions, 695 deletions
diff --git a/compiler/debug/elf_debug_line_writer.h b/compiler/debug/elf_debug_line_writer.h index 18a9165854..cdd1e53f94 100644 --- a/compiler/debug/elf_debug_line_writer.h +++ b/compiler/debug/elf_debug_line_writer.h @@ -104,10 +104,10 @@ class ElfDebugLineWriter { for (uint32_t s = 0; s < code_info.GetNumberOfStackMaps(encoding); s++) { StackMap stack_map = code_info.GetStackMapAt(s, encoding); DCHECK(stack_map.IsValid()); - const uint32_t pc = stack_map.GetNativePcOffset(encoding.stack_map_encoding, isa); - const int32_t dex = stack_map.GetDexPc(encoding.stack_map_encoding); + const uint32_t pc = stack_map.GetNativePcOffset(encoding.stack_map.encoding, isa); + const int32_t dex = stack_map.GetDexPc(encoding.stack_map.encoding); pc2dex_map.push_back({pc, dex}); - if (stack_map.HasDexRegisterMap(encoding.stack_map_encoding)) { + if (stack_map.HasDexRegisterMap(encoding.stack_map.encoding)) { // Guess that the first map with local variables is the end of prologue. prologue_end = std::min(prologue_end, pc); } diff --git a/compiler/debug/elf_debug_loc_writer.h b/compiler/debug/elf_debug_loc_writer.h index bce538743b..cbfdbddd1d 100644 --- a/compiler/debug/elf_debug_loc_writer.h +++ b/compiler/debug/elf_debug_loc_writer.h @@ -104,7 +104,7 @@ std::vector<VariableLocation> GetVariableLocations( for (uint32_t s = 0; s < code_info.GetNumberOfStackMaps(encoding); s++) { StackMap stack_map = code_info.GetStackMapAt(s, encoding); DCHECK(stack_map.IsValid()); - if (!stack_map.HasDexRegisterMap(encoding.stack_map_encoding)) { + if (!stack_map.HasDexRegisterMap(encoding.stack_map.encoding)) { // The compiler creates stackmaps without register maps at the start of // basic blocks in order to keep instruction-accurate line number mapping. // However, we never stop at those (breakpoint locations always have map). @@ -112,7 +112,7 @@ std::vector<VariableLocation> GetVariableLocations( // The main reason for this is to save space by avoiding undefined gaps. continue; } - const uint32_t pc_offset = stack_map.GetNativePcOffset(encoding.stack_map_encoding, isa); + const uint32_t pc_offset = stack_map.GetNativePcOffset(encoding.stack_map.encoding, isa); DCHECK_LE(pc_offset, method_info->code_size); DCHECK_LE(compilation_unit_code_address, method_info->code_address); const uint32_t low_pc = dchecked_integral_cast<uint32_t>( @@ -136,7 +136,7 @@ std::vector<VariableLocation> GetVariableLocations( } // Check that the stack map is in the requested range. - uint32_t dex_pc = stack_map.GetDexPc(encoding.stack_map_encoding); + uint32_t dex_pc = stack_map.GetDexPc(encoding.stack_map.encoding); if (!(dex_pc_low <= dex_pc && dex_pc < dex_pc_high)) { // The variable is not in scope at this PC. Therefore omit the entry. // Note that this is different to None() entry which means in scope, but unknown location. diff --git a/compiler/driver/compiler_driver-inl.h b/compiler/driver/compiler_driver-inl.h index f056dd3c00..f296851ebf 100644 --- a/compiler/driver/compiler_driver-inl.h +++ b/compiler/driver/compiler_driver-inl.h @@ -135,65 +135,6 @@ inline bool CompilerDriver::CanAccessResolvedMember<ArtMethod>( return referrer_class->CanAccessResolvedMethod(access_to, method, dex_cache, field_idx); } -template <typename ArtMember> -inline std::pair<bool, bool> CompilerDriver::IsClassOfStaticMemberAvailableToReferrer( - mirror::DexCache* dex_cache, - mirror::Class* referrer_class, - ArtMember* resolved_member, - uint16_t member_idx, - dex::TypeIndex* storage_index) { - DCHECK(resolved_member->IsStatic()); - if (LIKELY(referrer_class != nullptr)) { - ObjPtr<mirror::Class> members_class = resolved_member->GetDeclaringClass(); - if (members_class == referrer_class) { - *storage_index = members_class->GetDexTypeIndex(); - return std::make_pair(true, true); - } - if (CanAccessResolvedMember<ArtMember>( - referrer_class, members_class.Ptr(), resolved_member, dex_cache, member_idx)) { - // We have the resolved member, we must make it into a index for the referrer - // in its static storage (which may fail if it doesn't have a slot for it) - // TODO: for images we can elide the static storage base null check - // if we know there's a non-null entry in the image - const DexFile* dex_file = dex_cache->GetDexFile(); - dex::TypeIndex storage_idx(DexFile::kDexNoIndex16); - if (LIKELY(members_class->GetDexCache() == dex_cache)) { - // common case where the dex cache of both the referrer and the member are the same, - // no need to search the dex file - storage_idx = members_class->GetDexTypeIndex(); - } else { - // Search dex file for localized ssb index, may fail if member's class is a parent - // of the class mentioned in the dex file and there is no dex cache entry. - storage_idx = resolved_member->GetDeclaringClass()->FindTypeIndexInOtherDexFile(*dex_file); - } - if (storage_idx.IsValid()) { - *storage_index = storage_idx; - return std::make_pair(true, !resolved_member->IsFinal()); - } - } - } - // Conservative defaults. - *storage_index = dex::TypeIndex(DexFile::kDexNoIndex16); - return std::make_pair(false, false); -} - -inline std::pair<bool, bool> CompilerDriver::IsFastStaticField( - mirror::DexCache* dex_cache, mirror::Class* referrer_class, - ArtField* resolved_field, uint16_t field_idx, dex::TypeIndex* storage_index) { - return IsClassOfStaticMemberAvailableToReferrer( - dex_cache, referrer_class, resolved_field, field_idx, storage_index); -} - -inline bool CompilerDriver::IsClassOfStaticMethodAvailableToReferrer( - mirror::DexCache* dex_cache, mirror::Class* referrer_class, - ArtMethod* resolved_method, uint16_t method_idx, dex::TypeIndex* storage_index) { - std::pair<bool, bool> result = IsClassOfStaticMemberAvailableToReferrer( - dex_cache, referrer_class, resolved_method, method_idx, storage_index); - // Only the first member of `result` is meaningful, as there is no - // "write access" to a method. - return result.first; -} - inline ArtMethod* CompilerDriver::ResolveMethod( ScopedObjectAccess& soa, Handle<mirror::DexCache> dex_cache, Handle<mirror::ClassLoader> class_loader, const DexCompilationUnit* mUnit, diff --git a/compiler/driver/compiler_driver.cc b/compiler/driver/compiler_driver.cc index 1d4eaf8c5a..7af850a263 100644 --- a/compiler/driver/compiler_driver.cc +++ b/compiler/driver/compiler_driver.cc @@ -878,7 +878,7 @@ static void ResolveConstStrings(CompilerDriver* driver, MutableHandle<mirror::DexCache> dex_cache(hs.NewHandle<mirror::DexCache>(nullptr)); for (const DexFile* dex_file : dex_files) { - dex_cache.Assign(class_linker->FindDexCache(soa.Self(), *dex_file, false)); + dex_cache.Assign(class_linker->FindDexCache(soa.Self(), *dex_file)); TimingLogger::ScopedTiming t("Resolve const-string Strings", timings); size_t class_def_count = dex_file->NumClassDefs(); @@ -1182,10 +1182,12 @@ void CompilerDriver::LoadImageClasses(TimingLogger* timings) { Handle<mirror::DexCache> dex_cache(hs2.NewHandle(class_linker->RegisterDexFile(*dex_file, nullptr))); Handle<mirror::Class> klass(hs2.NewHandle( - class_linker->ResolveType(*dex_file, - exception_type_idx, - dex_cache, - ScopedNullHandle<mirror::ClassLoader>()))); + (dex_cache.Get() != nullptr) + ? class_linker->ResolveType(*dex_file, + exception_type_idx, + dex_cache, + ScopedNullHandle<mirror::ClassLoader>()) + : nullptr)); if (klass.Get() == nullptr) { const DexFile::TypeId& type_id = dex_file->GetTypeId(exception_type_idx); const char* descriptor = dex_file->GetTypeDescriptor(type_id); @@ -1776,7 +1778,7 @@ class ResolveClassFieldsAndMethodsVisitor : public CompilationVisitor { Handle<mirror::ClassLoader> class_loader( hs.NewHandle(soa.Decode<mirror::ClassLoader>(jclass_loader))); Handle<mirror::DexCache> dex_cache(hs.NewHandle(class_linker->FindDexCache( - soa.Self(), dex_file, false))); + soa.Self(), dex_file))); // Resolve the class. mirror::Class* klass = class_linker->ResolveType(dex_file, class_def.class_idx_, dex_cache, class_loader); @@ -1875,10 +1877,9 @@ class ResolveTypeVisitor : public CompilationVisitor { Handle<mirror::DexCache> dex_cache(hs.NewHandle(class_linker->RegisterDexFile( dex_file, class_loader.Get()))); - mirror::Class* klass = class_linker->ResolveType(dex_file, - dex::TypeIndex(type_idx), - dex_cache, - class_loader); + ObjPtr<mirror::Class> klass = (dex_cache.Get() != nullptr) + ? class_linker->ResolveType(dex_file, dex::TypeIndex(type_idx), dex_cache, class_loader) + : nullptr; if (klass == nullptr) { soa.Self()->AssertPendingException(); @@ -2135,7 +2136,7 @@ class VerifyClassVisitor : public CompilationVisitor { * will be rejected by the verifier and later skipped during compilation in the compiler. */ Handle<mirror::DexCache> dex_cache(hs.NewHandle(class_linker->FindDexCache( - soa.Self(), dex_file, false))); + soa.Self(), dex_file))); std::string error_msg; failure_kind = verifier::MethodVerifier::VerifyClass(soa.Self(), diff --git a/compiler/driver/compiler_driver.h b/compiler/driver/compiler_driver.h index 503fe3adfc..5b4c751c4a 100644 --- a/compiler/driver/compiler_driver.h +++ b/compiler/driver/compiler_driver.h @@ -233,27 +233,6 @@ class CompilerDriver { ArtField* resolved_field, uint16_t field_idx) REQUIRES_SHARED(Locks::mutator_lock_); - // Can we fast-path an SGET/SPUT access to a static field? If yes, compute the type index - // of the declaring class in the referrer's dex file. - std::pair<bool, bool> IsFastStaticField(mirror::DexCache* dex_cache, - mirror::Class* referrer_class, - ArtField* resolved_field, - uint16_t field_idx, - dex::TypeIndex* storage_index) - REQUIRES_SHARED(Locks::mutator_lock_); - - // Return whether the declaring class of `resolved_method` is - // available to `referrer_class`. If this is true, compute the type - // index of the declaring class in the referrer's dex file and - // return it through the out argument `storage_index`; otherwise - // return DexFile::kDexNoIndex through `storage_index`. - bool IsClassOfStaticMethodAvailableToReferrer(mirror::DexCache* dex_cache, - mirror::Class* referrer_class, - ArtMethod* resolved_method, - uint16_t method_idx, - dex::TypeIndex* storage_index) - REQUIRES_SHARED(Locks::mutator_lock_); - // Resolve a method. Returns null on failure, including incompatible class change. ArtMethod* ResolveMethod( ScopedObjectAccess& soa, Handle<mirror::DexCache> dex_cache, @@ -379,21 +358,6 @@ class CompilerDriver { } private: - // Return whether the declaring class of `resolved_member` is - // available to `referrer_class` for read or write access using two - // Boolean values returned as a pair. If is true at least for read - // access, compute the type index of the declaring class in the - // referrer's dex file and return it through the out argument - // `storage_index`; otherwise return DexFile::kDexNoIndex through - // `storage_index`. - template <typename ArtMember> - std::pair<bool, bool> IsClassOfStaticMemberAvailableToReferrer(mirror::DexCache* dex_cache, - mirror::Class* referrer_class, - ArtMember* resolved_member, - uint16_t member_idx, - dex::TypeIndex* storage_index) - REQUIRES_SHARED(Locks::mutator_lock_); - // Can `referrer_class` access the resolved `member`? // Dispatch call to mirror::Class::CanAccessResolvedField or // mirror::Class::CanAccessResolvedMember depending on the value of diff --git a/compiler/linker/arm64/relative_patcher_arm64.cc b/compiler/linker/arm64/relative_patcher_arm64.cc index 79e1785e91..9ddf200237 100644 --- a/compiler/linker/arm64/relative_patcher_arm64.cc +++ b/compiler/linker/arm64/relative_patcher_arm64.cc @@ -31,9 +31,7 @@ namespace linker { namespace { inline bool IsAdrpPatch(const LinkerPatch& patch) { - LinkerPatch::Type type = patch.GetType(); - return - (type == LinkerPatch::Type::kStringRelative || type == LinkerPatch::Type::kDexCacheArray) && + return (patch.IsPcRelative() && patch.GetType() != LinkerPatch::Type::kCallRelative) && patch.LiteralOffset() == patch.PcInsnOffset(); } @@ -214,11 +212,11 @@ void Arm64RelativePatcher::PatchPcRelativeReference(std::vector<uint8_t>* code, DCHECK(patch.GetType() == LinkerPatch::Type::kStringRelative || patch.GetType() == LinkerPatch::Type::kTypeRelative) << patch.GetType(); } else { - // With the read barrier (non-Baker) enabled, it could be kDexCacheArray in the - // HLoadString::LoadKind::kDexCachePcRelative case of VisitLoadString(). + // With the read barrier (non-Baker) enabled, it could be kStringBssEntry or kTypeBssEntry. DCHECK(patch.GetType() == LinkerPatch::Type::kStringRelative || patch.GetType() == LinkerPatch::Type::kTypeRelative || - patch.GetType() == LinkerPatch::Type::kDexCacheArray) << patch.GetType(); + patch.GetType() == LinkerPatch::Type::kStringBssEntry || + patch.GetType() == LinkerPatch::Type::kTypeBssEntry) << patch.GetType(); } shift = 0u; // No shift for ADD. } else { diff --git a/compiler/oat_test.cc b/compiler/oat_test.cc index d5842a8c9d..66111f6e23 100644 --- a/compiler/oat_test.cc +++ b/compiler/oat_test.cc @@ -149,11 +149,10 @@ class OatTest : public CommonCompilerTest { File* oat_file, const std::vector<const char*>& dex_filenames, SafeMap<std::string, std::string>& key_value_store, - bool verify) { + bool verify, + ProfileCompilationInfo* profile_compilation_info) { TimingLogger timings("WriteElf", false, false); - OatWriter oat_writer(/*compiling_boot_image*/false, - &timings, - /*profile_compilation_info*/nullptr); + OatWriter oat_writer(/*compiling_boot_image*/false, &timings, profile_compilation_info); for (const char* dex_filename : dex_filenames) { if (!oat_writer.AddDexFileSource(dex_filename, dex_filename)) { return false; @@ -264,7 +263,7 @@ class OatTest : public CommonCompilerTest { return true; } - void TestDexFileInput(bool verify, bool low_4gb); + void TestDexFileInput(bool verify, bool low_4gb, bool use_profile); void TestZipFileInput(bool verify); std::unique_ptr<const InstructionSetFeatures> insn_features_; @@ -568,7 +567,7 @@ static void MaybeModifyDexFileToFail(bool verify, std::unique_ptr<const DexFile> } } -void OatTest::TestDexFileInput(bool verify, bool low_4gb) { +void OatTest::TestDexFileInput(bool verify, bool low_4gb, bool use_profile) { TimingLogger timings("OatTest::DexFileInput", false, false); std::vector<const char*> input_filenames; @@ -606,11 +605,14 @@ void OatTest::TestDexFileInput(bool verify, bool low_4gb) { ScratchFile oat_file, vdex_file(oat_file, ".vdex"); SafeMap<std::string, std::string> key_value_store; key_value_store.Put(OatHeader::kImageLocationKey, "test.art"); + std::unique_ptr<ProfileCompilationInfo> + profile_compilation_info(use_profile ? new ProfileCompilationInfo() : nullptr); success = WriteElf(vdex_file.GetFile(), oat_file.GetFile(), input_filenames, key_value_store, - verify); + verify, + profile_compilation_info.get()); // In verify mode, we expect failure. if (verify) { @@ -654,15 +656,19 @@ void OatTest::TestDexFileInput(bool verify, bool low_4gb) { } TEST_F(OatTest, DexFileInputCheckOutput) { - TestDexFileInput(false, /*low_4gb*/false); + TestDexFileInput(/*verify*/false, /*low_4gb*/false, /*use_profile*/false); } TEST_F(OatTest, DexFileInputCheckOutputLow4GB) { - TestDexFileInput(false, /*low_4gb*/true); + TestDexFileInput(/*verify*/false, /*low_4gb*/true, /*use_profile*/false); } TEST_F(OatTest, DexFileInputCheckVerifier) { - TestDexFileInput(true, /*low_4gb*/false); + TestDexFileInput(/*verify*/true, /*low_4gb*/false, /*use_profile*/false); +} + +TEST_F(OatTest, DexFileFailsVerifierWithLayout) { + TestDexFileInput(/*verify*/true, /*low_4gb*/false, /*use_profile*/true); } void OatTest::TestZipFileInput(bool verify) { @@ -717,8 +723,8 @@ void OatTest::TestZipFileInput(bool verify) { std::vector<const char*> input_filenames { zip_file.GetFilename().c_str() }; // NOLINT [readability/braces] [4] ScratchFile oat_file, vdex_file(oat_file, ".vdex"); - success = WriteElf(vdex_file.GetFile(), oat_file.GetFile(), - input_filenames, key_value_store, verify); + success = WriteElf(vdex_file.GetFile(), oat_file.GetFile(), input_filenames, + key_value_store, verify, /*profile_compilation_info*/nullptr); if (verify) { ASSERT_FALSE(success); diff --git a/compiler/oat_writer.cc b/compiler/oat_writer.cc index bd2c5e3bfc..7c0cdbf270 100644 --- a/compiler/oat_writer.cc +++ b/compiler/oat_writer.cc @@ -1250,7 +1250,7 @@ class OatWriter::WriteCodeMethodVisitor : public OatDexMethodVisitor { const ScopedObjectAccess soa_; const ScopedAssertNoThreadSuspension no_thread_suspension_; ClassLinker* const class_linker_; - mirror::DexCache* dex_cache_; + ObjPtr<mirror::DexCache> dex_cache_; std::vector<uint8_t> patched_code_; void ReportWriteFailure(const char* what, const ClassDataItemIterator& it) { @@ -1261,7 +1261,7 @@ class OatWriter::WriteCodeMethodVisitor : public OatDexMethodVisitor { ArtMethod* GetTargetMethod(const LinkerPatch& patch) REQUIRES_SHARED(Locks::mutator_lock_) { MethodReference ref = patch.TargetMethod(); - mirror::DexCache* dex_cache = + ObjPtr<mirror::DexCache> dex_cache = (dex_file_ == ref.dex_file) ? dex_cache_ : class_linker_->FindDexCache( Thread::Current(), *ref.dex_file); ArtMethod* method = dex_cache->GetResolvedMethod( @@ -1295,7 +1295,7 @@ class OatWriter::WriteCodeMethodVisitor : public OatDexMethodVisitor { return target_offset; } - mirror::DexCache* GetDexCache(const DexFile* target_dex_file) + ObjPtr<mirror::DexCache> GetDexCache(const DexFile* target_dex_file) REQUIRES_SHARED(Locks::mutator_lock_) { return (target_dex_file == dex_file_) ? dex_cache_ @@ -1303,7 +1303,7 @@ class OatWriter::WriteCodeMethodVisitor : public OatDexMethodVisitor { } mirror::Class* GetTargetType(const LinkerPatch& patch) REQUIRES_SHARED(Locks::mutator_lock_) { - mirror::DexCache* dex_cache = GetDexCache(patch.TargetTypeDexFile()); + ObjPtr<mirror::DexCache> dex_cache = GetDexCache(patch.TargetTypeDexFile()); mirror::Class* type = dex_cache->GetResolvedType(patch.TargetTypeIndex()); CHECK(type != nullptr); return type; @@ -2266,6 +2266,10 @@ bool OatWriter::LayoutAndWriteDexFile(OutputStream* out, OatDexFile* oat_dex_fil File* raw_file = oat_dex_file->source_.GetRawFile(); dex_file = DexFile::OpenDex(raw_file->Fd(), location, /* verify_checksum */ true, &error_msg); } + if (dex_file == nullptr) { + LOG(ERROR) << "Failed to open dex file for layout: " << error_msg; + return false; + } Options options; options.output_to_memmap_ = true; DexLayout dex_layout(options, profile_compilation_info_, nullptr); diff --git a/compiler/optimizing/builder.h b/compiler/optimizing/builder.h index 8cf4089eba..e4ad4222fb 100644 --- a/compiler/optimizing/builder.h +++ b/compiler/optimizing/builder.h @@ -32,6 +32,8 @@ namespace art { +class CodeGenerator; + class HGraphBuilder : public ValueObject { public: HGraphBuilder(HGraph* graph, @@ -40,6 +42,7 @@ class HGraphBuilder : public ValueObject { const DexFile* dex_file, const DexFile::CodeItem& code_item, CompilerDriver* driver, + CodeGenerator* code_generator, OptimizingCompilerStats* compiler_stats, const uint8_t* interpreter_metadata, Handle<mirror::DexCache> dex_cache, @@ -61,6 +64,7 @@ class HGraphBuilder : public ValueObject { dex_compilation_unit, outer_compilation_unit, driver, + code_generator, interpreter_metadata, compiler_stats, dex_cache, @@ -89,6 +93,7 @@ class HGraphBuilder : public ValueObject { /* dex_compilation_unit */ nullptr, /* outer_compilation_unit */ nullptr, /* compiler_driver */ nullptr, + /* code_generator */ nullptr, /* interpreter_metadata */ nullptr, /* compiler_stats */ nullptr, null_dex_cache_, diff --git a/compiler/optimizing/code_generator_arm.cc b/compiler/optimizing/code_generator_arm.cc index f5b6ebef9c..759a951d6b 100644 --- a/compiler/optimizing/code_generator_arm.cc +++ b/compiler/optimizing/code_generator_arm.cc @@ -367,22 +367,37 @@ class BoundsCheckSlowPathARM : public SlowPathCodeARM { class LoadClassSlowPathARM : public SlowPathCodeARM { public: - LoadClassSlowPathARM(HLoadClass* cls, - HInstruction* at, - uint32_t dex_pc, - bool do_clinit) + LoadClassSlowPathARM(HLoadClass* cls, HInstruction* at, uint32_t dex_pc, bool do_clinit) : SlowPathCodeARM(at), cls_(cls), dex_pc_(dex_pc), do_clinit_(do_clinit) { DCHECK(at->IsLoadClass() || at->IsClinitCheck()); } void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { LocationSummary* locations = instruction_->GetLocations(); + Location out = locations->Out(); + constexpr bool call_saves_everything_except_r0 = (!kUseReadBarrier || kUseBakerReadBarrier); CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); __ Bind(GetEntryLabel()); SaveLiveRegisters(codegen, locations); InvokeRuntimeCallingConvention calling_convention; + // For HLoadClass/kBssEntry/kSaveEverything, make sure we preserve the address of the entry. + DCHECK_EQ(instruction_->IsLoadClass(), cls_ == instruction_); + bool is_load_class_bss_entry = + (cls_ == instruction_) && (cls_->GetLoadKind() == HLoadClass::LoadKind::kBssEntry); + Register entry_address = kNoRegister; + if (is_load_class_bss_entry && call_saves_everything_except_r0) { + Register temp = locations->GetTemp(0).AsRegister<Register>(); + // In the unlucky case that the `temp` is R0, we preserve the address in `out` across + // the kSaveEverything call. + bool temp_is_r0 = (temp == calling_convention.GetRegisterAt(0)); + entry_address = temp_is_r0 ? out.AsRegister<Register>() : temp; + DCHECK_NE(entry_address, calling_convention.GetRegisterAt(0)); + if (temp_is_r0) { + __ mov(entry_address, ShifterOperand(temp)); + } + } dex::TypeIndex type_index = cls_->GetTypeIndex(); __ LoadImmediate(calling_convention.GetRegisterAt(0), type_index.index_); QuickEntrypointEnum entrypoint = do_clinit_ ? kQuickInitializeStaticStorage @@ -394,30 +409,31 @@ class LoadClassSlowPathARM : public SlowPathCodeARM { CheckEntrypointTypes<kQuickInitializeType, void*, uint32_t>(); } + // For HLoadClass/kBssEntry, store the resolved Class to the BSS entry. + if (is_load_class_bss_entry) { + if (call_saves_everything_except_r0) { + // The class entry address was preserved in `entry_address` thanks to kSaveEverything. + __ str(R0, Address(entry_address)); + } else { + // For non-Baker read barrier, we need to re-calculate the address of the string entry. + Register temp = IP; + CodeGeneratorARM::PcRelativePatchInfo* labels = + arm_codegen->NewTypeBssEntryPatch(cls_->GetDexFile(), type_index); + __ BindTrackedLabel(&labels->movw_label); + __ movw(temp, /* placeholder */ 0u); + __ BindTrackedLabel(&labels->movt_label); + __ movt(temp, /* placeholder */ 0u); + __ BindTrackedLabel(&labels->add_pc_label); + __ add(temp, temp, ShifterOperand(PC)); + __ str(R0, Address(temp)); + } + } // Move the class to the desired location. - Location out = locations->Out(); if (out.IsValid()) { DCHECK(out.IsRegister() && !locations->GetLiveRegisters()->ContainsCoreRegister(out.reg())); arm_codegen->Move32(locations->Out(), Location::RegisterLocation(R0)); } RestoreLiveRegisters(codegen, locations); - // For HLoadClass/kBssEntry, store the resolved Class to the BSS entry. - DCHECK_EQ(instruction_->IsLoadClass(), cls_ == instruction_); - if (cls_ == instruction_ && cls_->GetLoadKind() == HLoadClass::LoadKind::kBssEntry) { - DCHECK(out.IsValid()); - // TODO: Change art_quick_initialize_type/art_quick_initialize_static_storage to - // kSaveEverything and use a temporary for the .bss entry address in the fast path, - // so that we can avoid another calculation here. - CodeGeneratorARM::PcRelativePatchInfo* labels = - arm_codegen->NewTypeBssEntryPatch(cls_->GetDexFile(), type_index); - __ BindTrackedLabel(&labels->movw_label); - __ movw(IP, /* placeholder */ 0u); - __ BindTrackedLabel(&labels->movt_label); - __ movt(IP, /* placeholder */ 0u); - __ BindTrackedLabel(&labels->add_pc_label); - __ add(IP, IP, ShifterOperand(PC)); - __ str(locations->Out().AsRegister<Register>(), Address(IP)); - } __ b(GetExitLabel()); } @@ -441,12 +457,13 @@ class LoadStringSlowPathARM : public SlowPathCodeARM { explicit LoadStringSlowPathARM(HLoadString* instruction) : SlowPathCodeARM(instruction) {} void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { + DCHECK(instruction_->IsLoadString()); + DCHECK_EQ(instruction_->AsLoadString()->GetLoadKind(), HLoadString::LoadKind::kBssEntry); LocationSummary* locations = instruction_->GetLocations(); DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg())); HLoadString* load = instruction_->AsLoadString(); const dex::StringIndex string_index = load->GetStringIndex(); Register out = locations->Out().AsRegister<Register>(); - Register temp = locations->GetTemp(0).AsRegister<Register>(); constexpr bool call_saves_everything_except_r0 = (!kUseReadBarrier || kUseBakerReadBarrier); CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); @@ -455,12 +472,16 @@ class LoadStringSlowPathARM : public SlowPathCodeARM { InvokeRuntimeCallingConvention calling_convention; // In the unlucky case that the `temp` is R0, we preserve the address in `out` across - // the kSaveEverything call (or use `out` for the address after non-kSaveEverything call). - bool temp_is_r0 = (temp == calling_convention.GetRegisterAt(0)); - Register entry_address = temp_is_r0 ? out : temp; - DCHECK_NE(entry_address, calling_convention.GetRegisterAt(0)); - if (call_saves_everything_except_r0 && temp_is_r0) { - __ mov(entry_address, ShifterOperand(temp)); + // the kSaveEverything call. + Register entry_address = kNoRegister; + if (call_saves_everything_except_r0) { + Register temp = locations->GetTemp(0).AsRegister<Register>(); + bool temp_is_r0 = (temp == calling_convention.GetRegisterAt(0)); + entry_address = temp_is_r0 ? out : temp; + DCHECK_NE(entry_address, calling_convention.GetRegisterAt(0)); + if (temp_is_r0) { + __ mov(entry_address, ShifterOperand(temp)); + } } __ LoadImmediate(calling_convention.GetRegisterAt(0), string_index.index_); @@ -473,15 +494,16 @@ class LoadStringSlowPathARM : public SlowPathCodeARM { __ str(R0, Address(entry_address)); } else { // For non-Baker read barrier, we need to re-calculate the address of the string entry. + Register temp = IP; CodeGeneratorARM::PcRelativePatchInfo* labels = arm_codegen->NewPcRelativeStringPatch(load->GetDexFile(), string_index); __ BindTrackedLabel(&labels->movw_label); - __ movw(entry_address, /* placeholder */ 0u); + __ movw(temp, /* placeholder */ 0u); __ BindTrackedLabel(&labels->movt_label); - __ movt(entry_address, /* placeholder */ 0u); + __ movt(temp, /* placeholder */ 0u); __ BindTrackedLabel(&labels->add_pc_label); - __ add(entry_address, entry_address, ShifterOperand(PC)); - __ str(R0, Address(entry_address)); + __ add(temp, temp, ShifterOperand(PC)); + __ str(R0, Address(temp)); } arm_codegen->Move32(locations->Out(), Location::RegisterLocation(R0)); @@ -624,6 +646,10 @@ class ArraySetSlowPathARM : public SlowPathCodeARM { // probably still be a from-space reference (unless it gets updated by // another thread, or if another thread installed another object // reference (different from `ref`) in `obj.field`). +// +// If `entrypoint` is a valid location it is assumed to already be +// holding the entrypoint. The case where the entrypoint is passed in +// is for the GcRoot read barrier. class ReadBarrierMarkSlowPathARM : public SlowPathCodeARM { public: ReadBarrierMarkSlowPathARM(HInstruction* instruction, @@ -3993,8 +4019,11 @@ void LocationsBuilderARM::VisitNewArray(HNewArray* instruction) { void InstructionCodeGeneratorARM::VisitNewArray(HNewArray* instruction) { // Note: if heap poisoning is enabled, the entry point takes cares // of poisoning the reference. - codegen_->InvokeRuntime(kQuickAllocArrayResolved, instruction, instruction->GetDexPc()); + QuickEntrypointEnum entrypoint = + CodeGenerator::GetArrayAllocationEntrypoint(instruction->GetLoadClass()->GetClass()); + codegen_->InvokeRuntime(entrypoint, instruction, instruction->GetDexPc()); CheckEntrypointTypes<kQuickAllocArrayResolved, void*, mirror::Class*, int32_t>(); + DCHECK(!codegen_->IsLeafMethod()); } void LocationsBuilderARM::VisitParameterValue(HParameterValue* instruction) { @@ -5719,6 +5748,9 @@ void ParallelMoveResolverARM::RestoreScratch(int reg) { HLoadClass::LoadKind CodeGeneratorARM::GetSupportedLoadClassKind( HLoadClass::LoadKind desired_class_load_kind) { switch (desired_class_load_kind) { + case HLoadClass::LoadKind::kInvalid: + LOG(FATAL) << "UNREACHABLE"; + UNREACHABLE(); case HLoadClass::LoadKind::kReferrersClass: break; case HLoadClass::LoadKind::kBootImageLinkTimeAddress: @@ -5749,6 +5781,7 @@ void LocationsBuilderARM::VisitLoadClass(HLoadClass* cls) { cls, Location::RegisterLocation(calling_convention.GetRegisterAt(0)), Location::RegisterLocation(R0)); + DCHECK_EQ(calling_convention.GetRegisterAt(0), R0); return; } DCHECK(!cls->NeedsAccessCheck()); @@ -5766,6 +5799,22 @@ void LocationsBuilderARM::VisitLoadClass(HLoadClass* cls) { locations->SetInAt(0, Location::RequiresRegister()); } locations->SetOut(Location::RequiresRegister()); + if (load_kind == HLoadClass::LoadKind::kBssEntry) { + if (!kUseReadBarrier || kUseBakerReadBarrier) { + // Rely on the type resolution or initialization and marking to save everything we need. + // Note that IP may be clobbered by saving/restoring the live register (only one thanks + // to the custom calling convention) or by marking, so we request a different temp. + locations->AddTemp(Location::RequiresRegister()); + RegisterSet caller_saves = RegisterSet::Empty(); + InvokeRuntimeCallingConvention calling_convention; + caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); + // TODO: Add GetReturnLocation() to the calling convention so that we can DCHECK() + // that the the kPrimNot result register is the same as the first argument register. + locations->SetCustomSlowPathCallerSaves(caller_saves); + } else { + // For non-Baker read barrier we have a temp-clobbering call. + } + } } // NO_THREAD_SAFETY_ANALYSIS as we manipulate handles whose internal object we know does not @@ -5828,15 +5877,18 @@ void InstructionCodeGeneratorARM::VisitLoadClass(HLoadClass* cls) NO_THREAD_SAFE break; } case HLoadClass::LoadKind::kBssEntry: { + Register temp = (!kUseReadBarrier || kUseBakerReadBarrier) + ? locations->GetTemp(0).AsRegister<Register>() + : out; CodeGeneratorARM::PcRelativePatchInfo* labels = codegen_->NewTypeBssEntryPatch(cls->GetDexFile(), cls->GetTypeIndex()); __ BindTrackedLabel(&labels->movw_label); - __ movw(out, /* placeholder */ 0u); + __ movw(temp, /* placeholder */ 0u); __ BindTrackedLabel(&labels->movt_label); - __ movt(out, /* placeholder */ 0u); + __ movt(temp, /* placeholder */ 0u); __ BindTrackedLabel(&labels->add_pc_label); - __ add(out, out, ShifterOperand(PC)); - GenerateGcRootFieldLoad(cls, out_loc, out, 0, kCompilerReadBarrierOption); + __ add(temp, temp, ShifterOperand(PC)); + GenerateGcRootFieldLoad(cls, out_loc, temp, /* offset */ 0, read_barrier_option); generate_null_check = true; break; } @@ -5845,10 +5897,11 @@ void InstructionCodeGeneratorARM::VisitLoadClass(HLoadClass* cls) NO_THREAD_SAFE cls->GetTypeIndex(), cls->GetClass())); // /* GcRoot<mirror::Class> */ out = *out - GenerateGcRootFieldLoad(cls, out_loc, out, /* offset */ 0, kCompilerReadBarrierOption); + GenerateGcRootFieldLoad(cls, out_loc, out, /* offset */ 0, read_barrier_option); break; } case HLoadClass::LoadKind::kDexCacheViaMethod: + case HLoadClass::LoadKind::kInvalid: LOG(FATAL) << "UNREACHABLE"; UNREACHABLE(); } @@ -5931,9 +5984,9 @@ void LocationsBuilderARM::VisitLoadString(HLoadString* load) { locations->SetOut(Location::RequiresRegister()); if (load_kind == HLoadString::LoadKind::kBssEntry) { if (!kUseReadBarrier || kUseBakerReadBarrier) { - // Rely on the pResolveString and/or marking to save everything, including temps. - // Note that IP may theoretically be clobbered by saving/restoring the live register - // (only one thanks to the custom calling convention), so we request a different temp. + // Rely on the pResolveString and marking to save everything we need, including temps. + // Note that IP may be clobbered by saving/restoring the live register (only one thanks + // to the custom calling convention) or by marking, so we request a different temp. locations->AddTemp(Location::RequiresRegister()); RegisterSet caller_saves = RegisterSet::Empty(); InvokeRuntimeCallingConvention calling_convention; @@ -5984,7 +6037,9 @@ void InstructionCodeGeneratorARM::VisitLoadString(HLoadString* load) NO_THREAD_S } case HLoadString::LoadKind::kBssEntry: { DCHECK(!codegen_->GetCompilerOptions().IsBootImage()); - Register temp = locations->GetTemp(0).AsRegister<Register>(); + Register temp = (!kUseReadBarrier || kUseBakerReadBarrier) + ? locations->GetTemp(0).AsRegister<Register>() + : out; CodeGeneratorARM::PcRelativePatchInfo* labels = codegen_->NewPcRelativeStringPatch(load->GetDexFile(), load->GetStringIndex()); __ BindTrackedLabel(&labels->movw_label); diff --git a/compiler/optimizing/code_generator_arm64.cc b/compiler/optimizing/code_generator_arm64.cc index 26c8254c76..27b42536ef 100644 --- a/compiler/optimizing/code_generator_arm64.cc +++ b/compiler/optimizing/code_generator_arm64.cc @@ -275,15 +275,37 @@ class LoadClassSlowPathARM64 : public SlowPathCodeARM64 { LoadClassSlowPathARM64(HLoadClass* cls, HInstruction* at, uint32_t dex_pc, - bool do_clinit) - : SlowPathCodeARM64(at), cls_(cls), dex_pc_(dex_pc), do_clinit_(do_clinit) { + bool do_clinit, + vixl::aarch64::Register bss_entry_temp = vixl::aarch64::Register(), + vixl::aarch64::Label* bss_entry_adrp_label = nullptr) + : SlowPathCodeARM64(at), + cls_(cls), + dex_pc_(dex_pc), + do_clinit_(do_clinit), + bss_entry_temp_(bss_entry_temp), + bss_entry_adrp_label_(bss_entry_adrp_label) { DCHECK(at->IsLoadClass() || at->IsClinitCheck()); } void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { LocationSummary* locations = instruction_->GetLocations(); + Location out = locations->Out(); + constexpr bool call_saves_everything_except_r0_ip0 = (!kUseReadBarrier || kUseBakerReadBarrier); CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen); + // For HLoadClass/kBssEntry/kSaveEverything, make sure we preserve the page address of + // the entry which is in a scratch register. Make sure it's not used for saving/restoring + // registers. Exclude the scratch register also for non-Baker read barrier for simplicity. + DCHECK_EQ(instruction_->IsLoadClass(), cls_ == instruction_); + bool is_load_class_bss_entry = + (cls_ == instruction_) && (cls_->GetLoadKind() == HLoadClass::LoadKind::kBssEntry); + UseScratchRegisterScope temps(arm64_codegen->GetVIXLAssembler()); + if (is_load_class_bss_entry) { + // This temp is a scratch register. + DCHECK(bss_entry_temp_.IsValid()); + temps.Exclude(bss_entry_temp_); + } + __ Bind(GetEntryLabel()); SaveLiveRegisters(codegen, locations); @@ -300,7 +322,6 @@ class LoadClassSlowPathARM64 : public SlowPathCodeARM64 { } // Move the class to the desired location. - Location out = locations->Out(); if (out.IsValid()) { DCHECK(out.IsRegister() && !locations->GetLiveRegisters()->ContainsCoreRegister(out.reg())); Primitive::Type type = instruction_->GetType(); @@ -308,25 +329,23 @@ class LoadClassSlowPathARM64 : public SlowPathCodeARM64 { } RestoreLiveRegisters(codegen, locations); // For HLoadClass/kBssEntry, store the resolved Class to the BSS entry. - DCHECK_EQ(instruction_->IsLoadClass(), cls_ == instruction_); - if (cls_ == instruction_ && cls_->GetLoadKind() == HLoadClass::LoadKind::kBssEntry) { + if (is_load_class_bss_entry) { DCHECK(out.IsValid()); - UseScratchRegisterScope temps(arm64_codegen->GetVIXLAssembler()); - Register temp = temps.AcquireX(); const DexFile& dex_file = cls_->GetDexFile(); - // TODO: Change art_quick_initialize_type/art_quick_initialize_static_storage to - // kSaveEverything and use a temporary for the ADRP in the fast path, so that we - // can avoid the ADRP here. - vixl::aarch64::Label* adrp_label = - arm64_codegen->NewBssEntryTypePatch(dex_file, type_index); - arm64_codegen->EmitAdrpPlaceholder(adrp_label, temp); + if (call_saves_everything_except_r0_ip0) { + // The class entry page address was preserved in bss_entry_temp_ thanks to kSaveEverything. + } else { + // For non-Baker read barrier, we need to re-calculate the address of the class entry page. + bss_entry_adrp_label_ = arm64_codegen->NewBssEntryTypePatch(dex_file, type_index); + arm64_codegen->EmitAdrpPlaceholder(bss_entry_adrp_label_, bss_entry_temp_); + } vixl::aarch64::Label* strp_label = - arm64_codegen->NewBssEntryTypePatch(dex_file, type_index, adrp_label); + arm64_codegen->NewBssEntryTypePatch(dex_file, type_index, bss_entry_adrp_label_); { SingleEmissionCheckScope guard(arm64_codegen->GetVIXLAssembler()); __ Bind(strp_label); __ str(RegisterFrom(locations->Out(), Primitive::kPrimNot), - MemOperand(temp, /* offset placeholder */ 0)); + MemOperand(bss_entry_temp_, /* offset placeholder */ 0)); } } __ B(GetExitLabel()); @@ -344,6 +363,10 @@ class LoadClassSlowPathARM64 : public SlowPathCodeARM64 { // Whether to initialize the class. const bool do_clinit_; + // For HLoadClass/kBssEntry, the temp register and the label of the ADRP where it was loaded. + vixl::aarch64::Register bss_entry_temp_; + vixl::aarch64::Label* bss_entry_adrp_label_; + DISALLOW_COPY_AND_ASSIGN(LoadClassSlowPathARM64); }; @@ -619,8 +642,10 @@ void JumpTableARM64::EmitTable(CodeGeneratorARM64* codegen) { // probably still be a from-space reference (unless it gets updated by // another thread, or if another thread installed another object // reference (different from `ref`) in `obj.field`). -// If entrypoint is a valid location it is assumed to already be holding the entrypoint. The case -// where the entrypoint is passed in is for the GcRoot read barrier. +// +// If `entrypoint` is a valid location it is assumed to already be +// holding the entrypoint. The case where the entrypoint is passed in +// is for the GcRoot read barrier. class ReadBarrierMarkSlowPathARM64 : public SlowPathCodeARM64 { public: ReadBarrierMarkSlowPathARM64(HInstruction* instruction, @@ -4360,6 +4385,9 @@ void InstructionCodeGeneratorARM64::VisitInvokeVirtual(HInvokeVirtual* invoke) { HLoadClass::LoadKind CodeGeneratorARM64::GetSupportedLoadClassKind( HLoadClass::LoadKind desired_class_load_kind) { switch (desired_class_load_kind) { + case HLoadClass::LoadKind::kInvalid: + LOG(FATAL) << "UNREACHABLE"; + UNREACHABLE(); case HLoadClass::LoadKind::kReferrersClass: break; case HLoadClass::LoadKind::kBootImageLinkTimeAddress: @@ -4390,6 +4418,7 @@ void LocationsBuilderARM64::VisitLoadClass(HLoadClass* cls) { cls, LocationFrom(calling_convention.GetRegisterAt(0)), LocationFrom(vixl::aarch64::x0)); + DCHECK(calling_convention.GetRegisterAt(0).Is(vixl::aarch64::x0)); return; } DCHECK(!cls->NeedsAccessCheck()); @@ -4407,6 +4436,22 @@ void LocationsBuilderARM64::VisitLoadClass(HLoadClass* cls) { locations->SetInAt(0, Location::RequiresRegister()); } locations->SetOut(Location::RequiresRegister()); + if (cls->GetLoadKind() == HLoadClass::LoadKind::kBssEntry) { + if (!kUseReadBarrier || kUseBakerReadBarrier) { + // Rely on the type resolution or initialization and marking to save everything we need. + // Note that IP0 may be clobbered by saving/restoring the live register (only one thanks + // to the custom calling convention) or by marking, so we shall use IP1. + RegisterSet caller_saves = RegisterSet::Empty(); + InvokeRuntimeCallingConvention calling_convention; + caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0).GetCode())); + DCHECK_EQ(calling_convention.GetRegisterAt(0).GetCode(), + RegisterFrom(calling_convention.GetReturnLocation(Primitive::kPrimNot), + Primitive::kPrimNot).GetCode()); + locations->SetCustomSlowPathCallerSaves(caller_saves); + } else { + // For non-Baker read barrier we have a temp-clobbering call. + } + } } // NO_THREAD_SAFETY_ANALYSIS as we manipulate handles whose internal object we know does not @@ -4421,6 +4466,8 @@ void InstructionCodeGeneratorARM64::VisitLoadClass(HLoadClass* cls) NO_THREAD_SA Location out_loc = cls->GetLocations()->Out(); Register out = OutputRegister(cls); + Register bss_entry_temp; + vixl::aarch64::Label* bss_entry_adrp_label = nullptr; const ReadBarrierOption read_barrier_option = cls->IsInBootImage() ? kWithoutReadBarrier @@ -4470,18 +4517,23 @@ void InstructionCodeGeneratorARM64::VisitLoadClass(HLoadClass* cls) NO_THREAD_SA // Add ADRP with its PC-relative Class .bss entry patch. const DexFile& dex_file = cls->GetDexFile(); dex::TypeIndex type_index = cls->GetTypeIndex(); - vixl::aarch64::Label* adrp_label = codegen_->NewBssEntryTypePatch(dex_file, type_index); - codegen_->EmitAdrpPlaceholder(adrp_label, out.X()); + // We can go to slow path even with non-zero reference and in that case marking + // can clobber IP0, so we need to use IP1 which shall be preserved. + bss_entry_temp = ip1; + UseScratchRegisterScope temps(codegen_->GetVIXLAssembler()); + temps.Exclude(bss_entry_temp); + bss_entry_adrp_label = codegen_->NewBssEntryTypePatch(dex_file, type_index); + codegen_->EmitAdrpPlaceholder(bss_entry_adrp_label, bss_entry_temp); // Add LDR with its PC-relative Class patch. vixl::aarch64::Label* ldr_label = - codegen_->NewBssEntryTypePatch(dex_file, type_index, adrp_label); + codegen_->NewBssEntryTypePatch(dex_file, type_index, bss_entry_adrp_label); // /* GcRoot<mirror::Class> */ out = *(base_address + offset) /* PC-relative */ GenerateGcRootFieldLoad(cls, - cls->GetLocations()->Out(), - out.X(), - /* placeholder */ 0u, + out_loc, + bss_entry_temp, + /* offset placeholder */ 0u, ldr_label, - kCompilerReadBarrierOption); + read_barrier_option); generate_null_check = true; break; } @@ -4494,18 +4546,20 @@ void InstructionCodeGeneratorARM64::VisitLoadClass(HLoadClass* cls) NO_THREAD_SA out.X(), /* offset */ 0, /* fixup_label */ nullptr, - kCompilerReadBarrierOption); + read_barrier_option); break; } case HLoadClass::LoadKind::kDexCacheViaMethod: + case HLoadClass::LoadKind::kInvalid: LOG(FATAL) << "UNREACHABLE"; UNREACHABLE(); } - if (generate_null_check || cls->MustGenerateClinitCheck()) { + bool do_clinit = cls->MustGenerateClinitCheck(); + if (generate_null_check || do_clinit) { DCHECK(cls->CanCallRuntime()); SlowPathCodeARM64* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathARM64( - cls, cls, cls->GetDexPc(), cls->MustGenerateClinitCheck()); + cls, cls, cls->GetDexPc(), do_clinit, bss_entry_temp, bss_entry_adrp_label); codegen_->AddSlowPath(slow_path); if (generate_null_check) { __ Cbz(out, slow_path->GetEntryLabel()); @@ -4573,7 +4627,9 @@ void LocationsBuilderARM64::VisitLoadString(HLoadString* load) { locations->SetOut(Location::RequiresRegister()); if (load->GetLoadKind() == HLoadString::LoadKind::kBssEntry) { if (!kUseReadBarrier || kUseBakerReadBarrier) { - // Rely on the pResolveString and/or marking to save everything, including temps. + // Rely on the pResolveString and marking to save everything we need. + // Note that IP0 may be clobbered by saving/restoring the live register (only one thanks + // to the custom calling convention) or by marking, so we shall use IP1. RegisterSet caller_saves = RegisterSet::Empty(); InvokeRuntimeCallingConvention calling_convention; caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0).GetCode())); @@ -4624,8 +4680,11 @@ void InstructionCodeGeneratorARM64::VisitLoadString(HLoadString* load) NO_THREAD const DexFile& dex_file = load->GetDexFile(); const dex::StringIndex string_index = load->GetStringIndex(); DCHECK(!codegen_->GetCompilerOptions().IsBootImage()); + // We could use IP0 as the marking shall not clobber IP0 if the reference is null and + // that's when we need the slow path. But let's not rely on such details and use IP1. + Register temp = ip1; UseScratchRegisterScope temps(codegen_->GetVIXLAssembler()); - Register temp = temps.AcquireX(); + temps.Exclude(temp); vixl::aarch64::Label* adrp_label = codegen_->NewPcRelativeStringPatch(dex_file, string_index); codegen_->EmitAdrpPlaceholder(adrp_label, temp); // Add LDR with its PC-relative String patch. diff --git a/compiler/optimizing/code_generator_arm_vixl.cc b/compiler/optimizing/code_generator_arm_vixl.cc index f4d3ec54ee..5c4ca5bc17 100644 --- a/compiler/optimizing/code_generator_arm_vixl.cc +++ b/compiler/optimizing/code_generator_arm_vixl.cc @@ -400,12 +400,30 @@ class LoadClassSlowPathARMVIXL : public SlowPathCodeARMVIXL { void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { LocationSummary* locations = instruction_->GetLocations(); + Location out = locations->Out(); + constexpr bool call_saves_everything_except_r0 = (!kUseReadBarrier || kUseBakerReadBarrier); CodeGeneratorARMVIXL* arm_codegen = down_cast<CodeGeneratorARMVIXL*>(codegen); __ Bind(GetEntryLabel()); SaveLiveRegisters(codegen, locations); InvokeRuntimeCallingConventionARMVIXL calling_convention; + // For HLoadClass/kBssEntry/kSaveEverything, make sure we preserve the address of the entry. + DCHECK_EQ(instruction_->IsLoadClass(), cls_ == instruction_); + bool is_load_class_bss_entry = + (cls_ == instruction_) && (cls_->GetLoadKind() == HLoadClass::LoadKind::kBssEntry); + vixl32::Register entry_address; + if (is_load_class_bss_entry && call_saves_everything_except_r0) { + vixl32::Register temp = RegisterFrom(locations->GetTemp(0)); + // In the unlucky case that the `temp` is R0, we preserve the address in `out` across + // the kSaveEverything call. + bool temp_is_r0 = temp.Is(calling_convention.GetRegisterAt(0)); + entry_address = temp_is_r0 ? RegisterFrom(out) : temp; + DCHECK(!entry_address.Is(calling_convention.GetRegisterAt(0))); + if (temp_is_r0) { + __ Mov(entry_address, temp); + } + } dex::TypeIndex type_index = cls_->GetTypeIndex(); __ Mov(calling_convention.GetRegisterAt(0), type_index.index_); QuickEntrypointEnum entrypoint = do_clinit_ ? kQuickInitializeStaticStorage @@ -417,27 +435,28 @@ class LoadClassSlowPathARMVIXL : public SlowPathCodeARMVIXL { CheckEntrypointTypes<kQuickInitializeType, void*, uint32_t>(); } + // For HLoadClass/kBssEntry, store the resolved Class to the BSS entry. + if (is_load_class_bss_entry) { + if (call_saves_everything_except_r0) { + // The class entry address was preserved in `entry_address` thanks to kSaveEverything. + __ Str(r0, MemOperand(entry_address)); + } else { + // For non-Baker read barrier, we need to re-calculate the address of the string entry. + UseScratchRegisterScope temps( + down_cast<CodeGeneratorARMVIXL*>(codegen)->GetVIXLAssembler()); + vixl32::Register temp = temps.Acquire(); + CodeGeneratorARMVIXL::PcRelativePatchInfo* labels = + arm_codegen->NewTypeBssEntryPatch(cls_->GetDexFile(), type_index); + arm_codegen->EmitMovwMovtPlaceholder(labels, temp); + __ Str(r0, MemOperand(temp)); + } + } // Move the class to the desired location. - Location out = locations->Out(); if (out.IsValid()) { DCHECK(out.IsRegister() && !locations->GetLiveRegisters()->ContainsCoreRegister(out.reg())); arm_codegen->Move32(locations->Out(), LocationFrom(r0)); } RestoreLiveRegisters(codegen, locations); - // For HLoadClass/kBssEntry, store the resolved Class to the BSS entry. - DCHECK_EQ(instruction_->IsLoadClass(), cls_ == instruction_); - if (cls_ == instruction_ && cls_->GetLoadKind() == HLoadClass::LoadKind::kBssEntry) { - DCHECK(out.IsValid()); - // TODO: Change art_quick_initialize_type/art_quick_initialize_static_storage to - // kSaveEverything and use a temporary for the .bss entry address in the fast path, - // so that we can avoid another calculation here. - UseScratchRegisterScope temps(down_cast<CodeGeneratorARMVIXL*>(codegen)->GetVIXLAssembler()); - vixl32::Register temp = temps.Acquire(); - CodeGeneratorARMVIXL::PcRelativePatchInfo* labels = - arm_codegen->NewTypeBssEntryPatch(cls_->GetDexFile(), type_index); - arm_codegen->EmitMovwMovtPlaceholder(labels, temp); - __ Str(OutputRegister(cls_), MemOperand(temp)); - } __ B(GetExitLabel()); } @@ -462,12 +481,13 @@ class LoadStringSlowPathARMVIXL : public SlowPathCodeARMVIXL { : SlowPathCodeARMVIXL(instruction) {} void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { + DCHECK(instruction_->IsLoadString()); + DCHECK_EQ(instruction_->AsLoadString()->GetLoadKind(), HLoadString::LoadKind::kBssEntry); LocationSummary* locations = instruction_->GetLocations(); DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg())); HLoadString* load = instruction_->AsLoadString(); const dex::StringIndex string_index = load->GetStringIndex(); vixl32::Register out = OutputRegister(load); - vixl32::Register temp = RegisterFrom(locations->GetTemp(0)); constexpr bool call_saves_everything_except_r0 = (!kUseReadBarrier || kUseBakerReadBarrier); CodeGeneratorARMVIXL* arm_codegen = down_cast<CodeGeneratorARMVIXL*>(codegen); @@ -476,12 +496,16 @@ class LoadStringSlowPathARMVIXL : public SlowPathCodeARMVIXL { InvokeRuntimeCallingConventionARMVIXL calling_convention; // In the unlucky case that the `temp` is R0, we preserve the address in `out` across - // the kSaveEverything call (or use `out` for the address after non-kSaveEverything call). - bool temp_is_r0 = (temp.Is(calling_convention.GetRegisterAt(0))); - vixl32::Register entry_address = temp_is_r0 ? out : temp; - DCHECK(!entry_address.Is(calling_convention.GetRegisterAt(0))); - if (call_saves_everything_except_r0 && temp_is_r0) { - __ Mov(entry_address, temp); + // the kSaveEverything call. + vixl32::Register entry_address; + if (call_saves_everything_except_r0) { + vixl32::Register temp = RegisterFrom(locations->GetTemp(0)); + bool temp_is_r0 = (temp.Is(calling_convention.GetRegisterAt(0))); + entry_address = temp_is_r0 ? out : temp; + DCHECK(!entry_address.Is(calling_convention.GetRegisterAt(0))); + if (temp_is_r0) { + __ Mov(entry_address, temp); + } } __ Mov(calling_convention.GetRegisterAt(0), string_index.index_); @@ -494,10 +518,13 @@ class LoadStringSlowPathARMVIXL : public SlowPathCodeARMVIXL { __ Str(r0, MemOperand(entry_address)); } else { // For non-Baker read barrier, we need to re-calculate the address of the string entry. + UseScratchRegisterScope temps( + down_cast<CodeGeneratorARMVIXL*>(codegen)->GetVIXLAssembler()); + vixl32::Register temp = temps.Acquire(); CodeGeneratorARMVIXL::PcRelativePatchInfo* labels = arm_codegen->NewPcRelativeStringPatch(load->GetDexFile(), string_index); - arm_codegen->EmitMovwMovtPlaceholder(labels, out); - __ Str(r0, MemOperand(entry_address)); + arm_codegen->EmitMovwMovtPlaceholder(labels, temp); + __ Str(r0, MemOperand(temp)); } arm_codegen->Move32(locations->Out(), LocationFrom(r0)); @@ -4005,8 +4032,11 @@ void LocationsBuilderARMVIXL::VisitNewArray(HNewArray* instruction) { void InstructionCodeGeneratorARMVIXL::VisitNewArray(HNewArray* instruction) { // Note: if heap poisoning is enabled, the entry point takes cares // of poisoning the reference. - codegen_->InvokeRuntime(kQuickAllocArrayResolved, instruction, instruction->GetDexPc()); + QuickEntrypointEnum entrypoint = + CodeGenerator::GetArrayAllocationEntrypoint(instruction->GetLoadClass()->GetClass()); + codegen_->InvokeRuntime(entrypoint, instruction, instruction->GetDexPc()); CheckEntrypointTypes<kQuickAllocArrayResolved, void*, mirror::Class*, int32_t>(); + DCHECK(!codegen_->IsLeafMethod()); } void LocationsBuilderARMVIXL::VisitParameterValue(HParameterValue* instruction) { @@ -5796,6 +5826,9 @@ void ParallelMoveResolverARMVIXL::RestoreScratch(int reg ATTRIBUTE_UNUSED) { HLoadClass::LoadKind CodeGeneratorARMVIXL::GetSupportedLoadClassKind( HLoadClass::LoadKind desired_class_load_kind) { switch (desired_class_load_kind) { + case HLoadClass::LoadKind::kInvalid: + LOG(FATAL) << "UNREACHABLE"; + UNREACHABLE(); case HLoadClass::LoadKind::kReferrersClass: break; case HLoadClass::LoadKind::kBootImageLinkTimeAddress: @@ -5826,6 +5859,7 @@ void LocationsBuilderARMVIXL::VisitLoadClass(HLoadClass* cls) { cls, LocationFrom(calling_convention.GetRegisterAt(0)), LocationFrom(r0)); + DCHECK(calling_convention.GetRegisterAt(0).Is(r0)); return; } DCHECK(!cls->NeedsAccessCheck()); @@ -5843,6 +5877,22 @@ void LocationsBuilderARMVIXL::VisitLoadClass(HLoadClass* cls) { locations->SetInAt(0, Location::RequiresRegister()); } locations->SetOut(Location::RequiresRegister()); + if (load_kind == HLoadClass::LoadKind::kBssEntry) { + if (!kUseReadBarrier || kUseBakerReadBarrier) { + // Rely on the type resolution or initialization and marking to save everything we need. + // Note that IP may be clobbered by saving/restoring the live register (only one thanks + // to the custom calling convention) or by marking, so we request a different temp. + locations->AddTemp(Location::RequiresRegister()); + RegisterSet caller_saves = RegisterSet::Empty(); + InvokeRuntimeCallingConventionARMVIXL calling_convention; + caller_saves.Add(LocationFrom(calling_convention.GetRegisterAt(0))); + // TODO: Add GetReturnLocation() to the calling convention so that we can DCHECK() + // that the the kPrimNot result register is the same as the first argument register. + locations->SetCustomSlowPathCallerSaves(caller_saves); + } else { + // For non-Baker read barrier we have a temp-clobbering call. + } + } } // NO_THREAD_SAFETY_ANALYSIS as we manipulate handles whose internal object we know does not @@ -5900,10 +5950,13 @@ void InstructionCodeGeneratorARMVIXL::VisitLoadClass(HLoadClass* cls) NO_THREAD_ break; } case HLoadClass::LoadKind::kBssEntry: { + vixl32::Register temp = (!kUseReadBarrier || kUseBakerReadBarrier) + ? RegisterFrom(locations->GetTemp(0)) + : out; CodeGeneratorARMVIXL::PcRelativePatchInfo* labels = codegen_->NewTypeBssEntryPatch(cls->GetDexFile(), cls->GetTypeIndex()); - codegen_->EmitMovwMovtPlaceholder(labels, out); - GenerateGcRootFieldLoad(cls, out_loc, out, 0, kCompilerReadBarrierOption); + codegen_->EmitMovwMovtPlaceholder(labels, temp); + GenerateGcRootFieldLoad(cls, out_loc, temp, /* offset */ 0, read_barrier_option); generate_null_check = true; break; } @@ -5912,10 +5965,11 @@ void InstructionCodeGeneratorARMVIXL::VisitLoadClass(HLoadClass* cls) NO_THREAD_ cls->GetTypeIndex(), cls->GetClass())); // /* GcRoot<mirror::Class> */ out = *out - GenerateGcRootFieldLoad(cls, out_loc, out, /* offset */ 0, kCompilerReadBarrierOption); + GenerateGcRootFieldLoad(cls, out_loc, out, /* offset */ 0, read_barrier_option); break; } case HLoadClass::LoadKind::kDexCacheViaMethod: + case HLoadClass::LoadKind::kInvalid: LOG(FATAL) << "UNREACHABLE"; UNREACHABLE(); } @@ -6005,9 +6059,9 @@ void LocationsBuilderARMVIXL::VisitLoadString(HLoadString* load) { locations->SetOut(Location::RequiresRegister()); if (load_kind == HLoadString::LoadKind::kBssEntry) { if (!kUseReadBarrier || kUseBakerReadBarrier) { - // Rely on the pResolveString and/or marking to save everything, including temps. - // Note that IP may theoretically be clobbered by saving/restoring the live register - // (only one thanks to the custom calling convention), so we request a different temp. + // Rely on the pResolveString and marking to save everything we need, including temps. + // Note that IP may be clobbered by saving/restoring the live register (only one thanks + // to the custom calling convention) or by marking, so we request a different temp. locations->AddTemp(Location::RequiresRegister()); RegisterSet caller_saves = RegisterSet::Empty(); InvokeRuntimeCallingConventionARMVIXL calling_convention; @@ -6052,7 +6106,9 @@ void InstructionCodeGeneratorARMVIXL::VisitLoadString(HLoadString* load) NO_THRE } case HLoadString::LoadKind::kBssEntry: { DCHECK(!codegen_->GetCompilerOptions().IsBootImage()); - vixl32::Register temp = RegisterFrom(locations->GetTemp(0)); + vixl32::Register temp = (!kUseReadBarrier || kUseBakerReadBarrier) + ? RegisterFrom(locations->GetTemp(0)) + : out; CodeGeneratorARMVIXL::PcRelativePatchInfo* labels = codegen_->NewPcRelativeStringPatch(load->GetDexFile(), load->GetStringIndex()); codegen_->EmitMovwMovtPlaceholder(labels, temp); @@ -7253,8 +7309,7 @@ vixl32::Register CodeGeneratorARMVIXL::GetInvokeStaticOrDirectExtraParameter( // save one load. However, since this is just an intrinsic slow path we prefer this // simple and more robust approach rather that trying to determine if that's the case. SlowPathCode* slow_path = GetCurrentSlowPath(); - DCHECK(slow_path != nullptr); // For intrinsified invokes the call is emitted on the slow path. - if (slow_path->IsCoreRegisterSaved(RegisterFrom(location).GetCode())) { + if (slow_path != nullptr && slow_path->IsCoreRegisterSaved(RegisterFrom(location).GetCode())) { int stack_offset = slow_path->GetStackOffsetOfCoreRegister(RegisterFrom(location).GetCode()); GetAssembler()->LoadFromOffset(kLoadWord, temp, sp, stack_offset); return temp; diff --git a/compiler/optimizing/code_generator_mips.cc b/compiler/optimizing/code_generator_mips.cc index a095970a1e..0677dad078 100644 --- a/compiler/optimizing/code_generator_mips.cc +++ b/compiler/optimizing/code_generator_mips.cc @@ -484,6 +484,8 @@ CodeGeneratorMIPS::CodeGeneratorMIPS(HGraph* graph, type_bss_entry_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), boot_image_address_patches_(std::less<uint32_t>(), graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + jit_string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + jit_class_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), clobbered_ra_(false) { // Save RA (containing the return address) to mimic Quick. AddAllocatedRegister(Location::RegisterLocation(RA)); @@ -704,9 +706,6 @@ bool CodeGeneratorMIPS::HasAllocatedCalleeSaveRegisters() const { // (this can happen in leaf methods), force CodeGenerator::InitializeCodeGeneration() // into the path that creates a stack frame so that RA can be explicitly saved and restored. // RA can't otherwise be saved/restored when it's the only spilled register. - // TODO: Can this be improved? It causes creation of a stack frame (while RA might be - // saved in an unused temporary register) and saving of RA and the current method pointer - // in the frame. return CodeGenerator::HasAllocatedCalleeSaveRegisters() || clobbered_ra_; } @@ -1160,6 +1159,67 @@ void CodeGeneratorMIPS::EmitPcRelativeAddressPlaceholderHigh(PcRelativePatchInfo // offset to `out` (e.g. lw, jialc, addiu). } +CodeGeneratorMIPS::JitPatchInfo* CodeGeneratorMIPS::NewJitRootStringPatch( + const DexFile& dex_file, + dex::StringIndex dex_index, + Handle<mirror::String> handle) { + jit_string_roots_.Overwrite(StringReference(&dex_file, dex_index), + reinterpret_cast64<uint64_t>(handle.GetReference())); + jit_string_patches_.emplace_back(dex_file, dex_index.index_); + return &jit_string_patches_.back(); +} + +CodeGeneratorMIPS::JitPatchInfo* CodeGeneratorMIPS::NewJitRootClassPatch( + const DexFile& dex_file, + dex::TypeIndex dex_index, + Handle<mirror::Class> handle) { + jit_class_roots_.Overwrite(TypeReference(&dex_file, dex_index), + reinterpret_cast64<uint64_t>(handle.GetReference())); + jit_class_patches_.emplace_back(dex_file, dex_index.index_); + return &jit_class_patches_.back(); +} + +void CodeGeneratorMIPS::PatchJitRootUse(uint8_t* code, + const uint8_t* roots_data, + const CodeGeneratorMIPS::JitPatchInfo& info, + uint64_t index_in_table) const { + uint32_t literal_offset = GetAssembler().GetLabelLocation(&info.high_label); + uintptr_t address = + reinterpret_cast<uintptr_t>(roots_data) + index_in_table * sizeof(GcRoot<mirror::Object>); + uint32_t addr32 = dchecked_integral_cast<uint32_t>(address); + // lui reg, addr32_high + DCHECK_EQ(code[literal_offset + 0], 0x34); + DCHECK_EQ(code[literal_offset + 1], 0x12); + DCHECK_EQ((code[literal_offset + 2] & 0xE0), 0x00); + DCHECK_EQ(code[literal_offset + 3], 0x3C); + // lw reg, reg, addr32_low + DCHECK_EQ(code[literal_offset + 4], 0x78); + DCHECK_EQ(code[literal_offset + 5], 0x56); + DCHECK_EQ((code[literal_offset + 7] & 0xFC), 0x8C); + addr32 += (addr32 & 0x8000) << 1; // Account for sign extension in "lw reg, reg, addr32_low". + // lui reg, addr32_high + code[literal_offset + 0] = static_cast<uint8_t>(addr32 >> 16); + code[literal_offset + 1] = static_cast<uint8_t>(addr32 >> 24); + // lw reg, reg, addr32_low + code[literal_offset + 4] = static_cast<uint8_t>(addr32 >> 0); + code[literal_offset + 5] = static_cast<uint8_t>(addr32 >> 8); +} + +void CodeGeneratorMIPS::EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) { + for (const JitPatchInfo& info : jit_string_patches_) { + const auto& it = jit_string_roots_.find(StringReference(&info.target_dex_file, + dex::StringIndex(info.index))); + DCHECK(it != jit_string_roots_.end()); + PatchJitRootUse(code, roots_data, info, it->second); + } + for (const JitPatchInfo& info : jit_class_patches_) { + const auto& it = jit_class_roots_.find(TypeReference(&info.target_dex_file, + dex::TypeIndex(info.index))); + DCHECK(it != jit_class_roots_.end()); + PatchJitRootUse(code, roots_data, info, it->second); + } +} + void CodeGeneratorMIPS::MarkGCCard(Register object, Register value, bool value_can_be_null) { @@ -5225,8 +5285,7 @@ HLoadString::LoadKind CodeGeneratorMIPS::GetSupportedLoadStringKind( break; case HLoadString::LoadKind::kJitTableAddress: DCHECK(Runtime::Current()->UseJitCompilation()); - // TODO: implement. - fallback_load = true; + fallback_load = false; break; case HLoadString::LoadKind::kDexCacheViaMethod: fallback_load = false; @@ -5249,6 +5308,9 @@ HLoadClass::LoadKind CodeGeneratorMIPS::GetSupportedLoadClassKind( bool is_r6 = GetInstructionSetFeatures().IsR6(); bool fallback_load = has_irreducible_loops && !is_r6; switch (desired_class_load_kind) { + case HLoadClass::LoadKind::kInvalid: + LOG(FATAL) << "UNREACHABLE"; + UNREACHABLE(); case HLoadClass::LoadKind::kReferrersClass: fallback_load = false; break; @@ -5265,8 +5327,7 @@ HLoadClass::LoadKind CodeGeneratorMIPS::GetSupportedLoadClassKind( break; case HLoadClass::LoadKind::kJitTableAddress: DCHECK(Runtime::Current()->UseJitCompilation()); - // TODO: implement. - fallback_load = true; + fallback_load = false; break; case HLoadClass::LoadKind::kDexCacheViaMethod: fallback_load = false; @@ -5591,10 +5652,18 @@ void InstructionCodeGeneratorMIPS::VisitLoadClass(HLoadClass* cls) NO_THREAD_SAF break; } case HLoadClass::LoadKind::kJitTableAddress: { - LOG(FATAL) << "Unimplemented"; + CodeGeneratorMIPS::JitPatchInfo* info = codegen_->NewJitRootClassPatch(cls->GetDexFile(), + cls->GetTypeIndex(), + cls->GetClass()); + bool reordering = __ SetReorder(false); + __ Bind(&info->high_label); + __ Lui(out, /* placeholder */ 0x1234); + GenerateGcRootFieldLoad(cls, out_loc, out, /* placeholder */ 0x5678); + __ SetReorder(reordering); break; } case HLoadClass::LoadKind::kDexCacheViaMethod: + case HLoadClass::LoadKind::kInvalid: LOG(FATAL) << "UNREACHABLE"; UNREACHABLE(); } @@ -5730,6 +5799,18 @@ void InstructionCodeGeneratorMIPS::VisitLoadString(HLoadString* load) NO_THREAD_ __ Bind(slow_path->GetExitLabel()); return; } + case HLoadString::LoadKind::kJitTableAddress: { + CodeGeneratorMIPS::JitPatchInfo* info = + codegen_->NewJitRootStringPatch(load->GetDexFile(), + load->GetStringIndex(), + load->GetString()); + bool reordering = __ SetReorder(false); + __ Bind(&info->high_label); + __ Lui(out, /* placeholder */ 0x1234); + GenerateGcRootFieldLoad(load, out_loc, out, /* placeholder */ 0x5678); + __ SetReorder(reordering); + return; + } default: break; } diff --git a/compiler/optimizing/code_generator_mips.h b/compiler/optimizing/code_generator_mips.h index e92eeef88f..47eba50248 100644 --- a/compiler/optimizing/code_generator_mips.h +++ b/compiler/optimizing/code_generator_mips.h @@ -352,6 +352,7 @@ class CodeGeneratorMIPS : public CodeGenerator { // Emit linker patches. void EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) OVERRIDE; + void EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) OVERRIDE; void MarkGCCard(Register object, Register value, bool value_can_be_null); @@ -465,6 +466,31 @@ class CodeGeneratorMIPS : public CodeGenerator { void EmitPcRelativeAddressPlaceholderHigh(PcRelativePatchInfo* info, Register out, Register base); + // The JitPatchInfo is used for JIT string and class loads. + struct JitPatchInfo { + JitPatchInfo(const DexFile& dex_file, uint64_t idx) + : target_dex_file(dex_file), index(idx) { } + JitPatchInfo(JitPatchInfo&& other) = default; + + const DexFile& target_dex_file; + // String/type index. + uint64_t index; + // Label for the instruction loading the most significant half of the address. + // The least significant half is loaded with the instruction that follows immediately. + MipsLabel high_label; + }; + + void PatchJitRootUse(uint8_t* code, + const uint8_t* roots_data, + const JitPatchInfo& info, + uint64_t index_in_table) const; + JitPatchInfo* NewJitRootStringPatch(const DexFile& dex_file, + dex::StringIndex dex_index, + Handle<mirror::String> handle); + JitPatchInfo* NewJitRootClassPatch(const DexFile& dex_file, + dex::TypeIndex dex_index, + Handle<mirror::Class> handle); + private: Register GetInvokeStaticOrDirectExtraParameter(HInvokeStaticOrDirect* invoke, Register temp); @@ -512,6 +538,10 @@ class CodeGeneratorMIPS : public CodeGenerator { ArenaDeque<PcRelativePatchInfo> type_bss_entry_patches_; // Deduplication map for patchable boot image addresses. Uint32ToLiteralMap boot_image_address_patches_; + // Patches for string root accesses in JIT compiled code. + ArenaDeque<JitPatchInfo> jit_string_patches_; + // Patches for class root accesses in JIT compiled code. + ArenaDeque<JitPatchInfo> jit_class_patches_; // PC-relative loads on R2 clobber RA, which may need to be preserved explicitly in leaf methods. // This is a flag set by pc_relative_fixups_mips and dex_cache_array_fixups_mips optimizations. diff --git a/compiler/optimizing/code_generator_mips64.cc b/compiler/optimizing/code_generator_mips64.cc index e96e3d75e1..4c8dabfede 100644 --- a/compiler/optimizing/code_generator_mips64.cc +++ b/compiler/optimizing/code_generator_mips64.cc @@ -91,9 +91,6 @@ Location InvokeDexCallingConventionVisitorMIPS64::GetNextLocation(Primitive::Typ // Space on the stack is reserved for all arguments. stack_index_ += Primitive::Is64BitType(type) ? 2 : 1; - // TODO: shouldn't we use a whole machine word per argument on the stack? - // Implicit 4-byte method pointer (and such) will cause misalignment. - return next_location; } @@ -434,7 +431,11 @@ CodeGeneratorMIPS64::CodeGeneratorMIPS64(HGraph* graph, pc_relative_type_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), type_bss_entry_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), boot_image_address_patches_(std::less<uint32_t>(), - graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) { + graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + jit_string_patches_(StringReferenceValueComparator(), + graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + jit_class_patches_(TypeReferenceValueComparator(), + graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) { // Save RA (containing the return address) to mimic Quick. AddAllocatedRegister(Location::RegisterLocation(RA)); } @@ -1055,6 +1056,49 @@ void CodeGeneratorMIPS64::EmitPcRelativeAddressPlaceholderHigh(PcRelativePatchIn // offset to `out` (e.g. ld, jialc, daddiu). } +Literal* CodeGeneratorMIPS64::DeduplicateJitStringLiteral(const DexFile& dex_file, + dex::StringIndex string_index, + Handle<mirror::String> handle) { + jit_string_roots_.Overwrite(StringReference(&dex_file, string_index), + reinterpret_cast64<uint64_t>(handle.GetReference())); + return jit_string_patches_.GetOrCreate( + StringReference(&dex_file, string_index), + [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); +} + +Literal* CodeGeneratorMIPS64::DeduplicateJitClassLiteral(const DexFile& dex_file, + dex::TypeIndex type_index, + Handle<mirror::Class> handle) { + jit_class_roots_.Overwrite(TypeReference(&dex_file, type_index), + reinterpret_cast64<uint64_t>(handle.GetReference())); + return jit_class_patches_.GetOrCreate( + TypeReference(&dex_file, type_index), + [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); +} + +void CodeGeneratorMIPS64::PatchJitRootUse(uint8_t* code, + const uint8_t* roots_data, + const Literal* literal, + uint64_t index_in_table) const { + uint32_t literal_offset = GetAssembler().GetLabelLocation(literal->GetLabel()); + uintptr_t address = + reinterpret_cast<uintptr_t>(roots_data) + index_in_table * sizeof(GcRoot<mirror::Object>); + reinterpret_cast<uint32_t*>(code + literal_offset)[0] = dchecked_integral_cast<uint32_t>(address); +} + +void CodeGeneratorMIPS64::EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) { + for (const auto& entry : jit_string_patches_) { + const auto& it = jit_string_roots_.find(entry.first); + DCHECK(it != jit_string_roots_.end()); + PatchJitRootUse(code, roots_data, entry.second, it->second); + } + for (const auto& entry : jit_class_patches_) { + const auto& it = jit_class_roots_.find(entry.first); + DCHECK(it != jit_class_roots_.end()); + PatchJitRootUse(code, roots_data, entry.second, it->second); + } +} + void CodeGeneratorMIPS64::SetupBlockedRegisters() const { // ZERO, K0, K1, GP, SP, RA are always reserved and can't be allocated. blocked_core_registers_[ZERO] = true; @@ -3309,8 +3353,6 @@ HLoadString::LoadKind CodeGeneratorMIPS64::GetSupportedLoadStringKind( break; case HLoadString::LoadKind::kJitTableAddress: DCHECK(Runtime::Current()->UseJitCompilation()); - // TODO: implement. - fallback_load = true; break; } if (fallback_load) { @@ -3326,6 +3368,9 @@ HLoadClass::LoadKind CodeGeneratorMIPS64::GetSupportedLoadClassKind( } bool fallback_load = false; switch (desired_class_load_kind) { + case HLoadClass::LoadKind::kInvalid: + LOG(FATAL) << "UNREACHABLE"; + UNREACHABLE(); case HLoadClass::LoadKind::kReferrersClass: break; case HLoadClass::LoadKind::kBootImageLinkTimeAddress: @@ -3341,8 +3386,6 @@ HLoadClass::LoadKind CodeGeneratorMIPS64::GetSupportedLoadClassKind( break; case HLoadClass::LoadKind::kJitTableAddress: DCHECK(Runtime::Current()->UseJitCompilation()); - // TODO: implement. - fallback_load = true; break; case HLoadClass::LoadKind::kDexCacheViaMethod: break; @@ -3580,11 +3623,16 @@ void InstructionCodeGeneratorMIPS64::VisitLoadClass(HLoadClass* cls) NO_THREAD_S generate_null_check = true; break; } - case HLoadClass::LoadKind::kJitTableAddress: { - LOG(FATAL) << "Unimplemented"; + case HLoadClass::LoadKind::kJitTableAddress: + __ LoadLiteral(out, + kLoadUnsignedWord, + codegen_->DeduplicateJitClassLiteral(cls->GetDexFile(), + cls->GetTypeIndex(), + cls->GetClass())); + GenerateGcRootFieldLoad(cls, out_loc, out, 0); break; - } case HLoadClass::LoadKind::kDexCacheViaMethod: + case HLoadClass::LoadKind::kInvalid: LOG(FATAL) << "UNREACHABLE"; UNREACHABLE(); } @@ -3685,6 +3733,14 @@ void InstructionCodeGeneratorMIPS64::VisitLoadString(HLoadString* load) NO_THREA __ Bind(slow_path->GetExitLabel()); return; } + case HLoadString::LoadKind::kJitTableAddress: + __ LoadLiteral(out, + kLoadUnsignedWord, + codegen_->DeduplicateJitStringLiteral(load->GetDexFile(), + load->GetStringIndex(), + load->GetString())); + GenerateGcRootFieldLoad(load, out_loc, out, 0); + return; default: break; } diff --git a/compiler/optimizing/code_generator_mips64.h b/compiler/optimizing/code_generator_mips64.h index 5ba8912134..26cc7dc788 100644 --- a/compiler/optimizing/code_generator_mips64.h +++ b/compiler/optimizing/code_generator_mips64.h @@ -52,7 +52,7 @@ static constexpr size_t kRuntimeParameterFpuRegistersLength = static constexpr GpuRegister kCoreCalleeSaves[] = - { S0, S1, S2, S3, S4, S5, S6, S7, GP, S8, RA }; // TODO: review + { S0, S1, S2, S3, S4, S5, S6, S7, GP, S8, RA }; static constexpr FpuRegister kFpuCalleeSaves[] = { F24, F25, F26, F27, F28, F29, F30, F31 }; @@ -312,6 +312,7 @@ class CodeGeneratorMIPS64 : public CodeGenerator { // Emit linker patches. void EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) OVERRIDE; + void EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) OVERRIDE; void MarkGCCard(GpuRegister object, GpuRegister value, bool value_can_be_null); @@ -425,10 +426,27 @@ class CodeGeneratorMIPS64 : public CodeGenerator { void EmitPcRelativeAddressPlaceholderHigh(PcRelativePatchInfo* info, GpuRegister out); + void PatchJitRootUse(uint8_t* code, + const uint8_t* roots_data, + const Literal* literal, + uint64_t index_in_table) const; + Literal* DeduplicateJitStringLiteral(const DexFile& dex_file, + dex::StringIndex string_index, + Handle<mirror::String> handle); + Literal* DeduplicateJitClassLiteral(const DexFile& dex_file, + dex::TypeIndex type_index, + Handle<mirror::Class> handle); + private: using Uint32ToLiteralMap = ArenaSafeMap<uint32_t, Literal*>; using Uint64ToLiteralMap = ArenaSafeMap<uint64_t, Literal*>; using MethodToLiteralMap = ArenaSafeMap<MethodReference, Literal*, MethodReferenceComparator>; + using StringToLiteralMap = ArenaSafeMap<StringReference, + Literal*, + StringReferenceValueComparator>; + using TypeToLiteralMap = ArenaSafeMap<TypeReference, + Literal*, + TypeReferenceValueComparator>; using BootStringToLiteralMap = ArenaSafeMap<StringReference, Literal*, StringReferenceValueComparator>; @@ -476,6 +494,10 @@ class CodeGeneratorMIPS64 : public CodeGenerator { ArenaDeque<PcRelativePatchInfo> type_bss_entry_patches_; // Deduplication map for patchable boot image addresses. Uint32ToLiteralMap boot_image_address_patches_; + // Patches for string root accesses in JIT compiled code. + StringToLiteralMap jit_string_patches_; + // Patches for class root accesses in JIT compiled code. + TypeToLiteralMap jit_class_patches_; DISALLOW_COPY_AND_ASSIGN(CodeGeneratorMIPS64); }; diff --git a/compiler/optimizing/code_generator_x86.cc b/compiler/optimizing/code_generator_x86.cc index 1b7431612d..09612c8dbf 100644 --- a/compiler/optimizing/code_generator_x86.cc +++ b/compiler/optimizing/code_generator_x86.cc @@ -4214,7 +4214,9 @@ void LocationsBuilderX86::VisitNewArray(HNewArray* instruction) { void InstructionCodeGeneratorX86::VisitNewArray(HNewArray* instruction) { // Note: if heap poisoning is enabled, the entry point takes cares // of poisoning the reference. - codegen_->InvokeRuntime(kQuickAllocArrayResolved, instruction, instruction->GetDexPc()); + QuickEntrypointEnum entrypoint = + CodeGenerator::GetArrayAllocationEntrypoint(instruction->GetLoadClass()->GetClass()); + codegen_->InvokeRuntime(entrypoint, instruction, instruction->GetDexPc()); CheckEntrypointTypes<kQuickAllocArrayResolved, void*, mirror::Class*, int32_t>(); DCHECK(!codegen_->IsLeafMethod()); } @@ -6022,6 +6024,9 @@ void ParallelMoveResolverX86::RestoreScratch(int reg) { HLoadClass::LoadKind CodeGeneratorX86::GetSupportedLoadClassKind( HLoadClass::LoadKind desired_class_load_kind) { switch (desired_class_load_kind) { + case HLoadClass::LoadKind::kInvalid: + LOG(FATAL) << "UNREACHABLE"; + UNREACHABLE(); case HLoadClass::LoadKind::kReferrersClass: break; case HLoadClass::LoadKind::kBootImageLinkTimeAddress: @@ -6052,6 +6057,7 @@ void LocationsBuilderX86::VisitLoadClass(HLoadClass* cls) { cls, Location::RegisterLocation(calling_convention.GetRegisterAt(0)), Location::RegisterLocation(EAX)); + DCHECK_EQ(calling_convention.GetRegisterAt(0), EAX); return; } DCHECK(!cls->NeedsAccessCheck()); @@ -6071,6 +6077,17 @@ void LocationsBuilderX86::VisitLoadClass(HLoadClass* cls) { locations->SetInAt(0, Location::RequiresRegister()); } locations->SetOut(Location::RequiresRegister()); + if (load_kind == HLoadClass::LoadKind::kBssEntry) { + if (!kUseReadBarrier || kUseBakerReadBarrier) { + // Rely on the type resolution and/or initialization to save everything. + RegisterSet caller_saves = RegisterSet::Empty(); + InvokeRuntimeCallingConvention calling_convention; + caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); + locations->SetCustomSlowPathCallerSaves(caller_saves); + } else { + // For non-Baker read barrier we have a temp-clobbering call. + } + } } Label* CodeGeneratorX86::NewJitRootClassPatch(const DexFile& dex_file, @@ -6153,10 +6170,11 @@ void InstructionCodeGeneratorX86::VisitLoadClass(HLoadClass* cls) NO_THREAD_SAFE Label* fixup_label = codegen_->NewJitRootClassPatch( cls->GetDexFile(), cls->GetTypeIndex(), cls->GetClass()); // /* GcRoot<mirror::Class> */ out = *address - GenerateGcRootFieldLoad(cls, out_loc, address, fixup_label, kCompilerReadBarrierOption); + GenerateGcRootFieldLoad(cls, out_loc, address, fixup_label, read_barrier_option); break; } case HLoadClass::LoadKind::kDexCacheViaMethod: + case HLoadClass::LoadKind::kInvalid: LOG(FATAL) << "UNREACHABLE"; UNREACHABLE(); } @@ -6244,7 +6262,7 @@ void LocationsBuilderX86::VisitLoadString(HLoadString* load) { locations->SetOut(Location::RequiresRegister()); if (load_kind == HLoadString::LoadKind::kBssEntry) { if (!kUseReadBarrier || kUseBakerReadBarrier) { - // Rely on the pResolveString and/or marking to save everything. + // Rely on the pResolveString to save everything. RegisterSet caller_saves = RegisterSet::Empty(); InvokeRuntimeCallingConvention calling_convention; caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); @@ -7130,9 +7148,10 @@ void InstructionCodeGeneratorX86::GenerateGcRootFieldLoad( // Fast path implementation of art::ReadBarrier::BarrierForRoot when // Baker's read barrier are used: // - // root = *address; - // if (Thread::Current()->GetIsGcMarking()) { - // root = ReadBarrier::Mark(root) + // root = obj.field; + // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg() + // if (temp != null) { + // root = temp(root) // } // /* GcRoot<mirror::Object> */ root = *address @@ -7153,8 +7172,11 @@ void InstructionCodeGeneratorX86::GenerateGcRootFieldLoad( instruction, root, /* unpoison_ref_before_marking */ false); codegen_->AddSlowPath(slow_path); - __ fs()->cmpl(Address::Absolute(Thread::IsGcMarkingOffset<kX86PointerSize>().Int32Value()), - Immediate(0)); + // Test the entrypoint (`Thread::Current()->pReadBarrierMarkReg ## root.reg()`). + const int32_t entry_point_offset = + CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kX86PointerSize>(root.reg()); + __ fs()->cmpl(Address::Absolute(entry_point_offset), Immediate(0)); + // The entrypoint is null when the GC is not marking. __ j(kNotEqual, slow_path->GetEntryLabel()); __ Bind(slow_path->GetExitLabel()); } else { diff --git a/compiler/optimizing/code_generator_x86_64.cc b/compiler/optimizing/code_generator_x86_64.cc index abd8246325..0879992e32 100644 --- a/compiler/optimizing/code_generator_x86_64.cc +++ b/compiler/optimizing/code_generator_x86_64.cc @@ -245,9 +245,8 @@ class LoadClassSlowPathX86_64 : public SlowPathCode { SaveLiveRegisters(codegen, locations); - InvokeRuntimeCallingConvention calling_convention; - __ movl(CpuRegister(calling_convention.GetRegisterAt(0)), - Immediate(cls_->GetTypeIndex().index_)); + // Custom calling convention: RAX serves as both input and output. + __ movl(CpuRegister(RAX), Immediate(cls_->GetTypeIndex().index_)); x86_64_codegen->InvokeRuntime(do_clinit_ ? kQuickInitializeStaticStorage : kQuickInitializeType, instruction_, dex_pc_, @@ -5427,6 +5426,9 @@ void InstructionCodeGeneratorX86_64::GenerateClassInitializationCheck( HLoadClass::LoadKind CodeGeneratorX86_64::GetSupportedLoadClassKind( HLoadClass::LoadKind desired_class_load_kind) { switch (desired_class_load_kind) { + case HLoadClass::LoadKind::kInvalid: + LOG(FATAL) << "UNREACHABLE"; + UNREACHABLE(); case HLoadClass::LoadKind::kReferrersClass: break; case HLoadClass::LoadKind::kBootImageLinkTimeAddress: @@ -5453,10 +5455,10 @@ HLoadClass::LoadKind CodeGeneratorX86_64::GetSupportedLoadClassKind( void LocationsBuilderX86_64::VisitLoadClass(HLoadClass* cls) { HLoadClass::LoadKind load_kind = cls->GetLoadKind(); if (load_kind == HLoadClass::LoadKind::kDexCacheViaMethod) { - InvokeRuntimeCallingConvention calling_convention; + // Custom calling convention: RAX serves as both input and output. CodeGenerator::CreateLoadClassRuntimeCallLocationSummary( cls, - Location::RegisterLocation(calling_convention.GetRegisterAt(0)), + Location::RegisterLocation(RAX), Location::RegisterLocation(RAX)); return; } @@ -5475,6 +5477,17 @@ void LocationsBuilderX86_64::VisitLoadClass(HLoadClass* cls) { locations->SetInAt(0, Location::RequiresRegister()); } locations->SetOut(Location::RequiresRegister()); + if (load_kind == HLoadClass::LoadKind::kBssEntry) { + if (!kUseReadBarrier || kUseBakerReadBarrier) { + // Rely on the type resolution and/or initialization to save everything. + // Custom calling convention: RAX serves as both input and output. + RegisterSet caller_saves = RegisterSet::Empty(); + caller_saves.Add(Location::RegisterLocation(RAX)); + locations->SetCustomSlowPathCallerSaves(caller_saves); + } else { + // For non-Baker read barrier we have a temp-clobbering call. + } + } } Label* CodeGeneratorX86_64::NewJitRootClassPatch(const DexFile& dex_file, @@ -5550,7 +5563,7 @@ void InstructionCodeGeneratorX86_64::VisitLoadClass(HLoadClass* cls) NO_THREAD_S Label* fixup_label = codegen_->NewJitRootClassPatch(cls->GetDexFile(), cls->GetTypeIndex(), cls->GetClass()); // /* GcRoot<mirror::Class> */ out = *address - GenerateGcRootFieldLoad(cls, out_loc, address, fixup_label, kCompilerReadBarrierOption); + GenerateGcRootFieldLoad(cls, out_loc, address, fixup_label, read_barrier_option); break; } default: @@ -5626,7 +5639,7 @@ void LocationsBuilderX86_64::VisitLoadString(HLoadString* load) { locations->SetOut(Location::RequiresRegister()); if (load->GetLoadKind() == HLoadString::LoadKind::kBssEntry) { if (!kUseReadBarrier || kUseBakerReadBarrier) { - // Rely on the pResolveString and/or marking to save everything. + // Rely on the pResolveString to save everything. // Custom calling convention: RAX serves as both input and output. RegisterSet caller_saves = RegisterSet::Empty(); caller_saves.Add(Location::RegisterLocation(RAX)); @@ -6498,9 +6511,10 @@ void InstructionCodeGeneratorX86_64::GenerateGcRootFieldLoad( // Fast path implementation of art::ReadBarrier::BarrierForRoot when // Baker's read barrier are used: // - // root = *address; - // if (Thread::Current()->GetIsGcMarking()) { - // root = ReadBarrier::Mark(root) + // root = obj.field; + // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg() + // if (temp != null) { + // root = temp(root) // } // /* GcRoot<mirror::Object> */ root = *address @@ -6521,9 +6535,11 @@ void InstructionCodeGeneratorX86_64::GenerateGcRootFieldLoad( instruction, root, /* unpoison_ref_before_marking */ false); codegen_->AddSlowPath(slow_path); - __ gs()->cmpl(Address::Absolute(Thread::IsGcMarkingOffset<kX86_64PointerSize>().Int32Value(), - /* no_rip */ true), - Immediate(0)); + // Test the `Thread::Current()->pReadBarrierMarkReg ## root.reg()` entrypoint. + const int32_t entry_point_offset = + CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kX86_64PointerSize>(root.reg()); + __ gs()->cmpl(Address::Absolute(entry_point_offset, /* no_rip */ true), Immediate(0)); + // The entrypoint is null when the GC is not marking. __ j(kNotEqual, slow_path->GetEntryLabel()); __ Bind(slow_path->GetExitLabel()); } else { diff --git a/compiler/optimizing/common_arm.h b/compiler/optimizing/common_arm.h index 21c3ae628a..ecb86875d6 100644 --- a/compiler/optimizing/common_arm.h +++ b/compiler/optimizing/common_arm.h @@ -146,6 +146,12 @@ inline vixl::aarch32::Register InputRegister(HInstruction* instr) { return InputRegisterAt(instr, 0); } +inline vixl::aarch32::DRegister DRegisterFromS(vixl::aarch32::SRegister s) { + vixl::aarch32::DRegister d = vixl::aarch32::DRegister(s.GetCode() / 2); + DCHECK(s.Is(d.GetLane(0)) || s.Is(d.GetLane(1))); + return d; +} + inline int32_t Int32ConstantFrom(HInstruction* instr) { if (instr->IsIntConstant()) { return instr->AsIntConstant()->GetValue(); diff --git a/compiler/optimizing/inliner.cc b/compiler/optimizing/inliner.cc index 7772e8f973..f0afccb782 100644 --- a/compiler/optimizing/inliner.cc +++ b/compiler/optimizing/inliner.cc @@ -558,9 +558,13 @@ HInstruction* HInliner::AddTypeGuard(HInstruction* receiver, is_referrer, invoke_instruction->GetDexPc(), /* needs_access_check */ false); + HLoadClass::LoadKind kind = HSharpening::SharpenClass( + load_class, codegen_, compiler_driver_, caller_compilation_unit_); + DCHECK(kind != HLoadClass::LoadKind::kInvalid) + << "We should always be able to reference a class for inline caches"; + // Insert before setting the kind, as setting the kind affects the inputs. bb_cursor->InsertInstructionAfter(load_class, receiver_class); - // Sharpen after adding the instruction, as the sharpening may remove inputs. - HSharpening::SharpenClass(load_class, codegen_, compiler_driver_); + load_class->SetLoadKind(kind); // TODO: Extend reference type propagation to understand the guard. HNotEqual* compare = new (graph_->GetArena()) HNotEqual(load_class, receiver_class); @@ -1286,6 +1290,7 @@ bool HInliner::TryBuildAndInlineHelper(HInvoke* invoke_instruction, resolved_method->GetDexFile(), *code_item, compiler_driver_, + codegen_, inline_stats.get(), resolved_method->GetQuickenedInfo(class_linker->GetImagePointerSize()), dex_cache, @@ -1416,10 +1421,13 @@ bool HInliner::TryBuildAndInlineHelper(HInvoke* invoke_instruction, return false; } - if (!same_dex_file && current->NeedsEnvironment()) { + if (current->NeedsEnvironment() && + !CanEncodeInlinedMethodInStackMap(*caller_compilation_unit_.GetDexFile(), + resolved_method)) { VLOG(compiler) << "Method " << callee_dex_file.PrettyMethod(method_index) << " could not be inlined because " << current->DebugName() - << " needs an environment and is in a different dex file"; + << " needs an environment, is in a different dex file" + << ", and cannot be encoded in the stack maps."; return false; } diff --git a/compiler/optimizing/instruction_builder.cc b/compiler/optimizing/instruction_builder.cc index cac385ce3c..a1c391f455 100644 --- a/compiler/optimizing/instruction_builder.cc +++ b/compiler/optimizing/instruction_builder.cc @@ -22,6 +22,7 @@ #include "dex_instruction-inl.h" #include "driver/compiler_options.h" #include "imtable-inl.h" +#include "sharpening.h" #include "scoped_thread_state_change-inl.h" namespace art { @@ -847,7 +848,7 @@ bool HInstructionBuilder::BuildInvoke(const Instruction& instruction, ScopedObjectAccess soa(Thread::Current()); if (invoke_type == kStatic) { clinit_check = ProcessClinitCheckForInvoke( - dex_pc, resolved_method, method_idx, &clinit_check_requirement); + dex_pc, resolved_method, &clinit_check_requirement); } else if (invoke_type == kSuper) { if (IsSameDexFile(*resolved_method->GetDexFile(), *dex_compilation_unit_->GetDexFile())) { // Update the method index to the one resolved. Note that this may be a no-op if @@ -933,15 +934,8 @@ bool HInstructionBuilder::BuildInvokePolymorphic(const Instruction& instruction bool HInstructionBuilder::BuildNewInstance(dex::TypeIndex type_index, uint32_t dex_pc) { ScopedObjectAccess soa(Thread::Current()); - Handle<mirror::DexCache> dex_cache = dex_compilation_unit_->GetDexCache(); - Handle<mirror::DexCache> outer_dex_cache = outer_compilation_unit_->GetDexCache(); - - if (outer_dex_cache.Get() != dex_cache.Get()) { - // We currently do not support inlining allocations across dex files. - return false; - } - HLoadClass* load_class = BuildLoadClass(type_index, dex_pc, /* check_access */ true); + HLoadClass* load_class = BuildLoadClass(type_index, dex_pc); HInstruction* cls = load_class; Handle<mirror::Class> klass = load_class->GetClass(); @@ -1005,39 +999,23 @@ bool HInstructionBuilder::IsInitialized(Handle<mirror::Class> cls) const { HClinitCheck* HInstructionBuilder::ProcessClinitCheckForInvoke( uint32_t dex_pc, ArtMethod* resolved_method, - uint32_t method_idx, HInvokeStaticOrDirect::ClinitCheckRequirement* clinit_check_requirement) { - Thread* self = Thread::Current(); - StackHandleScope<2> hs(self); - Handle<mirror::DexCache> dex_cache = dex_compilation_unit_->GetDexCache(); - Handle<mirror::DexCache> outer_dex_cache = outer_compilation_unit_->GetDexCache(); - Handle<mirror::Class> outer_class(hs.NewHandle(GetOutermostCompilingClass())); - Handle<mirror::Class> resolved_method_class(hs.NewHandle(resolved_method->GetDeclaringClass())); - - // The index at which the method's class is stored in the DexCache's type array. - dex::TypeIndex storage_index; - bool is_outer_class = (resolved_method->GetDeclaringClass() == outer_class.Get()); - if (is_outer_class) { - storage_index = outer_class->GetDexTypeIndex(); - } else if (outer_dex_cache.Get() == dex_cache.Get()) { - // Get `storage_index` from IsClassOfStaticMethodAvailableToReferrer. - compiler_driver_->IsClassOfStaticMethodAvailableToReferrer(outer_dex_cache.Get(), - GetCompilingClass(), - resolved_method, - method_idx, - &storage_index); - } + Handle<mirror::Class> klass = handles_->NewHandle(resolved_method->GetDeclaringClass()); HClinitCheck* clinit_check = nullptr; - - if (IsInitialized(resolved_method_class)) { + if (IsInitialized(klass)) { *clinit_check_requirement = HInvokeStaticOrDirect::ClinitCheckRequirement::kNone; - } else if (storage_index.IsValid()) { - *clinit_check_requirement = HInvokeStaticOrDirect::ClinitCheckRequirement::kExplicit; - HLoadClass* cls = BuildLoadClass( - storage_index, dex_pc, /* check_access */ false, /* outer */ true); - clinit_check = new (arena_) HClinitCheck(cls, dex_pc); - AppendInstruction(clinit_check); + } else { + HLoadClass* cls = BuildLoadClass(klass->GetDexTypeIndex(), + klass->GetDexFile(), + klass, + dex_pc, + /* needs_access_check */ false); + if (cls != nullptr) { + *clinit_check_requirement = HInvokeStaticOrDirect::ClinitCheckRequirement::kExplicit; + clinit_check = new (arena_) HClinitCheck(cls, dex_pc); + AppendInstruction(clinit_check); + } } return clinit_check; } @@ -1216,9 +1194,7 @@ bool HInstructionBuilder::BuildInstanceFieldAccess(const Instruction& instructio } ScopedObjectAccess soa(Thread::Current()); - ArtField* resolved_field = - compiler_driver_->ComputeInstanceFieldInfo(field_index, dex_compilation_unit_, is_put, soa); - + ArtField* resolved_field = ResolveField(field_index, /* is_static */ false, is_put); // Generate an explicit null check on the reference, unless the field access // is unresolved. In that case, we rely on the runtime to perform various @@ -1336,6 +1312,56 @@ void HInstructionBuilder::BuildUnresolvedStaticFieldAccess(const Instruction& in } } +ArtField* HInstructionBuilder::ResolveField(uint16_t field_idx, bool is_static, bool is_put) { + ScopedObjectAccess soa(Thread::Current()); + StackHandleScope<2> hs(soa.Self()); + + ClassLinker* class_linker = dex_compilation_unit_->GetClassLinker(); + Handle<mirror::ClassLoader> class_loader(hs.NewHandle( + soa.Decode<mirror::ClassLoader>(dex_compilation_unit_->GetClassLoader()))); + Handle<mirror::Class> compiling_class(hs.NewHandle(GetCompilingClass())); + + ArtField* resolved_field = class_linker->ResolveField(*dex_compilation_unit_->GetDexFile(), + field_idx, + dex_compilation_unit_->GetDexCache(), + class_loader, + is_static); + + if (UNLIKELY(resolved_field == nullptr)) { + // Clean up any exception left by type resolution. + soa.Self()->ClearException(); + return nullptr; + } + + // Check static/instance. The class linker has a fast path for looking into the dex cache + // and does not check static/instance if it hits it. + if (UNLIKELY(resolved_field->IsStatic() != is_static)) { + return nullptr; + } + + // Check access. + if (compiling_class.Get() == nullptr) { + if (!resolved_field->IsPublic()) { + return nullptr; + } + } else if (!compiling_class->CanAccessResolvedField(resolved_field->GetDeclaringClass(), + resolved_field, + dex_compilation_unit_->GetDexCache().Get(), + field_idx)) { + return nullptr; + } + + if (is_put && + resolved_field->IsFinal() && + (compiling_class.Get() != resolved_field->GetDeclaringClass())) { + // Final fields can only be updated within their own class. + // TODO: Only allow it in constructors. b/34966607. + return nullptr; + } + + return resolved_field; +} + bool HInstructionBuilder::BuildStaticFieldAccess(const Instruction& instruction, uint32_t dex_pc, bool is_put) { @@ -1343,12 +1369,7 @@ bool HInstructionBuilder::BuildStaticFieldAccess(const Instruction& instruction, uint16_t field_index = instruction.VRegB_21c(); ScopedObjectAccess soa(Thread::Current()); - StackHandleScope<3> hs(soa.Self()); - Handle<mirror::DexCache> dex_cache = dex_compilation_unit_->GetDexCache(); - Handle<mirror::ClassLoader> class_loader(hs.NewHandle( - soa.Decode<mirror::ClassLoader>(dex_compilation_unit_->GetClassLoader()))); - ArtField* resolved_field = compiler_driver_->ResolveField( - soa, dex_cache, class_loader, dex_compilation_unit_, field_index, true); + ArtField* resolved_field = ResolveField(field_index, /* is_static */ true, is_put); if (resolved_field == nullptr) { MaybeRecordStat(MethodCompilationStat::kUnresolvedField); @@ -1358,38 +1379,23 @@ bool HInstructionBuilder::BuildStaticFieldAccess(const Instruction& instruction, } Primitive::Type field_type = resolved_field->GetTypeAsPrimitiveType(); - Handle<mirror::DexCache> outer_dex_cache = outer_compilation_unit_->GetDexCache(); - Handle<mirror::Class> outer_class(hs.NewHandle(GetOutermostCompilingClass())); - // The index at which the field's class is stored in the DexCache's type array. - dex::TypeIndex storage_index; - bool is_outer_class = (outer_class.Get() == resolved_field->GetDeclaringClass()); - if (is_outer_class) { - storage_index = outer_class->GetDexTypeIndex(); - } else if (outer_dex_cache.Get() != dex_cache.Get()) { - // The compiler driver cannot currently understand multiple dex caches involved. Just bailout. - return false; - } else { - // TODO: This is rather expensive. Perf it and cache the results if needed. - std::pair<bool, bool> pair = compiler_driver_->IsFastStaticField( - outer_dex_cache.Get(), - GetCompilingClass(), - resolved_field, - field_index, - &storage_index); - bool can_easily_access = is_put ? pair.second : pair.first; - if (!can_easily_access) { - MaybeRecordStat(MethodCompilationStat::kUnresolvedFieldNotAFastAccess); - BuildUnresolvedStaticFieldAccess(instruction, dex_pc, is_put, field_type); - return true; - } + Handle<mirror::Class> klass = handles_->NewHandle(resolved_field->GetDeclaringClass()); + HLoadClass* constant = BuildLoadClass(klass->GetDexTypeIndex(), + klass->GetDexFile(), + klass, + dex_pc, + /* needs_access_check */ false); + + if (constant == nullptr) { + // The class cannot be referenced from this compiled code. Generate + // an unresolved access. + MaybeRecordStat(MethodCompilationStat::kUnresolvedFieldNotAFastAccess); + BuildUnresolvedStaticFieldAccess(instruction, dex_pc, is_put, field_type); + return true; } - HLoadClass* constant = BuildLoadClass( - storage_index, dex_pc, /* check_access */ false, /* outer */ true); - HInstruction* cls = constant; - Handle<mirror::Class> klass(hs.NewHandle(resolved_field->GetDeclaringClass())); if (!IsInitialized(klass)) { cls = new (arena_) HClinitCheck(constant, dex_pc); AppendInstruction(cls); @@ -1497,7 +1503,7 @@ void HInstructionBuilder::BuildFilledNewArray(uint32_t dex_pc, uint32_t* args, uint32_t register_index) { HInstruction* length = graph_->GetIntConstant(number_of_vreg_arguments, dex_pc); - HLoadClass* cls = BuildLoadClass(type_index, dex_pc, /* check_access */ true); + HLoadClass* cls = BuildLoadClass(type_index, dex_pc); HInstruction* object = new (arena_) HNewArray(cls, length, dex_pc); AppendInstruction(object); @@ -1627,44 +1633,68 @@ static TypeCheckKind ComputeTypeCheckKind(Handle<mirror::Class> cls) } } -HLoadClass* HInstructionBuilder::BuildLoadClass(dex::TypeIndex type_index, - uint32_t dex_pc, - bool check_access, - bool outer) { +HLoadClass* HInstructionBuilder::BuildLoadClass(dex::TypeIndex type_index, uint32_t dex_pc) { ScopedObjectAccess soa(Thread::Current()); - const DexCompilationUnit* compilation_unit = - outer ? outer_compilation_unit_ : dex_compilation_unit_; - const DexFile& dex_file = *compilation_unit->GetDexFile(); - StackHandleScope<1> hs(soa.Self()); + StackHandleScope<2> hs(soa.Self()); + const DexFile& dex_file = *dex_compilation_unit_->GetDexFile(); Handle<mirror::ClassLoader> class_loader(hs.NewHandle( soa.Decode<mirror::ClassLoader>(dex_compilation_unit_->GetClassLoader()))); Handle<mirror::Class> klass = handles_->NewHandle(compiler_driver_->ResolveClass( - soa, compilation_unit->GetDexCache(), class_loader, type_index, compilation_unit)); + soa, dex_compilation_unit_->GetDexCache(), class_loader, type_index, dex_compilation_unit_)); - bool is_accessible = false; - if (!check_access) { - is_accessible = true; - } else if (klass.Get() != nullptr) { + bool needs_access_check = true; + if (klass.Get() != nullptr) { if (klass->IsPublic()) { - is_accessible = true; + needs_access_check = false; } else { mirror::Class* compiling_class = GetCompilingClass(); if (compiling_class != nullptr && compiling_class->CanAccess(klass.Get())) { - is_accessible = true; + needs_access_check = false; } } } + return BuildLoadClass(type_index, dex_file, klass, dex_pc, needs_access_check); +} + +HLoadClass* HInstructionBuilder::BuildLoadClass(dex::TypeIndex type_index, + const DexFile& dex_file, + Handle<mirror::Class> klass, + uint32_t dex_pc, + bool needs_access_check) { + // Try to find a reference in the compiling dex file. + const DexFile* actual_dex_file = &dex_file; + if (!IsSameDexFile(dex_file, *dex_compilation_unit_->GetDexFile())) { + dex::TypeIndex local_type_index = + klass->FindTypeIndexInOtherDexFile(*dex_compilation_unit_->GetDexFile()); + if (local_type_index.IsValid()) { + type_index = local_type_index; + actual_dex_file = dex_compilation_unit_->GetDexFile(); + } + } + + // Note: `klass` must be from `handles_`. HLoadClass* load_class = new (arena_) HLoadClass( graph_->GetCurrentMethod(), type_index, - dex_file, + *actual_dex_file, klass, klass.Get() != nullptr && (klass.Get() == GetOutermostCompilingClass()), dex_pc, - !is_accessible); + needs_access_check); + HLoadClass::LoadKind load_kind = HSharpening::SharpenClass(load_class, + code_generator_, + compiler_driver_, + *dex_compilation_unit_); + + if (load_kind == HLoadClass::LoadKind::kInvalid) { + // We actually cannot reference this class, we're forced to bail. + return nullptr; + } + // Append the instruction first, as setting the load kind affects the inputs. AppendInstruction(load_class); + load_class->SetLoadKind(load_kind); return load_class; } @@ -1674,7 +1704,7 @@ void HInstructionBuilder::BuildTypeCheck(const Instruction& instruction, dex::TypeIndex type_index, uint32_t dex_pc) { HInstruction* object = LoadLocal(reference, Primitive::kPrimNot); - HLoadClass* cls = BuildLoadClass(type_index, dex_pc, /* check_access */ true); + HLoadClass* cls = BuildLoadClass(type_index, dex_pc); ScopedObjectAccess soa(Thread::Current()); TypeCheckKind check_kind = ComputeTypeCheckKind(cls->GetClass()); @@ -2498,7 +2528,7 @@ bool HInstructionBuilder::ProcessDexInstruction(const Instruction& instruction, case Instruction::NEW_ARRAY: { dex::TypeIndex type_index(instruction.VRegC_22c()); HInstruction* length = LoadLocal(instruction.VRegB_22c(), Primitive::kPrimInt); - HLoadClass* cls = BuildLoadClass(type_index, dex_pc, /* check_access */ true); + HLoadClass* cls = BuildLoadClass(type_index, dex_pc); AppendInstruction(new (arena_) HNewArray(cls, length, dex_pc)); UpdateLocal(instruction.VRegA_22c(), current_block_->GetLastInstruction()); break; @@ -2673,7 +2703,7 @@ bool HInstructionBuilder::ProcessDexInstruction(const Instruction& instruction, case Instruction::CONST_CLASS: { dex::TypeIndex type_index(instruction.VRegB_21c()); - BuildLoadClass(type_index, dex_pc, /* check_access */ true); + BuildLoadClass(type_index, dex_pc); UpdateLocal(instruction.VRegA_21c(), current_block_->GetLastInstruction()); break; } diff --git a/compiler/optimizing/instruction_builder.h b/compiler/optimizing/instruction_builder.h index 5efe95094c..3bb680ce44 100644 --- a/compiler/optimizing/instruction_builder.h +++ b/compiler/optimizing/instruction_builder.h @@ -31,6 +31,7 @@ namespace art { +class CodeGenerator; class Instruction; class HInstructionBuilder : public ValueObject { @@ -44,6 +45,7 @@ class HInstructionBuilder : public ValueObject { DexCompilationUnit* dex_compilation_unit, const DexCompilationUnit* const outer_compilation_unit, CompilerDriver* driver, + CodeGenerator* code_generator, const uint8_t* interpreter_metadata, OptimizingCompilerStats* compiler_stats, Handle<mirror::DexCache> dex_cache, @@ -61,6 +63,7 @@ class HInstructionBuilder : public ValueObject { current_locals_(nullptr), latest_result_(nullptr), compiler_driver_(driver), + code_generator_(code_generator), dex_compilation_unit_(dex_compilation_unit), outer_compilation_unit_(outer_compilation_unit), interpreter_metadata_(interpreter_metadata), @@ -228,10 +231,14 @@ class HInstructionBuilder : public ValueObject { // Builds a `HLoadClass` loading the given `type_index`. If `outer` is true, // this method will use the outer class's dex file to lookup the type at // `type_index`. + HLoadClass* BuildLoadClass(dex::TypeIndex type_index, uint32_t dex_pc); + HLoadClass* BuildLoadClass(dex::TypeIndex type_index, + const DexFile& dex_file, + Handle<mirror::Class> klass, uint32_t dex_pc, - bool check_access, - bool outer = false); + bool needs_access_check) + REQUIRES_SHARED(Locks::mutator_lock_); // Returns the outer-most compiling method's class. mirror::Class* GetOutermostCompilingClass() const; @@ -275,7 +282,6 @@ class HInstructionBuilder : public ValueObject { HClinitCheck* ProcessClinitCheckForInvoke( uint32_t dex_pc, ArtMethod* method, - uint32_t method_idx, HInvokeStaticOrDirect::ClinitCheckRequirement* clinit_check_requirement) REQUIRES_SHARED(Locks::mutator_lock_); @@ -290,6 +296,10 @@ class HInstructionBuilder : public ValueObject { // not be resolved. ArtMethod* ResolveMethod(uint16_t method_idx, InvokeType invoke_type); + // Try to resolve a field using the class linker. Return null if it could not + // be found. + ArtField* ResolveField(uint16_t field_idx, bool is_static, bool is_put); + ArenaAllocator* const arena_; HGraph* const graph_; VariableSizedHandleScope* handles_; @@ -311,6 +321,8 @@ class HInstructionBuilder : public ValueObject { CompilerDriver* const compiler_driver_; + CodeGenerator* const code_generator_; + // The compilation unit of the current method being compiled. Note that // it can be an inlined method. DexCompilationUnit* const dex_compilation_unit_; diff --git a/compiler/optimizing/intrinsics.h b/compiler/optimizing/intrinsics.h index 1e73cf67df..6425e1313f 100644 --- a/compiler/optimizing/intrinsics.h +++ b/compiler/optimizing/intrinsics.h @@ -31,6 +31,9 @@ class DexFile; static constexpr uint32_t kPositiveInfinityFloat = 0x7f800000U; static constexpr uint64_t kPositiveInfinityDouble = UINT64_C(0x7ff0000000000000); +static constexpr uint32_t kNanFloat = 0x7fc00000U; +static constexpr uint64_t kNanDouble = 0x7ff8000000000000; + // Recognize intrinsics from HInvoke nodes. class IntrinsicsRecognizer : public HOptimization { public: diff --git a/compiler/optimizing/intrinsics_arm_vixl.cc b/compiler/optimizing/intrinsics_arm_vixl.cc index 1a10173ed7..70a3d38c13 100644 --- a/compiler/optimizing/intrinsics_arm_vixl.cc +++ b/compiler/optimizing/intrinsics_arm_vixl.cc @@ -40,10 +40,12 @@ using helpers::LocationFrom; using helpers::LowRegisterFrom; using helpers::LowSRegisterFrom; using helpers::OutputDRegister; +using helpers::OutputSRegister; using helpers::OutputRegister; using helpers::OutputVRegister; using helpers::RegisterFrom; using helpers::SRegisterFrom; +using helpers::DRegisterFromS; using namespace vixl::aarch32; // NOLINT(build/namespaces) @@ -462,6 +464,214 @@ void IntrinsicCodeGeneratorARMVIXL::VisitMathAbsLong(HInvoke* invoke) { GenAbsInteger(invoke->GetLocations(), /* is64bit */ true, GetAssembler()); } +static void GenMinMaxFloat(HInvoke* invoke, bool is_min, ArmVIXLAssembler* assembler) { + Location op1_loc = invoke->GetLocations()->InAt(0); + Location op2_loc = invoke->GetLocations()->InAt(1); + Location out_loc = invoke->GetLocations()->Out(); + + // Optimization: don't generate any code if inputs are the same. + if (op1_loc.Equals(op2_loc)) { + DCHECK(out_loc.Equals(op1_loc)); // out_loc is set as SameAsFirstInput() in location builder. + return; + } + + vixl32::SRegister op1 = SRegisterFrom(op1_loc); + vixl32::SRegister op2 = SRegisterFrom(op2_loc); + vixl32::SRegister out = OutputSRegister(invoke); + UseScratchRegisterScope temps(assembler->GetVIXLAssembler()); + const vixl32::Register temp1 = temps.Acquire(); + vixl32::Register temp2 = RegisterFrom(invoke->GetLocations()->GetTemp(0)); + vixl32::Label nan, done; + + DCHECK(op1.Is(out)); + + __ Vcmp(op1, op2); + __ Vmrs(RegisterOrAPSR_nzcv(kPcCode), FPSCR); + __ B(vs, &nan, /* far_target */ false); // if un-ordered, go to NaN handling. + + // op1 <> op2 + vixl32::ConditionType cond = is_min ? gt : lt; + { + ExactAssemblyScope it_scope(assembler->GetVIXLAssembler(), + 2 * kMaxInstructionSizeInBytes, + CodeBufferCheckScope::kMaximumSize); + __ it(cond); + __ vmov(cond, F32, out, op2); + } + __ B(ne, &done, /* far_target */ false); // for <>(not equal), we've done min/max calculation. + + // handle op1 == op2, max(+0.0,-0.0), min(+0.0,-0.0). + __ Vmov(temp1, op1); + __ Vmov(temp2, op2); + if (is_min) { + __ Orr(temp1, temp1, temp2); + } else { + __ And(temp1, temp1, temp2); + } + __ Vmov(out, temp1); + __ B(&done); + + // handle NaN input. + __ Bind(&nan); + __ Movt(temp1, High16Bits(kNanFloat)); // 0x7FC0xxxx is a NaN. + __ Vmov(out, temp1); + + __ Bind(&done); +} + +static void CreateFPFPToFPLocations(ArenaAllocator* arena, HInvoke* invoke) { + LocationSummary* locations = new (arena) LocationSummary(invoke, + LocationSummary::kNoCall, + kIntrinsified); + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetInAt(1, Location::RequiresFpuRegister()); + locations->SetOut(Location::SameAsFirstInput()); +} + +void IntrinsicLocationsBuilderARMVIXL::VisitMathMinFloatFloat(HInvoke* invoke) { + CreateFPFPToFPLocations(arena_, invoke); + invoke->GetLocations()->AddTemp(Location::RequiresRegister()); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitMathMinFloatFloat(HInvoke* invoke) { + GenMinMaxFloat(invoke, /* is_min */ true, GetAssembler()); +} + +void IntrinsicLocationsBuilderARMVIXL::VisitMathMaxFloatFloat(HInvoke* invoke) { + CreateFPFPToFPLocations(arena_, invoke); + invoke->GetLocations()->AddTemp(Location::RequiresRegister()); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitMathMaxFloatFloat(HInvoke* invoke) { + GenMinMaxFloat(invoke, /* is_min */ false, GetAssembler()); +} + +static void GenMinMaxDouble(HInvoke* invoke, bool is_min, ArmVIXLAssembler* assembler) { + Location op1_loc = invoke->GetLocations()->InAt(0); + Location op2_loc = invoke->GetLocations()->InAt(1); + Location out_loc = invoke->GetLocations()->Out(); + + // Optimization: don't generate any code if inputs are the same. + if (op1_loc.Equals(op2_loc)) { + DCHECK(out_loc.Equals(op1_loc)); // out_loc is set as SameAsFirstInput() in. + return; + } + + vixl32::DRegister op1 = DRegisterFrom(op1_loc); + vixl32::DRegister op2 = DRegisterFrom(op2_loc); + vixl32::DRegister out = OutputDRegister(invoke); + vixl32::Label handle_nan_eq, done; + + DCHECK(op1.Is(out)); + + __ Vcmp(op1, op2); + __ Vmrs(RegisterOrAPSR_nzcv(kPcCode), FPSCR); + __ B(vs, &handle_nan_eq, /* far_target */ false); // if un-ordered, go to NaN handling. + + // op1 <> op2 + vixl32::ConditionType cond = is_min ? gt : lt; + { + ExactAssemblyScope it_scope(assembler->GetVIXLAssembler(), + 2 * kMaxInstructionSizeInBytes, + CodeBufferCheckScope::kMaximumSize); + __ it(cond); + __ vmov(cond, F64, out, op2); + } + __ B(ne, &done, /* far_target */ false); // for <>(not equal), we've done min/max calculation. + + // handle op1 == op2, max(+0.0,-0.0). + if (!is_min) { + __ Vand(F64, out, op1, op2); + __ B(&done); + } + + // handle op1 == op2, min(+0.0,-0.0), NaN input. + __ Bind(&handle_nan_eq); + __ Vorr(F64, out, op1, op2); // assemble op1/-0.0/NaN. + + __ Bind(&done); +} + +void IntrinsicLocationsBuilderARMVIXL::VisitMathMinDoubleDouble(HInvoke* invoke) { + CreateFPFPToFPLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitMathMinDoubleDouble(HInvoke* invoke) { + GenMinMaxDouble(invoke, /* is_min */ true , GetAssembler()); +} + +void IntrinsicLocationsBuilderARMVIXL::VisitMathMaxDoubleDouble(HInvoke* invoke) { + CreateFPFPToFPLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitMathMaxDoubleDouble(HInvoke* invoke) { + GenMinMaxDouble(invoke, /* is_min */ false, GetAssembler()); +} + +static void GenMinMaxLong(HInvoke* invoke, bool is_min, ArmVIXLAssembler* assembler) { + Location op1_loc = invoke->GetLocations()->InAt(0); + Location op2_loc = invoke->GetLocations()->InAt(1); + Location out_loc = invoke->GetLocations()->Out(); + + // Optimization: don't generate any code if inputs are the same. + if (op1_loc.Equals(op2_loc)) { + DCHECK(out_loc.Equals(op1_loc)); // out_loc is set as SameAsFirstInput() in location builder. + return; + } + + vixl32::Register op1_lo = LowRegisterFrom(op1_loc); + vixl32::Register op1_hi = HighRegisterFrom(op1_loc); + vixl32::Register op2_lo = LowRegisterFrom(op2_loc); + vixl32::Register op2_hi = HighRegisterFrom(op2_loc); + vixl32::Register out_lo = LowRegisterFrom(out_loc); + vixl32::Register out_hi = HighRegisterFrom(out_loc); + UseScratchRegisterScope temps(assembler->GetVIXLAssembler()); + const vixl32::Register temp = temps.Acquire(); + + DCHECK(op1_lo.Is(out_lo)); + DCHECK(op1_hi.Is(out_hi)); + + // Compare op1 >= op2, or op1 < op2. + __ Cmp(out_lo, op2_lo); + __ Sbcs(temp, out_hi, op2_hi); + + // Now GE/LT condition code is correct for the long comparison. + { + vixl32::ConditionType cond = is_min ? ge : lt; + ExactAssemblyScope it_scope(assembler->GetVIXLAssembler(), + 3 * kMaxInstructionSizeInBytes, + CodeBufferCheckScope::kMaximumSize); + __ itt(cond); + __ mov(cond, out_lo, op2_lo); + __ mov(cond, out_hi, op2_hi); + } +} + +static void CreateLongLongToLongLocations(ArenaAllocator* arena, HInvoke* invoke) { + LocationSummary* locations = new (arena) LocationSummary(invoke, + LocationSummary::kNoCall, + kIntrinsified); + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetInAt(1, Location::RequiresRegister()); + locations->SetOut(Location::SameAsFirstInput()); +} + +void IntrinsicLocationsBuilderARMVIXL::VisitMathMinLongLong(HInvoke* invoke) { + CreateLongLongToLongLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitMathMinLongLong(HInvoke* invoke) { + GenMinMaxLong(invoke, /* is_min */ true, GetAssembler()); +} + +void IntrinsicLocationsBuilderARMVIXL::VisitMathMaxLongLong(HInvoke* invoke) { + CreateLongLongToLongLocations(arena_, invoke); +} + +void IntrinsicCodeGeneratorARMVIXL::VisitMathMaxLongLong(HInvoke* invoke) { + GenMinMaxLong(invoke, /* is_min */ false, GetAssembler()); +} + static void GenMinMax(HInvoke* invoke, bool is_min, ArmVIXLAssembler* assembler) { vixl32::Register op1 = InputRegisterAt(invoke, 0); vixl32::Register op2 = InputRegisterAt(invoke, 1); @@ -2778,12 +2988,6 @@ void IntrinsicCodeGeneratorARMVIXL::VisitMathFloor(HInvoke* invoke) { __ Vrintm(F64, F64, OutputDRegister(invoke), InputDRegisterAt(invoke, 0)); } -UNIMPLEMENTED_INTRINSIC(ARMVIXL, MathMinDoubleDouble) -UNIMPLEMENTED_INTRINSIC(ARMVIXL, MathMinFloatFloat) -UNIMPLEMENTED_INTRINSIC(ARMVIXL, MathMaxDoubleDouble) -UNIMPLEMENTED_INTRINSIC(ARMVIXL, MathMaxFloatFloat) -UNIMPLEMENTED_INTRINSIC(ARMVIXL, MathMinLongLong) -UNIMPLEMENTED_INTRINSIC(ARMVIXL, MathMaxLongLong) UNIMPLEMENTED_INTRINSIC(ARMVIXL, MathRoundDouble) // Could be done by changing rounding mode, maybe? UNIMPLEMENTED_INTRINSIC(ARMVIXL, MathRoundFloat) // Could be done by changing rounding mode, maybe? UNIMPLEMENTED_INTRINSIC(ARMVIXL, UnsafeCASLong) // High register pressure. diff --git a/compiler/optimizing/nodes.cc b/compiler/optimizing/nodes.cc index 76900f23a9..abbb91a1a9 100644 --- a/compiler/optimizing/nodes.cc +++ b/compiler/optimizing/nodes.cc @@ -2464,16 +2464,15 @@ bool HLoadClass::InstructionDataEquals(const HInstruction* other) const { } } -void HLoadClass::SetLoadKindInternal(LoadKind load_kind) { - // Once sharpened, the load kind should not be changed again. - // Also, kReferrersClass should never be overwritten. - DCHECK_EQ(GetLoadKind(), LoadKind::kDexCacheViaMethod); +void HLoadClass::SetLoadKind(LoadKind load_kind) { SetPackedField<LoadKindField>(load_kind); - if (load_kind != LoadKind::kDexCacheViaMethod) { + if (load_kind != LoadKind::kDexCacheViaMethod && + load_kind != LoadKind::kReferrersClass) { RemoveAsUserOfInput(0u); SetRawInputAt(0u, nullptr); } + if (!NeedsEnvironment()) { RemoveEnvironment(); SetSideEffects(SideEffects::None()); diff --git a/compiler/optimizing/nodes.h b/compiler/optimizing/nodes.h index acf14aa726..96f9abafbf 100644 --- a/compiler/optimizing/nodes.h +++ b/compiler/optimizing/nodes.h @@ -4322,6 +4322,11 @@ class HInvokeInterface FINAL : public HInvoke { return (obj == InputAt(0)) && !GetLocations()->Intrinsified(); } + bool NeedsDexCacheOfDeclaringClass() const OVERRIDE { + // The assembly stub currently needs it. + return true; + } + uint32_t GetImtIndex() const { return imt_index_; } uint32_t GetDexMethodIndex() const { return dex_method_index_; } @@ -5508,6 +5513,9 @@ class HLoadClass FINAL : public HInstruction { public: // Determines how to load the Class. enum class LoadKind { + // We cannot load this class. See HSharpening::SharpenLoadClass. + kInvalid = -1, + // Use the Class* from the method's own ArtMethod*. kReferrersClass, @@ -5564,18 +5572,7 @@ class HLoadClass FINAL : public HInstruction { SetPackedFlag<kFlagGenerateClInitCheck>(false); } - void SetLoadKind(LoadKind load_kind) { - SetLoadKindInternal(load_kind); - } - - void SetLoadKindWithTypeReference(LoadKind load_kind, - const DexFile& dex_file, - dex::TypeIndex type_index) { - DCHECK(HasTypeReference(load_kind)); - DCHECK(IsSameDexFile(dex_file_, dex_file)); - DCHECK_EQ(type_index_, type_index); - SetLoadKindInternal(load_kind); - } + void SetLoadKind(LoadKind load_kind); LoadKind GetLoadKind() const { return GetPackedField<LoadKindField>(); @@ -5694,6 +5691,11 @@ class HLoadClass FINAL : public HInstruction { // for PC-relative loads, i.e. kBssEntry or kBootImageLinkTimePcRelative. HUserRecord<HInstruction*> special_input_; + // A type index and dex file where the class can be accessed. The dex file can be: + // - The compiling method's dex file if the class is defined there too. + // - The compiling method's dex file if the class is referenced there. + // - The dex file where the class is defined. When the load kind can only be + // kBssEntry or kDexCacheViaMethod, we cannot emit code for this `HLoadClass`. const dex::TypeIndex type_index_; const DexFile& dex_file_; diff --git a/compiler/optimizing/optimizing_compiler.cc b/compiler/optimizing/optimizing_compiler.cc index 1ab671022b..727ca7d893 100644 --- a/compiler/optimizing/optimizing_compiler.cc +++ b/compiler/optimizing/optimizing_compiler.cc @@ -999,6 +999,7 @@ CodeGenerator* OptimizingCompiler::TryCompile(ArenaAllocator* arena, &dex_file, *code_item, compiler_driver, + codegen.get(), compilation_stats_.get(), interpreter_metadata, dex_cache, @@ -1133,6 +1134,25 @@ bool IsCompilingWithCoreImage() { return false; } +bool EncodeArtMethodInInlineInfo(ArtMethod* method ATTRIBUTE_UNUSED) { + // Note: the runtime is null only for unit testing. + return Runtime::Current() == nullptr || !Runtime::Current()->IsAotCompiler(); +} + +bool CanEncodeInlinedMethodInStackMap(const DexFile& caller_dex_file, ArtMethod* callee) { + if (!Runtime::Current()->IsAotCompiler()) { + // JIT can always encode methods in stack maps. + return true; + } + if (IsSameDexFile(caller_dex_file, *callee->GetDexFile())) { + return true; + } + // TODO(ngeoffray): Support more AOT cases for inlining: + // - methods in multidex + // - methods in boot image for on-device non-PIC compilation. + return false; +} + bool OptimizingCompiler::JitCompile(Thread* self, jit::JitCodeCache* code_cache, ArtMethod* method, diff --git a/compiler/optimizing/optimizing_compiler.h b/compiler/optimizing/optimizing_compiler.h index 0c89da12e8..d8cea30a6b 100644 --- a/compiler/optimizing/optimizing_compiler.h +++ b/compiler/optimizing/optimizing_compiler.h @@ -17,10 +17,15 @@ #ifndef ART_COMPILER_OPTIMIZING_OPTIMIZING_COMPILER_H_ #define ART_COMPILER_OPTIMIZING_OPTIMIZING_COMPILER_H_ +#include "base/mutex.h" +#include "globals.h" + namespace art { +class ArtMethod; class Compiler; class CompilerDriver; +class DexFile; Compiler* CreateOptimizingCompiler(CompilerDriver* driver); @@ -29,6 +34,10 @@ Compiler* CreateOptimizingCompiler(CompilerDriver* driver); // information for checking invariants. bool IsCompilingWithCoreImage(); +bool EncodeArtMethodInInlineInfo(ArtMethod* method); +bool CanEncodeInlinedMethodInStackMap(const DexFile& caller_dex_file, ArtMethod* callee) + REQUIRES_SHARED(Locks::mutator_lock_); + } // namespace art #endif // ART_COMPILER_OPTIMIZING_OPTIMIZING_COMPILER_H_ diff --git a/compiler/optimizing/reference_type_propagation.cc b/compiler/optimizing/reference_type_propagation.cc index b02f2509ab..c55fccc7d3 100644 --- a/compiler/optimizing/reference_type_propagation.cc +++ b/compiler/optimizing/reference_type_propagation.cc @@ -24,9 +24,8 @@ namespace art { -static inline mirror::DexCache* FindDexCacheWithHint(Thread* self, - const DexFile& dex_file, - Handle<mirror::DexCache> hint_dex_cache) +static inline ObjPtr<mirror::DexCache> FindDexCacheWithHint( + Thread* self, const DexFile& dex_file, Handle<mirror::DexCache> hint_dex_cache) REQUIRES_SHARED(Locks::mutator_lock_) { if (LIKELY(hint_dex_cache->GetDexFile() == &dex_file)) { return hint_dex_cache.Get(); @@ -542,7 +541,7 @@ void ReferenceTypePropagation::RTPVisitor::UpdateReferenceTypeInfo(HInstruction* DCHECK_EQ(instr->GetType(), Primitive::kPrimNot); ScopedObjectAccess soa(Thread::Current()); - mirror::DexCache* dex_cache = FindDexCacheWithHint(soa.Self(), dex_file, hint_dex_cache_); + ObjPtr<mirror::DexCache> dex_cache = FindDexCacheWithHint(soa.Self(), dex_file, hint_dex_cache_); // Get type from dex cache assuming it was populated by the verifier. SetClassAsTypeInfo(instr, dex_cache->GetResolvedType(type_idx), is_exact); } @@ -562,7 +561,7 @@ static mirror::Class* GetClassFromDexCache(Thread* self, dex::TypeIndex type_idx, Handle<mirror::DexCache> hint_dex_cache) REQUIRES_SHARED(Locks::mutator_lock_) { - mirror::DexCache* dex_cache = FindDexCacheWithHint(self, dex_file, hint_dex_cache); + ObjPtr<mirror::DexCache> dex_cache = FindDexCacheWithHint(self, dex_file, hint_dex_cache); // Get type from dex cache assuming it was populated by the verifier. return dex_cache->GetResolvedType(type_idx); } diff --git a/compiler/optimizing/sharpening.cc b/compiler/optimizing/sharpening.cc index e745c73091..f07f02a719 100644 --- a/compiler/optimizing/sharpening.cc +++ b/compiler/optimizing/sharpening.cc @@ -42,8 +42,6 @@ void HSharpening::Run() { HInstruction* instruction = it.Current(); if (instruction->IsInvokeStaticOrDirect()) { ProcessInvokeStaticOrDirect(instruction->AsInvokeStaticOrDirect()); - } else if (instruction->IsLoadClass()) { - ProcessLoadClass(instruction->AsLoadClass()); } else if (instruction->IsLoadString()) { ProcessLoadString(instruction->AsLoadString()); } @@ -133,104 +131,93 @@ void HSharpening::ProcessInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) { invoke->SetDispatchInfo(dispatch_info); } -void HSharpening::ProcessLoadClass(HLoadClass* load_class) { - ScopedObjectAccess soa(Thread::Current()); - SharpenClass(load_class, codegen_, compiler_driver_); -} - -void HSharpening::SharpenClass(HLoadClass* load_class, - CodeGenerator* codegen, - CompilerDriver* compiler_driver) { +HLoadClass::LoadKind HSharpening::SharpenClass(HLoadClass* load_class, + CodeGenerator* codegen, + CompilerDriver* compiler_driver, + const DexCompilationUnit& dex_compilation_unit) { Handle<mirror::Class> klass = load_class->GetClass(); DCHECK(load_class->GetLoadKind() == HLoadClass::LoadKind::kDexCacheViaMethod || load_class->GetLoadKind() == HLoadClass::LoadKind::kReferrersClass) << load_class->GetLoadKind(); DCHECK(!load_class->IsInBootImage()) << "HLoadClass should not be optimized before sharpening."; + HLoadClass::LoadKind load_kind = load_class->GetLoadKind(); + if (load_class->NeedsAccessCheck()) { // We need to call the runtime anyway, so we simply get the class as that call's return value. - return; - } - - if (load_class->GetLoadKind() == HLoadClass::LoadKind::kReferrersClass) { + } else if (load_kind == HLoadClass::LoadKind::kReferrersClass) { // Loading from the ArtMethod* is the most efficient retrieval in code size. // TODO: This may not actually be true for all architectures and // locations of target classes. The additional register pressure // for using the ArtMethod* should be considered. - return; - } - - const DexFile& dex_file = load_class->GetDexFile(); - dex::TypeIndex type_index = load_class->GetTypeIndex(); + } else { + const DexFile& dex_file = load_class->GetDexFile(); + dex::TypeIndex type_index = load_class->GetTypeIndex(); - bool is_in_boot_image = false; - HLoadClass::LoadKind desired_load_kind = static_cast<HLoadClass::LoadKind>(-1); - Runtime* runtime = Runtime::Current(); - if (codegen->GetCompilerOptions().IsBootImage()) { - // Compiling boot image. Check if the class is a boot image class. - DCHECK(!runtime->UseJitCompilation()); - if (!compiler_driver->GetSupportBootImageFixup()) { - // compiler_driver_test. Do not sharpen. - desired_load_kind = HLoadClass::LoadKind::kDexCacheViaMethod; - } else if ((klass.Get() != nullptr) && compiler_driver->IsImageClass( - dex_file.StringDataByIdx(dex_file.GetTypeId(type_index).descriptor_idx_))) { - is_in_boot_image = true; - desired_load_kind = codegen->GetCompilerOptions().GetCompilePic() - ? HLoadClass::LoadKind::kBootImageLinkTimePcRelative - : HLoadClass::LoadKind::kBootImageLinkTimeAddress; + bool is_in_boot_image = false; + HLoadClass::LoadKind desired_load_kind = HLoadClass::LoadKind::kInvalid; + Runtime* runtime = Runtime::Current(); + if (codegen->GetCompilerOptions().IsBootImage()) { + // Compiling boot image. Check if the class is a boot image class. + DCHECK(!runtime->UseJitCompilation()); + if (!compiler_driver->GetSupportBootImageFixup()) { + // compiler_driver_test. Do not sharpen. + desired_load_kind = HLoadClass::LoadKind::kDexCacheViaMethod; + } else if ((klass.Get() != nullptr) && compiler_driver->IsImageClass( + dex_file.StringDataByIdx(dex_file.GetTypeId(type_index).descriptor_idx_))) { + is_in_boot_image = true; + desired_load_kind = codegen->GetCompilerOptions().GetCompilePic() + ? HLoadClass::LoadKind::kBootImageLinkTimePcRelative + : HLoadClass::LoadKind::kBootImageLinkTimeAddress; + } else { + // Not a boot image class. + DCHECK(ContainsElement(compiler_driver->GetDexFilesForOatFile(), &dex_file)); + desired_load_kind = HLoadClass::LoadKind::kBssEntry; + } } else { - // Not a boot image class. - DCHECK(ContainsElement(compiler_driver->GetDexFilesForOatFile(), &dex_file)); - desired_load_kind = HLoadClass::LoadKind::kBssEntry; - } - } else { - is_in_boot_image = (klass.Get() != nullptr) && - runtime->GetHeap()->ObjectIsInBootImageSpace(klass.Get()); - if (runtime->UseJitCompilation()) { - // TODO: Make sure we don't set the "compile PIC" flag for JIT as that's bogus. - // DCHECK(!codegen_->GetCompilerOptions().GetCompilePic()); - if (is_in_boot_image) { - // TODO: Use direct pointers for all non-moving spaces, not just boot image. Bug: 29530787 + is_in_boot_image = (klass.Get() != nullptr) && + runtime->GetHeap()->ObjectIsInBootImageSpace(klass.Get()); + if (runtime->UseJitCompilation()) { + // TODO: Make sure we don't set the "compile PIC" flag for JIT as that's bogus. + // DCHECK(!codegen_->GetCompilerOptions().GetCompilePic()); + if (is_in_boot_image) { + // TODO: Use direct pointers for all non-moving spaces, not just boot image. Bug: 29530787 + desired_load_kind = HLoadClass::LoadKind::kBootImageAddress; + } else if (klass.Get() != nullptr) { + desired_load_kind = HLoadClass::LoadKind::kJitTableAddress; + } else { + // Class not loaded yet. This happens when the dex code requesting + // this `HLoadClass` hasn't been executed in the interpreter. + // Fallback to the dex cache. + // TODO(ngeoffray): Generate HDeoptimize instead. + desired_load_kind = HLoadClass::LoadKind::kDexCacheViaMethod; + } + } else if (is_in_boot_image && !codegen->GetCompilerOptions().GetCompilePic()) { + // AOT app compilation. Check if the class is in the boot image. desired_load_kind = HLoadClass::LoadKind::kBootImageAddress; - } else if (klass.Get() != nullptr) { - desired_load_kind = HLoadClass::LoadKind::kJitTableAddress; } else { - // Class not loaded yet. This happens when the dex code requesting - // this `HLoadClass` hasn't been executed in the interpreter. - // Fallback to the dex cache. - // TODO(ngeoffray): Generate HDeoptimize instead. - desired_load_kind = HLoadClass::LoadKind::kDexCacheViaMethod; + // Not JIT and either the klass is not in boot image or we are compiling in PIC mode. + desired_load_kind = HLoadClass::LoadKind::kBssEntry; } - } else if (is_in_boot_image && !codegen->GetCompilerOptions().GetCompilePic()) { - // AOT app compilation. Check if the class is in the boot image. - desired_load_kind = HLoadClass::LoadKind::kBootImageAddress; - } else { - // Not JIT and either the klass is not in boot image or we are compiling in PIC mode. - desired_load_kind = HLoadClass::LoadKind::kBssEntry; } - } - DCHECK_NE(desired_load_kind, static_cast<HLoadClass::LoadKind>(-1)); + DCHECK_NE(desired_load_kind, HLoadClass::LoadKind::kInvalid); - if (is_in_boot_image) { - load_class->MarkInBootImage(); + if (is_in_boot_image) { + load_class->MarkInBootImage(); + } + load_kind = codegen->GetSupportedLoadClassKind(desired_load_kind); } - HLoadClass::LoadKind load_kind = codegen->GetSupportedLoadClassKind(desired_load_kind); - switch (load_kind) { - case HLoadClass::LoadKind::kBootImageLinkTimeAddress: - case HLoadClass::LoadKind::kBootImageLinkTimePcRelative: - case HLoadClass::LoadKind::kBssEntry: - case HLoadClass::LoadKind::kDexCacheViaMethod: - load_class->SetLoadKindWithTypeReference(load_kind, dex_file, type_index); - break; - case HLoadClass::LoadKind::kBootImageAddress: - case HLoadClass::LoadKind::kJitTableAddress: - load_class->SetLoadKind(load_kind); - break; - default: - LOG(FATAL) << "Unexpected load kind: " << load_kind; - UNREACHABLE(); + if (!IsSameDexFile(load_class->GetDexFile(), *dex_compilation_unit.GetDexFile())) { + if ((load_kind == HLoadClass::LoadKind::kDexCacheViaMethod) || + (load_kind == HLoadClass::LoadKind::kBssEntry)) { + // We actually cannot reference this class, we're forced to bail. + // We cannot reference this class with Bss, as the entrypoint will lookup the class + // in the caller's dex file, but that dex file does not reference the class. + return HLoadClass::LoadKind::kInvalid; + } } + return load_kind; } void HSharpening::ProcessLoadString(HLoadString* load_string) { diff --git a/compiler/optimizing/sharpening.h b/compiler/optimizing/sharpening.h index ae3d83ef2c..4240b2f339 100644 --- a/compiler/optimizing/sharpening.h +++ b/compiler/optimizing/sharpening.h @@ -17,6 +17,7 @@ #ifndef ART_COMPILER_OPTIMIZING_SHARPENING_H_ #define ART_COMPILER_OPTIMIZING_SHARPENING_H_ +#include "nodes.h" #include "optimization.h" namespace art { @@ -24,7 +25,6 @@ namespace art { class CodeGenerator; class CompilerDriver; class DexCompilationUnit; -class HInvokeStaticOrDirect; // Optimization that tries to improve the way we dispatch methods and access types, // fields, etc. Besides actual method sharpening based on receiver type (for example @@ -47,15 +47,15 @@ class HSharpening : public HOptimization { static constexpr const char* kSharpeningPassName = "sharpening"; - // Used internally but also by the inliner. - static void SharpenClass(HLoadClass* load_class, - CodeGenerator* codegen, - CompilerDriver* compiler_driver) + // Used by the builder and the inliner. + static HLoadClass::LoadKind SharpenClass(HLoadClass* load_class, + CodeGenerator* codegen, + CompilerDriver* compiler_driver, + const DexCompilationUnit& dex_compilation_unit) REQUIRES_SHARED(Locks::mutator_lock_); private: void ProcessInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke); - void ProcessLoadClass(HLoadClass* load_class); void ProcessLoadString(HLoadString* load_string); CodeGenerator* codegen_; diff --git a/compiler/optimizing/ssa_builder.cc b/compiler/optimizing/ssa_builder.cc index ae1e369999..487e4dd498 100644 --- a/compiler/optimizing/ssa_builder.cc +++ b/compiler/optimizing/ssa_builder.cc @@ -17,8 +17,10 @@ #include "ssa_builder.h" #include "bytecode_utils.h" +#include "mirror/class-inl.h" #include "nodes.h" #include "reference_type_propagation.h" +#include "scoped_thread_state_change-inl.h" #include "ssa_phi_elimination.h" namespace art { diff --git a/compiler/optimizing/stack_map_stream.cc b/compiler/optimizing/stack_map_stream.cc index 668108daa4..f8e01b7537 100644 --- a/compiler/optimizing/stack_map_stream.cc +++ b/compiler/optimizing/stack_map_stream.cc @@ -18,8 +18,9 @@ #include <unordered_map> +#include "art_method-inl.h" #include "base/stl_util.h" -#include "art_method.h" +#include "optimizing/optimizing_compiler.h" #include "runtime.h" #include "scoped_thread_state_change-inl.h" @@ -107,11 +108,6 @@ void StackMapStream::AddDexRegisterEntry(DexRegisterLocation::Kind kind, int32_t current_dex_register_++; } -static bool EncodeArtMethodInInlineInfo(ArtMethod* method ATTRIBUTE_UNUSED) { - // Note: the runtime is null only for unit testing. - return Runtime::Current() == nullptr || !Runtime::Current()->IsAotCompiler(); -} - void StackMapStream::BeginInlineInfoEntry(ArtMethod* method, uint32_t dex_pc, uint32_t num_dex_registers, @@ -157,56 +153,35 @@ CodeOffset StackMapStream::ComputeMaxNativePcCodeOffset() const { } size_t StackMapStream::PrepareForFillIn() { - const size_t stack_mask_size_in_bits = stack_mask_max_ + 1; // Need room for max element too. - const size_t number_of_stack_masks = PrepareStackMasks(stack_mask_size_in_bits); - const size_t register_mask_size_in_bits = MinimumBitsToStore(register_mask_max_); - const size_t number_of_register_masks = PrepareRegisterMasks(); - dex_register_maps_size_ = ComputeDexRegisterMapsSize(); - ComputeInlineInfoEncoding(); // needs dex_register_maps_size_. - inline_info_size_ = inline_infos_.size() * inline_info_encoding_.GetEntrySize(); + CodeInfoEncoding encoding; + encoding.dex_register_map.num_entries = 0; // TODO: Remove this field. + encoding.dex_register_map.num_bytes = ComputeDexRegisterMapsSize(); + encoding.location_catalog.num_entries = location_catalog_entries_.size(); + encoding.location_catalog.num_bytes = ComputeDexRegisterLocationCatalogSize(); + encoding.inline_info.num_entries = inline_infos_.size(); + ComputeInlineInfoEncoding(&encoding.inline_info.encoding, + encoding.dex_register_map.num_bytes); CodeOffset max_native_pc_offset = ComputeMaxNativePcCodeOffset(); - // The stack map contains compressed native PC offsets. - const size_t stack_map_size = stack_map_encoding_.SetFromSizes( + // Prepare the CodeInfo variable-sized encoding. + encoding.stack_mask.encoding.num_bits = stack_mask_max_ + 1; // Need room for max element too. + encoding.stack_mask.num_entries = PrepareStackMasks(encoding.stack_mask.encoding.num_bits); + encoding.register_mask.encoding.num_bits = MinimumBitsToStore(register_mask_max_); + encoding.register_mask.num_entries = PrepareRegisterMasks(); + encoding.stack_map.num_entries = stack_maps_.size(); + encoding.stack_map.encoding.SetFromSizes( + // The stack map contains compressed native PC offsets. max_native_pc_offset.CompressedValue(), dex_pc_max_, - dex_register_maps_size_, - inline_info_size_, - number_of_register_masks, - number_of_stack_masks); - stack_maps_size_ = RoundUp(stack_maps_.size() * stack_map_size, kBitsPerByte) / kBitsPerByte; - dex_register_location_catalog_size_ = ComputeDexRegisterLocationCatalogSize(); - const size_t stack_masks_bits = number_of_stack_masks * stack_mask_size_in_bits; - const size_t register_masks_bits = number_of_register_masks * register_mask_size_in_bits; - // Register masks are last, stack masks are right before that last. - // They are both bit packed / aligned. - const size_t non_header_size = - stack_maps_size_ + - dex_register_location_catalog_size_ + - dex_register_maps_size_ + - inline_info_size_ + - RoundUp(stack_masks_bits + register_masks_bits, kBitsPerByte) / kBitsPerByte; - - // Prepare the CodeInfo variable-sized encoding. - CodeInfoEncoding code_info_encoding; - code_info_encoding.non_header_size = non_header_size; - code_info_encoding.number_of_stack_maps = stack_maps_.size(); - code_info_encoding.number_of_stack_masks = number_of_stack_masks; - code_info_encoding.number_of_register_masks = number_of_register_masks; - code_info_encoding.stack_mask_size_in_bits = stack_mask_size_in_bits; - code_info_encoding.register_mask_size_in_bits = register_mask_size_in_bits; - code_info_encoding.stack_map_encoding = stack_map_encoding_; - code_info_encoding.inline_info_encoding = inline_info_encoding_; - code_info_encoding.number_of_location_catalog_entries = location_catalog_entries_.size(); - code_info_encoding.Compress(&code_info_encoding_); - - // TODO: Move the catalog at the end. It is currently too expensive at runtime - // to compute its size (note that we do not encode that size in the CodeInfo). - dex_register_location_catalog_start_ = code_info_encoding_.size() + stack_maps_size_; - dex_register_maps_start_ = - dex_register_location_catalog_start_ + dex_register_location_catalog_size_; - inline_infos_start_ = dex_register_maps_start_ + dex_register_maps_size_; - - needed_size_ = code_info_encoding_.size() + non_header_size; + encoding.dex_register_map.num_bytes, + encoding.inline_info.num_entries, + encoding.register_mask.num_entries, + encoding.stack_mask.num_entries); + DCHECK_EQ(code_info_encoding_.size(), 0u); + encoding.Compress(&code_info_encoding_); + encoding.ComputeTableOffsets(); + // Compute table offsets so we can get the non header size. + DCHECK_EQ(encoding.HeaderSize(), code_info_encoding_.size()); + needed_size_ = code_info_encoding_.size() + encoding.NonHeaderSize(); return needed_size_; } @@ -259,7 +234,8 @@ size_t StackMapStream::ComputeDexRegisterMapsSize() const { return size; } -void StackMapStream::ComputeInlineInfoEncoding() { +void StackMapStream::ComputeInlineInfoEncoding(InlineInfoEncoding* encoding, + size_t dex_register_maps_bytes) { uint32_t method_index_max = 0; uint32_t dex_pc_max = DexFile::kDexNoIndex; uint32_t extra_data_max = 0; @@ -285,10 +261,7 @@ void StackMapStream::ComputeInlineInfoEncoding() { } DCHECK_EQ(inline_info_index, inline_infos_.size()); - inline_info_encoding_.SetFromSizes(method_index_max, - dex_pc_max, - extra_data_max, - dex_register_maps_size_); + encoding->SetFromSizes(method_index_max, dex_pc_max, extra_data_max, dex_register_maps_bytes); } void StackMapStream::FillIn(MemoryRegion region) { @@ -303,19 +276,18 @@ void StackMapStream::FillIn(MemoryRegion region) { // Write the CodeInfo header. region.CopyFrom(0, MemoryRegion(code_info_encoding_.data(), code_info_encoding_.size())); - MemoryRegion dex_register_locations_region = region.Subregion( - dex_register_maps_start_, dex_register_maps_size_); - - MemoryRegion inline_infos_region = region.Subregion( - inline_infos_start_, inline_info_size_); - CodeInfo code_info(region); CodeInfoEncoding encoding = code_info.ExtractEncoding(); - DCHECK_EQ(code_info.GetStackMapsSize(encoding), stack_maps_size_); + DCHECK_EQ(encoding.stack_map.num_entries, stack_maps_.size()); + + MemoryRegion dex_register_locations_region = region.Subregion( + encoding.dex_register_map.byte_offset, + encoding.dex_register_map.num_bytes); // Set the Dex register location catalog. MemoryRegion dex_register_location_catalog_region = region.Subregion( - dex_register_location_catalog_start_, dex_register_location_catalog_size_); + encoding.location_catalog.byte_offset, + encoding.location_catalog.num_bytes); DexRegisterLocationCatalog dex_register_location_catalog(dex_register_location_catalog_region); // Offset in `dex_register_location_catalog` where to store the next // register location. @@ -329,27 +301,27 @@ void StackMapStream::FillIn(MemoryRegion region) { ArenaBitVector empty_bitmask(allocator_, 0, /* expandable */ false, kArenaAllocStackMapStream); uintptr_t next_dex_register_map_offset = 0; - uintptr_t next_inline_info_offset = 0; + uintptr_t next_inline_info_index = 0; for (size_t i = 0, e = stack_maps_.size(); i < e; ++i) { StackMap stack_map = code_info.GetStackMapAt(i, encoding); StackMapEntry entry = stack_maps_[i]; - stack_map.SetDexPc(stack_map_encoding_, entry.dex_pc); - stack_map.SetNativePcCodeOffset(stack_map_encoding_, entry.native_pc_code_offset); - stack_map.SetRegisterMaskIndex(stack_map_encoding_, entry.register_mask_index); - stack_map.SetStackMaskIndex(stack_map_encoding_, entry.stack_mask_index); + stack_map.SetDexPc(encoding.stack_map.encoding, entry.dex_pc); + stack_map.SetNativePcCodeOffset(encoding.stack_map.encoding, entry.native_pc_code_offset); + stack_map.SetRegisterMaskIndex(encoding.stack_map.encoding, entry.register_mask_index); + stack_map.SetStackMaskIndex(encoding.stack_map.encoding, entry.stack_mask_index); if (entry.num_dex_registers == 0 || (entry.live_dex_registers_mask->NumSetBits() == 0)) { // No dex map available. - stack_map.SetDexRegisterMapOffset(stack_map_encoding_, StackMap::kNoDexRegisterMap); + stack_map.SetDexRegisterMapOffset(encoding.stack_map.encoding, StackMap::kNoDexRegisterMap); } else { // Search for an entry with the same dex map. if (entry.same_dex_register_map_as_ != kNoSameDexMapFound) { // If we have a hit reuse the offset. stack_map.SetDexRegisterMapOffset( - stack_map_encoding_, + encoding.stack_map.encoding, code_info.GetStackMapAt(entry.same_dex_register_map_as_, encoding) - .GetDexRegisterMapOffset(stack_map_encoding_)); + .GetDexRegisterMapOffset(encoding.stack_map.encoding)); } else { // New dex registers maps should be added to the stack map. MemoryRegion register_region = dex_register_locations_region.Subregion( @@ -358,7 +330,8 @@ void StackMapStream::FillIn(MemoryRegion region) { next_dex_register_map_offset += register_region.size(); DexRegisterMap dex_register_map(register_region); stack_map.SetDexRegisterMapOffset( - stack_map_encoding_, register_region.begin() - dex_register_locations_region.begin()); + encoding.stack_map.encoding, + register_region.begin() - dex_register_locations_region.begin()); // Set the dex register location. FillInDexRegisterMap(dex_register_map, @@ -370,37 +343,37 @@ void StackMapStream::FillIn(MemoryRegion region) { // Set the inlining info. if (entry.inlining_depth != 0) { - MemoryRegion inline_region = inline_infos_region.Subregion( - next_inline_info_offset, - entry.inlining_depth * inline_info_encoding_.GetEntrySize()); - next_inline_info_offset += inline_region.size(); - InlineInfo inline_info(inline_region); + InlineInfo inline_info = code_info.GetInlineInfo(next_inline_info_index, encoding); - // Currently relative to the dex register map. - stack_map.SetInlineDescriptorOffset( - stack_map_encoding_, inline_region.begin() - dex_register_locations_region.begin()); + // Fill in the index. + stack_map.SetInlineInfoIndex(encoding.stack_map.encoding, next_inline_info_index); + DCHECK_EQ(next_inline_info_index, entry.inline_infos_start_index); + next_inline_info_index += entry.inlining_depth; - inline_info.SetDepth(inline_info_encoding_, entry.inlining_depth); + inline_info.SetDepth(encoding.inline_info.encoding, entry.inlining_depth); DCHECK_LE(entry.inline_infos_start_index + entry.inlining_depth, inline_infos_.size()); + for (size_t depth = 0; depth < entry.inlining_depth; ++depth) { InlineInfoEntry inline_entry = inline_infos_[depth + entry.inline_infos_start_index]; if (inline_entry.method != nullptr) { inline_info.SetMethodIndexAtDepth( - inline_info_encoding_, + encoding.inline_info.encoding, depth, High32Bits(reinterpret_cast<uintptr_t>(inline_entry.method))); inline_info.SetExtraDataAtDepth( - inline_info_encoding_, + encoding.inline_info.encoding, depth, Low32Bits(reinterpret_cast<uintptr_t>(inline_entry.method))); } else { - inline_info.SetMethodIndexAtDepth(inline_info_encoding_, depth, inline_entry.method_index); - inline_info.SetExtraDataAtDepth(inline_info_encoding_, depth, 1); + inline_info.SetMethodIndexAtDepth(encoding.inline_info.encoding, + depth, + inline_entry.method_index); + inline_info.SetExtraDataAtDepth(encoding.inline_info.encoding, depth, 1); } - inline_info.SetDexPcAtDepth(inline_info_encoding_, depth, inline_entry.dex_pc); + inline_info.SetDexPcAtDepth(encoding.inline_info.encoding, depth, inline_entry.dex_pc); if (inline_entry.num_dex_registers == 0) { // No dex map available. - inline_info.SetDexRegisterMapOffsetAtDepth(inline_info_encoding_, + inline_info.SetDexRegisterMapOffsetAtDepth(encoding.inline_info.encoding, depth, StackMap::kNoDexRegisterMap); DCHECK(inline_entry.live_dex_registers_mask == nullptr); @@ -412,8 +385,9 @@ void StackMapStream::FillIn(MemoryRegion region) { next_dex_register_map_offset += register_region.size(); DexRegisterMap dex_register_map(register_region); inline_info.SetDexRegisterMapOffsetAtDepth( - inline_info_encoding_, - depth, register_region.begin() - dex_register_locations_region.begin()); + encoding.inline_info.encoding, + depth, + register_region.begin() - dex_register_locations_region.begin()); FillInDexRegisterMap(dex_register_map, inline_entry.num_dex_registers, @@ -421,30 +395,28 @@ void StackMapStream::FillIn(MemoryRegion region) { inline_entry.dex_register_locations_start_index); } } - } else { - if (inline_info_size_ != 0) { - stack_map.SetInlineDescriptorOffset(stack_map_encoding_, StackMap::kNoInlineInfo); - } + } else if (encoding.stack_map.encoding.GetInlineInfoEncoding().BitSize() > 0) { + stack_map.SetInlineInfoIndex(encoding.stack_map.encoding, StackMap::kNoInlineInfo); } } // Write stack masks table. - size_t stack_mask_bits = encoding.stack_mask_size_in_bits; + const size_t stack_mask_bits = encoding.stack_mask.encoding.BitSize(); if (stack_mask_bits > 0) { size_t stack_mask_bytes = RoundUp(stack_mask_bits, kBitsPerByte) / kBitsPerByte; - for (size_t i = 0; i < encoding.number_of_stack_masks; ++i) { + for (size_t i = 0; i < encoding.stack_mask.num_entries; ++i) { MemoryRegion source(&stack_masks_[i * stack_mask_bytes], stack_mask_bytes); - BitMemoryRegion stack_mask = code_info.GetStackMask(encoding, i); - for (size_t bit_index = 0; bit_index < encoding.stack_mask_size_in_bits; ++bit_index) { + BitMemoryRegion stack_mask = code_info.GetStackMask(i, encoding); + for (size_t bit_index = 0; bit_index < stack_mask_bits; ++bit_index) { stack_mask.StoreBit(bit_index, source.LoadBit(bit_index)); } } } // Write register masks table. - for (size_t i = 0; i < encoding.number_of_register_masks; ++i) { - BitMemoryRegion register_mask = code_info.GetRegisterMask(encoding, i); - register_mask.StoreBits(0, register_masks_[i], encoding.register_mask_size_in_bits); + for (size_t i = 0; i < encoding.register_mask.num_entries; ++i) { + BitMemoryRegion register_mask = code_info.GetRegisterMask(i, encoding); + register_mask.StoreBits(0, register_masks_[i], encoding.register_mask.encoding.BitSize()); } // Verify all written data in debug build. @@ -546,7 +518,8 @@ void StackMapStream::CheckDexRegisterMap(const CodeInfo& code_info, } // Compare to the seen location. if (expected.GetKind() == DexRegisterLocation::Kind::kNone) { - DCHECK(!dex_register_map.IsValid() || !dex_register_map.IsDexRegisterLive(reg)); + DCHECK(!dex_register_map.IsValid() || !dex_register_map.IsDexRegisterLive(reg)) + << dex_register_map.IsValid() << " " << dex_register_map.IsDexRegisterLive(reg); } else { DCHECK(dex_register_map.IsDexRegisterLive(reg)); DexRegisterLocation seen = dex_register_map.GetDexRegisterLocation( @@ -599,7 +572,7 @@ void StackMapStream::CheckCodeInfo(MemoryRegion region) const { DCHECK_EQ(code_info.GetNumberOfStackMaps(encoding), stack_maps_.size()); for (size_t s = 0; s < stack_maps_.size(); ++s) { const StackMap stack_map = code_info.GetStackMapAt(s, encoding); - const StackMapEncoding& stack_map_encoding = encoding.stack_map_encoding; + const StackMapEncoding& stack_map_encoding = encoding.stack_map.encoding; StackMapEntry entry = stack_maps_[s]; // Check main stack map fields. @@ -633,18 +606,18 @@ void StackMapStream::CheckCodeInfo(MemoryRegion region) const { DCHECK_EQ(stack_map.HasInlineInfo(stack_map_encoding), (entry.inlining_depth != 0)); if (entry.inlining_depth != 0) { InlineInfo inline_info = code_info.GetInlineInfoOf(stack_map, encoding); - DCHECK_EQ(inline_info.GetDepth(encoding.inline_info_encoding), entry.inlining_depth); + DCHECK_EQ(inline_info.GetDepth(encoding.inline_info.encoding), entry.inlining_depth); for (size_t d = 0; d < entry.inlining_depth; ++d) { size_t inline_info_index = entry.inline_infos_start_index + d; DCHECK_LT(inline_info_index, inline_infos_.size()); InlineInfoEntry inline_entry = inline_infos_[inline_info_index]; - DCHECK_EQ(inline_info.GetDexPcAtDepth(encoding.inline_info_encoding, d), + DCHECK_EQ(inline_info.GetDexPcAtDepth(encoding.inline_info.encoding, d), inline_entry.dex_pc); - if (inline_info.EncodesArtMethodAtDepth(encoding.inline_info_encoding, d)) { - DCHECK_EQ(inline_info.GetArtMethodAtDepth(encoding.inline_info_encoding, d), + if (inline_info.EncodesArtMethodAtDepth(encoding.inline_info.encoding, d)) { + DCHECK_EQ(inline_info.GetArtMethodAtDepth(encoding.inline_info.encoding, d), inline_entry.method); } else { - DCHECK_EQ(inline_info.GetMethodIndexAtDepth(encoding.inline_info_encoding, d), + DCHECK_EQ(inline_info.GetMethodIndexAtDepth(encoding.inline_info.encoding, d), inline_entry.method_index); } diff --git a/compiler/optimizing/stack_map_stream.h b/compiler/optimizing/stack_map_stream.h index b1069a17be..08c1d3e3c0 100644 --- a/compiler/optimizing/stack_map_stream.h +++ b/compiler/optimizing/stack_map_stream.h @@ -79,13 +79,6 @@ class StackMapStream : public ValueObject { current_entry_(), current_inline_info_(), code_info_encoding_(allocator->Adapter(kArenaAllocStackMapStream)), - inline_info_size_(0), - dex_register_maps_size_(0), - stack_maps_size_(0), - dex_register_location_catalog_size_(0), - dex_register_location_catalog_start_(0), - dex_register_maps_start_(0), - inline_infos_start_(0), needed_size_(0), current_dex_register_(0), in_inline_frame_(false) { @@ -160,7 +153,8 @@ class StackMapStream : public ValueObject { size_t ComputeDexRegisterMapSize(uint32_t num_dex_registers, const BitVector* live_dex_registers_mask) const; size_t ComputeDexRegisterMapsSize() const; - void ComputeInlineInfoEncoding(); + void ComputeInlineInfoEncoding(InlineInfoEncoding* encoding, + size_t dex_register_maps_bytes); CodeOffset ComputeMaxNativePcCodeOffset() const; @@ -214,16 +208,7 @@ class StackMapStream : public ValueObject { StackMapEntry current_entry_; InlineInfoEntry current_inline_info_; - StackMapEncoding stack_map_encoding_; - InlineInfoEncoding inline_info_encoding_; ArenaVector<uint8_t> code_info_encoding_; - size_t inline_info_size_; - size_t dex_register_maps_size_; - size_t stack_maps_size_; - size_t dex_register_location_catalog_size_; - size_t dex_register_location_catalog_start_; - size_t dex_register_maps_start_; - size_t inline_infos_start_; size_t needed_size_; uint32_t current_dex_register_; bool in_inline_frame_; diff --git a/compiler/optimizing/stack_map_test.cc b/compiler/optimizing/stack_map_test.cc index ce6d5c2b22..bd0aa6dea7 100644 --- a/compiler/optimizing/stack_map_test.cc +++ b/compiler/optimizing/stack_map_test.cc @@ -32,10 +32,10 @@ static bool CheckStackMask( const StackMap& stack_map, const BitVector& bit_vector) { BitMemoryRegion stack_mask = code_info.GetStackMaskOf(encoding, stack_map); - if (bit_vector.GetNumberOfBits() > encoding.stack_mask_size_in_bits) { + if (bit_vector.GetNumberOfBits() > encoding.stack_mask.encoding.BitSize()) { return false; } - for (size_t i = 0; i < encoding.stack_mask_size_in_bits; ++i) { + for (size_t i = 0; i < encoding.stack_mask.encoding.BitSize(); ++i) { if (stack_mask.LoadBit(i) != bit_vector.IsBitSet(i)) { return false; } @@ -78,13 +78,13 @@ TEST(StackMapTest, Test1) { StackMap stack_map = code_info.GetStackMapAt(0, encoding); ASSERT_TRUE(stack_map.Equals(code_info.GetStackMapForDexPc(0, encoding))); ASSERT_TRUE(stack_map.Equals(code_info.GetStackMapForNativePcOffset(64, encoding))); - ASSERT_EQ(0u, stack_map.GetDexPc(encoding.stack_map_encoding)); - ASSERT_EQ(64u, stack_map.GetNativePcOffset(encoding.stack_map_encoding, kRuntimeISA)); + ASSERT_EQ(0u, stack_map.GetDexPc(encoding.stack_map.encoding)); + ASSERT_EQ(64u, stack_map.GetNativePcOffset(encoding.stack_map.encoding, kRuntimeISA)); ASSERT_EQ(0x3u, code_info.GetRegisterMaskOf(encoding, stack_map)); ASSERT_TRUE(CheckStackMask(code_info, encoding, stack_map, sp_mask)); - ASSERT_TRUE(stack_map.HasDexRegisterMap(encoding.stack_map_encoding)); + ASSERT_TRUE(stack_map.HasDexRegisterMap(encoding.stack_map.encoding)); DexRegisterMap dex_register_map = code_info.GetDexRegisterMapOf(stack_map, encoding, number_of_dex_registers); ASSERT_TRUE(dex_register_map.IsDexRegisterLive(0)); @@ -123,7 +123,7 @@ TEST(StackMapTest, Test1) { ASSERT_EQ(0, location0.GetValue()); ASSERT_EQ(-2, location1.GetValue()); - ASSERT_FALSE(stack_map.HasInlineInfo(encoding.stack_map_encoding)); + ASSERT_FALSE(stack_map.HasInlineInfo(encoding.stack_map.encoding)); } TEST(StackMapTest, Test2) { @@ -193,13 +193,13 @@ TEST(StackMapTest, Test2) { StackMap stack_map = code_info.GetStackMapAt(0, encoding); ASSERT_TRUE(stack_map.Equals(code_info.GetStackMapForDexPc(0, encoding))); ASSERT_TRUE(stack_map.Equals(code_info.GetStackMapForNativePcOffset(64, encoding))); - ASSERT_EQ(0u, stack_map.GetDexPc(encoding.stack_map_encoding)); - ASSERT_EQ(64u, stack_map.GetNativePcOffset(encoding.stack_map_encoding, kRuntimeISA)); + ASSERT_EQ(0u, stack_map.GetDexPc(encoding.stack_map.encoding)); + ASSERT_EQ(64u, stack_map.GetNativePcOffset(encoding.stack_map.encoding, kRuntimeISA)); ASSERT_EQ(0x3u, code_info.GetRegisterMaskOf(encoding, stack_map)); ASSERT_TRUE(CheckStackMask(code_info, encoding, stack_map, sp_mask1)); - ASSERT_TRUE(stack_map.HasDexRegisterMap(encoding.stack_map_encoding)); + ASSERT_TRUE(stack_map.HasDexRegisterMap(encoding.stack_map.encoding)); DexRegisterMap dex_register_map = code_info.GetDexRegisterMapOf(stack_map, encoding, number_of_dex_registers); ASSERT_TRUE(dex_register_map.IsDexRegisterLive(0)); @@ -238,13 +238,13 @@ TEST(StackMapTest, Test2) { ASSERT_EQ(0, location0.GetValue()); ASSERT_EQ(-2, location1.GetValue()); - ASSERT_TRUE(stack_map.HasInlineInfo(encoding.stack_map_encoding)); + ASSERT_TRUE(stack_map.HasInlineInfo(encoding.stack_map.encoding)); InlineInfo inline_info = code_info.GetInlineInfoOf(stack_map, encoding); - ASSERT_EQ(2u, inline_info.GetDepth(encoding.inline_info_encoding)); - ASSERT_EQ(3u, inline_info.GetDexPcAtDepth(encoding.inline_info_encoding, 0)); - ASSERT_EQ(2u, inline_info.GetDexPcAtDepth(encoding.inline_info_encoding, 1)); - ASSERT_TRUE(inline_info.EncodesArtMethodAtDepth(encoding.inline_info_encoding, 0)); - ASSERT_TRUE(inline_info.EncodesArtMethodAtDepth(encoding.inline_info_encoding, 1)); + ASSERT_EQ(2u, inline_info.GetDepth(encoding.inline_info.encoding)); + ASSERT_EQ(3u, inline_info.GetDexPcAtDepth(encoding.inline_info.encoding, 0)); + ASSERT_EQ(2u, inline_info.GetDexPcAtDepth(encoding.inline_info.encoding, 1)); + ASSERT_TRUE(inline_info.EncodesArtMethodAtDepth(encoding.inline_info.encoding, 0)); + ASSERT_TRUE(inline_info.EncodesArtMethodAtDepth(encoding.inline_info.encoding, 1)); } // Second stack map. @@ -252,13 +252,13 @@ TEST(StackMapTest, Test2) { StackMap stack_map = code_info.GetStackMapAt(1, encoding); ASSERT_TRUE(stack_map.Equals(code_info.GetStackMapForDexPc(1u, encoding))); ASSERT_TRUE(stack_map.Equals(code_info.GetStackMapForNativePcOffset(128u, encoding))); - ASSERT_EQ(1u, stack_map.GetDexPc(encoding.stack_map_encoding)); - ASSERT_EQ(128u, stack_map.GetNativePcOffset(encoding.stack_map_encoding, kRuntimeISA)); + ASSERT_EQ(1u, stack_map.GetDexPc(encoding.stack_map.encoding)); + ASSERT_EQ(128u, stack_map.GetNativePcOffset(encoding.stack_map.encoding, kRuntimeISA)); ASSERT_EQ(0xFFu, code_info.GetRegisterMaskOf(encoding, stack_map)); ASSERT_TRUE(CheckStackMask(code_info, encoding, stack_map, sp_mask2)); - ASSERT_TRUE(stack_map.HasDexRegisterMap(encoding.stack_map_encoding)); + ASSERT_TRUE(stack_map.HasDexRegisterMap(encoding.stack_map.encoding)); DexRegisterMap dex_register_map = code_info.GetDexRegisterMapOf(stack_map, encoding, number_of_dex_registers); ASSERT_TRUE(dex_register_map.IsDexRegisterLive(0)); @@ -298,7 +298,7 @@ TEST(StackMapTest, Test2) { ASSERT_EQ(18, location0.GetValue()); ASSERT_EQ(3, location1.GetValue()); - ASSERT_FALSE(stack_map.HasInlineInfo(encoding.stack_map_encoding)); + ASSERT_FALSE(stack_map.HasInlineInfo(encoding.stack_map.encoding)); } // Third stack map. @@ -306,13 +306,13 @@ TEST(StackMapTest, Test2) { StackMap stack_map = code_info.GetStackMapAt(2, encoding); ASSERT_TRUE(stack_map.Equals(code_info.GetStackMapForDexPc(2u, encoding))); ASSERT_TRUE(stack_map.Equals(code_info.GetStackMapForNativePcOffset(192u, encoding))); - ASSERT_EQ(2u, stack_map.GetDexPc(encoding.stack_map_encoding)); - ASSERT_EQ(192u, stack_map.GetNativePcOffset(encoding.stack_map_encoding, kRuntimeISA)); + ASSERT_EQ(2u, stack_map.GetDexPc(encoding.stack_map.encoding)); + ASSERT_EQ(192u, stack_map.GetNativePcOffset(encoding.stack_map.encoding, kRuntimeISA)); ASSERT_EQ(0xABu, code_info.GetRegisterMaskOf(encoding, stack_map)); ASSERT_TRUE(CheckStackMask(code_info, encoding, stack_map, sp_mask3)); - ASSERT_TRUE(stack_map.HasDexRegisterMap(encoding.stack_map_encoding)); + ASSERT_TRUE(stack_map.HasDexRegisterMap(encoding.stack_map.encoding)); DexRegisterMap dex_register_map = code_info.GetDexRegisterMapOf(stack_map, encoding, number_of_dex_registers); ASSERT_TRUE(dex_register_map.IsDexRegisterLive(0)); @@ -352,7 +352,7 @@ TEST(StackMapTest, Test2) { ASSERT_EQ(6, location0.GetValue()); ASSERT_EQ(8, location1.GetValue()); - ASSERT_FALSE(stack_map.HasInlineInfo(encoding.stack_map_encoding)); + ASSERT_FALSE(stack_map.HasInlineInfo(encoding.stack_map.encoding)); } // Fourth stack map. @@ -360,13 +360,13 @@ TEST(StackMapTest, Test2) { StackMap stack_map = code_info.GetStackMapAt(3, encoding); ASSERT_TRUE(stack_map.Equals(code_info.GetStackMapForDexPc(3u, encoding))); ASSERT_TRUE(stack_map.Equals(code_info.GetStackMapForNativePcOffset(256u, encoding))); - ASSERT_EQ(3u, stack_map.GetDexPc(encoding.stack_map_encoding)); - ASSERT_EQ(256u, stack_map.GetNativePcOffset(encoding.stack_map_encoding, kRuntimeISA)); + ASSERT_EQ(3u, stack_map.GetDexPc(encoding.stack_map.encoding)); + ASSERT_EQ(256u, stack_map.GetNativePcOffset(encoding.stack_map.encoding, kRuntimeISA)); ASSERT_EQ(0xCDu, code_info.GetRegisterMaskOf(encoding, stack_map)); ASSERT_TRUE(CheckStackMask(code_info, encoding, stack_map, sp_mask4)); - ASSERT_TRUE(stack_map.HasDexRegisterMap(encoding.stack_map_encoding)); + ASSERT_TRUE(stack_map.HasDexRegisterMap(encoding.stack_map.encoding)); DexRegisterMap dex_register_map = code_info.GetDexRegisterMapOf(stack_map, encoding, number_of_dex_registers); ASSERT_TRUE(dex_register_map.IsDexRegisterLive(0)); @@ -406,7 +406,7 @@ TEST(StackMapTest, Test2) { ASSERT_EQ(3, location0.GetValue()); ASSERT_EQ(1, location1.GetValue()); - ASSERT_FALSE(stack_map.HasInlineInfo(encoding.stack_map_encoding)); + ASSERT_FALSE(stack_map.HasInlineInfo(encoding.stack_map.encoding)); } } @@ -442,11 +442,11 @@ TEST(StackMapTest, TestNonLiveDexRegisters) { StackMap stack_map = code_info.GetStackMapAt(0, encoding); ASSERT_TRUE(stack_map.Equals(code_info.GetStackMapForDexPc(0, encoding))); ASSERT_TRUE(stack_map.Equals(code_info.GetStackMapForNativePcOffset(64, encoding))); - ASSERT_EQ(0u, stack_map.GetDexPc(encoding.stack_map_encoding)); - ASSERT_EQ(64u, stack_map.GetNativePcOffset(encoding.stack_map_encoding, kRuntimeISA)); + ASSERT_EQ(0u, stack_map.GetDexPc(encoding.stack_map.encoding)); + ASSERT_EQ(64u, stack_map.GetNativePcOffset(encoding.stack_map.encoding, kRuntimeISA)); ASSERT_EQ(0x3u, code_info.GetRegisterMaskOf(encoding, stack_map)); - ASSERT_TRUE(stack_map.HasDexRegisterMap(encoding.stack_map_encoding)); + ASSERT_TRUE(stack_map.HasDexRegisterMap(encoding.stack_map.encoding)); DexRegisterMap dex_register_map = code_info.GetDexRegisterMapOf(stack_map, encoding, number_of_dex_registers); ASSERT_FALSE(dex_register_map.IsDexRegisterLive(0)); @@ -483,7 +483,7 @@ TEST(StackMapTest, TestNonLiveDexRegisters) { ASSERT_EQ(0, location0.GetValue()); ASSERT_EQ(-2, location1.GetValue()); - ASSERT_FALSE(stack_map.HasInlineInfo(encoding.stack_map_encoding)); + ASSERT_FALSE(stack_map.HasInlineInfo(encoding.stack_map.encoding)); } // Generate a stack map whose dex register offset is @@ -543,13 +543,13 @@ TEST(StackMapTest, DexRegisterMapOffsetOverflow) { ASSERT_EQ(255u, dex_register_map0.Size()); StackMap stack_map1 = code_info.GetStackMapAt(1, encoding); - ASSERT_TRUE(stack_map1.HasDexRegisterMap(encoding.stack_map_encoding)); + ASSERT_TRUE(stack_map1.HasDexRegisterMap(encoding.stack_map.encoding)); // ...the offset of the second Dex register map (relative to the // beginning of the Dex register maps region) is 255 (i.e., // kNoDexRegisterMapSmallEncoding). - ASSERT_NE(stack_map1.GetDexRegisterMapOffset(encoding.stack_map_encoding), + ASSERT_NE(stack_map1.GetDexRegisterMapOffset(encoding.stack_map.encoding), StackMap::kNoDexRegisterMap); - ASSERT_EQ(stack_map1.GetDexRegisterMapOffset(encoding.stack_map_encoding), 0xFFu); + ASSERT_EQ(stack_map1.GetDexRegisterMapOffset(encoding.stack_map.encoding), 0xFFu); } TEST(StackMapTest, TestShareDexRegisterMap) { @@ -602,12 +602,12 @@ TEST(StackMapTest, TestShareDexRegisterMap) { ASSERT_EQ(-2, dex_registers2.GetConstant(1, number_of_dex_registers, ci, encoding)); // Verify dex register map offsets. - ASSERT_EQ(sm0.GetDexRegisterMapOffset(encoding.stack_map_encoding), - sm1.GetDexRegisterMapOffset(encoding.stack_map_encoding)); - ASSERT_NE(sm0.GetDexRegisterMapOffset(encoding.stack_map_encoding), - sm2.GetDexRegisterMapOffset(encoding.stack_map_encoding)); - ASSERT_NE(sm1.GetDexRegisterMapOffset(encoding.stack_map_encoding), - sm2.GetDexRegisterMapOffset(encoding.stack_map_encoding)); + ASSERT_EQ(sm0.GetDexRegisterMapOffset(encoding.stack_map.encoding), + sm1.GetDexRegisterMapOffset(encoding.stack_map.encoding)); + ASSERT_NE(sm0.GetDexRegisterMapOffset(encoding.stack_map.encoding), + sm2.GetDexRegisterMapOffset(encoding.stack_map.encoding)); + ASSERT_NE(sm1.GetDexRegisterMapOffset(encoding.stack_map.encoding), + sm2.GetDexRegisterMapOffset(encoding.stack_map.encoding)); } TEST(StackMapTest, TestNoDexRegisterMap) { @@ -641,22 +641,22 @@ TEST(StackMapTest, TestNoDexRegisterMap) { StackMap stack_map = code_info.GetStackMapAt(0, encoding); ASSERT_TRUE(stack_map.Equals(code_info.GetStackMapForDexPc(0, encoding))); ASSERT_TRUE(stack_map.Equals(code_info.GetStackMapForNativePcOffset(64, encoding))); - ASSERT_EQ(0u, stack_map.GetDexPc(encoding.stack_map_encoding)); - ASSERT_EQ(64u, stack_map.GetNativePcOffset(encoding.stack_map_encoding, kRuntimeISA)); + ASSERT_EQ(0u, stack_map.GetDexPc(encoding.stack_map.encoding)); + ASSERT_EQ(64u, stack_map.GetNativePcOffset(encoding.stack_map.encoding, kRuntimeISA)); ASSERT_EQ(0x3u, code_info.GetRegisterMaskOf(encoding, stack_map)); - ASSERT_FALSE(stack_map.HasDexRegisterMap(encoding.stack_map_encoding)); - ASSERT_FALSE(stack_map.HasInlineInfo(encoding.stack_map_encoding)); + ASSERT_FALSE(stack_map.HasDexRegisterMap(encoding.stack_map.encoding)); + ASSERT_FALSE(stack_map.HasInlineInfo(encoding.stack_map.encoding)); stack_map = code_info.GetStackMapAt(1, encoding); ASSERT_TRUE(stack_map.Equals(code_info.GetStackMapForDexPc(1, encoding))); ASSERT_TRUE(stack_map.Equals(code_info.GetStackMapForNativePcOffset(68, encoding))); - ASSERT_EQ(1u, stack_map.GetDexPc(encoding.stack_map_encoding)); - ASSERT_EQ(68u, stack_map.GetNativePcOffset(encoding.stack_map_encoding, kRuntimeISA)); + ASSERT_EQ(1u, stack_map.GetDexPc(encoding.stack_map.encoding)); + ASSERT_EQ(68u, stack_map.GetNativePcOffset(encoding.stack_map.encoding, kRuntimeISA)); ASSERT_EQ(0x4u, code_info.GetRegisterMaskOf(encoding, stack_map)); - ASSERT_FALSE(stack_map.HasDexRegisterMap(encoding.stack_map_encoding)); - ASSERT_FALSE(stack_map.HasInlineInfo(encoding.stack_map_encoding)); + ASSERT_FALSE(stack_map.HasDexRegisterMap(encoding.stack_map.encoding)); + ASSERT_FALSE(stack_map.HasInlineInfo(encoding.stack_map.encoding)); } TEST(StackMapTest, InlineTest) { @@ -743,11 +743,11 @@ TEST(StackMapTest, InlineTest) { ASSERT_EQ(4, dex_registers0.GetConstant(1, 2, ci, encoding)); InlineInfo if0 = ci.GetInlineInfoOf(sm0, encoding); - ASSERT_EQ(2u, if0.GetDepth(encoding.inline_info_encoding)); - ASSERT_EQ(2u, if0.GetDexPcAtDepth(encoding.inline_info_encoding, 0)); - ASSERT_TRUE(if0.EncodesArtMethodAtDepth(encoding.inline_info_encoding, 0)); - ASSERT_EQ(3u, if0.GetDexPcAtDepth(encoding.inline_info_encoding, 1)); - ASSERT_TRUE(if0.EncodesArtMethodAtDepth(encoding.inline_info_encoding, 1)); + ASSERT_EQ(2u, if0.GetDepth(encoding.inline_info.encoding)); + ASSERT_EQ(2u, if0.GetDexPcAtDepth(encoding.inline_info.encoding, 0)); + ASSERT_TRUE(if0.EncodesArtMethodAtDepth(encoding.inline_info.encoding, 0)); + ASSERT_EQ(3u, if0.GetDexPcAtDepth(encoding.inline_info.encoding, 1)); + ASSERT_TRUE(if0.EncodesArtMethodAtDepth(encoding.inline_info.encoding, 1)); DexRegisterMap dex_registers1 = ci.GetDexRegisterMapAtDepth(0, if0, encoding, 1); ASSERT_EQ(8, dex_registers1.GetStackOffsetInBytes(0, 1, ci, encoding)); @@ -767,13 +767,13 @@ TEST(StackMapTest, InlineTest) { ASSERT_EQ(0, dex_registers0.GetConstant(1, 2, ci, encoding)); InlineInfo if1 = ci.GetInlineInfoOf(sm1, encoding); - ASSERT_EQ(3u, if1.GetDepth(encoding.inline_info_encoding)); - ASSERT_EQ(2u, if1.GetDexPcAtDepth(encoding.inline_info_encoding, 0)); - ASSERT_TRUE(if1.EncodesArtMethodAtDepth(encoding.inline_info_encoding, 0)); - ASSERT_EQ(3u, if1.GetDexPcAtDepth(encoding.inline_info_encoding, 1)); - ASSERT_TRUE(if1.EncodesArtMethodAtDepth(encoding.inline_info_encoding, 1)); - ASSERT_EQ(5u, if1.GetDexPcAtDepth(encoding.inline_info_encoding, 2)); - ASSERT_TRUE(if1.EncodesArtMethodAtDepth(encoding.inline_info_encoding, 2)); + ASSERT_EQ(3u, if1.GetDepth(encoding.inline_info.encoding)); + ASSERT_EQ(2u, if1.GetDexPcAtDepth(encoding.inline_info.encoding, 0)); + ASSERT_TRUE(if1.EncodesArtMethodAtDepth(encoding.inline_info.encoding, 0)); + ASSERT_EQ(3u, if1.GetDexPcAtDepth(encoding.inline_info.encoding, 1)); + ASSERT_TRUE(if1.EncodesArtMethodAtDepth(encoding.inline_info.encoding, 1)); + ASSERT_EQ(5u, if1.GetDexPcAtDepth(encoding.inline_info.encoding, 2)); + ASSERT_TRUE(if1.EncodesArtMethodAtDepth(encoding.inline_info.encoding, 2)); DexRegisterMap dex_registers1 = ci.GetDexRegisterMapAtDepth(0, if1, encoding, 1); ASSERT_EQ(12, dex_registers1.GetStackOffsetInBytes(0, 1, ci, encoding)); @@ -783,7 +783,7 @@ TEST(StackMapTest, InlineTest) { ASSERT_EQ(10, dex_registers2.GetConstant(1, 3, ci, encoding)); ASSERT_EQ(5, dex_registers2.GetMachineRegister(2, 3, ci, encoding)); - ASSERT_FALSE(if1.HasDexRegisterMapAtDepth(encoding.inline_info_encoding, 2)); + ASSERT_FALSE(if1.HasDexRegisterMapAtDepth(encoding.inline_info.encoding, 2)); } { @@ -793,7 +793,7 @@ TEST(StackMapTest, InlineTest) { DexRegisterMap dex_registers0 = ci.GetDexRegisterMapOf(sm2, encoding, 2); ASSERT_FALSE(dex_registers0.IsDexRegisterLive(0)); ASSERT_EQ(4, dex_registers0.GetConstant(1, 2, ci, encoding)); - ASSERT_FALSE(sm2.HasInlineInfo(encoding.stack_map_encoding)); + ASSERT_FALSE(sm2.HasInlineInfo(encoding.stack_map.encoding)); } { @@ -805,15 +805,15 @@ TEST(StackMapTest, InlineTest) { ASSERT_EQ(0, dex_registers0.GetConstant(1, 2, ci, encoding)); InlineInfo if2 = ci.GetInlineInfoOf(sm3, encoding); - ASSERT_EQ(3u, if2.GetDepth(encoding.inline_info_encoding)); - ASSERT_EQ(2u, if2.GetDexPcAtDepth(encoding.inline_info_encoding, 0)); - ASSERT_TRUE(if2.EncodesArtMethodAtDepth(encoding.inline_info_encoding, 0)); - ASSERT_EQ(5u, if2.GetDexPcAtDepth(encoding.inline_info_encoding, 1)); - ASSERT_TRUE(if2.EncodesArtMethodAtDepth(encoding.inline_info_encoding, 1)); - ASSERT_EQ(10u, if2.GetDexPcAtDepth(encoding.inline_info_encoding, 2)); - ASSERT_TRUE(if2.EncodesArtMethodAtDepth(encoding.inline_info_encoding, 2)); + ASSERT_EQ(3u, if2.GetDepth(encoding.inline_info.encoding)); + ASSERT_EQ(2u, if2.GetDexPcAtDepth(encoding.inline_info.encoding, 0)); + ASSERT_TRUE(if2.EncodesArtMethodAtDepth(encoding.inline_info.encoding, 0)); + ASSERT_EQ(5u, if2.GetDexPcAtDepth(encoding.inline_info.encoding, 1)); + ASSERT_TRUE(if2.EncodesArtMethodAtDepth(encoding.inline_info.encoding, 1)); + ASSERT_EQ(10u, if2.GetDexPcAtDepth(encoding.inline_info.encoding, 2)); + ASSERT_TRUE(if2.EncodesArtMethodAtDepth(encoding.inline_info.encoding, 2)); - ASSERT_FALSE(if2.HasDexRegisterMapAtDepth(encoding.inline_info_encoding, 0)); + ASSERT_FALSE(if2.HasDexRegisterMapAtDepth(encoding.inline_info.encoding, 0)); DexRegisterMap dex_registers1 = ci.GetDexRegisterMapAtDepth(1, if2, encoding, 1); ASSERT_EQ(2, dex_registers1.GetMachineRegister(0, 1, ci, encoding)); @@ -865,8 +865,8 @@ TEST(StackMapTest, TestDeduplicateStackMask) { StackMap stack_map1 = code_info.GetStackMapForNativePcOffset(4, encoding); StackMap stack_map2 = code_info.GetStackMapForNativePcOffset(8, encoding); - EXPECT_EQ(stack_map1.GetStackMaskIndex(encoding.stack_map_encoding), - stack_map2.GetStackMaskIndex(encoding.stack_map_encoding)); + EXPECT_EQ(stack_map1.GetStackMaskIndex(encoding.stack_map.encoding), + stack_map2.GetStackMaskIndex(encoding.stack_map.encoding)); } } // namespace art |