diff options
212 files changed, 5745 insertions, 2422 deletions
diff --git a/build/Android.bp b/build/Android.bp index c54f436b35..ed6de3546f 100644 --- a/build/Android.bp +++ b/build/Android.bp @@ -59,10 +59,8 @@ art_global_defaults { "-Wunreachable-code-break", "-Wunreachable-code-return", - // Bug: http://b/29823425 Disable -Wconstant-conversion and - // -Wundefined-var-template for Clang update to r271374 + // Bug: http://b/29823425 Disable -Wconstant-conversion for Clang update to r271374 "-Wno-constant-conversion", - "-Wno-undefined-var-template", // Enable thread annotations for std::mutex, etc. "-D_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS", @@ -145,6 +143,15 @@ art_global_defaults { tidy_checks: [ "-google-default-arguments", + // We have local stores that are only used for debug checks. + "-clang-analyzer-deadcode.DeadStores", + // We are OK with some static globals and that they can, in theory, throw. + "-cert-err58-cpp", + // We have lots of C-style variadic functions, and are OK with them. JNI ensures + // that working around this warning would be extra-painful. + "-cert-dcl50-cpp", + // No exceptions. + "-misc-noexcept-move-constructor", ], } diff --git a/cmdline/cmdline_parser_test.cc b/cmdline/cmdline_parser_test.cc index 9f12f64a31..07639e8a7d 100644 --- a/cmdline/cmdline_parser_test.cc +++ b/cmdline/cmdline_parser_test.cc @@ -35,7 +35,7 @@ namespace art { return lhs.enabled_ == rhs.enabled_ && lhs.min_save_period_ms_ == rhs.min_save_period_ms_ && lhs.save_resolved_classes_delay_ms_ == rhs.save_resolved_classes_delay_ms_ && - lhs.startup_method_samples_ == rhs.startup_method_samples_ && + lhs.hot_startup_method_samples_ == rhs.hot_startup_method_samples_ && lhs.min_methods_to_save_ == rhs.min_methods_to_save_ && lhs.min_classes_to_save_ == rhs.min_classes_to_save_ && lhs.min_notification_before_wake_ == rhs.min_notification_before_wake_ && @@ -490,7 +490,7 @@ TEST_F(CmdlineParserTest, ProfileSaverOptions) { "-Xjitsaveprofilinginfo " "-Xps-min-save-period-ms:1 " "-Xps-save-resolved-classes-delay-ms:2 " - "-Xps-startup-method-samples:3 " + "-Xps-hot-startup-method-samples:3 " "-Xps-min-methods-to-save:4 " "-Xps-min-classes-to-save:5 " "-Xps-min-notification-before-wake:6 " diff --git a/cmdline/cmdline_types.h b/cmdline/cmdline_types.h index 0d2aed8ad1..185a0e403e 100644 --- a/cmdline/cmdline_types.h +++ b/cmdline/cmdline_types.h @@ -727,10 +727,10 @@ struct CmdlineType<ProfileSaverOptions> : CmdlineTypeParser<ProfileSaverOptions> &ProfileSaverOptions::save_resolved_classes_delay_ms_, type_parser.Parse(suffix)); } - if (android::base::StartsWith(option, "startup-method-samples:")) { + if (android::base::StartsWith(option, "hot-startup-method-samples:")) { CmdlineType<unsigned int> type_parser; return ParseInto(existing, - &ProfileSaverOptions::startup_method_samples_, + &ProfileSaverOptions::hot_startup_method_samples_, type_parser.Parse(suffix)); } if (android::base::StartsWith(option, "min-methods-to-save:")) { diff --git a/compiler/Android.bp b/compiler/Android.bp index 307a42cbba..a1269dcaf9 100644 --- a/compiler/Android.bp +++ b/compiler/Android.bp @@ -110,7 +110,6 @@ art_cc_defaults { "optimizing/code_generator_vector_arm.cc", "optimizing/code_generator_arm_vixl.cc", "optimizing/code_generator_vector_arm_vixl.cc", - "optimizing/dex_cache_array_fixups_arm.cc", "optimizing/instruction_simplifier_arm.cc", "optimizing/instruction_simplifier_shared.cc", "optimizing/intrinsics_arm.cc", @@ -145,7 +144,6 @@ art_cc_defaults { "linker/mips/relative_patcher_mips.cc", "optimizing/code_generator_mips.cc", "optimizing/code_generator_vector_mips.cc", - "optimizing/dex_cache_array_fixups_mips.cc", "optimizing/intrinsics_mips.cc", "optimizing/pc_relative_fixups_mips.cc", "utils/mips/assembler_mips.cc", @@ -342,6 +340,7 @@ art_cc_test { "image_test.cc", "image_write_read_test.cc", "jni/jni_compiler_test.cc", + "linker/method_bss_mapping_encoder_test.cc", "linker/multi_oat_relative_patcher_test.cc", "linker/output_stream_test.cc", "oat_test.cc", diff --git a/compiler/common_compiler_test.cc b/compiler/common_compiler_test.cc index a1ee68faeb..3683695a1b 100644 --- a/compiler/common_compiler_test.cc +++ b/compiler/common_compiler_test.cc @@ -19,6 +19,7 @@ #include "arch/instruction_set_features.h" #include "art_field-inl.h" #include "art_method-inl.h" +#include "base/callee_save_type.h" #include "base/enums.h" #include "class_linker.h" #include "compiled_method.h" @@ -166,8 +167,8 @@ void CommonCompilerTest::SetUp() { instruction_set_features_ = InstructionSetFeatures::FromCppDefines(); runtime_->SetInstructionSet(instruction_set); - for (int i = 0; i < Runtime::kLastCalleeSaveType; i++) { - Runtime::CalleeSaveType type = Runtime::CalleeSaveType(i); + for (uint32_t i = 0; i < static_cast<uint32_t>(CalleeSaveType::kLastCalleeSaveType); ++i) { + CalleeSaveType type = CalleeSaveType(i); if (!runtime_->HasCalleeSaveMethod(type)) { runtime_->SetCalleeSaveMethod(runtime_->CreateCalleeSaveMethod(), type); } diff --git a/compiler/compiled_method.h b/compiler/compiled_method.h index 0ca23a5c50..761e9e19a8 100644 --- a/compiler/compiled_method.h +++ b/compiler/compiled_method.h @@ -120,13 +120,13 @@ class LinkerPatch { // patch_type_ as an uintN_t and do explicit static_cast<>s. enum class Type : uint8_t { kMethodRelative, // NOTE: Actual patching is instruction_set-dependent. + kMethodBssEntry, // NOTE: Actual patching is instruction_set-dependent. kCall, kCallRelative, // NOTE: Actual patching is instruction_set-dependent. kTypeRelative, // NOTE: Actual patching is instruction_set-dependent. kTypeBssEntry, // NOTE: Actual patching is instruction_set-dependent. kStringRelative, // NOTE: Actual patching is instruction_set-dependent. kStringBssEntry, // NOTE: Actual patching is instruction_set-dependent. - kDexCacheArray, // NOTE: Actual patching is instruction_set-dependent. kBakerReadBarrierBranch, // NOTE: Actual patching is instruction_set-dependent. }; @@ -140,6 +140,16 @@ class LinkerPatch { return patch; } + static LinkerPatch MethodBssEntryPatch(size_t literal_offset, + const DexFile* target_dex_file, + uint32_t pc_insn_offset, + uint32_t target_method_idx) { + LinkerPatch patch(literal_offset, Type::kMethodBssEntry, target_dex_file); + patch.method_idx_ = target_method_idx; + patch.pc_insn_offset_ = pc_insn_offset; + return patch; + } + static LinkerPatch CodePatch(size_t literal_offset, const DexFile* target_dex_file, uint32_t target_method_idx) { @@ -196,16 +206,6 @@ class LinkerPatch { return patch; } - static LinkerPatch DexCacheArrayPatch(size_t literal_offset, - const DexFile* target_dex_file, - uint32_t pc_insn_offset, - uint32_t element_offset) { - LinkerPatch patch(literal_offset, Type::kDexCacheArray, target_dex_file); - patch.pc_insn_offset_ = pc_insn_offset; - patch.element_offset_ = element_offset; - return patch; - } - static LinkerPatch BakerReadBarrierBranchPatch(size_t literal_offset, uint32_t custom_value1 = 0u, uint32_t custom_value2 = 0u) { @@ -229,12 +229,12 @@ class LinkerPatch { bool IsPcRelative() const { switch (GetType()) { case Type::kMethodRelative: + case Type::kMethodBssEntry: case Type::kCallRelative: case Type::kTypeRelative: case Type::kTypeBssEntry: case Type::kStringRelative: case Type::kStringBssEntry: - case Type::kDexCacheArray: case Type::kBakerReadBarrierBranch: return true; default: @@ -244,6 +244,7 @@ class LinkerPatch { MethodReference TargetMethod() const { DCHECK(patch_type_ == Type::kMethodRelative || + patch_type_ == Type::kMethodBssEntry || patch_type_ == Type::kCall || patch_type_ == Type::kCallRelative); return MethodReference(target_dex_file_, method_idx_); @@ -273,23 +274,13 @@ class LinkerPatch { return dex::StringIndex(string_idx_); } - const DexFile* TargetDexCacheDexFile() const { - DCHECK(patch_type_ == Type::kDexCacheArray); - return target_dex_file_; - } - - size_t TargetDexCacheElementOffset() const { - DCHECK(patch_type_ == Type::kDexCacheArray); - return element_offset_; - } - uint32_t PcInsnOffset() const { DCHECK(patch_type_ == Type::kMethodRelative || + patch_type_ == Type::kMethodBssEntry || patch_type_ == Type::kTypeRelative || patch_type_ == Type::kTypeBssEntry || patch_type_ == Type::kStringRelative || - patch_type_ == Type::kStringBssEntry || - patch_type_ == Type::kDexCacheArray); + patch_type_ == Type::kStringBssEntry); return pc_insn_offset_; } @@ -324,12 +315,10 @@ class LinkerPatch { uint32_t method_idx_; // Method index for Call/Method patches. uint32_t type_idx_; // Type index for Type patches. uint32_t string_idx_; // String index for String patches. - uint32_t element_offset_; // Element offset in the dex cache arrays. uint32_t baker_custom_value1_; static_assert(sizeof(method_idx_) == sizeof(cmp1_), "needed by relational operators"); static_assert(sizeof(type_idx_) == sizeof(cmp1_), "needed by relational operators"); static_assert(sizeof(string_idx_) == sizeof(cmp1_), "needed by relational operators"); - static_assert(sizeof(element_offset_) == sizeof(cmp1_), "needed by relational operators"); static_assert(sizeof(baker_custom_value1_) == sizeof(cmp1_), "needed by relational operators"); }; union { diff --git a/compiler/compiled_method_test.cc b/compiler/compiled_method_test.cc index 72b2282ade..f4a72cf2cc 100644 --- a/compiler/compiled_method_test.cc +++ b/compiler/compiled_method_test.cc @@ -58,6 +58,14 @@ TEST(CompiledMethod, LinkerPatchOperators) { LinkerPatch::RelativeMethodPatch(16u, dex_file2, 3001u, 1000u), LinkerPatch::RelativeMethodPatch(16u, dex_file2, 3000u, 1001u), LinkerPatch::RelativeMethodPatch(16u, dex_file2, 3001u, 1001u), + LinkerPatch::MethodBssEntryPatch(16u, dex_file1, 3000u, 1000u), + LinkerPatch::MethodBssEntryPatch(16u, dex_file1, 3001u, 1000u), + LinkerPatch::MethodBssEntryPatch(16u, dex_file1, 3000u, 1001u), + LinkerPatch::MethodBssEntryPatch(16u, dex_file1, 3001u, 1001u), + LinkerPatch::MethodBssEntryPatch(16u, dex_file2, 3000u, 1000u), + LinkerPatch::MethodBssEntryPatch(16u, dex_file2, 3001u, 1000u), + LinkerPatch::MethodBssEntryPatch(16u, dex_file2, 3000u, 1001u), + LinkerPatch::MethodBssEntryPatch(16u, dex_file2, 3001u, 1001u), LinkerPatch::CodePatch(16u, dex_file1, 1000u), LinkerPatch::CodePatch(16u, dex_file1, 1001u), LinkerPatch::CodePatch(16u, dex_file2, 1000u), @@ -98,14 +106,6 @@ TEST(CompiledMethod, LinkerPatchOperators) { LinkerPatch::StringBssEntryPatch(16u, dex_file2, 3001u, 1000u), LinkerPatch::StringBssEntryPatch(16u, dex_file2, 3000u, 1001u), LinkerPatch::StringBssEntryPatch(16u, dex_file2, 3001u, 1001u), - LinkerPatch::DexCacheArrayPatch(16u, dex_file1, 3000u, 2000u), - LinkerPatch::DexCacheArrayPatch(16u, dex_file1, 3001u, 2000u), - LinkerPatch::DexCacheArrayPatch(16u, dex_file1, 3000u, 2001u), - LinkerPatch::DexCacheArrayPatch(16u, dex_file1, 3001u, 2001u), - LinkerPatch::DexCacheArrayPatch(16u, dex_file2, 3000u, 2000u), - LinkerPatch::DexCacheArrayPatch(16u, dex_file2, 3001u, 2000u), - LinkerPatch::DexCacheArrayPatch(16u, dex_file2, 3000u, 2001u), - LinkerPatch::DexCacheArrayPatch(16u, dex_file2, 3001u, 2001u), LinkerPatch::BakerReadBarrierBranchPatch(16u, 0u, 0u), LinkerPatch::BakerReadBarrierBranchPatch(16u, 0u, 1u), LinkerPatch::BakerReadBarrierBranchPatch(16u, 1u, 0u), @@ -119,6 +119,14 @@ TEST(CompiledMethod, LinkerPatchOperators) { LinkerPatch::RelativeMethodPatch(32u, dex_file2, 3001u, 1000u), LinkerPatch::RelativeMethodPatch(32u, dex_file2, 3000u, 1001u), LinkerPatch::RelativeMethodPatch(32u, dex_file2, 3001u, 1001u), + LinkerPatch::MethodBssEntryPatch(32u, dex_file1, 3000u, 1000u), + LinkerPatch::MethodBssEntryPatch(32u, dex_file1, 3001u, 1000u), + LinkerPatch::MethodBssEntryPatch(32u, dex_file1, 3000u, 1001u), + LinkerPatch::MethodBssEntryPatch(32u, dex_file1, 3001u, 1001u), + LinkerPatch::MethodBssEntryPatch(32u, dex_file2, 3000u, 1000u), + LinkerPatch::MethodBssEntryPatch(32u, dex_file2, 3001u, 1000u), + LinkerPatch::MethodBssEntryPatch(32u, dex_file2, 3000u, 1001u), + LinkerPatch::MethodBssEntryPatch(32u, dex_file2, 3001u, 1001u), LinkerPatch::CodePatch(32u, dex_file1, 1000u), LinkerPatch::CodePatch(32u, dex_file1, 1001u), LinkerPatch::CodePatch(32u, dex_file2, 1000u), @@ -159,14 +167,6 @@ TEST(CompiledMethod, LinkerPatchOperators) { LinkerPatch::StringBssEntryPatch(32u, dex_file2, 3001u, 1000u), LinkerPatch::StringBssEntryPatch(32u, dex_file2, 3000u, 1001u), LinkerPatch::StringBssEntryPatch(32u, dex_file2, 3001u, 1001u), - LinkerPatch::DexCacheArrayPatch(32u, dex_file1, 3000u, 2000u), - LinkerPatch::DexCacheArrayPatch(32u, dex_file1, 3001u, 2000u), - LinkerPatch::DexCacheArrayPatch(32u, dex_file1, 3000u, 2001u), - LinkerPatch::DexCacheArrayPatch(32u, dex_file1, 3001u, 2001u), - LinkerPatch::DexCacheArrayPatch(32u, dex_file2, 3000u, 2000u), - LinkerPatch::DexCacheArrayPatch(32u, dex_file2, 3001u, 2000u), - LinkerPatch::DexCacheArrayPatch(32u, dex_file2, 3000u, 2001u), - LinkerPatch::DexCacheArrayPatch(32u, dex_file2, 3001u, 2001u), LinkerPatch::BakerReadBarrierBranchPatch(32u, 0u, 0u), LinkerPatch::BakerReadBarrierBranchPatch(32u, 0u, 1u), LinkerPatch::BakerReadBarrierBranchPatch(32u, 1u, 0u), diff --git a/compiler/dex/verification_results.cc b/compiler/dex/verification_results.cc index b87cb61ed6..04ceca0513 100644 --- a/compiler/dex/verification_results.cc +++ b/compiler/dex/verification_results.cc @@ -110,12 +110,12 @@ void VerificationResults::CreateVerifiedMethodFor(MethodReference ref) { // This method should only be called for classes verified at compile time, // which have no verifier error, nor has methods that we know will throw // at runtime. - atomic_verified_methods_.Insert( - ref, - /*expected*/ nullptr, - new VerifiedMethod(/* encountered_error_types */ 0, /* has_runtime_throw */ false)); - // We don't check the result of `Insert` as we could insert twice for the same - // MethodReference in the presence of duplicate methods. + std::unique_ptr<VerifiedMethod> verified_method = std::make_unique<VerifiedMethod>( + /* encountered_error_types */ 0, /* has_runtime_throw */ false); + if (atomic_verified_methods_.Insert(ref, /*expected*/ nullptr, verified_method.get()) == + AtomicMap::InsertResult::kInsertResultSuccess) { + verified_method.release(); + } } void VerificationResults::AddRejectedClass(ClassReference ref) { diff --git a/compiler/driver/compiler_driver.cc b/compiler/driver/compiler_driver.cc index 93f678c64a..0d0769fe98 100644 --- a/compiler/driver/compiler_driver.cc +++ b/compiler/driver/compiler_driver.cc @@ -1000,8 +1000,9 @@ bool CompilerDriver::ShouldCompileBasedOnProfile(const MethodReference& method_r if (profile_compilation_info_ == nullptr) { return false; } - // TODO: Revisit compiling all startup methods. b/36457259 - bool result = profile_compilation_info_->IsStartupOrHotMethod(method_ref); + // Compile only hot methods, it is the profile saver's job to decide what startup methods to mark + // as hot. + bool result = profile_compilation_info_->ContainsHotMethod(method_ref); if (kDebugProfileGuidedCompilation) { LOG(INFO) << "[ProfileGuidedCompilation] " @@ -2292,18 +2293,9 @@ class InitializeClassVisitor : public CompilationVisitor { ObjectLock<mirror::Class> lock(soa.Self(), h_klass); // Attempt to initialize allowing initialization of parent classes but still not static // fields. - bool is_superclass_initialized = true; - if (!manager_->GetCompiler()->GetCompilerOptions().IsAppImage()) { - // If not an app image case, the compiler won't initialize too much things and do a fast - // fail, don't check dependencies. + bool is_superclass_initialized = InitializeDependencies(klass, class_loader, soa.Self()); + if (is_superclass_initialized) { manager_->GetClassLinker()->EnsureInitialized(soa.Self(), klass, false, true); - } else { - // For app images, do the initialization recursively and resolve types encountered to make - // sure the compiler runs without error. - is_superclass_initialized = InitializeDependencies(klass, class_loader, soa.Self()); - if (is_superclass_initialized) { - manager_->GetClassLinker()->EnsureInitialized(soa.Self(), klass, false, true); - } } old_status = klass->GetStatus(); // If superclass cannot be initialized, no need to proceed. @@ -2435,9 +2427,33 @@ class InitializeClassVisitor : public CompilationVisitor { } } + bool NoPotentialInternStrings(Handle<mirror::Class> klass, + Handle<mirror::ClassLoader>* class_loader) + REQUIRES_SHARED(Locks::mutator_lock_) { + StackHandleScope<1> hs(Thread::Current()); + Handle<mirror::DexCache> h_dex_cache = hs.NewHandle(klass->GetDexCache()); + const DexFile* dex_file = h_dex_cache->GetDexFile(); + const DexFile::ClassDef* class_def = klass->GetClassDef(); + annotations::RuntimeEncodedStaticFieldValueIterator value_it(*dex_file, + &h_dex_cache, + class_loader, + manager_->GetClassLinker(), + *class_def); + + const auto jString = annotations::RuntimeEncodedStaticFieldValueIterator::kString; + for ( ; value_it.HasNext(); value_it.Next()) { + if (value_it.GetValueType() == jString) { + // We don't want cache the static encoded strings which is a potential intern. + return false; + } + } + + return true; + } + bool ResolveTypesOfMethods(Thread* self, ArtMethod* m) REQUIRES_SHARED(Locks::mutator_lock_) { - auto rtn_type = m->GetReturnType(true); + auto rtn_type = m->GetReturnType(true); // return value is discarded because resolve will be done internally. if (rtn_type == nullptr) { self->ClearException(); return false; @@ -2548,8 +2564,9 @@ class InitializeClassVisitor : public CompilationVisitor { ObjPtr<mirror::Class> super_class = klass->GetSuperClass(); StackHandleScope<1> hs(self); Handle<mirror::Class> handle_scope_super(hs.NewHandle(super_class)); - if (!NoClinitInDependency(handle_scope_super, self, class_loader)) + if (!NoClinitInDependency(handle_scope_super, self, class_loader)) { return false; + } } uint32_t num_if = klass->NumDirectInterfaces(); @@ -2558,11 +2575,12 @@ class InitializeClassVisitor : public CompilationVisitor { interface = mirror::Class::GetDirectInterface(self, klass.Get(), i); StackHandleScope<1> hs(self); Handle<mirror::Class> handle_interface(hs.NewHandle(interface)); - if (!NoClinitInDependency(handle_interface, self, class_loader)) + if (!NoClinitInDependency(handle_interface, self, class_loader)) { return false; + } } - return true; + return NoPotentialInternStrings(klass, class_loader); } const ParallelCompilationManager* const manager_; diff --git a/compiler/elf_builder.h b/compiler/elf_builder.h index 7c02384ff2..2ef9fa1ccb 100644 --- a/compiler/elf_builder.h +++ b/compiler/elf_builder.h @@ -670,6 +670,7 @@ class ElfBuilder FINAL { Elf_Word rodata_size, Elf_Word text_size, Elf_Word bss_size, + Elf_Word bss_methods_offset, Elf_Word bss_roots_offset) { std::string soname(elf_file_path); size_t directory_separator_pos = soname.rfind('/'); @@ -715,9 +716,18 @@ class ElfBuilder FINAL { Elf_Word bss_index = rodata_index + 1u + (text_size != 0 ? 1u : 0u); Elf_Word oatbss = dynstr_.Add("oatbss"); dynsym_.Add(oatbss, bss_index, bss_address, bss_roots_offset, STB_GLOBAL, STT_OBJECT); + DCHECK_LE(bss_methods_offset, bss_roots_offset); + DCHECK_LE(bss_roots_offset, bss_size); + // Add a symbol marking the start of the methods part of the .bss, if not empty. + if (bss_methods_offset != bss_roots_offset) { + Elf_Word bss_methods_address = bss_address + bss_methods_offset; + Elf_Word bss_methods_size = bss_roots_offset - bss_methods_offset; + Elf_Word oatbssroots = dynstr_.Add("oatbssmethods"); + dynsym_.Add( + oatbssroots, bss_index, bss_methods_address, bss_methods_size, STB_GLOBAL, STT_OBJECT); + } // Add a symbol marking the start of the GC roots part of the .bss, if not empty. if (bss_roots_offset != bss_size) { - DCHECK_LT(bss_roots_offset, bss_size); Elf_Word bss_roots_address = bss_address + bss_roots_offset; Elf_Word bss_roots_size = bss_size - bss_roots_offset; Elf_Word oatbssroots = dynstr_.Add("oatbssroots"); diff --git a/compiler/elf_writer.h b/compiler/elf_writer.h index 7baae527ff..a8a5bc32b7 100644 --- a/compiler/elf_writer.h +++ b/compiler/elf_writer.h @@ -55,6 +55,7 @@ class ElfWriter { virtual void PrepareDynamicSection(size_t rodata_size, size_t text_size, size_t bss_size, + size_t bss_methods_offset, size_t bss_roots_offset) = 0; virtual void PrepareDebugInfo(const ArrayRef<const debug::MethodDebugInfo>& method_infos) = 0; virtual OutputStream* StartRoData() = 0; diff --git a/compiler/elf_writer_quick.cc b/compiler/elf_writer_quick.cc index 738f5a2b29..5d6dd2e1d7 100644 --- a/compiler/elf_writer_quick.cc +++ b/compiler/elf_writer_quick.cc @@ -22,7 +22,6 @@ #include "base/casts.h" #include "base/logging.h" -#include "base/stl_util.h" #include "compiled_method.h" #include "debug/elf_debug_writer.h" #include "debug/method_debug_info.h" @@ -80,7 +79,7 @@ class DebugInfoTask : public Task { const InstructionSetFeatures* instruction_set_features_; size_t rodata_section_size_; size_t text_section_size_; - const ArrayRef<const debug::MethodDebugInfo>& method_infos_; + const ArrayRef<const debug::MethodDebugInfo> method_infos_; std::vector<uint8_t> result_; }; @@ -97,6 +96,7 @@ class ElfWriterQuick FINAL : public ElfWriter { void PrepareDynamicSection(size_t rodata_size, size_t text_size, size_t bss_size, + size_t bss_methods_offset, size_t bss_roots_offset) OVERRIDE; void PrepareDebugInfo(const ArrayRef<const debug::MethodDebugInfo>& method_infos) OVERRIDE; OutputStream* StartRoData() OVERRIDE; @@ -136,15 +136,15 @@ std::unique_ptr<ElfWriter> CreateElfWriterQuick(InstructionSet instruction_set, const CompilerOptions* compiler_options, File* elf_file) { if (Is64BitInstructionSet(instruction_set)) { - return MakeUnique<ElfWriterQuick<ElfTypes64>>(instruction_set, - features, - compiler_options, - elf_file); + return std::make_unique<ElfWriterQuick<ElfTypes64>>(instruction_set, + features, + compiler_options, + elf_file); } else { - return MakeUnique<ElfWriterQuick<ElfTypes32>>(instruction_set, - features, - compiler_options, - elf_file); + return std::make_unique<ElfWriterQuick<ElfTypes32>>(instruction_set, + features, + compiler_options, + elf_file); } } @@ -160,7 +160,8 @@ ElfWriterQuick<ElfTypes>::ElfWriterQuick(InstructionSet instruction_set, rodata_size_(0u), text_size_(0u), bss_size_(0u), - output_stream_(MakeUnique<BufferedOutputStream>(MakeUnique<FileOutputStream>(elf_file))), + output_stream_( + std::make_unique<BufferedOutputStream>(std::make_unique<FileOutputStream>(elf_file))), builder_(new ElfBuilder<ElfTypes>(instruction_set, features, output_stream_.get())) {} template <typename ElfTypes> @@ -178,6 +179,7 @@ template <typename ElfTypes> void ElfWriterQuick<ElfTypes>::PrepareDynamicSection(size_t rodata_size, size_t text_size, size_t bss_size, + size_t bss_methods_offset, size_t bss_roots_offset) { DCHECK_EQ(rodata_size_, 0u); rodata_size_ = rodata_size; @@ -189,6 +191,7 @@ void ElfWriterQuick<ElfTypes>::PrepareDynamicSection(size_t rodata_size, rodata_size_, text_size_, bss_size_, + bss_methods_offset, bss_roots_offset); } diff --git a/compiler/exception_test.cc b/compiler/exception_test.cc index dc880b089e..b4777df0df 100644 --- a/compiler/exception_test.cc +++ b/compiler/exception_test.cc @@ -17,6 +17,7 @@ #include <memory> #include "base/arena_allocator.h" +#include "base/callee_save_type.h" #include "base/enums.h" #include "class_linker.h" #include "common_runtime_test.h" @@ -170,7 +171,7 @@ TEST_F(ExceptionTest, StackTraceElement) { Runtime* r = Runtime::Current(); r->SetInstructionSet(kRuntimeISA); ArtMethod* save_method = r->CreateCalleeSaveMethod(); - r->SetCalleeSaveMethod(save_method, Runtime::kSaveAllCalleeSaves); + r->SetCalleeSaveMethod(save_method, CalleeSaveType::kSaveAllCalleeSaves); QuickMethodFrameInfo frame_info = r->GetRuntimeMethodFrameInfo(save_method); ASSERT_EQ(kStackAlignment, 16U); diff --git a/compiler/image_test.h b/compiler/image_test.h index 2f15ff4815..3d89757d51 100644 --- a/compiler/image_test.h +++ b/compiler/image_test.h @@ -290,9 +290,9 @@ inline void CompilationHelper::Compile(CompilerDriver* driver, if (kIsVdexEnabled) { for (size_t i = 0, size = vdex_files.size(); i != size; ++i) { - std::unique_ptr<BufferedOutputStream> vdex_out( - MakeUnique<BufferedOutputStream>( - MakeUnique<FileOutputStream>(vdex_files[i].GetFile()))); + std::unique_ptr<BufferedOutputStream> vdex_out = + std::make_unique<BufferedOutputStream>( + std::make_unique<FileOutputStream>(vdex_files[i].GetFile())); oat_writers[i]->WriteVerifierDeps(vdex_out.get(), nullptr); oat_writers[i]->WriteChecksumsAndVdexHeader(vdex_out.get()); } @@ -311,6 +311,7 @@ inline void CompilationHelper::Compile(CompilerDriver* driver, elf_writer->PrepareDynamicSection(rodata_size, text_size, oat_writer->GetBssSize(), + oat_writer->GetBssMethodsOffset(), oat_writer->GetBssRootsOffset()); writer->UpdateOatFileLayout(i, diff --git a/compiler/image_writer.cc b/compiler/image_writer.cc index 4d6db4745f..406892e499 100644 --- a/compiler/image_writer.cc +++ b/compiler/image_writer.cc @@ -27,6 +27,8 @@ #include "art_field-inl.h" #include "art_method-inl.h" +#include "base/callee_save_type.h" +#include "base/enums.h" #include "base/logging.h" #include "base/unix_file/fd_file.h" #include "class_linker-inl.h" @@ -47,7 +49,6 @@ #include "globals.h" #include "image.h" #include "imt_conflict_table.h" -#include "intern_table.h" #include "jni_internal.h" #include "linear_alloc.h" #include "lock_word.h" @@ -1572,13 +1573,13 @@ void ImageWriter::CalculateNewObjectOffsets() { image_methods_[ImageHeader::kImtConflictMethod] = runtime->GetImtConflictMethod(); image_methods_[ImageHeader::kImtUnimplementedMethod] = runtime->GetImtUnimplementedMethod(); image_methods_[ImageHeader::kSaveAllCalleeSavesMethod] = - runtime->GetCalleeSaveMethod(Runtime::kSaveAllCalleeSaves); + runtime->GetCalleeSaveMethod(CalleeSaveType::kSaveAllCalleeSaves); image_methods_[ImageHeader::kSaveRefsOnlyMethod] = - runtime->GetCalleeSaveMethod(Runtime::kSaveRefsOnly); + runtime->GetCalleeSaveMethod(CalleeSaveType::kSaveRefsOnly); image_methods_[ImageHeader::kSaveRefsAndArgsMethod] = - runtime->GetCalleeSaveMethod(Runtime::kSaveRefsAndArgs); + runtime->GetCalleeSaveMethod(CalleeSaveType::kSaveRefsAndArgs); image_methods_[ImageHeader::kSaveEverythingMethod] = - runtime->GetCalleeSaveMethod(Runtime::kSaveEverything); + runtime->GetCalleeSaveMethod(CalleeSaveType::kSaveEverything); // Visit image methods first to have the main runtime methods in the first image. for (auto* m : image_methods_) { CHECK(m != nullptr); @@ -2482,8 +2483,8 @@ void ImageWriter::CopyAndFixupMethod(ArtMethod* orig, GetOatAddress(kOatAddressQuickResolutionTrampoline), target_ptr_size_); } else { bool found_one = false; - for (size_t i = 0; i < static_cast<size_t>(Runtime::kLastCalleeSaveType); ++i) { - auto idx = static_cast<Runtime::CalleeSaveType>(i); + for (size_t i = 0; i < static_cast<size_t>(CalleeSaveType::kLastCalleeSaveType); ++i) { + auto idx = static_cast<CalleeSaveType>(i); if (runtime->HasCalleeSaveMethod(idx) && runtime->GetCalleeSaveMethod(idx) == orig) { found_one = true; break; diff --git a/compiler/image_writer.h b/compiler/image_writer.h index 2283b39773..5e2db7d8f7 100644 --- a/compiler/image_writer.h +++ b/compiler/image_writer.h @@ -36,6 +36,7 @@ #include "class_table.h" #include "driver/compiler_driver.h" #include "image.h" +#include "intern_table.h" #include "lock_word.h" #include "mem_map.h" #include "mirror/dex_cache.h" @@ -106,19 +107,6 @@ class ImageWriter FINAL { ArtMethod* GetImageMethodAddress(ArtMethod* method) REQUIRES_SHARED(Locks::mutator_lock_); - template <typename PtrType> - PtrType GetDexCacheArrayElementImageAddress(const DexFile* dex_file, uint32_t offset) - const REQUIRES_SHARED(Locks::mutator_lock_) { - auto oat_it = dex_file_oat_index_map_.find(dex_file); - DCHECK(oat_it != dex_file_oat_index_map_.end()); - const ImageInfo& image_info = GetImageInfo(oat_it->second); - auto it = image_info.dex_cache_array_starts_.find(dex_file); - DCHECK(it != image_info.dex_cache_array_starts_.end()); - return reinterpret_cast<PtrType>( - image_info.image_begin_ + image_info.bin_slot_offsets_[kBinDexCacheArray] + - it->second + offset); - } - size_t GetOatFileOffset(size_t oat_index) const { return GetImageInfo(oat_index).oat_offset_; } diff --git a/compiler/jit/jit_compiler.cc b/compiler/jit/jit_compiler.cc index fed1f48d65..66135414f7 100644 --- a/compiler/jit/jit_compiler.cc +++ b/compiler/jit/jit_compiler.cc @@ -189,18 +189,12 @@ JitCompiler::~JitCompiler() { bool JitCompiler::CompileMethod(Thread* self, ArtMethod* method, bool osr) { DCHECK(!method->IsProxyMethod()); + DCHECK(method->GetDeclaringClass()->IsResolved()); + TimingLogger logger("JIT compiler timing logger", true, VLOG_IS_ON(jit)); - StackHandleScope<2> hs(self); self->AssertNoPendingException(); Runtime* runtime = Runtime::Current(); - // Ensure the class is initialized. - Handle<mirror::Class> h_class(hs.NewHandle(method->GetDeclaringClass())); - if (!runtime->GetClassLinker()->EnsureInitialized(self, h_class, true, true)) { - VLOG(jit) << "JIT failed to initialize " << method->PrettyMethod(); - return false; - } - // Do the compilation. bool success = false; { diff --git a/compiler/linker/arm/relative_patcher_arm_base.cc b/compiler/linker/arm/relative_patcher_arm_base.cc index c1ac230d43..18ff1c9bb6 100644 --- a/compiler/linker/arm/relative_patcher_arm_base.cc +++ b/compiler/linker/arm/relative_patcher_arm_base.cc @@ -16,6 +16,7 @@ #include "linker/arm/relative_patcher_arm_base.h" +#include "base/stl_util.h" #include "compiled_method.h" #include "linker/output_stream.h" #include "oat.h" diff --git a/compiler/linker/arm64/relative_patcher_arm64.cc b/compiler/linker/arm64/relative_patcher_arm64.cc index 117684a66b..bc21607c5b 100644 --- a/compiler/linker/arm64/relative_patcher_arm64.cc +++ b/compiler/linker/arm64/relative_patcher_arm64.cc @@ -59,11 +59,11 @@ inline bool IsAdrpPatch(const LinkerPatch& patch) { case LinkerPatch::Type::kBakerReadBarrierBranch: return false; case LinkerPatch::Type::kMethodRelative: + case LinkerPatch::Type::kMethodBssEntry: case LinkerPatch::Type::kTypeRelative: case LinkerPatch::Type::kTypeBssEntry: case LinkerPatch::Type::kStringRelative: case LinkerPatch::Type::kStringBssEntry: - case LinkerPatch::Type::kDexCacheArray: return patch.LiteralOffset() == patch.PcInsnOffset(); } } @@ -251,20 +251,20 @@ void Arm64RelativePatcher::PatchPcRelativeReference(std::vector<uint8_t>* code, // ADD immediate, 64-bit with imm12 == 0 (unset). if (!kEmitCompilerReadBarrier) { DCHECK(patch.GetType() == LinkerPatch::Type::kMethodRelative || - patch.GetType() == LinkerPatch::Type::kStringRelative || - patch.GetType() == LinkerPatch::Type::kTypeRelative) << patch.GetType(); + patch.GetType() == LinkerPatch::Type::kTypeRelative || + patch.GetType() == LinkerPatch::Type::kStringRelative) << patch.GetType(); } else { // With the read barrier (non-Baker) enabled, it could be kStringBssEntry or kTypeBssEntry. DCHECK(patch.GetType() == LinkerPatch::Type::kMethodRelative || - patch.GetType() == LinkerPatch::Type::kStringRelative || patch.GetType() == LinkerPatch::Type::kTypeRelative || - patch.GetType() == LinkerPatch::Type::kStringBssEntry || - patch.GetType() == LinkerPatch::Type::kTypeBssEntry) << patch.GetType(); + patch.GetType() == LinkerPatch::Type::kStringRelative || + patch.GetType() == LinkerPatch::Type::kTypeBssEntry || + patch.GetType() == LinkerPatch::Type::kStringBssEntry) << patch.GetType(); } shift = 0u; // No shift for ADD. } else { // LDR/STR 32-bit or 64-bit with imm12 == 0 (unset). - DCHECK(patch.GetType() == LinkerPatch::Type::kDexCacheArray || + DCHECK(patch.GetType() == LinkerPatch::Type::kMethodBssEntry || patch.GetType() == LinkerPatch::Type::kTypeBssEntry || patch.GetType() == LinkerPatch::Type::kStringBssEntry) << patch.GetType(); DCHECK_EQ(insn & 0xbfbffc00, 0xb9000000) << std::hex << insn; diff --git a/compiler/linker/method_bss_mapping_encoder.h b/compiler/linker/method_bss_mapping_encoder.h new file mode 100644 index 0000000000..b2922ec6d2 --- /dev/null +++ b/compiler/linker/method_bss_mapping_encoder.h @@ -0,0 +1,79 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_COMPILER_LINKER_METHOD_BSS_MAPPING_ENCODER_H_ +#define ART_COMPILER_LINKER_METHOD_BSS_MAPPING_ENCODER_H_ + +#include "base/enums.h" +#include "base/logging.h" +#include "dex_file.h" +#include "method_bss_mapping.h" + +namespace art { +namespace linker { + +// Helper class for encoding compressed MethodBssMapping. +class MethodBssMappingEncoder { + public: + explicit MethodBssMappingEncoder(PointerSize pointer_size) + : pointer_size_(static_cast<size_t>(pointer_size)) { + entry_.method_index = DexFile::kDexNoIndex16; + entry_.index_mask = 0u; + entry_.bss_offset = static_cast<uint32_t>(-1); + } + + // Try to merge the next method_index -> bss_offset mapping into the current entry. + // Return true on success, false on failure. + bool TryMerge(uint32_t method_index, uint32_t bss_offset) { + DCHECK_NE(method_index, entry_.method_index); + if (entry_.bss_offset + pointer_size_ != bss_offset) { + return false; + } + uint32_t diff = method_index - entry_.method_index; + if (diff > 16u) { + return false; + } + if ((entry_.index_mask & ~(static_cast<uint32_t>(-1) << diff)) != 0u) { + return false; + } + entry_.method_index = method_index; + // Insert the bit indicating the method index we've just overwritten + // and shift bits indicating method indexes before that. + entry_.index_mask = dchecked_integral_cast<uint16_t>( + (static_cast<uint32_t>(entry_.index_mask) | 0x10000u) >> diff); + entry_.bss_offset = bss_offset; + return true; + } + + void Reset(uint32_t method_index, uint32_t bss_offset) { + entry_.method_index = method_index; + entry_.index_mask = 0u; + entry_.bss_offset = bss_offset; + } + + MethodBssMappingEntry GetEntry() { + return entry_; + } + + private: + size_t pointer_size_; + MethodBssMappingEntry entry_; +}; + +} // namespace linker +} // namespace art + +#endif // ART_COMPILER_LINKER_METHOD_BSS_MAPPING_ENCODER_H_ diff --git a/compiler/linker/method_bss_mapping_encoder_test.cc b/compiler/linker/method_bss_mapping_encoder_test.cc new file mode 100644 index 0000000000..1240389bef --- /dev/null +++ b/compiler/linker/method_bss_mapping_encoder_test.cc @@ -0,0 +1,50 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "method_bss_mapping_encoder.h" + +#include "gtest/gtest.h" + +namespace art { +namespace linker { + +TEST(MethodBssMappingEncoder, TryMerge) { + for (PointerSize pointer_size : {PointerSize::k32, PointerSize::k64}) { + size_t raw_pointer_size = static_cast<size_t>(pointer_size); + MethodBssMappingEncoder encoder(pointer_size); + encoder.Reset(1u, 0u); + ASSERT_FALSE(encoder.TryMerge(5u, raw_pointer_size + 1)); // Wrong bss_offset difference. + ASSERT_FALSE(encoder.TryMerge(18u, raw_pointer_size)); // Method index out of range. + ASSERT_TRUE(encoder.TryMerge(5u, raw_pointer_size)); + ASSERT_TRUE(encoder.GetEntry().CoversIndex(1u)); + ASSERT_TRUE(encoder.GetEntry().CoversIndex(5u)); + ASSERT_FALSE(encoder.GetEntry().CoversIndex(17u)); + ASSERT_FALSE(encoder.TryMerge(17u, 2 * raw_pointer_size + 1)); // Wrong bss_offset difference. + ASSERT_FALSE(encoder.TryMerge(18u, 2 * raw_pointer_size)); // Method index out of range. + ASSERT_TRUE(encoder.TryMerge(17u, 2 * raw_pointer_size)); + ASSERT_TRUE(encoder.GetEntry().CoversIndex(1u)); + ASSERT_TRUE(encoder.GetEntry().CoversIndex(5u)); + ASSERT_TRUE(encoder.GetEntry().CoversIndex(17u)); + ASSERT_EQ(0u, encoder.GetEntry().GetBssOffset(1u, raw_pointer_size)); + ASSERT_EQ(raw_pointer_size, encoder.GetEntry().GetBssOffset(5u, raw_pointer_size)); + ASSERT_EQ(2 * raw_pointer_size, encoder.GetEntry().GetBssOffset(17u, raw_pointer_size)); + ASSERT_EQ(0x0011u, encoder.GetEntry().index_mask); + ASSERT_FALSE(encoder.TryMerge(18u, 2 * raw_pointer_size)); // Method index out of range. + } +} + +} // namespace linker +} // namespace art diff --git a/compiler/linker/mips/relative_patcher_mips.cc b/compiler/linker/mips/relative_patcher_mips.cc index 8da530f7cc..d99d237a23 100644 --- a/compiler/linker/mips/relative_patcher_mips.cc +++ b/compiler/linker/mips/relative_patcher_mips.cc @@ -50,7 +50,6 @@ void MipsRelativePatcher::PatchPcRelativeReference(std::vector<uint8_t>* code, uint32_t anchor_literal_offset = patch.PcInsnOffset(); uint32_t literal_offset = patch.LiteralOffset(); uint32_t literal_low_offset; - bool dex_cache_array = (patch.GetType() == LinkerPatch::Type::kDexCacheArray); // Perform basic sanity checks and initialize `literal_low_offset` to point // to the instruction containing the 16 least significant bits of the @@ -72,16 +71,8 @@ void MipsRelativePatcher::PatchPcRelativeReference(std::vector<uint8_t>* code, DCHECK_GE(code->size(), 16u); DCHECK_LE(literal_offset, code->size() - 12u); DCHECK_GE(literal_offset, 4u); - // The NAL instruction may not precede immediately as the PC+0 value may - // come from HMipsComputeBaseMethodAddress. - if (dex_cache_array) { - DCHECK_EQ(literal_offset + 4u, anchor_literal_offset); - // NAL - DCHECK_EQ((*code)[literal_offset - 4], 0x00); - DCHECK_EQ((*code)[literal_offset - 3], 0x00); - DCHECK_EQ((*code)[literal_offset - 2], 0x10); - DCHECK_EQ((*code)[literal_offset - 1], 0x04); - } + // The NAL instruction does not precede immediately as the PC+0 + // comes from HMipsComputeBaseMethodAddress. // LUI reg, offset_high DCHECK_EQ((*code)[literal_offset + 0], 0x34); DCHECK_EQ((*code)[literal_offset + 1], 0x12); @@ -90,10 +81,6 @@ void MipsRelativePatcher::PatchPcRelativeReference(std::vector<uint8_t>* code, // ADDU reg, reg, reg2 DCHECK_EQ((*code)[literal_offset + 4], 0x21); DCHECK_EQ(((*code)[literal_offset + 5] & 0x07), 0x00); - if (dex_cache_array) { - // reg2 is either RA or from HMipsComputeBaseMethodAddress. - DCHECK_EQ(((*code)[literal_offset + 6] & 0x1F), 0x1F); - } DCHECK_EQ(((*code)[literal_offset + 7] & 0xFC), 0x00); // instr reg(s), offset_low DCHECK_EQ((*code)[literal_offset + 8], 0x78); @@ -104,9 +91,6 @@ void MipsRelativePatcher::PatchPcRelativeReference(std::vector<uint8_t>* code, // Apply patch. uint32_t anchor_offset = patch_offset - literal_offset + anchor_literal_offset; uint32_t diff = target_offset - anchor_offset; - if (dex_cache_array && !is_r6) { - diff += kDexCacheArrayLwOffset; - } diff += (diff & 0x8000) << 1; // Account for sign extension in "instr reg(s), offset_low". // LUI reg, offset_high / AUIPC reg, offset_high diff --git a/compiler/linker/mips/relative_patcher_mips.h b/compiler/linker/mips/relative_patcher_mips.h index 852a345aa6..0b74bd33a4 100644 --- a/compiler/linker/mips/relative_patcher_mips.h +++ b/compiler/linker/mips/relative_patcher_mips.h @@ -46,9 +46,6 @@ class MipsRelativePatcher FINAL : public RelativePatcher { uint32_t patch_offset) OVERRIDE; private: - // We'll maximize the range of a single load instruction for dex cache array accesses - // by aligning offset -32768 with the offset of the first used element. - static constexpr uint32_t kDexCacheArrayLwOffset = 0x8000; bool is_r6; DISALLOW_COPY_AND_ASSIGN(MipsRelativePatcher); diff --git a/compiler/linker/mips/relative_patcher_mips_test.cc b/compiler/linker/mips/relative_patcher_mips_test.cc index 961b31266f..49af7c614b 100644 --- a/compiler/linker/mips/relative_patcher_mips_test.cc +++ b/compiler/linker/mips/relative_patcher_mips_test.cc @@ -61,7 +61,6 @@ void MipsRelativePatcherTest::CheckPcRelativePatch(const ArrayRef<const LinkerPa ASSERT_TRUE(result.first); uint32_t diff = target_offset - (result.second + kAnchorOffset); - CHECK_NE(patches[0].GetType(), LinkerPatch::Type::kDexCacheArray); diff += (diff & 0x8000) << 1; // Account for sign extension in addiu. const uint8_t expected_code[] = { diff --git a/compiler/linker/output_stream_test.cc b/compiler/linker/output_stream_test.cc index 84c76f2c6c..09fef29d48 100644 --- a/compiler/linker/output_stream_test.cc +++ b/compiler/linker/output_stream_test.cc @@ -19,7 +19,6 @@ #include "base/unix_file/fd_file.h" #include "base/logging.h" -#include "base/stl_util.h" #include "buffered_output_stream.h" #include "common_runtime_test.h" @@ -79,7 +78,7 @@ TEST_F(OutputStreamTest, File) { TEST_F(OutputStreamTest, Buffered) { ScratchFile tmp; { - BufferedOutputStream buffered_output_stream(MakeUnique<FileOutputStream>(tmp.GetFile())); + BufferedOutputStream buffered_output_stream(std::make_unique<FileOutputStream>(tmp.GetFile())); SetOutputStream(buffered_output_stream); GenerateTestOutput(); } @@ -125,7 +124,7 @@ TEST_F(OutputStreamTest, BufferedFlush) { bool flush_called; }; - std::unique_ptr<CheckingOutputStream> cos = MakeUnique<CheckingOutputStream>(); + std::unique_ptr<CheckingOutputStream> cos = std::make_unique<CheckingOutputStream>(); CheckingOutputStream* checking_output_stream = cos.get(); BufferedOutputStream buffered(std::move(cos)); ASSERT_FALSE(checking_output_stream->flush_called); diff --git a/compiler/oat_test.cc b/compiler/oat_test.cc index 1578c0cd3e..55d0bd95d7 100644 --- a/compiler/oat_test.cc +++ b/compiler/oat_test.cc @@ -19,6 +19,7 @@ #include "arch/instruction_set_features.h" #include "art_method-inl.h" #include "base/enums.h" +#include "base/stl_util.h" #include "base/unix_file/fd_file.h" #include "class_linker.h" #include "common_compiler_test.h" @@ -220,11 +221,12 @@ class OatTest : public CommonCompilerTest { elf_writer->PrepareDynamicSection(rodata_size, text_size, oat_writer.GetBssSize(), + oat_writer.GetBssMethodsOffset(), oat_writer.GetBssRootsOffset()); if (kIsVdexEnabled) { - std::unique_ptr<BufferedOutputStream> vdex_out( - MakeUnique<BufferedOutputStream>(MakeUnique<FileOutputStream>(vdex_file))); + std::unique_ptr<BufferedOutputStream> vdex_out = + std::make_unique<BufferedOutputStream>(std::make_unique<FileOutputStream>(vdex_file)); if (!oat_writer.WriteVerifierDeps(vdex_out.get(), nullptr)) { return false; } @@ -483,7 +485,7 @@ TEST_F(OatTest, WriteRead) { TEST_F(OatTest, OatHeaderSizeCheck) { // If this test is failing and you have to update these constants, // it is time to update OatHeader::kOatVersion - EXPECT_EQ(72U, sizeof(OatHeader)); + EXPECT_EQ(76U, sizeof(OatHeader)); EXPECT_EQ(4U, sizeof(OatMethodOffsets)); EXPECT_EQ(24U, sizeof(OatQuickMethodHeader)); EXPECT_EQ(161 * static_cast<size_t>(GetInstructionSetPointerSize(kRuntimeISA)), diff --git a/compiler/oat_writer.cc b/compiler/oat_writer.cc index fed2d34cdb..59daf5a09e 100644 --- a/compiler/oat_writer.cc +++ b/compiler/oat_writer.cc @@ -22,7 +22,7 @@ #include "arch/arm64/instruction_set_features_arm64.h" #include "art_method-inl.h" #include "base/allocator.h" -#include "base/bit_vector.h" +#include "base/bit_vector-inl.h" #include "base/enums.h" #include "base/file_magic.h" #include "base/stl_util.h" @@ -41,6 +41,7 @@ #include "image_writer.h" #include "linker/buffered_output_stream.h" #include "linker/file_output_stream.h" +#include "linker/method_bss_mapping_encoder.h" #include "linker/multi_oat_relative_patcher.h" #include "linker/output_stream.h" #include "mirror/array.h" @@ -230,12 +231,14 @@ class OatWriter::OatDexFile { return dex_file_location_data_; } - void ReserveClassOffsets(OatWriter* oat_writer); - size_t SizeOf() const; bool Write(OatWriter* oat_writer, OutputStream* out) const; bool WriteClassOffsets(OatWriter* oat_writer, OutputStream* out); + size_t GetClassOffsetsRawSize() const { + return class_offsets_.size() * sizeof(class_offsets_[0]); + } + // The source of the dex file. DexFileSource source_; @@ -256,15 +259,12 @@ class OatWriter::OatDexFile { uint32_t dex_file_offset_; uint32_t class_offsets_offset_; uint32_t lookup_table_offset_; + uint32_t method_bss_mapping_offset_; // Data to write to a separate section. dchecked_vector<uint32_t> class_offsets_; private: - size_t GetClassOffsetsRawSize() const { - return class_offsets_.size() * sizeof(class_offsets_[0]); - } - DISALLOW_COPY_AND_ASSIGN(OatDexFile); }; @@ -294,7 +294,10 @@ OatWriter::OatWriter(bool compiling_boot_image, TimingLogger* timings, ProfileCo oat_size_(0u), bss_start_(0u), bss_size_(0u), + bss_methods_offset_(0u), bss_roots_offset_(0u), + bss_method_entry_references_(), + bss_method_entries_(), bss_type_entries_(), bss_string_entries_(), oat_data_offset_(0u), @@ -331,6 +334,7 @@ OatWriter::OatWriter(bool compiling_boot_image, TimingLogger* timings, ProfileCo size_oat_dex_file_offset_(0), size_oat_dex_file_class_offsets_offset_(0), size_oat_dex_file_lookup_table_offset_(0), + size_oat_dex_file_method_bss_mapping_offset_(0), size_oat_lookup_table_alignment_(0), size_oat_lookup_table_(0), size_oat_class_offsets_alignment_(0), @@ -339,6 +343,7 @@ OatWriter::OatWriter(bool compiling_boot_image, TimingLogger* timings, ProfileCo size_oat_class_status_(0), size_oat_class_method_bitmaps_(0), size_oat_class_method_offsets_(0), + size_method_bss_mappings_(0u), relative_patcher_(nullptr), absolute_patch_locations_(), profile_compilation_info_(info) { @@ -502,17 +507,16 @@ bool OatWriter::WriteAndOpenDexFiles( // Reserve space for Vdex header and checksums. vdex_size_ = sizeof(VdexFile::Header) + oat_dex_files_.size() * sizeof(VdexFile::VdexChecksum); } - size_t oat_data_offset = InitOatHeader(instruction_set, - instruction_set_features, - dchecked_integral_cast<uint32_t>(oat_dex_files_.size()), - key_value_store); - oat_size_ = InitOatDexFiles(oat_data_offset); + oat_size_ = InitOatHeader(instruction_set, + instruction_set_features, + dchecked_integral_cast<uint32_t>(oat_dex_files_.size()), + key_value_store); ChecksumUpdatingOutputStream checksum_updating_rodata(oat_rodata, oat_header_.get()); if (kIsVdexEnabled) { - std::unique_ptr<BufferedOutputStream> vdex_out( - MakeUnique<BufferedOutputStream>(MakeUnique<FileOutputStream>(vdex_file))); + std::unique_ptr<BufferedOutputStream> vdex_out = + std::make_unique<BufferedOutputStream>(std::make_unique<FileOutputStream>(vdex_file)); // Write DEX files into VDEX, mmap and open them. if (!WriteDexFiles(vdex_out.get(), vdex_file, update_input_vdex) || !OpenDexFiles(vdex_file, verify, &dex_files_map, &dex_files)) { @@ -539,16 +543,6 @@ bool OatWriter::WriteAndOpenDexFiles( return false; } - // Reserve space for class offsets in OAT and update class_offsets_offset_. - for (OatDexFile& oat_dex_file : oat_dex_files_) { - oat_dex_file.ReserveClassOffsets(this); - } - - // Write OatDexFiles into OAT. Needs to be done last, once offsets are collected. - if (!WriteOatDexFiles(&checksum_updating_rodata)) { - return false; - } - *opened_dex_files_map = std::move(dex_files_map); *opened_dex_files = std::move(dex_files); write_state_ = WriteState::kPrepareLayout; @@ -567,16 +561,34 @@ void OatWriter::PrepareLayout(linker::MultiOatRelativePatcher* relative_patcher) InstructionSet instruction_set = compiler_driver_->GetInstructionSet(); CHECK_EQ(instruction_set, oat_header_->GetInstructionSet()); + { + TimingLogger::ScopedTiming split("InitBssLayout", timings_); + InitBssLayout(instruction_set); + } + uint32_t offset = oat_size_; { + TimingLogger::ScopedTiming split("InitClassOffsets", timings_); + offset = InitClassOffsets(offset); + } + { TimingLogger::ScopedTiming split("InitOatClasses", timings_); offset = InitOatClasses(offset); } { + TimingLogger::ScopedTiming split("InitMethodBssMappings", timings_); + offset = InitMethodBssMappings(offset); + } + { TimingLogger::ScopedTiming split("InitOatMaps", timings_); offset = InitOatMaps(offset); } { + TimingLogger::ScopedTiming split("InitOatDexFiles", timings_); + oat_header_->SetOatDexFilesOffset(offset); + offset = InitOatDexFiles(offset); + } + { TimingLogger::ScopedTiming split("InitOatCode", timings_); offset = InitOatCode(offset); } @@ -585,11 +597,7 @@ void OatWriter::PrepareLayout(linker::MultiOatRelativePatcher* relative_patcher) offset = InitOatCodeDexFiles(offset); } oat_size_ = offset; - - { - TimingLogger::ScopedTiming split("InitBssLayout", timings_); - InitBssLayout(instruction_set); - } + bss_start_ = (bss_size_ != 0u) ? RoundUp(oat_size_, kPageSize) : 0u; CHECK_EQ(dex_files_->size(), oat_dex_files_.size()); if (compiling_boot_image_) { @@ -606,11 +614,10 @@ OatWriter::~OatWriter() { class OatWriter::DexMethodVisitor { public: DexMethodVisitor(OatWriter* writer, size_t offset) - : writer_(writer), - offset_(offset), - dex_file_(nullptr), - class_def_index_(DexFile::kDexNoIndex) { - } + : writer_(writer), + offset_(offset), + dex_file_(nullptr), + class_def_index_(DexFile::kDexNoIndex) {} virtual bool StartClass(const DexFile* dex_file, size_t class_def_index) { DCHECK(dex_file_ == nullptr); @@ -650,19 +657,18 @@ class OatWriter::DexMethodVisitor { class OatWriter::OatDexMethodVisitor : public DexMethodVisitor { public: OatDexMethodVisitor(OatWriter* writer, size_t offset) - : DexMethodVisitor(writer, offset), - oat_class_index_(0u), - method_offsets_index_(0u) { - } + : DexMethodVisitor(writer, offset), + oat_class_index_(0u), + method_offsets_index_(0u) {} - bool StartClass(const DexFile* dex_file, size_t class_def_index) { + bool StartClass(const DexFile* dex_file, size_t class_def_index) OVERRIDE { DexMethodVisitor::StartClass(dex_file, class_def_index); DCHECK_LT(oat_class_index_, writer_->oat_classes_.size()); method_offsets_index_ = 0u; return true; } - bool EndClass() { + bool EndClass() OVERRIDE { ++oat_class_index_; return DexMethodVisitor::EndClass(); } @@ -672,21 +678,61 @@ class OatWriter::OatDexMethodVisitor : public DexMethodVisitor { size_t method_offsets_index_; }; +class OatWriter::InitBssLayoutMethodVisitor : public DexMethodVisitor { + public: + explicit InitBssLayoutMethodVisitor(OatWriter* writer) + : DexMethodVisitor(writer, /* offset */ 0u) {} + + bool VisitMethod(size_t class_def_method_index ATTRIBUTE_UNUSED, + const ClassDataItemIterator& it) OVERRIDE { + // Look for patches with .bss references and prepare maps with placeholders for their offsets. + CompiledMethod* compiled_method = writer_->compiler_driver_->GetCompiledMethod( + MethodReference(dex_file_, it.GetMemberIndex())); + if (compiled_method != nullptr) { + for (const LinkerPatch& patch : compiled_method->GetPatches()) { + if (patch.GetType() == LinkerPatch::Type::kMethodBssEntry) { + MethodReference target_method = patch.TargetMethod(); + auto refs_it = writer_->bss_method_entry_references_.find(target_method.dex_file); + if (refs_it == writer_->bss_method_entry_references_.end()) { + refs_it = writer_->bss_method_entry_references_.Put( + target_method.dex_file, + BitVector(target_method.dex_file->NumMethodIds(), + /* expandable */ false, + Allocator::GetMallocAllocator())); + refs_it->second.ClearAllBits(); + } + refs_it->second.SetBit(target_method.dex_method_index); + writer_->bss_method_entries_.Overwrite(target_method, /* placeholder */ 0u); + } else if (patch.GetType() == LinkerPatch::Type::kTypeBssEntry) { + TypeReference ref(patch.TargetTypeDexFile(), patch.TargetTypeIndex()); + writer_->bss_type_entries_.Overwrite(ref, /* placeholder */ 0u); + } else if (patch.GetType() == LinkerPatch::Type::kStringBssEntry) { + StringReference ref(patch.TargetStringDexFile(), patch.TargetStringIndex()); + writer_->bss_string_entries_.Overwrite(ref, /* placeholder */ 0u); + } + } + } + return true; + } +}; + class OatWriter::InitOatClassesMethodVisitor : public DexMethodVisitor { public: InitOatClassesMethodVisitor(OatWriter* writer, size_t offset) - : DexMethodVisitor(writer, offset), - compiled_methods_(), - num_non_null_compiled_methods_(0u) { + : DexMethodVisitor(writer, offset), + compiled_methods_(), + num_non_null_compiled_methods_(0u) { size_t num_classes = 0u; for (const OatDexFile& oat_dex_file : writer_->oat_dex_files_) { num_classes += oat_dex_file.class_offsets_.size(); } writer_->oat_classes_.reserve(num_classes); compiled_methods_.reserve(256u); + // If there are any classes, the class offsets allocation aligns the offset. + DCHECK(num_classes == 0u || IsAligned<4u>(offset)); } - bool StartClass(const DexFile* dex_file, size_t class_def_index) { + bool StartClass(const DexFile* dex_file, size_t class_def_index) OVERRIDE { DexMethodVisitor::StartClass(dex_file, class_def_index); compiled_methods_.clear(); num_non_null_compiled_methods_ = 0u; @@ -694,7 +740,7 @@ class OatWriter::InitOatClassesMethodVisitor : public DexMethodVisitor { } bool VisitMethod(size_t class_def_method_index ATTRIBUTE_UNUSED, - const ClassDataItemIterator& it) { + const ClassDataItemIterator& it) OVERRIDE { // Fill in the compiled_methods_ array for methods that have a // CompiledMethod. We track the number of non-null entries in // num_non_null_compiled_methods_ since we only want to allocate @@ -704,12 +750,12 @@ class OatWriter::InitOatClassesMethodVisitor : public DexMethodVisitor { writer_->compiler_driver_->GetCompiledMethod(MethodReference(dex_file_, method_idx)); compiled_methods_.push_back(compiled_method); if (compiled_method != nullptr) { - ++num_non_null_compiled_methods_; + ++num_non_null_compiled_methods_; } return true; } - bool EndClass() { + bool EndClass() OVERRIDE { ClassReference class_ref(dex_file_, class_def_index_); mirror::Class::Status status; bool found = writer_->compiler_driver_->GetCompiledClass(class_ref, &status); @@ -740,14 +786,14 @@ class OatWriter::InitOatClassesMethodVisitor : public DexMethodVisitor { class OatWriter::InitCodeMethodVisitor : public OatDexMethodVisitor { public: InitCodeMethodVisitor(OatWriter* writer, size_t offset, size_t quickening_info_offset) - : OatDexMethodVisitor(writer, offset), - debuggable_(writer->GetCompilerDriver()->GetCompilerOptions().GetDebuggable()), - current_quickening_info_offset_(quickening_info_offset) { + : OatDexMethodVisitor(writer, offset), + debuggable_(writer->GetCompilerDriver()->GetCompilerOptions().GetDebuggable()), + current_quickening_info_offset_(quickening_info_offset) { writer_->absolute_patch_locations_.reserve( writer_->compiler_driver_->GetNonRelativeLinkerPatchCount()); } - bool EndClass() { + bool EndClass() OVERRIDE { OatDexMethodVisitor::EndClass(); if (oat_class_index_ == writer_->oat_classes_.size()) { offset_ = writer_->relative_patcher_->ReserveSpaceEnd(offset_); @@ -755,7 +801,7 @@ class OatWriter::InitCodeMethodVisitor : public OatDexMethodVisitor { return true; } - bool VisitMethod(size_t class_def_method_index, const ClassDataItemIterator& it) + bool VisitMethod(size_t class_def_method_index, const ClassDataItemIterator& it) OVERRIDE REQUIRES_SHARED(Locks::mutator_lock_) { OatClass* oat_class = &writer_->oat_classes_[oat_class_index_]; CompiledMethod* compiled_method = oat_class->GetCompiledMethod(class_def_method_index); @@ -858,14 +904,6 @@ class OatWriter::InitCodeMethodVisitor : public OatDexMethodVisitor { if (!patch.IsPcRelative()) { writer_->absolute_patch_locations_.push_back(base_loc + patch.LiteralOffset()); } - if (patch.GetType() == LinkerPatch::Type::kTypeBssEntry) { - TypeReference ref(patch.TargetTypeDexFile(), patch.TargetTypeIndex()); - writer_->bss_type_entries_.Overwrite(ref, /* placeholder */ 0u); - } - if (patch.GetType() == LinkerPatch::Type::kStringBssEntry) { - StringReference ref(patch.TargetStringDexFile(), patch.TargetStringIndex()); - writer_->bss_string_entries_.Overwrite(ref, /* placeholder */ 0u); - } } } } @@ -950,11 +988,10 @@ class OatWriter::InitCodeMethodVisitor : public OatDexMethodVisitor { class OatWriter::InitMapMethodVisitor : public OatDexMethodVisitor { public: InitMapMethodVisitor(OatWriter* writer, size_t offset) - : OatDexMethodVisitor(writer, offset) { - } + : OatDexMethodVisitor(writer, offset) {} bool VisitMethod(size_t class_def_method_index, const ClassDataItemIterator& it ATTRIBUTE_UNUSED) - REQUIRES_SHARED(Locks::mutator_lock_) { + OVERRIDE REQUIRES_SHARED(Locks::mutator_lock_) { OatClass* oat_class = &writer_->oat_classes_[oat_class_index_]; CompiledMethod* compiled_method = oat_class->GetCompiledMethod(class_def_method_index); @@ -997,7 +1034,7 @@ class OatWriter::InitMethodInfoVisitor : public OatDexMethodVisitor { InitMethodInfoVisitor(OatWriter* writer, size_t offset) : OatDexMethodVisitor(writer, offset) {} bool VisitMethod(size_t class_def_method_index, const ClassDataItemIterator& it ATTRIBUTE_UNUSED) - REQUIRES_SHARED(Locks::mutator_lock_) { + OVERRIDE REQUIRES_SHARED(Locks::mutator_lock_) { OatClass* oat_class = &writer_->oat_classes_[oat_class_index_]; CompiledMethod* compiled_method = oat_class->GetCompiledMethod(class_def_method_index); @@ -1035,18 +1072,17 @@ class OatWriter::InitImageMethodVisitor : public OatDexMethodVisitor { InitImageMethodVisitor(OatWriter* writer, size_t offset, const std::vector<const DexFile*>* dex_files) - : OatDexMethodVisitor(writer, offset), - pointer_size_(GetInstructionSetPointerSize(writer_->compiler_driver_->GetInstructionSet())), - dex_files_(dex_files), - class_linker_(Runtime::Current()->GetClassLinker()) { - } + : OatDexMethodVisitor(writer, offset), + pointer_size_(GetInstructionSetPointerSize(writer_->compiler_driver_->GetInstructionSet())), + dex_files_(dex_files), + class_linker_(Runtime::Current()->GetClassLinker()) {} // Handle copied methods here. Copy pointer to quick code from // an origin method to a copied method only if they are // in the same oat file. If the origin and the copied methods are // in different oat files don't touch the copied method. // References to other oat files are not supported yet. - bool StartClass(const DexFile* dex_file, size_t class_def_index) + bool StartClass(const DexFile* dex_file, size_t class_def_index) OVERRIDE REQUIRES_SHARED(Locks::mutator_lock_) { OatDexMethodVisitor::StartClass(dex_file, class_def_index); // Skip classes that are not in the image. @@ -1085,7 +1121,7 @@ class OatWriter::InitImageMethodVisitor : public OatDexMethodVisitor { return true; } - bool VisitMethod(size_t class_def_method_index, const ClassDataItemIterator& it) + bool VisitMethod(size_t class_def_method_index, const ClassDataItemIterator& it) OVERRIDE REQUIRES_SHARED(Locks::mutator_lock_) { // Skip methods that are not in the image. if (!IsImageClass()) { @@ -1131,8 +1167,7 @@ class OatWriter::InitImageMethodVisitor : public OatDexMethodVisitor { // Should already have been resolved by the compiler, just peek into the dex cache. // It may not be resolved if the class failed to verify, in this case, don't set the // entrypoint. This is not fatal since the dex cache will contain a resolution method. - method = dex_cache->GetResolvedMethod(it.GetMemberIndex(), - class_linker_->GetImagePointerSize()); + method = dex_cache->GetResolvedMethod(it.GetMemberIndex(), pointer_size_); } if (method != nullptr && compiled_method != nullptr && @@ -1171,7 +1206,7 @@ class OatWriter::InitImageMethodVisitor : public OatDexMethodVisitor { } } - protected: + private: const PointerSize pointer_size_; const std::vector<const DexFile*>* dex_files_; ClassLinker* const class_linker_; @@ -1182,14 +1217,15 @@ class OatWriter::WriteCodeMethodVisitor : public OatDexMethodVisitor { public: WriteCodeMethodVisitor(OatWriter* writer, OutputStream* out, const size_t file_offset, size_t relative_offset) SHARED_LOCK_FUNCTION(Locks::mutator_lock_) - : OatDexMethodVisitor(writer, relative_offset), - class_loader_(writer->HasImage() ? writer->image_writer_->GetClassLoader() : nullptr), - out_(out), - file_offset_(file_offset), - soa_(Thread::Current()), - no_thread_suspension_("OatWriter patching"), - class_linker_(Runtime::Current()->GetClassLinker()), - dex_cache_(nullptr) { + : OatDexMethodVisitor(writer, relative_offset), + pointer_size_(GetInstructionSetPointerSize(writer_->compiler_driver_->GetInstructionSet())), + class_loader_(writer->HasImage() ? writer->image_writer_->GetClassLoader() : nullptr), + out_(out), + file_offset_(file_offset), + soa_(Thread::Current()), + no_thread_suspension_("OatWriter patching"), + class_linker_(Runtime::Current()->GetClassLinker()), + dex_cache_(nullptr) { patched_code_.reserve(16 * KB); if (writer_->HasBootImage()) { // If we're creating the image, the address space must be ready so that we can apply patches. @@ -1200,7 +1236,7 @@ class OatWriter::WriteCodeMethodVisitor : public OatDexMethodVisitor { ~WriteCodeMethodVisitor() UNLOCK_FUNCTION(Locks::mutator_lock_) { } - bool StartClass(const DexFile* dex_file, size_t class_def_index) + bool StartClass(const DexFile* dex_file, size_t class_def_index) OVERRIDE REQUIRES_SHARED(Locks::mutator_lock_) { OatDexMethodVisitor::StartClass(dex_file, class_def_index); if (dex_cache_ == nullptr || dex_cache_->GetDexFile() != dex_file) { @@ -1210,7 +1246,7 @@ class OatWriter::WriteCodeMethodVisitor : public OatDexMethodVisitor { return true; } - bool EndClass() REQUIRES_SHARED(Locks::mutator_lock_) { + bool EndClass() OVERRIDE REQUIRES_SHARED(Locks::mutator_lock_) { bool result = OatDexMethodVisitor::EndClass(); if (oat_class_index_ == writer_->oat_classes_.size()) { DCHECK(result); // OatDexMethodVisitor::EndClass() never fails. @@ -1223,7 +1259,7 @@ class OatWriter::WriteCodeMethodVisitor : public OatDexMethodVisitor { return result; } - bool VisitMethod(size_t class_def_method_index, const ClassDataItemIterator& it) + bool VisitMethod(size_t class_def_method_index, const ClassDataItemIterator& it) OVERRIDE REQUIRES_SHARED(Locks::mutator_lock_) { OatClass* oat_class = &writer_->oat_classes_[oat_class_index_]; const CompiledMethod* compiled_method = oat_class->GetCompiledMethod(class_def_method_index); @@ -1275,6 +1311,15 @@ class OatWriter::WriteCodeMethodVisitor : public OatDexMethodVisitor { for (const LinkerPatch& patch : compiled_method->GetPatches()) { uint32_t literal_offset = patch.LiteralOffset(); switch (patch.GetType()) { + case LinkerPatch::Type::kMethodBssEntry: { + uint32_t target_offset = + writer_->bss_start_ + writer_->bss_method_entries_.Get(patch.TargetMethod()); + writer_->relative_patcher_->PatchPcRelativeReference(&patched_code_, + patch, + offset_ + literal_offset, + target_offset); + break; + } case LinkerPatch::Type::kCallRelative: { // NOTE: Relative calls across oat files are not supported. uint32_t target_offset = GetTargetOffset(patch); @@ -1284,14 +1329,6 @@ class OatWriter::WriteCodeMethodVisitor : public OatDexMethodVisitor { target_offset); break; } - case LinkerPatch::Type::kDexCacheArray: { - uint32_t target_offset = GetDexCacheOffset(patch); - writer_->relative_patcher_->PatchPcRelativeReference(&patched_code_, - patch, - offset_ + literal_offset, - target_offset); - break; - } case LinkerPatch::Type::kStringRelative: { uint32_t target_offset = GetTargetObjectOffset(GetTargetString(patch)); writer_->relative_patcher_->PatchPcRelativeReference(&patched_code_, @@ -1302,7 +1339,8 @@ class OatWriter::WriteCodeMethodVisitor : public OatDexMethodVisitor { } case LinkerPatch::Type::kStringBssEntry: { StringReference ref(patch.TargetStringDexFile(), patch.TargetStringIndex()); - uint32_t target_offset = writer_->bss_string_entries_.Get(ref); + uint32_t target_offset = + writer_->bss_start_ + writer_->bss_string_entries_.Get(ref); writer_->relative_patcher_->PatchPcRelativeReference(&patched_code_, patch, offset_ + literal_offset, @@ -1319,7 +1357,7 @@ class OatWriter::WriteCodeMethodVisitor : public OatDexMethodVisitor { } case LinkerPatch::Type::kTypeBssEntry: { TypeReference ref(patch.TargetTypeDexFile(), patch.TargetTypeIndex()); - uint32_t target_offset = writer_->bss_type_entries_.Get(ref); + uint32_t target_offset = writer_->bss_start_ + writer_->bss_type_entries_.Get(ref); writer_->relative_patcher_->PatchPcRelativeReference(&patched_code_, patch, offset_ + literal_offset, @@ -1368,6 +1406,7 @@ class OatWriter::WriteCodeMethodVisitor : public OatDexMethodVisitor { } private: + const PointerSize pointer_size_; ObjPtr<mirror::ClassLoader> class_loader_; OutputStream* const out_; const size_t file_offset_; @@ -1388,8 +1427,7 @@ class OatWriter::WriteCodeMethodVisitor : public OatDexMethodVisitor { ObjPtr<mirror::DexCache> dex_cache = (dex_file_ == ref.dex_file) ? dex_cache_ : class_linker_->FindDexCache( Thread::Current(), *ref.dex_file); - ArtMethod* method = dex_cache->GetResolvedMethod( - ref.dex_method_index, class_linker_->GetImagePointerSize()); + ArtMethod* method = dex_cache->GetResolvedMethod(ref.dex_method_index, pointer_size_); CHECK(method != nullptr); return method; } @@ -1401,9 +1439,8 @@ class OatWriter::WriteCodeMethodVisitor : public OatDexMethodVisitor { if (UNLIKELY(target_offset == 0)) { ArtMethod* target = GetTargetMethod(patch); DCHECK(target != nullptr); - PointerSize size = - GetInstructionSetPointerSize(writer_->compiler_driver_->GetInstructionSet()); - const void* oat_code_offset = target->GetEntryPointFromQuickCompiledCodePtrSize(size); + const void* oat_code_offset = + target->GetEntryPointFromQuickCompiledCodePtrSize(pointer_size_); if (oat_code_offset != 0) { DCHECK(!writer_->HasBootImage()); DCHECK(!Runtime::Current()->GetClassLinker()->IsQuickResolutionStub(oat_code_offset)); @@ -1447,19 +1484,6 @@ class OatWriter::WriteCodeMethodVisitor : public OatDexMethodVisitor { return string; } - uint32_t GetDexCacheOffset(const LinkerPatch& patch) REQUIRES_SHARED(Locks::mutator_lock_) { - if (writer_->HasBootImage()) { - uintptr_t element = writer_->image_writer_->GetDexCacheArrayElementImageAddress<uintptr_t>( - patch.TargetDexCacheDexFile(), patch.TargetDexCacheElementOffset()); - size_t oat_index = writer_->image_writer_->GetOatIndexForDexCache(dex_cache_); - uintptr_t oat_data = writer_->image_writer_->GetOatDataBegin(oat_index); - return element - oat_data; - } else { - size_t start = writer_->dex_cache_arrays_offsets_.Get(patch.TargetDexCacheDexFile()); - return start + patch.TargetDexCacheElementOffset(); - } - } - uint32_t GetTargetMethodOffset(ArtMethod* method) REQUIRES_SHARED(Locks::mutator_lock_) { DCHECK(writer_->HasBootImage()); method = writer_->image_writer_->GetImageMethodAddress(method); @@ -1525,12 +1549,11 @@ class OatWriter::WriteMapMethodVisitor : public OatDexMethodVisitor { OutputStream* out, const size_t file_offset, size_t relative_offset) - : OatDexMethodVisitor(writer, relative_offset), - out_(out), - file_offset_(file_offset) { - } + : OatDexMethodVisitor(writer, relative_offset), + out_(out), + file_offset_(file_offset) {} - bool VisitMethod(size_t class_def_method_index, const ClassDataItemIterator& it) { + bool VisitMethod(size_t class_def_method_index, const ClassDataItemIterator& it) OVERRIDE { OatClass* oat_class = &writer_->oat_classes_[oat_class_index_]; const CompiledMethod* compiled_method = oat_class->GetCompiledMethod(class_def_method_index); @@ -1589,11 +1612,11 @@ class OatWriter::WriteMethodInfoVisitor : public OatDexMethodVisitor { OutputStream* out, const size_t file_offset, size_t relative_offset) - : OatDexMethodVisitor(writer, relative_offset), - out_(out), - file_offset_(file_offset) {} + : OatDexMethodVisitor(writer, relative_offset), + out_(out), + file_offset_(file_offset) {} - bool VisitMethod(size_t class_def_method_index, const ClassDataItemIterator& it) { + bool VisitMethod(size_t class_def_method_index, const ClassDataItemIterator& it) OVERRIDE { OatClass* oat_class = &writer_->oat_classes_[oat_class_index_]; const CompiledMethod* compiled_method = oat_class->GetCompiledMethod(class_def_method_index); @@ -1698,12 +1721,17 @@ size_t OatWriter::InitOatHeader(InstructionSet instruction_set, return oat_header_->GetHeaderSize(); } -size_t OatWriter::InitOatDexFiles(size_t offset) { - TimingLogger::ScopedTiming split("InitOatDexFiles", timings_); - // Initialize offsets of dex files. +size_t OatWriter::InitClassOffsets(size_t offset) { + // Reserve space for class offsets in OAT and update class_offsets_offset_. for (OatDexFile& oat_dex_file : oat_dex_files_) { - oat_dex_file.offset_ = offset; - offset += oat_dex_file.SizeOf(); + DCHECK_EQ(oat_dex_file.class_offsets_offset_, 0u); + if (!oat_dex_file.class_offsets_.empty()) { + // Class offsets are required to be 4 byte aligned. + offset = RoundUp(offset, 4u); + oat_dex_file.class_offsets_offset_ = offset; + offset += oat_dex_file.GetClassOffsetsRawSize(); + DCHECK_ALIGNED(offset, 4u); + } } return offset; } @@ -1748,6 +1776,50 @@ size_t OatWriter::InitOatMaps(size_t offset) { return offset; } +size_t OatWriter::InitMethodBssMappings(size_t offset) { + size_t number_of_dex_files = 0u; + for (size_t i = 0, size = dex_files_->size(); i != size; ++i) { + const DexFile* dex_file = (*dex_files_)[i]; + auto it = bss_method_entry_references_.find(dex_file); + if (it != bss_method_entry_references_.end()) { + const BitVector& method_indexes = it->second; + ++number_of_dex_files; + // If there are any classes, the class offsets allocation aligns the offset + // and we cannot have method bss mappings without class offsets. + static_assert(alignof(MethodBssMapping) == 4u, "MethodBssMapping alignment check."); + DCHECK_ALIGNED(offset, 4u); + oat_dex_files_[i].method_bss_mapping_offset_ = offset; + + linker::MethodBssMappingEncoder encoder( + GetInstructionSetPointerSize(oat_header_->GetInstructionSet())); + size_t number_of_entries = 0u; + bool first_index = true; + for (uint32_t method_index : method_indexes.Indexes()) { + uint32_t bss_offset = bss_method_entries_.Get(MethodReference(dex_file, method_index)); + if (first_index || !encoder.TryMerge(method_index, bss_offset)) { + encoder.Reset(method_index, bss_offset); + ++number_of_entries; + first_index = false; + } + } + DCHECK_NE(number_of_entries, 0u); + offset += MethodBssMapping::ComputeSize(number_of_entries); + } + } + // Check that all dex files targeted by method bss entries are in `*dex_files_`. + CHECK_EQ(number_of_dex_files, bss_method_entry_references_.size()); + return offset; +} + +size_t OatWriter::InitOatDexFiles(size_t offset) { + // Initialize offsets of oat dex files. + for (OatDexFile& oat_dex_file : oat_dex_files_) { + oat_dex_file.offset_ = offset; + offset += oat_dex_file.SizeOf(); + } + return offset; +} + size_t OatWriter::InitOatCode(size_t offset) { // calculate the offsets within OatHeader to executable code size_t old_offset = offset; @@ -1806,38 +1878,51 @@ size_t OatWriter::InitOatCodeDexFiles(size_t offset) { } void OatWriter::InitBssLayout(InstructionSet instruction_set) { + { + InitBssLayoutMethodVisitor visitor(this); + bool success = VisitDexMethods(&visitor); + DCHECK(success); + } + + DCHECK_EQ(bss_size_, 0u); if (HasBootImage()) { DCHECK(bss_string_entries_.empty()); - if (bss_type_entries_.empty()) { + if (bss_method_entries_.empty() && bss_type_entries_.empty()) { // Nothing to put to the .bss section. return; } } // Allocate space for app dex cache arrays in the .bss section. - bss_start_ = RoundUp(oat_size_, kPageSize); - bss_size_ = 0u; + PointerSize pointer_size = GetInstructionSetPointerSize(instruction_set); if (!HasBootImage()) { - PointerSize pointer_size = GetInstructionSetPointerSize(instruction_set); for (const DexFile* dex_file : *dex_files_) { - dex_cache_arrays_offsets_.Put(dex_file, bss_start_ + bss_size_); DexCacheArraysLayout layout(pointer_size, dex_file); bss_size_ += layout.Size(); } } + bss_methods_offset_ = bss_size_; + + // Prepare offsets for .bss ArtMethod entries. + for (auto& entry : bss_method_entries_) { + DCHECK_EQ(entry.second, 0u); + entry.second = bss_size_; + bss_size_ += static_cast<size_t>(pointer_size); + } + bss_roots_offset_ = bss_size_; // Prepare offsets for .bss Class entries. for (auto& entry : bss_type_entries_) { DCHECK_EQ(entry.second, 0u); - entry.second = bss_start_ + bss_size_; + entry.second = bss_size_; bss_size_ += sizeof(GcRoot<mirror::Class>); } // Prepare offsets for .bss String entries. for (auto& entry : bss_string_entries_) { DCHECK_EQ(entry.second, 0u); - entry.second = bss_start_ + bss_size_; + entry.second = bss_size_; bss_size_ += sizeof(GcRoot<mirror::String>); } } @@ -1845,30 +1930,45 @@ void OatWriter::InitBssLayout(InstructionSet instruction_set) { bool OatWriter::WriteRodata(OutputStream* out) { CHECK(write_state_ == WriteState::kWriteRoData); + size_t file_offset = oat_data_offset_; + off_t current_offset = out->Seek(0, kSeekCurrent); + if (current_offset == static_cast<off_t>(-1)) { + PLOG(ERROR) << "Failed to retrieve current position in " << out->GetLocation(); + } + DCHECK_GE(static_cast<size_t>(current_offset), file_offset + oat_header_->GetHeaderSize()); + size_t relative_offset = current_offset - file_offset; + // Wrap out to update checksum with each write. ChecksumUpdatingOutputStream checksum_updating_out(out, oat_header_.get()); out = &checksum_updating_out; - if (!WriteClassOffsets(out)) { - LOG(ERROR) << "Failed to write class offsets to " << out->GetLocation(); + relative_offset = WriteClassOffsets(out, file_offset, relative_offset); + if (relative_offset == 0) { + PLOG(ERROR) << "Failed to write class offsets to " << out->GetLocation(); return false; } - if (!WriteClasses(out)) { - LOG(ERROR) << "Failed to write classes to " << out->GetLocation(); + relative_offset = WriteClasses(out, file_offset, relative_offset); + if (relative_offset == 0) { + PLOG(ERROR) << "Failed to write classes to " << out->GetLocation(); return false; } - off_t tables_end_offset = out->Seek(0, kSeekCurrent); - if (tables_end_offset == static_cast<off_t>(-1)) { - LOG(ERROR) << "Failed to get oat code position in " << out->GetLocation(); + relative_offset = WriteMethodBssMappings(out, file_offset, relative_offset); + if (relative_offset == 0) { + PLOG(ERROR) << "Failed to write method bss mappings to " << out->GetLocation(); return false; } - size_t file_offset = oat_data_offset_; - size_t relative_offset = static_cast<size_t>(tables_end_offset) - file_offset; + relative_offset = WriteMaps(out, file_offset, relative_offset); if (relative_offset == 0) { - LOG(ERROR) << "Failed to write oat code to " << out->GetLocation(); + PLOG(ERROR) << "Failed to write oat code to " << out->GetLocation(); + return false; + } + + relative_offset = WriteOatDexFiles(out, file_offset, relative_offset); + if (relative_offset == 0) { + PLOG(ERROR) << "Failed to write oat dex information to " << out->GetLocation(); return false; } @@ -1891,12 +1991,12 @@ bool OatWriter::WriteRodata(OutputStream* out) { class OatWriter::WriteQuickeningInfoMethodVisitor : public DexMethodVisitor { public: WriteQuickeningInfoMethodVisitor(OatWriter* writer, OutputStream* out, uint32_t offset) - : DexMethodVisitor(writer, offset), - out_(out), - written_bytes_(0u) {} + : DexMethodVisitor(writer, offset), + out_(out), + written_bytes_(0u) {} bool VisitMethod(size_t class_def_method_index ATTRIBUTE_UNUSED, - const ClassDataItemIterator& it) { + const ClassDataItemIterator& it) OVERRIDE { if (it.GetMethodCodeItem() == nullptr) { // No CodeItem. Native or abstract method. return true; @@ -2092,6 +2192,7 @@ bool OatWriter::WriteCode(OutputStream* out) { DO_STAT(size_oat_dex_file_offset_); DO_STAT(size_oat_dex_file_class_offsets_offset_); DO_STAT(size_oat_dex_file_lookup_table_offset_); + DO_STAT(size_oat_dex_file_method_bss_mapping_offset_); DO_STAT(size_oat_lookup_table_alignment_); DO_STAT(size_oat_lookup_table_); DO_STAT(size_oat_class_offsets_alignment_); @@ -2100,6 +2201,7 @@ bool OatWriter::WriteCode(OutputStream* out) { DO_STAT(size_oat_class_status_); DO_STAT(size_oat_class_method_bitmaps_); DO_STAT(size_oat_class_method_offsets_); + DO_STAT(size_method_bss_mappings_); #undef DO_STAT VLOG(compiler) << "size_total=" << PrettySize(size_total) << " (" << size_total << "B)"; @@ -2172,35 +2274,41 @@ bool OatWriter::WriteHeader(OutputStream* out, return true; } -bool OatWriter::WriteClassOffsets(OutputStream* out) { +size_t OatWriter::WriteClassOffsets(OutputStream* out, size_t file_offset, size_t relative_offset) { for (OatDexFile& oat_dex_file : oat_dex_files_) { if (oat_dex_file.class_offsets_offset_ != 0u) { - uint32_t expected_offset = oat_data_offset_ + oat_dex_file.class_offsets_offset_; - off_t actual_offset = out->Seek(expected_offset, kSeekSet); - if (static_cast<uint32_t>(actual_offset) != expected_offset) { - PLOG(ERROR) << "Failed to seek to oat class offsets section. Actual: " << actual_offset - << " Expected: " << expected_offset << " File: " << oat_dex_file.GetLocation(); - return false; + // Class offsets are required to be 4 byte aligned. + if (UNLIKELY(!IsAligned<4u>(relative_offset))) { + size_t padding_size = RoundUp(relative_offset, 4u) - relative_offset; + if (!WriteUpTo16BytesAlignment(out, padding_size, &size_oat_class_offsets_alignment_)) { + return 0u; + } + relative_offset += padding_size; } + DCHECK_OFFSET(); if (!oat_dex_file.WriteClassOffsets(this, out)) { - return false; + return 0u; } + relative_offset += oat_dex_file.GetClassOffsetsRawSize(); } } - return true; + return relative_offset; } -bool OatWriter::WriteClasses(OutputStream* out) { +size_t OatWriter::WriteClasses(OutputStream* out, size_t file_offset, size_t relative_offset) { for (OatClass& oat_class : oat_classes_) { + // If there are any classes, the class offsets allocation aligns the offset. + DCHECK_ALIGNED(relative_offset, 4u); + DCHECK_OFFSET(); if (!oat_class.Write(this, out, oat_data_offset_)) { - PLOG(ERROR) << "Failed to write oat methods information to " << out->GetLocation(); - return false; + return 0u; } + relative_offset += oat_class.SizeOf(); } - return true; + return relative_offset; } -size_t OatWriter::WriteMaps(OutputStream* out, const size_t file_offset, size_t relative_offset) { +size_t OatWriter::WriteMaps(OutputStream* out, size_t file_offset, size_t relative_offset) { { size_t vmap_tables_offset = relative_offset; WriteMapMethodVisitor visitor(this, out, file_offset, relative_offset); @@ -2223,7 +2331,87 @@ size_t OatWriter::WriteMaps(OutputStream* out, const size_t file_offset, size_t return relative_offset; } -size_t OatWriter::WriteCode(OutputStream* out, const size_t file_offset, size_t relative_offset) { +size_t OatWriter::WriteMethodBssMappings(OutputStream* out, + size_t file_offset, + size_t relative_offset) { + TimingLogger::ScopedTiming split("WriteMethodBssMappings", timings_); + + for (size_t i = 0, size = dex_files_->size(); i != size; ++i) { + const DexFile* dex_file = (*dex_files_)[i]; + OatDexFile* oat_dex_file = &oat_dex_files_[i]; + auto it = bss_method_entry_references_.find(dex_file); + if (it != bss_method_entry_references_.end()) { + const BitVector& method_indexes = it->second; + // If there are any classes, the class offsets allocation aligns the offset + // and we cannot have method bss mappings without class offsets. + static_assert(alignof(MethodBssMapping) == sizeof(uint32_t), + "MethodBssMapping alignment check."); + DCHECK_ALIGNED(relative_offset, sizeof(uint32_t)); + + linker::MethodBssMappingEncoder encoder( + GetInstructionSetPointerSize(oat_header_->GetInstructionSet())); + // Allocate a sufficiently large MethodBssMapping. + size_t number_of_method_indexes = method_indexes.NumSetBits(); + DCHECK_NE(number_of_method_indexes, 0u); + size_t max_mappings_size = MethodBssMapping::ComputeSize(number_of_method_indexes); + DCHECK_ALIGNED(max_mappings_size, sizeof(uint32_t)); + std::unique_ptr<uint32_t[]> storage(new uint32_t[max_mappings_size / sizeof(uint32_t)]); + MethodBssMapping* mappings = new(storage.get()) MethodBssMapping(number_of_method_indexes); + mappings->ClearPadding(); + // Encode the MethodBssMapping. + auto init_it = mappings->begin(); + bool first_index = true; + for (uint32_t method_index : method_indexes.Indexes()) { + size_t bss_offset = bss_method_entries_.Get(MethodReference(dex_file, method_index)); + if (first_index) { + first_index = false; + encoder.Reset(method_index, bss_offset); + } else if (!encoder.TryMerge(method_index, bss_offset)) { + *init_it = encoder.GetEntry(); + ++init_it; + encoder.Reset(method_index, bss_offset); + } + } + // Store the last entry and shrink the mapping to the actual size. + *init_it = encoder.GetEntry(); + ++init_it; + DCHECK(init_it <= mappings->end()); + mappings->SetSize(std::distance(mappings->begin(), init_it)); + size_t mappings_size = MethodBssMapping::ComputeSize(mappings->size()); + + DCHECK_EQ(relative_offset, oat_dex_file->method_bss_mapping_offset_); + DCHECK_OFFSET(); + if (!out->WriteFully(storage.get(), mappings_size)) { + return 0u; + } + size_method_bss_mappings_ += mappings_size; + relative_offset += mappings_size; + } else { + DCHECK_EQ(0u, oat_dex_file->method_bss_mapping_offset_); + } + } + return relative_offset; +} + +size_t OatWriter::WriteOatDexFiles(OutputStream* out, size_t file_offset, size_t relative_offset) { + TimingLogger::ScopedTiming split("WriteOatDexFiles", timings_); + + for (size_t i = 0, size = oat_dex_files_.size(); i != size; ++i) { + OatDexFile* oat_dex_file = &oat_dex_files_[i]; + DCHECK_EQ(relative_offset, oat_dex_file->offset_); + DCHECK_OFFSET(); + + // Write OatDexFile. + if (!oat_dex_file->Write(this, out)) { + return 0u; + } + relative_offset += oat_dex_file->SizeOf(); + } + + return relative_offset; +} + +size_t OatWriter::WriteCode(OutputStream* out, size_t file_offset, size_t relative_offset) { if (compiler_driver_->GetCompilerOptions().IsBootImage()) { InstructionSet instruction_set = compiler_driver_->GetInstructionSet(); @@ -2253,7 +2441,7 @@ size_t OatWriter::WriteCode(OutputStream* out, const size_t file_offset, size_t } size_t OatWriter::WriteCodeDexFiles(OutputStream* out, - const size_t file_offset, + size_t file_offset, size_t relative_offset) { #define VISIT(VisitorType) \ do { \ @@ -2667,50 +2855,6 @@ bool OatWriter::WriteDexFile(OutputStream* out, return true; } -bool OatWriter::WriteOatDexFiles(OutputStream* rodata) { - TimingLogger::ScopedTiming split("WriteOatDexFiles", timings_); - - off_t initial_offset = rodata->Seek(0, kSeekCurrent); - if (initial_offset == static_cast<off_t>(-1)) { - LOG(ERROR) << "Failed to get current position in " << rodata->GetLocation(); - return false; - } - - // Seek to the start of OatDexFiles, i.e. to the end of the OatHeader. If there are - // no OatDexFiles, no data is actually written to .rodata before WriteHeader() and - // this Seek() ensures that we reserve the space for OatHeader in .rodata. - DCHECK(oat_dex_files_.empty() || oat_dex_files_[0u].offset_ == oat_header_->GetHeaderSize()); - uint32_t expected_offset = oat_data_offset_ + oat_header_->GetHeaderSize(); - off_t actual_offset = rodata->Seek(expected_offset, kSeekSet); - if (static_cast<uint32_t>(actual_offset) != expected_offset) { - PLOG(ERROR) << "Failed to seek to OatDexFile table section. Actual: " << actual_offset - << " Expected: " << expected_offset << " File: " << rodata->GetLocation(); - return false; - } - - for (size_t i = 0, size = oat_dex_files_.size(); i != size; ++i) { - OatDexFile* oat_dex_file = &oat_dex_files_[i]; - - DCHECK_EQ(oat_data_offset_ + oat_dex_file->offset_, - static_cast<size_t>(rodata->Seek(0, kSeekCurrent))); - - // Write OatDexFile. - if (!oat_dex_file->Write(this, rodata)) { - PLOG(ERROR) << "Failed to write oat dex information to " << rodata->GetLocation(); - return false; - } - } - - // Seek back to the initial position. - if (rodata->Seek(initial_offset, kSeekSet) != initial_offset) { - PLOG(ERROR) << "Failed to seek to initial position. Actual: " << actual_offset - << " Expected: " << initial_offset << " File: " << rodata->GetLocation(); - return false; - } - - return true; -} - bool OatWriter::OpenDexFiles( File* file, bool verify, @@ -2929,14 +3073,18 @@ bool OatWriter::WriteChecksumsAndVdexHeader(OutputStream* vdex_out) { } bool OatWriter::WriteCodeAlignment(OutputStream* out, uint32_t aligned_code_delta) { + return WriteUpTo16BytesAlignment(out, aligned_code_delta, &size_code_alignment_); +} + +bool OatWriter::WriteUpTo16BytesAlignment(OutputStream* out, uint32_t size, uint32_t* stat) { static const uint8_t kPadding[] = { 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u }; - DCHECK_LE(aligned_code_delta, sizeof(kPadding)); - if (UNLIKELY(!out->WriteFully(kPadding, aligned_code_delta))) { + DCHECK_LE(size, sizeof(kPadding)); + if (UNLIKELY(!out->WriteFully(kPadding, size))) { return false; } - size_code_alignment_ += aligned_code_delta; + *stat += size; return true; } @@ -2965,6 +3113,7 @@ OatWriter::OatDexFile::OatDexFile(const char* dex_file_location, dex_file_offset_(0u), class_offsets_offset_(0u), lookup_table_offset_(0u), + method_bss_mapping_offset_(0u), class_offsets_() { } @@ -2974,19 +3123,8 @@ size_t OatWriter::OatDexFile::SizeOf() const { + sizeof(dex_file_location_checksum_) + sizeof(dex_file_offset_) + sizeof(class_offsets_offset_) - + sizeof(lookup_table_offset_); -} - -void OatWriter::OatDexFile::ReserveClassOffsets(OatWriter* oat_writer) { - DCHECK_EQ(class_offsets_offset_, 0u); - if (!class_offsets_.empty()) { - // Class offsets are required to be 4 byte aligned. - size_t initial_offset = oat_writer->oat_size_; - size_t offset = RoundUp(initial_offset, 4); - oat_writer->size_oat_class_offsets_alignment_ += offset - initial_offset; - class_offsets_offset_ = offset; - oat_writer->oat_size_ = offset + GetClassOffsetsRawSize(); - } + + sizeof(lookup_table_offset_) + + sizeof(method_bss_mapping_offset_); } bool OatWriter::OatDexFile::Write(OatWriter* oat_writer, OutputStream* out) const { @@ -3029,6 +3167,12 @@ bool OatWriter::OatDexFile::Write(OatWriter* oat_writer, OutputStream* out) cons } oat_writer->size_oat_dex_file_lookup_table_offset_ += sizeof(lookup_table_offset_); + if (!out->WriteFully(&method_bss_mapping_offset_, sizeof(method_bss_mapping_offset_))) { + PLOG(ERROR) << "Failed to write method bss mapping offset to " << out->GetLocation(); + return false; + } + oat_writer->size_oat_dex_file_method_bss_mapping_offset_ += sizeof(method_bss_mapping_offset_); + return true; } diff --git a/compiler/oat_writer.h b/compiler/oat_writer.h index 66b70ade2e..9217701bc5 100644 --- a/compiler/oat_writer.h +++ b/compiler/oat_writer.h @@ -60,11 +60,6 @@ namespace verifier { // OatHeader variable length with count of D OatDexFiles // -// OatDexFile[0] one variable sized OatDexFile with offsets to Dex and OatClasses -// OatDexFile[1] -// ... -// OatDexFile[D] -// // TypeLookupTable[0] one descriptor to class def index hash table for each OatDexFile. // TypeLookupTable[1] // ... @@ -80,20 +75,25 @@ namespace verifier { // ... // OatClass[C] // -// GcMap one variable sized blob with GC map. -// GcMap GC maps are deduplicated. +// MethodBssMapping one variable sized MethodBssMapping for each dex file, optional. +// MethodBssMapping // ... -// GcMap +// MethodBssMapping // -// VmapTable one variable sized VmapTable blob (quick compiler only). +// VmapTable one variable sized VmapTable blob (CodeInfo or QuickeningInfo). // VmapTable VmapTables are deduplicated. // ... // VmapTable // -// MappingTable one variable sized blob with MappingTable (quick compiler only). -// MappingTable MappingTables are deduplicated. +// MethodInfo one variable sized blob with MethodInfo. +// MethodInfo MethodInfos are deduplicated. +// ... +// MethodInfo +// +// OatDexFile[0] one variable sized OatDexFile with offsets to Dex and OatClasses +// OatDexFile[1] // ... -// MappingTable +// OatDexFile[D] // // padding if necessary so that the following code will be page aligned // @@ -217,6 +217,10 @@ class OatWriter { return bss_size_; } + size_t GetBssMethodsOffset() const { + return bss_methods_offset_; + } + size_t GetBssRootsOffset() const { return bss_roots_offset_; } @@ -251,6 +255,7 @@ class OatWriter { // to actually write it. class DexMethodVisitor; class OatDexMethodVisitor; + class InitBssLayoutMethodVisitor; class InitOatClassesMethodVisitor; class InitCodeMethodVisitor; class InitMapMethodVisitor; @@ -295,26 +300,30 @@ class OatWriter { const InstructionSetFeatures* instruction_set_features, uint32_t num_dex_files, SafeMap<std::string, std::string>* key_value_store); - size_t InitOatDexFiles(size_t offset); + size_t InitClassOffsets(size_t offset); size_t InitOatClasses(size_t offset); size_t InitOatMaps(size_t offset); + size_t InitMethodBssMappings(size_t offset); + size_t InitOatDexFiles(size_t offset); size_t InitOatCode(size_t offset); size_t InitOatCodeDexFiles(size_t offset); void InitBssLayout(InstructionSet instruction_set); - bool WriteClassOffsets(OutputStream* out); - bool WriteClasses(OutputStream* out); - size_t WriteMaps(OutputStream* out, const size_t file_offset, size_t relative_offset); - size_t WriteCode(OutputStream* out, const size_t file_offset, size_t relative_offset); - size_t WriteCodeDexFiles(OutputStream* out, const size_t file_offset, size_t relative_offset); + size_t WriteClassOffsets(OutputStream* out, size_t file_offset, size_t relative_offset); + size_t WriteClasses(OutputStream* out, size_t file_offset, size_t relative_offset); + size_t WriteMaps(OutputStream* out, size_t file_offset, size_t relative_offset); + size_t WriteMethodBssMappings(OutputStream* out, size_t file_offset, size_t relative_offset); + size_t WriteOatDexFiles(OutputStream* out, size_t file_offset, size_t relative_offset); + size_t WriteCode(OutputStream* out, size_t file_offset, size_t relative_offset); + size_t WriteCodeDexFiles(OutputStream* out, size_t file_offset, size_t relative_offset); bool RecordOatDataOffset(OutputStream* out); bool ReadDexFileHeader(File* oat_file, OatDexFile* oat_dex_file); bool ValidateDexFileHeader(const uint8_t* raw_header, const char* location); - bool WriteOatDexFiles(OutputStream* oat_rodata); bool WriteTypeLookupTables(OutputStream* oat_rodata, const std::vector<std::unique_ptr<const DexFile>>& opened_dex_files); bool WriteCodeAlignment(OutputStream* out, uint32_t aligned_code_delta); + bool WriteUpTo16BytesAlignment(OutputStream* out, uint32_t size, uint32_t* stat); void SetMultiOatRelativePatcherAdjustment(); void CloseSources(); @@ -368,9 +377,20 @@ class OatWriter { // The size of the required .bss section holding the DexCache data and GC roots. size_t bss_size_; + // The offset of the methods in .bss section. + size_t bss_methods_offset_; + // The offset of the GC roots in .bss section. size_t bss_roots_offset_; + // Map for recording references to ArtMethod entries in .bss. + SafeMap<const DexFile*, BitVector> bss_method_entry_references_; + + // Map for allocating ArtMethod entries in .bss. Indexed by MethodReference for the target + // method in the dex file with the "method reference value comparator" for deduplication. + // The value is the target offset for patching, starting at `bss_start_ + bss_methods_offset_`. + SafeMap<MethodReference, size_t, MethodReferenceValueComparator> bss_method_entries_; + // Map for allocating Class entries in .bss. Indexed by TypeReference for the source // type in the dex file with the "type value comparator" for deduplication. The value // is the target offset for patching, starting at `bss_start_ + bss_roots_offset_`. @@ -381,10 +401,6 @@ class OatWriter { // is the target offset for patching, starting at `bss_start_ + bss_roots_offset_`. SafeMap<StringReference, size_t, StringReferenceValueComparator> bss_string_entries_; - // Offsets of the dex cache arrays for each app dex file. For the - // boot image, this information is provided by the ImageWriter. - SafeMap<const DexFile*, size_t> dex_cache_arrays_offsets_; // DexFiles not owned. - // Offset of the oat data from the start of the mmapped region of the elf file. size_t oat_data_offset_; @@ -434,6 +450,7 @@ class OatWriter { uint32_t size_oat_dex_file_offset_; uint32_t size_oat_dex_file_class_offsets_offset_; uint32_t size_oat_dex_file_lookup_table_offset_; + uint32_t size_oat_dex_file_method_bss_mapping_offset_; uint32_t size_oat_lookup_table_alignment_; uint32_t size_oat_lookup_table_; uint32_t size_oat_class_offsets_alignment_; @@ -442,6 +459,7 @@ class OatWriter { uint32_t size_oat_class_status_; uint32_t size_oat_class_method_bitmaps_; uint32_t size_oat_class_method_offsets_; + uint32_t size_method_bss_mappings_; // The helper for processing relative patches is external so that we can patch across oat files. linker::MultiOatRelativePatcher* relative_patcher_; diff --git a/compiler/optimizing/code_generator.cc b/compiler/optimizing/code_generator.cc index c918ee6687..93234f9630 100644 --- a/compiler/optimizing/code_generator.cc +++ b/compiler/optimizing/code_generator.cc @@ -41,6 +41,8 @@ #include "code_generator_mips64.h" #endif +#include "base/bit_utils.h" +#include "base/bit_utils_iterator.h" #include "bytecode_utils.h" #include "class_linker.h" #include "compiled_method.h" @@ -337,7 +339,7 @@ void CodeGenerator::CreateCommonInvokeLocationSummary( case HInvokeStaticOrDirect::MethodLoadKind::kRecursive: locations->SetInAt(call->GetSpecialInputIndex(), visitor->GetMethodLocation()); break; - case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: + case HInvokeStaticOrDirect::MethodLoadKind::kRuntimeCall: locations->AddTemp(visitor->GetMethodLocation()); locations->SetInAt(call->GetSpecialInputIndex(), Location::RequiresRegister()); break; @@ -350,6 +352,34 @@ void CodeGenerator::CreateCommonInvokeLocationSummary( } } +void CodeGenerator::GenerateInvokeStaticOrDirectRuntimeCall( + HInvokeStaticOrDirect* invoke, Location temp, SlowPathCode* slow_path) { + MoveConstant(temp, invoke->GetDexMethodIndex()); + + // The access check is unnecessary but we do not want to introduce + // extra entrypoints for the codegens that do not support some + // invoke type and fall back to the runtime call. + + // Initialize to anything to silent compiler warnings. + QuickEntrypointEnum entrypoint = kQuickInvokeStaticTrampolineWithAccessCheck; + switch (invoke->GetInvokeType()) { + case kStatic: + entrypoint = kQuickInvokeStaticTrampolineWithAccessCheck; + break; + case kDirect: + entrypoint = kQuickInvokeDirectTrampolineWithAccessCheck; + break; + case kSuper: + entrypoint = kQuickInvokeSuperTrampolineWithAccessCheck; + break; + case kVirtual: + case kInterface: + LOG(FATAL) << "Unexpected invoke type: " << invoke->GetInvokeType(); + UNREACHABLE(); + } + + InvokeRuntime(entrypoint, invoke, invoke->GetDexPc(), slow_path); +} void CodeGenerator::GenerateInvokeUnresolvedRuntimeCall(HInvokeUnresolved* invoke) { MoveConstant(invoke->GetLocations()->GetTemp(0), invoke->GetDexMethodIndex()); diff --git a/compiler/optimizing/code_generator.h b/compiler/optimizing/code_generator.h index c9ba5c3357..7bf43f7971 100644 --- a/compiler/optimizing/code_generator.h +++ b/compiler/optimizing/code_generator.h @@ -496,6 +496,8 @@ class CodeGenerator : public DeletableArenaObject<kArenaAllocCodeGenerator> { static void CreateCommonInvokeLocationSummary( HInvoke* invoke, InvokeDexCallingConventionVisitor* visitor); + void GenerateInvokeStaticOrDirectRuntimeCall( + HInvokeStaticOrDirect* invoke, Location temp, SlowPathCode* slow_path); void GenerateInvokeUnresolvedRuntimeCall(HInvokeUnresolved* invoke); void GenerateInvokePolymorphicCall(HInvokePolymorphic* invoke); @@ -564,9 +566,11 @@ class CodeGenerator : public DeletableArenaObject<kArenaAllocCodeGenerator> { HInvokeStaticOrDirect* invoke) = 0; // Generate a call to a static or direct method. - virtual void GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) = 0; + virtual void GenerateStaticOrDirectCall( + HInvokeStaticOrDirect* invoke, Location temp, SlowPathCode* slow_path = nullptr) = 0; // Generate a call to a virtual method. - virtual void GenerateVirtualCall(HInvokeVirtual* invoke, Location temp) = 0; + virtual void GenerateVirtualCall( + HInvokeVirtual* invoke, Location temp, SlowPathCode* slow_path = nullptr) = 0; // Copy the result of a call into the given target. virtual void MoveFromReturnRegister(Location trg, Primitive::Type type) = 0; diff --git a/compiler/optimizing/code_generator_arm.cc b/compiler/optimizing/code_generator_arm.cc index 097e4833d0..e4efbef394 100644 --- a/compiler/optimizing/code_generator_arm.cc +++ b/compiler/optimizing/code_generator_arm.cc @@ -19,6 +19,8 @@ #include "arch/arm/asm_support_arm.h" #include "arch/arm/instruction_set_features_arm.h" #include "art_method.h" +#include "base/bit_utils.h" +#include "base/bit_utils_iterator.h" #include "code_generator_utils.h" #include "common_arm.h" #include "compiled_method.h" @@ -47,7 +49,6 @@ static bool ExpectedPairLayout(Location location) { return ((location.low() & 1) == 0) && (location.low() + 1 == location.high()); } -static constexpr int kCurrentMethodStackOffset = 0; static constexpr Register kMethodRegisterArgument = R0; static constexpr Register kCoreAlwaysSpillRegister = R5; @@ -2396,8 +2397,8 @@ CodeGeneratorARM::CodeGeneratorARM(HGraph* graph, isa_features_(isa_features), uint32_literals_(std::less<uint32_t>(), graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), - pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), pc_relative_method_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + method_bss_entry_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), pc_relative_type_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), type_bss_entry_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), pc_relative_string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), @@ -3554,18 +3555,10 @@ void LocationsBuilderARM::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invok IntrinsicLocationsBuilderARM intrinsic(codegen_); if (intrinsic.TryDispatch(invoke)) { - if (invoke->GetLocations()->CanCall() && invoke->HasPcRelativeDexCache()) { - invoke->GetLocations()->SetInAt(invoke->GetSpecialInputIndex(), Location::Any()); - } return; } HandleInvoke(invoke); - - // For PC-relative dex cache the invoke has an extra input, the PC-relative address base. - if (invoke->HasPcRelativeDexCache()) { - invoke->GetLocations()->SetInAt(invoke->GetSpecialInputIndex(), Location::RequiresRegister()); - } } static bool TryGenerateIntrinsicCode(HInvoke* invoke, CodeGeneratorARM* codegen) { @@ -3589,7 +3582,6 @@ void InstructionCodeGeneratorARM::VisitInvokeStaticOrDirect(HInvokeStaticOrDirec LocationSummary* locations = invoke->GetLocations(); codegen_->GenerateStaticOrDirectCall( invoke, locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation()); - codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); } void LocationsBuilderARM::HandleInvoke(HInvoke* invoke) { @@ -3613,7 +3605,6 @@ void InstructionCodeGeneratorARM::VisitInvokeVirtual(HInvokeVirtual* invoke) { codegen_->GenerateVirtualCall(invoke, invoke->GetLocations()->GetTemp(0)); DCHECK(!codegen_->IsLeafMethod()); - codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); } void LocationsBuilderARM::VisitInvokeInterface(HInvokeInterface* invoke) { @@ -8955,7 +8946,8 @@ Register CodeGeneratorARM::GetInvokeStaticOrDirectExtraParameter(HInvokeStaticOr return location.AsRegister<Register>(); } -void CodeGeneratorARM::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) { +void CodeGeneratorARM::GenerateStaticOrDirectCall( + HInvokeStaticOrDirect* invoke, Location temp, SlowPathCode* slow_path) { Location callee_method = temp; // For all kinds except kRecursive, callee will be in temp. switch (invoke->GetMethodLoadKind()) { case HInvokeStaticOrDirect::MethodLoadKind::kStringInit: { @@ -8983,38 +8975,23 @@ void CodeGeneratorARM::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress: __ LoadImmediate(temp.AsRegister<Register>(), invoke->GetMethodAddress()); break; - case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: { - HArmDexCacheArraysBase* base = - invoke->InputAt(invoke->GetSpecialInputIndex())->AsArmDexCacheArraysBase(); - Register base_reg = GetInvokeStaticOrDirectExtraParameter(invoke, - temp.AsRegister<Register>()); - int32_t offset = invoke->GetDexCacheArrayOffset() - base->GetElementOffset(); - __ LoadFromOffset(kLoadWord, temp.AsRegister<Register>(), base_reg, offset); - break; - } - case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: { - Location current_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); - Register method_reg; - Register reg = temp.AsRegister<Register>(); - if (current_method.IsRegister()) { - method_reg = current_method.AsRegister<Register>(); - } else { - DCHECK(invoke->GetLocations()->Intrinsified()); - DCHECK(!current_method.IsValid()); - method_reg = reg; - __ LoadFromOffset(kLoadWord, reg, SP, kCurrentMethodStackOffset); - } - // /* ArtMethod*[] */ temp = temp.ptr_sized_fields_->dex_cache_resolved_methods_; - __ LoadFromOffset(kLoadWord, - reg, - method_reg, - ArtMethod::DexCacheResolvedMethodsOffset(kArmPointerSize).Int32Value()); - // temp = temp[index_in_cache]; - // Note: Don't use invoke->GetTargetMethod() as it may point to a different dex file. - uint32_t index_in_cache = invoke->GetDexMethodIndex(); - __ LoadFromOffset(kLoadWord, reg, reg, CodeGenerator::GetCachePointerOffset(index_in_cache)); + case HInvokeStaticOrDirect::MethodLoadKind::kBssEntry: { + Register temp_reg = temp.AsRegister<Register>(); + PcRelativePatchInfo* labels = NewMethodBssEntryPatch( + MethodReference(&GetGraph()->GetDexFile(), invoke->GetDexMethodIndex())); + __ BindTrackedLabel(&labels->movw_label); + __ movw(temp_reg, /* placeholder */ 0u); + __ BindTrackedLabel(&labels->movt_label); + __ movt(temp_reg, /* placeholder */ 0u); + __ BindTrackedLabel(&labels->add_pc_label); + __ add(temp_reg, temp_reg, ShifterOperand(PC)); + __ LoadFromOffset(kLoadWord, temp_reg, temp_reg, /* offset */ 0); break; } + case HInvokeStaticOrDirect::MethodLoadKind::kRuntimeCall: { + GenerateInvokeStaticOrDirectRuntimeCall(invoke, temp, slow_path); + return; // No code pointer retrieval; the runtime performs the call directly. + } } switch (invoke->GetCodePtrLocation()) { @@ -9030,11 +9007,13 @@ void CodeGeneratorARM::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, __ blx(LR); break; } + RecordPcInfo(invoke, invoke->GetDexPc(), slow_path); DCHECK(!IsLeafMethod()); } -void CodeGeneratorARM::GenerateVirtualCall(HInvokeVirtual* invoke, Location temp_location) { +void CodeGeneratorARM::GenerateVirtualCall( + HInvokeVirtual* invoke, Location temp_location, SlowPathCode* slow_path) { Register temp = temp_location.AsRegister<Register>(); uint32_t method_offset = mirror::Class::EmbeddedVTableEntryOffset( invoke->GetVTableIndex(), kArmPointerSize).Uint32Value(); @@ -9065,6 +9044,7 @@ void CodeGeneratorARM::GenerateVirtualCall(HInvokeVirtual* invoke, Location temp __ LoadFromOffset(kLoadWord, LR, temp, entry_point); // LR(); __ blx(LR); + RecordPcInfo(invoke, invoke->GetDexPc(), slow_path); } CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativeMethodPatch( @@ -9074,6 +9054,13 @@ CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativeMethodPatc &pc_relative_method_patches_); } +CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewMethodBssEntryPatch( + MethodReference target_method) { + return NewPcRelativePatch(*target_method.dex_file, + target_method.dex_method_index, + &method_bss_entry_patches_); +} + CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativeTypePatch( const DexFile& dex_file, dex::TypeIndex type_index) { return NewPcRelativePatch(dex_file, type_index.index_, &pc_relative_type_patches_); @@ -9089,11 +9076,6 @@ CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativeStringPatc return NewPcRelativePatch(dex_file, string_index.index_, &pc_relative_string_patches_); } -CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativeDexCacheArrayPatch( - const DexFile& dex_file, uint32_t element_offset) { - return NewPcRelativePatch(dex_file, element_offset, &pc_relative_dex_cache_patches_); -} - CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativePatch( const DexFile& dex_file, uint32_t offset_or_index, ArenaDeque<PcRelativePatchInfo>* patches) { patches->emplace_back(dex_file, offset_or_index); @@ -9152,15 +9134,13 @@ inline void CodeGeneratorARM::EmitPcRelativeLinkerPatches( void CodeGeneratorARM::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) { DCHECK(linker_patches->empty()); size_t size = - /* MOVW+MOVT for each entry */ 2u * pc_relative_dex_cache_patches_.size() + /* MOVW+MOVT for each entry */ 2u * pc_relative_method_patches_.size() + + /* MOVW+MOVT for each entry */ 2u * method_bss_entry_patches_.size() + /* MOVW+MOVT for each entry */ 2u * pc_relative_type_patches_.size() + /* MOVW+MOVT for each entry */ 2u * type_bss_entry_patches_.size() + /* MOVW+MOVT for each entry */ 2u * pc_relative_string_patches_.size() + baker_read_barrier_patches_.size(); linker_patches->reserve(size); - EmitPcRelativeLinkerPatches<LinkerPatch::DexCacheArrayPatch>(pc_relative_dex_cache_patches_, - linker_patches); if (GetCompilerOptions().IsBootImage()) { EmitPcRelativeLinkerPatches<LinkerPatch::RelativeMethodPatch>(pc_relative_method_patches_, linker_patches); @@ -9174,6 +9154,8 @@ void CodeGeneratorARM::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patche EmitPcRelativeLinkerPatches<LinkerPatch::StringBssEntryPatch>(pc_relative_string_patches_, linker_patches); } + EmitPcRelativeLinkerPatches<LinkerPatch::MethodBssEntryPatch>(method_bss_entry_patches_, + linker_patches); EmitPcRelativeLinkerPatches<LinkerPatch::TypeBssEntryPatch>(type_bss_entry_patches_, linker_patches); for (const BakerReadBarrierPatchInfo& info : baker_read_barrier_patches_) { @@ -9310,23 +9292,6 @@ void InstructionCodeGeneratorARM::VisitPackedSwitch(HPackedSwitch* switch_instr) } } -void LocationsBuilderARM::VisitArmDexCacheArraysBase(HArmDexCacheArraysBase* base) { - LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(base); - locations->SetOut(Location::RequiresRegister()); -} - -void InstructionCodeGeneratorARM::VisitArmDexCacheArraysBase(HArmDexCacheArraysBase* base) { - Register base_reg = base->GetLocations()->Out().AsRegister<Register>(); - CodeGeneratorARM::PcRelativePatchInfo* labels = - codegen_->NewPcRelativeDexCacheArrayPatch(base->GetDexFile(), base->GetElementOffset()); - __ BindTrackedLabel(&labels->movw_label); - __ movw(base_reg, /* placeholder */ 0u); - __ BindTrackedLabel(&labels->movt_label); - __ movt(base_reg, /* placeholder */ 0u); - __ BindTrackedLabel(&labels->add_pc_label); - __ add(base_reg, base_reg, ShifterOperand(PC)); -} - void CodeGeneratorARM::MoveFromReturnRegister(Location trg, Primitive::Type type) { if (!trg.IsValid()) { DCHECK_EQ(type, Primitive::kPrimVoid); diff --git a/compiler/optimizing/code_generator_arm.h b/compiler/optimizing/code_generator_arm.h index 5f37d3bff1..9280e6377c 100644 --- a/compiler/optimizing/code_generator_arm.h +++ b/compiler/optimizing/code_generator_arm.h @@ -455,8 +455,10 @@ class CodeGeneratorARM : public CodeGenerator { const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info, HInvokeStaticOrDirect* invoke) OVERRIDE; - void GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) OVERRIDE; - void GenerateVirtualCall(HInvokeVirtual* invoke, Location temp) OVERRIDE; + void GenerateStaticOrDirectCall( + HInvokeStaticOrDirect* invoke, Location temp, SlowPathCode* slow_path = nullptr) OVERRIDE; + void GenerateVirtualCall( + HInvokeVirtual* invoke, Location temp, SlowPathCode* slow_path = nullptr) OVERRIDE; void MoveFromReturnRegister(Location trg, Primitive::Type type) OVERRIDE; @@ -481,12 +483,11 @@ class CodeGeneratorARM : public CodeGenerator { }; PcRelativePatchInfo* NewPcRelativeMethodPatch(MethodReference target_method); + PcRelativePatchInfo* NewMethodBssEntryPatch(MethodReference target_method); PcRelativePatchInfo* NewPcRelativeTypePatch(const DexFile& dex_file, dex::TypeIndex type_index); PcRelativePatchInfo* NewTypeBssEntryPatch(const DexFile& dex_file, dex::TypeIndex type_index); PcRelativePatchInfo* NewPcRelativeStringPatch(const DexFile& dex_file, dex::StringIndex string_index); - PcRelativePatchInfo* NewPcRelativeDexCacheArrayPatch(const DexFile& dex_file, - uint32_t element_offset); // Add a new baker read barrier patch and return the label to be bound // before the BNE instruction. @@ -667,10 +668,10 @@ class CodeGeneratorARM : public CodeGenerator { // Deduplication map for 32-bit literals, used for non-patchable boot image addresses. Uint32ToLiteralMap uint32_literals_; - // PC-relative patch info for each HArmDexCacheArraysBase. - ArenaDeque<PcRelativePatchInfo> pc_relative_dex_cache_patches_; // PC-relative method patch info for kBootImageLinkTimePcRelative. ArenaDeque<PcRelativePatchInfo> pc_relative_method_patches_; + // PC-relative method patch info for kBssEntry. + ArenaDeque<PcRelativePatchInfo> method_bss_entry_patches_; // PC-relative type patch info for kBootImageLinkTimePcRelative. ArenaDeque<PcRelativePatchInfo> pc_relative_type_patches_; // PC-relative type patch info for kBssEntry. diff --git a/compiler/optimizing/code_generator_arm64.cc b/compiler/optimizing/code_generator_arm64.cc index d8e709c7a9..34397e66bc 100644 --- a/compiler/optimizing/code_generator_arm64.cc +++ b/compiler/optimizing/code_generator_arm64.cc @@ -19,6 +19,8 @@ #include "arch/arm64/asm_support_arm64.h" #include "arch/arm64/instruction_set_features_arm64.h" #include "art_method.h" +#include "base/bit_utils.h" +#include "base/bit_utils_iterator.h" #include "code_generator_utils.h" #include "compiled_method.h" #include "entrypoints/quick/quick_entrypoints.h" @@ -78,7 +80,6 @@ using helpers::VIXLRegCodeFromART; using helpers::WRegisterFrom; using helpers::XRegisterFrom; -static constexpr int kCurrentMethodStackOffset = 0; // The compare/jump sequence will generate about (1.5 * num_entries + 3) instructions. While jump // table version generates 7 instructions and num_entries literals. Compare/jump sequence will // generates less code/data with a small num_entries. @@ -1449,8 +1450,8 @@ CodeGeneratorARM64::CodeGeneratorARM64(HGraph* graph, graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), uint64_literals_(std::less<uint64_t>(), graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), - pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), pc_relative_method_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + method_bss_entry_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), pc_relative_type_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), type_bss_entry_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), pc_relative_string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), @@ -4497,7 +4498,8 @@ HInvokeStaticOrDirect::DispatchInfo CodeGeneratorARM64::GetSupportedInvokeStatic return desired_dispatch_info; } -void CodeGeneratorARM64::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) { +void CodeGeneratorARM64::GenerateStaticOrDirectCall( + HInvokeStaticOrDirect* invoke, Location temp, SlowPathCode* slow_path) { // Make sure that ArtMethod* is passed in kArtMethodRegister as per the calling convention. Location callee_method = temp; // For all kinds except kRecursive, callee will be in temp. switch (invoke->GetMethodLoadKind()) { @@ -4526,46 +4528,33 @@ void CodeGeneratorARM64::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invok // Load method address from literal pool. __ Ldr(XRegisterFrom(temp), DeduplicateUint64Literal(invoke->GetMethodAddress())); break; - case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: { + case HInvokeStaticOrDirect::MethodLoadKind::kBssEntry: { // Add ADRP with its PC-relative DexCache access patch. - const DexFile& dex_file = invoke->GetDexFileForPcRelativeDexCache(); - uint32_t element_offset = invoke->GetDexCacheArrayOffset(); - vixl::aarch64::Label* adrp_label = NewPcRelativeDexCacheArrayPatch(dex_file, element_offset); + MethodReference target_method(&GetGraph()->GetDexFile(), invoke->GetDexMethodIndex()); + vixl::aarch64::Label* adrp_label = NewMethodBssEntryPatch(target_method); EmitAdrpPlaceholder(adrp_label, XRegisterFrom(temp)); // Add LDR with its PC-relative DexCache access patch. vixl::aarch64::Label* ldr_label = - NewPcRelativeDexCacheArrayPatch(dex_file, element_offset, adrp_label); + NewMethodBssEntryPatch(target_method, adrp_label); EmitLdrOffsetPlaceholder(ldr_label, XRegisterFrom(temp), XRegisterFrom(temp)); break; } - case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: { - Location current_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); - Register reg = XRegisterFrom(temp); - Register method_reg; - if (current_method.IsRegister()) { - method_reg = XRegisterFrom(current_method); - } else { - DCHECK(invoke->GetLocations()->Intrinsified()); - DCHECK(!current_method.IsValid()); - method_reg = reg; - __ Ldr(reg.X(), MemOperand(sp, kCurrentMethodStackOffset)); - } - - // /* ArtMethod*[] */ temp = temp.ptr_sized_fields_->dex_cache_resolved_methods_; - __ Ldr(reg.X(), - MemOperand(method_reg.X(), - ArtMethod::DexCacheResolvedMethodsOffset(kArm64PointerSize).Int32Value())); - // temp = temp[index_in_cache]; - // Note: Don't use invoke->GetTargetMethod() as it may point to a different dex file. - uint32_t index_in_cache = invoke->GetDexMethodIndex(); - __ Ldr(reg.X(), MemOperand(reg.X(), GetCachePointerOffset(index_in_cache))); - break; + case HInvokeStaticOrDirect::MethodLoadKind::kRuntimeCall: { + GenerateInvokeStaticOrDirectRuntimeCall(invoke, temp, slow_path); + return; // No code pointer retrieval; the runtime performs the call directly. } } switch (invoke->GetCodePtrLocation()) { case HInvokeStaticOrDirect::CodePtrLocation::kCallSelf: - __ Bl(&frame_entry_label_); + { + // Use a scope to help guarantee that `RecordPcInfo()` records the correct pc. + ExactAssemblyScope eas(GetVIXLAssembler(), + kInstructionSize, + CodeBufferCheckScope::kExactSize); + __ bl(&frame_entry_label_); + RecordPcInfo(invoke, invoke->GetDexPc(), slow_path); + } break; case HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod: // LR = callee_method->entry_point_from_quick_compiled_code_; @@ -4573,14 +4562,13 @@ void CodeGeneratorARM64::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invok XRegisterFrom(callee_method), ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArm64PointerSize).Int32Value())); { - // To ensure that the pc position is recorded immediately after the `blr` instruction - // BLR must be the last instruction emitted in this function. - // Recording the pc will occur right after returning from this function. + // Use a scope to help guarantee that `RecordPcInfo()` records the correct pc. ExactAssemblyScope eas(GetVIXLAssembler(), kInstructionSize, CodeBufferCheckScope::kExactSize); // lr() __ blr(lr); + RecordPcInfo(invoke, invoke->GetDexPc(), slow_path); } break; } @@ -4588,7 +4576,8 @@ void CodeGeneratorARM64::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invok DCHECK(!IsLeafMethod()); } -void CodeGeneratorARM64::GenerateVirtualCall(HInvokeVirtual* invoke, Location temp_in) { +void CodeGeneratorARM64::GenerateVirtualCall( + HInvokeVirtual* invoke, Location temp_in, SlowPathCode* slow_path) { // Use the calling convention instead of the location of the receiver, as // intrinsics may have put the receiver in a different register. In the intrinsics // slow path, the arguments have been moved to the right place, so here we are @@ -4622,12 +4611,11 @@ void CodeGeneratorARM64::GenerateVirtualCall(HInvokeVirtual* invoke, Location te // lr = temp->GetEntryPoint(); __ Ldr(lr, MemOperand(temp, entry_point.SizeValue())); { - // To ensure that the pc position is recorded immediately after the `blr` instruction - // BLR should be the last instruction emitted in this function. - // Recording the pc will occur right after returning from this function. + // Use a scope to help guarantee that `RecordPcInfo()` records the correct pc. ExactAssemblyScope eas(GetVIXLAssembler(), kInstructionSize, CodeBufferCheckScope::kExactSize); // lr(); __ blr(lr); + RecordPcInfo(invoke, invoke->GetDexPc(), slow_path); } } @@ -4648,6 +4636,15 @@ vixl::aarch64::Label* CodeGeneratorARM64::NewPcRelativeMethodPatch( &pc_relative_method_patches_); } +vixl::aarch64::Label* CodeGeneratorARM64::NewMethodBssEntryPatch( + MethodReference target_method, + vixl::aarch64::Label* adrp_label) { + return NewPcRelativePatch(*target_method.dex_file, + target_method.dex_method_index, + adrp_label, + &method_bss_entry_patches_); +} + vixl::aarch64::Label* CodeGeneratorARM64::NewPcRelativeTypePatch( const DexFile& dex_file, dex::TypeIndex type_index, @@ -4670,13 +4667,6 @@ vixl::aarch64::Label* CodeGeneratorARM64::NewPcRelativeStringPatch( NewPcRelativePatch(dex_file, string_index.index_, adrp_label, &pc_relative_string_patches_); } -vixl::aarch64::Label* CodeGeneratorARM64::NewPcRelativeDexCacheArrayPatch( - const DexFile& dex_file, - uint32_t element_offset, - vixl::aarch64::Label* adrp_label) { - return NewPcRelativePatch(dex_file, element_offset, adrp_label, &pc_relative_dex_cache_patches_); -} - vixl::aarch64::Label* CodeGeneratorARM64::NewBakerReadBarrierPatch(uint32_t custom_data) { baker_read_barrier_patches_.emplace_back(custom_data); return &baker_read_barrier_patches_.back().label; @@ -4698,7 +4688,7 @@ vixl::aarch64::Label* CodeGeneratorARM64::NewPcRelativePatch( vixl::aarch64::Literal<uint32_t>* CodeGeneratorARM64::DeduplicateBootImageAddressLiteral( uint64_t address) { - return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address), &uint32_literals_); + return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address)); } vixl::aarch64::Literal<uint32_t>* CodeGeneratorARM64::DeduplicateJitStringLiteral( @@ -4761,19 +4751,13 @@ inline void CodeGeneratorARM64::EmitPcRelativeLinkerPatches( void CodeGeneratorARM64::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) { DCHECK(linker_patches->empty()); size_t size = - pc_relative_dex_cache_patches_.size() + pc_relative_method_patches_.size() + + method_bss_entry_patches_.size() + pc_relative_type_patches_.size() + type_bss_entry_patches_.size() + pc_relative_string_patches_.size() + baker_read_barrier_patches_.size(); linker_patches->reserve(size); - for (const PcRelativePatchInfo& info : pc_relative_dex_cache_patches_) { - linker_patches->push_back(LinkerPatch::DexCacheArrayPatch(info.label.GetLocation(), - &info.target_dex_file, - info.pc_insn_label->GetLocation(), - info.offset_or_index)); - } if (GetCompilerOptions().IsBootImage()) { EmitPcRelativeLinkerPatches<LinkerPatch::RelativeMethodPatch>(pc_relative_method_patches_, linker_patches); @@ -4787,6 +4771,8 @@ void CodeGeneratorARM64::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patc EmitPcRelativeLinkerPatches<LinkerPatch::StringBssEntryPatch>(pc_relative_string_patches_, linker_patches); } + EmitPcRelativeLinkerPatches<LinkerPatch::MethodBssEntryPatch>(method_bss_entry_patches_, + linker_patches); EmitPcRelativeLinkerPatches<LinkerPatch::TypeBssEntryPatch>(type_bss_entry_patches_, linker_patches); for (const BakerReadBarrierPatchInfo& info : baker_read_barrier_patches_) { @@ -4796,9 +4782,8 @@ void CodeGeneratorARM64::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patc DCHECK_EQ(size, linker_patches->size()); } -vixl::aarch64::Literal<uint32_t>* CodeGeneratorARM64::DeduplicateUint32Literal(uint32_t value, - Uint32ToLiteralMap* map) { - return map->GetOrCreate( +vixl::aarch64::Literal<uint32_t>* CodeGeneratorARM64::DeduplicateUint32Literal(uint32_t value) { + return uint32_literals_.GetOrCreate( value, [this, value]() { return __ CreateLiteralDestroyedWithPool<uint32_t>(value); }); } @@ -4824,7 +4809,6 @@ void InstructionCodeGeneratorARM64::VisitInvokeStaticOrDirect(HInvokeStaticOrDir LocationSummary* locations = invoke->GetLocations(); codegen_->GenerateStaticOrDirectCall( invoke, locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation()); - codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); } void InstructionCodeGeneratorARM64::VisitInvokeVirtual(HInvokeVirtual* invoke) { @@ -4837,7 +4821,6 @@ void InstructionCodeGeneratorARM64::VisitInvokeVirtual(HInvokeVirtual* invoke) { EmissionCheckScope guard(GetVIXLAssembler(), kInvokeCodeMarginSizeInBytes); codegen_->GenerateVirtualCall(invoke, invoke->GetLocations()->GetTemp(0)); DCHECK(!codegen_->IsLeafMethod()); - codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); } HLoadClass::LoadKind CodeGeneratorARM64::GetSupportedLoadClassKind( diff --git a/compiler/optimizing/code_generator_arm64.h b/compiler/optimizing/code_generator_arm64.h index 747fc9f0b1..d9c49d19bb 100644 --- a/compiler/optimizing/code_generator_arm64.h +++ b/compiler/optimizing/code_generator_arm64.h @@ -540,8 +540,10 @@ class CodeGeneratorARM64 : public CodeGenerator { const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info, HInvokeStaticOrDirect* invoke) OVERRIDE; - void GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) OVERRIDE; - void GenerateVirtualCall(HInvokeVirtual* invoke, Location temp) OVERRIDE; + void GenerateStaticOrDirectCall( + HInvokeStaticOrDirect* invoke, Location temp, SlowPathCode* slow_path = nullptr) OVERRIDE; + void GenerateVirtualCall( + HInvokeVirtual* invoke, Location temp, SlowPathCode* slow_path = nullptr) OVERRIDE; void MoveFromReturnRegister(Location trg ATTRIBUTE_UNUSED, Primitive::Type type ATTRIBUTE_UNUSED) OVERRIDE { @@ -555,6 +557,13 @@ class CodeGeneratorARM64 : public CodeGenerator { vixl::aarch64::Label* NewPcRelativeMethodPatch(MethodReference target_method, vixl::aarch64::Label* adrp_label = nullptr); + // Add a new .bss entry method patch for an instruction and return + // the label to be bound before the instruction. The instruction will be + // either the ADRP (pass `adrp_label = null`) or the LDR (pass `adrp_label` + // pointing to the associated ADRP patch label). + vixl::aarch64::Label* NewMethodBssEntryPatch(MethodReference target_method, + vixl::aarch64::Label* adrp_label = nullptr); + // Add a new PC-relative type patch for an instruction and return the label // to be bound before the instruction. The instruction will be either the // ADRP (pass `adrp_label = null`) or the ADD (pass `adrp_label` pointing @@ -579,15 +588,6 @@ class CodeGeneratorARM64 : public CodeGenerator { dex::StringIndex string_index, vixl::aarch64::Label* adrp_label = nullptr); - // Add a new PC-relative dex cache array patch for an instruction and return - // the label to be bound before the instruction. The instruction will be - // either the ADRP (pass `adrp_label = null`) or the LDR (pass `adrp_label` - // pointing to the associated ADRP patch label). - vixl::aarch64::Label* NewPcRelativeDexCacheArrayPatch( - const DexFile& dex_file, - uint32_t element_offset, - vixl::aarch64::Label* adrp_label = nullptr); - // Add a new baker read barrier patch and return the label to be bound // before the CBNZ instruction. vixl::aarch64::Label* NewBakerReadBarrierPatch(uint32_t custom_data); @@ -739,8 +739,7 @@ class CodeGeneratorARM64 : public CodeGenerator { vixl::aarch64::Literal<uint32_t>*, TypeReferenceValueComparator>; - vixl::aarch64::Literal<uint32_t>* DeduplicateUint32Literal(uint32_t value, - Uint32ToLiteralMap* map); + vixl::aarch64::Literal<uint32_t>* DeduplicateUint32Literal(uint32_t value); vixl::aarch64::Literal<uint64_t>* DeduplicateUint64Literal(uint64_t value); // The PcRelativePatchInfo is used for PC-relative addressing of dex cache arrays @@ -791,10 +790,10 @@ class CodeGeneratorARM64 : public CodeGenerator { Uint32ToLiteralMap uint32_literals_; // Deduplication map for 64-bit literals, used for non-patchable method address or method code. Uint64ToLiteralMap uint64_literals_; - // PC-relative DexCache access info. - ArenaDeque<PcRelativePatchInfo> pc_relative_dex_cache_patches_; // PC-relative method patch info for kBootImageLinkTimePcRelative. ArenaDeque<PcRelativePatchInfo> pc_relative_method_patches_; + // PC-relative method patch info for kBssEntry. + ArenaDeque<PcRelativePatchInfo> method_bss_entry_patches_; // PC-relative type patch info for kBootImageLinkTimePcRelative. ArenaDeque<PcRelativePatchInfo> pc_relative_type_patches_; // PC-relative type patch info for kBssEntry. diff --git a/compiler/optimizing/code_generator_arm_vixl.cc b/compiler/optimizing/code_generator_arm_vixl.cc index 4d5f88e14a..c6bd871bc5 100644 --- a/compiler/optimizing/code_generator_arm_vixl.cc +++ b/compiler/optimizing/code_generator_arm_vixl.cc @@ -19,6 +19,8 @@ #include "arch/arm/asm_support_arm.h" #include "arch/arm/instruction_set_features_arm.h" #include "art_method.h" +#include "base/bit_utils.h" +#include "base/bit_utils_iterator.h" #include "code_generator_utils.h" #include "common_arm.h" #include "compiled_method.h" @@ -76,7 +78,6 @@ static bool ExpectedPairLayout(Location location) { // Use a local definition to prevent copying mistakes. static constexpr size_t kArmWordSize = static_cast<size_t>(kArmPointerSize); static constexpr size_t kArmBitsPerWord = kArmWordSize * kBitsPerByte; -static constexpr int kCurrentMethodStackOffset = 0; static constexpr uint32_t kPackedSwitchCompareJumpThreshold = 7; // Reference load (except object array loads) is using LDR Rt, [Rn, #offset] which can handle @@ -2500,8 +2501,8 @@ CodeGeneratorARMVIXL::CodeGeneratorARMVIXL(HGraph* graph, isa_features_(isa_features), uint32_literals_(std::less<uint32_t>(), graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), - pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), pc_relative_method_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + method_bss_entry_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), pc_relative_type_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), type_bss_entry_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), pc_relative_string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), @@ -3643,18 +3644,10 @@ void LocationsBuilderARMVIXL::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* i IntrinsicLocationsBuilderARMVIXL intrinsic(codegen_); if (intrinsic.TryDispatch(invoke)) { - if (invoke->GetLocations()->CanCall() && invoke->HasPcRelativeDexCache()) { - invoke->GetLocations()->SetInAt(invoke->GetSpecialInputIndex(), Location::Any()); - } return; } HandleInvoke(invoke); - - // For PC-relative dex cache the invoke has an extra input, the PC-relative address base. - if (invoke->HasPcRelativeDexCache()) { - invoke->GetLocations()->SetInAt(invoke->GetSpecialInputIndex(), Location::RequiresRegister()); - } } static bool TryGenerateIntrinsicCode(HInvoke* invoke, CodeGeneratorARMVIXL* codegen) { @@ -3678,7 +3671,6 @@ void InstructionCodeGeneratorARMVIXL::VisitInvokeStaticOrDirect(HInvokeStaticOrD LocationSummary* locations = invoke->GetLocations(); codegen_->GenerateStaticOrDirectCall( invoke, locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation()); - codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); } void LocationsBuilderARMVIXL::HandleInvoke(HInvoke* invoke) { @@ -3701,7 +3693,6 @@ void InstructionCodeGeneratorARMVIXL::VisitInvokeVirtual(HInvokeVirtual* invoke) } codegen_->GenerateVirtualCall(invoke, invoke->GetLocations()->GetTemp(0)); - codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); DCHECK(!codegen_->IsLeafMethod()); } @@ -9120,7 +9111,7 @@ vixl32::Register CodeGeneratorARMVIXL::GetInvokeStaticOrDirectExtraParameter( } void CodeGeneratorARMVIXL::GenerateStaticOrDirectCall( - HInvokeStaticOrDirect* invoke, Location temp) { + HInvokeStaticOrDirect* invoke, Location temp, SlowPathCode* slow_path) { Location callee_method = temp; // For all kinds except kRecursive, callee will be in temp. switch (invoke->GetMethodLoadKind()) { case HInvokeStaticOrDirect::MethodLoadKind::kStringInit: { @@ -9143,44 +9134,30 @@ void CodeGeneratorARMVIXL::GenerateStaticOrDirectCall( case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress: __ Mov(RegisterFrom(temp), Operand::From(invoke->GetMethodAddress())); break; - case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: { - HArmDexCacheArraysBase* base = - invoke->InputAt(invoke->GetSpecialInputIndex())->AsArmDexCacheArraysBase(); - vixl32::Register base_reg = GetInvokeStaticOrDirectExtraParameter(invoke, RegisterFrom(temp)); - int32_t offset = invoke->GetDexCacheArrayOffset() - base->GetElementOffset(); - GetAssembler()->LoadFromOffset(kLoadWord, RegisterFrom(temp), base_reg, offset); + case HInvokeStaticOrDirect::MethodLoadKind::kBssEntry: { + PcRelativePatchInfo* labels = NewMethodBssEntryPatch( + MethodReference(&GetGraph()->GetDexFile(), invoke->GetDexMethodIndex())); + vixl32::Register temp_reg = RegisterFrom(temp); + EmitMovwMovtPlaceholder(labels, temp_reg); + GetAssembler()->LoadFromOffset(kLoadWord, temp_reg, temp_reg, /* offset*/ 0); break; } - case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: { - Location current_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); - vixl32::Register method_reg; - vixl32::Register reg = RegisterFrom(temp); - if (current_method.IsRegister()) { - method_reg = RegisterFrom(current_method); - } else { - DCHECK(invoke->GetLocations()->Intrinsified()); - DCHECK(!current_method.IsValid()); - method_reg = reg; - GetAssembler()->LoadFromOffset(kLoadWord, reg, sp, kCurrentMethodStackOffset); - } - // /* ArtMethod*[] */ temp = temp.ptr_sized_fields_->dex_cache_resolved_methods_; - GetAssembler()->LoadFromOffset( - kLoadWord, - reg, - method_reg, - ArtMethod::DexCacheResolvedMethodsOffset(kArmPointerSize).Int32Value()); - // temp = temp[index_in_cache]; - // Note: Don't use invoke->GetTargetMethod() as it may point to a different dex file. - uint32_t index_in_cache = invoke->GetDexMethodIndex(); - GetAssembler()->LoadFromOffset( - kLoadWord, reg, reg, CodeGenerator::GetCachePointerOffset(index_in_cache)); - break; + case HInvokeStaticOrDirect::MethodLoadKind::kRuntimeCall: { + GenerateInvokeStaticOrDirectRuntimeCall(invoke, temp, slow_path); + return; // No code pointer retrieval; the runtime performs the call directly. } } switch (invoke->GetCodePtrLocation()) { case HInvokeStaticOrDirect::CodePtrLocation::kCallSelf: - __ Bl(GetFrameEntryLabel()); + { + // Use a scope to help guarantee that `RecordPcInfo()` records the correct pc. + ExactAssemblyScope aas(GetVIXLAssembler(), + vixl32::k32BitT32InstructionSizeInBytes, + CodeBufferCheckScope::kMaximumSize); + __ bl(GetFrameEntryLabel()); + RecordPcInfo(invoke, invoke->GetDexPc(), slow_path); + } break; case HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod: // LR = callee_method->entry_point_from_quick_compiled_code_ @@ -9190,12 +9167,14 @@ void CodeGeneratorARMVIXL::GenerateStaticOrDirectCall( RegisterFrom(callee_method), ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmPointerSize).Int32Value()); { + // Use a scope to help guarantee that `RecordPcInfo()` records the correct pc. // blx in T32 has only 16bit encoding that's why a stricter check for the scope is used. ExactAssemblyScope aas(GetVIXLAssembler(), vixl32::k16BitT32InstructionSizeInBytes, CodeBufferCheckScope::kExactSize); // LR() __ blx(lr); + RecordPcInfo(invoke, invoke->GetDexPc(), slow_path); } break; } @@ -9203,7 +9182,8 @@ void CodeGeneratorARMVIXL::GenerateStaticOrDirectCall( DCHECK(!IsLeafMethod()); } -void CodeGeneratorARMVIXL::GenerateVirtualCall(HInvokeVirtual* invoke, Location temp_location) { +void CodeGeneratorARMVIXL::GenerateVirtualCall( + HInvokeVirtual* invoke, Location temp_location, SlowPathCode* slow_path) { vixl32::Register temp = RegisterFrom(temp_location); uint32_t method_offset = mirror::Class::EmbeddedVTableEntryOffset( invoke->GetVTableIndex(), kArmPointerSize).Uint32Value(); @@ -9239,15 +9219,16 @@ void CodeGeneratorARMVIXL::GenerateVirtualCall(HInvokeVirtual* invoke, Location GetAssembler()->LoadFromOffset(kLoadWord, temp, temp, method_offset); // LR = temp->GetEntryPoint(); GetAssembler()->LoadFromOffset(kLoadWord, lr, temp, entry_point); - // LR(); - // This `blx` *must* be the *last* instruction generated by this stub, so that calls to - // `RecordPcInfo()` immediately following record the correct pc. Use a scope to help guarantee - // that. - // blx in T32 has only 16bit encoding that's why a stricter check for the scope is used. - ExactAssemblyScope aas(GetVIXLAssembler(), - vixl32::k16BitT32InstructionSizeInBytes, - CodeBufferCheckScope::kExactSize); - __ blx(lr); + { + // Use a scope to help guarantee that `RecordPcInfo()` records the correct pc. + // blx in T32 has only 16bit encoding that's why a stricter check for the scope is used. + ExactAssemblyScope aas(GetVIXLAssembler(), + vixl32::k16BitT32InstructionSizeInBytes, + CodeBufferCheckScope::kExactSize); + // LR(); + __ blx(lr); + RecordPcInfo(invoke, invoke->GetDexPc(), slow_path); + } } CodeGeneratorARMVIXL::PcRelativePatchInfo* CodeGeneratorARMVIXL::NewPcRelativeMethodPatch( @@ -9257,6 +9238,13 @@ CodeGeneratorARMVIXL::PcRelativePatchInfo* CodeGeneratorARMVIXL::NewPcRelativeMe &pc_relative_method_patches_); } +CodeGeneratorARMVIXL::PcRelativePatchInfo* CodeGeneratorARMVIXL::NewMethodBssEntryPatch( + MethodReference target_method) { + return NewPcRelativePatch(*target_method.dex_file, + target_method.dex_method_index, + &method_bss_entry_patches_); +} + CodeGeneratorARMVIXL::PcRelativePatchInfo* CodeGeneratorARMVIXL::NewPcRelativeTypePatch( const DexFile& dex_file, dex::TypeIndex type_index) { return NewPcRelativePatch(dex_file, type_index.index_, &pc_relative_type_patches_); @@ -9272,11 +9260,6 @@ CodeGeneratorARMVIXL::PcRelativePatchInfo* CodeGeneratorARMVIXL::NewPcRelativeSt return NewPcRelativePatch(dex_file, string_index.index_, &pc_relative_string_patches_); } -CodeGeneratorARMVIXL::PcRelativePatchInfo* CodeGeneratorARMVIXL::NewPcRelativeDexCacheArrayPatch( - const DexFile& dex_file, uint32_t element_offset) { - return NewPcRelativePatch(dex_file, element_offset, &pc_relative_dex_cache_patches_); -} - CodeGeneratorARMVIXL::PcRelativePatchInfo* CodeGeneratorARMVIXL::NewPcRelativePatch( const DexFile& dex_file, uint32_t offset_or_index, ArenaDeque<PcRelativePatchInfo>* patches) { patches->emplace_back(dex_file, offset_or_index); @@ -9340,15 +9323,13 @@ inline void CodeGeneratorARMVIXL::EmitPcRelativeLinkerPatches( void CodeGeneratorARMVIXL::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) { DCHECK(linker_patches->empty()); size_t size = - /* MOVW+MOVT for each entry */ 2u * pc_relative_dex_cache_patches_.size() + /* MOVW+MOVT for each entry */ 2u * pc_relative_method_patches_.size() + + /* MOVW+MOVT for each entry */ 2u * method_bss_entry_patches_.size() + /* MOVW+MOVT for each entry */ 2u * pc_relative_type_patches_.size() + /* MOVW+MOVT for each entry */ 2u * type_bss_entry_patches_.size() + /* MOVW+MOVT for each entry */ 2u * pc_relative_string_patches_.size() + baker_read_barrier_patches_.size(); linker_patches->reserve(size); - EmitPcRelativeLinkerPatches<LinkerPatch::DexCacheArrayPatch>(pc_relative_dex_cache_patches_, - linker_patches); if (GetCompilerOptions().IsBootImage()) { EmitPcRelativeLinkerPatches<LinkerPatch::RelativeMethodPatch>(pc_relative_method_patches_, linker_patches); @@ -9362,6 +9343,8 @@ void CodeGeneratorARMVIXL::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_pa EmitPcRelativeLinkerPatches<LinkerPatch::StringBssEntryPatch>(pc_relative_string_patches_, linker_patches); } + EmitPcRelativeLinkerPatches<LinkerPatch::MethodBssEntryPatch>(method_bss_entry_patches_, + linker_patches); EmitPcRelativeLinkerPatches<LinkerPatch::TypeBssEntryPatch>(type_bss_entry_patches_, linker_patches); for (const BakerReadBarrierPatchInfo& info : baker_read_barrier_patches_) { @@ -9511,17 +9494,6 @@ void InstructionCodeGeneratorARMVIXL::VisitPackedSwitch(HPackedSwitch* switch_in } } } -void LocationsBuilderARMVIXL::VisitArmDexCacheArraysBase(HArmDexCacheArraysBase* base) { - LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(base); - locations->SetOut(Location::RequiresRegister()); -} - -void InstructionCodeGeneratorARMVIXL::VisitArmDexCacheArraysBase(HArmDexCacheArraysBase* base) { - vixl32::Register base_reg = OutputRegister(base); - CodeGeneratorARMVIXL::PcRelativePatchInfo* labels = - codegen_->NewPcRelativeDexCacheArrayPatch(base->GetDexFile(), base->GetElementOffset()); - codegen_->EmitMovwMovtPlaceholder(labels, base_reg); -} // Copy the result of a call into the given target. void CodeGeneratorARMVIXL::MoveFromReturnRegister(Location trg, Primitive::Type type) { diff --git a/compiler/optimizing/code_generator_arm_vixl.h b/compiler/optimizing/code_generator_arm_vixl.h index f6e4de33a8..805a3f4366 100644 --- a/compiler/optimizing/code_generator_arm_vixl.h +++ b/compiler/optimizing/code_generator_arm_vixl.h @@ -538,8 +538,10 @@ class CodeGeneratorARMVIXL : public CodeGenerator { const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info, HInvokeStaticOrDirect* invoke) OVERRIDE; - void GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) OVERRIDE; - void GenerateVirtualCall(HInvokeVirtual* invoke, Location temp) OVERRIDE; + void GenerateStaticOrDirectCall( + HInvokeStaticOrDirect* invoke, Location temp, SlowPathCode* slow_path = nullptr) OVERRIDE; + void GenerateVirtualCall( + HInvokeVirtual* invoke, Location temp, SlowPathCode* slow_path = nullptr) OVERRIDE; void MoveFromReturnRegister(Location trg, Primitive::Type type) OVERRIDE; @@ -564,12 +566,11 @@ class CodeGeneratorARMVIXL : public CodeGenerator { }; PcRelativePatchInfo* NewPcRelativeMethodPatch(MethodReference target_method); + PcRelativePatchInfo* NewMethodBssEntryPatch(MethodReference target_method); PcRelativePatchInfo* NewPcRelativeTypePatch(const DexFile& dex_file, dex::TypeIndex type_index); PcRelativePatchInfo* NewTypeBssEntryPatch(const DexFile& dex_file, dex::TypeIndex type_index); PcRelativePatchInfo* NewPcRelativeStringPatch(const DexFile& dex_file, dex::StringIndex string_index); - PcRelativePatchInfo* NewPcRelativeDexCacheArrayPatch(const DexFile& dex_file, - uint32_t element_offset); // Add a new baker read barrier patch and return the label to be bound // before the BNE instruction. @@ -764,10 +765,10 @@ class CodeGeneratorARMVIXL : public CodeGenerator { // Deduplication map for 32-bit literals, used for non-patchable boot image addresses. Uint32ToLiteralMap uint32_literals_; - // PC-relative patch info for each HArmDexCacheArraysBase. - ArenaDeque<PcRelativePatchInfo> pc_relative_dex_cache_patches_; // PC-relative method patch info for kBootImageLinkTimePcRelative. ArenaDeque<PcRelativePatchInfo> pc_relative_method_patches_; + // PC-relative method patch info for kBssEntry. + ArenaDeque<PcRelativePatchInfo> method_bss_entry_patches_; // PC-relative type patch info for kBootImageLinkTimePcRelative. ArenaDeque<PcRelativePatchInfo> pc_relative_type_patches_; // PC-relative type patch info for kBssEntry. diff --git a/compiler/optimizing/code_generator_mips.cc b/compiler/optimizing/code_generator_mips.cc index 951d75a708..b39d412ac2 100644 --- a/compiler/optimizing/code_generator_mips.cc +++ b/compiler/optimizing/code_generator_mips.cc @@ -40,10 +40,6 @@ namespace mips { static constexpr int kCurrentMethodStackOffset = 0; static constexpr Register kMethodRegisterArgument = A0; -// We'll maximize the range of a single load instruction for dex cache array accesses -// by aligning offset -32768 with the offset of the first used element. -static constexpr uint32_t kDexCacheArrayLwOffset = 0x8000; - Location MipsReturnLocation(Primitive::Type return_type) { switch (return_type) { case Primitive::kPrimBoolean: @@ -1060,8 +1056,8 @@ CodeGeneratorMIPS::CodeGeneratorMIPS(HGraph* graph, isa_features_(isa_features), uint32_literals_(std::less<uint32_t>(), graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), - pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), pc_relative_method_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + method_bss_entry_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), pc_relative_type_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), type_bss_entry_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), pc_relative_string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), @@ -1602,14 +1598,12 @@ inline void CodeGeneratorMIPS::EmitPcRelativeLinkerPatches( void CodeGeneratorMIPS::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) { DCHECK(linker_patches->empty()); size_t size = - pc_relative_dex_cache_patches_.size() + pc_relative_method_patches_.size() + + method_bss_entry_patches_.size() + pc_relative_type_patches_.size() + type_bss_entry_patches_.size() + pc_relative_string_patches_.size(); linker_patches->reserve(size); - EmitPcRelativeLinkerPatches<LinkerPatch::DexCacheArrayPatch>(pc_relative_dex_cache_patches_, - linker_patches); if (GetCompilerOptions().IsBootImage()) { EmitPcRelativeLinkerPatches<LinkerPatch::RelativeMethodPatch>(pc_relative_method_patches_, linker_patches); @@ -1623,6 +1617,8 @@ void CodeGeneratorMIPS::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patch EmitPcRelativeLinkerPatches<LinkerPatch::StringBssEntryPatch>(pc_relative_string_patches_, linker_patches); } + EmitPcRelativeLinkerPatches<LinkerPatch::MethodBssEntryPatch>(method_bss_entry_patches_, + linker_patches); EmitPcRelativeLinkerPatches<LinkerPatch::TypeBssEntryPatch>(type_bss_entry_patches_, linker_patches); DCHECK_EQ(size, linker_patches->size()); @@ -1635,6 +1631,13 @@ CodeGeneratorMIPS::PcRelativePatchInfo* CodeGeneratorMIPS::NewPcRelativeMethodPa &pc_relative_method_patches_); } +CodeGeneratorMIPS::PcRelativePatchInfo* CodeGeneratorMIPS::NewMethodBssEntryPatch( + MethodReference target_method) { + return NewPcRelativePatch(*target_method.dex_file, + target_method.dex_method_index, + &method_bss_entry_patches_); +} + CodeGeneratorMIPS::PcRelativePatchInfo* CodeGeneratorMIPS::NewPcRelativeTypePatch( const DexFile& dex_file, dex::TypeIndex type_index) { return NewPcRelativePatch(dex_file, type_index.index_, &pc_relative_type_patches_); @@ -1650,11 +1653,6 @@ CodeGeneratorMIPS::PcRelativePatchInfo* CodeGeneratorMIPS::NewPcRelativeStringPa return NewPcRelativePatch(dex_file, string_index.index_, &pc_relative_string_patches_); } -CodeGeneratorMIPS::PcRelativePatchInfo* CodeGeneratorMIPS::NewPcRelativeDexCacheArrayPatch( - const DexFile& dex_file, uint32_t element_offset) { - return NewPcRelativePatch(dex_file, element_offset, &pc_relative_dex_cache_patches_); -} - CodeGeneratorMIPS::PcRelativePatchInfo* CodeGeneratorMIPS::NewPcRelativePatch( const DexFile& dex_file, uint32_t offset_or_index, ArenaDeque<PcRelativePatchInfo>* patches) { patches->emplace_back(dex_file, offset_or_index); @@ -7000,7 +6998,7 @@ HLoadString::LoadKind CodeGeneratorMIPS::GetSupportedLoadStringKind( HLoadString::LoadKind desired_string_load_kind) { // We disable PC-relative load on pre-R6 when there is an irreducible loop, as the optimization // is incompatible with it. - // TODO: Create as many MipsDexCacheArraysBase instructions as needed for methods + // TODO: Create as many HMipsComputeBaseMethodAddress instructions as needed for methods // with irreducible loops. bool has_irreducible_loops = GetGraph()->HasIrreducibleLoops(); bool is_r6 = GetInstructionSetFeatures().IsR6(); @@ -7030,6 +7028,8 @@ HLoadClass::LoadKind CodeGeneratorMIPS::GetSupportedLoadClassKind( HLoadClass::LoadKind desired_class_load_kind) { // We disable PC-relative load on pre-R6 when there is an irreducible loop, as the optimization // is incompatible with it. + // TODO: Create as many HMipsComputeBaseMethodAddress instructions as needed for methods + // with irreducible loops. bool has_irreducible_loops = GetGraph()->HasIrreducibleLoops(); bool is_r6 = GetInstructionSetFeatures().IsR6(); bool fallback_load = has_irreducible_loops && !is_r6; @@ -7093,25 +7093,28 @@ HInvokeStaticOrDirect::DispatchInfo CodeGeneratorMIPS::GetSupportedInvokeStaticO HInvokeStaticOrDirect::DispatchInfo dispatch_info = desired_dispatch_info; // We disable PC-relative load on pre-R6 when there is an irreducible loop, as the optimization // is incompatible with it. + // TODO: Create as many HMipsComputeBaseMethodAddress instructions as needed for methods + // with irreducible loops. bool has_irreducible_loops = GetGraph()->HasIrreducibleLoops(); bool is_r6 = GetInstructionSetFeatures().IsR6(); bool fallback_load = has_irreducible_loops && !is_r6; switch (dispatch_info.method_load_kind) { case HInvokeStaticOrDirect::MethodLoadKind::kBootImageLinkTimePcRelative: - case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: + case HInvokeStaticOrDirect::MethodLoadKind::kBssEntry: break; default: fallback_load = false; break; } if (fallback_load) { - dispatch_info.method_load_kind = HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod; + dispatch_info.method_load_kind = HInvokeStaticOrDirect::MethodLoadKind::kRuntimeCall; dispatch_info.method_load_data = 0; } return dispatch_info; } -void CodeGeneratorMIPS::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) { +void CodeGeneratorMIPS::GenerateStaticOrDirectCall( + HInvokeStaticOrDirect* invoke, Location temp, SlowPathCode* slow_path) { // All registers are assumed to be correctly set up per the calling convention. Location callee_method = temp; // For all kinds except kRecursive, callee will be in temp. HInvokeStaticOrDirect::MethodLoadKind method_load_kind = invoke->GetMethodLoadKind(); @@ -7148,51 +7151,20 @@ void CodeGeneratorMIPS::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress: __ LoadConst32(temp.AsRegister<Register>(), invoke->GetMethodAddress()); break; - case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: - if (is_r6) { - uint32_t offset = invoke->GetDexCacheArrayOffset(); - CodeGeneratorMIPS::PcRelativePatchInfo* info = - NewPcRelativeDexCacheArrayPatch(invoke->GetDexFileForPcRelativeDexCache(), offset); - bool reordering = __ SetReorder(false); - EmitPcRelativeAddressPlaceholderHigh(info, TMP, ZERO); - __ Lw(temp.AsRegister<Register>(), TMP, /* placeholder */ 0x5678); - __ SetReorder(reordering); - } else { - HMipsDexCacheArraysBase* base = - invoke->InputAt(invoke->GetSpecialInputIndex())->AsMipsDexCacheArraysBase(); - int32_t offset = - invoke->GetDexCacheArrayOffset() - base->GetElementOffset() - kDexCacheArrayLwOffset; - __ LoadFromOffset(kLoadWord, temp.AsRegister<Register>(), base_reg, offset); - } - break; - case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: { - Location current_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); - Register reg = temp.AsRegister<Register>(); - Register method_reg; - if (current_method.IsRegister()) { - method_reg = current_method.AsRegister<Register>(); - } else { - // TODO: use the appropriate DCHECK() here if possible. - // DCHECK(invoke->GetLocations()->Intrinsified()); - DCHECK(!current_method.IsValid()); - method_reg = reg; - __ Lw(reg, SP, kCurrentMethodStackOffset); - } - - // temp = temp->dex_cache_resolved_methods_; - __ LoadFromOffset(kLoadWord, - reg, - method_reg, - ArtMethod::DexCacheResolvedMethodsOffset(kMipsPointerSize).Int32Value()); - // temp = temp[index_in_cache]; - // Note: Don't use invoke->GetTargetMethod() as it may point to a different dex file. - uint32_t index_in_cache = invoke->GetDexMethodIndex(); - __ LoadFromOffset(kLoadWord, - reg, - reg, - CodeGenerator::GetCachePointerOffset(index_in_cache)); + case HInvokeStaticOrDirect::MethodLoadKind::kBssEntry: { + PcRelativePatchInfo* info = NewMethodBssEntryPatch( + MethodReference(&GetGraph()->GetDexFile(), invoke->GetDexMethodIndex())); + Register temp_reg = temp.AsRegister<Register>(); + bool reordering = __ SetReorder(false); + EmitPcRelativeAddressPlaceholderHigh(info, TMP, base_reg); + __ Lw(temp_reg, TMP, /* placeholder */ 0x5678); + __ SetReorder(reordering); break; } + case HInvokeStaticOrDirect::MethodLoadKind::kRuntimeCall: { + GenerateInvokeStaticOrDirectRuntimeCall(invoke, temp, slow_path); + return; // No code pointer retrieval; the runtime performs the call directly. + } } switch (code_ptr_location) { @@ -7211,6 +7183,8 @@ void CodeGeneratorMIPS::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke __ NopIfNoReordering(); break; } + RecordPcInfo(invoke, invoke->GetDexPc(), slow_path); + DCHECK(!IsLeafMethod()); } @@ -7228,10 +7202,10 @@ void InstructionCodeGeneratorMIPS::VisitInvokeStaticOrDirect(HInvokeStaticOrDire locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation()); - codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); } -void CodeGeneratorMIPS::GenerateVirtualCall(HInvokeVirtual* invoke, Location temp_location) { +void CodeGeneratorMIPS::GenerateVirtualCall( + HInvokeVirtual* invoke, Location temp_location, SlowPathCode* slow_path) { // Use the calling convention instead of the location of the receiver, as // intrinsics may have put the receiver in a different register. In the intrinsics // slow path, the arguments have been moved to the right place, so here we are @@ -7263,6 +7237,7 @@ void CodeGeneratorMIPS::GenerateVirtualCall(HInvokeVirtual* invoke, Location tem // T9(); __ Jalr(T9); __ NopIfNoReordering(); + RecordPcInfo(invoke, invoke->GetDexPc(), slow_path); } void InstructionCodeGeneratorMIPS::VisitInvokeVirtual(HInvokeVirtual* invoke) { @@ -7272,7 +7247,6 @@ void InstructionCodeGeneratorMIPS::VisitInvokeVirtual(HInvokeVirtual* invoke) { codegen_->GenerateVirtualCall(invoke, invoke->GetLocations()->GetTemp(0)); DCHECK(!codegen_->IsLeafMethod()); - codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); } void LocationsBuilderMIPS::VisitLoadClass(HLoadClass* cls) { @@ -8732,29 +8706,11 @@ void InstructionCodeGeneratorMIPS::VisitMipsComputeBaseMethodAddress( __ Nal(); // Grab the return address off RA. __ Move(reg, RA); - // TODO: Can we share this code with that of VisitMipsDexCacheArraysBase()? // Remember this offset (the obtained PC value) for later use with constant area. __ BindPcRelBaseLabel(); } -void LocationsBuilderMIPS::VisitMipsDexCacheArraysBase(HMipsDexCacheArraysBase* base) { - LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(base); - locations->SetOut(Location::RequiresRegister()); -} - -void InstructionCodeGeneratorMIPS::VisitMipsDexCacheArraysBase(HMipsDexCacheArraysBase* base) { - Register reg = base->GetLocations()->Out().AsRegister<Register>(); - CodeGeneratorMIPS::PcRelativePatchInfo* info = - codegen_->NewPcRelativeDexCacheArrayPatch(base->GetDexFile(), base->GetElementOffset()); - CHECK(!codegen_->GetInstructionSetFeatures().IsR6()); - bool reordering = __ SetReorder(false); - // TODO: Reuse MipsComputeBaseMethodAddress on R2 instead of passing ZERO to force emitting NAL. - codegen_->EmitPcRelativeAddressPlaceholderHigh(info, reg, ZERO); - __ Addiu(reg, reg, /* placeholder */ 0x5678); - __ SetReorder(reordering); -} - void LocationsBuilderMIPS::VisitInvokeUnresolved(HInvokeUnresolved* invoke) { // The trampoline uses the same calling convention as dex calling conventions, // except instead of loading arg0/r0 with the target Method*, arg0/r0 will contain diff --git a/compiler/optimizing/code_generator_mips.h b/compiler/optimizing/code_generator_mips.h index 736b5070d9..e72e838dd9 100644 --- a/compiler/optimizing/code_generator_mips.h +++ b/compiler/optimizing/code_generator_mips.h @@ -552,8 +552,10 @@ class CodeGeneratorMIPS : public CodeGenerator { const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info, HInvokeStaticOrDirect* invoke) OVERRIDE; - void GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp); - void GenerateVirtualCall(HInvokeVirtual* invoke, Location temp) OVERRIDE; + void GenerateStaticOrDirectCall( + HInvokeStaticOrDirect* invoke, Location temp, SlowPathCode* slow_path = nullptr) OVERRIDE; + void GenerateVirtualCall( + HInvokeVirtual* invoke, Location temp, SlowPathCode* slow_path = nullptr) OVERRIDE; void MoveFromReturnRegister(Location trg ATTRIBUTE_UNUSED, Primitive::Type type ATTRIBUTE_UNUSED) OVERRIDE { @@ -583,12 +585,11 @@ class CodeGeneratorMIPS : public CodeGenerator { }; PcRelativePatchInfo* NewPcRelativeMethodPatch(MethodReference target_method); + PcRelativePatchInfo* NewMethodBssEntryPatch(MethodReference target_method); PcRelativePatchInfo* NewPcRelativeTypePatch(const DexFile& dex_file, dex::TypeIndex type_index); PcRelativePatchInfo* NewTypeBssEntryPatch(const DexFile& dex_file, dex::TypeIndex type_index); PcRelativePatchInfo* NewPcRelativeStringPatch(const DexFile& dex_file, dex::StringIndex string_index); - PcRelativePatchInfo* NewPcRelativeDexCacheArrayPatch(const DexFile& dex_file, - uint32_t element_offset); Literal* DeduplicateBootImageAddressLiteral(uint32_t address); void EmitPcRelativeAddressPlaceholderHigh(PcRelativePatchInfo* info, Register out, Register base); @@ -643,10 +644,10 @@ class CodeGeneratorMIPS : public CodeGenerator { // Deduplication map for 32-bit literals, used for non-patchable boot image addresses. Uint32ToLiteralMap uint32_literals_; - // PC-relative patch info for each HMipsDexCacheArraysBase. - ArenaDeque<PcRelativePatchInfo> pc_relative_dex_cache_patches_; // PC-relative method patch info for kBootImageLinkTimePcRelative. ArenaDeque<PcRelativePatchInfo> pc_relative_method_patches_; + // PC-relative method patch info for kBssEntry. + ArenaDeque<PcRelativePatchInfo> method_bss_entry_patches_; // PC-relative type patch info for kBootImageLinkTimePcRelative. ArenaDeque<PcRelativePatchInfo> pc_relative_type_patches_; // PC-relative type patch info for kBssEntry. diff --git a/compiler/optimizing/code_generator_mips64.cc b/compiler/optimizing/code_generator_mips64.cc index 6026814f04..e4f1cbd600 100644 --- a/compiler/optimizing/code_generator_mips64.cc +++ b/compiler/optimizing/code_generator_mips64.cc @@ -957,8 +957,8 @@ CodeGeneratorMIPS64::CodeGeneratorMIPS64(HGraph* graph, graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), uint64_literals_(std::less<uint64_t>(), graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), - pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), pc_relative_method_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + method_bss_entry_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), pc_relative_type_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), type_bss_entry_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), pc_relative_string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), @@ -1440,14 +1440,12 @@ inline void CodeGeneratorMIPS64::EmitPcRelativeLinkerPatches( void CodeGeneratorMIPS64::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) { DCHECK(linker_patches->empty()); size_t size = - pc_relative_dex_cache_patches_.size() + pc_relative_method_patches_.size() + + method_bss_entry_patches_.size() + pc_relative_type_patches_.size() + type_bss_entry_patches_.size() + pc_relative_string_patches_.size(); linker_patches->reserve(size); - EmitPcRelativeLinkerPatches<LinkerPatch::DexCacheArrayPatch>(pc_relative_dex_cache_patches_, - linker_patches); if (GetCompilerOptions().IsBootImage()) { EmitPcRelativeLinkerPatches<LinkerPatch::RelativeMethodPatch>(pc_relative_method_patches_, linker_patches); @@ -1461,6 +1459,8 @@ void CodeGeneratorMIPS64::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_pat EmitPcRelativeLinkerPatches<LinkerPatch::StringBssEntryPatch>(pc_relative_string_patches_, linker_patches); } + EmitPcRelativeLinkerPatches<LinkerPatch::MethodBssEntryPatch>(method_bss_entry_patches_, + linker_patches); EmitPcRelativeLinkerPatches<LinkerPatch::TypeBssEntryPatch>(type_bss_entry_patches_, linker_patches); DCHECK_EQ(size, linker_patches->size()); @@ -1473,6 +1473,13 @@ CodeGeneratorMIPS64::PcRelativePatchInfo* CodeGeneratorMIPS64::NewPcRelativeMeth &pc_relative_method_patches_); } +CodeGeneratorMIPS64::PcRelativePatchInfo* CodeGeneratorMIPS64::NewMethodBssEntryPatch( + MethodReference target_method) { + return NewPcRelativePatch(*target_method.dex_file, + target_method.dex_method_index, + &method_bss_entry_patches_); +} + CodeGeneratorMIPS64::PcRelativePatchInfo* CodeGeneratorMIPS64::NewPcRelativeTypePatch( const DexFile& dex_file, dex::TypeIndex type_index) { return NewPcRelativePatch(dex_file, type_index.index_, &pc_relative_type_patches_); @@ -1488,11 +1495,6 @@ CodeGeneratorMIPS64::PcRelativePatchInfo* CodeGeneratorMIPS64::NewPcRelativeStri return NewPcRelativePatch(dex_file, string_index.index_, &pc_relative_string_patches_); } -CodeGeneratorMIPS64::PcRelativePatchInfo* CodeGeneratorMIPS64::NewPcRelativeDexCacheArrayPatch( - const DexFile& dex_file, uint32_t element_offset) { - return NewPcRelativePatch(dex_file, element_offset, &pc_relative_dex_cache_patches_); -} - CodeGeneratorMIPS64::PcRelativePatchInfo* CodeGeneratorMIPS64::NewPcRelativePatch( const DexFile& dex_file, uint32_t offset_or_index, ArenaDeque<PcRelativePatchInfo>* patches) { patches->emplace_back(dex_file, offset_or_index); @@ -4915,7 +4917,8 @@ HInvokeStaticOrDirect::DispatchInfo CodeGeneratorMIPS64::GetSupportedInvokeStati return desired_dispatch_info; } -void CodeGeneratorMIPS64::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) { +void CodeGeneratorMIPS64::GenerateStaticOrDirectCall( + HInvokeStaticOrDirect* invoke, Location temp, SlowPathCode* slow_path) { // All registers are assumed to be correctly set up per the calling convention. Location callee_method = temp; // For all kinds except kRecursive, callee will be in temp. HInvokeStaticOrDirect::MethodLoadKind method_load_kind = invoke->GetMethodLoadKind(); @@ -4948,41 +4951,16 @@ void CodeGeneratorMIPS64::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invo kLoadDoubleword, DeduplicateUint64Literal(invoke->GetMethodAddress())); break; - case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: { - uint32_t offset = invoke->GetDexCacheArrayOffset(); - CodeGeneratorMIPS64::PcRelativePatchInfo* info = - NewPcRelativeDexCacheArrayPatch(invoke->GetDexFileForPcRelativeDexCache(), offset); + case HInvokeStaticOrDirect::MethodLoadKind::kBssEntry: { + PcRelativePatchInfo* info = NewMethodBssEntryPatch( + MethodReference(&GetGraph()->GetDexFile(), invoke->GetDexMethodIndex())); EmitPcRelativeAddressPlaceholderHigh(info, AT); __ Ld(temp.AsRegister<GpuRegister>(), AT, /* placeholder */ 0x5678); break; } - case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: { - Location current_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); - GpuRegister reg = temp.AsRegister<GpuRegister>(); - GpuRegister method_reg; - if (current_method.IsRegister()) { - method_reg = current_method.AsRegister<GpuRegister>(); - } else { - // TODO: use the appropriate DCHECK() here if possible. - // DCHECK(invoke->GetLocations()->Intrinsified()); - DCHECK(!current_method.IsValid()); - method_reg = reg; - __ Ld(reg, SP, kCurrentMethodStackOffset); - } - - // temp = temp->dex_cache_resolved_methods_; - __ LoadFromOffset(kLoadDoubleword, - reg, - method_reg, - ArtMethod::DexCacheResolvedMethodsOffset(kMips64PointerSize).Int32Value()); - // temp = temp[index_in_cache]; - // Note: Don't use invoke->GetTargetMethod() as it may point to a different dex file. - uint32_t index_in_cache = invoke->GetDexMethodIndex(); - __ LoadFromOffset(kLoadDoubleword, - reg, - reg, - CodeGenerator::GetCachePointerOffset(index_in_cache)); - break; + case HInvokeStaticOrDirect::MethodLoadKind::kRuntimeCall: { + GenerateInvokeStaticOrDirectRuntimeCall(invoke, temp, slow_path); + return; // No code pointer retrieval; the runtime performs the call directly. } } @@ -5002,6 +4980,8 @@ void CodeGeneratorMIPS64::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invo __ Nop(); break; } + RecordPcInfo(invoke, invoke->GetDexPc(), slow_path); + DCHECK(!IsLeafMethod()); } @@ -5019,10 +4999,10 @@ void InstructionCodeGeneratorMIPS64::VisitInvokeStaticOrDirect(HInvokeStaticOrDi locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation()); - codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); } -void CodeGeneratorMIPS64::GenerateVirtualCall(HInvokeVirtual* invoke, Location temp_location) { +void CodeGeneratorMIPS64::GenerateVirtualCall( + HInvokeVirtual* invoke, Location temp_location, SlowPathCode* slow_path) { // Use the calling convention instead of the location of the receiver, as // intrinsics may have put the receiver in a different register. In the intrinsics // slow path, the arguments have been moved to the right place, so here we are @@ -5054,6 +5034,7 @@ void CodeGeneratorMIPS64::GenerateVirtualCall(HInvokeVirtual* invoke, Location t // T9(); __ Jalr(T9); __ Nop(); + RecordPcInfo(invoke, invoke->GetDexPc(), slow_path); } void InstructionCodeGeneratorMIPS64::VisitInvokeVirtual(HInvokeVirtual* invoke) { @@ -5063,7 +5044,6 @@ void InstructionCodeGeneratorMIPS64::VisitInvokeVirtual(HInvokeVirtual* invoke) codegen_->GenerateVirtualCall(invoke, invoke->GetLocations()->GetTemp(0)); DCHECK(!codegen_->IsLeafMethod()); - codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); } void LocationsBuilderMIPS64::VisitLoadClass(HLoadClass* cls) { diff --git a/compiler/optimizing/code_generator_mips64.h b/compiler/optimizing/code_generator_mips64.h index 9c6b6f62cb..6260c73614 100644 --- a/compiler/optimizing/code_generator_mips64.h +++ b/compiler/optimizing/code_generator_mips64.h @@ -521,8 +521,10 @@ class CodeGeneratorMIPS64 : public CodeGenerator { const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info, HInvokeStaticOrDirect* invoke) OVERRIDE; - void GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) OVERRIDE; - void GenerateVirtualCall(HInvokeVirtual* invoke, Location temp) OVERRIDE; + void GenerateStaticOrDirectCall( + HInvokeStaticOrDirect* invoke, Location temp, SlowPathCode* slow_path = nullptr) OVERRIDE; + void GenerateVirtualCall( + HInvokeVirtual* invoke, Location temp, SlowPathCode* slow_path = nullptr) OVERRIDE; void MoveFromReturnRegister(Location trg ATTRIBUTE_UNUSED, Primitive::Type type ATTRIBUTE_UNUSED) OVERRIDE { @@ -549,12 +551,11 @@ class CodeGeneratorMIPS64 : public CodeGenerator { }; PcRelativePatchInfo* NewPcRelativeMethodPatch(MethodReference target_method); + PcRelativePatchInfo* NewMethodBssEntryPatch(MethodReference target_method); PcRelativePatchInfo* NewPcRelativeTypePatch(const DexFile& dex_file, dex::TypeIndex type_index); PcRelativePatchInfo* NewTypeBssEntryPatch(const DexFile& dex_file, dex::TypeIndex type_index); PcRelativePatchInfo* NewPcRelativeStringPatch(const DexFile& dex_file, dex::StringIndex string_index); - PcRelativePatchInfo* NewPcRelativeDexCacheArrayPatch(const DexFile& dex_file, - uint32_t element_offset); PcRelativePatchInfo* NewPcRelativeCallPatch(const DexFile& dex_file, uint32_t method_index); Literal* DeduplicateBootImageAddressLiteral(uint64_t address); @@ -607,10 +608,10 @@ class CodeGeneratorMIPS64 : public CodeGenerator { // Deduplication map for 64-bit literals, used for non-patchable method address or method code // address. Uint64ToLiteralMap uint64_literals_; - // PC-relative patch info. - ArenaDeque<PcRelativePatchInfo> pc_relative_dex_cache_patches_; // PC-relative method patch info for kBootImageLinkTimePcRelative. ArenaDeque<PcRelativePatchInfo> pc_relative_method_patches_; + // PC-relative method patch info for kBssEntry. + ArenaDeque<PcRelativePatchInfo> method_bss_entry_patches_; // PC-relative type patch info for kBootImageLinkTimePcRelative. ArenaDeque<PcRelativePatchInfo> pc_relative_type_patches_; // PC-relative type patch info for kBssEntry. diff --git a/compiler/optimizing/code_generator_vector_arm64.cc b/compiler/optimizing/code_generator_vector_arm64.cc index a41adca02c..f422b9fc8b 100644 --- a/compiler/optimizing/code_generator_vector_arm64.cc +++ b/compiler/optimizing/code_generator_vector_arm64.cc @@ -22,6 +22,8 @@ using namespace vixl::aarch64; // NOLINT(build/namespaces) namespace art { namespace arm64 { +using helpers::ARM64EncodableConstantOrRegister; +using helpers::Arm64CanEncodeConstantAsImmediate; using helpers::DRegisterFrom; using helpers::VRegisterFrom; using helpers::HeapOperand; @@ -34,6 +36,7 @@ using helpers::WRegisterFrom; void LocationsBuilderARM64::VisitVecReplicateScalar(HVecReplicateScalar* instruction) { LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction); + HInstruction* input = instruction->InputAt(0); switch (instruction->GetPackedType()) { case Primitive::kPrimBoolean: case Primitive::kPrimByte: @@ -41,13 +44,19 @@ void LocationsBuilderARM64::VisitVecReplicateScalar(HVecReplicateScalar* instruc case Primitive::kPrimShort: case Primitive::kPrimInt: case Primitive::kPrimLong: - locations->SetInAt(0, Location::RequiresRegister()); + locations->SetInAt(0, ARM64EncodableConstantOrRegister(input, instruction)); locations->SetOut(Location::RequiresFpuRegister()); break; case Primitive::kPrimFloat: case Primitive::kPrimDouble: - locations->SetInAt(0, Location::RequiresFpuRegister()); - locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); + if (input->IsConstant() && + Arm64CanEncodeConstantAsImmediate(input->AsConstant(), instruction)) { + locations->SetInAt(0, Location::ConstantLocation(input->AsConstant())); + locations->SetOut(Location::RequiresFpuRegister()); + } else { + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); + } break; default: LOG(FATAL) << "Unsupported SIMD type"; @@ -57,33 +66,58 @@ void LocationsBuilderARM64::VisitVecReplicateScalar(HVecReplicateScalar* instruc void InstructionCodeGeneratorARM64::VisitVecReplicateScalar(HVecReplicateScalar* instruction) { LocationSummary* locations = instruction->GetLocations(); + Location src_loc = locations->InAt(0); VRegister dst = VRegisterFrom(locations->Out()); switch (instruction->GetPackedType()) { case Primitive::kPrimBoolean: case Primitive::kPrimByte: DCHECK_EQ(16u, instruction->GetVectorLength()); - __ Dup(dst.V16B(), InputRegisterAt(instruction, 0)); + if (src_loc.IsConstant()) { + __ Movi(dst.V16B(), Int64ConstantFrom(src_loc)); + } else { + __ Dup(dst.V16B(), InputRegisterAt(instruction, 0)); + } break; case Primitive::kPrimChar: case Primitive::kPrimShort: DCHECK_EQ(8u, instruction->GetVectorLength()); - __ Dup(dst.V8H(), InputRegisterAt(instruction, 0)); + if (src_loc.IsConstant()) { + __ Movi(dst.V8H(), Int64ConstantFrom(src_loc)); + } else { + __ Dup(dst.V8H(), InputRegisterAt(instruction, 0)); + } break; case Primitive::kPrimInt: DCHECK_EQ(4u, instruction->GetVectorLength()); - __ Dup(dst.V4S(), InputRegisterAt(instruction, 0)); + if (src_loc.IsConstant()) { + __ Movi(dst.V4S(), Int64ConstantFrom(src_loc)); + } else { + __ Dup(dst.V4S(), InputRegisterAt(instruction, 0)); + } break; case Primitive::kPrimLong: DCHECK_EQ(2u, instruction->GetVectorLength()); - __ Dup(dst.V2D(), XRegisterFrom(locations->InAt(0))); + if (src_loc.IsConstant()) { + __ Movi(dst.V2D(), Int64ConstantFrom(src_loc)); + } else { + __ Dup(dst.V2D(), XRegisterFrom(src_loc)); + } break; case Primitive::kPrimFloat: DCHECK_EQ(4u, instruction->GetVectorLength()); - __ Dup(dst.V4S(), VRegisterFrom(locations->InAt(0)).V4S(), 0); + if (src_loc.IsConstant()) { + __ Fmov(dst.V4S(), src_loc.GetConstant()->AsFloatConstant()->GetValue()); + } else { + __ Dup(dst.V4S(), VRegisterFrom(src_loc).V4S(), 0); + } break; case Primitive::kPrimDouble: DCHECK_EQ(2u, instruction->GetVectorLength()); - __ Dup(dst.V2D(), VRegisterFrom(locations->InAt(0)).V2D(), 0); + if (src_loc.IsConstant()) { + __ Fmov(dst.V2D(), src_loc.GetConstant()->AsDoubleConstant()->GetValue()); + } else { + __ Dup(dst.V2D(), VRegisterFrom(src_loc).V2D(), 0); + } break; default: LOG(FATAL) << "Unsupported SIMD type"; diff --git a/compiler/optimizing/code_generator_x86.cc b/compiler/optimizing/code_generator_x86.cc index b8465cd9d5..83a261d334 100644 --- a/compiler/optimizing/code_generator_x86.cc +++ b/compiler/optimizing/code_generator_x86.cc @@ -1032,8 +1032,8 @@ CodeGeneratorX86::CodeGeneratorX86(HGraph* graph, move_resolver_(graph->GetArena(), this), assembler_(graph->GetArena()), isa_features_(isa_features), - pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), boot_image_method_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + method_bss_entry_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), boot_image_type_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), type_bss_entry_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), @@ -2204,7 +2204,6 @@ void InstructionCodeGeneratorX86::VisitInvokeStaticOrDirect(HInvokeStaticOrDirec LocationSummary* locations = invoke->GetLocations(); codegen_->GenerateStaticOrDirectCall( invoke, locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation()); - codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); } void LocationsBuilderX86::VisitInvokeVirtual(HInvokeVirtual* invoke) { @@ -2228,7 +2227,6 @@ void InstructionCodeGeneratorX86::VisitInvokeVirtual(HInvokeVirtual* invoke) { codegen_->GenerateVirtualCall(invoke, invoke->GetLocations()->GetTemp(0)); DCHECK(!codegen_->IsLeafMethod()); - codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); } void LocationsBuilderX86::VisitInvokeInterface(HInvokeInterface* invoke) { @@ -4530,7 +4528,8 @@ Register CodeGeneratorX86::GetInvokeStaticOrDirectExtraParameter(HInvokeStaticOr return location.AsRegister<Register>(); } -void CodeGeneratorX86::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) { +void CodeGeneratorX86::GenerateStaticOrDirectCall( + HInvokeStaticOrDirect* invoke, Location temp, SlowPathCode* slow_path) { Location callee_method = temp; // For all kinds except kRecursive, callee will be in temp. switch (invoke->GetMethodLoadKind()) { case HInvokeStaticOrDirect::MethodLoadKind::kStringInit: { @@ -4554,38 +4553,19 @@ void CodeGeneratorX86::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress: __ movl(temp.AsRegister<Register>(), Immediate(invoke->GetMethodAddress())); break; - case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: { + case HInvokeStaticOrDirect::MethodLoadKind::kBssEntry: { Register base_reg = GetInvokeStaticOrDirectExtraParameter(invoke, temp.AsRegister<Register>()); __ movl(temp.AsRegister<Register>(), Address(base_reg, kDummy32BitOffset)); // Bind a new fixup label at the end of the "movl" insn. - uint32_t offset = invoke->GetDexCacheArrayOffset(); - __ Bind(NewPcRelativeDexCacheArrayPatch( + __ Bind(NewMethodBssEntryPatch( invoke->InputAt(invoke->GetSpecialInputIndex())->AsX86ComputeBaseMethodAddress(), - invoke->GetDexFileForPcRelativeDexCache(), - offset)); + MethodReference(&GetGraph()->GetDexFile(), invoke->GetDexMethodIndex()))); break; } - case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: { - Location current_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); - Register method_reg; - Register reg = temp.AsRegister<Register>(); - if (current_method.IsRegister()) { - method_reg = current_method.AsRegister<Register>(); - } else { - DCHECK(invoke->GetLocations()->Intrinsified()); - DCHECK(!current_method.IsValid()); - method_reg = reg; - __ movl(reg, Address(ESP, kCurrentMethodStackOffset)); - } - // /* ArtMethod*[] */ temp = temp.ptr_sized_fields_->dex_cache_resolved_methods_; - __ movl(reg, Address(method_reg, - ArtMethod::DexCacheResolvedMethodsOffset(kX86PointerSize).Int32Value())); - // temp = temp[index_in_cache]; - // Note: Don't use invoke->GetTargetMethod() as it may point to a different dex file. - uint32_t index_in_cache = invoke->GetDexMethodIndex(); - __ movl(reg, Address(reg, CodeGenerator::GetCachePointerOffset(index_in_cache))); - break; + case HInvokeStaticOrDirect::MethodLoadKind::kRuntimeCall: { + GenerateInvokeStaticOrDirectRuntimeCall(invoke, temp, slow_path); + return; // No code pointer retrieval; the runtime performs the call directly. } } @@ -4600,11 +4580,13 @@ void CodeGeneratorX86::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, kX86PointerSize).Int32Value())); break; } + RecordPcInfo(invoke, invoke->GetDexPc(), slow_path); DCHECK(!IsLeafMethod()); } -void CodeGeneratorX86::GenerateVirtualCall(HInvokeVirtual* invoke, Location temp_in) { +void CodeGeneratorX86::GenerateVirtualCall( + HInvokeVirtual* invoke, Location temp_in, SlowPathCode* slow_path) { Register temp = temp_in.AsRegister<Register>(); uint32_t method_offset = mirror::Class::EmbeddedVTableEntryOffset( invoke->GetVTableIndex(), kX86PointerSize).Uint32Value(); @@ -4632,6 +4614,7 @@ void CodeGeneratorX86::GenerateVirtualCall(HInvokeVirtual* invoke, Location temp // call temp->GetEntryPoint(); __ call(Address( temp, ArtMethod::EntryPointFromQuickCompiledCodeOffset(kX86PointerSize).Int32Value())); + RecordPcInfo(invoke, invoke->GetDexPc(), slow_path); } void CodeGeneratorX86::RecordBootMethodPatch(HInvokeStaticOrDirect* invoke) { @@ -4644,6 +4627,16 @@ void CodeGeneratorX86::RecordBootMethodPatch(HInvokeStaticOrDirect* invoke) { __ Bind(&boot_image_method_patches_.back().label); } +Label* CodeGeneratorX86::NewMethodBssEntryPatch( + HX86ComputeBaseMethodAddress* method_address, + MethodReference target_method) { + // Add the patch entry and bind its label at the end of the instruction. + method_bss_entry_patches_.emplace_back(method_address, + *target_method.dex_file, + target_method.dex_method_index); + return &method_bss_entry_patches_.back().label; +} + void CodeGeneratorX86::RecordBootTypePatch(HLoadClass* load_class) { HX86ComputeBaseMethodAddress* address = load_class->InputAt(0)->AsX86ComputeBaseMethodAddress(); boot_image_type_patches_.emplace_back(address, @@ -4678,15 +4671,6 @@ Label* CodeGeneratorX86::NewStringBssEntryPatch(HLoadString* load_string) { return &string_patches_.back().label; } -Label* CodeGeneratorX86::NewPcRelativeDexCacheArrayPatch( - HX86ComputeBaseMethodAddress* method_address, - const DexFile& dex_file, - uint32_t element_offset) { - // Add the patch entry and bind its label at the end of the instruction. - pc_relative_dex_cache_patches_.emplace_back(method_address, dex_file, element_offset); - return &pc_relative_dex_cache_patches_.back().label; -} - // The label points to the end of the "movl" or another instruction but the literal offset // for method patch needs to point to the embedded constant which occupies the last 4 bytes. constexpr uint32_t kLabelPositionToLiteralOffsetAdjustment = 4u; @@ -4705,14 +4689,12 @@ inline void CodeGeneratorX86::EmitPcRelativeLinkerPatches( void CodeGeneratorX86::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) { DCHECK(linker_patches->empty()); size_t size = - pc_relative_dex_cache_patches_.size() + boot_image_method_patches_.size() + + method_bss_entry_patches_.size() + boot_image_type_patches_.size() + type_bss_entry_patches_.size() + string_patches_.size(); linker_patches->reserve(size); - EmitPcRelativeLinkerPatches<LinkerPatch::DexCacheArrayPatch>(pc_relative_dex_cache_patches_, - linker_patches); if (GetCompilerOptions().IsBootImage()) { EmitPcRelativeLinkerPatches<LinkerPatch::RelativeMethodPatch>(boot_image_method_patches_, linker_patches); @@ -4724,6 +4706,8 @@ void CodeGeneratorX86::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patche DCHECK(boot_image_type_patches_.empty()); EmitPcRelativeLinkerPatches<LinkerPatch::StringBssEntryPatch>(string_patches_, linker_patches); } + EmitPcRelativeLinkerPatches<LinkerPatch::MethodBssEntryPatch>(method_bss_entry_patches_, + linker_patches); EmitPcRelativeLinkerPatches<LinkerPatch::TypeBssEntryPatch>(type_bss_entry_patches_, linker_patches); DCHECK_EQ(size, linker_patches->size()); diff --git a/compiler/optimizing/code_generator_x86.h b/compiler/optimizing/code_generator_x86.h index 8130bd9d25..f48753b614 100644 --- a/compiler/optimizing/code_generator_x86.h +++ b/compiler/optimizing/code_generator_x86.h @@ -408,18 +408,19 @@ class CodeGeneratorX86 : public CodeGenerator { HInvokeStaticOrDirect* invoke) OVERRIDE; // Generate a call to a static or direct method. - void GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) OVERRIDE; + void GenerateStaticOrDirectCall( + HInvokeStaticOrDirect* invoke, Location temp, SlowPathCode* slow_path = nullptr) OVERRIDE; // Generate a call to a virtual method. - void GenerateVirtualCall(HInvokeVirtual* invoke, Location temp) OVERRIDE; + void GenerateVirtualCall( + HInvokeVirtual* invoke, Location temp, SlowPathCode* slow_path = nullptr) OVERRIDE; void RecordBootMethodPatch(HInvokeStaticOrDirect* invoke); + Label* NewMethodBssEntryPatch(HX86ComputeBaseMethodAddress* method_address, + MethodReference target_method); void RecordBootTypePatch(HLoadClass* load_class); Label* NewTypeBssEntryPatch(HLoadClass* load_class); void RecordBootStringPatch(HLoadString* load_string); Label* NewStringBssEntryPatch(HLoadString* load_string); - Label* NewPcRelativeDexCacheArrayPatch(HX86ComputeBaseMethodAddress* method_address, - const DexFile& dex_file, - uint32_t element_offset); Label* NewJitRootStringPatch(const DexFile& dex_file, dex::StringIndex dex_index, Handle<mirror::String> handle); @@ -631,10 +632,10 @@ class CodeGeneratorX86 : public CodeGenerator { X86Assembler assembler_; const X86InstructionSetFeatures& isa_features_; - // PC-relative DexCache access info. - ArenaDeque<X86PcRelativePatchInfo> pc_relative_dex_cache_patches_; // PC-relative method patch info for kBootImageLinkTimePcRelative. ArenaDeque<X86PcRelativePatchInfo> boot_image_method_patches_; + // PC-relative method patch info for kBssEntry. + ArenaDeque<X86PcRelativePatchInfo> method_bss_entry_patches_; // PC-relative type patch info for kBootImageLinkTimePcRelative. ArenaDeque<X86PcRelativePatchInfo> boot_image_type_patches_; // Type patch locations for kBssEntry. diff --git a/compiler/optimizing/code_generator_x86_64.cc b/compiler/optimizing/code_generator_x86_64.cc index 8dde298267..7331a9e98e 100644 --- a/compiler/optimizing/code_generator_x86_64.cc +++ b/compiler/optimizing/code_generator_x86_64.cc @@ -977,8 +977,8 @@ HInvokeStaticOrDirect::DispatchInfo CodeGeneratorX86_64::GetSupportedInvokeStati return desired_dispatch_info; } -void CodeGeneratorX86_64::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, - Location temp) { +void CodeGeneratorX86_64::GenerateStaticOrDirectCall( + HInvokeStaticOrDirect* invoke, Location temp, SlowPathCode* slow_path) { // All registers are assumed to be correctly set up. Location callee_method = temp; // For all kinds except kRecursive, callee will be in temp. @@ -1002,35 +1002,17 @@ void CodeGeneratorX86_64::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invo case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress: Load64BitValue(temp.AsRegister<CpuRegister>(), invoke->GetMethodAddress()); break; - case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: { + case HInvokeStaticOrDirect::MethodLoadKind::kBssEntry: { __ movq(temp.AsRegister<CpuRegister>(), Address::Absolute(kDummy32BitOffset, /* no_rip */ false)); // Bind a new fixup label at the end of the "movl" insn. - uint32_t offset = invoke->GetDexCacheArrayOffset(); - __ Bind(NewPcRelativeDexCacheArrayPatch(invoke->GetDexFileForPcRelativeDexCache(), offset)); + __ Bind(NewMethodBssEntryPatch( + MethodReference(&GetGraph()->GetDexFile(), invoke->GetDexMethodIndex()))); break; } - case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: { - Location current_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); - Register method_reg; - CpuRegister reg = temp.AsRegister<CpuRegister>(); - if (current_method.IsRegister()) { - method_reg = current_method.AsRegister<Register>(); - } else { - DCHECK(invoke->GetLocations()->Intrinsified()); - DCHECK(!current_method.IsValid()); - method_reg = reg.AsRegister(); - __ movq(reg, Address(CpuRegister(RSP), kCurrentMethodStackOffset)); - } - // /* ArtMethod*[] */ temp = temp.ptr_sized_fields_->dex_cache_resolved_methods_; - __ movq(reg, - Address(CpuRegister(method_reg), - ArtMethod::DexCacheResolvedMethodsOffset(kX86_64PointerSize).SizeValue())); - // temp = temp[index_in_cache]; - // Note: Don't use invoke->GetTargetMethod() as it may point to a different dex file. - uint32_t index_in_cache = invoke->GetDexMethodIndex(); - __ movq(reg, Address(reg, CodeGenerator::GetCachePointerOffset(index_in_cache))); - break; + case HInvokeStaticOrDirect::MethodLoadKind::kRuntimeCall: { + GenerateInvokeStaticOrDirectRuntimeCall(invoke, temp, slow_path); + return; // No code pointer retrieval; the runtime performs the call directly. } } @@ -1045,11 +1027,13 @@ void CodeGeneratorX86_64::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invo kX86_64PointerSize).SizeValue())); break; } + RecordPcInfo(invoke, invoke->GetDexPc(), slow_path); DCHECK(!IsLeafMethod()); } -void CodeGeneratorX86_64::GenerateVirtualCall(HInvokeVirtual* invoke, Location temp_in) { +void CodeGeneratorX86_64::GenerateVirtualCall( + HInvokeVirtual* invoke, Location temp_in, SlowPathCode* slow_path) { CpuRegister temp = temp_in.AsRegister<CpuRegister>(); size_t method_offset = mirror::Class::EmbeddedVTableEntryOffset( invoke->GetVTableIndex(), kX86_64PointerSize).SizeValue(); @@ -1078,6 +1062,7 @@ void CodeGeneratorX86_64::GenerateVirtualCall(HInvokeVirtual* invoke, Location t // call temp->GetEntryPoint(); __ call(Address(temp, ArtMethod::EntryPointFromQuickCompiledCodeOffset( kX86_64PointerSize).SizeValue())); + RecordPcInfo(invoke, invoke->GetDexPc(), slow_path); } void CodeGeneratorX86_64::RecordBootMethodPatch(HInvokeStaticOrDirect* invoke) { @@ -1086,6 +1071,12 @@ void CodeGeneratorX86_64::RecordBootMethodPatch(HInvokeStaticOrDirect* invoke) { __ Bind(&boot_image_method_patches_.back().label); } +Label* CodeGeneratorX86_64::NewMethodBssEntryPatch(MethodReference target_method) { + // Add a patch entry and return the label. + method_bss_entry_patches_.emplace_back(*target_method.dex_file, target_method.dex_method_index); + return &method_bss_entry_patches_.back().label; +} + void CodeGeneratorX86_64::RecordBootTypePatch(HLoadClass* load_class) { boot_image_type_patches_.emplace_back(load_class->GetDexFile(), load_class->GetTypeIndex().index_); @@ -1109,13 +1100,6 @@ Label* CodeGeneratorX86_64::NewStringBssEntryPatch(HLoadString* load_string) { return &string_patches_.back().label; } -Label* CodeGeneratorX86_64::NewPcRelativeDexCacheArrayPatch(const DexFile& dex_file, - uint32_t element_offset) { - // Add a patch entry and return the label. - pc_relative_dex_cache_patches_.emplace_back(dex_file, element_offset); - return &pc_relative_dex_cache_patches_.back().label; -} - // The label points to the end of the "movl" or another instruction but the literal offset // for method patch needs to point to the embedded constant which occupies the last 4 bytes. constexpr uint32_t kLabelPositionToLiteralOffsetAdjustment = 4u; @@ -1134,14 +1118,12 @@ inline void CodeGeneratorX86_64::EmitPcRelativeLinkerPatches( void CodeGeneratorX86_64::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) { DCHECK(linker_patches->empty()); size_t size = - pc_relative_dex_cache_patches_.size() + boot_image_method_patches_.size() + + method_bss_entry_patches_.size() + boot_image_type_patches_.size() + type_bss_entry_patches_.size() + string_patches_.size(); linker_patches->reserve(size); - EmitPcRelativeLinkerPatches<LinkerPatch::DexCacheArrayPatch>(pc_relative_dex_cache_patches_, - linker_patches); if (GetCompilerOptions().IsBootImage()) { EmitPcRelativeLinkerPatches<LinkerPatch::RelativeMethodPatch>(boot_image_method_patches_, linker_patches); @@ -1153,6 +1135,8 @@ void CodeGeneratorX86_64::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_pat DCHECK(boot_image_type_patches_.empty()); EmitPcRelativeLinkerPatches<LinkerPatch::StringBssEntryPatch>(string_patches_, linker_patches); } + EmitPcRelativeLinkerPatches<LinkerPatch::MethodBssEntryPatch>(method_bss_entry_patches_, + linker_patches); EmitPcRelativeLinkerPatches<LinkerPatch::TypeBssEntryPatch>(type_bss_entry_patches_, linker_patches); DCHECK_EQ(size, linker_patches->size()); @@ -1241,8 +1225,8 @@ CodeGeneratorX86_64::CodeGeneratorX86_64(HGraph* graph, assembler_(graph->GetArena()), isa_features_(isa_features), constant_area_start_(0), - pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), boot_image_method_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), + method_bss_entry_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), boot_image_type_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), type_bss_entry_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), @@ -2387,7 +2371,6 @@ void InstructionCodeGeneratorX86_64::VisitInvokeStaticOrDirect(HInvokeStaticOrDi LocationSummary* locations = invoke->GetLocations(); codegen_->GenerateStaticOrDirectCall( invoke, locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation()); - codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); } void LocationsBuilderX86_64::HandleInvoke(HInvoke* invoke) { @@ -2411,7 +2394,6 @@ void InstructionCodeGeneratorX86_64::VisitInvokeVirtual(HInvokeVirtual* invoke) codegen_->GenerateVirtualCall(invoke, invoke->GetLocations()->GetTemp(0)); DCHECK(!codegen_->IsLeafMethod()); - codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); } void LocationsBuilderX86_64::VisitInvokeInterface(HInvokeInterface* invoke) { diff --git a/compiler/optimizing/code_generator_x86_64.h b/compiler/optimizing/code_generator_x86_64.h index 25479814d0..33c64290d4 100644 --- a/compiler/optimizing/code_generator_x86_64.h +++ b/compiler/optimizing/code_generator_x86_64.h @@ -404,15 +404,17 @@ class CodeGeneratorX86_64 : public CodeGenerator { const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info, HInvokeStaticOrDirect* invoke) OVERRIDE; - void GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) OVERRIDE; - void GenerateVirtualCall(HInvokeVirtual* invoke, Location temp) OVERRIDE; + void GenerateStaticOrDirectCall( + HInvokeStaticOrDirect* invoke, Location temp, SlowPathCode* slow_path = nullptr) OVERRIDE; + void GenerateVirtualCall( + HInvokeVirtual* invoke, Location temp, SlowPathCode* slow_path = nullptr) OVERRIDE; void RecordBootMethodPatch(HInvokeStaticOrDirect* invoke); + Label* NewMethodBssEntryPatch(MethodReference target_method); void RecordBootTypePatch(HLoadClass* load_class); Label* NewTypeBssEntryPatch(HLoadClass* load_class); void RecordBootStringPatch(HLoadString* load_string); Label* NewStringBssEntryPatch(HLoadString* load_string); - Label* NewPcRelativeDexCacheArrayPatch(const DexFile& dex_file, uint32_t element_offset); Label* NewJitRootStringPatch(const DexFile& dex_file, dex::StringIndex dex_index, Handle<mirror::String> handle); @@ -601,10 +603,10 @@ class CodeGeneratorX86_64 : public CodeGenerator { // Used for fixups to the constant area. int constant_area_start_; - // PC-relative DexCache access info. - ArenaDeque<PatchInfo<Label>> pc_relative_dex_cache_patches_; // PC-relative method patch info for kBootImageLinkTimePcRelative. ArenaDeque<PatchInfo<Label>> boot_image_method_patches_; + // PC-relative method patch info for kBssEntry. + ArenaDeque<PatchInfo<Label>> method_bss_entry_patches_; // PC-relative type patch info for kBootImageLinkTimePcRelative. ArenaDeque<PatchInfo<Label>> boot_image_type_patches_; // Type patch locations for kBssEntry. diff --git a/compiler/optimizing/common_arm64.h b/compiler/optimizing/common_arm64.h index 721f74eeee..e73fd7ddc8 100644 --- a/compiler/optimizing/common_arm64.h +++ b/compiler/optimizing/common_arm64.h @@ -234,9 +234,20 @@ inline vixl::aarch64::Operand OperandFromMemOperand( } } -inline bool CanEncodeConstantAsImmediate(HConstant* constant, HInstruction* instr) { - DCHECK(constant->IsIntConstant() || constant->IsLongConstant() || constant->IsNullConstant()) - << constant->DebugName(); +inline bool Arm64CanEncodeConstantAsImmediate(HConstant* constant, HInstruction* instr) { + int64_t value = CodeGenerator::GetInt64ValueOf(constant); + + // TODO: Improve this when IsSIMDConstantEncodable method is implemented in VIXL. + if (instr->IsVecReplicateScalar()) { + if (constant->IsLongConstant()) { + return false; + } else if (constant->IsFloatConstant()) { + return vixl::aarch64::Assembler::IsImmFP32(constant->AsFloatConstant()->GetValue()); + } else if (constant->IsDoubleConstant()) { + return vixl::aarch64::Assembler::IsImmFP64(constant->AsDoubleConstant()->GetValue()); + } + return IsUint<8>(value); + } // For single uses we let VIXL handle the constant generation since it will // use registers that are not managed by the register allocator (wip0, wip1). @@ -249,8 +260,6 @@ inline bool CanEncodeConstantAsImmediate(HConstant* constant, HInstruction* inst return true; } - int64_t value = CodeGenerator::GetInt64ValueOf(constant); - if (instr->IsAnd() || instr->IsOr() || instr->IsXor()) { // Uses logical operations. return vixl::aarch64::Assembler::IsImmLogical(value, vixl::aarch64::kXRegSize); @@ -276,7 +285,7 @@ inline bool CanEncodeConstantAsImmediate(HConstant* constant, HInstruction* inst inline Location ARM64EncodableConstantOrRegister(HInstruction* constant, HInstruction* instr) { if (constant->IsConstant() - && CanEncodeConstantAsImmediate(constant->AsConstant(), instr)) { + && Arm64CanEncodeConstantAsImmediate(constant->AsConstant(), instr)) { return Location::ConstantLocation(constant->AsConstant()); } diff --git a/compiler/optimizing/dex_cache_array_fixups_arm.cc b/compiler/optimizing/dex_cache_array_fixups_arm.cc deleted file mode 100644 index 0c832a5c35..0000000000 --- a/compiler/optimizing/dex_cache_array_fixups_arm.cc +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Copyright (C) 2015 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "dex_cache_array_fixups_arm.h" - -#include "base/arena_containers.h" -#ifdef ART_USE_OLD_ARM_BACKEND -#include "code_generator_arm.h" -#include "intrinsics_arm.h" -#else -#include "code_generator_arm_vixl.h" -#include "intrinsics_arm_vixl.h" -#endif -#include "utils/dex_cache_arrays_layout-inl.h" - -namespace art { -namespace arm { -#ifdef ART_USE_OLD_ARM_BACKEND -typedef CodeGeneratorARM CodeGeneratorARMType; -typedef IntrinsicLocationsBuilderARM IntrinsicLocationsBuilderARMType; -#else -typedef CodeGeneratorARMVIXL CodeGeneratorARMType; -typedef IntrinsicLocationsBuilderARMVIXL IntrinsicLocationsBuilderARMType; -#endif - -/** - * Finds instructions that need the dex cache arrays base as an input. - */ -class DexCacheArrayFixupsVisitor : public HGraphVisitor { - public: - DexCacheArrayFixupsVisitor(HGraph* graph, CodeGenerator* codegen) - : HGraphVisitor(graph), - codegen_(down_cast<CodeGeneratorARMType*>(codegen)), - dex_cache_array_bases_(std::less<const DexFile*>(), - // Attribute memory use to code generator. - graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) {} - - void MoveBasesIfNeeded() { - for (const auto& entry : dex_cache_array_bases_) { - // Bring the base closer to the first use (previously, it was in the - // entry block) and relieve some pressure on the register allocator - // while avoiding recalculation of the base in a loop. - HArmDexCacheArraysBase* base = entry.second; - base->MoveBeforeFirstUserAndOutOfLoops(); - } - } - - private: - void VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) OVERRIDE { - // If this is an invoke with PC-relative access to the dex cache methods array, - // we need to add the dex cache arrays base as the special input. - if (invoke->HasPcRelativeDexCache() && - !IsCallFreeIntrinsic<IntrinsicLocationsBuilderARMType>(invoke, codegen_)) { - HArmDexCacheArraysBase* base = - GetOrCreateDexCacheArrayBase(invoke, invoke->GetDexFileForPcRelativeDexCache()); - // Update the element offset in base. - DexCacheArraysLayout layout(kArmPointerSize, &invoke->GetDexFileForPcRelativeDexCache()); - base->UpdateElementOffset(layout.MethodOffset(invoke->GetDexMethodIndex())); - // Add the special argument base to the method. - DCHECK(!invoke->HasCurrentMethodInput()); - invoke->AddSpecialInput(base); - } - } - - HArmDexCacheArraysBase* GetOrCreateDexCacheArrayBase(HInstruction* cursor, - const DexFile& dex_file) { - if (GetGraph()->HasIrreducibleLoops()) { - HArmDexCacheArraysBase* base = new (GetGraph()->GetArena()) HArmDexCacheArraysBase(dex_file); - cursor->GetBlock()->InsertInstructionBefore(base, cursor); - return base; - } else { - // Ensure we only initialize the pointer once for each dex file. - auto lb = dex_cache_array_bases_.lower_bound(&dex_file); - if (lb != dex_cache_array_bases_.end() && - !dex_cache_array_bases_.key_comp()(&dex_file, lb->first)) { - return lb->second; - } - - // Insert the base at the start of the entry block, move it to a better - // position later in MoveBaseIfNeeded(). - HArmDexCacheArraysBase* base = new (GetGraph()->GetArena()) HArmDexCacheArraysBase(dex_file); - HBasicBlock* entry_block = GetGraph()->GetEntryBlock(); - entry_block->InsertInstructionBefore(base, entry_block->GetFirstInstruction()); - dex_cache_array_bases_.PutBefore(lb, &dex_file, base); - return base; - } - } - - CodeGeneratorARMType* codegen_; - - using DexCacheArraysBaseMap = - ArenaSafeMap<const DexFile*, HArmDexCacheArraysBase*, std::less<const DexFile*>>; - DexCacheArraysBaseMap dex_cache_array_bases_; -}; - -void DexCacheArrayFixups::Run() { - DexCacheArrayFixupsVisitor visitor(graph_, codegen_); - visitor.VisitInsertionOrder(); - visitor.MoveBasesIfNeeded(); -} - -} // namespace arm -} // namespace art diff --git a/compiler/optimizing/dex_cache_array_fixups_arm.h b/compiler/optimizing/dex_cache_array_fixups_arm.h deleted file mode 100644 index 9d67a319b9..0000000000 --- a/compiler/optimizing/dex_cache_array_fixups_arm.h +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (C) 2015 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef ART_COMPILER_OPTIMIZING_DEX_CACHE_ARRAY_FIXUPS_ARM_H_ -#define ART_COMPILER_OPTIMIZING_DEX_CACHE_ARRAY_FIXUPS_ARM_H_ - -#include "nodes.h" -#include "optimization.h" - -namespace art { - -class CodeGenerator; - -namespace arm { - -class DexCacheArrayFixups : public HOptimization { - public: - DexCacheArrayFixups(HGraph* graph, CodeGenerator* codegen, OptimizingCompilerStats* stats) - : HOptimization(graph, kDexCacheArrayFixupsArmPassName, stats), - codegen_(codegen) {} - - static constexpr const char* kDexCacheArrayFixupsArmPassName = "dex_cache_array_fixups_arm"; - - void Run() OVERRIDE; - - private: - CodeGenerator* codegen_; -}; - -} // namespace arm -} // namespace art - -#endif // ART_COMPILER_OPTIMIZING_DEX_CACHE_ARRAY_FIXUPS_ARM_H_ diff --git a/compiler/optimizing/dex_cache_array_fixups_mips.cc b/compiler/optimizing/dex_cache_array_fixups_mips.cc deleted file mode 100644 index 7734f9197d..0000000000 --- a/compiler/optimizing/dex_cache_array_fixups_mips.cc +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright (C) 2016 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "code_generator_mips.h" -#include "dex_cache_array_fixups_mips.h" - -#include "base/arena_containers.h" -#include "intrinsics_mips.h" -#include "utils/dex_cache_arrays_layout-inl.h" - -namespace art { -namespace mips { - -/** - * Finds instructions that need the dex cache arrays base as an input. - */ -class DexCacheArrayFixupsVisitor : public HGraphVisitor { - public: - explicit DexCacheArrayFixupsVisitor(HGraph* graph, CodeGenerator* codegen) - : HGraphVisitor(graph), - codegen_(down_cast<CodeGeneratorMIPS*>(codegen)), - dex_cache_array_bases_(std::less<const DexFile*>(), - // Attribute memory use to code generator. - graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) {} - - void MoveBasesIfNeeded() { - for (const auto& entry : dex_cache_array_bases_) { - // Bring the base closer to the first use (previously, it was in the - // entry block) and relieve some pressure on the register allocator - // while avoiding recalculation of the base in a loop. - HMipsDexCacheArraysBase* base = entry.second; - base->MoveBeforeFirstUserAndOutOfLoops(); - } - // Computing the dex cache base for PC-relative accesses will clobber RA with - // the NAL instruction on R2. Take a note of this before generating the method - // entry. - if (!dex_cache_array_bases_.empty()) { - codegen_->ClobberRA(); - } - } - - private: - void VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) OVERRIDE { - // If this is an invoke with PC-relative access to the dex cache methods array, - // we need to add the dex cache arrays base as the special input. - if (invoke->HasPcRelativeDexCache() && - !IsCallFreeIntrinsic<IntrinsicLocationsBuilderMIPS>(invoke, codegen_)) { - // Initialize base for target method dex file if needed. - HMipsDexCacheArraysBase* base = - GetOrCreateDexCacheArrayBase(invoke->GetDexFileForPcRelativeDexCache()); - // Update the element offset in base. - DexCacheArraysLayout layout(kMipsPointerSize, &invoke->GetDexFileForPcRelativeDexCache()); - base->UpdateElementOffset(layout.MethodOffset(invoke->GetDexMethodIndex())); - // Add the special argument base to the method. - DCHECK(!invoke->HasCurrentMethodInput()); - invoke->AddSpecialInput(base); - } - } - - HMipsDexCacheArraysBase* GetOrCreateDexCacheArrayBase(const DexFile& dex_file) { - return dex_cache_array_bases_.GetOrCreate( - &dex_file, - [this, &dex_file]() { - HMipsDexCacheArraysBase* base = - new (GetGraph()->GetArena()) HMipsDexCacheArraysBase(dex_file); - HBasicBlock* entry_block = GetGraph()->GetEntryBlock(); - // Insert the base at the start of the entry block, move it to a better - // position later in MoveBaseIfNeeded(). - entry_block->InsertInstructionBefore(base, entry_block->GetFirstInstruction()); - return base; - }); - } - - CodeGeneratorMIPS* codegen_; - - using DexCacheArraysBaseMap = - ArenaSafeMap<const DexFile*, HMipsDexCacheArraysBase*, std::less<const DexFile*>>; - DexCacheArraysBaseMap dex_cache_array_bases_; -}; - -void DexCacheArrayFixups::Run() { - CodeGeneratorMIPS* mips_codegen = down_cast<CodeGeneratorMIPS*>(codegen_); - if (mips_codegen->GetInstructionSetFeatures().IsR6()) { - // Do nothing for R6 because it has PC-relative addressing. - return; - } - if (graph_->HasIrreducibleLoops()) { - // Do not run this optimization, as irreducible loops do not work with an instruction - // that can be live-in at the irreducible loop header. - return; - } - DexCacheArrayFixupsVisitor visitor(graph_, codegen_); - visitor.VisitInsertionOrder(); - visitor.MoveBasesIfNeeded(); -} - -} // namespace mips -} // namespace art diff --git a/compiler/optimizing/dex_cache_array_fixups_mips.h b/compiler/optimizing/dex_cache_array_fixups_mips.h deleted file mode 100644 index 861a199d6c..0000000000 --- a/compiler/optimizing/dex_cache_array_fixups_mips.h +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (C) 2016 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef ART_COMPILER_OPTIMIZING_DEX_CACHE_ARRAY_FIXUPS_MIPS_H_ -#define ART_COMPILER_OPTIMIZING_DEX_CACHE_ARRAY_FIXUPS_MIPS_H_ - -#include "nodes.h" -#include "optimization.h" - -namespace art { - -class CodeGenerator; - -namespace mips { - -class DexCacheArrayFixups : public HOptimization { - public: - DexCacheArrayFixups(HGraph* graph, CodeGenerator* codegen, OptimizingCompilerStats* stats) - : HOptimization(graph, kDexCacheArrayFixupsMipsPassName, stats), - codegen_(codegen) {} - - static constexpr const char* kDexCacheArrayFixupsMipsPassName = "dex_cache_array_fixups_mips"; - - void Run() OVERRIDE; - - private: - CodeGenerator* codegen_; -}; - -} // namespace mips -} // namespace art - -#endif // ART_COMPILER_OPTIMIZING_DEX_CACHE_ARRAY_FIXUPS_MIPS_H_ diff --git a/compiler/optimizing/induction_var_analysis.cc b/compiler/optimizing/induction_var_analysis.cc index 88473f02e5..84b20f65e3 100644 --- a/compiler/optimizing/induction_var_analysis.cc +++ b/compiler/optimizing/induction_var_analysis.cc @@ -695,8 +695,8 @@ HInductionVarAnalysis::InductionInfo* HInductionVarAnalysis::SolveOp(HLoopInform /*fetch*/ nullptr, type_); default: - CHECK(false) << op; - break; + LOG(FATAL) << op; + UNREACHABLE(); } } } diff --git a/compiler/optimizing/inliner.cc b/compiler/optimizing/inliner.cc index 9be6a512f5..142c95780e 100644 --- a/compiler/optimizing/inliner.cc +++ b/compiler/optimizing/inliner.cc @@ -56,7 +56,7 @@ static constexpr size_t kMaximumNumberOfInstructionsForSmallMethod = 3; // Limit the number of dex registers that we accumulate while inlining // to avoid creating large amount of nested environments. -static constexpr size_t kMaximumNumberOfCumulatedDexRegisters = 64; +static constexpr size_t kMaximumNumberOfCumulatedDexRegisters = 32; // Limit recursive call inlining, which do not benefit from too // much inlining compared to code locality. diff --git a/compiler/optimizing/instruction_builder.cc b/compiler/optimizing/instruction_builder.cc index df9e7164ed..a73b1246d8 100644 --- a/compiler/optimizing/instruction_builder.cc +++ b/compiler/optimizing/instruction_builder.cc @@ -888,7 +888,7 @@ bool HInstructionBuilder::BuildInvoke(const Instruction& instruction, } HInvokeStaticOrDirect::DispatchInfo dispatch_info = { - HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod, + HInvokeStaticOrDirect::MethodLoadKind::kRuntimeCall, HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod, 0u }; diff --git a/compiler/optimizing/intrinsics_arm64.cc b/compiler/optimizing/intrinsics_arm64.cc index 990a773a95..37d79814be 100644 --- a/compiler/optimizing/intrinsics_arm64.cc +++ b/compiler/optimizing/intrinsics_arm64.cc @@ -124,12 +124,12 @@ class IntrinsicSlowPathARM64 : public SlowPathCodeARM64 { // are no pools emitted. vixl::EmissionCheckScope guard(codegen->GetVIXLAssembler(), kInvokeCodeMarginSizeInBytes); if (invoke_->IsInvokeStaticOrDirect()) { - codegen->GenerateStaticOrDirectCall(invoke_->AsInvokeStaticOrDirect(), - LocationFrom(kArtMethodRegister)); + codegen->GenerateStaticOrDirectCall( + invoke_->AsInvokeStaticOrDirect(), LocationFrom(kArtMethodRegister), this); } else { - codegen->GenerateVirtualCall(invoke_->AsInvokeVirtual(), LocationFrom(kArtMethodRegister)); + codegen->GenerateVirtualCall( + invoke_->AsInvokeVirtual(), LocationFrom(kArtMethodRegister), this); } - codegen->RecordPcInfo(invoke_, invoke_->GetDexPc(), this); } // Copy the result back to the expected output. diff --git a/compiler/optimizing/intrinsics_arm_vixl.cc b/compiler/optimizing/intrinsics_arm_vixl.cc index 0e04b9a950..3c9b613803 100644 --- a/compiler/optimizing/intrinsics_arm_vixl.cc +++ b/compiler/optimizing/intrinsics_arm_vixl.cc @@ -97,11 +97,10 @@ class IntrinsicSlowPathARMVIXL : public SlowPathCodeARMVIXL { Location method_loc = MoveArguments(codegen); if (invoke_->IsInvokeStaticOrDirect()) { - codegen->GenerateStaticOrDirectCall(invoke_->AsInvokeStaticOrDirect(), method_loc); + codegen->GenerateStaticOrDirectCall(invoke_->AsInvokeStaticOrDirect(), method_loc, this); } else { - codegen->GenerateVirtualCall(invoke_->AsInvokeVirtual(), method_loc); + codegen->GenerateVirtualCall(invoke_->AsInvokeVirtual(), method_loc, this); } - codegen->RecordPcInfo(invoke_, invoke_->GetDexPc(), this); // Copy the result back to the expected output. Location out = invoke_->GetLocations()->Out(); diff --git a/compiler/optimizing/intrinsics_mips.cc b/compiler/optimizing/intrinsics_mips.cc index ea3e9e5ec9..4cea6dfdfb 100644 --- a/compiler/optimizing/intrinsics_mips.cc +++ b/compiler/optimizing/intrinsics_mips.cc @@ -112,12 +112,12 @@ class IntrinsicSlowPathMIPS : public SlowPathCodeMIPS { MoveArguments(invoke_, codegen); if (invoke_->IsInvokeStaticOrDirect()) { - codegen->GenerateStaticOrDirectCall(invoke_->AsInvokeStaticOrDirect(), - Location::RegisterLocation(A0)); + codegen->GenerateStaticOrDirectCall( + invoke_->AsInvokeStaticOrDirect(), Location::RegisterLocation(A0), this); } else { - codegen->GenerateVirtualCall(invoke_->AsInvokeVirtual(), Location::RegisterLocation(A0)); + codegen->GenerateVirtualCall( + invoke_->AsInvokeVirtual(), Location::RegisterLocation(A0), this); } - codegen->RecordPcInfo(invoke_, invoke_->GetDexPc(), this); // Copy the result back to the expected output. Location out = invoke_->GetLocations()->Out(); diff --git a/compiler/optimizing/intrinsics_mips64.cc b/compiler/optimizing/intrinsics_mips64.cc index 2ecb1a3b02..d785567e0f 100644 --- a/compiler/optimizing/intrinsics_mips64.cc +++ b/compiler/optimizing/intrinsics_mips64.cc @@ -101,12 +101,12 @@ class IntrinsicSlowPathMIPS64 : public SlowPathCodeMIPS64 { MoveArguments(invoke_, codegen); if (invoke_->IsInvokeStaticOrDirect()) { - codegen->GenerateStaticOrDirectCall(invoke_->AsInvokeStaticOrDirect(), - Location::RegisterLocation(A0)); + codegen->GenerateStaticOrDirectCall( + invoke_->AsInvokeStaticOrDirect(), Location::RegisterLocation(A0), this); } else { - codegen->GenerateVirtualCall(invoke_->AsInvokeVirtual(), Location::RegisterLocation(A0)); + codegen->GenerateVirtualCall( + invoke_->AsInvokeVirtual(), Location::RegisterLocation(A0), this); } - codegen->RecordPcInfo(invoke_, invoke_->GetDexPc(), this); // Copy the result back to the expected output. Location out = invoke_->GetLocations()->Out(); diff --git a/compiler/optimizing/intrinsics_utils.h b/compiler/optimizing/intrinsics_utils.h index c1f9ae6425..8c69d9b643 100644 --- a/compiler/optimizing/intrinsics_utils.h +++ b/compiler/optimizing/intrinsics_utils.h @@ -56,11 +56,10 @@ class IntrinsicSlowPath : public SlowPathCode { Location method_loc = MoveArguments(codegen); if (invoke_->IsInvokeStaticOrDirect()) { - codegen->GenerateStaticOrDirectCall(invoke_->AsInvokeStaticOrDirect(), method_loc); + codegen->GenerateStaticOrDirectCall(invoke_->AsInvokeStaticOrDirect(), method_loc, this); } else { - codegen->GenerateVirtualCall(invoke_->AsInvokeVirtual(), method_loc); + codegen->GenerateVirtualCall(invoke_->AsInvokeVirtual(), method_loc, this); } - codegen->RecordPcInfo(invoke_, invoke_->GetDexPc(), this); // Copy the result back to the expected output. Location out = invoke_->GetLocations()->Out(); diff --git a/compiler/optimizing/intrinsics_x86.cc b/compiler/optimizing/intrinsics_x86.cc index a9da15d2ce..6b4851d541 100644 --- a/compiler/optimizing/intrinsics_x86.cc +++ b/compiler/optimizing/intrinsics_x86.cc @@ -796,7 +796,6 @@ static void InvokeOutOfLineIntrinsic(CodeGeneratorX86* codegen, HInvoke* invoke) DCHECK(invoke->IsInvokeStaticOrDirect()); codegen->GenerateStaticOrDirectCall(invoke->AsInvokeStaticOrDirect(), Location::RegisterLocation(EAX)); - codegen->RecordPcInfo(invoke, invoke->GetDexPc()); // Copy the result back to the expected output. Location out = invoke->GetLocations()->Out(); diff --git a/compiler/optimizing/intrinsics_x86_64.cc b/compiler/optimizing/intrinsics_x86_64.cc index 8100645e54..ef98b7be30 100644 --- a/compiler/optimizing/intrinsics_x86_64.cc +++ b/compiler/optimizing/intrinsics_x86_64.cc @@ -567,7 +567,6 @@ static void InvokeOutOfLineIntrinsic(CodeGeneratorX86_64* codegen, HInvoke* invo DCHECK(invoke->IsInvokeStaticOrDirect()); codegen->GenerateStaticOrDirectCall( invoke->AsInvokeStaticOrDirect(), Location::RegisterLocation(RDI)); - codegen->RecordPcInfo(invoke, invoke->GetDexPc()); // Copy the result back to the expected output. Location out = invoke->GetLocations()->Out(); diff --git a/compiler/optimizing/loop_optimization.cc b/compiler/optimizing/loop_optimization.cc index cfcafa5c24..9c8a632d40 100644 --- a/compiler/optimizing/loop_optimization.cc +++ b/compiler/optimizing/loop_optimization.cc @@ -499,6 +499,7 @@ void HLoopOptimization::OptimizeInnerLoop(LoopNode* node) { body = it.Current(); } } + CHECK(body != nullptr); // Ensure there is only a single exit point. if (header->GetSuccessors().size() != 2) { return; diff --git a/compiler/optimizing/nodes.cc b/compiler/optimizing/nodes.cc index 9a91287670..d0047c54f2 100644 --- a/compiler/optimizing/nodes.cc +++ b/compiler/optimizing/nodes.cc @@ -2623,7 +2623,7 @@ const DexFile& HInvokeStaticOrDirect::GetDexFileForPcRelativeDexCache() const { } bool HInvokeStaticOrDirect::NeedsDexCacheOfDeclaringClass() const { - if (GetMethodLoadKind() != MethodLoadKind::kDexCacheViaMethod) { + if (GetMethodLoadKind() != MethodLoadKind::kRuntimeCall) { return false; } if (!IsIntrinsic()) { @@ -2643,10 +2643,10 @@ std::ostream& operator<<(std::ostream& os, HInvokeStaticOrDirect::MethodLoadKind return os << "BootImageLinkTimePcRelative"; case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress: return os << "DirectAddress"; - case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: - return os << "DexCachePcRelative"; - case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: - return os << "DexCacheViaMethod"; + case HInvokeStaticOrDirect::MethodLoadKind::kBssEntry: + return os << "BssEntry"; + case HInvokeStaticOrDirect::MethodLoadKind::kRuntimeCall: + return os << "RuntimeCall"; default: LOG(FATAL) << "Unknown MethodLoadKind: " << static_cast<int>(rhs); UNREACHABLE(); diff --git a/compiler/optimizing/nodes.h b/compiler/optimizing/nodes.h index befd0ff97b..2867797e20 100644 --- a/compiler/optimizing/nodes.h +++ b/compiler/optimizing/nodes.h @@ -1410,12 +1410,7 @@ class HLoopInformationOutwardIterator : public ValueObject { M(IntermediateAddressIndex, Instruction) #endif -#ifndef ART_ENABLE_CODEGEN_arm #define FOR_EACH_CONCRETE_INSTRUCTION_ARM(M) -#else -#define FOR_EACH_CONCRETE_INSTRUCTION_ARM(M) \ - M(ArmDexCacheArraysBase, Instruction) -#endif #define FOR_EACH_CONCRETE_INSTRUCTION_ARM64(M) @@ -1424,7 +1419,6 @@ class HLoopInformationOutwardIterator : public ValueObject { #else #define FOR_EACH_CONCRETE_INSTRUCTION_MIPS(M) \ M(MipsComputeBaseMethodAddress, Instruction) \ - M(MipsDexCacheArraysBase, Instruction) \ M(MipsPackedSwitch, Instruction) #endif @@ -4166,17 +4160,13 @@ class HInvokeStaticOrDirect FINAL : public HInvoke { // Used for app->boot calls with non-relocatable image and for JIT-compiled calls. kDirectAddress, - // Load from resolved methods array in the dex cache using a PC-relative load. - // Used when we need to use the dex cache, for example for invoke-static that - // may cause class initialization (the entry may point to a resolution method), - // and we know that we can access the dex cache arrays using a PC-relative load. - kDexCachePcRelative, - - // Use ArtMethod* from the resolved methods of the compiled method's own ArtMethod*. - // Used for JIT when we need to use the dex cache. This is also the last-resort-kind - // used when other kinds are unavailable (say, dex cache arrays are not PC-relative) - // or unimplemented or impractical (i.e. slow) on a particular architecture. - kDexCacheViaMethod, + // Load from an entry in the .bss section using a PC-relative load. + // Used for classes outside boot image when .bss is accessible with a PC-relative load. + kBssEntry, + + // Make a runtime call to resolve and call the method. This is the last-resort-kind + // used when other kinds are unimplemented on a particular architecture. + kRuntimeCall, }; // Determines the location of the code pointer. @@ -4197,7 +4187,6 @@ class HInvokeStaticOrDirect FINAL : public HInvoke { // - thread entrypoint offset for kStringInit method if this is a string init invoke. // Note that there are multiple string init methods, each having its own offset. // - the method address for kDirectAddress - // - the dex cache arrays offset for kDexCachePcRel. uint64_t method_load_data; }; @@ -4298,12 +4287,9 @@ class HInvokeStaticOrDirect FINAL : public HInvoke { bool NeedsDexCacheOfDeclaringClass() const OVERRIDE; bool IsStringInit() const { return GetMethodLoadKind() == MethodLoadKind::kStringInit; } bool HasMethodAddress() const { return GetMethodLoadKind() == MethodLoadKind::kDirectAddress; } - bool HasPcRelativeDexCache() const { - return GetMethodLoadKind() == MethodLoadKind::kDexCachePcRelative; - } bool HasPcRelativeMethodLoadKind() const { return GetMethodLoadKind() == MethodLoadKind::kBootImageLinkTimePcRelative || - GetMethodLoadKind() == MethodLoadKind::kDexCachePcRelative; + GetMethodLoadKind() == MethodLoadKind::kBssEntry; } bool HasCurrentMethodInput() const { // This function can be called only after the invoke has been fully initialized by the builder. @@ -4327,11 +4313,6 @@ class HInvokeStaticOrDirect FINAL : public HInvoke { return dispatch_info_.method_load_data; } - uint32_t GetDexCacheArrayOffset() const { - DCHECK(HasPcRelativeDexCache()); - return dispatch_info_.method_load_data; - } - const DexFile& GetDexFileForPcRelativeDexCache() const; ClinitCheckRequirement GetClinitCheckRequirement() const { @@ -4376,7 +4357,7 @@ class HInvokeStaticOrDirect FINAL : public HInvoke { // Does this method load kind need the current method as an input? static bool NeedsCurrentMethodInput(MethodLoadKind kind) { - return kind == MethodLoadKind::kRecursive || kind == MethodLoadKind::kDexCacheViaMethod; + return kind == MethodLoadKind::kRecursive || kind == MethodLoadKind::kRuntimeCall; } DECLARE_INSTRUCTION(InvokeStaticOrDirect); @@ -6881,9 +6862,6 @@ class HParallelMove FINAL : public HTemplateInstruction<0> { #if defined(ART_ENABLE_CODEGEN_arm) || defined(ART_ENABLE_CODEGEN_arm64) #include "nodes_shared.h" #endif -#ifdef ART_ENABLE_CODEGEN_arm -#include "nodes_arm.h" -#endif #ifdef ART_ENABLE_CODEGEN_mips #include "nodes_mips.h" #endif diff --git a/compiler/optimizing/nodes_arm.h b/compiler/optimizing/nodes_arm.h deleted file mode 100644 index d9f9740e73..0000000000 --- a/compiler/optimizing/nodes_arm.h +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (C) 2015 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef ART_COMPILER_OPTIMIZING_NODES_ARM_H_ -#define ART_COMPILER_OPTIMIZING_NODES_ARM_H_ - -namespace art { - -class HArmDexCacheArraysBase FINAL : public HExpression<0> { - public: - explicit HArmDexCacheArraysBase(const DexFile& dex_file) - : HExpression(Primitive::kPrimInt, SideEffects::None(), kNoDexPc), - dex_file_(&dex_file), - element_offset_(static_cast<size_t>(-1)) { } - - bool CanBeMoved() const OVERRIDE { return true; } - - void UpdateElementOffset(size_t element_offset) { - // Use the lowest offset from the requested elements so that all offsets from - // this base are non-negative because our assemblers emit negative-offset loads - // as a sequence of two or more instructions. (However, positive offsets beyond - // 4KiB also require two or more instructions, so this simple heuristic could - // be improved for cases where there is a dense cluster of elements far from - // the lowest offset. This is expected to be rare enough though, so we choose - // not to spend compile time on elaborate calculations.) - element_offset_ = std::min(element_offset_, element_offset); - } - - const DexFile& GetDexFile() const { - return *dex_file_; - } - - size_t GetElementOffset() const { - return element_offset_; - } - - DECLARE_INSTRUCTION(ArmDexCacheArraysBase); - - private: - const DexFile* dex_file_; - size_t element_offset_; - - DISALLOW_COPY_AND_ASSIGN(HArmDexCacheArraysBase); -}; - -} // namespace art - -#endif // ART_COMPILER_OPTIMIZING_NODES_ARM_H_ diff --git a/compiler/optimizing/nodes_mips.h b/compiler/optimizing/nodes_mips.h index 36431c1fb9..8e439d9621 100644 --- a/compiler/optimizing/nodes_mips.h +++ b/compiler/optimizing/nodes_mips.h @@ -34,38 +34,6 @@ class HMipsComputeBaseMethodAddress : public HExpression<0> { DISALLOW_COPY_AND_ASSIGN(HMipsComputeBaseMethodAddress); }; -class HMipsDexCacheArraysBase : public HExpression<0> { - public: - explicit HMipsDexCacheArraysBase(const DexFile& dex_file) - : HExpression(Primitive::kPrimInt, SideEffects::None(), kNoDexPc), - dex_file_(&dex_file), - element_offset_(static_cast<size_t>(-1)) { } - - bool CanBeMoved() const OVERRIDE { return true; } - - void UpdateElementOffset(size_t element_offset) { - // We'll maximize the range of a single load instruction for dex cache array accesses - // by aligning offset -32768 with the offset of the first used element. - element_offset_ = std::min(element_offset_, element_offset); - } - - const DexFile& GetDexFile() const { - return *dex_file_; - } - - size_t GetElementOffset() const { - return element_offset_; - } - - DECLARE_INSTRUCTION(MipsDexCacheArraysBase); - - private: - const DexFile* dex_file_; - size_t element_offset_; - - DISALLOW_COPY_AND_ASSIGN(HMipsDexCacheArraysBase); -}; - // Mips version of HPackedSwitch that holds a pointer to the base method address. class HMipsPackedSwitch FINAL : public HTemplateInstruction<2> { public: diff --git a/compiler/optimizing/optimizing_compiler.cc b/compiler/optimizing/optimizing_compiler.cc index e5ab00bce3..890ba674b5 100644 --- a/compiler/optimizing/optimizing_compiler.cc +++ b/compiler/optimizing/optimizing_compiler.cc @@ -24,16 +24,11 @@ #include "android-base/strings.h" -#ifdef ART_ENABLE_CODEGEN_arm -#include "dex_cache_array_fixups_arm.h" -#endif - #ifdef ART_ENABLE_CODEGEN_arm64 #include "instruction_simplifier_arm64.h" #endif #ifdef ART_ENABLE_CODEGEN_mips -#include "dex_cache_array_fixups_mips.h" #include "pc_relative_fixups_mips.h" #endif @@ -522,8 +517,6 @@ static HOptimization* BuildOptimization( } else if (opt_name == CodeSinking::kCodeSinkingPassName) { return new (arena) CodeSinking(graph, stats); #ifdef ART_ENABLE_CODEGEN_arm - } else if (opt_name == arm::DexCacheArrayFixups::kDexCacheArrayFixupsArmPassName) { - return new (arena) arm::DexCacheArrayFixups(graph, codegen, stats); } else if (opt_name == arm::InstructionSimplifierArm::kInstructionSimplifierArmPassName) { return new (arena) arm::InstructionSimplifierArm(graph, stats); #endif @@ -532,8 +525,6 @@ static HOptimization* BuildOptimization( return new (arena) arm64::InstructionSimplifierArm64(graph, stats); #endif #ifdef ART_ENABLE_CODEGEN_mips - } else if (opt_name == mips::DexCacheArrayFixups::kDexCacheArrayFixupsMipsPassName) { - return new (arena) mips::DexCacheArrayFixups(graph, codegen, stats); } else if (opt_name == mips::PcRelativeFixups::kPcRelativeFixupsMipsPassName) { return new (arena) mips::PcRelativeFixups(graph, codegen, stats); #endif @@ -641,8 +632,6 @@ void OptimizingCompiler::RunArchOptimizations(InstructionSet instruction_set, #if defined(ART_ENABLE_CODEGEN_arm) case kThumb2: case kArm: { - arm::DexCacheArrayFixups* fixups = - new (arena) arm::DexCacheArrayFixups(graph, codegen, stats); arm::InstructionSimplifierArm* simplifier = new (arena) arm::InstructionSimplifierArm(graph, stats); SideEffectsAnalysis* side_effects = new (arena) SideEffectsAnalysis(graph); @@ -653,7 +642,6 @@ void OptimizingCompiler::RunArchOptimizations(InstructionSet instruction_set, simplifier, side_effects, gvn, - fixups, scheduling, }; RunOptimizations(arm_optimizations, arraysize(arm_optimizations), pass_observer); @@ -682,11 +670,8 @@ void OptimizingCompiler::RunArchOptimizations(InstructionSet instruction_set, case kMips: { mips::PcRelativeFixups* pc_relative_fixups = new (arena) mips::PcRelativeFixups(graph, codegen, stats); - mips::DexCacheArrayFixups* dex_cache_array_fixups = - new (arena) mips::DexCacheArrayFixups(graph, codegen, stats); HOptimization* mips_optimizations[] = { pc_relative_fixups, - dex_cache_array_fixups }; RunOptimizations(mips_optimizations, arraysize(mips_optimizations), pass_observer); break; diff --git a/compiler/optimizing/pc_relative_fixups_mips.cc b/compiler/optimizing/pc_relative_fixups_mips.cc index bce54bf49a..21b645279e 100644 --- a/compiler/optimizing/pc_relative_fixups_mips.cc +++ b/compiler/optimizing/pc_relative_fixups_mips.cc @@ -59,10 +59,9 @@ class PCRelativeHandlerVisitor : public HGraphVisitor { } void VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) OVERRIDE { - // If this is an invoke with PC-relative pointer to a method, + // If this is an invoke with PC-relative load kind, // we need to add the base as the special input. - if (invoke->GetMethodLoadKind() == - HInvokeStaticOrDirect::MethodLoadKind::kBootImageLinkTimePcRelative && + if (invoke->HasPcRelativeMethodLoadKind() && !IsCallFreeIntrinsic<IntrinsicLocationsBuilderMIPS>(invoke, codegen_)) { InitializePCRelativeBasePointer(); // Add the special argument base to the method. diff --git a/compiler/optimizing/scheduler_arm.cc b/compiler/optimizing/scheduler_arm.cc index 832a7e1571..e78cd78aa2 100644 --- a/compiler/optimizing/scheduler_arm.cc +++ b/compiler/optimizing/scheduler_arm.cc @@ -818,10 +818,5 @@ void SchedulingLatencyVisitorARM::VisitTypeConversion(HTypeConversion* instr) { } } -void SchedulingLatencyVisitorARM::VisitArmDexCacheArraysBase(art::HArmDexCacheArraysBase*) { - last_visited_internal_latency_ = kArmIntegerOpLatency; - last_visited_latency_ = kArmIntegerOpLatency; -} - } // namespace arm } // namespace art diff --git a/compiler/optimizing/sharpening.cc b/compiler/optimizing/sharpening.cc index 106b709eda..8bd568befd 100644 --- a/compiler/optimizing/sharpening.cc +++ b/compiler/optimizing/sharpening.cc @@ -128,15 +128,8 @@ void HSharpening::SharpenInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke, method_load_kind = HInvokeStaticOrDirect::MethodLoadKind::kBootImageLinkTimePcRelative; code_ptr_location = HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod; } else { - // Use PC-relative access to the dex cache arrays. - method_load_kind = HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative; - // Note: we use the invoke's graph instead of the codegen graph, which are - // different when inlining (the codegen graph is the most outer graph). The - // invoke's dex method index is relative to the dex file where the invoke's graph - // was built from. - DexCacheArraysLayout layout(GetInstructionSetPointerSize(codegen->GetInstructionSet()), - &invoke->GetBlock()->GetGraph()->GetDexFile()); - method_load_data = layout.MethodOffset(invoke->GetDexMethodIndex()); + // Use PC-relative access to the .bss methods arrays. + method_load_kind = HInvokeStaticOrDirect::MethodLoadKind::kBssEntry; code_ptr_location = HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod; } diff --git a/compiler/utils/arm/assembler_arm.h b/compiler/utils/arm/assembler_arm.h index 0f24e81be2..bb23a29064 100644 --- a/compiler/utils/arm/assembler_arm.h +++ b/compiler/utils/arm/assembler_arm.h @@ -25,7 +25,7 @@ #include "base/bit_utils.h" #include "base/enums.h" #include "base/logging.h" -#include "base/stl_util.h" +#include "base/stl_util_identity.h" #include "base/value_object.h" #include "constants_arm.h" #include "utils/arm/assembler_arm_shared.h" diff --git a/compiler/utils/arm/assembler_arm_vixl.cc b/compiler/utils/arm/assembler_arm_vixl.cc index 6afc3ddecb..eb3f870432 100644 --- a/compiler/utils/arm/assembler_arm_vixl.cc +++ b/compiler/utils/arm/assembler_arm_vixl.cc @@ -18,6 +18,8 @@ #include <type_traits> #include "assembler_arm_vixl.h" +#include "base/bit_utils.h" +#include "base/bit_utils_iterator.h" #include "entrypoints/quick/quick_entrypoints.h" #include "thread.h" diff --git a/compiler/utils/mips/assembler_mips.cc b/compiler/utils/mips/assembler_mips.cc index a99d02d4d0..0b05b752da 100644 --- a/compiler/utils/mips/assembler_mips.cc +++ b/compiler/utils/mips/assembler_mips.cc @@ -404,6 +404,129 @@ uint32_t MipsAssembler::EmitFI(int opcode, int fmt, FRegister ft, uint16_t imm) return encoding; } +uint32_t MipsAssembler::EmitMsa3R(int operation, + int df, + VectorRegister wt, + VectorRegister ws, + VectorRegister wd, + int minor_opcode) { + CHECK_NE(wt, kNoVectorRegister); + CHECK_NE(ws, kNoVectorRegister); + CHECK_NE(wd, kNoVectorRegister); + uint32_t encoding = static_cast<uint32_t>(kMsaMajorOpcode) << kOpcodeShift | + operation << kMsaOperationShift | + df << kDfShift | + static_cast<uint32_t>(wt) << kWtShift | + static_cast<uint32_t>(ws) << kWsShift | + static_cast<uint32_t>(wd) << kWdShift | + minor_opcode; + Emit(encoding); + return encoding; +} + +uint32_t MipsAssembler::EmitMsaBIT(int operation, + int df_m, + VectorRegister ws, + VectorRegister wd, + int minor_opcode) { + CHECK_NE(ws, kNoVectorRegister); + CHECK_NE(wd, kNoVectorRegister); + uint32_t encoding = static_cast<uint32_t>(kMsaMajorOpcode) << kOpcodeShift | + operation << kMsaOperationShift | + df_m << kDfMShift | + static_cast<uint32_t>(ws) << kWsShift | + static_cast<uint32_t>(wd) << kWdShift | + minor_opcode; + Emit(encoding); + return encoding; +} + +uint32_t MipsAssembler::EmitMsaELM(int operation, + int df_n, + VectorRegister ws, + VectorRegister wd, + int minor_opcode) { + CHECK_NE(ws, kNoVectorRegister); + CHECK_NE(wd, kNoVectorRegister); + uint32_t encoding = static_cast<uint32_t>(kMsaMajorOpcode) << kOpcodeShift | + operation << kMsaELMOperationShift | + df_n << kDfNShift | + static_cast<uint32_t>(ws) << kWsShift | + static_cast<uint32_t>(wd) << kWdShift | + minor_opcode; + Emit(encoding); + return encoding; +} + +uint32_t MipsAssembler::EmitMsaMI10(int s10, + Register rs, + VectorRegister wd, + int minor_opcode, + int df) { + CHECK_NE(rs, kNoRegister); + CHECK_NE(wd, kNoVectorRegister); + CHECK(IsUint<10>(s10)) << s10; + uint32_t encoding = static_cast<uint32_t>(kMsaMajorOpcode) << kOpcodeShift | + s10 << kS10Shift | + static_cast<uint32_t>(rs) << kWsShift | + static_cast<uint32_t>(wd) << kWdShift | + minor_opcode << kS10MinorShift | + df; + Emit(encoding); + return encoding; +} + +uint32_t MipsAssembler::EmitMsaI10(int operation, + int df, + int i10, + VectorRegister wd, + int minor_opcode) { + CHECK_NE(wd, kNoVectorRegister); + CHECK(IsUint<10>(i10)) << i10; + uint32_t encoding = static_cast<uint32_t>(kMsaMajorOpcode) << kOpcodeShift | + operation << kMsaOperationShift | + df << kDfShift | + i10 << kI10Shift | + static_cast<uint32_t>(wd) << kWdShift | + minor_opcode; + Emit(encoding); + return encoding; +} + +uint32_t MipsAssembler::EmitMsa2R(int operation, + int df, + VectorRegister ws, + VectorRegister wd, + int minor_opcode) { + CHECK_NE(ws, kNoVectorRegister); + CHECK_NE(wd, kNoVectorRegister); + uint32_t encoding = static_cast<uint32_t>(kMsaMajorOpcode) << kOpcodeShift | + operation << kMsa2ROperationShift | + df << kDf2RShift | + static_cast<uint32_t>(ws) << kWsShift | + static_cast<uint32_t>(wd) << kWdShift | + minor_opcode; + Emit(encoding); + return encoding; +} + +uint32_t MipsAssembler::EmitMsa2RF(int operation, + int df, + VectorRegister ws, + VectorRegister wd, + int minor_opcode) { + CHECK_NE(ws, kNoVectorRegister); + CHECK_NE(wd, kNoVectorRegister); + uint32_t encoding = static_cast<uint32_t>(kMsaMajorOpcode) << kOpcodeShift | + operation << kMsa2RFOperationShift | + df << kDf2RShift | + static_cast<uint32_t>(ws) << kWsShift | + static_cast<uint32_t>(wd) << kWdShift | + minor_opcode; + Emit(encoding); + return encoding; +} + void MipsAssembler::Addu(Register rd, Register rs, Register rt) { DsFsmInstrRrr(EmitR(0, rs, rt, rd, 0, 0x21), rd, rs, rt); } @@ -635,9 +758,8 @@ void MipsAssembler::Ins(Register rd, Register rt, int pos, int size) { DsFsmInstrRrr(EmitR(0x1f, rt, rd, static_cast<Register>(pos + size - 1), pos, 0x04), rd, rd, rt); } -// TODO: This instruction is available in both R6 and MSA and it should be used when available. void MipsAssembler::Lsa(Register rd, Register rs, Register rt, int saPlusOne) { - CHECK(IsR6()); + CHECK(IsR6() || HasMsa()); CHECK(1 <= saPlusOne && saPlusOne <= 4) << saPlusOne; int sa = saPlusOne - 1; DsFsmInstrRrr(EmitR(0x0, rs, rt, rd, sa, 0x05), rd, rs, rt); @@ -653,7 +775,7 @@ void MipsAssembler::ShiftAndAdd(Register dst, if (shamt == TIMES_1) { // Catch the special case where the shift amount is zero (0). Addu(dst, src_base, src_idx); - } else if (IsR6()) { + } else if (IsR6() || HasMsa()) { Lsa(dst, src_idx, src_base, shamt); } else { Sll(tmp, src_idx, shamt); @@ -1709,6 +1831,1079 @@ void MipsAssembler::PopAndReturn(Register rd, Register rt) { SetReorder(reordering); } +void MipsAssembler::AndV(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x0, 0x0, wt, ws, wd, 0x1e), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::OrV(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x0, 0x1, wt, ws, wd, 0x1e), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::NorV(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x0, 0x2, wt, ws, wd, 0x1e), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::XorV(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x0, 0x3, wt, ws, wd, 0x1e), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::AddvB(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x0, 0x0, wt, ws, wd, 0xe), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::AddvH(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x0, 0x1, wt, ws, wd, 0xe), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::AddvW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x0, 0x2, wt, ws, wd, 0xe), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::AddvD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x0, 0x3, wt, ws, wd, 0xe), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::SubvB(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x1, 0x0, wt, ws, wd, 0xe), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::SubvH(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x1, 0x1, wt, ws, wd, 0xe), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::SubvW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x1, 0x2, wt, ws, wd, 0xe), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::SubvD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x1, 0x3, wt, ws, wd, 0xe), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::MulvB(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x0, 0x0, wt, ws, wd, 0x12), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::MulvH(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x0, 0x1, wt, ws, wd, 0x12), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::MulvW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x0, 0x2, wt, ws, wd, 0x12), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::MulvD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x0, 0x3, wt, ws, wd, 0x12), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Div_sB(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x4, 0x0, wt, ws, wd, 0x12), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Div_sH(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x4, 0x1, wt, ws, wd, 0x12), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Div_sW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x4, 0x2, wt, ws, wd, 0x12), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Div_sD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x4, 0x3, wt, ws, wd, 0x12), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Div_uB(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x5, 0x0, wt, ws, wd, 0x12), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Div_uH(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x5, 0x1, wt, ws, wd, 0x12), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Div_uW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x5, 0x2, wt, ws, wd, 0x12), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Div_uD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x5, 0x3, wt, ws, wd, 0x12), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Mod_sB(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x6, 0x0, wt, ws, wd, 0x12), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Mod_sH(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x6, 0x1, wt, ws, wd, 0x12), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Mod_sW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x6, 0x2, wt, ws, wd, 0x12), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Mod_sD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x6, 0x3, wt, ws, wd, 0x12), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Mod_uB(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x7, 0x0, wt, ws, wd, 0x12), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Mod_uH(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x7, 0x1, wt, ws, wd, 0x12), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Mod_uW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x7, 0x2, wt, ws, wd, 0x12), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Mod_uD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x7, 0x3, wt, ws, wd, 0x12), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Add_aB(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x0, 0x0, wt, ws, wd, 0x10), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Add_aH(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x0, 0x1, wt, ws, wd, 0x10), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Add_aW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x0, 0x2, wt, ws, wd, 0x10), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Add_aD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x0, 0x3, wt, ws, wd, 0x10), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Ave_sB(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x4, 0x0, wt, ws, wd, 0x10), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Ave_sH(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x4, 0x1, wt, ws, wd, 0x10), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Ave_sW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x4, 0x2, wt, ws, wd, 0x10), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Ave_sD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x4, 0x3, wt, ws, wd, 0x10), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Ave_uB(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x5, 0x0, wt, ws, wd, 0x10), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Ave_uH(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x5, 0x1, wt, ws, wd, 0x10), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Ave_uW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x5, 0x2, wt, ws, wd, 0x10), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Ave_uD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x5, 0x3, wt, ws, wd, 0x10), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Aver_sB(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x6, 0x0, wt, ws, wd, 0x10), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Aver_sH(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x6, 0x1, wt, ws, wd, 0x10), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Aver_sW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x6, 0x2, wt, ws, wd, 0x10), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Aver_sD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x6, 0x3, wt, ws, wd, 0x10), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Aver_uB(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x7, 0x0, wt, ws, wd, 0x10), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Aver_uH(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x7, 0x1, wt, ws, wd, 0x10), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Aver_uW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x7, 0x2, wt, ws, wd, 0x10), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Aver_uD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x7, 0x3, wt, ws, wd, 0x10), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Max_sB(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x2, 0x0, wt, ws, wd, 0xe), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Max_sH(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x2, 0x1, wt, ws, wd, 0xe), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Max_sW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x2, 0x2, wt, ws, wd, 0xe), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Max_sD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x2, 0x3, wt, ws, wd, 0xe), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Max_uB(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x3, 0x0, wt, ws, wd, 0xe), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Max_uH(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x3, 0x1, wt, ws, wd, 0xe), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Max_uW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x3, 0x2, wt, ws, wd, 0xe), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Max_uD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x3, 0x3, wt, ws, wd, 0xe), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Min_sB(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x4, 0x0, wt, ws, wd, 0xe), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Min_sH(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x4, 0x1, wt, ws, wd, 0xe), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Min_sW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x4, 0x2, wt, ws, wd, 0xe), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Min_sD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x4, 0x3, wt, ws, wd, 0xe), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Min_uB(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x5, 0x0, wt, ws, wd, 0xe), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Min_uH(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x5, 0x1, wt, ws, wd, 0xe), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Min_uW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x5, 0x2, wt, ws, wd, 0xe), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Min_uD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x5, 0x3, wt, ws, wd, 0xe), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::FaddW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x0, 0x0, wt, ws, wd, 0x1b), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::FaddD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x0, 0x1, wt, ws, wd, 0x1b), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::FsubW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x0, 0x2, wt, ws, wd, 0x1b), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::FsubD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x0, 0x3, wt, ws, wd, 0x1b), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::FmulW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x1, 0x0, wt, ws, wd, 0x1b), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::FmulD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x1, 0x1, wt, ws, wd, 0x1b), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::FdivW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x1, 0x2, wt, ws, wd, 0x1b), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::FdivD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x1, 0x3, wt, ws, wd, 0x1b), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::FmaxW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x7, 0x0, wt, ws, wd, 0x1b), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::FmaxD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x7, 0x1, wt, ws, wd, 0x1b), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::FminW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x6, 0x0, wt, ws, wd, 0x1b), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::FminD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x6, 0x1, wt, ws, wd, 0x1b), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::Ffint_sW(VectorRegister wd, VectorRegister ws) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa2RF(0x19e, 0x0, ws, wd, 0x1e), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(ws)); +} + +void MipsAssembler::Ffint_sD(VectorRegister wd, VectorRegister ws) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa2RF(0x19e, 0x1, ws, wd, 0x1e), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(ws)); +} + +void MipsAssembler::Ftint_sW(VectorRegister wd, VectorRegister ws) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa2RF(0x19c, 0x0, ws, wd, 0x1e), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(ws)); +} + +void MipsAssembler::Ftint_sD(VectorRegister wd, VectorRegister ws) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa2RF(0x19c, 0x1, ws, wd, 0x1e), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(ws)); +} + +void MipsAssembler::SllB(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x0, 0x0, wt, ws, wd, 0xd), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::SllH(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x0, 0x1, wt, ws, wd, 0xd), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::SllW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x0, 0x2, wt, ws, wd, 0xd), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::SllD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x0, 0x3, wt, ws, wd, 0xd), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::SraB(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x1, 0x0, wt, ws, wd, 0xd), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::SraH(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x1, 0x1, wt, ws, wd, 0xd), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::SraW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x1, 0x2, wt, ws, wd, 0xd), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::SraD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x1, 0x3, wt, ws, wd, 0xd), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::SrlB(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x2, 0x0, wt, ws, wd, 0xd), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::SrlH(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x2, 0x1, wt, ws, wd, 0xd), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::SrlW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x2, 0x2, wt, ws, wd, 0xd), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::SrlD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x2, 0x3, wt, ws, wd, 0xd), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::SlliB(VectorRegister wd, VectorRegister ws, int shamt3) { + CHECK(HasMsa()); + CHECK(IsUint<3>(shamt3)) << shamt3; + DsFsmInstrFff(EmitMsaBIT(0x0, shamt3 | kMsaDfMByteMask, ws, wd, 0x9), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(ws)); +} + +void MipsAssembler::SlliH(VectorRegister wd, VectorRegister ws, int shamt4) { + CHECK(HasMsa()); + CHECK(IsUint<4>(shamt4)) << shamt4; + DsFsmInstrFff(EmitMsaBIT(0x0, shamt4 | kMsaDfMHalfwordMask, ws, wd, 0x9), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(ws)); +} + +void MipsAssembler::SlliW(VectorRegister wd, VectorRegister ws, int shamt5) { + CHECK(HasMsa()); + CHECK(IsUint<5>(shamt5)) << shamt5; + DsFsmInstrFff(EmitMsaBIT(0x0, shamt5 | kMsaDfMWordMask, ws, wd, 0x9), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(ws)); +} + +void MipsAssembler::SlliD(VectorRegister wd, VectorRegister ws, int shamt6) { + CHECK(HasMsa()); + CHECK(IsUint<6>(shamt6)) << shamt6; + DsFsmInstrFff(EmitMsaBIT(0x0, shamt6 | kMsaDfMDoublewordMask, ws, wd, 0x9), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(ws)); +} + +void MipsAssembler::SraiB(VectorRegister wd, VectorRegister ws, int shamt3) { + CHECK(HasMsa()); + CHECK(IsUint<3>(shamt3)) << shamt3; + DsFsmInstrFff(EmitMsaBIT(0x1, shamt3 | kMsaDfMByteMask, ws, wd, 0x9), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(ws)); +} + +void MipsAssembler::SraiH(VectorRegister wd, VectorRegister ws, int shamt4) { + CHECK(HasMsa()); + CHECK(IsUint<4>(shamt4)) << shamt4; + DsFsmInstrFff(EmitMsaBIT(0x1, shamt4 | kMsaDfMHalfwordMask, ws, wd, 0x9), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(ws)); +} + +void MipsAssembler::SraiW(VectorRegister wd, VectorRegister ws, int shamt5) { + CHECK(HasMsa()); + CHECK(IsUint<5>(shamt5)) << shamt5; + DsFsmInstrFff(EmitMsaBIT(0x1, shamt5 | kMsaDfMWordMask, ws, wd, 0x9), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(ws)); +} + +void MipsAssembler::SraiD(VectorRegister wd, VectorRegister ws, int shamt6) { + CHECK(HasMsa()); + CHECK(IsUint<6>(shamt6)) << shamt6; + DsFsmInstrFff(EmitMsaBIT(0x1, shamt6 | kMsaDfMDoublewordMask, ws, wd, 0x9), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(ws)); +} + +void MipsAssembler::SrliB(VectorRegister wd, VectorRegister ws, int shamt3) { + CHECK(HasMsa()); + CHECK(IsUint<3>(shamt3)) << shamt3; + DsFsmInstrFff(EmitMsaBIT(0x2, shamt3 | kMsaDfMByteMask, ws, wd, 0x9), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(ws)); +} + +void MipsAssembler::SrliH(VectorRegister wd, VectorRegister ws, int shamt4) { + CHECK(HasMsa()); + CHECK(IsUint<4>(shamt4)) << shamt4; + DsFsmInstrFff(EmitMsaBIT(0x2, shamt4 | kMsaDfMHalfwordMask, ws, wd, 0x9), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(ws)); +} + +void MipsAssembler::SrliW(VectorRegister wd, VectorRegister ws, int shamt5) { + CHECK(HasMsa()); + CHECK(IsUint<5>(shamt5)) << shamt5; + DsFsmInstrFff(EmitMsaBIT(0x2, shamt5 | kMsaDfMWordMask, ws, wd, 0x9), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(ws)); +} + +void MipsAssembler::SrliD(VectorRegister wd, VectorRegister ws, int shamt6) { + CHECK(HasMsa()); + CHECK(IsUint<6>(shamt6)) << shamt6; + DsFsmInstrFff(EmitMsaBIT(0x2, shamt6 | kMsaDfMDoublewordMask, ws, wd, 0x9), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(ws)); +} + +void MipsAssembler::MoveV(VectorRegister wd, VectorRegister ws) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsaBIT(0x1, 0x3e, ws, wd, 0x19), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(ws)); +} + +void MipsAssembler::SplatiB(VectorRegister wd, VectorRegister ws, int n4) { + CHECK(HasMsa()); + CHECK(IsUint<4>(n4)) << n4; + DsFsmInstrFff(EmitMsaELM(0x1, n4 | kMsaDfNByteMask, ws, wd, 0x19), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(ws)); +} + +void MipsAssembler::SplatiH(VectorRegister wd, VectorRegister ws, int n3) { + CHECK(HasMsa()); + CHECK(IsUint<3>(n3)) << n3; + DsFsmInstrFff(EmitMsaELM(0x1, n3 | kMsaDfNHalfwordMask, ws, wd, 0x19), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(ws)); +} + +void MipsAssembler::SplatiW(VectorRegister wd, VectorRegister ws, int n2) { + CHECK(HasMsa()); + CHECK(IsUint<2>(n2)) << n2; + DsFsmInstrFff(EmitMsaELM(0x1, n2 | kMsaDfNWordMask, ws, wd, 0x19), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(ws)); +} + +void MipsAssembler::SplatiD(VectorRegister wd, VectorRegister ws, int n1) { + CHECK(HasMsa()); + CHECK(IsUint<1>(n1)) << n1; + DsFsmInstrFff(EmitMsaELM(0x1, n1 | kMsaDfNDoublewordMask, ws, wd, 0x19), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(ws)); +} + +void MipsAssembler::FillB(VectorRegister wd, Register rs) { + CHECK(HasMsa()); + DsFsmInstrFr(EmitMsa2R(0xc0, 0x0, static_cast<VectorRegister>(rs), wd, 0x1e), + static_cast<FRegister>(wd), + rs); +} + +void MipsAssembler::FillH(VectorRegister wd, Register rs) { + CHECK(HasMsa()); + DsFsmInstrFr(EmitMsa2R(0xc0, 0x1, static_cast<VectorRegister>(rs), wd, 0x1e), + static_cast<FRegister>(wd), + rs); +} + +void MipsAssembler::FillW(VectorRegister wd, Register rs) { + CHECK(HasMsa()); + DsFsmInstrFr(EmitMsa2R(0xc0, 0x2, static_cast<VectorRegister>(rs), wd, 0x1e), + static_cast<FRegister>(wd), + rs); +} + +void MipsAssembler::LdiB(VectorRegister wd, int imm8) { + CHECK(HasMsa()); + CHECK(IsInt<8>(imm8)) << imm8; + DsFsmInstrFr(EmitMsaI10(0x6, 0x0, imm8 & kMsaS10Mask, wd, 0x7), + static_cast<FRegister>(wd), + ZERO); +} + +void MipsAssembler::LdiH(VectorRegister wd, int imm10) { + CHECK(HasMsa()); + CHECK(IsInt<10>(imm10)) << imm10; + DsFsmInstrFr(EmitMsaI10(0x6, 0x1, imm10 & kMsaS10Mask, wd, 0x7), + static_cast<FRegister>(wd), + ZERO); +} + +void MipsAssembler::LdiW(VectorRegister wd, int imm10) { + CHECK(HasMsa()); + CHECK(IsInt<10>(imm10)) << imm10; + DsFsmInstrFr(EmitMsaI10(0x6, 0x2, imm10 & kMsaS10Mask, wd, 0x7), + static_cast<FRegister>(wd), + ZERO); +} + +void MipsAssembler::LdiD(VectorRegister wd, int imm10) { + CHECK(HasMsa()); + CHECK(IsInt<10>(imm10)) << imm10; + DsFsmInstrFr(EmitMsaI10(0x6, 0x3, imm10 & kMsaS10Mask, wd, 0x7), + static_cast<FRegister>(wd), + ZERO); +} + +void MipsAssembler::LdB(VectorRegister wd, Register rs, int offset) { + CHECK(HasMsa()); + CHECK(IsInt<10>(offset)) << offset; + DsFsmInstrFr(EmitMsaMI10(offset & kMsaS10Mask, rs, wd, 0x8, 0x0), + static_cast<FRegister>(wd), + rs); +} + +void MipsAssembler::LdH(VectorRegister wd, Register rs, int offset) { + CHECK(HasMsa()); + CHECK(IsInt<11>(offset)) << offset; + CHECK_ALIGNED(offset, kMipsHalfwordSize); + DsFsmInstrFr(EmitMsaMI10((offset >> TIMES_2) & kMsaS10Mask, rs, wd, 0x8, 0x1), + static_cast<FRegister>(wd), + rs); +} + +void MipsAssembler::LdW(VectorRegister wd, Register rs, int offset) { + CHECK(HasMsa()); + CHECK(IsInt<12>(offset)) << offset; + CHECK_ALIGNED(offset, kMipsWordSize); + DsFsmInstrFr(EmitMsaMI10((offset >> TIMES_4) & kMsaS10Mask, rs, wd, 0x8, 0x2), + static_cast<FRegister>(wd), + rs); +} + +void MipsAssembler::LdD(VectorRegister wd, Register rs, int offset) { + CHECK(HasMsa()); + CHECK(IsInt<13>(offset)) << offset; + CHECK_ALIGNED(offset, kMipsDoublewordSize); + DsFsmInstrFr(EmitMsaMI10((offset >> TIMES_8) & kMsaS10Mask, rs, wd, 0x8, 0x3), + static_cast<FRegister>(wd), + rs); +} + +void MipsAssembler::StB(VectorRegister wd, Register rs, int offset) { + CHECK(HasMsa()); + CHECK(IsInt<10>(offset)) << offset; + DsFsmInstrFR(EmitMsaMI10(offset & kMsaS10Mask, rs, wd, 0x9, 0x0), static_cast<FRegister>(wd), rs); +} + +void MipsAssembler::StH(VectorRegister wd, Register rs, int offset) { + CHECK(HasMsa()); + CHECK(IsInt<11>(offset)) << offset; + CHECK_ALIGNED(offset, kMipsHalfwordSize); + DsFsmInstrFR(EmitMsaMI10((offset >> TIMES_2) & kMsaS10Mask, rs, wd, 0x9, 0x1), + static_cast<FRegister>(wd), + rs); +} + +void MipsAssembler::StW(VectorRegister wd, Register rs, int offset) { + CHECK(HasMsa()); + CHECK(IsInt<12>(offset)) << offset; + CHECK_ALIGNED(offset, kMipsWordSize); + DsFsmInstrFR(EmitMsaMI10((offset >> TIMES_4) & kMsaS10Mask, rs, wd, 0x9, 0x2), + static_cast<FRegister>(wd), + rs); +} + +void MipsAssembler::StD(VectorRegister wd, Register rs, int offset) { + CHECK(HasMsa()); + CHECK(IsInt<13>(offset)) << offset; + CHECK_ALIGNED(offset, kMipsDoublewordSize); + DsFsmInstrFR(EmitMsaMI10((offset >> TIMES_8) & kMsaS10Mask, rs, wd, 0x9, 0x3), + static_cast<FRegister>(wd), + rs); +} + +void MipsAssembler::IlvrB(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x5, 0x0, wt, ws, wd, 0x14), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::IlvrH(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x5, 0x1, wt, ws, wd, 0x14), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::IlvrW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x5, 0x2, wt, ws, wd, 0x14), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + +void MipsAssembler::IlvrD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + DsFsmInstrFff(EmitMsa3R(0x5, 0x3, wt, ws, wd, 0x14), + static_cast<FRegister>(wd), + static_cast<FRegister>(ws), + static_cast<FRegister>(wt)); +} + void MipsAssembler::LoadConst32(Register rd, int32_t value) { if (IsUint<16>(value)) { // Use OR with (unsigned) immediate to encode 16b unsigned int. diff --git a/compiler/utils/mips/assembler_mips.h b/compiler/utils/mips/assembler_mips.h index 463daeb5d7..dd4ce6dc80 100644 --- a/compiler/utils/mips/assembler_mips.h +++ b/compiler/utils/mips/assembler_mips.h @@ -25,6 +25,7 @@ #include "base/arena_containers.h" #include "base/enums.h" #include "base/macros.h" +#include "base/stl_util_identity.h" #include "constants_mips.h" #include "globals.h" #include "managed_register_mips.h" @@ -36,6 +37,7 @@ namespace art { namespace mips { +static constexpr size_t kMipsHalfwordSize = 2; static constexpr size_t kMipsWordSize = 4; static constexpr size_t kMipsDoublewordSize = 8; @@ -194,6 +196,7 @@ class MipsAssembler FINAL : public Assembler, public JNIMacroAssembler<PointerSi last_position_adjustment_(0), last_old_position_(0), last_branch_id_(0), + has_msa_(instruction_set_features != nullptr ? instruction_set_features->HasMsa() : false), isa_features_(instruction_set_features) { cfi().DelayEmittingAdvancePCs(); } @@ -464,6 +467,149 @@ class MipsAssembler FINAL : public Assembler, public JNIMacroAssembler<PointerSi void Clear(Register rd); void Not(Register rd, Register rs); + // MSA instructions. + void AndV(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void OrV(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void NorV(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void XorV(VectorRegister wd, VectorRegister ws, VectorRegister wt); + + void AddvB(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void AddvH(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void AddvW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void AddvD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void SubvB(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void SubvH(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void SubvW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void SubvD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void MulvB(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void MulvH(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void MulvW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void MulvD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Div_sB(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Div_sH(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Div_sW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Div_sD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Div_uB(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Div_uH(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Div_uW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Div_uD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Mod_sB(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Mod_sH(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Mod_sW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Mod_sD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Mod_uB(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Mod_uH(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Mod_uW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Mod_uD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Add_aB(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Add_aH(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Add_aW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Add_aD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Ave_sB(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Ave_sH(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Ave_sW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Ave_sD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Ave_uB(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Ave_uH(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Ave_uW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Ave_uD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Aver_sB(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Aver_sH(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Aver_sW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Aver_sD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Aver_uB(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Aver_uH(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Aver_uW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Aver_uD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Max_sB(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Max_sH(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Max_sW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Max_sD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Max_uB(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Max_uH(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Max_uW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Max_uD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Min_sB(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Min_sH(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Min_sW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Min_sD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Min_uB(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Min_uH(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Min_uW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Min_uD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + + void FaddW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void FaddD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void FsubW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void FsubD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void FmulW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void FmulD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void FdivW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void FdivD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void FmaxW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void FmaxD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void FminW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void FminD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + + void Ffint_sW(VectorRegister wd, VectorRegister ws); + void Ffint_sD(VectorRegister wd, VectorRegister ws); + void Ftint_sW(VectorRegister wd, VectorRegister ws); + void Ftint_sD(VectorRegister wd, VectorRegister ws); + + void SllB(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void SllH(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void SllW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void SllD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void SraB(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void SraH(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void SraW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void SraD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void SrlB(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void SrlH(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void SrlW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void SrlD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + + // Immediate shift instructions, where shamtN denotes shift amount (must be between 0 and 2^N-1). + void SlliB(VectorRegister wd, VectorRegister ws, int shamt3); + void SlliH(VectorRegister wd, VectorRegister ws, int shamt4); + void SlliW(VectorRegister wd, VectorRegister ws, int shamt5); + void SlliD(VectorRegister wd, VectorRegister ws, int shamt6); + void SraiB(VectorRegister wd, VectorRegister ws, int shamt3); + void SraiH(VectorRegister wd, VectorRegister ws, int shamt4); + void SraiW(VectorRegister wd, VectorRegister ws, int shamt5); + void SraiD(VectorRegister wd, VectorRegister ws, int shamt6); + void SrliB(VectorRegister wd, VectorRegister ws, int shamt3); + void SrliH(VectorRegister wd, VectorRegister ws, int shamt4); + void SrliW(VectorRegister wd, VectorRegister ws, int shamt5); + void SrliD(VectorRegister wd, VectorRegister ws, int shamt6); + + void MoveV(VectorRegister wd, VectorRegister ws); + void SplatiB(VectorRegister wd, VectorRegister ws, int n4); + void SplatiH(VectorRegister wd, VectorRegister ws, int n3); + void SplatiW(VectorRegister wd, VectorRegister ws, int n2); + void SplatiD(VectorRegister wd, VectorRegister ws, int n1); + void FillB(VectorRegister wd, Register rs); + void FillH(VectorRegister wd, Register rs); + void FillW(VectorRegister wd, Register rs); + + void LdiB(VectorRegister wd, int imm8); + void LdiH(VectorRegister wd, int imm10); + void LdiW(VectorRegister wd, int imm10); + void LdiD(VectorRegister wd, int imm10); + void LdB(VectorRegister wd, Register rs, int offset); + void LdH(VectorRegister wd, Register rs, int offset); + void LdW(VectorRegister wd, Register rs, int offset); + void LdD(VectorRegister wd, Register rs, int offset); + void StB(VectorRegister wd, Register rs, int offset); + void StH(VectorRegister wd, Register rs, int offset); + void StW(VectorRegister wd, Register rs, int offset); + void StD(VectorRegister wd, Register rs, int offset); + + void IlvrB(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void IlvrH(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void IlvrW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void IlvrD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + // Higher level composite instructions. void LoadConst32(Register rd, int32_t value); void LoadConst64(Register reg_hi, Register reg_lo, int64_t value); @@ -1282,6 +1428,30 @@ class MipsAssembler FINAL : public Assembler, public JNIMacroAssembler<PointerSi uint32_t EmitFI(int opcode, int fmt, FRegister rt, uint16_t imm); void EmitBcondR2(BranchCondition cond, Register rs, Register rt, uint16_t imm16); void EmitBcondR6(BranchCondition cond, Register rs, Register rt, uint32_t imm16_21); + uint32_t EmitMsa3R(int operation, + int df, + VectorRegister wt, + VectorRegister ws, + VectorRegister wd, + int minor_opcode); + uint32_t EmitMsaBIT(int operation, + int df_m, + VectorRegister ws, + VectorRegister wd, + int minor_opcode); + uint32_t EmitMsaELM(int operation, + int df_n, + VectorRegister ws, + VectorRegister wd, + int minor_opcode); + uint32_t EmitMsaMI10(int s10, Register rs, VectorRegister wd, int minor_opcode, int df); + uint32_t EmitMsaI10(int operation, int df, int i10, VectorRegister wd, int minor_opcode); + uint32_t EmitMsa2R(int operation, int df, VectorRegister ws, VectorRegister wd, int minor_opcode); + uint32_t EmitMsa2RF(int operation, + int df, + VectorRegister ws, + VectorRegister wd, + int minor_opcode); void Buncond(MipsLabel* label); void Bcond(MipsLabel* label, BranchCondition condition, Register lhs, Register rhs = ZERO); @@ -1332,6 +1502,10 @@ class MipsAssembler FINAL : public Assembler, public JNIMacroAssembler<PointerSi // Emits exception block. void EmitExceptionPoll(MipsExceptionSlowPath* exception); + bool HasMsa() const { + return has_msa_; + } + bool IsR6() const { if (isa_features_ != nullptr) { return isa_features_->IsR6(); @@ -1386,6 +1560,8 @@ class MipsAssembler FINAL : public Assembler, public JNIMacroAssembler<PointerSi uint32_t last_old_position_; uint32_t last_branch_id_; + const bool has_msa_; + const MipsInstructionSetFeatures* isa_features_; DISALLOW_COPY_AND_ASSIGN(MipsAssembler); diff --git a/compiler/utils/mips/assembler_mips32r6_test.cc b/compiler/utils/mips/assembler_mips32r6_test.cc index 30667efa38..d4642607ad 100644 --- a/compiler/utils/mips/assembler_mips32r6_test.cc +++ b/compiler/utils/mips/assembler_mips32r6_test.cc @@ -34,9 +34,14 @@ struct MIPSCpuRegisterCompare { class AssemblerMIPS32r6Test : public AssemblerTest<mips::MipsAssembler, mips::Register, mips::FRegister, - uint32_t> { + uint32_t, + mips::VectorRegister> { public: - typedef AssemblerTest<mips::MipsAssembler, mips::Register, mips::FRegister, uint32_t> Base; + typedef AssemblerTest<mips::MipsAssembler, + mips::Register, + mips::FRegister, + uint32_t, + mips::VectorRegister> Base; AssemblerMIPS32r6Test() : instruction_set_features_(MipsInstructionSetFeatures::FromVariant("mips32r6", nullptr)) { @@ -61,7 +66,7 @@ class AssemblerMIPS32r6Test : public AssemblerTest<mips::MipsAssembler, // We use "-modd-spreg" so we can use odd-numbered single precision FPU registers. // We put the code at address 0x1000000 (instead of 0) to avoid overlapping with the // .MIPS.abiflags section (there doesn't seem to be a way to suppress its generation easily). - return " -march=mips32r6 -modd-spreg -Wa,--no-warn" + return " -march=mips32r6 -mmsa -modd-spreg -Wa,--no-warn" " -Wl,-Ttext=0x1000000 -Wl,-e0x1000000 -nostdlib"; } @@ -182,6 +187,39 @@ class AssemblerMIPS32r6Test : public AssemblerTest<mips::MipsAssembler, fp_registers_.push_back(new mips::FRegister(mips::F29)); fp_registers_.push_back(new mips::FRegister(mips::F30)); fp_registers_.push_back(new mips::FRegister(mips::F31)); + + vec_registers_.push_back(new mips::VectorRegister(mips::W0)); + vec_registers_.push_back(new mips::VectorRegister(mips::W1)); + vec_registers_.push_back(new mips::VectorRegister(mips::W2)); + vec_registers_.push_back(new mips::VectorRegister(mips::W3)); + vec_registers_.push_back(new mips::VectorRegister(mips::W4)); + vec_registers_.push_back(new mips::VectorRegister(mips::W5)); + vec_registers_.push_back(new mips::VectorRegister(mips::W6)); + vec_registers_.push_back(new mips::VectorRegister(mips::W7)); + vec_registers_.push_back(new mips::VectorRegister(mips::W8)); + vec_registers_.push_back(new mips::VectorRegister(mips::W9)); + vec_registers_.push_back(new mips::VectorRegister(mips::W10)); + vec_registers_.push_back(new mips::VectorRegister(mips::W11)); + vec_registers_.push_back(new mips::VectorRegister(mips::W12)); + vec_registers_.push_back(new mips::VectorRegister(mips::W13)); + vec_registers_.push_back(new mips::VectorRegister(mips::W14)); + vec_registers_.push_back(new mips::VectorRegister(mips::W15)); + vec_registers_.push_back(new mips::VectorRegister(mips::W16)); + vec_registers_.push_back(new mips::VectorRegister(mips::W17)); + vec_registers_.push_back(new mips::VectorRegister(mips::W18)); + vec_registers_.push_back(new mips::VectorRegister(mips::W19)); + vec_registers_.push_back(new mips::VectorRegister(mips::W20)); + vec_registers_.push_back(new mips::VectorRegister(mips::W21)); + vec_registers_.push_back(new mips::VectorRegister(mips::W22)); + vec_registers_.push_back(new mips::VectorRegister(mips::W23)); + vec_registers_.push_back(new mips::VectorRegister(mips::W24)); + vec_registers_.push_back(new mips::VectorRegister(mips::W25)); + vec_registers_.push_back(new mips::VectorRegister(mips::W26)); + vec_registers_.push_back(new mips::VectorRegister(mips::W27)); + vec_registers_.push_back(new mips::VectorRegister(mips::W28)); + vec_registers_.push_back(new mips::VectorRegister(mips::W29)); + vec_registers_.push_back(new mips::VectorRegister(mips::W30)); + vec_registers_.push_back(new mips::VectorRegister(mips::W31)); } } @@ -189,6 +227,7 @@ class AssemblerMIPS32r6Test : public AssemblerTest<mips::MipsAssembler, AssemblerTest::TearDown(); STLDeleteElements(®isters_); STLDeleteElements(&fp_registers_); + STLDeleteElements(&vec_registers_); } std::vector<mips::Register*> GetRegisters() OVERRIDE { @@ -199,6 +238,10 @@ class AssemblerMIPS32r6Test : public AssemblerTest<mips::MipsAssembler, return fp_registers_; } + std::vector<mips::VectorRegister*> GetVectorRegisters() OVERRIDE { + return vec_registers_; + } + uint32_t CreateImmediate(int64_t imm_value) OVERRIDE { return imm_value; } @@ -250,6 +293,7 @@ class AssemblerMIPS32r6Test : public AssemblerTest<mips::MipsAssembler, std::map<mips::Register, std::string, MIPSCpuRegisterCompare> secondary_register_names_; std::vector<mips::FRegister*> fp_registers_; + std::vector<mips::VectorRegister*> vec_registers_; std::unique_ptr<const MipsInstructionSetFeatures> instruction_set_features_; }; @@ -328,13 +372,11 @@ TEST_F(AssemblerMIPS32r6Test, Lsa) { } TEST_F(AssemblerMIPS32r6Test, Seleqz) { - DriverStr(RepeatRRR(&mips::MipsAssembler::Seleqz, "seleqz ${reg1}, ${reg2}, ${reg3}"), - "seleqz"); + DriverStr(RepeatRRR(&mips::MipsAssembler::Seleqz, "seleqz ${reg1}, ${reg2}, ${reg3}"), "seleqz"); } TEST_F(AssemblerMIPS32r6Test, Selnez) { - DriverStr(RepeatRRR(&mips::MipsAssembler::Selnez, "selnez ${reg1}, ${reg2}, ${reg3}"), - "selnez"); + DriverStr(RepeatRRR(&mips::MipsAssembler::Selnez, "selnez ${reg1}, ${reg2}, ${reg3}"), "selnez"); } TEST_F(AssemblerMIPS32r6Test, ClzR6) { @@ -914,6 +956,566 @@ TEST_F(AssemblerMIPS32r6Test, LongBranchReorder) { // AssemblerMIPS32r6Test.Bltu // AssemblerMIPS32r6Test.Bgeu +// MSA instructions. + +TEST_F(AssemblerMIPS32r6Test, AndV) { + DriverStr(RepeatVVV(&mips::MipsAssembler::AndV, "and.v ${reg1}, ${reg2}, ${reg3}"), "and.v"); +} + +TEST_F(AssemblerMIPS32r6Test, OrV) { + DriverStr(RepeatVVV(&mips::MipsAssembler::OrV, "or.v ${reg1}, ${reg2}, ${reg3}"), "or.v"); +} + +TEST_F(AssemblerMIPS32r6Test, NorV) { + DriverStr(RepeatVVV(&mips::MipsAssembler::NorV, "nor.v ${reg1}, ${reg2}, ${reg3}"), "nor.v"); +} + +TEST_F(AssemblerMIPS32r6Test, XorV) { + DriverStr(RepeatVVV(&mips::MipsAssembler::XorV, "xor.v ${reg1}, ${reg2}, ${reg3}"), "xor.v"); +} + +TEST_F(AssemblerMIPS32r6Test, AddvB) { + DriverStr(RepeatVVV(&mips::MipsAssembler::AddvB, "addv.b ${reg1}, ${reg2}, ${reg3}"), "addv.b"); +} + +TEST_F(AssemblerMIPS32r6Test, AddvH) { + DriverStr(RepeatVVV(&mips::MipsAssembler::AddvH, "addv.h ${reg1}, ${reg2}, ${reg3}"), "addv.h"); +} + +TEST_F(AssemblerMIPS32r6Test, AddvW) { + DriverStr(RepeatVVV(&mips::MipsAssembler::AddvW, "addv.w ${reg1}, ${reg2}, ${reg3}"), "addv.w"); +} + +TEST_F(AssemblerMIPS32r6Test, AddvD) { + DriverStr(RepeatVVV(&mips::MipsAssembler::AddvD, "addv.d ${reg1}, ${reg2}, ${reg3}"), "addv.d"); +} + +TEST_F(AssemblerMIPS32r6Test, SubvB) { + DriverStr(RepeatVVV(&mips::MipsAssembler::SubvB, "subv.b ${reg1}, ${reg2}, ${reg3}"), "subv.b"); +} + +TEST_F(AssemblerMIPS32r6Test, SubvH) { + DriverStr(RepeatVVV(&mips::MipsAssembler::SubvH, "subv.h ${reg1}, ${reg2}, ${reg3}"), "subv.h"); +} + +TEST_F(AssemblerMIPS32r6Test, SubvW) { + DriverStr(RepeatVVV(&mips::MipsAssembler::SubvW, "subv.w ${reg1}, ${reg2}, ${reg3}"), "subv.w"); +} + +TEST_F(AssemblerMIPS32r6Test, SubvD) { + DriverStr(RepeatVVV(&mips::MipsAssembler::SubvD, "subv.d ${reg1}, ${reg2}, ${reg3}"), "subv.d"); +} + +TEST_F(AssemblerMIPS32r6Test, MulvB) { + DriverStr(RepeatVVV(&mips::MipsAssembler::MulvB, "mulv.b ${reg1}, ${reg2}, ${reg3}"), "mulv.b"); +} + +TEST_F(AssemblerMIPS32r6Test, MulvH) { + DriverStr(RepeatVVV(&mips::MipsAssembler::MulvH, "mulv.h ${reg1}, ${reg2}, ${reg3}"), "mulv.h"); +} + +TEST_F(AssemblerMIPS32r6Test, MulvW) { + DriverStr(RepeatVVV(&mips::MipsAssembler::MulvW, "mulv.w ${reg1}, ${reg2}, ${reg3}"), "mulv.w"); +} + +TEST_F(AssemblerMIPS32r6Test, MulvD) { + DriverStr(RepeatVVV(&mips::MipsAssembler::MulvD, "mulv.d ${reg1}, ${reg2}, ${reg3}"), "mulv.d"); +} + +TEST_F(AssemblerMIPS32r6Test, Div_sB) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Div_sB, "div_s.b ${reg1}, ${reg2}, ${reg3}"), + "div_s.b"); +} + +TEST_F(AssemblerMIPS32r6Test, Div_sH) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Div_sH, "div_s.h ${reg1}, ${reg2}, ${reg3}"), + "div_s.h"); +} + +TEST_F(AssemblerMIPS32r6Test, Div_sW) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Div_sW, "div_s.w ${reg1}, ${reg2}, ${reg3}"), + "div_s.w"); +} + +TEST_F(AssemblerMIPS32r6Test, Div_sD) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Div_sD, "div_s.d ${reg1}, ${reg2}, ${reg3}"), + "div_s.d"); +} + +TEST_F(AssemblerMIPS32r6Test, Div_uB) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Div_uB, "div_u.b ${reg1}, ${reg2}, ${reg3}"), + "div_u.b"); +} + +TEST_F(AssemblerMIPS32r6Test, Div_uH) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Div_uH, "div_u.h ${reg1}, ${reg2}, ${reg3}"), + "div_u.h"); +} + +TEST_F(AssemblerMIPS32r6Test, Div_uW) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Div_uW, "div_u.w ${reg1}, ${reg2}, ${reg3}"), + "div_u.w"); +} + +TEST_F(AssemblerMIPS32r6Test, Div_uD) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Div_uD, "div_u.d ${reg1}, ${reg2}, ${reg3}"), + "div_u.d"); +} + +TEST_F(AssemblerMIPS32r6Test, Mod_sB) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Mod_sB, "mod_s.b ${reg1}, ${reg2}, ${reg3}"), + "mod_s.b"); +} + +TEST_F(AssemblerMIPS32r6Test, Mod_sH) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Mod_sH, "mod_s.h ${reg1}, ${reg2}, ${reg3}"), + "mod_s.h"); +} + +TEST_F(AssemblerMIPS32r6Test, Mod_sW) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Mod_sW, "mod_s.w ${reg1}, ${reg2}, ${reg3}"), + "mod_s.w"); +} + +TEST_F(AssemblerMIPS32r6Test, Mod_sD) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Mod_sD, "mod_s.d ${reg1}, ${reg2}, ${reg3}"), + "mod_s.d"); +} + +TEST_F(AssemblerMIPS32r6Test, Mod_uB) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Mod_uB, "mod_u.b ${reg1}, ${reg2}, ${reg3}"), + "mod_u.b"); +} + +TEST_F(AssemblerMIPS32r6Test, Mod_uH) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Mod_uH, "mod_u.h ${reg1}, ${reg2}, ${reg3}"), + "mod_u.h"); +} + +TEST_F(AssemblerMIPS32r6Test, Mod_uW) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Mod_uW, "mod_u.w ${reg1}, ${reg2}, ${reg3}"), + "mod_u.w"); +} + +TEST_F(AssemblerMIPS32r6Test, Mod_uD) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Mod_uD, "mod_u.d ${reg1}, ${reg2}, ${reg3}"), + "mod_u.d"); +} + +TEST_F(AssemblerMIPS32r6Test, Add_aB) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Add_aB, "add_a.b ${reg1}, ${reg2}, ${reg3}"), + "add_a.b"); +} + +TEST_F(AssemblerMIPS32r6Test, Add_aH) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Add_aH, "add_a.h ${reg1}, ${reg2}, ${reg3}"), + "add_a.h"); +} + +TEST_F(AssemblerMIPS32r6Test, Add_aW) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Add_aW, "add_a.w ${reg1}, ${reg2}, ${reg3}"), + "add_a.w"); +} + +TEST_F(AssemblerMIPS32r6Test, Add_aD) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Add_aD, "add_a.d ${reg1}, ${reg2}, ${reg3}"), + "add_a.d"); +} + +TEST_F(AssemblerMIPS32r6Test, Ave_sB) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Ave_sB, "ave_s.b ${reg1}, ${reg2}, ${reg3}"), + "ave_s.b"); +} + +TEST_F(AssemblerMIPS32r6Test, Ave_sH) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Ave_sH, "ave_s.h ${reg1}, ${reg2}, ${reg3}"), + "ave_s.h"); +} + +TEST_F(AssemblerMIPS32r6Test, Ave_sW) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Ave_sW, "ave_s.w ${reg1}, ${reg2}, ${reg3}"), + "ave_s.w"); +} + +TEST_F(AssemblerMIPS32r6Test, Ave_sD) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Ave_sD, "ave_s.d ${reg1}, ${reg2}, ${reg3}"), + "ave_s.d"); +} + +TEST_F(AssemblerMIPS32r6Test, Ave_uB) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Ave_uB, "ave_u.b ${reg1}, ${reg2}, ${reg3}"), + "ave_u.b"); +} + +TEST_F(AssemblerMIPS32r6Test, Ave_uH) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Ave_uH, "ave_u.h ${reg1}, ${reg2}, ${reg3}"), + "ave_u.h"); +} + +TEST_F(AssemblerMIPS32r6Test, Ave_uW) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Ave_uW, "ave_u.w ${reg1}, ${reg2}, ${reg3}"), + "ave_u.w"); +} + +TEST_F(AssemblerMIPS32r6Test, Ave_uD) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Ave_uD, "ave_u.d ${reg1}, ${reg2}, ${reg3}"), + "ave_u.d"); +} + +TEST_F(AssemblerMIPS32r6Test, Aver_sB) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Aver_sB, "aver_s.b ${reg1}, ${reg2}, ${reg3}"), + "aver_s.b"); +} + +TEST_F(AssemblerMIPS32r6Test, Aver_sH) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Aver_sH, "aver_s.h ${reg1}, ${reg2}, ${reg3}"), + "aver_s.h"); +} + +TEST_F(AssemblerMIPS32r6Test, Aver_sW) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Aver_sW, "aver_s.w ${reg1}, ${reg2}, ${reg3}"), + "aver_s.w"); +} + +TEST_F(AssemblerMIPS32r6Test, Aver_sD) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Aver_sD, "aver_s.d ${reg1}, ${reg2}, ${reg3}"), + "aver_s.d"); +} + +TEST_F(AssemblerMIPS32r6Test, Aver_uB) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Aver_uB, "aver_u.b ${reg1}, ${reg2}, ${reg3}"), + "aver_u.b"); +} + +TEST_F(AssemblerMIPS32r6Test, Aver_uH) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Aver_uH, "aver_u.h ${reg1}, ${reg2}, ${reg3}"), + "aver_u.h"); +} + +TEST_F(AssemblerMIPS32r6Test, Aver_uW) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Aver_uW, "aver_u.w ${reg1}, ${reg2}, ${reg3}"), + "aver_u.w"); +} + +TEST_F(AssemblerMIPS32r6Test, Aver_uD) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Aver_uD, "aver_u.d ${reg1}, ${reg2}, ${reg3}"), + "aver_u.d"); +} + +TEST_F(AssemblerMIPS32r6Test, Max_sB) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Max_sB, "max_s.b ${reg1}, ${reg2}, ${reg3}"), + "max_s.b"); +} + +TEST_F(AssemblerMIPS32r6Test, Max_sH) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Max_sH, "max_s.h ${reg1}, ${reg2}, ${reg3}"), + "max_s.h"); +} + +TEST_F(AssemblerMIPS32r6Test, Max_sW) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Max_sW, "max_s.w ${reg1}, ${reg2}, ${reg3}"), + "max_s.w"); +} + +TEST_F(AssemblerMIPS32r6Test, Max_sD) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Max_sD, "max_s.d ${reg1}, ${reg2}, ${reg3}"), + "max_s.d"); +} + +TEST_F(AssemblerMIPS32r6Test, Max_uB) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Max_uB, "max_u.b ${reg1}, ${reg2}, ${reg3}"), + "max_u.b"); +} + +TEST_F(AssemblerMIPS32r6Test, Max_uH) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Max_uH, "max_u.h ${reg1}, ${reg2}, ${reg3}"), + "max_u.h"); +} + +TEST_F(AssemblerMIPS32r6Test, Max_uW) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Max_uW, "max_u.w ${reg1}, ${reg2}, ${reg3}"), + "max_u.w"); +} + +TEST_F(AssemblerMIPS32r6Test, Max_uD) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Max_uD, "max_u.d ${reg1}, ${reg2}, ${reg3}"), + "max_u.d"); +} + +TEST_F(AssemblerMIPS32r6Test, Min_sB) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Min_sB, "min_s.b ${reg1}, ${reg2}, ${reg3}"), + "min_s.b"); +} + +TEST_F(AssemblerMIPS32r6Test, Min_sH) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Min_sH, "min_s.h ${reg1}, ${reg2}, ${reg3}"), + "min_s.h"); +} + +TEST_F(AssemblerMIPS32r6Test, Min_sW) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Min_sW, "min_s.w ${reg1}, ${reg2}, ${reg3}"), + "min_s.w"); +} + +TEST_F(AssemblerMIPS32r6Test, Min_sD) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Min_sD, "min_s.d ${reg1}, ${reg2}, ${reg3}"), + "min_s.d"); +} + +TEST_F(AssemblerMIPS32r6Test, Min_uB) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Min_uB, "min_u.b ${reg1}, ${reg2}, ${reg3}"), + "min_u.b"); +} + +TEST_F(AssemblerMIPS32r6Test, Min_uH) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Min_uH, "min_u.h ${reg1}, ${reg2}, ${reg3}"), + "min_u.h"); +} + +TEST_F(AssemblerMIPS32r6Test, Min_uW) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Min_uW, "min_u.w ${reg1}, ${reg2}, ${reg3}"), + "min_u.w"); +} + +TEST_F(AssemblerMIPS32r6Test, Min_uD) { + DriverStr(RepeatVVV(&mips::MipsAssembler::Min_uD, "min_u.d ${reg1}, ${reg2}, ${reg3}"), + "min_u.d"); +} + +TEST_F(AssemblerMIPS32r6Test, FaddW) { + DriverStr(RepeatVVV(&mips::MipsAssembler::FaddW, "fadd.w ${reg1}, ${reg2}, ${reg3}"), "fadd.w"); +} + +TEST_F(AssemblerMIPS32r6Test, FaddD) { + DriverStr(RepeatVVV(&mips::MipsAssembler::FaddD, "fadd.d ${reg1}, ${reg2}, ${reg3}"), "fadd.d"); +} + +TEST_F(AssemblerMIPS32r6Test, FsubW) { + DriverStr(RepeatVVV(&mips::MipsAssembler::FsubW, "fsub.w ${reg1}, ${reg2}, ${reg3}"), "fsub.w"); +} + +TEST_F(AssemblerMIPS32r6Test, FsubD) { + DriverStr(RepeatVVV(&mips::MipsAssembler::FsubD, "fsub.d ${reg1}, ${reg2}, ${reg3}"), "fsub.d"); +} + +TEST_F(AssemblerMIPS32r6Test, FmulW) { + DriverStr(RepeatVVV(&mips::MipsAssembler::FmulW, "fmul.w ${reg1}, ${reg2}, ${reg3}"), "fmul.w"); +} + +TEST_F(AssemblerMIPS32r6Test, FmulD) { + DriverStr(RepeatVVV(&mips::MipsAssembler::FmulD, "fmul.d ${reg1}, ${reg2}, ${reg3}"), "fmul.d"); +} + +TEST_F(AssemblerMIPS32r6Test, FdivW) { + DriverStr(RepeatVVV(&mips::MipsAssembler::FdivW, "fdiv.w ${reg1}, ${reg2}, ${reg3}"), "fdiv.w"); +} + +TEST_F(AssemblerMIPS32r6Test, FdivD) { + DriverStr(RepeatVVV(&mips::MipsAssembler::FdivD, "fdiv.d ${reg1}, ${reg2}, ${reg3}"), "fdiv.d"); +} + +TEST_F(AssemblerMIPS32r6Test, FmaxW) { + DriverStr(RepeatVVV(&mips::MipsAssembler::FmaxW, "fmax.w ${reg1}, ${reg2}, ${reg3}"), "fmax.w"); +} + +TEST_F(AssemblerMIPS32r6Test, FmaxD) { + DriverStr(RepeatVVV(&mips::MipsAssembler::FmaxD, "fmax.d ${reg1}, ${reg2}, ${reg3}"), "fmax.d"); +} + +TEST_F(AssemblerMIPS32r6Test, FminW) { + DriverStr(RepeatVVV(&mips::MipsAssembler::FminW, "fmin.w ${reg1}, ${reg2}, ${reg3}"), "fmin.w"); +} + +TEST_F(AssemblerMIPS32r6Test, FminD) { + DriverStr(RepeatVVV(&mips::MipsAssembler::FminD, "fmin.d ${reg1}, ${reg2}, ${reg3}"), "fmin.d"); +} + +TEST_F(AssemblerMIPS32r6Test, Ffint_sW) { + DriverStr(RepeatVV(&mips::MipsAssembler::Ffint_sW, "ffint_s.w ${reg1}, ${reg2}"), "ffint_s.w"); +} + +TEST_F(AssemblerMIPS32r6Test, Ffint_sD) { + DriverStr(RepeatVV(&mips::MipsAssembler::Ffint_sD, "ffint_s.d ${reg1}, ${reg2}"), "ffint_s.d"); +} + +TEST_F(AssemblerMIPS32r6Test, Ftint_sW) { + DriverStr(RepeatVV(&mips::MipsAssembler::Ftint_sW, "ftint_s.w ${reg1}, ${reg2}"), "ftint_s.w"); +} + +TEST_F(AssemblerMIPS32r6Test, Ftint_sD) { + DriverStr(RepeatVV(&mips::MipsAssembler::Ftint_sD, "ftint_s.d ${reg1}, ${reg2}"), "ftint_s.d"); +} + +TEST_F(AssemblerMIPS32r6Test, SllB) { + DriverStr(RepeatVVV(&mips::MipsAssembler::SllB, "sll.b ${reg1}, ${reg2}, ${reg3}"), "sll.b"); +} + +TEST_F(AssemblerMIPS32r6Test, SllH) { + DriverStr(RepeatVVV(&mips::MipsAssembler::SllH, "sll.h ${reg1}, ${reg2}, ${reg3}"), "sll.h"); +} + +TEST_F(AssemblerMIPS32r6Test, SllW) { + DriverStr(RepeatVVV(&mips::MipsAssembler::SllW, "sll.w ${reg1}, ${reg2}, ${reg3}"), "sll.w"); +} + +TEST_F(AssemblerMIPS32r6Test, SllD) { + DriverStr(RepeatVVV(&mips::MipsAssembler::SllD, "sll.d ${reg1}, ${reg2}, ${reg3}"), "sll.d"); +} + +TEST_F(AssemblerMIPS32r6Test, SraB) { + DriverStr(RepeatVVV(&mips::MipsAssembler::SraB, "sra.b ${reg1}, ${reg2}, ${reg3}"), "sra.b"); +} + +TEST_F(AssemblerMIPS32r6Test, SraH) { + DriverStr(RepeatVVV(&mips::MipsAssembler::SraH, "sra.h ${reg1}, ${reg2}, ${reg3}"), "sra.h"); +} + +TEST_F(AssemblerMIPS32r6Test, SraW) { + DriverStr(RepeatVVV(&mips::MipsAssembler::SraW, "sra.w ${reg1}, ${reg2}, ${reg3}"), "sra.w"); +} + +TEST_F(AssemblerMIPS32r6Test, SraD) { + DriverStr(RepeatVVV(&mips::MipsAssembler::SraD, "sra.d ${reg1}, ${reg2}, ${reg3}"), "sra.d"); +} + +TEST_F(AssemblerMIPS32r6Test, SrlB) { + DriverStr(RepeatVVV(&mips::MipsAssembler::SrlB, "srl.b ${reg1}, ${reg2}, ${reg3}"), "srl.b"); +} + +TEST_F(AssemblerMIPS32r6Test, SrlH) { + DriverStr(RepeatVVV(&mips::MipsAssembler::SrlH, "srl.h ${reg1}, ${reg2}, ${reg3}"), "srl.h"); +} + +TEST_F(AssemblerMIPS32r6Test, SrlW) { + DriverStr(RepeatVVV(&mips::MipsAssembler::SrlW, "srl.w ${reg1}, ${reg2}, ${reg3}"), "srl.w"); +} + +TEST_F(AssemblerMIPS32r6Test, SrlD) { + DriverStr(RepeatVVV(&mips::MipsAssembler::SrlD, "srl.d ${reg1}, ${reg2}, ${reg3}"), "srl.d"); +} + +TEST_F(AssemblerMIPS32r6Test, SlliB) { + DriverStr(RepeatVVIb(&mips::MipsAssembler::SlliB, 3, "slli.b ${reg1}, ${reg2}, {imm}"), "slli.b"); +} + +TEST_F(AssemblerMIPS32r6Test, SlliH) { + DriverStr(RepeatVVIb(&mips::MipsAssembler::SlliH, 4, "slli.h ${reg1}, ${reg2}, {imm}"), "slli.h"); +} + +TEST_F(AssemblerMIPS32r6Test, SlliW) { + DriverStr(RepeatVVIb(&mips::MipsAssembler::SlliW, 5, "slli.w ${reg1}, ${reg2}, {imm}"), "slli.w"); +} + +TEST_F(AssemblerMIPS32r6Test, SlliD) { + DriverStr(RepeatVVIb(&mips::MipsAssembler::SlliD, 6, "slli.d ${reg1}, ${reg2}, {imm}"), "slli.d"); +} + +TEST_F(AssemblerMIPS32r6Test, MoveV) { + DriverStr(RepeatVV(&mips::MipsAssembler::MoveV, "move.v ${reg1}, ${reg2}"), "move.v"); +} + +TEST_F(AssemblerMIPS32r6Test, SplatiB) { + DriverStr(RepeatVVIb(&mips::MipsAssembler::SplatiB, 4, "splati.b ${reg1}, ${reg2}[{imm}]"), + "splati.b"); +} + +TEST_F(AssemblerMIPS32r6Test, SplatiH) { + DriverStr(RepeatVVIb(&mips::MipsAssembler::SplatiH, 3, "splati.h ${reg1}, ${reg2}[{imm}]"), + "splati.h"); +} + +TEST_F(AssemblerMIPS32r6Test, SplatiW) { + DriverStr(RepeatVVIb(&mips::MipsAssembler::SplatiW, 2, "splati.w ${reg1}, ${reg2}[{imm}]"), + "splati.w"); +} + +TEST_F(AssemblerMIPS32r6Test, SplatiD) { + DriverStr(RepeatVVIb(&mips::MipsAssembler::SplatiD, 1, "splati.d ${reg1}, ${reg2}[{imm}]"), + "splati.d"); +} + +TEST_F(AssemblerMIPS32r6Test, FillB) { + DriverStr(RepeatVR(&mips::MipsAssembler::FillB, "fill.b ${reg1}, ${reg2}"), "fill.b"); +} + +TEST_F(AssemblerMIPS32r6Test, FillH) { + DriverStr(RepeatVR(&mips::MipsAssembler::FillH, "fill.h ${reg1}, ${reg2}"), "fill.h"); +} + +TEST_F(AssemblerMIPS32r6Test, FillW) { + DriverStr(RepeatVR(&mips::MipsAssembler::FillW, "fill.w ${reg1}, ${reg2}"), "fill.w"); +} + +TEST_F(AssemblerMIPS32r6Test, LdiB) { + DriverStr(RepeatVIb(&mips::MipsAssembler::LdiB, -8, "ldi.b ${reg}, {imm}"), "ldi.b"); +} + +TEST_F(AssemblerMIPS32r6Test, LdiH) { + DriverStr(RepeatVIb(&mips::MipsAssembler::LdiH, -10, "ldi.h ${reg}, {imm}"), "ldi.h"); +} + +TEST_F(AssemblerMIPS32r6Test, LdiW) { + DriverStr(RepeatVIb(&mips::MipsAssembler::LdiW, -10, "ldi.w ${reg}, {imm}"), "ldi.w"); +} + +TEST_F(AssemblerMIPS32r6Test, LdiD) { + DriverStr(RepeatVIb(&mips::MipsAssembler::LdiD, -10, "ldi.d ${reg}, {imm}"), "ldi.d"); +} + +TEST_F(AssemblerMIPS32r6Test, LdB) { + DriverStr(RepeatVRIb(&mips::MipsAssembler::LdB, -10, "ld.b ${reg1}, {imm}(${reg2})"), "ld.b"); +} + +TEST_F(AssemblerMIPS32r6Test, LdH) { + DriverStr(RepeatVRIb(&mips::MipsAssembler::LdH, -10, "ld.h ${reg1}, {imm}(${reg2})", 0, 2), + "ld.h"); +} + +TEST_F(AssemblerMIPS32r6Test, LdW) { + DriverStr(RepeatVRIb(&mips::MipsAssembler::LdW, -10, "ld.w ${reg1}, {imm}(${reg2})", 0, 4), + "ld.w"); +} + +TEST_F(AssemblerMIPS32r6Test, LdD) { + DriverStr(RepeatVRIb(&mips::MipsAssembler::LdD, -10, "ld.d ${reg1}, {imm}(${reg2})", 0, 8), + "ld.d"); +} + +TEST_F(AssemblerMIPS32r6Test, StB) { + DriverStr(RepeatVRIb(&mips::MipsAssembler::StB, -10, "st.b ${reg1}, {imm}(${reg2})"), "st.b"); +} + +TEST_F(AssemblerMIPS32r6Test, StH) { + DriverStr(RepeatVRIb(&mips::MipsAssembler::StH, -10, "st.h ${reg1}, {imm}(${reg2})", 0, 2), + "st.h"); +} + +TEST_F(AssemblerMIPS32r6Test, StW) { + DriverStr(RepeatVRIb(&mips::MipsAssembler::StW, -10, "st.w ${reg1}, {imm}(${reg2})", 0, 4), + "st.w"); +} + +TEST_F(AssemblerMIPS32r6Test, StD) { + DriverStr(RepeatVRIb(&mips::MipsAssembler::StD, -10, "st.d ${reg1}, {imm}(${reg2})", 0, 8), + "st.d"); +} + +TEST_F(AssemblerMIPS32r6Test, IlvrB) { + DriverStr(RepeatVVV(&mips::MipsAssembler::IlvrB, "ilvr.b ${reg1}, ${reg2}, ${reg3}"), "ilvr.b"); +} + +TEST_F(AssemblerMIPS32r6Test, IlvrH) { + DriverStr(RepeatVVV(&mips::MipsAssembler::IlvrH, "ilvr.h ${reg1}, ${reg2}, ${reg3}"), "ilvr.h"); +} + +TEST_F(AssemblerMIPS32r6Test, IlvrW) { + DriverStr(RepeatVVV(&mips::MipsAssembler::IlvrW, "ilvr.w ${reg1}, ${reg2}, ${reg3}"), "ilvr.w"); +} + +TEST_F(AssemblerMIPS32r6Test, IlvrD) { + DriverStr(RepeatVVV(&mips::MipsAssembler::IlvrD, "ilvr.d ${reg1}, ${reg2}, ${reg3}"), "ilvr.d"); +} + #undef __ } // namespace art diff --git a/compiler/utils/mips/constants_mips.h b/compiler/utils/mips/constants_mips.h index 44ed5cc124..b4dfdbd8d3 100644 --- a/compiler/utils/mips/constants_mips.h +++ b/compiler/utils/mips/constants_mips.h @@ -75,8 +75,37 @@ enum InstructionFields { kFdShift = 6, kFdBits = 5, + kMsaOperationShift = 23, + kMsaELMOperationShift = 22, + kMsa2ROperationShift = 18, + kMsa2RFOperationShift = 17, + kDfShift = 21, + kDfMShift = 16, + kDf2RShift = 16, + kDfNShift = 16, + kWtShift = 16, + kWtBits = 5, + kWsShift = 11, + kWsBits = 5, + kWdShift = 6, + kWdBits = 5, + kS10Shift = 16, + kI10Shift = 11, + kS10MinorShift = 2, + kBranchOffsetMask = 0x0000ffff, kJumpOffsetMask = 0x03ffffff, + + kMsaMajorOpcode = 0x1e, + kMsaDfMByteMask = 0x70, + kMsaDfMHalfwordMask = 0x60, + kMsaDfMWordMask = 0x40, + kMsaDfMDoublewordMask = 0x00, + kMsaDfNByteMask = 0x00, + kMsaDfNHalfwordMask = 0x20, + kMsaDfNWordMask = 0x30, + kMsaDfNDoublewordMask = 0x38, + kMsaS10Mask = 0x3ff, }; enum ScaleFactor { diff --git a/compiler/utils/mips64/assembler_mips64.cc b/compiler/utils/mips64/assembler_mips64.cc index b8b800abe3..24900a7f10 100644 --- a/compiler/utils/mips64/assembler_mips64.cc +++ b/compiler/utils/mips64/assembler_mips64.cc @@ -1456,6 +1456,86 @@ void Mips64Assembler::Aver_uD(VectorRegister wd, VectorRegister ws, VectorRegist EmitMsa3R(0x7, 0x3, wt, ws, wd, 0x10); } +void Mips64Assembler::Max_sB(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + EmitMsa3R(0x2, 0x0, wt, ws, wd, 0xe); +} + +void Mips64Assembler::Max_sH(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + EmitMsa3R(0x2, 0x1, wt, ws, wd, 0xe); +} + +void Mips64Assembler::Max_sW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + EmitMsa3R(0x2, 0x2, wt, ws, wd, 0xe); +} + +void Mips64Assembler::Max_sD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + EmitMsa3R(0x2, 0x3, wt, ws, wd, 0xe); +} + +void Mips64Assembler::Max_uB(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + EmitMsa3R(0x3, 0x0, wt, ws, wd, 0xe); +} + +void Mips64Assembler::Max_uH(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + EmitMsa3R(0x3, 0x1, wt, ws, wd, 0xe); +} + +void Mips64Assembler::Max_uW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + EmitMsa3R(0x3, 0x2, wt, ws, wd, 0xe); +} + +void Mips64Assembler::Max_uD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + EmitMsa3R(0x3, 0x3, wt, ws, wd, 0xe); +} + +void Mips64Assembler::Min_sB(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + EmitMsa3R(0x4, 0x0, wt, ws, wd, 0xe); +} + +void Mips64Assembler::Min_sH(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + EmitMsa3R(0x4, 0x1, wt, ws, wd, 0xe); +} + +void Mips64Assembler::Min_sW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + EmitMsa3R(0x4, 0x2, wt, ws, wd, 0xe); +} + +void Mips64Assembler::Min_sD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + EmitMsa3R(0x4, 0x3, wt, ws, wd, 0xe); +} + +void Mips64Assembler::Min_uB(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + EmitMsa3R(0x5, 0x0, wt, ws, wd, 0xe); +} + +void Mips64Assembler::Min_uH(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + EmitMsa3R(0x5, 0x1, wt, ws, wd, 0xe); +} + +void Mips64Assembler::Min_uW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + EmitMsa3R(0x5, 0x2, wt, ws, wd, 0xe); +} + +void Mips64Assembler::Min_uD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + EmitMsa3R(0x5, 0x3, wt, ws, wd, 0xe); +} + void Mips64Assembler::FaddW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { CHECK(HasMsa()); EmitMsa3R(0x0, 0x0, wt, ws, wd, 0x1b); @@ -1496,6 +1576,26 @@ void Mips64Assembler::FdivD(VectorRegister wd, VectorRegister ws, VectorRegister EmitMsa3R(0x1, 0x3, wt, ws, wd, 0x1b); } +void Mips64Assembler::FmaxW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + EmitMsa3R(0x7, 0x0, wt, ws, wd, 0x1b); +} + +void Mips64Assembler::FmaxD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + EmitMsa3R(0x7, 0x1, wt, ws, wd, 0x1b); +} + +void Mips64Assembler::FminW(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + EmitMsa3R(0x6, 0x0, wt, ws, wd, 0x1b); +} + +void Mips64Assembler::FminD(VectorRegister wd, VectorRegister ws, VectorRegister wt) { + CHECK(HasMsa()); + EmitMsa3R(0x6, 0x1, wt, ws, wd, 0x1b); +} + void Mips64Assembler::Ffint_sW(VectorRegister wd, VectorRegister ws) { CHECK(HasMsa()); EmitMsa2RF(0x19e, 0x0, ws, wd, 0x1e); diff --git a/compiler/utils/mips64/assembler_mips64.h b/compiler/utils/mips64/assembler_mips64.h index 9b4064543f..773db9b208 100644 --- a/compiler/utils/mips64/assembler_mips64.h +++ b/compiler/utils/mips64/assembler_mips64.h @@ -25,6 +25,7 @@ #include "base/arena_containers.h" #include "base/enums.h" #include "base/macros.h" +#include "base/stl_util_identity.h" #include "constants_mips64.h" #include "globals.h" #include "managed_register_mips64.h" @@ -704,6 +705,22 @@ class Mips64Assembler FINAL : public Assembler, public JNIMacroAssembler<Pointer void Aver_uH(VectorRegister wd, VectorRegister ws, VectorRegister wt); void Aver_uW(VectorRegister wd, VectorRegister ws, VectorRegister wt); void Aver_uD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Max_sB(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Max_sH(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Max_sW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Max_sD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Max_uB(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Max_uH(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Max_uW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Max_uD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Min_sB(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Min_sH(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Min_sW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Min_sD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Min_uB(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Min_uH(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Min_uW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void Min_uD(VectorRegister wd, VectorRegister ws, VectorRegister wt); void FaddW(VectorRegister wd, VectorRegister ws, VectorRegister wt); void FaddD(VectorRegister wd, VectorRegister ws, VectorRegister wt); @@ -713,6 +730,10 @@ class Mips64Assembler FINAL : public Assembler, public JNIMacroAssembler<Pointer void FmulD(VectorRegister wd, VectorRegister ws, VectorRegister wt); void FdivW(VectorRegister wd, VectorRegister ws, VectorRegister wt); void FdivD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void FmaxW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void FmaxD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void FminW(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void FminD(VectorRegister wd, VectorRegister ws, VectorRegister wt); void Ffint_sW(VectorRegister wd, VectorRegister ws); void Ffint_sD(VectorRegister wd, VectorRegister ws); diff --git a/compiler/utils/mips64/assembler_mips64_test.cc b/compiler/utils/mips64/assembler_mips64_test.cc index fbebe0ce15..bdf9598ee7 100644 --- a/compiler/utils/mips64/assembler_mips64_test.cc +++ b/compiler/utils/mips64/assembler_mips64_test.cc @@ -2998,6 +2998,86 @@ TEST_F(AssemblerMIPS64Test, Aver_uD) { "aver_u.d"); } +TEST_F(AssemblerMIPS64Test, Max_sB) { + DriverStr(RepeatVVV(&mips64::Mips64Assembler::Max_sB, "max_s.b ${reg1}, ${reg2}, ${reg3}"), + "max_s.b"); +} + +TEST_F(AssemblerMIPS64Test, Max_sH) { + DriverStr(RepeatVVV(&mips64::Mips64Assembler::Max_sH, "max_s.h ${reg1}, ${reg2}, ${reg3}"), + "max_s.h"); +} + +TEST_F(AssemblerMIPS64Test, Max_sW) { + DriverStr(RepeatVVV(&mips64::Mips64Assembler::Max_sW, "max_s.w ${reg1}, ${reg2}, ${reg3}"), + "max_s.w"); +} + +TEST_F(AssemblerMIPS64Test, Max_sD) { + DriverStr(RepeatVVV(&mips64::Mips64Assembler::Max_sD, "max_s.d ${reg1}, ${reg2}, ${reg3}"), + "max_s.d"); +} + +TEST_F(AssemblerMIPS64Test, Max_uB) { + DriverStr(RepeatVVV(&mips64::Mips64Assembler::Max_uB, "max_u.b ${reg1}, ${reg2}, ${reg3}"), + "max_u.b"); +} + +TEST_F(AssemblerMIPS64Test, Max_uH) { + DriverStr(RepeatVVV(&mips64::Mips64Assembler::Max_uH, "max_u.h ${reg1}, ${reg2}, ${reg3}"), + "max_u.h"); +} + +TEST_F(AssemblerMIPS64Test, Max_uW) { + DriverStr(RepeatVVV(&mips64::Mips64Assembler::Max_uW, "max_u.w ${reg1}, ${reg2}, ${reg3}"), + "max_u.w"); +} + +TEST_F(AssemblerMIPS64Test, Max_uD) { + DriverStr(RepeatVVV(&mips64::Mips64Assembler::Max_uD, "max_u.d ${reg1}, ${reg2}, ${reg3}"), + "max_u.d"); +} + +TEST_F(AssemblerMIPS64Test, Min_sB) { + DriverStr(RepeatVVV(&mips64::Mips64Assembler::Min_sB, "min_s.b ${reg1}, ${reg2}, ${reg3}"), + "min_s.b"); +} + +TEST_F(AssemblerMIPS64Test, Min_sH) { + DriverStr(RepeatVVV(&mips64::Mips64Assembler::Min_sH, "min_s.h ${reg1}, ${reg2}, ${reg3}"), + "min_s.h"); +} + +TEST_F(AssemblerMIPS64Test, Min_sW) { + DriverStr(RepeatVVV(&mips64::Mips64Assembler::Min_sW, "min_s.w ${reg1}, ${reg2}, ${reg3}"), + "min_s.w"); +} + +TEST_F(AssemblerMIPS64Test, Min_sD) { + DriverStr(RepeatVVV(&mips64::Mips64Assembler::Min_sD, "min_s.d ${reg1}, ${reg2}, ${reg3}"), + "min_s.d"); +} + +TEST_F(AssemblerMIPS64Test, Min_uB) { + DriverStr(RepeatVVV(&mips64::Mips64Assembler::Min_uB, "min_u.b ${reg1}, ${reg2}, ${reg3}"), + "min_u.b"); +} + +TEST_F(AssemblerMIPS64Test, Min_uH) { + DriverStr(RepeatVVV(&mips64::Mips64Assembler::Min_uH, "min_u.h ${reg1}, ${reg2}, ${reg3}"), + "min_u.h"); +} + +TEST_F(AssemblerMIPS64Test, Min_uW) { + DriverStr(RepeatVVV(&mips64::Mips64Assembler::Min_uW, "min_u.w ${reg1}, ${reg2}, ${reg3}"), + "min_u.w"); +} + +TEST_F(AssemblerMIPS64Test, Min_uD) { + DriverStr(RepeatVVV(&mips64::Mips64Assembler::Min_uD, "min_u.d ${reg1}, ${reg2}, ${reg3}"), + "min_u.d"); +} + TEST_F(AssemblerMIPS64Test, FaddW) { DriverStr(RepeatVVV(&mips64::Mips64Assembler::FaddW, "fadd.w ${reg1}, ${reg2}, ${reg3}"), "fadd.w"); @@ -3038,6 +3118,26 @@ TEST_F(AssemblerMIPS64Test, FdivD) { "fdiv.d"); } +TEST_F(AssemblerMIPS64Test, FmaxW) { + DriverStr(RepeatVVV(&mips64::Mips64Assembler::FmaxW, "fmax.w ${reg1}, ${reg2}, ${reg3}"), + "fmax.w"); +} + +TEST_F(AssemblerMIPS64Test, FmaxD) { + DriverStr(RepeatVVV(&mips64::Mips64Assembler::FmaxD, "fmax.d ${reg1}, ${reg2}, ${reg3}"), + "fmax.d"); +} + +TEST_F(AssemblerMIPS64Test, FminW) { + DriverStr(RepeatVVV(&mips64::Mips64Assembler::FminW, "fmin.w ${reg1}, ${reg2}, ${reg3}"), + "fmin.w"); +} + +TEST_F(AssemblerMIPS64Test, FminD) { + DriverStr(RepeatVVV(&mips64::Mips64Assembler::FminD, "fmin.d ${reg1}, ${reg2}, ${reg3}"), + "fmin.d"); +} + TEST_F(AssemblerMIPS64Test, Ffint_sW) { DriverStr(RepeatVV(&mips64::Mips64Assembler::Ffint_sW, "ffint_s.w ${reg1}, ${reg2}"), "ffint_s.w"); diff --git a/compiler/utils/swap_space.cc b/compiler/utils/swap_space.cc index 4f6c915142..621a652f0a 100644 --- a/compiler/utils/swap_space.cc +++ b/compiler/utils/swap_space.cc @@ -20,6 +20,7 @@ #include <numeric> #include <sys/mman.h> +#include "base/bit_utils.h" #include "base/logging.h" #include "base/macros.h" #include "base/mutex.h" diff --git a/dex2oat/dex2oat.cc b/dex2oat/dex2oat.cc index dcdf3bca02..3dd07031d5 100644 --- a/dex2oat/dex2oat.cc +++ b/dex2oat/dex2oat.cc @@ -39,6 +39,7 @@ #include "arch/instruction_set_features.h" #include "arch/mips/instruction_set_features_mips.h" #include "art_method-inl.h" +#include "base/callee_save_type.h" #include "base/dumpable.h" #include "base/macros.h" #include "base/scoped_flock.h" @@ -113,13 +114,18 @@ static std::string CommandLine() { static std::string StrippedCommandLine() { std::vector<std::string> command; - // Do a pre-pass to look for zip-fd. + // Do a pre-pass to look for zip-fd and the compiler filter. bool saw_zip_fd = false; + bool saw_compiler_filter = false; for (int i = 0; i < original_argc; ++i) { if (android::base::StartsWith(original_argv[i], "--zip-fd=")) { saw_zip_fd = true; break; } + if (android::base::StartsWith(original_argv[i], "--compiler-filter=")) { + saw_compiler_filter = true; + break; + } } // Now filter out things. @@ -162,6 +168,11 @@ static std::string StrippedCommandLine() { command.push_back(original_argv[i]); } + if (!saw_compiler_filter) { + command.push_back("--compiler-filter=" + + CompilerFilter::NameOfFilter(CompilerFilter::kDefaultCompilerFilter)); + } + // Construct the final output. if (command.size() <= 1U) { // It seems only "/system/bin/dex2oat" is left, or not even that. Use a pretty line. @@ -514,13 +525,14 @@ class WatchDog { CHECK_WATCH_DOG_PTHREAD_CALL(pthread_mutex_unlock, (&mutex_), reason); } - const int64_t timeout_in_milliseconds_; - bool shutting_down_; // TODO: Switch to Mutex when we can guarantee it won't prevent shutdown in error cases. pthread_mutex_t mutex_; pthread_cond_t cond_; pthread_attr_t attr_; pthread_t pthread_; + + const int64_t timeout_in_milliseconds_; + bool shutting_down_; }; class Dex2Oat FINAL { @@ -1390,8 +1402,8 @@ class Dex2Oat FINAL { // Note: we're only invalidating the magic data in the file, as dex2oat needs the rest of // the information to remain valid. if (update_input_vdex_) { - std::unique_ptr<BufferedOutputStream> vdex_out(MakeUnique<BufferedOutputStream>( - MakeUnique<FileOutputStream>(vdex_files_.back().get()))); + std::unique_ptr<BufferedOutputStream> vdex_out = std::make_unique<BufferedOutputStream>( + std::make_unique<FileOutputStream>(vdex_files_.back().get())); if (!vdex_out->WriteFully(&VdexFile::Header::kVdexInvalidMagic, arraysize(VdexFile::Header::kVdexInvalidMagic))) { PLOG(ERROR) << "Failed to invalidate vdex header. File: " << vdex_out->GetLocation(); @@ -1888,8 +1900,8 @@ class Dex2Oat FINAL { verifier::VerifierDeps* verifier_deps = callbacks_->GetVerifierDeps(); for (size_t i = 0, size = oat_files_.size(); i != size; ++i) { File* vdex_file = vdex_files_[i].get(); - std::unique_ptr<BufferedOutputStream> vdex_out( - MakeUnique<BufferedOutputStream>(MakeUnique<FileOutputStream>(vdex_file))); + std::unique_ptr<BufferedOutputStream> vdex_out = + std::make_unique<BufferedOutputStream>(std::make_unique<FileOutputStream>(vdex_file)); if (!oat_writers_[i]->WriteVerifierDeps(vdex_out.get(), verifier_deps)) { LOG(ERROR) << "Failed to write verifier dependencies into VDEX " << vdex_file->GetPath(); @@ -1923,6 +1935,7 @@ class Dex2Oat FINAL { elf_writer->PrepareDynamicSection(rodata_size, text_size, oat_writer->GetBssSize(), + oat_writer->GetBssMethodsOffset(), oat_writer->GetBssRootsOffset()); if (IsImage()) { @@ -2500,8 +2513,8 @@ class Dex2Oat FINAL { runtime_.reset(Runtime::Current()); runtime_->SetInstructionSet(instruction_set_); - for (int i = 0; i < Runtime::kLastCalleeSaveType; i++) { - Runtime::CalleeSaveType type = Runtime::CalleeSaveType(i); + for (uint32_t i = 0; i < static_cast<uint32_t>(CalleeSaveType::kLastCalleeSaveType); ++i) { + CalleeSaveType type = CalleeSaveType(i); if (!runtime_->HasCalleeSaveMethod(type)) { runtime_->SetCalleeSaveMethod(runtime_->CreateCalleeSaveMethod(), type); } @@ -2920,7 +2933,7 @@ static dex2oat::ReturnCode Dex2oat(int argc, char** argv) { // might produce a stack frame too large for this function or for // functions inlining it (such as main), that would not fit the // requirements of the `-Wframe-larger-than` option. - std::unique_ptr<Dex2Oat> dex2oat = MakeUnique<Dex2Oat>(&timings); + std::unique_ptr<Dex2Oat> dex2oat = std::make_unique<Dex2Oat>(&timings); // Parse arguments. Argument mistakes will lead to exit(EXIT_FAILURE) in UsageError. dex2oat->ParseArgs(argc, argv); diff --git a/dexdump/dexdump.cc b/dexdump/dexdump.cc index 5656ddd59c..1541d7b39e 100644 --- a/dexdump/dexdump.cc +++ b/dexdump/dexdump.cc @@ -1747,9 +1747,8 @@ static void dumpCallSite(const DexFile* pDexFile, u4 idx) { case EncodedArrayValueIterator::ValueType::kArray: case EncodedArrayValueIterator::ValueType::kAnnotation: // Unreachable based on current EncodedArrayValueIterator::Next(). - UNIMPLEMENTED(FATAL) << " type " << type; + UNIMPLEMENTED(FATAL) << " type " << it.GetValueType(); UNREACHABLE(); - break; case EncodedArrayValueIterator::ValueType::kNull: type = "Null"; value = "null"; diff --git a/dexlayout/dex_ir.cc b/dexlayout/dex_ir.cc index cf453b9a16..62ee445085 100644 --- a/dexlayout/dex_ir.cc +++ b/dexlayout/dex_ir.cc @@ -57,31 +57,6 @@ static void GetLocalsCb(void* context, const DexFile::LocalInfo& entry) { entry.reg_))); } -static uint32_t GetCodeItemSize(const DexFile::CodeItem& disk_code_item) { - uintptr_t code_item_start = reinterpret_cast<uintptr_t>(&disk_code_item); - uint32_t insns_size = disk_code_item.insns_size_in_code_units_; - uint32_t tries_size = disk_code_item.tries_size_; - if (tries_size == 0) { - uintptr_t insns_end = reinterpret_cast<uintptr_t>(&disk_code_item.insns_[insns_size]); - return insns_end - code_item_start; - } else { - // Get the start of the handler data. - const uint8_t* handler_data = DexFile::GetCatchHandlerData(disk_code_item, 0); - uint32_t handlers_size = DecodeUnsignedLeb128(&handler_data); - // Manually read each handler. - for (uint32_t i = 0; i < handlers_size; ++i) { - int32_t uleb128_count = DecodeSignedLeb128(&handler_data) * 2; - if (uleb128_count <= 0) { - uleb128_count = -uleb128_count + 1; - } - for (int32_t j = 0; j < uleb128_count; ++j) { - DecodeUnsignedLeb128(&handler_data); - } - } - return reinterpret_cast<uintptr_t>(handler_data) - code_item_start; - } -} - static uint32_t GetDebugInfoStreamSize(const uint8_t* debug_info_stream) { const uint8_t* stream = debug_info_stream; DecodeUnsignedLeb128(&stream); // line_start @@ -686,7 +661,7 @@ CodeItem* Collections::CreateCodeItem(const DexFile& dex_file, } } - uint32_t size = GetCodeItemSize(disk_code_item); + uint32_t size = DexFile::GetCodeItemSize(disk_code_item); CodeItem* code_item = new CodeItem( registers_size, ins_size, outs_size, debug_info, insns_size, insns, tries, handler_list); code_item->SetSize(size); diff --git a/dexlayout/dex_ir.h b/dexlayout/dex_ir.h index 5692eb2b39..95e64bf3e7 100644 --- a/dexlayout/dex_ir.h +++ b/dexlayout/dex_ir.h @@ -23,6 +23,7 @@ #include <vector> #include <stdint.h> +#include "base/stl_util.h" #include "dex_file-inl.h" #include "leb128.h" #include "utf.h" diff --git a/dexlayout/dex_visualize.cc b/dexlayout/dex_visualize.cc index 829e9feda8..d279bcb65c 100644 --- a/dexlayout/dex_visualize.cc +++ b/dexlayout/dex_visualize.cc @@ -174,7 +174,7 @@ class Dumper { ProfileCompilationInfo* profile_info) { if (profile_info != nullptr) { uint32_t method_idx = method->GetMethodId()->GetIndex(); - if (!profile_info->ContainsMethod(MethodReference(dex_file, method_idx))) { + if (!profile_info->ContainsHotMethod(MethodReference(dex_file, method_idx))) { return; } } diff --git a/dexlayout/dexlayout.cc b/dexlayout/dexlayout.cc index db227676c2..50dda88c55 100644 --- a/dexlayout/dexlayout.cc +++ b/dexlayout/dexlayout.cc @@ -1665,8 +1665,9 @@ int32_t DexLayout::LayoutCodeItems(const DexFile* dex_file, enum CodeItemKind { kMethodNotExecuted = 0, - kMethodExecuted = 1, - kSize = 2, + kMethodClinit = 1, + kMethodExecuted = 2, + kSize = 3, }; static constexpr InvokeType invoke_types[] = { @@ -1694,26 +1695,28 @@ int32_t DexLayout::LayoutCodeItems(const DexFile* dex_file, continue; } // Separate executed methods (clinits and profiled methods) from unexecuted methods. - // TODO: clinits are executed only once, consider separating them further. const bool is_clinit = is_profile_class && (method->GetAccessFlags() & kAccConstructor) != 0 && (method->GetAccessFlags() & kAccStatic) != 0; - const bool is_method_executed = is_clinit || + const bool is_method_executed = info_->IsStartupOrHotMethod(MethodReference(dex_file, method_id->GetIndex())); - code_items[is_method_executed - ? CodeItemKind::kMethodExecuted - : CodeItemKind::kMethodNotExecuted] - .insert(code_item); + CodeItemKind code_item_kind = CodeItemKind::kMethodNotExecuted; + if (is_clinit) { + code_item_kind = CodeItemKind::kMethodClinit; + } else if (is_method_executed) { + code_item_kind = CodeItemKind::kMethodExecuted; + } + code_items[code_item_kind].insert(code_item); } } } - // total_diff includes diffs generated by both executed and non-executed methods. + // Total_diff includes diffs generated by clinits, executed, and non-executed methods. int32_t total_diff = 0; // The relative placement has no effect on correctness; it is used to ensure // the layout is deterministic for (std::unordered_set<dex_ir::CodeItem*>& code_items_set : code_items) { - // diff is reset for executed and non-executed methods. + // diff is reset for each class of code items. int32_t diff = 0; for (dex_ir::ClassData* data : new_class_data_order) { data->SetOffset(data->GetOffset() + diff); diff --git a/disassembler/disassembler_mips.cc b/disassembler/disassembler_mips.cc index 8894cc9899..91203cb9f9 100644 --- a/disassembler/disassembler_mips.cc +++ b/disassembler/disassembler_mips.cc @@ -438,10 +438,16 @@ static const MipsInstruction gMipsInstructions[] = { { kMsaMask | (0x7 << 23), kMsa | (0x5 << 23) | 0x10, "ave_u", "Vkmn" }, { kMsaMask | (0x7 << 23), kMsa | (0x6 << 23) | 0x10, "aver_s", "Vkmn" }, { kMsaMask | (0x7 << 23), kMsa | (0x7 << 23) | 0x10, "aver_u", "Vkmn" }, + { kMsaMask | (0x7 << 23), kMsa | (0x2 << 23) | 0xe, "max_s", "Vkmn" }, + { kMsaMask | (0x7 << 23), kMsa | (0x3 << 23) | 0xe, "max_u", "Vkmn" }, + { kMsaMask | (0x7 << 23), kMsa | (0x4 << 23) | 0xe, "min_s", "Vkmn" }, + { kMsaMask | (0x7 << 23), kMsa | (0x5 << 23) | 0xe, "min_u", "Vkmn" }, { kMsaMask | (0xf << 22), kMsa | (0x0 << 22) | 0x1b, "fadd", "Ukmn" }, { kMsaMask | (0xf << 22), kMsa | (0x1 << 22) | 0x1b, "fsub", "Ukmn" }, { kMsaMask | (0xf << 22), kMsa | (0x2 << 22) | 0x1b, "fmul", "Ukmn" }, { kMsaMask | (0xf << 22), kMsa | (0x3 << 22) | 0x1b, "fdiv", "Ukmn" }, + { kMsaMask | (0xf << 22), kMsa | (0xe << 22) | 0x1b, "fmax", "Ukmn" }, + { kMsaMask | (0xf << 22), kMsa | (0xc << 22) | 0x1b, "fmin", "Ukmn" }, { kMsaMask | (0x1ff << 17), kMsa | (0x19e << 17) | 0x1e, "ffint_s", "ukm" }, { kMsaMask | (0x1ff << 17), kMsa | (0x19c << 17) | 0x1e, "ftint_s", "ukm" }, { kMsaMask | (0x7 << 23), kMsa | (0x0 << 23) | 0xd, "sll", "Vkmn" }, diff --git a/oatdump/oatdump.cc b/oatdump/oatdump.cc index a79b408a40..9b95de2fb0 100644 --- a/oatdump/oatdump.cc +++ b/oatdump/oatdump.cc @@ -130,8 +130,8 @@ class OatSymbolizer FINAL { if (elf_file == nullptr) { return false; } - std::unique_ptr<BufferedOutputStream> output_stream( - MakeUnique<BufferedOutputStream>(MakeUnique<FileOutputStream>(elf_file.get()))); + std::unique_ptr<BufferedOutputStream> output_stream = + std::make_unique<BufferedOutputStream>(std::make_unique<FileOutputStream>(elf_file.get())); builder_.reset(new ElfBuilder<ElfTypes>(isa, features.get(), output_stream.get())); builder_->Start(); @@ -171,6 +171,7 @@ class OatSymbolizer FINAL { rodata_size, text_size, oat_file_->BssSize(), + oat_file_->BssMethodsOffset(), oat_file_->BssRootsOffset()); builder_->WriteDynamicSection(); diff --git a/patchoat/patchoat.cc b/patchoat/patchoat.cc index e750ede8fa..ec3481b622 100644 --- a/patchoat/patchoat.cc +++ b/patchoat/patchoat.cc @@ -40,6 +40,7 @@ #include "elf_file_impl.h" #include "gc/space/image_space.h" #include "image-inl.h" +#include "intern_table.h" #include "mirror/dex_cache.h" #include "mirror/executable.h" #include "mirror/object-inl.h" diff --git a/runtime/Android.bp b/runtime/Android.bp index c5508e32d4..d1e124fd35 100644 --- a/runtime/Android.bp +++ b/runtime/Android.bp @@ -429,6 +429,7 @@ gensrcs { srcs: [ "arch/instruction_set.h", "base/allocator.h", + "base/callee_save_type.h", "base/enums.h", "base/mutex.h", "debugger.h", @@ -456,7 +457,6 @@ gensrcs { "oat.h", "object_callbacks.h", "process_state.h", - "runtime.h", "stack.h", "thread.h", "thread_state.h", diff --git a/runtime/arch/arch_test.cc b/runtime/arch/arch_test.cc index d6056c0ece..838ae40838 100644 --- a/runtime/arch/arch_test.cc +++ b/runtime/arch/arch_test.cc @@ -17,11 +17,30 @@ #include <stdint.h> #include "art_method-inl.h" +#include "base/callee_save_type.h" #include "common_runtime_test.h" #include "quick/quick_method_frame_info.h" -// Common tests are declared next to the constants. -#define ADD_TEST_EQ(x, y) EXPECT_EQ(x, y); -#include "asm_support.h" + + +// asm_support.h declares tests next to the #defines. We use asm_support_check.h to (safely) +// generate CheckAsmSupportOffsetsAndSizes using gtest's EXPECT for the tests. We also use the +// RETURN_TYPE, HEADER and FOOTER defines from asm_support_check.h to try to ensure that any +// tests are actually generated. + +// Let CheckAsmSupportOffsetsAndSizes return a size_t (the count). +#define ASM_SUPPORT_CHECK_RETURN_TYPE size_t + +// Declare the counter that will be updated per test. +#define ASM_SUPPORT_CHECK_HEADER size_t count = 0; + +// Use EXPECT_EQ for tests, and increment the counter. +#define ADD_TEST_EQ(x, y) EXPECT_EQ(x, y); count++; + +// Return the counter at the end of CheckAsmSupportOffsetsAndSizes. +#define ASM_SUPPORT_CHECK_FOOTER return count; + +// Generate CheckAsmSupportOffsetsAndSizes(). +#include "asm_support_check.h" namespace art { @@ -40,7 +59,7 @@ class ArchTest : public CommonRuntimeTest { ASSERT_EQ(InstructionSet::kX86_64, Runtime::Current()->GetInstructionSet()); } - static void CheckFrameSize(InstructionSet isa, Runtime::CalleeSaveType type, uint32_t save_size) + static void CheckFrameSize(InstructionSet isa, CalleeSaveType type, uint32_t save_size) NO_THREAD_SAFETY_ANALYSIS { Runtime* const runtime = Runtime::Current(); Thread* const self = Thread::Current(); @@ -57,7 +76,8 @@ class ArchTest : public CommonRuntimeTest { }; TEST_F(ArchTest, CheckCommonOffsetsAndSizes) { - CheckAsmSupportOffsetsAndSizes(); + size_t test_count = CheckAsmSupportOffsetsAndSizes(); + EXPECT_GT(test_count, 0u); } // Grab architecture specific constants. @@ -151,16 +171,16 @@ static constexpr size_t kFrameSizeSaveEverything = FRAME_SIZE_SAVE_EVERYTHING; #define TEST_ARCH(Arch, arch) \ TEST_F(ArchTest, Arch) { \ CheckFrameSize(InstructionSet::k##Arch, \ - Runtime::kSaveAllCalleeSaves, \ + CalleeSaveType::kSaveAllCalleeSaves, \ arch::kFrameSizeSaveAllCalleeSaves); \ CheckFrameSize(InstructionSet::k##Arch, \ - Runtime::kSaveRefsOnly, \ + CalleeSaveType::kSaveRefsOnly, \ arch::kFrameSizeSaveRefsOnly); \ CheckFrameSize(InstructionSet::k##Arch, \ - Runtime::kSaveRefsAndArgs, \ + CalleeSaveType::kSaveRefsAndArgs, \ arch::kFrameSizeSaveRefsAndArgs); \ CheckFrameSize(InstructionSet::k##Arch, \ - Runtime::kSaveEverything, \ + CalleeSaveType::kSaveEverything, \ arch::kFrameSizeSaveEverything); \ } TEST_ARCH(Arm, arm) diff --git a/runtime/arch/arm/context_arm.cc b/runtime/arch/arm/context_arm.cc index 817dcf5783..0db14fb8a5 100644 --- a/runtime/arch/arm/context_arm.cc +++ b/runtime/arch/arm/context_arm.cc @@ -17,6 +17,7 @@ #include "context_arm.h" #include "base/bit_utils.h" +#include "base/bit_utils_iterator.h" #include "quick/quick_method_frame_info.h" #include "thread-current-inl.h" diff --git a/runtime/arch/arm/entrypoints_init_arm.cc b/runtime/arch/arm/entrypoints_init_arm.cc index 919b0afc40..8a8d26466f 100644 --- a/runtime/arch/arm/entrypoints_init_arm.cc +++ b/runtime/arch/arm/entrypoints_init_arm.cc @@ -18,6 +18,7 @@ #include <string.h> #include "arch/arm/asm_support_arm.h" +#include "base/bit_utils.h" #include "entrypoints/jni/jni_entrypoints.h" #include "entrypoints/quick/quick_alloc_entrypoints.h" #include "entrypoints/quick/quick_default_externs.h" diff --git a/runtime/arch/arm/quick_method_frame_info_arm.h b/runtime/arch/arm/quick_method_frame_info_arm.h index 35f1948138..39061f0d4d 100644 --- a/runtime/arch/arm/quick_method_frame_info_arm.h +++ b/runtime/arch/arm/quick_method_frame_info_arm.h @@ -17,10 +17,12 @@ #ifndef ART_RUNTIME_ARCH_ARM_QUICK_METHOD_FRAME_INFO_ARM_H_ #define ART_RUNTIME_ARCH_ARM_QUICK_METHOD_FRAME_INFO_ARM_H_ +#include "arch/instruction_set.h" #include "base/bit_utils.h" +#include "base/callee_save_type.h" +#include "base/enums.h" #include "quick/quick_method_frame_info.h" #include "registers_arm.h" -#include "runtime.h" // for Runtime::CalleeSaveType. namespace art { namespace arm { @@ -53,44 +55,44 @@ static constexpr uint32_t kArmCalleeSaveFpAllSpills = static constexpr uint32_t kArmCalleeSaveFpEverythingSpills = kArmCalleeSaveFpArgSpills | kArmCalleeSaveFpAllSpills; -constexpr uint32_t ArmCalleeSaveCoreSpills(Runtime::CalleeSaveType type) { +constexpr uint32_t ArmCalleeSaveCoreSpills(CalleeSaveType type) { return kArmCalleeSaveAlwaysSpills | kArmCalleeSaveRefSpills | - (type == Runtime::kSaveRefsAndArgs ? kArmCalleeSaveArgSpills : 0) | - (type == Runtime::kSaveAllCalleeSaves ? kArmCalleeSaveAllSpills : 0) | - (type == Runtime::kSaveEverything ? kArmCalleeSaveEverythingSpills : 0); + (type == CalleeSaveType::kSaveRefsAndArgs ? kArmCalleeSaveArgSpills : 0) | + (type == CalleeSaveType::kSaveAllCalleeSaves ? kArmCalleeSaveAllSpills : 0) | + (type == CalleeSaveType::kSaveEverything ? kArmCalleeSaveEverythingSpills : 0); } -constexpr uint32_t ArmCalleeSaveFpSpills(Runtime::CalleeSaveType type) { +constexpr uint32_t ArmCalleeSaveFpSpills(CalleeSaveType type) { return kArmCalleeSaveFpAlwaysSpills | kArmCalleeSaveFpRefSpills | - (type == Runtime::kSaveRefsAndArgs ? kArmCalleeSaveFpArgSpills : 0) | - (type == Runtime::kSaveAllCalleeSaves ? kArmCalleeSaveFpAllSpills : 0) | - (type == Runtime::kSaveEverything ? kArmCalleeSaveFpEverythingSpills : 0); + (type == CalleeSaveType::kSaveRefsAndArgs ? kArmCalleeSaveFpArgSpills : 0) | + (type == CalleeSaveType::kSaveAllCalleeSaves ? kArmCalleeSaveFpAllSpills : 0) | + (type == CalleeSaveType::kSaveEverything ? kArmCalleeSaveFpEverythingSpills : 0); } -constexpr uint32_t ArmCalleeSaveFrameSize(Runtime::CalleeSaveType type) { +constexpr uint32_t ArmCalleeSaveFrameSize(CalleeSaveType type) { return RoundUp((POPCOUNT(ArmCalleeSaveCoreSpills(type)) /* gprs */ + POPCOUNT(ArmCalleeSaveFpSpills(type)) /* fprs */ + 1 /* Method* */) * static_cast<size_t>(kArmPointerSize), kStackAlignment); } -constexpr QuickMethodFrameInfo ArmCalleeSaveMethodFrameInfo(Runtime::CalleeSaveType type) { +constexpr QuickMethodFrameInfo ArmCalleeSaveMethodFrameInfo(CalleeSaveType type) { return QuickMethodFrameInfo(ArmCalleeSaveFrameSize(type), ArmCalleeSaveCoreSpills(type), ArmCalleeSaveFpSpills(type)); } -constexpr size_t ArmCalleeSaveFpr1Offset(Runtime::CalleeSaveType type) { +constexpr size_t ArmCalleeSaveFpr1Offset(CalleeSaveType type) { return ArmCalleeSaveFrameSize(type) - (POPCOUNT(ArmCalleeSaveCoreSpills(type)) + POPCOUNT(ArmCalleeSaveFpSpills(type))) * static_cast<size_t>(kArmPointerSize); } -constexpr size_t ArmCalleeSaveGpr1Offset(Runtime::CalleeSaveType type) { +constexpr size_t ArmCalleeSaveGpr1Offset(CalleeSaveType type) { return ArmCalleeSaveFrameSize(type) - POPCOUNT(ArmCalleeSaveCoreSpills(type)) * static_cast<size_t>(kArmPointerSize); } -constexpr size_t ArmCalleeSaveLrOffset(Runtime::CalleeSaveType type) { +constexpr size_t ArmCalleeSaveLrOffset(CalleeSaveType type) { return ArmCalleeSaveFrameSize(type) - POPCOUNT(ArmCalleeSaveCoreSpills(type) & (-(1 << LR))) * static_cast<size_t>(kArmPointerSize); } diff --git a/runtime/arch/arm64/context_arm64.cc b/runtime/arch/arm64/context_arm64.cc index a8f034eaf4..0465c1e79d 100644 --- a/runtime/arch/arm64/context_arm64.cc +++ b/runtime/arch/arm64/context_arm64.cc @@ -19,6 +19,7 @@ #include "context_arm64.h" #include "base/bit_utils.h" +#include "base/bit_utils_iterator.h" #include "quick/quick_method_frame_info.h" #include "thread-current-inl.h" diff --git a/runtime/arch/arm64/entrypoints_init_arm64.cc b/runtime/arch/arm64/entrypoints_init_arm64.cc index 610cdee683..9bbcef307e 100644 --- a/runtime/arch/arm64/entrypoints_init_arm64.cc +++ b/runtime/arch/arm64/entrypoints_init_arm64.cc @@ -18,6 +18,7 @@ #include <string.h> #include "arch/arm64/asm_support_arm64.h" +#include "base/bit_utils.h" #include "entrypoints/jni/jni_entrypoints.h" #include "entrypoints/quick/quick_alloc_entrypoints.h" #include "entrypoints/quick/quick_default_externs.h" diff --git a/runtime/arch/arm64/quick_method_frame_info_arm64.h b/runtime/arch/arm64/quick_method_frame_info_arm64.h index 32d9d08739..c231d4d3d4 100644 --- a/runtime/arch/arm64/quick_method_frame_info_arm64.h +++ b/runtime/arch/arm64/quick_method_frame_info_arm64.h @@ -17,10 +17,13 @@ #ifndef ART_RUNTIME_ARCH_ARM64_QUICK_METHOD_FRAME_INFO_ARM64_H_ #define ART_RUNTIME_ARCH_ARM64_QUICK_METHOD_FRAME_INFO_ARM64_H_ +#include "arch/instruction_set.h" #include "base/bit_utils.h" +#include "base/callee_save_type.h" +#include "base/enums.h" +#include "globals.h" #include "quick/quick_method_frame_info.h" #include "registers_arm64.h" -#include "runtime.h" // for Runtime::CalleeSaveType. namespace art { namespace arm64 { @@ -76,44 +79,44 @@ static constexpr uint32_t kArm64CalleeSaveFpEverythingSpills = (1 << art::arm64::D27) | (1 << art::arm64::D28) | (1 << art::arm64::D29) | (1 << art::arm64::D30) | (1 << art::arm64::D31); -constexpr uint32_t Arm64CalleeSaveCoreSpills(Runtime::CalleeSaveType type) { +constexpr uint32_t Arm64CalleeSaveCoreSpills(CalleeSaveType type) { return kArm64CalleeSaveAlwaysSpills | kArm64CalleeSaveRefSpills | - (type == Runtime::kSaveRefsAndArgs ? kArm64CalleeSaveArgSpills : 0) | - (type == Runtime::kSaveAllCalleeSaves ? kArm64CalleeSaveAllSpills : 0) | - (type == Runtime::kSaveEverything ? kArm64CalleeSaveEverythingSpills : 0); + (type == CalleeSaveType::kSaveRefsAndArgs ? kArm64CalleeSaveArgSpills : 0) | + (type == CalleeSaveType::kSaveAllCalleeSaves ? kArm64CalleeSaveAllSpills : 0) | + (type == CalleeSaveType::kSaveEverything ? kArm64CalleeSaveEverythingSpills : 0); } -constexpr uint32_t Arm64CalleeSaveFpSpills(Runtime::CalleeSaveType type) { +constexpr uint32_t Arm64CalleeSaveFpSpills(CalleeSaveType type) { return kArm64CalleeSaveFpAlwaysSpills | kArm64CalleeSaveFpRefSpills | - (type == Runtime::kSaveRefsAndArgs ? kArm64CalleeSaveFpArgSpills : 0) | - (type == Runtime::kSaveAllCalleeSaves ? kArm64CalleeSaveFpAllSpills : 0) | - (type == Runtime::kSaveEverything ? kArm64CalleeSaveFpEverythingSpills : 0); + (type == CalleeSaveType::kSaveRefsAndArgs ? kArm64CalleeSaveFpArgSpills : 0) | + (type == CalleeSaveType::kSaveAllCalleeSaves ? kArm64CalleeSaveFpAllSpills : 0) | + (type == CalleeSaveType::kSaveEverything ? kArm64CalleeSaveFpEverythingSpills : 0); } -constexpr uint32_t Arm64CalleeSaveFrameSize(Runtime::CalleeSaveType type) { +constexpr uint32_t Arm64CalleeSaveFrameSize(CalleeSaveType type) { return RoundUp((POPCOUNT(Arm64CalleeSaveCoreSpills(type)) /* gprs */ + POPCOUNT(Arm64CalleeSaveFpSpills(type)) /* fprs */ + 1 /* Method* */) * static_cast<size_t>(kArm64PointerSize), kStackAlignment); } -constexpr QuickMethodFrameInfo Arm64CalleeSaveMethodFrameInfo(Runtime::CalleeSaveType type) { +constexpr QuickMethodFrameInfo Arm64CalleeSaveMethodFrameInfo(CalleeSaveType type) { return QuickMethodFrameInfo(Arm64CalleeSaveFrameSize(type), Arm64CalleeSaveCoreSpills(type), Arm64CalleeSaveFpSpills(type)); } -constexpr size_t Arm64CalleeSaveFpr1Offset(Runtime::CalleeSaveType type) { +constexpr size_t Arm64CalleeSaveFpr1Offset(CalleeSaveType type) { return Arm64CalleeSaveFrameSize(type) - (POPCOUNT(Arm64CalleeSaveCoreSpills(type)) + POPCOUNT(Arm64CalleeSaveFpSpills(type))) * static_cast<size_t>(kArm64PointerSize); } -constexpr size_t Arm64CalleeSaveGpr1Offset(Runtime::CalleeSaveType type) { +constexpr size_t Arm64CalleeSaveGpr1Offset(CalleeSaveType type) { return Arm64CalleeSaveFrameSize(type) - POPCOUNT(Arm64CalleeSaveCoreSpills(type)) * static_cast<size_t>(kArm64PointerSize); } -constexpr size_t Arm64CalleeSaveLrOffset(Runtime::CalleeSaveType type) { +constexpr size_t Arm64CalleeSaveLrOffset(CalleeSaveType type) { return Arm64CalleeSaveFrameSize(type) - POPCOUNT(Arm64CalleeSaveCoreSpills(type) & (-(1 << LR))) * static_cast<size_t>(kArm64PointerSize); diff --git a/runtime/arch/mips/context_mips.cc b/runtime/arch/mips/context_mips.cc index 98ed5e60e6..ca1de0ae2a 100644 --- a/runtime/arch/mips/context_mips.cc +++ b/runtime/arch/mips/context_mips.cc @@ -17,6 +17,7 @@ #include "context_mips.h" #include "base/bit_utils.h" +#include "base/bit_utils_iterator.h" #include "quick/quick_method_frame_info.h" namespace art { diff --git a/runtime/arch/mips/fault_handler_mips.cc b/runtime/arch/mips/fault_handler_mips.cc index 25e442c3e6..52a3df55d6 100644 --- a/runtime/arch/mips/fault_handler_mips.cc +++ b/runtime/arch/mips/fault_handler_mips.cc @@ -18,6 +18,7 @@ #include <sys/ucontext.h> #include "art_method.h" +#include "base/callee_save_type.h" #include "base/hex_dump.h" #include "base/logging.h" #include "base/macros.h" @@ -80,7 +81,7 @@ bool NullPointerHandler::Action(int sig ATTRIBUTE_UNUSED, siginfo_t* info, void* // Decrement $sp by the frame size of the kSaveEverything method and store // the fault address in the padding right after the ArtMethod*. - sc->sc_regs[mips::SP] -= mips::MipsCalleeSaveFrameSize(Runtime::kSaveEverything); + sc->sc_regs[mips::SP] -= mips::MipsCalleeSaveFrameSize(CalleeSaveType::kSaveEverything); uintptr_t* padding = reinterpret_cast<uintptr_t*>(sc->sc_regs[mips::SP]) + /* ArtMethod* */ 1; *padding = reinterpret_cast<uintptr_t>(info->si_addr); diff --git a/runtime/arch/mips/quick_method_frame_info_mips.h b/runtime/arch/mips/quick_method_frame_info_mips.h index 6f16352d91..01879a5cea 100644 --- a/runtime/arch/mips/quick_method_frame_info_mips.h +++ b/runtime/arch/mips/quick_method_frame_info_mips.h @@ -17,10 +17,12 @@ #ifndef ART_RUNTIME_ARCH_MIPS_QUICK_METHOD_FRAME_INFO_MIPS_H_ #define ART_RUNTIME_ARCH_MIPS_QUICK_METHOD_FRAME_INFO_MIPS_H_ +#include "arch/instruction_set.h" #include "base/bit_utils.h" +#include "base/callee_save_type.h" +#include "base/enums.h" #include "quick/quick_method_frame_info.h" #include "registers_mips.h" -#include "runtime.h" // for Runtime::CalleeSaveType. namespace art { namespace mips { @@ -62,27 +64,27 @@ static constexpr uint32_t kMipsCalleeSaveFpEverythingSpills = (1 << art::mips::F24) | (1 << art::mips::F25) | (1 << art::mips::F26) | (1 << art::mips::F27) | (1 << art::mips::F28) | (1 << art::mips::F29) | (1 << art::mips::F30) | (1u << art::mips::F31); -constexpr uint32_t MipsCalleeSaveCoreSpills(Runtime::CalleeSaveType type) { +constexpr uint32_t MipsCalleeSaveCoreSpills(CalleeSaveType type) { return kMipsCalleeSaveAlwaysSpills | kMipsCalleeSaveRefSpills | - (type == Runtime::kSaveRefsAndArgs ? kMipsCalleeSaveArgSpills : 0) | - (type == Runtime::kSaveAllCalleeSaves ? kMipsCalleeSaveAllSpills : 0) | - (type == Runtime::kSaveEverything ? kMipsCalleeSaveEverythingSpills : 0); + (type == CalleeSaveType::kSaveRefsAndArgs ? kMipsCalleeSaveArgSpills : 0) | + (type == CalleeSaveType::kSaveAllCalleeSaves ? kMipsCalleeSaveAllSpills : 0) | + (type == CalleeSaveType::kSaveEverything ? kMipsCalleeSaveEverythingSpills : 0); } -constexpr uint32_t MipsCalleeSaveFPSpills(Runtime::CalleeSaveType type) { +constexpr uint32_t MipsCalleeSaveFPSpills(CalleeSaveType type) { return kMipsCalleeSaveFpAlwaysSpills | kMipsCalleeSaveFpRefSpills | - (type == Runtime::kSaveRefsAndArgs ? kMipsCalleeSaveFpArgSpills : 0) | - (type == Runtime::kSaveAllCalleeSaves ? kMipsCalleeSaveAllFPSpills : 0) | - (type == Runtime::kSaveEverything ? kMipsCalleeSaveFpEverythingSpills : 0); + (type == CalleeSaveType::kSaveRefsAndArgs ? kMipsCalleeSaveFpArgSpills : 0) | + (type == CalleeSaveType::kSaveAllCalleeSaves ? kMipsCalleeSaveAllFPSpills : 0) | + (type == CalleeSaveType::kSaveEverything ? kMipsCalleeSaveFpEverythingSpills : 0); } -constexpr uint32_t MipsCalleeSaveFrameSize(Runtime::CalleeSaveType type) { +constexpr uint32_t MipsCalleeSaveFrameSize(CalleeSaveType type) { return RoundUp((POPCOUNT(MipsCalleeSaveCoreSpills(type)) /* gprs */ + POPCOUNT(MipsCalleeSaveFPSpills(type)) /* fprs */ + 1 /* Method* */) * static_cast<size_t>(kMipsPointerSize), kStackAlignment); } -constexpr QuickMethodFrameInfo MipsCalleeSaveMethodFrameInfo(Runtime::CalleeSaveType type) { +constexpr QuickMethodFrameInfo MipsCalleeSaveMethodFrameInfo(CalleeSaveType type) { return QuickMethodFrameInfo(MipsCalleeSaveFrameSize(type), MipsCalleeSaveCoreSpills(type), MipsCalleeSaveFPSpills(type)); diff --git a/runtime/arch/mips/registers_mips.cc b/runtime/arch/mips/registers_mips.cc index 5d31f2f910..92c2746d2a 100644 --- a/runtime/arch/mips/registers_mips.cc +++ b/runtime/arch/mips/registers_mips.cc @@ -45,5 +45,14 @@ std::ostream& operator<<(std::ostream& os, const FRegister& rhs) { return os; } +std::ostream& operator<<(std::ostream& os, const VectorRegister& rhs) { + if (rhs >= W0 && rhs < kNumberOfVectorRegisters) { + os << "w" << static_cast<int>(rhs); + } else { + os << "VectorRegister[" << static_cast<int>(rhs) << "]"; + } + return os; +} + } // namespace mips } // namespace art diff --git a/runtime/arch/mips/registers_mips.h b/runtime/arch/mips/registers_mips.h index 555f3f0473..57af150b33 100644 --- a/runtime/arch/mips/registers_mips.h +++ b/runtime/arch/mips/registers_mips.h @@ -106,6 +106,45 @@ enum FRegister { }; std::ostream& operator<<(std::ostream& os, const FRegister& rhs); +// Values for vector registers. +enum VectorRegister { + W0 = 0, + W1 = 1, + W2 = 2, + W3 = 3, + W4 = 4, + W5 = 5, + W6 = 6, + W7 = 7, + W8 = 8, + W9 = 9, + W10 = 10, + W11 = 11, + W12 = 12, + W13 = 13, + W14 = 14, + W15 = 15, + W16 = 16, + W17 = 17, + W18 = 18, + W19 = 19, + W20 = 20, + W21 = 21, + W22 = 22, + W23 = 23, + W24 = 24, + W25 = 25, + W26 = 26, + W27 = 27, + W28 = 28, + W29 = 29, + W30 = 30, + W31 = 31, + kNumberOfVectorRegisters = 32, + kNoVectorRegister = -1, +}; +std::ostream& operator<<(std::ostream& os, const VectorRegister& rhs); + } // namespace mips } // namespace art diff --git a/runtime/arch/mips64/context_mips64.cc b/runtime/arch/mips64/context_mips64.cc index bd1ac3b0a7..b14908fa04 100644 --- a/runtime/arch/mips64/context_mips64.cc +++ b/runtime/arch/mips64/context_mips64.cc @@ -17,6 +17,7 @@ #include "context_mips64.h" #include "base/bit_utils.h" +#include "base/bit_utils_iterator.h" #include "quick/quick_method_frame_info.h" namespace art { diff --git a/runtime/arch/mips64/fault_handler_mips64.cc b/runtime/arch/mips64/fault_handler_mips64.cc index 69d73b09c2..9d77ebcd22 100644 --- a/runtime/arch/mips64/fault_handler_mips64.cc +++ b/runtime/arch/mips64/fault_handler_mips64.cc @@ -19,6 +19,7 @@ #include <sys/ucontext.h> #include "art_method.h" +#include "base/callee_save_type.h" #include "base/hex_dump.h" #include "base/logging.h" #include "base/macros.h" @@ -82,7 +83,7 @@ bool NullPointerHandler::Action(int sig ATTRIBUTE_UNUSED, siginfo_t* info, void* // Decrement $sp by the frame size of the kSaveEverything method and store // the fault address in the padding right after the ArtMethod*. - sc->sc_regs[mips64::SP] -= mips64::Mips64CalleeSaveFrameSize(Runtime::kSaveEverything); + sc->sc_regs[mips64::SP] -= mips64::Mips64CalleeSaveFrameSize(CalleeSaveType::kSaveEverything); uintptr_t* padding = reinterpret_cast<uintptr_t*>(sc->sc_regs[mips64::SP]) + /* ArtMethod* */ 1; *padding = reinterpret_cast<uintptr_t>(info->si_addr); diff --git a/runtime/arch/mips64/quick_method_frame_info_mips64.h b/runtime/arch/mips64/quick_method_frame_info_mips64.h index d774473289..a55ab0e196 100644 --- a/runtime/arch/mips64/quick_method_frame_info_mips64.h +++ b/runtime/arch/mips64/quick_method_frame_info_mips64.h @@ -17,10 +17,12 @@ #ifndef ART_RUNTIME_ARCH_MIPS64_QUICK_METHOD_FRAME_INFO_MIPS64_H_ #define ART_RUNTIME_ARCH_MIPS64_QUICK_METHOD_FRAME_INFO_MIPS64_H_ +#include "arch/instruction_set.h" #include "base/bit_utils.h" +#include "base/callee_save_type.h" +#include "base/enums.h" #include "quick/quick_method_frame_info.h" #include "registers_mips64.h" -#include "runtime.h" // for Runtime::CalleeSaveType. namespace art { namespace mips64 { @@ -69,27 +71,27 @@ static constexpr uint32_t kMips64CalleeSaveFpEverythingSpills = (1 << art::mips64::F27) | (1 << art::mips64::F28) | (1 << art::mips64::F29) | (1 << art::mips64::F30) | (1 << art::mips64::F31); -constexpr uint32_t Mips64CalleeSaveCoreSpills(Runtime::CalleeSaveType type) { +constexpr uint32_t Mips64CalleeSaveCoreSpills(CalleeSaveType type) { return kMips64CalleeSaveAlwaysSpills | kMips64CalleeSaveRefSpills | - (type == Runtime::kSaveRefsAndArgs ? kMips64CalleeSaveArgSpills : 0) | - (type == Runtime::kSaveAllCalleeSaves ? kMips64CalleeSaveAllSpills : 0) | - (type == Runtime::kSaveEverything ? kMips64CalleeSaveEverythingSpills : 0); + (type == CalleeSaveType::kSaveRefsAndArgs ? kMips64CalleeSaveArgSpills : 0) | + (type == CalleeSaveType::kSaveAllCalleeSaves ? kMips64CalleeSaveAllSpills : 0) | + (type == CalleeSaveType::kSaveEverything ? kMips64CalleeSaveEverythingSpills : 0); } -constexpr uint32_t Mips64CalleeSaveFpSpills(Runtime::CalleeSaveType type) { +constexpr uint32_t Mips64CalleeSaveFpSpills(CalleeSaveType type) { return kMips64CalleeSaveFpRefSpills | - (type == Runtime::kSaveRefsAndArgs ? kMips64CalleeSaveFpArgSpills : 0) | - (type == Runtime::kSaveAllCalleeSaves ? kMips64CalleeSaveFpAllSpills : 0) | - (type == Runtime::kSaveEverything ? kMips64CalleeSaveFpEverythingSpills : 0); + (type == CalleeSaveType::kSaveRefsAndArgs ? kMips64CalleeSaveFpArgSpills : 0) | + (type == CalleeSaveType::kSaveAllCalleeSaves ? kMips64CalleeSaveFpAllSpills : 0) | + (type == CalleeSaveType::kSaveEverything ? kMips64CalleeSaveFpEverythingSpills : 0); } -constexpr uint32_t Mips64CalleeSaveFrameSize(Runtime::CalleeSaveType type) { +constexpr uint32_t Mips64CalleeSaveFrameSize(CalleeSaveType type) { return RoundUp((POPCOUNT(Mips64CalleeSaveCoreSpills(type)) /* gprs */ + POPCOUNT(Mips64CalleeSaveFpSpills(type)) /* fprs */ + + 1 /* Method* */) * static_cast<size_t>(kMips64PointerSize), kStackAlignment); } -constexpr QuickMethodFrameInfo Mips64CalleeSaveMethodFrameInfo(Runtime::CalleeSaveType type) { +constexpr QuickMethodFrameInfo Mips64CalleeSaveMethodFrameInfo(CalleeSaveType type) { return QuickMethodFrameInfo(Mips64CalleeSaveFrameSize(type), Mips64CalleeSaveCoreSpills(type), Mips64CalleeSaveFpSpills(type)); diff --git a/runtime/arch/stub_test.cc b/runtime/arch/stub_test.cc index 207bf9d365..bd51809c22 100644 --- a/runtime/arch/stub_test.cc +++ b/runtime/arch/stub_test.cc @@ -18,6 +18,7 @@ #include "art_field-inl.h" #include "art_method-inl.h" +#include "base/callee_save_type.h" #include "base/enums.h" #include "class_linker-inl.h" #include "common_runtime_test.h" @@ -43,8 +44,8 @@ class StubTest : public CommonRuntimeTest { // Create callee-save methods ScopedObjectAccess soa(Thread::Current()); runtime_->SetInstructionSet(kRuntimeISA); - for (int i = 0; i < Runtime::kLastCalleeSaveType; i++) { - Runtime::CalleeSaveType type = Runtime::CalleeSaveType(i); + for (uint32_t i = 0; i < static_cast<uint32_t>(CalleeSaveType::kLastCalleeSaveType); ++i) { + CalleeSaveType type = CalleeSaveType(i); if (!runtime_->HasCalleeSaveMethod(type)) { runtime_->SetCalleeSaveMethod(runtime_->CreateCalleeSaveMethod(), type); } diff --git a/runtime/arch/x86/context_x86.cc b/runtime/arch/x86/context_x86.cc index cb3dfec5f5..5c3171299c 100644 --- a/runtime/arch/x86/context_x86.cc +++ b/runtime/arch/x86/context_x86.cc @@ -17,6 +17,7 @@ #include "context_x86.h" #include "base/bit_utils.h" +#include "base/bit_utils_iterator.h" #include "base/memory_tool.h" #include "quick/quick_method_frame_info.h" diff --git a/runtime/arch/x86/quick_method_frame_info_x86.h b/runtime/arch/x86/quick_method_frame_info_x86.h index 9fcde35cec..8342c9fe03 100644 --- a/runtime/arch/x86/quick_method_frame_info_x86.h +++ b/runtime/arch/x86/quick_method_frame_info_x86.h @@ -17,10 +17,12 @@ #ifndef ART_RUNTIME_ARCH_X86_QUICK_METHOD_FRAME_INFO_X86_H_ #define ART_RUNTIME_ARCH_X86_QUICK_METHOD_FRAME_INFO_X86_H_ +#include "arch/instruction_set.h" #include "base/bit_utils.h" +#include "base/callee_save_type.h" +#include "base/enums.h" #include "quick/quick_method_frame_info.h" #include "registers_x86.h" -#include "runtime.h" // for Runtime::CalleeSaveType. namespace art { namespace x86 { @@ -54,24 +56,24 @@ static constexpr uint32_t kX86CalleeSaveFpEverythingSpills = (1 << art::x86::XMM4) | (1 << art::x86::XMM5) | (1 << art::x86::XMM6) | (1 << art::x86::XMM7); -constexpr uint32_t X86CalleeSaveCoreSpills(Runtime::CalleeSaveType type) { +constexpr uint32_t X86CalleeSaveCoreSpills(CalleeSaveType type) { return kX86CalleeSaveAlwaysSpills | kX86CalleeSaveRefSpills | - (type == Runtime::kSaveRefsAndArgs ? kX86CalleeSaveArgSpills : 0) | - (type == Runtime::kSaveEverything ? kX86CalleeSaveEverythingSpills : 0); + (type == CalleeSaveType::kSaveRefsAndArgs ? kX86CalleeSaveArgSpills : 0) | + (type == CalleeSaveType::kSaveEverything ? kX86CalleeSaveEverythingSpills : 0); } -constexpr uint32_t X86CalleeSaveFpSpills(Runtime::CalleeSaveType type) { - return (type == Runtime::kSaveRefsAndArgs ? kX86CalleeSaveFpArgSpills : 0) | - (type == Runtime::kSaveEverything ? kX86CalleeSaveFpEverythingSpills : 0); +constexpr uint32_t X86CalleeSaveFpSpills(CalleeSaveType type) { + return (type == CalleeSaveType::kSaveRefsAndArgs ? kX86CalleeSaveFpArgSpills : 0) | + (type == CalleeSaveType::kSaveEverything ? kX86CalleeSaveFpEverythingSpills : 0); } -constexpr uint32_t X86CalleeSaveFrameSize(Runtime::CalleeSaveType type) { +constexpr uint32_t X86CalleeSaveFrameSize(CalleeSaveType type) { return RoundUp((POPCOUNT(X86CalleeSaveCoreSpills(type)) /* gprs */ + 2 * POPCOUNT(X86CalleeSaveFpSpills(type)) /* fprs */ + 1 /* Method* */) * static_cast<size_t>(kX86PointerSize), kStackAlignment); } -constexpr QuickMethodFrameInfo X86CalleeSaveMethodFrameInfo(Runtime::CalleeSaveType type) { +constexpr QuickMethodFrameInfo X86CalleeSaveMethodFrameInfo(CalleeSaveType type) { return QuickMethodFrameInfo(X86CalleeSaveFrameSize(type), X86CalleeSaveCoreSpills(type), X86CalleeSaveFpSpills(type)); diff --git a/runtime/arch/x86_64/context_x86_64.cc b/runtime/arch/x86_64/context_x86_64.cc index 7c49e9c2b2..a4db223b1a 100644 --- a/runtime/arch/x86_64/context_x86_64.cc +++ b/runtime/arch/x86_64/context_x86_64.cc @@ -17,6 +17,7 @@ #include "context_x86_64.h" #include "base/bit_utils.h" +#include "base/bit_utils_iterator.h" #include "quick/quick_method_frame_info.h" namespace art { diff --git a/runtime/arch/x86_64/quick_method_frame_info_x86_64.h b/runtime/arch/x86_64/quick_method_frame_info_x86_64.h index 867522f3c1..425d616e76 100644 --- a/runtime/arch/x86_64/quick_method_frame_info_x86_64.h +++ b/runtime/arch/x86_64/quick_method_frame_info_x86_64.h @@ -17,10 +17,12 @@ #ifndef ART_RUNTIME_ARCH_X86_64_QUICK_METHOD_FRAME_INFO_X86_64_H_ #define ART_RUNTIME_ARCH_X86_64_QUICK_METHOD_FRAME_INFO_X86_64_H_ +#include "arch/instruction_set.h" #include "base/bit_utils.h" +#include "base/callee_save_type.h" +#include "base/enums.h" #include "quick/quick_method_frame_info.h" #include "registers_x86_64.h" -#include "runtime.h" // for Runtime::CalleeSaveType. namespace art { namespace x86_64 { @@ -53,25 +55,25 @@ static constexpr uint32_t kX86_64CalleeSaveFpEverythingSpills = (1 << art::x86_64::XMM8) | (1 << art::x86_64::XMM9) | (1 << art::x86_64::XMM10) | (1 << art::x86_64::XMM11); -constexpr uint32_t X86_64CalleeSaveCoreSpills(Runtime::CalleeSaveType type) { +constexpr uint32_t X86_64CalleeSaveCoreSpills(CalleeSaveType type) { return kX86_64CalleeSaveAlwaysSpills | kX86_64CalleeSaveRefSpills | - (type == Runtime::kSaveRefsAndArgs ? kX86_64CalleeSaveArgSpills : 0) | - (type == Runtime::kSaveEverything ? kX86_64CalleeSaveEverythingSpills : 0); + (type == CalleeSaveType::kSaveRefsAndArgs ? kX86_64CalleeSaveArgSpills : 0) | + (type == CalleeSaveType::kSaveEverything ? kX86_64CalleeSaveEverythingSpills : 0); } -constexpr uint32_t X86_64CalleeSaveFpSpills(Runtime::CalleeSaveType type) { +constexpr uint32_t X86_64CalleeSaveFpSpills(CalleeSaveType type) { return kX86_64CalleeSaveFpSpills | - (type == Runtime::kSaveRefsAndArgs ? kX86_64CalleeSaveFpArgSpills : 0) | - (type == Runtime::kSaveEverything ? kX86_64CalleeSaveFpEverythingSpills : 0); + (type == CalleeSaveType::kSaveRefsAndArgs ? kX86_64CalleeSaveFpArgSpills : 0) | + (type == CalleeSaveType::kSaveEverything ? kX86_64CalleeSaveFpEverythingSpills : 0); } -constexpr uint32_t X86_64CalleeSaveFrameSize(Runtime::CalleeSaveType type) { +constexpr uint32_t X86_64CalleeSaveFrameSize(CalleeSaveType type) { return RoundUp((POPCOUNT(X86_64CalleeSaveCoreSpills(type)) /* gprs */ + POPCOUNT(X86_64CalleeSaveFpSpills(type)) /* fprs */ + 1 /* Method* */) * static_cast<size_t>(kX86_64PointerSize), kStackAlignment); } -constexpr QuickMethodFrameInfo X86_64CalleeSaveMethodFrameInfo(Runtime::CalleeSaveType type) { +constexpr QuickMethodFrameInfo X86_64CalleeSaveMethodFrameInfo(CalleeSaveType type) { return QuickMethodFrameInfo(X86_64CalleeSaveFrameSize(type), X86_64CalleeSaveCoreSpills(type), X86_64CalleeSaveFpSpills(type)); diff --git a/runtime/art_method-inl.h b/runtime/art_method-inl.h index d1afcb8dd3..40d7e5c3f3 100644 --- a/runtime/art_method-inl.h +++ b/runtime/art_method-inl.h @@ -20,6 +20,7 @@ #include "art_method.h" #include "art_field.h" +#include "base/callee_save_type.h" #include "base/logging.h" #include "class_linker-inl.h" #include "common_throws.h" @@ -36,7 +37,9 @@ #include "mirror/string.h" #include "oat.h" #include "obj_ptr-inl.h" +#include "primitive.h" #include "quick/quick_method_frame_info.h" +#include "read_barrier-inl.h" #include "runtime-inl.h" #include "scoped_thread_state_change-inl.h" #include "thread-current-inl.h" @@ -199,8 +202,8 @@ inline bool ArtMethod::IsCalleeSaveMethod() { } Runtime* runtime = Runtime::Current(); bool result = false; - for (int i = 0; i < Runtime::kLastCalleeSaveType; i++) { - if (this == runtime->GetCalleeSaveMethod(Runtime::CalleeSaveType(i))) { + for (uint32_t i = 0; i < static_cast<uint32_t>(CalleeSaveType::kLastCalleeSaveType); i++) { + if (this == runtime->GetCalleeSaveMethod(CalleeSaveType(i))) { result = true; break; } @@ -271,12 +274,14 @@ inline const char* ArtMethod::GetName() { return "<runtime internal resolution method>"; } else if (this == runtime->GetImtConflictMethod()) { return "<runtime internal imt conflict method>"; - } else if (this == runtime->GetCalleeSaveMethod(Runtime::kSaveAllCalleeSaves)) { + } else if (this == runtime->GetCalleeSaveMethod(CalleeSaveType::kSaveAllCalleeSaves)) { return "<runtime internal callee-save all registers method>"; - } else if (this == runtime->GetCalleeSaveMethod(Runtime::kSaveRefsOnly)) { + } else if (this == runtime->GetCalleeSaveMethod(CalleeSaveType::kSaveRefsOnly)) { return "<runtime internal callee-save reference registers method>"; - } else if (this == runtime->GetCalleeSaveMethod(Runtime::kSaveRefsAndArgs)) { + } else if (this == runtime->GetCalleeSaveMethod(CalleeSaveType::kSaveRefsAndArgs)) { return "<runtime internal callee-save reference and argument registers method>"; + } else if (this == runtime->GetCalleeSaveMethod(CalleeSaveType::kSaveEverything)) { + return "<runtime internal save-every-register method>"; } else { return "<unknown runtime internal method>"; } @@ -340,6 +345,10 @@ inline const char* ArtMethod::GetReturnTypeDescriptor() { return dex_file->GetTypeDescriptor(dex_file->GetTypeId(proto_id.return_type_idx_)); } +inline Primitive::Type ArtMethod::GetReturnTypePrimitive() { + return Primitive::GetType(GetReturnTypeDescriptor()[0]); +} + inline const char* ArtMethod::GetTypeDescriptorFromTypeIdx(dex::TypeIndex type_idx) { DCHECK(!IsProxyMethod()); const DexFile* dex_file = GetDexFile(); diff --git a/runtime/art_method.h b/runtime/art_method.h index d8dfdd7959..3a8d279606 100644 --- a/runtime/art_method.h +++ b/runtime/art_method.h @@ -27,6 +27,7 @@ #include "modifiers.h" #include "obj_ptr.h" #include "offsets.h" +#include "primitive.h" #include "read_barrier_option.h" namespace art { @@ -569,6 +570,8 @@ class ArtMethod FINAL { const char* GetReturnTypeDescriptor() REQUIRES_SHARED(Locks::mutator_lock_); + ALWAYS_INLINE Primitive::Type GetReturnTypePrimitive() REQUIRES_SHARED(Locks::mutator_lock_); + const char* GetTypeDescriptorFromTypeIdx(dex::TypeIndex type_idx) REQUIRES_SHARED(Locks::mutator_lock_); diff --git a/runtime/asm_support.h b/runtime/asm_support.h index 1ce7fd33bb..44c0661e3f 100644 --- a/runtime/asm_support.h +++ b/runtime/asm_support.h @@ -17,23 +17,6 @@ #ifndef ART_RUNTIME_ASM_SUPPORT_H_ #define ART_RUNTIME_ASM_SUPPORT_H_ -#if defined(__cplusplus) -#include "art_method.h" -#include "base/bit_utils.h" -#include "gc/accounting/card_table.h" -#include "gc/allocator/rosalloc.h" -#include "gc/heap.h" -#include "jit/jit.h" -#include "lock_word.h" -#include "mirror/class.h" -#include "mirror/dex_cache.h" -#include "mirror/string.h" -#include "utils/dex_cache_arrays_layout.h" -#include "runtime.h" -#include "stack.h" -#include "thread.h" -#endif - #include "read_barrier_c.h" #if defined(__arm__) || defined(__mips__) @@ -50,14 +33,10 @@ #define SUSPEND_CHECK_INTERVAL 96 #endif -#if defined(__cplusplus) - +// To generate tests related to the constants in this header, either define ADD_TEST_EQ before +// including, or use asm_support_check.h. #ifndef ADD_TEST_EQ // Allow #include-r to replace with their own. -#define ADD_TEST_EQ(x, y) CHECK_EQ(x, y); -#endif - -static inline void CheckAsmSupportOffsetsAndSizes() { -#else +#define DEFINED_ADD_TEST_EQ 1 #define ADD_TEST_EQ(x, y) #endif @@ -75,6 +54,7 @@ ADD_TEST_EQ(static_cast<size_t>(1U << POINTER_SIZE_SHIFT), // Export new defines (for assembly use) by editing cpp-define-generator def files. #define DEFINE_CHECK_EQ ADD_TEST_EQ #include "asm_support_gen.h" +#undef DEFINE_CHECK_EQ // Offset of field Thread::tlsPtr_.exception. #define THREAD_EXCEPTION_OFFSET (THREAD_CARD_TABLE_OFFSET + __SIZEOF_POINTER__) @@ -251,8 +231,9 @@ ADD_TEST_EQ(MIRROR_STRING_VALUE_OFFSET, art::mirror::String::ValueOffset().Int32 #define STRING_COMPRESSION_FEATURE 1 ADD_TEST_EQ(STRING_COMPRESSION_FEATURE, art::mirror::kUseStringCompression); -#if defined(__cplusplus) -} // End of CheckAsmSupportOffsets. +#ifdef DEFINED_ADD_TEST_EQ +#undef ADD_TEST_EQ +#undef DEFINED_ADD_TEST_EQ #endif #endif // ART_RUNTIME_ASM_SUPPORT_H_ diff --git a/runtime/asm_support_check.h b/runtime/asm_support_check.h new file mode 100644 index 0000000000..cc6a578313 --- /dev/null +++ b/runtime/asm_support_check.h @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2011 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_RUNTIME_ASM_SUPPORT_CHECK_H_ +#define ART_RUNTIME_ASM_SUPPORT_CHECK_H_ + +#include "art_method.h" +#include "base/bit_utils.h" +#include "base/callee_save_type.h" +#include "gc/accounting/card_table.h" +#include "gc/allocator/rosalloc.h" +#include "gc/heap.h" +#include "jit/jit.h" +#include "lock_word.h" +#include "mirror/class.h" +#include "mirror/dex_cache.h" +#include "mirror/string.h" +#include "utils/dex_cache_arrays_layout.h" +#include "runtime.h" +#include "stack.h" +#include "thread.h" + +#ifndef ADD_TEST_EQ +#define ADD_TEST_EQ(x, y) CHECK_EQ(x, y); +#endif + +#ifndef ASM_SUPPORT_CHECK_RETURN_TYPE +#define ASM_SUPPORT_CHECK_RETURN_TYPE void +#endif + +// Prepare for re-include of asm_support.h. +#ifdef ART_RUNTIME_ASM_SUPPORT_H_ +#undef ART_RUNTIME_ASM_SUPPORT_H_ +#endif + +namespace art { + +static inline ASM_SUPPORT_CHECK_RETURN_TYPE CheckAsmSupportOffsetsAndSizes() { +#ifdef ASM_SUPPORT_CHECK_HEADER + ASM_SUPPORT_CHECK_HEADER +#endif + +#include "asm_support.h" + +#ifdef ASM_SUPPORT_CHECK_FOOTER + ASM_SUPPORT_CHECK_FOOTER +#endif +} + +} // namespace art + +#endif // ART_RUNTIME_ASM_SUPPORT_CHECK_H_ diff --git a/runtime/base/allocator.h b/runtime/base/allocator.h index 8d1c982f3d..fba9308e8e 100644 --- a/runtime/base/allocator.h +++ b/runtime/base/allocator.h @@ -17,12 +17,7 @@ #ifndef ART_RUNTIME_BASE_ALLOCATOR_H_ #define ART_RUNTIME_BASE_ALLOCATOR_H_ -#include <map> -#include <set> -#include <unordered_map> - #include "atomic.h" -#include "base/hash_map.h" #include "base/macros.h" #include "base/mutex.h" #include "base/type_static_if.h" @@ -156,29 +151,6 @@ using TrackingAllocator = typename TypeStaticIf<kEnableTrackingAllocator, TrackingAllocatorImpl<T, kTag>, std::allocator<T>>::type; -template<class Key, class T, AllocatorTag kTag, class Compare = std::less<Key>> -using AllocationTrackingMultiMap = std::multimap< - Key, T, Compare, TrackingAllocator<std::pair<const Key, T>, kTag>>; - -template<class Key, AllocatorTag kTag, class Compare = std::less<Key>> -using AllocationTrackingSet = std::set<Key, Compare, TrackingAllocator<Key, kTag>>; - -template<class Key, - class T, - AllocatorTag kTag, - class Hash = std::hash<Key>, - class Pred = std::equal_to<Key>> -using AllocationTrackingUnorderedMap = std::unordered_map< - Key, T, Hash, Pred, TrackingAllocator<std::pair<const Key, T>, kTag>>; - -template<class Key, - class T, - class EmptyFn, - AllocatorTag kTag, - class Hash = std::hash<Key>, - class Pred = std::equal_to<Key>> -using AllocationTrackingHashMap = HashMap< - Key, T, EmptyFn, Hash, Pred, TrackingAllocator<std::pair<Key, T>, kTag>>; } // namespace art #endif // ART_RUNTIME_BASE_ALLOCATOR_H_ diff --git a/runtime/base/bit_utils.h b/runtime/base/bit_utils.h index f536c72bae..0844678b74 100644 --- a/runtime/base/bit_utils.h +++ b/runtime/base/bit_utils.h @@ -17,13 +17,11 @@ #ifndef ART_RUNTIME_BASE_BIT_UTILS_H_ #define ART_RUNTIME_BASE_BIT_UTILS_H_ -#include <iterator> #include <limits> #include <type_traits> -#include "base/iteration_range.h" #include "base/logging.h" -#include "base/stl_util.h" +#include "base/stl_util_identity.h" namespace art { @@ -312,85 +310,6 @@ constexpr T MinInt(size_t bits) { : static_cast<T>(0); } -// Using the Curiously Recurring Template Pattern to implement everything shared -// by LowToHighBitIterator and HighToLowBitIterator, i.e. everything but operator*(). -template <typename T, typename Iter> -class BitIteratorBase - : public std::iterator<std::forward_iterator_tag, uint32_t, ptrdiff_t, void, void> { - static_assert(std::is_integral<T>::value, "T must be integral"); - static_assert(std::is_unsigned<T>::value, "T must be unsigned"); - - static_assert(sizeof(T) == sizeof(uint32_t) || sizeof(T) == sizeof(uint64_t), "Unsupported size"); - - public: - BitIteratorBase() : bits_(0u) { } - explicit BitIteratorBase(T bits) : bits_(bits) { } - - Iter& operator++() { - DCHECK_NE(bits_, 0u); - uint32_t bit = *static_cast<Iter&>(*this); - bits_ &= ~(static_cast<T>(1u) << bit); - return static_cast<Iter&>(*this); - } - - Iter& operator++(int) { - Iter tmp(static_cast<Iter&>(*this)); - ++*this; - return tmp; - } - - protected: - T bits_; - - template <typename U, typename I> - friend bool operator==(const BitIteratorBase<U, I>& lhs, const BitIteratorBase<U, I>& rhs); -}; - -template <typename T, typename Iter> -bool operator==(const BitIteratorBase<T, Iter>& lhs, const BitIteratorBase<T, Iter>& rhs) { - return lhs.bits_ == rhs.bits_; -} - -template <typename T, typename Iter> -bool operator!=(const BitIteratorBase<T, Iter>& lhs, const BitIteratorBase<T, Iter>& rhs) { - return !(lhs == rhs); -} - -template <typename T> -class LowToHighBitIterator : public BitIteratorBase<T, LowToHighBitIterator<T>> { - public: - using BitIteratorBase<T, LowToHighBitIterator<T>>::BitIteratorBase; - - uint32_t operator*() const { - DCHECK_NE(this->bits_, 0u); - return CTZ(this->bits_); - } -}; - -template <typename T> -class HighToLowBitIterator : public BitIteratorBase<T, HighToLowBitIterator<T>> { - public: - using BitIteratorBase<T, HighToLowBitIterator<T>>::BitIteratorBase; - - uint32_t operator*() const { - DCHECK_NE(this->bits_, 0u); - static_assert(std::numeric_limits<T>::radix == 2, "Unexpected radix!"); - return std::numeric_limits<T>::digits - 1u - CLZ(this->bits_); - } -}; - -template <typename T> -IterationRange<LowToHighBitIterator<T>> LowToHighBits(T bits) { - return IterationRange<LowToHighBitIterator<T>>( - LowToHighBitIterator<T>(bits), LowToHighBitIterator<T>()); -} - -template <typename T> -IterationRange<HighToLowBitIterator<T>> HighToLowBits(T bits) { - return IterationRange<HighToLowBitIterator<T>>( - HighToLowBitIterator<T>(bits), HighToLowBitIterator<T>()); -} - // Returns value with bit set in lowest one-bit position or 0 if 0. (java.lang.X.lowestOneBit). template <typename kind> inline static kind LowestOneBitValue(kind opnd) { diff --git a/runtime/base/bit_utils_iterator.h b/runtime/base/bit_utils_iterator.h new file mode 100644 index 0000000000..8514de6b75 --- /dev/null +++ b/runtime/base/bit_utils_iterator.h @@ -0,0 +1,112 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_RUNTIME_BASE_BIT_UTILS_ITERATOR_H_ +#define ART_RUNTIME_BASE_BIT_UTILS_ITERATOR_H_ + +#include <iterator> +#include <limits> +#include <type_traits> + +#include "base/bit_utils.h" +#include "base/iteration_range.h" +#include "base/logging.h" +#include "base/stl_util.h" + +namespace art { + +// Using the Curiously Recurring Template Pattern to implement everything shared +// by LowToHighBitIterator and HighToLowBitIterator, i.e. everything but operator*(). +template <typename T, typename Iter> +class BitIteratorBase + : public std::iterator<std::forward_iterator_tag, uint32_t, ptrdiff_t, void, void> { + static_assert(std::is_integral<T>::value, "T must be integral"); + static_assert(std::is_unsigned<T>::value, "T must be unsigned"); + + static_assert(sizeof(T) == sizeof(uint32_t) || sizeof(T) == sizeof(uint64_t), "Unsupported size"); + + public: + BitIteratorBase() : bits_(0u) { } + explicit BitIteratorBase(T bits) : bits_(bits) { } + + Iter& operator++() { + DCHECK_NE(bits_, 0u); + uint32_t bit = *static_cast<Iter&>(*this); + bits_ &= ~(static_cast<T>(1u) << bit); + return static_cast<Iter&>(*this); + } + + Iter& operator++(int) { + Iter tmp(static_cast<Iter&>(*this)); + ++*this; + return tmp; + } + + protected: + T bits_; + + template <typename U, typename I> + friend bool operator==(const BitIteratorBase<U, I>& lhs, const BitIteratorBase<U, I>& rhs); +}; + +template <typename T, typename Iter> +bool operator==(const BitIteratorBase<T, Iter>& lhs, const BitIteratorBase<T, Iter>& rhs) { + return lhs.bits_ == rhs.bits_; +} + +template <typename T, typename Iter> +bool operator!=(const BitIteratorBase<T, Iter>& lhs, const BitIteratorBase<T, Iter>& rhs) { + return !(lhs == rhs); +} + +template <typename T> +class LowToHighBitIterator : public BitIteratorBase<T, LowToHighBitIterator<T>> { + public: + using BitIteratorBase<T, LowToHighBitIterator<T>>::BitIteratorBase; + + uint32_t operator*() const { + DCHECK_NE(this->bits_, 0u); + return CTZ(this->bits_); + } +}; + +template <typename T> +class HighToLowBitIterator : public BitIteratorBase<T, HighToLowBitIterator<T>> { + public: + using BitIteratorBase<T, HighToLowBitIterator<T>>::BitIteratorBase; + + uint32_t operator*() const { + DCHECK_NE(this->bits_, 0u); + static_assert(std::numeric_limits<T>::radix == 2, "Unexpected radix!"); + return std::numeric_limits<T>::digits - 1u - CLZ(this->bits_); + } +}; + +template <typename T> +IterationRange<LowToHighBitIterator<T>> LowToHighBits(T bits) { + return IterationRange<LowToHighBitIterator<T>>( + LowToHighBitIterator<T>(bits), LowToHighBitIterator<T>()); +} + +template <typename T> +IterationRange<HighToLowBitIterator<T>> HighToLowBits(T bits) { + return IterationRange<HighToLowBitIterator<T>>( + HighToLowBitIterator<T>(bits), HighToLowBitIterator<T>()); +} + +} // namespace art + +#endif // ART_RUNTIME_BASE_BIT_UTILS_ITERATOR_H_ diff --git a/runtime/base/bit_utils_test.cc b/runtime/base/bit_utils_test.cc index 77bd0b815e..9f22fb4670 100644 --- a/runtime/base/bit_utils_test.cc +++ b/runtime/base/bit_utils_test.cc @@ -17,6 +17,7 @@ #include <vector> #include "bit_utils.h" +#include "bit_utils_iterator.h" #include "gtest/gtest.h" diff --git a/runtime/base/callee_save_type.h b/runtime/base/callee_save_type.h new file mode 100644 index 0000000000..501b296d4f --- /dev/null +++ b/runtime/base/callee_save_type.h @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_RUNTIME_BASE_CALLEE_SAVE_TYPE_H_ +#define ART_RUNTIME_BASE_CALLEE_SAVE_TYPE_H_ + +#include <cstddef> +#include <ostream> + +namespace art { + +// Returns a special method that describes all callee saves being spilled to the stack. +enum class CalleeSaveType : uint32_t { + kSaveAllCalleeSaves, // All callee-save registers. + kSaveRefsOnly, // Only those callee-save registers that can hold references. + kSaveRefsAndArgs, // References (see above) and arguments (usually caller-save registers). + kSaveEverything, // All registers, including both callee-save and caller-save. + kLastCalleeSaveType // Value used for iteration. +}; +std::ostream& operator<<(std::ostream& os, const CalleeSaveType& rhs); + +} // namespace art + +#endif // ART_RUNTIME_BASE_CALLEE_SAVE_TYPE_H_ diff --git a/runtime/base/stl_util.h b/runtime/base/stl_util.h index cfe27f3811..b27297241d 100644 --- a/runtime/base/stl_util.h +++ b/runtime/base/stl_util.h @@ -25,13 +25,6 @@ namespace art { -// Sort and remove duplicates of an STL vector or deque. -template<class T> -void STLSortAndRemoveDuplicates(T* v) { - std::sort(v->begin(), v->end()); - v->erase(std::unique(v->begin(), v->end()), v->end()); -} - // STLDeleteContainerPointers() // For a range within a container of pointers, calls delete // (non-array version) on these pointers. @@ -83,20 +76,6 @@ void STLDeleteValues(T *v) { } } -template <class T> -std::string ToString(const T& v) { - std::ostringstream os; - os << "["; - for (size_t i = 0; i < v.size(); ++i) { - os << v[i]; - if (i < v.size() - 1) { - os << ", "; - } - } - os << "]"; - return os.str(); -} - // Deleter using free() for use with std::unique_ptr<>. See also UniqueCPtr<> below. struct FreeDelete { // NOTE: Deleting a const object is valid but free() takes a non-const pointer. @@ -109,13 +88,6 @@ struct FreeDelete { template <typename T> using UniqueCPtr = std::unique_ptr<T, FreeDelete>; -// C++14 from-the-future import (std::make_unique) -// Invoke the constructor of 'T' with the provided args, and wrap the result in a unique ptr. -template <typename T, typename ... Args> -std::unique_ptr<T> MakeUnique(Args&& ... args) { - return std::unique_ptr<T>(new T(std::forward<Args>(args)...)); -} - // Find index of the first element with the specified value known to be in the container. template <typename Container, typename T> size_t IndexOfElement(const Container& container, const T& value) { @@ -150,13 +122,6 @@ bool ContainsElement(const Container& container, const T& value, size_t start_po return it != container.end(); } -// const char* compare function suitable for std::map or std::set. -struct CStringLess { - bool operator()(const char* lhs, const char* rhs) const { - return strcmp(lhs, rhs) < 0; - } -}; - // 32-bit FNV-1a hash function suitable for std::unordered_map. // It can be used with any container which works with range-based for loop. // See http://en.wikipedia.org/wiki/Fowler%E2%80%93Noll%E2%80%93Vo_hash_function @@ -171,23 +136,6 @@ struct FNVHash { } }; -// Use to suppress type deduction for a function argument. -// See std::identity<> for more background: -// http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2005/n1856.html#20.2.2 - move/forward helpers -// -// e.g. "template <typename X> void bar(identity<X>::type foo); -// bar(5); // compilation error -// bar<int>(5); // ok -// or "template <typename T> void foo(T* x, typename Identity<T*>::type y); -// Base b; -// Derived d; -// foo(&b, &d); // Use implicit Derived* -> Base* conversion. -// If T was deduced from both &b and &d, there would be a mismatch, i.e. deduction failure. -template <typename T> -struct Identity { - using type = T; -}; - // Merge `other` entries into `to_update`. template <typename T> static inline void MergeSets(std::set<T>& to_update, const std::set<T>& other) { diff --git a/runtime/base/stl_util_identity.h b/runtime/base/stl_util_identity.h new file mode 100644 index 0000000000..40a93f79c5 --- /dev/null +++ b/runtime/base/stl_util_identity.h @@ -0,0 +1,41 @@ +/* + * Copyright (C) 2011 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_RUNTIME_BASE_STL_UTIL_IDENTITY_H_ +#define ART_RUNTIME_BASE_STL_UTIL_IDENTITY_H_ + +namespace art { + +// Use to suppress type deduction for a function argument. +// See std::identity<> for more background: +// http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2005/n1856.html#20.2.2 - move/forward helpers +// +// e.g. "template <typename X> void bar(identity<X>::type foo); +// bar(5); // compilation error +// bar<int>(5); // ok +// or "template <typename T> void foo(T* x, typename Identity<T*>::type y); +// Base b; +// Derived d; +// foo(&b, &d); // Use implicit Derived* -> Base* conversion. +// If T was deduced from both &b and &d, there would be a mismatch, i.e. deduction failure. +template <typename T> +struct Identity { + using type = T; +}; + +} // namespace art + +#endif // ART_RUNTIME_BASE_STL_UTIL_IDENTITY_H_ diff --git a/runtime/base/strlcpy.h b/runtime/base/strlcpy.h new file mode 100644 index 0000000000..de135ea990 --- /dev/null +++ b/runtime/base/strlcpy.h @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_RUNTIME_BASE_STRLCPY_H_ +#define ART_RUNTIME_BASE_STRLCPY_H_ + +#include <cstdio> +#include <cstring> + +// Expose a simple implementation of strlcpy when we're not compiling against bionic. This is to +// make static analyzers happy not using strcpy. +// +// Bionic exposes this function, but the host glibc does not. Remove this shim when we compile +// against bionic on the host, also. + +#if !defined(__BIONIC__) && !defined(__APPLE__) + +static inline size_t strlcpy(char* dst, const char* src, size_t size) { + // Extra-lazy implementation: this is only a host shim, and we don't have to call this often. + return snprintf(dst, size, "%s", src); +} + +#endif + +#endif // ART_RUNTIME_BASE_STRLCPY_H_ diff --git a/runtime/base/variant_map.h b/runtime/base/variant_map.h index 531cb37355..fdb60c4141 100644 --- a/runtime/base/variant_map.h +++ b/runtime/base/variant_map.h @@ -22,7 +22,7 @@ #include <type_traits> #include <utility> -#include "base/stl_util.h" +#include "base/stl_util_identity.h" namespace art { diff --git a/runtime/class_linker.cc b/runtime/class_linker.cc index a81c832450..0fa25d15d2 100644 --- a/runtime/class_linker.cc +++ b/runtime/class_linker.cc @@ -55,7 +55,6 @@ #include "gc_root-inl.h" #include "gc/accounting/card_table-inl.h" #include "gc/accounting/heap_bitmap-inl.h" -#include "gc/accounting/space_bitmap-inl.h" #include "gc/heap.h" #include "gc/scoped_gc_critical_section.h" #include "gc/space/image_space.h" @@ -89,7 +88,6 @@ #include "mirror/method_handles_lookup.h" #include "mirror/object-inl.h" #include "mirror/object_array-inl.h" -#include "mirror/object-refvisitor-inl.h" #include "mirror/proxy.h" #include "mirror/reference-inl.h" #include "mirror/stack_trace_element.h" @@ -1195,63 +1193,6 @@ class VerifyDeclaringClassVisitor : public ArtMethodVisitor { gc::accounting::HeapBitmap* const live_bitmap_; }; -class FixupInternVisitor { - public: - ALWAYS_INLINE ObjPtr<mirror::Object> TryInsertIntern(mirror::Object* obj) const - REQUIRES_SHARED(Locks::mutator_lock_) { - if (obj != nullptr && obj->IsString()) { - const auto intern = Runtime::Current()->GetInternTable()->InternStrong(obj->AsString()); - return intern; - } - return obj; - } - - ALWAYS_INLINE void VisitRootIfNonNull( - mirror::CompressedReference<mirror::Object>* root) const - REQUIRES_SHARED(Locks::mutator_lock_) { - if (!root->IsNull()) { - VisitRoot(root); - } - } - - ALWAYS_INLINE void VisitRoot(mirror::CompressedReference<mirror::Object>* root) const - REQUIRES_SHARED(Locks::mutator_lock_) { - root->Assign(TryInsertIntern(root->AsMirrorPtr())); - } - - // Visit Class Fields - ALWAYS_INLINE void operator()(ObjPtr<mirror::Object> obj, - MemberOffset offset, - bool is_static ATTRIBUTE_UNUSED) const - REQUIRES_SHARED(Locks::mutator_lock_) { - // There could be overlap between ranges, we must avoid visiting the same reference twice. - // Avoid the class field since we already fixed it up in FixupClassVisitor. - if (offset.Uint32Value() != mirror::Object::ClassOffset().Uint32Value()) { - // Updating images, don't do a read barrier. - // Only string fields are fixed, don't do a verify. - mirror::Object* ref = obj->GetFieldObject<mirror::Object, kVerifyNone, kWithoutReadBarrier>( - offset); - obj->SetFieldObject<false, false>(offset, TryInsertIntern(ref)); - } - } - - void operator()(ObjPtr<mirror::Class> klass ATTRIBUTE_UNUSED, - ObjPtr<mirror::Reference> ref) const - REQUIRES_SHARED(Locks::mutator_lock_) REQUIRES(Locks::heap_bitmap_lock_) { - this->operator()(ref, mirror::Reference::ReferentOffset(), false); - } - - void operator()(mirror::Object* obj) const - REQUIRES_SHARED(Locks::mutator_lock_) { - if (obj->IsDexCache()) { - obj->VisitReferences<true, kVerifyNone, kWithoutReadBarrier>(*this, *this); - } else { - // Don't visit native roots for non-dex-cache - obj->VisitReferences<false, kVerifyNone, kWithoutReadBarrier>(*this, *this); - } - } -}; - // Copies data from one array to another array at the same position // if pred returns false. If there is a page of continuous data in // the src array for which pred consistently returns true then @@ -1344,7 +1285,6 @@ bool AppImageClassLoadersAndDexCachesHelper::Update( return false; } } - // Only add the classes to the class loader after the points where we can return false. for (size_t i = 0; i < num_dex_caches; i++) { ObjPtr<mirror::DexCache> dex_cache = dex_caches->Get(i); @@ -1508,21 +1448,6 @@ bool AppImageClassLoadersAndDexCachesHelper::Update( } } } - { - // Fixup all the literal strings happens at app images which are supposed to be interned. - ScopedTrace timing("Fixup String Intern in image and dex_cache"); - const auto& image_header = space->GetImageHeader(); - const auto bitmap = space->GetMarkBitmap(); // bitmap of objects - const uint8_t* target_base = space->GetMemMap()->Begin(); - const ImageSection& objects_section = - image_header.GetImageSection(ImageHeader::kSectionObjects); - - uintptr_t objects_begin = reinterpret_cast<uintptr_t>(target_base + objects_section.Offset()); - uintptr_t objects_end = reinterpret_cast<uintptr_t>(target_base + objects_section.End()); - - FixupInternVisitor fixup_intern_visitor; - bitmap->VisitMarkedRange(objects_begin, objects_end, fixup_intern_visitor); - } if (*out_forward_dex_cache_array) { ScopedTrace timing("Fixup ArtMethod dex cache arrays"); FixupArtMethodArrayVisitor visitor(header); @@ -3492,8 +3417,11 @@ void ClassLinker::RegisterDexFileLocked(const DexFile& dex_file, // Example dex_cache location is SettingsProvider.apk and // dex file location is /system/priv-app/SettingsProvider/SettingsProvider.apk CHECK_EQ(dex_cache_location, dex_file_suffix); - // Clean up pass to remove null dex caches. + const OatFile* oat_file = + (dex_file.GetOatDexFile() != nullptr) ? dex_file.GetOatDexFile()->GetOatFile() : nullptr; + // Clean up pass to remove null dex caches. Also check if we need to initialize OatFile .bss. // Null dex caches can occur due to class unloading and we are lazily removing null entries. + bool initialize_oat_file_bss = (oat_file != nullptr); JavaVMExt* const vm = self->GetJniEnv()->vm; for (auto it = dex_caches_.begin(); it != dex_caches_.end(); ) { DexCacheData data = *it; @@ -3501,9 +3429,21 @@ void ClassLinker::RegisterDexFileLocked(const DexFile& dex_file, vm->DeleteWeakGlobalRef(self, data.weak_root); it = dex_caches_.erase(it); } else { + if (initialize_oat_file_bss && + it->dex_file->GetOatDexFile() != nullptr && + it->dex_file->GetOatDexFile()->GetOatFile() == oat_file) { + initialize_oat_file_bss = false; // Already initialized. + } ++it; } } + if (initialize_oat_file_bss) { + // TODO: Pre-initialize from boot/app image? + ArtMethod* resolution_method = Runtime::Current()->GetResolutionMethod(); + for (ArtMethod*& entry : oat_file->GetBssMethods()) { + entry = resolution_method; + } + } jweak dex_cache_jweak = vm->AddWeakGlobalRef(self, dex_cache); dex_cache->SetDexFile(&dex_file); DexCacheData data; diff --git a/runtime/class_table.cc b/runtime/class_table.cc index b71610ac7f..c45bbe5334 100644 --- a/runtime/class_table.cc +++ b/runtime/class_table.cc @@ -16,6 +16,7 @@ #include "class_table-inl.h" +#include "base/stl_util.h" #include "mirror/class-inl.h" #include "oat_file.h" diff --git a/runtime/common_runtime_test.cc b/runtime/common_runtime_test.cc index 01c6641ae9..f9259944b4 100644 --- a/runtime/common_runtime_test.cc +++ b/runtime/common_runtime_test.cc @@ -728,13 +728,3 @@ void CheckJniAbortCatcher::Hook(void* data, const std::string& reason) { } } // namespace art - -namespace std { - -template <typename T> -std::ostream& operator<<(std::ostream& os, const std::vector<T>& rhs) { -os << ::art::ToString(rhs); -return os; -} - -} // namespace std diff --git a/runtime/common_runtime_test.h b/runtime/common_runtime_test.h index 24dbd058ec..1274a3623b 100644 --- a/runtime/common_runtime_test.h +++ b/runtime/common_runtime_test.h @@ -258,12 +258,4 @@ class CheckJniAbortCatcher { } // namespace art -namespace std { - -// TODO: isn't gtest supposed to be able to print STL types for itself? -template <typename T> -std::ostream& operator<<(std::ostream& os, const std::vector<T>& rhs); - -} // namespace std - #endif // ART_RUNTIME_COMMON_RUNTIME_TEST_H_ diff --git a/runtime/debugger.cc b/runtime/debugger.cc index cfa56a5769..7e70b7564c 100644 --- a/runtime/debugger.cc +++ b/runtime/debugger.cc @@ -149,7 +149,9 @@ class DebugInstrumentationListener FINAL : public instrumentation::Instrumentati DebugInstrumentationListener() {} virtual ~DebugInstrumentationListener() {} - void MethodEntered(Thread* thread, mirror::Object* this_object, ArtMethod* method, + void MethodEntered(Thread* thread, + Handle<mirror::Object> this_object, + ArtMethod* method, uint32_t dex_pc) OVERRIDE REQUIRES_SHARED(Locks::mutator_lock_) { if (method->IsNative()) { @@ -171,12 +173,15 @@ class DebugInstrumentationListener FINAL : public instrumentation::Instrumentati // also group it with other events for this location like BREAKPOINT or SINGLE_STEP. thread->SetDebugMethodEntry(); } else { - Dbg::UpdateDebugger(thread, this_object, method, 0, Dbg::kMethodEntry, nullptr); + Dbg::UpdateDebugger(thread, this_object.Get(), method, 0, Dbg::kMethodEntry, nullptr); } } - void MethodExited(Thread* thread, mirror::Object* this_object, ArtMethod* method, - uint32_t dex_pc, const JValue& return_value) + void MethodExited(Thread* thread, + Handle<mirror::Object> this_object, + ArtMethod* method, + uint32_t dex_pc, + const JValue& return_value) OVERRIDE REQUIRES_SHARED(Locks::mutator_lock_) { if (method->IsNative()) { // TODO: post location events is a suspension point and native method entry stubs aren't. @@ -189,18 +194,22 @@ class DebugInstrumentationListener FINAL : public instrumentation::Instrumentati events |= Dbg::kMethodEntry; thread->ClearDebugMethodEntry(); } - Dbg::UpdateDebugger(thread, this_object, method, dex_pc, events, &return_value); + Dbg::UpdateDebugger(thread, this_object.Get(), method, dex_pc, events, &return_value); } - void MethodUnwind(Thread* thread ATTRIBUTE_UNUSED, mirror::Object* this_object ATTRIBUTE_UNUSED, - ArtMethod* method, uint32_t dex_pc) + void MethodUnwind(Thread* thread ATTRIBUTE_UNUSED, + Handle<mirror::Object> this_object ATTRIBUTE_UNUSED, + ArtMethod* method, + uint32_t dex_pc) OVERRIDE REQUIRES_SHARED(Locks::mutator_lock_) { // We're not recorded to listen to this kind of event, so complain. LOG(ERROR) << "Unexpected method unwind event in debugger " << ArtMethod::PrettyMethod(method) << " " << dex_pc; } - void DexPcMoved(Thread* thread, mirror::Object* this_object, ArtMethod* method, + void DexPcMoved(Thread* thread, + Handle<mirror::Object> this_object, + ArtMethod* method, uint32_t new_dex_pc) OVERRIDE REQUIRES_SHARED(Locks::mutator_lock_) { if (IsListeningToMethodExit() && IsReturn(method, new_dex_pc)) { @@ -217,26 +226,33 @@ class DebugInstrumentationListener FINAL : public instrumentation::Instrumentati events = Dbg::kMethodEntry; thread->ClearDebugMethodEntry(); } - Dbg::UpdateDebugger(thread, this_object, method, new_dex_pc, events, nullptr); + Dbg::UpdateDebugger(thread, this_object.Get(), method, new_dex_pc, events, nullptr); } } - void FieldRead(Thread* thread ATTRIBUTE_UNUSED, mirror::Object* this_object, - ArtMethod* method, uint32_t dex_pc, ArtField* field) + void FieldRead(Thread* thread ATTRIBUTE_UNUSED, + Handle<mirror::Object> this_object, + ArtMethod* method, + uint32_t dex_pc, + ArtField* field) OVERRIDE REQUIRES_SHARED(Locks::mutator_lock_) { - Dbg::PostFieldAccessEvent(method, dex_pc, this_object, field); + Dbg::PostFieldAccessEvent(method, dex_pc, this_object.Get(), field); } - void FieldWritten(Thread* thread ATTRIBUTE_UNUSED, mirror::Object* this_object, - ArtMethod* method, uint32_t dex_pc, ArtField* field, + void FieldWritten(Thread* thread ATTRIBUTE_UNUSED, + Handle<mirror::Object> this_object, + ArtMethod* method, + uint32_t dex_pc, + ArtField* field, const JValue& field_value) OVERRIDE REQUIRES_SHARED(Locks::mutator_lock_) { - Dbg::PostFieldModificationEvent(method, dex_pc, this_object, field, &field_value); + Dbg::PostFieldModificationEvent(method, dex_pc, this_object.Get(), field, &field_value); } - void ExceptionCaught(Thread* thread ATTRIBUTE_UNUSED, mirror::Throwable* exception_object) + void ExceptionCaught(Thread* thread ATTRIBUTE_UNUSED, + Handle<mirror::Throwable> exception_object) OVERRIDE REQUIRES_SHARED(Locks::mutator_lock_) { - Dbg::PostException(exception_object); + Dbg::PostException(exception_object.Get()); } // We only care about branches in the Jit. @@ -248,10 +264,10 @@ class DebugInstrumentationListener FINAL : public instrumentation::Instrumentati // We only care about invokes in the Jit. void InvokeVirtualOrInterface(Thread* thread ATTRIBUTE_UNUSED, - mirror::Object*, + Handle<mirror::Object> this_object ATTRIBUTE_UNUSED, ArtMethod* method, uint32_t dex_pc, - ArtMethod*) + ArtMethod* target ATTRIBUTE_UNUSED) OVERRIDE REQUIRES_SHARED(Locks::mutator_lock_) { LOG(ERROR) << "Unexpected invoke event in debugger " << ArtMethod::PrettyMethod(method) << " " << dex_pc; diff --git a/runtime/dex2oat_environment_test.h b/runtime/dex2oat_environment_test.h index 6765407949..93daa45519 100644 --- a/runtime/dex2oat_environment_test.h +++ b/runtime/dex2oat_environment_test.h @@ -23,6 +23,7 @@ #include <gtest/gtest.h> +#include "base/stl_util.h" #include "common_runtime_test.h" #include "compiler_callbacks.h" #include "exec_utils.h" diff --git a/runtime/dex_file.cc b/runtime/dex_file.cc index aace8eba43..b267e5f22a 100644 --- a/runtime/dex_file.cc +++ b/runtime/dex_file.cc @@ -35,6 +35,7 @@ #include "base/enums.h" #include "base/file_magic.h" #include "base/logging.h" +#include "base/stl_util.h" #include "base/systrace.h" #include "base/unix_file/fd_file.h" #include "dex_file-inl.h" @@ -678,6 +679,32 @@ uint32_t DexFile::FindCodeItemOffset(const DexFile::ClassDef& class_def, UNREACHABLE(); } +uint32_t DexFile::GetCodeItemSize(const DexFile::CodeItem& code_item) { + uintptr_t code_item_start = reinterpret_cast<uintptr_t>(&code_item); + uint32_t insns_size = code_item.insns_size_in_code_units_; + uint32_t tries_size = code_item.tries_size_; + const uint8_t* handler_data = GetCatchHandlerData(code_item, 0); + + if (tries_size == 0 || handler_data == nullptr) { + uintptr_t insns_end = reinterpret_cast<uintptr_t>(&code_item.insns_[insns_size]); + return insns_end - code_item_start; + } else { + // Get the start of the handler data. + uint32_t handlers_size = DecodeUnsignedLeb128(&handler_data); + // Manually read each handler. + for (uint32_t i = 0; i < handlers_size; ++i) { + int32_t uleb128_count = DecodeSignedLeb128(&handler_data) * 2; + if (uleb128_count <= 0) { + uleb128_count = -uleb128_count + 1; + } + for (int32_t j = 0; j < uleb128_count; ++j) { + DecodeUnsignedLeb128(&handler_data); + } + } + return reinterpret_cast<uintptr_t>(handler_data) - code_item_start; + } +} + const DexFile::FieldId* DexFile::FindFieldId(const DexFile::TypeId& declaring_klass, const DexFile::StringId& name, const DexFile::TypeId& type) const { diff --git a/runtime/dex_file.h b/runtime/dex_file.h index 3de78ed5a3..3249edbe83 100644 --- a/runtime/dex_file.h +++ b/runtime/dex_file.h @@ -636,6 +636,8 @@ class DexFile { uint32_t FindCodeItemOffset(const DexFile::ClassDef& class_def, uint32_t dex_method_idx) const; + static uint32_t GetCodeItemSize(const DexFile::CodeItem& disk_code_item); + // Returns the declaring class descriptor string of a field id. const char* GetFieldDeclaringClassDescriptor(const FieldId& field_id) const { const DexFile::TypeId& type_id = GetTypeId(field_id.class_idx_); diff --git a/runtime/dex_file_tracking_registrar.cc b/runtime/dex_file_tracking_registrar.cc index cfbca3d00f..f41a50bb80 100644 --- a/runtime/dex_file_tracking_registrar.cc +++ b/runtime/dex_file_tracking_registrar.cc @@ -16,6 +16,9 @@ #include "dex_file_tracking_registrar.h" +#include <deque> +#include <tuple> + // For dex tracking through poisoning. Note: Requires forcing sanitization. This is the reason for // the ifdefs and early include. #ifdef ART_DEX_FILE_ACCESS_TRACKING @@ -26,6 +29,7 @@ #include "base/memory_tool.h" #include "base/logging.h" +#include "dex_file-inl.h" namespace art { namespace dex { @@ -39,11 +43,161 @@ static constexpr bool kDexFileAccessTracking = false; #endif -void RegisterDexFile(const DexFile* const dex_file) { - if (kDexFileAccessTracking && dex_file != nullptr) { - LOG(ERROR) << dex_file->GetLocation() + " @ " << std::hex - << reinterpret_cast<uintptr_t>(dex_file->Begin()); - MEMORY_TOOL_MAKE_NOACCESS(dex_file->Begin(), dex_file->Size()); +// The following are configurations of poisoning certain sections of a Dex File. +// More will be added +enum DexTrackingType { + // Poisons all of a Dex File when set. + kWholeDexTracking, + // Poisons all Code Items of a Dex File when set. + kCodeItemTracking, + // Poisons all subsections of a Code Item, except the Insns bytecode array + // section, when set for all Code Items in a Dex File. + kCodeItemNonInsnsTracking, + // Poisons all subsections of a Code Item, except the Insns bytecode array + // section, when set for all Code Items in a Dex File. + // Additionally unpoisons the entire Code Item when method is a class + // initializer. + kCodeItemNonInsnsNoClinitTracking, + // Poisons based on a custom tracking system which can be specified in + // SetDexSections + kCustomTracking, +}; + +// Intended for local changes only. +// Represents the current configuration being run. +static constexpr DexTrackingType kCurrentTrackingSystem = kWholeDexTracking; + +// Intended for local changes only. +void DexFileTrackingRegistrar::SetDexSections() { + if (kDexFileAccessTracking || dex_file_ != nullptr) { + switch (kCurrentTrackingSystem) { + case kWholeDexTracking: + SetDexFileRegistration(true); + break; + case kCodeItemTracking: + SetAllCodeItemRegistration(true); + break; + case kCodeItemNonInsnsTracking: + SetAllCodeItemRegistration(true); + SetAllInsnsRegistration(false); + break; + case kCodeItemNonInsnsNoClinitTracking: + SetAllCodeItemRegistration(true); + SetAllInsnsRegistration(false); + SetCodeItemRegistration("<clinit>", false); + break; + case kCustomTracking: + // TODO: Add/remove additional calls here to (un)poison sections of + // dex_file_ + break; + } + } +} + +void RegisterDexFile(const DexFile* dex_file) { + DexFileTrackingRegistrar dex_tracking_registrar(dex_file); + dex_tracking_registrar.SetDexSections(); + dex_tracking_registrar.SetCurrentRanges(); +} + +inline void SetRegistrationRange(const void* begin, size_t size, bool should_poison) { + if (should_poison) { + MEMORY_TOOL_MAKE_NOACCESS(begin, size); + } else { + // Note: MEMORY_TOOL_MAKE_UNDEFINED has the same functionality with Address + // Sanitizer. The difference has not been tested with Valgrind + MEMORY_TOOL_MAKE_DEFINED(begin, size); + } +} + +void DexFileTrackingRegistrar::SetCurrentRanges() { + // This also empties range_values_ to avoid redundant (un)poisoning upon + // subsequent calls. + while (!range_values_.empty()) { + const std::tuple<const void*, size_t, bool>& current_range = range_values_.front(); + SetRegistrationRange(std::get<0>(current_range), + std::get<1>(current_range), + std::get<2>(current_range)); + range_values_.pop_front(); + } +} + +void DexFileTrackingRegistrar::SetDexFileRegistration(bool should_poison) { + const void* dex_file_begin = reinterpret_cast<const void*>(dex_file_->Begin()); + size_t dex_file_size = dex_file_->Size(); + range_values_.push_back(std::make_tuple(dex_file_begin, dex_file_size, should_poison)); +} + +void DexFileTrackingRegistrar::SetAllCodeItemRegistration(bool should_poison) { + for (size_t classdef_ctr = 0; classdef_ctr < dex_file_->NumClassDefs(); ++classdef_ctr) { + const DexFile::ClassDef& cd = dex_file_->GetClassDef(classdef_ctr); + const uint8_t* class_data = dex_file_->GetClassData(cd); + if (class_data != nullptr) { + ClassDataItemIterator cdit(*dex_file_, class_data); + // Skipping Fields + while (cdit.HasNextStaticField() || cdit.HasNextInstanceField()) { + cdit.Next(); + } + while (cdit.HasNextDirectMethod()) { + const DexFile::CodeItem* code_item = cdit.GetMethodCodeItem(); + if (code_item != nullptr) { + const void* code_item_begin = reinterpret_cast<const void*>(code_item); + size_t code_item_size = DexFile::GetCodeItemSize(*code_item); + range_values_.push_back(std::make_tuple(code_item_begin, code_item_size, should_poison)); + } + cdit.Next(); + } + } + } +} + +void DexFileTrackingRegistrar::SetAllInsnsRegistration(bool should_poison) { + for (size_t classdef_ctr = 0; classdef_ctr < dex_file_->NumClassDefs(); ++classdef_ctr) { + const DexFile::ClassDef& cd = dex_file_->GetClassDef(classdef_ctr); + const uint8_t* class_data = dex_file_->GetClassData(cd); + if (class_data != nullptr) { + ClassDataItemIterator cdit(*dex_file_, class_data); + // Skipping Fields + while (cdit.HasNextStaticField() || cdit.HasNextInstanceField()) { + cdit.Next(); + } + while (cdit.HasNextDirectMethod()) { + const DexFile::CodeItem* code_item = cdit.GetMethodCodeItem(); + if (code_item != nullptr) { + const void* insns_begin = reinterpret_cast<const void*>(&code_item->insns_); + // Member insns_size_in_code_units_ is in 2-byte units + size_t insns_size = code_item->insns_size_in_code_units_ * 2; + range_values_.push_back(std::make_tuple(insns_begin, insns_size, should_poison)); + } + cdit.Next(); + } + } + } +} + +void DexFileTrackingRegistrar::SetCodeItemRegistration(const char* class_name, bool should_poison) { + for (size_t classdef_ctr = 0; classdef_ctr < dex_file_->NumClassDefs(); ++classdef_ctr) { + const DexFile::ClassDef& cd = dex_file_->GetClassDef(classdef_ctr); + const uint8_t* class_data = dex_file_->GetClassData(cd); + if (class_data != nullptr) { + ClassDataItemIterator cdit(*dex_file_, class_data); + // Skipping Fields + while (cdit.HasNextStaticField() || cdit.HasNextInstanceField()) { + cdit.Next(); + } + while (cdit.HasNextDirectMethod()) { + const DexFile::MethodId& methodid_item = dex_file_->GetMethodId(cdit.GetMemberIndex()); + const char * methodid_name = dex_file_->GetMethodName(methodid_item); + const DexFile::CodeItem* code_item = cdit.GetMethodCodeItem(); + if (code_item != nullptr && strcmp(methodid_name, class_name) == 0) { + const void* code_item_begin = reinterpret_cast<const void*>(code_item); + size_t code_item_size = DexFile::GetCodeItemSize(*code_item); + range_values_.push_back( + std::make_tuple(code_item_begin, code_item_size, should_poison)); + } + cdit.Next(); + } + } } } diff --git a/runtime/dex_file_tracking_registrar.h b/runtime/dex_file_tracking_registrar.h index 7d5d78d6df..b0fa275b38 100644 --- a/runtime/dex_file_tracking_registrar.h +++ b/runtime/dex_file_tracking_registrar.h @@ -17,13 +17,53 @@ #ifndef ART_RUNTIME_DEX_FILE_TRACKING_REGISTRAR_H_ #define ART_RUNTIME_DEX_FILE_TRACKING_REGISTRAR_H_ +#include <deque> +#include <tuple> + #include "dex_file.h" namespace art { namespace dex { namespace tracking { -void RegisterDexFile(const DexFile* const dex_file); +// Class for (un)poisoning various sections of Dex Files +// +// This class provides the means to log accesses only of sections whose +// accesses are needed. All accesses are displayed as stack traces in +// logcat. +class DexFileTrackingRegistrar { + public: + explicit DexFileTrackingRegistrar(const DexFile* const dex_file) + : dex_file_(dex_file) { + } + + // This function is where the functions below it are called to actually + // poison sections. + void SetDexSections(); + + // Uses data contained inside range_values_ to poison memory through the + // memory tool. + void SetCurrentRanges(); + + private: + void SetDexFileRegistration(bool should_poison); + + // Set of functions concerning Code Items of dex_file_ + void SetAllCodeItemRegistration(bool should_poison); + // Sets the insns_ section of all code items. + void SetAllInsnsRegistration(bool should_poison); + // This function finds the code item of a class based on class name. + void SetCodeItemRegistration(const char* class_name, bool should_poison); + + // Contains tuples of all ranges of memory that need to be explicitly + // (un)poisoned by the memory tool. + std::deque<std::tuple<const void *, size_t, bool>> range_values_; + + const DexFile* const dex_file_; +}; + +// This function is meant to called externally to use DexfileTrackingRegistrar +void RegisterDexFile(const DexFile* dex_file); } // namespace tracking } // namespace dex diff --git a/runtime/dex_file_verifier.h b/runtime/dex_file_verifier.h index d1043c6841..74f82254b3 100644 --- a/runtime/dex_file_verifier.h +++ b/runtime/dex_file_verifier.h @@ -19,6 +19,8 @@ #include <unordered_set> +#include "base/allocator.h" +#include "base/hash_map.h" #include "dex_file.h" #include "dex_file_types.h" #include "safe_map.h" @@ -226,6 +228,15 @@ class DexFileVerifier { } }; // Map from offset to dex file type, HashMap for performance reasons. + template<class Key, + class T, + class EmptyFn, + AllocatorTag kTag, + class Hash = std::hash<Key>, + class Pred = std::equal_to<Key>> + using AllocationTrackingHashMap = HashMap< + Key, T, EmptyFn, Hash, Pred, TrackingAllocator<std::pair<Key, T>, kTag>>; + AllocationTrackingHashMap<uint32_t, uint16_t, OffsetTypeMapEmptyFn, diff --git a/runtime/dex_reference_collection.h b/runtime/dex_reference_collection.h new file mode 100644 index 0000000000..76355d6335 --- /dev/null +++ b/runtime/dex_reference_collection.h @@ -0,0 +1,84 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_RUNTIME_DEX_REFERENCE_COLLECTION_H_ +#define ART_RUNTIME_DEX_REFERENCE_COLLECTION_H_ + +#include "base/macros.h" + +#include <vector> +#include <map> + +namespace art { + +class DexFile; + +// Collection of dex references that is more memory efficient than a vector of <dex, index> pairs. +// Also allows quick lookups of all of the references for a single dex. +template <class IndexType, template<typename Type> class Allocator> +class DexReferenceCollection { + public: + using VectorAllocator = Allocator<IndexType>; + using IndexVector = std::vector<IndexType, VectorAllocator>; + using MapAllocator = Allocator<std::pair<const DexFile*, IndexVector>>; + using DexFileMap = std::map< + const DexFile*, + IndexVector, + std::less<const DexFile*>, + Allocator<std::pair<const DexFile* const, IndexVector>>>; + + DexReferenceCollection(const MapAllocator& map_allocator = MapAllocator(), + const VectorAllocator& vector_allocator = VectorAllocator()) + : map_(map_allocator), + vector_allocator_(vector_allocator) {} + + void AddReference(const DexFile* dex, IndexType index) { + GetOrInsertVector(dex)->push_back(index); + } + + DexFileMap& GetMap() { + return map_; + } + + size_t NumReferences() const { + size_t ret = 0; + for (auto&& pair : map_) { + ret += pair.second.size(); + } + return ret; + } + + private: + DexFileMap map_; + // Optimize for adding to same vector in succession. + const DexFile* current_dex_file_ = nullptr; + IndexVector* current_vector_ = nullptr; + VectorAllocator vector_allocator_; + + ALWAYS_INLINE IndexVector* GetOrInsertVector(const DexFile* dex) { + if (UNLIKELY(current_dex_file_ != dex)) { + // There is an assumption that constructing an empty vector wont do any allocations. If this + // incorrect, this might leak for the arena case. + current_vector_ = &map_.emplace(dex, IndexVector(vector_allocator_)).first->second; + current_dex_file_ = dex; + } + return current_vector_; + } +}; + +} // namespace art + +#endif // ART_RUNTIME_DEX_REFERENCE_COLLECTION_H_ diff --git a/runtime/entrypoints/entrypoint_utils.cc b/runtime/entrypoints/entrypoint_utils.cc index c340a885c3..88a5a13246 100644 --- a/runtime/entrypoints/entrypoint_utils.cc +++ b/runtime/entrypoints/entrypoint_utils.cc @@ -177,7 +177,7 @@ bool FillArrayData(ObjPtr<mirror::Object> obj, const Instruction::ArrayDataPaylo } static inline std::pair<ArtMethod*, uintptr_t> DoGetCalleeSaveMethodOuterCallerAndPc( - ArtMethod** sp, Runtime::CalleeSaveType type) REQUIRES_SHARED(Locks::mutator_lock_) { + ArtMethod** sp, CalleeSaveType type) REQUIRES_SHARED(Locks::mutator_lock_) { DCHECK_EQ(*sp, Runtime::Current()->GetCalleeSaveMethod(type)); const size_t callee_frame_size = GetCalleeSaveFrameSize(kRuntimeISA, type); @@ -232,9 +232,7 @@ static inline ArtMethod* DoGetCalleeSaveMethodCaller(ArtMethod* outer_method, return caller; } -ArtMethod* GetCalleeSaveMethodCaller(ArtMethod** sp, - Runtime::CalleeSaveType type, - bool do_caller_check) +ArtMethod* GetCalleeSaveMethodCaller(ArtMethod** sp, CalleeSaveType type, bool do_caller_check) REQUIRES_SHARED(Locks::mutator_lock_) { ScopedAssertNoThreadSuspension ants(__FUNCTION__); auto outer_caller_and_pc = DoGetCalleeSaveMethodOuterCallerAndPc(sp, type); @@ -244,8 +242,7 @@ ArtMethod* GetCalleeSaveMethodCaller(ArtMethod** sp, return caller; } -CallerAndOuterMethod GetCalleeSaveMethodCallerAndOuterMethod(Thread* self, - Runtime::CalleeSaveType type) { +CallerAndOuterMethod GetCalleeSaveMethodCallerAndOuterMethod(Thread* self, CalleeSaveType type) { CallerAndOuterMethod result; ScopedAssertNoThreadSuspension ants(__FUNCTION__); ArtMethod** sp = self->GetManagedStack()->GetTopQuickFrame(); @@ -257,7 +254,7 @@ CallerAndOuterMethod GetCalleeSaveMethodCallerAndOuterMethod(Thread* self, return result; } -ArtMethod* GetCalleeSaveOuterMethod(Thread* self, Runtime::CalleeSaveType type) { +ArtMethod* GetCalleeSaveOuterMethod(Thread* self, CalleeSaveType type) { ScopedAssertNoThreadSuspension ants(__FUNCTION__); ArtMethod** sp = self->GetManagedStack()->GetTopQuickFrame(); return DoGetCalleeSaveMethodOuterCallerAndPc(sp, type).first; diff --git a/runtime/entrypoints/entrypoint_utils.h b/runtime/entrypoints/entrypoint_utils.h index 69ee3ebe75..eed08aabad 100644 --- a/runtime/entrypoints/entrypoint_utils.h +++ b/runtime/entrypoints/entrypoint_utils.h @@ -20,6 +20,7 @@ #include <jni.h> #include <stdint.h> +#include "base/callee_save_type.h" #include "base/macros.h" #include "base/mutex.h" #include "dex_instruction.h" @@ -28,7 +29,6 @@ #include "handle.h" #include "invoke_type.h" #include "jvalue.h" -#include "runtime.h" namespace art { @@ -178,7 +178,7 @@ template <typename INT_TYPE, typename FLOAT_TYPE> inline INT_TYPE art_float_to_integral(FLOAT_TYPE f); ArtMethod* GetCalleeSaveMethodCaller(ArtMethod** sp, - Runtime::CalleeSaveType type, + CalleeSaveType type, bool do_caller_check = false) REQUIRES_SHARED(Locks::mutator_lock_); @@ -187,11 +187,10 @@ struct CallerAndOuterMethod { ArtMethod* outer_method; }; -CallerAndOuterMethod GetCalleeSaveMethodCallerAndOuterMethod(Thread* self, - Runtime::CalleeSaveType type) +CallerAndOuterMethod GetCalleeSaveMethodCallerAndOuterMethod(Thread* self, CalleeSaveType type) REQUIRES_SHARED(Locks::mutator_lock_); -ArtMethod* GetCalleeSaveOuterMethod(Thread* self, Runtime::CalleeSaveType type) +ArtMethod* GetCalleeSaveOuterMethod(Thread* self, CalleeSaveType type) REQUIRES_SHARED(Locks::mutator_lock_); } // namespace art diff --git a/runtime/entrypoints/quick/callee_save_frame.h b/runtime/entrypoints/quick/callee_save_frame.h index c94bf4a318..69e3fc1045 100644 --- a/runtime/entrypoints/quick/callee_save_frame.h +++ b/runtime/entrypoints/quick/callee_save_frame.h @@ -18,10 +18,10 @@ #define ART_RUNTIME_ENTRYPOINTS_QUICK_CALLEE_SAVE_FRAME_H_ #include "arch/instruction_set.h" +#include "base/callee_save_type.h" #include "base/enums.h" #include "base/mutex.h" -#include "runtime.h" -#include "thread.h" +#include "thread-inl.h" // Specific frame size code is in architecture-specific files. We include this to compile-time // specialize the code. @@ -67,7 +67,7 @@ class ScopedQuickEntrypointChecks { bool exit_check_; }; -static constexpr size_t GetCalleeSaveFrameSize(InstructionSet isa, Runtime::CalleeSaveType type) { +static constexpr size_t GetCalleeSaveFrameSize(InstructionSet isa, CalleeSaveType type) { // constexpr must be a return statement. return (isa == kArm || isa == kThumb2) ? arm::ArmCalleeSaveFrameSize(type) : isa == kArm64 ? arm64::Arm64CalleeSaveFrameSize(type) : @@ -93,8 +93,7 @@ static constexpr PointerSize GetConstExprPointerSize(InstructionSet isa) { } // Note: this specialized statement is sanity-checked in the quick-trampoline gtest. -static constexpr size_t GetCalleeSaveReturnPcOffset(InstructionSet isa, - Runtime::CalleeSaveType type) { +static constexpr size_t GetCalleeSaveReturnPcOffset(InstructionSet isa, CalleeSaveType type) { return GetCalleeSaveFrameSize(isa, type) - static_cast<size_t>(GetConstExprPointerSize(isa)); } diff --git a/runtime/entrypoints/quick/quick_deoptimization_entrypoints.cc b/runtime/entrypoints/quick/quick_deoptimization_entrypoints.cc index 5762e4f00a..53f0727a5f 100644 --- a/runtime/entrypoints/quick/quick_deoptimization_entrypoints.cc +++ b/runtime/entrypoints/quick/quick_deoptimization_entrypoints.cc @@ -21,6 +21,7 @@ #include "interpreter/interpreter.h" #include "obj_ptr-inl.h" // TODO: Find the other include that isn't complete, and clean this up. #include "quick_exception_handler.h" +#include "runtime.h" #include "thread.h" namespace art { diff --git a/runtime/entrypoints/quick/quick_dexcache_entrypoints.cc b/runtime/entrypoints/quick/quick_dexcache_entrypoints.cc index 6b965678c3..fe565430fe 100644 --- a/runtime/entrypoints/quick/quick_dexcache_entrypoints.cc +++ b/runtime/entrypoints/quick/quick_dexcache_entrypoints.cc @@ -15,6 +15,7 @@ */ #include "art_method-inl.h" +#include "base/callee_save_type.h" #include "callee_save_frame.h" #include "entrypoints/entrypoint_utils-inl.h" #include "class_linker-inl.h" @@ -64,7 +65,8 @@ extern "C" mirror::Class* artInitializeStaticStorageFromCode(uint32_t type_idx, // A class may be accessing another class' fields when it doesn't have access, as access has been // given by inheritance. ScopedQuickEntrypointChecks sqec(self); - auto caller_and_outer = GetCalleeSaveMethodCallerAndOuterMethod(self, Runtime::kSaveEverything); + auto caller_and_outer = GetCalleeSaveMethodCallerAndOuterMethod(self, + CalleeSaveType::kSaveEverything); ArtMethod* caller = caller_and_outer.caller; mirror::Class* result = ResolveVerifyAndClinit(dex::TypeIndex(type_idx), caller, self, true, false); @@ -78,7 +80,8 @@ extern "C" mirror::Class* artInitializeTypeFromCode(uint32_t type_idx, Thread* s REQUIRES_SHARED(Locks::mutator_lock_) { // Called when method->dex_cache_resolved_types_[] misses. ScopedQuickEntrypointChecks sqec(self); - auto caller_and_outer = GetCalleeSaveMethodCallerAndOuterMethod(self, Runtime::kSaveEverything); + auto caller_and_outer = GetCalleeSaveMethodCallerAndOuterMethod(self, + CalleeSaveType::kSaveEverything); ArtMethod* caller = caller_and_outer.caller; mirror::Class* result = ResolveVerifyAndClinit(dex::TypeIndex(type_idx), caller, self, false, false); @@ -93,7 +96,8 @@ extern "C" mirror::Class* artInitializeTypeAndVerifyAccessFromCode(uint32_t type // Called when caller isn't guaranteed to have access to a type and the dex cache may be // unpopulated. ScopedQuickEntrypointChecks sqec(self); - auto caller_and_outer = GetCalleeSaveMethodCallerAndOuterMethod(self, Runtime::kSaveEverything); + auto caller_and_outer = GetCalleeSaveMethodCallerAndOuterMethod(self, + CalleeSaveType::kSaveEverything); ArtMethod* caller = caller_and_outer.caller; mirror::Class* result = ResolveVerifyAndClinit(dex::TypeIndex(type_idx), caller, self, false, true); @@ -106,7 +110,8 @@ extern "C" mirror::Class* artInitializeTypeAndVerifyAccessFromCode(uint32_t type extern "C" mirror::String* artResolveStringFromCode(int32_t string_idx, Thread* self) REQUIRES_SHARED(Locks::mutator_lock_) { ScopedQuickEntrypointChecks sqec(self); - auto caller_and_outer = GetCalleeSaveMethodCallerAndOuterMethod(self, Runtime::kSaveEverything); + auto caller_and_outer = GetCalleeSaveMethodCallerAndOuterMethod(self, + CalleeSaveType::kSaveEverything); ArtMethod* caller = caller_and_outer.caller; mirror::String* result = ResolveStringFromCode(caller, dex::StringIndex(string_idx)); if (LIKELY(result != nullptr)) { diff --git a/runtime/entrypoints/quick/quick_field_entrypoints.cc b/runtime/entrypoints/quick/quick_field_entrypoints.cc index 822c5a8d9d..726bddd334 100644 --- a/runtime/entrypoints/quick/quick_field_entrypoints.cc +++ b/runtime/entrypoints/quick/quick_field_entrypoints.cc @@ -18,6 +18,7 @@ #include "art_field-inl.h" #include "art_method-inl.h" +#include "base/callee_save_type.h" #include "callee_save_frame.h" #include "dex_file-inl.h" #include "entrypoints/entrypoint_utils-inl.h" @@ -59,12 +60,12 @@ static ArtMethod* GetReferrer(Thread* self) REQUIRES_SHARED(Locks::mutator_lock_ if (kIsDebugBuild) { // stub_test doesn't call this code with a proper frame, so get the outer, and if // it does not have compiled code return it. - ArtMethod* outer = GetCalleeSaveOuterMethod(self, Runtime::kSaveRefsOnly); + ArtMethod* outer = GetCalleeSaveOuterMethod(self, CalleeSaveType::kSaveRefsOnly); if (outer->GetEntryPointFromQuickCompiledCode() == nullptr) { return outer; } } - return GetCalleeSaveMethodCallerAndOuterMethod(self, Runtime::kSaveRefsOnly).caller; + return GetCalleeSaveMethodCallerAndOuterMethod(self, CalleeSaveType::kSaveRefsOnly).caller; } #define ART_GET_FIELD_FROM_CODE(Kind, PrimitiveType, RetType, SetType, \ diff --git a/runtime/entrypoints/quick/quick_instrumentation_entrypoints.cc b/runtime/entrypoints/quick/quick_instrumentation_entrypoints.cc index aa1ebb7ee7..0d2f6d02b6 100644 --- a/runtime/entrypoints/quick/quick_instrumentation_entrypoints.cc +++ b/runtime/entrypoints/quick/quick_instrumentation_entrypoints.cc @@ -15,6 +15,7 @@ */ #include "art_method.h" +#include "base/callee_save_type.h" #include "base/enums.h" #include "callee_save_frame.h" #include "entrypoints/runtime_asm_entrypoints.h" @@ -56,7 +57,7 @@ extern "C" TwoWordReturn artInstrumentationMethodExitFromCode(Thread* self, ArtM CHECK(!self->IsExceptionPending()) << "Enter instrumentation exit stub with pending exception " << self->GetException()->Dump(); // Compute address of return PC and sanity check that it currently holds 0. - size_t return_pc_offset = GetCalleeSaveReturnPcOffset(kRuntimeISA, Runtime::kSaveRefsOnly); + size_t return_pc_offset = GetCalleeSaveReturnPcOffset(kRuntimeISA, CalleeSaveType::kSaveRefsOnly); uintptr_t* return_pc = reinterpret_cast<uintptr_t*>(reinterpret_cast<uint8_t*>(sp) + return_pc_offset); CHECK_EQ(*return_pc, 0U); diff --git a/runtime/entrypoints/quick/quick_trampoline_entrypoints.cc b/runtime/entrypoints/quick/quick_trampoline_entrypoints.cc index 90231e289e..6fcb711716 100644 --- a/runtime/entrypoints/quick/quick_trampoline_entrypoints.cc +++ b/runtime/entrypoints/quick/quick_trampoline_entrypoints.cc @@ -15,6 +15,7 @@ */ #include "art_method-inl.h" +#include "base/callee_save_type.h" #include "base/enums.h" #include "callee_save_frame.h" #include "common_throws.h" @@ -28,6 +29,7 @@ #include "imtable-inl.h" #include "interpreter/interpreter.h" #include "linear_alloc.h" +#include "method_bss_mapping.h" #include "method_handles.h" #include "method_reference.h" #include "mirror/class-inl.h" @@ -36,6 +38,7 @@ #include "mirror/method_handle_impl.h" #include "mirror/object-inl.h" #include "mirror/object_array-inl.h" +#include "oat_file.h" #include "oat_quick_method_header.h" #include "quick_exception_handler.h" #include "runtime.h" @@ -46,13 +49,13 @@ namespace art { -// Visits the arguments as saved to the stack by a Runtime::kRefAndArgs callee save frame. +// Visits the arguments as saved to the stack by a CalleeSaveType::kRefAndArgs callee save frame. class QuickArgumentVisitor { // Number of bytes for each out register in the caller method's frame. static constexpr size_t kBytesStackArgLocation = 4; // Frame size in bytes of a callee-save frame for RefsAndArgs. static constexpr size_t kQuickCalleeSaveFrame_RefAndArgs_FrameSize = - GetCalleeSaveFrameSize(kRuntimeISA, Runtime::kSaveRefsAndArgs); + GetCalleeSaveFrameSize(kRuntimeISA, CalleeSaveType::kSaveRefsAndArgs); #if defined(__arm__) // The callee save frame is pointed to by SP. // | argN | | @@ -81,11 +84,11 @@ class QuickArgumentVisitor { static constexpr size_t kNumQuickFprArgs = kArm32QuickCodeUseSoftFloat ? 0 : 16; static constexpr bool kGprFprLockstep = false; static constexpr size_t kQuickCalleeSaveFrame_RefAndArgs_Fpr1Offset = - arm::ArmCalleeSaveFpr1Offset(Runtime::kSaveRefsAndArgs); // Offset of first FPR arg. + arm::ArmCalleeSaveFpr1Offset(CalleeSaveType::kSaveRefsAndArgs); // Offset of first FPR arg. static constexpr size_t kQuickCalleeSaveFrame_RefAndArgs_Gpr1Offset = - arm::ArmCalleeSaveGpr1Offset(Runtime::kSaveRefsAndArgs); // Offset of first GPR arg. + arm::ArmCalleeSaveGpr1Offset(CalleeSaveType::kSaveRefsAndArgs); // Offset of first GPR arg. static constexpr size_t kQuickCalleeSaveFrame_RefAndArgs_LrOffset = - arm::ArmCalleeSaveLrOffset(Runtime::kSaveRefsAndArgs); // Offset of return address. + arm::ArmCalleeSaveLrOffset(CalleeSaveType::kSaveRefsAndArgs); // Offset of return address. static size_t GprIndexToGprOffset(uint32_t gpr_index) { return gpr_index * GetBytesPerGprSpillLocation(kRuntimeISA); } @@ -118,12 +121,15 @@ class QuickArgumentVisitor { static constexpr size_t kNumQuickGprArgs = 7; // 7 arguments passed in GPRs. static constexpr size_t kNumQuickFprArgs = 8; // 8 arguments passed in FPRs. static constexpr bool kGprFprLockstep = false; + // Offset of first FPR arg. static constexpr size_t kQuickCalleeSaveFrame_RefAndArgs_Fpr1Offset = - arm64::Arm64CalleeSaveFpr1Offset(Runtime::kSaveRefsAndArgs); // Offset of first FPR arg. + arm64::Arm64CalleeSaveFpr1Offset(CalleeSaveType::kSaveRefsAndArgs); + // Offset of first GPR arg. static constexpr size_t kQuickCalleeSaveFrame_RefAndArgs_Gpr1Offset = - arm64::Arm64CalleeSaveGpr1Offset(Runtime::kSaveRefsAndArgs); // Offset of first GPR arg. + arm64::Arm64CalleeSaveGpr1Offset(CalleeSaveType::kSaveRefsAndArgs); + // Offset of return address. static constexpr size_t kQuickCalleeSaveFrame_RefAndArgs_LrOffset = - arm64::Arm64CalleeSaveLrOffset(Runtime::kSaveRefsAndArgs); // Offset of return address. + arm64::Arm64CalleeSaveLrOffset(CalleeSaveType::kSaveRefsAndArgs); static size_t GprIndexToGprOffset(uint32_t gpr_index) { return gpr_index * GetBytesPerGprSpillLocation(kRuntimeISA); } @@ -323,7 +329,7 @@ class QuickArgumentVisitor { static ArtMethod* GetCallingMethod(ArtMethod** sp) REQUIRES_SHARED(Locks::mutator_lock_) { DCHECK((*sp)->IsCalleeSaveMethod()); - return GetCalleeSaveMethodCaller(sp, Runtime::kSaveRefsAndArgs); + return GetCalleeSaveMethodCaller(sp, CalleeSaveType::kSaveRefsAndArgs); } static ArtMethod* GetOuterMethod(ArtMethod** sp) REQUIRES_SHARED(Locks::mutator_lock_) { @@ -335,7 +341,8 @@ class QuickArgumentVisitor { static uint32_t GetCallingDexPc(ArtMethod** sp) REQUIRES_SHARED(Locks::mutator_lock_) { DCHECK((*sp)->IsCalleeSaveMethod()); - const size_t callee_frame_size = GetCalleeSaveFrameSize(kRuntimeISA, Runtime::kSaveRefsAndArgs); + const size_t callee_frame_size = GetCalleeSaveFrameSize(kRuntimeISA, + CalleeSaveType::kSaveRefsAndArgs); ArtMethod** caller_sp = reinterpret_cast<ArtMethod**>( reinterpret_cast<uintptr_t>(sp) + callee_frame_size); uintptr_t outer_pc = QuickArgumentVisitor::GetCallingPc(sp); @@ -362,7 +369,8 @@ class QuickArgumentVisitor { static bool GetInvokeType(ArtMethod** sp, InvokeType* invoke_type, uint32_t* dex_method_index) REQUIRES_SHARED(Locks::mutator_lock_) { DCHECK((*sp)->IsCalleeSaveMethod()); - const size_t callee_frame_size = GetCalleeSaveFrameSize(kRuntimeISA, Runtime::kSaveRefsAndArgs); + const size_t callee_frame_size = GetCalleeSaveFrameSize(kRuntimeISA, + CalleeSaveType::kSaveRefsAndArgs); ArtMethod** caller_sp = reinterpret_cast<ArtMethod**>( reinterpret_cast<uintptr_t>(sp) + callee_frame_size); uintptr_t outer_pc = QuickArgumentVisitor::GetCallingPc(sp); @@ -1105,6 +1113,32 @@ extern "C" const void* artQuickResolutionTrampoline( DCHECK_EQ(caller->GetDexFile(), called_method.dex_file); called = linker->ResolveMethod<ClassLinker::kForceICCECheck>( self, called_method.dex_method_index, caller, invoke_type); + + // Update .bss entry in oat file if any. + if (called != nullptr && called_method.dex_file->GetOatDexFile() != nullptr) { + const MethodBssMapping* mapping = + called_method.dex_file->GetOatDexFile()->GetMethodBssMapping(); + if (mapping != nullptr) { + auto pp = std::partition_point( + mapping->begin(), + mapping->end(), + [called_method](const MethodBssMappingEntry& entry) { + return entry.method_index < called_method.dex_method_index; + }); + if (pp != mapping->end() && pp->CoversIndex(called_method.dex_method_index)) { + size_t bss_offset = pp->GetBssOffset(called_method.dex_method_index, + static_cast<size_t>(kRuntimePointerSize)); + DCHECK_ALIGNED(bss_offset, static_cast<size_t>(kRuntimePointerSize)); + const OatFile* oat_file = called_method.dex_file->GetOatDexFile()->GetOatFile(); + ArtMethod** method_entry = reinterpret_cast<ArtMethod**>(const_cast<uint8_t*>( + oat_file->BssBegin() + bss_offset)); + DCHECK_GE(method_entry, oat_file->GetBssMethods().data()); + DCHECK_LT(method_entry, + oat_file->GetBssMethods().data() + oat_file->GetBssMethods().size()); + *method_entry = called; + } + } + } } const void* code = nullptr; if (LIKELY(!self->IsExceptionPending())) { @@ -2236,7 +2270,7 @@ static TwoWordReturn artInvokeCommon(uint32_t method_idx, Thread* self, ArtMethod** sp) { ScopedQuickEntrypointChecks sqec(self); - DCHECK_EQ(*sp, Runtime::Current()->GetCalleeSaveMethod(Runtime::kSaveRefsAndArgs)); + DCHECK_EQ(*sp, Runtime::Current()->GetCalleeSaveMethod(CalleeSaveType::kSaveRefsAndArgs)); ArtMethod* caller_method = QuickArgumentVisitor::GetCallingMethod(sp); ArtMethod* method = FindMethodFast(method_idx, this_object, caller_method, access_check, type); if (UNLIKELY(method == nullptr)) { @@ -2457,7 +2491,7 @@ extern "C" uintptr_t artInvokePolymorphic( ArtMethod** sp) REQUIRES_SHARED(Locks::mutator_lock_) { ScopedQuickEntrypointChecks sqec(self); - DCHECK_EQ(*sp, Runtime::Current()->GetCalleeSaveMethod(Runtime::kSaveRefsAndArgs)); + DCHECK_EQ(*sp, Runtime::Current()->GetCalleeSaveMethod(CalleeSaveType::kSaveRefsAndArgs)); // Start new JNI local reference state JNIEnvExt* env = self->GetJniEnv(); diff --git a/runtime/entrypoints/quick/quick_trampoline_entrypoints_test.cc b/runtime/entrypoints/quick/quick_trampoline_entrypoints_test.cc index 1cd641b962..7e08b7ace0 100644 --- a/runtime/entrypoints/quick/quick_trampoline_entrypoints_test.cc +++ b/runtime/entrypoints/quick/quick_trampoline_entrypoints_test.cc @@ -17,6 +17,7 @@ #include <stdint.h> #include "art_method-inl.h" +#include "base/callee_save_type.h" #include "callee_save_frame.h" #include "common_runtime_test.h" #include "quick/quick_method_frame_info.h" @@ -38,7 +39,7 @@ class QuickTrampolineEntrypointsTest : public CommonRuntimeTest { ASSERT_EQ(InstructionSet::kX86_64, Runtime::Current()->GetInstructionSet()); } - static ArtMethod* CreateCalleeSaveMethod(InstructionSet isa, Runtime::CalleeSaveType type) + static ArtMethod* CreateCalleeSaveMethod(InstructionSet isa, CalleeSaveType type) NO_THREAD_SAFETY_ANALYSIS { Runtime* r = Runtime::Current(); @@ -53,7 +54,7 @@ class QuickTrampolineEntrypointsTest : public CommonRuntimeTest { return save_method; } - static void CheckFrameSize(InstructionSet isa, Runtime::CalleeSaveType type, uint32_t save_size) + static void CheckFrameSize(InstructionSet isa, CalleeSaveType type, uint32_t save_size) NO_THREAD_SAFETY_ANALYSIS { ArtMethod* save_method = CreateCalleeSaveMethod(isa, type); QuickMethodFrameInfo frame_info = Runtime::Current()->GetRuntimeMethodFrameInfo(save_method); @@ -62,7 +63,7 @@ class QuickTrampolineEntrypointsTest : public CommonRuntimeTest { << frame_info.FpSpillMask() << std::dec << " ISA " << isa; } - static void CheckPCOffset(InstructionSet isa, Runtime::CalleeSaveType type, size_t pc_offset) + static void CheckPCOffset(InstructionSet isa, CalleeSaveType type, size_t pc_offset) NO_THREAD_SAFETY_ANALYSIS { ArtMethod* save_method = CreateCalleeSaveMethod(isa, type); QuickMethodFrameInfo frame_info = Runtime::Current()->GetRuntimeMethodFrameInfo(save_method); @@ -80,16 +81,16 @@ class QuickTrampolineEntrypointsTest : public CommonRuntimeTest { // This test ensures that kQuickCalleeSaveFrame_RefAndArgs_FrameSize is correct. TEST_F(QuickTrampolineEntrypointsTest, FrameSize) { // We have to use a define here as the callee_save_frame.h functions are constexpr. -#define CHECK_FRAME_SIZE(isa) \ - CheckFrameSize(isa, \ - Runtime::kSaveRefsAndArgs, \ - GetCalleeSaveFrameSize(isa, Runtime::kSaveRefsAndArgs)); \ - CheckFrameSize(isa, \ - Runtime::kSaveRefsOnly, \ - GetCalleeSaveFrameSize(isa, Runtime::kSaveRefsOnly)); \ - CheckFrameSize(isa, \ - Runtime::kSaveAllCalleeSaves, \ - GetCalleeSaveFrameSize(isa, Runtime::kSaveAllCalleeSaves)) +#define CHECK_FRAME_SIZE(isa) \ + CheckFrameSize(isa, \ + CalleeSaveType::kSaveRefsAndArgs, \ + GetCalleeSaveFrameSize(isa, CalleeSaveType::kSaveRefsAndArgs)); \ + CheckFrameSize(isa, \ + CalleeSaveType::kSaveRefsOnly, \ + GetCalleeSaveFrameSize(isa, CalleeSaveType::kSaveRefsOnly)); \ + CheckFrameSize(isa, \ + CalleeSaveType::kSaveAllCalleeSaves, \ + GetCalleeSaveFrameSize(isa, CalleeSaveType::kSaveAllCalleeSaves)) CHECK_FRAME_SIZE(kArm); CHECK_FRAME_SIZE(kArm64); @@ -116,12 +117,12 @@ TEST_F(QuickTrampolineEntrypointsTest, ReturnPC) { // Ensure that the computation in callee_save_frame.h correct. // Note: we can only check against the kRuntimeISA, because the ArtMethod computation uses // sizeof(void*), which is wrong when the target bitwidth is not the same as the host's. - CheckPCOffset(kRuntimeISA, Runtime::kSaveRefsAndArgs, - GetCalleeSaveReturnPcOffset(kRuntimeISA, Runtime::kSaveRefsAndArgs)); - CheckPCOffset(kRuntimeISA, Runtime::kSaveRefsOnly, - GetCalleeSaveReturnPcOffset(kRuntimeISA, Runtime::kSaveRefsOnly)); - CheckPCOffset(kRuntimeISA, Runtime::kSaveAllCalleeSaves, - GetCalleeSaveReturnPcOffset(kRuntimeISA, Runtime::kSaveAllCalleeSaves)); + CheckPCOffset(kRuntimeISA, CalleeSaveType::kSaveRefsAndArgs, + GetCalleeSaveReturnPcOffset(kRuntimeISA, CalleeSaveType::kSaveRefsAndArgs)); + CheckPCOffset(kRuntimeISA, CalleeSaveType::kSaveRefsOnly, + GetCalleeSaveReturnPcOffset(kRuntimeISA, CalleeSaveType::kSaveRefsOnly)); + CheckPCOffset(kRuntimeISA, CalleeSaveType::kSaveAllCalleeSaves, + GetCalleeSaveReturnPcOffset(kRuntimeISA, CalleeSaveType::kSaveAllCalleeSaves)); } } // namespace art diff --git a/runtime/exec_utils.cc b/runtime/exec_utils.cc index 9efb1a353c..db1baa76f9 100644 --- a/runtime/exec_utils.cc +++ b/runtime/exec_utils.cc @@ -28,7 +28,6 @@ namespace art { -using android::base::StringAppendF; using android::base::StringPrintf; int ExecAndReturnCode(std::vector<std::string>& arg_vector, std::string* error_msg) { diff --git a/runtime/gc/accounting/atomic_stack.h b/runtime/gc/accounting/atomic_stack.h index 351798efd1..3d0e8172b6 100644 --- a/runtime/gc/accounting/atomic_stack.h +++ b/runtime/gc/accounting/atomic_stack.h @@ -29,6 +29,13 @@ #include "mem_map.h" #include "stack_reference.h" +// This implements a double-ended queue (deque) with various flavors of PushBack operations, +// as well as PopBack and PopFront operations. We expect that all calls are performed +// by a single thread (normally the GC). There is one exception, which accounts for the +// name: +// - Multiple calls to AtomicPushBack*() and AtomicBumpBack() may be made concurrently, +// provided no other calls are made at the same time. + namespace art { namespace gc { namespace accounting { @@ -150,7 +157,7 @@ class AtomicStack { // Pop a number of elements. void PopBackCount(int32_t n) { DCHECK_GE(Size(), static_cast<size_t>(n)); - back_index_.FetchAndSubSequentiallyConsistent(n); + back_index_.StoreRelaxed(back_index_.LoadRelaxed() - n); } bool IsEmpty() const { diff --git a/runtime/gc/allocator/rosalloc.h b/runtime/gc/allocator/rosalloc.h index 562fc750ed..b85d7dff5c 100644 --- a/runtime/gc/allocator/rosalloc.h +++ b/runtime/gc/allocator/rosalloc.h @@ -707,6 +707,9 @@ class RosAlloc { // the end of the memory region that's ever managed by this allocator. size_t max_capacity_; + template<class Key, AllocatorTag kTag, class Compare = std::less<Key>> + using AllocationTrackingSet = std::set<Key, Compare, TrackingAllocator<Key, kTag>>; + // The run sets that hold the runs whose slots are not all // full. non_full_runs_[i] is guarded by size_bracket_locks_[i]. AllocationTrackingSet<Run*, kAllocatorTagRosAlloc> non_full_runs_[kNumOfSizeBrackets]; diff --git a/runtime/gc/heap.cc b/runtime/gc/heap.cc index 1af3b57830..d944ce4904 100644 --- a/runtime/gc/heap.cc +++ b/runtime/gc/heap.cc @@ -90,8 +90,6 @@ namespace art { namespace gc { -using android::base::StringPrintf; - static constexpr size_t kCollectorTransitionStressIterations = 0; static constexpr size_t kCollectorTransitionStressWait = 10 * 1000; // Microseconds // Minimum amount of remaining bytes before a concurrent GC is triggered. diff --git a/runtime/gc/space/image_space.cc b/runtime/gc/space/image_space.cc index b6bff5b17c..9da2876416 100644 --- a/runtime/gc/space/image_space.cc +++ b/runtime/gc/space/image_space.cc @@ -27,6 +27,7 @@ #include "art_field-inl.h" #include "art_method-inl.h" +#include "base/callee_save_type.h" #include "base/enums.h" #include "base/macros.h" #include "base/stl_util.h" @@ -714,13 +715,13 @@ class ImageSpaceLoader { image_header->GetImageMethod(ImageHeader::kImtConflictMethod)); CHECK_EQ(runtime->GetImtUnimplementedMethod(), image_header->GetImageMethod(ImageHeader::kImtUnimplementedMethod)); - CHECK_EQ(runtime->GetCalleeSaveMethod(Runtime::kSaveAllCalleeSaves), + CHECK_EQ(runtime->GetCalleeSaveMethod(CalleeSaveType::kSaveAllCalleeSaves), image_header->GetImageMethod(ImageHeader::kSaveAllCalleeSavesMethod)); - CHECK_EQ(runtime->GetCalleeSaveMethod(Runtime::kSaveRefsOnly), + CHECK_EQ(runtime->GetCalleeSaveMethod(CalleeSaveType::kSaveRefsOnly), image_header->GetImageMethod(ImageHeader::kSaveRefsOnlyMethod)); - CHECK_EQ(runtime->GetCalleeSaveMethod(Runtime::kSaveRefsAndArgs), + CHECK_EQ(runtime->GetCalleeSaveMethod(CalleeSaveType::kSaveRefsAndArgs), image_header->GetImageMethod(ImageHeader::kSaveRefsAndArgsMethod)); - CHECK_EQ(runtime->GetCalleeSaveMethod(Runtime::kSaveEverything), + CHECK_EQ(runtime->GetCalleeSaveMethod(CalleeSaveType::kSaveEverything), image_header->GetImageMethod(ImageHeader::kSaveEverythingMethod)); } else if (!runtime->HasResolutionMethod()) { runtime->SetInstructionSet(space->oat_file_non_owned_->GetOatHeader().GetInstructionSet()); @@ -730,14 +731,16 @@ class ImageSpaceLoader { image_header->GetImageMethod(ImageHeader::kImtUnimplementedMethod)); runtime->SetCalleeSaveMethod( image_header->GetImageMethod(ImageHeader::kSaveAllCalleeSavesMethod), - Runtime::kSaveAllCalleeSaves); + CalleeSaveType::kSaveAllCalleeSaves); runtime->SetCalleeSaveMethod( - image_header->GetImageMethod(ImageHeader::kSaveRefsOnlyMethod), Runtime::kSaveRefsOnly); + image_header->GetImageMethod(ImageHeader::kSaveRefsOnlyMethod), + CalleeSaveType::kSaveRefsOnly); runtime->SetCalleeSaveMethod( image_header->GetImageMethod(ImageHeader::kSaveRefsAndArgsMethod), - Runtime::kSaveRefsAndArgs); + CalleeSaveType::kSaveRefsAndArgs); runtime->SetCalleeSaveMethod( - image_header->GetImageMethod(ImageHeader::kSaveEverythingMethod), Runtime::kSaveEverything); + image_header->GetImageMethod(ImageHeader::kSaveEverythingMethod), + CalleeSaveType::kSaveEverything); } VLOG(image) << "ImageSpace::Init exiting " << *space.get(); diff --git a/runtime/generated/asm_support_gen.h b/runtime/generated/asm_support_gen.h index 2de4f19222..3a7f21de67 100644 --- a/runtime/generated/asm_support_gen.h +++ b/runtime/generated/asm_support_gen.h @@ -27,13 +27,13 @@ DEFINE_CHECK_EQ(static_cast<size_t>(COMPRESSED_REFERENCE_SIZE), (static_cast<siz #define COMPRESSED_REFERENCE_SIZE_SHIFT 0x2 DEFINE_CHECK_EQ(static_cast<size_t>(COMPRESSED_REFERENCE_SIZE_SHIFT), (static_cast<size_t>(art::WhichPowerOf2(sizeof(art::mirror::CompressedReference<art::mirror::Object>))))) #define RUNTIME_SAVE_ALL_CALLEE_SAVES_METHOD_OFFSET 0 -DEFINE_CHECK_EQ(static_cast<size_t>(RUNTIME_SAVE_ALL_CALLEE_SAVES_METHOD_OFFSET), (static_cast<size_t>(art::Runtime::GetCalleeSaveMethodOffset(art::Runtime:: kSaveAllCalleeSaves)))) +DEFINE_CHECK_EQ(static_cast<size_t>(RUNTIME_SAVE_ALL_CALLEE_SAVES_METHOD_OFFSET), (static_cast<size_t>(art::Runtime::GetCalleeSaveMethodOffset(art::CalleeSaveType::kSaveAllCalleeSaves)))) #define RUNTIME_SAVE_REFS_ONLY_METHOD_OFFSET 0x8 -DEFINE_CHECK_EQ(static_cast<size_t>(RUNTIME_SAVE_REFS_ONLY_METHOD_OFFSET), (static_cast<size_t>(art::Runtime::GetCalleeSaveMethodOffset(art::Runtime:: kSaveRefsOnly)))) +DEFINE_CHECK_EQ(static_cast<size_t>(RUNTIME_SAVE_REFS_ONLY_METHOD_OFFSET), (static_cast<size_t>(art::Runtime::GetCalleeSaveMethodOffset(art::CalleeSaveType::kSaveRefsOnly)))) #define RUNTIME_SAVE_REFS_AND_ARGS_METHOD_OFFSET 0x10 -DEFINE_CHECK_EQ(static_cast<size_t>(RUNTIME_SAVE_REFS_AND_ARGS_METHOD_OFFSET), (static_cast<size_t>(art::Runtime::GetCalleeSaveMethodOffset(art::Runtime:: kSaveRefsAndArgs)))) +DEFINE_CHECK_EQ(static_cast<size_t>(RUNTIME_SAVE_REFS_AND_ARGS_METHOD_OFFSET), (static_cast<size_t>(art::Runtime::GetCalleeSaveMethodOffset(art::CalleeSaveType::kSaveRefsAndArgs)))) #define RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET 0x18 -DEFINE_CHECK_EQ(static_cast<size_t>(RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET), (static_cast<size_t>(art::Runtime::GetCalleeSaveMethodOffset(art::Runtime:: kSaveEverything)))) +DEFINE_CHECK_EQ(static_cast<size_t>(RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET), (static_cast<size_t>(art::Runtime::GetCalleeSaveMethodOffset(art::CalleeSaveType::kSaveEverything)))) #define THREAD_FLAGS_OFFSET 0 DEFINE_CHECK_EQ(static_cast<int32_t>(THREAD_FLAGS_OFFSET), (static_cast<int32_t>(art::Thread:: ThreadFlagsOffset<art::kRuntimePointerSize>().Int32Value()))) #define THREAD_ID_OFFSET 12 diff --git a/runtime/instrumentation.cc b/runtime/instrumentation.cc index d862ff2708..9b9c70f040 100644 --- a/runtime/instrumentation.cc +++ b/runtime/instrumentation.cc @@ -20,7 +20,9 @@ #include "arch/context.h" #include "art_method-inl.h" +#include "art_field-inl.h" #include "atomic.h" +#include "base/callee_save_type.h" #include "class_linker.h" #include "debugger.h" #include "dex_file-inl.h" @@ -31,6 +33,7 @@ #include "interpreter/interpreter.h" #include "jit/jit.h" #include "jit/jit_code_cache.h" +#include "jvalue-inl.h" #include "mirror/class-inl.h" #include "mirror/dex_cache.h" #include "mirror/object_array-inl.h" @@ -45,6 +48,30 @@ namespace instrumentation { constexpr bool kVerboseInstrumentation = false; +void InstrumentationListener::MethodExited(Thread* thread, + Handle<mirror::Object> this_object, + ArtMethod* method, + uint32_t dex_pc, + Handle<mirror::Object> return_value) { + DCHECK_EQ(method->GetInterfaceMethodIfProxy(kRuntimePointerSize)->GetReturnTypePrimitive(), + Primitive::kPrimNot); + JValue v; + v.SetL(return_value.Get()); + MethodExited(thread, this_object, method, dex_pc, v); +} + +void InstrumentationListener::FieldWritten(Thread* thread, + Handle<mirror::Object> this_object, + ArtMethod* method, + uint32_t dex_pc, + ArtField* field, + Handle<mirror::Object> field_value) { + DCHECK(!field->IsPrimitiveType()); + JValue v; + v.SetL(field_value.Get()); + FieldWritten(thread, this_object, method, dex_pc, field, v); +} + // Instrumentation works on non-inlined frames by updating returned PCs // of compiled frames. static constexpr StackVisitor::StackWalkKind kInstrumentationStackWalk = @@ -357,7 +384,7 @@ static void InstrumentationRestoreStack(Thread* thread, void* arg) LOG(INFO) << " Removing exit stub in " << DescribeLocation(); } if (instrumentation_frame.interpreter_entry_) { - CHECK(m == Runtime::Current()->GetCalleeSaveMethod(Runtime::kSaveRefsAndArgs)); + CHECK(m == Runtime::Current()->GetCalleeSaveMethod(CalleeSaveType::kSaveRefsAndArgs)); } else { CHECK(m == instrumentation_frame.method_) << ArtMethod::PrettyMethod(m); } @@ -916,48 +943,75 @@ const void* Instrumentation::GetQuickCodeFor(ArtMethod* method, PointerSize poin return class_linker->GetQuickOatCodeFor(method); } -void Instrumentation::MethodEnterEventImpl(Thread* thread, mirror::Object* this_object, +void Instrumentation::MethodEnterEventImpl(Thread* thread, + ObjPtr<mirror::Object> this_object, ArtMethod* method, uint32_t dex_pc) const { if (HasMethodEntryListeners()) { + Thread* self = Thread::Current(); + StackHandleScope<1> hs(self); + Handle<mirror::Object> thiz(hs.NewHandle(this_object)); for (InstrumentationListener* listener : method_entry_listeners_) { if (listener != nullptr) { - listener->MethodEntered(thread, this_object, method, dex_pc); + listener->MethodEntered(thread, thiz, method, dex_pc); } } } } -void Instrumentation::MethodExitEventImpl(Thread* thread, mirror::Object* this_object, +void Instrumentation::MethodExitEventImpl(Thread* thread, + ObjPtr<mirror::Object> this_object, ArtMethod* method, - uint32_t dex_pc, const JValue& return_value) const { + uint32_t dex_pc, + const JValue& return_value) const { if (HasMethodExitListeners()) { - for (InstrumentationListener* listener : method_exit_listeners_) { - if (listener != nullptr) { - listener->MethodExited(thread, this_object, method, dex_pc, return_value); + Thread* self = Thread::Current(); + StackHandleScope<2> hs(self); + Handle<mirror::Object> thiz(hs.NewHandle(this_object)); + if (method->GetInterfaceMethodIfProxy(kRuntimePointerSize) + ->GetReturnTypePrimitive() != Primitive::kPrimNot) { + for (InstrumentationListener* listener : method_exit_listeners_) { + if (listener != nullptr) { + listener->MethodExited(thread, thiz, method, dex_pc, return_value); + } + } + } else { + Handle<mirror::Object> ret(hs.NewHandle(return_value.GetL())); + for (InstrumentationListener* listener : method_exit_listeners_) { + if (listener != nullptr) { + listener->MethodExited(thread, thiz, method, dex_pc, ret); + } } } } } -void Instrumentation::MethodUnwindEvent(Thread* thread, mirror::Object* this_object, +void Instrumentation::MethodUnwindEvent(Thread* thread, + mirror::Object* this_object, ArtMethod* method, uint32_t dex_pc) const { if (HasMethodUnwindListeners()) { + Thread* self = Thread::Current(); + StackHandleScope<1> hs(self); + Handle<mirror::Object> thiz(hs.NewHandle(this_object)); for (InstrumentationListener* listener : method_unwind_listeners_) { if (listener != nullptr) { - listener->MethodUnwind(thread, this_object, method, dex_pc); + listener->MethodUnwind(thread, thiz, method, dex_pc); } } } } -void Instrumentation::DexPcMovedEventImpl(Thread* thread, mirror::Object* this_object, +void Instrumentation::DexPcMovedEventImpl(Thread* thread, + ObjPtr<mirror::Object> this_object, ArtMethod* method, uint32_t dex_pc) const { + Thread* self = Thread::Current(); + StackHandleScope<1> hs(self); + Handle<mirror::Object> thiz(hs.NewHandle(this_object)); for (InstrumentationListener* listener : dex_pc_listeners_) { if (listener != nullptr) { - listener->DexPcMoved(thread, this_object, method, dex_pc); + listener->DexPcMoved(thread, thiz, method, dex_pc); } } } @@ -974,36 +1028,56 @@ void Instrumentation::BranchImpl(Thread* thread, } void Instrumentation::InvokeVirtualOrInterfaceImpl(Thread* thread, - mirror::Object* this_object, + ObjPtr<mirror::Object> this_object, ArtMethod* caller, uint32_t dex_pc, ArtMethod* callee) const { - // We cannot have thread suspension since that would cause the this_object parameter to - // potentially become a dangling pointer. An alternative could be to put it in a handle instead. - ScopedAssertNoThreadSuspension ants(__FUNCTION__); + Thread* self = Thread::Current(); + StackHandleScope<1> hs(self); + Handle<mirror::Object> thiz(hs.NewHandle(this_object)); for (InstrumentationListener* listener : invoke_virtual_or_interface_listeners_) { if (listener != nullptr) { - listener->InvokeVirtualOrInterface(thread, this_object, caller, dex_pc, callee); + listener->InvokeVirtualOrInterface(thread, thiz, caller, dex_pc, callee); } } } -void Instrumentation::FieldReadEventImpl(Thread* thread, mirror::Object* this_object, - ArtMethod* method, uint32_t dex_pc, +void Instrumentation::FieldReadEventImpl(Thread* thread, + ObjPtr<mirror::Object> this_object, + ArtMethod* method, + uint32_t dex_pc, ArtField* field) const { + Thread* self = Thread::Current(); + StackHandleScope<1> hs(self); + Handle<mirror::Object> thiz(hs.NewHandle(this_object)); for (InstrumentationListener* listener : field_read_listeners_) { if (listener != nullptr) { - listener->FieldRead(thread, this_object, method, dex_pc, field); + listener->FieldRead(thread, thiz, method, dex_pc, field); } } } -void Instrumentation::FieldWriteEventImpl(Thread* thread, mirror::Object* this_object, - ArtMethod* method, uint32_t dex_pc, - ArtField* field, const JValue& field_value) const { - for (InstrumentationListener* listener : field_write_listeners_) { - if (listener != nullptr) { - listener->FieldWritten(thread, this_object, method, dex_pc, field, field_value); +void Instrumentation::FieldWriteEventImpl(Thread* thread, + ObjPtr<mirror::Object> this_object, + ArtMethod* method, + uint32_t dex_pc, + ArtField* field, + const JValue& field_value) const { + Thread* self = Thread::Current(); + StackHandleScope<2> hs(self); + Handle<mirror::Object> thiz(hs.NewHandle(this_object)); + if (field->IsPrimitiveType()) { + for (InstrumentationListener* listener : field_write_listeners_) { + if (listener != nullptr) { + listener->FieldWritten(thread, thiz, method, dex_pc, field, field_value); + } + } + } else { + Handle<mirror::Object> val(hs.NewHandle(field_value.GetL())); + for (InstrumentationListener* listener : field_write_listeners_) { + if (listener != nullptr) { + listener->FieldWritten(thread, thiz, method, dex_pc, field, val); + } } } } @@ -1018,7 +1092,7 @@ void Instrumentation::ExceptionCaughtEvent(Thread* thread, thread->ClearException(); for (InstrumentationListener* listener : exception_caught_listeners_) { if (listener != nullptr) { - listener->ExceptionCaught(thread, h_exception.Get()); + listener->ExceptionCaught(thread, h_exception); } } thread->SetException(h_exception.Get()); diff --git a/runtime/instrumentation.h b/runtime/instrumentation.h index 01071a541f..363985fd62 100644 --- a/runtime/instrumentation.h +++ b/runtime/instrumentation.h @@ -36,6 +36,7 @@ namespace mirror { } // namespace mirror class ArtField; class ArtMethod; +template <typename T> class Handle; union JValue; class Thread; @@ -62,37 +63,70 @@ struct InstrumentationListener { virtual ~InstrumentationListener() {} // Call-back for when a method is entered. - virtual void MethodEntered(Thread* thread, mirror::Object* this_object, + virtual void MethodEntered(Thread* thread, + Handle<mirror::Object> this_object, ArtMethod* method, uint32_t dex_pc) REQUIRES_SHARED(Locks::mutator_lock_) = 0; - // Call-back for when a method is exited. - virtual void MethodExited(Thread* thread, mirror::Object* this_object, - ArtMethod* method, uint32_t dex_pc, + virtual void MethodExited(Thread* thread, + Handle<mirror::Object> this_object, + ArtMethod* method, + uint32_t dex_pc, + Handle<mirror::Object> return_value) + REQUIRES_SHARED(Locks::mutator_lock_); + + // Call-back for when a method is exited. The implementor should either handler-ize the return + // value (if appropriate) or use the alternate MethodExited callback instead if they need to + // go through a suspend point. + virtual void MethodExited(Thread* thread, + Handle<mirror::Object> this_object, + ArtMethod* method, + uint32_t dex_pc, const JValue& return_value) REQUIRES_SHARED(Locks::mutator_lock_) = 0; // Call-back for when a method is popped due to an exception throw. A method will either cause a // MethodExited call-back or a MethodUnwind call-back when its activation is removed. - virtual void MethodUnwind(Thread* thread, mirror::Object* this_object, - ArtMethod* method, uint32_t dex_pc) + virtual void MethodUnwind(Thread* thread, + Handle<mirror::Object> this_object, + ArtMethod* method, + uint32_t dex_pc) REQUIRES_SHARED(Locks::mutator_lock_) = 0; // Call-back for when the dex pc moves in a method. - virtual void DexPcMoved(Thread* thread, mirror::Object* this_object, - ArtMethod* method, uint32_t new_dex_pc) + virtual void DexPcMoved(Thread* thread, + Handle<mirror::Object> this_object, + ArtMethod* method, + uint32_t new_dex_pc) REQUIRES_SHARED(Locks::mutator_lock_) = 0; // Call-back for when we read from a field. - virtual void FieldRead(Thread* thread, mirror::Object* this_object, ArtMethod* method, - uint32_t dex_pc, ArtField* field) = 0; + virtual void FieldRead(Thread* thread, + Handle<mirror::Object> this_object, + ArtMethod* method, + uint32_t dex_pc, + ArtField* field) = 0; + + virtual void FieldWritten(Thread* thread, + Handle<mirror::Object> this_object, + ArtMethod* method, + uint32_t dex_pc, + ArtField* field, + Handle<mirror::Object> field_value) + REQUIRES_SHARED(Locks::mutator_lock_); // Call-back for when we write into a field. - virtual void FieldWritten(Thread* thread, mirror::Object* this_object, ArtMethod* method, - uint32_t dex_pc, ArtField* field, const JValue& field_value) = 0; + virtual void FieldWritten(Thread* thread, + Handle<mirror::Object> this_object, + ArtMethod* method, + uint32_t dex_pc, + ArtField* field, + const JValue& field_value) + REQUIRES_SHARED(Locks::mutator_lock_) = 0; // Call-back when an exception is caught. - virtual void ExceptionCaught(Thread* thread, mirror::Throwable* exception_object) + virtual void ExceptionCaught(Thread* thread, + Handle<mirror::Throwable> exception_object) REQUIRES_SHARED(Locks::mutator_lock_) = 0; // Call-back for when we execute a branch. @@ -104,11 +138,10 @@ struct InstrumentationListener { // Call-back for when we get an invokevirtual or an invokeinterface. virtual void InvokeVirtualOrInterface(Thread* thread, - mirror::Object* this_object, + Handle<mirror::Object> this_object, ArtMethod* caller, uint32_t dex_pc, ArtMethod* callee) - REQUIRES(Roles::uninterruptible_) REQUIRES_SHARED(Locks::mutator_lock_) = 0; }; @@ -323,8 +356,10 @@ class Instrumentation { } // Inform listeners that a method has been exited. - void MethodExitEvent(Thread* thread, mirror::Object* this_object, - ArtMethod* method, uint32_t dex_pc, + void MethodExitEvent(Thread* thread, + mirror::Object* this_object, + ArtMethod* method, + uint32_t dex_pc, const JValue& return_value) const REQUIRES_SHARED(Locks::mutator_lock_) { if (UNLIKELY(HasMethodExitListeners())) { @@ -465,31 +500,42 @@ class Instrumentation { // exclusive access to mutator lock which you can't get if the runtime isn't started. void SetEntrypointsInstrumented(bool instrumented) NO_THREAD_SAFETY_ANALYSIS; - void MethodEnterEventImpl(Thread* thread, mirror::Object* this_object, - ArtMethod* method, uint32_t dex_pc) const + void MethodEnterEventImpl(Thread* thread, + ObjPtr<mirror::Object> this_object, + ArtMethod* method, + uint32_t dex_pc) const REQUIRES_SHARED(Locks::mutator_lock_); - void MethodExitEventImpl(Thread* thread, mirror::Object* this_object, + void MethodExitEventImpl(Thread* thread, + ObjPtr<mirror::Object> this_object, ArtMethod* method, - uint32_t dex_pc, const JValue& return_value) const + uint32_t dex_pc, + const JValue& return_value) const REQUIRES_SHARED(Locks::mutator_lock_); - void DexPcMovedEventImpl(Thread* thread, mirror::Object* this_object, - ArtMethod* method, uint32_t dex_pc) const + void DexPcMovedEventImpl(Thread* thread, + ObjPtr<mirror::Object> this_object, + ArtMethod* method, + uint32_t dex_pc) const REQUIRES_SHARED(Locks::mutator_lock_); void BranchImpl(Thread* thread, ArtMethod* method, uint32_t dex_pc, int32_t offset) const REQUIRES_SHARED(Locks::mutator_lock_); void InvokeVirtualOrInterfaceImpl(Thread* thread, - mirror::Object* this_object, + ObjPtr<mirror::Object> this_object, ArtMethod* caller, uint32_t dex_pc, ArtMethod* callee) const REQUIRES_SHARED(Locks::mutator_lock_); - void FieldReadEventImpl(Thread* thread, mirror::Object* this_object, - ArtMethod* method, uint32_t dex_pc, - ArtField* field) const + void FieldReadEventImpl(Thread* thread, + ObjPtr<mirror::Object> this_object, + ArtMethod* method, + uint32_t dex_pc, + ArtField* field) const REQUIRES_SHARED(Locks::mutator_lock_); - void FieldWriteEventImpl(Thread* thread, mirror::Object* this_object, - ArtMethod* method, uint32_t dex_pc, - ArtField* field, const JValue& field_value) const + void FieldWriteEventImpl(Thread* thread, + ObjPtr<mirror::Object> this_object, + ArtMethod* method, + uint32_t dex_pc, + ArtField* field, + const JValue& field_value) const REQUIRES_SHARED(Locks::mutator_lock_); // Read barrier-aware utility functions for accessing deoptimized_methods_ diff --git a/runtime/instrumentation_test.cc b/runtime/instrumentation_test.cc index 9926ee7386..2a601c9cf2 100644 --- a/runtime/instrumentation_test.cc +++ b/runtime/instrumentation_test.cc @@ -23,11 +23,13 @@ #include "dex_file.h" #include "gc/scoped_gc_critical_section.h" #include "handle_scope-inl.h" +#include "jni_internal.h" #include "jvalue.h" #include "runtime.h" #include "scoped_thread_state_change-inl.h" #include "thread_list.h" -#include "thread-current-inl.h" +#include "thread-inl.h" +#include "well_known_classes.h" namespace art { namespace instrumentation { @@ -35,16 +37,22 @@ namespace instrumentation { class TestInstrumentationListener FINAL : public instrumentation::InstrumentationListener { public: TestInstrumentationListener() - : received_method_enter_event(false), received_method_exit_event(false), - received_method_unwind_event(false), received_dex_pc_moved_event(false), - received_field_read_event(false), received_field_written_event(false), - received_exception_caught_event(false), received_branch_event(false), + : received_method_enter_event(false), + received_method_exit_event(false), + received_method_exit_object_event(false), + received_method_unwind_event(false), + received_dex_pc_moved_event(false), + received_field_read_event(false), + received_field_written_event(false), + received_field_written_object_event(false), + received_exception_caught_event(false), + received_branch_event(false), received_invoke_virtual_or_interface_event(false) {} virtual ~TestInstrumentationListener() {} void MethodEntered(Thread* thread ATTRIBUTE_UNUSED, - mirror::Object* this_object ATTRIBUTE_UNUSED, + Handle<mirror::Object> this_object ATTRIBUTE_UNUSED, ArtMethod* method ATTRIBUTE_UNUSED, uint32_t dex_pc ATTRIBUTE_UNUSED) OVERRIDE REQUIRES_SHARED(Locks::mutator_lock_) { @@ -52,7 +60,16 @@ class TestInstrumentationListener FINAL : public instrumentation::Instrumentatio } void MethodExited(Thread* thread ATTRIBUTE_UNUSED, - mirror::Object* this_object ATTRIBUTE_UNUSED, + Handle<mirror::Object> this_object ATTRIBUTE_UNUSED, + ArtMethod* method ATTRIBUTE_UNUSED, + uint32_t dex_pc ATTRIBUTE_UNUSED, + Handle<mirror::Object> return_value ATTRIBUTE_UNUSED) + OVERRIDE REQUIRES_SHARED(Locks::mutator_lock_) { + received_method_exit_object_event = true; + } + + void MethodExited(Thread* thread ATTRIBUTE_UNUSED, + Handle<mirror::Object> this_object ATTRIBUTE_UNUSED, ArtMethod* method ATTRIBUTE_UNUSED, uint32_t dex_pc ATTRIBUTE_UNUSED, const JValue& return_value ATTRIBUTE_UNUSED) @@ -61,7 +78,7 @@ class TestInstrumentationListener FINAL : public instrumentation::Instrumentatio } void MethodUnwind(Thread* thread ATTRIBUTE_UNUSED, - mirror::Object* this_object ATTRIBUTE_UNUSED, + Handle<mirror::Object> this_object ATTRIBUTE_UNUSED, ArtMethod* method ATTRIBUTE_UNUSED, uint32_t dex_pc ATTRIBUTE_UNUSED) OVERRIDE REQUIRES_SHARED(Locks::mutator_lock_) { @@ -69,7 +86,7 @@ class TestInstrumentationListener FINAL : public instrumentation::Instrumentatio } void DexPcMoved(Thread* thread ATTRIBUTE_UNUSED, - mirror::Object* this_object ATTRIBUTE_UNUSED, + Handle<mirror::Object> this_object ATTRIBUTE_UNUSED, ArtMethod* method ATTRIBUTE_UNUSED, uint32_t new_dex_pc ATTRIBUTE_UNUSED) OVERRIDE REQUIRES_SHARED(Locks::mutator_lock_) { @@ -77,7 +94,7 @@ class TestInstrumentationListener FINAL : public instrumentation::Instrumentatio } void FieldRead(Thread* thread ATTRIBUTE_UNUSED, - mirror::Object* this_object ATTRIBUTE_UNUSED, + Handle<mirror::Object> this_object ATTRIBUTE_UNUSED, ArtMethod* method ATTRIBUTE_UNUSED, uint32_t dex_pc ATTRIBUTE_UNUSED, ArtField* field ATTRIBUTE_UNUSED) @@ -86,7 +103,17 @@ class TestInstrumentationListener FINAL : public instrumentation::Instrumentatio } void FieldWritten(Thread* thread ATTRIBUTE_UNUSED, - mirror::Object* this_object ATTRIBUTE_UNUSED, + Handle<mirror::Object> this_object ATTRIBUTE_UNUSED, + ArtMethod* method ATTRIBUTE_UNUSED, + uint32_t dex_pc ATTRIBUTE_UNUSED, + ArtField* field ATTRIBUTE_UNUSED, + Handle<mirror::Object> field_value ATTRIBUTE_UNUSED) + OVERRIDE REQUIRES_SHARED(Locks::mutator_lock_) { + received_field_written_object_event = true; + } + + void FieldWritten(Thread* thread ATTRIBUTE_UNUSED, + Handle<mirror::Object> this_object ATTRIBUTE_UNUSED, ArtMethod* method ATTRIBUTE_UNUSED, uint32_t dex_pc ATTRIBUTE_UNUSED, ArtField* field ATTRIBUTE_UNUSED, @@ -96,7 +123,7 @@ class TestInstrumentationListener FINAL : public instrumentation::Instrumentatio } void ExceptionCaught(Thread* thread ATTRIBUTE_UNUSED, - mirror::Throwable* exception_object ATTRIBUTE_UNUSED) + Handle<mirror::Throwable> exception_object ATTRIBUTE_UNUSED) OVERRIDE REQUIRES_SHARED(Locks::mutator_lock_) { received_exception_caught_event = true; } @@ -110,7 +137,7 @@ class TestInstrumentationListener FINAL : public instrumentation::Instrumentatio } void InvokeVirtualOrInterface(Thread* thread ATTRIBUTE_UNUSED, - mirror::Object* this_object ATTRIBUTE_UNUSED, + Handle<mirror::Object> this_object ATTRIBUTE_UNUSED, ArtMethod* caller ATTRIBUTE_UNUSED, uint32_t dex_pc ATTRIBUTE_UNUSED, ArtMethod* callee ATTRIBUTE_UNUSED) @@ -121,10 +148,12 @@ class TestInstrumentationListener FINAL : public instrumentation::Instrumentatio void Reset() { received_method_enter_event = false; received_method_exit_event = false; + received_method_exit_object_event = false; received_method_unwind_event = false; received_dex_pc_moved_event = false; received_field_read_event = false; received_field_written_event = false; + received_field_written_object_event = false; received_exception_caught_event = false; received_branch_event = false; received_invoke_virtual_or_interface_event = false; @@ -132,10 +161,12 @@ class TestInstrumentationListener FINAL : public instrumentation::Instrumentatio bool received_method_enter_event; bool received_method_exit_event; + bool received_method_exit_object_event; bool received_method_unwind_event; bool received_dex_pc_moved_event; bool received_field_read_event; bool received_field_written_event; + bool received_field_written_object_event; bool received_exception_caught_event; bool received_branch_event; bool received_invoke_virtual_or_interface_event; @@ -171,6 +202,13 @@ class InstrumentationTest : public CommonRuntimeTest { } void TestEvent(uint32_t instrumentation_event) { + TestEvent(instrumentation_event, nullptr, nullptr, false); + } + + void TestEvent(uint32_t instrumentation_event, + ArtMethod* event_method, + ArtField* event_field, + bool with_object) { ScopedObjectAccess soa(Thread::Current()); instrumentation::Instrumentation* instr = Runtime::Current()->GetInstrumentation(); TestInstrumentationListener listener; @@ -180,15 +218,20 @@ class InstrumentationTest : public CommonRuntimeTest { instr->AddListener(&listener, instrumentation_event); } - ArtMethod* const event_method = nullptr; mirror::Object* const event_obj = nullptr; const uint32_t event_dex_pc = 0; // Check the listener is registered and is notified of the event. EXPECT_TRUE(HasEventListener(instr, instrumentation_event)); - EXPECT_FALSE(DidListenerReceiveEvent(listener, instrumentation_event)); - ReportEvent(instr, instrumentation_event, soa.Self(), event_method, event_obj, event_dex_pc); - EXPECT_TRUE(DidListenerReceiveEvent(listener, instrumentation_event)); + EXPECT_FALSE(DidListenerReceiveEvent(listener, instrumentation_event, with_object)); + ReportEvent(instr, + instrumentation_event, + soa.Self(), + event_method, + event_obj, + event_field, + event_dex_pc); + EXPECT_TRUE(DidListenerReceiveEvent(listener, instrumentation_event, with_object)); listener.Reset(); { @@ -199,9 +242,15 @@ class InstrumentationTest : public CommonRuntimeTest { // Check the listener is not registered and is not notified of the event. EXPECT_FALSE(HasEventListener(instr, instrumentation_event)); - EXPECT_FALSE(DidListenerReceiveEvent(listener, instrumentation_event)); - ReportEvent(instr, instrumentation_event, soa.Self(), event_method, event_obj, event_dex_pc); - EXPECT_FALSE(DidListenerReceiveEvent(listener, instrumentation_event)); + EXPECT_FALSE(DidListenerReceiveEvent(listener, instrumentation_event, with_object)); + ReportEvent(instr, + instrumentation_event, + soa.Self(), + event_method, + event_obj, + event_field, + event_dex_pc); + EXPECT_FALSE(DidListenerReceiveEvent(listener, instrumentation_event, with_object)); } void DeoptimizeMethod(Thread* self, ArtMethod* method, bool enable_deoptimization) @@ -317,8 +366,12 @@ class InstrumentationTest : public CommonRuntimeTest { } } - static void ReportEvent(const instrumentation::Instrumentation* instr, uint32_t event_type, - Thread* self, ArtMethod* method, mirror::Object* obj, + static void ReportEvent(const instrumentation::Instrumentation* instr, + uint32_t event_type, + Thread* self, + ArtMethod* method, + mirror::Object* obj, + ArtField* field, uint32_t dex_pc) REQUIRES_SHARED(Locks::mutator_lock_) { switch (event_type) { @@ -337,11 +390,11 @@ class InstrumentationTest : public CommonRuntimeTest { instr->DexPcMovedEvent(self, obj, method, dex_pc); break; case instrumentation::Instrumentation::kFieldRead: - instr->FieldReadEvent(self, obj, method, dex_pc, nullptr); + instr->FieldReadEvent(self, obj, method, dex_pc, field); break; case instrumentation::Instrumentation::kFieldWritten: { JValue value; - instr->FieldWriteEvent(self, obj, method, dex_pc, nullptr, value); + instr->FieldWriteEvent(self, obj, method, dex_pc, field, value); break; } case instrumentation::Instrumentation::kExceptionCaught: { @@ -364,12 +417,14 @@ class InstrumentationTest : public CommonRuntimeTest { } static bool DidListenerReceiveEvent(const TestInstrumentationListener& listener, - uint32_t event_type) { + uint32_t event_type, + bool with_object) { switch (event_type) { case instrumentation::Instrumentation::kMethodEntered: return listener.received_method_enter_event; case instrumentation::Instrumentation::kMethodExited: - return listener.received_method_exit_event; + return (!with_object && listener.received_method_exit_event) || + (with_object && listener.received_method_exit_object_event); case instrumentation::Instrumentation::kMethodUnwind: return listener.received_method_unwind_event; case instrumentation::Instrumentation::kDexPcMoved: @@ -377,7 +432,8 @@ class InstrumentationTest : public CommonRuntimeTest { case instrumentation::Instrumentation::kFieldRead: return listener.received_field_read_event; case instrumentation::Instrumentation::kFieldWritten: - return listener.received_field_written_event; + return (!with_object && listener.received_field_written_event) || + (with_object && listener.received_field_written_object_event); case instrumentation::Instrumentation::kExceptionCaught: return listener.received_exception_caught_event; case instrumentation::Instrumentation::kBranch: @@ -419,8 +475,42 @@ TEST_F(InstrumentationTest, MethodEntryEvent) { TestEvent(instrumentation::Instrumentation::kMethodEntered); } -TEST_F(InstrumentationTest, MethodExitEvent) { - TestEvent(instrumentation::Instrumentation::kMethodExited); +TEST_F(InstrumentationTest, MethodExitObjectEvent) { + ScopedObjectAccess soa(Thread::Current()); + jobject class_loader = LoadDex("Instrumentation"); + Runtime* const runtime = Runtime::Current(); + ClassLinker* class_linker = runtime->GetClassLinker(); + StackHandleScope<1> hs(soa.Self()); + Handle<mirror::ClassLoader> loader(hs.NewHandle(soa.Decode<mirror::ClassLoader>(class_loader))); + mirror::Class* klass = class_linker->FindClass(soa.Self(), "LInstrumentation;", loader); + ASSERT_TRUE(klass != nullptr); + ArtMethod* method = klass->FindDeclaredDirectMethod("returnReference", + "()Ljava/lang/Object;", + kRuntimePointerSize); + ASSERT_TRUE(method != nullptr); + TestEvent(instrumentation::Instrumentation::kMethodExited, + /*event_method*/ method, + /*event_field*/ nullptr, + /*with_object*/ true); +} + +TEST_F(InstrumentationTest, MethodExitPrimEvent) { + ScopedObjectAccess soa(Thread::Current()); + jobject class_loader = LoadDex("Instrumentation"); + Runtime* const runtime = Runtime::Current(); + ClassLinker* class_linker = runtime->GetClassLinker(); + StackHandleScope<1> hs(soa.Self()); + Handle<mirror::ClassLoader> loader(hs.NewHandle(soa.Decode<mirror::ClassLoader>(class_loader))); + mirror::Class* klass = class_linker->FindClass(soa.Self(), "LInstrumentation;", loader); + ASSERT_TRUE(klass != nullptr); + ArtMethod* method = klass->FindDeclaredDirectMethod("returnPrimitive", + "()I", + kRuntimePointerSize); + ASSERT_TRUE(method != nullptr); + TestEvent(instrumentation::Instrumentation::kMethodExited, + /*event_method*/ method, + /*event_field*/ nullptr, + /*with_object*/ false); } TEST_F(InstrumentationTest, MethodUnwindEvent) { @@ -435,8 +525,40 @@ TEST_F(InstrumentationTest, FieldReadEvent) { TestEvent(instrumentation::Instrumentation::kFieldRead); } -TEST_F(InstrumentationTest, FieldWriteEvent) { - TestEvent(instrumentation::Instrumentation::kFieldWritten); +TEST_F(InstrumentationTest, FieldWriteObjectEvent) { + ScopedObjectAccess soa(Thread::Current()); + jobject class_loader = LoadDex("Instrumentation"); + Runtime* const runtime = Runtime::Current(); + ClassLinker* class_linker = runtime->GetClassLinker(); + StackHandleScope<1> hs(soa.Self()); + Handle<mirror::ClassLoader> loader(hs.NewHandle(soa.Decode<mirror::ClassLoader>(class_loader))); + mirror::Class* klass = class_linker->FindClass(soa.Self(), "LInstrumentation;", loader); + ASSERT_TRUE(klass != nullptr); + ArtField* field = klass->FindDeclaredStaticField("referenceField", "Ljava/lang/Object;"); + ASSERT_TRUE(field != nullptr); + + TestEvent(instrumentation::Instrumentation::kFieldWritten, + /*event_method*/ nullptr, + /*event_field*/ field, + /*with_object*/ true); +} + +TEST_F(InstrumentationTest, FieldWritePrimEvent) { + ScopedObjectAccess soa(Thread::Current()); + jobject class_loader = LoadDex("Instrumentation"); + Runtime* const runtime = Runtime::Current(); + ClassLinker* class_linker = runtime->GetClassLinker(); + StackHandleScope<1> hs(soa.Self()); + Handle<mirror::ClassLoader> loader(hs.NewHandle(soa.Decode<mirror::ClassLoader>(class_loader))); + mirror::Class* klass = class_linker->FindClass(soa.Self(), "LInstrumentation;", loader); + ASSERT_TRUE(klass != nullptr); + ArtField* field = klass->FindDeclaredStaticField("primitiveField", "I"); + ASSERT_TRUE(field != nullptr); + + TestEvent(instrumentation::Instrumentation::kFieldWritten, + /*event_method*/ nullptr, + /*event_field*/ field, + /*with_object*/ false); } TEST_F(InstrumentationTest, ExceptionCaughtEvent) { diff --git a/runtime/interpreter/interpreter_switch_impl.cc b/runtime/interpreter/interpreter_switch_impl.cc index b191dd79a1..32a23783b7 100644 --- a/runtime/interpreter/interpreter_switch_impl.cc +++ b/runtime/interpreter/interpreter_switch_impl.cc @@ -307,6 +307,8 @@ JValue ExecuteSwitchImpl(Thread* self, const DexFile::CodeItem* code_item, instrumentation->MethodExitEvent(self, shadow_frame.GetThisObject(code_item->ins_size_), shadow_frame.GetMethod(), inst->GetDexPc(insns), result); + // Re-load since it might have moved during the MethodExitEvent. + result.SetL(shadow_frame.GetVRegReference(ref_idx)); } if (interpret_one_instruction) { /* Signal mterp to return to caller */ diff --git a/runtime/jdwp/jdwp_event.cc b/runtime/jdwp/jdwp_event.cc index 96249f9b58..4ab3d69e35 100644 --- a/runtime/jdwp/jdwp_event.cc +++ b/runtime/jdwp/jdwp_event.cc @@ -500,8 +500,8 @@ static bool ModsMatch(JdwpEvent* pEvent, const ModBasket& basket) } break; case MK_CONDITIONAL: - CHECK(false); // should not be getting these - break; + LOG(FATAL) << "Unexpected MK_CONDITIONAL"; // should not be getting these + UNREACHABLE(); case MK_THREAD_ONLY: if (!Dbg::MatchThread(pMod->threadOnly.threadId, basket.thread)) { return false; diff --git a/runtime/jdwp/jdwp_expand_buf.cc b/runtime/jdwp/jdwp_expand_buf.cc index 961dd369c8..f0b8c918dc 100644 --- a/runtime/jdwp/jdwp_expand_buf.cc +++ b/runtime/jdwp/jdwp_expand_buf.cc @@ -152,7 +152,9 @@ void expandBufAdd8BE(ExpandBuf* pBuf, uint64_t val) { static void SetUtf8String(uint8_t* buf, const char* str, size_t strLen) { Set4BE(buf, strLen); - memcpy(buf + sizeof(uint32_t), str, strLen); + if (str != nullptr) { + memcpy(buf + sizeof(uint32_t), str, strLen); + } } /* diff --git a/runtime/jit/jit_code_cache.cc b/runtime/jit/jit_code_cache.cc index 0cafac7380..6d20b49c37 100644 --- a/runtime/jit/jit_code_cache.cc +++ b/runtime/jit/jit_code_cache.cc @@ -29,6 +29,7 @@ #include "entrypoints/runtime_asm_entrypoints.h" #include "gc/accounting/bitmap-inl.h" #include "gc/scoped_gc_critical_section.h" +#include "intern_table.h" #include "jit/jit.h" #include "jit/profiling_info.h" #include "linear_alloc.h" diff --git a/runtime/jit/profile_compilation_info-inl.h b/runtime/jit/profile_compilation_info-inl.h new file mode 100644 index 0000000000..8a067a5870 --- /dev/null +++ b/runtime/jit/profile_compilation_info-inl.h @@ -0,0 +1,69 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_RUNTIME_JIT_PROFILE_COMPILATION_INFO_INL_H_ +#define ART_RUNTIME_JIT_PROFILE_COMPILATION_INFO_INL_H_ + +#include "profile_compilation_info.h" + +namespace art { + +template <class Iterator> +inline bool ProfileCompilationInfo::AddSampledMethodsForDex(bool startup, + const DexFile* dex_file, + Iterator index_begin, + Iterator index_end) { + DexFileData* data = GetOrAddDexFileData(dex_file); + if (data == nullptr) { + return false; + } + for (auto it = index_begin; it != index_end; ++it) { + DCHECK_LT(*it, data->num_method_ids); + data->AddSampledMethod(startup, *it); + } + return true; +} + +template <class Iterator> +inline bool ProfileCompilationInfo::AddHotMethodsForDex(const DexFile* dex_file, + Iterator index_begin, + Iterator index_end) { + DexFileData* data = GetOrAddDexFileData(dex_file); + if (data == nullptr) { + return false; + } + for (auto it = index_begin; it != index_end; ++it) { + DCHECK_LT(*it, data->num_method_ids); + data->FindOrAddMethod(*it); + } + return true; +} + +template <class Iterator> +inline bool ProfileCompilationInfo::AddClassesForDex(const DexFile* dex_file, + Iterator index_begin, + Iterator index_end) { + DexFileData* data = GetOrAddDexFileData(dex_file); + if (data == nullptr) { + return false; + } + data->class_set.insert(index_begin, index_end); + return true; +} + +} // namespace art + +#endif // ART_RUNTIME_JIT_PROFILE_COMPILATION_INFO_INL_H_ diff --git a/runtime/jit/profile_compilation_info.cc b/runtime/jit/profile_compilation_info.cc index a292a6eab9..ea27d3b300 100644 --- a/runtime/jit/profile_compilation_info.cc +++ b/runtime/jit/profile_compilation_info.cc @@ -47,8 +47,8 @@ namespace art { const uint8_t ProfileCompilationInfo::kProfileMagic[] = { 'p', 'r', 'o', '\0' }; -// Last profile version: Instead of method index, put the difference with the last -// method's index. +// Last profile version: Move startup methods to use a bitmap. Also add support for post-startup +// methods. const uint8_t ProfileCompilationInfo::kProfileVersion[] = { '0', '0', '8', '\0' }; static constexpr uint16_t kMaxDexFileKeyLength = PATH_MAX; @@ -147,18 +147,6 @@ bool ProfileCompilationInfo::AddSampledMethod(bool startup, return true; } -bool ProfileCompilationInfo::AddSampledMethods(bool startup, - std::vector<MethodReference>& methods) { - for (const MethodReference& ref : methods) { - DexFileData* data = GetOrAddDexFileData(ref.dex_file); - if (data == nullptr) { - return false; - } - data->AddSampledMethod(startup, ref.dex_method_index); - } - return true; -} - bool ProfileCompilationInfo::AddMethodsAndClasses( const std::vector<ProfileMethodInfo>& methods, const std::set<DexCacheResolvedClasses>& resolved_classes) { @@ -284,10 +272,13 @@ static constexpr size_t kLineHeaderSize = /** * Serialization format: * magic,version,number_of_dex_files,uncompressed_size_of_zipped_data,compressed_data_size, - * zipped[dex_location1,number_of_classes1,methods_region_size,dex_location_checksum1, \ + * zipped[dex_location1,number_of_classes1,methods_region_size,dex_location_checksum1 + * num_method_ids, * method_encoding_11,method_encoding_12...,class_id1,class_id2... - * dex_location2,number_of_classes2,methods_region_size,dex_location_checksum2, \ + * startup/post startup bitmap, + * dex_location2,number_of_classes2,methods_region_size,dex_location_checksum2, num_method_ids, * method_encoding_21,method_encoding_22...,,class_id1,class_id2... + * startup/post startup bitmap, * .....] * The method_encoding is: * method_id,number_of_inline_caches,inline_cache1,inline_cache2... @@ -995,15 +986,6 @@ bool ProfileCompilationInfo::Load(int fd) { } } -void ProfileCompilationInfo::DexFileData::CreateBitmap() { - const size_t num_bits = num_method_ids * kMethodBitCount; - bitmap_storage.resize(RoundUp(num_bits, kBitsPerByte) / kBitsPerByte); - if (!bitmap_storage.empty()) { - method_bitmap = - BitMemoryRegion(MemoryRegion(&bitmap_storage[0], bitmap_storage.size()), 0, num_bits); - } -} - // TODO(calin): fail fast if the dex checksums don't match. ProfileCompilationInfo::ProfileLoadSatus ProfileCompilationInfo::LoadInternal( int fd, std::string* error) { @@ -1256,7 +1238,7 @@ bool ProfileCompilationInfo::IsStartupOrHotMethod(const std::string& dex_locatio return method_it != methods.end(); } -bool ProfileCompilationInfo::ContainsMethod(const MethodReference& method_ref) const { +bool ProfileCompilationInfo::ContainsHotMethod(const MethodReference& method_ref) const { return FindMethod(method_ref.dex_file->GetLocation(), method_ref.dex_file->GetLocationChecksum(), method_ref.dex_method_index) != nullptr; diff --git a/runtime/jit/profile_compilation_info.h b/runtime/jit/profile_compilation_info.h index 2b89a41dd8..bd1b9d651f 100644 --- a/runtime/jit/profile_compilation_info.h +++ b/runtime/jit/profile_compilation_info.h @@ -197,6 +197,10 @@ class ProfileCompilationInfo { bool AddMethodsAndClasses(const std::vector<ProfileMethodInfo>& methods, const std::set<DexCacheResolvedClasses>& resolved_classes); + // Iterator is type for ids not class defs. + template <class Iterator> + bool AddClassesForDex(const DexFile* dex_file, Iterator index_begin, Iterator index_end); + // Add a method index to the profile (without inline caches). bool AddMethodIndex(const std::string& dex_location, uint32_t checksum, @@ -207,13 +211,24 @@ class ProfileCompilationInfo { bool AddMethod(const ProfileMethodInfo& pmi); // Add methods that have samples but are are not necessarily hot. These are partitioned into two - // possibly interesecting sets startup and post startup. - bool AddSampledMethods(bool startup, std::vector<MethodReference>& methods); + // possibly intersecting sets startup and post startup. bool AddSampledMethod(bool startup, const std::string& dex_location, uint32_t checksum, uint16_t method_idx, uint32_t num_method_ids); + // Bulk add sampled methods for a single dex, fast since it only has one GetOrAddDexFileData call. + template <class Iterator> + bool AddSampledMethodsForDex(bool startup, + const DexFile* dex_file, + Iterator index_begin, + Iterator index_end); + + // Bulk add hot methods for a single dex, fast since it only has one GetOrAddDexFileData call. + template <class Iterator> + bool AddHotMethodsForDex(const DexFile* dex_file, + Iterator index_begin, + Iterator index_end); // Load profile information from the given file descriptor. // If the current profile is non-empty the load will fail. @@ -246,8 +261,8 @@ class ProfileCompilationInfo { uint32_t dex_checksum, uint16_t dex_method_index) const; - // Return true if the method reference is present in the profiling info. - bool ContainsMethod(const MethodReference& method_ref) const; + // Return true if the method reference iS present and hot in the profiling info. + bool ContainsHotMethod(const MethodReference& method_ref) const; // Return true if the class's type is present in the profiling info. bool ContainsClass(const DexFile& dex_file, dex::TypeIndex type_idx) const; @@ -343,7 +358,12 @@ class ProfileCompilationInfo { class_set(std::less<dex::TypeIndex>(), arena->Adapter(kArenaAllocProfile)), num_method_ids(num_methods), bitmap_storage(arena->Adapter(kArenaAllocProfile)) { - CreateBitmap(); + const size_t num_bits = num_method_ids * kBitmapCount; + bitmap_storage.resize(RoundUp(num_bits, kBitsPerByte) / kBitsPerByte); + if (!bitmap_storage.empty()) { + method_bitmap = + BitMemoryRegion(MemoryRegion(&bitmap_storage[0], bitmap_storage.size()), 0, num_bits); + } } bool operator==(const DexFileData& other) const { @@ -359,6 +379,13 @@ class ProfileCompilationInfo { return method_bitmap.LoadBit(MethodBitIndex(startup, index)); } + void MergeBitmap(const DexFileData& other) { + DCHECK_EQ(bitmap_storage.size(), other.bitmap_storage.size()); + for (size_t i = 0; i < bitmap_storage.size(); ++i) { + bitmap_storage[i] |= other.bitmap_storage[i]; + } + } + // The arena used to allocate new inline cache maps. ArenaAllocator* arena_; // The profile key this data belongs to. @@ -380,28 +407,21 @@ class ProfileCompilationInfo { ArenaVector<uint8_t> bitmap_storage; BitMemoryRegion method_bitmap; - void CreateBitmap(); - - void MergeBitmap(const DexFileData& other) { - DCHECK_EQ(bitmap_storage.size(), other.bitmap_storage.size()); - for (size_t i = 0; i < bitmap_storage.size(); ++i) { - bitmap_storage[i] |= other.bitmap_storage[i]; - } - } - private: - enum Bits { - kMethodBitStartup, - kMethodBitAfterStartup, - kMethodBitCount, + enum BitmapIndex { + kBitmapStartup, + kBitmapPostStartup, + kBitmapCount, }; size_t MethodBitIndex(bool startup, size_t index) const { DCHECK_LT(index, num_method_ids); - if (!startup) { - index += num_method_ids; - } - return index; + // The format is [startup bitmap][post startup bitmap] + // This compresses better than ([startup bit][post statup bit])* + + return index + (startup + ? kBitmapStartup * num_method_ids + : kBitmapPostStartup * num_method_ids); } }; diff --git a/runtime/jit/profile_compilation_info_test.cc b/runtime/jit/profile_compilation_info_test.cc index 615149feb3..39670afb0c 100644 --- a/runtime/jit/profile_compilation_info_test.cc +++ b/runtime/jit/profile_compilation_info_test.cc @@ -298,7 +298,8 @@ TEST_F(ProfileCompilationInfoTest, SaveArtMethods) { { ScopedObjectAccess soa(self); for (ArtMethod* m : main_methods) { - ASSERT_TRUE(info1.ContainsMethod(MethodReference(m->GetDexFile(), m->GetDexMethodIndex()))); + ASSERT_TRUE(info1.ContainsHotMethod( + MethodReference(m->GetDexFile(), m->GetDexMethodIndex()))); } } @@ -314,10 +315,12 @@ TEST_F(ProfileCompilationInfoTest, SaveArtMethods) { { ScopedObjectAccess soa(self); for (ArtMethod* m : main_methods) { - ASSERT_TRUE(info2.ContainsMethod(MethodReference(m->GetDexFile(), m->GetDexMethodIndex()))); + ASSERT_TRUE( + info2.ContainsHotMethod(MethodReference(m->GetDexFile(), m->GetDexMethodIndex()))); } for (ArtMethod* m : second_methods) { - ASSERT_TRUE(info2.ContainsMethod(MethodReference(m->GetDexFile(), m->GetDexMethodIndex()))); + ASSERT_TRUE( + info2.ContainsHotMethod(MethodReference(m->GetDexFile(), m->GetDexMethodIndex()))); } } } @@ -663,7 +666,8 @@ TEST_F(ProfileCompilationInfoTest, SaveArtMethodsWithInlineCaches) { { ScopedObjectAccess soa(self); for (ArtMethod* m : main_methods) { - ASSERT_TRUE(info.ContainsMethod(MethodReference(m->GetDexFile(), m->GetDexMethodIndex()))); + ASSERT_TRUE( + info.ContainsHotMethod(MethodReference(m->GetDexFile(), m->GetDexMethodIndex()))); const ProfileMethodInfo& pmi = profile_methods_map.find(m)->second; std::unique_ptr<ProfileCompilationInfo::OfflineProfileMethodInfo> offline_pmi = info.GetMethod(m->GetDexFile()->GetLocation(), diff --git a/runtime/jit/profile_saver.cc b/runtime/jit/profile_saver.cc index c96ca88874..68f33bde03 100644 --- a/runtime/jit/profile_saver.cc +++ b/runtime/jit/profile_saver.cc @@ -25,12 +25,16 @@ #include "art_method-inl.h" #include "base/enums.h" +#include "base/scoped_arena_containers.h" +#include "base/stl_util.h" #include "base/systrace.h" #include "base/time_utils.h" #include "compiler_filter.h" +#include "dex_reference_collection.h" #include "gc/collector_type.h" #include "gc/gc_cause.h" #include "gc/scoped_gc_critical_section.h" +#include "jit/profile_compilation_info-inl.h" #include "oat_file_manager.h" #include "scoped_thread_state_change-inl.h" @@ -179,31 +183,45 @@ void ProfileSaver::NotifyJitActivityInternal() { } } +using MethodReferenceCollection = DexReferenceCollection<uint16_t, ScopedArenaAllocatorAdapter>; +using TypeReferenceCollection = DexReferenceCollection<dex::TypeIndex, + ScopedArenaAllocatorAdapter>; + // Get resolved methods that have a profile info or more than kStartupMethodSamples samples. // Excludes native methods and classes in the boot image. -class GetMethodsVisitor : public ClassVisitor { +class GetClassesAndMethodsVisitor : public ClassVisitor { public: - GetMethodsVisitor(std::vector<MethodReference>* hot_methods, - std::vector<MethodReference>* startup_methods, - uint32_t startup_method_samples) + GetClassesAndMethodsVisitor(MethodReferenceCollection* hot_methods, + MethodReferenceCollection* sampled_methods, + TypeReferenceCollection* resolved_classes, + uint32_t hot_method_sample_threshold) : hot_methods_(hot_methods), - startup_methods_(startup_methods), - startup_method_samples_(startup_method_samples) {} + sampled_methods_(sampled_methods), + resolved_classes_(resolved_classes), + hot_method_sample_threshold_(hot_method_sample_threshold) {} virtual bool operator()(ObjPtr<mirror::Class> klass) REQUIRES_SHARED(Locks::mutator_lock_) { - if (Runtime::Current()->GetHeap()->ObjectIsInBootImageSpace(klass)) { + if (klass->IsProxyClass() || + klass->IsArrayClass() || + !klass->IsResolved() || + klass->IsErroneousResolved() || + klass->GetClassLoader() == nullptr) { return true; } + DCHECK(klass->GetDexCache() != nullptr) << klass->PrettyClass(); + resolved_classes_->AddReference(&klass->GetDexFile(), klass->GetDexTypeIndex()); for (ArtMethod& method : klass->GetMethods(kRuntimePointerSize)) { - if (!method.IsNative() && !method.IsProxyMethod()) { + if (!method.IsNative()) { + DCHECK(!method.IsProxyMethod()); const uint16_t counter = method.GetCounter(); - MethodReference ref(method.GetDexFile(), method.GetDexMethodIndex()); + // Mark startup methods as hot if they have more than hot_method_sample_threshold_ samples. + // This means they will get compiled by the compiler driver. if (method.GetProfilingInfo(kRuntimePointerSize) != nullptr || - (method.GetAccessFlags() & kAccPreviouslyWarm) != 0) { - hot_methods_->push_back(ref); - startup_methods_->push_back(ref); - } else if (counter >= startup_method_samples_) { - startup_methods_->push_back(ref); + (method.GetAccessFlags() & kAccPreviouslyWarm) != 0 || + counter >= hot_method_sample_threshold_) { + hot_methods_->AddReference(method.GetDexFile(), method.GetDexMethodIndex()); + } else if (counter != 0) { + sampled_methods_->AddReference(method.GetDexFile(), method.GetDexMethodIndex()); } } else { CHECK_EQ(method.GetCounter(), 0u); @@ -213,83 +231,95 @@ class GetMethodsVisitor : public ClassVisitor { } private: - std::vector<MethodReference>* const hot_methods_; - std::vector<MethodReference>* const startup_methods_; - uint32_t startup_method_samples_; + MethodReferenceCollection* const hot_methods_; + MethodReferenceCollection* const sampled_methods_; + TypeReferenceCollection* const resolved_classes_; + uint32_t hot_method_sample_threshold_; }; void ProfileSaver::FetchAndCacheResolvedClassesAndMethods() { ScopedTrace trace(__PRETTY_FUNCTION__); + const uint64_t start_time = NanoTime(); // Resolve any new registered locations. ResolveTrackedLocations(); Thread* const self = Thread::Current(); - std::vector<MethodReference> hot_methods; - std::vector<MethodReference> startup_methods; - std::set<DexCacheResolvedClasses> resolved_classes; + Runtime* const runtime = Runtime::Current(); + ArenaStack stack(runtime->GetArenaPool()); + ScopedArenaAllocator allocator(&stack); + MethodReferenceCollection hot_methods(allocator.Adapter(), allocator.Adapter()); + MethodReferenceCollection startup_methods(allocator.Adapter(), allocator.Adapter()); + TypeReferenceCollection resolved_classes(allocator.Adapter(), allocator.Adapter()); + const size_t hot_threshold = options_.GetHotStartupMethodSamples(); { ScopedObjectAccess soa(self); gc::ScopedGCCriticalSection sgcs(self, gc::kGcCauseProfileSaver, gc::kCollectorTypeCriticalSection); - - ClassLinker* const class_linker = Runtime::Current()->GetClassLinker(); - resolved_classes = class_linker->GetResolvedClasses(/*ignore boot classes*/ true); - { ScopedTrace trace2("Get hot methods"); - GetMethodsVisitor visitor(&hot_methods, - &startup_methods, - options_.GetStartupMethodSamples()); - class_linker->VisitClasses(&visitor); - VLOG(profiler) << "Profile saver recorded " << hot_methods.size() << " hot methods and " - << startup_methods.size() << " startup methods with threshold " - << options_.GetStartupMethodSamples(); + GetClassesAndMethodsVisitor visitor(&hot_methods, + &startup_methods, + &resolved_classes, + hot_threshold); + runtime->GetClassLinker()->VisitClasses(&visitor); } } + MutexLock mu(self, *Locks::profiler_lock_); uint64_t total_number_of_profile_entries_cached = 0; for (const auto& it : tracked_dex_base_locations_) { std::set<DexCacheResolvedClasses> resolved_classes_for_location; const std::string& filename = it.first; + auto info_it = profile_cache_.Put( + filename, + new ProfileCompilationInfo(Runtime::Current()->GetArenaPool())); + ProfileCompilationInfo* cached_info = info_it->second; + const std::set<std::string>& locations = it.second; - std::vector<ProfileMethodInfo> profile_methods_for_location; - for (const MethodReference& ref : hot_methods) { - if (locations.find(ref.dex_file->GetBaseLocation()) != locations.end()) { - profile_methods_for_location.emplace_back(ref.dex_file, ref.dex_method_index); + for (const auto& pair : hot_methods.GetMap()) { + const DexFile* const dex_file = pair.first; + if (locations.find(dex_file->GetBaseLocation()) != locations.end()) { + cached_info->AddSampledMethodsForDex(/*startup*/ true, + dex_file, + pair.second.begin(), + pair.second.end()); + // Adding hot methods is a bit slow, TODO: optimize. + cached_info->AddHotMethodsForDex(dex_file, pair.second.begin(), pair.second.end()); } } - std::vector<MethodReference> startup_methods_for_locations; - for (const MethodReference& ref : startup_methods) { - if (locations.find(ref.dex_file->GetBaseLocation()) != locations.end()) { - startup_methods_for_locations.push_back(ref); + for (const auto& pair : startup_methods.GetMap()) { + const DexFile* const dex_file = pair.first; + if (locations.find(dex_file->GetBaseLocation()) != locations.end()) { + cached_info->AddSampledMethodsForDex(/*startup*/ true, + dex_file, + pair.second.begin(), + pair.second.end()); } } - - for (const DexCacheResolvedClasses& classes : resolved_classes) { - if (locations.find(classes.GetBaseLocation()) != locations.end()) { - VLOG(profiler) << "Added " << classes.GetClasses().size() << " classes for location " - << classes.GetBaseLocation() << " (" << classes.GetDexLocation() << ")"; - resolved_classes_for_location.insert(classes); + for (const auto& pair : resolved_classes.GetMap()) { + const DexFile* const dex_file = pair.first; + if (locations.find(dex_file->GetBaseLocation()) != locations.end()) { + const TypeReferenceCollection::IndexVector& classes = pair.second; + VLOG(profiler) << "Added " << classes.size() << " classes for location " + << dex_file->GetBaseLocation() + << " (" << dex_file->GetLocation() << ")"; + cached_info->AddClassesForDex(dex_file, classes.begin(), classes.end()); } else { - VLOG(profiler) << "Location not found " << classes.GetBaseLocation() - << " (" << classes.GetDexLocation() << ")"; + VLOG(profiler) << "Location not found " << dex_file->GetBaseLocation() + << " (" << dex_file->GetLocation() << ")"; } } - auto info_it = profile_cache_.Put( - filename, - new ProfileCompilationInfo(Runtime::Current()->GetArenaPool())); - - ProfileCompilationInfo* cached_info = info_it->second; - cached_info->AddMethodsAndClasses(profile_methods_for_location, resolved_classes_for_location); - cached_info->AddSampledMethods(/*startup*/ true, startup_methods_for_locations); total_number_of_profile_entries_cached += resolved_classes_for_location.size(); } max_number_of_profile_entries_cached_ = std::max( max_number_of_profile_entries_cached_, total_number_of_profile_entries_cached); + VLOG(profiler) << "Profile saver recorded " << hot_methods.NumReferences() << " hot methods and " + << startup_methods.NumReferences() << " startup methods with threshold " + << hot_threshold << " in " << PrettyDuration(NanoTime() - start_time); } bool ProfileSaver::ProcessProfilingInfo(bool force_save, /*out*/uint16_t* number_of_new_methods) { @@ -618,7 +648,7 @@ bool ProfileSaver::HasSeenMethod(const std::string& profile, if (!info.Load(profile, /*clear_if_invalid*/false)) { return false; } - return info.ContainsMethod(MethodReference(dex_file, method_idx)); + return info.ContainsHotMethod(MethodReference(dex_file, method_idx)); } return false; } diff --git a/runtime/jit/profile_saver_options.h b/runtime/jit/profile_saver_options.h index 07aeb66eb6..455bc1aae4 100644 --- a/runtime/jit/profile_saver_options.h +++ b/runtime/jit/profile_saver_options.h @@ -22,8 +22,8 @@ struct ProfileSaverOptions { public: static constexpr uint32_t kMinSavePeriodMs = 40 * 1000; // 40 seconds static constexpr uint32_t kSaveResolvedClassesDelayMs = 5 * 1000; // 5 seconds - // Minimum number of JIT samples during launch to include a method into the profile. - static constexpr uint32_t kStartupMethodSamples = 1; + // Minimum number of JIT samples during launch to mark a method as hot in the profile. + static constexpr uint32_t kHotStartupMethodSamples = 1; static constexpr uint32_t kMinMethodsToSave = 10; static constexpr uint32_t kMinClassesToSave = 10; static constexpr uint32_t kMinNotificationBeforeWake = 10; @@ -33,7 +33,7 @@ struct ProfileSaverOptions { enabled_(false), min_save_period_ms_(kMinSavePeriodMs), save_resolved_classes_delay_ms_(kSaveResolvedClassesDelayMs), - startup_method_samples_(kStartupMethodSamples), + hot_startup_method_samples_(kHotStartupMethodSamples), min_methods_to_save_(kMinMethodsToSave), min_classes_to_save_(kMinClassesToSave), min_notification_before_wake_(kMinNotificationBeforeWake), @@ -44,7 +44,7 @@ struct ProfileSaverOptions { bool enabled, uint32_t min_save_period_ms, uint32_t save_resolved_classes_delay_ms, - uint32_t startup_method_samples, + uint32_t hot_startup_method_samples, uint32_t min_methods_to_save, uint32_t min_classes_to_save, uint32_t min_notification_before_wake, @@ -53,7 +53,7 @@ struct ProfileSaverOptions { enabled_(enabled), min_save_period_ms_(min_save_period_ms), save_resolved_classes_delay_ms_(save_resolved_classes_delay_ms), - startup_method_samples_(startup_method_samples), + hot_startup_method_samples_(hot_startup_method_samples), min_methods_to_save_(min_methods_to_save), min_classes_to_save_(min_classes_to_save), min_notification_before_wake_(min_notification_before_wake), @@ -73,8 +73,8 @@ struct ProfileSaverOptions { uint32_t GetSaveResolvedClassesDelayMs() const { return save_resolved_classes_delay_ms_; } - uint32_t GetStartupMethodSamples() const { - return startup_method_samples_; + uint32_t GetHotStartupMethodSamples() const { + return hot_startup_method_samples_; } uint32_t GetMinMethodsToSave() const { return min_methods_to_save_; @@ -96,7 +96,7 @@ struct ProfileSaverOptions { os << "enabled_" << pso.enabled_ << ", min_save_period_ms_" << pso.min_save_period_ms_ << ", save_resolved_classes_delay_ms_" << pso.save_resolved_classes_delay_ms_ - << ", startup_method_samples_" << pso.startup_method_samples_ + << ", hot_startup_method_samples_" << pso.hot_startup_method_samples_ << ", min_methods_to_save_" << pso.min_methods_to_save_ << ", min_classes_to_save_" << pso.min_classes_to_save_ << ", min_notification_before_wake_" << pso.min_notification_before_wake_ @@ -107,7 +107,7 @@ struct ProfileSaverOptions { bool enabled_; uint32_t min_save_period_ms_; uint32_t save_resolved_classes_delay_ms_; - uint32_t startup_method_samples_; + uint32_t hot_startup_method_samples_; uint32_t min_methods_to_save_; uint32_t min_classes_to_save_; uint32_t min_notification_before_wake_; diff --git a/runtime/mem_map.cc b/runtime/mem_map.cc index 12793e433e..c847942fd1 100644 --- a/runtime/mem_map.cc +++ b/runtime/mem_map.cc @@ -23,6 +23,7 @@ #include <sys/resource.h> #endif +#include <map> #include <memory> #include <sstream> @@ -32,6 +33,7 @@ #include "cutils/ashmem.h" #include "base/allocator.h" +#include "base/bit_utils.h" #include "base/memory_tool.h" #include "globals.h" #include "utils.h" @@ -46,6 +48,10 @@ namespace art { using android::base::StringPrintf; using android::base::unique_fd; +template<class Key, class T, AllocatorTag kTag, class Compare = std::less<Key>> +using AllocationTrackingMultiMap = + std::multimap<Key, T, Compare, TrackingAllocator<std::pair<const Key, T>, kTag>>; + using Maps = AllocationTrackingMultiMap<void*, MemMap*, kAllocatorTagMaps>; // All the non-empty MemMaps. Use a multimap as we do a reserve-and-divide (eg ElfMap::Load()). @@ -187,7 +193,7 @@ static bool CheckNonOverlapping(uintptr_t begin, *error_msg = StringPrintf("Failed to build process map"); return false; } - ScopedBacktraceMapIteratorLock(map.get()); + ScopedBacktraceMapIteratorLock lock(map.get()); for (BacktraceMap::const_iterator it = map->begin(); it != map->end(); ++it) { if ((begin >= it->start && begin < it->end) // start of new within old || (end > it->start && end < it->end) // end of new within old diff --git a/runtime/method_bss_mapping.h b/runtime/method_bss_mapping.h new file mode 100644 index 0000000000..1476f93e21 --- /dev/null +++ b/runtime/method_bss_mapping.h @@ -0,0 +1,57 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_RUNTIME_METHOD_BSS_MAPPING_H_ +#define ART_RUNTIME_METHOD_BSS_MAPPING_H_ + +#include "base/bit_utils.h" +#include "base/length_prefixed_array.h" + +namespace art { + +// MethodBssMappingEntry describes a mapping of up to 17 method indexes to their offsets +// in the .bss. The highest index and its associated .bss offset are stored in plain form +// as `method_index` and `bss_offset`, respectively, while the additional indexes can be +// stored in compressed form if their associated .bss entries are consecutive and in the +// method index order. Each of the 16 bits of the `index_mask` corresponds to one of the +// previous 16 method indexes and indicates whether there is a .bss entry for that index. +// +struct MethodBssMappingEntry { + bool CoversIndex(uint32_t method_idx) const { + uint32_t diff = method_index - method_idx; + return (diff == 0) || (diff <= 16 && ((index_mask >> (16u - diff)) & 1u) != 0); + } + + uint32_t GetBssOffset(uint32_t method_idx, size_t entry_size) const { + DCHECK(CoversIndex(method_idx)); + uint32_t diff = method_index - method_idx; + if (diff == 0) { + return bss_offset; + } else { + return bss_offset - POPCOUNT(index_mask >> (16u - diff)) * entry_size; + } + } + + uint16_t method_index; + uint16_t index_mask; + uint32_t bss_offset; +}; + +using MethodBssMapping = LengthPrefixedArray<MethodBssMappingEntry>; + +} // namespace art + +#endif // ART_RUNTIME_METHOD_BSS_MAPPING_H_ diff --git a/runtime/method_reference.h b/runtime/method_reference.h index 0b0afe64a6..3948ed5bb9 100644 --- a/runtime/method_reference.h +++ b/runtime/method_reference.h @@ -44,6 +44,56 @@ struct MethodReferenceComparator { } }; +// Compare the actual referenced method signatures. Used for method reference deduplication. +struct MethodReferenceValueComparator { + bool operator()(MethodReference mr1, MethodReference mr2) const { + if (mr1.dex_file == mr2.dex_file) { + DCHECK_EQ(mr1.dex_method_index < mr2.dex_method_index, SlowCompare(mr1, mr2)); + return mr1.dex_method_index < mr2.dex_method_index; + } else { + return SlowCompare(mr1, mr2); + } + } + + bool SlowCompare(MethodReference mr1, MethodReference mr2) const { + // The order is the same as for method ids in a single dex file. + // Compare the class descriptors first. + const DexFile::MethodId& mid1 = mr1.dex_file->GetMethodId(mr1.dex_method_index); + const DexFile::MethodId& mid2 = mr2.dex_file->GetMethodId(mr2.dex_method_index); + int descriptor_diff = strcmp(mr1.dex_file->StringByTypeIdx(mid1.class_idx_), + mr2.dex_file->StringByTypeIdx(mid2.class_idx_)); + if (descriptor_diff != 0) { + return descriptor_diff < 0; + } + // Compare names second. + int name_diff = strcmp(mr1.dex_file->GetMethodName(mid1), mr2.dex_file->GetMethodName(mid2)); + if (name_diff != 0) { + return name_diff < 0; + } + // And then compare proto ids, starting with return type comparison. + const DexFile::ProtoId& prid1 = mr1.dex_file->GetProtoId(mid1.proto_idx_); + const DexFile::ProtoId& prid2 = mr2.dex_file->GetProtoId(mid2.proto_idx_); + int return_type_diff = strcmp(mr1.dex_file->StringByTypeIdx(prid1.return_type_idx_), + mr2.dex_file->StringByTypeIdx(prid2.return_type_idx_)); + if (return_type_diff != 0) { + return return_type_diff < 0; + } + // And finishing with lexicographical parameter comparison. + const DexFile::TypeList* params1 = mr1.dex_file->GetProtoParameters(prid1); + size_t param1_size = (params1 != nullptr) ? params1->Size() : 0u; + const DexFile::TypeList* params2 = mr2.dex_file->GetProtoParameters(prid2); + size_t param2_size = (params2 != nullptr) ? params2->Size() : 0u; + for (size_t i = 0, num = std::min(param1_size, param2_size); i != num; ++i) { + int param_diff = strcmp(mr1.dex_file->StringByTypeIdx(params1->GetTypeItem(i).type_idx_), + mr2.dex_file->StringByTypeIdx(params2->GetTypeItem(i).type_idx_)); + if (param_diff != 0) { + return param_diff < 0; + } + } + return param1_size < param2_size; + } +}; + } // namespace art #endif // ART_RUNTIME_METHOD_REFERENCE_H_ diff --git a/runtime/mirror/array.h b/runtime/mirror/array.h index 7287a92fa5..99565c6f5d 100644 --- a/runtime/mirror/array.h +++ b/runtime/mirror/array.h @@ -188,6 +188,16 @@ class MANAGED PrimitiveArray : public Array { DISALLOW_IMPLICIT_CONSTRUCTORS(PrimitiveArray); }; +// Declare the different primitive arrays. Instantiations will be in array.cc. +extern template class PrimitiveArray<uint8_t>; // BooleanArray +extern template class PrimitiveArray<int8_t>; // ByteArray +extern template class PrimitiveArray<uint16_t>; // CharArray +extern template class PrimitiveArray<double>; // DoubleArray +extern template class PrimitiveArray<float>; // FloatArray +extern template class PrimitiveArray<int32_t>; // IntArray +extern template class PrimitiveArray<int64_t>; // LongArray +extern template class PrimitiveArray<int16_t>; // ShortArray + // Either an IntArray or a LongArray. class PointerArray : public Array { public: diff --git a/runtime/mirror/class.cc b/runtime/mirror/class.cc index 06ee3d36fe..e4b53209e9 100644 --- a/runtime/mirror/class.cc +++ b/runtime/mirror/class.cc @@ -1143,9 +1143,7 @@ uint32_t Class::Depth() { dex::TypeIndex Class::FindTypeIndexInOtherDexFile(const DexFile& dex_file) { std::string temp; const DexFile::TypeId* type_id = dex_file.FindTypeId(GetDescriptor(&temp)); - return (type_id == nullptr) - ? dex::TypeIndex(DexFile::kDexNoIndex) - : dex_file.GetIndexForTypeId(*type_id); + return (type_id == nullptr) ? dex::TypeIndex() : dex_file.GetIndexForTypeId(*type_id); } template <PointerSize kPointerSize, bool kTransactionActive> diff --git a/runtime/mirror/class.h b/runtime/mirror/class.h index dfdd16240b..913ab796a1 100644 --- a/runtime/mirror/class.h +++ b/runtime/mirror/class.h @@ -17,6 +17,7 @@ #ifndef ART_RUNTIME_MIRROR_CLASS_H_ #define ART_RUNTIME_MIRROR_CLASS_H_ +#include "base/bit_utils.h" #include "base/enums.h" #include "base/iteration_range.h" #include "dex_file.h" diff --git a/runtime/mirror/string-inl.h b/runtime/mirror/string-inl.h index 57b20a193b..75606391ad 100644 --- a/runtime/mirror/string-inl.h +++ b/runtime/mirror/string-inl.h @@ -26,7 +26,6 @@ #include "common_throws.h" #include "gc/heap-inl.h" #include "globals.h" -#include "intern_table.h" #include "runtime.h" #include "thread.h" #include "utf.h" @@ -161,10 +160,6 @@ class SetStringCountAndValueVisitorFromString { const int32_t offset_; }; -inline ObjPtr<String> String::Intern() { - return Runtime::Current()->GetInternTable()->InternWeak(this); -} - inline uint16_t String::CharAt(int32_t index) { int32_t count = GetLength(); if (UNLIKELY((index < 0) || (index >= count))) { diff --git a/runtime/mirror/string.cc b/runtime/mirror/string.cc index 80745d271d..82ff6ddead 100644 --- a/runtime/mirror/string.cc +++ b/runtime/mirror/string.cc @@ -19,6 +19,7 @@ #include "arch/memcmp16.h" #include "array.h" #include "base/array_ref.h" +#include "base/stl_util.h" #include "class-inl.h" #include "gc/accounting/card_table-inl.h" #include "gc_root-inl.h" @@ -420,5 +421,9 @@ std::string String::PrettyStringDescriptor() { return PrettyDescriptor(ToModifiedUtf8().c_str()); } +ObjPtr<String> String::Intern() { + return Runtime::Current()->GetInternTable()->InternWeak(this); +} + } // namespace mirror } // namespace art diff --git a/runtime/oat.cc b/runtime/oat.cc index d14b399a9a..21e20e9b74 100644 --- a/runtime/oat.cc +++ b/runtime/oat.cc @@ -23,6 +23,7 @@ #include "arch/instruction_set_features.h" #include "base/bit_utils.h" +#include "base/strlcpy.h" namespace art { @@ -71,6 +72,7 @@ OatHeader::OatHeader(InstructionSet instruction_set, instruction_set_(instruction_set), instruction_set_features_bitmap_(instruction_set_features->AsBitmap()), dex_file_count_(dex_file_count), + oat_dex_files_offset_(0), executable_offset_(0), interpreter_to_interpreter_bridge_offset_(0), interpreter_to_compiled_code_bridge_offset_(0), @@ -203,6 +205,20 @@ uint32_t OatHeader::GetInstructionSetFeaturesBitmap() const { return instruction_set_features_bitmap_; } +uint32_t OatHeader::GetOatDexFilesOffset() const { + DCHECK(IsValid()); + DCHECK_GT(oat_dex_files_offset_, sizeof(OatHeader)); + return oat_dex_files_offset_; +} + +void OatHeader::SetOatDexFilesOffset(uint32_t oat_dex_files_offset) { + DCHECK_GT(oat_dex_files_offset, sizeof(OatHeader)); + DCHECK(IsValid()); + DCHECK_EQ(oat_dex_files_offset_, 0u); + + oat_dex_files_offset_ = oat_dex_files_offset; +} + uint32_t OatHeader::GetExecutableOffset() const { DCHECK(IsValid()); DCHECK_ALIGNED(executable_offset_, kPageSize); @@ -505,9 +521,9 @@ void OatHeader::Flatten(const SafeMap<std::string, std::string>* key_value_store SafeMap<std::string, std::string>::const_iterator it = key_value_store->begin(); SafeMap<std::string, std::string>::const_iterator end = key_value_store->end(); for ( ; it != end; ++it) { - strcpy(data_ptr, it->first.c_str()); + strlcpy(data_ptr, it->first.c_str(), it->first.length() + 1); data_ptr += it->first.length() + 1; - strcpy(data_ptr, it->second.c_str()); + strlcpy(data_ptr, it->second.c_str(), it->second.length() + 1); data_ptr += it->second.length() + 1; } } diff --git a/runtime/oat.h b/runtime/oat.h index 57c2f9f6e6..521cc40764 100644 --- a/runtime/oat.h +++ b/runtime/oat.h @@ -32,7 +32,7 @@ class InstructionSetFeatures; class PACKED(4) OatHeader { public: static constexpr uint8_t kOatMagic[] = { 'o', 'a', 't', '\n' }; - static constexpr uint8_t kOatVersion[] = { '1', '2', '6', '\0' }; // Shuffle access flags. + static constexpr uint8_t kOatVersion[] = { '1', '2', '7', '\0' }; // .bss ArtMethod* section. static constexpr const char* kImageLocationKey = "image-location"; static constexpr const char* kDex2OatCmdLineKey = "dex2oat-cmdline"; @@ -64,6 +64,8 @@ class PACKED(4) OatHeader { DCHECK(IsValid()); return dex_file_count_; } + uint32_t GetOatDexFilesOffset() const; + void SetOatDexFilesOffset(uint32_t oat_dex_files_offset); uint32_t GetExecutableOffset() const; void SetExecutableOffset(uint32_t executable_offset); @@ -135,6 +137,7 @@ class PACKED(4) OatHeader { InstructionSet instruction_set_; uint32_t instruction_set_features_bitmap_; uint32_t dex_file_count_; + uint32_t oat_dex_files_offset_; uint32_t executable_offset_; uint32_t interpreter_to_interpreter_bridge_offset_; uint32_t interpreter_to_compiled_code_bridge_offset_; diff --git a/runtime/oat_file.cc b/runtime/oat_file.cc index a816522b9e..888de457dc 100644 --- a/runtime/oat_file.cc +++ b/runtime/oat_file.cc @@ -246,6 +246,9 @@ bool OatFileBase::ComputeFields(uint8_t* requested_base, } // Readjust to be non-inclusive upper bound. bss_end_ += sizeof(uint32_t); + // Find bss methods if present. + bss_methods_ = + const_cast<uint8_t*>(FindDynamicSymbolAddress("oatbssmethods", &symbol_error_msg)); // Find bss roots if present. bss_roots_ = const_cast<uint8_t*>(FindDynamicSymbolAddress("oatbssroots", &symbol_error_msg)); } @@ -311,51 +314,63 @@ bool OatFileBase::Setup(const char* abs_dex_location, std::string* error_msg) { cause.c_str()); return false; } - const uint8_t* oat = Begin(); - oat += sizeof(OatHeader); - if (oat > End()) { - *error_msg = StringPrintf("In oat file '%s' found truncated OatHeader", GetLocation().c_str()); + PointerSize pointer_size = GetInstructionSetPointerSize(GetOatHeader().GetInstructionSet()); + size_t key_value_store_size = + (Size() >= sizeof(OatHeader)) ? GetOatHeader().GetKeyValueStoreSize() : 0u; + if (Size() < sizeof(OatHeader) + key_value_store_size) { + *error_msg = StringPrintf("In oat file '%s' found truncated OatHeader, " + "size = %zu < %zu + %zu", + GetLocation().c_str(), + Size(), + sizeof(OatHeader), + key_value_store_size); return false; } - oat += GetOatHeader().GetKeyValueStoreSize(); - if (oat > End()) { - *error_msg = StringPrintf("In oat file '%s' found truncated variable-size data: " - "%p + %zu + %u <= %p", + size_t oat_dex_files_offset = GetOatHeader().GetOatDexFilesOffset(); + if (oat_dex_files_offset < GetOatHeader().GetHeaderSize() || oat_dex_files_offset > Size()) { + *error_msg = StringPrintf("In oat file '%s' found invalid oat dex files offset: " + "%zu is not in [%zu, %zu]", GetLocation().c_str(), - Begin(), - sizeof(OatHeader), - GetOatHeader().GetKeyValueStoreSize(), - End()); + oat_dex_files_offset, + GetOatHeader().GetHeaderSize(), + Size()); return false; } + const uint8_t* oat = Begin() + oat_dex_files_offset; // Jump to the OatDexFile records. - if (!IsAligned<alignof(GcRoot<mirror::Object>)>(bss_begin_) || - !IsAligned<alignof(GcRoot<mirror::Object>)>(bss_roots_) || + DCHECK_GE(static_cast<size_t>(pointer_size), alignof(GcRoot<mirror::Object>)); + if (!IsAligned<kPageSize>(bss_begin_) || + !IsAlignedParam(bss_methods_, static_cast<size_t>(pointer_size)) || + !IsAlignedParam(bss_roots_, static_cast<size_t>(pointer_size)) || !IsAligned<alignof(GcRoot<mirror::Object>)>(bss_end_)) { *error_msg = StringPrintf("In oat file '%s' found unaligned bss symbol(s): " - "begin = %p, roots = %p, end = %p", + "begin = %p, methods_ = %p, roots = %p, end = %p", GetLocation().c_str(), bss_begin_, + bss_methods_, bss_roots_, bss_end_); return false; } - if (bss_roots_ != nullptr && (bss_roots_ < bss_begin_ || bss_roots_ > bss_end_)) { - *error_msg = StringPrintf("In oat file '%s' found bss roots outside .bss: " - "%p is outside range [%p, %p]", + if ((bss_methods_ != nullptr && (bss_methods_ < bss_begin_ || bss_methods_ > bss_end_)) || + (bss_roots_ != nullptr && (bss_roots_ < bss_begin_ || bss_roots_ > bss_end_)) || + (bss_methods_ != nullptr && bss_roots_ != nullptr && bss_methods_ > bss_roots_)) { + *error_msg = StringPrintf("In oat file '%s' found bss symbol(s) outside .bss or unordered: " + "begin = %p, methods_ = %p, roots = %p, end = %p", GetLocation().c_str(), - bss_roots_, bss_begin_, + bss_methods_, + bss_roots_, bss_end_); return false; } - PointerSize pointer_size = GetInstructionSetPointerSize(GetOatHeader().GetInstructionSet()); - uint8_t* dex_cache_arrays = (bss_begin_ == bss_roots_) ? nullptr : bss_begin_; + uint8_t* after_arrays = (bss_methods_ != nullptr) ? bss_methods_ : bss_roots_; // May be null. + uint8_t* dex_cache_arrays = (bss_begin_ == after_arrays) ? nullptr : bss_begin_; uint8_t* dex_cache_arrays_end = - (bss_begin_ == bss_roots_) ? nullptr : (bss_roots_ != nullptr) ? bss_roots_ : bss_end_; + (bss_begin_ == after_arrays) ? nullptr : (after_arrays != nullptr) ? after_arrays : bss_end_; DCHECK_EQ(dex_cache_arrays != nullptr, dex_cache_arrays_end != nullptr); uint32_t dex_file_count = GetOatHeader().GetDexFileCount(); oat_dex_files_storage_.reserve(dex_file_count); @@ -529,6 +544,55 @@ bool OatFileBase::Setup(const char* abs_dex_location, std::string* error_msg) { return false; } + uint32_t method_bss_mapping_offset; + if (UNLIKELY(!ReadOatDexFileData(*this, &oat, &method_bss_mapping_offset))) { + *error_msg = StringPrintf("In oat file '%s' found OatDexFile #%zd for '%s' truncated " + "after method bss mapping offset", + GetLocation().c_str(), + i, + dex_file_location.c_str()); + return false; + } + const bool readable_method_bss_mapping_size = + method_bss_mapping_offset != 0u && + method_bss_mapping_offset <= Size() && + IsAligned<alignof(MethodBssMapping)>(method_bss_mapping_offset) && + Size() - method_bss_mapping_offset >= MethodBssMapping::ComputeSize(0); + const MethodBssMapping* method_bss_mapping = readable_method_bss_mapping_size + ? reinterpret_cast<const MethodBssMapping*>(Begin() + method_bss_mapping_offset) + : nullptr; + if (method_bss_mapping_offset != 0u && + (UNLIKELY(method_bss_mapping == nullptr) || + UNLIKELY(method_bss_mapping->size() == 0u) || + UNLIKELY(Size() - method_bss_mapping_offset < + MethodBssMapping::ComputeSize(method_bss_mapping->size())))) { + *error_msg = StringPrintf("In oat file '%s' found OatDexFile #%zu for '%s' with unaligned or " + " truncated method bss mapping, offset %u of %zu, length %zu", + GetLocation().c_str(), + i, + dex_file_location.c_str(), + method_bss_mapping_offset, + Size(), + method_bss_mapping != nullptr ? method_bss_mapping->size() : 0u); + return false; + } + if (kIsDebugBuild && method_bss_mapping != nullptr) { + const MethodBssMappingEntry* prev_entry = nullptr; + for (const MethodBssMappingEntry& entry : *method_bss_mapping) { + CHECK_ALIGNED_PARAM(entry.bss_offset, static_cast<size_t>(pointer_size)); + CHECK_LT(entry.bss_offset, BssSize()); + CHECK_LE(POPCOUNT(entry.index_mask) * static_cast<size_t>(pointer_size), entry.bss_offset); + size_t index_mask_span = (entry.index_mask != 0u) ? 16u - CTZ(entry.index_mask) : 0u; + CHECK_LE(index_mask_span, entry.method_index); + if (prev_entry != nullptr) { + CHECK_LT(prev_entry->method_index, entry.method_index - index_mask_span); + } + prev_entry = &entry; + } + CHECK_LT(prev_entry->method_index, + reinterpret_cast<const DexFile::Header*>(dex_file_pointer)->method_ids_size_); + } + uint8_t* current_dex_cache_arrays = nullptr; if (dex_cache_arrays != nullptr) { // All DexCache types except for CallSite have their instance counts in the @@ -569,6 +633,7 @@ bool OatFileBase::Setup(const char* abs_dex_location, std::string* error_msg) { dex_file_checksum, dex_file_pointer, lookup_table_data, + method_bss_mapping, class_offsets_pointer, current_dex_cache_arrays); oat_dex_files_storage_.push_back(oat_dex_file); @@ -1158,6 +1223,7 @@ OatFile::OatFile(const std::string& location, bool is_executable) end_(nullptr), bss_begin_(nullptr), bss_end_(nullptr), + bss_methods_(nullptr), bss_roots_(nullptr), is_executable_(is_executable), secondary_lookup_lock_("OatFile secondary lookup lock", kOatFileSecondaryLookupLock) { @@ -1198,6 +1264,17 @@ const uint8_t* OatFile::DexEnd() const { return kIsVdexEnabled ? vdex_->End() : End(); } +ArrayRef<ArtMethod*> OatFile::GetBssMethods() const { + if (bss_methods_ != nullptr) { + ArtMethod** methods = reinterpret_cast<ArtMethod**>(bss_methods_); + ArtMethod** methods_end = + reinterpret_cast<ArtMethod**>(bss_roots_ != nullptr ? bss_roots_ : bss_end_); + return ArrayRef<ArtMethod*>(methods, methods_end - methods); + } else { + return ArrayRef<ArtMethod*>(); + } +} + ArrayRef<GcRoot<mirror::Object>> OatFile::GetBssGcRoots() const { if (bss_roots_ != nullptr) { auto* roots = reinterpret_cast<GcRoot<mirror::Object>*>(bss_roots_); @@ -1283,6 +1360,7 @@ OatFile::OatDexFile::OatDexFile(const OatFile* oat_file, uint32_t dex_file_location_checksum, const uint8_t* dex_file_pointer, const uint8_t* lookup_table_data, + const MethodBssMapping* method_bss_mapping_data, const uint32_t* oat_class_offsets_pointer, uint8_t* dex_cache_arrays) : oat_file_(oat_file), @@ -1291,6 +1369,7 @@ OatFile::OatDexFile::OatDexFile(const OatFile* oat_file, dex_file_location_checksum_(dex_file_location_checksum), dex_file_pointer_(dex_file_pointer), lookup_table_data_(lookup_table_data), + method_bss_mapping_(method_bss_mapping_data), oat_class_offsets_pointer_(oat_class_offsets_pointer), dex_cache_arrays_(dex_cache_arrays) { // Initialize TypeLookupTable. diff --git a/runtime/oat_file.h b/runtime/oat_file.h index a6d2eba354..66ed44f1b9 100644 --- a/runtime/oat_file.h +++ b/runtime/oat_file.h @@ -26,6 +26,7 @@ #include "base/stringpiece.h" #include "compiler_filter.h" #include "dex_file.h" +#include "method_bss_mapping.h" #include "mirror/class.h" #include "oat.h" #include "os.h" @@ -257,8 +258,14 @@ class OatFile { return BssEnd() - BssBegin(); } + size_t BssMethodsOffset() const { + // Note: This is used only for symbolizer and needs to return a valid .bss offset. + return (bss_methods_ != nullptr) ? bss_methods_ - BssBegin() : BssRootsOffset(); + } + size_t BssRootsOffset() const { - return bss_roots_ - BssBegin(); + // Note: This is used only for symbolizer and needs to return a valid .bss offset. + return (bss_roots_ != nullptr) ? bss_roots_ - BssBegin() : BssSize(); } size_t DexSize() const { @@ -274,6 +281,7 @@ class OatFile { const uint8_t* DexBegin() const; const uint8_t* DexEnd() const; + ArrayRef<ArtMethod*> GetBssMethods() const; ArrayRef<GcRoot<mirror::Object>> GetBssGcRoots() const; // Returns the absolute dex location for the encoded relative dex location. @@ -325,6 +333,9 @@ class OatFile { // Pointer to the end of the .bss section, if present, otherwise null. uint8_t* bss_end_; + // Pointer to the beginning of the ArtMethod*s in .bss section, if present, otherwise null. + uint8_t* bss_methods_; + // Pointer to the beginning of the GC roots in .bss section, if present, otherwise null. uint8_t* bss_roots_; @@ -422,6 +433,10 @@ class OatDexFile FINAL { return lookup_table_data_; } + const MethodBssMapping* GetMethodBssMapping() const { + return method_bss_mapping_; + } + const uint8_t* GetDexFilePointer() const { return dex_file_pointer_; } @@ -448,6 +463,7 @@ class OatDexFile FINAL { uint32_t dex_file_checksum, const uint8_t* dex_file_pointer, const uint8_t* lookup_table_data, + const MethodBssMapping* method_bss_mapping, const uint32_t* oat_class_offsets_pointer, uint8_t* dex_cache_arrays); @@ -458,7 +474,8 @@ class OatDexFile FINAL { const std::string canonical_dex_file_location_; const uint32_t dex_file_location_checksum_ = 0u; const uint8_t* const dex_file_pointer_ = nullptr; - const uint8_t* lookup_table_data_ = nullptr; + const uint8_t* const lookup_table_data_ = nullptr; + const MethodBssMapping* const method_bss_mapping_ = nullptr; const uint32_t* const oat_class_offsets_pointer_ = 0u; uint8_t* const dex_cache_arrays_ = nullptr; mutable std::unique_ptr<TypeLookupTable> lookup_table_; diff --git a/runtime/oat_file_assistant.cc b/runtime/oat_file_assistant.cc index 9e08b34b27..2e2e8c3ef6 100644 --- a/runtime/oat_file_assistant.cc +++ b/runtime/oat_file_assistant.cc @@ -24,6 +24,7 @@ #include "android-base/strings.h" #include "base/logging.h" +#include "base/stl_util.h" #include "compiler_filter.h" #include "class_linker.h" #include "exec_utils.h" @@ -239,7 +240,7 @@ OatFileAssistant::MakeUpToDate(bool profile_changed, std::string* error_msg) { case kDex2OatForBootImage: case kDex2OatForRelocation: case kDex2OatForFilter: - return GenerateOatFileNoChecks(info, error_msg); + return GenerateOatFileNoChecks(info, target, error_msg); } UNREACHABLE(); } @@ -614,7 +615,7 @@ static bool PrepareOdexDirectories(const std::string& dex_location, } OatFileAssistant::ResultOfAttemptToUpdate OatFileAssistant::GenerateOatFileNoChecks( - OatFileAssistant::OatFileInfo& info, std::string* error_msg) { + OatFileAssistant::OatFileInfo& info, CompilerFilter::Filter filter, std::string* error_msg) { CHECK(error_msg != nullptr); Runtime* runtime = Runtime::Current(); @@ -689,6 +690,7 @@ OatFileAssistant::ResultOfAttemptToUpdate OatFileAssistant::GenerateOatFileNoChe args.push_back("--output-vdex-fd=" + std::to_string(vdex_file->Fd())); args.push_back("--oat-fd=" + std::to_string(oat_file->Fd())); args.push_back("--oat-location=" + oat_file_name); + args.push_back("--compiler-filter=" + CompilerFilter::NameOfFilter(filter)); if (!Dex2Oat(args, error_msg)) { // Manually delete the oat and vdex files. This ensures there is no garbage diff --git a/runtime/oat_file_assistant.h b/runtime/oat_file_assistant.h index 7e2385ec6c..03d9ca38a8 100644 --- a/runtime/oat_file_assistant.h +++ b/runtime/oat_file_assistant.h @@ -366,14 +366,16 @@ class OatFileAssistant { }; // Generate the oat file for the given info from the dex file using the - // current runtime compiler options. + // current runtime compiler options and the specified filter. // This does not check the current status before attempting to generate the // oat file. // // If the result is not kUpdateSucceeded, the value of error_msg will be set // to a string describing why there was a failure or the update was not // attempted. error_msg must not be null. - ResultOfAttemptToUpdate GenerateOatFileNoChecks(OatFileInfo& info, std::string* error_msg); + ResultOfAttemptToUpdate GenerateOatFileNoChecks(OatFileInfo& info, + CompilerFilter::Filter target, + std::string* error_msg); // Return info for the best oat file. OatFileInfo& GetBestInfo(); diff --git a/runtime/oat_file_assistant_test.cc b/runtime/oat_file_assistant_test.cc index c2029165ad..3619129718 100644 --- a/runtime/oat_file_assistant_test.cc +++ b/runtime/oat_file_assistant_test.cc @@ -1232,6 +1232,25 @@ TEST_F(OatFileAssistantTest, DexOptStatusValues) { } } +// Verify that when no compiler filter is passed the default one from OatFileAssistant is used. +TEST_F(OatFileAssistantTest, DefaultMakeUpToDateFilter) { + std::string dex_location = GetScratchDir() + "/TestDex.jar"; + Copy(GetDexSrc1(), dex_location); + + OatFileAssistant oat_file_assistant(dex_location.c_str(), kRuntimeISA, false); + + const CompilerFilter::Filter default_filter = + OatFileAssistant::kDefaultCompilerFilterForDexLoading; + std::string error_msg; + EXPECT_EQ(OatFileAssistant::kUpdateSucceeded, + oat_file_assistant.MakeUpToDate(false, &error_msg)) << error_msg; + EXPECT_EQ(-OatFileAssistant::kNoDexOptNeeded, + oat_file_assistant.GetDexOptNeeded(default_filter)); + std::unique_ptr<OatFile> oat_file = oat_file_assistant.GetBestOatFile(); + EXPECT_NE(nullptr, oat_file.get()); + EXPECT_EQ(default_filter, oat_file->GetCompilerFilter()); +} + // TODO: More Tests: // * Test class linker falls back to unquickened dex for DexNoOat // * Test class linker falls back to unquickened dex for MultiDexNoOat diff --git a/runtime/obj_ptr-inl.h b/runtime/obj_ptr-inl.h index 3d9b3c6cf7..f1e3b5053b 100644 --- a/runtime/obj_ptr-inl.h +++ b/runtime/obj_ptr-inl.h @@ -17,6 +17,7 @@ #ifndef ART_RUNTIME_OBJ_PTR_INL_H_ #define ART_RUNTIME_OBJ_PTR_INL_H_ +#include "base/bit_utils.h" #include "obj_ptr.h" #include "thread-current-inl.h" diff --git a/runtime/openjdkjvmti/art_jvmti.h b/runtime/openjdkjvmti/art_jvmti.h index 2a2aa4c199..4b83b7c873 100644 --- a/runtime/openjdkjvmti/art_jvmti.h +++ b/runtime/openjdkjvmti/art_jvmti.h @@ -41,6 +41,7 @@ #include "base/casts.h" #include "base/logging.h" #include "base/macros.h" +#include "base/strlcpy.h" #include "events.h" #include "java_vm_ext.h" #include "jni_env_ext.h" @@ -187,7 +188,7 @@ static inline JvmtiUniquePtr<char[]> CopyString(jvmtiEnv* env, const char* src, size_t len = strlen(src) + 1; JvmtiUniquePtr<char[]> ret = AllocJvmtiUniquePtr<char[]>(env, len, error); if (ret != nullptr) { - strcpy(ret.get(), src); + strlcpy(ret.get(), src, len); } return ret; } diff --git a/runtime/openjdkjvmti/ti_class.cc b/runtime/openjdkjvmti/ti_class.cc index 0aa93dfb57..ed54cd13c3 100644 --- a/runtime/openjdkjvmti/ti_class.cc +++ b/runtime/openjdkjvmti/ti_class.cc @@ -103,7 +103,8 @@ static std::unique_ptr<const art::DexFile> MakeSingleDexFile(art::Thread* self, return nullptr; } uint32_t checksum = reinterpret_cast<const art::DexFile::Header*>(map->Begin())->checksum_; - std::unique_ptr<const art::DexFile> dex_file(art::DexFile::Open(map->GetName(), + std::string map_name = map->GetName(); + std::unique_ptr<const art::DexFile> dex_file(art::DexFile::Open(map_name, checksum, std::move(map), /*verify*/true, diff --git a/runtime/parsed_options.cc b/runtime/parsed_options.cc index ef4957c0ba..abb6f8c018 100644 --- a/runtime/parsed_options.cc +++ b/runtime/parsed_options.cc @@ -709,7 +709,7 @@ void ParsedOptions::Usage(const char* fmt, ...) { UsageMessage(stream, " -Xmethod-trace-file-size:integervalue\n"); UsageMessage(stream, " -Xps-min-save-period-ms:integervalue\n"); UsageMessage(stream, " -Xps-save-resolved-classes-delay-ms:integervalue\n"); - UsageMessage(stream, " -Xps-startup-method-samples:integervalue\n"); + UsageMessage(stream, " -Xps-hot-startup-method-samples:integervalue\n"); UsageMessage(stream, " -Xps-min-methods-to-save:integervalue\n"); UsageMessage(stream, " -Xps-min-classes-to-save:integervalue\n"); UsageMessage(stream, " -Xps-min-notification-before-wake:integervalue\n"); diff --git a/runtime/reflection.cc b/runtime/reflection.cc index e16ef1d77c..532da2b16e 100644 --- a/runtime/reflection.cc +++ b/runtime/reflection.cc @@ -31,6 +31,7 @@ #include "nth_caller_visitor.h" #include "scoped_thread_state_change-inl.h" #include "stack_reference.h" +#include "ScopedLocalRef.h" #include "well_known_classes.h" namespace art { @@ -668,7 +669,7 @@ jobject InvokeMethod(const ScopedObjectAccessAlreadyRunnable& soa, jobject javaM // Wrap any exception with "Ljava/lang/reflect/InvocationTargetException;" and return early. if (soa.Self()->IsExceptionPending()) { // If we get another exception when we are trying to wrap, then just use that instead. - jthrowable th = soa.Env()->ExceptionOccurred(); + ScopedLocalRef<jthrowable> th(soa.Env(), soa.Env()->ExceptionOccurred()); soa.Self()->ClearException(); jclass exception_class = soa.Env()->FindClass("java/lang/reflect/InvocationTargetException"); if (exception_class == nullptr) { @@ -677,7 +678,7 @@ jobject InvokeMethod(const ScopedObjectAccessAlreadyRunnable& soa, jobject javaM } jmethodID mid = soa.Env()->GetMethodID(exception_class, "<init>", "(Ljava/lang/Throwable;)V"); CHECK(mid != nullptr); - jobject exception_instance = soa.Env()->NewObject(exception_class, mid, th); + jobject exception_instance = soa.Env()->NewObject(exception_class, mid, th.get()); if (exception_instance == nullptr) { soa.Self()->AssertPendingException(); return nullptr; diff --git a/runtime/runtime-inl.h b/runtime/runtime-inl.h index 9281577b3e..609f0d658d 100644 --- a/runtime/runtime-inl.h +++ b/runtime/runtime-inl.h @@ -20,6 +20,7 @@ #include "runtime.h" #include "art_method.h" +#include "base/callee_save_type.h" #include "gc_root-inl.h" #include "obj_ptr-inl.h" @@ -41,15 +42,15 @@ inline QuickMethodFrameInfo Runtime::GetRuntimeMethodFrameInfo(ArtMethod* method DCHECK_NE(method, GetImtConflictMethod()); DCHECK_NE(method, GetResolutionMethod()); // Don't use GetCalleeSaveMethod(), some tests don't set all callee save methods. - if (method == GetCalleeSaveMethodUnchecked(Runtime::kSaveRefsAndArgs)) { - return GetCalleeSaveMethodFrameInfo(Runtime::kSaveRefsAndArgs); - } else if (method == GetCalleeSaveMethodUnchecked(Runtime::kSaveAllCalleeSaves)) { - return GetCalleeSaveMethodFrameInfo(Runtime::kSaveAllCalleeSaves); - } else if (method == GetCalleeSaveMethodUnchecked(Runtime::kSaveRefsOnly)) { - return GetCalleeSaveMethodFrameInfo(Runtime::kSaveRefsOnly); + if (method == GetCalleeSaveMethodUnchecked(CalleeSaveType::kSaveRefsAndArgs)) { + return GetCalleeSaveMethodFrameInfo(CalleeSaveType::kSaveRefsAndArgs); + } else if (method == GetCalleeSaveMethodUnchecked(CalleeSaveType::kSaveAllCalleeSaves)) { + return GetCalleeSaveMethodFrameInfo(CalleeSaveType::kSaveAllCalleeSaves); + } else if (method == GetCalleeSaveMethodUnchecked(CalleeSaveType::kSaveRefsOnly)) { + return GetCalleeSaveMethodFrameInfo(CalleeSaveType::kSaveRefsOnly); } else { - DCHECK_EQ(method, GetCalleeSaveMethodUnchecked(Runtime::kSaveEverything)); - return GetCalleeSaveMethodFrameInfo(Runtime::kSaveEverything); + DCHECK_EQ(method, GetCalleeSaveMethodUnchecked(CalleeSaveType::kSaveEverything)); + return GetCalleeSaveMethodFrameInfo(CalleeSaveType::kSaveEverything); } } @@ -76,7 +77,7 @@ inline ArtMethod* Runtime::GetCalleeSaveMethod(CalleeSaveType type) inline ArtMethod* Runtime::GetCalleeSaveMethodUnchecked(CalleeSaveType type) REQUIRES_SHARED(Locks::mutator_lock_) { - return reinterpret_cast<ArtMethod*>(callee_save_methods_[type]); + return reinterpret_cast<ArtMethod*>(callee_save_methods_[static_cast<size_t>(type)]); } } // namespace art diff --git a/runtime/runtime.cc b/runtime/runtime.cc index 74e291e2d0..c11e4bd448 100644 --- a/runtime/runtime.cc +++ b/runtime/runtime.cc @@ -57,6 +57,7 @@ #include "art_field-inl.h" #include "art_method-inl.h" #include "asm_support.h" +#include "asm_support_check.h" #include "atomic.h" #include "base/arena_allocator.h" #include "base/dumpable.h" @@ -217,6 +218,7 @@ Runtime::Runtime() intern_table_(nullptr), class_linker_(nullptr), signal_catcher_(nullptr), + use_tombstoned_traces_(false), java_vm_(nullptr), fault_message_lock_("Fault message lock"), fault_message_(""), @@ -259,6 +261,9 @@ Runtime::Runtime() process_state_(kProcessStateJankPerceptible), zygote_no_threads_(false), cha_(nullptr) { + static_assert(Runtime::kCalleeSaveSize == + static_cast<uint32_t>(CalleeSaveType::kLastCalleeSaveType), "Unexpected size"); + CheckAsmSupportOffsetsAndSizes(); std::fill(callee_save_methods_, callee_save_methods_ + arraysize(callee_save_methods_), 0u); interpreter::CheckInterpreterAsmConstants(); @@ -1331,8 +1336,8 @@ bool Runtime::Init(RuntimeArgumentMap&& runtime_options_in) { // TODO: Should we move the following to InitWithoutImage? SetInstructionSet(instruction_set_); - for (int i = 0; i < Runtime::kLastCalleeSaveType; i++) { - Runtime::CalleeSaveType type = Runtime::CalleeSaveType(i); + for (uint32_t i = 0; i < kCalleeSaveSize; i++) { + CalleeSaveType type = CalleeSaveType(i); if (!HasCalleeSaveMethod(type)) { SetCalleeSaveMethod(CreateCalleeSaveMethod(), type); } @@ -1797,7 +1802,7 @@ void Runtime::VisitConstantRoots(RootVisitor* visitor) { if (imt_unimplemented_method_ != nullptr) { imt_unimplemented_method_->VisitRoots(buffered_visitor, pointer_size); } - for (size_t i = 0; i < kLastCalleeSaveType; ++i) { + for (uint32_t i = 0; i < kCalleeSaveSize; ++i) { auto* m = reinterpret_cast<ArtMethod*>(callee_save_methods_[i]); if (m != nullptr) { m->VisitRoots(buffered_visitor, pointer_size); @@ -1973,32 +1978,32 @@ void Runtime::BroadcastForNewSystemWeaks(bool broadcast_for_checkpoint) { void Runtime::SetInstructionSet(InstructionSet instruction_set) { instruction_set_ = instruction_set; if ((instruction_set_ == kThumb2) || (instruction_set_ == kArm)) { - for (int i = 0; i != kLastCalleeSaveType; ++i) { + for (int i = 0; i != kCalleeSaveSize; ++i) { CalleeSaveType type = static_cast<CalleeSaveType>(i); callee_save_method_frame_infos_[i] = arm::ArmCalleeSaveMethodFrameInfo(type); } } else if (instruction_set_ == kMips) { - for (int i = 0; i != kLastCalleeSaveType; ++i) { + for (int i = 0; i != kCalleeSaveSize; ++i) { CalleeSaveType type = static_cast<CalleeSaveType>(i); callee_save_method_frame_infos_[i] = mips::MipsCalleeSaveMethodFrameInfo(type); } } else if (instruction_set_ == kMips64) { - for (int i = 0; i != kLastCalleeSaveType; ++i) { + for (int i = 0; i != kCalleeSaveSize; ++i) { CalleeSaveType type = static_cast<CalleeSaveType>(i); callee_save_method_frame_infos_[i] = mips64::Mips64CalleeSaveMethodFrameInfo(type); } } else if (instruction_set_ == kX86) { - for (int i = 0; i != kLastCalleeSaveType; ++i) { + for (int i = 0; i != kCalleeSaveSize; ++i) { CalleeSaveType type = static_cast<CalleeSaveType>(i); callee_save_method_frame_infos_[i] = x86::X86CalleeSaveMethodFrameInfo(type); } } else if (instruction_set_ == kX86_64) { - for (int i = 0; i != kLastCalleeSaveType; ++i) { + for (int i = 0; i != kCalleeSaveSize; ++i) { CalleeSaveType type = static_cast<CalleeSaveType>(i); callee_save_method_frame_infos_[i] = x86_64::X86_64CalleeSaveMethodFrameInfo(type); } } else if (instruction_set_ == kArm64) { - for (int i = 0; i != kLastCalleeSaveType; ++i) { + for (int i = 0; i != kCalleeSaveSize; ++i) { CalleeSaveType type = static_cast<CalleeSaveType>(i); callee_save_method_frame_infos_[i] = arm64::Arm64CalleeSaveMethodFrameInfo(type); } @@ -2012,15 +2017,14 @@ void Runtime::ClearInstructionSet() { } void Runtime::SetCalleeSaveMethod(ArtMethod* method, CalleeSaveType type) { - DCHECK_LT(static_cast<int>(type), static_cast<int>(kLastCalleeSaveType)); + DCHECK_LT(static_cast<uint32_t>(type), kCalleeSaveSize); CHECK(method != nullptr); - callee_save_methods_[type] = reinterpret_cast<uintptr_t>(method); + callee_save_methods_[static_cast<size_t>(type)] = reinterpret_cast<uintptr_t>(method); } void Runtime::ClearCalleeSaveMethods() { - for (size_t i = 0; i < static_cast<size_t>(kLastCalleeSaveType); ++i) { - CalleeSaveType type = static_cast<CalleeSaveType>(i); - callee_save_methods_[type] = reinterpret_cast<uintptr_t>(nullptr); + for (size_t i = 0; i < kCalleeSaveSize; ++i) { + callee_save_methods_[i] = reinterpret_cast<uintptr_t>(nullptr); } } diff --git a/runtime/runtime.h b/runtime/runtime.h index 992c5c827f..2505d8706e 100644 --- a/runtime/runtime.h +++ b/runtime/runtime.h @@ -71,6 +71,7 @@ namespace verifier { } // namespace verifier class ArenaPool; class ArtMethod; +enum class CalleeSaveType: uint32_t; class ClassHierarchyAnalysis; class ClassLinker; class CompilerCallbacks; @@ -377,17 +378,8 @@ class Runtime { imt_unimplemented_method_ = nullptr; } - // Returns a special method that describes all callee saves being spilled to the stack. - enum CalleeSaveType { - kSaveAllCalleeSaves, // All callee-save registers. - kSaveRefsOnly, // Only those callee-save registers that can hold references. - kSaveRefsAndArgs, // References (see above) and arguments (usually caller-save registers). - kSaveEverything, // All registers, including both callee-save and caller-save. - kLastCalleeSaveType // Value used for iteration - }; - bool HasCalleeSaveMethod(CalleeSaveType type) const { - return callee_save_methods_[type] != 0u; + return callee_save_methods_[static_cast<size_t>(type)] != 0u; } ArtMethod* GetCalleeSaveMethod(CalleeSaveType type) @@ -397,14 +389,14 @@ class Runtime { REQUIRES_SHARED(Locks::mutator_lock_); QuickMethodFrameInfo GetCalleeSaveMethodFrameInfo(CalleeSaveType type) const { - return callee_save_method_frame_infos_[type]; + return callee_save_method_frame_infos_[static_cast<size_t>(type)]; } QuickMethodFrameInfo GetRuntimeMethodFrameInfo(ArtMethod* method) REQUIRES_SHARED(Locks::mutator_lock_); static size_t GetCalleeSaveMethodOffset(CalleeSaveType type) { - return OFFSETOF_MEMBER(Runtime, callee_save_methods_[type]); + return OFFSETOF_MEMBER(Runtime, callee_save_methods_[static_cast<size_t>(type)]); } InstructionSet GetInstructionSet() const { @@ -724,8 +716,10 @@ class Runtime { static constexpr int kProfileForground = 0; static constexpr int kProfileBackground = 1; + static constexpr uint32_t kCalleeSaveSize = 4u; + // 64 bit so that we can share the same asm offsets for both 32 and 64 bits. - uint64_t callee_save_methods_[kLastCalleeSaveType]; + uint64_t callee_save_methods_[kCalleeSaveSize]; GcRoot<mirror::Throwable> pre_allocated_OutOfMemoryError_; GcRoot<mirror::Throwable> pre_allocated_NoClassDefFoundError_; ArtMethod* resolution_method_; @@ -739,7 +733,7 @@ class Runtime { GcRoot<mirror::Object> sentinel_; InstructionSet instruction_set_; - QuickMethodFrameInfo callee_save_method_frame_infos_[kLastCalleeSaveType]; + QuickMethodFrameInfo callee_save_method_frame_infos_[kCalleeSaveSize]; CompilerCallbacks* compiler_callbacks_; bool is_zygote_; @@ -959,7 +953,6 @@ class Runtime { DISALLOW_COPY_AND_ASSIGN(Runtime); }; -std::ostream& operator<<(std::ostream& os, const Runtime::CalleeSaveType& rhs); } // namespace art diff --git a/runtime/stack.cc b/runtime/stack.cc index aedcc1eb03..eec0460015 100644 --- a/runtime/stack.cc +++ b/runtime/stack.cc @@ -20,6 +20,7 @@ #include "arch/context.h" #include "art_method-inl.h" +#include "base/callee_save_type.h" #include "base/enums.h" #include "base/hex_dump.h" #include "entrypoints/entrypoint_utils-inl.h" @@ -734,7 +735,7 @@ QuickMethodFrameInfo StackVisitor::GetCurrentQuickFrameInfo() const { Runtime* runtime = Runtime::Current(); if (method->IsAbstract()) { - return runtime->GetCalleeSaveMethodFrameInfo(Runtime::kSaveRefsAndArgs); + return runtime->GetCalleeSaveMethodFrameInfo(CalleeSaveType::kSaveRefsAndArgs); } // This goes before IsProxyMethod since runtime methods have a null declaring class. @@ -748,7 +749,7 @@ QuickMethodFrameInfo StackVisitor::GetCurrentQuickFrameInfo() const { // compiled method without any stubs. Therefore the method must have a OatQuickMethodHeader. DCHECK(!method->IsDirect() && !method->IsConstructor()) << "Constructors of proxy classes must have a OatQuickMethodHeader"; - return runtime->GetCalleeSaveMethodFrameInfo(Runtime::kSaveRefsAndArgs); + return runtime->GetCalleeSaveMethodFrameInfo(CalleeSaveType::kSaveRefsAndArgs); } // The only remaining case is if the method is native and uses the generic JNI stub. @@ -761,7 +762,7 @@ QuickMethodFrameInfo StackVisitor::GetCurrentQuickFrameInfo() const { uint32_t handle_refs = GetNumberOfReferenceArgsWithoutReceiver(method) + 1; size_t scope_size = HandleScope::SizeOf(handle_refs); QuickMethodFrameInfo callee_info = - runtime->GetCalleeSaveMethodFrameInfo(Runtime::kSaveRefsAndArgs); + runtime->GetCalleeSaveMethodFrameInfo(CalleeSaveType::kSaveRefsAndArgs); // Callee saves + handle scope + method ref + alignment // Note: -sizeof(void*) since callee-save frame stores a whole method pointer. @@ -844,11 +845,11 @@ void StackVisitor::WalkStack(bool include_transitions) { thread_->GetInstrumentationStack()->at(instrumentation_stack_depth); instrumentation_stack_depth++; if (GetMethod() == - Runtime::Current()->GetCalleeSaveMethod(Runtime::kSaveAllCalleeSaves)) { + Runtime::Current()->GetCalleeSaveMethod(CalleeSaveType::kSaveAllCalleeSaves)) { // Skip runtime save all callee frames which are used to deliver exceptions. } else if (instrumentation_frame.interpreter_entry_) { ArtMethod* callee = - Runtime::Current()->GetCalleeSaveMethod(Runtime::kSaveRefsAndArgs); + Runtime::Current()->GetCalleeSaveMethod(CalleeSaveType::kSaveRefsAndArgs); CHECK_EQ(GetMethod(), callee) << "Expected: " << ArtMethod::PrettyMethod(callee) << " Found: " << ArtMethod::PrettyMethod(GetMethod()); } else { diff --git a/runtime/stride_iterator.h b/runtime/stride_iterator.h index ac04c3b403..0560c33eee 100644 --- a/runtime/stride_iterator.h +++ b/runtime/stride_iterator.h @@ -24,8 +24,11 @@ namespace art { template<typename T> -class StrideIterator : public std::iterator<std::forward_iterator_tag, T> { +class StrideIterator : public std::iterator<std::random_access_iterator_tag, T> { public: + using difference_type = + typename std::iterator<std::random_access_iterator_tag, T>::difference_type; + StrideIterator(const StrideIterator&) = default; StrideIterator(StrideIterator&&) = default; StrideIterator& operator=(const StrideIterator&) = default; @@ -44,28 +47,56 @@ class StrideIterator : public std::iterator<std::forward_iterator_tag, T> { return !(*this == other); } - StrideIterator operator++() { // Value after modification. + StrideIterator& operator++() { // Value after modification. ptr_ += stride_; return *this; } StrideIterator operator++(int) { StrideIterator<T> temp = *this; - ptr_ += stride_; + ++*this; return temp; } - StrideIterator operator+(ssize_t delta) const { + StrideIterator& operator--() { // Value after modification. + ptr_ -= stride_; + return *this; + } + + StrideIterator operator--(int) { StrideIterator<T> temp = *this; - temp += delta; + --*this; return temp; } - StrideIterator& operator+=(ssize_t delta) { + StrideIterator& operator+=(difference_type delta) { ptr_ += static_cast<ssize_t>(stride_) * delta; return *this; } + StrideIterator operator+(difference_type delta) const { + StrideIterator<T> temp = *this; + temp += delta; + return temp; + } + + StrideIterator& operator-=(difference_type delta) { + ptr_ -= static_cast<ssize_t>(stride_) * delta; + return *this; + } + + StrideIterator operator-(difference_type delta) const { + StrideIterator<T> temp = *this; + temp -= delta; + return temp; + } + + difference_type operator-(const StrideIterator& rhs) { + DCHECK_EQ(stride_, rhs.stride_); + DCHECK_EQ((ptr_ - rhs.ptr_) % stride_, 0u); + return (ptr_ - rhs.ptr_) / stride_; + } + T& operator*() const { return *reinterpret_cast<T*>(ptr_); } @@ -74,12 +105,46 @@ class StrideIterator : public std::iterator<std::forward_iterator_tag, T> { return &**this; } + T& operator[](difference_type n) { + return *(*this + n); + } + private: uintptr_t ptr_; // Not const for operator=. size_t stride_; + + template <typename U> + friend bool operator<(const StrideIterator<U>& lhs, const StrideIterator<U>& rhs); }; +template <typename T> +StrideIterator<T> operator+(typename StrideIterator<T>::difference_type dist, + const StrideIterator<T>& it) { + return it + dist; +} + +template <typename T> +bool operator<(const StrideIterator<T>& lhs, const StrideIterator<T>& rhs) { + DCHECK_EQ(lhs.stride_, rhs.stride_); + return lhs.ptr_ < rhs.ptr_; +} + +template <typename T> +bool operator>(const StrideIterator<T>& lhs, const StrideIterator<T>& rhs) { + return rhs < lhs; +} + +template <typename T> +bool operator<=(const StrideIterator<T>& lhs, const StrideIterator<T>& rhs) { + return !(rhs < lhs); +} + +template <typename T> +bool operator>=(const StrideIterator<T>& lhs, const StrideIterator<T>& rhs) { + return !(lhs < rhs); +} + } // namespace art #endif // ART_RUNTIME_STRIDE_ITERATOR_H_ diff --git a/runtime/ti/agent.cc b/runtime/ti/agent.cc index 86f5282664..82b9af33d5 100644 --- a/runtime/ti/agent.cc +++ b/runtime/ti/agent.cc @@ -18,6 +18,7 @@ #include "android-base/stringprintf.h" +#include "base/strlcpy.h" #include "java_vm_ext.h" #include "runtime.h" @@ -57,7 +58,7 @@ Agent::LoadError Agent::DoLoadHelper(bool attaching, } // Need to let the function fiddle with the array. std::unique_ptr<char[]> copied_args(new char[args_.size() + 1]); - strcpy(copied_args.get(), args_.c_str()); + strlcpy(copied_args.get(), args_.c_str(), args_.size() + 1); // TODO Need to do some checks that we are at a good spot etc. *call_res = callback(Runtime::Current()->GetJavaVM(), copied_args.get(), diff --git a/runtime/trace.cc b/runtime/trace.cc index 3550d56bd8..cabd1620a7 100644 --- a/runtime/trace.cc +++ b/runtime/trace.cc @@ -740,7 +740,7 @@ void Trace::FinishTracing() { } void Trace::DexPcMoved(Thread* thread ATTRIBUTE_UNUSED, - mirror::Object* this_object ATTRIBUTE_UNUSED, + Handle<mirror::Object> this_object ATTRIBUTE_UNUSED, ArtMethod* method, uint32_t new_dex_pc) { // We're not recorded to listen to this kind of event, so complain. @@ -749,7 +749,7 @@ void Trace::DexPcMoved(Thread* thread ATTRIBUTE_UNUSED, } void Trace::FieldRead(Thread* thread ATTRIBUTE_UNUSED, - mirror::Object* this_object ATTRIBUTE_UNUSED, + Handle<mirror::Object> this_object ATTRIBUTE_UNUSED, ArtMethod* method, uint32_t dex_pc, ArtField* field ATTRIBUTE_UNUSED) @@ -760,7 +760,7 @@ void Trace::FieldRead(Thread* thread ATTRIBUTE_UNUSED, } void Trace::FieldWritten(Thread* thread ATTRIBUTE_UNUSED, - mirror::Object* this_object ATTRIBUTE_UNUSED, + Handle<mirror::Object> this_object ATTRIBUTE_UNUSED, ArtMethod* method, uint32_t dex_pc, ArtField* field ATTRIBUTE_UNUSED, @@ -771,8 +771,10 @@ void Trace::FieldWritten(Thread* thread ATTRIBUTE_UNUSED, << " " << dex_pc; } -void Trace::MethodEntered(Thread* thread, mirror::Object* this_object ATTRIBUTE_UNUSED, - ArtMethod* method, uint32_t dex_pc ATTRIBUTE_UNUSED) { +void Trace::MethodEntered(Thread* thread, + Handle<mirror::Object> this_object ATTRIBUTE_UNUSED, + ArtMethod* method, + uint32_t dex_pc ATTRIBUTE_UNUSED) { uint32_t thread_clock_diff = 0; uint32_t wall_clock_diff = 0; ReadClocks(thread, &thread_clock_diff, &wall_clock_diff); @@ -780,8 +782,10 @@ void Trace::MethodEntered(Thread* thread, mirror::Object* this_object ATTRIBUTE_ thread_clock_diff, wall_clock_diff); } -void Trace::MethodExited(Thread* thread, mirror::Object* this_object ATTRIBUTE_UNUSED, - ArtMethod* method, uint32_t dex_pc ATTRIBUTE_UNUSED, +void Trace::MethodExited(Thread* thread, + Handle<mirror::Object> this_object ATTRIBUTE_UNUSED, + ArtMethod* method, + uint32_t dex_pc ATTRIBUTE_UNUSED, const JValue& return_value ATTRIBUTE_UNUSED) { uint32_t thread_clock_diff = 0; uint32_t wall_clock_diff = 0; @@ -790,8 +794,10 @@ void Trace::MethodExited(Thread* thread, mirror::Object* this_object ATTRIBUTE_U thread_clock_diff, wall_clock_diff); } -void Trace::MethodUnwind(Thread* thread, mirror::Object* this_object ATTRIBUTE_UNUSED, - ArtMethod* method, uint32_t dex_pc ATTRIBUTE_UNUSED) { +void Trace::MethodUnwind(Thread* thread, + Handle<mirror::Object> this_object ATTRIBUTE_UNUSED, + ArtMethod* method, + uint32_t dex_pc ATTRIBUTE_UNUSED) { uint32_t thread_clock_diff = 0; uint32_t wall_clock_diff = 0; ReadClocks(thread, &thread_clock_diff, &wall_clock_diff); @@ -800,7 +806,7 @@ void Trace::MethodUnwind(Thread* thread, mirror::Object* this_object ATTRIBUTE_U } void Trace::ExceptionCaught(Thread* thread ATTRIBUTE_UNUSED, - mirror::Throwable* exception_object ATTRIBUTE_UNUSED) + Handle<mirror::Throwable> exception_object ATTRIBUTE_UNUSED) REQUIRES_SHARED(Locks::mutator_lock_) { LOG(ERROR) << "Unexpected exception caught event in tracing"; } @@ -812,7 +818,7 @@ void Trace::Branch(Thread* /*thread*/, ArtMethod* method, } void Trace::InvokeVirtualOrInterface(Thread*, - mirror::Object*, + Handle<mirror::Object>, ArtMethod* method, uint32_t dex_pc, ArtMethod*) { diff --git a/runtime/trace.h b/runtime/trace.h index 485e9a133a..ad1025045c 100644 --- a/runtime/trace.h +++ b/runtime/trace.h @@ -140,36 +140,54 @@ class Trace FINAL : public instrumentation::InstrumentationListener { REQUIRES_SHARED(Locks::mutator_lock_) REQUIRES(!*unique_methods_lock_, !*streaming_lock_); // InstrumentationListener implementation. - void MethodEntered(Thread* thread, mirror::Object* this_object, - ArtMethod* method, uint32_t dex_pc) + void MethodEntered(Thread* thread, + Handle<mirror::Object> this_object, + ArtMethod* method, + uint32_t dex_pc) REQUIRES_SHARED(Locks::mutator_lock_) REQUIRES(!*unique_methods_lock_, !*streaming_lock_) OVERRIDE; - void MethodExited(Thread* thread, mirror::Object* this_object, - ArtMethod* method, uint32_t dex_pc, + void MethodExited(Thread* thread, + Handle<mirror::Object> this_object, + ArtMethod* method, + uint32_t dex_pc, const JValue& return_value) REQUIRES_SHARED(Locks::mutator_lock_) REQUIRES(!*unique_methods_lock_, !*streaming_lock_) OVERRIDE; - void MethodUnwind(Thread* thread, mirror::Object* this_object, - ArtMethod* method, uint32_t dex_pc) + void MethodUnwind(Thread* thread, + Handle<mirror::Object> this_object, + ArtMethod* method, + uint32_t dex_pc) REQUIRES_SHARED(Locks::mutator_lock_) REQUIRES(!*unique_methods_lock_, !*streaming_lock_) OVERRIDE; - void DexPcMoved(Thread* thread, mirror::Object* this_object, - ArtMethod* method, uint32_t new_dex_pc) + void DexPcMoved(Thread* thread, + Handle<mirror::Object> this_object, + ArtMethod* method, + uint32_t new_dex_pc) REQUIRES_SHARED(Locks::mutator_lock_) REQUIRES(!*unique_methods_lock_, !*streaming_lock_) OVERRIDE; - void FieldRead(Thread* thread, mirror::Object* this_object, - ArtMethod* method, uint32_t dex_pc, ArtField* field) + void FieldRead(Thread* thread, + Handle<mirror::Object> this_object, + ArtMethod* method, + uint32_t dex_pc, + ArtField* field) REQUIRES_SHARED(Locks::mutator_lock_) REQUIRES(!*unique_methods_lock_) OVERRIDE; - void FieldWritten(Thread* thread, mirror::Object* this_object, - ArtMethod* method, uint32_t dex_pc, ArtField* field, + void FieldWritten(Thread* thread, + Handle<mirror::Object> this_object, + ArtMethod* method, + uint32_t dex_pc, + ArtField* field, const JValue& field_value) REQUIRES_SHARED(Locks::mutator_lock_) REQUIRES(!*unique_methods_lock_) OVERRIDE; - void ExceptionCaught(Thread* thread, mirror::Throwable* exception_object) + void ExceptionCaught(Thread* thread, + Handle<mirror::Throwable> exception_object) REQUIRES_SHARED(Locks::mutator_lock_) REQUIRES(!*unique_methods_lock_) OVERRIDE; - void Branch(Thread* thread, ArtMethod* method, uint32_t dex_pc, int32_t dex_pc_offset) + void Branch(Thread* thread, + ArtMethod* method, + uint32_t dex_pc, + int32_t dex_pc_offset) REQUIRES_SHARED(Locks::mutator_lock_) REQUIRES(!*unique_methods_lock_) OVERRIDE; void InvokeVirtualOrInterface(Thread* thread, - mirror::Object* this_object, + Handle<mirror::Object> this_object, ArtMethod* caller, uint32_t dex_pc, ArtMethod* callee) diff --git a/runtime/utils.cc b/runtime/utils.cc index 20a53b7c69..c4b044110c 100644 --- a/runtime/utils.cc +++ b/runtime/utils.cc @@ -303,7 +303,7 @@ std::string PrintableChar(uint16_t ch) { if (NeedsEscaping(ch)) { StringAppendF(&result, "\\u%04x", ch); } else { - result += ch; + result += static_cast<std::string::value_type>(ch); } result += '\''; return result; @@ -330,7 +330,7 @@ std::string PrintableString(const char* utf) { if (NeedsEscaping(leading)) { StringAppendF(&result, "\\u%04x", leading); } else { - result += leading; + result += static_cast<std::string::value_type>(leading); } const uint32_t trailing = GetTrailingUtf16Char(ch); diff --git a/runtime/verifier/method_verifier.h b/runtime/verifier/method_verifier.h index cb208f44a5..46fdc5419d 100644 --- a/runtime/verifier/method_verifier.h +++ b/runtime/verifier/method_verifier.h @@ -24,7 +24,6 @@ #include "base/arena_allocator.h" #include "base/macros.h" #include "base/scoped_arena_containers.h" -#include "base/stl_util.h" #include "base/value_object.h" #include "dex_file.h" #include "dex_file_types.h" diff --git a/runtime/zip_archive.cc b/runtime/zip_archive.cc index 0d0d5c73ae..df1012ea3c 100644 --- a/runtime/zip_archive.cc +++ b/runtime/zip_archive.cc @@ -25,6 +25,7 @@ #include <vector> #include "android-base/stringprintf.h" +#include "base/bit_utils.h" #include "base/unix_file/fd_file.h" namespace art { diff --git a/test/551-checker-shifter-operand/src/Main.java b/test/551-checker-shifter-operand/src/Main.java index bf09a6aa5e..951889ab9f 100644 --- a/test/551-checker-shifter-operand/src/Main.java +++ b/test/551-checker-shifter-operand/src/Main.java @@ -234,8 +234,8 @@ public class Main { /// CHECK-START-ARM: void Main.$opt$noinline$testAnd(long, long) disassembly (after) /// CHECK: and lsl /// CHECK: sbfx - /// CHECK: asr - /// CHECK: and + /// CHECK: asr{{s?}} + /// CHECK: and{{s?}} /// CHECK-START-ARM64: void Main.$opt$noinline$testAnd(long, long) instruction_simplifier_arm64 (after) /// CHECK: DataProcWithShifterOp @@ -259,7 +259,7 @@ public class Main { /// CHECK-START-ARM: void Main.$opt$noinline$testOr(int, int) disassembly (after) /// CHECK: orr asr /// CHECK: ubfx - /// CHECK: orr + /// CHECK: orr{{s?}} /// CHECK-START-ARM64: void Main.$opt$noinline$testOr(int, int) instruction_simplifier_arm64 (after) /// CHECK: DataProcWithShifterOp @@ -282,9 +282,8 @@ public class Main { /// CHECK-START-ARM: void Main.$opt$noinline$testXor(long, long) disassembly (after) /// CHECK: eor lsr - /// CHECK: mov - /// CHECK: asr - /// CHECK: eor + /// CHECK: asr{{s?}} + /// CHECK: eor{{s?}} /// CHECK-START-ARM64: void Main.$opt$noinline$testXor(long, long) instruction_simplifier_arm64 (after) /// CHECK: DataProcWithShifterOp diff --git a/test/552-checker-sharpening/src/Main.java b/test/552-checker-sharpening/src/Main.java index 196831b921..7408e6d263 100644 --- a/test/552-checker-sharpening/src/Main.java +++ b/test/552-checker-sharpening/src/Main.java @@ -42,31 +42,30 @@ public class Main { } /// CHECK-START: int Main.testSimple(int) sharpening (before) - /// CHECK: InvokeStaticOrDirect method_load_kind:DexCacheViaMethod + /// CHECK: InvokeStaticOrDirect method_load_kind:RuntimeCall /// CHECK-START-ARM: int Main.testSimple(int) sharpening (after) - /// CHECK-NOT: ArmDexCacheArraysBase - /// CHECK: InvokeStaticOrDirect method_load_kind:DexCachePcRelative + /// CHECK: InvokeStaticOrDirect method_load_kind:BssEntry /// CHECK-START-ARM64: int Main.testSimple(int) sharpening (after) - /// CHECK: InvokeStaticOrDirect method_load_kind:DexCachePcRelative + /// CHECK: InvokeStaticOrDirect method_load_kind:BssEntry /// CHECK-START-MIPS: int Main.testSimple(int) sharpening (after) - /// CHECK: InvokeStaticOrDirect method_load_kind:DexCachePcRelative + /// CHECK: InvokeStaticOrDirect method_load_kind:BssEntry /// CHECK-START-MIPS64: int Main.testSimple(int) sharpening (after) - /// CHECK: InvokeStaticOrDirect method_load_kind:DexCachePcRelative + /// CHECK: InvokeStaticOrDirect method_load_kind:BssEntry /// CHECK-START-X86: int Main.testSimple(int) sharpening (after) /// CHECK-NOT: X86ComputeBaseMethodAddress - /// CHECK: InvokeStaticOrDirect method_load_kind:DexCachePcRelative + /// CHECK: InvokeStaticOrDirect method_load_kind:BssEntry /// CHECK-START-X86_64: int Main.testSimple(int) sharpening (after) - /// CHECK: InvokeStaticOrDirect method_load_kind:DexCachePcRelative + /// CHECK: InvokeStaticOrDirect method_load_kind:BssEntry - /// CHECK-START-ARM: int Main.testSimple(int) dex_cache_array_fixups_arm (after) - /// CHECK: ArmDexCacheArraysBase - /// CHECK-NOT: ArmDexCacheArraysBase + /// CHECK-START-MIPS: int Main.testSimple(int) pc_relative_fixups_mips (after) + /// CHECK: MipsComputeBaseMethodAddress + /// CHECK-NOT: MipsComputeBaseMethodAddress /// CHECK-START-X86: int Main.testSimple(int) pc_relative_fixups_x86 (after) /// CHECK: X86ComputeBaseMethodAddress @@ -78,40 +77,39 @@ public class Main { } /// CHECK-START: int Main.testDiamond(boolean, int) sharpening (before) - /// CHECK: InvokeStaticOrDirect method_load_kind:DexCacheViaMethod + /// CHECK: InvokeStaticOrDirect method_load_kind:RuntimeCall /// CHECK-START-ARM: int Main.testDiamond(boolean, int) sharpening (after) - /// CHECK-NOT: ArmDexCacheArraysBase - /// CHECK: InvokeStaticOrDirect method_load_kind:DexCachePcRelative - /// CHECK: InvokeStaticOrDirect method_load_kind:DexCachePcRelative + /// CHECK: InvokeStaticOrDirect method_load_kind:BssEntry + /// CHECK: InvokeStaticOrDirect method_load_kind:BssEntry /// CHECK-START-ARM64: int Main.testDiamond(boolean, int) sharpening (after) - /// CHECK: InvokeStaticOrDirect method_load_kind:DexCachePcRelative - /// CHECK: InvokeStaticOrDirect method_load_kind:DexCachePcRelative + /// CHECK: InvokeStaticOrDirect method_load_kind:BssEntry + /// CHECK: InvokeStaticOrDirect method_load_kind:BssEntry /// CHECK-START-MIPS: int Main.testDiamond(boolean, int) sharpening (after) - /// CHECK: InvokeStaticOrDirect method_load_kind:DexCachePcRelative - /// CHECK: InvokeStaticOrDirect method_load_kind:DexCachePcRelative + /// CHECK: InvokeStaticOrDirect method_load_kind:BssEntry + /// CHECK: InvokeStaticOrDirect method_load_kind:BssEntry /// CHECK-START-MIPS64: int Main.testDiamond(boolean, int) sharpening (after) - /// CHECK: InvokeStaticOrDirect method_load_kind:DexCachePcRelative - /// CHECK: InvokeStaticOrDirect method_load_kind:DexCachePcRelative + /// CHECK: InvokeStaticOrDirect method_load_kind:BssEntry + /// CHECK: InvokeStaticOrDirect method_load_kind:BssEntry /// CHECK-START-X86: int Main.testDiamond(boolean, int) sharpening (after) /// CHECK-NOT: X86ComputeBaseMethodAddress - /// CHECK: InvokeStaticOrDirect method_load_kind:DexCachePcRelative - /// CHECK: InvokeStaticOrDirect method_load_kind:DexCachePcRelative + /// CHECK: InvokeStaticOrDirect method_load_kind:BssEntry + /// CHECK: InvokeStaticOrDirect method_load_kind:BssEntry /// CHECK-START-X86_64: int Main.testDiamond(boolean, int) sharpening (after) - /// CHECK: InvokeStaticOrDirect method_load_kind:DexCachePcRelative - /// CHECK: InvokeStaticOrDirect method_load_kind:DexCachePcRelative + /// CHECK: InvokeStaticOrDirect method_load_kind:BssEntry + /// CHECK: InvokeStaticOrDirect method_load_kind:BssEntry - /// CHECK-START-ARM: int Main.testDiamond(boolean, int) dex_cache_array_fixups_arm (after) - /// CHECK: ArmDexCacheArraysBase - /// CHECK-NOT: ArmDexCacheArraysBase + /// CHECK-START-MIPS: int Main.testDiamond(boolean, int) pc_relative_fixups_mips (after) + /// CHECK: MipsComputeBaseMethodAddress + /// CHECK-NOT: MipsComputeBaseMethodAddress - /// CHECK-START-ARM: int Main.testDiamond(boolean, int) dex_cache_array_fixups_arm (after) - /// CHECK: ArmDexCacheArraysBase + /// CHECK-START-MIPS: int Main.testDiamond(boolean, int) pc_relative_fixups_mips (after) + /// CHECK: MipsComputeBaseMethodAddress /// CHECK-NEXT: If /// CHECK-START-X86: int Main.testDiamond(boolean, int) pc_relative_fixups_x86 (after) @@ -123,8 +121,8 @@ public class Main { /// CHECK-NEXT: If public static int testDiamond(boolean negate, int x) { - // These calls should use PC-relative dex cache array loads to retrieve the target method. - // PC-relative bases used by ARM, MIPS and X86 should be pulled before the If. + // These calls should use PC-relative loads to retrieve the target method. + // PC-relative bases used by MIPS and X86 should be pulled before the If. if (negate) { return $noinline$foo(-x); } else { @@ -132,72 +130,72 @@ public class Main { } } - /// CHECK-START-X86: int Main.testLoop(int[], int) pc_relative_fixups_x86 (before) - /// CHECK-NOT: X86ComputeBaseMethodAddress + /// CHECK-START-MIPS: int Main.testLoop(int[], int) pc_relative_fixups_mips (before) + /// CHECK-NOT: MipsComputeBaseMethodAddress - /// CHECK-START-X86: int Main.testLoop(int[], int) pc_relative_fixups_x86 (after) - /// CHECK: X86ComputeBaseMethodAddress - /// CHECK-NOT: X86ComputeBaseMethodAddress + /// CHECK-START-MIPS: int Main.testLoop(int[], int) pc_relative_fixups_mips (after) + /// CHECK: MipsComputeBaseMethodAddress + /// CHECK-NOT: MipsComputeBaseMethodAddress - /// CHECK-START-X86: int Main.testLoop(int[], int) pc_relative_fixups_x86 (after) + /// CHECK-START-MIPS: int Main.testLoop(int[], int) pc_relative_fixups_mips (after) /// CHECK: InvokeStaticOrDirect /// CHECK-NOT: InvokeStaticOrDirect - /// CHECK-START-X86: int Main.testLoop(int[], int) pc_relative_fixups_x86 (after) + /// CHECK-START-MIPS: int Main.testLoop(int[], int) pc_relative_fixups_mips (after) /// CHECK: ArrayLength - /// CHECK-NEXT: X86ComputeBaseMethodAddress + /// CHECK-NEXT: MipsComputeBaseMethodAddress /// CHECK-NEXT: Goto /// CHECK: begin_block - /// CHECK: InvokeStaticOrDirect method_load_kind:DexCachePcRelative + /// CHECK: InvokeStaticOrDirect method_load_kind:BssEntry - /// CHECK-START-ARM: int Main.testLoop(int[], int) dex_cache_array_fixups_arm (before) - /// CHECK-NOT: ArmDexCacheArraysBase + /// CHECK-START-X86: int Main.testLoop(int[], int) pc_relative_fixups_x86 (before) + /// CHECK-NOT: X86ComputeBaseMethodAddress - /// CHECK-START-ARM: int Main.testLoop(int[], int) dex_cache_array_fixups_arm (after) - /// CHECK: ArmDexCacheArraysBase - /// CHECK-NOT: ArmDexCacheArraysBase + /// CHECK-START-X86: int Main.testLoop(int[], int) pc_relative_fixups_x86 (after) + /// CHECK: X86ComputeBaseMethodAddress + /// CHECK-NOT: X86ComputeBaseMethodAddress - /// CHECK-START-ARM: int Main.testLoop(int[], int) dex_cache_array_fixups_arm (after) + /// CHECK-START-X86: int Main.testLoop(int[], int) pc_relative_fixups_x86 (after) /// CHECK: InvokeStaticOrDirect /// CHECK-NOT: InvokeStaticOrDirect - /// CHECK-START-ARM: int Main.testLoop(int[], int) dex_cache_array_fixups_arm (after) + /// CHECK-START-X86: int Main.testLoop(int[], int) pc_relative_fixups_x86 (after) /// CHECK: ArrayLength - /// CHECK-NEXT: ArmDexCacheArraysBase + /// CHECK-NEXT: X86ComputeBaseMethodAddress /// CHECK-NEXT: Goto /// CHECK: begin_block - /// CHECK: InvokeStaticOrDirect method_load_kind:DexCachePcRelative + /// CHECK: InvokeStaticOrDirect method_load_kind:BssEntry public static int testLoop(int[] array, int x) { - // PC-relative bases used by ARM, MIPS and X86 should be pulled before the loop. + // PC-relative bases used by MIPS and X86 should be pulled before the loop. for (int i : array) { x += $noinline$foo(i); } return x; } - /// CHECK-START-X86: int Main.testLoopWithDiamond(int[], boolean, int) pc_relative_fixups_x86 (before) - /// CHECK-NOT: X86ComputeBaseMethodAddress + /// CHECK-START-MIPS: int Main.testLoopWithDiamond(int[], boolean, int) pc_relative_fixups_mips (before) + /// CHECK-NOT: MipsComputeBaseMethodAddress - /// CHECK-START-X86: int Main.testLoopWithDiamond(int[], boolean, int) pc_relative_fixups_x86 (after) + /// CHECK-START-MIPS: int Main.testLoopWithDiamond(int[], boolean, int) pc_relative_fixups_mips (after) /// CHECK: If /// CHECK: begin_block /// CHECK: ArrayLength - /// CHECK-NEXT: X86ComputeBaseMethodAddress + /// CHECK-NEXT: MipsComputeBaseMethodAddress /// CHECK-NEXT: Goto - /// CHECK-START-ARM: int Main.testLoopWithDiamond(int[], boolean, int) dex_cache_array_fixups_arm (before) - /// CHECK-NOT: ArmDexCacheArraysBase + /// CHECK-START-X86: int Main.testLoopWithDiamond(int[], boolean, int) pc_relative_fixups_x86 (before) + /// CHECK-NOT: X86ComputeBaseMethodAddress - /// CHECK-START-ARM: int Main.testLoopWithDiamond(int[], boolean, int) dex_cache_array_fixups_arm (after) + /// CHECK-START-X86: int Main.testLoopWithDiamond(int[], boolean, int) pc_relative_fixups_x86 (after) /// CHECK: If /// CHECK: begin_block /// CHECK: ArrayLength - /// CHECK-NEXT: ArmDexCacheArraysBase + /// CHECK-NEXT: X86ComputeBaseMethodAddress /// CHECK-NEXT: Goto public static int testLoopWithDiamond(int[] array, boolean negate, int x) { - // PC-relative bases used by ARM, MIPS and X86 should be pulled before the loop + // PC-relative bases used by MIPS and X86 should be pulled before the loop // but not outside the if. if (array != null) { for (int i : array) { diff --git a/test/564-checker-irreducible-loop/smali/IrreducibleLoop.smali b/test/564-checker-irreducible-loop/smali/IrreducibleLoop.smali index e4bf236266..5f73bbe759 100644 --- a/test/564-checker-irreducible-loop/smali/IrreducibleLoop.smali +++ b/test/564-checker-irreducible-loop/smali/IrreducibleLoop.smali @@ -18,8 +18,8 @@ ## CHECK-START-X86: int IrreducibleLoop.simpleLoop(int) dead_code_elimination$initial (before) ## CHECK-DAG: <<Constant:i\d+>> IntConstant 42 -## CHECK-DAG: InvokeStaticOrDirect [<<Constant>>] loop:{{B\d+}} irreducible:true -## CHECK-DAG: InvokeStaticOrDirect [<<Constant>>] loop:none +## CHECK-DAG: InvokeStaticOrDirect [<<Constant>>{{(,[ij]\d+)?}}] loop:{{B\d+}} irreducible:true +## CHECK-DAG: InvokeStaticOrDirect [<<Constant>>{{(,[ij]\d+)?}}] loop:none .method public static simpleLoop(I)I .registers 3 const/16 v0, 42 diff --git a/test/588-checker-irreducib-lifetime-hole/smali/IrreducibleLoop.smali b/test/588-checker-irreducib-lifetime-hole/smali/IrreducibleLoop.smali index 9b8aa510a4..3058358033 100644 --- a/test/588-checker-irreducib-lifetime-hole/smali/IrreducibleLoop.smali +++ b/test/588-checker-irreducib-lifetime-hole/smali/IrreducibleLoop.smali @@ -19,8 +19,8 @@ ## CHECK-START-X86: int IrreducibleLoop.simpleLoop1(int) dead_code_elimination$initial (before) ## CHECK-DAG: <<Constant:i\d+>> IntConstant 42 ## CHECK-DAG: Goto irreducible:true -## CHECK-DAG: InvokeStaticOrDirect [<<Constant>>] loop:none -## CHECK-DAG: InvokeStaticOrDirect [{{i\d+}}] loop:none +## CHECK-DAG: InvokeStaticOrDirect [<<Constant>>{{(,[ij]\d+)?}}] loop:none +## CHECK-DAG: InvokeStaticOrDirect [{{i\d+}}{{(,[ij]\d+)?}}] loop:none .method public static simpleLoop1(I)I .registers 3 const/16 v0, 42 @@ -59,8 +59,8 @@ ## CHECK-START-X86: int IrreducibleLoop.simpleLoop2(int) dead_code_elimination$initial (before) ## CHECK-DAG: <<Constant:i\d+>> IntConstant 42 ## CHECK-DAG: Goto irreducible:true -## CHECK-DAG: InvokeStaticOrDirect [<<Constant>>] loop:none -## CHECK-DAG: InvokeStaticOrDirect [{{i\d+}}] loop:none +## CHECK-DAG: InvokeStaticOrDirect [<<Constant>>{{(,[ij]\d+)?}}] loop:none +## CHECK-DAG: InvokeStaticOrDirect [{{i\d+}}{{(,[ij]\d+)?}}] loop:none .method public static simpleLoop2(I)I .registers 3 const/16 v0, 42 diff --git a/test/596-app-images/src/Main.java b/test/596-app-images/src/Main.java index 88d95f4162..8ee3c888b0 100644 --- a/test/596-app-images/src/Main.java +++ b/test/596-app-images/src/Main.java @@ -14,10 +14,6 @@ * limitations under the License. */ -import java.lang.reflect.Field; -import java.lang.reflect.Constructor; -import java.lang.reflect.Method; - class Main { static class Inner { final public static int abc = 10; @@ -50,76 +46,13 @@ class Main { if (!checkInitialized(StaticFieldsInit.class)) System.out.println("StaticFieldsInit class is not initialized!"); - if (!checkInitialized(StaticInternString.class)) - System.out.println("StaticInternString class is not initialized!"); - - StringBuffer sb = new StringBuffer(); - sb.append("java."); - sb.append("abc."); - sb.append("Action"); - - String tmp = sb.toString(); - String intern = tmp.intern(); - - assertNotEqual(tmp, intern, "Dynamically constructed String, not interned."); - assertEqual(intern, StaticInternString.intent, "Static encoded literal String not interned."); - assertEqual(BootInternedString.boot, BootInternedString.boot.intern(), - "Static encoded literal String not moved back to runtime intern table."); - - try { - Field f = StaticInternString.class.getDeclaredField("intent"); - assertEqual(intern, f.get(null), "String Literals are not interned properly."); - - } catch (Exception e) { - System.out.println("Exception"); - } - - assertEqual(StaticInternString.getIntent(), StaticInternString2.getIntent(), - "String Literals are not intenred properly, App image static strings duplicated."); - - // reload the class StaticInternString, check whether static strings interned properly - final String DEX_FILE = System.getenv("DEX_LOCATION") + "/596-app-images.jar"; - final String LIBRARY_SEARCH_PATH = System.getProperty("java.library.path"); - - try { - Class<?> pathClassLoader = Class.forName("dalvik.system.PathClassLoader"); - if (pathClassLoader == null) { - throw new AssertionError("Counldn't find path class loader class"); - } - Constructor<?> ctor = - pathClassLoader.getDeclaredConstructor(String.class, String.class, ClassLoader.class); - ClassLoader loader = (ClassLoader) ctor.newInstance( - DEX_FILE, LIBRARY_SEARCH_PATH, null); - - Class<?> staticInternString = loader.loadClass("StaticInternString"); - - if (!checkAppImageContains(staticInternString)) { - System.out.println("Not loaded again."); - } - Method getIntent = staticInternString.getDeclaredMethod("getIntent"); - - assertEqual(StaticInternString.getIntent(), getIntent.invoke(staticInternString), - "Dynamically loaded app image's literal strings not interned properly."); - } catch (Exception e) { - e.printStackTrace(System.out); - } - + if (checkInitialized(StaticInternString.class)) + System.out.println("StaticInternString class is initialized!"); } public static native boolean checkAppImageLoaded(); public static native boolean checkAppImageContains(Class<?> klass); public static native boolean checkInitialized(Class<?> klass); - - public static void assertEqual(Object a, Object b, String msg) { - if (a != b) - System.out.println(msg); - } - - public static void assertNotEqual(Object a, Object b, String msg) { - if (a == b) - System.out.println(msg); - } - } class StaticFields{ @@ -135,21 +68,6 @@ class StaticFieldsInit{ } class StaticInternString { - final public static String intent = "java.abc.Action"; - static public String getIntent() { - return intent; - } -} - -class BootInternedString { - final public static String boot = "double"; -} - -class StaticInternString2 { - final public static String intent = "java.abc.Action"; - - static String getIntent() { - return intent; - } + final public static String intern = "java.abc.Action"; } diff --git a/test/655-checker-simd-arm-opt/expected.txt b/test/655-checker-simd-arm-opt/expected.txt new file mode 100644 index 0000000000..b0aad4deb5 --- /dev/null +++ b/test/655-checker-simd-arm-opt/expected.txt @@ -0,0 +1 @@ +passed diff --git a/test/655-checker-simd-arm-opt/info.txt b/test/655-checker-simd-arm-opt/info.txt new file mode 100644 index 0000000000..198cc952b2 --- /dev/null +++ b/test/655-checker-simd-arm-opt/info.txt @@ -0,0 +1 @@ +Checker test for arm and arm64 simd optimizations. diff --git a/test/655-checker-simd-arm-opt/src/Main.java b/test/655-checker-simd-arm-opt/src/Main.java new file mode 100644 index 0000000000..7b61dd7951 --- /dev/null +++ b/test/655-checker-simd-arm-opt/src/Main.java @@ -0,0 +1,99 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Checker test for arm and arm64 simd optimizations. + */ +public class Main { + + private static void expectEquals(int expected, int result) { + if (expected != result) { + throw new Error("Expected: " + expected + ", found: " + result); + } + } + + /// CHECK-START-ARM64: void Main.encodableConstants(byte[], short[], char[], int[], long[], float[], double[]) disassembly (after) + /// CHECK-DAG: <<C1:i\d+>> IntConstant 1 + /// CHECK-DAG: <<C2:i\d+>> IntConstant 2 + /// CHECK-DAG: <<C3:i\d+>> IntConstant 3 + /// CHECK-DAG: <<C4:i\d+>> IntConstant 4 + /// CHECK-DAG: <<L5:j\d+>> LongConstant 5 + /// CHECK-DAG: <<F2:f\d+>> FloatConstant 2 + /// CHECK-DAG: <<D20:d\d+>> DoubleConstant 20 + // + /// CHECK-DAG: VecReplicateScalar [<<C1>>] + /// CHECK-DAG: movi v{{[0-9]+}}.16b, #0x1 + /// CHECK-DAG: VecReplicateScalar [<<C2>>] + /// CHECK-DAG: movi v{{[0-9]+}}.8h, #0x2, lsl #0 + /// CHECK-DAG: VecReplicateScalar [<<C3>>] + /// CHECK-DAG: movi v{{[0-9]+}}.8h, #0x3, lsl #0 + /// CHECK-DAG: VecReplicateScalar [<<C4>>] + /// CHECK-DAG: movi v{{[0-9]+}}.4s, #0x4, lsl #0 + /// CHECK-DAG: VecReplicateScalar [<<L5>>] + /// CHECK-DAG: dup v{{[0-9]+}}.2d, x{{[0-9]+}} + /// CHECK-DAG: VecReplicateScalar [<<F2>>] + /// CHECK-DAG: fmov v{{[0-9]+}}.4s, #0x0 + /// CHECK-DAG: VecReplicateScalar [<<D20>>] + /// CHECK-DAG: fmov v{{[0-9]+}}.2d, #0x34 + private static void encodableConstants(byte[] b, short[] s, char[] c, int[] a, long[] l, float[] f, double[] d) { + for (int i = 0; i < ARRAY_SIZE; i++) { + b[i] += 1; + } + for (int i = 0; i < ARRAY_SIZE; i++) { + s[i] += 2; + } + for (int i = 0; i < ARRAY_SIZE; i++) { + c[i] += 3; + } + for (int i = 0; i < ARRAY_SIZE; i++) { + a[i] += 4; + } + for (int i = 0; i < ARRAY_SIZE; i++) { + l[i] += 5; + } + for (int i = 0; i < ARRAY_SIZE; i++) { + f[i] += 2.0f; + } + for (int i = 0; i < ARRAY_SIZE; i++) { + d[i] += 20.0; + } + } + + private static int sumArray(byte[] b, short[] s, char[] c, int[] a, long[] l, float[] f, double[] d) { + int sum = 0; + for (int i = 0; i < ARRAY_SIZE; i++) { + sum += b[i] + s[i] + c[i] + a[i] + l[i] + f[i] + d[i]; + } + return sum; + } + + public static final int ARRAY_SIZE = 100; + + public static void main(String[] args) { + byte[] b = new byte[ARRAY_SIZE]; + short[] s = new short[ARRAY_SIZE]; + char[] c = new char[ARRAY_SIZE]; + int[] a = new int[ARRAY_SIZE]; + long[] l = new long[ARRAY_SIZE]; + float[] f = new float[ARRAY_SIZE]; + double[] d = new double[ARRAY_SIZE]; + + encodableConstants(b, s, c, a, l, f, d); + expectEquals(3700, sumArray(b, s, c, a, l, f, d)); + + System.out.println("passed"); + } +} diff --git a/test/655-jit-clinit/expected.txt b/test/655-jit-clinit/expected.txt new file mode 100644 index 0000000000..6a5618ebc6 --- /dev/null +++ b/test/655-jit-clinit/expected.txt @@ -0,0 +1 @@ +JNI_OnLoad called diff --git a/test/655-jit-clinit/info.txt b/test/655-jit-clinit/info.txt new file mode 100644 index 0000000000..5c81d9b014 --- /dev/null +++ b/test/655-jit-clinit/info.txt @@ -0,0 +1,3 @@ +Regression test for the JIT compiler, which used to wait +on a class object, meaning application code could just block +all JIT compilations. diff --git a/test/655-jit-clinit/src/Main.java b/test/655-jit-clinit/src/Main.java new file mode 100644 index 0000000000..44b315478f --- /dev/null +++ b/test/655-jit-clinit/src/Main.java @@ -0,0 +1,51 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +public class Main { + public static void main(String[] args) throws Exception { + System.loadLibrary(args[0]); + if (!hasJit()) { + return; + } + Foo.hotMethod(); + } + + public native static boolean isJitCompiled(Class<?> cls, String methodName); + private native static boolean hasJit(); +} + +class Foo { + static void hotMethod() { + for (int i = 0; i < array.length; ++i) { + array[i] = array; + } + } + + static { + array = new Object[10000]; + while (!Main.isJitCompiled(Foo.class, "hotMethod")) { + Foo.hotMethod(); + try { + // Sleep to give a chance for the JIT to compile `hotMethod`. + Thread.sleep(100); + } catch (Exception e) { + // Ignore + } + } + } + + static Object[] array; +} diff --git a/test/909-attach-agent/attach.cc b/test/909-attach-agent/attach.cc index 0150e0962f..3a6788a8e3 100644 --- a/test/909-attach-agent/attach.cc +++ b/test/909-attach-agent/attach.cc @@ -27,18 +27,22 @@ namespace art { namespace Test909AttachAgent { +static void Println(const char* c) { + fprintf(stdout, "%s\n", c); + fflush(stdout); +} + jint OnAttach(JavaVM* vm, char* options ATTRIBUTE_UNUSED, void* reserved ATTRIBUTE_UNUSED) { - fprintf(stderr, "Attached Agent for test 909-attach-agent\n"); - fsync(1); + Println("Attached Agent for test 909-attach-agent"); jvmtiEnv* env = nullptr; jvmtiEnv* env2 = nullptr; #define CHECK_CALL_SUCCESS(c) \ do { \ if ((c) != JNI_OK) { \ - fprintf(stderr, "call " #c " did not succeed\n"); \ + Println("call " #c " did not succeed"); \ return -1; \ } \ } while (false) @@ -46,7 +50,7 @@ jint OnAttach(JavaVM* vm, CHECK_CALL_SUCCESS(vm->GetEnv(reinterpret_cast<void**>(&env), JVMTI_VERSION_1_0)); CHECK_CALL_SUCCESS(vm->GetEnv(reinterpret_cast<void**>(&env2), JVMTI_VERSION_1_0)); if (env == env2) { - fprintf(stderr, "GetEnv returned same environment twice!\n"); + Println("GetEnv returned same environment twice!"); return -1; } unsigned char* local_data = nullptr; @@ -56,19 +60,19 @@ jint OnAttach(JavaVM* vm, unsigned char* get_data = nullptr; CHECK_CALL_SUCCESS(env->GetEnvironmentLocalStorage(reinterpret_cast<void**>(&get_data))); if (get_data != local_data) { - fprintf(stderr, "Got different data from local storage then what was set!\n"); + Println("Got different data from local storage then what was set!"); return -1; } CHECK_CALL_SUCCESS(env2->GetEnvironmentLocalStorage(reinterpret_cast<void**>(&get_data))); if (get_data != nullptr) { - fprintf(stderr, "env2 did not have nullptr local storage.\n"); + Println("env2 did not have nullptr local storage."); return -1; } CHECK_CALL_SUCCESS(env->Deallocate(local_data)); jint version = 0; CHECK_CALL_SUCCESS(env->GetVersionNumber(&version)); if ((version & JVMTI_VERSION_1) != JVMTI_VERSION_1) { - fprintf(stderr, "Unexpected version number!\n"); + Println("Unexpected version number!"); return -1; } CHECK_CALL_SUCCESS(env->DisposeEnvironment()); diff --git a/test/Instrumentation/Instrumentation.java b/test/Instrumentation/Instrumentation.java index 09d434213b..b44f78fc3b 100644 --- a/test/Instrumentation/Instrumentation.java +++ b/test/Instrumentation/Instrumentation.java @@ -15,8 +15,21 @@ */ public class Instrumentation { + private static int primitiveField; + private static Object referenceField; + // Direct method private void instanceMethod() { System.out.println("instanceMethod"); } + + private Object returnReference() { + System.out.println("returnReference"); + return null; + } + + private int returnPrimitive() { + System.out.println("returnPrimitive"); + return 0; + } } diff --git a/test/common/runtime_state.cc b/test/common/runtime_state.cc index 7ac019e5d3..d8e5b571bd 100644 --- a/test/common/runtime_state.cc +++ b/test/common/runtime_state.cc @@ -148,12 +148,24 @@ extern "C" JNIEXPORT jboolean JNICALL Java_Main_isAotCompiled(JNIEnv* env, CHECK(chars.c_str() != nullptr); ArtMethod* method = soa.Decode<mirror::Class>(cls)->FindDeclaredDirectMethodByName( chars.c_str(), kRuntimePointerSize); - const void* code = method->GetOatMethodQuickCode(kRuntimePointerSize); - jit::Jit* jit = Runtime::Current()->GetJit(); - if (jit != nullptr && jit->GetCodeCache()->ContainsPc(code)) { - return true; + return method->GetOatMethodQuickCode(kRuntimePointerSize) != nullptr; +} + +extern "C" JNIEXPORT jboolean JNICALL Java_Main_isJitCompiled(JNIEnv* env, + jclass, + jclass cls, + jstring method_name) { + jit::Jit* jit = GetJitIfEnabled(); + if (jit == nullptr) { + return false; } - return code != nullptr; + Thread* self = Thread::Current(); + ScopedObjectAccess soa(self); + ScopedUtfChars chars(env, method_name); + CHECK(chars.c_str() != nullptr); + ArtMethod* method = soa.Decode<mirror::Class>(cls)->FindDeclaredDirectMethodByName( + chars.c_str(), kRuntimePointerSize); + return jit->GetCodeCache()->ContainsPc(method->GetEntryPointFromQuickCompiledCode()); } extern "C" JNIEXPORT void JNICALL Java_Main_ensureJitCompiled(JNIEnv* env, diff --git a/test/knownfailures.json b/test/knownfailures.json index 214b827f6a..dee3b8dbc9 100644 --- a/test/knownfailures.json +++ b/test/knownfailures.json @@ -706,5 +706,16 @@ "variant": "host", "env_vars": {"SANITIZE_HOST": "address"}, "bug": "b/62350406" + }, + { + "tests": "655-jit-clinit", + "description": [ "Currently broken in debug mode"], + "variant": "debug" + }, + { + "tests": ["137-cfi", "629-vdex-speed"], + "description": [ "Tests require speed compilation which is no longer the default for", + "no-prebuild configs."], + "variant": "no-prebuild" } ] diff --git a/test/run-test b/test/run-test index 933a7febac..08be213c89 100755 --- a/test/run-test +++ b/test/run-test @@ -728,10 +728,8 @@ export TEST_NAME=`basename ${test_dir}` # Checker when compiled with Optimizing on host. if [[ "$TEST_NAME" =~ ^[0-9]+-checker- ]]; then if [ "$runtime" = "art" -a "$image_suffix" = "" -a "$USE_JACK" = "true" ]; then - # In no-prebuild mode, the compiler is only invoked if both dex2oat and - # patchoat are available. Disable Checker otherwise (b/22552692). - if [ "$prebuild_mode" = "yes" ] \ - || [ "$have_patchoat" = "yes" -a "$have_dex2oat" = "yes" ]; then + # In no-prebuild or no-image mode, the compiler only quickens so disable the checker. + if [ "$prebuild_mode" = "yes" -a "$have_image" = "yes" ]; then run_checker="yes" if [ "$target_mode" = "no" ]; then diff --git a/test/testrunner/target_config.py b/test/testrunner/target_config.py index 6e47c5eb7a..97d450945c 100644 --- a/test/testrunner/target_config.py +++ b/test/testrunner/target_config.py @@ -316,6 +316,23 @@ target_config = { } }, + # ASAN (host) configurations. + + 'art-gtest-asan': { + 'make' : 'test-art-host-gtest', + 'env': { + 'SANITIZE_HOST' : 'address' + } + }, + 'art-run-test-asan': { + 'run-test' : ['--interpreter', + '--optimizing', + '--jit'], + 'env': { + 'SANITIZE_HOST' : 'address' + } + }, + # ART Golem build targets used by go/lem (continuous ART benchmarking), # (art-opt-cc is used by default since it mimics the default preopt config), # diff --git a/tools/cpp-define-generator/offset_runtime.def b/tools/cpp-define-generator/offset_runtime.def index 17167a0605..41e7e40af5 100644 --- a/tools/cpp-define-generator/offset_runtime.def +++ b/tools/cpp-define-generator/offset_runtime.def @@ -17,7 +17,8 @@ // Offsets within ShadowFrame. #if defined(DEFINE_INCLUDE_DEPENDENCIES) -#include "runtime.h" // art::Runtime +#include "base/callee_save_type.h" // art::CalleeSaveType +#include "runtime.h" // art::Runtime #endif #include "common.def" // DEFINE_OFFSET_EXPR @@ -25,17 +26,20 @@ // Note: these callee save methods loads require read barriers. #define DEFINE_RUNTIME_CALLEE_SAVE_OFFSET(field_name, constant_name) \ - DEFINE_OFFSET_EXPR(Runtime, field_name ## _METHOD, size_t, art::Runtime::GetCalleeSaveMethodOffset(art::Runtime:: constant_name)) + DEFINE_OFFSET_EXPR(Runtime, \ + field_name ## _METHOD, \ + size_t, \ + art::Runtime::GetCalleeSaveMethodOffset(constant_name)) // Macro substring Constant name // Offset of field Runtime::callee_save_methods_[kSaveAllCalleeSaves] -DEFINE_RUNTIME_CALLEE_SAVE_OFFSET(SAVE_ALL_CALLEE_SAVES, kSaveAllCalleeSaves) +DEFINE_RUNTIME_CALLEE_SAVE_OFFSET(SAVE_ALL_CALLEE_SAVES, art::CalleeSaveType::kSaveAllCalleeSaves) // Offset of field Runtime::callee_save_methods_[kSaveRefsOnly] -DEFINE_RUNTIME_CALLEE_SAVE_OFFSET(SAVE_REFS_ONLY, kSaveRefsOnly) +DEFINE_RUNTIME_CALLEE_SAVE_OFFSET(SAVE_REFS_ONLY, art::CalleeSaveType::kSaveRefsOnly) // Offset of field Runtime::callee_save_methods_[kSaveRefsAndArgs] -DEFINE_RUNTIME_CALLEE_SAVE_OFFSET(SAVE_REFS_AND_ARGS, kSaveRefsAndArgs) +DEFINE_RUNTIME_CALLEE_SAVE_OFFSET(SAVE_REFS_AND_ARGS, art::CalleeSaveType::kSaveRefsAndArgs) // Offset of field Runtime::callee_save_methods_[kSaveEverything] -DEFINE_RUNTIME_CALLEE_SAVE_OFFSET(SAVE_EVERYTHING, kSaveEverything) +DEFINE_RUNTIME_CALLEE_SAVE_OFFSET(SAVE_EVERYTHING, art::CalleeSaveType::kSaveEverything) #undef DEFINE_RUNTIME_CALLEE_SAVE_OFFSET #include "common_undef.def" // undef DEFINE_OFFSET_EXPR diff --git a/tools/dexfuzz/src/dexfuzz/program/Mutation.java b/tools/dexfuzz/src/dexfuzz/program/Mutation.java index 2eba7181c9..436fcc4a98 100644 --- a/tools/dexfuzz/src/dexfuzz/program/Mutation.java +++ b/tools/dexfuzz/src/dexfuzz/program/Mutation.java @@ -39,8 +39,21 @@ public abstract class Mutation { this.mutatableCode = mutatableCode; this.mutatableCodeIdx = mutatableCode.mutatableCodeIdx; } - + /** + * Serializes the field(s) of the mutation to string format. + * The fields are separated by a space. + * @return the serialized string representation of the field(s) of the mutation. + */ public abstract String getString(); + /** + * Deserializes the strings back to the field(s) of the mutation, + * given a string array as its argument. The string array + * contains the individual elements which were previously constructed by + * getstring() method. elements[0] stores the class name and elements[1] + * stores the mutable code index which are predefined in MutationSerializer.java. + * Users can deserialize the string representation in elements[2] and so forth. + * @param elements string array with serialized representations of the field(s) of the mutation. + */ public abstract void parseString(String[] elements); }
\ No newline at end of file |