diff options
279 files changed, 16310 insertions, 2674 deletions
diff --git a/PREUPLOAD.cfg b/PREUPLOAD.cfg index cf1832beb4..0ed230c408 100644 --- a/PREUPLOAD.cfg +++ b/PREUPLOAD.cfg @@ -1,2 +1,3 @@ [Hook Scripts] check_generated_files_up_to_date = tools/cpp-define-generator/presubmit-check-files-up-to-date +check_cpplint_on_changed_files = tools/cpplint_presubmit.py diff --git a/build/Android.common_test.mk b/build/Android.common_test.mk index 1876efcd58..1ae79ac8cd 100644 --- a/build/Android.common_test.mk +++ b/build/Android.common_test.mk @@ -66,6 +66,9 @@ ART_TEST_OPTIMIZING ?= true # Do you want to test the optimizing compiler with graph coloring register allocation? ART_TEST_OPTIMIZING_GRAPH_COLOR ?= $(ART_TEST_FULL) +# Do you want to do run-tests with profiles? +ART_TEST_SPEED_PROFILE ?= $(ART_TEST_FULL) + # Do we want to test PIC-compiled tests ("apps")? ART_TEST_PIC_TEST ?= $(ART_TEST_FULL) diff --git a/cmdline/cmdline_types.h b/cmdline/cmdline_types.h index 71c4e95921..cd19fa415c 100644 --- a/cmdline/cmdline_types.h +++ b/cmdline/cmdline_types.h @@ -401,19 +401,19 @@ struct CmdlineType<std::vector<Plugin>> : CmdlineTypeParser<std::vector<Plugin>> }; template <> -struct CmdlineType<std::vector<ti::Agent>> : CmdlineTypeParser<std::vector<ti::Agent>> { +struct CmdlineType<std::list<ti::Agent>> : CmdlineTypeParser<std::list<ti::Agent>> { Result Parse(const std::string& args) { - assert(false && "Use AppendValues() for an Agent vector type"); - return Result::Failure("Unconditional failure: Agent vector must be appended: " + args); + assert(false && "Use AppendValues() for an Agent list type"); + return Result::Failure("Unconditional failure: Agent list must be appended: " + args); } Result ParseAndAppend(const std::string& args, - std::vector<ti::Agent>& existing_value) { + std::list<ti::Agent>& existing_value) { existing_value.emplace_back(args); return Result::SuccessNoValue(); } - static const char* Name() { return "std::vector<ti::Agent>"; } + static const char* Name() { return "std::list<ti::Agent>"; } }; template <> diff --git a/compiler/Android.bp b/compiler/Android.bp index d57f301ff9..312fc7b35a 100644 --- a/compiler/Android.bp +++ b/compiler/Android.bp @@ -106,7 +106,9 @@ art_cc_defaults { "linker/arm/relative_patcher_arm_base.cc", "linker/arm/relative_patcher_thumb2.cc", "optimizing/code_generator_arm.cc", + "optimizing/code_generator_vector_arm.cc", "optimizing/code_generator_arm_vixl.cc", + "optimizing/code_generator_vector_arm_vixl.cc", "optimizing/dex_cache_array_fixups_arm.cc", "optimizing/instruction_simplifier_arm.cc", "optimizing/instruction_simplifier_shared.cc", @@ -126,6 +128,7 @@ art_cc_defaults { "jni/quick/arm64/calling_convention_arm64.cc", "linker/arm64/relative_patcher_arm64.cc", "optimizing/code_generator_arm64.cc", + "optimizing/code_generator_vector_arm64.cc", "optimizing/scheduler_arm64.cc", "optimizing/instruction_simplifier_arm64.cc", "optimizing/intrinsics_arm64.cc", @@ -139,6 +142,7 @@ art_cc_defaults { "jni/quick/mips/calling_convention_mips.cc", "linker/mips/relative_patcher_mips.cc", "optimizing/code_generator_mips.cc", + "optimizing/code_generator_vector_mips.cc", "optimizing/dex_cache_array_fixups_mips.cc", "optimizing/intrinsics_mips.cc", "optimizing/pc_relative_fixups_mips.cc", @@ -151,6 +155,7 @@ art_cc_defaults { "jni/quick/mips64/calling_convention_mips64.cc", "linker/mips64/relative_patcher_mips64.cc", "optimizing/code_generator_mips64.cc", + "optimizing/code_generator_vector_mips64.cc", "optimizing/intrinsics_mips64.cc", "utils/mips64/assembler_mips64.cc", "utils/mips64/managed_register_mips64.cc", @@ -162,6 +167,7 @@ art_cc_defaults { "linker/x86/relative_patcher_x86.cc", "linker/x86/relative_patcher_x86_base.cc", "optimizing/code_generator_x86.cc", + "optimizing/code_generator_vector_x86.cc", "optimizing/intrinsics_x86.cc", "optimizing/pc_relative_fixups_x86.cc", "optimizing/x86_memory_gen.cc", @@ -176,6 +182,7 @@ art_cc_defaults { "linker/x86_64/relative_patcher_x86_64.cc", "optimizing/intrinsics_x86_64.cc", "optimizing/code_generator_x86_64.cc", + "optimizing/code_generator_vector_x86_64.cc", "utils/x86_64/assembler_x86_64.cc", "utils/x86_64/jni_macro_assembler_x86_64.cc", "utils/x86_64/managed_register_x86_64.cc", @@ -391,6 +398,7 @@ art_cc_test { mips64: { srcs: [ "linker/mips64/relative_patcher_mips64_test.cc", + "utils/mips64/managed_register_mips64_test.cc", ], }, x86: { diff --git a/compiler/dex/dex_to_dex_compiler.cc b/compiler/dex/dex_to_dex_compiler.cc index 808e28c9ea..538fe93793 100644 --- a/compiler/dex/dex_to_dex_compiler.cc +++ b/compiler/dex/dex_to_dex_compiler.cc @@ -70,10 +70,6 @@ class DexCompiler { return *unit_.GetDexFile(); } - bool PerformOptimizations() const { - return dex_to_dex_compilation_level_ >= DexToDexCompilationLevel::kOptimize; - } - // Compiles a RETURN-VOID into a RETURN-VOID-BARRIER within a constructor where // a barrier is required. void CompileReturnVoid(Instruction* inst, uint32_t dex_pc); @@ -114,7 +110,7 @@ class DexCompiler { }; void DexCompiler::Compile() { - DCHECK_GE(dex_to_dex_compilation_level_, DexToDexCompilationLevel::kRequired); + DCHECK_EQ(dex_to_dex_compilation_level_, DexToDexCompilationLevel::kOptimize); const DexFile::CodeItem* code_item = unit_.GetCodeItem(); const uint16_t* insns = code_item->insns_; const uint32_t insns_size = code_item->insns_size_in_code_units_; @@ -221,7 +217,7 @@ void DexCompiler::CompileReturnVoid(Instruction* inst, uint32_t dex_pc) { } Instruction* DexCompiler::CompileCheckCast(Instruction* inst, uint32_t dex_pc) { - if (!kEnableCheckCastEllision || !PerformOptimizations()) { + if (!kEnableCheckCastEllision) { return inst; } if (!driver_.IsSafeCast(&unit_, dex_pc)) { @@ -254,7 +250,7 @@ void DexCompiler::CompileInstanceFieldAccess(Instruction* inst, uint32_t dex_pc, Instruction::Code new_opcode, bool is_put) { - if (!kEnableQuickening || !PerformOptimizations()) { + if (!kEnableQuickening) { return; } uint32_t field_idx = inst->VRegC_22c(); @@ -279,7 +275,7 @@ void DexCompiler::CompileInstanceFieldAccess(Instruction* inst, void DexCompiler::CompileInvokeVirtual(Instruction* inst, uint32_t dex_pc, Instruction::Code new_opcode, bool is_range) { - if (!kEnableQuickening || !PerformOptimizations()) { + if (!kEnableQuickening) { return; } uint32_t method_idx = is_range ? inst->VRegB_3rc() : inst->VRegB_35c(); diff --git a/compiler/dex/dex_to_dex_compiler.h b/compiler/dex/dex_to_dex_compiler.h index 00c596d60e..87ddb395ad 100644 --- a/compiler/dex/dex_to_dex_compiler.h +++ b/compiler/dex/dex_to_dex_compiler.h @@ -34,8 +34,7 @@ namespace optimizer { enum class DexToDexCompilationLevel { kDontDexToDexCompile, // Only meaning wrt image time interpretation. - kRequired, // Dex-to-dex compilation required for correctness. - kOptimize // Perform required transformation and peep-hole optimizations. + kOptimize // Perform peep-hole optimizations. }; std::ostream& operator<<(std::ostream& os, const DexToDexCompilationLevel& rhs); diff --git a/compiler/driver/compiler_driver.cc b/compiler/driver/compiler_driver.cc index 995098799c..e823f67d3c 100644 --- a/compiler/driver/compiler_driver.cc +++ b/compiler/driver/compiler_driver.cc @@ -532,16 +532,13 @@ static optimizer::DexToDexCompilationLevel GetDexToDexCompilationLevel( if (driver.GetCompilerOptions().GetDebuggable()) { // We are debuggable so definitions of classes might be changed. We don't want to do any // optimizations that could break that. - max_level = optimizer::DexToDexCompilationLevel::kRequired; + max_level = optimizer::DexToDexCompilationLevel::kDontDexToDexCompile; } if (klass->IsVerified()) { // Class is verified so we can enable DEX-to-DEX compilation for performance. return max_level; - } else if (klass->ShouldVerifyAtRuntime()) { - // Class verification has soft-failed. Anyway, ensure at least correctness. - return optimizer::DexToDexCompilationLevel::kRequired; } else { - // Class verification has failed: do not run DEX-to-DEX compilation. + // Class verification has failed: do not run DEX-to-DEX optimizations. return optimizer::DexToDexCompilationLevel::kDontDexToDexCompile; } } @@ -611,7 +608,7 @@ static void CompileMethod(Thread* self, dex_file, (verified_method != nullptr) ? dex_to_dex_compilation_level - : optimizer::DexToDexCompilationLevel::kRequired); + : optimizer::DexToDexCompilationLevel::kDontDexToDexCompile); } } else if ((access_flags & kAccNative) != 0) { // Are we extracting only and have support for generic JNI down calls? diff --git a/compiler/image_writer.cc b/compiler/image_writer.cc index d156644484..d129249d63 100644 --- a/compiler/image_writer.cc +++ b/compiler/image_writer.cc @@ -1338,21 +1338,20 @@ mirror::Object* ImageWriter::TryAssignBinSlot(WorkStack& work_stack, // live. if (as_klass->ShouldHaveImt()) { ImTable* imt = as_klass->GetImt(target_ptr_size_); - for (size_t i = 0; i < ImTable::kSize; ++i) { - ArtMethod* imt_method = imt->Get(i, target_ptr_size_); - DCHECK(imt_method != nullptr); - if (imt_method->IsRuntimeMethod() && - !IsInBootImage(imt_method) && - !NativeRelocationAssigned(imt_method)) { - AssignMethodOffset(imt_method, kNativeObjectRelocationTypeRuntimeMethod, oat_index); + if (TryAssignImTableOffset(imt, oat_index)) { + // Since imt's can be shared only do this the first time to not double count imt method + // fixups. + for (size_t i = 0; i < ImTable::kSize; ++i) { + ArtMethod* imt_method = imt->Get(i, target_ptr_size_); + DCHECK(imt_method != nullptr); + if (imt_method->IsRuntimeMethod() && + !IsInBootImage(imt_method) && + !NativeRelocationAssigned(imt_method)) { + AssignMethodOffset(imt_method, kNativeObjectRelocationTypeRuntimeMethod, oat_index); + } } } } - - if (as_klass->ShouldHaveImt()) { - ImTable* imt = as_klass->GetImt(target_ptr_size_); - TryAssignImTableOffset(imt, oat_index); - } } else if (obj->IsClassLoader()) { // Register the class loader if it has a class table. // The fake boot class loader should not get registered and we should end up with only one @@ -1386,10 +1385,10 @@ bool ImageWriter::NativeRelocationAssigned(void* ptr) const { return native_object_relocations_.find(ptr) != native_object_relocations_.end(); } -void ImageWriter::TryAssignImTableOffset(ImTable* imt, size_t oat_index) { +bool ImageWriter::TryAssignImTableOffset(ImTable* imt, size_t oat_index) { // No offset, or already assigned. if (imt == nullptr || IsInBootImage(imt) || NativeRelocationAssigned(imt)) { - return; + return false; } // If the method is a conflict method we also want to assign the conflict table offset. ImageInfo& image_info = GetImageInfo(oat_index); @@ -1401,6 +1400,7 @@ void ImageWriter::TryAssignImTableOffset(ImTable* imt, size_t oat_index) { image_info.bin_slot_sizes_[kBinImTable], kNativeObjectRelocationTypeIMTable}); image_info.bin_slot_sizes_[kBinImTable] += size; + return true; } void ImageWriter::TryAssignConflictTableOffset(ImtConflictTable* table, size_t oat_index) { @@ -1499,8 +1499,7 @@ class ImageWriter::VisitReferencesVisitor { ALWAYS_INLINE void operator() (ObjPtr<mirror::Class> klass ATTRIBUTE_UNUSED, ObjPtr<mirror::Reference> ref) const REQUIRES_SHARED(Locks::mutator_lock_) { - ref->SetReferent</*kTransactionActive*/false>( - VisitReference(ref->GetReferent<kWithoutReadBarrier>())); + operator()(ref, mirror::Reference::ReferentOffset(), /* is_static */ false); } private: @@ -1658,7 +1657,7 @@ void ImageWriter::CalculateNewObjectOffsets() { // Calculate size of the dex cache arrays slot and prepare offsets. PrepareDexCacheArraySlots(); - // Calculate the sizes of the intern tables and class tables. + // Calculate the sizes of the intern tables, class tables, and fixup tables. for (ImageInfo& image_info : image_infos_) { // Calculate how big the intern table will be after being serialized. InternTable* const intern_table = image_info.intern_table_.get(); @@ -1666,6 +1665,7 @@ void ImageWriter::CalculateNewObjectOffsets() { if (intern_table->StrongSize() != 0u) { image_info.intern_table_bytes_ = intern_table->WriteToMemory(nullptr); } + // Calculate the size of the class table. ReaderMutexLock mu(self, *Locks::classlinker_classes_lock_); DCHECK_EQ(image_info.class_table_->NumReferencedZygoteClasses(), 0u); @@ -1718,8 +1718,6 @@ void ImageWriter::CalculateNewObjectOffsets() { // Transform each object's bin slot into an offset which will be used to do the final copy. heap->VisitObjects(UnbinObjectsIntoOffsetCallback, this); - // DCHECK_EQ(image_end_, GetBinSizeSum(kBinMirrorCount) + image_objects_offset_begin_); - size_t i = 0; for (ImageInfo& image_info : image_infos_) { image_info.image_roots_address_ = PointerToLowMemUInt32(GetImageAddress(image_roots[i].Get())); @@ -1733,8 +1731,6 @@ void ImageWriter::CalculateNewObjectOffsets() { ImageInfo& image_info = GetImageInfo(relocation.oat_index); relocation.offset += image_info.bin_slot_offsets_[bin_type]; } - - // Note that image_info.image_end_ is left at end of used mirror object section. } size_t ImageWriter::ImageInfo::CreateImageSections(ImageSection* out_sections) const { @@ -1776,7 +1772,6 @@ size_t ImageWriter::ImageInfo::CreateImageSections(ImageSection* out_sections) c ImageSection* dex_cache_arrays_section = &out_sections[ImageHeader::kSectionDexCacheArrays]; *dex_cache_arrays_section = ImageSection(bin_slot_offsets_[kBinDexCacheArray], bin_slot_sizes_[kBinDexCacheArray]); - // Round up to the alignment the string table expects. See HashSet::WriteToMemory. size_t cur_pos = RoundUp(dex_cache_arrays_section->End(), sizeof(uint64_t)); // Calculate the size of the interned strings. @@ -1868,18 +1863,18 @@ class ImageWriter::FixupRootVisitor : public RootVisitor { explicit FixupRootVisitor(ImageWriter* image_writer) : image_writer_(image_writer) { } - void VisitRoots(mirror::Object*** roots, size_t count, const RootInfo& info ATTRIBUTE_UNUSED) + void VisitRoots(mirror::Object*** roots ATTRIBUTE_UNUSED, + size_t count ATTRIBUTE_UNUSED, + const RootInfo& info ATTRIBUTE_UNUSED) OVERRIDE REQUIRES_SHARED(Locks::mutator_lock_) { - for (size_t i = 0; i < count; ++i) { - *roots[i] = image_writer_->GetImageAddress(*roots[i]); - } + LOG(FATAL) << "Unsupported"; } void VisitRoots(mirror::CompressedReference<mirror::Object>** roots, size_t count, const RootInfo& info ATTRIBUTE_UNUSED) OVERRIDE REQUIRES_SHARED(Locks::mutator_lock_) { for (size_t i = 0; i < count; ++i) { - roots[i]->Assign(image_writer_->GetImageAddress(roots[i]->AsMirrorPtr())); + image_writer_->CopyReference(roots[i], roots[i]->AsMirrorPtr()); } } @@ -1890,7 +1885,9 @@ class ImageWriter::FixupRootVisitor : public RootVisitor { void ImageWriter::CopyAndFixupImTable(ImTable* orig, ImTable* copy) { for (size_t i = 0; i < ImTable::kSize; ++i) { ArtMethod* method = orig->Get(i, target_ptr_size_); - copy->Set(i, NativeLocationInImage(method), target_ptr_size_); + void** address = reinterpret_cast<void**>(copy->AddressOfElement(i, target_ptr_size_)); + CopyAndFixupPointer(address, method); + DCHECK_EQ(copy->Get(i, target_ptr_size_), NativeLocationInImage(method)); } } @@ -1899,10 +1896,13 @@ void ImageWriter::CopyAndFixupImtConflictTable(ImtConflictTable* orig, ImtConfli for (size_t i = 0; i < count; ++i) { ArtMethod* interface_method = orig->GetInterfaceMethod(i, target_ptr_size_); ArtMethod* implementation_method = orig->GetImplementationMethod(i, target_ptr_size_); - copy->SetInterfaceMethod(i, target_ptr_size_, NativeLocationInImage(interface_method)); - copy->SetImplementationMethod(i, - target_ptr_size_, - NativeLocationInImage(implementation_method)); + CopyAndFixupPointer(copy->AddressOfInterfaceMethod(i, target_ptr_size_), interface_method); + CopyAndFixupPointer(copy->AddressOfImplementationMethod(i, target_ptr_size_), + implementation_method); + DCHECK_EQ(copy->GetInterfaceMethod(i, target_ptr_size_), + NativeLocationInImage(interface_method)); + DCHECK_EQ(copy->GetImplementationMethod(i, target_ptr_size_), + NativeLocationInImage(implementation_method)); } } @@ -1921,8 +1921,9 @@ void ImageWriter::CopyAndFixupNativeData(size_t oat_index) { switch (relocation.type) { case kNativeObjectRelocationTypeArtField: { memcpy(dest, pair.first, sizeof(ArtField)); - reinterpret_cast<ArtField*>(dest)->SetDeclaringClass( - GetImageAddress(reinterpret_cast<ArtField*>(pair.first)->GetDeclaringClass().Ptr())); + CopyReference( + reinterpret_cast<ArtField*>(dest)->GetDeclaringClassAddressWithoutBarrier(), + reinterpret_cast<ArtField*>(pair.first)->GetDeclaringClass().Ptr()); break; } case kNativeObjectRelocationTypeRuntimeMethod: @@ -2039,8 +2040,10 @@ void ImageWriter::CopyAndFixupObjectsCallback(Object* obj, void* arg) { reinterpret_cast<ImageWriter*>(arg)->CopyAndFixupObject(obj); } -void ImageWriter::FixupPointerArray(mirror::Object* dst, mirror::PointerArray* arr, - mirror::Class* klass, Bin array_type) { +void ImageWriter::FixupPointerArray(mirror::Object* dst, + mirror::PointerArray* arr, + mirror::Class* klass, + Bin array_type) { CHECK(klass->IsArrayClass()); CHECK(arr->IsIntArray() || arr->IsLongArray()) << klass->PrettyClass() << " " << arr; // Fixup int and long pointers for the ArtMethod or ArtField arrays. @@ -2049,7 +2052,7 @@ void ImageWriter::FixupPointerArray(mirror::Object* dst, mirror::PointerArray* a auto* dest_array = down_cast<mirror::PointerArray*>(dst); for (size_t i = 0, count = num_elements; i < count; ++i) { void* elem = arr->GetElementPtrSize<void*>(i, target_ptr_size_); - if (elem != nullptr && !IsInBootImage(elem)) { + if (kIsDebugBuild && elem != nullptr && !IsInBootImage(elem)) { auto it = native_object_relocations_.find(elem); if (UNLIKELY(it == native_object_relocations_.end())) { if (it->second.IsArtMethodRelocation()) { @@ -2065,12 +2068,9 @@ void ImageWriter::FixupPointerArray(mirror::Object* dst, mirror::PointerArray* a << Class::PrettyClass(field->GetDeclaringClass()); } UNREACHABLE(); - } else { - ImageInfo& image_info = GetImageInfo(it->second.oat_index); - elem = image_info.image_begin_ + it->second.offset; } } - dest_array->SetElementPtrSize<false, true>(i, elem, target_ptr_size_); + CopyAndFixupPointer(dest_array->ElementAddress(i, target_ptr_size_), elem); } } @@ -2118,22 +2118,19 @@ class ImageWriter::FixupVisitor { void operator()(ObjPtr<Object> obj, MemberOffset offset, bool is_static ATTRIBUTE_UNUSED) const - REQUIRES(Locks::mutator_lock_, Locks::heap_bitmap_lock_) { + REQUIRES_SHARED(Locks::mutator_lock_) REQUIRES(Locks::heap_bitmap_lock_) { ObjPtr<Object> ref = obj->GetFieldObject<Object, kVerifyNone>(offset); - // Use SetFieldObjectWithoutWriteBarrier to avoid card marking since we are writing to the - // image. - copy_->SetFieldObjectWithoutWriteBarrier<false, true, kVerifyNone>( - offset, - image_writer_->GetImageAddress(ref.Ptr())); + // Copy the reference and record the fixup if necessary. + image_writer_->CopyReference( + copy_->GetFieldObjectReferenceAddr<kVerifyNone>(offset), + ref.Ptr()); } // java.lang.ref.Reference visitor. void operator()(ObjPtr<mirror::Class> klass ATTRIBUTE_UNUSED, ObjPtr<mirror::Reference> ref) const REQUIRES_SHARED(Locks::mutator_lock_) REQUIRES(Locks::heap_bitmap_lock_) { - copy_->SetFieldObjectWithoutWriteBarrier<false, true, kVerifyNone>( - mirror::Reference::ReferentOffset(), - image_writer_->GetImageAddress(ref->GetReferent())); + operator()(ref, mirror::Reference::ReferentOffset(), /* is_static */ false); } protected: @@ -2211,7 +2208,10 @@ class ImageWriter::NativeLocationVisitor { explicit NativeLocationVisitor(ImageWriter* image_writer) : image_writer_(image_writer) {} template <typename T> - T* operator()(T* ptr) const REQUIRES_SHARED(Locks::mutator_lock_) { + T* operator()(T* ptr, void** dest_addr = nullptr) const REQUIRES_SHARED(Locks::mutator_lock_) { + if (dest_addr != nullptr) { + image_writer_->CopyAndFixupPointer(dest_addr, ptr); + } return image_writer_->NativeLocationInImage(ptr); } @@ -2274,10 +2274,10 @@ void ImageWriter::FixupObject(Object* orig, Object* copy) { } } - -class ImageAddressVisitor { +class ImageWriter::ImageAddressVisitorForDexCacheArray { public: - explicit ImageAddressVisitor(ImageWriter* image_writer) : image_writer_(image_writer) {} + explicit ImageAddressVisitorForDexCacheArray(ImageWriter* image_writer) + : image_writer_(image_writer) {} template <typename T> T* operator()(T* ptr) const REQUIRES_SHARED(Locks::mutator_lock_) { @@ -2288,9 +2288,9 @@ class ImageAddressVisitor { ImageWriter* const image_writer_; }; - void ImageWriter::FixupDexCache(mirror::DexCache* orig_dex_cache, mirror::DexCache* copy_dex_cache) { + ImageAddressVisitorForDexCacheArray fixup_visitor(this); // Though the DexCache array fields are usually treated as native pointers, we set the full // 64-bit values here, clearing the top 32 bits for 32-bit targets. The zero-extension is // done by casting to the unsigned type uintptr_t before casting to int64_t, i.e. @@ -2300,8 +2300,7 @@ void ImageWriter::FixupDexCache(mirror::DexCache* orig_dex_cache, copy_dex_cache->SetFieldPtrWithSize<false>(mirror::DexCache::StringsOffset(), NativeLocationInImage(orig_strings), PointerSize::k64); - orig_dex_cache->FixupStrings(NativeCopyLocation(orig_strings, orig_dex_cache), - ImageAddressVisitor(this)); + orig_dex_cache->FixupStrings(NativeCopyLocation(orig_strings, orig_dex_cache), fixup_visitor); } mirror::TypeDexCacheType* orig_types = orig_dex_cache->GetResolvedTypes(); if (orig_types != nullptr) { @@ -2309,7 +2308,7 @@ void ImageWriter::FixupDexCache(mirror::DexCache* orig_dex_cache, NativeLocationInImage(orig_types), PointerSize::k64); orig_dex_cache->FixupResolvedTypes(NativeCopyLocation(orig_types, orig_dex_cache), - ImageAddressVisitor(this)); + fixup_visitor); } ArtMethod** orig_methods = orig_dex_cache->GetResolvedMethods(); if (orig_methods != nullptr) { @@ -2333,7 +2332,8 @@ void ImageWriter::FixupDexCache(mirror::DexCache* orig_dex_cache, for (size_t i = 0, num = orig_dex_cache->NumResolvedFields(); i != num; ++i) { mirror::FieldDexCachePair orig = mirror::DexCache::GetNativePairPtrSize(orig_fields, i, target_ptr_size_); - mirror::FieldDexCachePair copy(NativeLocationInImage(orig.object), orig.index); + mirror::FieldDexCachePair copy = orig; + copy.object = NativeLocationInImage(orig.object); mirror::DexCache::SetNativePairPtrSize(copy_fields, i, copy, target_ptr_size_); } } @@ -2343,7 +2343,7 @@ void ImageWriter::FixupDexCache(mirror::DexCache* orig_dex_cache, NativeLocationInImage(orig_method_types), PointerSize::k64); orig_dex_cache->FixupResolvedMethodTypes(NativeCopyLocation(orig_method_types, orig_dex_cache), - ImageAddressVisitor(this)); + fixup_visitor); } GcRoot<mirror::CallSite>* orig_call_sites = orig_dex_cache->GetResolvedCallSites(); if (orig_call_sites != nullptr) { @@ -2351,7 +2351,7 @@ void ImageWriter::FixupDexCache(mirror::DexCache* orig_dex_cache, NativeLocationInImage(orig_call_sites), PointerSize::k64); orig_dex_cache->FixupResolvedCallSites(NativeCopyLocation(orig_call_sites, orig_dex_cache), - ImageAddressVisitor(this)); + fixup_visitor); } // Remove the DexFile pointers. They will be fixed up when the runtime loads the oat file. Leaving @@ -2459,7 +2459,8 @@ void ImageWriter::CopyAndFixupMethod(ArtMethod* orig, memcpy(copy, orig, ArtMethod::Size(target_ptr_size_)); - copy->SetDeclaringClass(GetImageAddress(orig->GetDeclaringClassUnchecked())); + CopyReference(copy->GetDeclaringClassAddressWithoutBarrier(), orig->GetDeclaringClassUnchecked()); + ArtMethod** orig_resolved_methods = orig->GetDexCacheResolvedMethods(target_ptr_size_); copy->SetDexCacheResolvedMethods(NativeLocationInImage(orig_resolved_methods), target_ptr_size_); @@ -2571,7 +2572,7 @@ size_t ImageWriter::GetOatIndex(mirror::Object* obj) const { return GetDefaultOatIndex(); } auto it = oat_index_map_.find(obj); - DCHECK(it != oat_index_map_.end()); + DCHECK(it != oat_index_map_.end()) << obj; return it->second; } @@ -2672,4 +2673,31 @@ ImageWriter::ImageInfo::ImageInfo() : intern_table_(new InternTable), class_table_(new ClassTable) {} +void ImageWriter::CopyReference(mirror::HeapReference<mirror::Object>* dest, + ObjPtr<mirror::Object> src) { + dest->Assign(GetImageAddress(src.Ptr())); +} + +void ImageWriter::CopyReference(mirror::CompressedReference<mirror::Object>* dest, + ObjPtr<mirror::Object> src) { + dest->Assign(GetImageAddress(src.Ptr())); +} + +void ImageWriter::CopyAndFixupPointer(void** target, void* value) { + void* new_value = value; + if (value != nullptr && !IsInBootImage(value)) { + auto it = native_object_relocations_.find(value); + CHECK(it != native_object_relocations_.end()) << value; + const NativeObjectRelocation& relocation = it->second; + ImageInfo& image_info = GetImageInfo(relocation.oat_index); + new_value = reinterpret_cast<void*>(image_info.image_begin_ + relocation.offset); + } + if (target_ptr_size_ == PointerSize::k32) { + *reinterpret_cast<uint32_t*>(target) = PointerToLowMemUInt32(new_value); + } else { + *reinterpret_cast<uint64_t*>(target) = reinterpret_cast<uintptr_t>(new_value); + } +} + + } // namespace art diff --git a/compiler/image_writer.h b/compiler/image_writer.h index 16aff61dab..39113c8143 100644 --- a/compiler/image_writer.h +++ b/compiler/image_writer.h @@ -38,8 +38,9 @@ #include "image.h" #include "lock_word.h" #include "mem_map.h" -#include "oat_file.h" #include "mirror/dex_cache.h" +#include "obj_ptr.h" +#include "oat_file.h" #include "os.h" #include "safe_map.h" #include "utils.h" @@ -317,6 +318,12 @@ class ImageWriter FINAL { // Number of image class table bytes. size_t class_table_bytes_ = 0; + // Number of object fixup bytes. + size_t object_fixup_bytes_ = 0; + + // Number of pointer fixup bytes. + size_t pointer_fixup_bytes_ = 0; + // Intern table associated with this image for serialization. std::unique_ptr<InternTable> intern_table_; @@ -464,7 +471,8 @@ class ImageWriter FINAL { size_t oat_index) REQUIRES_SHARED(Locks::mutator_lock_); - void TryAssignImTableOffset(ImTable* imt, size_t oat_index) REQUIRES_SHARED(Locks::mutator_lock_); + // Return true if imt was newly inserted. + bool TryAssignImTableOffset(ImTable* imt, size_t oat_index) REQUIRES_SHARED(Locks::mutator_lock_); // Assign the offset for an IMT conflict table. Does nothing if the table already has a native // relocation. @@ -534,6 +542,14 @@ class ImageWriter FINAL { // Return true if there already exists a native allocation for an object. bool NativeRelocationAssigned(void* ptr) const; + void CopyReference(mirror::HeapReference<mirror::Object>* dest, ObjPtr<mirror::Object> src) + REQUIRES_SHARED(Locks::mutator_lock_); + + void CopyReference(mirror::CompressedReference<mirror::Object>* dest, ObjPtr<mirror::Object> src) + REQUIRES_SHARED(Locks::mutator_lock_); + + void CopyAndFixupPointer(void** target, void* value); + const CompilerDriver& compiler_driver_; // Beginning target image address for the first image. @@ -608,9 +624,11 @@ class ImageWriter FINAL { class FixupRootVisitor; class FixupVisitor; class GetRootsVisitor; + class ImageAddressVisitorForDexCacheArray; class NativeLocationVisitor; class PruneClassesVisitor; class PruneClassLoaderClassesVisitor; + class RegisterBootClassPathClassesVisitor; class VisitReferencesVisitor; DISALLOW_COPY_AND_ASSIGN(ImageWriter); diff --git a/compiler/optimizing/bounds_check_elimination.cc b/compiler/optimizing/bounds_check_elimination.cc index 2ee4db923a..476906a768 100644 --- a/compiler/optimizing/bounds_check_elimination.cc +++ b/compiler/optimizing/bounds_check_elimination.cc @@ -528,7 +528,8 @@ class BCEVisitor : public HGraphVisitor { has_dom_based_dynamic_bce_(false), initial_block_size_(graph->GetBlocks().size()), side_effects_(side_effects), - induction_range_(induction_analysis) {} + induction_range_(induction_analysis), + next_(nullptr) {} void VisitBasicBlock(HBasicBlock* block) OVERRIDE { DCHECK(!IsAddedBlock(block)); @@ -1618,8 +1619,8 @@ class BCEVisitor : public HGraphVisitor { void InsertDeoptInLoop(HLoopInformation* loop, HBasicBlock* block, HInstruction* condition) { HInstruction* suspend = loop->GetSuspendCheck(); block->InsertInstructionBefore(condition, block->GetLastInstruction()); - HDeoptimize* deoptimize = - new (GetGraph()->GetArena()) HDeoptimize(condition, suspend->GetDexPc()); + HDeoptimize* deoptimize = new (GetGraph()->GetArena()) HDeoptimize( + GetGraph()->GetArena(), condition, HDeoptimize::Kind::kBCE, suspend->GetDexPc()); block->InsertInstructionBefore(deoptimize, block->GetLastInstruction()); if (suspend->HasEnvironment()) { deoptimize->CopyEnvironmentFromWithLoopPhiAdjustment( @@ -1631,8 +1632,8 @@ class BCEVisitor : public HGraphVisitor { void InsertDeoptInBlock(HBoundsCheck* bounds_check, HInstruction* condition) { HBasicBlock* block = bounds_check->GetBlock(); block->InsertInstructionBefore(condition, bounds_check); - HDeoptimize* deoptimize = - new (GetGraph()->GetArena()) HDeoptimize(condition, bounds_check->GetDexPc()); + HDeoptimize* deoptimize = new (GetGraph()->GetArena()) HDeoptimize( + GetGraph()->GetArena(), condition, HDeoptimize::Kind::kBCE, bounds_check->GetDexPc()); block->InsertInstructionBefore(deoptimize, bounds_check); deoptimize->CopyEnvironmentFrom(bounds_check->GetEnvironment()); } diff --git a/compiler/optimizing/cha_guard_optimization.cc b/compiler/optimizing/cha_guard_optimization.cc index fe423012ca..048073e37a 100644 --- a/compiler/optimizing/cha_guard_optimization.cc +++ b/compiler/optimizing/cha_guard_optimization.cc @@ -36,7 +36,8 @@ class CHAGuardVisitor : HGraphVisitor { : HGraphVisitor(graph), block_has_cha_guard_(GetGraph()->GetBlocks().size(), 0, - graph->GetArena()->Adapter(kArenaAllocCHA)) { + graph->GetArena()->Adapter(kArenaAllocCHA)), + instruction_iterator_(nullptr) { number_of_guards_to_visit_ = GetGraph()->GetNumberOfCHAGuards(); DCHECK_NE(number_of_guards_to_visit_, 0u); // Will recount number of guards during guard optimization. @@ -201,8 +202,8 @@ bool CHAGuardVisitor::HoistGuard(HShouldDeoptimizeFlag* flag, HInstruction* suspend = loop_info->GetSuspendCheck(); // Need a new deoptimize instruction that copies the environment // of the suspend instruction for the loop. - HDeoptimize* deoptimize = - new (GetGraph()->GetArena()) HDeoptimize(compare, suspend->GetDexPc()); + HDeoptimize* deoptimize = new (GetGraph()->GetArena()) HDeoptimize( + GetGraph()->GetArena(), compare, HDeoptimize::Kind::kInline, suspend->GetDexPc()); pre_header->InsertInstructionBefore(deoptimize, pre_header->GetLastInstruction()); deoptimize->CopyEnvironmentFromWithLoopPhiAdjustment( suspend->GetEnvironment(), loop_info->GetHeader()); diff --git a/compiler/optimizing/code_generator_arm.cc b/compiler/optimizing/code_generator_arm.cc index d735b27090..d7cc577580 100644 --- a/compiler/optimizing/code_generator_arm.cc +++ b/compiler/optimizing/code_generator_arm.cc @@ -1134,7 +1134,7 @@ class ReadBarrierForHeapReferenceSlowPathARM : public SlowPathCodeARM { instruction_->IsArrayGet() || instruction_->IsInstanceOf() || instruction_->IsCheckCast() || - (instruction_->IsInvokeVirtual()) && instruction_->GetLocations()->Intrinsified()) + (instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified())) << "Unexpected instruction in read barrier for heap reference slow path: " << instruction_->DebugName(); // The read barrier instrumentation of object ArrayGet diff --git a/compiler/optimizing/code_generator_arm64.cc b/compiler/optimizing/code_generator_arm64.cc index 28cc942dfb..d463830ff6 100644 --- a/compiler/optimizing/code_generator_arm64.cc +++ b/compiler/optimizing/code_generator_arm64.cc @@ -1150,7 +1150,7 @@ class ReadBarrierForHeapReferenceSlowPathARM64 : public SlowPathCodeARM64 { instruction_->IsArrayGet() || instruction_->IsInstanceOf() || instruction_->IsCheckCast() || - (instruction_->IsInvokeVirtual()) && instruction_->GetLocations()->Intrinsified()) + (instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified())) << "Unexpected instruction in read barrier for heap reference slow path: " << instruction_->DebugName(); // The read barrier instrumentation of object ArrayGet @@ -3281,7 +3281,7 @@ void InstructionCodeGeneratorARM64::GenerateDivRemWithAnyConstant(HBinaryOperati void InstructionCodeGeneratorARM64::GenerateDivRemIntegral(HBinaryOperation* instruction) { DCHECK(instruction->IsDiv() || instruction->IsRem()); Primitive::Type type = instruction->GetResultType(); - DCHECK(type == Primitive::kPrimInt || Primitive::kPrimLong); + DCHECK(type == Primitive::kPrimInt || type == Primitive::kPrimLong); LocationSummary* locations = instruction->GetLocations(); Register out = OutputRegister(instruction); diff --git a/compiler/optimizing/code_generator_arm64.h b/compiler/optimizing/code_generator_arm64.h index 7471cd5f12..10d8b841f8 100644 --- a/compiler/optimizing/code_generator_arm64.h +++ b/compiler/optimizing/code_generator_arm64.h @@ -318,6 +318,11 @@ class InstructionCodeGeneratorARM64 : public InstructionCodeGenerator { void GenerateDivRemIntegral(HBinaryOperation* instruction); void HandleGoto(HInstruction* got, HBasicBlock* successor); + vixl::aarch64::MemOperand CreateVecMemRegisters( + HVecMemoryOperation* instruction, + Location* reg_loc, + bool is_load); + Arm64Assembler* const assembler_; CodeGeneratorARM64* const codegen_; diff --git a/compiler/optimizing/code_generator_arm_vixl.cc b/compiler/optimizing/code_generator_arm_vixl.cc index a1c3da9e9c..cce412b314 100644 --- a/compiler/optimizing/code_generator_arm_vixl.cc +++ b/compiler/optimizing/code_generator_arm_vixl.cc @@ -1175,7 +1175,7 @@ class ReadBarrierForHeapReferenceSlowPathARMVIXL : public SlowPathCodeARMVIXL { instruction_->IsArrayGet() || instruction_->IsInstanceOf() || instruction_->IsCheckCast() || - (instruction_->IsInvokeVirtual()) && instruction_->GetLocations()->Intrinsified()) + (instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified())) << "Unexpected instruction in read barrier for heap reference slow path: " << instruction_->DebugName(); // The read barrier instrumentation of object ArrayGet diff --git a/compiler/optimizing/code_generator_mips.cc b/compiler/optimizing/code_generator_mips.cc index 5f02a52417..287891feae 100644 --- a/compiler/optimizing/code_generator_mips.cc +++ b/compiler/optimizing/code_generator_mips.cc @@ -461,6 +461,536 @@ class DeoptimizationSlowPathMIPS : public SlowPathCodeMIPS { DISALLOW_COPY_AND_ASSIGN(DeoptimizationSlowPathMIPS); }; +class ArraySetSlowPathMIPS : public SlowPathCodeMIPS { + public: + explicit ArraySetSlowPathMIPS(HInstruction* instruction) : SlowPathCodeMIPS(instruction) {} + + void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { + LocationSummary* locations = instruction_->GetLocations(); + __ Bind(GetEntryLabel()); + SaveLiveRegisters(codegen, locations); + + InvokeRuntimeCallingConvention calling_convention; + HParallelMove parallel_move(codegen->GetGraph()->GetArena()); + parallel_move.AddMove( + locations->InAt(0), + Location::RegisterLocation(calling_convention.GetRegisterAt(0)), + Primitive::kPrimNot, + nullptr); + parallel_move.AddMove( + locations->InAt(1), + Location::RegisterLocation(calling_convention.GetRegisterAt(1)), + Primitive::kPrimInt, + nullptr); + parallel_move.AddMove( + locations->InAt(2), + Location::RegisterLocation(calling_convention.GetRegisterAt(2)), + Primitive::kPrimNot, + nullptr); + codegen->GetMoveResolver()->EmitNativeCode(¶llel_move); + + CodeGeneratorMIPS* mips_codegen = down_cast<CodeGeneratorMIPS*>(codegen); + mips_codegen->InvokeRuntime(kQuickAputObject, instruction_, instruction_->GetDexPc(), this); + CheckEntrypointTypes<kQuickAputObject, void, mirror::Array*, int32_t, mirror::Object*>(); + RestoreLiveRegisters(codegen, locations); + __ B(GetExitLabel()); + } + + const char* GetDescription() const OVERRIDE { return "ArraySetSlowPathMIPS"; } + + private: + DISALLOW_COPY_AND_ASSIGN(ArraySetSlowPathMIPS); +}; + +// Slow path marking an object reference `ref` during a read +// barrier. The field `obj.field` in the object `obj` holding this +// reference does not get updated by this slow path after marking (see +// ReadBarrierMarkAndUpdateFieldSlowPathMIPS below for that). +// +// This means that after the execution of this slow path, `ref` will +// always be up-to-date, but `obj.field` may not; i.e., after the +// flip, `ref` will be a to-space reference, but `obj.field` will +// probably still be a from-space reference (unless it gets updated by +// another thread, or if another thread installed another object +// reference (different from `ref`) in `obj.field`). +// +// If `entrypoint` is a valid location it is assumed to already be +// holding the entrypoint. The case where the entrypoint is passed in +// is for the GcRoot read barrier. +class ReadBarrierMarkSlowPathMIPS : public SlowPathCodeMIPS { + public: + ReadBarrierMarkSlowPathMIPS(HInstruction* instruction, + Location ref, + Location entrypoint = Location::NoLocation()) + : SlowPathCodeMIPS(instruction), ref_(ref), entrypoint_(entrypoint) { + DCHECK(kEmitCompilerReadBarrier); + } + + const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkSlowPathMIPS"; } + + void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { + LocationSummary* locations = instruction_->GetLocations(); + Register ref_reg = ref_.AsRegister<Register>(); + DCHECK(locations->CanCall()); + DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg)) << ref_reg; + DCHECK(instruction_->IsInstanceFieldGet() || + instruction_->IsStaticFieldGet() || + instruction_->IsArrayGet() || + instruction_->IsArraySet() || + instruction_->IsLoadClass() || + instruction_->IsLoadString() || + instruction_->IsInstanceOf() || + instruction_->IsCheckCast() || + (instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified()) || + (instruction_->IsInvokeStaticOrDirect() && instruction_->GetLocations()->Intrinsified())) + << "Unexpected instruction in read barrier marking slow path: " + << instruction_->DebugName(); + + __ Bind(GetEntryLabel()); + // No need to save live registers; it's taken care of by the + // entrypoint. Also, there is no need to update the stack mask, + // as this runtime call will not trigger a garbage collection. + CodeGeneratorMIPS* mips_codegen = down_cast<CodeGeneratorMIPS*>(codegen); + DCHECK((V0 <= ref_reg && ref_reg <= T7) || + (S2 <= ref_reg && ref_reg <= S7) || + (ref_reg == FP)) << ref_reg; + // "Compact" slow path, saving two moves. + // + // Instead of using the standard runtime calling convention (input + // and output in A0 and V0 respectively): + // + // A0 <- ref + // V0 <- ReadBarrierMark(A0) + // ref <- V0 + // + // we just use rX (the register containing `ref`) as input and output + // of a dedicated entrypoint: + // + // rX <- ReadBarrierMarkRegX(rX) + // + if (entrypoint_.IsValid()) { + mips_codegen->ValidateInvokeRuntimeWithoutRecordingPcInfo(instruction_, this); + DCHECK_EQ(entrypoint_.AsRegister<Register>(), T9); + __ Jalr(entrypoint_.AsRegister<Register>()); + __ NopIfNoReordering(); + } else { + int32_t entry_point_offset = + CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kMipsPointerSize>(ref_reg - 1); + // This runtime call does not require a stack map. + mips_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, + instruction_, + this, + /* direct */ false); + } + __ B(GetExitLabel()); + } + + private: + // The location (register) of the marked object reference. + const Location ref_; + + // The location of the entrypoint if already loaded. + const Location entrypoint_; + + DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkSlowPathMIPS); +}; + +// Slow path marking an object reference `ref` during a read barrier, +// and if needed, atomically updating the field `obj.field` in the +// object `obj` holding this reference after marking (contrary to +// ReadBarrierMarkSlowPathMIPS above, which never tries to update +// `obj.field`). +// +// This means that after the execution of this slow path, both `ref` +// and `obj.field` will be up-to-date; i.e., after the flip, both will +// hold the same to-space reference (unless another thread installed +// another object reference (different from `ref`) in `obj.field`). +class ReadBarrierMarkAndUpdateFieldSlowPathMIPS : public SlowPathCodeMIPS { + public: + ReadBarrierMarkAndUpdateFieldSlowPathMIPS(HInstruction* instruction, + Location ref, + Register obj, + Location field_offset, + Register temp1) + : SlowPathCodeMIPS(instruction), + ref_(ref), + obj_(obj), + field_offset_(field_offset), + temp1_(temp1) { + DCHECK(kEmitCompilerReadBarrier); + } + + const char* GetDescription() const OVERRIDE { + return "ReadBarrierMarkAndUpdateFieldSlowPathMIPS"; + } + + void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { + LocationSummary* locations = instruction_->GetLocations(); + Register ref_reg = ref_.AsRegister<Register>(); + DCHECK(locations->CanCall()); + DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg)) << ref_reg; + // This slow path is only used by the UnsafeCASObject intrinsic. + DCHECK((instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified())) + << "Unexpected instruction in read barrier marking and field updating slow path: " + << instruction_->DebugName(); + DCHECK(instruction_->GetLocations()->Intrinsified()); + DCHECK_EQ(instruction_->AsInvoke()->GetIntrinsic(), Intrinsics::kUnsafeCASObject); + DCHECK(field_offset_.IsRegisterPair()) << field_offset_; + + __ Bind(GetEntryLabel()); + + // Save the old reference. + // Note that we cannot use AT or TMP to save the old reference, as those + // are used by the code that follows, but we need the old reference after + // the call to the ReadBarrierMarkRegX entry point. + DCHECK_NE(temp1_, AT); + DCHECK_NE(temp1_, TMP); + __ Move(temp1_, ref_reg); + + // No need to save live registers; it's taken care of by the + // entrypoint. Also, there is no need to update the stack mask, + // as this runtime call will not trigger a garbage collection. + CodeGeneratorMIPS* mips_codegen = down_cast<CodeGeneratorMIPS*>(codegen); + DCHECK((V0 <= ref_reg && ref_reg <= T7) || + (S2 <= ref_reg && ref_reg <= S7) || + (ref_reg == FP)) << ref_reg; + // "Compact" slow path, saving two moves. + // + // Instead of using the standard runtime calling convention (input + // and output in A0 and V0 respectively): + // + // A0 <- ref + // V0 <- ReadBarrierMark(A0) + // ref <- V0 + // + // we just use rX (the register containing `ref`) as input and output + // of a dedicated entrypoint: + // + // rX <- ReadBarrierMarkRegX(rX) + // + int32_t entry_point_offset = + CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kMipsPointerSize>(ref_reg - 1); + // This runtime call does not require a stack map. + mips_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, + instruction_, + this, + /* direct */ false); + + // If the new reference is different from the old reference, + // update the field in the holder (`*(obj_ + field_offset_)`). + // + // Note that this field could also hold a different object, if + // another thread had concurrently changed it. In that case, the + // the compare-and-set (CAS) loop below would abort, leaving the + // field as-is. + MipsLabel done; + __ Beq(temp1_, ref_reg, &done); + + // Update the the holder's field atomically. This may fail if + // mutator updates before us, but it's OK. This is achieved + // using a strong compare-and-set (CAS) operation with relaxed + // memory synchronization ordering, where the expected value is + // the old reference and the desired value is the new reference. + + // Convenience aliases. + Register base = obj_; + // The UnsafeCASObject intrinsic uses a register pair as field + // offset ("long offset"), of which only the low part contains + // data. + Register offset = field_offset_.AsRegisterPairLow<Register>(); + Register expected = temp1_; + Register value = ref_reg; + Register tmp_ptr = TMP; // Pointer to actual memory. + Register tmp = AT; // Value in memory. + + __ Addu(tmp_ptr, base, offset); + + if (kPoisonHeapReferences) { + __ PoisonHeapReference(expected); + // Do not poison `value` if it is the same register as + // `expected`, which has just been poisoned. + if (value != expected) { + __ PoisonHeapReference(value); + } + } + + // do { + // tmp = [r_ptr] - expected; + // } while (tmp == 0 && failure([r_ptr] <- r_new_value)); + + bool is_r6 = mips_codegen->GetInstructionSetFeatures().IsR6(); + MipsLabel loop_head, exit_loop; + __ Bind(&loop_head); + if (is_r6) { + __ LlR6(tmp, tmp_ptr); + } else { + __ LlR2(tmp, tmp_ptr); + } + __ Bne(tmp, expected, &exit_loop); + __ Move(tmp, value); + if (is_r6) { + __ ScR6(tmp, tmp_ptr); + } else { + __ ScR2(tmp, tmp_ptr); + } + __ Beqz(tmp, &loop_head); + __ Bind(&exit_loop); + + if (kPoisonHeapReferences) { + __ UnpoisonHeapReference(expected); + // Do not unpoison `value` if it is the same register as + // `expected`, which has just been unpoisoned. + if (value != expected) { + __ UnpoisonHeapReference(value); + } + } + + __ Bind(&done); + __ B(GetExitLabel()); + } + + private: + // The location (register) of the marked object reference. + const Location ref_; + // The register containing the object holding the marked object reference field. + const Register obj_; + // The location of the offset of the marked reference field within `obj_`. + Location field_offset_; + + const Register temp1_; + + DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkAndUpdateFieldSlowPathMIPS); +}; + +// Slow path generating a read barrier for a heap reference. +class ReadBarrierForHeapReferenceSlowPathMIPS : public SlowPathCodeMIPS { + public: + ReadBarrierForHeapReferenceSlowPathMIPS(HInstruction* instruction, + Location out, + Location ref, + Location obj, + uint32_t offset, + Location index) + : SlowPathCodeMIPS(instruction), + out_(out), + ref_(ref), + obj_(obj), + offset_(offset), + index_(index) { + DCHECK(kEmitCompilerReadBarrier); + // If `obj` is equal to `out` or `ref`, it means the initial object + // has been overwritten by (or after) the heap object reference load + // to be instrumented, e.g.: + // + // __ LoadFromOffset(kLoadWord, out, out, offset); + // codegen_->GenerateReadBarrierSlow(instruction, out_loc, out_loc, out_loc, offset); + // + // In that case, we have lost the information about the original + // object, and the emitted read barrier cannot work properly. + DCHECK(!obj.Equals(out)) << "obj=" << obj << " out=" << out; + DCHECK(!obj.Equals(ref)) << "obj=" << obj << " ref=" << ref; + } + + void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { + CodeGeneratorMIPS* mips_codegen = down_cast<CodeGeneratorMIPS*>(codegen); + LocationSummary* locations = instruction_->GetLocations(); + Register reg_out = out_.AsRegister<Register>(); + DCHECK(locations->CanCall()); + DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out)); + DCHECK(instruction_->IsInstanceFieldGet() || + instruction_->IsStaticFieldGet() || + instruction_->IsArrayGet() || + instruction_->IsInstanceOf() || + instruction_->IsCheckCast() || + (instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified())) + << "Unexpected instruction in read barrier for heap reference slow path: " + << instruction_->DebugName(); + + __ Bind(GetEntryLabel()); + SaveLiveRegisters(codegen, locations); + + // We may have to change the index's value, but as `index_` is a + // constant member (like other "inputs" of this slow path), + // introduce a copy of it, `index`. + Location index = index_; + if (index_.IsValid()) { + // Handle `index_` for HArrayGet and UnsafeGetObject/UnsafeGetObjectVolatile intrinsics. + if (instruction_->IsArrayGet()) { + // Compute the actual memory offset and store it in `index`. + Register index_reg = index_.AsRegister<Register>(); + DCHECK(locations->GetLiveRegisters()->ContainsCoreRegister(index_reg)); + if (codegen->IsCoreCalleeSaveRegister(index_reg)) { + // We are about to change the value of `index_reg` (see the + // calls to art::mips::MipsAssembler::Sll and + // art::mips::MipsAssembler::Addiu32 below), but it has + // not been saved by the previous call to + // art::SlowPathCode::SaveLiveRegisters, as it is a + // callee-save register -- + // art::SlowPathCode::SaveLiveRegisters does not consider + // callee-save registers, as it has been designed with the + // assumption that callee-save registers are supposed to be + // handled by the called function. So, as a callee-save + // register, `index_reg` _would_ eventually be saved onto + // the stack, but it would be too late: we would have + // changed its value earlier. Therefore, we manually save + // it here into another freely available register, + // `free_reg`, chosen of course among the caller-save + // registers (as a callee-save `free_reg` register would + // exhibit the same problem). + // + // Note we could have requested a temporary register from + // the register allocator instead; but we prefer not to, as + // this is a slow path, and we know we can find a + // caller-save register that is available. + Register free_reg = FindAvailableCallerSaveRegister(codegen); + __ Move(free_reg, index_reg); + index_reg = free_reg; + index = Location::RegisterLocation(index_reg); + } else { + // The initial register stored in `index_` has already been + // saved in the call to art::SlowPathCode::SaveLiveRegisters + // (as it is not a callee-save register), so we can freely + // use it. + } + // Shifting the index value contained in `index_reg` by the scale + // factor (2) cannot overflow in practice, as the runtime is + // unable to allocate object arrays with a size larger than + // 2^26 - 1 (that is, 2^28 - 4 bytes). + __ Sll(index_reg, index_reg, TIMES_4); + static_assert( + sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), + "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); + __ Addiu32(index_reg, index_reg, offset_); + } else { + // In the case of the UnsafeGetObject/UnsafeGetObjectVolatile + // intrinsics, `index_` is not shifted by a scale factor of 2 + // (as in the case of ArrayGet), as it is actually an offset + // to an object field within an object. + DCHECK(instruction_->IsInvoke()) << instruction_->DebugName(); + DCHECK(instruction_->GetLocations()->Intrinsified()); + DCHECK((instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObject) || + (instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile)) + << instruction_->AsInvoke()->GetIntrinsic(); + DCHECK_EQ(offset_, 0U); + DCHECK(index_.IsRegisterPair()); + // UnsafeGet's offset location is a register pair, the low + // part contains the correct offset. + index = index_.ToLow(); + } + } + + // We're moving two or three locations to locations that could + // overlap, so we need a parallel move resolver. + InvokeRuntimeCallingConvention calling_convention; + HParallelMove parallel_move(codegen->GetGraph()->GetArena()); + parallel_move.AddMove(ref_, + Location::RegisterLocation(calling_convention.GetRegisterAt(0)), + Primitive::kPrimNot, + nullptr); + parallel_move.AddMove(obj_, + Location::RegisterLocation(calling_convention.GetRegisterAt(1)), + Primitive::kPrimNot, + nullptr); + if (index.IsValid()) { + parallel_move.AddMove(index, + Location::RegisterLocation(calling_convention.GetRegisterAt(2)), + Primitive::kPrimInt, + nullptr); + codegen->GetMoveResolver()->EmitNativeCode(¶llel_move); + } else { + codegen->GetMoveResolver()->EmitNativeCode(¶llel_move); + __ LoadConst32(calling_convention.GetRegisterAt(2), offset_); + } + mips_codegen->InvokeRuntime(kQuickReadBarrierSlow, + instruction_, + instruction_->GetDexPc(), + this); + CheckEntrypointTypes< + kQuickReadBarrierSlow, mirror::Object*, mirror::Object*, mirror::Object*, uint32_t>(); + mips_codegen->Move32(out_, calling_convention.GetReturnLocation(Primitive::kPrimNot)); + + RestoreLiveRegisters(codegen, locations); + __ B(GetExitLabel()); + } + + const char* GetDescription() const OVERRIDE { return "ReadBarrierForHeapReferenceSlowPathMIPS"; } + + private: + Register FindAvailableCallerSaveRegister(CodeGenerator* codegen) { + size_t ref = static_cast<int>(ref_.AsRegister<Register>()); + size_t obj = static_cast<int>(obj_.AsRegister<Register>()); + for (size_t i = 0, e = codegen->GetNumberOfCoreRegisters(); i < e; ++i) { + if (i != ref && + i != obj && + !codegen->IsCoreCalleeSaveRegister(i) && + !codegen->IsBlockedCoreRegister(i)) { + return static_cast<Register>(i); + } + } + // We shall never fail to find a free caller-save register, as + // there are more than two core caller-save registers on MIPS + // (meaning it is possible to find one which is different from + // `ref` and `obj`). + DCHECK_GT(codegen->GetNumberOfCoreCallerSaveRegisters(), 2u); + LOG(FATAL) << "Could not find a free caller-save register"; + UNREACHABLE(); + } + + const Location out_; + const Location ref_; + const Location obj_; + const uint32_t offset_; + // An additional location containing an index to an array. + // Only used for HArrayGet and the UnsafeGetObject & + // UnsafeGetObjectVolatile intrinsics. + const Location index_; + + DISALLOW_COPY_AND_ASSIGN(ReadBarrierForHeapReferenceSlowPathMIPS); +}; + +// Slow path generating a read barrier for a GC root. +class ReadBarrierForRootSlowPathMIPS : public SlowPathCodeMIPS { + public: + ReadBarrierForRootSlowPathMIPS(HInstruction* instruction, Location out, Location root) + : SlowPathCodeMIPS(instruction), out_(out), root_(root) { + DCHECK(kEmitCompilerReadBarrier); + } + + void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { + LocationSummary* locations = instruction_->GetLocations(); + Register reg_out = out_.AsRegister<Register>(); + DCHECK(locations->CanCall()); + DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out)); + DCHECK(instruction_->IsLoadClass() || instruction_->IsLoadString()) + << "Unexpected instruction in read barrier for GC root slow path: " + << instruction_->DebugName(); + + __ Bind(GetEntryLabel()); + SaveLiveRegisters(codegen, locations); + + InvokeRuntimeCallingConvention calling_convention; + CodeGeneratorMIPS* mips_codegen = down_cast<CodeGeneratorMIPS*>(codegen); + mips_codegen->Move32(Location::RegisterLocation(calling_convention.GetRegisterAt(0)), root_); + mips_codegen->InvokeRuntime(kQuickReadBarrierForRootSlow, + instruction_, + instruction_->GetDexPc(), + this); + CheckEntrypointTypes<kQuickReadBarrierForRootSlow, mirror::Object*, GcRoot<mirror::Object>*>(); + mips_codegen->Move32(out_, calling_convention.GetReturnLocation(Primitive::kPrimNot)); + + RestoreLiveRegisters(codegen, locations); + __ B(GetExitLabel()); + } + + const char* GetDescription() const OVERRIDE { return "ReadBarrierForRootSlowPathMIPS"; } + + private: + const Location out_; + const Location root_; + + DISALLOW_COPY_AND_ASSIGN(ReadBarrierForRootSlowPathMIPS); +}; + CodeGeneratorMIPS::CodeGeneratorMIPS(HGraph* graph, const MipsInstructionSetFeatures& isa_features, const CompilerOptions& compiler_options, @@ -1310,10 +1840,26 @@ void CodeGeneratorMIPS::InvokeRuntime(QuickEntrypointEnum entrypoint, uint32_t dex_pc, SlowPathCode* slow_path) { ValidateInvokeRuntime(entrypoint, instruction, slow_path); + GenerateInvokeRuntime(GetThreadOffset<kMipsPointerSize>(entrypoint).Int32Value(), + IsDirectEntrypoint(entrypoint)); + if (EntrypointRequiresStackMap(entrypoint)) { + RecordPcInfo(instruction, dex_pc, slow_path); + } +} + +void CodeGeneratorMIPS::InvokeRuntimeWithoutRecordingPcInfo(int32_t entry_point_offset, + HInstruction* instruction, + SlowPathCode* slow_path, + bool direct) { + ValidateInvokeRuntimeWithoutRecordingPcInfo(instruction, slow_path); + GenerateInvokeRuntime(entry_point_offset, direct); +} + +void CodeGeneratorMIPS::GenerateInvokeRuntime(int32_t entry_point_offset, bool direct) { bool reordering = __ SetReorder(false); - __ LoadFromOffset(kLoadWord, T9, TR, GetThreadOffset<kMipsPointerSize>(entrypoint).Int32Value()); + __ LoadFromOffset(kLoadWord, T9, TR, entry_point_offset); __ Jalr(T9); - if (IsDirectEntrypoint(entrypoint)) { + if (direct) { // Reserve argument space on stack (for $a0-$a3) for // entrypoints that directly reference native implementations. // Called function may use this space to store $a0-$a3 regs. @@ -1323,9 +1869,6 @@ void CodeGeneratorMIPS::InvokeRuntime(QuickEntrypointEnum entrypoint, __ Nop(); // In delay slot. } __ SetReorder(reordering); - if (EntrypointRequiresStackMap(entrypoint)) { - RecordPcInfo(instruction, dex_pc, slow_path); - } } void InstructionCodeGeneratorMIPS::GenerateClassInitializationCheck(SlowPathCodeMIPS* slow_path, @@ -1885,14 +2428,31 @@ void InstructionCodeGeneratorMIPS::VisitAnd(HAnd* instruction) { } void LocationsBuilderMIPS::VisitArrayGet(HArrayGet* instruction) { + Primitive::Type type = instruction->GetType(); + bool object_array_get_with_read_barrier = + kEmitCompilerReadBarrier && (type == Primitive::kPrimNot); LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); + new (GetGraph()->GetArena()) LocationSummary(instruction, + object_array_get_with_read_barrier + ? LocationSummary::kCallOnSlowPath + : LocationSummary::kNoCall); locations->SetInAt(0, Location::RequiresRegister()); locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); - if (Primitive::IsFloatingPointType(instruction->GetType())) { + if (Primitive::IsFloatingPointType(type)) { locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); } else { - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); + // The output overlaps in the case of an object array get with + // read barriers enabled: we do not want the move to overwrite the + // array's location, as we need it to emit the read barrier. + locations->SetOut(Location::RequiresRegister(), + object_array_get_with_read_barrier + ? Location::kOutputOverlap + : Location::kNoOutputOverlap); + } + // We need a temporary register for the read barrier marking slow + // path in CodeGeneratorMIPS::GenerateArrayLoadWithBakerReadBarrier. + if (object_array_get_with_read_barrier && kUseBakerReadBarrier) { + locations->AddTemp(Location::RequiresRegister()); } } @@ -1905,7 +2465,9 @@ static auto GetImplicitNullChecker(HInstruction* instruction, CodeGeneratorMIPS* void InstructionCodeGeneratorMIPS::VisitArrayGet(HArrayGet* instruction) { LocationSummary* locations = instruction->GetLocations(); - Register obj = locations->InAt(0).AsRegister<Register>(); + Location obj_loc = locations->InAt(0); + Register obj = obj_loc.AsRegister<Register>(); + Location out_loc = locations->Out(); Location index = locations->InAt(1); uint32_t data_offset = CodeGenerator::GetArrayDataOffset(instruction); auto null_checker = GetImplicitNullChecker(instruction, codegen_); @@ -1915,7 +2477,7 @@ void InstructionCodeGeneratorMIPS::VisitArrayGet(HArrayGet* instruction) { instruction->IsStringCharAt(); switch (type) { case Primitive::kPrimBoolean: { - Register out = locations->Out().AsRegister<Register>(); + Register out = out_loc.AsRegister<Register>(); if (index.IsConstant()) { size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset; @@ -1928,7 +2490,7 @@ void InstructionCodeGeneratorMIPS::VisitArrayGet(HArrayGet* instruction) { } case Primitive::kPrimByte: { - Register out = locations->Out().AsRegister<Register>(); + Register out = out_loc.AsRegister<Register>(); if (index.IsConstant()) { size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset; @@ -1941,7 +2503,7 @@ void InstructionCodeGeneratorMIPS::VisitArrayGet(HArrayGet* instruction) { } case Primitive::kPrimShort: { - Register out = locations->Out().AsRegister<Register>(); + Register out = out_loc.AsRegister<Register>(); if (index.IsConstant()) { size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset; @@ -1955,7 +2517,7 @@ void InstructionCodeGeneratorMIPS::VisitArrayGet(HArrayGet* instruction) { } case Primitive::kPrimChar: { - Register out = locations->Out().AsRegister<Register>(); + Register out = out_loc.AsRegister<Register>(); if (maybe_compressed_char_at) { uint32_t count_offset = mirror::String::CountOffset().Uint32Value(); __ LoadFromOffset(kLoadWord, TMP, obj, count_offset, null_checker); @@ -2008,10 +2570,9 @@ void InstructionCodeGeneratorMIPS::VisitArrayGet(HArrayGet* instruction) { break; } - case Primitive::kPrimInt: - case Primitive::kPrimNot: { + case Primitive::kPrimInt: { DCHECK_EQ(sizeof(mirror::HeapReference<mirror::Object>), sizeof(int32_t)); - Register out = locations->Out().AsRegister<Register>(); + Register out = out_loc.AsRegister<Register>(); if (index.IsConstant()) { size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; @@ -2024,8 +2585,53 @@ void InstructionCodeGeneratorMIPS::VisitArrayGet(HArrayGet* instruction) { break; } + case Primitive::kPrimNot: { + static_assert( + sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), + "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); + // /* HeapReference<Object> */ out = + // *(obj + data_offset + index * sizeof(HeapReference<Object>)) + if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { + Location temp = locations->GetTemp(0); + // Note that a potential implicit null check is handled in this + // CodeGeneratorMIPS::GenerateArrayLoadWithBakerReadBarrier call. + codegen_->GenerateArrayLoadWithBakerReadBarrier(instruction, + out_loc, + obj, + data_offset, + index, + temp, + /* needs_null_check */ true); + } else { + Register out = out_loc.AsRegister<Register>(); + if (index.IsConstant()) { + size_t offset = + (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; + __ LoadFromOffset(kLoadWord, out, obj, offset, null_checker); + // If read barriers are enabled, emit read barriers other than + // Baker's using a slow path (and also unpoison the loaded + // reference, if heap poisoning is enabled). + codegen_->MaybeGenerateReadBarrierSlow(instruction, out_loc, out_loc, obj_loc, offset); + } else { + __ Sll(TMP, index.AsRegister<Register>(), TIMES_4); + __ Addu(TMP, obj, TMP); + __ LoadFromOffset(kLoadWord, out, TMP, data_offset, null_checker); + // If read barriers are enabled, emit read barriers other than + // Baker's using a slow path (and also unpoison the loaded + // reference, if heap poisoning is enabled). + codegen_->MaybeGenerateReadBarrierSlow(instruction, + out_loc, + out_loc, + obj_loc, + data_offset, + index); + } + } + break; + } + case Primitive::kPrimLong: { - Register out = locations->Out().AsRegisterPairLow<Register>(); + Register out = out_loc.AsRegisterPairLow<Register>(); if (index.IsConstant()) { size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; @@ -2039,7 +2645,7 @@ void InstructionCodeGeneratorMIPS::VisitArrayGet(HArrayGet* instruction) { } case Primitive::kPrimFloat: { - FRegister out = locations->Out().AsFpuRegister<FRegister>(); + FRegister out = out_loc.AsFpuRegister<FRegister>(); if (index.IsConstant()) { size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; @@ -2053,7 +2659,7 @@ void InstructionCodeGeneratorMIPS::VisitArrayGet(HArrayGet* instruction) { } case Primitive::kPrimDouble: { - FRegister out = locations->Out().AsFpuRegister<FRegister>(); + FRegister out = out_loc.AsFpuRegister<FRegister>(); if (index.IsConstant()) { size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; @@ -2070,11 +2676,6 @@ void InstructionCodeGeneratorMIPS::VisitArrayGet(HArrayGet* instruction) { LOG(FATAL) << "Unreachable type " << instruction->GetType(); UNREACHABLE(); } - - if (type == Primitive::kPrimNot) { - Register out = locations->Out().AsRegister<Register>(); - __ MaybeUnpoisonHeapReference(out); - } } void LocationsBuilderMIPS::VisitArrayLength(HArrayLength* instruction) { @@ -2116,23 +2717,28 @@ Location LocationsBuilderMIPS::FpuRegisterOrConstantForStore(HInstruction* instr } void LocationsBuilderMIPS::VisitArraySet(HArraySet* instruction) { - bool needs_runtime_call = instruction->NeedsTypeCheck(); + Primitive::Type value_type = instruction->GetComponentType(); + + bool needs_write_barrier = + CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue()); + bool may_need_runtime_call_for_type_check = instruction->NeedsTypeCheck(); + LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary( instruction, - needs_runtime_call ? LocationSummary::kCallOnMainOnly : LocationSummary::kNoCall); - if (needs_runtime_call) { - InvokeRuntimeCallingConvention calling_convention; - locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); - locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); - locations->SetInAt(2, Location::RegisterLocation(calling_convention.GetRegisterAt(2))); + may_need_runtime_call_for_type_check ? + LocationSummary::kCallOnSlowPath : + LocationSummary::kNoCall); + + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); + if (Primitive::IsFloatingPointType(instruction->InputAt(2)->GetType())) { + locations->SetInAt(2, FpuRegisterOrConstantForStore(instruction->InputAt(2))); } else { - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); - if (Primitive::IsFloatingPointType(instruction->InputAt(2)->GetType())) { - locations->SetInAt(2, FpuRegisterOrConstantForStore(instruction->InputAt(2))); - } else { - locations->SetInAt(2, RegisterOrZeroConstant(instruction->InputAt(2))); - } + locations->SetInAt(2, RegisterOrZeroConstant(instruction->InputAt(2))); + } + if (needs_write_barrier) { + // Temporary register for the write barrier. + locations->AddTemp(Location::RequiresRegister()); // Possibly used for ref. poisoning too. } } @@ -2142,7 +2748,7 @@ void InstructionCodeGeneratorMIPS::VisitArraySet(HArraySet* instruction) { Location index = locations->InAt(1); Location value_location = locations->InAt(2); Primitive::Type value_type = instruction->GetComponentType(); - bool needs_runtime_call = locations->WillCall(); + bool may_need_runtime_call_for_type_check = instruction->NeedsTypeCheck(); bool needs_write_barrier = CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue()); auto null_checker = GetImplicitNullChecker(instruction, codegen_); @@ -2186,9 +2792,27 @@ void InstructionCodeGeneratorMIPS::VisitArraySet(HArraySet* instruction) { break; } - case Primitive::kPrimInt: + case Primitive::kPrimInt: { + uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value(); + if (index.IsConstant()) { + data_offset += index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4; + } else { + __ Sll(base_reg, index.AsRegister<Register>(), TIMES_4); + __ Addu(base_reg, obj, base_reg); + } + if (value_location.IsConstant()) { + int32_t value = CodeGenerator::GetInt32ValueOf(value_location.GetConstant()); + __ StoreConstToOffset(kStoreWord, value, base_reg, data_offset, TMP, null_checker); + } else { + Register value = value_location.AsRegister<Register>(); + __ StoreToOffset(kStoreWord, value, base_reg, data_offset, null_checker); + } + break; + } + case Primitive::kPrimNot: { - if (!needs_runtime_call) { + if (value_location.IsConstant()) { + // Just setting null. uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value(); if (index.IsConstant()) { data_offset += index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4; @@ -2196,48 +2820,110 @@ void InstructionCodeGeneratorMIPS::VisitArraySet(HArraySet* instruction) { __ Sll(base_reg, index.AsRegister<Register>(), TIMES_4); __ Addu(base_reg, obj, base_reg); } - if (value_location.IsConstant()) { - int32_t value = CodeGenerator::GetInt32ValueOf(value_location.GetConstant()); - __ StoreConstToOffset(kStoreWord, value, base_reg, data_offset, TMP, null_checker); - DCHECK(!needs_write_barrier); - } else { - Register value = value_location.AsRegister<Register>(); - if (kPoisonHeapReferences && needs_write_barrier) { - // Note that in the case where `value` is a null reference, - // we do not enter this block, as a null reference does not - // need poisoning. - DCHECK_EQ(value_type, Primitive::kPrimNot); - // Use Sw() instead of StoreToOffset() in order to be able to - // hold the poisoned reference in AT and thus avoid allocating - // yet another temporary register. - if (index.IsConstant()) { - if (!IsInt<16>(static_cast<int32_t>(data_offset))) { - int16_t low = Low16Bits(data_offset); - uint32_t high = data_offset - low; - __ Addiu32(TMP, obj, high); - base_reg = TMP; - data_offset = low; - } - } else { - DCHECK(IsInt<16>(static_cast<int32_t>(data_offset))); - } - __ PoisonHeapReference(AT, value); - __ Sw(AT, base_reg, data_offset); - null_checker(); + int32_t value = CodeGenerator::GetInt32ValueOf(value_location.GetConstant()); + DCHECK_EQ(value, 0); + __ StoreConstToOffset(kStoreWord, value, base_reg, data_offset, TMP, null_checker); + DCHECK(!needs_write_barrier); + DCHECK(!may_need_runtime_call_for_type_check); + break; + } + + DCHECK(needs_write_barrier); + Register value = value_location.AsRegister<Register>(); + Register temp1 = locations->GetTemp(0).AsRegister<Register>(); + Register temp2 = TMP; // Doesn't need to survive slow path. + uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); + uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); + uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); + MipsLabel done; + SlowPathCodeMIPS* slow_path = nullptr; + + if (may_need_runtime_call_for_type_check) { + slow_path = new (GetGraph()->GetArena()) ArraySetSlowPathMIPS(instruction); + codegen_->AddSlowPath(slow_path); + if (instruction->GetValueCanBeNull()) { + MipsLabel non_zero; + __ Bnez(value, &non_zero); + uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value(); + if (index.IsConstant()) { + data_offset += index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4; } else { - __ StoreToOffset(kStoreWord, value, base_reg, data_offset, null_checker); - } - if (needs_write_barrier) { - DCHECK_EQ(value_type, Primitive::kPrimNot); - codegen_->MarkGCCard(obj, value, instruction->GetValueCanBeNull()); + __ Sll(base_reg, index.AsRegister<Register>(), TIMES_4); + __ Addu(base_reg, obj, base_reg); } + __ StoreToOffset(kStoreWord, value, base_reg, data_offset, null_checker); + __ B(&done); + __ Bind(&non_zero); } + + // Note that when read barriers are enabled, the type checks + // are performed without read barriers. This is fine, even in + // the case where a class object is in the from-space after + // the flip, as a comparison involving such a type would not + // produce a false positive; it may of course produce a false + // negative, in which case we would take the ArraySet slow + // path. + + // /* HeapReference<Class> */ temp1 = obj->klass_ + __ LoadFromOffset(kLoadWord, temp1, obj, class_offset, null_checker); + __ MaybeUnpoisonHeapReference(temp1); + + // /* HeapReference<Class> */ temp1 = temp1->component_type_ + __ LoadFromOffset(kLoadWord, temp1, temp1, component_offset); + // /* HeapReference<Class> */ temp2 = value->klass_ + __ LoadFromOffset(kLoadWord, temp2, value, class_offset); + // If heap poisoning is enabled, no need to unpoison `temp1` + // nor `temp2`, as we are comparing two poisoned references. + + if (instruction->StaticTypeOfArrayIsObjectArray()) { + MipsLabel do_put; + __ Beq(temp1, temp2, &do_put); + // If heap poisoning is enabled, the `temp1` reference has + // not been unpoisoned yet; unpoison it now. + __ MaybeUnpoisonHeapReference(temp1); + + // /* HeapReference<Class> */ temp1 = temp1->super_class_ + __ LoadFromOffset(kLoadWord, temp1, temp1, super_offset); + // If heap poisoning is enabled, no need to unpoison + // `temp1`, as we are comparing against null below. + __ Bnez(temp1, slow_path->GetEntryLabel()); + __ Bind(&do_put); + } else { + __ Bne(temp1, temp2, slow_path->GetEntryLabel()); + } + } + + Register source = value; + if (kPoisonHeapReferences) { + // Note that in the case where `value` is a null reference, + // we do not enter this block, as a null reference does not + // need poisoning. + __ Move(temp1, value); + __ PoisonHeapReference(temp1); + source = temp1; + } + + uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value(); + if (index.IsConstant()) { + data_offset += index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4; } else { - DCHECK_EQ(value_type, Primitive::kPrimNot); - // Note: if heap poisoning is enabled, pAputObject takes care - // of poisoning the reference. - codegen_->InvokeRuntime(kQuickAputObject, instruction, instruction->GetDexPc()); - CheckEntrypointTypes<kQuickAputObject, void, mirror::Array*, int32_t, mirror::Object*>(); + __ Sll(base_reg, index.AsRegister<Register>(), TIMES_4); + __ Addu(base_reg, obj, base_reg); + } + __ StoreToOffset(kStoreWord, source, base_reg, data_offset); + + if (!may_need_runtime_call_for_type_check) { + codegen_->MaybeRecordImplicitNullCheck(instruction); + } + + codegen_->MarkGCCard(obj, value, instruction->GetValueCanBeNull()); + + if (done.IsLinked()) { + __ Bind(&done); + } + + if (slow_path != nullptr) { + __ Bind(slow_path->GetExitLabel()); } break; } @@ -2327,6 +3013,23 @@ void InstructionCodeGeneratorMIPS::VisitBoundsCheck(HBoundsCheck* instruction) { __ Bgeu(index, length, slow_path->GetEntryLabel()); } +// Temp is used for read barrier. +static size_t NumberOfInstanceOfTemps(TypeCheckKind type_check_kind) { + if (kEmitCompilerReadBarrier && + (kUseBakerReadBarrier || + type_check_kind == TypeCheckKind::kAbstractClassCheck || + type_check_kind == TypeCheckKind::kClassHierarchyCheck || + type_check_kind == TypeCheckKind::kArrayObjectCheck)) { + return 1; + } + return 0; +} + +// Extra temp is used for read barrier. +static size_t NumberOfCheckCastTemps(TypeCheckKind type_check_kind) { + return 1 + NumberOfInstanceOfTemps(type_check_kind); +} + void LocationsBuilderMIPS::VisitCheckCast(HCheckCast* instruction) { LocationSummary::CallKind call_kind = LocationSummary::kNoCall; bool throws_into_catch = instruction->CanThrowIntoCatchBlock(); @@ -2337,7 +3040,7 @@ void LocationsBuilderMIPS::VisitCheckCast(HCheckCast* instruction) { case TypeCheckKind::kAbstractClassCheck: case TypeCheckKind::kClassHierarchyCheck: case TypeCheckKind::kArrayObjectCheck: - call_kind = throws_into_catch + call_kind = (throws_into_catch || kEmitCompilerReadBarrier) ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall; // In fact, call on a fatal (non-returning) slow path. break; @@ -2351,15 +3054,20 @@ void LocationsBuilderMIPS::VisitCheckCast(HCheckCast* instruction) { LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind); locations->SetInAt(0, Location::RequiresRegister()); locations->SetInAt(1, Location::RequiresRegister()); - locations->AddTemp(Location::RequiresRegister()); + locations->AddRegisterTemps(NumberOfCheckCastTemps(type_check_kind)); } void InstructionCodeGeneratorMIPS::VisitCheckCast(HCheckCast* instruction) { TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); LocationSummary* locations = instruction->GetLocations(); - Register obj = locations->InAt(0).AsRegister<Register>(); + Location obj_loc = locations->InAt(0); + Register obj = obj_loc.AsRegister<Register>(); Register cls = locations->InAt(1).AsRegister<Register>(); - Register temp = locations->GetTemp(0).AsRegister<Register>(); + Location temp_loc = locations->GetTemp(0); + Register temp = temp_loc.AsRegister<Register>(); + const size_t num_temps = NumberOfCheckCastTemps(type_check_kind); + DCHECK_LE(num_temps, 2u); + Location maybe_temp2_loc = (num_temps >= 2) ? locations->GetTemp(1) : Location::NoLocation(); const uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); const uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); const uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); @@ -2396,8 +3104,12 @@ void InstructionCodeGeneratorMIPS::VisitCheckCast(HCheckCast* instruction) { case TypeCheckKind::kExactCheck: case TypeCheckKind::kArrayCheck: { // /* HeapReference<Class> */ temp = obj->klass_ - __ LoadFromOffset(kLoadWord, temp, obj, class_offset); - __ MaybeUnpoisonHeapReference(temp); + GenerateReferenceLoadTwoRegisters(instruction, + temp_loc, + obj_loc, + class_offset, + maybe_temp2_loc, + kWithoutReadBarrier); // Jump to slow path for throwing the exception or doing a // more involved array check. __ Bne(temp, cls, slow_path->GetEntryLabel()); @@ -2406,15 +3118,22 @@ void InstructionCodeGeneratorMIPS::VisitCheckCast(HCheckCast* instruction) { case TypeCheckKind::kAbstractClassCheck: { // /* HeapReference<Class> */ temp = obj->klass_ - __ LoadFromOffset(kLoadWord, temp, obj, class_offset); - __ MaybeUnpoisonHeapReference(temp); + GenerateReferenceLoadTwoRegisters(instruction, + temp_loc, + obj_loc, + class_offset, + maybe_temp2_loc, + kWithoutReadBarrier); // If the class is abstract, we eagerly fetch the super class of the // object to avoid doing a comparison we know will fail. MipsLabel loop; __ Bind(&loop); // /* HeapReference<Class> */ temp = temp->super_class_ - __ LoadFromOffset(kLoadWord, temp, temp, super_offset); - __ MaybeUnpoisonHeapReference(temp); + GenerateReferenceLoadOneRegister(instruction, + temp_loc, + super_offset, + maybe_temp2_loc, + kWithoutReadBarrier); // If the class reference currently in `temp` is null, jump to the slow path to throw the // exception. __ Beqz(temp, slow_path->GetEntryLabel()); @@ -2425,15 +3144,22 @@ void InstructionCodeGeneratorMIPS::VisitCheckCast(HCheckCast* instruction) { case TypeCheckKind::kClassHierarchyCheck: { // /* HeapReference<Class> */ temp = obj->klass_ - __ LoadFromOffset(kLoadWord, temp, obj, class_offset); - __ MaybeUnpoisonHeapReference(temp); + GenerateReferenceLoadTwoRegisters(instruction, + temp_loc, + obj_loc, + class_offset, + maybe_temp2_loc, + kWithoutReadBarrier); // Walk over the class hierarchy to find a match. MipsLabel loop; __ Bind(&loop); __ Beq(temp, cls, &done); // /* HeapReference<Class> */ temp = temp->super_class_ - __ LoadFromOffset(kLoadWord, temp, temp, super_offset); - __ MaybeUnpoisonHeapReference(temp); + GenerateReferenceLoadOneRegister(instruction, + temp_loc, + super_offset, + maybe_temp2_loc, + kWithoutReadBarrier); // If the class reference currently in `temp` is null, jump to the slow path to throw the // exception. Otherwise, jump to the beginning of the loop. __ Bnez(temp, &loop); @@ -2443,14 +3169,21 @@ void InstructionCodeGeneratorMIPS::VisitCheckCast(HCheckCast* instruction) { case TypeCheckKind::kArrayObjectCheck: { // /* HeapReference<Class> */ temp = obj->klass_ - __ LoadFromOffset(kLoadWord, temp, obj, class_offset); - __ MaybeUnpoisonHeapReference(temp); + GenerateReferenceLoadTwoRegisters(instruction, + temp_loc, + obj_loc, + class_offset, + maybe_temp2_loc, + kWithoutReadBarrier); // Do an exact check. __ Beq(temp, cls, &done); // Otherwise, we need to check that the object's class is a non-primitive array. // /* HeapReference<Class> */ temp = temp->component_type_ - __ LoadFromOffset(kLoadWord, temp, temp, component_offset); - __ MaybeUnpoisonHeapReference(temp); + GenerateReferenceLoadOneRegister(instruction, + temp_loc, + component_offset, + maybe_temp2_loc, + kWithoutReadBarrier); // If the component type is null, jump to the slow path to throw the exception. __ Beqz(temp, slow_path->GetEntryLabel()); // Otherwise, the object is indeed an array, further check that this component @@ -2477,11 +3210,19 @@ void InstructionCodeGeneratorMIPS::VisitCheckCast(HCheckCast* instruction) { // Avoid read barriers to improve performance of the fast path. We can not get false // positives by doing this. // /* HeapReference<Class> */ temp = obj->klass_ - __ LoadFromOffset(kLoadWord, temp, obj, class_offset); - __ MaybeUnpoisonHeapReference(temp); + GenerateReferenceLoadTwoRegisters(instruction, + temp_loc, + obj_loc, + class_offset, + maybe_temp2_loc, + kWithoutReadBarrier); // /* HeapReference<Class> */ temp = temp->iftable_ - __ LoadFromOffset(kLoadWord, temp, temp, iftable_offset); - __ MaybeUnpoisonHeapReference(temp); + GenerateReferenceLoadTwoRegisters(instruction, + temp_loc, + temp_loc, + iftable_offset, + maybe_temp2_loc, + kWithoutReadBarrier); // Iftable is never null. __ Lw(TMP, temp, array_length_offset); // Loop through the iftable and check if any class matches. @@ -5032,8 +5773,15 @@ void LocationsBuilderMIPS::HandleFieldGet(HInstruction* instruction, const Field Primitive::Type field_type = field_info.GetFieldType(); bool is_wide = (field_type == Primitive::kPrimLong) || (field_type == Primitive::kPrimDouble); bool generate_volatile = field_info.IsVolatile() && is_wide; + bool object_field_get_with_read_barrier = + kEmitCompilerReadBarrier && (field_type == Primitive::kPrimNot); LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary( - instruction, generate_volatile ? LocationSummary::kCallOnMainOnly : LocationSummary::kNoCall); + instruction, + generate_volatile + ? LocationSummary::kCallOnMainOnly + : (object_field_get_with_read_barrier + ? LocationSummary::kCallOnSlowPath + : LocationSummary::kNoCall)); locations->SetInAt(0, Location::RequiresRegister()); if (generate_volatile) { @@ -5054,7 +5802,18 @@ void LocationsBuilderMIPS::HandleFieldGet(HInstruction* instruction, const Field if (Primitive::IsFloatingPointType(instruction->GetType())) { locations->SetOut(Location::RequiresFpuRegister()); } else { - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); + // The output overlaps in the case of an object field get with + // read barriers enabled: we do not want the move to overwrite the + // object's location, as we need it to emit the read barrier. + locations->SetOut(Location::RequiresRegister(), + object_field_get_with_read_barrier + ? Location::kOutputOverlap + : Location::kNoOutputOverlap); + } + if (object_field_get_with_read_barrier && kUseBakerReadBarrier) { + // We need a temporary register for the read barrier marking slow + // path in CodeGeneratorMIPS::GenerateFieldLoadWithBakerReadBarrier. + locations->AddTemp(Location::RequiresRegister()); } } } @@ -5064,7 +5823,9 @@ void InstructionCodeGeneratorMIPS::HandleFieldGet(HInstruction* instruction, uint32_t dex_pc) { Primitive::Type type = field_info.GetFieldType(); LocationSummary* locations = instruction->GetLocations(); - Register obj = locations->InAt(0).AsRegister<Register>(); + Location obj_loc = locations->InAt(0); + Register obj = obj_loc.AsRegister<Register>(); + Location dst_loc = locations->Out(); LoadOperandType load_type = kLoadUnsignedByte; bool is_volatile = field_info.IsVolatile(); uint32_t offset = field_info.GetFieldOffset().Uint32Value(); @@ -5107,40 +5868,61 @@ void InstructionCodeGeneratorMIPS::HandleFieldGet(HInstruction* instruction, CheckEntrypointTypes<kQuickA64Load, int64_t, volatile const int64_t*>(); if (type == Primitive::kPrimDouble) { // FP results are returned in core registers. Need to move them. - Location out = locations->Out(); - if (out.IsFpuRegister()) { - __ Mtc1(locations->GetTemp(1).AsRegister<Register>(), out.AsFpuRegister<FRegister>()); + if (dst_loc.IsFpuRegister()) { + __ Mtc1(locations->GetTemp(1).AsRegister<Register>(), dst_loc.AsFpuRegister<FRegister>()); __ MoveToFpuHigh(locations->GetTemp(2).AsRegister<Register>(), - out.AsFpuRegister<FRegister>()); + dst_loc.AsFpuRegister<FRegister>()); } else { - DCHECK(out.IsDoubleStackSlot()); + DCHECK(dst_loc.IsDoubleStackSlot()); __ StoreToOffset(kStoreWord, locations->GetTemp(1).AsRegister<Register>(), SP, - out.GetStackIndex()); + dst_loc.GetStackIndex()); __ StoreToOffset(kStoreWord, locations->GetTemp(2).AsRegister<Register>(), SP, - out.GetStackIndex() + 4); + dst_loc.GetStackIndex() + 4); } } } else { - if (!Primitive::IsFloatingPointType(type)) { + if (type == Primitive::kPrimNot) { + // /* HeapReference<Object> */ dst = *(obj + offset) + if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { + Location temp_loc = locations->GetTemp(0); + // Note that a potential implicit null check is handled in this + // CodeGeneratorMIPS::GenerateFieldLoadWithBakerReadBarrier call. + codegen_->GenerateFieldLoadWithBakerReadBarrier(instruction, + dst_loc, + obj, + offset, + temp_loc, + /* needs_null_check */ true); + if (is_volatile) { + GenerateMemoryBarrier(MemBarrierKind::kLoadAny); + } + } else { + __ LoadFromOffset(kLoadWord, dst_loc.AsRegister<Register>(), obj, offset, null_checker); + if (is_volatile) { + GenerateMemoryBarrier(MemBarrierKind::kLoadAny); + } + // If read barriers are enabled, emit read barriers other than + // Baker's using a slow path (and also unpoison the loaded + // reference, if heap poisoning is enabled). + codegen_->MaybeGenerateReadBarrierSlow(instruction, dst_loc, dst_loc, obj_loc, offset); + } + } else if (!Primitive::IsFloatingPointType(type)) { Register dst; if (type == Primitive::kPrimLong) { - DCHECK(locations->Out().IsRegisterPair()); - dst = locations->Out().AsRegisterPairLow<Register>(); + DCHECK(dst_loc.IsRegisterPair()); + dst = dst_loc.AsRegisterPairLow<Register>(); } else { - DCHECK(locations->Out().IsRegister()); - dst = locations->Out().AsRegister<Register>(); + DCHECK(dst_loc.IsRegister()); + dst = dst_loc.AsRegister<Register>(); } __ LoadFromOffset(load_type, dst, obj, offset, null_checker); - if (type == Primitive::kPrimNot) { - __ MaybeUnpoisonHeapReference(dst); - } } else { - DCHECK(locations->Out().IsFpuRegister()); - FRegister dst = locations->Out().AsFpuRegister<FRegister>(); + DCHECK(dst_loc.IsFpuRegister()); + FRegister dst = dst_loc.AsFpuRegister<FRegister>(); if (type == Primitive::kPrimFloat) { __ LoadSFromOffset(dst, obj, offset, null_checker); } else { @@ -5149,7 +5931,9 @@ void InstructionCodeGeneratorMIPS::HandleFieldGet(HInstruction* instruction, } } - if (is_volatile) { + // Memory barriers, in the case of references, are handled in the + // previous switch statement. + if (is_volatile && (type != Primitive::kPrimNot)) { GenerateMemoryBarrier(MemBarrierKind::kLoadAny); } } @@ -5290,7 +6074,6 @@ void InstructionCodeGeneratorMIPS::HandleFieldSet(HInstruction* instruction, } } - // TODO: memory barriers? if (needs_write_barrier) { Register src = value_location.AsRegister<Register>(); codegen_->MarkGCCard(obj, src, value_can_be_null); @@ -5320,14 +6103,133 @@ void InstructionCodeGeneratorMIPS::VisitInstanceFieldSet(HInstanceFieldSet* inst instruction->GetValueCanBeNull()); } -void InstructionCodeGeneratorMIPS::GenerateGcRootFieldLoad( - HInstruction* instruction ATTRIBUTE_UNUSED, - Location root, - Register obj, - uint32_t offset) { +void InstructionCodeGeneratorMIPS::GenerateReferenceLoadOneRegister( + HInstruction* instruction, + Location out, + uint32_t offset, + Location maybe_temp, + ReadBarrierOption read_barrier_option) { + Register out_reg = out.AsRegister<Register>(); + if (read_barrier_option == kWithReadBarrier) { + CHECK(kEmitCompilerReadBarrier); + DCHECK(maybe_temp.IsRegister()) << maybe_temp; + if (kUseBakerReadBarrier) { + // Load with fast path based Baker's read barrier. + // /* HeapReference<Object> */ out = *(out + offset) + codegen_->GenerateFieldLoadWithBakerReadBarrier(instruction, + out, + out_reg, + offset, + maybe_temp, + /* needs_null_check */ false); + } else { + // Load with slow path based read barrier. + // Save the value of `out` into `maybe_temp` before overwriting it + // in the following move operation, as we will need it for the + // read barrier below. + __ Move(maybe_temp.AsRegister<Register>(), out_reg); + // /* HeapReference<Object> */ out = *(out + offset) + __ LoadFromOffset(kLoadWord, out_reg, out_reg, offset); + codegen_->GenerateReadBarrierSlow(instruction, out, out, maybe_temp, offset); + } + } else { + // Plain load with no read barrier. + // /* HeapReference<Object> */ out = *(out + offset) + __ LoadFromOffset(kLoadWord, out_reg, out_reg, offset); + __ MaybeUnpoisonHeapReference(out_reg); + } +} + +void InstructionCodeGeneratorMIPS::GenerateReferenceLoadTwoRegisters( + HInstruction* instruction, + Location out, + Location obj, + uint32_t offset, + Location maybe_temp, + ReadBarrierOption read_barrier_option) { + Register out_reg = out.AsRegister<Register>(); + Register obj_reg = obj.AsRegister<Register>(); + if (read_barrier_option == kWithReadBarrier) { + CHECK(kEmitCompilerReadBarrier); + if (kUseBakerReadBarrier) { + DCHECK(maybe_temp.IsRegister()) << maybe_temp; + // Load with fast path based Baker's read barrier. + // /* HeapReference<Object> */ out = *(obj + offset) + codegen_->GenerateFieldLoadWithBakerReadBarrier(instruction, + out, + obj_reg, + offset, + maybe_temp, + /* needs_null_check */ false); + } else { + // Load with slow path based read barrier. + // /* HeapReference<Object> */ out = *(obj + offset) + __ LoadFromOffset(kLoadWord, out_reg, obj_reg, offset); + codegen_->GenerateReadBarrierSlow(instruction, out, out, obj, offset); + } + } else { + // Plain load with no read barrier. + // /* HeapReference<Object> */ out = *(obj + offset) + __ LoadFromOffset(kLoadWord, out_reg, obj_reg, offset); + __ MaybeUnpoisonHeapReference(out_reg); + } +} + +void InstructionCodeGeneratorMIPS::GenerateGcRootFieldLoad(HInstruction* instruction, + Location root, + Register obj, + uint32_t offset, + ReadBarrierOption read_barrier_option) { Register root_reg = root.AsRegister<Register>(); - if (kEmitCompilerReadBarrier) { - UNIMPLEMENTED(FATAL) << "for read barrier"; + if (read_barrier_option == kWithReadBarrier) { + DCHECK(kEmitCompilerReadBarrier); + if (kUseBakerReadBarrier) { + // Fast path implementation of art::ReadBarrier::BarrierForRoot when + // Baker's read barrier are used: + // + // root = obj.field; + // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg() + // if (temp != null) { + // root = temp(root) + // } + + // /* GcRoot<mirror::Object> */ root = *(obj + offset) + __ LoadFromOffset(kLoadWord, root_reg, obj, offset); + static_assert( + sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(GcRoot<mirror::Object>), + "art::mirror::CompressedReference<mirror::Object> and art::GcRoot<mirror::Object> " + "have different sizes."); + static_assert(sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(int32_t), + "art::mirror::CompressedReference<mirror::Object> and int32_t " + "have different sizes."); + + // Slow path marking the GC root `root`. + Location temp = Location::RegisterLocation(T9); + SlowPathCodeMIPS* slow_path = + new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathMIPS( + instruction, + root, + /*entrypoint*/ temp); + codegen_->AddSlowPath(slow_path); + + // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg() + const int32_t entry_point_offset = + CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kMipsPointerSize>(root.reg() - 1); + // Loading the entrypoint does not require a load acquire since it is only changed when + // threads are suspended or running a checkpoint. + __ LoadFromOffset(kLoadWord, temp.AsRegister<Register>(), TR, entry_point_offset); + // The entrypoint is null when the GC is not marking, this prevents one load compared to + // checking GetIsGcMarking. + __ Bnez(temp.AsRegister<Register>(), slow_path->GetEntryLabel()); + __ Bind(slow_path->GetExitLabel()); + } else { + // GC root loaded through a slow path for read barriers other + // than Baker's. + // /* GcRoot<mirror::Object>* */ root = obj + offset + __ Addiu32(root_reg, obj, offset); + // /* mirror::Object* */ root = root->Read() + codegen_->GenerateReadBarrierForRootSlow(instruction, root, root); + } } else { // Plain GC root load with no read barrier. // /* GcRoot<mirror::Object> */ root = *(obj + offset) @@ -5337,6 +6239,226 @@ void InstructionCodeGeneratorMIPS::GenerateGcRootFieldLoad( } } +void CodeGeneratorMIPS::GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction, + Location ref, + Register obj, + uint32_t offset, + Location temp, + bool needs_null_check) { + DCHECK(kEmitCompilerReadBarrier); + DCHECK(kUseBakerReadBarrier); + + // /* HeapReference<Object> */ ref = *(obj + offset) + Location no_index = Location::NoLocation(); + ScaleFactor no_scale_factor = TIMES_1; + GenerateReferenceLoadWithBakerReadBarrier(instruction, + ref, + obj, + offset, + no_index, + no_scale_factor, + temp, + needs_null_check); +} + +void CodeGeneratorMIPS::GenerateArrayLoadWithBakerReadBarrier(HInstruction* instruction, + Location ref, + Register obj, + uint32_t data_offset, + Location index, + Location temp, + bool needs_null_check) { + DCHECK(kEmitCompilerReadBarrier); + DCHECK(kUseBakerReadBarrier); + + static_assert( + sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), + "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); + // /* HeapReference<Object> */ ref = + // *(obj + data_offset + index * sizeof(HeapReference<Object>)) + ScaleFactor scale_factor = TIMES_4; + GenerateReferenceLoadWithBakerReadBarrier(instruction, + ref, + obj, + data_offset, + index, + scale_factor, + temp, + needs_null_check); +} + +void CodeGeneratorMIPS::GenerateReferenceLoadWithBakerReadBarrier(HInstruction* instruction, + Location ref, + Register obj, + uint32_t offset, + Location index, + ScaleFactor scale_factor, + Location temp, + bool needs_null_check, + bool always_update_field) { + DCHECK(kEmitCompilerReadBarrier); + DCHECK(kUseBakerReadBarrier); + + // In slow path based read barriers, the read barrier call is + // inserted after the original load. However, in fast path based + // Baker's read barriers, we need to perform the load of + // mirror::Object::monitor_ *before* the original reference load. + // This load-load ordering is required by the read barrier. + // The fast path/slow path (for Baker's algorithm) should look like: + // + // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState(); + // lfence; // Load fence or artificial data dependency to prevent load-load reordering + // HeapReference<Object> ref = *src; // Original reference load. + // bool is_gray = (rb_state == ReadBarrier::GrayState()); + // if (is_gray) { + // ref = ReadBarrier::Mark(ref); // Performed by runtime entrypoint slow path. + // } + // + // Note: the original implementation in ReadBarrier::Barrier is + // slightly more complex as it performs additional checks that we do + // not do here for performance reasons. + + Register ref_reg = ref.AsRegister<Register>(); + Register temp_reg = temp.AsRegister<Register>(); + uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value(); + + // /* int32_t */ monitor = obj->monitor_ + __ LoadFromOffset(kLoadWord, temp_reg, obj, monitor_offset); + if (needs_null_check) { + MaybeRecordImplicitNullCheck(instruction); + } + // /* LockWord */ lock_word = LockWord(monitor) + static_assert(sizeof(LockWord) == sizeof(int32_t), + "art::LockWord and int32_t have different sizes."); + + __ Sync(0); // Barrier to prevent load-load reordering. + + // The actual reference load. + if (index.IsValid()) { + // Load types involving an "index": ArrayGet, + // UnsafeGetObject/UnsafeGetObjectVolatile and UnsafeCASObject + // intrinsics. + // /* HeapReference<Object> */ ref = *(obj + offset + (index << scale_factor)) + if (index.IsConstant()) { + size_t computed_offset = + (index.GetConstant()->AsIntConstant()->GetValue() << scale_factor) + offset; + __ LoadFromOffset(kLoadWord, ref_reg, obj, computed_offset); + } else { + // Handle the special case of the + // UnsafeGetObject/UnsafeGetObjectVolatile and UnsafeCASObject + // intrinsics, which use a register pair as index ("long + // offset"), of which only the low part contains data. + Register index_reg = index.IsRegisterPair() + ? index.AsRegisterPairLow<Register>() + : index.AsRegister<Register>(); + __ Sll(TMP, index_reg, scale_factor); + __ Addu(TMP, obj, TMP); + __ LoadFromOffset(kLoadWord, ref_reg, TMP, offset); + } + } else { + // /* HeapReference<Object> */ ref = *(obj + offset) + __ LoadFromOffset(kLoadWord, ref_reg, obj, offset); + } + + // Object* ref = ref_addr->AsMirrorPtr() + __ MaybeUnpoisonHeapReference(ref_reg); + + // Slow path marking the object `ref` when it is gray. + SlowPathCodeMIPS* slow_path; + if (always_update_field) { + // ReadBarrierMarkAndUpdateFieldSlowPathMIPS only supports address + // of the form `obj + field_offset`, where `obj` is a register and + // `field_offset` is a register pair (of which only the lower half + // is used). Thus `offset` and `scale_factor` above are expected + // to be null in this code path. + DCHECK_EQ(offset, 0u); + DCHECK_EQ(scale_factor, ScaleFactor::TIMES_1); + slow_path = new (GetGraph()->GetArena()) + ReadBarrierMarkAndUpdateFieldSlowPathMIPS(instruction, + ref, + obj, + /* field_offset */ index, + temp_reg); + } else { + slow_path = new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathMIPS(instruction, ref); + } + AddSlowPath(slow_path); + + // if (rb_state == ReadBarrier::GrayState()) + // ref = ReadBarrier::Mark(ref); + // Given the numeric representation, it's enough to check the low bit of the + // rb_state. We do that by shifting the bit into the sign bit (31) and + // performing a branch on less than zero. + static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0"); + static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1"); + static_assert(LockWord::kReadBarrierStateSize == 1, "Expecting 1-bit read barrier state size"); + __ Sll(temp_reg, temp_reg, 31 - LockWord::kReadBarrierStateShift); + __ Bltz(temp_reg, slow_path->GetEntryLabel()); + __ Bind(slow_path->GetExitLabel()); +} + +void CodeGeneratorMIPS::GenerateReadBarrierSlow(HInstruction* instruction, + Location out, + Location ref, + Location obj, + uint32_t offset, + Location index) { + DCHECK(kEmitCompilerReadBarrier); + + // Insert a slow path based read barrier *after* the reference load. + // + // If heap poisoning is enabled, the unpoisoning of the loaded + // reference will be carried out by the runtime within the slow + // path. + // + // Note that `ref` currently does not get unpoisoned (when heap + // poisoning is enabled), which is alright as the `ref` argument is + // not used by the artReadBarrierSlow entry point. + // + // TODO: Unpoison `ref` when it is used by artReadBarrierSlow. + SlowPathCodeMIPS* slow_path = new (GetGraph()->GetArena()) + ReadBarrierForHeapReferenceSlowPathMIPS(instruction, out, ref, obj, offset, index); + AddSlowPath(slow_path); + + __ B(slow_path->GetEntryLabel()); + __ Bind(slow_path->GetExitLabel()); +} + +void CodeGeneratorMIPS::MaybeGenerateReadBarrierSlow(HInstruction* instruction, + Location out, + Location ref, + Location obj, + uint32_t offset, + Location index) { + if (kEmitCompilerReadBarrier) { + // Baker's read barriers shall be handled by the fast path + // (CodeGeneratorMIPS::GenerateReferenceLoadWithBakerReadBarrier). + DCHECK(!kUseBakerReadBarrier); + // If heap poisoning is enabled, unpoisoning will be taken care of + // by the runtime within the slow path. + GenerateReadBarrierSlow(instruction, out, ref, obj, offset, index); + } else if (kPoisonHeapReferences) { + __ UnpoisonHeapReference(out.AsRegister<Register>()); + } +} + +void CodeGeneratorMIPS::GenerateReadBarrierForRootSlow(HInstruction* instruction, + Location out, + Location root) { + DCHECK(kEmitCompilerReadBarrier); + + // Insert a slow path based read barrier *after* the GC root load. + // + // Note that GC roots are not affected by heap poisoning, so we do + // not need to do anything special for this here. + SlowPathCodeMIPS* slow_path = + new (GetGraph()->GetArena()) ReadBarrierForRootSlowPathMIPS(instruction, out, root); + AddSlowPath(slow_path); + + __ B(slow_path->GetEntryLabel()); + __ Bind(slow_path->GetExitLabel()); +} + void LocationsBuilderMIPS::VisitInstanceOf(HInstanceOf* instruction) { LocationSummary::CallKind call_kind = LocationSummary::kNoCall; TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); @@ -5345,7 +6467,8 @@ void LocationsBuilderMIPS::VisitInstanceOf(HInstanceOf* instruction) { case TypeCheckKind::kAbstractClassCheck: case TypeCheckKind::kClassHierarchyCheck: case TypeCheckKind::kArrayObjectCheck: - call_kind = LocationSummary::kNoCall; + call_kind = + kEmitCompilerReadBarrier ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall; break; case TypeCheckKind::kArrayCheck: case TypeCheckKind::kUnresolvedCheck: @@ -5360,14 +6483,20 @@ void LocationsBuilderMIPS::VisitInstanceOf(HInstanceOf* instruction) { // The output does overlap inputs. // Note that TypeCheckSlowPathMIPS uses this register too. locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); + locations->AddRegisterTemps(NumberOfInstanceOfTemps(type_check_kind)); } void InstructionCodeGeneratorMIPS::VisitInstanceOf(HInstanceOf* instruction) { TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); LocationSummary* locations = instruction->GetLocations(); - Register obj = locations->InAt(0).AsRegister<Register>(); + Location obj_loc = locations->InAt(0); + Register obj = obj_loc.AsRegister<Register>(); Register cls = locations->InAt(1).AsRegister<Register>(); - Register out = locations->Out().AsRegister<Register>(); + Location out_loc = locations->Out(); + Register out = out_loc.AsRegister<Register>(); + const size_t num_temps = NumberOfInstanceOfTemps(type_check_kind); + DCHECK_LE(num_temps, 1u); + Location maybe_temp_loc = (num_temps >= 1) ? locations->GetTemp(0) : Location::NoLocation(); uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); @@ -5385,8 +6514,12 @@ void InstructionCodeGeneratorMIPS::VisitInstanceOf(HInstanceOf* instruction) { switch (type_check_kind) { case TypeCheckKind::kExactCheck: { // /* HeapReference<Class> */ out = obj->klass_ - __ LoadFromOffset(kLoadWord, out, obj, class_offset); - __ MaybeUnpoisonHeapReference(out); + GenerateReferenceLoadTwoRegisters(instruction, + out_loc, + obj_loc, + class_offset, + maybe_temp_loc, + kCompilerReadBarrierOption); // Classes must be equal for the instanceof to succeed. __ Xor(out, out, cls); __ Sltiu(out, out, 1); @@ -5395,15 +6528,22 @@ void InstructionCodeGeneratorMIPS::VisitInstanceOf(HInstanceOf* instruction) { case TypeCheckKind::kAbstractClassCheck: { // /* HeapReference<Class> */ out = obj->klass_ - __ LoadFromOffset(kLoadWord, out, obj, class_offset); - __ MaybeUnpoisonHeapReference(out); + GenerateReferenceLoadTwoRegisters(instruction, + out_loc, + obj_loc, + class_offset, + maybe_temp_loc, + kCompilerReadBarrierOption); // If the class is abstract, we eagerly fetch the super class of the // object to avoid doing a comparison we know will fail. MipsLabel loop; __ Bind(&loop); // /* HeapReference<Class> */ out = out->super_class_ - __ LoadFromOffset(kLoadWord, out, out, super_offset); - __ MaybeUnpoisonHeapReference(out); + GenerateReferenceLoadOneRegister(instruction, + out_loc, + super_offset, + maybe_temp_loc, + kCompilerReadBarrierOption); // If `out` is null, we use it for the result, and jump to `done`. __ Beqz(out, &done); __ Bne(out, cls, &loop); @@ -5413,15 +6553,22 @@ void InstructionCodeGeneratorMIPS::VisitInstanceOf(HInstanceOf* instruction) { case TypeCheckKind::kClassHierarchyCheck: { // /* HeapReference<Class> */ out = obj->klass_ - __ LoadFromOffset(kLoadWord, out, obj, class_offset); - __ MaybeUnpoisonHeapReference(out); + GenerateReferenceLoadTwoRegisters(instruction, + out_loc, + obj_loc, + class_offset, + maybe_temp_loc, + kCompilerReadBarrierOption); // Walk over the class hierarchy to find a match. MipsLabel loop, success; __ Bind(&loop); __ Beq(out, cls, &success); // /* HeapReference<Class> */ out = out->super_class_ - __ LoadFromOffset(kLoadWord, out, out, super_offset); - __ MaybeUnpoisonHeapReference(out); + GenerateReferenceLoadOneRegister(instruction, + out_loc, + super_offset, + maybe_temp_loc, + kCompilerReadBarrierOption); __ Bnez(out, &loop); // If `out` is null, we use it for the result, and jump to `done`. __ B(&done); @@ -5432,15 +6579,22 @@ void InstructionCodeGeneratorMIPS::VisitInstanceOf(HInstanceOf* instruction) { case TypeCheckKind::kArrayObjectCheck: { // /* HeapReference<Class> */ out = obj->klass_ - __ LoadFromOffset(kLoadWord, out, obj, class_offset); - __ MaybeUnpoisonHeapReference(out); + GenerateReferenceLoadTwoRegisters(instruction, + out_loc, + obj_loc, + class_offset, + maybe_temp_loc, + kCompilerReadBarrierOption); // Do an exact check. MipsLabel success; __ Beq(out, cls, &success); // Otherwise, we need to check that the object's class is a non-primitive array. // /* HeapReference<Class> */ out = out->component_type_ - __ LoadFromOffset(kLoadWord, out, out, component_offset); - __ MaybeUnpoisonHeapReference(out); + GenerateReferenceLoadOneRegister(instruction, + out_loc, + component_offset, + maybe_temp_loc, + kCompilerReadBarrierOption); // If `out` is null, we use it for the result, and jump to `done`. __ Beqz(out, &done); __ LoadFromOffset(kLoadUnsignedHalfword, out, out, primitive_offset); @@ -5455,8 +6609,12 @@ void InstructionCodeGeneratorMIPS::VisitInstanceOf(HInstanceOf* instruction) { case TypeCheckKind::kArrayCheck: { // No read barrier since the slow path will retry upon failure. // /* HeapReference<Class> */ out = obj->klass_ - __ LoadFromOffset(kLoadWord, out, obj, class_offset); - __ MaybeUnpoisonHeapReference(out); + GenerateReferenceLoadTwoRegisters(instruction, + out_loc, + obj_loc, + class_offset, + maybe_temp_loc, + kWithoutReadBarrier); DCHECK(locations->OnlyCallsOnSlowPath()); slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathMIPS(instruction, /* is_fatal */ false); @@ -5627,9 +6785,6 @@ static bool TryGenerateIntrinsicCode(HInvoke* invoke, CodeGeneratorMIPS* codegen HLoadString::LoadKind CodeGeneratorMIPS::GetSupportedLoadStringKind( HLoadString::LoadKind desired_string_load_kind) { - if (kEmitCompilerReadBarrier) { - UNIMPLEMENTED(FATAL) << "for read barrier"; - } // We disable PC-relative load on pre-R6 when there is an irreducible loop, as the optimization // is incompatible with it. // TODO: Create as many MipsDexCacheArraysBase instructions as needed for methods @@ -5665,9 +6820,6 @@ HLoadString::LoadKind CodeGeneratorMIPS::GetSupportedLoadStringKind( HLoadClass::LoadKind CodeGeneratorMIPS::GetSupportedLoadClassKind( HLoadClass::LoadKind desired_class_load_kind) { - if (kEmitCompilerReadBarrier) { - UNIMPLEMENTED(FATAL) << "for read barrier"; - } // We disable PC-relative load on pre-R6 when there is an irreducible loop, as the optimization // is incompatible with it. bool has_irreducible_loops = GetGraph()->HasIrreducibleLoops(); @@ -5916,12 +7068,13 @@ void LocationsBuilderMIPS::VisitLoadClass(HLoadClass* cls) { CodeGenerator::CreateLoadClassRuntimeCallLocationSummary( cls, Location::RegisterLocation(calling_convention.GetRegisterAt(0)), - Location::RegisterLocation(V0)); + calling_convention.GetReturnLocation(Primitive::kPrimNot)); return; } DCHECK(!cls->NeedsAccessCheck()); - LocationSummary::CallKind call_kind = (cls->NeedsEnvironment() || kEmitCompilerReadBarrier) + const bool requires_read_barrier = kEmitCompilerReadBarrier && !cls->IsInBootImage(); + LocationSummary::CallKind call_kind = (cls->NeedsEnvironment() || requires_read_barrier) ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall; LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(cls, call_kind); @@ -5976,6 +7129,9 @@ void InstructionCodeGeneratorMIPS::VisitLoadClass(HLoadClass* cls) NO_THREAD_SAF break; } + const ReadBarrierOption read_barrier_option = cls->IsInBootImage() + ? kWithoutReadBarrier + : kCompilerReadBarrierOption; bool generate_null_check = false; switch (load_kind) { case HLoadClass::LoadKind::kReferrersClass: { @@ -5985,11 +7141,13 @@ void InstructionCodeGeneratorMIPS::VisitLoadClass(HLoadClass* cls) NO_THREAD_SAF GenerateGcRootFieldLoad(cls, out_loc, base_or_current_method_reg, - ArtMethod::DeclaringClassOffset().Int32Value()); + ArtMethod::DeclaringClassOffset().Int32Value(), + read_barrier_option); break; } case HLoadClass::LoadKind::kBootImageLinkTimeAddress: DCHECK(codegen_->GetCompilerOptions().IsBootImage()); + DCHECK_EQ(read_barrier_option, kWithoutReadBarrier); __ LoadLiteral(out, base_or_current_method_reg, codegen_->DeduplicateBootImageTypeLiteral(cls->GetDexFile(), @@ -5997,6 +7155,7 @@ void InstructionCodeGeneratorMIPS::VisitLoadClass(HLoadClass* cls) NO_THREAD_SAF break; case HLoadClass::LoadKind::kBootImageLinkTimePcRelative: { DCHECK(codegen_->GetCompilerOptions().IsBootImage()); + DCHECK_EQ(read_barrier_option, kWithoutReadBarrier); CodeGeneratorMIPS::PcRelativePatchInfo* info = codegen_->NewPcRelativeTypePatch(cls->GetDexFile(), cls->GetTypeIndex()); bool reordering = __ SetReorder(false); @@ -6006,7 +7165,7 @@ void InstructionCodeGeneratorMIPS::VisitLoadClass(HLoadClass* cls) NO_THREAD_SAF break; } case HLoadClass::LoadKind::kBootImageAddress: { - DCHECK(!kEmitCompilerReadBarrier); + DCHECK_EQ(read_barrier_option, kWithoutReadBarrier); uint32_t address = dchecked_integral_cast<uint32_t>( reinterpret_cast<uintptr_t>(cls->GetClass().Get())); DCHECK_NE(address, 0u); @@ -6020,7 +7179,7 @@ void InstructionCodeGeneratorMIPS::VisitLoadClass(HLoadClass* cls) NO_THREAD_SAF codegen_->NewTypeBssEntryPatch(cls->GetDexFile(), cls->GetTypeIndex()); bool reordering = __ SetReorder(false); codegen_->EmitPcRelativeAddressPlaceholderHigh(info, out, base_or_current_method_reg); - GenerateGcRootFieldLoad(cls, out_loc, out, /* placeholder */ 0x5678); + GenerateGcRootFieldLoad(cls, out_loc, out, /* placeholder */ 0x5678, read_barrier_option); __ SetReorder(reordering); generate_null_check = true; break; @@ -6032,7 +7191,7 @@ void InstructionCodeGeneratorMIPS::VisitLoadClass(HLoadClass* cls) NO_THREAD_SAF bool reordering = __ SetReorder(false); __ Bind(&info->high_label); __ Lui(out, /* placeholder */ 0x1234); - GenerateGcRootFieldLoad(cls, out_loc, out, /* placeholder */ 0x5678); + GenerateGcRootFieldLoad(cls, out_loc, out, /* placeholder */ 0x5678, read_barrier_option); __ SetReorder(reordering); break; } @@ -6165,7 +7324,11 @@ void InstructionCodeGeneratorMIPS::VisitLoadString(HLoadString* load) NO_THREAD_ codegen_->NewPcRelativeStringPatch(load->GetDexFile(), load->GetStringIndex()); bool reordering = __ SetReorder(false); codegen_->EmitPcRelativeAddressPlaceholderHigh(info, out, base_or_current_method_reg); - GenerateGcRootFieldLoad(load, out_loc, out, /* placeholder */ 0x5678); + GenerateGcRootFieldLoad(load, + out_loc, + out, + /* placeholder */ 0x5678, + kCompilerReadBarrierOption); __ SetReorder(reordering); SlowPathCodeMIPS* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathMIPS(load); codegen_->AddSlowPath(slow_path); @@ -6181,7 +7344,11 @@ void InstructionCodeGeneratorMIPS::VisitLoadString(HLoadString* load) NO_THREAD_ bool reordering = __ SetReorder(false); __ Bind(&info->high_label); __ Lui(out, /* placeholder */ 0x1234); - GenerateGcRootFieldLoad(load, out_loc, out, /* placeholder */ 0x5678); + GenerateGcRootFieldLoad(load, + out_loc, + out, + /* placeholder */ 0x5678, + kCompilerReadBarrierOption); __ SetReorder(reordering); return; } diff --git a/compiler/optimizing/code_generator_mips.h b/compiler/optimizing/code_generator_mips.h index 98fee24a74..3875c4bdba 100644 --- a/compiler/optimizing/code_generator_mips.h +++ b/compiler/optimizing/code_generator_mips.h @@ -241,6 +241,38 @@ class InstructionCodeGeneratorMIPS : public InstructionCodeGenerator { uint32_t dex_pc, bool value_can_be_null); void HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info, uint32_t dex_pc); + + // Generate a heap reference load using one register `out`: + // + // out <- *(out + offset) + // + // while honoring heap poisoning and/or read barriers (if any). + // + // Location `maybe_temp` is used when generating a read barrier and + // shall be a register in that case; it may be an invalid location + // otherwise. + void GenerateReferenceLoadOneRegister(HInstruction* instruction, + Location out, + uint32_t offset, + Location maybe_temp, + ReadBarrierOption read_barrier_option); + // Generate a heap reference load using two different registers + // `out` and `obj`: + // + // out <- *(obj + offset) + // + // while honoring heap poisoning and/or read barriers (if any). + // + // Location `maybe_temp` is used when generating a Baker's (fast + // path) read barrier and shall be a register in that case; it may + // be an invalid location otherwise. + void GenerateReferenceLoadTwoRegisters(HInstruction* instruction, + Location out, + Location obj, + uint32_t offset, + Location maybe_temp, + ReadBarrierOption read_barrier_option); + // Generate a GC root reference load: // // root <- *(obj + offset) @@ -249,7 +281,9 @@ class InstructionCodeGeneratorMIPS : public InstructionCodeGenerator { void GenerateGcRootFieldLoad(HInstruction* instruction, Location root, Register obj, - uint32_t offset); + uint32_t offset, + ReadBarrierOption read_barrier_option); + void GenerateIntCompare(IfCondition cond, LocationSummary* locations); // When the function returns `false` it means that the condition holds if `dst` is non-zero // and doesn't hold if `dst` is zero. If it returns `true`, the roles of zero and non-zero @@ -353,6 +387,91 @@ class CodeGeneratorMIPS : public CodeGenerator { void EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) OVERRIDE; void EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) OVERRIDE; + // Fast path implementation of ReadBarrier::Barrier for a heap + // reference field load when Baker's read barriers are used. + void GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction, + Location ref, + Register obj, + uint32_t offset, + Location temp, + bool needs_null_check); + // Fast path implementation of ReadBarrier::Barrier for a heap + // reference array load when Baker's read barriers are used. + void GenerateArrayLoadWithBakerReadBarrier(HInstruction* instruction, + Location ref, + Register obj, + uint32_t data_offset, + Location index, + Location temp, + bool needs_null_check); + + // Factored implementation, used by GenerateFieldLoadWithBakerReadBarrier, + // GenerateArrayLoadWithBakerReadBarrier and some intrinsics. + // + // Load the object reference located at the address + // `obj + offset + (index << scale_factor)`, held by object `obj`, into + // `ref`, and mark it if needed. + // + // If `always_update_field` is true, the value of the reference is + // atomically updated in the holder (`obj`). + void GenerateReferenceLoadWithBakerReadBarrier(HInstruction* instruction, + Location ref, + Register obj, + uint32_t offset, + Location index, + ScaleFactor scale_factor, + Location temp, + bool needs_null_check, + bool always_update_field = false); + + // Generate a read barrier for a heap reference within `instruction` + // using a slow path. + // + // A read barrier for an object reference read from the heap is + // implemented as a call to the artReadBarrierSlow runtime entry + // point, which is passed the values in locations `ref`, `obj`, and + // `offset`: + // + // mirror::Object* artReadBarrierSlow(mirror::Object* ref, + // mirror::Object* obj, + // uint32_t offset); + // + // The `out` location contains the value returned by + // artReadBarrierSlow. + // + // When `index` is provided (i.e. for array accesses), the offset + // value passed to artReadBarrierSlow is adjusted to take `index` + // into account. + void GenerateReadBarrierSlow(HInstruction* instruction, + Location out, + Location ref, + Location obj, + uint32_t offset, + Location index = Location::NoLocation()); + + // If read barriers are enabled, generate a read barrier for a heap + // reference using a slow path. If heap poisoning is enabled, also + // unpoison the reference in `out`. + void MaybeGenerateReadBarrierSlow(HInstruction* instruction, + Location out, + Location ref, + Location obj, + uint32_t offset, + Location index = Location::NoLocation()); + + // Generate a read barrier for a GC root within `instruction` using + // a slow path. + // + // A read barrier for an object reference GC root is implemented as + // a call to the artReadBarrierForRootSlow runtime entry point, + // which is passed the value in location `root`: + // + // mirror::Object* artReadBarrierForRootSlow(GcRoot<mirror::Object>* root); + // + // The `out` location contains the value returned by + // artReadBarrierForRootSlow. + void GenerateReadBarrierForRootSlow(HInstruction* instruction, Location out, Location root); + void MarkGCCard(Register object, Register value, bool value_can_be_null); // Register allocation. @@ -400,6 +519,15 @@ class CodeGeneratorMIPS : public CodeGenerator { uint32_t dex_pc, SlowPathCode* slow_path = nullptr) OVERRIDE; + // Generate code to invoke a runtime entry point, but do not record + // PC-related information in a stack map. + void InvokeRuntimeWithoutRecordingPcInfo(int32_t entry_point_offset, + HInstruction* instruction, + SlowPathCode* slow_path, + bool direct); + + void GenerateInvokeRuntime(int32_t entry_point_offset, bool direct); + ParallelMoveResolver* GetMoveResolver() OVERRIDE { return &move_resolver_; } bool NeedsTwoRegisters(Primitive::Type type) const OVERRIDE { diff --git a/compiler/optimizing/code_generator_mips64.cc b/compiler/optimizing/code_generator_mips64.cc index c82533bc7d..78b31e9e86 100644 --- a/compiler/optimizing/code_generator_mips64.cc +++ b/compiler/optimizing/code_generator_mips64.cc @@ -407,6 +407,528 @@ class DeoptimizationSlowPathMIPS64 : public SlowPathCodeMIPS64 { DISALLOW_COPY_AND_ASSIGN(DeoptimizationSlowPathMIPS64); }; +class ArraySetSlowPathMIPS64 : public SlowPathCodeMIPS64 { + public: + explicit ArraySetSlowPathMIPS64(HInstruction* instruction) : SlowPathCodeMIPS64(instruction) {} + + void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { + LocationSummary* locations = instruction_->GetLocations(); + __ Bind(GetEntryLabel()); + SaveLiveRegisters(codegen, locations); + + InvokeRuntimeCallingConvention calling_convention; + HParallelMove parallel_move(codegen->GetGraph()->GetArena()); + parallel_move.AddMove( + locations->InAt(0), + Location::RegisterLocation(calling_convention.GetRegisterAt(0)), + Primitive::kPrimNot, + nullptr); + parallel_move.AddMove( + locations->InAt(1), + Location::RegisterLocation(calling_convention.GetRegisterAt(1)), + Primitive::kPrimInt, + nullptr); + parallel_move.AddMove( + locations->InAt(2), + Location::RegisterLocation(calling_convention.GetRegisterAt(2)), + Primitive::kPrimNot, + nullptr); + codegen->GetMoveResolver()->EmitNativeCode(¶llel_move); + + CodeGeneratorMIPS64* mips64_codegen = down_cast<CodeGeneratorMIPS64*>(codegen); + mips64_codegen->InvokeRuntime(kQuickAputObject, instruction_, instruction_->GetDexPc(), this); + CheckEntrypointTypes<kQuickAputObject, void, mirror::Array*, int32_t, mirror::Object*>(); + RestoreLiveRegisters(codegen, locations); + __ Bc(GetExitLabel()); + } + + const char* GetDescription() const OVERRIDE { return "ArraySetSlowPathMIPS64"; } + + private: + DISALLOW_COPY_AND_ASSIGN(ArraySetSlowPathMIPS64); +}; + +// Slow path marking an object reference `ref` during a read +// barrier. The field `obj.field` in the object `obj` holding this +// reference does not get updated by this slow path after marking (see +// ReadBarrierMarkAndUpdateFieldSlowPathMIPS64 below for that). +// +// This means that after the execution of this slow path, `ref` will +// always be up-to-date, but `obj.field` may not; i.e., after the +// flip, `ref` will be a to-space reference, but `obj.field` will +// probably still be a from-space reference (unless it gets updated by +// another thread, or if another thread installed another object +// reference (different from `ref`) in `obj.field`). +// +// If `entrypoint` is a valid location it is assumed to already be +// holding the entrypoint. The case where the entrypoint is passed in +// is for the GcRoot read barrier. +class ReadBarrierMarkSlowPathMIPS64 : public SlowPathCodeMIPS64 { + public: + ReadBarrierMarkSlowPathMIPS64(HInstruction* instruction, + Location ref, + Location entrypoint = Location::NoLocation()) + : SlowPathCodeMIPS64(instruction), ref_(ref), entrypoint_(entrypoint) { + DCHECK(kEmitCompilerReadBarrier); + } + + const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkSlowPathMIPS"; } + + void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { + LocationSummary* locations = instruction_->GetLocations(); + GpuRegister ref_reg = ref_.AsRegister<GpuRegister>(); + DCHECK(locations->CanCall()); + DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg)) << ref_reg; + DCHECK(instruction_->IsInstanceFieldGet() || + instruction_->IsStaticFieldGet() || + instruction_->IsArrayGet() || + instruction_->IsArraySet() || + instruction_->IsLoadClass() || + instruction_->IsLoadString() || + instruction_->IsInstanceOf() || + instruction_->IsCheckCast() || + (instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified()) || + (instruction_->IsInvokeStaticOrDirect() && instruction_->GetLocations()->Intrinsified())) + << "Unexpected instruction in read barrier marking slow path: " + << instruction_->DebugName(); + + __ Bind(GetEntryLabel()); + // No need to save live registers; it's taken care of by the + // entrypoint. Also, there is no need to update the stack mask, + // as this runtime call will not trigger a garbage collection. + CodeGeneratorMIPS64* mips64_codegen = down_cast<CodeGeneratorMIPS64*>(codegen); + DCHECK((V0 <= ref_reg && ref_reg <= T2) || + (S2 <= ref_reg && ref_reg <= S7) || + (ref_reg == S8)) << ref_reg; + // "Compact" slow path, saving two moves. + // + // Instead of using the standard runtime calling convention (input + // and output in A0 and V0 respectively): + // + // A0 <- ref + // V0 <- ReadBarrierMark(A0) + // ref <- V0 + // + // we just use rX (the register containing `ref`) as input and output + // of a dedicated entrypoint: + // + // rX <- ReadBarrierMarkRegX(rX) + // + if (entrypoint_.IsValid()) { + mips64_codegen->ValidateInvokeRuntimeWithoutRecordingPcInfo(instruction_, this); + DCHECK_EQ(entrypoint_.AsRegister<GpuRegister>(), T9); + __ Jalr(entrypoint_.AsRegister<GpuRegister>()); + __ Nop(); + } else { + int32_t entry_point_offset = + CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kMips64PointerSize>(ref_reg - 1); + // This runtime call does not require a stack map. + mips64_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, + instruction_, + this); + } + __ Bc(GetExitLabel()); + } + + private: + // The location (register) of the marked object reference. + const Location ref_; + + // The location of the entrypoint if already loaded. + const Location entrypoint_; + + DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkSlowPathMIPS64); +}; + +// Slow path marking an object reference `ref` during a read barrier, +// and if needed, atomically updating the field `obj.field` in the +// object `obj` holding this reference after marking (contrary to +// ReadBarrierMarkSlowPathMIPS64 above, which never tries to update +// `obj.field`). +// +// This means that after the execution of this slow path, both `ref` +// and `obj.field` will be up-to-date; i.e., after the flip, both will +// hold the same to-space reference (unless another thread installed +// another object reference (different from `ref`) in `obj.field`). +class ReadBarrierMarkAndUpdateFieldSlowPathMIPS64 : public SlowPathCodeMIPS64 { + public: + ReadBarrierMarkAndUpdateFieldSlowPathMIPS64(HInstruction* instruction, + Location ref, + GpuRegister obj, + Location field_offset, + GpuRegister temp1) + : SlowPathCodeMIPS64(instruction), + ref_(ref), + obj_(obj), + field_offset_(field_offset), + temp1_(temp1) { + DCHECK(kEmitCompilerReadBarrier); + } + + const char* GetDescription() const OVERRIDE { + return "ReadBarrierMarkAndUpdateFieldSlowPathMIPS64"; + } + + void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { + LocationSummary* locations = instruction_->GetLocations(); + GpuRegister ref_reg = ref_.AsRegister<GpuRegister>(); + DCHECK(locations->CanCall()); + DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg)) << ref_reg; + // This slow path is only used by the UnsafeCASObject intrinsic. + DCHECK((instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified())) + << "Unexpected instruction in read barrier marking and field updating slow path: " + << instruction_->DebugName(); + DCHECK(instruction_->GetLocations()->Intrinsified()); + DCHECK_EQ(instruction_->AsInvoke()->GetIntrinsic(), Intrinsics::kUnsafeCASObject); + DCHECK(field_offset_.IsRegister()) << field_offset_; + + __ Bind(GetEntryLabel()); + + // Save the old reference. + // Note that we cannot use AT or TMP to save the old reference, as those + // are used by the code that follows, but we need the old reference after + // the call to the ReadBarrierMarkRegX entry point. + DCHECK_NE(temp1_, AT); + DCHECK_NE(temp1_, TMP); + __ Move(temp1_, ref_reg); + + // No need to save live registers; it's taken care of by the + // entrypoint. Also, there is no need to update the stack mask, + // as this runtime call will not trigger a garbage collection. + CodeGeneratorMIPS64* mips64_codegen = down_cast<CodeGeneratorMIPS64*>(codegen); + DCHECK((V0 <= ref_reg && ref_reg <= T2) || + (S2 <= ref_reg && ref_reg <= S7) || + (ref_reg == S8)) << ref_reg; + // "Compact" slow path, saving two moves. + // + // Instead of using the standard runtime calling convention (input + // and output in A0 and V0 respectively): + // + // A0 <- ref + // V0 <- ReadBarrierMark(A0) + // ref <- V0 + // + // we just use rX (the register containing `ref`) as input and output + // of a dedicated entrypoint: + // + // rX <- ReadBarrierMarkRegX(rX) + // + int32_t entry_point_offset = + CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kMips64PointerSize>(ref_reg - 1); + // This runtime call does not require a stack map. + mips64_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, + instruction_, + this); + + // If the new reference is different from the old reference, + // update the field in the holder (`*(obj_ + field_offset_)`). + // + // Note that this field could also hold a different object, if + // another thread had concurrently changed it. In that case, the + // the compare-and-set (CAS) loop below would abort, leaving the + // field as-is. + Mips64Label done; + __ Beqc(temp1_, ref_reg, &done); + + // Update the the holder's field atomically. This may fail if + // mutator updates before us, but it's OK. This is achieved + // using a strong compare-and-set (CAS) operation with relaxed + // memory synchronization ordering, where the expected value is + // the old reference and the desired value is the new reference. + + // Convenience aliases. + GpuRegister base = obj_; + GpuRegister offset = field_offset_.AsRegister<GpuRegister>(); + GpuRegister expected = temp1_; + GpuRegister value = ref_reg; + GpuRegister tmp_ptr = TMP; // Pointer to actual memory. + GpuRegister tmp = AT; // Value in memory. + + __ Daddu(tmp_ptr, base, offset); + + if (kPoisonHeapReferences) { + __ PoisonHeapReference(expected); + // Do not poison `value` if it is the same register as + // `expected`, which has just been poisoned. + if (value != expected) { + __ PoisonHeapReference(value); + } + } + + // do { + // tmp = [r_ptr] - expected; + // } while (tmp == 0 && failure([r_ptr] <- r_new_value)); + + Mips64Label loop_head, exit_loop; + __ Bind(&loop_head); + __ Ll(tmp, tmp_ptr); + // The LL instruction sign-extends the 32-bit value, but + // 32-bit references must be zero-extended. Zero-extend `tmp`. + __ Dext(tmp, tmp, 0, 32); + __ Bnec(tmp, expected, &exit_loop); + __ Move(tmp, value); + __ Sc(tmp, tmp_ptr); + __ Beqzc(tmp, &loop_head); + __ Bind(&exit_loop); + + if (kPoisonHeapReferences) { + __ UnpoisonHeapReference(expected); + // Do not unpoison `value` if it is the same register as + // `expected`, which has just been unpoisoned. + if (value != expected) { + __ UnpoisonHeapReference(value); + } + } + + __ Bind(&done); + __ Bc(GetExitLabel()); + } + + private: + // The location (register) of the marked object reference. + const Location ref_; + // The register containing the object holding the marked object reference field. + const GpuRegister obj_; + // The location of the offset of the marked reference field within `obj_`. + Location field_offset_; + + const GpuRegister temp1_; + + DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkAndUpdateFieldSlowPathMIPS64); +}; + +// Slow path generating a read barrier for a heap reference. +class ReadBarrierForHeapReferenceSlowPathMIPS64 : public SlowPathCodeMIPS64 { + public: + ReadBarrierForHeapReferenceSlowPathMIPS64(HInstruction* instruction, + Location out, + Location ref, + Location obj, + uint32_t offset, + Location index) + : SlowPathCodeMIPS64(instruction), + out_(out), + ref_(ref), + obj_(obj), + offset_(offset), + index_(index) { + DCHECK(kEmitCompilerReadBarrier); + // If `obj` is equal to `out` or `ref`, it means the initial object + // has been overwritten by (or after) the heap object reference load + // to be instrumented, e.g.: + // + // __ LoadFromOffset(kLoadWord, out, out, offset); + // codegen_->GenerateReadBarrierSlow(instruction, out_loc, out_loc, out_loc, offset); + // + // In that case, we have lost the information about the original + // object, and the emitted read barrier cannot work properly. + DCHECK(!obj.Equals(out)) << "obj=" << obj << " out=" << out; + DCHECK(!obj.Equals(ref)) << "obj=" << obj << " ref=" << ref; + } + + void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { + CodeGeneratorMIPS64* mips64_codegen = down_cast<CodeGeneratorMIPS64*>(codegen); + LocationSummary* locations = instruction_->GetLocations(); + Primitive::Type type = Primitive::kPrimNot; + GpuRegister reg_out = out_.AsRegister<GpuRegister>(); + DCHECK(locations->CanCall()); + DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out)); + DCHECK(instruction_->IsInstanceFieldGet() || + instruction_->IsStaticFieldGet() || + instruction_->IsArrayGet() || + instruction_->IsInstanceOf() || + instruction_->IsCheckCast() || + (instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified())) + << "Unexpected instruction in read barrier for heap reference slow path: " + << instruction_->DebugName(); + + __ Bind(GetEntryLabel()); + SaveLiveRegisters(codegen, locations); + + // We may have to change the index's value, but as `index_` is a + // constant member (like other "inputs" of this slow path), + // introduce a copy of it, `index`. + Location index = index_; + if (index_.IsValid()) { + // Handle `index_` for HArrayGet and UnsafeGetObject/UnsafeGetObjectVolatile intrinsics. + if (instruction_->IsArrayGet()) { + // Compute the actual memory offset and store it in `index`. + GpuRegister index_reg = index_.AsRegister<GpuRegister>(); + DCHECK(locations->GetLiveRegisters()->ContainsCoreRegister(index_reg)); + if (codegen->IsCoreCalleeSaveRegister(index_reg)) { + // We are about to change the value of `index_reg` (see the + // calls to art::mips64::Mips64Assembler::Sll and + // art::mips64::MipsAssembler::Addiu32 below), but it has + // not been saved by the previous call to + // art::SlowPathCode::SaveLiveRegisters, as it is a + // callee-save register -- + // art::SlowPathCode::SaveLiveRegisters does not consider + // callee-save registers, as it has been designed with the + // assumption that callee-save registers are supposed to be + // handled by the called function. So, as a callee-save + // register, `index_reg` _would_ eventually be saved onto + // the stack, but it would be too late: we would have + // changed its value earlier. Therefore, we manually save + // it here into another freely available register, + // `free_reg`, chosen of course among the caller-save + // registers (as a callee-save `free_reg` register would + // exhibit the same problem). + // + // Note we could have requested a temporary register from + // the register allocator instead; but we prefer not to, as + // this is a slow path, and we know we can find a + // caller-save register that is available. + GpuRegister free_reg = FindAvailableCallerSaveRegister(codegen); + __ Move(free_reg, index_reg); + index_reg = free_reg; + index = Location::RegisterLocation(index_reg); + } else { + // The initial register stored in `index_` has already been + // saved in the call to art::SlowPathCode::SaveLiveRegisters + // (as it is not a callee-save register), so we can freely + // use it. + } + // Shifting the index value contained in `index_reg` by the scale + // factor (2) cannot overflow in practice, as the runtime is + // unable to allocate object arrays with a size larger than + // 2^26 - 1 (that is, 2^28 - 4 bytes). + __ Sll(index_reg, index_reg, TIMES_4); + static_assert( + sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), + "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); + __ Addiu32(index_reg, index_reg, offset_); + } else { + // In the case of the UnsafeGetObject/UnsafeGetObjectVolatile + // intrinsics, `index_` is not shifted by a scale factor of 2 + // (as in the case of ArrayGet), as it is actually an offset + // to an object field within an object. + DCHECK(instruction_->IsInvoke()) << instruction_->DebugName(); + DCHECK(instruction_->GetLocations()->Intrinsified()); + DCHECK((instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObject) || + (instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile)) + << instruction_->AsInvoke()->GetIntrinsic(); + DCHECK_EQ(offset_, 0U); + DCHECK(index_.IsRegister()); + } + } + + // We're moving two or three locations to locations that could + // overlap, so we need a parallel move resolver. + InvokeRuntimeCallingConvention calling_convention; + HParallelMove parallel_move(codegen->GetGraph()->GetArena()); + parallel_move.AddMove(ref_, + Location::RegisterLocation(calling_convention.GetRegisterAt(0)), + Primitive::kPrimNot, + nullptr); + parallel_move.AddMove(obj_, + Location::RegisterLocation(calling_convention.GetRegisterAt(1)), + Primitive::kPrimNot, + nullptr); + if (index.IsValid()) { + parallel_move.AddMove(index, + Location::RegisterLocation(calling_convention.GetRegisterAt(2)), + Primitive::kPrimInt, + nullptr); + codegen->GetMoveResolver()->EmitNativeCode(¶llel_move); + } else { + codegen->GetMoveResolver()->EmitNativeCode(¶llel_move); + __ LoadConst32(calling_convention.GetRegisterAt(2), offset_); + } + mips64_codegen->InvokeRuntime(kQuickReadBarrierSlow, + instruction_, + instruction_->GetDexPc(), + this); + CheckEntrypointTypes< + kQuickReadBarrierSlow, mirror::Object*, mirror::Object*, mirror::Object*, uint32_t>(); + mips64_codegen->MoveLocation(out_, calling_convention.GetReturnLocation(type), type); + + RestoreLiveRegisters(codegen, locations); + __ Bc(GetExitLabel()); + } + + const char* GetDescription() const OVERRIDE { + return "ReadBarrierForHeapReferenceSlowPathMIPS64"; + } + + private: + GpuRegister FindAvailableCallerSaveRegister(CodeGenerator* codegen) { + size_t ref = static_cast<int>(ref_.AsRegister<GpuRegister>()); + size_t obj = static_cast<int>(obj_.AsRegister<GpuRegister>()); + for (size_t i = 0, e = codegen->GetNumberOfCoreRegisters(); i < e; ++i) { + if (i != ref && + i != obj && + !codegen->IsCoreCalleeSaveRegister(i) && + !codegen->IsBlockedCoreRegister(i)) { + return static_cast<GpuRegister>(i); + } + } + // We shall never fail to find a free caller-save register, as + // there are more than two core caller-save registers on MIPS64 + // (meaning it is possible to find one which is different from + // `ref` and `obj`). + DCHECK_GT(codegen->GetNumberOfCoreCallerSaveRegisters(), 2u); + LOG(FATAL) << "Could not find a free caller-save register"; + UNREACHABLE(); + } + + const Location out_; + const Location ref_; + const Location obj_; + const uint32_t offset_; + // An additional location containing an index to an array. + // Only used for HArrayGet and the UnsafeGetObject & + // UnsafeGetObjectVolatile intrinsics. + const Location index_; + + DISALLOW_COPY_AND_ASSIGN(ReadBarrierForHeapReferenceSlowPathMIPS64); +}; + +// Slow path generating a read barrier for a GC root. +class ReadBarrierForRootSlowPathMIPS64 : public SlowPathCodeMIPS64 { + public: + ReadBarrierForRootSlowPathMIPS64(HInstruction* instruction, Location out, Location root) + : SlowPathCodeMIPS64(instruction), out_(out), root_(root) { + DCHECK(kEmitCompilerReadBarrier); + } + + void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { + LocationSummary* locations = instruction_->GetLocations(); + Primitive::Type type = Primitive::kPrimNot; + GpuRegister reg_out = out_.AsRegister<GpuRegister>(); + DCHECK(locations->CanCall()); + DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out)); + DCHECK(instruction_->IsLoadClass() || instruction_->IsLoadString()) + << "Unexpected instruction in read barrier for GC root slow path: " + << instruction_->DebugName(); + + __ Bind(GetEntryLabel()); + SaveLiveRegisters(codegen, locations); + + InvokeRuntimeCallingConvention calling_convention; + CodeGeneratorMIPS64* mips64_codegen = down_cast<CodeGeneratorMIPS64*>(codegen); + mips64_codegen->MoveLocation(Location::RegisterLocation(calling_convention.GetRegisterAt(0)), + root_, + Primitive::kPrimNot); + mips64_codegen->InvokeRuntime(kQuickReadBarrierForRootSlow, + instruction_, + instruction_->GetDexPc(), + this); + CheckEntrypointTypes<kQuickReadBarrierForRootSlow, mirror::Object*, GcRoot<mirror::Object>*>(); + mips64_codegen->MoveLocation(out_, calling_convention.GetReturnLocation(type), type); + + RestoreLiveRegisters(codegen, locations); + __ Bc(GetExitLabel()); + } + + const char* GetDescription() const OVERRIDE { return "ReadBarrierForRootSlowPathMIPS64"; } + + private: + const Location out_; + const Location root_; + + DISALLOW_COPY_AND_ASSIGN(ReadBarrierForRootSlowPathMIPS64); +}; + CodeGeneratorMIPS64::CodeGeneratorMIPS64(HGraph* graph, const Mips64InstructionSetFeatures& isa_features, const CompilerOptions& compiler_options, @@ -1140,23 +1662,32 @@ void CodeGeneratorMIPS64::InvokeRuntime(QuickEntrypointEnum entrypoint, uint32_t dex_pc, SlowPathCode* slow_path) { ValidateInvokeRuntime(entrypoint, instruction, slow_path); - __ LoadFromOffset(kLoadDoubleword, - T9, - TR, - GetThreadOffset<kMips64PointerSize>(entrypoint).Int32Value()); - __ Jalr(T9); - __ Nop(); + GenerateInvokeRuntime(GetThreadOffset<kMips64PointerSize>(entrypoint).Int32Value()); if (EntrypointRequiresStackMap(entrypoint)) { RecordPcInfo(instruction, dex_pc, slow_path); } } +void CodeGeneratorMIPS64::InvokeRuntimeWithoutRecordingPcInfo(int32_t entry_point_offset, + HInstruction* instruction, + SlowPathCode* slow_path) { + ValidateInvokeRuntimeWithoutRecordingPcInfo(instruction, slow_path); + GenerateInvokeRuntime(entry_point_offset); +} + +void CodeGeneratorMIPS64::GenerateInvokeRuntime(int32_t entry_point_offset) { + __ LoadFromOffset(kLoadDoubleword, T9, TR, entry_point_offset); + __ Jalr(T9); + __ Nop(); +} + void InstructionCodeGeneratorMIPS64::GenerateClassInitializationCheck(SlowPathCodeMIPS64* slow_path, GpuRegister class_reg) { __ LoadFromOffset(kLoadWord, TMP, class_reg, mirror::Class::StatusOffset().Int32Value()); __ LoadConst32(AT, mirror::Class::kStatusInitialized); __ Bltc(TMP, AT, slow_path->GetEntryLabel()); - // TODO: barrier needed? + // Even if the initialized flag is set, we need to ensure consistent memory ordering. + __ Sync(0); __ Bind(slow_path->GetExitLabel()); } @@ -1447,14 +1978,31 @@ void InstructionCodeGeneratorMIPS64::VisitAnd(HAnd* instruction) { } void LocationsBuilderMIPS64::VisitArrayGet(HArrayGet* instruction) { + Primitive::Type type = instruction->GetType(); + bool object_array_get_with_read_barrier = + kEmitCompilerReadBarrier && (type == Primitive::kPrimNot); LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); + new (GetGraph()->GetArena()) LocationSummary(instruction, + object_array_get_with_read_barrier + ? LocationSummary::kCallOnSlowPath + : LocationSummary::kNoCall); locations->SetInAt(0, Location::RequiresRegister()); locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); - if (Primitive::IsFloatingPointType(instruction->GetType())) { + if (Primitive::IsFloatingPointType(type)) { locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); } else { - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); + // The output overlaps in the case of an object array get with + // read barriers enabled: we do not want the move to overwrite the + // array's location, as we need it to emit the read barrier. + locations->SetOut(Location::RequiresRegister(), + object_array_get_with_read_barrier + ? Location::kOutputOverlap + : Location::kNoOutputOverlap); + } + // We need a temporary register for the read barrier marking slow + // path in CodeGeneratorMIPS64::GenerateArrayLoadWithBakerReadBarrier. + if (object_array_get_with_read_barrier && kUseBakerReadBarrier) { + locations->AddTemp(Location::RequiresRegister()); } } @@ -1467,7 +2015,9 @@ static auto GetImplicitNullChecker(HInstruction* instruction, CodeGeneratorMIPS6 void InstructionCodeGeneratorMIPS64::VisitArrayGet(HArrayGet* instruction) { LocationSummary* locations = instruction->GetLocations(); - GpuRegister obj = locations->InAt(0).AsRegister<GpuRegister>(); + Location obj_loc = locations->InAt(0); + GpuRegister obj = obj_loc.AsRegister<GpuRegister>(); + Location out_loc = locations->Out(); Location index = locations->InAt(1); uint32_t data_offset = CodeGenerator::GetArrayDataOffset(instruction); auto null_checker = GetImplicitNullChecker(instruction, codegen_); @@ -1477,7 +2027,7 @@ void InstructionCodeGeneratorMIPS64::VisitArrayGet(HArrayGet* instruction) { instruction->IsStringCharAt(); switch (type) { case Primitive::kPrimBoolean: { - GpuRegister out = locations->Out().AsRegister<GpuRegister>(); + GpuRegister out = out_loc.AsRegister<GpuRegister>(); if (index.IsConstant()) { size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset; @@ -1490,7 +2040,7 @@ void InstructionCodeGeneratorMIPS64::VisitArrayGet(HArrayGet* instruction) { } case Primitive::kPrimByte: { - GpuRegister out = locations->Out().AsRegister<GpuRegister>(); + GpuRegister out = out_loc.AsRegister<GpuRegister>(); if (index.IsConstant()) { size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset; @@ -1503,7 +2053,7 @@ void InstructionCodeGeneratorMIPS64::VisitArrayGet(HArrayGet* instruction) { } case Primitive::kPrimShort: { - GpuRegister out = locations->Out().AsRegister<GpuRegister>(); + GpuRegister out = out_loc.AsRegister<GpuRegister>(); if (index.IsConstant()) { size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset; @@ -1517,7 +2067,7 @@ void InstructionCodeGeneratorMIPS64::VisitArrayGet(HArrayGet* instruction) { } case Primitive::kPrimChar: { - GpuRegister out = locations->Out().AsRegister<GpuRegister>(); + GpuRegister out = out_loc.AsRegister<GpuRegister>(); if (maybe_compressed_char_at) { uint32_t count_offset = mirror::String::CountOffset().Uint32Value(); __ LoadFromOffset(kLoadWord, TMP, obj, count_offset, null_checker); @@ -1570,10 +2120,9 @@ void InstructionCodeGeneratorMIPS64::VisitArrayGet(HArrayGet* instruction) { break; } - case Primitive::kPrimInt: - case Primitive::kPrimNot: { + case Primitive::kPrimInt: { DCHECK_EQ(sizeof(mirror::HeapReference<mirror::Object>), sizeof(int32_t)); - GpuRegister out = locations->Out().AsRegister<GpuRegister>(); + GpuRegister out = out_loc.AsRegister<GpuRegister>(); LoadOperandType load_type = (type == Primitive::kPrimNot) ? kLoadUnsignedWord : kLoadWord; if (index.IsConstant()) { size_t offset = @@ -1587,8 +2136,53 @@ void InstructionCodeGeneratorMIPS64::VisitArrayGet(HArrayGet* instruction) { break; } + case Primitive::kPrimNot: { + static_assert( + sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), + "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); + // /* HeapReference<Object> */ out = + // *(obj + data_offset + index * sizeof(HeapReference<Object>)) + if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { + Location temp = locations->GetTemp(0); + // Note that a potential implicit null check is handled in this + // CodeGeneratorMIPS64::GenerateArrayLoadWithBakerReadBarrier call. + codegen_->GenerateArrayLoadWithBakerReadBarrier(instruction, + out_loc, + obj, + data_offset, + index, + temp, + /* needs_null_check */ true); + } else { + GpuRegister out = out_loc.AsRegister<GpuRegister>(); + if (index.IsConstant()) { + size_t offset = + (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; + __ LoadFromOffset(kLoadUnsignedWord, out, obj, offset, null_checker); + // If read barriers are enabled, emit read barriers other than + // Baker's using a slow path (and also unpoison the loaded + // reference, if heap poisoning is enabled). + codegen_->MaybeGenerateReadBarrierSlow(instruction, out_loc, out_loc, obj_loc, offset); + } else { + __ Sll(TMP, index.AsRegister<GpuRegister>(), TIMES_4); + __ Addu(TMP, obj, TMP); + __ LoadFromOffset(kLoadUnsignedWord, out, TMP, data_offset, null_checker); + // If read barriers are enabled, emit read barriers other than + // Baker's using a slow path (and also unpoison the loaded + // reference, if heap poisoning is enabled). + codegen_->MaybeGenerateReadBarrierSlow(instruction, + out_loc, + out_loc, + obj_loc, + data_offset, + index); + } + } + break; + } + case Primitive::kPrimLong: { - GpuRegister out = locations->Out().AsRegister<GpuRegister>(); + GpuRegister out = out_loc.AsRegister<GpuRegister>(); if (index.IsConstant()) { size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; @@ -1602,7 +2196,7 @@ void InstructionCodeGeneratorMIPS64::VisitArrayGet(HArrayGet* instruction) { } case Primitive::kPrimFloat: { - FpuRegister out = locations->Out().AsFpuRegister<FpuRegister>(); + FpuRegister out = out_loc.AsFpuRegister<FpuRegister>(); if (index.IsConstant()) { size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; @@ -1616,7 +2210,7 @@ void InstructionCodeGeneratorMIPS64::VisitArrayGet(HArrayGet* instruction) { } case Primitive::kPrimDouble: { - FpuRegister out = locations->Out().AsFpuRegister<FpuRegister>(); + FpuRegister out = out_loc.AsFpuRegister<FpuRegister>(); if (index.IsConstant()) { size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; @@ -1633,11 +2227,6 @@ void InstructionCodeGeneratorMIPS64::VisitArrayGet(HArrayGet* instruction) { LOG(FATAL) << "Unreachable type " << instruction->GetType(); UNREACHABLE(); } - - if (type == Primitive::kPrimNot) { - GpuRegister out = locations->Out().AsRegister<GpuRegister>(); - __ MaybeUnpoisonHeapReference(out); - } } void LocationsBuilderMIPS64::VisitArrayLength(HArrayLength* instruction) { @@ -1679,23 +2268,28 @@ Location LocationsBuilderMIPS64::FpuRegisterOrConstantForStore(HInstruction* ins } void LocationsBuilderMIPS64::VisitArraySet(HArraySet* instruction) { - bool needs_runtime_call = instruction->NeedsTypeCheck(); + Primitive::Type value_type = instruction->GetComponentType(); + + bool needs_write_barrier = + CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue()); + bool may_need_runtime_call_for_type_check = instruction->NeedsTypeCheck(); + LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary( instruction, - needs_runtime_call ? LocationSummary::kCallOnMainOnly : LocationSummary::kNoCall); - if (needs_runtime_call) { - InvokeRuntimeCallingConvention calling_convention; - locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); - locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); - locations->SetInAt(2, Location::RegisterLocation(calling_convention.GetRegisterAt(2))); + may_need_runtime_call_for_type_check ? + LocationSummary::kCallOnSlowPath : + LocationSummary::kNoCall); + + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); + if (Primitive::IsFloatingPointType(instruction->InputAt(2)->GetType())) { + locations->SetInAt(2, FpuRegisterOrConstantForStore(instruction->InputAt(2))); } else { - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); - if (Primitive::IsFloatingPointType(instruction->InputAt(2)->GetType())) { - locations->SetInAt(2, FpuRegisterOrConstantForStore(instruction->InputAt(2))); - } else { - locations->SetInAt(2, RegisterOrZeroConstant(instruction->InputAt(2))); - } + locations->SetInAt(2, RegisterOrZeroConstant(instruction->InputAt(2))); + } + if (needs_write_barrier) { + // Temporary register for the write barrier. + locations->AddTemp(Location::RequiresRegister()); // Possibly used for ref. poisoning too. } } @@ -1705,7 +2299,7 @@ void InstructionCodeGeneratorMIPS64::VisitArraySet(HArraySet* instruction) { Location index = locations->InAt(1); Location value_location = locations->InAt(2); Primitive::Type value_type = instruction->GetComponentType(); - bool needs_runtime_call = locations->WillCall(); + bool may_need_runtime_call_for_type_check = instruction->NeedsTypeCheck(); bool needs_write_barrier = CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue()); auto null_checker = GetImplicitNullChecker(instruction, codegen_); @@ -1749,68 +2343,138 @@ void InstructionCodeGeneratorMIPS64::VisitArraySet(HArraySet* instruction) { break; } - case Primitive::kPrimInt: + case Primitive::kPrimInt: { + uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value(); + if (index.IsConstant()) { + data_offset += index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4; + } else { + __ Dsll(base_reg, index.AsRegister<GpuRegister>(), TIMES_4); + __ Daddu(base_reg, obj, base_reg); + } + if (value_location.IsConstant()) { + int32_t value = CodeGenerator::GetInt32ValueOf(value_location.GetConstant()); + __ StoreConstToOffset(kStoreWord, value, base_reg, data_offset, TMP, null_checker); + } else { + GpuRegister value = value_location.AsRegister<GpuRegister>(); + __ StoreToOffset(kStoreWord, value, base_reg, data_offset, null_checker); + } + break; + } + case Primitive::kPrimNot: { - if (!needs_runtime_call) { + if (value_location.IsConstant()) { + // Just setting null. uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value(); if (index.IsConstant()) { data_offset += index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4; } else { - DCHECK(index.IsRegister()) << index; __ Dsll(base_reg, index.AsRegister<GpuRegister>(), TIMES_4); __ Daddu(base_reg, obj, base_reg); } - if (value_location.IsConstant()) { - int32_t value = CodeGenerator::GetInt32ValueOf(value_location.GetConstant()); - __ StoreConstToOffset(kStoreWord, value, base_reg, data_offset, TMP, null_checker); - DCHECK(!needs_write_barrier); - } else { - GpuRegister value = value_location.AsRegister<GpuRegister>(); - if (kPoisonHeapReferences && needs_write_barrier) { - // Note that in the case where `value` is a null reference, - // we do not enter this block, as a null reference does not - // need poisoning. - DCHECK_EQ(value_type, Primitive::kPrimNot); - // Use Sw() instead of StoreToOffset() in order to be able to - // hold the poisoned reference in AT and thus avoid allocating - // yet another temporary register. - if (index.IsConstant()) { - if (!IsInt<16>(static_cast<int32_t>(data_offset))) { - int16_t low16 = Low16Bits(data_offset); - // For consistency with StoreToOffset() and such treat data_offset as int32_t. - uint64_t high48 = static_cast<uint64_t>(static_cast<int32_t>(data_offset)) - low16; - int16_t upper16 = High16Bits(high48); - // Allow the full [-2GB,+2GB) range in case `low16` is negative and needs a - // compensatory 64KB added, which may push `high48` above 2GB and require - // the dahi instruction. - int16_t higher16 = High32Bits(high48) + ((upper16 < 0) ? 1 : 0); - __ Daui(TMP, obj, upper16); - if (higher16 != 0) { - __ Dahi(TMP, higher16); - } - base_reg = TMP; - data_offset = low16; - } - } else { - DCHECK(IsInt<16>(static_cast<int32_t>(data_offset))); - } - __ PoisonHeapReference(AT, value); - __ Sw(AT, base_reg, data_offset); - null_checker(); + int32_t value = CodeGenerator::GetInt32ValueOf(value_location.GetConstant()); + DCHECK_EQ(value, 0); + __ StoreConstToOffset(kStoreWord, value, base_reg, data_offset, TMP, null_checker); + DCHECK(!needs_write_barrier); + DCHECK(!may_need_runtime_call_for_type_check); + break; + } + + DCHECK(needs_write_barrier); + GpuRegister value = value_location.AsRegister<GpuRegister>(); + GpuRegister temp1 = locations->GetTemp(0).AsRegister<GpuRegister>(); + GpuRegister temp2 = TMP; // Doesn't need to survive slow path. + uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); + uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); + uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); + Mips64Label done; + SlowPathCodeMIPS64* slow_path = nullptr; + + if (may_need_runtime_call_for_type_check) { + slow_path = new (GetGraph()->GetArena()) ArraySetSlowPathMIPS64(instruction); + codegen_->AddSlowPath(slow_path); + if (instruction->GetValueCanBeNull()) { + Mips64Label non_zero; + __ Bnezc(value, &non_zero); + uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value(); + if (index.IsConstant()) { + data_offset += index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4; } else { - __ StoreToOffset(kStoreWord, value, base_reg, data_offset, null_checker); - } - if (needs_write_barrier) { - DCHECK_EQ(value_type, Primitive::kPrimNot); - codegen_->MarkGCCard(obj, value, instruction->GetValueCanBeNull()); + __ Dsll(base_reg, index.AsRegister<GpuRegister>(), TIMES_4); + __ Daddu(base_reg, obj, base_reg); } + __ StoreToOffset(kStoreWord, value, base_reg, data_offset, null_checker); + __ Bc(&done); + __ Bind(&non_zero); + } + + // Note that when read barriers are enabled, the type checks + // are performed without read barriers. This is fine, even in + // the case where a class object is in the from-space after + // the flip, as a comparison involving such a type would not + // produce a false positive; it may of course produce a false + // negative, in which case we would take the ArraySet slow + // path. + + // /* HeapReference<Class> */ temp1 = obj->klass_ + __ LoadFromOffset(kLoadUnsignedWord, temp1, obj, class_offset, null_checker); + __ MaybeUnpoisonHeapReference(temp1); + + // /* HeapReference<Class> */ temp1 = temp1->component_type_ + __ LoadFromOffset(kLoadUnsignedWord, temp1, temp1, component_offset); + // /* HeapReference<Class> */ temp2 = value->klass_ + __ LoadFromOffset(kLoadUnsignedWord, temp2, value, class_offset); + // If heap poisoning is enabled, no need to unpoison `temp1` + // nor `temp2`, as we are comparing two poisoned references. + + if (instruction->StaticTypeOfArrayIsObjectArray()) { + Mips64Label do_put; + __ Beqc(temp1, temp2, &do_put); + // If heap poisoning is enabled, the `temp1` reference has + // not been unpoisoned yet; unpoison it now. + __ MaybeUnpoisonHeapReference(temp1); + + // /* HeapReference<Class> */ temp1 = temp1->super_class_ + __ LoadFromOffset(kLoadUnsignedWord, temp1, temp1, super_offset); + // If heap poisoning is enabled, no need to unpoison + // `temp1`, as we are comparing against null below. + __ Bnezc(temp1, slow_path->GetEntryLabel()); + __ Bind(&do_put); + } else { + __ Bnec(temp1, temp2, slow_path->GetEntryLabel()); } + } + + GpuRegister source = value; + if (kPoisonHeapReferences) { + // Note that in the case where `value` is a null reference, + // we do not enter this block, as a null reference does not + // need poisoning. + __ Move(temp1, value); + __ PoisonHeapReference(temp1); + source = temp1; + } + + uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value(); + if (index.IsConstant()) { + data_offset += index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4; } else { - DCHECK_EQ(value_type, Primitive::kPrimNot); - // Note: if heap poisoning is enabled, pAputObject takes care - // of poisoning the reference. - codegen_->InvokeRuntime(kQuickAputObject, instruction, instruction->GetDexPc()); - CheckEntrypointTypes<kQuickAputObject, void, mirror::Array*, int32_t, mirror::Object*>(); + __ Dsll(base_reg, index.AsRegister<GpuRegister>(), TIMES_4); + __ Daddu(base_reg, obj, base_reg); + } + __ StoreToOffset(kStoreWord, source, base_reg, data_offset); + + if (!may_need_runtime_call_for_type_check) { + codegen_->MaybeRecordImplicitNullCheck(instruction); + } + + codegen_->MarkGCCard(obj, value, instruction->GetValueCanBeNull()); + + if (done.IsLinked()) { + __ Bind(&done); + } + + if (slow_path != nullptr) { + __ Bind(slow_path->GetExitLabel()); } break; } @@ -1900,6 +2564,23 @@ void InstructionCodeGeneratorMIPS64::VisitBoundsCheck(HBoundsCheck* instruction) __ Bgeuc(index, length, slow_path->GetEntryLabel()); } +// Temp is used for read barrier. +static size_t NumberOfInstanceOfTemps(TypeCheckKind type_check_kind) { + if (kEmitCompilerReadBarrier && + (kUseBakerReadBarrier || + type_check_kind == TypeCheckKind::kAbstractClassCheck || + type_check_kind == TypeCheckKind::kClassHierarchyCheck || + type_check_kind == TypeCheckKind::kArrayObjectCheck)) { + return 1; + } + return 0; +} + +// Extra temp is used for read barrier. +static size_t NumberOfCheckCastTemps(TypeCheckKind type_check_kind) { + return 1 + NumberOfInstanceOfTemps(type_check_kind); +} + void LocationsBuilderMIPS64::VisitCheckCast(HCheckCast* instruction) { LocationSummary::CallKind call_kind = LocationSummary::kNoCall; bool throws_into_catch = instruction->CanThrowIntoCatchBlock(); @@ -1910,7 +2591,7 @@ void LocationsBuilderMIPS64::VisitCheckCast(HCheckCast* instruction) { case TypeCheckKind::kAbstractClassCheck: case TypeCheckKind::kClassHierarchyCheck: case TypeCheckKind::kArrayObjectCheck: - call_kind = throws_into_catch + call_kind = (throws_into_catch || kEmitCompilerReadBarrier) ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall; // In fact, call on a fatal (non-returning) slow path. break; @@ -1924,15 +2605,20 @@ void LocationsBuilderMIPS64::VisitCheckCast(HCheckCast* instruction) { LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind); locations->SetInAt(0, Location::RequiresRegister()); locations->SetInAt(1, Location::RequiresRegister()); - locations->AddTemp(Location::RequiresRegister()); + locations->AddRegisterTemps(NumberOfCheckCastTemps(type_check_kind)); } void InstructionCodeGeneratorMIPS64::VisitCheckCast(HCheckCast* instruction) { TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); LocationSummary* locations = instruction->GetLocations(); - GpuRegister obj = locations->InAt(0).AsRegister<GpuRegister>(); + Location obj_loc = locations->InAt(0); + GpuRegister obj = obj_loc.AsRegister<GpuRegister>(); GpuRegister cls = locations->InAt(1).AsRegister<GpuRegister>(); - GpuRegister temp = locations->GetTemp(0).AsRegister<GpuRegister>(); + Location temp_loc = locations->GetTemp(0); + GpuRegister temp = temp_loc.AsRegister<GpuRegister>(); + const size_t num_temps = NumberOfCheckCastTemps(type_check_kind); + DCHECK_LE(num_temps, 2u); + Location maybe_temp2_loc = (num_temps >= 2) ? locations->GetTemp(1) : Location::NoLocation(); const uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); const uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); const uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); @@ -1969,8 +2655,12 @@ void InstructionCodeGeneratorMIPS64::VisitCheckCast(HCheckCast* instruction) { case TypeCheckKind::kExactCheck: case TypeCheckKind::kArrayCheck: { // /* HeapReference<Class> */ temp = obj->klass_ - __ LoadFromOffset(kLoadUnsignedWord, temp, obj, class_offset); - __ MaybeUnpoisonHeapReference(temp); + GenerateReferenceLoadTwoRegisters(instruction, + temp_loc, + obj_loc, + class_offset, + maybe_temp2_loc, + kWithoutReadBarrier); // Jump to slow path for throwing the exception or doing a // more involved array check. __ Bnec(temp, cls, slow_path->GetEntryLabel()); @@ -1979,15 +2669,22 @@ void InstructionCodeGeneratorMIPS64::VisitCheckCast(HCheckCast* instruction) { case TypeCheckKind::kAbstractClassCheck: { // /* HeapReference<Class> */ temp = obj->klass_ - __ LoadFromOffset(kLoadUnsignedWord, temp, obj, class_offset); - __ MaybeUnpoisonHeapReference(temp); + GenerateReferenceLoadTwoRegisters(instruction, + temp_loc, + obj_loc, + class_offset, + maybe_temp2_loc, + kWithoutReadBarrier); // If the class is abstract, we eagerly fetch the super class of the // object to avoid doing a comparison we know will fail. Mips64Label loop; __ Bind(&loop); // /* HeapReference<Class> */ temp = temp->super_class_ - __ LoadFromOffset(kLoadUnsignedWord, temp, temp, super_offset); - __ MaybeUnpoisonHeapReference(temp); + GenerateReferenceLoadOneRegister(instruction, + temp_loc, + super_offset, + maybe_temp2_loc, + kWithoutReadBarrier); // If the class reference currently in `temp` is null, jump to the slow path to throw the // exception. __ Beqzc(temp, slow_path->GetEntryLabel()); @@ -1998,15 +2695,22 @@ void InstructionCodeGeneratorMIPS64::VisitCheckCast(HCheckCast* instruction) { case TypeCheckKind::kClassHierarchyCheck: { // /* HeapReference<Class> */ temp = obj->klass_ - __ LoadFromOffset(kLoadUnsignedWord, temp, obj, class_offset); - __ MaybeUnpoisonHeapReference(temp); + GenerateReferenceLoadTwoRegisters(instruction, + temp_loc, + obj_loc, + class_offset, + maybe_temp2_loc, + kWithoutReadBarrier); // Walk over the class hierarchy to find a match. Mips64Label loop; __ Bind(&loop); __ Beqc(temp, cls, &done); // /* HeapReference<Class> */ temp = temp->super_class_ - __ LoadFromOffset(kLoadUnsignedWord, temp, temp, super_offset); - __ MaybeUnpoisonHeapReference(temp); + GenerateReferenceLoadOneRegister(instruction, + temp_loc, + super_offset, + maybe_temp2_loc, + kWithoutReadBarrier); // If the class reference currently in `temp` is null, jump to the slow path to throw the // exception. Otherwise, jump to the beginning of the loop. __ Bnezc(temp, &loop); @@ -2016,14 +2720,21 @@ void InstructionCodeGeneratorMIPS64::VisitCheckCast(HCheckCast* instruction) { case TypeCheckKind::kArrayObjectCheck: { // /* HeapReference<Class> */ temp = obj->klass_ - __ LoadFromOffset(kLoadUnsignedWord, temp, obj, class_offset); - __ MaybeUnpoisonHeapReference(temp); + GenerateReferenceLoadTwoRegisters(instruction, + temp_loc, + obj_loc, + class_offset, + maybe_temp2_loc, + kWithoutReadBarrier); // Do an exact check. __ Beqc(temp, cls, &done); // Otherwise, we need to check that the object's class is a non-primitive array. // /* HeapReference<Class> */ temp = temp->component_type_ - __ LoadFromOffset(kLoadUnsignedWord, temp, temp, component_offset); - __ MaybeUnpoisonHeapReference(temp); + GenerateReferenceLoadOneRegister(instruction, + temp_loc, + component_offset, + maybe_temp2_loc, + kWithoutReadBarrier); // If the component type is null, jump to the slow path to throw the exception. __ Beqzc(temp, slow_path->GetEntryLabel()); // Otherwise, the object is indeed an array, further check that this component @@ -2050,11 +2761,19 @@ void InstructionCodeGeneratorMIPS64::VisitCheckCast(HCheckCast* instruction) { // Avoid read barriers to improve performance of the fast path. We can not get false // positives by doing this. // /* HeapReference<Class> */ temp = obj->klass_ - __ LoadFromOffset(kLoadUnsignedWord, temp, obj, class_offset); - __ MaybeUnpoisonHeapReference(temp); + GenerateReferenceLoadTwoRegisters(instruction, + temp_loc, + obj_loc, + class_offset, + maybe_temp2_loc, + kWithoutReadBarrier); // /* HeapReference<Class> */ temp = temp->iftable_ - __ LoadFromOffset(kLoadUnsignedWord, temp, temp, iftable_offset); - __ MaybeUnpoisonHeapReference(temp); + GenerateReferenceLoadTwoRegisters(instruction, + temp_loc, + temp_loc, + iftable_offset, + maybe_temp2_loc, + kWithoutReadBarrier); // Iftable is never null. __ Lw(TMP, temp, array_length_offset); // Loop through the iftable and check if any class matches. @@ -3270,14 +3989,31 @@ void CodeGeneratorMIPS64::GenerateNop() { } void LocationsBuilderMIPS64::HandleFieldGet(HInstruction* instruction, - const FieldInfo& field_info ATTRIBUTE_UNUSED) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); + const FieldInfo& field_info) { + Primitive::Type field_type = field_info.GetFieldType(); + bool object_field_get_with_read_barrier = + kEmitCompilerReadBarrier && (field_type == Primitive::kPrimNot); + LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary( + instruction, + object_field_get_with_read_barrier + ? LocationSummary::kCallOnSlowPath + : LocationSummary::kNoCall); locations->SetInAt(0, Location::RequiresRegister()); if (Primitive::IsFloatingPointType(instruction->GetType())) { locations->SetOut(Location::RequiresFpuRegister()); } else { - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); + // The output overlaps in the case of an object field get with + // read barriers enabled: we do not want the move to overwrite the + // object's location, as we need it to emit the read barrier. + locations->SetOut(Location::RequiresRegister(), + object_field_get_with_read_barrier + ? Location::kOutputOverlap + : Location::kNoOutputOverlap); + } + if (object_field_get_with_read_barrier && kUseBakerReadBarrier) { + // We need a temporary register for the read barrier marking slow + // path in CodeGeneratorMIPS64::GenerateFieldLoadWithBakerReadBarrier. + locations->AddTemp(Location::RequiresRegister()); } } @@ -3285,8 +4021,11 @@ void InstructionCodeGeneratorMIPS64::HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info) { Primitive::Type type = field_info.GetFieldType(); LocationSummary* locations = instruction->GetLocations(); - GpuRegister obj = locations->InAt(0).AsRegister<GpuRegister>(); + Location obj_loc = locations->InAt(0); + GpuRegister obj = obj_loc.AsRegister<GpuRegister>(); + Location dst_loc = locations->Out(); LoadOperandType load_type = kLoadUnsignedByte; + bool is_volatile = field_info.IsVolatile(); uint32_t offset = field_info.GetFieldOffset().Uint32Value(); auto null_checker = GetImplicitNullChecker(instruction, codegen_); @@ -3319,19 +4058,46 @@ void InstructionCodeGeneratorMIPS64::HandleFieldGet(HInstruction* instruction, UNREACHABLE(); } if (!Primitive::IsFloatingPointType(type)) { - DCHECK(locations->Out().IsRegister()); - GpuRegister dst = locations->Out().AsRegister<GpuRegister>(); - __ LoadFromOffset(load_type, dst, obj, offset, null_checker); + DCHECK(dst_loc.IsRegister()); + GpuRegister dst = dst_loc.AsRegister<GpuRegister>(); + if (type == Primitive::kPrimNot) { + // /* HeapReference<Object> */ dst = *(obj + offset) + if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { + Location temp_loc = locations->GetTemp(0); + // Note that a potential implicit null check is handled in this + // CodeGeneratorMIPS64::GenerateFieldLoadWithBakerReadBarrier call. + codegen_->GenerateFieldLoadWithBakerReadBarrier(instruction, + dst_loc, + obj, + offset, + temp_loc, + /* needs_null_check */ true); + if (is_volatile) { + GenerateMemoryBarrier(MemBarrierKind::kLoadAny); + } + } else { + __ LoadFromOffset(kLoadUnsignedWord, dst, obj, offset, null_checker); + if (is_volatile) { + GenerateMemoryBarrier(MemBarrierKind::kLoadAny); + } + // If read barriers are enabled, emit read barriers other than + // Baker's using a slow path (and also unpoison the loaded + // reference, if heap poisoning is enabled). + codegen_->MaybeGenerateReadBarrierSlow(instruction, dst_loc, dst_loc, obj_loc, offset); + } + } else { + __ LoadFromOffset(load_type, dst, obj, offset, null_checker); + } } else { - DCHECK(locations->Out().IsFpuRegister()); - FpuRegister dst = locations->Out().AsFpuRegister<FpuRegister>(); + DCHECK(dst_loc.IsFpuRegister()); + FpuRegister dst = dst_loc.AsFpuRegister<FpuRegister>(); __ LoadFpuFromOffset(load_type, dst, obj, offset, null_checker); } - // TODO: memory barrier? - if (type == Primitive::kPrimNot) { - GpuRegister dst = locations->Out().AsRegister<GpuRegister>(); - __ MaybeUnpoisonHeapReference(dst); + // Memory barriers, in the case of references, are handled in the + // previous switch statement. + if (is_volatile && (type != Primitive::kPrimNot)) { + GenerateMemoryBarrier(MemBarrierKind::kLoadAny); } } @@ -3355,6 +4121,7 @@ void InstructionCodeGeneratorMIPS64::HandleFieldSet(HInstruction* instruction, GpuRegister obj = locations->InAt(0).AsRegister<GpuRegister>(); Location value_location = locations->InAt(1); StoreOperandType store_type = kStoreByte; + bool is_volatile = field_info.IsVolatile(); uint32_t offset = field_info.GetFieldOffset().Uint32Value(); bool needs_write_barrier = CodeGenerator::StoreNeedsWriteBarrier(type, instruction->InputAt(1)); auto null_checker = GetImplicitNullChecker(instruction, codegen_); @@ -3382,6 +4149,10 @@ void InstructionCodeGeneratorMIPS64::HandleFieldSet(HInstruction* instruction, UNREACHABLE(); } + if (is_volatile) { + GenerateMemoryBarrier(MemBarrierKind::kAnyStore); + } + if (value_location.IsConstant()) { int64_t value = CodeGenerator::GetInt64ValueOf(value_location.GetConstant()); __ StoreConstToOffset(store_type, value, obj, offset, TMP, null_checker); @@ -3405,12 +4176,16 @@ void InstructionCodeGeneratorMIPS64::HandleFieldSet(HInstruction* instruction, __ StoreFpuToOffset(store_type, src, obj, offset, null_checker); } } - // TODO: memory barriers? + if (needs_write_barrier) { DCHECK(value_location.IsRegister()); GpuRegister src = value_location.AsRegister<GpuRegister>(); codegen_->MarkGCCard(obj, src, value_can_be_null); } + + if (is_volatile) { + GenerateMemoryBarrier(MemBarrierKind::kAnyAny); + } } void LocationsBuilderMIPS64::VisitInstanceFieldGet(HInstanceFieldGet* instruction) { @@ -3429,14 +4204,134 @@ void InstructionCodeGeneratorMIPS64::VisitInstanceFieldSet(HInstanceFieldSet* in HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull()); } +void InstructionCodeGeneratorMIPS64::GenerateReferenceLoadOneRegister( + HInstruction* instruction, + Location out, + uint32_t offset, + Location maybe_temp, + ReadBarrierOption read_barrier_option) { + GpuRegister out_reg = out.AsRegister<GpuRegister>(); + if (read_barrier_option == kWithReadBarrier) { + CHECK(kEmitCompilerReadBarrier); + DCHECK(maybe_temp.IsRegister()) << maybe_temp; + if (kUseBakerReadBarrier) { + // Load with fast path based Baker's read barrier. + // /* HeapReference<Object> */ out = *(out + offset) + codegen_->GenerateFieldLoadWithBakerReadBarrier(instruction, + out, + out_reg, + offset, + maybe_temp, + /* needs_null_check */ false); + } else { + // Load with slow path based read barrier. + // Save the value of `out` into `maybe_temp` before overwriting it + // in the following move operation, as we will need it for the + // read barrier below. + __ Move(maybe_temp.AsRegister<GpuRegister>(), out_reg); + // /* HeapReference<Object> */ out = *(out + offset) + __ LoadFromOffset(kLoadUnsignedWord, out_reg, out_reg, offset); + codegen_->GenerateReadBarrierSlow(instruction, out, out, maybe_temp, offset); + } + } else { + // Plain load with no read barrier. + // /* HeapReference<Object> */ out = *(out + offset) + __ LoadFromOffset(kLoadUnsignedWord, out_reg, out_reg, offset); + __ MaybeUnpoisonHeapReference(out_reg); + } +} + +void InstructionCodeGeneratorMIPS64::GenerateReferenceLoadTwoRegisters( + HInstruction* instruction, + Location out, + Location obj, + uint32_t offset, + Location maybe_temp, + ReadBarrierOption read_barrier_option) { + GpuRegister out_reg = out.AsRegister<GpuRegister>(); + GpuRegister obj_reg = obj.AsRegister<GpuRegister>(); + if (read_barrier_option == kWithReadBarrier) { + CHECK(kEmitCompilerReadBarrier); + if (kUseBakerReadBarrier) { + DCHECK(maybe_temp.IsRegister()) << maybe_temp; + // Load with fast path based Baker's read barrier. + // /* HeapReference<Object> */ out = *(obj + offset) + codegen_->GenerateFieldLoadWithBakerReadBarrier(instruction, + out, + obj_reg, + offset, + maybe_temp, + /* needs_null_check */ false); + } else { + // Load with slow path based read barrier. + // /* HeapReference<Object> */ out = *(obj + offset) + __ LoadFromOffset(kLoadUnsignedWord, out_reg, obj_reg, offset); + codegen_->GenerateReadBarrierSlow(instruction, out, out, obj, offset); + } + } else { + // Plain load with no read barrier. + // /* HeapReference<Object> */ out = *(obj + offset) + __ LoadFromOffset(kLoadUnsignedWord, out_reg, obj_reg, offset); + __ MaybeUnpoisonHeapReference(out_reg); + } +} + void InstructionCodeGeneratorMIPS64::GenerateGcRootFieldLoad( - HInstruction* instruction ATTRIBUTE_UNUSED, + HInstruction* instruction, Location root, GpuRegister obj, - uint32_t offset) { + uint32_t offset, + ReadBarrierOption read_barrier_option) { GpuRegister root_reg = root.AsRegister<GpuRegister>(); - if (kEmitCompilerReadBarrier) { - UNIMPLEMENTED(FATAL) << "for read barrier"; + if (read_barrier_option == kWithReadBarrier) { + DCHECK(kEmitCompilerReadBarrier); + if (kUseBakerReadBarrier) { + // Fast path implementation of art::ReadBarrier::BarrierForRoot when + // Baker's read barrier are used: + // + // root = obj.field; + // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg() + // if (temp != null) { + // root = temp(root) + // } + + // /* GcRoot<mirror::Object> */ root = *(obj + offset) + __ LoadFromOffset(kLoadUnsignedWord, root_reg, obj, offset); + static_assert( + sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(GcRoot<mirror::Object>), + "art::mirror::CompressedReference<mirror::Object> and art::GcRoot<mirror::Object> " + "have different sizes."); + static_assert(sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(int32_t), + "art::mirror::CompressedReference<mirror::Object> and int32_t " + "have different sizes."); + + // Slow path marking the GC root `root`. + Location temp = Location::RegisterLocation(T9); + SlowPathCodeMIPS64* slow_path = + new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathMIPS64( + instruction, + root, + /*entrypoint*/ temp); + codegen_->AddSlowPath(slow_path); + + // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg() + const int32_t entry_point_offset = + CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kMips64PointerSize>(root.reg() - 1); + // Loading the entrypoint does not require a load acquire since it is only changed when + // threads are suspended or running a checkpoint. + __ LoadFromOffset(kLoadDoubleword, temp.AsRegister<GpuRegister>(), TR, entry_point_offset); + // The entrypoint is null when the GC is not marking, this prevents one load compared to + // checking GetIsGcMarking. + __ Bnezc(temp.AsRegister<GpuRegister>(), slow_path->GetEntryLabel()); + __ Bind(slow_path->GetExitLabel()); + } else { + // GC root loaded through a slow path for read barriers other + // than Baker's. + // /* GcRoot<mirror::Object>* */ root = obj + offset + __ Daddiu64(root_reg, obj, static_cast<int32_t>(offset)); + // /* mirror::Object* */ root = root->Read() + codegen_->GenerateReadBarrierForRootSlow(instruction, root, root); + } } else { // Plain GC root load with no read barrier. // /* GcRoot<mirror::Object> */ root = *(obj + offset) @@ -3446,6 +4341,219 @@ void InstructionCodeGeneratorMIPS64::GenerateGcRootFieldLoad( } } +void CodeGeneratorMIPS64::GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction, + Location ref, + GpuRegister obj, + uint32_t offset, + Location temp, + bool needs_null_check) { + DCHECK(kEmitCompilerReadBarrier); + DCHECK(kUseBakerReadBarrier); + + // /* HeapReference<Object> */ ref = *(obj + offset) + Location no_index = Location::NoLocation(); + ScaleFactor no_scale_factor = TIMES_1; + GenerateReferenceLoadWithBakerReadBarrier(instruction, + ref, + obj, + offset, + no_index, + no_scale_factor, + temp, + needs_null_check); +} + +void CodeGeneratorMIPS64::GenerateArrayLoadWithBakerReadBarrier(HInstruction* instruction, + Location ref, + GpuRegister obj, + uint32_t data_offset, + Location index, + Location temp, + bool needs_null_check) { + DCHECK(kEmitCompilerReadBarrier); + DCHECK(kUseBakerReadBarrier); + + static_assert( + sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), + "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); + // /* HeapReference<Object> */ ref = + // *(obj + data_offset + index * sizeof(HeapReference<Object>)) + ScaleFactor scale_factor = TIMES_4; + GenerateReferenceLoadWithBakerReadBarrier(instruction, + ref, + obj, + data_offset, + index, + scale_factor, + temp, + needs_null_check); +} + +void CodeGeneratorMIPS64::GenerateReferenceLoadWithBakerReadBarrier(HInstruction* instruction, + Location ref, + GpuRegister obj, + uint32_t offset, + Location index, + ScaleFactor scale_factor, + Location temp, + bool needs_null_check, + bool always_update_field) { + DCHECK(kEmitCompilerReadBarrier); + DCHECK(kUseBakerReadBarrier); + + // In slow path based read barriers, the read barrier call is + // inserted after the original load. However, in fast path based + // Baker's read barriers, we need to perform the load of + // mirror::Object::monitor_ *before* the original reference load. + // This load-load ordering is required by the read barrier. + // The fast path/slow path (for Baker's algorithm) should look like: + // + // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState(); + // lfence; // Load fence or artificial data dependency to prevent load-load reordering + // HeapReference<Object> ref = *src; // Original reference load. + // bool is_gray = (rb_state == ReadBarrier::GrayState()); + // if (is_gray) { + // ref = ReadBarrier::Mark(ref); // Performed by runtime entrypoint slow path. + // } + // + // Note: the original implementation in ReadBarrier::Barrier is + // slightly more complex as it performs additional checks that we do + // not do here for performance reasons. + + GpuRegister ref_reg = ref.AsRegister<GpuRegister>(); + GpuRegister temp_reg = temp.AsRegister<GpuRegister>(); + uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value(); + + // /* int32_t */ monitor = obj->monitor_ + __ LoadFromOffset(kLoadWord, temp_reg, obj, monitor_offset); + if (needs_null_check) { + MaybeRecordImplicitNullCheck(instruction); + } + // /* LockWord */ lock_word = LockWord(monitor) + static_assert(sizeof(LockWord) == sizeof(int32_t), + "art::LockWord and int32_t have different sizes."); + + __ Sync(0); // Barrier to prevent load-load reordering. + + // The actual reference load. + if (index.IsValid()) { + // Load types involving an "index": ArrayGet, + // UnsafeGetObject/UnsafeGetObjectVolatile and UnsafeCASObject + // intrinsics. + // /* HeapReference<Object> */ ref = *(obj + offset + (index << scale_factor)) + if (index.IsConstant()) { + size_t computed_offset = + (index.GetConstant()->AsIntConstant()->GetValue() << scale_factor) + offset; + __ LoadFromOffset(kLoadUnsignedWord, ref_reg, obj, computed_offset); + } else { + GpuRegister index_reg = index.AsRegister<GpuRegister>(); + __ Dsll(TMP, index_reg, scale_factor); + __ Daddu(TMP, obj, TMP); + __ LoadFromOffset(kLoadUnsignedWord, ref_reg, TMP, offset); + } + } else { + // /* HeapReference<Object> */ ref = *(obj + offset) + __ LoadFromOffset(kLoadUnsignedWord, ref_reg, obj, offset); + } + + // Object* ref = ref_addr->AsMirrorPtr() + __ MaybeUnpoisonHeapReference(ref_reg); + + // Slow path marking the object `ref` when it is gray. + SlowPathCodeMIPS64* slow_path; + if (always_update_field) { + // ReadBarrierMarkAndUpdateFieldSlowPathMIPS64 only supports address + // of the form `obj + field_offset`, where `obj` is a register and + // `field_offset` is a register. Thus `offset` and `scale_factor` + // above are expected to be null in this code path. + DCHECK_EQ(offset, 0u); + DCHECK_EQ(scale_factor, ScaleFactor::TIMES_1); + slow_path = new (GetGraph()->GetArena()) + ReadBarrierMarkAndUpdateFieldSlowPathMIPS64(instruction, + ref, + obj, + /* field_offset */ index, + temp_reg); + } else { + slow_path = new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathMIPS64(instruction, ref); + } + AddSlowPath(slow_path); + + // if (rb_state == ReadBarrier::GrayState()) + // ref = ReadBarrier::Mark(ref); + // Given the numeric representation, it's enough to check the low bit of the + // rb_state. We do that by shifting the bit into the sign bit (31) and + // performing a branch on less than zero. + static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0"); + static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1"); + static_assert(LockWord::kReadBarrierStateSize == 1, "Expecting 1-bit read barrier state size"); + __ Sll(temp_reg, temp_reg, 31 - LockWord::kReadBarrierStateShift); + __ Bltzc(temp_reg, slow_path->GetEntryLabel()); + __ Bind(slow_path->GetExitLabel()); +} + +void CodeGeneratorMIPS64::GenerateReadBarrierSlow(HInstruction* instruction, + Location out, + Location ref, + Location obj, + uint32_t offset, + Location index) { + DCHECK(kEmitCompilerReadBarrier); + + // Insert a slow path based read barrier *after* the reference load. + // + // If heap poisoning is enabled, the unpoisoning of the loaded + // reference will be carried out by the runtime within the slow + // path. + // + // Note that `ref` currently does not get unpoisoned (when heap + // poisoning is enabled), which is alright as the `ref` argument is + // not used by the artReadBarrierSlow entry point. + // + // TODO: Unpoison `ref` when it is used by artReadBarrierSlow. + SlowPathCodeMIPS64* slow_path = new (GetGraph()->GetArena()) + ReadBarrierForHeapReferenceSlowPathMIPS64(instruction, out, ref, obj, offset, index); + AddSlowPath(slow_path); + + __ Bc(slow_path->GetEntryLabel()); + __ Bind(slow_path->GetExitLabel()); +} + +void CodeGeneratorMIPS64::MaybeGenerateReadBarrierSlow(HInstruction* instruction, + Location out, + Location ref, + Location obj, + uint32_t offset, + Location index) { + if (kEmitCompilerReadBarrier) { + // Baker's read barriers shall be handled by the fast path + // (CodeGeneratorMIPS64::GenerateReferenceLoadWithBakerReadBarrier). + DCHECK(!kUseBakerReadBarrier); + // If heap poisoning is enabled, unpoisoning will be taken care of + // by the runtime within the slow path. + GenerateReadBarrierSlow(instruction, out, ref, obj, offset, index); + } else if (kPoisonHeapReferences) { + __ UnpoisonHeapReference(out.AsRegister<GpuRegister>()); + } +} + +void CodeGeneratorMIPS64::GenerateReadBarrierForRootSlow(HInstruction* instruction, + Location out, + Location root) { + DCHECK(kEmitCompilerReadBarrier); + + // Insert a slow path based read barrier *after* the GC root load. + // + // Note that GC roots are not affected by heap poisoning, so we do + // not need to do anything special for this here. + SlowPathCodeMIPS64* slow_path = + new (GetGraph()->GetArena()) ReadBarrierForRootSlowPathMIPS64(instruction, out, root); + AddSlowPath(slow_path); + + __ Bc(slow_path->GetEntryLabel()); + __ Bind(slow_path->GetExitLabel()); +} + void LocationsBuilderMIPS64::VisitInstanceOf(HInstanceOf* instruction) { LocationSummary::CallKind call_kind = LocationSummary::kNoCall; TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); @@ -3454,7 +4562,8 @@ void LocationsBuilderMIPS64::VisitInstanceOf(HInstanceOf* instruction) { case TypeCheckKind::kAbstractClassCheck: case TypeCheckKind::kClassHierarchyCheck: case TypeCheckKind::kArrayObjectCheck: - call_kind = LocationSummary::kNoCall; + call_kind = + kEmitCompilerReadBarrier ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall; break; case TypeCheckKind::kArrayCheck: case TypeCheckKind::kUnresolvedCheck: @@ -3469,14 +4578,20 @@ void LocationsBuilderMIPS64::VisitInstanceOf(HInstanceOf* instruction) { // The output does overlap inputs. // Note that TypeCheckSlowPathMIPS64 uses this register too. locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); + locations->AddRegisterTemps(NumberOfInstanceOfTemps(type_check_kind)); } void InstructionCodeGeneratorMIPS64::VisitInstanceOf(HInstanceOf* instruction) { TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); LocationSummary* locations = instruction->GetLocations(); - GpuRegister obj = locations->InAt(0).AsRegister<GpuRegister>(); + Location obj_loc = locations->InAt(0); + GpuRegister obj = obj_loc.AsRegister<GpuRegister>(); GpuRegister cls = locations->InAt(1).AsRegister<GpuRegister>(); - GpuRegister out = locations->Out().AsRegister<GpuRegister>(); + Location out_loc = locations->Out(); + GpuRegister out = out_loc.AsRegister<GpuRegister>(); + const size_t num_temps = NumberOfInstanceOfTemps(type_check_kind); + DCHECK_LE(num_temps, 1u); + Location maybe_temp_loc = (num_temps >= 1) ? locations->GetTemp(0) : Location::NoLocation(); uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); @@ -3494,8 +4609,12 @@ void InstructionCodeGeneratorMIPS64::VisitInstanceOf(HInstanceOf* instruction) { switch (type_check_kind) { case TypeCheckKind::kExactCheck: { // /* HeapReference<Class> */ out = obj->klass_ - __ LoadFromOffset(kLoadUnsignedWord, out, obj, class_offset); - __ MaybeUnpoisonHeapReference(out); + GenerateReferenceLoadTwoRegisters(instruction, + out_loc, + obj_loc, + class_offset, + maybe_temp_loc, + kCompilerReadBarrierOption); // Classes must be equal for the instanceof to succeed. __ Xor(out, out, cls); __ Sltiu(out, out, 1); @@ -3504,15 +4623,22 @@ void InstructionCodeGeneratorMIPS64::VisitInstanceOf(HInstanceOf* instruction) { case TypeCheckKind::kAbstractClassCheck: { // /* HeapReference<Class> */ out = obj->klass_ - __ LoadFromOffset(kLoadUnsignedWord, out, obj, class_offset); - __ MaybeUnpoisonHeapReference(out); + GenerateReferenceLoadTwoRegisters(instruction, + out_loc, + obj_loc, + class_offset, + maybe_temp_loc, + kCompilerReadBarrierOption); // If the class is abstract, we eagerly fetch the super class of the // object to avoid doing a comparison we know will fail. Mips64Label loop; __ Bind(&loop); // /* HeapReference<Class> */ out = out->super_class_ - __ LoadFromOffset(kLoadUnsignedWord, out, out, super_offset); - __ MaybeUnpoisonHeapReference(out); + GenerateReferenceLoadOneRegister(instruction, + out_loc, + super_offset, + maybe_temp_loc, + kCompilerReadBarrierOption); // If `out` is null, we use it for the result, and jump to `done`. __ Beqzc(out, &done); __ Bnec(out, cls, &loop); @@ -3522,15 +4648,22 @@ void InstructionCodeGeneratorMIPS64::VisitInstanceOf(HInstanceOf* instruction) { case TypeCheckKind::kClassHierarchyCheck: { // /* HeapReference<Class> */ out = obj->klass_ - __ LoadFromOffset(kLoadUnsignedWord, out, obj, class_offset); - __ MaybeUnpoisonHeapReference(out); + GenerateReferenceLoadTwoRegisters(instruction, + out_loc, + obj_loc, + class_offset, + maybe_temp_loc, + kCompilerReadBarrierOption); // Walk over the class hierarchy to find a match. Mips64Label loop, success; __ Bind(&loop); __ Beqc(out, cls, &success); // /* HeapReference<Class> */ out = out->super_class_ - __ LoadFromOffset(kLoadUnsignedWord, out, out, super_offset); - __ MaybeUnpoisonHeapReference(out); + GenerateReferenceLoadOneRegister(instruction, + out_loc, + super_offset, + maybe_temp_loc, + kCompilerReadBarrierOption); __ Bnezc(out, &loop); // If `out` is null, we use it for the result, and jump to `done`. __ Bc(&done); @@ -3541,15 +4674,22 @@ void InstructionCodeGeneratorMIPS64::VisitInstanceOf(HInstanceOf* instruction) { case TypeCheckKind::kArrayObjectCheck: { // /* HeapReference<Class> */ out = obj->klass_ - __ LoadFromOffset(kLoadUnsignedWord, out, obj, class_offset); - __ MaybeUnpoisonHeapReference(out); + GenerateReferenceLoadTwoRegisters(instruction, + out_loc, + obj_loc, + class_offset, + maybe_temp_loc, + kCompilerReadBarrierOption); // Do an exact check. Mips64Label success; __ Beqc(out, cls, &success); // Otherwise, we need to check that the object's class is a non-primitive array. // /* HeapReference<Class> */ out = out->component_type_ - __ LoadFromOffset(kLoadUnsignedWord, out, out, component_offset); - __ MaybeUnpoisonHeapReference(out); + GenerateReferenceLoadOneRegister(instruction, + out_loc, + component_offset, + maybe_temp_loc, + kCompilerReadBarrierOption); // If `out` is null, we use it for the result, and jump to `done`. __ Beqzc(out, &done); __ LoadFromOffset(kLoadUnsignedHalfword, out, out, primitive_offset); @@ -3564,8 +4704,12 @@ void InstructionCodeGeneratorMIPS64::VisitInstanceOf(HInstanceOf* instruction) { case TypeCheckKind::kArrayCheck: { // No read barrier since the slow path will retry upon failure. // /* HeapReference<Class> */ out = obj->klass_ - __ LoadFromOffset(kLoadUnsignedWord, out, obj, class_offset); - __ MaybeUnpoisonHeapReference(out); + GenerateReferenceLoadTwoRegisters(instruction, + out_loc, + obj_loc, + class_offset, + maybe_temp_loc, + kWithoutReadBarrier); DCHECK(locations->OnlyCallsOnSlowPath()); slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathMIPS64(instruction, /* is_fatal */ false); @@ -3735,9 +4879,6 @@ static bool TryGenerateIntrinsicCode(HInvoke* invoke, CodeGeneratorMIPS64* codeg HLoadString::LoadKind CodeGeneratorMIPS64::GetSupportedLoadStringKind( HLoadString::LoadKind desired_string_load_kind) { - if (kEmitCompilerReadBarrier) { - UNIMPLEMENTED(FATAL) << "for read barrier"; - } bool fallback_load = false; switch (desired_string_load_kind) { case HLoadString::LoadKind::kBootImageLinkTimeAddress: @@ -3765,9 +4906,6 @@ HLoadString::LoadKind CodeGeneratorMIPS64::GetSupportedLoadStringKind( HLoadClass::LoadKind CodeGeneratorMIPS64::GetSupportedLoadClassKind( HLoadClass::LoadKind desired_class_load_kind) { - if (kEmitCompilerReadBarrier) { - UNIMPLEMENTED(FATAL) << "for read barrier"; - } bool fallback_load = false; switch (desired_class_load_kind) { case HLoadClass::LoadKind::kInvalid: @@ -3960,7 +5098,8 @@ void LocationsBuilderMIPS64::VisitLoadClass(HLoadClass* cls) { } DCHECK(!cls->NeedsAccessCheck()); - LocationSummary::CallKind call_kind = (cls->NeedsEnvironment() || kEmitCompilerReadBarrier) + const bool requires_read_barrier = kEmitCompilerReadBarrier && !cls->IsInBootImage(); + LocationSummary::CallKind call_kind = (cls->NeedsEnvironment() || requires_read_barrier) ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall; LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(cls, call_kind); @@ -3989,6 +5128,9 @@ void InstructionCodeGeneratorMIPS64::VisitLoadClass(HLoadClass* cls) NO_THREAD_S current_method_reg = locations->InAt(0).AsRegister<GpuRegister>(); } + const ReadBarrierOption read_barrier_option = cls->IsInBootImage() + ? kWithoutReadBarrier + : kCompilerReadBarrierOption; bool generate_null_check = false; switch (load_kind) { case HLoadClass::LoadKind::kReferrersClass: @@ -3998,10 +5140,12 @@ void InstructionCodeGeneratorMIPS64::VisitLoadClass(HLoadClass* cls) NO_THREAD_S GenerateGcRootFieldLoad(cls, out_loc, current_method_reg, - ArtMethod::DeclaringClassOffset().Int32Value()); + ArtMethod::DeclaringClassOffset().Int32Value(), + read_barrier_option); break; case HLoadClass::LoadKind::kBootImageLinkTimeAddress: DCHECK(codegen_->GetCompilerOptions().IsBootImage()); + DCHECK_EQ(read_barrier_option, kWithoutReadBarrier); __ LoadLiteral(out, kLoadUnsignedWord, codegen_->DeduplicateBootImageTypeLiteral(cls->GetDexFile(), @@ -4009,6 +5153,7 @@ void InstructionCodeGeneratorMIPS64::VisitLoadClass(HLoadClass* cls) NO_THREAD_S break; case HLoadClass::LoadKind::kBootImageLinkTimePcRelative: { DCHECK(codegen_->GetCompilerOptions().IsBootImage()); + DCHECK_EQ(read_barrier_option, kWithoutReadBarrier); CodeGeneratorMIPS64::PcRelativePatchInfo* info = codegen_->NewPcRelativeTypePatch(cls->GetDexFile(), cls->GetTypeIndex()); codegen_->EmitPcRelativeAddressPlaceholderHigh(info, AT); @@ -4016,7 +5161,7 @@ void InstructionCodeGeneratorMIPS64::VisitLoadClass(HLoadClass* cls) NO_THREAD_S break; } case HLoadClass::LoadKind::kBootImageAddress: { - DCHECK(!kEmitCompilerReadBarrier); + DCHECK_EQ(read_barrier_option, kWithoutReadBarrier); uint32_t address = dchecked_integral_cast<uint32_t>( reinterpret_cast<uintptr_t>(cls->GetClass().Get())); DCHECK_NE(address, 0u); @@ -4029,7 +5174,7 @@ void InstructionCodeGeneratorMIPS64::VisitLoadClass(HLoadClass* cls) NO_THREAD_S CodeGeneratorMIPS64::PcRelativePatchInfo* info = codegen_->NewTypeBssEntryPatch(cls->GetDexFile(), cls->GetTypeIndex()); codegen_->EmitPcRelativeAddressPlaceholderHigh(info, out); - GenerateGcRootFieldLoad(cls, out_loc, out, /* placeholder */ 0x5678); + GenerateGcRootFieldLoad(cls, out_loc, out, /* placeholder */ 0x5678, read_barrier_option); generate_null_check = true; break; } @@ -4039,7 +5184,7 @@ void InstructionCodeGeneratorMIPS64::VisitLoadClass(HLoadClass* cls) NO_THREAD_S codegen_->DeduplicateJitClassLiteral(cls->GetDexFile(), cls->GetTypeIndex(), cls->GetClass())); - GenerateGcRootFieldLoad(cls, out_loc, out, 0); + GenerateGcRootFieldLoad(cls, out_loc, out, 0, read_barrier_option); break; case HLoadClass::LoadKind::kDexCacheViaMethod: case HLoadClass::LoadKind::kInvalid: @@ -4136,7 +5281,11 @@ void InstructionCodeGeneratorMIPS64::VisitLoadString(HLoadString* load) NO_THREA CodeGeneratorMIPS64::PcRelativePatchInfo* info = codegen_->NewPcRelativeStringPatch(load->GetDexFile(), load->GetStringIndex()); codegen_->EmitPcRelativeAddressPlaceholderHigh(info, out); - GenerateGcRootFieldLoad(load, out_loc, out, /* placeholder */ 0x5678); + GenerateGcRootFieldLoad(load, + out_loc, + out, + /* placeholder */ 0x5678, + kCompilerReadBarrierOption); SlowPathCodeMIPS64* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathMIPS64(load); codegen_->AddSlowPath(slow_path); __ Beqzc(out, slow_path->GetEntryLabel()); @@ -4149,7 +5298,7 @@ void InstructionCodeGeneratorMIPS64::VisitLoadString(HLoadString* load) NO_THREA codegen_->DeduplicateJitStringLiteral(load->GetDexFile(), load->GetStringIndex(), load->GetString())); - GenerateGcRootFieldLoad(load, out_loc, out, 0); + GenerateGcRootFieldLoad(load, out_loc, out, 0, kCompilerReadBarrierOption); return; default: break; diff --git a/compiler/optimizing/code_generator_mips64.h b/compiler/optimizing/code_generator_mips64.h index 6040dc9492..fd1a174608 100644 --- a/compiler/optimizing/code_generator_mips64.h +++ b/compiler/optimizing/code_generator_mips64.h @@ -237,6 +237,38 @@ class InstructionCodeGeneratorMIPS64 : public InstructionCodeGenerator { const FieldInfo& field_info, bool value_can_be_null); void HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info); + + // Generate a heap reference load using one register `out`: + // + // out <- *(out + offset) + // + // while honoring heap poisoning and/or read barriers (if any). + // + // Location `maybe_temp` is used when generating a read barrier and + // shall be a register in that case; it may be an invalid location + // otherwise. + void GenerateReferenceLoadOneRegister(HInstruction* instruction, + Location out, + uint32_t offset, + Location maybe_temp, + ReadBarrierOption read_barrier_option); + // Generate a heap reference load using two different registers + // `out` and `obj`: + // + // out <- *(obj + offset) + // + // while honoring heap poisoning and/or read barriers (if any). + // + // Location `maybe_temp` is used when generating a Baker's (fast + // path) read barrier and shall be a register in that case; it may + // be an invalid location otherwise. + void GenerateReferenceLoadTwoRegisters(HInstruction* instruction, + Location out, + Location obj, + uint32_t offset, + Location maybe_temp, + ReadBarrierOption read_barrier_option); + // Generate a GC root reference load: // // root <- *(obj + offset) @@ -245,7 +277,9 @@ class InstructionCodeGeneratorMIPS64 : public InstructionCodeGenerator { void GenerateGcRootFieldLoad(HInstruction* instruction, Location root, GpuRegister obj, - uint32_t offset); + uint32_t offset, + ReadBarrierOption read_barrier_option); + void GenerateTestAndBranch(HInstruction* instruction, size_t condition_input_index, Mips64Label* true_target, @@ -316,6 +350,91 @@ class CodeGeneratorMIPS64 : public CodeGenerator { void EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) OVERRIDE; void EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) OVERRIDE; + // Fast path implementation of ReadBarrier::Barrier for a heap + // reference field load when Baker's read barriers are used. + void GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction, + Location ref, + GpuRegister obj, + uint32_t offset, + Location temp, + bool needs_null_check); + // Fast path implementation of ReadBarrier::Barrier for a heap + // reference array load when Baker's read barriers are used. + void GenerateArrayLoadWithBakerReadBarrier(HInstruction* instruction, + Location ref, + GpuRegister obj, + uint32_t data_offset, + Location index, + Location temp, + bool needs_null_check); + + // Factored implementation, used by GenerateFieldLoadWithBakerReadBarrier, + // GenerateArrayLoadWithBakerReadBarrier and some intrinsics. + // + // Load the object reference located at the address + // `obj + offset + (index << scale_factor)`, held by object `obj`, into + // `ref`, and mark it if needed. + // + // If `always_update_field` is true, the value of the reference is + // atomically updated in the holder (`obj`). + void GenerateReferenceLoadWithBakerReadBarrier(HInstruction* instruction, + Location ref, + GpuRegister obj, + uint32_t offset, + Location index, + ScaleFactor scale_factor, + Location temp, + bool needs_null_check, + bool always_update_field = false); + + // Generate a read barrier for a heap reference within `instruction` + // using a slow path. + // + // A read barrier for an object reference read from the heap is + // implemented as a call to the artReadBarrierSlow runtime entry + // point, which is passed the values in locations `ref`, `obj`, and + // `offset`: + // + // mirror::Object* artReadBarrierSlow(mirror::Object* ref, + // mirror::Object* obj, + // uint32_t offset); + // + // The `out` location contains the value returned by + // artReadBarrierSlow. + // + // When `index` is provided (i.e. for array accesses), the offset + // value passed to artReadBarrierSlow is adjusted to take `index` + // into account. + void GenerateReadBarrierSlow(HInstruction* instruction, + Location out, + Location ref, + Location obj, + uint32_t offset, + Location index = Location::NoLocation()); + + // If read barriers are enabled, generate a read barrier for a heap + // reference using a slow path. If heap poisoning is enabled, also + // unpoison the reference in `out`. + void MaybeGenerateReadBarrierSlow(HInstruction* instruction, + Location out, + Location ref, + Location obj, + uint32_t offset, + Location index = Location::NoLocation()); + + // Generate a read barrier for a GC root within `instruction` using + // a slow path. + // + // A read barrier for an object reference GC root is implemented as + // a call to the artReadBarrierForRootSlow runtime entry point, + // which is passed the value in location `root`: + // + // mirror::Object* artReadBarrierForRootSlow(GcRoot<mirror::Object>* root); + // + // The `out` location contains the value returned by + // artReadBarrierForRootSlow. + void GenerateReadBarrierForRootSlow(HInstruction* instruction, Location out, Location root); + void MarkGCCard(GpuRegister object, GpuRegister value, bool value_can_be_null); // Register allocation. @@ -366,6 +485,14 @@ class CodeGeneratorMIPS64 : public CodeGenerator { uint32_t dex_pc, SlowPathCode* slow_path = nullptr) OVERRIDE; + // Generate code to invoke a runtime entry point, but do not record + // PC-related information in a stack map. + void InvokeRuntimeWithoutRecordingPcInfo(int32_t entry_point_offset, + HInstruction* instruction, + SlowPathCode* slow_path); + + void GenerateInvokeRuntime(int32_t entry_point_offset); + ParallelMoveResolver* GetMoveResolver() OVERRIDE { return &move_resolver_; } bool NeedsTwoRegisters(Primitive::Type type ATTRIBUTE_UNUSED) const OVERRIDE { return false; } diff --git a/compiler/optimizing/code_generator_vector_arm.cc b/compiler/optimizing/code_generator_vector_arm.cc new file mode 100644 index 0000000000..ba2b2cb2c9 --- /dev/null +++ b/compiler/optimizing/code_generator_vector_arm.cc @@ -0,0 +1,235 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "code_generator_arm.h" + +namespace art { +namespace arm { + +// NOLINT on __ macro to suppress wrong warning/fix (misc-macro-parentheses) from clang-tidy. +#define __ down_cast<ArmAssembler*>(GetAssembler())-> // NOLINT + +void LocationsBuilderARM::VisitVecReplicateScalar(HVecReplicateScalar* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void InstructionCodeGeneratorARM::VisitVecReplicateScalar(HVecReplicateScalar* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderARM::VisitVecSetScalars(HVecSetScalars* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void InstructionCodeGeneratorARM::VisitVecSetScalars(HVecSetScalars* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderARM::VisitVecSumReduce(HVecSumReduce* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void InstructionCodeGeneratorARM::VisitVecSumReduce(HVecSumReduce* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +// Helper to set up locations for vector unary operations. +static void CreateVecUnOpLocations(ArenaAllocator* arena, HVecUnaryOperation* instruction) { + LocationSummary* locations = new (arena) LocationSummary(instruction); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + case Primitive::kPrimFloat: + case Primitive::kPrimDouble: + DCHECK(locations); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void LocationsBuilderARM::VisitVecCnv(HVecCnv* instruction) { + CreateVecUnOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorARM::VisitVecCnv(HVecCnv* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderARM::VisitVecNeg(HVecNeg* instruction) { + CreateVecUnOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorARM::VisitVecNeg(HVecNeg* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderARM::VisitVecNot(HVecNot* instruction) { + CreateVecUnOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorARM::VisitVecNot(HVecNot* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +// Helper to set up locations for vector binary operations. +static void CreateVecBinOpLocations(ArenaAllocator* arena, HVecBinaryOperation* instruction) { + LocationSummary* locations = new (arena) LocationSummary(instruction); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + case Primitive::kPrimFloat: + case Primitive::kPrimDouble: + DCHECK(locations); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void LocationsBuilderARM::VisitVecAdd(HVecAdd* instruction) { + CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorARM::VisitVecAdd(HVecAdd* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderARM::VisitVecSub(HVecSub* instruction) { + CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorARM::VisitVecSub(HVecSub* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderARM::VisitVecMul(HVecMul* instruction) { + CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorARM::VisitVecMul(HVecMul* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderARM::VisitVecDiv(HVecDiv* instruction) { + CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorARM::VisitVecDiv(HVecDiv* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderARM::VisitVecAnd(HVecAnd* instruction) { + CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorARM::VisitVecAnd(HVecAnd* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderARM::VisitVecAndNot(HVecAndNot* instruction) { + CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorARM::VisitVecAndNot(HVecAndNot* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderARM::VisitVecOr(HVecOr* instruction) { + CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorARM::VisitVecOr(HVecOr* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderARM::VisitVecXor(HVecXor* instruction) { + CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorARM::VisitVecXor(HVecXor* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +// Helper to set up locations for vector shift operations. +static void CreateVecShiftLocations(ArenaAllocator* arena, HVecBinaryOperation* instruction) { + LocationSummary* locations = new (arena) LocationSummary(instruction); + switch (instruction->GetPackedType()) { + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + case Primitive::kPrimLong: + DCHECK(locations); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void LocationsBuilderARM::VisitVecShl(HVecShl* instruction) { + CreateVecShiftLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorARM::VisitVecShl(HVecShl* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderARM::VisitVecShr(HVecShr* instruction) { + CreateVecShiftLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorARM::VisitVecShr(HVecShr* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderARM::VisitVecUShr(HVecUShr* instruction) { + CreateVecShiftLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorARM::VisitVecUShr(HVecUShr* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderARM::VisitVecLoad(HVecLoad* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void InstructionCodeGeneratorARM::VisitVecLoad(HVecLoad* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderARM::VisitVecStore(HVecStore* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void InstructionCodeGeneratorARM::VisitVecStore(HVecStore* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +#undef __ + +} // namespace arm +} // namespace art diff --git a/compiler/optimizing/code_generator_vector_arm64.cc b/compiler/optimizing/code_generator_vector_arm64.cc new file mode 100644 index 0000000000..96d00210b8 --- /dev/null +++ b/compiler/optimizing/code_generator_vector_arm64.cc @@ -0,0 +1,641 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "code_generator_arm64.h" +#include "mirror/array-inl.h" + +using namespace vixl::aarch64; // NOLINT(build/namespaces) + +namespace art { +namespace arm64 { + +using helpers::DRegisterFrom; +using helpers::HeapOperand; +using helpers::InputRegisterAt; +using helpers::Int64ConstantFrom; +using helpers::XRegisterFrom; + +#define __ GetVIXLAssembler()-> + +void LocationsBuilderARM64::VisitVecReplicateScalar(HVecReplicateScalar* instruction) { + LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetOut(Location::RequiresFpuRegister()); + break; + case Primitive::kPrimFloat: + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void InstructionCodeGeneratorARM64::VisitVecReplicateScalar(HVecReplicateScalar* instruction) { + LocationSummary* locations = instruction->GetLocations(); + FPRegister dst = DRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ Dup(dst.V8B(), InputRegisterAt(instruction, 0)); + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ Dup(dst.V4H(), InputRegisterAt(instruction, 0)); + break; + case Primitive::kPrimInt: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ Dup(dst.V2S(), InputRegisterAt(instruction, 0)); + break; + case Primitive::kPrimFloat: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ Dup(dst.V2S(), DRegisterFrom(locations->InAt(0)).V2S(), 0); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void LocationsBuilderARM64::VisitVecSetScalars(HVecSetScalars* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void InstructionCodeGeneratorARM64::VisitVecSetScalars(HVecSetScalars* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderARM64::VisitVecSumReduce(HVecSumReduce* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void InstructionCodeGeneratorARM64::VisitVecSumReduce(HVecSumReduce* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +// Helper to set up locations for vector unary operations. +static void CreateVecUnOpLocations(ArenaAllocator* arena, HVecUnaryOperation* instruction) { + LocationSummary* locations = new (arena) LocationSummary(instruction); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetOut(Location::RequiresFpuRegister(), + instruction->IsVecNot() ? Location::kOutputOverlap + : Location::kNoOutputOverlap); + break; + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + case Primitive::kPrimFloat: + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void LocationsBuilderARM64::VisitVecCnv(HVecCnv* instruction) { + CreateVecUnOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorARM64::VisitVecCnv(HVecCnv* instruction) { + LocationSummary* locations = instruction->GetLocations(); + FPRegister src = DRegisterFrom(locations->InAt(0)); + FPRegister dst = DRegisterFrom(locations->Out()); + Primitive::Type from = instruction->GetInputType(); + Primitive::Type to = instruction->GetResultType(); + if (from == Primitive::kPrimInt && to == Primitive::kPrimFloat) { + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ Scvtf(dst.V2S(), src.V2S()); + } else { + LOG(FATAL) << "Unsupported SIMD type"; + } +} + +void LocationsBuilderARM64::VisitVecNeg(HVecNeg* instruction) { + CreateVecUnOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorARM64::VisitVecNeg(HVecNeg* instruction) { + LocationSummary* locations = instruction->GetLocations(); + FPRegister src = DRegisterFrom(locations->InAt(0)); + FPRegister dst = DRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case Primitive::kPrimByte: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ Neg(dst.V8B(), src.V8B()); + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ Neg(dst.V4H(), src.V4H()); + break; + case Primitive::kPrimInt: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ Neg(dst.V2S(), src.V2S()); + break; + case Primitive::kPrimFloat: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ Fneg(dst.V2S(), src.V2S()); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void LocationsBuilderARM64::VisitVecNot(HVecNot* instruction) { + CreateVecUnOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorARM64::VisitVecNot(HVecNot* instruction) { + LocationSummary* locations = instruction->GetLocations(); + FPRegister src = DRegisterFrom(locations->InAt(0)); + FPRegister dst = DRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: // special case boolean-not + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ Movi(dst.V8B(), 1); + __ Eor(dst.V8B(), dst.V8B(), src.V8B()); + break; + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + __ Not(dst.V8B(), src.V8B()); // lanes do not matter + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +// Helper to set up locations for vector binary operations. +static void CreateVecBinOpLocations(ArenaAllocator* arena, HVecBinaryOperation* instruction) { + LocationSummary* locations = new (arena) LocationSummary(instruction); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + case Primitive::kPrimFloat: + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetInAt(1, Location::RequiresFpuRegister()); + locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void LocationsBuilderARM64::VisitVecAdd(HVecAdd* instruction) { + CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorARM64::VisitVecAdd(HVecAdd* instruction) { + LocationSummary* locations = instruction->GetLocations(); + FPRegister lhs = DRegisterFrom(locations->InAt(0)); + FPRegister rhs = DRegisterFrom(locations->InAt(1)); + FPRegister dst = DRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case Primitive::kPrimByte: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ Add(dst.V8B(), lhs.V8B(), rhs.V8B()); + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ Add(dst.V4H(), lhs.V4H(), rhs.V4H()); + break; + case Primitive::kPrimInt: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ Add(dst.V2S(), lhs.V2S(), rhs.V2S()); + break; + case Primitive::kPrimFloat: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ Fadd(dst.V2S(), lhs.V2S(), rhs.V2S()); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void LocationsBuilderARM64::VisitVecSub(HVecSub* instruction) { + CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorARM64::VisitVecSub(HVecSub* instruction) { + LocationSummary* locations = instruction->GetLocations(); + FPRegister lhs = DRegisterFrom(locations->InAt(0)); + FPRegister rhs = DRegisterFrom(locations->InAt(1)); + FPRegister dst = DRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case Primitive::kPrimByte: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ Sub(dst.V8B(), lhs.V8B(), rhs.V8B()); + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ Sub(dst.V4H(), lhs.V4H(), rhs.V4H()); + break; + case Primitive::kPrimInt: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ Sub(dst.V2S(), lhs.V2S(), rhs.V2S()); + break; + case Primitive::kPrimFloat: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ Fsub(dst.V2S(), lhs.V2S(), rhs.V2S()); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void LocationsBuilderARM64::VisitVecMul(HVecMul* instruction) { + CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorARM64::VisitVecMul(HVecMul* instruction) { + LocationSummary* locations = instruction->GetLocations(); + FPRegister lhs = DRegisterFrom(locations->InAt(0)); + FPRegister rhs = DRegisterFrom(locations->InAt(1)); + FPRegister dst = DRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case Primitive::kPrimByte: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ Mul(dst.V8B(), lhs.V8B(), rhs.V8B()); + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ Mul(dst.V4H(), lhs.V4H(), rhs.V4H()); + break; + case Primitive::kPrimInt: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ Mul(dst.V2S(), lhs.V2S(), rhs.V2S()); + break; + case Primitive::kPrimFloat: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ Fmul(dst.V2S(), lhs.V2S(), rhs.V2S()); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void LocationsBuilderARM64::VisitVecDiv(HVecDiv* instruction) { + CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorARM64::VisitVecDiv(HVecDiv* instruction) { + LocationSummary* locations = instruction->GetLocations(); + FPRegister lhs = DRegisterFrom(locations->InAt(0)); + FPRegister rhs = DRegisterFrom(locations->InAt(1)); + FPRegister dst = DRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case Primitive::kPrimFloat: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ Fdiv(dst.V2S(), lhs.V2S(), rhs.V2S()); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void LocationsBuilderARM64::VisitVecAnd(HVecAnd* instruction) { + CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorARM64::VisitVecAnd(HVecAnd* instruction) { + LocationSummary* locations = instruction->GetLocations(); + FPRegister lhs = DRegisterFrom(locations->InAt(0)); + FPRegister rhs = DRegisterFrom(locations->InAt(1)); + FPRegister dst = DRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + case Primitive::kPrimFloat: + __ And(dst.V8B(), lhs.V8B(), rhs.V8B()); // lanes do not matter + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void LocationsBuilderARM64::VisitVecAndNot(HVecAndNot* instruction) { + LOG(FATAL) << "Unsupported SIMD instruction " << instruction->GetId(); +} + +void InstructionCodeGeneratorARM64::VisitVecAndNot(HVecAndNot* instruction) { + LOG(FATAL) << "Unsupported SIMD instruction " << instruction->GetId(); +} + +void LocationsBuilderARM64::VisitVecOr(HVecOr* instruction) { + CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorARM64::VisitVecOr(HVecOr* instruction) { + LocationSummary* locations = instruction->GetLocations(); + FPRegister lhs = DRegisterFrom(locations->InAt(0)); + FPRegister rhs = DRegisterFrom(locations->InAt(1)); + FPRegister dst = DRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + case Primitive::kPrimFloat: + __ Orr(dst.V8B(), lhs.V8B(), rhs.V8B()); // lanes do not matter + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void LocationsBuilderARM64::VisitVecXor(HVecXor* instruction) { + CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorARM64::VisitVecXor(HVecXor* instruction) { + LocationSummary* locations = instruction->GetLocations(); + FPRegister lhs = DRegisterFrom(locations->InAt(0)); + FPRegister rhs = DRegisterFrom(locations->InAt(1)); + FPRegister dst = DRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + case Primitive::kPrimFloat: + __ Eor(dst.V8B(), lhs.V8B(), rhs.V8B()); // lanes do not matter + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +// Helper to set up locations for vector shift operations. +static void CreateVecShiftLocations(ArenaAllocator* arena, HVecBinaryOperation* instruction) { + LocationSummary* locations = new (arena) LocationSummary(instruction); + switch (instruction->GetPackedType()) { + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetInAt(1, Location::ConstantLocation(instruction->InputAt(1)->AsConstant())); + locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void LocationsBuilderARM64::VisitVecShl(HVecShl* instruction) { + CreateVecShiftLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorARM64::VisitVecShl(HVecShl* instruction) { + LocationSummary* locations = instruction->GetLocations(); + FPRegister lhs = DRegisterFrom(locations->InAt(0)); + FPRegister dst = DRegisterFrom(locations->Out()); + int32_t value = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue(); + switch (instruction->GetPackedType()) { + case Primitive::kPrimByte: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ Shl(dst.V8B(), lhs.V8B(), value); + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ Shl(dst.V4H(), lhs.V4H(), value); + break; + case Primitive::kPrimInt: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ Shl(dst.V2S(), lhs.V2S(), value); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void LocationsBuilderARM64::VisitVecShr(HVecShr* instruction) { + CreateVecShiftLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorARM64::VisitVecShr(HVecShr* instruction) { + LocationSummary* locations = instruction->GetLocations(); + FPRegister lhs = DRegisterFrom(locations->InAt(0)); + FPRegister dst = DRegisterFrom(locations->Out()); + int32_t value = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue(); + switch (instruction->GetPackedType()) { + case Primitive::kPrimByte: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ Sshr(dst.V8B(), lhs.V8B(), value); + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ Sshr(dst.V4H(), lhs.V4H(), value); + break; + case Primitive::kPrimInt: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ Sshr(dst.V2S(), lhs.V2S(), value); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void LocationsBuilderARM64::VisitVecUShr(HVecUShr* instruction) { + CreateVecShiftLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorARM64::VisitVecUShr(HVecUShr* instruction) { + LocationSummary* locations = instruction->GetLocations(); + FPRegister lhs = DRegisterFrom(locations->InAt(0)); + FPRegister dst = DRegisterFrom(locations->Out()); + int32_t value = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue(); + switch (instruction->GetPackedType()) { + case Primitive::kPrimByte: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ Ushr(dst.V8B(), lhs.V8B(), value); + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ Ushr(dst.V4H(), lhs.V4H(), value); + break; + case Primitive::kPrimInt: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ Ushr(dst.V2S(), lhs.V2S(), value); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +// Helper to set up locations for vector memory operations. +static void CreateVecMemLocations(ArenaAllocator* arena, + HVecMemoryOperation* instruction, + bool is_load) { + LocationSummary* locations = new (arena) LocationSummary(instruction); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + case Primitive::kPrimFloat: + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); + if (is_load) { + locations->SetOut(Location::RequiresFpuRegister()); + } else { + locations->SetInAt(2, Location::RequiresFpuRegister()); + } + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +// Helper to set up registers and address for vector memory operations. +MemOperand InstructionCodeGeneratorARM64::CreateVecMemRegisters( + HVecMemoryOperation* instruction, + Location* reg_loc, + bool is_load) { + LocationSummary* locations = instruction->GetLocations(); + Register base = InputRegisterAt(instruction, 0); + Location index = locations->InAt(1); + *reg_loc = is_load ? locations->Out() : locations->InAt(2); + + Primitive::Type packed_type = instruction->GetPackedType(); + uint32_t offset = mirror::Array::DataOffset(Primitive::ComponentSize(packed_type)).Uint32Value(); + size_t shift = Primitive::ComponentSizeShift(packed_type); + + UseScratchRegisterScope temps(GetVIXLAssembler()); + Register temp = temps.AcquireSameSizeAs(base); + if (index.IsConstant()) { + offset += Int64ConstantFrom(index) << shift; + __ Add(temp, base, offset); + } else { + if (instruction->InputAt(0)->IsIntermediateAddress()) { + temp = base; + } else { + __ Add(temp, base, offset); + } + __ Add(temp.X(), temp.X(), Operand(XRegisterFrom(index), LSL, shift)); + } + return HeapOperand(temp); +} + +void LocationsBuilderARM64::VisitVecLoad(HVecLoad* instruction) { + CreateVecMemLocations(GetGraph()->GetArena(), instruction, /*is_load*/ true); +} + +void InstructionCodeGeneratorARM64::VisitVecLoad(HVecLoad* instruction) { + Location reg_loc = Location::NoLocation(); + MemOperand mem = CreateVecMemRegisters(instruction, ®_loc, /*is_load*/ true); + FPRegister reg = DRegisterFrom(reg_loc); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ Ld1(reg.V8B(), mem); + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ Ld1(reg.V4H(), mem); + break; + case Primitive::kPrimInt: + case Primitive::kPrimFloat: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ Ld1(reg.V2S(), mem); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void LocationsBuilderARM64::VisitVecStore(HVecStore* instruction) { + CreateVecMemLocations(GetGraph()->GetArena(), instruction, /*is_load*/ false); +} + +void InstructionCodeGeneratorARM64::VisitVecStore(HVecStore* instruction) { + Location reg_loc = Location::NoLocation(); + MemOperand mem = CreateVecMemRegisters(instruction, ®_loc, /*is_load*/ false); + FPRegister reg = DRegisterFrom(reg_loc); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ St1(reg.V8B(), mem); + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ St1(reg.V4H(), mem); + break; + case Primitive::kPrimInt: + case Primitive::kPrimFloat: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ St1(reg.V2S(), mem); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +#undef __ + +} // namespace arm64 +} // namespace art diff --git a/compiler/optimizing/code_generator_vector_arm_vixl.cc b/compiler/optimizing/code_generator_vector_arm_vixl.cc new file mode 100644 index 0000000000..171198902d --- /dev/null +++ b/compiler/optimizing/code_generator_vector_arm_vixl.cc @@ -0,0 +1,235 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "code_generator_arm_vixl.h" + +namespace art { +namespace arm { + +// NOLINT on __ macro to suppress wrong warning/fix (misc-macro-parentheses) from clang-tidy. +#define __ reinterpret_cast<ArmVIXLAssembler*>(GetAssembler())->GetVIXLAssembler()-> // NOLINT + +void LocationsBuilderARMVIXL::VisitVecReplicateScalar(HVecReplicateScalar* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void InstructionCodeGeneratorARMVIXL::VisitVecReplicateScalar(HVecReplicateScalar* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderARMVIXL::VisitVecSetScalars(HVecSetScalars* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void InstructionCodeGeneratorARMVIXL::VisitVecSetScalars(HVecSetScalars* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderARMVIXL::VisitVecSumReduce(HVecSumReduce* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void InstructionCodeGeneratorARMVIXL::VisitVecSumReduce(HVecSumReduce* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +// Helper to set up locations for vector unary operations. +static void CreateVecUnOpLocations(ArenaAllocator* arena, HVecUnaryOperation* instruction) { + LocationSummary* locations = new (arena) LocationSummary(instruction); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + case Primitive::kPrimFloat: + case Primitive::kPrimDouble: + DCHECK(locations); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void LocationsBuilderARMVIXL::VisitVecCnv(HVecCnv* instruction) { + CreateVecUnOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorARMVIXL::VisitVecCnv(HVecCnv* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderARMVIXL::VisitVecNeg(HVecNeg* instruction) { + CreateVecUnOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorARMVIXL::VisitVecNeg(HVecNeg* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderARMVIXL::VisitVecNot(HVecNot* instruction) { + CreateVecUnOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorARMVIXL::VisitVecNot(HVecNot* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +// Helper to set up locations for vector binary operations. +static void CreateVecBinOpLocations(ArenaAllocator* arena, HVecBinaryOperation* instruction) { + LocationSummary* locations = new (arena) LocationSummary(instruction); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + case Primitive::kPrimFloat: + case Primitive::kPrimDouble: + DCHECK(locations); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void LocationsBuilderARMVIXL::VisitVecAdd(HVecAdd* instruction) { + CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorARMVIXL::VisitVecAdd(HVecAdd* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderARMVIXL::VisitVecSub(HVecSub* instruction) { + CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorARMVIXL::VisitVecSub(HVecSub* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderARMVIXL::VisitVecMul(HVecMul* instruction) { + CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorARMVIXL::VisitVecMul(HVecMul* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderARMVIXL::VisitVecDiv(HVecDiv* instruction) { + CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorARMVIXL::VisitVecDiv(HVecDiv* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderARMVIXL::VisitVecAnd(HVecAnd* instruction) { + CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorARMVIXL::VisitVecAnd(HVecAnd* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderARMVIXL::VisitVecAndNot(HVecAndNot* instruction) { + CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorARMVIXL::VisitVecAndNot(HVecAndNot* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderARMVIXL::VisitVecOr(HVecOr* instruction) { + CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorARMVIXL::VisitVecOr(HVecOr* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderARMVIXL::VisitVecXor(HVecXor* instruction) { + CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorARMVIXL::VisitVecXor(HVecXor* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +// Helper to set up locations for vector shift operations. +static void CreateVecShiftLocations(ArenaAllocator* arena, HVecBinaryOperation* instruction) { + LocationSummary* locations = new (arena) LocationSummary(instruction); + switch (instruction->GetPackedType()) { + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + case Primitive::kPrimLong: + DCHECK(locations); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void LocationsBuilderARMVIXL::VisitVecShl(HVecShl* instruction) { + CreateVecShiftLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorARMVIXL::VisitVecShl(HVecShl* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderARMVIXL::VisitVecShr(HVecShr* instruction) { + CreateVecShiftLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorARMVIXL::VisitVecShr(HVecShr* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderARMVIXL::VisitVecUShr(HVecUShr* instruction) { + CreateVecShiftLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorARMVIXL::VisitVecUShr(HVecUShr* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderARMVIXL::VisitVecLoad(HVecLoad* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void InstructionCodeGeneratorARMVIXL::VisitVecLoad(HVecLoad* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderARMVIXL::VisitVecStore(HVecStore* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void InstructionCodeGeneratorARMVIXL::VisitVecStore(HVecStore* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +#undef __ + +} // namespace arm +} // namespace art diff --git a/compiler/optimizing/code_generator_vector_mips.cc b/compiler/optimizing/code_generator_vector_mips.cc new file mode 100644 index 0000000000..6f5fe0d2a4 --- /dev/null +++ b/compiler/optimizing/code_generator_vector_mips.cc @@ -0,0 +1,235 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "code_generator_mips.h" + +namespace art { +namespace mips { + +// NOLINT on __ macro to suppress wrong warning/fix (misc-macro-parentheses) from clang-tidy. +#define __ down_cast<MipsAssembler*>(GetAssembler())-> // NOLINT + +void LocationsBuilderMIPS::VisitVecReplicateScalar(HVecReplicateScalar* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void InstructionCodeGeneratorMIPS::VisitVecReplicateScalar(HVecReplicateScalar* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderMIPS::VisitVecSetScalars(HVecSetScalars* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void InstructionCodeGeneratorMIPS::VisitVecSetScalars(HVecSetScalars* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderMIPS::VisitVecSumReduce(HVecSumReduce* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void InstructionCodeGeneratorMIPS::VisitVecSumReduce(HVecSumReduce* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +// Helper to set up locations for vector unary operations. +static void CreateVecUnOpLocations(ArenaAllocator* arena, HVecUnaryOperation* instruction) { + LocationSummary* locations = new (arena) LocationSummary(instruction); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + case Primitive::kPrimFloat: + case Primitive::kPrimDouble: + DCHECK(locations); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void LocationsBuilderMIPS::VisitVecCnv(HVecCnv* instruction) { + CreateVecUnOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorMIPS::VisitVecCnv(HVecCnv* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderMIPS::VisitVecNeg(HVecNeg* instruction) { + CreateVecUnOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorMIPS::VisitVecNeg(HVecNeg* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderMIPS::VisitVecNot(HVecNot* instruction) { + CreateVecUnOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorMIPS::VisitVecNot(HVecNot* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +// Helper to set up locations for vector binary operations. +static void CreateVecBinOpLocations(ArenaAllocator* arena, HVecBinaryOperation* instruction) { + LocationSummary* locations = new (arena) LocationSummary(instruction); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + case Primitive::kPrimFloat: + case Primitive::kPrimDouble: + DCHECK(locations); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void LocationsBuilderMIPS::VisitVecAdd(HVecAdd* instruction) { + CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorMIPS::VisitVecAdd(HVecAdd* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderMIPS::VisitVecSub(HVecSub* instruction) { + CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorMIPS::VisitVecSub(HVecSub* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderMIPS::VisitVecMul(HVecMul* instruction) { + CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorMIPS::VisitVecMul(HVecMul* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderMIPS::VisitVecDiv(HVecDiv* instruction) { + CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorMIPS::VisitVecDiv(HVecDiv* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderMIPS::VisitVecAnd(HVecAnd* instruction) { + CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorMIPS::VisitVecAnd(HVecAnd* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderMIPS::VisitVecAndNot(HVecAndNot* instruction) { + CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorMIPS::VisitVecAndNot(HVecAndNot* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderMIPS::VisitVecOr(HVecOr* instruction) { + CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorMIPS::VisitVecOr(HVecOr* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderMIPS::VisitVecXor(HVecXor* instruction) { + CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorMIPS::VisitVecXor(HVecXor* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +// Helper to set up locations for vector shift operations. +static void CreateVecShiftLocations(ArenaAllocator* arena, HVecBinaryOperation* instruction) { + LocationSummary* locations = new (arena) LocationSummary(instruction); + switch (instruction->GetPackedType()) { + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + case Primitive::kPrimLong: + DCHECK(locations); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void LocationsBuilderMIPS::VisitVecShl(HVecShl* instruction) { + CreateVecShiftLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorMIPS::VisitVecShl(HVecShl* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderMIPS::VisitVecShr(HVecShr* instruction) { + CreateVecShiftLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorMIPS::VisitVecShr(HVecShr* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderMIPS::VisitVecUShr(HVecUShr* instruction) { + CreateVecShiftLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorMIPS::VisitVecUShr(HVecUShr* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderMIPS::VisitVecLoad(HVecLoad* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void InstructionCodeGeneratorMIPS::VisitVecLoad(HVecLoad* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderMIPS::VisitVecStore(HVecStore* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void InstructionCodeGeneratorMIPS::VisitVecStore(HVecStore* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +#undef __ + +} // namespace mips +} // namespace art diff --git a/compiler/optimizing/code_generator_vector_mips64.cc b/compiler/optimizing/code_generator_vector_mips64.cc new file mode 100644 index 0000000000..2ee7ac91cf --- /dev/null +++ b/compiler/optimizing/code_generator_vector_mips64.cc @@ -0,0 +1,235 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "code_generator_mips64.h" + +namespace art { +namespace mips64 { + +// NOLINT on __ macro to suppress wrong warning/fix (misc-macro-parentheses) from clang-tidy. +#define __ down_cast<Mips64Assembler*>(GetAssembler())-> // NOLINT + +void LocationsBuilderMIPS64::VisitVecReplicateScalar(HVecReplicateScalar* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void InstructionCodeGeneratorMIPS64::VisitVecReplicateScalar(HVecReplicateScalar* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderMIPS64::VisitVecSetScalars(HVecSetScalars* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void InstructionCodeGeneratorMIPS64::VisitVecSetScalars(HVecSetScalars* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderMIPS64::VisitVecSumReduce(HVecSumReduce* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void InstructionCodeGeneratorMIPS64::VisitVecSumReduce(HVecSumReduce* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +// Helper to set up locations for vector unary operations. +static void CreateVecUnOpLocations(ArenaAllocator* arena, HVecUnaryOperation* instruction) { + LocationSummary* locations = new (arena) LocationSummary(instruction); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + case Primitive::kPrimFloat: + case Primitive::kPrimDouble: + DCHECK(locations); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void LocationsBuilderMIPS64::VisitVecCnv(HVecCnv* instruction) { + CreateVecUnOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorMIPS64::VisitVecCnv(HVecCnv* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderMIPS64::VisitVecNeg(HVecNeg* instruction) { + CreateVecUnOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorMIPS64::VisitVecNeg(HVecNeg* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderMIPS64::VisitVecNot(HVecNot* instruction) { + CreateVecUnOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorMIPS64::VisitVecNot(HVecNot* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +// Helper to set up locations for vector binary operations. +static void CreateVecBinOpLocations(ArenaAllocator* arena, HVecBinaryOperation* instruction) { + LocationSummary* locations = new (arena) LocationSummary(instruction); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + case Primitive::kPrimFloat: + case Primitive::kPrimDouble: + DCHECK(locations); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void LocationsBuilderMIPS64::VisitVecAdd(HVecAdd* instruction) { + CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorMIPS64::VisitVecAdd(HVecAdd* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderMIPS64::VisitVecSub(HVecSub* instruction) { + CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorMIPS64::VisitVecSub(HVecSub* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderMIPS64::VisitVecMul(HVecMul* instruction) { + CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorMIPS64::VisitVecMul(HVecMul* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderMIPS64::VisitVecDiv(HVecDiv* instruction) { + CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorMIPS64::VisitVecDiv(HVecDiv* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderMIPS64::VisitVecAnd(HVecAnd* instruction) { + CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorMIPS64::VisitVecAnd(HVecAnd* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderMIPS64::VisitVecAndNot(HVecAndNot* instruction) { + CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorMIPS64::VisitVecAndNot(HVecAndNot* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderMIPS64::VisitVecOr(HVecOr* instruction) { + CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorMIPS64::VisitVecOr(HVecOr* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderMIPS64::VisitVecXor(HVecXor* instruction) { + CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorMIPS64::VisitVecXor(HVecXor* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +// Helper to set up locations for vector shift operations. +static void CreateVecShiftLocations(ArenaAllocator* arena, HVecBinaryOperation* instruction) { + LocationSummary* locations = new (arena) LocationSummary(instruction); + switch (instruction->GetPackedType()) { + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + case Primitive::kPrimLong: + DCHECK(locations); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void LocationsBuilderMIPS64::VisitVecShl(HVecShl* instruction) { + CreateVecShiftLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorMIPS64::VisitVecShl(HVecShl* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderMIPS64::VisitVecShr(HVecShr* instruction) { + CreateVecShiftLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorMIPS64::VisitVecShr(HVecShr* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderMIPS64::VisitVecUShr(HVecUShr* instruction) { + CreateVecShiftLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorMIPS64::VisitVecUShr(HVecUShr* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderMIPS64::VisitVecLoad(HVecLoad* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void InstructionCodeGeneratorMIPS64::VisitVecLoad(HVecLoad* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderMIPS64::VisitVecStore(HVecStore* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void InstructionCodeGeneratorMIPS64::VisitVecStore(HVecStore* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +#undef __ + +} // namespace mips64 +} // namespace art diff --git a/compiler/optimizing/code_generator_vector_x86.cc b/compiler/optimizing/code_generator_vector_x86.cc new file mode 100644 index 0000000000..4f3988ee2e --- /dev/null +++ b/compiler/optimizing/code_generator_vector_x86.cc @@ -0,0 +1,767 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "code_generator_x86.h" +#include "mirror/array-inl.h" + +namespace art { +namespace x86 { + +// NOLINT on __ macro to suppress wrong warning/fix (misc-macro-parentheses) from clang-tidy. +#define __ down_cast<X86Assembler*>(GetAssembler())-> // NOLINT + +void LocationsBuilderX86::VisitVecReplicateScalar(HVecReplicateScalar* instruction) { + LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction); + switch (instruction->GetPackedType()) { + case Primitive::kPrimLong: + // Long needs extra temporary to load the register pair. + locations->AddTemp(Location::RequiresFpuRegister()); + FALLTHROUGH_INTENDED; + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetOut(Location::RequiresFpuRegister()); + break; + case Primitive::kPrimFloat: + case Primitive::kPrimDouble: + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetOut(Location::SameAsFirstInput()); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void InstructionCodeGeneratorX86::VisitVecReplicateScalar(HVecReplicateScalar* instruction) { + LocationSummary* locations = instruction->GetLocations(); + XmmRegister reg = locations->Out().AsFpuRegister<XmmRegister>(); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + DCHECK_EQ(16u, instruction->GetVectorLength()); + __ movd(reg, locations->InAt(0).AsRegister<Register>()); + __ punpcklbw(reg, reg); + __ punpcklwd(reg, reg); + __ pshufd(reg, reg, Immediate(0)); + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ movd(reg, locations->InAt(0).AsRegister<Register>()); + __ punpcklwd(reg, reg); + __ pshufd(reg, reg, Immediate(0)); + break; + case Primitive::kPrimInt: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ movd(reg, locations->InAt(0).AsRegister<Register>()); + __ pshufd(reg, reg, Immediate(0)); + break; + case Primitive::kPrimLong: { + XmmRegister tmp = locations->GetTemp(0).AsFpuRegister<XmmRegister>(); + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ movd(reg, locations->InAt(0).AsRegisterPairLow<Register>()); + __ movd(tmp, locations->InAt(0).AsRegisterPairHigh<Register>()); + __ punpckldq(reg, tmp); + __ punpcklqdq(reg, reg); + break; + } + case Primitive::kPrimFloat: + DCHECK(locations->InAt(0).Equals(locations->Out())); + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ shufps(reg, reg, Immediate(0)); + break; + case Primitive::kPrimDouble: + DCHECK(locations->InAt(0).Equals(locations->Out())); + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ shufpd(reg, reg, Immediate(0)); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void LocationsBuilderX86::VisitVecSetScalars(HVecSetScalars* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void InstructionCodeGeneratorX86::VisitVecSetScalars(HVecSetScalars* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderX86::VisitVecSumReduce(HVecSumReduce* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void InstructionCodeGeneratorX86::VisitVecSumReduce(HVecSumReduce* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +// Helper to set up locations for vector unary operations. +static void CreateVecUnOpLocations(ArenaAllocator* arena, HVecUnaryOperation* instruction) { + LocationSummary* locations = new (arena) LocationSummary(instruction); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + case Primitive::kPrimLong: + case Primitive::kPrimFloat: + case Primitive::kPrimDouble: + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetOut(Location::RequiresFpuRegister()); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void LocationsBuilderX86::VisitVecCnv(HVecCnv* instruction) { + CreateVecUnOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorX86::VisitVecCnv(HVecCnv* instruction) { + LocationSummary* locations = instruction->GetLocations(); + XmmRegister src = locations->InAt(0).AsFpuRegister<XmmRegister>(); + XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); + Primitive::Type from = instruction->GetInputType(); + Primitive::Type to = instruction->GetResultType(); + if (from == Primitive::kPrimInt && to == Primitive::kPrimFloat) { + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ cvtdq2ps(dst, src); + } else { + LOG(FATAL) << "Unsupported SIMD type"; + } +} + +void LocationsBuilderX86::VisitVecNeg(HVecNeg* instruction) { + CreateVecUnOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorX86::VisitVecNeg(HVecNeg* instruction) { + LocationSummary* locations = instruction->GetLocations(); + XmmRegister src = locations->InAt(0).AsFpuRegister<XmmRegister>(); + XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); + switch (instruction->GetPackedType()) { + case Primitive::kPrimByte: + DCHECK_EQ(16u, instruction->GetVectorLength()); + __ pxor(dst, dst); + __ psubb(dst, src); + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ pxor(dst, dst); + __ psubw(dst, src); + break; + case Primitive::kPrimInt: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ pxor(dst, dst); + __ psubd(dst, src); + break; + case Primitive::kPrimLong: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ pxor(dst, dst); + __ psubq(dst, src); + break; + case Primitive::kPrimFloat: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ xorps(dst, dst); + __ subps(dst, src); + break; + case Primitive::kPrimDouble: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ xorpd(dst, dst); + __ subpd(dst, src); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void LocationsBuilderX86::VisitVecNot(HVecNot* instruction) { + CreateVecUnOpLocations(GetGraph()->GetArena(), instruction); + // Boolean-not requires a temporary to construct the 16 x one. + if (instruction->GetPackedType() == Primitive::kPrimBoolean) { + instruction->GetLocations()->AddTemp(Location::RequiresFpuRegister()); + } +} + +void InstructionCodeGeneratorX86::VisitVecNot(HVecNot* instruction) { + LocationSummary* locations = instruction->GetLocations(); + XmmRegister src = locations->InAt(0).AsFpuRegister<XmmRegister>(); + XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: { // special case boolean-not + DCHECK_EQ(16u, instruction->GetVectorLength()); + XmmRegister tmp = locations->GetTemp(0).AsFpuRegister<XmmRegister>(); + __ pxor(dst, dst); + __ pcmpeqb(tmp, tmp); // all ones + __ psubb(dst, tmp); // 16 x one + __ pxor(dst, src); + break; + } + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + case Primitive::kPrimLong: + DCHECK_LE(2u, instruction->GetVectorLength()); + DCHECK_LE(instruction->GetVectorLength(), 16u); + __ pcmpeqb(dst, dst); // all ones + __ pxor(dst, src); + break; + case Primitive::kPrimFloat: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ pcmpeqb(dst, dst); // all ones + __ xorps(dst, src); + break; + case Primitive::kPrimDouble: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ pcmpeqb(dst, dst); // all ones + __ xorpd(dst, src); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +// Helper to set up locations for vector binary operations. +static void CreateVecBinOpLocations(ArenaAllocator* arena, HVecBinaryOperation* instruction) { + LocationSummary* locations = new (arena) LocationSummary(instruction); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + case Primitive::kPrimLong: + case Primitive::kPrimFloat: + case Primitive::kPrimDouble: + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetInAt(1, Location::RequiresFpuRegister()); + locations->SetOut(Location::SameAsFirstInput()); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void LocationsBuilderX86::VisitVecAdd(HVecAdd* instruction) { + CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorX86::VisitVecAdd(HVecAdd* instruction) { + LocationSummary* locations = instruction->GetLocations(); + DCHECK(locations->InAt(0).Equals(locations->Out())); + XmmRegister src = locations->InAt(1).AsFpuRegister<XmmRegister>(); + XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); + switch (instruction->GetPackedType()) { + case Primitive::kPrimByte: + DCHECK_EQ(16u, instruction->GetVectorLength()); + __ paddb(dst, src); + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ paddw(dst, src); + break; + case Primitive::kPrimInt: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ paddd(dst, src); + break; + case Primitive::kPrimLong: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ paddq(dst, src); + break; + case Primitive::kPrimFloat: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ addps(dst, src); + break; + case Primitive::kPrimDouble: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ addpd(dst, src); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void LocationsBuilderX86::VisitVecSub(HVecSub* instruction) { + CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorX86::VisitVecSub(HVecSub* instruction) { + LocationSummary* locations = instruction->GetLocations(); + DCHECK(locations->InAt(0).Equals(locations->Out())); + XmmRegister src = locations->InAt(1).AsFpuRegister<XmmRegister>(); + XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); + switch (instruction->GetPackedType()) { + case Primitive::kPrimByte: + DCHECK_EQ(16u, instruction->GetVectorLength()); + __ psubb(dst, src); + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ psubw(dst, src); + break; + case Primitive::kPrimInt: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ psubd(dst, src); + break; + case Primitive::kPrimLong: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ psubq(dst, src); + break; + case Primitive::kPrimFloat: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ subps(dst, src); + break; + case Primitive::kPrimDouble: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ subpd(dst, src); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void LocationsBuilderX86::VisitVecMul(HVecMul* instruction) { + CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorX86::VisitVecMul(HVecMul* instruction) { + LocationSummary* locations = instruction->GetLocations(); + DCHECK(locations->InAt(0).Equals(locations->Out())); + XmmRegister src = locations->InAt(1).AsFpuRegister<XmmRegister>(); + XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); + switch (instruction->GetPackedType()) { + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ pmullw(dst, src); + break; + case Primitive::kPrimInt: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ pmulld(dst, src); + break; + case Primitive::kPrimFloat: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ mulps(dst, src); + break; + case Primitive::kPrimDouble: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ mulpd(dst, src); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void LocationsBuilderX86::VisitVecDiv(HVecDiv* instruction) { + CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorX86::VisitVecDiv(HVecDiv* instruction) { + LocationSummary* locations = instruction->GetLocations(); + DCHECK(locations->InAt(0).Equals(locations->Out())); + XmmRegister src = locations->InAt(1).AsFpuRegister<XmmRegister>(); + XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); + switch (instruction->GetPackedType()) { + case Primitive::kPrimFloat: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ divps(dst, src); + break; + case Primitive::kPrimDouble: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ divpd(dst, src); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void LocationsBuilderX86::VisitVecAnd(HVecAnd* instruction) { + CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorX86::VisitVecAnd(HVecAnd* instruction) { + LocationSummary* locations = instruction->GetLocations(); + DCHECK(locations->InAt(0).Equals(locations->Out())); + XmmRegister src = locations->InAt(1).AsFpuRegister<XmmRegister>(); + XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + case Primitive::kPrimLong: + DCHECK_LE(2u, instruction->GetVectorLength()); + DCHECK_LE(instruction->GetVectorLength(), 16u); + __ pand(dst, src); + break; + case Primitive::kPrimFloat: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ andps(dst, src); + break; + case Primitive::kPrimDouble: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ andpd(dst, src); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void LocationsBuilderX86::VisitVecAndNot(HVecAndNot* instruction) { + CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorX86::VisitVecAndNot(HVecAndNot* instruction) { + LocationSummary* locations = instruction->GetLocations(); + DCHECK(locations->InAt(0).Equals(locations->Out())); + XmmRegister src = locations->InAt(1).AsFpuRegister<XmmRegister>(); + XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + case Primitive::kPrimLong: + DCHECK_LE(2u, instruction->GetVectorLength()); + DCHECK_LE(instruction->GetVectorLength(), 16u); + __ pandn(dst, src); + break; + case Primitive::kPrimFloat: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ andnps(dst, src); + break; + case Primitive::kPrimDouble: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ andnpd(dst, src); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void LocationsBuilderX86::VisitVecOr(HVecOr* instruction) { + CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorX86::VisitVecOr(HVecOr* instruction) { + LocationSummary* locations = instruction->GetLocations(); + DCHECK(locations->InAt(0).Equals(locations->Out())); + XmmRegister src = locations->InAt(1).AsFpuRegister<XmmRegister>(); + XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + case Primitive::kPrimLong: + DCHECK_LE(2u, instruction->GetVectorLength()); + DCHECK_LE(instruction->GetVectorLength(), 16u); + __ por(dst, src); + break; + case Primitive::kPrimFloat: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ orps(dst, src); + break; + case Primitive::kPrimDouble: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ orpd(dst, src); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void LocationsBuilderX86::VisitVecXor(HVecXor* instruction) { + CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorX86::VisitVecXor(HVecXor* instruction) { + LocationSummary* locations = instruction->GetLocations(); + DCHECK(locations->InAt(0).Equals(locations->Out())); + XmmRegister src = locations->InAt(1).AsFpuRegister<XmmRegister>(); + XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + case Primitive::kPrimLong: + DCHECK_LE(2u, instruction->GetVectorLength()); + DCHECK_LE(instruction->GetVectorLength(), 16u); + __ pxor(dst, src); + break; + case Primitive::kPrimFloat: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ xorps(dst, src); + break; + case Primitive::kPrimDouble: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ xorpd(dst, src); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +// Helper to set up locations for vector shift operations. +static void CreateVecShiftLocations(ArenaAllocator* arena, HVecBinaryOperation* instruction) { + LocationSummary* locations = new (arena) LocationSummary(instruction); + switch (instruction->GetPackedType()) { + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + case Primitive::kPrimLong: + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetInAt(1, Location::ConstantLocation(instruction->InputAt(1)->AsConstant())); + locations->SetOut(Location::SameAsFirstInput()); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void LocationsBuilderX86::VisitVecShl(HVecShl* instruction) { + CreateVecShiftLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorX86::VisitVecShl(HVecShl* instruction) { + LocationSummary* locations = instruction->GetLocations(); + DCHECK(locations->InAt(0).Equals(locations->Out())); + int32_t value = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue(); + XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); + switch (instruction->GetPackedType()) { + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ psllw(dst, Immediate(static_cast<uint8_t>(value))); + break; + case Primitive::kPrimInt: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ pslld(dst, Immediate(static_cast<uint8_t>(value))); + break; + case Primitive::kPrimLong: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ psllq(dst, Immediate(static_cast<uint8_t>(value))); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void LocationsBuilderX86::VisitVecShr(HVecShr* instruction) { + CreateVecShiftLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorX86::VisitVecShr(HVecShr* instruction) { + LocationSummary* locations = instruction->GetLocations(); + DCHECK(locations->InAt(0).Equals(locations->Out())); + int32_t value = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue(); + XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); + switch (instruction->GetPackedType()) { + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ psraw(dst, Immediate(static_cast<uint8_t>(value))); + break; + case Primitive::kPrimInt: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ psrad(dst, Immediate(static_cast<uint8_t>(value))); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void LocationsBuilderX86::VisitVecUShr(HVecUShr* instruction) { + CreateVecShiftLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorX86::VisitVecUShr(HVecUShr* instruction) { + LocationSummary* locations = instruction->GetLocations(); + DCHECK(locations->InAt(0).Equals(locations->Out())); + int32_t value = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue(); + XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); + switch (instruction->GetPackedType()) { + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ psrlw(dst, Immediate(static_cast<uint8_t>(value))); + break; + case Primitive::kPrimInt: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ psrld(dst, Immediate(static_cast<uint8_t>(value))); + break; + case Primitive::kPrimLong: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ psrlq(dst, Immediate(static_cast<uint8_t>(value))); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +// Helper to set up locations for vector memory operations. +static void CreateVecMemLocations(ArenaAllocator* arena, + HVecMemoryOperation* instruction, + bool is_load) { + LocationSummary* locations = new (arena) LocationSummary(instruction); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + case Primitive::kPrimLong: + case Primitive::kPrimFloat: + case Primitive::kPrimDouble: + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); + if (is_load) { + locations->SetOut(Location::RequiresFpuRegister()); + } else { + locations->SetInAt(2, Location::RequiresFpuRegister()); + } + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +// Helper to set up registers and address for vector memory operations. +static Address CreateVecMemRegisters(HVecMemoryOperation* instruction, + Location* reg_loc, + bool is_load) { + LocationSummary* locations = instruction->GetLocations(); + Location base = locations->InAt(0); + Location index = locations->InAt(1); + *reg_loc = is_load ? locations->Out() : locations->InAt(2); + size_t size = Primitive::ComponentSize(instruction->GetPackedType()); + uint32_t offset = mirror::Array::DataOffset(size).Uint32Value(); + ScaleFactor scale = TIMES_1; + switch (size) { + case 2: scale = TIMES_2; break; + case 4: scale = TIMES_4; break; + case 8: scale = TIMES_8; break; + default: break; + } + return CodeGeneratorX86::ArrayAddress(base.AsRegister<Register>(), index, scale, offset); +} + +void LocationsBuilderX86::VisitVecLoad(HVecLoad* instruction) { + CreateVecMemLocations(GetGraph()->GetArena(), instruction, /*is_load*/ true); +} + +void InstructionCodeGeneratorX86::VisitVecLoad(HVecLoad* instruction) { + Location reg_loc = Location::NoLocation(); + Address address = CreateVecMemRegisters(instruction, ®_loc, /*is_load*/ true); + XmmRegister reg = reg_loc.AsFpuRegister<XmmRegister>(); + bool is_aligned16 = instruction->GetAlignment().IsAlignedAt(16); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + case Primitive::kPrimLong: + DCHECK_LE(2u, instruction->GetVectorLength()); + DCHECK_LE(instruction->GetVectorLength(), 16u); + is_aligned16 ? __ movdqa(reg, address) : __ movdqu(reg, address); + break; + case Primitive::kPrimFloat: + DCHECK_EQ(4u, instruction->GetVectorLength()); + is_aligned16 ? __ movaps(reg, address) : __ movups(reg, address); + break; + case Primitive::kPrimDouble: + DCHECK_EQ(2u, instruction->GetVectorLength()); + is_aligned16 ? __ movapd(reg, address) : __ movupd(reg, address); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void LocationsBuilderX86::VisitVecStore(HVecStore* instruction) { + CreateVecMemLocations(GetGraph()->GetArena(), instruction, /*is_load*/ false); +} + +void InstructionCodeGeneratorX86::VisitVecStore(HVecStore* instruction) { + Location reg_loc = Location::NoLocation(); + Address address = CreateVecMemRegisters(instruction, ®_loc, /*is_load*/ false); + XmmRegister reg = reg_loc.AsFpuRegister<XmmRegister>(); + bool is_aligned16 = instruction->GetAlignment().IsAlignedAt(16); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + case Primitive::kPrimLong: + DCHECK_LE(2u, instruction->GetVectorLength()); + DCHECK_LE(instruction->GetVectorLength(), 16u); + is_aligned16 ? __ movdqa(address, reg) : __ movdqu(address, reg); + break; + case Primitive::kPrimFloat: + DCHECK_EQ(4u, instruction->GetVectorLength()); + is_aligned16 ? __ movaps(address, reg) : __ movups(address, reg); + break; + case Primitive::kPrimDouble: + DCHECK_EQ(2u, instruction->GetVectorLength()); + is_aligned16 ? __ movapd(address, reg) : __ movupd(address, reg); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +#undef __ + +} // namespace x86 +} // namespace art diff --git a/compiler/optimizing/code_generator_vector_x86_64.cc b/compiler/optimizing/code_generator_vector_x86_64.cc new file mode 100644 index 0000000000..b1c1494f6b --- /dev/null +++ b/compiler/optimizing/code_generator_vector_x86_64.cc @@ -0,0 +1,760 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "code_generator_x86_64.h" +#include "mirror/array-inl.h" + +namespace art { +namespace x86_64 { + +// NOLINT on __ macro to suppress wrong warning/fix (misc-macro-parentheses) from clang-tidy. +#define __ down_cast<X86_64Assembler*>(GetAssembler())-> // NOLINT + +void LocationsBuilderX86_64::VisitVecReplicateScalar(HVecReplicateScalar* instruction) { + LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + case Primitive::kPrimLong: + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetOut(Location::RequiresFpuRegister()); + break; + case Primitive::kPrimFloat: + case Primitive::kPrimDouble: + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetOut(Location::SameAsFirstInput()); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void InstructionCodeGeneratorX86_64::VisitVecReplicateScalar(HVecReplicateScalar* instruction) { + LocationSummary* locations = instruction->GetLocations(); + XmmRegister reg = locations->Out().AsFpuRegister<XmmRegister>(); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + DCHECK_EQ(16u, instruction->GetVectorLength()); + __ movd(reg, locations->InAt(0).AsRegister<CpuRegister>()); + __ punpcklbw(reg, reg); + __ punpcklwd(reg, reg); + __ pshufd(reg, reg, Immediate(0)); + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ movd(reg, locations->InAt(0).AsRegister<CpuRegister>()); + __ punpcklwd(reg, reg); + __ pshufd(reg, reg, Immediate(0)); + break; + case Primitive::kPrimInt: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ movd(reg, locations->InAt(0).AsRegister<CpuRegister>()); + __ pshufd(reg, reg, Immediate(0)); + break; + case Primitive::kPrimLong: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ movd(reg, locations->InAt(0).AsRegister<CpuRegister>()); // is 64-bit + __ punpcklqdq(reg, reg); + break; + case Primitive::kPrimFloat: + DCHECK(locations->InAt(0).Equals(locations->Out())); + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ shufps(reg, reg, Immediate(0)); + break; + case Primitive::kPrimDouble: + DCHECK(locations->InAt(0).Equals(locations->Out())); + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ shufpd(reg, reg, Immediate(0)); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void LocationsBuilderX86_64::VisitVecSetScalars(HVecSetScalars* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void InstructionCodeGeneratorX86_64::VisitVecSetScalars(HVecSetScalars* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void LocationsBuilderX86_64::VisitVecSumReduce(HVecSumReduce* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +void InstructionCodeGeneratorX86_64::VisitVecSumReduce(HVecSumReduce* instruction) { + LOG(FATAL) << "No SIMD for " << instruction->GetId(); +} + +// Helper to set up locations for vector unary operations. +static void CreateVecUnOpLocations(ArenaAllocator* arena, HVecUnaryOperation* instruction) { + LocationSummary* locations = new (arena) LocationSummary(instruction); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + case Primitive::kPrimLong: + case Primitive::kPrimFloat: + case Primitive::kPrimDouble: + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetOut(Location::RequiresFpuRegister()); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void LocationsBuilderX86_64::VisitVecCnv(HVecCnv* instruction) { + CreateVecUnOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorX86_64::VisitVecCnv(HVecCnv* instruction) { + LocationSummary* locations = instruction->GetLocations(); + XmmRegister src = locations->InAt(0).AsFpuRegister<XmmRegister>(); + XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); + Primitive::Type from = instruction->GetInputType(); + Primitive::Type to = instruction->GetResultType(); + if (from == Primitive::kPrimInt && to == Primitive::kPrimFloat) { + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ cvtdq2ps(dst, src); + } else { + LOG(FATAL) << "Unsupported SIMD type"; + } +} + +void LocationsBuilderX86_64::VisitVecNeg(HVecNeg* instruction) { + CreateVecUnOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorX86_64::VisitVecNeg(HVecNeg* instruction) { + LocationSummary* locations = instruction->GetLocations(); + XmmRegister src = locations->InAt(0).AsFpuRegister<XmmRegister>(); + XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); + switch (instruction->GetPackedType()) { + case Primitive::kPrimByte: + DCHECK_EQ(16u, instruction->GetVectorLength()); + __ pxor(dst, dst); + __ psubb(dst, src); + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ pxor(dst, dst); + __ psubw(dst, src); + break; + case Primitive::kPrimInt: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ pxor(dst, dst); + __ psubd(dst, src); + break; + case Primitive::kPrimLong: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ pxor(dst, dst); + __ psubq(dst, src); + break; + case Primitive::kPrimFloat: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ xorps(dst, dst); + __ subps(dst, src); + break; + case Primitive::kPrimDouble: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ xorpd(dst, dst); + __ subpd(dst, src); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void LocationsBuilderX86_64::VisitVecNot(HVecNot* instruction) { + CreateVecUnOpLocations(GetGraph()->GetArena(), instruction); + // Boolean-not requires a temporary to construct the 16 x one. + if (instruction->GetPackedType() == Primitive::kPrimBoolean) { + instruction->GetLocations()->AddTemp(Location::RequiresFpuRegister()); + } +} + +void InstructionCodeGeneratorX86_64::VisitVecNot(HVecNot* instruction) { + LocationSummary* locations = instruction->GetLocations(); + XmmRegister src = locations->InAt(0).AsFpuRegister<XmmRegister>(); + XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: { // special case boolean-not + DCHECK_EQ(16u, instruction->GetVectorLength()); + XmmRegister tmp = locations->GetTemp(0).AsFpuRegister<XmmRegister>(); + __ pxor(dst, dst); + __ pcmpeqb(tmp, tmp); // all ones + __ psubb(dst, tmp); // 16 x one + __ pxor(dst, src); + break; + } + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + case Primitive::kPrimLong: + DCHECK_LE(2u, instruction->GetVectorLength()); + DCHECK_LE(instruction->GetVectorLength(), 16u); + __ pcmpeqb(dst, dst); // all ones + __ pxor(dst, src); + break; + case Primitive::kPrimFloat: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ pcmpeqb(dst, dst); // all ones + __ xorps(dst, src); + break; + case Primitive::kPrimDouble: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ pcmpeqb(dst, dst); // all ones + __ xorpd(dst, src); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +// Helper to set up locations for vector binary operations. +static void CreateVecBinOpLocations(ArenaAllocator* arena, HVecBinaryOperation* instruction) { + LocationSummary* locations = new (arena) LocationSummary(instruction); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + case Primitive::kPrimLong: + case Primitive::kPrimFloat: + case Primitive::kPrimDouble: + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetInAt(1, Location::RequiresFpuRegister()); + locations->SetOut(Location::SameAsFirstInput()); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void LocationsBuilderX86_64::VisitVecAdd(HVecAdd* instruction) { + CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorX86_64::VisitVecAdd(HVecAdd* instruction) { + LocationSummary* locations = instruction->GetLocations(); + DCHECK(locations->InAt(0).Equals(locations->Out())); + XmmRegister src = locations->InAt(1).AsFpuRegister<XmmRegister>(); + XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); + switch (instruction->GetPackedType()) { + case Primitive::kPrimByte: + DCHECK_EQ(16u, instruction->GetVectorLength()); + __ paddb(dst, src); + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ paddw(dst, src); + break; + case Primitive::kPrimInt: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ paddd(dst, src); + break; + case Primitive::kPrimLong: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ paddq(dst, src); + break; + case Primitive::kPrimFloat: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ addps(dst, src); + break; + case Primitive::kPrimDouble: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ addpd(dst, src); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void LocationsBuilderX86_64::VisitVecSub(HVecSub* instruction) { + CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorX86_64::VisitVecSub(HVecSub* instruction) { + LocationSummary* locations = instruction->GetLocations(); + DCHECK(locations->InAt(0).Equals(locations->Out())); + XmmRegister src = locations->InAt(1).AsFpuRegister<XmmRegister>(); + XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); + switch (instruction->GetPackedType()) { + case Primitive::kPrimByte: + DCHECK_EQ(16u, instruction->GetVectorLength()); + __ psubb(dst, src); + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ psubw(dst, src); + break; + case Primitive::kPrimInt: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ psubd(dst, src); + break; + case Primitive::kPrimLong: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ psubq(dst, src); + break; + case Primitive::kPrimFloat: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ subps(dst, src); + break; + case Primitive::kPrimDouble: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ subpd(dst, src); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void LocationsBuilderX86_64::VisitVecMul(HVecMul* instruction) { + CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorX86_64::VisitVecMul(HVecMul* instruction) { + LocationSummary* locations = instruction->GetLocations(); + DCHECK(locations->InAt(0).Equals(locations->Out())); + XmmRegister src = locations->InAt(1).AsFpuRegister<XmmRegister>(); + XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); + switch (instruction->GetPackedType()) { + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ pmullw(dst, src); + break; + case Primitive::kPrimInt: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ pmulld(dst, src); + break; + case Primitive::kPrimFloat: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ mulps(dst, src); + break; + case Primitive::kPrimDouble: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ mulpd(dst, src); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void LocationsBuilderX86_64::VisitVecDiv(HVecDiv* instruction) { + CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorX86_64::VisitVecDiv(HVecDiv* instruction) { + LocationSummary* locations = instruction->GetLocations(); + DCHECK(locations->InAt(0).Equals(locations->Out())); + XmmRegister src = locations->InAt(1).AsFpuRegister<XmmRegister>(); + XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); + switch (instruction->GetPackedType()) { + case Primitive::kPrimFloat: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ divps(dst, src); + break; + case Primitive::kPrimDouble: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ divpd(dst, src); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void LocationsBuilderX86_64::VisitVecAnd(HVecAnd* instruction) { + CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorX86_64::VisitVecAnd(HVecAnd* instruction) { + LocationSummary* locations = instruction->GetLocations(); + DCHECK(locations->InAt(0).Equals(locations->Out())); + XmmRegister src = locations->InAt(1).AsFpuRegister<XmmRegister>(); + XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + case Primitive::kPrimLong: + DCHECK_LE(2u, instruction->GetVectorLength()); + DCHECK_LE(instruction->GetVectorLength(), 16u); + __ pand(dst, src); + break; + case Primitive::kPrimFloat: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ andps(dst, src); + break; + case Primitive::kPrimDouble: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ andpd(dst, src); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void LocationsBuilderX86_64::VisitVecAndNot(HVecAndNot* instruction) { + CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorX86_64::VisitVecAndNot(HVecAndNot* instruction) { + LocationSummary* locations = instruction->GetLocations(); + DCHECK(locations->InAt(0).Equals(locations->Out())); + XmmRegister src = locations->InAt(1).AsFpuRegister<XmmRegister>(); + XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + case Primitive::kPrimLong: + DCHECK_LE(2u, instruction->GetVectorLength()); + DCHECK_LE(instruction->GetVectorLength(), 16u); + __ pandn(dst, src); + break; + case Primitive::kPrimFloat: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ andnps(dst, src); + break; + case Primitive::kPrimDouble: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ andnpd(dst, src); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void LocationsBuilderX86_64::VisitVecOr(HVecOr* instruction) { + CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorX86_64::VisitVecOr(HVecOr* instruction) { + LocationSummary* locations = instruction->GetLocations(); + DCHECK(locations->InAt(0).Equals(locations->Out())); + XmmRegister src = locations->InAt(1).AsFpuRegister<XmmRegister>(); + XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + case Primitive::kPrimLong: + DCHECK_LE(2u, instruction->GetVectorLength()); + DCHECK_LE(instruction->GetVectorLength(), 16u); + __ por(dst, src); + break; + case Primitive::kPrimFloat: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ orps(dst, src); + break; + case Primitive::kPrimDouble: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ orpd(dst, src); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void LocationsBuilderX86_64::VisitVecXor(HVecXor* instruction) { + CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorX86_64::VisitVecXor(HVecXor* instruction) { + LocationSummary* locations = instruction->GetLocations(); + DCHECK(locations->InAt(0).Equals(locations->Out())); + XmmRegister src = locations->InAt(1).AsFpuRegister<XmmRegister>(); + XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + case Primitive::kPrimLong: + DCHECK_LE(2u, instruction->GetVectorLength()); + DCHECK_LE(instruction->GetVectorLength(), 16u); + __ pxor(dst, src); + break; + case Primitive::kPrimFloat: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ xorps(dst, src); + break; + case Primitive::kPrimDouble: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ xorpd(dst, src); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +// Helper to set up locations for vector shift operations. +static void CreateVecShiftLocations(ArenaAllocator* arena, HVecBinaryOperation* instruction) { + LocationSummary* locations = new (arena) LocationSummary(instruction); + switch (instruction->GetPackedType()) { + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + case Primitive::kPrimLong: + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetInAt(1, Location::ConstantLocation(instruction->InputAt(1)->AsConstant())); + locations->SetOut(Location::SameAsFirstInput()); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void LocationsBuilderX86_64::VisitVecShl(HVecShl* instruction) { + CreateVecShiftLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorX86_64::VisitVecShl(HVecShl* instruction) { + LocationSummary* locations = instruction->GetLocations(); + DCHECK(locations->InAt(0).Equals(locations->Out())); + int32_t value = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue(); + XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); + switch (instruction->GetPackedType()) { + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ psllw(dst, Immediate(static_cast<int8_t>(value))); + break; + case Primitive::kPrimInt: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ pslld(dst, Immediate(static_cast<int8_t>(value))); + break; + case Primitive::kPrimLong: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ psllq(dst, Immediate(static_cast<int8_t>(value))); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void LocationsBuilderX86_64::VisitVecShr(HVecShr* instruction) { + CreateVecShiftLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorX86_64::VisitVecShr(HVecShr* instruction) { + LocationSummary* locations = instruction->GetLocations(); + DCHECK(locations->InAt(0).Equals(locations->Out())); + int32_t value = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue(); + XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); + switch (instruction->GetPackedType()) { + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ psraw(dst, Immediate(static_cast<int8_t>(value))); + break; + case Primitive::kPrimInt: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ psrad(dst, Immediate(static_cast<int8_t>(value))); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void LocationsBuilderX86_64::VisitVecUShr(HVecUShr* instruction) { + CreateVecShiftLocations(GetGraph()->GetArena(), instruction); +} + +void InstructionCodeGeneratorX86_64::VisitVecUShr(HVecUShr* instruction) { + LocationSummary* locations = instruction->GetLocations(); + DCHECK(locations->InAt(0).Equals(locations->Out())); + int32_t value = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue(); + XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); + switch (instruction->GetPackedType()) { + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ psrlw(dst, Immediate(static_cast<int8_t>(value))); + break; + case Primitive::kPrimInt: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ psrld(dst, Immediate(static_cast<int8_t>(value))); + break; + case Primitive::kPrimLong: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ psrlq(dst, Immediate(static_cast<int8_t>(value))); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +// Helper to set up locations for vector memory operations. +static void CreateVecMemLocations(ArenaAllocator* arena, + HVecMemoryOperation* instruction, + bool is_load) { + LocationSummary* locations = new (arena) LocationSummary(instruction); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + case Primitive::kPrimLong: + case Primitive::kPrimFloat: + case Primitive::kPrimDouble: + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); + if (is_load) { + locations->SetOut(Location::RequiresFpuRegister()); + } else { + locations->SetInAt(2, Location::RequiresFpuRegister()); + } + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +// Helper to set up registers and address for vector memory operations. +static Address CreateVecMemRegisters(HVecMemoryOperation* instruction, + Location* reg_loc, + bool is_load) { + LocationSummary* locations = instruction->GetLocations(); + Location base = locations->InAt(0); + Location index = locations->InAt(1); + *reg_loc = is_load ? locations->Out() : locations->InAt(2); + size_t size = Primitive::ComponentSize(instruction->GetPackedType()); + uint32_t offset = mirror::Array::DataOffset(size).Uint32Value(); + ScaleFactor scale = TIMES_1; + switch (size) { + case 2: scale = TIMES_2; break; + case 4: scale = TIMES_4; break; + case 8: scale = TIMES_8; break; + default: break; + } + return CodeGeneratorX86_64::ArrayAddress(base.AsRegister<CpuRegister>(), index, scale, offset); +} + +void LocationsBuilderX86_64::VisitVecLoad(HVecLoad* instruction) { + CreateVecMemLocations(GetGraph()->GetArena(), instruction, /*is_load*/ true); +} + +void InstructionCodeGeneratorX86_64::VisitVecLoad(HVecLoad* instruction) { + Location reg_loc = Location::NoLocation(); + Address address = CreateVecMemRegisters(instruction, ®_loc, /*is_load*/ true); + XmmRegister reg = reg_loc.AsFpuRegister<XmmRegister>(); + bool is_aligned16 = instruction->GetAlignment().IsAlignedAt(16); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + case Primitive::kPrimLong: + DCHECK_LE(2u, instruction->GetVectorLength()); + DCHECK_LE(instruction->GetVectorLength(), 16u); + is_aligned16 ? __ movdqa(reg, address) : __ movdqu(reg, address); + break; + case Primitive::kPrimFloat: + DCHECK_EQ(4u, instruction->GetVectorLength()); + is_aligned16 ? __ movaps(reg, address) : __ movups(reg, address); + break; + case Primitive::kPrimDouble: + DCHECK_EQ(2u, instruction->GetVectorLength()); + is_aligned16 ? __ movapd(reg, address) : __ movupd(reg, address); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +void LocationsBuilderX86_64::VisitVecStore(HVecStore* instruction) { + CreateVecMemLocations(GetGraph()->GetArena(), instruction, /*is_load*/ false); +} + +void InstructionCodeGeneratorX86_64::VisitVecStore(HVecStore* instruction) { + Location reg_loc = Location::NoLocation(); + Address address = CreateVecMemRegisters(instruction, ®_loc, /*is_load*/ false); + XmmRegister reg = reg_loc.AsFpuRegister<XmmRegister>(); + bool is_aligned16 = instruction->GetAlignment().IsAlignedAt(16); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + case Primitive::kPrimLong: + DCHECK_LE(2u, instruction->GetVectorLength()); + DCHECK_LE(instruction->GetVectorLength(), 16u); + is_aligned16 ? __ movdqa(address, reg) : __ movdqu(address, reg); + break; + case Primitive::kPrimFloat: + DCHECK_EQ(4u, instruction->GetVectorLength()); + is_aligned16 ? __ movaps(address, reg) : __ movups(address, reg); + break; + case Primitive::kPrimDouble: + DCHECK_EQ(2u, instruction->GetVectorLength()); + is_aligned16 ? __ movapd(address, reg) : __ movupd(address, reg); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +#undef __ + +} // namespace x86_64 +} // namespace art diff --git a/compiler/optimizing/code_generator_x86.cc b/compiler/optimizing/code_generator_x86.cc index 4db4796985..80776e8b78 100644 --- a/compiler/optimizing/code_generator_x86.cc +++ b/compiler/optimizing/code_generator_x86.cc @@ -723,7 +723,7 @@ class ReadBarrierForHeapReferenceSlowPathX86 : public SlowPathCode { instruction_->IsArrayGet() || instruction_->IsInstanceOf() || instruction_->IsCheckCast() || - (instruction_->IsInvokeVirtual()) && instruction_->GetLocations()->Intrinsified()) + (instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified())) << "Unexpected instruction in read barrier for heap reference slow path: " << instruction_->DebugName(); diff --git a/compiler/optimizing/code_generator_x86_64.cc b/compiler/optimizing/code_generator_x86_64.cc index 2ffc398287..49f099f6a9 100644 --- a/compiler/optimizing/code_generator_x86_64.cc +++ b/compiler/optimizing/code_generator_x86_64.cc @@ -744,7 +744,7 @@ class ReadBarrierForHeapReferenceSlowPathX86_64 : public SlowPathCode { instruction_->IsArrayGet() || instruction_->IsInstanceOf() || instruction_->IsCheckCast() || - (instruction_->IsInvokeVirtual()) && instruction_->GetLocations()->Intrinsified()) + (instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified())) << "Unexpected instruction in read barrier for heap reference slow path: " << instruction_->DebugName(); @@ -3660,7 +3660,7 @@ void InstructionCodeGeneratorX86_64::GenerateDivRemWithAnyConstant(HBinaryOperat void InstructionCodeGeneratorX86_64::GenerateDivRemIntegral(HBinaryOperation* instruction) { DCHECK(instruction->IsDiv() || instruction->IsRem()); Primitive::Type type = instruction->GetResultType(); - DCHECK(type == Primitive::kPrimInt || Primitive::kPrimLong); + DCHECK(type == Primitive::kPrimInt || type == Primitive::kPrimLong); bool is_div = instruction->IsDiv(); LocationSummary* locations = instruction->GetLocations(); diff --git a/compiler/optimizing/codegen_test_utils.h b/compiler/optimizing/codegen_test_utils.h index cd954043f5..31cd204c9f 100644 --- a/compiler/optimizing/codegen_test_utils.h +++ b/compiler/optimizing/codegen_test_utils.h @@ -74,7 +74,6 @@ class CodegenTargetConfig { } private: - CodegenTargetConfig() {} InstructionSet isa_; CreateCodegenFn create_codegen_; }; diff --git a/compiler/optimizing/common_arm.h b/compiler/optimizing/common_arm.h index e184745520..01304ac35b 100644 --- a/compiler/optimizing/common_arm.h +++ b/compiler/optimizing/common_arm.h @@ -66,6 +66,11 @@ inline vixl::aarch32::SRegister LowSRegisterFrom(Location location) { return vixl::aarch32::SRegister(location.AsFpuRegisterPairLow<vixl::aarch32::SRegister>()); } +inline vixl::aarch32::SRegister HighSRegisterFrom(Location location) { + DCHECK(location.IsFpuRegisterPair()) << location; + return vixl::aarch32::SRegister(location.AsFpuRegisterPairHigh<vixl::aarch32::SRegister>()); +} + inline vixl::aarch32::Register RegisterFrom(Location location) { DCHECK(location.IsRegister()) << location; return vixl::aarch32::Register(location.reg()); diff --git a/compiler/optimizing/graph_visualizer.cc b/compiler/optimizing/graph_visualizer.cc index 0dfae11465..cc3c143b15 100644 --- a/compiler/optimizing/graph_visualizer.cc +++ b/compiler/optimizing/graph_visualizer.cc @@ -505,6 +505,10 @@ class HGraphVisualizerPrinter : public HGraphDelegateVisitor { StartAttributeStream("kind") << (try_boundary->IsEntry() ? "entry" : "exit"); } + void VisitDeoptimize(HDeoptimize* deoptimize) OVERRIDE { + StartAttributeStream("kind") << deoptimize->GetKind(); + } + #if defined(ART_ENABLE_CODEGEN_arm) || defined(ART_ENABLE_CODEGEN_arm64) void VisitMultiplyAccumulate(HMultiplyAccumulate* instruction) OVERRIDE { StartAttributeStream("kind") << instruction->GetOpKind(); diff --git a/compiler/optimizing/induction_var_analysis_test.cc b/compiler/optimizing/induction_var_analysis_test.cc index 82ee93d5c2..9516ccb385 100644 --- a/compiler/optimizing/induction_var_analysis_test.cc +++ b/compiler/optimizing/induction_var_analysis_test.cc @@ -29,7 +29,21 @@ namespace art { */ class InductionVarAnalysisTest : public CommonCompilerTest { public: - InductionVarAnalysisTest() : pool_(), allocator_(&pool_) { + InductionVarAnalysisTest() + : pool_(), + allocator_(&pool_), + iva_(nullptr), + entry_(nullptr), + return_(nullptr), + exit_(nullptr), + parameter_(nullptr), + constant0_(nullptr), + constant1_(nullptr), + constant2_(nullptr), + constant7_(nullptr), + constant100_(nullptr), + constantm1_(nullptr), + float_constant0_(nullptr) { graph_ = CreateGraph(&allocator_); } diff --git a/compiler/optimizing/inliner.cc b/compiler/optimizing/inliner.cc index f7331452c6..79cd7048a5 100644 --- a/compiler/optimizing/inliner.cc +++ b/compiler/optimizing/inliner.cc @@ -63,7 +63,7 @@ static constexpr size_t kMaximumNumberOfCumulatedDexRegisters = 64; static constexpr size_t kMaximumNumberOfRecursiveCalls = 4; // Controls the use of inline caches in AOT mode. -static constexpr bool kUseAOTInlineCaches = false; +static constexpr bool kUseAOTInlineCaches = true; // We check for line numbers to make sure the DepthString implementation // aligns the output nicely. @@ -672,6 +672,32 @@ HInstanceFieldGet* HInliner::BuildGetReceiverClass(ClassLinker* class_linker, return result; } +static ArtMethod* ResolveMethodFromInlineCache(Handle<mirror::Class> klass, + ArtMethod* resolved_method, + HInstruction* invoke_instruction, + PointerSize pointer_size) + REQUIRES_SHARED(Locks::mutator_lock_) { + if (Runtime::Current()->IsAotCompiler()) { + // We can get unrelated types when working with profiles (corruption, + // systme updates, or anyone can write to it). So first check if the class + // actually implements the declaring class of the method that is being + // called in bytecode. + // Note: the lookup methods used below require to have assignable types. + if (!resolved_method->GetDeclaringClass()->IsAssignableFrom(klass.Get())) { + return nullptr; + } + } + + if (invoke_instruction->IsInvokeInterface()) { + resolved_method = klass->FindVirtualMethodForInterface(resolved_method, pointer_size); + } else { + DCHECK(invoke_instruction->IsInvokeVirtual()); + resolved_method = klass->FindVirtualMethodForVirtual(resolved_method, pointer_size); + } + DCHECK(resolved_method != nullptr); + return resolved_method; +} + bool HInliner::TryInlineMonomorphicCall(HInvoke* invoke_instruction, ArtMethod* resolved_method, Handle<mirror::ObjectArray<mirror::Class>> classes) { @@ -690,20 +716,20 @@ bool HInliner::TryInlineMonomorphicCall(HInvoke* invoke_instruction, ClassLinker* class_linker = caller_compilation_unit_.GetClassLinker(); PointerSize pointer_size = class_linker->GetImagePointerSize(); - if (invoke_instruction->IsInvokeInterface()) { - resolved_method = GetMonomorphicType(classes)->FindVirtualMethodForInterface( - resolved_method, pointer_size); - } else { - DCHECK(invoke_instruction->IsInvokeVirtual()); - resolved_method = GetMonomorphicType(classes)->FindVirtualMethodForVirtual( - resolved_method, pointer_size); - } + Handle<mirror::Class> monomorphic_type = handles_->NewHandle(GetMonomorphicType(classes)); + resolved_method = ResolveMethodFromInlineCache( + monomorphic_type, resolved_method, invoke_instruction, pointer_size); + LOG_NOTE() << "Try inline monomorphic call to " << resolved_method->PrettyMethod(); - DCHECK(resolved_method != nullptr); + if (resolved_method == nullptr) { + // Bogus AOT profile, bail. + DCHECK(Runtime::Current()->IsAotCompiler()); + return false; + } + HInstruction* receiver = invoke_instruction->InputAt(0); HInstruction* cursor = invoke_instruction->GetPrevious(); HBasicBlock* bb_cursor = invoke_instruction->GetBlock(); - Handle<mirror::Class> monomorphic_type = handles_->NewHandle(GetMonomorphicType(classes)); if (!TryInlineAndReplace(invoke_instruction, resolved_method, ReferenceTypeInfo::Create(monomorphic_type, /* is_exact */ true), @@ -742,7 +768,8 @@ void HInliner::AddCHAGuard(HInstruction* invoke_instruction, HShouldDeoptimizeFlag(graph_->GetArena(), dex_pc); HInstruction* compare = new (graph_->GetArena()) HNotEqual( deopt_flag, graph_->GetIntConstant(0, dex_pc)); - HInstruction* deopt = new (graph_->GetArena()) HDeoptimize(compare, dex_pc); + HInstruction* deopt = new (graph_->GetArena()) HDeoptimize( + graph_->GetArena(), compare, HDeoptimize::Kind::kInline, dex_pc); if (cursor != nullptr) { bb_cursor->InsertInstructionAfter(deopt_flag, cursor); @@ -806,9 +833,16 @@ HInstruction* HInliner::AddTypeGuard(HInstruction* receiver, bb_cursor->InsertInstructionAfter(compare, load_class); if (with_deoptimization) { HDeoptimize* deoptimize = new (graph_->GetArena()) HDeoptimize( - compare, invoke_instruction->GetDexPc()); + graph_->GetArena(), + compare, + receiver, + HDeoptimize::Kind::kInline, + invoke_instruction->GetDexPc()); bb_cursor->InsertInstructionAfter(deoptimize, compare); deoptimize->CopyEnvironmentFrom(invoke_instruction->GetEnvironment()); + DCHECK_EQ(invoke_instruction->InputAt(0), receiver); + receiver->ReplaceUsesDominatedBy(deoptimize, deoptimize); + deoptimize->SetReferenceTypeInfo(receiver->GetReferenceTypeInfo()); } return compare; } @@ -835,11 +869,14 @@ bool HInliner::TryInlinePolymorphicCall(HInvoke* invoke_instruction, ArtMethod* method = nullptr; Handle<mirror::Class> handle = handles_->NewHandle(classes->Get(i)); - if (invoke_instruction->IsInvokeInterface()) { - method = handle->FindVirtualMethodForInterface(resolved_method, pointer_size); - } else { - DCHECK(invoke_instruction->IsInvokeVirtual()); - method = handle->FindVirtualMethodForVirtual(resolved_method, pointer_size); + method = ResolveMethodFromInlineCache( + handle, resolved_method, invoke_instruction, pointer_size); + if (method == nullptr) { + DCHECK(Runtime::Current()->IsAotCompiler()); + // AOT profile is bogus. This loop expects to iterate over all entries, + // so just just continue. + all_targets_inlined = false; + continue; } HInstruction* receiver = invoke_instruction->InputAt(0); @@ -884,7 +921,7 @@ bool HInliner::TryInlinePolymorphicCall(HInvoke* invoke_instruction, } invoke_instruction->GetBlock()->RemoveInstruction(invoke_instruction); // Because the inline cache data can be populated concurrently, we force the end of the - // iteration. Otherhwise, we could see a new receiver type. + // iteration. Otherwise, we could see a new receiver type. break; } else { CreateDiamondPatternForPolymorphicInline(compare, return_replacement, invoke_instruction); @@ -1083,13 +1120,19 @@ bool HInliner::TryInlinePolymorphicCallToSameTarget( CreateDiamondPatternForPolymorphicInline(compare, return_replacement, invoke_instruction); } else { HDeoptimize* deoptimize = new (graph_->GetArena()) HDeoptimize( - compare, invoke_instruction->GetDexPc()); + graph_->GetArena(), + compare, + receiver, + HDeoptimize::Kind::kInline, + invoke_instruction->GetDexPc()); bb_cursor->InsertInstructionAfter(deoptimize, compare); deoptimize->CopyEnvironmentFrom(invoke_instruction->GetEnvironment()); if (return_replacement != nullptr) { invoke_instruction->ReplaceWith(return_replacement); } + receiver->ReplaceUsesDominatedBy(deoptimize, deoptimize); invoke_instruction->GetBlock()->RemoveInstruction(invoke_instruction); + deoptimize->SetReferenceTypeInfo(receiver->GetReferenceTypeInfo()); } // Run type propagation to get the guard typed. diff --git a/compiler/optimizing/instruction_simplifier.cc b/compiler/optimizing/instruction_simplifier.cc index 17421fc364..60790e5b84 100644 --- a/compiler/optimizing/instruction_simplifier.cc +++ b/compiler/optimizing/instruction_simplifier.cc @@ -2132,6 +2132,9 @@ void InstructionSimplifierVisitor::VisitDeoptimize(HDeoptimize* deoptimize) { if (cond->IsConstant()) { if (cond->AsIntConstant()->IsFalse()) { // Never deopt: instruction can be removed. + if (deoptimize->GuardsAnInput()) { + deoptimize->ReplaceWith(deoptimize->GuardedInput()); + } deoptimize->GetBlock()->RemoveInstruction(deoptimize); } else { // Always deopt. diff --git a/compiler/optimizing/intrinsics_arm_vixl.cc b/compiler/optimizing/intrinsics_arm_vixl.cc index b25bad7170..0d933eaf82 100644 --- a/compiler/optimizing/intrinsics_arm_vixl.cc +++ b/compiler/optimizing/intrinsics_arm_vixl.cc @@ -39,6 +39,7 @@ using helpers::Int32ConstantFrom; using helpers::LocationFrom; using helpers::LowRegisterFrom; using helpers::LowSRegisterFrom; +using helpers::HighSRegisterFrom; using helpers::OutputDRegister; using helpers::OutputSRegister; using helpers::OutputRegister; @@ -794,6 +795,58 @@ void IntrinsicCodeGeneratorARMVIXL::VisitMathRint(HInvoke* invoke) { __ Vrintn(F64, F64, OutputDRegister(invoke), InputDRegisterAt(invoke, 0)); } +void IntrinsicLocationsBuilderARMVIXL::VisitMathRoundFloat(HInvoke* invoke) { + if (features_.HasARMv8AInstructions()) { + LocationSummary* locations = new (arena_) LocationSummary(invoke, + LocationSummary::kNoCall, + kIntrinsified); + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetOut(Location::RequiresRegister()); + locations->AddTemp(Location::RequiresFpuRegister()); + } +} + +void IntrinsicCodeGeneratorARMVIXL::VisitMathRoundFloat(HInvoke* invoke) { + DCHECK(codegen_->GetInstructionSetFeatures().HasARMv8AInstructions()); + + ArmVIXLAssembler* assembler = GetAssembler(); + vixl32::SRegister in_reg = InputSRegisterAt(invoke, 0); + vixl32::Register out_reg = OutputRegister(invoke); + vixl32::SRegister temp1 = LowSRegisterFrom(invoke->GetLocations()->GetTemp(0)); + vixl32::SRegister temp2 = HighSRegisterFrom(invoke->GetLocations()->GetTemp(0)); + vixl32::Label done; + vixl32::Label* final_label = codegen_->GetFinalLabel(invoke, &done); + + // Round to nearest integer, ties away from zero. + __ Vcvta(S32, F32, temp1, in_reg); + __ Vmov(out_reg, temp1); + + // For positive, zero or NaN inputs, rounding is done. + __ Cmp(out_reg, 0); + __ B(ge, final_label, /* far_target */ false); + + // Handle input < 0 cases. + // If input is negative but not a tie, previous result (round to nearest) is valid. + // If input is a negative tie, change rounding direction to positive infinity, out_reg += 1. + __ Vrinta(F32, F32, temp1, in_reg); + __ Vmov(temp2, 0.5); + __ Vsub(F32, temp1, in_reg, temp1); + __ Vcmp(F32, temp1, temp2); + __ Vmrs(RegisterOrAPSR_nzcv(kPcCode), FPSCR); + { + // Use ExactAsemblyScope here because we are using IT. + ExactAssemblyScope it_scope(assembler->GetVIXLAssembler(), + 2 * kMaxInstructionSizeInBytes, + CodeBufferCheckScope::kMaximumSize); + __ it(eq); + __ add(eq, out_reg, out_reg, 1); + } + + if (done.IsReferenced()) { + __ Bind(&done); + } +} + void IntrinsicLocationsBuilderARMVIXL::VisitMemoryPeekByte(HInvoke* invoke) { CreateIntToIntLocations(arena_, invoke); } @@ -3100,7 +3153,6 @@ void IntrinsicCodeGeneratorARMVIXL::VisitIntegerValueOf(HInvoke* invoke) { } UNIMPLEMENTED_INTRINSIC(ARMVIXL, MathRoundDouble) // Could be done by changing rounding mode, maybe? -UNIMPLEMENTED_INTRINSIC(ARMVIXL, MathRoundFloat) // Could be done by changing rounding mode, maybe? UNIMPLEMENTED_INTRINSIC(ARMVIXL, UnsafeCASLong) // High register pressure. UNIMPLEMENTED_INTRINSIC(ARMVIXL, SystemArrayCopyChar) UNIMPLEMENTED_INTRINSIC(ARMVIXL, IntegerHighestOneBit) diff --git a/compiler/optimizing/intrinsics_mips.cc b/compiler/optimizing/intrinsics_mips.cc index bf85b1989e..b67793c4ed 100644 --- a/compiler/optimizing/intrinsics_mips.cc +++ b/compiler/optimizing/intrinsics_mips.cc @@ -1514,21 +1514,31 @@ void IntrinsicCodeGeneratorMIPS::VisitThreadCurrentThread(HInvoke* invoke) { Thread::PeerOffset<kMipsPointerSize>().Int32Value()); } -static void CreateIntIntIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) { - bool can_call = - invoke->GetIntrinsic() == Intrinsics::kUnsafeGetObject || - invoke->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile; +static void CreateIntIntIntToIntLocations(ArenaAllocator* arena, + HInvoke* invoke, + Primitive::Type type) { + bool can_call = kEmitCompilerReadBarrier && + (invoke->GetIntrinsic() == Intrinsics::kUnsafeGetObject || + invoke->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile); LocationSummary* locations = new (arena) LocationSummary(invoke, - can_call ? - LocationSummary::kCallOnSlowPath : - LocationSummary::kNoCall, + (can_call + ? LocationSummary::kCallOnSlowPath + : LocationSummary::kNoCall), kIntrinsified); locations->SetInAt(0, Location::NoLocation()); // Unused receiver. locations->SetInAt(1, Location::RequiresRegister()); locations->SetInAt(2, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); + locations->SetOut(Location::RequiresRegister(), + (can_call ? Location::kOutputOverlap : Location::kNoOutputOverlap)); + if (type == Primitive::kPrimNot && kEmitCompilerReadBarrier && kUseBakerReadBarrier) { + // We need a temporary register for the read barrier marking slow + // path in InstructionCodeGeneratorMIPS::GenerateReferenceLoadWithBakerReadBarrier. + locations->AddTemp(Location::RequiresRegister()); + } } +// Note that the caller must supply a properly aligned memory address. +// If they do not, the behavior is undefined (atomicity not guaranteed, exception may occur). static void GenUnsafeGet(HInvoke* invoke, Primitive::Type type, bool is_volatile, @@ -1539,49 +1549,109 @@ static void GenUnsafeGet(HInvoke* invoke, (type == Primitive::kPrimLong) || (type == Primitive::kPrimNot)) << type; MipsAssembler* assembler = codegen->GetAssembler(); + // Target register. + Location trg_loc = locations->Out(); // Object pointer. - Register base = locations->InAt(1).AsRegister<Register>(); + Location base_loc = locations->InAt(1); + Register base = base_loc.AsRegister<Register>(); // The "offset" argument is passed as a "long". Since this code is for // a 32-bit processor, we can only use 32-bit addresses, so we only // need the low 32-bits of offset. - Register offset_lo = invoke->GetLocations()->InAt(2).AsRegisterPairLow<Register>(); + Location offset_loc = locations->InAt(2); + Register offset_lo = offset_loc.AsRegisterPairLow<Register>(); - __ Addu(TMP, base, offset_lo); - if (is_volatile) { - __ Sync(0); + if (!(kEmitCompilerReadBarrier && kUseBakerReadBarrier && (type == Primitive::kPrimNot))) { + __ Addu(TMP, base, offset_lo); } - if (type == Primitive::kPrimLong) { - Register trg_lo = locations->Out().AsRegisterPairLow<Register>(); - Register trg_hi = locations->Out().AsRegisterPairHigh<Register>(); - if (is_R6) { - __ Lw(trg_lo, TMP, 0); - __ Lw(trg_hi, TMP, 4); - } else { - __ Lwr(trg_lo, TMP, 0); - __ Lwl(trg_lo, TMP, 3); - __ Lwr(trg_hi, TMP, 4); - __ Lwl(trg_hi, TMP, 7); + switch (type) { + case Primitive::kPrimLong: { + Register trg_lo = trg_loc.AsRegisterPairLow<Register>(); + Register trg_hi = trg_loc.AsRegisterPairHigh<Register>(); + CHECK(!is_volatile); // TODO: support atomic 8-byte volatile loads. + if (is_R6) { + __ Lw(trg_lo, TMP, 0); + __ Lw(trg_hi, TMP, 4); + } else { + __ Lwr(trg_lo, TMP, 0); + __ Lwl(trg_lo, TMP, 3); + __ Lwr(trg_hi, TMP, 4); + __ Lwl(trg_hi, TMP, 7); + } + break; } - } else { - Register trg = locations->Out().AsRegister<Register>(); - if (is_R6) { - __ Lw(trg, TMP, 0); - } else { - __ Lwr(trg, TMP, 0); - __ Lwl(trg, TMP, 3); + case Primitive::kPrimInt: { + Register trg = trg_loc.AsRegister<Register>(); + if (is_R6) { + __ Lw(trg, TMP, 0); + } else { + __ Lwr(trg, TMP, 0); + __ Lwl(trg, TMP, 3); + } + if (is_volatile) { + __ Sync(0); + } + break; } - if (type == Primitive::kPrimNot) { - __ MaybeUnpoisonHeapReference(trg); + case Primitive::kPrimNot: { + Register trg = trg_loc.AsRegister<Register>(); + if (kEmitCompilerReadBarrier) { + if (kUseBakerReadBarrier) { + Location temp = locations->GetTemp(0); + codegen->GenerateReferenceLoadWithBakerReadBarrier(invoke, + trg_loc, + base, + /* offset */ 0U, + /* index */ offset_loc, + TIMES_1, + temp, + /* needs_null_check */ false); + if (is_volatile) { + __ Sync(0); + } + } else { + if (is_R6) { + __ Lw(trg, TMP, 0); + } else { + __ Lwr(trg, TMP, 0); + __ Lwl(trg, TMP, 3); + } + if (is_volatile) { + __ Sync(0); + } + codegen->GenerateReadBarrierSlow(invoke, + trg_loc, + trg_loc, + base_loc, + /* offset */ 0U, + /* index */ offset_loc); + } + } else { + if (is_R6) { + __ Lw(trg, TMP, 0); + } else { + __ Lwr(trg, TMP, 0); + __ Lwl(trg, TMP, 3); + } + if (is_volatile) { + __ Sync(0); + } + __ MaybeUnpoisonHeapReference(trg); + } + break; } + + default: + LOG(FATAL) << "Unexpected type " << type; + UNREACHABLE(); } } // int sun.misc.Unsafe.getInt(Object o, long offset) void IntrinsicLocationsBuilderMIPS::VisitUnsafeGet(HInvoke* invoke) { - CreateIntIntIntToIntLocations(arena_, invoke); + CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimInt); } void IntrinsicCodeGeneratorMIPS::VisitUnsafeGet(HInvoke* invoke) { @@ -1590,7 +1660,7 @@ void IntrinsicCodeGeneratorMIPS::VisitUnsafeGet(HInvoke* invoke) { // int sun.misc.Unsafe.getIntVolatile(Object o, long offset) void IntrinsicLocationsBuilderMIPS::VisitUnsafeGetVolatile(HInvoke* invoke) { - CreateIntIntIntToIntLocations(arena_, invoke); + CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimInt); } void IntrinsicCodeGeneratorMIPS::VisitUnsafeGetVolatile(HInvoke* invoke) { @@ -1599,25 +1669,16 @@ void IntrinsicCodeGeneratorMIPS::VisitUnsafeGetVolatile(HInvoke* invoke) { // long sun.misc.Unsafe.getLong(Object o, long offset) void IntrinsicLocationsBuilderMIPS::VisitUnsafeGetLong(HInvoke* invoke) { - CreateIntIntIntToIntLocations(arena_, invoke); + CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimLong); } void IntrinsicCodeGeneratorMIPS::VisitUnsafeGetLong(HInvoke* invoke) { GenUnsafeGet(invoke, Primitive::kPrimLong, /* is_volatile */ false, IsR6(), codegen_); } -// long sun.misc.Unsafe.getLongVolatile(Object o, long offset) -void IntrinsicLocationsBuilderMIPS::VisitUnsafeGetLongVolatile(HInvoke* invoke) { - CreateIntIntIntToIntLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorMIPS::VisitUnsafeGetLongVolatile(HInvoke* invoke) { - GenUnsafeGet(invoke, Primitive::kPrimLong, /* is_volatile */ true, IsR6(), codegen_); -} - // Object sun.misc.Unsafe.getObject(Object o, long offset) void IntrinsicLocationsBuilderMIPS::VisitUnsafeGetObject(HInvoke* invoke) { - CreateIntIntIntToIntLocations(arena_, invoke); + CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimNot); } void IntrinsicCodeGeneratorMIPS::VisitUnsafeGetObject(HInvoke* invoke) { @@ -1626,7 +1687,7 @@ void IntrinsicCodeGeneratorMIPS::VisitUnsafeGetObject(HInvoke* invoke) { // Object sun.misc.Unsafe.getObjectVolatile(Object o, long offset) void IntrinsicLocationsBuilderMIPS::VisitUnsafeGetObjectVolatile(HInvoke* invoke) { - CreateIntIntIntToIntLocations(arena_, invoke); + CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimNot); } void IntrinsicCodeGeneratorMIPS::VisitUnsafeGetObjectVolatile(HInvoke* invoke) { @@ -1643,6 +1704,8 @@ static void CreateIntIntIntIntToVoidLocations(ArenaAllocator* arena, HInvoke* in locations->SetInAt(3, Location::RequiresRegister()); } +// Note that the caller must supply a properly aligned memory address. +// If they do not, the behavior is undefined (atomicity not guaranteed, exception may occur). static void GenUnsafePut(LocationSummary* locations, Primitive::Type type, bool is_volatile, @@ -1681,7 +1744,7 @@ static void GenUnsafePut(LocationSummary* locations, } else { Register value_lo = locations->InAt(3).AsRegisterPairLow<Register>(); Register value_hi = locations->InAt(3).AsRegisterPairHigh<Register>(); - + CHECK(!is_volatile); // TODO: support atomic 8-byte volatile stores. if (is_R6) { __ Sw(value_lo, TMP, 0); __ Sw(value_hi, TMP, 4); @@ -1815,50 +1878,71 @@ void IntrinsicCodeGeneratorMIPS::VisitUnsafePutLongOrdered(HInvoke* invoke) { codegen_); } -// void sun.misc.Unsafe.putLongVolatile(Object o, long offset, long x) -void IntrinsicLocationsBuilderMIPS::VisitUnsafePutLongVolatile(HInvoke* invoke) { - CreateIntIntIntIntToVoidLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorMIPS::VisitUnsafePutLongVolatile(HInvoke* invoke) { - GenUnsafePut(invoke->GetLocations(), - Primitive::kPrimLong, - /* is_volatile */ true, - /* is_ordered */ false, - IsR6(), - codegen_); -} - -static void CreateIntIntIntIntIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) { +static void CreateIntIntIntIntIntToIntPlusTemps(ArenaAllocator* arena, HInvoke* invoke) { + bool can_call = kEmitCompilerReadBarrier && + kUseBakerReadBarrier && + (invoke->GetIntrinsic() == Intrinsics::kUnsafeCASObject); LocationSummary* locations = new (arena) LocationSummary(invoke, - LocationSummary::kNoCall, + (can_call + ? LocationSummary::kCallOnSlowPath + : LocationSummary::kNoCall), kIntrinsified); locations->SetInAt(0, Location::NoLocation()); // Unused receiver. locations->SetInAt(1, Location::RequiresRegister()); locations->SetInAt(2, Location::RequiresRegister()); locations->SetInAt(3, Location::RequiresRegister()); locations->SetInAt(4, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister()); + + // Temporary register used in CAS by (Baker) read barrier. + if (can_call) { + locations->AddTemp(Location::RequiresRegister()); + } } -static void GenCas(LocationSummary* locations, Primitive::Type type, CodeGeneratorMIPS* codegen) { +// Note that the caller must supply a properly aligned memory address. +// If they do not, the behavior is undefined (atomicity not guaranteed, exception may occur). +static void GenCas(HInvoke* invoke, Primitive::Type type, CodeGeneratorMIPS* codegen) { MipsAssembler* assembler = codegen->GetAssembler(); + LocationSummary* locations = invoke->GetLocations(); bool isR6 = codegen->GetInstructionSetFeatures().IsR6(); Register base = locations->InAt(1).AsRegister<Register>(); - Register offset_lo = locations->InAt(2).AsRegisterPairLow<Register>(); + Location offset_loc = locations->InAt(2); + Register offset_lo = offset_loc.AsRegisterPairLow<Register>(); Register expected = locations->InAt(3).AsRegister<Register>(); Register value = locations->InAt(4).AsRegister<Register>(); - Register out = locations->Out().AsRegister<Register>(); + Location out_loc = locations->Out(); + Register out = out_loc.AsRegister<Register>(); DCHECK_NE(base, out); DCHECK_NE(offset_lo, out); DCHECK_NE(expected, out); if (type == Primitive::kPrimNot) { - // Mark card for object assuming new value is stored. + // The only read barrier implementation supporting the + // UnsafeCASObject intrinsic is the Baker-style read barriers. + DCHECK(!kEmitCompilerReadBarrier || kUseBakerReadBarrier); + + // Mark card for object assuming new value is stored. Worst case we will mark an unchanged + // object and scan the receiver at the next GC for nothing. bool value_can_be_null = true; // TODO: Worth finding out this information? codegen->MarkGCCard(base, value, value_can_be_null); + + if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { + Location temp = locations->GetTemp(0); + // Need to make sure the reference stored in the field is a to-space + // one before attempting the CAS or the CAS could fail incorrectly. + codegen->GenerateReferenceLoadWithBakerReadBarrier( + invoke, + out_loc, // Unused, used only as a "temporary" within the read barrier. + base, + /* offset */ 0u, + /* index */ offset_loc, + ScaleFactor::TIMES_1, + temp, + /* needs_null_check */ false, + /* always_update_field */ true); + } } MipsLabel loop_head, exit_loop; @@ -1926,20 +2010,30 @@ static void GenCas(LocationSummary* locations, Primitive::Type type, CodeGenerat // boolean sun.misc.Unsafe.compareAndSwapInt(Object o, long offset, int expected, int x) void IntrinsicLocationsBuilderMIPS::VisitUnsafeCASInt(HInvoke* invoke) { - CreateIntIntIntIntIntToIntLocations(arena_, invoke); + CreateIntIntIntIntIntToIntPlusTemps(arena_, invoke); } void IntrinsicCodeGeneratorMIPS::VisitUnsafeCASInt(HInvoke* invoke) { - GenCas(invoke->GetLocations(), Primitive::kPrimInt, codegen_); + GenCas(invoke, Primitive::kPrimInt, codegen_); } // boolean sun.misc.Unsafe.compareAndSwapObject(Object o, long offset, Object expected, Object x) void IntrinsicLocationsBuilderMIPS::VisitUnsafeCASObject(HInvoke* invoke) { - CreateIntIntIntIntIntToIntLocations(arena_, invoke); + // The only read barrier implementation supporting the + // UnsafeCASObject intrinsic is the Baker-style read barriers. + if (kEmitCompilerReadBarrier && !kUseBakerReadBarrier) { + return; + } + + CreateIntIntIntIntIntToIntPlusTemps(arena_, invoke); } void IntrinsicCodeGeneratorMIPS::VisitUnsafeCASObject(HInvoke* invoke) { - GenCas(invoke->GetLocations(), Primitive::kPrimNot, codegen_); + // The only read barrier implementation supporting the + // UnsafeCASObject intrinsic is the Baker-style read barriers. + DCHECK(!kEmitCompilerReadBarrier || kUseBakerReadBarrier); + + GenCas(invoke, Primitive::kPrimNot, codegen_); } // int java.lang.String.compareTo(String anotherString) @@ -2664,6 +2758,8 @@ UNIMPLEMENTED_INTRINSIC(MIPS, MathCeil) UNIMPLEMENTED_INTRINSIC(MIPS, MathFloor) UNIMPLEMENTED_INTRINSIC(MIPS, MathRint) UNIMPLEMENTED_INTRINSIC(MIPS, MathRoundDouble) +UNIMPLEMENTED_INTRINSIC(MIPS, UnsafeGetLongVolatile); +UNIMPLEMENTED_INTRINSIC(MIPS, UnsafePutLongVolatile); UNIMPLEMENTED_INTRINSIC(MIPS, UnsafeCASLong) UNIMPLEMENTED_INTRINSIC(MIPS, ReferenceGetReferent) diff --git a/compiler/optimizing/intrinsics_mips64.cc b/compiler/optimizing/intrinsics_mips64.cc index 1ee89cf127..6098767aae 100644 --- a/compiler/optimizing/intrinsics_mips64.cc +++ b/compiler/optimizing/intrinsics_mips64.cc @@ -1151,16 +1151,31 @@ void IntrinsicCodeGeneratorMIPS64::VisitThreadCurrentThread(HInvoke* invoke) { Thread::PeerOffset<kMips64PointerSize>().Int32Value()); } -static void CreateIntIntIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) { +static void CreateIntIntIntToIntLocations(ArenaAllocator* arena, + HInvoke* invoke, + Primitive::Type type) { + bool can_call = kEmitCompilerReadBarrier && + (invoke->GetIntrinsic() == Intrinsics::kUnsafeGetObject || + invoke->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile); LocationSummary* locations = new (arena) LocationSummary(invoke, - LocationSummary::kNoCall, + (can_call + ? LocationSummary::kCallOnSlowPath + : LocationSummary::kNoCall), kIntrinsified); locations->SetInAt(0, Location::NoLocation()); // Unused receiver. locations->SetInAt(1, Location::RequiresRegister()); locations->SetInAt(2, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); + locations->SetOut(Location::RequiresRegister(), + (can_call ? Location::kOutputOverlap : Location::kNoOutputOverlap)); + if (type == Primitive::kPrimNot && kEmitCompilerReadBarrier && kUseBakerReadBarrier) { + // We need a temporary register for the read barrier marking slow + // path in InstructionCodeGeneratorMIPS64::GenerateReferenceLoadWithBakerReadBarrier. + locations->AddTemp(Location::RequiresRegister()); + } } +// Note that the caller must supply a properly aligned memory address. +// If they do not, the behavior is undefined (atomicity not guaranteed, exception may occur). static void GenUnsafeGet(HInvoke* invoke, Primitive::Type type, bool is_volatile, @@ -1168,30 +1183,71 @@ static void GenUnsafeGet(HInvoke* invoke, LocationSummary* locations = invoke->GetLocations(); DCHECK((type == Primitive::kPrimInt) || (type == Primitive::kPrimLong) || - (type == Primitive::kPrimNot)); + (type == Primitive::kPrimNot)) << type; Mips64Assembler* assembler = codegen->GetAssembler(); + // Target register. + Location trg_loc = locations->Out(); + GpuRegister trg = trg_loc.AsRegister<GpuRegister>(); // Object pointer. - GpuRegister base = locations->InAt(1).AsRegister<GpuRegister>(); + Location base_loc = locations->InAt(1); + GpuRegister base = base_loc.AsRegister<GpuRegister>(); // Long offset. - GpuRegister offset = locations->InAt(2).AsRegister<GpuRegister>(); - GpuRegister trg = locations->Out().AsRegister<GpuRegister>(); + Location offset_loc = locations->InAt(2); + GpuRegister offset = offset_loc.AsRegister<GpuRegister>(); - __ Daddu(TMP, base, offset); - if (is_volatile) { - __ Sync(0); + if (!(kEmitCompilerReadBarrier && kUseBakerReadBarrier && (type == Primitive::kPrimNot))) { + __ Daddu(TMP, base, offset); } + switch (type) { + case Primitive::kPrimLong: + __ Ld(trg, TMP, 0); + if (is_volatile) { + __ Sync(0); + } + break; + case Primitive::kPrimInt: __ Lw(trg, TMP, 0); + if (is_volatile) { + __ Sync(0); + } break; case Primitive::kPrimNot: - __ Lwu(trg, TMP, 0); - __ MaybeUnpoisonHeapReference(trg); - break; - - case Primitive::kPrimLong: - __ Ld(trg, TMP, 0); + if (kEmitCompilerReadBarrier) { + if (kUseBakerReadBarrier) { + Location temp = locations->GetTemp(0); + codegen->GenerateReferenceLoadWithBakerReadBarrier(invoke, + trg_loc, + base, + /* offset */ 0U, + /* index */ offset_loc, + TIMES_1, + temp, + /* needs_null_check */ false); + if (is_volatile) { + __ Sync(0); + } + } else { + __ Lwu(trg, TMP, 0); + if (is_volatile) { + __ Sync(0); + } + codegen->GenerateReadBarrierSlow(invoke, + trg_loc, + trg_loc, + base_loc, + /* offset */ 0U, + /* index */ offset_loc); + } + } else { + __ Lwu(trg, TMP, 0); + if (is_volatile) { + __ Sync(0); + } + __ MaybeUnpoisonHeapReference(trg); + } break; default: @@ -1202,7 +1258,7 @@ static void GenUnsafeGet(HInvoke* invoke, // int sun.misc.Unsafe.getInt(Object o, long offset) void IntrinsicLocationsBuilderMIPS64::VisitUnsafeGet(HInvoke* invoke) { - CreateIntIntIntToIntLocations(arena_, invoke); + CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimInt); } void IntrinsicCodeGeneratorMIPS64::VisitUnsafeGet(HInvoke* invoke) { @@ -1211,7 +1267,7 @@ void IntrinsicCodeGeneratorMIPS64::VisitUnsafeGet(HInvoke* invoke) { // int sun.misc.Unsafe.getIntVolatile(Object o, long offset) void IntrinsicLocationsBuilderMIPS64::VisitUnsafeGetVolatile(HInvoke* invoke) { - CreateIntIntIntToIntLocations(arena_, invoke); + CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimInt); } void IntrinsicCodeGeneratorMIPS64::VisitUnsafeGetVolatile(HInvoke* invoke) { @@ -1220,7 +1276,7 @@ void IntrinsicCodeGeneratorMIPS64::VisitUnsafeGetVolatile(HInvoke* invoke) { // long sun.misc.Unsafe.getLong(Object o, long offset) void IntrinsicLocationsBuilderMIPS64::VisitUnsafeGetLong(HInvoke* invoke) { - CreateIntIntIntToIntLocations(arena_, invoke); + CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimLong); } void IntrinsicCodeGeneratorMIPS64::VisitUnsafeGetLong(HInvoke* invoke) { @@ -1229,7 +1285,7 @@ void IntrinsicCodeGeneratorMIPS64::VisitUnsafeGetLong(HInvoke* invoke) { // long sun.misc.Unsafe.getLongVolatile(Object o, long offset) void IntrinsicLocationsBuilderMIPS64::VisitUnsafeGetLongVolatile(HInvoke* invoke) { - CreateIntIntIntToIntLocations(arena_, invoke); + CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimLong); } void IntrinsicCodeGeneratorMIPS64::VisitUnsafeGetLongVolatile(HInvoke* invoke) { @@ -1238,7 +1294,7 @@ void IntrinsicCodeGeneratorMIPS64::VisitUnsafeGetLongVolatile(HInvoke* invoke) { // Object sun.misc.Unsafe.getObject(Object o, long offset) void IntrinsicLocationsBuilderMIPS64::VisitUnsafeGetObject(HInvoke* invoke) { - CreateIntIntIntToIntLocations(arena_, invoke); + CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimNot); } void IntrinsicCodeGeneratorMIPS64::VisitUnsafeGetObject(HInvoke* invoke) { @@ -1247,7 +1303,7 @@ void IntrinsicCodeGeneratorMIPS64::VisitUnsafeGetObject(HInvoke* invoke) { // Object sun.misc.Unsafe.getObjectVolatile(Object o, long offset) void IntrinsicLocationsBuilderMIPS64::VisitUnsafeGetObjectVolatile(HInvoke* invoke) { - CreateIntIntIntToIntLocations(arena_, invoke); + CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimNot); } void IntrinsicCodeGeneratorMIPS64::VisitUnsafeGetObjectVolatile(HInvoke* invoke) { @@ -1264,6 +1320,8 @@ static void CreateIntIntIntIntToVoid(ArenaAllocator* arena, HInvoke* invoke) { locations->SetInAt(3, Location::RequiresRegister()); } +// Note that the caller must supply a properly aligned memory address. +// If they do not, the behavior is undefined (atomicity not guaranteed, exception may occur). static void GenUnsafePut(LocationSummary* locations, Primitive::Type type, bool is_volatile, @@ -1429,35 +1487,70 @@ void IntrinsicCodeGeneratorMIPS64::VisitUnsafePutLongVolatile(HInvoke* invoke) { codegen_); } -static void CreateIntIntIntIntIntToInt(ArenaAllocator* arena, HInvoke* invoke) { +static void CreateIntIntIntIntIntToIntPlusTemps(ArenaAllocator* arena, HInvoke* invoke) { + bool can_call = kEmitCompilerReadBarrier && + kUseBakerReadBarrier && + (invoke->GetIntrinsic() == Intrinsics::kUnsafeCASObject); LocationSummary* locations = new (arena) LocationSummary(invoke, - LocationSummary::kNoCall, + (can_call + ? LocationSummary::kCallOnSlowPath + : LocationSummary::kNoCall), kIntrinsified); locations->SetInAt(0, Location::NoLocation()); // Unused receiver. locations->SetInAt(1, Location::RequiresRegister()); locations->SetInAt(2, Location::RequiresRegister()); locations->SetInAt(3, Location::RequiresRegister()); locations->SetInAt(4, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister()); + + // Temporary register used in CAS by (Baker) read barrier. + if (can_call) { + locations->AddTemp(Location::RequiresRegister()); + } } -static void GenCas(LocationSummary* locations, Primitive::Type type, CodeGeneratorMIPS64* codegen) { +// Note that the caller must supply a properly aligned memory address. +// If they do not, the behavior is undefined (atomicity not guaranteed, exception may occur). +static void GenCas(HInvoke* invoke, Primitive::Type type, CodeGeneratorMIPS64* codegen) { Mips64Assembler* assembler = codegen->GetAssembler(); + LocationSummary* locations = invoke->GetLocations(); GpuRegister base = locations->InAt(1).AsRegister<GpuRegister>(); - GpuRegister offset = locations->InAt(2).AsRegister<GpuRegister>(); + Location offset_loc = locations->InAt(2); + GpuRegister offset = offset_loc.AsRegister<GpuRegister>(); GpuRegister expected = locations->InAt(3).AsRegister<GpuRegister>(); GpuRegister value = locations->InAt(4).AsRegister<GpuRegister>(); - GpuRegister out = locations->Out().AsRegister<GpuRegister>(); + Location out_loc = locations->Out(); + GpuRegister out = out_loc.AsRegister<GpuRegister>(); DCHECK_NE(base, out); DCHECK_NE(offset, out); DCHECK_NE(expected, out); if (type == Primitive::kPrimNot) { - // Mark card for object assuming new value is stored. + // The only read barrier implementation supporting the + // UnsafeCASObject intrinsic is the Baker-style read barriers. + DCHECK(!kEmitCompilerReadBarrier || kUseBakerReadBarrier); + + // Mark card for object assuming new value is stored. Worst case we will mark an unchanged + // object and scan the receiver at the next GC for nothing. bool value_can_be_null = true; // TODO: Worth finding out this information? codegen->MarkGCCard(base, value, value_can_be_null); + + if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { + Location temp = locations->GetTemp(0); + // Need to make sure the reference stored in the field is a to-space + // one before attempting the CAS or the CAS could fail incorrectly. + codegen->GenerateReferenceLoadWithBakerReadBarrier( + invoke, + out_loc, // Unused, used only as a "temporary" within the read barrier. + base, + /* offset */ 0u, + /* index */ offset_loc, + ScaleFactor::TIMES_1, + temp, + /* needs_null_check */ false, + /* always_update_field */ true); + } } Mips64Label loop_head, exit_loop; @@ -1521,29 +1614,39 @@ static void GenCas(LocationSummary* locations, Primitive::Type type, CodeGenerat // boolean sun.misc.Unsafe.compareAndSwapInt(Object o, long offset, int expected, int x) void IntrinsicLocationsBuilderMIPS64::VisitUnsafeCASInt(HInvoke* invoke) { - CreateIntIntIntIntIntToInt(arena_, invoke); + CreateIntIntIntIntIntToIntPlusTemps(arena_, invoke); } void IntrinsicCodeGeneratorMIPS64::VisitUnsafeCASInt(HInvoke* invoke) { - GenCas(invoke->GetLocations(), Primitive::kPrimInt, codegen_); + GenCas(invoke, Primitive::kPrimInt, codegen_); } // boolean sun.misc.Unsafe.compareAndSwapLong(Object o, long offset, long expected, long x) void IntrinsicLocationsBuilderMIPS64::VisitUnsafeCASLong(HInvoke* invoke) { - CreateIntIntIntIntIntToInt(arena_, invoke); + CreateIntIntIntIntIntToIntPlusTemps(arena_, invoke); } void IntrinsicCodeGeneratorMIPS64::VisitUnsafeCASLong(HInvoke* invoke) { - GenCas(invoke->GetLocations(), Primitive::kPrimLong, codegen_); + GenCas(invoke, Primitive::kPrimLong, codegen_); } // boolean sun.misc.Unsafe.compareAndSwapObject(Object o, long offset, Object expected, Object x) void IntrinsicLocationsBuilderMIPS64::VisitUnsafeCASObject(HInvoke* invoke) { - CreateIntIntIntIntIntToInt(arena_, invoke); + // The only read barrier implementation supporting the + // UnsafeCASObject intrinsic is the Baker-style read barriers. + if (kEmitCompilerReadBarrier && !kUseBakerReadBarrier) { + return; + } + + CreateIntIntIntIntIntToIntPlusTemps(arena_, invoke); } void IntrinsicCodeGeneratorMIPS64::VisitUnsafeCASObject(HInvoke* invoke) { - GenCas(invoke->GetLocations(), Primitive::kPrimNot, codegen_); + // The only read barrier implementation supporting the + // UnsafeCASObject intrinsic is the Baker-style read barriers. + DCHECK(!kEmitCompilerReadBarrier || kUseBakerReadBarrier); + + GenCas(invoke, Primitive::kPrimNot, codegen_); } // int java.lang.String.compareTo(String anotherString) diff --git a/compiler/optimizing/licm_test.cc b/compiler/optimizing/licm_test.cc index 5bcfa4c98b..8d15f78cce 100644 --- a/compiler/optimizing/licm_test.cc +++ b/compiler/optimizing/licm_test.cc @@ -28,7 +28,18 @@ namespace art { */ class LICMTest : public CommonCompilerTest { public: - LICMTest() : pool_(), allocator_(&pool_) { + LICMTest() + : pool_(), + allocator_(&pool_), + entry_(nullptr), + loop_preheader_(nullptr), + loop_header_(nullptr), + loop_body_(nullptr), + return_(nullptr), + exit_(nullptr), + parameter_(nullptr), + int_constant_(nullptr), + float_constant_(nullptr) { graph_ = CreateGraph(&allocator_); } diff --git a/compiler/optimizing/loop_optimization.cc b/compiler/optimizing/loop_optimization.cc index 8df513f410..42ed04dfa3 100644 --- a/compiler/optimizing/loop_optimization.cc +++ b/compiler/optimizing/loop_optimization.cc @@ -16,11 +16,21 @@ #include "loop_optimization.h" +#include "arch/instruction_set.h" +#include "arch/arm/instruction_set_features_arm.h" +#include "arch/arm64/instruction_set_features_arm64.h" +#include "arch/mips/instruction_set_features_mips.h" +#include "arch/mips64/instruction_set_features_mips64.h" +#include "arch/x86/instruction_set_features_x86.h" +#include "arch/x86_64/instruction_set_features_x86_64.h" #include "driver/compiler_driver.h" #include "linear_order.h" namespace art { +// Enables vectorization (SIMDization) in the loop optimizer. +static constexpr bool kEnableVectorization = true; + // Remove the instruction from the graph. A bit more elaborate than the usual // instruction removal, since there may be a cycle in the use structure. static void RemoveFromCycle(HInstruction* instruction) { @@ -53,6 +63,19 @@ static bool IsEarlyExit(HLoopInformation* loop_info) { return false; } +// Test vector restrictions. +static bool HasVectorRestrictions(uint64_t restrictions, uint64_t tested) { + return (restrictions & tested) != 0; +} + +// Inserts an instruction. +static HInstruction* Insert(HBasicBlock* block, HInstruction* instruction) { + DCHECK(block != nullptr); + DCHECK(instruction != nullptr); + block->InsertInstructionBefore(instruction, block->GetLastInstruction()); + return instruction; +} + // // Class methods. // @@ -64,11 +87,15 @@ HLoopOptimization::HLoopOptimization(HGraph* graph, compiler_driver_(compiler_driver), induction_range_(induction_analysis), loop_allocator_(nullptr), + global_allocator_(graph_->GetArena()), top_loop_(nullptr), last_loop_(nullptr), iset_(nullptr), induction_simplication_count_(0), - simplified_(false) { + simplified_(false), + vector_length_(0), + vector_refs_(nullptr), + vector_map_(nullptr) { } void HLoopOptimization::Run() { @@ -81,15 +108,13 @@ void HLoopOptimization::Run() { // Phase-local allocator that draws from the global pool. Since the allocator // itself resides on the stack, it is destructed on exiting Run(), which // implies its underlying memory is released immediately. - ArenaAllocator allocator(graph_->GetArena()->GetArenaPool()); + ArenaAllocator allocator(global_allocator_->GetArenaPool()); loop_allocator_ = &allocator; // Perform loop optimizations. LocalRun(); - if (top_loop_ == nullptr) { - // All loops have been eliminated. - graph_->SetHasLoops(false); + graph_->SetHasLoops(false); // no more loops } // Detach. @@ -111,18 +136,29 @@ void HLoopOptimization::LocalRun() { } // Traverse the loop hierarchy inner-to-outer and optimize. Traversal can use - // a temporary set that stores instructions using the phase-local allocator. + // temporary data structures using the phase-local allocator. All new HIR + // should use the global allocator. if (top_loop_ != nullptr) { ArenaSet<HInstruction*> iset(loop_allocator_->Adapter(kArenaAllocLoopOptimization)); + ArenaSet<ArrayReference> refs(loop_allocator_->Adapter(kArenaAllocLoopOptimization)); + ArenaSafeMap<HInstruction*, HInstruction*> map( + std::less<HInstruction*>(), loop_allocator_->Adapter(kArenaAllocLoopOptimization)); + // Attach. iset_ = &iset; + vector_refs_ = &refs; + vector_map_ = ↦ + // Traverse. TraverseLoopsInnerToOuter(top_loop_); - iset_ = nullptr; // detach + // Detach. + iset_ = nullptr; + vector_refs_ = nullptr; + vector_map_ = nullptr; } } void HLoopOptimization::AddLoop(HLoopInformation* loop_info) { DCHECK(loop_info != nullptr); - LoopNode* node = new (loop_allocator_) LoopNode(loop_info); // phase-local allocator + LoopNode* node = new (loop_allocator_) LoopNode(loop_info); if (last_loop_ == nullptr) { // First loop. DCHECK(top_loop_ == nullptr); @@ -170,7 +206,7 @@ void HLoopOptimization::RemoveLoop(LoopNode* node) { void HLoopOptimization::TraverseLoopsInnerToOuter(LoopNode* node) { for ( ; node != nullptr; node = node->next) { // Visit inner loops first. - int current_induction_simplification_count = induction_simplication_count_; + uint32_t current_induction_simplification_count = induction_simplication_count_; if (node->inner != nullptr) { TraverseLoopsInnerToOuter(node->inner); } @@ -179,7 +215,7 @@ void HLoopOptimization::TraverseLoopsInnerToOuter(LoopNode* node) { if (current_induction_simplification_count != induction_simplication_count_) { induction_range_.ReVisit(node->loop_info); } - // Repeat simplifications in the body of this loop until no more changes occur. + // Repeat simplifications in the loop-body until no more changes occur. // Note that since each simplification consists of eliminating code (without // introducing new code), this process is always finite. do { @@ -187,13 +223,17 @@ void HLoopOptimization::TraverseLoopsInnerToOuter(LoopNode* node) { SimplifyInduction(node); SimplifyBlocks(node); } while (simplified_); - // Simplify inner loop. + // Optimize inner loop. if (node->inner == nullptr) { - SimplifyInnerLoop(node); + OptimizeInnerLoop(node); } } } +// +// Optimization. +// + void HLoopOptimization::SimplifyInduction(LoopNode* node) { HBasicBlock* header = node->loop_info->GetHeader(); HBasicBlock* preheader = node->loop_info->GetPreHeader(); @@ -204,13 +244,9 @@ void HLoopOptimization::SimplifyInduction(LoopNode* node) { // for (int i = 0; i < 10; i++, k++) { .... no k .... } return k; for (HInstructionIterator it(header->GetPhis()); !it.Done(); it.Advance()) { HPhi* phi = it.Current()->AsPhi(); - iset_->clear(); - int32_t use_count = 0; - if (IsPhiInduction(phi) && - IsOnlyUsedAfterLoop(node->loop_info, phi, /*collect_loop_uses*/ false, &use_count) && - // No uses, or no early-exit with proper replacement. - (use_count == 0 || - (!IsEarlyExit(node->loop_info) && TryReplaceWithLastValue(phi, preheader)))) { + iset_->clear(); // prepare phi induction + if (TrySetPhiInduction(phi, /*restrict_uses*/ true) && + TryAssignLastValue(node->loop_info, phi, preheader, /*collect_loop_uses*/ false)) { for (HInstruction* i : *iset_) { RemoveFromCycle(i); } @@ -256,49 +292,47 @@ void HLoopOptimization::SimplifyBlocks(LoopNode* node) { } } -bool HLoopOptimization::SimplifyInnerLoop(LoopNode* node) { +void HLoopOptimization::OptimizeInnerLoop(LoopNode* node) { HBasicBlock* header = node->loop_info->GetHeader(); HBasicBlock* preheader = node->loop_info->GetPreHeader(); // Ensure loop header logic is finite. - int64_t tc = 0; - if (!induction_range_.IsFinite(node->loop_info, &tc)) { - return false; + int64_t trip_count = 0; + if (!induction_range_.IsFinite(node->loop_info, &trip_count)) { + return; } + // Ensure there is only a single loop-body (besides the header). HBasicBlock* body = nullptr; for (HBlocksInLoopIterator it(*node->loop_info); !it.Done(); it.Advance()) { if (it.Current() != header) { if (body != nullptr) { - return false; + return; } body = it.Current(); } } // Ensure there is only a single exit point. if (header->GetSuccessors().size() != 2) { - return false; + return; } HBasicBlock* exit = (header->GetSuccessors()[0] == body) ? header->GetSuccessors()[1] : header->GetSuccessors()[0]; // Ensure exit can only be reached by exiting loop. if (exit->GetPredecessors().size() != 1) { - return false; + return; } // Detect either an empty loop (no side effects other than plain iteration) or // a trivial loop (just iterating once). Replace subsequent index uses, if any, // with the last value and remove the loop, possibly after unrolling its body. HInstruction* phi = header->GetFirstPhi(); - iset_->clear(); - int32_t use_count = 0; - if (IsEmptyHeader(header)) { + iset_->clear(); // prepare phi induction + if (TrySetSimpleLoopHeader(header)) { bool is_empty = IsEmptyBody(body); - if ((is_empty || tc == 1) && - IsOnlyUsedAfterLoop(node->loop_info, phi, /*collect_loop_uses*/ true, &use_count) && - // No uses, or proper replacement. - (use_count == 0 || TryReplaceWithLastValue(phi, preheader))) { + if ((is_empty || trip_count == 1) && + TryAssignLastValue(node->loop_info, phi, preheader, /*collect_loop_uses*/ true)) { if (!is_empty) { - // Unroll the loop body, which sees initial value of the index. + // Unroll the loop-body, which sees initial value of the index. phi->ReplaceWith(phi->InputAt(0)); preheader->MergeInstructionsWith(body); } @@ -308,28 +342,649 @@ bool HLoopOptimization::SimplifyInnerLoop(LoopNode* node) { header->RemoveDominatedBlock(exit); header->DisconnectAndDelete(); preheader->AddSuccessor(exit); - preheader->AddInstruction(new (graph_->GetArena()) HGoto()); // global allocator + preheader->AddInstruction(new (global_allocator_) HGoto()); preheader->AddDominatedBlock(exit); exit->SetDominator(preheader); RemoveLoop(node); // update hierarchy + return; + } + } + + // Vectorize loop, if possible and valid. + if (kEnableVectorization) { + iset_->clear(); // prepare phi induction + if (TrySetSimpleLoopHeader(header) && + CanVectorize(node, body, trip_count) && + TryAssignLastValue(node->loop_info, phi, preheader, /*collect_loop_uses*/ true)) { + Vectorize(node, body, exit, trip_count); + graph_->SetHasSIMD(true); // flag SIMD usage + return; + } + } +} + +// +// Loop vectorization. The implementation is based on the book by Aart J.C. Bik: +// "The Software Vectorization Handbook. Applying Multimedia Extensions for Maximum Performance." +// Intel Press, June, 2004 (http://www.aartbik.com/). +// + +bool HLoopOptimization::CanVectorize(LoopNode* node, HBasicBlock* block, int64_t trip_count) { + // Reset vector bookkeeping. + vector_length_ = 0; + vector_refs_->clear(); + vector_runtime_test_a_ = + vector_runtime_test_b_= nullptr; + + // Phis in the loop-body prevent vectorization. + if (!block->GetPhis().IsEmpty()) { + return false; + } + + // Scan the loop-body, starting a right-hand-side tree traversal at each left-hand-side + // occurrence, which allows passing down attributes down the use tree. + for (HInstructionIterator it(block->GetInstructions()); !it.Done(); it.Advance()) { + if (!VectorizeDef(node, it.Current(), /*generate_code*/ false)) { + return false; // failure to vectorize a left-hand-side + } + } + + // Heuristics. Does vectorization seem profitable? + // TODO: refine + if (vector_length_ == 0) { + return false; // nothing found + } else if (0 < trip_count && trip_count < vector_length_) { + return false; // insufficient iterations + } + + // Data dependence analysis. Find each pair of references with same type, where + // at least one is a write. Each such pair denotes a possible data dependence. + // This analysis exploits the property that differently typed arrays cannot be + // aliased, as well as the property that references either point to the same + // array or to two completely disjoint arrays, i.e., no partial aliasing. + // Other than a few simply heuristics, no detailed subscript analysis is done. + for (auto i = vector_refs_->begin(); i != vector_refs_->end(); ++i) { + for (auto j = i; ++j != vector_refs_->end(); ) { + if (i->type == j->type && (i->lhs || j->lhs)) { + // Found same-typed a[i+x] vs. b[i+y], where at least one is a write. + HInstruction* a = i->base; + HInstruction* b = j->base; + HInstruction* x = i->offset; + HInstruction* y = j->offset; + if (a == b) { + // Found a[i+x] vs. a[i+y]. Accept if x == y (loop-independent data dependence). + // Conservatively assume a loop-carried data dependence otherwise, and reject. + if (x != y) { + return false; + } + } else { + // Found a[i+x] vs. b[i+y]. Accept if x == y (at worst loop-independent data dependence). + // Conservatively assume a potential loop-carried data dependence otherwise, avoided by + // generating an explicit a != b disambiguation runtime test on the two references. + if (x != y) { + // For now, we reject after one test to avoid excessive overhead. + if (vector_runtime_test_a_ != nullptr) { + return false; + } + vector_runtime_test_a_ = a; + vector_runtime_test_b_ = b; + } + } + } + } + } + + // Success! + return true; +} + +void HLoopOptimization::Vectorize(LoopNode* node, + HBasicBlock* block, + HBasicBlock* exit, + int64_t trip_count) { + Primitive::Type induc_type = Primitive::kPrimInt; + HBasicBlock* header = node->loop_info->GetHeader(); + HBasicBlock* preheader = node->loop_info->GetPreHeader(); + + // A cleanup is needed for any unknown trip count or for a known trip count + // with remainder iterations after vectorization. + bool needs_cleanup = trip_count == 0 || (trip_count % vector_length_) != 0; + + // Adjust vector bookkeeping. + iset_->clear(); // prepare phi induction + bool is_simple_loop_header = TrySetSimpleLoopHeader(header); // fills iset_ + DCHECK(is_simple_loop_header); + + // Generate preheader: + // stc = <trip-count>; + // vtc = stc - stc % VL; + HInstruction* stc = induction_range_.GenerateTripCount(node->loop_info, graph_, preheader); + HInstruction* vtc = stc; + if (needs_cleanup) { + DCHECK(IsPowerOfTwo(vector_length_)); + HInstruction* rem = Insert( + preheader, new (global_allocator_) HAnd(induc_type, + stc, + graph_->GetIntConstant(vector_length_ - 1))); + vtc = Insert(preheader, new (global_allocator_) HSub(induc_type, stc, rem)); + } + + // Generate runtime disambiguation test: + // vtc = a != b ? vtc : 0; + if (vector_runtime_test_a_ != nullptr) { + HInstruction* rt = Insert( + preheader, + new (global_allocator_) HNotEqual(vector_runtime_test_a_, vector_runtime_test_b_)); + vtc = Insert(preheader, + new (global_allocator_) HSelect(rt, vtc, graph_->GetIntConstant(0), kNoDexPc)); + needs_cleanup = true; + } + + // Generate vector loop: + // for (i = 0; i < vtc; i += VL) + // <vectorized-loop-body> + vector_mode_ = kVector; + GenerateNewLoop(node, + block, + graph_->TransformLoopForVectorization(header, block, exit), + graph_->GetIntConstant(0), + vtc, + graph_->GetIntConstant(vector_length_)); + HLoopInformation* vloop = vector_header_->GetLoopInformation(); + + // Generate cleanup loop, if needed: + // for ( ; i < stc; i += 1) + // <loop-body> + if (needs_cleanup) { + vector_mode_ = kSequential; + GenerateNewLoop(node, + block, + graph_->TransformLoopForVectorization(vector_header_, vector_body_, exit), + vector_phi_, + stc, + graph_->GetIntConstant(1)); + } + + // Remove the original loop by disconnecting the body block + // and removing all instructions from the header. + block->DisconnectAndDelete(); + while (!header->GetFirstInstruction()->IsGoto()) { + header->RemoveInstruction(header->GetFirstInstruction()); + } + // Update loop hierarchy: the old header now resides in the + // same outer loop as the old preheader. + header->SetLoopInformation(preheader->GetLoopInformation()); // outward + node->loop_info = vloop; +} + +void HLoopOptimization::GenerateNewLoop(LoopNode* node, + HBasicBlock* block, + HBasicBlock* new_preheader, + HInstruction* lo, + HInstruction* hi, + HInstruction* step) { + Primitive::Type induc_type = Primitive::kPrimInt; + // Prepare new loop. + vector_map_->clear(); + vector_preheader_ = new_preheader, + vector_header_ = vector_preheader_->GetSingleSuccessor(); + vector_body_ = vector_header_->GetSuccessors()[1]; + vector_phi_ = new (global_allocator_) HPhi(global_allocator_, + kNoRegNumber, + 0, + HPhi::ToPhiType(induc_type)); + // Generate header. + // for (i = lo; i < hi; i += step) + // <loop-body> + HInstruction* cond = new (global_allocator_) HAboveOrEqual(vector_phi_, hi); + vector_header_->AddPhi(vector_phi_); + vector_header_->AddInstruction(cond); + vector_header_->AddInstruction(new (global_allocator_) HIf(cond)); + // Suspend check and environment. + HInstruction* suspend = vector_header_->GetFirstInstruction(); + suspend->CopyEnvironmentFromWithLoopPhiAdjustment( + node->loop_info->GetSuspendCheck()->GetEnvironment(), vector_header_); + // Generate body. + for (HInstructionIterator it(block->GetInstructions()); !it.Done(); it.Advance()) { + bool vectorized_def = VectorizeDef(node, it.Current(), /*generate_code*/ true); + DCHECK(vectorized_def); + } + for (HInstructionIterator it(block->GetInstructions()); !it.Done(); it.Advance()) { + auto i = vector_map_->find(it.Current()); + if (i != vector_map_->end() && !i->second->IsInBlock()) { + Insert(vector_body_, i->second); // lays out in original order + if (i->second->NeedsEnvironment()) { + i->second->CopyEnvironmentFromWithLoopPhiAdjustment( + suspend->GetEnvironment(), vector_header_); + } + } + } + // Finalize increment and phi. + HInstruction* inc = new (global_allocator_) HAdd(induc_type, vector_phi_, step); + vector_phi_->AddInput(lo); + vector_phi_->AddInput(Insert(vector_body_, inc)); +} + +// TODO: accept reductions at left-hand-side, mixed-type store idioms, etc. +bool HLoopOptimization::VectorizeDef(LoopNode* node, + HInstruction* instruction, + bool generate_code) { + // Accept a left-hand-side array base[index] for + // (1) supported vector type, + // (2) loop-invariant base, + // (3) unit stride index, + // (4) vectorizable right-hand-side value. + uint64_t restrictions = kNone; + if (instruction->IsArraySet()) { + Primitive::Type type = instruction->AsArraySet()->GetComponentType(); + HInstruction* base = instruction->InputAt(0); + HInstruction* index = instruction->InputAt(1); + HInstruction* value = instruction->InputAt(2); + HInstruction* offset = nullptr; + if (TrySetVectorType(type, &restrictions) && + node->loop_info->IsDefinedOutOfTheLoop(base) && + induction_range_.IsUnitStride(index, &offset) && + VectorizeUse(node, value, generate_code, type, restrictions)) { + if (generate_code) { + GenerateVecSub(index, offset); + GenerateVecMem(instruction, vector_map_->Get(index), vector_map_->Get(value), type); + } else { + vector_refs_->insert(ArrayReference(base, offset, type, /*lhs*/ true)); + } return true; } + return false; + } + // Branch back okay. + if (instruction->IsGoto()) { + return true; + } + // Otherwise accept only expressions with no effects outside the immediate loop-body. + // Note that actual uses are inspected during right-hand-side tree traversal. + return !IsUsedOutsideLoop(node->loop_info, instruction) && !instruction->DoesAnyWrite(); +} + +// TODO: more operations and intrinsics, detect saturation arithmetic, etc. +bool HLoopOptimization::VectorizeUse(LoopNode* node, + HInstruction* instruction, + bool generate_code, + Primitive::Type type, + uint64_t restrictions) { + // Accept anything for which code has already been generated. + if (generate_code) { + if (vector_map_->find(instruction) != vector_map_->end()) { + return true; + } + } + // Continue the right-hand-side tree traversal, passing in proper + // types and vector restrictions along the way. During code generation, + // all new nodes are drawn from the global allocator. + if (node->loop_info->IsDefinedOutOfTheLoop(instruction)) { + // Accept invariant use, using scalar expansion. + if (generate_code) { + GenerateVecInv(instruction, type); + } + return true; + } else if (instruction->IsArrayGet()) { + // Accept a right-hand-side array base[index] for + // (1) exact matching vector type, + // (2) loop-invariant base, + // (3) unit stride index, + // (4) vectorizable right-hand-side value. + HInstruction* base = instruction->InputAt(0); + HInstruction* index = instruction->InputAt(1); + HInstruction* offset = nullptr; + if (type == instruction->GetType() && + node->loop_info->IsDefinedOutOfTheLoop(base) && + induction_range_.IsUnitStride(index, &offset)) { + if (generate_code) { + GenerateVecSub(index, offset); + GenerateVecMem(instruction, vector_map_->Get(index), nullptr, type); + } else { + vector_refs_->insert(ArrayReference(base, offset, type, /*lhs*/ false)); + } + return true; + } + } else if (instruction->IsTypeConversion()) { + // Accept particular type conversions. + HTypeConversion* conversion = instruction->AsTypeConversion(); + HInstruction* opa = conversion->InputAt(0); + Primitive::Type from = conversion->GetInputType(); + Primitive::Type to = conversion->GetResultType(); + if ((to == Primitive::kPrimByte || + to == Primitive::kPrimChar || + to == Primitive::kPrimShort) && from == Primitive::kPrimInt) { + // Accept a "narrowing" type conversion from a "wider" computation for + // (1) conversion into final required type, + // (2) vectorizable operand, + // (3) "wider" operations cannot bring in higher order bits. + if (to == type && VectorizeUse(node, opa, generate_code, type, restrictions | kNoHiBits)) { + if (generate_code) { + if (vector_mode_ == kVector) { + vector_map_->Put(instruction, vector_map_->Get(opa)); // operand pass-through + } else { + GenerateVecOp(instruction, vector_map_->Get(opa), nullptr, type); + } + } + return true; + } + } else if (to == Primitive::kPrimFloat && from == Primitive::kPrimInt) { + DCHECK_EQ(to, type); + // Accept int to float conversion for + // (1) supported int, + // (2) vectorizable operand. + if (TrySetVectorType(from, &restrictions) && + VectorizeUse(node, opa, generate_code, from, restrictions)) { + if (generate_code) { + GenerateVecOp(instruction, vector_map_->Get(opa), nullptr, type); + } + return true; + } + } + return false; + } else if (instruction->IsNeg() || instruction->IsNot() || instruction->IsBooleanNot()) { + // Accept unary operator for vectorizable operand. + HInstruction* opa = instruction->InputAt(0); + if (VectorizeUse(node, opa, generate_code, type, restrictions)) { + if (generate_code) { + GenerateVecOp(instruction, vector_map_->Get(opa), nullptr, type); + } + return true; + } + } else if (instruction->IsAdd() || instruction->IsSub() || + instruction->IsMul() || instruction->IsDiv() || + instruction->IsAnd() || instruction->IsOr() || instruction->IsXor()) { + // Deal with vector restrictions. + if ((instruction->IsMul() && HasVectorRestrictions(restrictions, kNoMul)) || + (instruction->IsDiv() && HasVectorRestrictions(restrictions, kNoDiv))) { + return false; + } + // Accept binary operator for vectorizable operands. + HInstruction* opa = instruction->InputAt(0); + HInstruction* opb = instruction->InputAt(1); + if (VectorizeUse(node, opa, generate_code, type, restrictions) && + VectorizeUse(node, opb, generate_code, type, restrictions)) { + if (generate_code) { + GenerateVecOp(instruction, vector_map_->Get(opa), vector_map_->Get(opb), type); + } + return true; + } + } else if (instruction->IsShl() || instruction->IsShr() || instruction->IsUShr()) { + // Deal with vector restrictions. + if ((HasVectorRestrictions(restrictions, kNoShift)) || + (instruction->IsShr() && HasVectorRestrictions(restrictions, kNoShr))) { + return false; // unsupported instruction + } else if ((instruction->IsShr() || instruction->IsUShr()) && + HasVectorRestrictions(restrictions, kNoHiBits)) { + return false; // hibits may impact lobits; TODO: we can do better! + } + // Accept shift operator for vectorizable/invariant operands. + // TODO: accept symbolic, albeit loop invariant shift factors. + HInstruction* opa = instruction->InputAt(0); + HInstruction* opb = instruction->InputAt(1); + if (VectorizeUse(node, opa, generate_code, type, restrictions) && opb->IsIntConstant()) { + if (generate_code) { + // Make sure shift factor only looks at lower bits, as defined for sequential shifts. + // Note that even the narrower SIMD shifts do the right thing after that. + int32_t mask = (instruction->GetType() == Primitive::kPrimLong) + ? kMaxLongShiftDistance + : kMaxIntShiftDistance; + HInstruction* s = graph_->GetIntConstant(opb->AsIntConstant()->GetValue() & mask); + GenerateVecOp(instruction, vector_map_->Get(opa), s, type); + } + return true; + } + } else if (instruction->IsInvokeStaticOrDirect()) { + // TODO: coming soon. + return false; } return false; } -bool HLoopOptimization::IsPhiInduction(HPhi* phi) { +bool HLoopOptimization::TrySetVectorType(Primitive::Type type, uint64_t* restrictions) { + const InstructionSetFeatures* features = compiler_driver_->GetInstructionSetFeatures(); + switch (compiler_driver_->GetInstructionSet()) { + case kArm: + case kThumb2: + return false; + case kArm64: + // Allow vectorization for all ARM devices, because Android assumes that + // ARMv8 AArch64 always supports advanced SIMD. For now, only D registers + // (64-bit vectors) not Q registers (128-bit vectors). + switch (type) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + *restrictions |= kNoDiv; + return TrySetVectorLength(8); + case Primitive::kPrimChar: + case Primitive::kPrimShort: + *restrictions |= kNoDiv; + return TrySetVectorLength(4); + case Primitive::kPrimInt: + *restrictions |= kNoDiv; + return TrySetVectorLength(2); + case Primitive::kPrimFloat: + return TrySetVectorLength(2); + default: + return false; + } + case kX86: + case kX86_64: + // Allow vectorization for SSE4-enabled X86 devices only (128-bit vectors). + if (features->AsX86InstructionSetFeatures()->HasSSE4_1()) { + switch (type) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + *restrictions |= kNoMul | kNoDiv | kNoShift; + return TrySetVectorLength(16); + case Primitive::kPrimChar: + case Primitive::kPrimShort: + *restrictions |= kNoDiv; + return TrySetVectorLength(8); + case Primitive::kPrimInt: + *restrictions |= kNoDiv; + return TrySetVectorLength(4); + case Primitive::kPrimLong: + *restrictions |= kNoMul | kNoDiv | kNoShr; + return TrySetVectorLength(2); + case Primitive::kPrimFloat: + return TrySetVectorLength(4); + case Primitive::kPrimDouble: + return TrySetVectorLength(2); + default: + break; + } // switch type + } + return false; + case kMips: + case kMips64: + // TODO: implement MIPS SIMD. + return false; + default: + return false; + } // switch instruction set +} + +bool HLoopOptimization::TrySetVectorLength(uint32_t length) { + DCHECK(IsPowerOfTwo(length) && length >= 2u); + // First time set? + if (vector_length_ == 0) { + vector_length_ = length; + } + // Different types are acceptable within a loop-body, as long as all the corresponding vector + // lengths match exactly to obtain a uniform traversal through the vector iteration space + // (idiomatic exceptions to this rule can be handled by further unrolling sub-expressions). + return vector_length_ == length; +} + +void HLoopOptimization::GenerateVecInv(HInstruction* org, Primitive::Type type) { + if (vector_map_->find(org) == vector_map_->end()) { + // In scalar code, just use a self pass-through for scalar invariants + // (viz. expression remains itself). + if (vector_mode_ == kSequential) { + vector_map_->Put(org, org); + return; + } + // In vector code, explicit scalar expansion is needed. + HInstruction* vector = new (global_allocator_) HVecReplicateScalar( + global_allocator_, org, type, vector_length_); + vector_map_->Put(org, Insert(vector_preheader_, vector)); + } +} + +void HLoopOptimization::GenerateVecSub(HInstruction* org, HInstruction* offset) { + if (vector_map_->find(org) == vector_map_->end()) { + HInstruction* subscript = vector_phi_; + if (offset != nullptr) { + subscript = new (global_allocator_) HAdd(Primitive::kPrimInt, subscript, offset); + if (org->IsPhi()) { + Insert(vector_body_, subscript); // lacks layout placeholder + } + } + vector_map_->Put(org, subscript); + } +} + +void HLoopOptimization::GenerateVecMem(HInstruction* org, + HInstruction* opa, + HInstruction* opb, + Primitive::Type type) { + HInstruction* vector = nullptr; + if (vector_mode_ == kVector) { + // Vector store or load. + if (opb != nullptr) { + vector = new (global_allocator_) HVecStore( + global_allocator_, org->InputAt(0), opa, opb, type, vector_length_); + } else { + vector = new (global_allocator_) HVecLoad( + global_allocator_, org->InputAt(0), opa, type, vector_length_); + } + } else { + // Scalar store or load. + DCHECK(vector_mode_ == kSequential); + if (opb != nullptr) { + vector = new (global_allocator_) HArraySet(org->InputAt(0), opa, opb, type, kNoDexPc); + } else { + vector = new (global_allocator_) HArrayGet(org->InputAt(0), opa, type, kNoDexPc); + } + } + vector_map_->Put(org, vector); +} + +#define GENERATE_VEC(x, y) \ + if (vector_mode_ == kVector) { \ + vector = (x); \ + } else { \ + DCHECK(vector_mode_ == kSequential); \ + vector = (y); \ + } \ + break; + +void HLoopOptimization::GenerateVecOp(HInstruction* org, + HInstruction* opa, + HInstruction* opb, + Primitive::Type type) { + if (vector_mode_ == kSequential) { + // Scalar code follows implicit integral promotion. + if (type == Primitive::kPrimBoolean || + type == Primitive::kPrimByte || + type == Primitive::kPrimChar || + type == Primitive::kPrimShort) { + type = Primitive::kPrimInt; + } + } + HInstruction* vector = nullptr; + switch (org->GetKind()) { + case HInstruction::kNeg: + DCHECK(opb == nullptr); + GENERATE_VEC( + new (global_allocator_) HVecNeg(global_allocator_, opa, type, vector_length_), + new (global_allocator_) HNeg(type, opa)); + case HInstruction::kNot: + DCHECK(opb == nullptr); + GENERATE_VEC( + new (global_allocator_) HVecNot(global_allocator_, opa, type, vector_length_), + new (global_allocator_) HNot(type, opa)); + case HInstruction::kBooleanNot: + DCHECK(opb == nullptr); + GENERATE_VEC( + new (global_allocator_) HVecNot(global_allocator_, opa, type, vector_length_), + new (global_allocator_) HBooleanNot(opa)); + case HInstruction::kTypeConversion: + DCHECK(opb == nullptr); + GENERATE_VEC( + new (global_allocator_) HVecCnv(global_allocator_, opa, type, vector_length_), + new (global_allocator_) HTypeConversion(type, opa, kNoDexPc)); + case HInstruction::kAdd: + GENERATE_VEC( + new (global_allocator_) HVecAdd(global_allocator_, opa, opb, type, vector_length_), + new (global_allocator_) HAdd(type, opa, opb)); + case HInstruction::kSub: + GENERATE_VEC( + new (global_allocator_) HVecSub(global_allocator_, opa, opb, type, vector_length_), + new (global_allocator_) HSub(type, opa, opb)); + case HInstruction::kMul: + GENERATE_VEC( + new (global_allocator_) HVecMul(global_allocator_, opa, opb, type, vector_length_), + new (global_allocator_) HMul(type, opa, opb)); + case HInstruction::kDiv: + GENERATE_VEC( + new (global_allocator_) HVecDiv(global_allocator_, opa, opb, type, vector_length_), + new (global_allocator_) HDiv(type, opa, opb, kNoDexPc)); + case HInstruction::kAnd: + GENERATE_VEC( + new (global_allocator_) HVecAnd(global_allocator_, opa, opb, type, vector_length_), + new (global_allocator_) HAnd(type, opa, opb)); + case HInstruction::kOr: + GENERATE_VEC( + new (global_allocator_) HVecOr(global_allocator_, opa, opb, type, vector_length_), + new (global_allocator_) HOr(type, opa, opb)); + case HInstruction::kXor: + GENERATE_VEC( + new (global_allocator_) HVecXor(global_allocator_, opa, opb, type, vector_length_), + new (global_allocator_) HXor(type, opa, opb)); + case HInstruction::kShl: + GENERATE_VEC( + new (global_allocator_) HVecShl(global_allocator_, opa, opb, type, vector_length_), + new (global_allocator_) HShl(type, opa, opb)); + case HInstruction::kShr: + GENERATE_VEC( + new (global_allocator_) HVecShr(global_allocator_, opa, opb, type, vector_length_), + new (global_allocator_) HShr(type, opa, opb)); + case HInstruction::kUShr: + GENERATE_VEC( + new (global_allocator_) HVecUShr(global_allocator_, opa, opb, type, vector_length_), + new (global_allocator_) HUShr(type, opa, opb)); + case HInstruction::kInvokeStaticOrDirect: { + // TODO: coming soon. + break; + } + default: + break; + } // switch + CHECK(vector != nullptr) << "Unsupported SIMD operator"; + vector_map_->Put(org, vector); +} + +#undef GENERATE_VEC + +// +// Helpers. +// + +bool HLoopOptimization::TrySetPhiInduction(HPhi* phi, bool restrict_uses) { + DCHECK(iset_->empty()); ArenaSet<HInstruction*>* set = induction_range_.LookupCycle(phi); if (set != nullptr) { - DCHECK(iset_->empty()); for (HInstruction* i : *set) { // Check that, other than instructions that are no longer in the graph (removed earlier) - // each instruction is removable and, other than the phi, uses are contained in the cycle. + // each instruction is removable and, when restrict uses are requested, other than for phi, + // all uses are contained within the cycle. if (!i->IsInBlock()) { continue; } else if (!i->IsRemovable()) { return false; - } else if (i != phi) { + } else if (i != phi && restrict_uses) { for (const HUseListNode<HInstruction*>& use : i->GetUses()) { if (set->find(use.GetUser()) == set->end()) { return false; @@ -348,10 +1003,12 @@ bool HLoopOptimization::IsPhiInduction(HPhi* phi) { // c: Condition(phi, bound) // i: If(c) // TODO: Find a less pattern matching approach? -bool HLoopOptimization::IsEmptyHeader(HBasicBlock* block) { +bool HLoopOptimization::TrySetSimpleLoopHeader(HBasicBlock* block) { DCHECK(iset_->empty()); HInstruction* phi = block->GetFirstPhi(); - if (phi != nullptr && phi->GetNext() == nullptr && IsPhiInduction(phi->AsPhi())) { + if (phi != nullptr && + phi->GetNext() == nullptr && + TrySetPhiInduction(phi->AsPhi(), /*restrict_uses*/ false)) { HInstruction* s = block->GetFirstInstruction(); if (s != nullptr && s->IsSuspendCheck()) { HInstruction* c = s->GetNext(); @@ -369,14 +1026,24 @@ bool HLoopOptimization::IsEmptyHeader(HBasicBlock* block) { } bool HLoopOptimization::IsEmptyBody(HBasicBlock* block) { - if (block->GetFirstPhi() == nullptr) { - for (HInstructionIterator it(block->GetInstructions()); !it.Done(); it.Advance()) { - HInstruction* instruction = it.Current(); - if (!instruction->IsGoto() && iset_->find(instruction) == iset_->end()) { - return false; - } + if (!block->GetPhis().IsEmpty()) { + return false; + } + for (HInstructionIterator it(block->GetInstructions()); !it.Done(); it.Advance()) { + HInstruction* instruction = it.Current(); + if (!instruction->IsGoto() && iset_->find(instruction) == iset_->end()) { + return false; + } + } + return true; +} + +bool HLoopOptimization::IsUsedOutsideLoop(HLoopInformation* loop_info, + HInstruction* instruction) { + for (const HUseListNode<HInstruction*>& use : instruction->GetUses()) { + if (use.GetUser()->GetBlock()->GetLoopInformation() != loop_info) { + return true; } - return true; } return false; } @@ -438,6 +1105,19 @@ bool HLoopOptimization::TryReplaceWithLastValue(HInstruction* instruction, HBasi return false; } +bool HLoopOptimization::TryAssignLastValue(HLoopInformation* loop_info, + HInstruction* instruction, + HBasicBlock* block, + bool collect_loop_uses) { + // Assigning the last value is always successful if there are no uses. + // Otherwise, it succeeds in a no early-exit loop by generating the + // proper last value assignment. + int32_t use_count = 0; + return IsOnlyUsedAfterLoop(loop_info, instruction, collect_loop_uses, &use_count) && + (use_count == 0 || + (!IsEarlyExit(loop_info) && TryReplaceWithLastValue(instruction, block))); +} + void HLoopOptimization::RemoveDeadInstructions(const HInstructionList& list) { for (HBackwardInstructionIterator i(list); !i.Done(); i.Advance()) { HInstruction* instruction = i.Current(); diff --git a/compiler/optimizing/loop_optimization.h b/compiler/optimizing/loop_optimization.h index 0b798fc7a9..16f7691af2 100644 --- a/compiler/optimizing/loop_optimization.h +++ b/compiler/optimizing/loop_optimization.h @@ -27,7 +27,8 @@ class CompilerDriver; /** * Loop optimizations. Builds a loop hierarchy and applies optimizations to - * the detected nested loops, such as removal of dead induction and empty loops. + * the detected nested loops, such as removal of dead induction and empty loops + * and inner loop vectorization. */ class HLoopOptimization : public HOptimization { public: @@ -50,34 +51,105 @@ class HLoopOptimization : public HOptimization { inner(nullptr), previous(nullptr), next(nullptr) {} - HLoopInformation* const loop_info; + HLoopInformation* loop_info; LoopNode* outer; LoopNode* inner; LoopNode* previous; LoopNode* next; }; - void LocalRun(); + /* + * Vectorization restrictions (bit mask). + */ + enum VectorRestrictions { + kNone = 0, // no restrictions + kNoMul = 1, // no multiplication + kNoDiv = 2, // no division + kNoShift = 4, // no shift + kNoShr = 8, // no arithmetic shift right + kNoHiBits = 16, // "wider" operations cannot bring in higher order bits + }; + + /* + * Vectorization mode during synthesis + * (sequential peeling/cleanup loop or vector loop). + */ + enum VectorMode { + kSequential, + kVector + }; + + /* + * Representation of a unit-stride array reference. + */ + struct ArrayReference { + ArrayReference(HInstruction* b, HInstruction* o, Primitive::Type t, bool l) + : base(b), offset(o), type(t), lhs(l) { } + bool operator<(const ArrayReference& other) const { + return + (base < other.base) || + (base == other.base && + (offset < other.offset || (offset == other.offset && + (type < other.type || + (type == other.type && lhs < other.lhs))))); + } + HInstruction* base; // base address + HInstruction* offset; // offset + i + Primitive::Type type; // component type + bool lhs; // def/use + }; + // Loop setup and traversal. + void LocalRun(); void AddLoop(HLoopInformation* loop_info); void RemoveLoop(LoopNode* node); - void TraverseLoopsInnerToOuter(LoopNode* node); - // Simplification. + // Optimization. void SimplifyInduction(LoopNode* node); void SimplifyBlocks(LoopNode* node); - bool SimplifyInnerLoop(LoopNode* node); + void OptimizeInnerLoop(LoopNode* node); + + // Vectorization analysis and synthesis. + bool CanVectorize(LoopNode* node, HBasicBlock* block, int64_t trip_count); + void Vectorize(LoopNode* node, HBasicBlock* block, HBasicBlock* exit, int64_t trip_count); + void GenerateNewLoop(LoopNode* node, + HBasicBlock* block, + HBasicBlock* new_preheader, + HInstruction* lo, + HInstruction* hi, + HInstruction* step); + bool VectorizeDef(LoopNode* node, HInstruction* instruction, bool generate_code); + bool VectorizeUse(LoopNode* node, + HInstruction* instruction, + bool generate_code, + Primitive::Type type, + uint64_t restrictions); + bool TrySetVectorType(Primitive::Type type, /*out*/ uint64_t* restrictions); + bool TrySetVectorLength(uint32_t length); + void GenerateVecInv(HInstruction* org, Primitive::Type type); + void GenerateVecSub(HInstruction* org, HInstruction* off); + void GenerateVecMem(HInstruction* org, + HInstruction* opa, + HInstruction* opb, + Primitive::Type type); + void GenerateVecOp(HInstruction* org, HInstruction* opa, HInstruction* opb, Primitive::Type type); // Helpers. - bool IsPhiInduction(HPhi* phi); - bool IsEmptyHeader(HBasicBlock* block); + bool TrySetPhiInduction(HPhi* phi, bool restrict_uses); + bool TrySetSimpleLoopHeader(HBasicBlock* block); bool IsEmptyBody(HBasicBlock* block); bool IsOnlyUsedAfterLoop(HLoopInformation* loop_info, HInstruction* instruction, bool collect_loop_uses, /*out*/ int32_t* use_count); + bool IsUsedOutsideLoop(HLoopInformation* loop_info, + HInstruction* instruction); bool TryReplaceWithLastValue(HInstruction* instruction, HBasicBlock* block); + bool TryAssignLastValue(HLoopInformation* loop_info, + HInstruction* instruction, + HBasicBlock* block, + bool collect_loop_uses); void RemoveDeadInstructions(const HInstructionList& list); // Compiler driver (to query ISA features). @@ -90,6 +162,9 @@ class HLoopOptimization : public HOptimization { // through this allocator is immediately released when the loop optimizer is done. ArenaAllocator* loop_allocator_; + // Global heap memory allocator. Used to build HIR. + ArenaAllocator* global_allocator_; + // Entries into the loop hierarchy representation. The hierarchy resides // in phase-local heap memory. LoopNode* top_loop_; @@ -102,11 +177,33 @@ class HLoopOptimization : public HOptimization { // Counter that tracks how many induction cycles have been simplified. Useful // to trigger incremental updates of induction variable analysis of outer loops // when the induction of inner loops has changed. - int32_t induction_simplication_count_; + uint32_t induction_simplication_count_; // Flag that tracks if any simplifications have occurred. bool simplified_; + // Number of "lanes" for selected packed type. + uint32_t vector_length_; + + // Set of array references in the vector loop. + // Contents reside in phase-local heap memory. + ArenaSet<ArrayReference>* vector_refs_; + + // Mapping used during vectorization synthesis for both the scalar peeling/cleanup + // loop (simd_ is false) and the actual vector loop (simd_ is true). The data + // structure maps original instructions into the new instructions. + // Contents reside in phase-local heap memory. + ArenaSafeMap<HInstruction*, HInstruction*>* vector_map_; + + // Temporary vectorization bookkeeping. + HBasicBlock* vector_preheader_; // preheader of the new loop + HBasicBlock* vector_header_; // header of the new loop + HBasicBlock* vector_body_; // body of the new loop + HInstruction* vector_runtime_test_a_; + HInstruction* vector_runtime_test_b_; // defines a != b runtime test + HPhi* vector_phi_; // the Phi representing the normalized loop index + VectorMode vector_mode_; // selects synthesis mode + friend class LoopOptimizationTest; DISALLOW_COPY_AND_ASSIGN(HLoopOptimization); diff --git a/compiler/optimizing/nodes.cc b/compiler/optimizing/nodes.cc index ec706e6694..5617e4bfcb 100644 --- a/compiler/optimizing/nodes.cc +++ b/compiler/optimizing/nodes.cc @@ -1088,6 +1088,19 @@ void HInstruction::ReplaceWith(HInstruction* other) { DCHECK(env_uses_.empty()); } +void HInstruction::ReplaceUsesDominatedBy(HInstruction* dominator, HInstruction* replacement) { + const HUseList<HInstruction*>& uses = GetUses(); + for (auto it = uses.begin(), end = uses.end(); it != end; /* ++it below */) { + HInstruction* user = it->GetUser(); + size_t index = it->GetIndex(); + // Increment `it` now because `*it` may disappear thanks to user->ReplaceInput(). + ++it; + if (dominator->StrictlyDominates(user)) { + user->ReplaceInput(replacement, index); + } + } +} + void HInstruction::ReplaceInput(HInstruction* replacement, size_t index) { HUserRecord<HInstruction*> input_use = InputRecordAt(index); if (input_use.GetInstruction() == replacement) { @@ -1323,6 +1336,18 @@ std::ostream& operator<<(std::ostream& os, const ComparisonBias& rhs) { } } +std::ostream& operator<<(std::ostream& os, const HDeoptimize::Kind& rhs) { + switch (rhs) { + case HDeoptimize::Kind::kBCE: + return os << "bce"; + case HDeoptimize::Kind::kInline: + return os << "inline"; + default: + LOG(FATAL) << "Unknown Deoptimization kind: " << static_cast<int>(rhs); + UNREACHABLE(); + } +} + bool HCondition::IsBeforeWhenDisregardMoves(HInstruction* instruction) const { return this == instruction->GetPreviousDisregardingMoves(); } @@ -2315,6 +2340,66 @@ void HGraph::TransformLoopHeaderForBCE(HBasicBlock* header) { new_pre_header, old_pre_header, /* replace_if_back_edge */ false); } +HBasicBlock* HGraph::TransformLoopForVectorization(HBasicBlock* header, + HBasicBlock* body, + HBasicBlock* exit) { + DCHECK(header->IsLoopHeader()); + HLoopInformation* loop = header->GetLoopInformation(); + + // Add new loop blocks. + HBasicBlock* new_pre_header = new (arena_) HBasicBlock(this, header->GetDexPc()); + HBasicBlock* new_header = new (arena_) HBasicBlock(this, header->GetDexPc()); + HBasicBlock* new_body = new (arena_) HBasicBlock(this, header->GetDexPc()); + AddBlock(new_pre_header); + AddBlock(new_header); + AddBlock(new_body); + + // Set up control flow. + header->ReplaceSuccessor(exit, new_pre_header); + new_pre_header->AddSuccessor(new_header); + new_header->AddSuccessor(exit); + new_header->AddSuccessor(new_body); + new_body->AddSuccessor(new_header); + + // Set up dominators. + header->ReplaceDominatedBlock(exit, new_pre_header); + new_pre_header->SetDominator(header); + new_pre_header->dominated_blocks_.push_back(new_header); + new_header->SetDominator(new_pre_header); + new_header->dominated_blocks_.push_back(new_body); + new_body->SetDominator(new_header); + new_header->dominated_blocks_.push_back(exit); + exit->SetDominator(new_header); + + // Fix reverse post order. + size_t index_of_header = IndexOfElement(reverse_post_order_, header); + MakeRoomFor(&reverse_post_order_, 2, index_of_header); + reverse_post_order_[++index_of_header] = new_pre_header; + reverse_post_order_[++index_of_header] = new_header; + size_t index_of_body = IndexOfElement(reverse_post_order_, body); + MakeRoomFor(&reverse_post_order_, 1, index_of_body - 1); + reverse_post_order_[index_of_body] = new_body; + + // Add gotos and suspend check (client must add conditional in header and copy environment). + new_pre_header->AddInstruction(new (arena_) HGoto()); + HSuspendCheck* suspend_check = new (arena_) HSuspendCheck(header->GetDexPc()); + new_header->AddInstruction(suspend_check); + new_body->AddInstruction(new (arena_) HGoto()); + + // Update loop information. + new_header->AddBackEdge(new_body); + new_header->GetLoopInformation()->SetSuspendCheck(suspend_check); + new_header->GetLoopInformation()->Populate(); + new_pre_header->SetLoopInformation(loop->GetPreHeader()->GetLoopInformation()); // outward + HLoopInformationOutwardIterator it(*new_header); + for (it.Advance(); !it.Done(); it.Advance()) { + it.Current()->Add(new_pre_header); + it.Current()->Add(new_header); + it.Current()->Add(new_body); + } + return new_pre_header; +} + static void CheckAgainstUpperBound(ReferenceTypeInfo rti, ReferenceTypeInfo upper_bound_rti) REQUIRES_SHARED(Locks::mutator_lock_) { if (rti.IsValid()) { diff --git a/compiler/optimizing/nodes.h b/compiler/optimizing/nodes.h index fb0c889792..52a02c2285 100644 --- a/compiler/optimizing/nodes.h +++ b/compiler/optimizing/nodes.h @@ -400,6 +400,12 @@ class HGraph : public ArenaObject<kArenaAllocGraph> { // put deoptimization instructions, etc. void TransformLoopHeaderForBCE(HBasicBlock* header); + // Adds a new loop directly after the loop with the given header and exit. + // Returns the new preheader. + HBasicBlock* TransformLoopForVectorization(HBasicBlock* header, + HBasicBlock* body, + HBasicBlock* exit); + // Removes `block` from the graph. Assumes `block` has been disconnected from // other blocks and has no instructions or phis. void DeleteDeadEmptyBlock(HBasicBlock* block); @@ -1363,6 +1369,25 @@ class HLoopInformationOutwardIterator : public ValueObject { M(TypeConversion, Instruction) \ M(UShr, BinaryOperation) \ M(Xor, BinaryOperation) \ + M(VecReplicateScalar, VecUnaryOperation) \ + M(VecSetScalars, VecUnaryOperation) \ + M(VecSumReduce, VecUnaryOperation) \ + M(VecCnv, VecUnaryOperation) \ + M(VecNeg, VecUnaryOperation) \ + M(VecNot, VecUnaryOperation) \ + M(VecAdd, VecBinaryOperation) \ + M(VecSub, VecBinaryOperation) \ + M(VecMul, VecBinaryOperation) \ + M(VecDiv, VecBinaryOperation) \ + M(VecAnd, VecBinaryOperation) \ + M(VecAndNot, VecBinaryOperation) \ + M(VecOr, VecBinaryOperation) \ + M(VecXor, VecBinaryOperation) \ + M(VecShl, VecBinaryOperation) \ + M(VecShr, VecBinaryOperation) \ + M(VecUShr, VecBinaryOperation) \ + M(VecLoad, VecMemoryOperation) \ + M(VecStore, VecMemoryOperation) \ /* * Instructions, shared across several (not all) architectures. @@ -1424,7 +1449,11 @@ class HLoopInformationOutwardIterator : public ValueObject { M(Constant, Instruction) \ M(UnaryOperation, Instruction) \ M(BinaryOperation, Instruction) \ - M(Invoke, Instruction) + M(Invoke, Instruction) \ + M(VecOperation, Instruction) \ + M(VecUnaryOperation, VecOperation) \ + M(VecBinaryOperation, VecOperation) \ + M(VecMemoryOperation, VecOperation) #define FOR_EACH_INSTRUCTION(M) \ FOR_EACH_CONCRETE_INSTRUCTION(M) \ @@ -2081,6 +2110,7 @@ class HInstruction : public ArenaObject<kArenaAllocInstruction> { void SetLocations(LocationSummary* locations) { locations_ = locations; } void ReplaceWith(HInstruction* instruction); + void ReplaceUsesDominatedBy(HInstruction* dominator, HInstruction* replacement); void ReplaceInput(HInstruction* replacement, size_t index); // This is almost the same as doing `ReplaceWith()`. But in this helper, the @@ -2944,28 +2974,97 @@ class HTryBoundary FINAL : public HTemplateInstruction<0> { }; // Deoptimize to interpreter, upon checking a condition. -class HDeoptimize FINAL : public HTemplateInstruction<1> { +class HDeoptimize FINAL : public HVariableInputSizeInstruction { public: + enum class Kind { + kBCE, + kInline, + kLast = kInline + }; + + // Use this constructor when the `HDeoptimize` acts as a barrier, where no code can move + // across. + HDeoptimize(ArenaAllocator* arena, HInstruction* cond, Kind kind, uint32_t dex_pc) + : HVariableInputSizeInstruction( + SideEffects::All(), + dex_pc, + arena, + /* number_of_inputs */ 1, + kArenaAllocMisc) { + SetPackedFlag<kFieldCanBeMoved>(false); + SetPackedField<DeoptimizeKindField>(kind); + SetRawInputAt(0, cond); + } + + // Use this constructor when the `HDeoptimize` guards an instruction, and any user + // that relies on the deoptimization to pass should have its input be the `HDeoptimize` + // instead of `guard`. // We set CanTriggerGC to prevent any intermediate address to be live // at the point of the `HDeoptimize`. - HDeoptimize(HInstruction* cond, uint32_t dex_pc) - : HTemplateInstruction(SideEffects::CanTriggerGC(), dex_pc) { + HDeoptimize(ArenaAllocator* arena, + HInstruction* cond, + HInstruction* guard, + Kind kind, + uint32_t dex_pc) + : HVariableInputSizeInstruction( + SideEffects::CanTriggerGC(), + dex_pc, + arena, + /* number_of_inputs */ 2, + kArenaAllocMisc) { + SetPackedFlag<kFieldCanBeMoved>(true); + SetPackedField<DeoptimizeKindField>(kind); SetRawInputAt(0, cond); + SetRawInputAt(1, guard); } - bool CanBeMoved() const OVERRIDE { return true; } - bool InstructionDataEquals(const HInstruction* other ATTRIBUTE_UNUSED) const OVERRIDE { - return true; + bool CanBeMoved() const OVERRIDE { return GetPackedFlag<kFieldCanBeMoved>(); } + + bool InstructionDataEquals(const HInstruction* other) const OVERRIDE { + return (other->CanBeMoved() == CanBeMoved()) && (other->AsDeoptimize()->GetKind() == GetKind()); } + bool NeedsEnvironment() const OVERRIDE { return true; } + bool CanThrow() const OVERRIDE { return true; } + Kind GetKind() const { return GetPackedField<DeoptimizeKindField>(); } + + Primitive::Type GetType() const OVERRIDE { + return GuardsAnInput() ? GuardedInput()->GetType() : Primitive::kPrimVoid; + } + + bool GuardsAnInput() const { + return InputCount() == 2; + } + + HInstruction* GuardedInput() const { + DCHECK(GuardsAnInput()); + return InputAt(1); + } + + void RemoveGuard() { + RemoveInputAt(1); + } + DECLARE_INSTRUCTION(Deoptimize); private: + static constexpr size_t kFieldCanBeMoved = kNumberOfGenericPackedBits; + static constexpr size_t kFieldDeoptimizeKind = kNumberOfGenericPackedBits + 1; + static constexpr size_t kFieldDeoptimizeKindSize = + MinimumBitsToStore(static_cast<size_t>(Kind::kLast)); + static constexpr size_t kNumberOfDeoptimizePackedBits = + kFieldDeoptimizeKind + kFieldDeoptimizeKindSize; + static_assert(kNumberOfDeoptimizePackedBits <= kMaxNumberOfPackedBits, + "Too many packed fields."); + using DeoptimizeKindField = BitField<Kind, kFieldDeoptimizeKind, kFieldDeoptimizeKindSize>; + DISALLOW_COPY_AND_ASSIGN(HDeoptimize); }; +std::ostream& operator<<(std::ostream& os, const HDeoptimize::Kind& rhs); + // Represents a should_deoptimize flag. Currently used for CHA-based devirtualization. // The compiled code checks this flag value in a guard before devirtualized call and // if it's true, starts to do deoptimization. @@ -6619,6 +6718,8 @@ class HParallelMove FINAL : public HTemplateInstruction<0> { } // namespace art +#include "nodes_vector.h" + #if defined(ART_ENABLE_CODEGEN_arm) || defined(ART_ENABLE_CODEGEN_arm64) #include "nodes_shared.h" #endif diff --git a/compiler/optimizing/nodes_vector.h b/compiler/optimizing/nodes_vector.h new file mode 100644 index 0000000000..9f9b918f17 --- /dev/null +++ b/compiler/optimizing/nodes_vector.h @@ -0,0 +1,585 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_COMPILER_OPTIMIZING_NODES_VECTOR_H_ +#define ART_COMPILER_OPTIMIZING_NODES_VECTOR_H_ + +// This #include should never be used by compilation, because this header file (nodes_vector.h) +// is included in the header file nodes.h itself. However it gives editing tools better context. +#include "nodes.h" + +namespace art { + +// Memory alignment, represented as an offset relative to a base, where 0 <= offset < base, +// and base is a power of two. For example, the value Alignment(16, 0) means memory is +// perfectly aligned at a 16-byte boundary, whereas the value Alignment(16, 4) means +// memory is always exactly 4 bytes above such a boundary. +class Alignment { + public: + Alignment(size_t base, size_t offset) : base_(base), offset_(offset) { + DCHECK_LT(offset, base); + DCHECK(IsPowerOfTwo(base)); + } + + // Returns true if memory is "at least" aligned at the given boundary. + // Assumes requested base is power of two. + bool IsAlignedAt(size_t base) const { + DCHECK_NE(0u, base); + DCHECK(IsPowerOfTwo(base)); + return ((offset_ | base_) & (base - 1u)) == 0; + } + + std::string ToString() const { + return "ALIGN(" + std::to_string(base_) + "," + std::to_string(offset_) + ")"; + } + + private: + size_t base_; + size_t offset_; +}; + +// +// Definitions of abstract vector operations in HIR. +// + +// Abstraction of a vector operation, i.e., an operation that performs +// GetVectorLength() x GetPackedType() operations simultaneously. +class HVecOperation : public HVariableInputSizeInstruction { + public: + HVecOperation(ArenaAllocator* arena, + Primitive::Type packed_type, + SideEffects side_effects, + size_t number_of_inputs, + size_t vector_length, + uint32_t dex_pc) + : HVariableInputSizeInstruction(side_effects, + dex_pc, + arena, + number_of_inputs, + kArenaAllocVectorNode), + vector_length_(vector_length) { + SetPackedField<TypeField>(packed_type); + DCHECK_LT(1u, vector_length); + } + + // Returns the number of elements packed in a vector. + size_t GetVectorLength() const { + return vector_length_; + } + + // Returns the number of bytes in a full vector. + size_t GetVectorNumberOfBytes() const { + return vector_length_ * Primitive::ComponentSize(GetPackedType()); + } + + // Returns the type of the vector operation: a SIMD operation looks like a FPU location. + // TODO: we could introduce SIMD types in HIR. + Primitive::Type GetType() const OVERRIDE { + return Primitive::kPrimDouble; + } + + // Returns the true component type packed in a vector. + Primitive::Type GetPackedType() const { + return GetPackedField<TypeField>(); + } + + DECLARE_ABSTRACT_INSTRUCTION(VecOperation); + + private: + // Additional packed bits. + static constexpr size_t kFieldType = HInstruction::kNumberOfGenericPackedBits; + static constexpr size_t kFieldTypeSize = + MinimumBitsToStore(static_cast<size_t>(Primitive::kPrimLast)); + static constexpr size_t kNumberOfVectorOpPackedBits = kFieldType + kFieldTypeSize; + static_assert(kNumberOfVectorOpPackedBits <= kMaxNumberOfPackedBits, "Too many packed fields."); + using TypeField = BitField<Primitive::Type, kFieldType, kFieldTypeSize>; + + const size_t vector_length_; + + DISALLOW_COPY_AND_ASSIGN(HVecOperation); +}; + +// Abstraction of a unary vector operation. +class HVecUnaryOperation : public HVecOperation { + public: + HVecUnaryOperation(ArenaAllocator* arena, + Primitive::Type packed_type, + size_t vector_length, + uint32_t dex_pc) + : HVecOperation(arena, + packed_type, + SideEffects::None(), + /*number_of_inputs*/ 1, + vector_length, + dex_pc) { } + DECLARE_ABSTRACT_INSTRUCTION(VecUnaryOperation); + private: + DISALLOW_COPY_AND_ASSIGN(HVecUnaryOperation); +}; + +// Abstraction of a binary vector operation. +class HVecBinaryOperation : public HVecOperation { + public: + HVecBinaryOperation(ArenaAllocator* arena, + Primitive::Type packed_type, + size_t vector_length, + uint32_t dex_pc) + : HVecOperation(arena, + packed_type, + SideEffects::None(), + /*number_of_inputs*/ 2, + vector_length, + dex_pc) { } + DECLARE_ABSTRACT_INSTRUCTION(VecBinaryOperation); + private: + DISALLOW_COPY_AND_ASSIGN(HVecBinaryOperation); +}; + +// Abstraction of a vector operation that references memory, with an alignment. +// The Android runtime guarantees at least "component size" alignment for array +// elements and, thus, vectors. +class HVecMemoryOperation : public HVecOperation { + public: + HVecMemoryOperation(ArenaAllocator* arena, + Primitive::Type packed_type, + SideEffects side_effects, + size_t number_of_inputs, + size_t vector_length, + uint32_t dex_pc) + : HVecOperation(arena, packed_type, side_effects, number_of_inputs, vector_length, dex_pc), + alignment_(Primitive::ComponentSize(packed_type), 0) { } + + void SetAlignment(Alignment alignment) { alignment_ = alignment; } + + Alignment GetAlignment() const { return alignment_; } + + DECLARE_ABSTRACT_INSTRUCTION(VecMemoryOperation); + + private: + Alignment alignment_; + + DISALLOW_COPY_AND_ASSIGN(HVecMemoryOperation); +}; + +// +// Definitions of concrete vector operations in HIR. +// + +// Replicates the given scalar into a vector, +// viz. replicate(x) = [ x, .. , x ]. +class HVecReplicateScalar FINAL : public HVecUnaryOperation { + public: + HVecReplicateScalar(ArenaAllocator* arena, + HInstruction* scalar, + Primitive::Type packed_type, + size_t vector_length, + uint32_t dex_pc = kNoDexPc) + : HVecUnaryOperation(arena, packed_type, vector_length, dex_pc) { + SetRawInputAt(0, scalar); + } + DECLARE_INSTRUCTION(VecReplicateScalar); + private: + DISALLOW_COPY_AND_ASSIGN(HVecReplicateScalar); +}; + +// Assigns the given scalar elements to a vector, +// viz. set( array(x1, .., xn) ) = [ x1, .. , xn ]. +class HVecSetScalars FINAL : public HVecUnaryOperation { + HVecSetScalars(ArenaAllocator* arena, + HInstruction** scalars, // array + Primitive::Type packed_type, + size_t vector_length, + uint32_t dex_pc = kNoDexPc) + : HVecUnaryOperation(arena, packed_type, vector_length, dex_pc) { + for (size_t i = 0; i < vector_length; i++) { + SetRawInputAt(0, scalars[i]); + } + } + DECLARE_INSTRUCTION(VecSetScalars); + private: + DISALLOW_COPY_AND_ASSIGN(HVecSetScalars); +}; + +// Sum-reduces the given vector into a shorter vector (m < n) or scalar (m = 1), +// viz. sum-reduce[ x1, .. , xn ] = [ y1, .., ym ], where yi = sum_j x_j. +class HVecSumReduce FINAL : public HVecUnaryOperation { + HVecSumReduce(ArenaAllocator* arena, + HInstruction* input, + Primitive::Type packed_type, + size_t vector_length, + uint32_t dex_pc = kNoDexPc) + : HVecUnaryOperation(arena, packed_type, vector_length, dex_pc) { + DCHECK(input->IsVecOperation()); + DCHECK_EQ(input->AsVecOperation()->GetPackedType(), packed_type); + SetRawInputAt(0, input); + } + + // TODO: probably integral promotion + Primitive::Type GetType() const OVERRIDE { return GetPackedType(); } + + DECLARE_INSTRUCTION(VecSumReduce); + private: + DISALLOW_COPY_AND_ASSIGN(HVecSumReduce); +}; + +// Converts every component in the vector, +// viz. cnv[ x1, .. , xn ] = [ cnv(x1), .. , cnv(xn) ]. +class HVecCnv FINAL : public HVecUnaryOperation { + public: + HVecCnv(ArenaAllocator* arena, + HInstruction* input, + Primitive::Type packed_type, + size_t vector_length, + uint32_t dex_pc = kNoDexPc) + : HVecUnaryOperation(arena, packed_type, vector_length, dex_pc) { + DCHECK(input->IsVecOperation()); + DCHECK_NE(input->AsVecOperation()->GetPackedType(), packed_type); // actual convert + SetRawInputAt(0, input); + } + + Primitive::Type GetInputType() const { return InputAt(0)->AsVecOperation()->GetPackedType(); } + Primitive::Type GetResultType() const { return GetPackedType(); } + + DECLARE_INSTRUCTION(VecCnv); + + private: + DISALLOW_COPY_AND_ASSIGN(HVecCnv); +}; + +// Negates every component in the vector, +// viz. neg[ x1, .. , xn ] = [ -x1, .. , -xn ]. +class HVecNeg FINAL : public HVecUnaryOperation { + public: + HVecNeg(ArenaAllocator* arena, + HInstruction* input, + Primitive::Type packed_type, + size_t vector_length, + uint32_t dex_pc = kNoDexPc) + : HVecUnaryOperation(arena, packed_type, vector_length, dex_pc) { + DCHECK(input->IsVecOperation()); + DCHECK_EQ(input->AsVecOperation()->GetPackedType(), packed_type); + SetRawInputAt(0, input); + } + DECLARE_INSTRUCTION(VecNeg); + private: + DISALLOW_COPY_AND_ASSIGN(HVecNeg); +}; + +// Bitwise- or boolean-nots every component in the vector, +// viz. not[ x1, .. , xn ] = [ ~x1, .. , ~xn ], or +// not[ x1, .. , xn ] = [ !x1, .. , !xn ] for boolean. +class HVecNot FINAL : public HVecUnaryOperation { + public: + HVecNot(ArenaAllocator* arena, + HInstruction* input, + Primitive::Type packed_type, + size_t vector_length, + uint32_t dex_pc = kNoDexPc) + : HVecUnaryOperation(arena, packed_type, vector_length, dex_pc) { + DCHECK(input->IsVecOperation()); + SetRawInputAt(0, input); + } + DECLARE_INSTRUCTION(VecNot); + private: + DISALLOW_COPY_AND_ASSIGN(HVecNot); +}; + +// Adds every component in the two vectors, +// viz. [ x1, .. , xn ] + [ y1, .. , yn ] = [ x1 + y1, .. , xn + yn ]. +class HVecAdd FINAL : public HVecBinaryOperation { + public: + HVecAdd(ArenaAllocator* arena, + HInstruction* left, + HInstruction* right, + Primitive::Type packed_type, + size_t vector_length, + uint32_t dex_pc = kNoDexPc) + : HVecBinaryOperation(arena, packed_type, vector_length, dex_pc) { + DCHECK(left->IsVecOperation() && right->IsVecOperation()); + DCHECK_EQ(left->AsVecOperation()->GetPackedType(), packed_type); + DCHECK_EQ(right->AsVecOperation()->GetPackedType(), packed_type); + SetRawInputAt(0, left); + SetRawInputAt(1, right); + } + DECLARE_INSTRUCTION(VecAdd); + private: + DISALLOW_COPY_AND_ASSIGN(HVecAdd); +}; + +// Subtracts every component in the two vectors, +// viz. [ x1, .. , xn ] - [ y1, .. , yn ] = [ x1 - y1, .. , xn - yn ]. +class HVecSub FINAL : public HVecBinaryOperation { + public: + HVecSub(ArenaAllocator* arena, + HInstruction* left, + HInstruction* right, + Primitive::Type packed_type, + size_t vector_length, + uint32_t dex_pc = kNoDexPc) + : HVecBinaryOperation(arena, packed_type, vector_length, dex_pc) { + DCHECK(left->IsVecOperation() && right->IsVecOperation()); + DCHECK_EQ(left->AsVecOperation()->GetPackedType(), packed_type); + DCHECK_EQ(right->AsVecOperation()->GetPackedType(), packed_type); + SetRawInputAt(0, left); + SetRawInputAt(1, right); + } + DECLARE_INSTRUCTION(VecSub); + private: + DISALLOW_COPY_AND_ASSIGN(HVecSub); +}; + +// Multiplies every component in the two vectors, +// viz. [ x1, .. , xn ] * [ y1, .. , yn ] = [ x1 * y1, .. , xn * yn ]. +class HVecMul FINAL : public HVecBinaryOperation { + public: + HVecMul(ArenaAllocator* arena, + HInstruction* left, + HInstruction* right, + Primitive::Type packed_type, + size_t vector_length, + uint32_t dex_pc = kNoDexPc) + : HVecBinaryOperation(arena, packed_type, vector_length, dex_pc) { + DCHECK(left->IsVecOperation() && right->IsVecOperation()); + DCHECK_EQ(left->AsVecOperation()->GetPackedType(), packed_type); + DCHECK_EQ(right->AsVecOperation()->GetPackedType(), packed_type); + SetRawInputAt(0, left); + SetRawInputAt(1, right); + } + DECLARE_INSTRUCTION(VecMul); + private: + DISALLOW_COPY_AND_ASSIGN(HVecMul); +}; + +// Divides every component in the two vectors, +// viz. [ x1, .. , xn ] / [ y1, .. , yn ] = [ x1 / y1, .. , xn / yn ]. +class HVecDiv FINAL : public HVecBinaryOperation { + public: + HVecDiv(ArenaAllocator* arena, + HInstruction* left, + HInstruction* right, + Primitive::Type packed_type, + size_t vector_length, + uint32_t dex_pc = kNoDexPc) + : HVecBinaryOperation(arena, packed_type, vector_length, dex_pc) { + DCHECK(left->IsVecOperation() && right->IsVecOperation()); + DCHECK_EQ(left->AsVecOperation()->GetPackedType(), packed_type); + DCHECK_EQ(right->AsVecOperation()->GetPackedType(), packed_type); + SetRawInputAt(0, left); + SetRawInputAt(1, right); + } + DECLARE_INSTRUCTION(VecDiv); + private: + DISALLOW_COPY_AND_ASSIGN(HVecDiv); +}; + +// Bitwise-ands every component in the two vectors, +// viz. [ x1, .. , xn ] & [ y1, .. , yn ] = [ x1 & y1, .. , xn & yn ]. +class HVecAnd FINAL : public HVecBinaryOperation { + public: + HVecAnd(ArenaAllocator* arena, + HInstruction* left, + HInstruction* right, + Primitive::Type packed_type, + size_t vector_length, + uint32_t dex_pc = kNoDexPc) + : HVecBinaryOperation(arena, packed_type, vector_length, dex_pc) { + DCHECK(left->IsVecOperation() && right->IsVecOperation()); + SetRawInputAt(0, left); + SetRawInputAt(1, right); + } + DECLARE_INSTRUCTION(VecAnd); + private: + DISALLOW_COPY_AND_ASSIGN(HVecAnd); +}; + +// Bitwise-and-nots every component in the two vectors, +// viz. [ x1, .. , xn ] and-not [ y1, .. , yn ] = [ ~x1 & y1, .. , ~xn & yn ]. +class HVecAndNot FINAL : public HVecBinaryOperation { + public: + HVecAndNot(ArenaAllocator* arena, + HInstruction* left, + HInstruction* right, + Primitive::Type packed_type, + size_t vector_length, + uint32_t dex_pc = kNoDexPc) + : HVecBinaryOperation(arena, packed_type, vector_length, dex_pc) { + DCHECK(left->IsVecOperation() && right->IsVecOperation()); + SetRawInputAt(0, left); + SetRawInputAt(1, right); + } + DECLARE_INSTRUCTION(VecAndNot); + private: + DISALLOW_COPY_AND_ASSIGN(HVecAndNot); +}; + +// Bitwise-ors every component in the two vectors, +// viz. [ x1, .. , xn ] | [ y1, .. , yn ] = [ x1 | y1, .. , xn | yn ]. +class HVecOr FINAL : public HVecBinaryOperation { + public: + HVecOr(ArenaAllocator* arena, + HInstruction* left, + HInstruction* right, + Primitive::Type packed_type, + size_t vector_length, + uint32_t dex_pc = kNoDexPc) + : HVecBinaryOperation(arena, packed_type, vector_length, dex_pc) { + DCHECK(left->IsVecOperation() && right->IsVecOperation()); + SetRawInputAt(0, left); + SetRawInputAt(1, right); + } + DECLARE_INSTRUCTION(VecOr); + private: + DISALLOW_COPY_AND_ASSIGN(HVecOr); +}; + +// Bitwise-xors every component in the two vectors, +// viz. [ x1, .. , xn ] ^ [ y1, .. , yn ] = [ x1 ^ y1, .. , xn ^ yn ]. +class HVecXor FINAL : public HVecBinaryOperation { + public: + HVecXor(ArenaAllocator* arena, + HInstruction* left, + HInstruction* right, + Primitive::Type packed_type, + size_t vector_length, + uint32_t dex_pc = kNoDexPc) + : HVecBinaryOperation(arena, packed_type, vector_length, dex_pc) { + DCHECK(left->IsVecOperation() && right->IsVecOperation()); + SetRawInputAt(0, left); + SetRawInputAt(1, right); + } + DECLARE_INSTRUCTION(VecXor); + private: + DISALLOW_COPY_AND_ASSIGN(HVecXor); +}; + +// Logically shifts every component in the vector left by the given distance, +// viz. [ x1, .. , xn ] << d = [ x1 << d, .. , xn << d ]. +class HVecShl FINAL : public HVecBinaryOperation { + public: + HVecShl(ArenaAllocator* arena, + HInstruction* left, + HInstruction* right, + Primitive::Type packed_type, + size_t vector_length, + uint32_t dex_pc = kNoDexPc) + : HVecBinaryOperation(arena, packed_type, vector_length, dex_pc) { + DCHECK(left->IsVecOperation()); + DCHECK_EQ(left->AsVecOperation()->GetPackedType(), packed_type); + SetRawInputAt(0, left); + SetRawInputAt(1, right); + } + DECLARE_INSTRUCTION(VecShl); + private: + DISALLOW_COPY_AND_ASSIGN(HVecShl); +}; + +// Arithmetically shifts every component in the vector right by the given distance, +// viz. [ x1, .. , xn ] >> d = [ x1 >> d, .. , xn >> d ]. +class HVecShr FINAL : public HVecBinaryOperation { + public: + HVecShr(ArenaAllocator* arena, + HInstruction* left, + HInstruction* right, + Primitive::Type packed_type, + size_t vector_length, + uint32_t dex_pc = kNoDexPc) + : HVecBinaryOperation(arena, packed_type, vector_length, dex_pc) { + DCHECK(left->IsVecOperation()); + DCHECK_EQ(left->AsVecOperation()->GetPackedType(), packed_type); + SetRawInputAt(0, left); + SetRawInputAt(1, right); + } + DECLARE_INSTRUCTION(VecShr); + private: + DISALLOW_COPY_AND_ASSIGN(HVecShr); +}; + +// Logically shifts every component in the vector right by the given distance, +// viz. [ x1, .. , xn ] >>> d = [ x1 >>> d, .. , xn >>> d ]. +class HVecUShr FINAL : public HVecBinaryOperation { + public: + HVecUShr(ArenaAllocator* arena, + HInstruction* left, + HInstruction* right, + Primitive::Type packed_type, + size_t vector_length, + uint32_t dex_pc = kNoDexPc) + : HVecBinaryOperation(arena, packed_type, vector_length, dex_pc) { + DCHECK(left->IsVecOperation()); + DCHECK_EQ(left->AsVecOperation()->GetPackedType(), packed_type); + SetRawInputAt(0, left); + SetRawInputAt(1, right); + } + DECLARE_INSTRUCTION(VecUShr); + private: + DISALLOW_COPY_AND_ASSIGN(HVecUShr); +}; + +// Loads a vector from memory, viz. load(mem, 1) +// yield the vector [ mem(1), .. , mem(n) ]. +class HVecLoad FINAL : public HVecMemoryOperation { + public: + HVecLoad(ArenaAllocator* arena, + HInstruction* base, + HInstruction* index, + Primitive::Type packed_type, + size_t vector_length, + uint32_t dex_pc = kNoDexPc) + : HVecMemoryOperation(arena, + packed_type, + SideEffects::ArrayReadOfType(packed_type), + /*number_of_inputs*/ 2, + vector_length, + dex_pc) { + SetRawInputAt(0, base); + SetRawInputAt(1, index); + } + DECLARE_INSTRUCTION(VecLoad); + private: + DISALLOW_COPY_AND_ASSIGN(HVecLoad); +}; + +// Stores a vector to memory, viz. store(m, 1, [x1, .. , xn] ) +// sets mem(1) = x1, .. , mem(n) = xn. +class HVecStore FINAL : public HVecMemoryOperation { + public: + HVecStore(ArenaAllocator* arena, + HInstruction* base, + HInstruction* index, + HInstruction* value, + Primitive::Type packed_type, + size_t vector_length, + uint32_t dex_pc = kNoDexPc) + : HVecMemoryOperation(arena, + packed_type, + SideEffects::ArrayWriteOfType(packed_type), + /*number_of_inputs*/ 3, + vector_length, + dex_pc) { + DCHECK(value->IsVecOperation()); + DCHECK_EQ(value->AsVecOperation()->GetPackedType(), packed_type); + SetRawInputAt(0, base); + SetRawInputAt(1, index); + SetRawInputAt(2, value); + } + DECLARE_INSTRUCTION(VecStore); + private: + DISALLOW_COPY_AND_ASSIGN(HVecStore); +}; + +} // namespace art + +#endif // ART_COMPILER_OPTIMIZING_NODES_VECTOR_H_ diff --git a/compiler/optimizing/optimizing_compiler.cc b/compiler/optimizing/optimizing_compiler.cc index 3c6d2d64a9..eb88fdee84 100644 --- a/compiler/optimizing/optimizing_compiler.cc +++ b/compiler/optimizing/optimizing_compiler.cc @@ -454,6 +454,8 @@ static bool IsInstructionSetSupported(InstructionSet instruction_set) { static bool InstructionSetSupportsReadBarrier(InstructionSet instruction_set) { return instruction_set == kArm64 || instruction_set == kThumb2 + || instruction_set == kMips + || instruction_set == kMips64 || instruction_set == kX86 || instruction_set == kX86_64; } diff --git a/compiler/optimizing/prepare_for_register_allocation.cc b/compiler/optimizing/prepare_for_register_allocation.cc index efbaf6c221..66bfea9860 100644 --- a/compiler/optimizing/prepare_for_register_allocation.cc +++ b/compiler/optimizing/prepare_for_register_allocation.cc @@ -40,6 +40,14 @@ void PrepareForRegisterAllocation::VisitDivZeroCheck(HDivZeroCheck* check) { check->ReplaceWith(check->InputAt(0)); } +void PrepareForRegisterAllocation::VisitDeoptimize(HDeoptimize* deoptimize) { + if (deoptimize->GuardsAnInput()) { + // Replace the uses with the actual guarded instruction. + deoptimize->ReplaceWith(deoptimize->GuardedInput()); + deoptimize->RemoveGuard(); + } +} + void PrepareForRegisterAllocation::VisitBoundsCheck(HBoundsCheck* check) { check->ReplaceWith(check->InputAt(0)); if (check->IsStringCharAt()) { diff --git a/compiler/optimizing/prepare_for_register_allocation.h b/compiler/optimizing/prepare_for_register_allocation.h index c128227654..7ffbe44ef6 100644 --- a/compiler/optimizing/prepare_for_register_allocation.h +++ b/compiler/optimizing/prepare_for_register_allocation.h @@ -44,6 +44,7 @@ class PrepareForRegisterAllocation : public HGraphDelegateVisitor { void VisitClinitCheck(HClinitCheck* check) OVERRIDE; void VisitCondition(HCondition* condition) OVERRIDE; void VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) OVERRIDE; + void VisitDeoptimize(HDeoptimize* deoptimize) OVERRIDE; bool CanMoveClinitCheck(HInstruction* input, HInstruction* user) const; bool CanEmitConditionAt(HCondition* condition, HInstruction* user) const; diff --git a/compiler/optimizing/reference_type_propagation.cc b/compiler/optimizing/reference_type_propagation.cc index 6e332ca59b..d5637b9b75 100644 --- a/compiler/optimizing/reference_type_propagation.cc +++ b/compiler/optimizing/reference_type_propagation.cc @@ -310,8 +310,8 @@ static void BoundTypeForClassCheck(HInstruction* check) { BoundTypeIn(receiver, trueBlock, /* start_instruction */ nullptr, class_rti); } else { DCHECK(check->IsDeoptimize()); - if (compare->IsEqual()) { - BoundTypeIn(receiver, check->GetBlock(), check, class_rti); + if (compare->IsEqual() && check->AsDeoptimize()->GuardsAnInput()) { + check->SetReferenceTypeInfo(class_rti); } } } diff --git a/compiler/optimizing/reference_type_propagation_test.cc b/compiler/optimizing/reference_type_propagation_test.cc index 84a4bab1a9..0b49ce1a4c 100644 --- a/compiler/optimizing/reference_type_propagation_test.cc +++ b/compiler/optimizing/reference_type_propagation_test.cc @@ -29,7 +29,7 @@ namespace art { */ class ReferenceTypePropagationTest : public CommonCompilerTest { public: - ReferenceTypePropagationTest() : pool_(), allocator_(&pool_) { + ReferenceTypePropagationTest() : pool_(), allocator_(&pool_), propagation_(nullptr) { graph_ = CreateGraph(&allocator_); } diff --git a/compiler/optimizing/scheduler.h b/compiler/optimizing/scheduler.h index ab0dad4300..9236a0e4fa 100644 --- a/compiler/optimizing/scheduler.h +++ b/compiler/optimizing/scheduler.h @@ -315,7 +315,10 @@ class SchedulingLatencyVisitor : public HGraphDelegateVisitor { // This class and its sub-classes will never be used to drive a visit of an // `HGraph` but only to visit `HInstructions` one at a time, so we do not need // to pass a valid graph to `HGraphDelegateVisitor()`. - SchedulingLatencyVisitor() : HGraphDelegateVisitor(nullptr) {} + SchedulingLatencyVisitor() + : HGraphDelegateVisitor(nullptr), + last_visited_latency_(0), + last_visited_internal_latency_(0) {} void VisitInstruction(HInstruction* instruction) OVERRIDE { LOG(FATAL) << "Error visiting " << instruction->DebugName() << ". " @@ -413,6 +416,7 @@ class HScheduler { selector_(selector), only_optimize_loop_blocks_(true), scheduling_graph_(this, arena), + cursor_(nullptr), candidates_(arena_->Adapter(kArenaAllocScheduler)) {} virtual ~HScheduler() {} diff --git a/compiler/optimizing/ssa_liveness_analysis.cc b/compiler/optimizing/ssa_liveness_analysis.cc index 36ee5a903a..b538a89a06 100644 --- a/compiler/optimizing/ssa_liveness_analysis.cc +++ b/compiler/optimizing/ssa_liveness_analysis.cc @@ -470,7 +470,12 @@ bool LiveInterval::SameRegisterKind(Location other) const { } size_t LiveInterval::NumberOfSpillSlotsNeeded() const { - // TODO: detect vector operation. + // For a SIMD operation, compute the number of needed spill slots. + // TODO: do through vector type? + HInstruction* definition = GetParent()->GetDefinedBy(); + if (definition != nullptr && definition->IsVecOperation()) { + return definition->AsVecOperation()->GetVectorNumberOfBytes() / kVRegSize; + } // Return number of needed spill slots based on type. return (type_ == Primitive::kPrimLong || type_ == Primitive::kPrimDouble) ? 2 : 1; } diff --git a/compiler/optimizing/ssa_liveness_analysis_test.cc b/compiler/optimizing/ssa_liveness_analysis_test.cc index 1916c73ca4..a1016d1d47 100644 --- a/compiler/optimizing/ssa_liveness_analysis_test.cc +++ b/compiler/optimizing/ssa_liveness_analysis_test.cc @@ -189,13 +189,14 @@ TEST_F(SsaLivenessAnalysisTest, TestDeoptimize) { // Use HAboveOrEqual+HDeoptimize as the bounds check. HInstruction* ae = new (&allocator_) HAboveOrEqual(index, length); block->AddInstruction(ae); - HInstruction* deoptimize = new(&allocator_) HDeoptimize(ae, /* dex_pc */ 0u); + HInstruction* deoptimize = + new(&allocator_) HDeoptimize(&allocator_, ae, HDeoptimize::Kind::kBCE, /* dex_pc */ 0u); block->AddInstruction(deoptimize); HEnvironment* deoptimize_env = new (&allocator_) HEnvironment(&allocator_, - /* number_of_vregs */ 5, - /* method */ nullptr, - /* dex_pc */ 0u, - deoptimize); + /* number_of_vregs */ 5, + /* method */ nullptr, + /* dex_pc */ 0u, + deoptimize); deoptimize_env->CopyFrom(args); deoptimize->SetRawEnvironment(deoptimize_env); HInstruction* array_set = diff --git a/compiler/utils/assembler_test.h b/compiler/utils/assembler_test.h index d265a44092..f655994bd3 100644 --- a/compiler/utils/assembler_test.h +++ b/compiler/utils/assembler_test.h @@ -309,7 +309,7 @@ class AssemblerTest : public testing::Test { template <typename RegType, typename ImmType> std::string RepeatTemplatedRegisterImmBits(void (Ass::*f)(RegType, ImmType), int imm_bits, - const std::vector<Reg*> registers, + const std::vector<RegType*> registers, std::string (AssemblerTest::*GetName)(const RegType&), const std::string& fmt, int bias) { @@ -573,6 +573,19 @@ class AssemblerTest : public testing::Test { } template <typename ImmType> + std::string RepeatVIb(void (Ass::*f)(VecReg, ImmType), + int imm_bits, + std::string fmt, + int bias = 0) { + return RepeatTemplatedRegisterImmBits<VecReg, ImmType>(f, + imm_bits, + GetVectorRegisters(), + &AssemblerTest::GetVecRegName, + fmt, + bias); + } + + template <typename ImmType> std::string RepeatVRIb(void (Ass::*f)(VecReg, Reg, ImmType), int imm_bits, const std::string& fmt, diff --git a/compiler/utils/atomic_method_ref_map-inl.h b/compiler/utils/atomic_method_ref_map-inl.h index d71c2fe997..ad3a099eb6 100644 --- a/compiler/utils/atomic_method_ref_map-inl.h +++ b/compiler/utils/atomic_method_ref_map-inl.h @@ -42,7 +42,7 @@ template <typename T> inline bool AtomicMethodRefMap<T>::Get(MethodReference ref, T* out) const { const ElementArray* const array = GetArray(ref.dex_file); if (array == nullptr) { - return kInsertResultInvalidDexFile; + return false; } *out = (*array)[ref.dex_method_index].LoadRelaxed(); return true; diff --git a/compiler/utils/mips64/assembler_mips64.cc b/compiler/utils/mips64/assembler_mips64.cc index 8a5ae754df..0cff44d830 100644 --- a/compiler/utils/mips64/assembler_mips64.cc +++ b/compiler/utils/mips64/assembler_mips64.cc @@ -252,6 +252,22 @@ void Mips64Assembler::EmitMsaMI10(int s10, Emit(encoding); } +void Mips64Assembler::EmitMsaI10(int operation, + int df, + int i10, + VectorRegister wd, + int minor_opcode) { + CHECK_NE(wd, kNoVectorRegister); + CHECK(IsUint<10>(i10)) << i10; + uint32_t encoding = static_cast<uint32_t>(kMsaMajorOpcode) << kOpcodeShift | + operation << kMsaOperationShift | + df << kDfShift | + i10 << kI10Shift | + static_cast<uint32_t>(wd) << kWdShift | + minor_opcode; + Emit(encoding); +} + void Mips64Assembler::EmitMsa2R(int operation, int df, VectorRegister ws, @@ -1581,6 +1597,30 @@ void Mips64Assembler::FillD(VectorRegister wd, GpuRegister rs) { EmitMsa2R(0xc0, 0x3, static_cast<VectorRegister>(rs), wd, 0x1e); } +void Mips64Assembler::LdiB(VectorRegister wd, int imm8) { + CHECK(HasMsa()); + CHECK(IsInt<8>(imm8)) << imm8; + EmitMsaI10(0x6, 0x0, imm8 & kMsaS10Mask, wd, 0x7); +} + +void Mips64Assembler::LdiH(VectorRegister wd, int imm10) { + CHECK(HasMsa()); + CHECK(IsInt<10>(imm10)) << imm10; + EmitMsaI10(0x6, 0x1, imm10 & kMsaS10Mask, wd, 0x7); +} + +void Mips64Assembler::LdiW(VectorRegister wd, int imm10) { + CHECK(HasMsa()); + CHECK(IsInt<10>(imm10)) << imm10; + EmitMsaI10(0x6, 0x2, imm10 & kMsaS10Mask, wd, 0x7); +} + +void Mips64Assembler::LdiD(VectorRegister wd, int imm10) { + CHECK(HasMsa()); + CHECK(IsInt<10>(imm10)) << imm10; + EmitMsaI10(0x6, 0x3, imm10 & kMsaS10Mask, wd, 0x7); +} + void Mips64Assembler::LdB(VectorRegister wd, GpuRegister rs, int offset) { CHECK(HasMsa()); CHECK(IsInt<10>(offset)) << offset; @@ -1661,6 +1701,7 @@ void Mips64Assembler::Addiu32(GpuRegister rt, GpuRegister rs, int32_t value) { } } +// TODO: don't use rtmp, use daui, dahi, dati. void Mips64Assembler::Daddiu64(GpuRegister rt, GpuRegister rs, int64_t value, GpuRegister rtmp) { if (IsInt<16>(value)) { Daddiu(rt, rs, value); diff --git a/compiler/utils/mips64/assembler_mips64.h b/compiler/utils/mips64/assembler_mips64.h index a8035b6da4..666c6935a1 100644 --- a/compiler/utils/mips64/assembler_mips64.h +++ b/compiler/utils/mips64/assembler_mips64.h @@ -734,6 +734,10 @@ class Mips64Assembler FINAL : public Assembler, public JNIMacroAssembler<Pointer void FillW(VectorRegister wd, GpuRegister rs); void FillD(VectorRegister wd, GpuRegister rs); + void LdiB(VectorRegister wd, int imm8); + void LdiH(VectorRegister wd, int imm10); + void LdiW(VectorRegister wd, int imm10); + void LdiD(VectorRegister wd, int imm10); void LdB(VectorRegister wd, GpuRegister rs, int offset); void LdH(VectorRegister wd, GpuRegister rs, int offset); void LdW(VectorRegister wd, GpuRegister rs, int offset); @@ -1457,6 +1461,7 @@ class Mips64Assembler FINAL : public Assembler, public JNIMacroAssembler<Pointer void EmitMsaBIT(int operation, int df_m, VectorRegister ws, VectorRegister wd, int minor_opcode); void EmitMsaELM(int operation, int df_n, VectorRegister ws, VectorRegister wd, int minor_opcode); void EmitMsaMI10(int s10, GpuRegister rs, VectorRegister wd, int minor_opcode, int df); + void EmitMsaI10(int operation, int df, int i10, VectorRegister wd, int minor_opcode); void EmitMsa2R(int operation, int df, VectorRegister ws, VectorRegister wd, int minor_opcode); void EmitMsa2RF(int operation, int df, VectorRegister ws, VectorRegister wd, int minor_opcode); diff --git a/compiler/utils/mips64/assembler_mips64_test.cc b/compiler/utils/mips64/assembler_mips64_test.cc index cadbe27819..f2e3b1610c 100644 --- a/compiler/utils/mips64/assembler_mips64_test.cc +++ b/compiler/utils/mips64/assembler_mips64_test.cc @@ -2836,6 +2836,22 @@ TEST_F(AssemblerMIPS64Test, FillD) { DriverStr(RepeatVR(&mips64::Mips64Assembler::FillD, "fill.d ${reg1}, ${reg2}"), "fill.d"); } +TEST_F(AssemblerMIPS64Test, LdiB) { + DriverStr(RepeatVIb(&mips64::Mips64Assembler::LdiB, -8, "ldi.b ${reg}, {imm}"), "ldi.b"); +} + +TEST_F(AssemblerMIPS64Test, LdiH) { + DriverStr(RepeatVIb(&mips64::Mips64Assembler::LdiH, -10, "ldi.h ${reg}, {imm}"), "ldi.h"); +} + +TEST_F(AssemblerMIPS64Test, LdiW) { + DriverStr(RepeatVIb(&mips64::Mips64Assembler::LdiW, -10, "ldi.w ${reg}, {imm}"), "ldi.w"); +} + +TEST_F(AssemblerMIPS64Test, LdiD) { + DriverStr(RepeatVIb(&mips64::Mips64Assembler::LdiD, -10, "ldi.d ${reg}, {imm}"), "ldi.d"); +} + TEST_F(AssemblerMIPS64Test, LdB) { DriverStr(RepeatVRIb(&mips64::Mips64Assembler::LdB, -10, "ld.b ${reg1}, {imm}(${reg2})"), "ld.b"); } diff --git a/compiler/utils/mips64/constants_mips64.h b/compiler/utils/mips64/constants_mips64.h index 5ae9c73589..bc8e40b437 100644 --- a/compiler/utils/mips64/constants_mips64.h +++ b/compiler/utils/mips64/constants_mips64.h @@ -66,6 +66,7 @@ enum InstructionFields { kWdShift = 6, kWdBits = 5, kS10Shift = 16, + kI10Shift = 11, kS10MinorShift = 2, kBranchOffsetMask = 0x0000ffff, diff --git a/compiler/utils/mips64/managed_register_mips64.cc b/compiler/utils/mips64/managed_register_mips64.cc index dea396e4a7..42d061ec15 100644 --- a/compiler/utils/mips64/managed_register_mips64.cc +++ b/compiler/utils/mips64/managed_register_mips64.cc @@ -26,6 +26,11 @@ bool Mips64ManagedRegister::Overlaps(const Mips64ManagedRegister& other) const { CHECK(IsValidManagedRegister()); CHECK(other.IsValidManagedRegister()); if (Equals(other)) return true; + if (IsFpuRegister() && other.IsVectorRegister()) { + return (AsFpuRegister() == other.AsOverlappingFpuRegister()); + } else if (IsVectorRegister() && other.IsFpuRegister()) { + return (AsVectorRegister() == other.AsOverlappingVectorRegister()); + } return false; } @@ -36,6 +41,8 @@ void Mips64ManagedRegister::Print(std::ostream& os) const { os << "GPU: " << static_cast<int>(AsGpuRegister()); } else if (IsFpuRegister()) { os << "FpuRegister: " << static_cast<int>(AsFpuRegister()); + } else if (IsVectorRegister()) { + os << "VectorRegister: " << static_cast<int>(AsVectorRegister()); } else { os << "??: " << RegId(); } diff --git a/compiler/utils/mips64/managed_register_mips64.h b/compiler/utils/mips64/managed_register_mips64.h index c9f95569cf..3980199b1e 100644 --- a/compiler/utils/mips64/managed_register_mips64.h +++ b/compiler/utils/mips64/managed_register_mips64.h @@ -30,11 +30,27 @@ const int kNumberOfGpuAllocIds = kNumberOfGpuRegisters; const int kNumberOfFpuRegIds = kNumberOfFpuRegisters; const int kNumberOfFpuAllocIds = kNumberOfFpuRegisters; -const int kNumberOfRegIds = kNumberOfGpuRegIds + kNumberOfFpuRegIds; -const int kNumberOfAllocIds = kNumberOfGpuAllocIds + kNumberOfFpuAllocIds; - -// An instance of class 'ManagedRegister' represents a single GPU register (enum -// Register) or a double precision FP register (enum FpuRegister) +const int kNumberOfVecRegIds = kNumberOfVectorRegisters; +const int kNumberOfVecAllocIds = kNumberOfVectorRegisters; + +const int kNumberOfRegIds = kNumberOfGpuRegIds + kNumberOfFpuRegIds + kNumberOfVecRegIds; +const int kNumberOfAllocIds = kNumberOfGpuAllocIds + kNumberOfFpuAllocIds + kNumberOfVecAllocIds; + +// Register ids map: +// [0..R[ core registers (enum GpuRegister) +// [R..F[ floating-point registers (enum FpuRegister) +// [F..W[ MSA vector registers (enum VectorRegister) +// where +// R = kNumberOfGpuRegIds +// F = R + kNumberOfFpuRegIds +// W = F + kNumberOfVecRegIds + +// An instance of class 'ManagedRegister' represents a single Mips64 register. +// A register can be one of the following: +// * core register (enum GpuRegister) +// * floating-point register (enum FpuRegister) +// * MSA vector register (enum VectorRegister) +// // 'ManagedRegister::NoRegister()' provides an invalid register. // There is a one-to-one mapping between ManagedRegister and register id. class Mips64ManagedRegister : public ManagedRegister { @@ -49,6 +65,21 @@ class Mips64ManagedRegister : public ManagedRegister { return static_cast<FpuRegister>(id_ - kNumberOfGpuRegIds); } + constexpr VectorRegister AsVectorRegister() const { + CHECK(IsVectorRegister()); + return static_cast<VectorRegister>(id_ - (kNumberOfGpuRegIds + kNumberOfFpuRegisters)); + } + + constexpr FpuRegister AsOverlappingFpuRegister() const { + CHECK(IsValidManagedRegister()); + return static_cast<FpuRegister>(AsVectorRegister()); + } + + constexpr VectorRegister AsOverlappingVectorRegister() const { + CHECK(IsValidManagedRegister()); + return static_cast<VectorRegister>(AsFpuRegister()); + } + constexpr bool IsGpuRegister() const { CHECK(IsValidManagedRegister()); return (0 <= id_) && (id_ < kNumberOfGpuRegIds); @@ -60,6 +91,12 @@ class Mips64ManagedRegister : public ManagedRegister { return (0 <= test) && (test < kNumberOfFpuRegIds); } + constexpr bool IsVectorRegister() const { + CHECK(IsValidManagedRegister()); + const int test = id_ - (kNumberOfGpuRegIds + kNumberOfFpuRegIds); + return (0 <= test) && (test < kNumberOfVecRegIds); + } + void Print(std::ostream& os) const; // Returns true if the two managed-registers ('this' and 'other') overlap. @@ -77,6 +114,11 @@ class Mips64ManagedRegister : public ManagedRegister { return FromRegId(r + kNumberOfGpuRegIds); } + static constexpr Mips64ManagedRegister FromVectorRegister(VectorRegister r) { + CHECK_NE(r, kNoVectorRegister); + return FromRegId(r + kNumberOfGpuRegIds + kNumberOfFpuRegIds); + } + private: constexpr bool IsValidManagedRegister() const { return (0 <= id_) && (id_ < kNumberOfRegIds); diff --git a/compiler/utils/mips64/managed_register_mips64_test.cc b/compiler/utils/mips64/managed_register_mips64_test.cc new file mode 100644 index 0000000000..8b72d7e61d --- /dev/null +++ b/compiler/utils/mips64/managed_register_mips64_test.cc @@ -0,0 +1,480 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "managed_register_mips64.h" +#include "globals.h" +#include "gtest/gtest.h" + +namespace art { +namespace mips64 { + +TEST(Mips64ManagedRegister, NoRegister) { + Mips64ManagedRegister reg = ManagedRegister::NoRegister().AsMips64(); + EXPECT_TRUE(reg.IsNoRegister()); + EXPECT_FALSE(reg.Overlaps(reg)); +} + +TEST(Mips64ManagedRegister, GpuRegister) { + Mips64ManagedRegister reg = Mips64ManagedRegister::FromGpuRegister(ZERO); + EXPECT_FALSE(reg.IsNoRegister()); + EXPECT_TRUE(reg.IsGpuRegister()); + EXPECT_FALSE(reg.IsFpuRegister()); + EXPECT_FALSE(reg.IsVectorRegister()); + EXPECT_EQ(ZERO, reg.AsGpuRegister()); + + reg = Mips64ManagedRegister::FromGpuRegister(AT); + EXPECT_FALSE(reg.IsNoRegister()); + EXPECT_TRUE(reg.IsGpuRegister()); + EXPECT_FALSE(reg.IsFpuRegister()); + EXPECT_FALSE(reg.IsVectorRegister()); + EXPECT_EQ(AT, reg.AsGpuRegister()); + + reg = Mips64ManagedRegister::FromGpuRegister(V0); + EXPECT_FALSE(reg.IsNoRegister()); + EXPECT_TRUE(reg.IsGpuRegister()); + EXPECT_FALSE(reg.IsFpuRegister()); + EXPECT_FALSE(reg.IsVectorRegister()); + EXPECT_EQ(V0, reg.AsGpuRegister()); + + reg = Mips64ManagedRegister::FromGpuRegister(A0); + EXPECT_FALSE(reg.IsNoRegister()); + EXPECT_TRUE(reg.IsGpuRegister()); + EXPECT_FALSE(reg.IsFpuRegister()); + EXPECT_FALSE(reg.IsVectorRegister()); + EXPECT_EQ(A0, reg.AsGpuRegister()); + + reg = Mips64ManagedRegister::FromGpuRegister(A7); + EXPECT_FALSE(reg.IsNoRegister()); + EXPECT_TRUE(reg.IsGpuRegister()); + EXPECT_FALSE(reg.IsFpuRegister()); + EXPECT_FALSE(reg.IsVectorRegister()); + EXPECT_EQ(A7, reg.AsGpuRegister()); + + reg = Mips64ManagedRegister::FromGpuRegister(T0); + EXPECT_FALSE(reg.IsNoRegister()); + EXPECT_TRUE(reg.IsGpuRegister()); + EXPECT_FALSE(reg.IsFpuRegister()); + EXPECT_FALSE(reg.IsVectorRegister()); + EXPECT_EQ(T0, reg.AsGpuRegister()); + + reg = Mips64ManagedRegister::FromGpuRegister(T3); + EXPECT_FALSE(reg.IsNoRegister()); + EXPECT_TRUE(reg.IsGpuRegister()); + EXPECT_FALSE(reg.IsFpuRegister()); + EXPECT_FALSE(reg.IsVectorRegister()); + EXPECT_EQ(T3, reg.AsGpuRegister()); + + reg = Mips64ManagedRegister::FromGpuRegister(S0); + EXPECT_FALSE(reg.IsNoRegister()); + EXPECT_TRUE(reg.IsGpuRegister()); + EXPECT_FALSE(reg.IsFpuRegister()); + EXPECT_FALSE(reg.IsVectorRegister()); + EXPECT_EQ(S0, reg.AsGpuRegister()); + + reg = Mips64ManagedRegister::FromGpuRegister(GP); + EXPECT_FALSE(reg.IsNoRegister()); + EXPECT_TRUE(reg.IsGpuRegister()); + EXPECT_FALSE(reg.IsFpuRegister()); + EXPECT_FALSE(reg.IsVectorRegister()); + EXPECT_EQ(GP, reg.AsGpuRegister()); + + reg = Mips64ManagedRegister::FromGpuRegister(SP); + EXPECT_FALSE(reg.IsNoRegister()); + EXPECT_TRUE(reg.IsGpuRegister()); + EXPECT_FALSE(reg.IsFpuRegister()); + EXPECT_FALSE(reg.IsVectorRegister()); + EXPECT_EQ(SP, reg.AsGpuRegister()); + + reg = Mips64ManagedRegister::FromGpuRegister(RA); + EXPECT_FALSE(reg.IsNoRegister()); + EXPECT_TRUE(reg.IsGpuRegister()); + EXPECT_FALSE(reg.IsFpuRegister()); + EXPECT_FALSE(reg.IsVectorRegister()); + EXPECT_EQ(RA, reg.AsGpuRegister()); +} + +TEST(Mips64ManagedRegister, FpuRegister) { + Mips64ManagedRegister reg = Mips64ManagedRegister::FromFpuRegister(F0); + Mips64ManagedRegister vreg = Mips64ManagedRegister::FromVectorRegister(W0); + EXPECT_FALSE(reg.IsNoRegister()); + EXPECT_FALSE(reg.IsGpuRegister()); + EXPECT_TRUE(reg.IsFpuRegister()); + EXPECT_FALSE(reg.IsVectorRegister()); + EXPECT_TRUE(reg.Overlaps(vreg)); + EXPECT_EQ(F0, reg.AsFpuRegister()); + EXPECT_EQ(W0, reg.AsOverlappingVectorRegister()); + EXPECT_TRUE(reg.Equals(Mips64ManagedRegister::FromFpuRegister(F0))); + + reg = Mips64ManagedRegister::FromFpuRegister(F1); + vreg = Mips64ManagedRegister::FromVectorRegister(W1); + EXPECT_FALSE(reg.IsNoRegister()); + EXPECT_FALSE(reg.IsGpuRegister()); + EXPECT_TRUE(reg.IsFpuRegister()); + EXPECT_FALSE(reg.IsVectorRegister()); + EXPECT_TRUE(reg.Overlaps(vreg)); + EXPECT_EQ(F1, reg.AsFpuRegister()); + EXPECT_EQ(W1, reg.AsOverlappingVectorRegister()); + EXPECT_TRUE(reg.Equals(Mips64ManagedRegister::FromFpuRegister(F1))); + + reg = Mips64ManagedRegister::FromFpuRegister(F20); + vreg = Mips64ManagedRegister::FromVectorRegister(W20); + EXPECT_FALSE(reg.IsNoRegister()); + EXPECT_FALSE(reg.IsGpuRegister()); + EXPECT_TRUE(reg.IsFpuRegister()); + EXPECT_FALSE(reg.IsVectorRegister()); + EXPECT_TRUE(reg.Overlaps(vreg)); + EXPECT_EQ(F20, reg.AsFpuRegister()); + EXPECT_EQ(W20, reg.AsOverlappingVectorRegister()); + EXPECT_TRUE(reg.Equals(Mips64ManagedRegister::FromFpuRegister(F20))); + + reg = Mips64ManagedRegister::FromFpuRegister(F31); + vreg = Mips64ManagedRegister::FromVectorRegister(W31); + EXPECT_FALSE(reg.IsNoRegister()); + EXPECT_FALSE(reg.IsGpuRegister()); + EXPECT_TRUE(reg.IsFpuRegister()); + EXPECT_FALSE(reg.IsVectorRegister()); + EXPECT_TRUE(reg.Overlaps(vreg)); + EXPECT_EQ(F31, reg.AsFpuRegister()); + EXPECT_EQ(W31, reg.AsOverlappingVectorRegister()); + EXPECT_TRUE(reg.Equals(Mips64ManagedRegister::FromFpuRegister(F31))); +} + +TEST(Mips64ManagedRegister, VectorRegister) { + Mips64ManagedRegister reg = Mips64ManagedRegister::FromVectorRegister(W0); + Mips64ManagedRegister freg = Mips64ManagedRegister::FromFpuRegister(F0); + EXPECT_FALSE(reg.IsNoRegister()); + EXPECT_FALSE(reg.IsGpuRegister()); + EXPECT_FALSE(reg.IsFpuRegister()); + EXPECT_TRUE(reg.IsVectorRegister()); + EXPECT_TRUE(reg.Overlaps(freg)); + EXPECT_EQ(W0, reg.AsVectorRegister()); + EXPECT_EQ(F0, reg.AsOverlappingFpuRegister()); + EXPECT_TRUE(reg.Equals(Mips64ManagedRegister::FromVectorRegister(W0))); + + reg = Mips64ManagedRegister::FromVectorRegister(W2); + freg = Mips64ManagedRegister::FromFpuRegister(F2); + EXPECT_FALSE(reg.IsNoRegister()); + EXPECT_FALSE(reg.IsGpuRegister()); + EXPECT_FALSE(reg.IsFpuRegister()); + EXPECT_TRUE(reg.IsVectorRegister()); + EXPECT_TRUE(reg.Overlaps(freg)); + EXPECT_EQ(W2, reg.AsVectorRegister()); + EXPECT_EQ(F2, reg.AsOverlappingFpuRegister()); + EXPECT_TRUE(reg.Equals(Mips64ManagedRegister::FromVectorRegister(W2))); + + reg = Mips64ManagedRegister::FromVectorRegister(W13); + freg = Mips64ManagedRegister::FromFpuRegister(F13); + EXPECT_FALSE(reg.IsNoRegister()); + EXPECT_FALSE(reg.IsGpuRegister()); + EXPECT_FALSE(reg.IsFpuRegister()); + EXPECT_TRUE(reg.IsVectorRegister()); + EXPECT_TRUE(reg.Overlaps(freg)); + EXPECT_EQ(W13, reg.AsVectorRegister()); + EXPECT_EQ(F13, reg.AsOverlappingFpuRegister()); + EXPECT_TRUE(reg.Equals(Mips64ManagedRegister::FromVectorRegister(W13))); + + reg = Mips64ManagedRegister::FromVectorRegister(W29); + freg = Mips64ManagedRegister::FromFpuRegister(F29); + EXPECT_FALSE(reg.IsNoRegister()); + EXPECT_FALSE(reg.IsGpuRegister()); + EXPECT_FALSE(reg.IsFpuRegister()); + EXPECT_TRUE(reg.IsVectorRegister()); + EXPECT_TRUE(reg.Overlaps(freg)); + EXPECT_EQ(W29, reg.AsVectorRegister()); + EXPECT_EQ(F29, reg.AsOverlappingFpuRegister()); + EXPECT_TRUE(reg.Equals(Mips64ManagedRegister::FromVectorRegister(W29))); +} + +TEST(Mips64ManagedRegister, Equals) { + ManagedRegister no_reg = ManagedRegister::NoRegister(); + EXPECT_TRUE(no_reg.Equals(Mips64ManagedRegister::NoRegister())); + EXPECT_FALSE(no_reg.Equals(Mips64ManagedRegister::FromGpuRegister(ZERO))); + EXPECT_FALSE(no_reg.Equals(Mips64ManagedRegister::FromGpuRegister(A1))); + EXPECT_FALSE(no_reg.Equals(Mips64ManagedRegister::FromGpuRegister(S2))); + EXPECT_FALSE(no_reg.Equals(Mips64ManagedRegister::FromFpuRegister(F0))); + EXPECT_FALSE(no_reg.Equals(Mips64ManagedRegister::FromVectorRegister(W0))); + + Mips64ManagedRegister reg_ZERO = Mips64ManagedRegister::FromGpuRegister(ZERO); + EXPECT_FALSE(reg_ZERO.Equals(Mips64ManagedRegister::NoRegister())); + EXPECT_TRUE(reg_ZERO.Equals(Mips64ManagedRegister::FromGpuRegister(ZERO))); + EXPECT_FALSE(reg_ZERO.Equals(Mips64ManagedRegister::FromGpuRegister(A1))); + EXPECT_FALSE(reg_ZERO.Equals(Mips64ManagedRegister::FromGpuRegister(S2))); + EXPECT_FALSE(reg_ZERO.Equals(Mips64ManagedRegister::FromFpuRegister(F0))); + EXPECT_FALSE(reg_ZERO.Equals(Mips64ManagedRegister::FromVectorRegister(W0))); + + Mips64ManagedRegister reg_A1 = Mips64ManagedRegister::FromGpuRegister(A1); + EXPECT_FALSE(reg_A1.Equals(Mips64ManagedRegister::NoRegister())); + EXPECT_FALSE(reg_A1.Equals(Mips64ManagedRegister::FromGpuRegister(ZERO))); + EXPECT_FALSE(reg_A1.Equals(Mips64ManagedRegister::FromGpuRegister(A0))); + EXPECT_TRUE(reg_A1.Equals(Mips64ManagedRegister::FromGpuRegister(A1))); + EXPECT_FALSE(reg_A1.Equals(Mips64ManagedRegister::FromGpuRegister(S2))); + EXPECT_FALSE(reg_A1.Equals(Mips64ManagedRegister::FromFpuRegister(F0))); + EXPECT_FALSE(reg_A1.Equals(Mips64ManagedRegister::FromVectorRegister(W0))); + + Mips64ManagedRegister reg_S2 = Mips64ManagedRegister::FromGpuRegister(S2); + EXPECT_FALSE(reg_S2.Equals(Mips64ManagedRegister::NoRegister())); + EXPECT_FALSE(reg_S2.Equals(Mips64ManagedRegister::FromGpuRegister(ZERO))); + EXPECT_FALSE(reg_S2.Equals(Mips64ManagedRegister::FromGpuRegister(A1))); + EXPECT_FALSE(reg_S2.Equals(Mips64ManagedRegister::FromGpuRegister(S1))); + EXPECT_TRUE(reg_S2.Equals(Mips64ManagedRegister::FromGpuRegister(S2))); + EXPECT_FALSE(reg_S2.Equals(Mips64ManagedRegister::FromFpuRegister(F0))); + EXPECT_FALSE(reg_S2.Equals(Mips64ManagedRegister::FromVectorRegister(W0))); + + Mips64ManagedRegister reg_F0 = Mips64ManagedRegister::FromFpuRegister(F0); + EXPECT_FALSE(reg_F0.Equals(Mips64ManagedRegister::NoRegister())); + EXPECT_FALSE(reg_F0.Equals(Mips64ManagedRegister::FromGpuRegister(ZERO))); + EXPECT_FALSE(reg_F0.Equals(Mips64ManagedRegister::FromGpuRegister(A1))); + EXPECT_FALSE(reg_F0.Equals(Mips64ManagedRegister::FromGpuRegister(S2))); + EXPECT_TRUE(reg_F0.Equals(Mips64ManagedRegister::FromFpuRegister(F0))); + EXPECT_FALSE(reg_F0.Equals(Mips64ManagedRegister::FromFpuRegister(F1))); + EXPECT_FALSE(reg_F0.Equals(Mips64ManagedRegister::FromFpuRegister(F31))); + EXPECT_FALSE(reg_F0.Equals(Mips64ManagedRegister::FromVectorRegister(W0))); + + Mips64ManagedRegister reg_F31 = Mips64ManagedRegister::FromFpuRegister(F31); + EXPECT_FALSE(reg_F31.Equals(Mips64ManagedRegister::NoRegister())); + EXPECT_FALSE(reg_F31.Equals(Mips64ManagedRegister::FromGpuRegister(ZERO))); + EXPECT_FALSE(reg_F31.Equals(Mips64ManagedRegister::FromGpuRegister(A1))); + EXPECT_FALSE(reg_F31.Equals(Mips64ManagedRegister::FromGpuRegister(S2))); + EXPECT_FALSE(reg_F31.Equals(Mips64ManagedRegister::FromFpuRegister(F0))); + EXPECT_FALSE(reg_F31.Equals(Mips64ManagedRegister::FromFpuRegister(F1))); + EXPECT_TRUE(reg_F31.Equals(Mips64ManagedRegister::FromFpuRegister(F31))); + EXPECT_FALSE(reg_F31.Equals(Mips64ManagedRegister::FromVectorRegister(W0))); + + Mips64ManagedRegister reg_W0 = Mips64ManagedRegister::FromVectorRegister(W0); + EXPECT_FALSE(reg_W0.Equals(Mips64ManagedRegister::NoRegister())); + EXPECT_FALSE(reg_W0.Equals(Mips64ManagedRegister::FromGpuRegister(ZERO))); + EXPECT_FALSE(reg_W0.Equals(Mips64ManagedRegister::FromGpuRegister(A1))); + EXPECT_FALSE(reg_W0.Equals(Mips64ManagedRegister::FromGpuRegister(S1))); + EXPECT_FALSE(reg_W0.Equals(Mips64ManagedRegister::FromFpuRegister(F0))); + EXPECT_TRUE(reg_W0.Equals(Mips64ManagedRegister::FromVectorRegister(W0))); + EXPECT_FALSE(reg_W0.Equals(Mips64ManagedRegister::FromVectorRegister(W1))); + EXPECT_FALSE(reg_W0.Equals(Mips64ManagedRegister::FromVectorRegister(W31))); + + Mips64ManagedRegister reg_W31 = Mips64ManagedRegister::FromVectorRegister(W31); + EXPECT_FALSE(reg_W31.Equals(Mips64ManagedRegister::NoRegister())); + EXPECT_FALSE(reg_W31.Equals(Mips64ManagedRegister::FromGpuRegister(ZERO))); + EXPECT_FALSE(reg_W31.Equals(Mips64ManagedRegister::FromGpuRegister(A1))); + EXPECT_FALSE(reg_W31.Equals(Mips64ManagedRegister::FromGpuRegister(S1))); + EXPECT_FALSE(reg_W31.Equals(Mips64ManagedRegister::FromFpuRegister(F0))); + EXPECT_FALSE(reg_W31.Equals(Mips64ManagedRegister::FromVectorRegister(W0))); + EXPECT_FALSE(reg_W31.Equals(Mips64ManagedRegister::FromVectorRegister(W1))); + EXPECT_TRUE(reg_W31.Equals(Mips64ManagedRegister::FromVectorRegister(W31))); +} + +TEST(Mips64ManagedRegister, Overlaps) { + Mips64ManagedRegister reg = Mips64ManagedRegister::FromFpuRegister(F0); + Mips64ManagedRegister reg_o = Mips64ManagedRegister::FromVectorRegister(W0); + EXPECT_TRUE(reg.Overlaps(reg_o)); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromGpuRegister(ZERO))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromGpuRegister(A0))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromGpuRegister(S0))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromGpuRegister(RA))); + EXPECT_EQ(F0, reg_o.AsOverlappingFpuRegister()); + EXPECT_EQ(W0, reg.AsOverlappingVectorRegister()); + EXPECT_TRUE(reg.Overlaps(Mips64ManagedRegister::FromFpuRegister(F0))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromFpuRegister(F4))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromFpuRegister(F16))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromFpuRegister(F31))); + EXPECT_TRUE(reg.Overlaps(Mips64ManagedRegister::FromVectorRegister(W0))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromVectorRegister(W4))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromVectorRegister(W16))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromVectorRegister(W31))); + + reg = Mips64ManagedRegister::FromFpuRegister(F4); + reg_o = Mips64ManagedRegister::FromVectorRegister(W4); + EXPECT_TRUE(reg.Overlaps(reg_o)); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromGpuRegister(ZERO))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromGpuRegister(A0))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromGpuRegister(S0))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromGpuRegister(RA))); + EXPECT_EQ(F4, reg_o.AsOverlappingFpuRegister()); + EXPECT_EQ(W4, reg.AsOverlappingVectorRegister()); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromFpuRegister(F0))); + EXPECT_TRUE(reg.Overlaps(Mips64ManagedRegister::FromFpuRegister(F4))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromFpuRegister(F16))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromFpuRegister(F31))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromVectorRegister(W0))); + EXPECT_TRUE(reg.Overlaps(Mips64ManagedRegister::FromVectorRegister(W4))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromVectorRegister(W16))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromVectorRegister(W31))); + + reg = Mips64ManagedRegister::FromFpuRegister(F16); + reg_o = Mips64ManagedRegister::FromVectorRegister(W16); + EXPECT_TRUE(reg.Overlaps(reg_o)); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromGpuRegister(ZERO))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromGpuRegister(A0))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromGpuRegister(S0))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromGpuRegister(RA))); + EXPECT_EQ(F16, reg_o.AsOverlappingFpuRegister()); + EXPECT_EQ(W16, reg.AsOverlappingVectorRegister()); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromFpuRegister(F0))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromFpuRegister(F4))); + EXPECT_TRUE(reg.Overlaps(Mips64ManagedRegister::FromFpuRegister(F16))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromFpuRegister(F31))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromVectorRegister(W0))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromVectorRegister(W4))); + EXPECT_TRUE(reg.Overlaps(Mips64ManagedRegister::FromVectorRegister(W16))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromVectorRegister(W31))); + + reg = Mips64ManagedRegister::FromFpuRegister(F31); + reg_o = Mips64ManagedRegister::FromVectorRegister(W31); + EXPECT_TRUE(reg.Overlaps(reg_o)); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromGpuRegister(ZERO))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromGpuRegister(A0))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromGpuRegister(S0))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromGpuRegister(RA))); + EXPECT_EQ(F31, reg_o.AsOverlappingFpuRegister()); + EXPECT_EQ(W31, reg.AsOverlappingVectorRegister()); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromFpuRegister(F0))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromFpuRegister(F4))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromFpuRegister(F16))); + EXPECT_TRUE(reg.Overlaps(Mips64ManagedRegister::FromFpuRegister(F31))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromVectorRegister(W0))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromVectorRegister(W4))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromVectorRegister(W16))); + EXPECT_TRUE(reg.Overlaps(Mips64ManagedRegister::FromVectorRegister(W31))); + + reg = Mips64ManagedRegister::FromVectorRegister(W0); + reg_o = Mips64ManagedRegister::FromFpuRegister(F0); + EXPECT_TRUE(reg.Overlaps(reg_o)); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromGpuRegister(ZERO))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromGpuRegister(A0))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromGpuRegister(S0))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromGpuRegister(RA))); + EXPECT_EQ(W0, reg_o.AsOverlappingVectorRegister()); + EXPECT_EQ(F0, reg.AsOverlappingFpuRegister()); + EXPECT_TRUE(reg.Overlaps(Mips64ManagedRegister::FromFpuRegister(F0))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromFpuRegister(F4))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromFpuRegister(F16))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromFpuRegister(F31))); + EXPECT_TRUE(reg.Overlaps(Mips64ManagedRegister::FromVectorRegister(W0))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromVectorRegister(W4))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromVectorRegister(W16))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromVectorRegister(W31))); + + reg = Mips64ManagedRegister::FromVectorRegister(W4); + reg_o = Mips64ManagedRegister::FromFpuRegister(F4); + EXPECT_TRUE(reg.Overlaps(reg_o)); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromGpuRegister(ZERO))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromGpuRegister(A0))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromGpuRegister(S0))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromGpuRegister(RA))); + EXPECT_EQ(W4, reg_o.AsOverlappingVectorRegister()); + EXPECT_EQ(F4, reg.AsOverlappingFpuRegister()); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromFpuRegister(F0))); + EXPECT_TRUE(reg.Overlaps(Mips64ManagedRegister::FromFpuRegister(F4))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromFpuRegister(F16))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromFpuRegister(F31))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromVectorRegister(W0))); + EXPECT_TRUE(reg.Overlaps(Mips64ManagedRegister::FromVectorRegister(W4))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromVectorRegister(W16))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromVectorRegister(W31))); + + reg = Mips64ManagedRegister::FromVectorRegister(W16); + reg_o = Mips64ManagedRegister::FromFpuRegister(F16); + EXPECT_TRUE(reg.Overlaps(reg_o)); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromGpuRegister(ZERO))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromGpuRegister(A0))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromGpuRegister(S0))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromGpuRegister(RA))); + EXPECT_EQ(W16, reg_o.AsOverlappingVectorRegister()); + EXPECT_EQ(F16, reg.AsOverlappingFpuRegister()); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromFpuRegister(F0))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromFpuRegister(F4))); + EXPECT_TRUE(reg.Overlaps(Mips64ManagedRegister::FromFpuRegister(F16))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromFpuRegister(F31))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromVectorRegister(W0))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromVectorRegister(W4))); + EXPECT_TRUE(reg.Overlaps(Mips64ManagedRegister::FromVectorRegister(W16))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromVectorRegister(W31))); + + reg = Mips64ManagedRegister::FromVectorRegister(W31); + reg_o = Mips64ManagedRegister::FromFpuRegister(F31); + EXPECT_TRUE(reg.Overlaps(reg_o)); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromGpuRegister(ZERO))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromGpuRegister(A0))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromGpuRegister(S0))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromGpuRegister(RA))); + EXPECT_EQ(W31, reg_o.AsOverlappingVectorRegister()); + EXPECT_EQ(F31, reg.AsOverlappingFpuRegister()); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromFpuRegister(F0))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromFpuRegister(F4))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromFpuRegister(F16))); + EXPECT_TRUE(reg.Overlaps(Mips64ManagedRegister::FromFpuRegister(F31))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromVectorRegister(W0))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromVectorRegister(W4))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromVectorRegister(W16))); + EXPECT_TRUE(reg.Overlaps(Mips64ManagedRegister::FromVectorRegister(W31))); + + reg = Mips64ManagedRegister::FromGpuRegister(ZERO); + EXPECT_TRUE(reg.Overlaps(Mips64ManagedRegister::FromGpuRegister(ZERO))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromGpuRegister(A0))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromGpuRegister(S0))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromGpuRegister(RA))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromFpuRegister(F0))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromFpuRegister(F4))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromFpuRegister(F16))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromFpuRegister(F31))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromVectorRegister(W0))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromVectorRegister(W4))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromVectorRegister(W16))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromVectorRegister(W31))); + + reg = Mips64ManagedRegister::FromGpuRegister(A0); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromGpuRegister(ZERO))); + EXPECT_TRUE(reg.Overlaps(Mips64ManagedRegister::FromGpuRegister(A0))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromGpuRegister(S0))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromGpuRegister(RA))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromFpuRegister(F0))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromFpuRegister(F4))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromFpuRegister(F16))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromFpuRegister(F31))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromVectorRegister(W0))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromVectorRegister(W4))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromVectorRegister(W16))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromVectorRegister(W31))); + + reg = Mips64ManagedRegister::FromGpuRegister(S0); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromGpuRegister(ZERO))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromGpuRegister(A0))); + EXPECT_TRUE(reg.Overlaps(Mips64ManagedRegister::FromGpuRegister(S0))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromGpuRegister(RA))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromFpuRegister(F0))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromFpuRegister(F4))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromFpuRegister(F16))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromFpuRegister(F31))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromVectorRegister(W0))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromVectorRegister(W4))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromVectorRegister(W16))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromVectorRegister(W31))); + + reg = Mips64ManagedRegister::FromGpuRegister(RA); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromGpuRegister(ZERO))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromGpuRegister(A0))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromGpuRegister(S0))); + EXPECT_TRUE(reg.Overlaps(Mips64ManagedRegister::FromGpuRegister(RA))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromFpuRegister(F0))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromFpuRegister(F4))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromFpuRegister(F16))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromFpuRegister(F31))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromVectorRegister(W0))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromVectorRegister(W4))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromVectorRegister(W16))); + EXPECT_FALSE(reg.Overlaps(Mips64ManagedRegister::FromVectorRegister(W31))); +} + +} // namespace mips64 +} // namespace art diff --git a/compiler/utils/x86/assembler_x86.cc b/compiler/utils/x86/assembler_x86.cc index 5307dc09d9..9c934b7f39 100644 --- a/compiler/utils/x86/assembler_x86.cc +++ b/compiler/utils/x86/assembler_x86.cc @@ -1221,6 +1221,24 @@ void X86Assembler::por(XmmRegister dst, XmmRegister src) { } +void X86Assembler::pavgb(XmmRegister dst, XmmRegister src) { + AssemblerBuffer::EnsureCapacity ensured(&buffer_); + EmitUint8(0x66); + EmitUint8(0x0F); + EmitUint8(0xE0); + EmitXmmRegisterOperand(dst, src); +} + + +void X86Assembler::pavgw(XmmRegister dst, XmmRegister src) { + AssemblerBuffer::EnsureCapacity ensured(&buffer_); + EmitUint8(0x66); + EmitUint8(0x0F); + EmitUint8(0xE3); + EmitXmmRegisterOperand(dst, src); +} + + void X86Assembler::pcmpeqb(XmmRegister dst, XmmRegister src) { AssemblerBuffer::EnsureCapacity ensured(&buffer_); EmitUint8(0x66); diff --git a/compiler/utils/x86/assembler_x86.h b/compiler/utils/x86/assembler_x86.h index f52cf16c8b..b87522a017 100644 --- a/compiler/utils/x86/assembler_x86.h +++ b/compiler/utils/x86/assembler_x86.h @@ -495,6 +495,9 @@ class X86Assembler FINAL : public Assembler { void orps(XmmRegister dst, XmmRegister src); void por(XmmRegister dst, XmmRegister src); + void pavgb(XmmRegister dst, XmmRegister src); // no addr variant (for now) + void pavgw(XmmRegister dst, XmmRegister src); + void pcmpeqb(XmmRegister dst, XmmRegister src); void pcmpeqw(XmmRegister dst, XmmRegister src); void pcmpeqd(XmmRegister dst, XmmRegister src); diff --git a/compiler/utils/x86/assembler_x86_test.cc b/compiler/utils/x86/assembler_x86_test.cc index 23049079e0..a01eb6dc23 100644 --- a/compiler/utils/x86/assembler_x86_test.cc +++ b/compiler/utils/x86/assembler_x86_test.cc @@ -605,6 +605,14 @@ TEST_F(AssemblerX86Test, POr) { DriverStr(RepeatFF(&x86::X86Assembler::por, "por %{reg2}, %{reg1}"), "por"); } +TEST_F(AssemblerX86Test, PAvgB) { + DriverStr(RepeatFF(&x86::X86Assembler::pavgb, "pavgb %{reg2}, %{reg1}"), "pavgb"); +} + +TEST_F(AssemblerX86Test, PAvgW) { + DriverStr(RepeatFF(&x86::X86Assembler::pavgw, "pavgw %{reg2}, %{reg1}"), "pavgw"); +} + TEST_F(AssemblerX86Test, PCmpeqB) { DriverStr(RepeatFF(&x86::X86Assembler::pcmpeqb, "pcmpeqb %{reg2}, %{reg1}"), "cmpeqb"); } diff --git a/compiler/utils/x86_64/assembler_x86_64.cc b/compiler/utils/x86_64/assembler_x86_64.cc index d20a6965c3..488c75de41 100644 --- a/compiler/utils/x86_64/assembler_x86_64.cc +++ b/compiler/utils/x86_64/assembler_x86_64.cc @@ -1427,6 +1427,24 @@ void X86_64Assembler::por(XmmRegister dst, XmmRegister src) { EmitXmmRegisterOperand(dst.LowBits(), src); } +void X86_64Assembler::pavgb(XmmRegister dst, XmmRegister src) { + AssemblerBuffer::EnsureCapacity ensured(&buffer_); + EmitUint8(0x66); + EmitOptionalRex32(dst, src); + EmitUint8(0x0F); + EmitUint8(0xE0); + EmitXmmRegisterOperand(dst.LowBits(), src); +} + +void X86_64Assembler::pavgw(XmmRegister dst, XmmRegister src) { + AssemblerBuffer::EnsureCapacity ensured(&buffer_); + EmitUint8(0x66); + EmitOptionalRex32(dst, src); + EmitUint8(0x0F); + EmitUint8(0xE3); + EmitXmmRegisterOperand(dst.LowBits(), src); +} + void X86_64Assembler::pcmpeqb(XmmRegister dst, XmmRegister src) { AssemblerBuffer::EnsureCapacity ensured(&buffer_); EmitUint8(0x66); diff --git a/compiler/utils/x86_64/assembler_x86_64.h b/compiler/utils/x86_64/assembler_x86_64.h index 08e17e81e5..fc2b117f71 100644 --- a/compiler/utils/x86_64/assembler_x86_64.h +++ b/compiler/utils/x86_64/assembler_x86_64.h @@ -523,6 +523,9 @@ class X86_64Assembler FINAL : public Assembler { void orps(XmmRegister dst, XmmRegister src); void por(XmmRegister dst, XmmRegister src); + void pavgb(XmmRegister dst, XmmRegister src); // no addr variant (for now) + void pavgw(XmmRegister dst, XmmRegister src); + void pcmpeqb(XmmRegister dst, XmmRegister src); void pcmpeqw(XmmRegister dst, XmmRegister src); void pcmpeqd(XmmRegister dst, XmmRegister src); diff --git a/compiler/utils/x86_64/assembler_x86_64_test.cc b/compiler/utils/x86_64/assembler_x86_64_test.cc index 20062fdb07..4adf210e47 100644 --- a/compiler/utils/x86_64/assembler_x86_64_test.cc +++ b/compiler/utils/x86_64/assembler_x86_64_test.cc @@ -1293,6 +1293,14 @@ TEST_F(AssemblerX86_64Test, Por) { DriverStr(RepeatFF(&x86_64::X86_64Assembler::por, "por %{reg2}, %{reg1}"), "por"); } +TEST_F(AssemblerX86_64Test, Pavgb) { + DriverStr(RepeatFF(&x86_64::X86_64Assembler::pavgb, "pavgb %{reg2}, %{reg1}"), "pavgb"); +} + +TEST_F(AssemblerX86_64Test, Pavgw) { + DriverStr(RepeatFF(&x86_64::X86_64Assembler::pavgw, "pavgw %{reg2}, %{reg1}"), "pavgw"); +} + TEST_F(AssemblerX86_64Test, PCmpeqb) { DriverStr(RepeatFF(&x86_64::X86_64Assembler::pcmpeqb, "pcmpeqb %{reg2}, %{reg1}"), "pcmpeqb"); } diff --git a/compiler/verifier_deps_test.cc b/compiler/verifier_deps_test.cc index 4bfc84990d..fa7e98586c 100644 --- a/compiler/verifier_deps_test.cc +++ b/compiler/verifier_deps_test.cc @@ -18,21 +18,21 @@ #include "verifier/verifier_deps.h" #include "class_linker.h" -#include "compiler/common_compiler_test.h" -#include "compiler/dex/verification_results.h" -#include "compiler/dex/verified_method.h" -#include "compiler/driver/compiler_options.h" -#include "compiler/driver/compiler_driver.h" -#include "compiler/utils/atomic_method_ref_map-inl.h" +#include "common_compiler_test.h" #include "compiler_callbacks.h" +#include "dex/verification_results.h" +#include "dex/verified_method.h" #include "dex_file.h" #include "dex_file_types.h" +#include "driver/compiler_options.h" +#include "driver/compiler_driver.h" #include "handle_scope-inl.h" #include "verifier/method_verifier-inl.h" #include "mirror/class_loader.h" #include "runtime.h" #include "thread.h" #include "scoped_thread_state_change-inl.h" +#include "utils/atomic_method_ref_map-inl.h" namespace art { namespace verifier { diff --git a/dexlayout/dex_ir.cc b/dexlayout/dex_ir.cc index 4228503a8f..4502626917 100644 --- a/dexlayout/dex_ir.cc +++ b/dexlayout/dex_ir.cc @@ -763,5 +763,138 @@ ClassData* Collections::CreateClassData( return class_data; } +static uint32_t HeaderOffset(const dex_ir::Collections& collections ATTRIBUTE_UNUSED) { + return 0; +} + +static uint32_t HeaderSize(const dex_ir::Collections& collections ATTRIBUTE_UNUSED) { + // Size is in elements, so there is only one header. + return 1; +} + +// The description of each dex file section type. +struct FileSectionDescriptor { + public: + std::string name; + uint16_t type; + // A function that when applied to a collection object, gives the size of the section. + std::function<uint32_t(const dex_ir::Collections&)> size_fn; + // A function that when applied to a collection object, gives the offset of the section. + std::function<uint32_t(const dex_ir::Collections&)> offset_fn; +}; + +static const std::vector<FileSectionDescriptor> kFileSectionDescriptors = { + { + "Header", + DexFile::kDexTypeHeaderItem, + &HeaderSize, + &HeaderOffset, + }, { + "StringId", + DexFile::kDexTypeStringIdItem, + &dex_ir::Collections::StringIdsSize, + &dex_ir::Collections::StringIdsOffset + }, { + "TypeId", + DexFile::kDexTypeTypeIdItem, + &dex_ir::Collections::TypeIdsSize, + &dex_ir::Collections::TypeIdsOffset + }, { + "ProtoId", + DexFile::kDexTypeProtoIdItem, + &dex_ir::Collections::ProtoIdsSize, + &dex_ir::Collections::ProtoIdsOffset + }, { + "FieldId", + DexFile::kDexTypeFieldIdItem, + &dex_ir::Collections::FieldIdsSize, + &dex_ir::Collections::FieldIdsOffset + }, { + "MethodId", + DexFile::kDexTypeMethodIdItem, + &dex_ir::Collections::MethodIdsSize, + &dex_ir::Collections::MethodIdsOffset + }, { + "ClassDef", + DexFile::kDexTypeClassDefItem, + &dex_ir::Collections::ClassDefsSize, + &dex_ir::Collections::ClassDefsOffset + }, { + "StringData", + DexFile::kDexTypeStringDataItem, + &dex_ir::Collections::StringDatasSize, + &dex_ir::Collections::StringDatasOffset + }, { + "TypeList", + DexFile::kDexTypeTypeList, + &dex_ir::Collections::TypeListsSize, + &dex_ir::Collections::TypeListsOffset + }, { + "EncArr", + DexFile::kDexTypeEncodedArrayItem, + &dex_ir::Collections::EncodedArrayItemsSize, + &dex_ir::Collections::EncodedArrayItemsOffset + }, { + "Annotation", + DexFile::kDexTypeAnnotationItem, + &dex_ir::Collections::AnnotationItemsSize, + &dex_ir::Collections::AnnotationItemsOffset + }, { + "AnnoSet", + DexFile::kDexTypeAnnotationSetItem, + &dex_ir::Collections::AnnotationSetItemsSize, + &dex_ir::Collections::AnnotationSetItemsOffset + }, { + "AnnoSetRL", + DexFile::kDexTypeAnnotationSetRefList, + &dex_ir::Collections::AnnotationSetRefListsSize, + &dex_ir::Collections::AnnotationSetRefListsOffset + }, { + "AnnoDir", + DexFile::kDexTypeAnnotationsDirectoryItem, + &dex_ir::Collections::AnnotationsDirectoryItemsSize, + &dex_ir::Collections::AnnotationsDirectoryItemsOffset + }, { + "DebugInfo", + DexFile::kDexTypeDebugInfoItem, + &dex_ir::Collections::DebugInfoItemsSize, + &dex_ir::Collections::DebugInfoItemsOffset + }, { + "CodeItem", + DexFile::kDexTypeCodeItem, + &dex_ir::Collections::CodeItemsSize, + &dex_ir::Collections::CodeItemsOffset + }, { + "ClassData", + DexFile::kDexTypeClassDataItem, + &dex_ir::Collections::ClassDatasSize, + &dex_ir::Collections::ClassDatasOffset + } +}; + +std::vector<dex_ir::DexFileSection> GetSortedDexFileSections(dex_ir::Header* header, + dex_ir::SortDirection direction) { + const dex_ir::Collections& collections = header->GetCollections(); + std::vector<dex_ir::DexFileSection> sorted_sections; + // Build the table that will map from offset to color + for (const FileSectionDescriptor& s : kFileSectionDescriptors) { + sorted_sections.push_back(dex_ir::DexFileSection(s.name, + s.type, + s.size_fn(collections), + s.offset_fn(collections))); + } + // Sort by offset. + std::sort(sorted_sections.begin(), + sorted_sections.end(), + [=](dex_ir::DexFileSection& a, dex_ir::DexFileSection& b) { + if (direction == SortDirection::kSortDescending) { + return a.offset > b.offset; + } else { + return a.offset < b.offset; + } + }); + return sorted_sections; +} + } // namespace dex_ir } // namespace art diff --git a/dexlayout/dex_ir.h b/dexlayout/dex_ir.h index 78ddde84a1..cad039550a 100644 --- a/dexlayout/dex_ir.h +++ b/dexlayout/dex_ir.h @@ -1104,6 +1104,28 @@ class MapItem : public Item { DISALLOW_COPY_AND_ASSIGN(MapItem); }; +// Interface for building a vector of file sections for use by other clients. +struct DexFileSection { + public: + DexFileSection(const std::string& name, uint16_t type, uint32_t size, uint32_t offset) + : name(name), type(type), size(size), offset(offset) { } + std::string name; + // The type (DexFile::MapItemType). + uint16_t type; + // The size (in elements, not bytes). + uint32_t size; + // The byte offset from the start of the file. + uint32_t offset; +}; + +enum class SortDirection { + kSortAscending, + kSortDescending +}; + +std::vector<DexFileSection> GetSortedDexFileSections(dex_ir::Header* header, + SortDirection direction); + } // namespace dex_ir } // namespace art diff --git a/dexlayout/dex_visualize.cc b/dexlayout/dex_visualize.cc index 452f51b28b..829e9feda8 100644 --- a/dexlayout/dex_visualize.cc +++ b/dexlayout/dex_visualize.cc @@ -41,142 +41,13 @@ static std::string MultidexName(const std::string& prefix, return prefix + ((dex_file_index > 0) ? std::to_string(dex_file_index + 1) : "") + suffix; } -struct FileSection { - public: - std::string name_; - uint16_t type_; - std::function<uint32_t(const dex_ir::Collections&)> size_fn_; - std::function<uint32_t(const dex_ir::Collections&)> offset_fn_; -}; - -static uint32_t HeaderOffset(const dex_ir::Collections& collections ATTRIBUTE_UNUSED) { - return 0; -} - -static uint32_t HeaderSize(const dex_ir::Collections& collections ATTRIBUTE_UNUSED) { - // Size is in elements, so there is only one header. - return 1; -} - -static const std::vector<FileSection> kFileSections = { - { - "Header", - DexFile::kDexTypeHeaderItem, - &HeaderSize, - &HeaderOffset, - }, { - "StringId", - DexFile::kDexTypeStringIdItem, - &dex_ir::Collections::StringIdsSize, - &dex_ir::Collections::StringIdsOffset - }, { - "TypeId", - DexFile::kDexTypeTypeIdItem, - &dex_ir::Collections::TypeIdsSize, - &dex_ir::Collections::TypeIdsOffset - }, { - "ProtoId", - DexFile::kDexTypeProtoIdItem, - &dex_ir::Collections::ProtoIdsSize, - &dex_ir::Collections::ProtoIdsOffset - }, { - "FieldId", - DexFile::kDexTypeFieldIdItem, - &dex_ir::Collections::FieldIdsSize, - &dex_ir::Collections::FieldIdsOffset - }, { - "MethodId", - DexFile::kDexTypeMethodIdItem, - &dex_ir::Collections::MethodIdsSize, - &dex_ir::Collections::MethodIdsOffset - }, { - "ClassDef", - DexFile::kDexTypeClassDefItem, - &dex_ir::Collections::ClassDefsSize, - &dex_ir::Collections::ClassDefsOffset - }, { - "StringData", - DexFile::kDexTypeStringDataItem, - &dex_ir::Collections::StringDatasSize, - &dex_ir::Collections::StringDatasOffset - }, { - "TypeList", - DexFile::kDexTypeTypeList, - &dex_ir::Collections::TypeListsSize, - &dex_ir::Collections::TypeListsOffset - }, { - "EncArr", - DexFile::kDexTypeEncodedArrayItem, - &dex_ir::Collections::EncodedArrayItemsSize, - &dex_ir::Collections::EncodedArrayItemsOffset - }, { - "Annotation", - DexFile::kDexTypeAnnotationItem, - &dex_ir::Collections::AnnotationItemsSize, - &dex_ir::Collections::AnnotationItemsOffset - }, { - "AnnoSet", - DexFile::kDexTypeAnnotationSetItem, - &dex_ir::Collections::AnnotationSetItemsSize, - &dex_ir::Collections::AnnotationSetItemsOffset - }, { - "AnnoSetRL", - DexFile::kDexTypeAnnotationSetRefList, - &dex_ir::Collections::AnnotationSetRefListsSize, - &dex_ir::Collections::AnnotationSetRefListsOffset - }, { - "AnnoDir", - DexFile::kDexTypeAnnotationsDirectoryItem, - &dex_ir::Collections::AnnotationsDirectoryItemsSize, - &dex_ir::Collections::AnnotationsDirectoryItemsOffset - }, { - "DebugInfo", - DexFile::kDexTypeDebugInfoItem, - &dex_ir::Collections::DebugInfoItemsSize, - &dex_ir::Collections::DebugInfoItemsOffset - }, { - "CodeItem", - DexFile::kDexTypeCodeItem, - &dex_ir::Collections::CodeItemsSize, - &dex_ir::Collections::CodeItemsOffset - }, { - "ClassData", - DexFile::kDexTypeClassDataItem, - &dex_ir::Collections::ClassDatasSize, - &dex_ir::Collections::ClassDatasOffset - } -}; - -static constexpr bool kSortAscending = false; -static constexpr bool kSortDescending = true; - -static std::vector<const FileSection*> GetSortedSections( - const dex_ir::Collections& collections, - bool sort_descending) { - std::vector<const FileSection*> sorted_sections; - // Build the table that will map from offset to color - for (const FileSection& s : kFileSections) { - sorted_sections.push_back(&s); - } - // Sort by offset. - std::sort(sorted_sections.begin(), - sorted_sections.end(), - [&](const FileSection* a, const FileSection* b) { - if (sort_descending) { - return a->offset_fn_(collections) > b->offset_fn_(collections); - } else { - return a->offset_fn_(collections) < b->offset_fn_(collections); - } - }); - return sorted_sections; -} - class Dumper { public: // Colors are based on the type of the section in MapList. - explicit Dumper(const dex_ir::Collections& collections) - : collections_(collections), out_file_(nullptr), - sorted_sections_(GetSortedSections(collections, kSortDescending)) { } + explicit Dumper(dex_ir::Header* header) + : out_file_(nullptr), + sorted_sections_( + dex_ir::GetSortedDexFileSections(header, dex_ir::SortDirection::kSortDescending)) { } bool OpenAndPrintHeader(size_t dex_index) { // Open the file and emit the gnuplot prologue. @@ -191,12 +62,13 @@ class Dumper { fprintf(out_file_, "set ylabel \"ClassDef index\"\n"); fprintf(out_file_, "set xtics rotate out ("); bool printed_one = false; - for (const FileSection& s : kFileSections) { - if (s.size_fn_(collections_) > 0) { + + for (const dex_ir::DexFileSection& s : sorted_sections_) { + if (s.size > 0) { if (printed_one) { fprintf(out_file_, ", "); } - fprintf(out_file_, "\"%s\" %d", s.name_.c_str(), s.offset_fn_(collections_) / kPageSize); + fprintf(out_file_, "\"%s\" %d", s.name.c_str(), s.offset / kPageSize); printed_one = true; } } @@ -209,9 +81,9 @@ class Dumper { int GetColor(uint32_t offset) const { // The dread linear search to find the right section for the reference. uint16_t section = 0; - for (const FileSection* file_section : sorted_sections_) { - if (file_section->offset_fn_(collections_) < offset) { - section = file_section->type_; + for (const dex_ir::DexFileSection& file_section : sorted_sections_) { + if (file_section.offset < offset) { + section = file_section.type; break; } } @@ -362,9 +234,8 @@ class Dumper { { DexFile::kDexTypeAnnotationsDirectoryItem, 16 } }; - const dex_ir::Collections& collections_; FILE* out_file_; - std::vector<const FileSection*> sorted_sections_; + std::vector<dex_ir::DexFileSection> sorted_sections_; DISALLOW_COPY_AND_ASSIGN(Dumper); }; @@ -377,7 +248,7 @@ void VisualizeDexLayout(dex_ir::Header* header, const DexFile* dex_file, size_t dex_file_index, ProfileCompilationInfo* profile_info) { - std::unique_ptr<Dumper> dumper(new Dumper(header->GetCollections())); + std::unique_ptr<Dumper> dumper(new Dumper(header)); if (!dumper->OpenAndPrintHeader(dex_file_index)) { fprintf(stderr, "Could not open output file.\n"); return; @@ -433,13 +304,12 @@ void VisualizeDexLayout(dex_ir::Header* header, } static uint32_t FindNextByteAfterSection(dex_ir::Header* header, - const dex_ir::Collections& collections, - std::vector<const FileSection*>& sorted_sections, + const std::vector<dex_ir::DexFileSection>& sorted_sections, size_t section_index) { for (size_t i = section_index + 1; i < sorted_sections.size(); ++i) { - const FileSection* section = sorted_sections[i]; - if (section->size_fn_(collections) != 0) { - return section->offset_fn_(collections); + const dex_ir::DexFileSection& section = sorted_sections.at(i); + if (section.size != 0) { + return section.offset; } } return header->FileSize(); @@ -454,19 +324,22 @@ void ShowDexSectionStatistics(dex_ir::Header* header, size_t dex_file_index) { MultidexName("classes", dex_file_index, ".dex").c_str(), header->FileSize()); fprintf(stdout, "section offset items bytes pages pct\n"); - const dex_ir::Collections& collections = header->GetCollections(); - std::vector<const FileSection*> sorted_sections(GetSortedSections(collections, kSortAscending)); + std::vector<dex_ir::DexFileSection> sorted_sections = + GetSortedDexFileSections(header, dex_ir::SortDirection::kSortAscending); for (size_t i = 0; i < sorted_sections.size(); ++i) { - const FileSection* file_section = sorted_sections[i]; - const char* name = file_section->name_.c_str(); - uint32_t offset = file_section->offset_fn_(collections); - uint32_t items = file_section->size_fn_(collections); + const dex_ir::DexFileSection& file_section = sorted_sections[i]; uint32_t bytes = 0; - if (items > 0) { - bytes = FindNextByteAfterSection(header, collections, sorted_sections, i) - offset; + if (file_section.size > 0) { + bytes = FindNextByteAfterSection(header, sorted_sections, i) - file_section.offset; } - fprintf(stdout, "%-10s %8d %8d %8d %8d %%%02d\n", name, offset, items, bytes, - (bytes + kPageSize - 1) / kPageSize, 100 * bytes / header->FileSize()); + fprintf(stdout, + "%-10s %8d %8d %8d %8d %%%02d\n", + file_section.name.c_str(), + file_section.offset, + file_section.size, + bytes, + RoundUp(bytes, kPageSize) / kPageSize, + 100 * bytes / header->FileSize()); } fprintf(stdout, "\n"); } diff --git a/dexlayout/dexlayout.cc b/dexlayout/dexlayout.cc index 615bcf92ea..0536f3223f 100644 --- a/dexlayout/dexlayout.cc +++ b/dexlayout/dexlayout.cc @@ -1528,6 +1528,111 @@ std::vector<dex_ir::ClassData*> DexLayout::LayoutClassDefsAndClassData(const Dex return new_class_data_order; } +void DexLayout::LayoutStringData(const DexFile* dex_file) { + const size_t num_strings = header_->GetCollections().StringIds().size(); + std::vector<bool> is_shorty(num_strings, false); + std::vector<bool> from_hot_method(num_strings, false); + for (std::unique_ptr<dex_ir::ClassDef>& class_def : header_->GetCollections().ClassDefs()) { + // A name of a profile class is probably going to get looked up by ClassTable::Lookup, mark it + // as hot. + const bool is_profile_class = + info_->ContainsClass(*dex_file, dex::TypeIndex(class_def->ClassType()->GetIndex())); + if (is_profile_class) { + from_hot_method[class_def->ClassType()->GetStringId()->GetIndex()] = true; + } + dex_ir::ClassData* data = class_def->GetClassData(); + if (data == nullptr) { + continue; + } + for (size_t i = 0; i < 2; ++i) { + for (auto& method : *(i == 0 ? data->DirectMethods() : data->VirtualMethods())) { + const dex_ir::MethodId* method_id = method->GetMethodId(); + dex_ir::CodeItem* code_item = method->GetCodeItem(); + if (code_item == nullptr) { + continue; + } + const bool is_clinit = is_profile_class && + (method->GetAccessFlags() & kAccConstructor) != 0 && + (method->GetAccessFlags() & kAccStatic) != 0; + const bool method_executed = is_clinit || + info_->ContainsMethod(MethodReference(dex_file, method_id->GetIndex())); + if (!method_executed) { + continue; + } + is_shorty[method_id->Proto()->Shorty()->GetIndex()] = true; + dex_ir::CodeFixups* fixups = code_item->GetCodeFixups(); + if (fixups == nullptr) { + continue; + } + if (fixups->StringIds() != nullptr) { + // Add const-strings. + for (dex_ir::StringId* id : *fixups->StringIds()) { + from_hot_method[id->GetIndex()] = true; + } + } + // TODO: Only visit field ids from static getters and setters. + for (dex_ir::FieldId* id : *fixups->FieldIds()) { + // Add the field names and types from getters and setters. + from_hot_method[id->Name()->GetIndex()] = true; + from_hot_method[id->Type()->GetStringId()->GetIndex()] = true; + } + } + } + } + // Sort string data by specified order. + std::vector<dex_ir::StringId*> string_ids; + size_t min_offset = std::numeric_limits<size_t>::max(); + size_t max_offset = 0; + size_t hot_bytes = 0; + for (auto& string_id : header_->GetCollections().StringIds()) { + string_ids.push_back(string_id.get()); + const size_t cur_offset = string_id->DataItem()->GetOffset(); + CHECK_NE(cur_offset, 0u); + min_offset = std::min(min_offset, cur_offset); + dex_ir::StringData* data = string_id->DataItem(); + const size_t element_size = data->GetSize() + 1; // Add one extra for null. + size_t end_offset = cur_offset + element_size; + if (is_shorty[string_id->GetIndex()] || from_hot_method[string_id->GetIndex()]) { + hot_bytes += element_size; + } + max_offset = std::max(max_offset, end_offset); + } + VLOG(compiler) << "Hot string data bytes " << hot_bytes << "/" << max_offset - min_offset; + std::sort(string_ids.begin(), + string_ids.end(), + [&is_shorty, &from_hot_method](const dex_ir::StringId* a, + const dex_ir::StringId* b) { + const bool a_is_hot = from_hot_method[a->GetIndex()]; + const bool b_is_hot = from_hot_method[b->GetIndex()]; + if (a_is_hot != b_is_hot) { + return a_is_hot < b_is_hot; + } + // After hot methods are partitioned, subpartition shorties. + const bool a_is_shorty = is_shorty[a->GetIndex()]; + const bool b_is_shorty = is_shorty[b->GetIndex()]; + if (a_is_shorty != b_is_shorty) { + return a_is_shorty < b_is_shorty; + } + // Preserve order. + return a->DataItem()->GetOffset() < b->DataItem()->GetOffset(); + }); + // Now we know what order we want the string data, reorder the offsets. + size_t offset = min_offset; + for (dex_ir::StringId* string_id : string_ids) { + dex_ir::StringData* data = string_id->DataItem(); + data->SetOffset(offset); + offset += data->GetSize() + 1; // Add one extra for null. + } + if (offset > max_offset) { + const uint32_t diff = offset - max_offset; + // If we expanded the string data section, we need to update the offsets or else we will + // corrupt the next section when writing out. + FixupSections(header_->GetCollections().StringDatasOffset(), diff); + // Update file size. + header_->SetFileSize(header_->FileSize() + diff); + } +} + // Orders code items according to specified class data ordering. // NOTE: If the section following the code items is byte aligned, the last code item is left in // place to preserve alignment. Layout needs an overhaul to handle movement of other sections. @@ -1686,6 +1791,7 @@ void DexLayout::FixupSections(uint32_t offset, uint32_t diff) { } void DexLayout::LayoutOutputFile(const DexFile* dex_file) { + LayoutStringData(dex_file); std::vector<dex_ir::ClassData*> new_class_data_order = LayoutClassDefsAndClassData(dex_file); int32_t diff = LayoutCodeItems(new_class_data_order); // Move sections after ClassData by diff bytes. @@ -1700,7 +1806,7 @@ void DexLayout::OutputDexFile(const DexFile* dex_file) { std::unique_ptr<File> new_file; if (!options_.output_to_memmap_) { std::string output_location(options_.output_dex_directory_); - size_t last_slash = dex_file_location.rfind("/"); + size_t last_slash = dex_file_location.rfind('/'); std::string dex_file_directory = dex_file_location.substr(0, last_slash + 1); if (output_location == dex_file_directory) { output_location = dex_file_location + ".new"; diff --git a/dexlayout/dexlayout.h b/dexlayout/dexlayout.h index f26b423847..69117ad763 100644 --- a/dexlayout/dexlayout.h +++ b/dexlayout/dexlayout.h @@ -109,6 +109,7 @@ class DexLayout { std::vector<dex_ir::ClassData*> LayoutClassDefsAndClassData(const DexFile* dex_file); int32_t LayoutCodeItems(std::vector<dex_ir::ClassData*> new_class_data_order); + void LayoutStringData(const DexFile* dex_file); bool IsNextSectionCodeItemAligned(uint32_t offset); template<class T> void FixupSection(std::map<uint32_t, std::unique_ptr<T>>& map, uint32_t diff); void FixupSections(uint32_t offset, uint32_t diff); diff --git a/dexlayout/dexlayout_test.cc b/dexlayout/dexlayout_test.cc index bd6548e65b..4f7040427e 100644 --- a/dexlayout/dexlayout_test.cc +++ b/dexlayout/dexlayout_test.cc @@ -41,19 +41,7 @@ static const char kDexFileLayoutInputDex[] = "AAAAdQEAAAAQAAABAAAAjAEAAA=="; static const char kDexFileLayoutInputProfile[] = - "cHJvADAwNAABCwABAAAAAAD1KW3+Y2xhc3Nlcy5kZXgBAA=="; - -static const char kDexFileLayoutExpectedOutputDex[] = - "ZGV4CjAzNQD1KW3+B8NAB0f2A/ZVIBJ0aHrGIqcpVTAUAgAAcAAAAHhWNBIAAAAAAAAAAIwBAAAH" - "AAAAcAAAAAQAAACMAAAAAQAAAJwAAAAAAAAAAAAAAAMAAACoAAAAAgAAAMAAAAAUAQAAAAEAADAB" - "AAA4AQAAQAEAAEgBAABNAQAAUgEAAGYBAAADAAAABAAAAAUAAAAGAAAABgAAAAMAAAAAAAAAAAAA" - "AAAAAAABAAAAAAAAAAIAAAAAAAAAAQAAAAAAAAACAAAAAAAAAAIAAAAAAAAAdQEAAAAAAAAAAAAA" - "AAAAAAIAAAAAAAAAAQAAAAAAAAB/AQAAAAAAAAEAAQABAAAAbwEAAAQAAABwEAIAAAAOAAEAAQAB" - "AAAAaQEAAAQAAABwEAIAAAAOAAY8aW5pdD4ABkEuamF2YQAGQi5qYXZhAANMQTsAA0xCOwASTGph" - "dmEvbGFuZy9PYmplY3Q7AAFWAAQABw48AAQABw48AAAAAQABgIAEgAIAAAEAAICABJgCAAAACwAA" - "AAAAAAABAAAAAAAAAAEAAAAHAAAAcAAAAAIAAAAEAAAAjAAAAAMAAAABAAAAnAAAAAUAAAADAAAA" - "qAAAAAYAAAACAAAAwAAAAAEgAAACAAAAAAEAAAIgAAAHAAAAMAEAAAMgAAACAAAAaQEAAAAgAAAC" - "AAAAdQEAAAAQAAABAAAAjAEAAA=="; + "cHJvADAwNQABCwABAAAAAAD1KW3+Y2xhc3Nlcy5kZXgBAA=="; // Dex file with catch handler unreferenced by try blocks. // Constructed by building a dex file with try/catch blocks and hex editing. @@ -314,26 +302,21 @@ class DexLayoutTest : public CommonRuntimeTest { WriteFileBase64(kDexFileLayoutInputDex, dex_file.c_str()); std::string profile_file = tmp_dir + "primary.prof"; WriteFileBase64(kDexFileLayoutInputProfile, profile_file.c_str()); - std::string expected_output = tmp_dir + "expected.dex"; - WriteFileBase64(kDexFileLayoutExpectedOutputDex, expected_output.c_str()); std::string output_dex = tmp_dir + "classes.dex.new"; std::string dexlayout = GetTestAndroidRoot() + "/bin/dexlayout"; EXPECT_TRUE(OS::FileExists(dexlayout.c_str())) << dexlayout << " should be a valid file path"; std::vector<std::string> dexlayout_exec_argv = - { dexlayout, "-w", tmp_dir, "-o", tmp_name, "-p", profile_file, dex_file }; + { dexlayout, "-v", "-w", tmp_dir, "-o", tmp_name, "-p", profile_file, dex_file }; if (!::art::Exec(dexlayout_exec_argv, error_msg)) { return false; } - std::vector<std::string> diff_exec_argv = - { "/usr/bin/diff", expected_output, output_dex }; - if (!::art::Exec(diff_exec_argv, error_msg)) { - return false; - } + + // -v makes sure that the layout did not corrupt the dex file. std::vector<std::string> rm_exec_argv = - { "/bin/rm", dex_file, profile_file, expected_output, output_dex }; + { "/bin/rm", dex_file, profile_file, output_dex }; if (!::art::Exec(rm_exec_argv, error_msg)) { return false; } diff --git a/disassembler/disassembler_mips.cc b/disassembler/disassembler_mips.cc index fc6c18b1df..eb57d339af 100644 --- a/disassembler/disassembler_mips.cc +++ b/disassembler/disassembler_mips.cc @@ -448,6 +448,7 @@ static const MipsInstruction gMipsInstructions[] = { { kMsaMask | (0x3ff << 16), kMsa | (0xbe << 16) | 0x19, "move.v", "km" }, { kMsaMask | (0xf << 22), kMsa | (0x1 << 22) | 0x19, "splati", "kX" }, { kMsaMask | (0xff << 18), kMsa | (0xc0 << 18) | 0x1e, "fill", "vkD" }, + { kMsaMask | (0x7 << 23), kMsa | (0x6 << 23) | 0x7, "ldi", "kx" }, { kMsaSpecialMask | (0xf << 2), kMsa | (0x8 << 2), "ld", "kw" }, { kMsaSpecialMask | (0xf << 2), kMsa | (0x9 << 2), "st", "kw" }, }; @@ -697,6 +698,20 @@ size_t DisassemblerMips::Dump(std::ostream& os, const uint8_t* instr_ptr) { } break; } + case 'x': // MSA i10. + { + int32_t df = (instruction >> 21) & 0x3; + int32_t i10 = (instruction >> 11) & 0x3ff; + i10 -= (i10 & 0x200) << 1; // Sign-extend i10. + switch (df) { + case 0: opcode += ".b"; break; + case 1: opcode += ".h"; break; + case 2: opcode += ".w"; break; + case 3: opcode += ".d"; break; + } + args << i10; + break; + } } if (*(args_fmt + 1)) { args << ", "; diff --git a/disassembler/disassembler_x86.cc b/disassembler/disassembler_x86.cc index 77ed3c6a22..f5c3ad20cc 100644 --- a/disassembler/disassembler_x86.cc +++ b/disassembler/disassembler_x86.cc @@ -1101,6 +1101,22 @@ DISASSEMBLER_ENTRY(cmp, opcode1 = opcode_tmp.c_str(); } break; + case 0xE0: + case 0xE3: + if (prefix[2] == 0x66) { + src_reg_file = dst_reg_file = SSE; + prefix[2] = 0; // clear prefix now it's served its purpose as part of the opcode + } else { + src_reg_file = dst_reg_file = MMX; + } + switch (*instr) { + case 0xE0: opcode1 = "pavgb"; break; + case 0xE3: opcode1 = "pavgw"; break; + } + prefix[2] = 0; + has_modrm = true; + load = true; + break; case 0xEB: if (prefix[2] == 0x66) { src_reg_file = dst_reg_file = SSE; diff --git a/patchoat/patchoat.cc b/patchoat/patchoat.cc index dfaae7d864..0c2717f207 100644 --- a/patchoat/patchoat.cc +++ b/patchoat/patchoat.cc @@ -485,7 +485,7 @@ class PatchOat::RelocatedPointerVisitor { explicit RelocatedPointerVisitor(PatchOat* patch_oat) : patch_oat_(patch_oat) {} template <typename T> - T* operator()(T* ptr) const { + T* operator()(T* ptr, void** dest_addr ATTRIBUTE_UNUSED = 0) const { return patch_oat_->RelocatedAddressOfPointer(ptr); } diff --git a/profman/profile_assistant.cc b/profman/profile_assistant.cc index a25460e427..b9a85bc9af 100644 --- a/profman/profile_assistant.cc +++ b/profman/profile_assistant.cc @@ -44,10 +44,15 @@ ProfileAssistant::ProcessingResult ProfileAssistant::ProcessProfilesInternal( // Merge all current profiles. for (size_t i = 0; i < profile_files.size(); i++) { - if (!info.Load(profile_files[i].GetFile()->Fd())) { + ProfileCompilationInfo cur_info; + if (!cur_info.Load(profile_files[i].GetFile()->Fd())) { LOG(WARNING) << "Could not load profile file at index " << i; return kErrorBadProfiles; } + if (!info.MergeWith(cur_info)) { + LOG(WARNING) << "Could not merge profile file at index " << i; + return kErrorBadProfiles; + } } // Check if there is enough new information added by the current profiles. diff --git a/profman/profile_assistant_test.cc b/profman/profile_assistant_test.cc index 1a8a614a4a..94f6e708fe 100644 --- a/profman/profile_assistant_test.cc +++ b/profman/profile_assistant_test.cc @@ -37,14 +37,25 @@ class ProfileAssistantTest : public CommonRuntimeTest { uint16_t number_of_classes, const ScratchFile& profile, ProfileCompilationInfo* info, - uint16_t start_method_index = 0) { + uint16_t start_method_index = 0, + bool reverse_dex_write_order = false) { std::string dex_location1 = "location1" + id; uint32_t dex_location_checksum1 = checksum; std::string dex_location2 = "location2" + id; uint32_t dex_location_checksum2 = 10 * checksum; for (uint16_t i = start_method_index; i < start_method_index + number_of_methods; i++) { - ASSERT_TRUE(info->AddMethodIndex(dex_location1, dex_location_checksum1, i)); - ASSERT_TRUE(info->AddMethodIndex(dex_location2, dex_location_checksum2, i)); + // reverse_dex_write_order controls the order in which the dex files will be added to + // the profile and thus written to disk. + ProfileCompilationInfo::OfflineProfileMethodInfo pmi = + GetOfflineProfileMethodInfo(dex_location1, dex_location_checksum1, + dex_location2, dex_location_checksum2); + if (reverse_dex_write_order) { + ASSERT_TRUE(info->AddMethod(dex_location2, dex_location_checksum2, i, pmi)); + ASSERT_TRUE(info->AddMethod(dex_location1, dex_location_checksum1, i, pmi)); + } else { + ASSERT_TRUE(info->AddMethod(dex_location1, dex_location_checksum1, i, pmi)); + ASSERT_TRUE(info->AddMethod(dex_location2, dex_location_checksum2, i, pmi)); + } } for (uint16_t i = 0; i < number_of_classes; i++) { ASSERT_TRUE(info->AddClassIndex(dex_location1, dex_location_checksum1, dex::TypeIndex(i))); @@ -55,6 +66,43 @@ class ProfileAssistantTest : public CommonRuntimeTest { ASSERT_TRUE(profile.GetFile()->ResetOffset()); } + ProfileCompilationInfo::OfflineProfileMethodInfo GetOfflineProfileMethodInfo( + const std::string& dex_location1, uint32_t dex_checksum1, + const std::string& dex_location2, uint32_t dex_checksum2) { + ProfileCompilationInfo::OfflineProfileMethodInfo pmi; + pmi.dex_references.emplace_back(dex_location1, dex_checksum1); + pmi.dex_references.emplace_back(dex_location2, dex_checksum2); + + // Monomorphic + for (uint16_t dex_pc = 0; dex_pc < 11; dex_pc++) { + ProfileCompilationInfo::DexPcData dex_pc_data; + dex_pc_data.AddClass(0, dex::TypeIndex(0)); + pmi.inline_caches.Put(dex_pc, dex_pc_data); + } + // Polymorphic + for (uint16_t dex_pc = 11; dex_pc < 22; dex_pc++) { + ProfileCompilationInfo::DexPcData dex_pc_data; + dex_pc_data.AddClass(0, dex::TypeIndex(0)); + dex_pc_data.AddClass(1, dex::TypeIndex(1)); + + pmi.inline_caches.Put(dex_pc, dex_pc_data); + } + // Megamorphic + for (uint16_t dex_pc = 22; dex_pc < 33; dex_pc++) { + ProfileCompilationInfo::DexPcData dex_pc_data; + dex_pc_data.SetIsMegamorphic(); + pmi.inline_caches.Put(dex_pc, dex_pc_data); + } + // Missing types + for (uint16_t dex_pc = 33; dex_pc < 44; dex_pc++) { + ProfileCompilationInfo::DexPcData dex_pc_data; + dex_pc_data.SetIsMissingTypes(); + pmi.inline_caches.Put(dex_pc, dex_pc_data); + } + + return pmi; + } + int GetFd(const ScratchFile& file) const { return static_cast<int>(file.GetFd()); } @@ -99,6 +147,18 @@ class ProfileAssistantTest : public CommonRuntimeTest { return ExecAndReturnCode(argv_str, &error); } + bool GenerateTestProfileWithInputDex(const std::string& filename) { + std::string profman_cmd = GetProfmanCmd(); + std::vector<std::string> argv_str; + argv_str.push_back(profman_cmd); + argv_str.push_back("--generate-test-profile=" + filename); + argv_str.push_back("--generate-test-profile-seed=0"); + argv_str.push_back("--apk=" + GetLibCoreDexFileNames()[0]); + argv_str.push_back("--dex-location=" + GetLibCoreDexFileNames()[0]); + std::string error; + return ExecAndReturnCode(argv_str, &error); + } + bool CreateProfile(std::string profile_file_contents, const std::string& filename, const std::string& dex_location) { @@ -425,6 +485,17 @@ TEST_F(ProfileAssistantTest, TestProfileGeneration) { ASSERT_TRUE(info.Load(GetFd(profile))); } +TEST_F(ProfileAssistantTest, TestProfileGenerationWithIndexDex) { + ScratchFile profile; + // Generate a test profile passing in a dex file as reference. + GenerateTestProfileWithInputDex(profile.GetFilename()); + + // Verify that the generated profile is valid and can be loaded. + ASSERT_TRUE(profile.GetFile()->ResetOffset()); + ProfileCompilationInfo info; + ASSERT_TRUE(info.Load(GetFd(profile))); +} + TEST_F(ProfileAssistantTest, TestProfileCreationAllMatch) { // Class names put here need to be in sorted order. std::vector<std::string> class_names = { @@ -629,4 +700,44 @@ TEST_F(ProfileAssistantTest, TestProfileCreateInlineCache) { } } +TEST_F(ProfileAssistantTest, MergeProfilesWithDifferentDexOrder) { + ScratchFile profile1; + ScratchFile reference_profile; + + std::vector<int> profile_fds({GetFd(profile1)}); + int reference_profile_fd = GetFd(reference_profile); + + // The new profile info will contain the methods with indices 0-100. + const uint16_t kNumberOfMethodsToEnableCompilation = 100; + ProfileCompilationInfo info1; + SetupProfile("p1", 1, kNumberOfMethodsToEnableCompilation, 0, profile1, &info1, + /*start_method_index*/0, /*reverse_dex_write_order*/false); + + // The reference profile info will contain the methods with indices 50-150. + // When setting up the profile reverse the order in which the dex files + // are added to the profile. This will verify that profman merges profiles + // with a different dex order correctly. + const uint16_t kNumberOfMethodsAlreadyCompiled = 100; + ProfileCompilationInfo reference_info; + SetupProfile("p1", 1, kNumberOfMethodsAlreadyCompiled, 0, reference_profile, + &reference_info, kNumberOfMethodsToEnableCompilation / 2, /*reverse_dex_write_order*/true); + + // We should advise compilation. + ASSERT_EQ(ProfileAssistant::kCompile, + ProcessProfiles(profile_fds, reference_profile_fd)); + + // The resulting compilation info must be equal to the merge of the inputs. + ProfileCompilationInfo result; + ASSERT_TRUE(reference_profile.GetFile()->ResetOffset()); + ASSERT_TRUE(result.Load(reference_profile_fd)); + + ProfileCompilationInfo expected; + ASSERT_TRUE(expected.MergeWith(reference_info)); + ASSERT_TRUE(expected.MergeWith(info1)); + ASSERT_TRUE(expected.Equals(result)); + + // The information from profile must remain the same. + CheckProfileInfo(profile1, info1); +} + } // namespace art diff --git a/profman/profman.cc b/profman/profman.cc index fdb9a75a6f..5504695a96 100644 --- a/profman/profman.cc +++ b/profman/profman.cc @@ -117,6 +117,8 @@ NO_RETURN static void Usage(const char *fmt, ...) { UsageError(" number of methods that should be generated. Defaults to 5."); UsageError(" --generate-test-profile-class-ratio=<number>: the percentage from the maximum"); UsageError(" number of classes that should be generated. Defaults to 5."); + UsageError(" --generate-test-profile-seed=<number>: seed for random number generator used when"); + UsageError(" generating random test profiles. Defaults to using NanoTime."); UsageError(""); UsageError(" --create-profile-from=<filename>: creates a profile from a list of classes."); UsageError(""); @@ -156,6 +158,7 @@ class ProfMan FINAL { test_profile_num_dex_(kDefaultTestProfileNumDex), test_profile_method_ratio_(kDefaultTestProfileMethodRatio), test_profile_class_ratio_(kDefaultTestProfileClassRatio), + test_profile_seed_(NanoTime()), start_ns_(NanoTime()) {} ~ProfMan() { @@ -221,6 +224,8 @@ class ProfMan FINAL { "--generate-test-profile-class-ratio", &test_profile_class_ratio_, Usage); + } else if (option.starts_with("--generate-test-profile-seed=")) { + ParseUintOption(option, "--generate-test-profile-seed", &test_profile_seed_, Usage); } else { Usage("Unknown argument '%s'", option.data()); } @@ -418,7 +423,7 @@ class ProfMan FINAL { return true; } - bool GetClassNames(std::string profile_file, + bool GetClassNames(const std::string& profile_file, std::vector<std::unique_ptr<const DexFile>>* dex_files, std::set<std::string>* class_names) { int fd = open(profile_file.c_str(), O_RDONLY); @@ -711,7 +716,7 @@ class ProfMan FINAL { } std::vector<ProfileMethodInfo::ProfileClassReference> classes(inline_cache_elems.size()); size_t class_it = 0; - for (const std::string ic_class : inline_cache_elems) { + for (const std::string& ic_class : inline_cache_elems) { if (!FindClass(dex_files, ic_class, &(classes[class_it++]))) { LOG(ERROR) << "Could not find class: " << ic_class; return false; @@ -798,17 +803,39 @@ class ProfMan FINAL { if (test_profile_class_ratio_ > 100) { Usage("Invalid ratio for --generate-test-profile-class-ratio"); } + // If given APK files or DEX locations, check that they're ok. + if (!apk_files_.empty() || !apks_fd_.empty() || !dex_locations_.empty()) { + if (apk_files_.empty() && apks_fd_.empty()) { + Usage("APK files must be specified when passing DEX locations to --generate-test-profile"); + } + if (dex_locations_.empty()) { + Usage("DEX locations must be specified when passing APK files to --generate-test-profile"); + } + } // ShouldGenerateTestProfile confirms !test_profile_.empty(). int profile_test_fd = open(test_profile_.c_str(), O_CREAT | O_TRUNC | O_WRONLY, 0644); if (profile_test_fd < 0) { LOG(ERROR) << "Cannot open " << test_profile_ << strerror(errno); return -1; } - - bool result = ProfileCompilationInfo::GenerateTestProfile(profile_test_fd, - test_profile_num_dex_, - test_profile_method_ratio_, - test_profile_class_ratio_); + bool result; + if (apk_files_.empty() && apks_fd_.empty() && dex_locations_.empty()) { + result = ProfileCompilationInfo::GenerateTestProfile(profile_test_fd, + test_profile_num_dex_, + test_profile_method_ratio_, + test_profile_class_ratio_, + test_profile_seed_); + } else { + // Initialize MemMap for ZipArchive::OpenFromFd. + MemMap::Init(); + // Open the dex files to look up classes and methods. + std::vector<std::unique_ptr<const DexFile>> dex_files; + OpenApkFilesFromLocations(&dex_files); + // Create a random profile file based on the set of dex files. + result = ProfileCompilationInfo::GenerateTestProfile(profile_test_fd, + dex_files, + test_profile_seed_); + } close(profile_test_fd); // ignore close result. return result ? 0 : -1; } @@ -857,6 +884,7 @@ class ProfMan FINAL { uint16_t test_profile_num_dex_; uint16_t test_profile_method_ratio_; uint16_t test_profile_class_ratio_; + uint32_t test_profile_seed_; uint64_t start_ns_; }; diff --git a/runtime/arch/arm64/quick_entrypoints_arm64.S b/runtime/arch/arm64/quick_entrypoints_arm64.S index 5b5d2ef0dc..26622f0482 100644 --- a/runtime/arch/arm64/quick_entrypoints_arm64.S +++ b/runtime/arch/arm64/quick_entrypoints_arm64.S @@ -2379,9 +2379,8 @@ ENTRY \name ret .Lnot_marked_rb_\name: // Check if the top two bits are one, if this is the case it is a forwarding address. - mvn wIP0, wIP0 - cmp wzr, wIP0, lsr #30 - beq .Lret_forwarding_address\name + tst wIP0, wIP0, lsl #1 + bmi .Lret_forwarding_address\name .Lslow_rb_\name: /* * Allocate 44 stack slots * 8 = 352 bytes: @@ -2452,10 +2451,9 @@ ENTRY \name DECREASE_FRAME 352 ret .Lret_forwarding_address\name: - mvn wIP0, wIP0 // Shift left by the forwarding address shift. This clears out the state bits since they are // in the top 2 bits of the lock word. - lsl \wreg, wIP0, #LOCK_WORD_STATE_FORWARDING_ADDRESS_SHIFT + lsl \wreg, wIP0, #LOCK_WORD_STATE_FORWARDING_ADDRESS_SHIFT ret END \name .endm diff --git a/runtime/arch/mips/entrypoints_init_mips.cc b/runtime/arch/mips/entrypoints_init_mips.cc index 36f9ea78e1..2349620c1d 100644 --- a/runtime/arch/mips/entrypoints_init_mips.cc +++ b/runtime/arch/mips/entrypoints_init_mips.cc @@ -32,6 +32,33 @@ namespace art { // Cast entrypoints. extern "C" size_t artInstanceOfFromCode(mirror::Object* obj, mirror::Class* ref_class); +// Read barrier entrypoints. +// art_quick_read_barrier_mark_regXX uses a non-standard calling +// convention: it expects its input in register XX+1 and returns its +// result in that same register, and saves and restores all +// caller-save registers. +extern "C" mirror::Object* art_quick_read_barrier_mark_reg01(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg02(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg03(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg04(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg05(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg06(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg07(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg08(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg09(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg10(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg11(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg12(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg13(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg14(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg17(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg18(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg19(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg20(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg21(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg22(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg29(mirror::Object*); + // Math entrypoints. extern int32_t CmpgDouble(double a, double b); extern int32_t CmplDouble(double a, double b); @@ -59,9 +86,71 @@ extern "C" double fmod(double a, double b); // REM_DOUBLE[_2ADDR] extern "C" int64_t __divdi3(int64_t, int64_t); extern "C" int64_t __moddi3(int64_t, int64_t); -// No read barrier entrypoints for marking registers. -void UpdateReadBarrierEntrypoints(QuickEntryPoints* qpoints ATTRIBUTE_UNUSED, - bool is_marking ATTRIBUTE_UNUSED) {} +void UpdateReadBarrierEntrypoints(QuickEntryPoints* qpoints, bool is_marking) { + qpoints->pReadBarrierMarkReg01 = is_marking ? art_quick_read_barrier_mark_reg01 : nullptr; + static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg01), + "Non-direct C stub marked direct."); + qpoints->pReadBarrierMarkReg02 = is_marking ? art_quick_read_barrier_mark_reg02 : nullptr; + static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg02), + "Non-direct C stub marked direct."); + qpoints->pReadBarrierMarkReg03 = is_marking ? art_quick_read_barrier_mark_reg03 : nullptr; + static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg03), + "Non-direct C stub marked direct."); + qpoints->pReadBarrierMarkReg04 = is_marking ? art_quick_read_barrier_mark_reg04 : nullptr; + static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg04), + "Non-direct C stub marked direct."); + qpoints->pReadBarrierMarkReg05 = is_marking ? art_quick_read_barrier_mark_reg05 : nullptr; + static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg05), + "Non-direct C stub marked direct."); + qpoints->pReadBarrierMarkReg06 = is_marking ? art_quick_read_barrier_mark_reg06 : nullptr; + static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg06), + "Non-direct C stub marked direct."); + qpoints->pReadBarrierMarkReg07 = is_marking ? art_quick_read_barrier_mark_reg07 : nullptr; + static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg07), + "Non-direct C stub marked direct."); + qpoints->pReadBarrierMarkReg08 = is_marking ? art_quick_read_barrier_mark_reg08 : nullptr; + static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg08), + "Non-direct C stub marked direct."); + qpoints->pReadBarrierMarkReg09 = is_marking ? art_quick_read_barrier_mark_reg09 : nullptr; + static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg09), + "Non-direct C stub marked direct."); + qpoints->pReadBarrierMarkReg10 = is_marking ? art_quick_read_barrier_mark_reg10 : nullptr; + static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg10), + "Non-direct C stub marked direct."); + qpoints->pReadBarrierMarkReg11 = is_marking ? art_quick_read_barrier_mark_reg11 : nullptr; + static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg11), + "Non-direct C stub marked direct."); + qpoints->pReadBarrierMarkReg12 = is_marking ? art_quick_read_barrier_mark_reg12 : nullptr; + static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg12), + "Non-direct C stub marked direct."); + qpoints->pReadBarrierMarkReg13 = is_marking ? art_quick_read_barrier_mark_reg13 : nullptr; + static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg13), + "Non-direct C stub marked direct."); + qpoints->pReadBarrierMarkReg14 = is_marking ? art_quick_read_barrier_mark_reg14 : nullptr; + static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg14), + "Non-direct C stub marked direct."); + qpoints->pReadBarrierMarkReg17 = is_marking ? art_quick_read_barrier_mark_reg17 : nullptr; + static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg17), + "Non-direct C stub marked direct."); + qpoints->pReadBarrierMarkReg18 = is_marking ? art_quick_read_barrier_mark_reg18 : nullptr; + static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg18), + "Non-direct C stub marked direct."); + qpoints->pReadBarrierMarkReg19 = is_marking ? art_quick_read_barrier_mark_reg19 : nullptr; + static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg19), + "Non-direct C stub marked direct."); + qpoints->pReadBarrierMarkReg20 = is_marking ? art_quick_read_barrier_mark_reg20 : nullptr; + static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg20), + "Non-direct C stub marked direct."); + qpoints->pReadBarrierMarkReg21 = is_marking ? art_quick_read_barrier_mark_reg21 : nullptr; + static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg21), + "Non-direct C stub marked direct."); + qpoints->pReadBarrierMarkReg22 = is_marking ? art_quick_read_barrier_mark_reg22 : nullptr; + static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg22), + "Non-direct C stub marked direct."); + qpoints->pReadBarrierMarkReg29 = is_marking ? art_quick_read_barrier_mark_reg29 : nullptr; + static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg29), + "Non-direct C stub marked direct."); +} void InitEntryPoints(JniEntryPoints* jpoints, QuickEntryPoints* qpoints) { // Note: MIPS has asserts checking for the type of entrypoint. Don't move it @@ -287,77 +376,19 @@ void InitEntryPoints(JniEntryPoints* jpoints, QuickEntryPoints* qpoints) { // Read barrier. qpoints->pReadBarrierJni = ReadBarrierJni; static_assert(IsDirectEntrypoint(kQuickReadBarrierJni), "Direct C stub not marked direct."); - // Read barriers (and these entry points in particular) are not - // supported in the compiler on MIPS32. + UpdateReadBarrierEntrypoints(qpoints, /*is_marking*/ false); + // Cannot use the following registers to pass arguments: + // 0(ZERO), 1(AT), 16(S0), 17(S1), 24(T8), 25(T9), 26(K0), 27(K1), 28(GP), 29(SP), 31(RA). + // Note that there are 30 entry points only: 00 for register 1(AT), ..., 29 for register 30(S8). qpoints->pReadBarrierMarkReg00 = nullptr; static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg00), "Non-direct C stub marked direct."); - qpoints->pReadBarrierMarkReg01 = nullptr; - static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg01), - "Non-direct C stub marked direct."); - qpoints->pReadBarrierMarkReg02 = nullptr; - static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg02), - "Non-direct C stub marked direct."); - qpoints->pReadBarrierMarkReg03 = nullptr; - static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg03), - "Non-direct C stub marked direct."); - qpoints->pReadBarrierMarkReg04 = nullptr; - static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg04), - "Non-direct C stub marked direct."); - qpoints->pReadBarrierMarkReg05 = nullptr; - static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg05), - "Non-direct C stub marked direct."); - qpoints->pReadBarrierMarkReg06 = nullptr; - static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg06), - "Non-direct C stub marked direct."); - qpoints->pReadBarrierMarkReg07 = nullptr; - static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg07), - "Non-direct C stub marked direct."); - qpoints->pReadBarrierMarkReg08 = nullptr; - static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg08), - "Non-direct C stub marked direct."); - qpoints->pReadBarrierMarkReg09 = nullptr; - static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg09), - "Non-direct C stub marked direct."); - qpoints->pReadBarrierMarkReg10 = nullptr; - static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg10), - "Non-direct C stub marked direct."); - qpoints->pReadBarrierMarkReg11 = nullptr; - static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg11), - "Non-direct C stub marked direct."); - qpoints->pReadBarrierMarkReg12 = nullptr; - static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg12), - "Non-direct C stub marked direct."); - qpoints->pReadBarrierMarkReg13 = nullptr; - static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg13), - "Non-direct C stub marked direct."); - qpoints->pReadBarrierMarkReg14 = nullptr; - static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg14), - "Non-direct C stub marked direct."); qpoints->pReadBarrierMarkReg15 = nullptr; static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg15), "Non-direct C stub marked direct."); qpoints->pReadBarrierMarkReg16 = nullptr; static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg16), "Non-direct C stub marked direct."); - qpoints->pReadBarrierMarkReg17 = nullptr; - static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg17), - "Non-direct C stub marked direct."); - qpoints->pReadBarrierMarkReg18 = nullptr; - static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg18), - "Non-direct C stub marked direct."); - qpoints->pReadBarrierMarkReg19 = nullptr; - static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg19), - "Non-direct C stub marked direct."); - qpoints->pReadBarrierMarkReg20 = nullptr; - static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg20), - "Non-direct C stub marked direct."); - qpoints->pReadBarrierMarkReg21 = nullptr; - static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg21), - "Non-direct C stub marked direct."); - qpoints->pReadBarrierMarkReg22 = nullptr; - static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg22), - "Non-direct C stub marked direct."); qpoints->pReadBarrierMarkReg23 = nullptr; static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg23), "Non-direct C stub marked direct."); @@ -376,9 +407,6 @@ void InitEntryPoints(JniEntryPoints* jpoints, QuickEntryPoints* qpoints) { qpoints->pReadBarrierMarkReg28 = nullptr; static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg28), "Non-direct C stub marked direct."); - qpoints->pReadBarrierMarkReg29 = nullptr; - static_assert(!IsDirectEntrypoint(kQuickReadBarrierMarkReg29), - "Non-direct C stub marked direct."); qpoints->pReadBarrierSlow = artReadBarrierSlow; static_assert(IsDirectEntrypoint(kQuickReadBarrierSlow), "Direct C stub not marked direct."); qpoints->pReadBarrierForRootSlow = artReadBarrierForRootSlow; diff --git a/runtime/arch/mips/quick_entrypoints_mips.S b/runtime/arch/mips/quick_entrypoints_mips.S index 5d6153949a..808536b722 100644 --- a/runtime/arch/mips/quick_entrypoints_mips.S +++ b/runtime/arch/mips/quick_entrypoints_mips.S @@ -2198,6 +2198,151 @@ ENTRY_NO_GP art_quick_string_compareto subu $v0, $t0, $t1 # return (this.charAt(i) - anotherString.charAt(i)) END art_quick_string_compareto + /* + * Create a function `name` calling the ReadBarrier::Mark routine, + * getting its argument and returning its result through register + * `reg`, saving and restoring all caller-save registers. + */ +.macro READ_BARRIER_MARK_REG name, reg +ENTRY \name + /* TODO: optimizations: mark bit, forwarding. */ + addiu $sp, $sp, -160 # includes 16 bytes of space for argument registers a0-a3 + .cfi_adjust_cfa_offset 160 + + sw $ra, 156($sp) + .cfi_rel_offset 31, 156 + sw $t8, 152($sp) + .cfi_rel_offset 24, 152 + sw $t7, 148($sp) + .cfi_rel_offset 15, 148 + sw $t6, 144($sp) + .cfi_rel_offset 14, 144 + sw $t5, 140($sp) + .cfi_rel_offset 13, 140 + sw $t4, 136($sp) + .cfi_rel_offset 12, 136 + sw $t3, 132($sp) + .cfi_rel_offset 11, 132 + sw $t2, 128($sp) + .cfi_rel_offset 10, 128 + sw $t1, 124($sp) + .cfi_rel_offset 9, 124 + sw $t0, 120($sp) + .cfi_rel_offset 8, 120 + sw $a3, 116($sp) + .cfi_rel_offset 7, 116 + sw $a2, 112($sp) + .cfi_rel_offset 6, 112 + sw $a1, 108($sp) + .cfi_rel_offset 5, 108 + sw $a0, 104($sp) + .cfi_rel_offset 4, 104 + sw $v1, 100($sp) + .cfi_rel_offset 3, 100 + sw $v0, 96($sp) + .cfi_rel_offset 2, 96 + + la $t9, artReadBarrierMark + + sdc1 $f18, 88($sp) + sdc1 $f16, 80($sp) + sdc1 $f14, 72($sp) + sdc1 $f12, 64($sp) + sdc1 $f10, 56($sp) + sdc1 $f8, 48($sp) + sdc1 $f6, 40($sp) + sdc1 $f4, 32($sp) + sdc1 $f2, 24($sp) + + .ifnc \reg, $a0 + move $a0, \reg # pass obj from `reg` in a0 + .endif + jalr $t9 # v0 <- artReadBarrierMark(obj) + sdc1 $f0, 16($sp) # in delay slot + + lw $ra, 156($sp) + .cfi_restore 31 + lw $t8, 152($sp) + .cfi_restore 24 + lw $t7, 148($sp) + .cfi_restore 15 + lw $t6, 144($sp) + .cfi_restore 14 + lw $t5, 140($sp) + .cfi_restore 13 + lw $t4, 136($sp) + .cfi_restore 12 + lw $t3, 132($sp) + .cfi_restore 11 + lw $t2, 128($sp) + .cfi_restore 10 + lw $t1, 124($sp) + .cfi_restore 9 + lw $t0, 120($sp) + .cfi_restore 8 + lw $a3, 116($sp) + .cfi_restore 7 + lw $a2, 112($sp) + .cfi_restore 6 + lw $a1, 108($sp) + .cfi_restore 5 + lw $a0, 104($sp) + .cfi_restore 4 + lw $v1, 100($sp) + .cfi_restore 3 + + .ifnc \reg, $v0 + move \reg, $v0 # `reg` <- v0 + lw $v0, 96($sp) + .cfi_restore 2 + .endif + + ldc1 $f18, 88($sp) + ldc1 $f16, 80($sp) + ldc1 $f14, 72($sp) + ldc1 $f12, 64($sp) + ldc1 $f10, 56($sp) + ldc1 $f8, 48($sp) + ldc1 $f6, 40($sp) + ldc1 $f4, 32($sp) + ldc1 $f2, 24($sp) + ldc1 $f0, 16($sp) + + jalr $zero, $ra + addiu $sp, $sp, 160 + .cfi_adjust_cfa_offset -160 +END \name +.endm + +// Note that art_quick_read_barrier_mark_regXX corresponds to register XX+1. +// ZERO (register 0) is reserved. +// AT (register 1) is reserved as a temporary/scratch register. +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg01, $v0 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg02, $v1 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg03, $a0 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg04, $a1 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg05, $a2 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg06, $a3 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg07, $t0 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg08, $t1 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg09, $t2 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg10, $t3 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg11, $t4 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg12, $t5 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg13, $t6 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg14, $t7 +// S0 and S1 (registers 16 and 17) are reserved as suspended and thread registers. +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg17, $s2 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg18, $s3 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg19, $s4 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg20, $s5 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg21, $s6 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg22, $s7 +// T8 and T9 (registers 24 and 25) are reserved as temporary/scratch registers. +// K0, K1, GP, SP (registers 26 - 29) are reserved. +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg29, $s8 +// RA (register 31) is reserved. + .extern artInvokePolymorphic ENTRY art_quick_invoke_polymorphic SETUP_SAVE_REFS_AND_ARGS_FRAME diff --git a/runtime/arch/mips64/entrypoints_init_mips64.cc b/runtime/arch/mips64/entrypoints_init_mips64.cc index bc17d47366..66405cbe50 100644 --- a/runtime/arch/mips64/entrypoints_init_mips64.cc +++ b/runtime/arch/mips64/entrypoints_init_mips64.cc @@ -32,6 +32,32 @@ namespace art { // Cast entrypoints. extern "C" size_t artInstanceOfFromCode(mirror::Object* obj, mirror::Class* ref_class); +// Read barrier entrypoints. +// art_quick_read_barrier_mark_regXX uses a non-standard calling +// convention: it expects its input in register XX+1 and returns its +// result in that same register, and saves and restores all +// caller-save registers. +extern "C" mirror::Object* art_quick_read_barrier_mark_reg01(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg02(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg03(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg04(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg05(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg06(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg07(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg08(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg09(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg10(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg11(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg12(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg13(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg17(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg18(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg19(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg20(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg21(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg22(mirror::Object*); +extern "C" mirror::Object* art_quick_read_barrier_mark_reg29(mirror::Object*); + // Math entrypoints. extern int32_t CmpgDouble(double a, double b); extern int32_t CmplDouble(double a, double b); @@ -60,8 +86,28 @@ extern "C" int64_t __divdi3(int64_t, int64_t); extern "C" int64_t __moddi3(int64_t, int64_t); // No read barrier entrypoints for marking registers. -void UpdateReadBarrierEntrypoints(QuickEntryPoints* qpoints ATTRIBUTE_UNUSED, - bool is_marking ATTRIBUTE_UNUSED) {} +void UpdateReadBarrierEntrypoints(QuickEntryPoints* qpoints, bool is_marking) { + qpoints->pReadBarrierMarkReg01 = is_marking ? art_quick_read_barrier_mark_reg01 : nullptr; + qpoints->pReadBarrierMarkReg02 = is_marking ? art_quick_read_barrier_mark_reg02 : nullptr; + qpoints->pReadBarrierMarkReg03 = is_marking ? art_quick_read_barrier_mark_reg03 : nullptr; + qpoints->pReadBarrierMarkReg04 = is_marking ? art_quick_read_barrier_mark_reg04 : nullptr; + qpoints->pReadBarrierMarkReg05 = is_marking ? art_quick_read_barrier_mark_reg05 : nullptr; + qpoints->pReadBarrierMarkReg06 = is_marking ? art_quick_read_barrier_mark_reg06 : nullptr; + qpoints->pReadBarrierMarkReg07 = is_marking ? art_quick_read_barrier_mark_reg07 : nullptr; + qpoints->pReadBarrierMarkReg08 = is_marking ? art_quick_read_barrier_mark_reg08 : nullptr; + qpoints->pReadBarrierMarkReg09 = is_marking ? art_quick_read_barrier_mark_reg09 : nullptr; + qpoints->pReadBarrierMarkReg10 = is_marking ? art_quick_read_barrier_mark_reg10 : nullptr; + qpoints->pReadBarrierMarkReg11 = is_marking ? art_quick_read_barrier_mark_reg11 : nullptr; + qpoints->pReadBarrierMarkReg12 = is_marking ? art_quick_read_barrier_mark_reg12 : nullptr; + qpoints->pReadBarrierMarkReg13 = is_marking ? art_quick_read_barrier_mark_reg13 : nullptr; + qpoints->pReadBarrierMarkReg17 = is_marking ? art_quick_read_barrier_mark_reg17 : nullptr; + qpoints->pReadBarrierMarkReg18 = is_marking ? art_quick_read_barrier_mark_reg18 : nullptr; + qpoints->pReadBarrierMarkReg19 = is_marking ? art_quick_read_barrier_mark_reg19 : nullptr; + qpoints->pReadBarrierMarkReg20 = is_marking ? art_quick_read_barrier_mark_reg20 : nullptr; + qpoints->pReadBarrierMarkReg21 = is_marking ? art_quick_read_barrier_mark_reg21 : nullptr; + qpoints->pReadBarrierMarkReg22 = is_marking ? art_quick_read_barrier_mark_reg22 : nullptr; + qpoints->pReadBarrierMarkReg29 = is_marking ? art_quick_read_barrier_mark_reg29 : nullptr; +} void InitEntryPoints(JniEntryPoints* jpoints, QuickEntryPoints* qpoints) { DefaultInitEntryPoints(jpoints, qpoints); @@ -103,38 +149,20 @@ void InitEntryPoints(JniEntryPoints* jpoints, QuickEntryPoints* qpoints) { // Read barrier. qpoints->pReadBarrierJni = ReadBarrierJni; - // Read barriers (and these entry points in particular) are not - // supported in the compiler on MIPS64. + UpdateReadBarrierEntrypoints(qpoints, /*is_marking*/ false); + // Cannot use the following registers to pass arguments: + // 0(ZERO), 1(AT), 15(T3), 16(S0), 17(S1), 24(T8), 25(T9), 26(K0), 27(K1), 28(GP), 29(SP), 31(RA). + // Note that there are 30 entry points only: 00 for register 1(AT), ..., 29 for register 30(S8). qpoints->pReadBarrierMarkReg00 = nullptr; - qpoints->pReadBarrierMarkReg01 = nullptr; - qpoints->pReadBarrierMarkReg02 = nullptr; - qpoints->pReadBarrierMarkReg03 = nullptr; - qpoints->pReadBarrierMarkReg04 = nullptr; - qpoints->pReadBarrierMarkReg05 = nullptr; - qpoints->pReadBarrierMarkReg06 = nullptr; - qpoints->pReadBarrierMarkReg07 = nullptr; - qpoints->pReadBarrierMarkReg08 = nullptr; - qpoints->pReadBarrierMarkReg09 = nullptr; - qpoints->pReadBarrierMarkReg10 = nullptr; - qpoints->pReadBarrierMarkReg11 = nullptr; - qpoints->pReadBarrierMarkReg12 = nullptr; - qpoints->pReadBarrierMarkReg13 = nullptr; qpoints->pReadBarrierMarkReg14 = nullptr; qpoints->pReadBarrierMarkReg15 = nullptr; qpoints->pReadBarrierMarkReg16 = nullptr; - qpoints->pReadBarrierMarkReg17 = nullptr; - qpoints->pReadBarrierMarkReg18 = nullptr; - qpoints->pReadBarrierMarkReg19 = nullptr; - qpoints->pReadBarrierMarkReg20 = nullptr; - qpoints->pReadBarrierMarkReg21 = nullptr; - qpoints->pReadBarrierMarkReg22 = nullptr; qpoints->pReadBarrierMarkReg23 = nullptr; qpoints->pReadBarrierMarkReg24 = nullptr; qpoints->pReadBarrierMarkReg25 = nullptr; qpoints->pReadBarrierMarkReg26 = nullptr; qpoints->pReadBarrierMarkReg27 = nullptr; qpoints->pReadBarrierMarkReg28 = nullptr; - qpoints->pReadBarrierMarkReg29 = nullptr; qpoints->pReadBarrierSlow = artReadBarrierSlow; qpoints->pReadBarrierForRootSlow = artReadBarrierForRootSlow; }; diff --git a/runtime/arch/mips64/quick_entrypoints_mips64.S b/runtime/arch/mips64/quick_entrypoints_mips64.S index 3ee9c4a9c8..9c92805adf 100644 --- a/runtime/arch/mips64/quick_entrypoints_mips64.S +++ b/runtime/arch/mips64/quick_entrypoints_mips64.S @@ -2053,6 +2053,180 @@ ENTRY_NO_GP art_quick_indexof #endif END art_quick_indexof + /* + * Create a function `name` calling the ReadBarrier::Mark routine, + * getting its argument and returning its result through register + * `reg`, saving and restoring all caller-save registers. + */ +.macro READ_BARRIER_MARK_REG name, reg +ENTRY \name + /* TODO: optimizations: mark bit, forwarding. */ + daddiu $sp, $sp, -320 + .cfi_adjust_cfa_offset 320 + + sd $ra, 312($sp) + .cfi_rel_offset 31, 312 + sd $t8, 304($sp) # save t8 holding caller's gp + .cfi_rel_offset 24, 304 + sd $t3, 296($sp) + .cfi_rel_offset 15, 296 + sd $t2, 288($sp) + .cfi_rel_offset 14, 288 + sd $t1, 280($sp) + .cfi_rel_offset 13, 280 + sd $t0, 272($sp) + .cfi_rel_offset 12, 272 + sd $a7, 264($sp) + .cfi_rel_offset 11, 264 + sd $a6, 256($sp) + .cfi_rel_offset 10, 256 + sd $a5, 248($sp) + .cfi_rel_offset 9, 248 + sd $a4, 240($sp) + .cfi_rel_offset 8, 240 + sd $a3, 232($sp) + .cfi_rel_offset 7, 232 + sd $a2, 224($sp) + .cfi_rel_offset 6, 224 + sd $a1, 216($sp) + .cfi_rel_offset 5, 216 + sd $a0, 208($sp) + .cfi_rel_offset 4, 208 + sd $v1, 200($sp) + .cfi_rel_offset 3, 200 + sd $v0, 192($sp) + .cfi_rel_offset 2, 192 + + dla $t9, artReadBarrierMark + + sdc1 $f23, 184($sp) + sdc1 $f22, 176($sp) + sdc1 $f21, 168($sp) + sdc1 $f20, 160($sp) + sdc1 $f19, 152($sp) + sdc1 $f18, 144($sp) + sdc1 $f17, 136($sp) + sdc1 $f16, 128($sp) + sdc1 $f15, 120($sp) + sdc1 $f14, 112($sp) + sdc1 $f13, 104($sp) + sdc1 $f12, 96($sp) + sdc1 $f11, 88($sp) + sdc1 $f10, 80($sp) + sdc1 $f9, 72($sp) + sdc1 $f8, 64($sp) + sdc1 $f7, 56($sp) + sdc1 $f6, 48($sp) + sdc1 $f5, 40($sp) + sdc1 $f4, 32($sp) + sdc1 $f3, 24($sp) + sdc1 $f2, 16($sp) + sdc1 $f1, 8($sp) + + .ifnc \reg, $a0 + move $a0, \reg # pass obj from `reg` in a0 + .endif + jalr $t9 # v0 <- artReadBarrierMark(obj) + sdc1 $f0, 0($sp) # in delay slot + + ld $ra, 312($sp) + .cfi_restore 31 + ld $t8, 304($sp) # restore t8 holding caller's gp + .cfi_restore 24 + ld $t3, 296($sp) + .cfi_restore 15 + ld $t2, 288($sp) + .cfi_restore 14 + ld $t1, 280($sp) + .cfi_restore 13 + ld $t0, 272($sp) + .cfi_restore 12 + ld $a7, 264($sp) + .cfi_restore 11 + ld $a6, 256($sp) + .cfi_restore 10 + ld $a5, 248($sp) + .cfi_restore 9 + ld $a4, 240($sp) + .cfi_restore 8 + ld $a3, 232($sp) + .cfi_restore 7 + ld $a2, 224($sp) + .cfi_restore 6 + ld $a1, 216($sp) + .cfi_restore 5 + ld $a0, 208($sp) + .cfi_restore 4 + ld $v1, 200($sp) + .cfi_restore 3 + + .ifnc \reg, $v0 + move \reg, $v0 # `reg` <- v0 + ld $v0, 192($sp) + .cfi_restore 2 + .endif + + ldc1 $f23, 184($sp) + ldc1 $f22, 176($sp) + ldc1 $f21, 168($sp) + ldc1 $f20, 160($sp) + ldc1 $f19, 152($sp) + ldc1 $f18, 144($sp) + ldc1 $f17, 136($sp) + ldc1 $f16, 128($sp) + ldc1 $f15, 120($sp) + ldc1 $f14, 112($sp) + ldc1 $f13, 104($sp) + ldc1 $f12, 96($sp) + ldc1 $f11, 88($sp) + ldc1 $f10, 80($sp) + ldc1 $f9, 72($sp) + ldc1 $f8, 64($sp) + ldc1 $f7, 56($sp) + ldc1 $f6, 48($sp) + ldc1 $f5, 40($sp) + ldc1 $f4, 32($sp) + ldc1 $f3, 24($sp) + ldc1 $f2, 16($sp) + ldc1 $f1, 8($sp) + ldc1 $f0, 0($sp) + + .cpreturn # restore caller's gp from t8 + jalr $zero, $ra + daddiu $sp, $sp, 320 + .cfi_adjust_cfa_offset -320 +END \name +.endm + +// Note that art_quick_read_barrier_mark_regXX corresponds to register XX+1. +// ZERO (register 0) is reserved. +// AT (register 1) is reserved as a temporary/scratch register. +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg01, $v0 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg02, $v1 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg03, $a0 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg04, $a1 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg05, $a2 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg06, $a3 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg07, $a4 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg08, $a5 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg09, $a6 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg10, $a7 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg11, $t0 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg12, $t1 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg13, $t2 +// T3 (register 15) is reserved as a temporary/scratch register. +// S0 and S1 (registers 16 and 17) are reserved as suspended and thread registers. +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg17, $s2 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg18, $s3 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg19, $s4 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg20, $s5 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg21, $s6 +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg22, $s7 +// T8 and T9 (registers 24 and 25) are reserved as temporary/scratch registers. +// K0, K1, GP, SP (registers 26 - 29) are reserved. +READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg29, $s8 +// RA (register 31) is reserved. + .extern artInvokePolymorphic ENTRY art_quick_invoke_polymorphic SETUP_SAVE_REFS_AND_ARGS_FRAME diff --git a/runtime/art_field.h b/runtime/art_field.h index 75dd981136..666ed8a868 100644 --- a/runtime/art_field.h +++ b/runtime/art_field.h @@ -47,6 +47,10 @@ class ArtField FINAL { void SetDeclaringClass(ObjPtr<mirror::Class> new_declaring_class) REQUIRES_SHARED(Locks::mutator_lock_); + mirror::CompressedReference<mirror::Object>* GetDeclaringClassAddressWithoutBarrier() { + return declaring_class_.AddressWithoutBarrier(); + } + uint32_t GetAccessFlags() REQUIRES_SHARED(Locks::mutator_lock_); void SetAccessFlags(uint32_t new_access_flags) REQUIRES_SHARED(Locks::mutator_lock_) { diff --git a/runtime/art_method-inl.h b/runtime/art_method-inl.h index b47f8f0fc2..5cf0e0f90c 100644 --- a/runtime/art_method-inl.h +++ b/runtime/art_method-inl.h @@ -32,6 +32,7 @@ #include "mirror/dex_cache-inl.h" #include "mirror/object-inl.h" #include "mirror/object_array.h" +#include "mirror/string.h" #include "oat.h" #include "obj_ptr-inl.h" #include "quick/quick_method_frame_info.h" @@ -56,8 +57,10 @@ inline mirror::Class* ArtMethod::GetDeclaringClass() { if (!IsRuntimeMethod()) { CHECK(result != nullptr) << this; if (kCheckDeclaringClassState) { - CHECK(result->IsIdxLoaded() || result->IsErroneous()) - << result->GetStatus() << " " << result->PrettyClass(); + if (!(result->IsIdxLoaded() || result->IsErroneous())) { + LOG(FATAL_WITHOUT_ABORT) << "Class status: " << result->GetStatus(); + LOG(FATAL) << result->PrettyClass(); + } } } else { CHECK(result == nullptr) << this; diff --git a/runtime/art_method.h b/runtime/art_method.h index 8f09cc6d03..51b65760a1 100644 --- a/runtime/art_method.h +++ b/runtime/art_method.h @@ -73,6 +73,10 @@ class ArtMethod FINAL { ALWAYS_INLINE mirror::Class* GetDeclaringClassUnchecked() REQUIRES_SHARED(Locks::mutator_lock_); + mirror::CompressedReference<mirror::Object>* GetDeclaringClassAddressWithoutBarrier() { + return declaring_class_.AddressWithoutBarrier(); + } + void SetDeclaringClass(ObjPtr<mirror::Class> new_declaring_class) REQUIRES_SHARED(Locks::mutator_lock_); diff --git a/runtime/base/arena_allocator.cc b/runtime/base/arena_allocator.cc index 5aede38334..e763e439b9 100644 --- a/runtime/base/arena_allocator.cc +++ b/runtime/base/arena_allocator.cc @@ -166,7 +166,7 @@ void ArenaAllocatorMemoryTool::DoMakeInaccessible(void* ptr, size_t size) { MEMORY_TOOL_MAKE_NOACCESS(ptr, size); } -Arena::Arena() : bytes_allocated_(0), next_(nullptr) { +Arena::Arena() : bytes_allocated_(0), memory_(nullptr), size_(0), next_(nullptr) { } class MallocArena FINAL : public Arena { diff --git a/runtime/base/histogram-inl.h b/runtime/base/histogram-inl.h index ca9a694144..b28eb729d8 100644 --- a/runtime/base/histogram-inl.h +++ b/runtime/base/histogram-inl.h @@ -48,7 +48,8 @@ template <class Value> inline Histogram<Value>::Histogram(const char* name) : kAdjust(0), kInitialBucketCount(0), name_(name), - max_buckets_(0) { + max_buckets_(0), + sample_size_(0) { } template <class Value> diff --git a/runtime/base/mutex.h b/runtime/base/mutex.h index 038aeb33e7..2414b5f937 100644 --- a/runtime/base/mutex.h +++ b/runtime/base/mutex.h @@ -62,10 +62,11 @@ enum LockLevel { kJdwpAdbStateLock, kJdwpSocketLock, kRegionSpaceRegionLock, + kMarkSweepMarkStackLock, kRosAllocGlobalLock, kRosAllocBracketLock, kRosAllocBulkFreeLock, - kMarkSweepMarkStackLock, + kTaggingLockLevel, kTransactionLogLock, kJniFunctionTableLock, kJniWeakGlobalsLock, diff --git a/runtime/class_linker.cc b/runtime/class_linker.cc index 8162a820e0..3c18704440 100644 --- a/runtime/class_linker.cc +++ b/runtime/class_linker.cc @@ -3505,13 +3505,12 @@ ObjPtr<mirror::DexCache> ClassLinker::FindDexCache(Thread* self, const DexFile& return dex_cache; } // Failure, dump diagnostic and abort. - std::string location(dex_file.GetLocation()); for (const DexCacheData& data : dex_caches_) { if (DecodeDexCache(self, data) != nullptr) { - LOG(ERROR) << "Registered dex file " << data.dex_file->GetLocation(); + LOG(FATAL_WITHOUT_ABORT) << "Registered dex file " << data.dex_file->GetLocation(); } } - LOG(FATAL) << "Failed to find DexCache for DexFile " << location; + LOG(FATAL) << "Failed to find DexCache for DexFile " << dex_file.GetLocation(); UNREACHABLE(); } @@ -6724,10 +6723,11 @@ static void CheckClassOwnsVTableEntries(Thread* self, ArtMethod* m = check_vtable->GetElementPtrSize<ArtMethod*>(i, pointer_size); CHECK(m != nullptr); - CHECK_EQ(m->GetMethodIndexDuringLinking(), i) - << m->PrettyMethod() - << " has an unexpected method index for its spot in the vtable for class" - << klass->PrettyClass(); + if (m->GetMethodIndexDuringLinking() != i) { + LOG(WARNING) << m->PrettyMethod() + << " has an unexpected method index for its spot in the vtable for class" + << klass->PrettyClass(); + } ArraySlice<ArtMethod> virtuals = klass->GetVirtualMethodsSliceUnchecked(pointer_size); auto is_same_method = [m] (const ArtMethod& meth) { return &meth == m; diff --git a/runtime/class_table.h b/runtime/class_table.h index 79f5aea399..430edbba4e 100644 --- a/runtime/class_table.h +++ b/runtime/class_table.h @@ -256,6 +256,7 @@ class ClassTable { } private: + // Only copies classes. void CopyWithoutLocks(const ClassTable& source_table) NO_THREAD_SAFETY_ANALYSIS; void InsertWithoutLocks(ObjPtr<mirror::Class> klass) NO_THREAD_SAFETY_ANALYSIS; diff --git a/runtime/dex_file.cc b/runtime/dex_file.cc index 35e9d5db29..85100ae49d 100644 --- a/runtime/dex_file.cc +++ b/runtime/dex_file.cc @@ -23,6 +23,7 @@ #include <string.h> #include <sys/file.h> #include <sys/stat.h> +#include <zlib.h> #include <memory> #include <sstream> @@ -67,6 +68,12 @@ const uint8_t DexFile::kDexMagicVersions[DexFile::kNumDexVersions][DexFile::kDex {'0', '3', '8', '\0'} }; +uint32_t DexFile::CalculateChecksum() const { + const uint32_t non_sum = OFFSETOF_MEMBER(DexFile::Header, signature_); + const uint8_t* non_sum_ptr = Begin() + non_sum; + return adler32(adler32(0L, Z_NULL, 0), non_sum_ptr, Size() - non_sum); +} + struct DexFile::AnnotationValue { JValue value_; uint8_t type_; diff --git a/runtime/dex_file.h b/runtime/dex_file.h index 58b8e792ee..1b18d21cb1 100644 --- a/runtime/dex_file.h +++ b/runtime/dex_file.h @@ -1088,6 +1088,9 @@ class DexFile { static int64_t ReadSignedLong(const uint8_t* ptr, int zwidth); static uint64_t ReadUnsignedLong(const uint8_t* ptr, int zwidth, bool fill_on_right); + // Recalculates the checksum of the dex file. Does not use the current value in the header. + uint32_t CalculateChecksum() const; + // Returns a human-readable form of the method at an index. std::string PrettyMethod(uint32_t method_idx, bool with_signature = true) const; // Returns a human-readable form of the field at an index. @@ -1320,6 +1323,9 @@ class ClassDataItemIterator { uint32_t NumVirtualMethods() const { return header_.virtual_methods_size_; } + bool IsAtMethod() const { + return pos_ >= EndOfInstanceFieldsPos(); + } bool HasNextStaticField() const { return pos_ < EndOfStaticFieldsPos(); } diff --git a/runtime/dex_file_verifier.cc b/runtime/dex_file_verifier.cc index 0b3f16a3cb..11b3cd025a 100644 --- a/runtime/dex_file_verifier.cc +++ b/runtime/dex_file_verifier.cc @@ -17,7 +17,6 @@ #include "dex_file_verifier.h" #include <inttypes.h> -#include <zlib.h> #include <limits> #include <memory> @@ -368,11 +367,8 @@ bool DexFileVerifier::CheckHeader() { return false; } + uint32_t adler_checksum = dex_file_->CalculateChecksum(); // Compute and verify the checksum in the header. - uint32_t adler_checksum = adler32(0L, Z_NULL, 0); - const uint32_t non_sum = sizeof(header_->magic_) + sizeof(header_->checksum_); - const uint8_t* non_sum_ptr = reinterpret_cast<const uint8_t*>(header_) + non_sum; - adler_checksum = adler32(adler_checksum, non_sum_ptr, expected_size - non_sum); if (adler_checksum != header_->checksum_) { if (verify_checksum_) { ErrorStringPrintf("Bad checksum (%08x, expected %08x)", adler_checksum, header_->checksum_); diff --git a/runtime/fault_handler.cc b/runtime/fault_handler.cc index 64128cc61e..4220250c38 100644 --- a/runtime/fault_handler.cc +++ b/runtime/fault_handler.cc @@ -38,8 +38,8 @@ extern "C" __attribute__((visibility("default"))) void art_sigsegv_fault() { } // Signal handler called on SIGSEGV. -static void art_fault_handler(int sig, siginfo_t* info, void* context) { - fault_manager.HandleFault(sig, info, context); +static bool art_fault_handler(int sig, siginfo_t* info, void* context) { + return fault_manager.HandleFault(sig, info, context); } FaultManager::FaultManager() : initialized_(false) { @@ -49,43 +49,15 @@ FaultManager::FaultManager() : initialized_(false) { FaultManager::~FaultManager() { } -static void SetUpArtAction(struct sigaction* action) { - action->sa_sigaction = art_fault_handler; - sigemptyset(&action->sa_mask); - action->sa_flags = SA_SIGINFO | SA_ONSTACK; -#if !defined(__APPLE__) && !defined(__mips__) - action->sa_restorer = nullptr; -#endif -} - -void FaultManager::EnsureArtActionInFrontOfSignalChain() { - if (initialized_) { - struct sigaction action; - SetUpArtAction(&action); - EnsureFrontOfChain(SIGSEGV, &action); - } else { - LOG(WARNING) << "Can't call " << __FUNCTION__ << " due to unitialized fault manager"; - } -} - void FaultManager::Init() { CHECK(!initialized_); - struct sigaction action; - SetUpArtAction(&action); - - // Set our signal handler now. - int e = sigaction(SIGSEGV, &action, &oldaction_); - if (e != 0) { - VLOG(signals) << "Failed to claim SEGV: " << strerror(errno); - } - // Make sure our signal handler is called before any user handlers. - ClaimSignalChain(SIGSEGV, &oldaction_); + AddSpecialSignalHandlerFn(SIGSEGV, art_fault_handler); initialized_ = true; } void FaultManager::Release() { if (initialized_) { - UnclaimSignalChain(SIGSEGV); + RemoveSpecialSignalHandlerFn(SIGSEGV, art_fault_handler); initialized_ = false; } } @@ -118,93 +90,36 @@ bool FaultManager::HandleFaultByOtherHandlers(int sig, siginfo_t* info, void* co return false; } -class ScopedSignalUnblocker { - public: - explicit ScopedSignalUnblocker(const std::initializer_list<int>& signals) { - sigset_t new_mask; - sigemptyset(&new_mask); - for (int signal : signals) { - sigaddset(&new_mask, signal); - } - if (sigprocmask(SIG_UNBLOCK, &new_mask, &previous_mask_) != 0) { - PLOG(FATAL) << "failed to unblock signals"; - } - } - - ~ScopedSignalUnblocker() { - if (sigprocmask(SIG_SETMASK, &previous_mask_, nullptr) != 0) { - PLOG(FATAL) << "failed to unblock signals"; - } - } - - private: - sigset_t previous_mask_; -}; - -class ScopedHandlingSignalSetter { - public: - explicit ScopedHandlingSignalSetter(Thread* thread) : thread_(thread) { - CHECK(!thread->HandlingSignal()); - thread_->SetHandlingSignal(true); - } - - ~ScopedHandlingSignalSetter() { - CHECK(thread_->HandlingSignal()); - thread_->SetHandlingSignal(false); - } - - private: - Thread* thread_; -}; - -void FaultManager::HandleFault(int sig, siginfo_t* info, void* context) { - // BE CAREFUL ALLOCATING HERE INCLUDING USING LOG(...) - // - // If malloc calls abort, it will be holding its lock. - // If the handler tries to call malloc, it will deadlock. - - // Use a thread local field to track whether we're recursing, and fall back. - // (e.g.. if one of our handlers crashed) - Thread* thread = Thread::Current(); - - if (thread != nullptr && !thread->HandlingSignal()) { - // Unblock some signals and set thread->handling_signal_ to true, - // so that we can catch crashes in our signal handler. - ScopedHandlingSignalSetter setter(thread); - ScopedSignalUnblocker unblocker { SIGABRT, SIGBUS, SIGSEGV }; // NOLINT - - VLOG(signals) << "Handling fault"; +bool FaultManager::HandleFault(int sig, siginfo_t* info, void* context) { + VLOG(signals) << "Handling fault"; #ifdef TEST_NESTED_SIGNAL - // Simulate a crash in a handler. - raise(SIGSEGV); + // Simulate a crash in a handler. + raise(SIGSEGV); #endif - if (IsInGeneratedCode(info, context, true)) { - VLOG(signals) << "in generated code, looking for handler"; - for (const auto& handler : generated_code_handlers_) { - VLOG(signals) << "invoking Action on handler " << handler; - if (handler->Action(sig, info, context)) { - // We have handled a signal so it's time to return from the - // signal handler to the appropriate place. - return; - } + if (IsInGeneratedCode(info, context, true)) { + VLOG(signals) << "in generated code, looking for handler"; + for (const auto& handler : generated_code_handlers_) { + VLOG(signals) << "invoking Action on handler " << handler; + if (handler->Action(sig, info, context)) { + // We have handled a signal so it's time to return from the + // signal handler to the appropriate place. + return true; } + } - // We hit a signal we didn't handle. This might be something for which - // we can give more information about so call all registered handlers to - // see if it is. - if (HandleFaultByOtherHandlers(sig, info, context)) { - return; - } + // We hit a signal we didn't handle. This might be something for which + // we can give more information about so call all registered handlers to + // see if it is. + if (HandleFaultByOtherHandlers(sig, info, context)) { + return true; } } // Set a breakpoint in this function to catch unhandled signals. art_sigsegv_fault(); - - // Pass this on to the next handler in the chain, or the default if none. - InvokeUserSignalHandler(sig, info, context); + return false; } void FaultManager::AddHandler(FaultHandler* handler, bool generated_code) { diff --git a/runtime/fault_handler.h b/runtime/fault_handler.h index ce59ba7e64..d56cf17861 100644 --- a/runtime/fault_handler.h +++ b/runtime/fault_handler.h @@ -42,9 +42,9 @@ class FaultManager { // Unclaim signals and delete registered handlers. void Shutdown(); - void EnsureArtActionInFrontOfSignalChain(); - void HandleFault(int sig, siginfo_t* info, void* context); + // Try to handle a fault, returns true if successful. + bool HandleFault(int sig, siginfo_t* info, void* context); // Added handlers are owned by the fault handler and will be freed on Shutdown(). void AddHandler(FaultHandler* handler, bool generated_code); diff --git a/runtime/gc/allocator/rosalloc.h b/runtime/gc/allocator/rosalloc.h index 1fa2d1ac8a..562fc750ed 100644 --- a/runtime/gc/allocator/rosalloc.h +++ b/runtime/gc/allocator/rosalloc.h @@ -141,7 +141,7 @@ class RosAlloc { template<bool kUseTail = true> class SlotFreeList { public: - SlotFreeList() : head_(0U), tail_(0), size_(0) {} + SlotFreeList() : head_(0U), tail_(0), size_(0), padding_(0) {} Slot* Head() const { return reinterpret_cast<Slot*>(head_); } diff --git a/runtime/gc/collector/concurrent_copying.cc b/runtime/gc/collector/concurrent_copying.cc index d2ab41d409..24ba52f0c5 100644 --- a/runtime/gc/collector/concurrent_copying.cc +++ b/runtime/gc/collector/concurrent_copying.cc @@ -72,12 +72,19 @@ ConcurrentCopying::ConcurrentCopying(Heap* heap, rb_mark_bit_stack_full_(false), mark_stack_lock_("concurrent copying mark stack lock", kMarkSweepMarkStackLock), thread_running_gc_(nullptr), - is_marking_(false), is_active_(false), is_asserting_to_space_invariant_(false), + is_marking_(false), + is_active_(false), + is_asserting_to_space_invariant_(false), region_space_bitmap_(nullptr), - heap_mark_bitmap_(nullptr), live_stack_freeze_size_(0), mark_stack_mode_(kMarkStackModeOff), + heap_mark_bitmap_(nullptr), + live_stack_freeze_size_(0), + from_space_num_objects_at_first_pause_(0), + from_space_num_bytes_at_first_pause_(0), + mark_stack_mode_(kMarkStackModeOff), weak_ref_access_enabled_(true), skipped_blocks_lock_("concurrent copying bytes blocks lock", kMarkSweepMarkStackLock), measure_read_barrier_slow_path_(measure_read_barrier_slow_path), + mark_from_read_barrier_measurements_(false), rb_slow_path_ns_(0), rb_slow_path_count_(0), rb_slow_path_count_gc_(0), @@ -87,6 +94,7 @@ ConcurrentCopying::ConcurrentCopying(Heap* heap, rb_slow_path_count_gc_total_(0), rb_table_(heap_->GetReadBarrierTable()), force_evacuate_all_(false), + gc_grays_immune_objects_(false), immune_gray_stack_lock_("concurrent copying immune gray stack lock", kMarkSweepMarkStackLock) { static_assert(space::RegionSpace::kRegionSize == accounting::ReadBarrierTable::kRegionSize, diff --git a/runtime/gc/collector/mark_compact.cc b/runtime/gc/collector/mark_compact.cc index 00393881e9..c61f69dad3 100644 --- a/runtime/gc/collector/mark_compact.cc +++ b/runtime/gc/collector/mark_compact.cc @@ -52,8 +52,12 @@ void MarkCompact::BindBitmaps() { MarkCompact::MarkCompact(Heap* heap, const std::string& name_prefix) : GarbageCollector(heap, name_prefix + (name_prefix.empty() ? "" : " ") + "mark compact"), + mark_stack_(nullptr), space_(nullptr), + mark_bitmap_(nullptr), collector_name_(name_), + bump_pointer_(nullptr), + live_objects_in_space_(0), updating_references_(false) {} void MarkCompact::RunPhases() { diff --git a/runtime/gc/space/image_space.cc b/runtime/gc/space/image_space.cc index 568f8d62a8..662efe2c8d 100644 --- a/runtime/gc/space/image_space.cc +++ b/runtime/gc/space/image_space.cc @@ -886,7 +886,7 @@ class ImageSpaceLoader { explicit FixupObjectAdapter(Args... args) : FixupVisitor(args...) {} template <typename T> - T* operator()(T* obj) const { + T* operator()(T* obj, void** dest_addr ATTRIBUTE_UNUSED = nullptr) const { return ForwardObject(obj); } }; @@ -976,7 +976,8 @@ class ImageSpaceLoader { ForwardObject(obj)); } - void operator()(mirror::Object* obj) const NO_THREAD_SAFETY_ANALYSIS { + void operator()(mirror::Object* obj) const + NO_THREAD_SAFETY_ANALYSIS { if (visited_->Test(obj)) { // Already visited. return; diff --git a/runtime/gc/space/large_object_space.cc b/runtime/gc/space/large_object_space.cc index 4c6b5bfadd..3988073de8 100644 --- a/runtime/gc/space/large_object_space.cc +++ b/runtime/gc/space/large_object_space.cc @@ -16,13 +16,12 @@ #include "large_object_space.h" -#include <valgrind.h> #include <memory> -#include <memcheck/memcheck.h> #include "gc/accounting/heap_bitmap-inl.h" #include "gc/accounting/space_bitmap-inl.h" #include "base/logging.h" +#include "base/memory_tool.h" #include "base/mutex-inl.h" #include "base/stl_util.h" #include "image.h" diff --git a/runtime/imt_conflict_table.h b/runtime/imt_conflict_table.h index fdd10fefc4..35868642e1 100644 --- a/runtime/imt_conflict_table.h +++ b/runtime/imt_conflict_table.h @@ -81,6 +81,14 @@ class ImtConflictTable { return GetMethod(index * kMethodCount + kMethodImplementation, pointer_size); } + void** AddressOfInterfaceMethod(size_t index, PointerSize pointer_size) { + return AddressOfMethod(index * kMethodCount + kMethodInterface, pointer_size); + } + + void** AddressOfImplementationMethod(size_t index, PointerSize pointer_size) { + return AddressOfMethod(index * kMethodCount + kMethodImplementation, pointer_size); + } + // Return true if two conflict tables are the same. bool Equals(ImtConflictTable* other, PointerSize pointer_size) const { size_t num = NumEntries(pointer_size); @@ -169,6 +177,14 @@ class ImtConflictTable { } private: + void** AddressOfMethod(size_t index, PointerSize pointer_size) { + if (pointer_size == PointerSize::k64) { + return reinterpret_cast<void**>(&data64_[index]); + } else { + return reinterpret_cast<void**>(&data32_[index]); + } + } + ArtMethod* GetMethod(size_t index, PointerSize pointer_size) const { if (pointer_size == PointerSize::k64) { return reinterpret_cast<ArtMethod*>(static_cast<uintptr_t>(data64_[index])); diff --git a/runtime/imtable.h b/runtime/imtable.h index b7066bd521..aa0a5043b5 100644 --- a/runtime/imtable.h +++ b/runtime/imtable.h @@ -37,9 +37,13 @@ class ImTable { // (non-marker) interfaces. static constexpr size_t kSize = IMT_SIZE; + uint8_t* AddressOfElement(size_t index, PointerSize pointer_size) { + return reinterpret_cast<uint8_t*>(this) + OffsetOfElement(index, pointer_size); + } + ArtMethod* Get(size_t index, PointerSize pointer_size) { DCHECK_LT(index, kSize); - uint8_t* ptr = reinterpret_cast<uint8_t*>(this) + OffsetOfElement(index, pointer_size); + uint8_t* ptr = AddressOfElement(index, pointer_size); if (pointer_size == PointerSize::k32) { uint32_t value = *reinterpret_cast<uint32_t*>(ptr); return reinterpret_cast<ArtMethod*>(value); @@ -51,7 +55,7 @@ class ImTable { void Set(size_t index, ArtMethod* method, PointerSize pointer_size) { DCHECK_LT(index, kSize); - uint8_t* ptr = reinterpret_cast<uint8_t*>(this) + OffsetOfElement(index, pointer_size); + uint8_t* ptr = AddressOfElement(index, pointer_size); if (pointer_size == PointerSize::k32) { uintptr_t value = reinterpret_cast<uintptr_t>(method); DCHECK_EQ(static_cast<uint32_t>(value), value); // Check that we dont lose any non 0 bits. diff --git a/runtime/interpreter/interpreter.cc b/runtime/interpreter/interpreter.cc index 1b3d339f36..67e949f8c8 100644 --- a/runtime/interpreter/interpreter.cc +++ b/runtime/interpreter/interpreter.cc @@ -232,12 +232,6 @@ enum InterpreterImplKind { kSwitchImplKind, // Switch-based interpreter implementation. kMterpImplKind // Assembly interpreter }; -static std::ostream& operator<<(std::ostream& os, const InterpreterImplKind& rhs) { - os << ((rhs == kSwitchImplKind) - ? "Switch-based interpreter" - : "Asm interpreter"); - return os; -} static constexpr InterpreterImplKind kInterpreterImplKind = kMterpImplKind; diff --git a/runtime/interpreter/interpreter_intrinsics.cc b/runtime/interpreter/interpreter_intrinsics.cc index 0ae7307052..869d43061b 100644 --- a/runtime/interpreter/interpreter_intrinsics.cc +++ b/runtime/interpreter/interpreter_intrinsics.cc @@ -14,9 +14,12 @@ * limitations under the License. */ -#include "interpreter/interpreter_common.h" #include "interpreter/interpreter_intrinsics.h" +#include "compiler/intrinsics_enum.h" +#include "dex_instruction.h" +#include "interpreter/interpreter_common.h" + namespace art { namespace interpreter { diff --git a/runtime/interpreter/interpreter_intrinsics.h b/runtime/interpreter/interpreter_intrinsics.h index ae45679b8a..2a23002d05 100644 --- a/runtime/interpreter/interpreter_intrinsics.h +++ b/runtime/interpreter/interpreter_intrinsics.h @@ -17,10 +17,14 @@ #ifndef ART_RUNTIME_INTERPRETER_INTERPRETER_INTRINSICS_H_ #define ART_RUNTIME_INTERPRETER_INTERPRETER_INTRINSICS_H_ -#include "compiler/intrinsics_enum.h" -#include "dex_instruction.h" +#include "jvalue.h" namespace art { + +class ArtMethod; +class Instruction; +class ShadowFrame; + namespace interpreter { // Invokes to methods identified as intrinics are routed here. If there is diff --git a/runtime/java_vm_ext.cc b/runtime/java_vm_ext.cc index a341cdb89f..b93b8f2a97 100644 --- a/runtime/java_vm_ext.cc +++ b/runtime/java_vm_ext.cc @@ -39,6 +39,7 @@ #include "runtime_options.h" #include "ScopedLocalRef.h" #include "scoped_thread_state_change-inl.h" +#include "sigchain.h" #include "thread-inl.h" #include "thread_list.h" @@ -900,7 +901,8 @@ bool JavaVMExt::LoadNativeLibrary(JNIEnv* env, int version = (*jni_on_load)(this, nullptr); if (runtime_->GetTargetSdkVersion() != 0 && runtime_->GetTargetSdkVersion() <= 21) { - fault_manager.EnsureArtActionInFrontOfSignalChain(); + // Make sure that sigchain owns SIGSEGV. + EnsureFrontOfChain(SIGSEGV); } self->SetClassLoaderOverride(old_class_loader.get()); diff --git a/runtime/jit/jit.h b/runtime/jit/jit.h index 5da1ea1196..4f5bebfbf9 100644 --- a/runtime/jit/jit.h +++ b/runtime/jit/jit.h @@ -279,6 +279,10 @@ class JitOptions { code_cache_initial_capacity_(0), code_cache_max_capacity_(0), compile_threshold_(0), + warmup_threshold_(0), + osr_threshold_(0), + priority_thread_weight_(0), + invoke_transition_weight_(0), dump_info_on_shutdown_(false) {} DISALLOW_COPY_AND_ASSIGN(JitOptions); diff --git a/runtime/jit/profile_compilation_info.cc b/runtime/jit/profile_compilation_info.cc index b23a86313f..24ea27529a 100644 --- a/runtime/jit/profile_compilation_info.cc +++ b/runtime/jit/profile_compilation_info.cc @@ -37,8 +37,9 @@ namespace art { const uint8_t ProfileCompilationInfo::kProfileMagic[] = { 'p', 'r', 'o', '\0' }; -// Last profile version: fix the order of dex files in the profile. -const uint8_t ProfileCompilationInfo::kProfileVersion[] = { '0', '0', '4', '\0' }; +// Last profile version: fix profman merges. Update profile version to force +// regeneration of possibly faulty profiles. +const uint8_t ProfileCompilationInfo::kProfileVersion[] = { '0', '0', '5', '\0' }; static constexpr uint16_t kMaxDexFileKeyLength = PATH_MAX; @@ -57,6 +58,14 @@ static_assert(InlineCache::kIndividualCacheSize < kIsMegamorphicEncoding, static_assert(InlineCache::kIndividualCacheSize < kIsMissingTypesEncoding, "InlineCache::kIndividualCacheSize is larger than expected"); +ProfileCompilationInfo::ProfileCompilationInfo(const ProfileCompilationInfo& pci) { + MergeWith(pci); +} + +ProfileCompilationInfo::~ProfileCompilationInfo() { + ClearProfile(); +} + void ProfileCompilationInfo::DexPcData::AddClass(uint16_t dex_profile_idx, const dex::TypeIndex& type_idx) { if (is_megamorphic || is_missing_types) { @@ -227,28 +236,21 @@ bool ProfileCompilationInfo::Save(int fd) { DCHECK_LE(info_.size(), std::numeric_limits<uint8_t>::max()); AddUintToBuffer(&buffer, static_cast<uint8_t>(info_.size())); - // Make sure we write the dex files in order of their profile index. This + // Dex files must be written in the order of their profile index. This // avoids writing the index in the output file and simplifies the parsing logic. - std::vector<const std::string*> ordered_info_location(info_.size()); - std::vector<const DexFileData*> ordered_info_data(info_.size()); - for (const auto& it : info_) { - ordered_info_location[it.second.profile_index] = &(it.first); - ordered_info_data[it.second.profile_index] = &(it.second); - } - for (size_t i = 0; i < info_.size(); i++) { + for (const DexFileData* dex_data_ptr : info_) { + const DexFileData& dex_data = *dex_data_ptr; if (buffer.size() > kMaxSizeToKeepBeforeWriting) { if (!WriteBuffer(fd, buffer.data(), buffer.size())) { return false; } buffer.clear(); } - const std::string& dex_location = *ordered_info_location[i]; - const DexFileData& dex_data = *ordered_info_data[i]; // Note that we allow dex files without any methods or classes, so that // inline caches can refer valid dex files. - if (dex_location.size() >= kMaxDexFileKeyLength) { + if (dex_data.profile_key.size() >= kMaxDexFileKeyLength) { LOG(WARNING) << "DexFileKey exceeds allocated limit"; return false; } @@ -258,19 +260,19 @@ bool ProfileCompilationInfo::Save(int fd) { uint32_t methods_region_size = GetMethodsRegionSize(dex_data); size_t required_capacity = buffer.size() + kLineHeaderSize + - dex_location.size() + + dex_data.profile_key.size() + sizeof(uint16_t) * dex_data.class_set.size() + methods_region_size; buffer.reserve(required_capacity); - DCHECK_LE(dex_location.size(), std::numeric_limits<uint16_t>::max()); + DCHECK_LE(dex_data.profile_key.size(), std::numeric_limits<uint16_t>::max()); DCHECK_LE(dex_data.class_set.size(), std::numeric_limits<uint16_t>::max()); - AddUintToBuffer(&buffer, static_cast<uint16_t>(dex_location.size())); + AddUintToBuffer(&buffer, static_cast<uint16_t>(dex_data.profile_key.size())); AddUintToBuffer(&buffer, static_cast<uint16_t>(dex_data.class_set.size())); AddUintToBuffer(&buffer, methods_region_size); // uint32_t AddUintToBuffer(&buffer, dex_data.checksum); // uint32_t - AddStringToBuffer(&buffer, dex_location); + AddStringToBuffer(&buffer, dex_data.profile_key); for (const auto& method_it : dex_data.method_map) { AddUintToBuffer(&buffer, method_it.first); @@ -375,23 +377,52 @@ void ProfileCompilationInfo::GroupClassesByDex( } ProfileCompilationInfo::DexFileData* ProfileCompilationInfo::GetOrAddDexFileData( - const std::string& dex_location, + const std::string& profile_key, uint32_t checksum) { - auto info_it = info_.FindOrAdd(dex_location, DexFileData(checksum, info_.size())); - if (info_.size() > std::numeric_limits<uint8_t>::max()) { + const auto& profile_index_it = profile_key_map_.FindOrAdd(profile_key, profile_key_map_.size()); + if (profile_key_map_.size() > std::numeric_limits<uint8_t>::max()) { // Allow only 255 dex files to be profiled. This allows us to save bytes // when encoding. The number is well above what we expect for normal applications. if (kIsDebugBuild) { - LOG(WARNING) << "Exceeded the maximum number of dex files (255). Something went wrong"; + LOG(ERROR) << "Exceeded the maximum number of dex files (255). Something went wrong"; } - info_.erase(dex_location); + profile_key_map_.erase(profile_key); return nullptr; } - if (info_it->second.checksum != checksum) { - LOG(WARNING) << "Checksum mismatch for dex " << dex_location; + + uint8_t profile_index = profile_index_it->second; + if (info_.size() <= profile_index) { + // This is a new addition. Add it to the info_ array. + info_.emplace_back(new DexFileData(profile_key, checksum, profile_index)); + } + DexFileData* result = info_[profile_index]; + // DCHECK that profile info map key is consistent with the one stored in the dex file data. + // This should always be the case since since the cache map is managed by ProfileCompilationInfo. + DCHECK_EQ(profile_key, result->profile_key); + DCHECK_EQ(profile_index, result->profile_index); + + // Check that the checksum matches. + // This may different if for example the dex file was updated and + // we had a record of the old one. + if (result->checksum != checksum) { + LOG(WARNING) << "Checksum mismatch for dex " << profile_key; + return nullptr; + } + return result; +} + +const ProfileCompilationInfo::DexFileData* ProfileCompilationInfo::FindDexData( + const std::string& profile_key) const { + const auto& profile_index_it = profile_key_map_.find(profile_key); + if (profile_index_it == profile_key_map_.end()) { return nullptr; } - return &info_it->second; + + uint8_t profile_index = profile_index_it->second; + const DexFileData* result = info_[profile_index]; + DCHECK_EQ(profile_key, result->profile_key); + DCHECK_EQ(profile_index, result->profile_index); + return result; } bool ProfileCompilationInfo::AddResolvedClasses(const DexCacheResolvedClasses& classes) { @@ -415,9 +446,7 @@ bool ProfileCompilationInfo::AddMethod(const std::string& dex_location, uint32_t dex_checksum, uint16_t method_index, const OfflineProfileMethodInfo& pmi) { - DexFileData* const data = GetOrAddDexFileData( - GetProfileDexFileKey(dex_location), - dex_checksum); + DexFileData* const data = GetOrAddDexFileData(GetProfileDexFileKey(dex_location), dex_checksum); if (data == nullptr) { // checksum mismatch return false; } @@ -498,13 +527,13 @@ bool ProfileCompilationInfo::AddClassIndex(const std::string& dex_location, return true; } -#define READ_UINT(type, buffer, dest, error) \ - do { \ - if (!buffer.ReadUintAndAdvance<type>(&dest)) { \ - *error = "Could not read "#dest; \ - return false; \ - } \ - } \ +#define READ_UINT(type, buffer, dest, error) \ + do { \ + if (!(buffer).ReadUintAndAdvance<type>(&(dest))) { \ + *(error) = "Could not read "#dest; \ + return false; \ + } \ + } \ while (false) bool ProfileCompilationInfo::ReadInlineCache(SafeBuffer& buffer, @@ -753,6 +782,8 @@ ProfileCompilationInfo::ProfileLoadSatus ProfileCompilationInfo::ReadProfileLine return kProfileLoadSuccess; } +// TODO(calin): Fix this API. ProfileCompilationInfo::Load should be static and +// return a unique pointer to a ProfileCompilationInfo upon success. bool ProfileCompilationInfo::Load(int fd) { std::string error; ProfileLoadSatus status = LoadInternal(fd, &error); @@ -770,6 +801,10 @@ ProfileCompilationInfo::ProfileLoadSatus ProfileCompilationInfo::LoadInternal( ScopedTrace trace(__PRETTY_FUNCTION__); DCHECK_GE(fd, 0); + if (!IsEmpty()) { + return kProfileLoadWouldOverwiteData; + } + struct stat stat_buffer; if (fstat(fd, &stat_buffer) != 0) { return kProfileLoadIOError; @@ -820,10 +855,10 @@ bool ProfileCompilationInfo::MergeWith(const ProfileCompilationInfo& other) { // the current profile info. // Note that the number of elements should be very small, so this should not // be a performance issue. - for (const auto& other_it : other.info_) { - auto info_it = info_.find(other_it.first); - if ((info_it != info_.end()) && (info_it->second.checksum != other_it.second.checksum)) { - LOG(WARNING) << "Checksum mismatch for dex " << other_it.first; + for (const DexFileData* other_dex_data : other.info_) { + const DexFileData* dex_data = FindDexData(other_dex_data->profile_key); + if ((dex_data != nullptr) && (dex_data->checksum != other_dex_data->checksum)) { + LOG(WARNING) << "Checksum mismatch for dex " << other_dex_data->profile_key; return false; } } @@ -840,32 +875,28 @@ bool ProfileCompilationInfo::MergeWith(const ProfileCompilationInfo& other) { // First, build a mapping from other_dex_profile_index to this_dex_profile_index. // This will make sure that the ClassReferences will point to the correct dex file. SafeMap<uint8_t, uint8_t> dex_profile_index_remap; - for (const auto& other_it : other.info_) { - const std::string& other_dex_location = other_it.first; - uint32_t other_checksum = other_it.second.checksum; - const DexFileData& other_dex_data = other_it.second; - const DexFileData* dex_data = GetOrAddDexFileData(other_dex_location, other_checksum); + for (const DexFileData* other_dex_data : other.info_) { + const DexFileData* dex_data = GetOrAddDexFileData(other_dex_data->profile_key, + other_dex_data->checksum); if (dex_data == nullptr) { return false; // Could happen if we exceed the number of allowed dex files. } - dex_profile_index_remap.Put(other_dex_data.profile_index, dex_data->profile_index); + dex_profile_index_remap.Put(other_dex_data->profile_index, dex_data->profile_index); } // Merge the actual profile data. - for (const auto& other_it : other.info_) { - const std::string& other_dex_location = other_it.first; - const DexFileData& other_dex_data = other_it.second; - auto info_it = info_.find(other_dex_location); - DCHECK(info_it != info_.end()); + for (const DexFileData* other_dex_data : other.info_) { + DexFileData* dex_data = const_cast<DexFileData*>(FindDexData(other_dex_data->profile_key)); + DCHECK(dex_data != nullptr); // Merge the classes. - info_it->second.class_set.insert(other_dex_data.class_set.begin(), - other_dex_data.class_set.end()); + dex_data->class_set.insert(other_dex_data->class_set.begin(), + other_dex_data->class_set.end()); // Merge the methods and the inline caches. - for (const auto& other_method_it : other_dex_data.method_map) { + for (const auto& other_method_it : other_dex_data->method_map) { uint16_t other_method_index = other_method_it.first; - auto method_it = info_it->second.method_map.FindOrAdd(other_method_index); + auto method_it = dex_data->method_map.FindOrAdd(other_method_index); const auto& other_inline_cache = other_method_it.second; for (const auto& other_ic_it : other_inline_cache) { uint16_t other_dex_pc = other_ic_it.first; @@ -905,28 +936,18 @@ const ProfileCompilationInfo::InlineCacheMap* ProfileCompilationInfo::FindMethod(const std::string& dex_location, uint32_t dex_checksum, uint16_t dex_method_index) const { - auto info_it = info_.find(GetProfileDexFileKey(dex_location)); - if (info_it != info_.end()) { - if (!ChecksumMatch(dex_checksum, info_it->second.checksum)) { + const DexFileData* dex_data = FindDexData(GetProfileDexFileKey(dex_location)); + if (dex_data != nullptr) { + if (!ChecksumMatch(dex_checksum, dex_data->checksum)) { return nullptr; } - const MethodMap& methods = info_it->second.method_map; + const MethodMap& methods = dex_data->method_map; const auto method_it = methods.find(dex_method_index); return method_it == methods.end() ? nullptr : &(method_it->second); } return nullptr; } -void ProfileCompilationInfo::DexFileToProfileIndex( - /*out*/std::vector<DexReference>* dex_references) const { - dex_references->resize(info_.size()); - for (const auto& info_it : info_) { - DexReference& dex_ref = (*dex_references)[info_it.second.profile_index]; - dex_ref.dex_location = info_it.first; - dex_ref.dex_checksum = info_it.second.checksum; - } -} - bool ProfileCompilationInfo::GetMethod(const std::string& dex_location, uint32_t dex_checksum, uint16_t dex_method_index, @@ -936,7 +957,12 @@ bool ProfileCompilationInfo::GetMethod(const std::string& dex_location, return false; } - DexFileToProfileIndex(&pmi->dex_references); + pmi->dex_references.resize(info_.size()); + for (const DexFileData* dex_data : info_) { + pmi->dex_references[dex_data->profile_index].dex_location = dex_data->profile_key; + pmi->dex_references[dex_data->profile_index].dex_checksum = dex_data->checksum; + } + // TODO(calin): maybe expose a direct pointer to avoid copying pmi->inline_caches = *inline_caches; return true; @@ -944,12 +970,12 @@ bool ProfileCompilationInfo::GetMethod(const std::string& dex_location, bool ProfileCompilationInfo::ContainsClass(const DexFile& dex_file, dex::TypeIndex type_idx) const { - auto info_it = info_.find(GetProfileDexFileKey(dex_file.GetLocation())); - if (info_it != info_.end()) { - if (!ChecksumMatch(dex_file, info_it->second.checksum)) { + const DexFileData* dex_data = FindDexData(GetProfileDexFileKey(dex_file.GetLocation())); + if (dex_data != nullptr) { + if (!ChecksumMatch(dex_file, dex_data->checksum)) { return false; } - const std::set<dex::TypeIndex>& classes = info_it->second.class_set; + const std::set<dex::TypeIndex>& classes = dex_data->class_set; return classes.find(type_idx) != classes.end(); } return false; @@ -957,16 +983,16 @@ bool ProfileCompilationInfo::ContainsClass(const DexFile& dex_file, dex::TypeInd uint32_t ProfileCompilationInfo::GetNumberOfMethods() const { uint32_t total = 0; - for (const auto& it : info_) { - total += it.second.method_map.size(); + for (const DexFileData* dex_data : info_) { + total += dex_data->method_map.size(); } return total; } uint32_t ProfileCompilationInfo::GetNumberOfResolvedClasses() const { uint32_t total = 0; - for (const auto& it : info_) { - total += it.second.class_set.size(); + for (const DexFileData* dex_data : info_) { + total += dex_data->class_set.size(); } return total; } @@ -999,35 +1025,27 @@ std::string ProfileCompilationInfo::DumpInfo(const std::vector<const DexFile*>* os << "ProfileInfo:"; const std::string kFirstDexFileKeySubstitute = ":classes.dex"; - // Write the entries in profile index order. - std::vector<const std::string*> ordered_info_location(info_.size()); - std::vector<const DexFileData*> ordered_info_data(info_.size()); - for (const auto& it : info_) { - ordered_info_location[it.second.profile_index] = &(it.first); - ordered_info_data[it.second.profile_index] = &(it.second); - } - for (size_t profile_index = 0; profile_index < info_.size(); profile_index++) { + + for (const DexFileData* dex_data : info_) { os << "\n"; - const std::string& location = *ordered_info_location[profile_index]; - const DexFileData& dex_data = *ordered_info_data[profile_index]; if (print_full_dex_location) { - os << location; + os << dex_data->profile_key; } else { // Replace the (empty) multidex suffix of the first key with a substitute for easier reading. - std::string multidex_suffix = DexFile::GetMultiDexSuffix(location); + std::string multidex_suffix = DexFile::GetMultiDexSuffix(dex_data->profile_key); os << (multidex_suffix.empty() ? kFirstDexFileKeySubstitute : multidex_suffix); } - os << " [index=" << static_cast<uint32_t>(dex_data.profile_index) << "]"; + os << " [index=" << static_cast<uint32_t>(dex_data->profile_index) << "]"; const DexFile* dex_file = nullptr; if (dex_files != nullptr) { for (size_t i = 0; i < dex_files->size(); i++) { - if (location == (*dex_files)[i]->GetLocation()) { + if (dex_data->profile_key == (*dex_files)[i]->GetLocation()) { dex_file = (*dex_files)[i]; } } } os << "\n\tmethods: "; - for (const auto method_it : dex_data.method_map) { + for (const auto& method_it : dex_data->method_map) { if (dex_file != nullptr) { os << "\n\t\t" << dex_file->PrettyMethod(method_it.first, true); } else { @@ -1052,7 +1070,7 @@ std::string ProfileCompilationInfo::DumpInfo(const std::vector<const DexFile*>* os << "], "; } os << "\n\tclasses: "; - for (const auto class_it : dex_data.class_set) { + for (const auto class_it : dex_data->class_set) { if (dex_file != nullptr) { os << "\n\t\t" << dex_file->PrettyType(class_it); } else { @@ -1076,19 +1094,17 @@ void ProfileCompilationInfo::GetClassNames(const std::vector<const DexFile*>* de if (info_.empty()) { return; } - for (const auto& it : info_) { - const std::string& location = it.first; - const DexFileData& dex_data = it.second; + for (const DexFileData* dex_data : info_) { const DexFile* dex_file = nullptr; if (dex_files != nullptr) { for (size_t i = 0; i < dex_files->size(); i++) { - if (location == GetProfileDexFileKey((*dex_files)[i]->GetLocation()) && - dex_data.checksum == (*dex_files)[i]->GetLocationChecksum()) { + if (dex_data->profile_key == GetProfileDexFileKey((*dex_files)[i]->GetLocation()) && + dex_data->checksum == (*dex_files)[i]->GetLocationChecksum()) { dex_file = (*dex_files)[i]; } } } - for (const auto class_it : dex_data.class_set) { + for (const auto class_it : dex_data->class_set) { if (dex_file != nullptr) { class_names->insert(std::string(dex_file->PrettyType(class_it))); } @@ -1097,7 +1113,19 @@ void ProfileCompilationInfo::GetClassNames(const std::vector<const DexFile*>* de } bool ProfileCompilationInfo::Equals(const ProfileCompilationInfo& other) { - return info_.Equals(other.info_); + // No need to compare profile_key_map_. That's only a cache for fast search. + // All the information is already in the info_ vector. + if (info_.size() != other.info_.size()) { + return false; + } + for (size_t i = 0; i < info_.size(); i++) { + const DexFileData& dex_data = *info_[i]; + const DexFileData& other_dex_data = *other.info_[i]; + if (!(dex_data == other_dex_data)) { + return false; + } + } + return true; } std::set<DexCacheResolvedClasses> ProfileCompilationInfo::GetResolvedClasses( @@ -1107,13 +1135,11 @@ std::set<DexCacheResolvedClasses> ProfileCompilationInfo::GetResolvedClasses( key_to_location_map.emplace(GetProfileDexFileKey(location), location); } std::set<DexCacheResolvedClasses> ret; - for (auto&& pair : info_) { - const std::string& profile_key = pair.first; - auto it = key_to_location_map.find(profile_key); + for (const DexFileData* dex_data : info_) { + const auto& it = key_to_location_map.find(dex_data->profile_key); if (it != key_to_location_map.end()) { - const DexFileData& data = pair.second; - DexCacheResolvedClasses classes(it->second, it->second, data.checksum); - classes.AddClasses(data.class_set.begin(), data.class_set.end()); + DexCacheResolvedClasses classes(it->second, it->second, dex_data->checksum); + classes.AddClasses(dex_data->class_set.begin(), dex_data->class_set.end()); ret.insert(classes); } } @@ -1121,8 +1147,8 @@ std::set<DexCacheResolvedClasses> ProfileCompilationInfo::GetResolvedClasses( } void ProfileCompilationInfo::ClearResolvedClasses() { - for (auto& pair : info_) { - pair.second.class_set.clear(); + for (DexFileData* dex_data : info_) { + dex_data->class_set.clear(); } } @@ -1130,7 +1156,8 @@ void ProfileCompilationInfo::ClearResolvedClasses() { bool ProfileCompilationInfo::GenerateTestProfile(int fd, uint16_t number_of_dex_files, uint16_t method_ratio, - uint16_t class_ratio) { + uint16_t class_ratio, + uint32_t random_seed) { const std::string base_dex_location = "base.apk"; ProfileCompilationInfo info; // The limits are defined by the dex specification. @@ -1139,7 +1166,7 @@ bool ProfileCompilationInfo::GenerateTestProfile(int fd, uint16_t number_of_methods = max_method * method_ratio / 100; uint16_t number_of_classes = max_classes * class_ratio / 100; - srand(MicroTime()); + std::srand(random_seed); // Make sure we generate more samples with a low index value. // This makes it more likely to hit valid method/class indices in small apps. @@ -1169,6 +1196,32 @@ bool ProfileCompilationInfo::GenerateTestProfile(int fd, return info.Save(fd); } +// Naive implementation to generate a random profile file suitable for testing. +bool ProfileCompilationInfo::GenerateTestProfile( + int fd, + std::vector<std::unique_ptr<const DexFile>>& dex_files, + uint32_t random_seed) { + std::srand(random_seed); + ProfileCompilationInfo info; + for (std::unique_ptr<const DexFile>& dex_file : dex_files) { + const std::string& location = dex_file->GetLocation(); + uint32_t checksum = dex_file->GetLocationChecksum(); + for (uint32_t i = 0; i < dex_file->NumClassDefs(); ++i) { + // Randomly add a class from the dex file (with 50% chance). + if (std::rand() % 2 != 0) { + info.AddClassIndex(location, checksum, dex::TypeIndex(dex_file->GetClassDef(i).class_idx_)); + } + } + for (uint32_t i = 0; i < dex_file->NumMethodIds(); ++i) { + // Randomly add a method from the dex file (with 50% chance). + if (std::rand() % 2 != 0) { + info.AddMethodIndex(location, checksum, i); + } + } + } + return info.Save(fd); +} + bool ProfileCompilationInfo::OfflineProfileMethodInfo::operator==( const OfflineProfileMethodInfo& other) const { if (inline_caches.size() != other.inline_caches.size()) { @@ -1210,4 +1263,17 @@ bool ProfileCompilationInfo::OfflineProfileMethodInfo::operator==( return true; } +void ProfileCompilationInfo::ClearProfile() { + for (DexFileData* dex_data : info_) { + delete dex_data; + } + info_.clear(); + profile_key_map_.clear(); +} + +bool ProfileCompilationInfo::IsEmpty() const { + DCHECK_EQ(info_.empty(), profile_key_map_.empty()); + return info_.empty(); +} + } // namespace art diff --git a/runtime/jit/profile_compilation_info.h b/runtime/jit/profile_compilation_info.h index 6ad528c805..87f763686e 100644 --- a/runtime/jit/profile_compilation_info.h +++ b/runtime/jit/profile_compilation_info.h @@ -86,7 +86,7 @@ class ProfileCompilationInfo { // A dex location together with its checksum. struct DexReference { - DexReference() {} + DexReference() : dex_checksum(0) {} DexReference(const std::string& location, uint32_t checksum) : dex_location(location), dex_checksum(checksum) {} @@ -181,11 +181,16 @@ class ProfileCompilationInfo { // Public methods to create, extend or query the profile. + ProfileCompilationInfo() {} + ProfileCompilationInfo(const ProfileCompilationInfo& pci); + ~ProfileCompilationInfo(); + // Add the given methods and classes to the current profile object. bool AddMethodsAndClasses(const std::vector<ProfileMethodInfo>& methods, const std::set<DexCacheResolvedClasses>& resolved_classes); // Load profile information from the given file descriptor. + // If the current profile is non-empty the load will fail. bool Load(int fd); // Merge the data from another ProfileCompilationInfo into the current object. @@ -253,7 +258,14 @@ class ProfileCompilationInfo { static bool GenerateTestProfile(int fd, uint16_t number_of_dex_files, uint16_t method_ratio, - uint16_t class_ratio); + uint16_t class_ratio, + uint32_t random_seed); + + // Generate a test profile which will randomly contain classes and methods from + // the provided list of dex files. + static bool GenerateTestProfile(int fd, + std::vector<std::unique_ptr<const DexFile>>& dex_files, + uint32_t random_seed); // Check that the given profile method info contain the same data. static bool Equals(const ProfileCompilationInfo::OfflineProfileMethodInfo& pmi1, @@ -261,6 +273,7 @@ class ProfileCompilationInfo { private: enum ProfileLoadSatus { + kProfileLoadWouldOverwiteData, kProfileLoadIOError, kProfileLoadVersionMismatch, kProfileLoadBadData, @@ -271,14 +284,21 @@ class ProfileCompilationInfo { using MethodMap = SafeMap<uint16_t, InlineCacheMap>; // Internal representation of the profile information belonging to a dex file. + // Note that we could do without profile_key (the key used to encode the dex + // file in the profile) and profile_index (the index of the dex file in the + // profile) fields in this struct because we can infer them from + // profile_key_map_ and info_. However, it makes the profiles logic much + // simpler if we have references here as well. struct DexFileData { - DexFileData(uint32_t location_checksum, uint16_t index) - : profile_index(index), checksum(location_checksum) {} - // The profile index of this dex file (matches ClassReference#dex_profile_index) + DexFileData(const std::string& key, uint32_t location_checksum, uint16_t index) + : profile_key(key), profile_index(index), checksum(location_checksum) {} + // The profile key this data belongs to. + std::string profile_key; + // The profile index of this dex file (matches ClassReference#dex_profile_index). uint8_t profile_index; - // The dex checksum + // The dex checksum. uint32_t checksum; - // The methonds' profile information + // The methonds' profile information. MethodMap method_map; // The classes which have been profiled. Note that these don't necessarily include // all the classes that can be found in the inline caches reference. @@ -289,12 +309,9 @@ class ProfileCompilationInfo { } }; - // Maps dex file to their profile information. - using DexFileToProfileInfoMap = SafeMap<const std::string, DexFileData>; - - // Return the profile data for the given dex location or null if the dex location + // Return the profile data for the given profile key or null if the dex location // already exists but has a different checksum - DexFileData* GetOrAddDexFileData(const std::string& dex_location, uint32_t checksum); + DexFileData* GetOrAddDexFileData(const std::string& profile_key, uint32_t checksum); // Add a method index to the profile (without inline caches). bool AddMethodIndex(const std::string& dex_location, uint32_t checksum, uint16_t method_idx); @@ -325,6 +342,16 @@ class ProfileCompilationInfo { // be the same as the profile index of the dex file (used to encode the ClassReferences). void DexFileToProfileIndex(/*out*/std::vector<DexReference>* dex_references) const; + // Return the dex data associated with the given profile key or null if the profile + // doesn't contain the key. + const DexFileData* FindDexData(const std::string& profile_key) const; + + // Clear all the profile data. + void ClearProfile(); + + // Checks if the profile is empty. + bool IsEmpty() const; + // Parsing functionality. // The information present in the header of each profile line. @@ -431,7 +458,15 @@ class ProfileCompilationInfo { friend class ProfileAssistantTest; friend class Dex2oatLayoutTest; - DexFileToProfileInfoMap info_; + // Vector containing the actual profile info. + // The vector index is the profile index of the dex data and + // matched DexFileData::profile_index. + std::vector<DexFileData*> info_; + + // Cache mapping profile keys to profile index. + // This is used to speed up searches since it avoids iterating + // over the info_ vector when searching by profile key. + SafeMap<const std::string, uint8_t> profile_key_map_; }; } // namespace art diff --git a/runtime/jit/profile_compilation_info_test.cc b/runtime/jit/profile_compilation_info_test.cc index 5cd8e8fef5..c9f2d0e153 100644 --- a/runtime/jit/profile_compilation_info_test.cc +++ b/runtime/jit/profile_compilation_info_test.cc @@ -195,7 +195,7 @@ class ProfileCompilationInfoTest : public CommonRuntimeTest { dex_pc_data.AddClass(1, dex::TypeIndex(1)); dex_pc_data.AddClass(2, dex::TypeIndex(2)); - pmi.inline_caches.Put(dex_pc, dex_pc_data); + pmi.inline_caches.Put(dex_pc, dex_pc_data); } // Megamorphic for (uint16_t dex_pc = 22; dex_pc < 33; dex_pc++) { @@ -787,4 +787,26 @@ TEST_F(ProfileCompilationInfoTest, MissingTypesInlineCachesMerge) { ASSERT_TRUE(info_no_inline_cache.Save(GetFd(profile))); } +TEST_F(ProfileCompilationInfoTest, LoadShouldClearExistingDataFromProfiles) { + ScratchFile profile; + + ProfileCompilationInfo saved_info; + // Save a few methods. + for (uint16_t i = 0; i < 10; i++) { + ASSERT_TRUE(AddMethod("dex_location1", /* checksum */ 1, /* method_idx */ i, &saved_info)); + } + ASSERT_TRUE(saved_info.Save(GetFd(profile))); + ASSERT_EQ(0, profile.GetFile()->Flush()); + ASSERT_TRUE(profile.GetFile()->ResetOffset()); + + // Add a bunch of methods to test_info. + ProfileCompilationInfo test_info; + for (uint16_t i = 0; i < 10; i++) { + ASSERT_TRUE(AddMethod("dex_location2", /* checksum */ 2, /* method_idx */ i, &test_info)); + } + + // Attempt to load the saved profile into test_info. + // This should fail since the test_info already contains data and the load would overwrite it. + ASSERT_FALSE(test_info.Load(GetFd(profile))); +} } // namespace art diff --git a/runtime/method_handles.cc b/runtime/method_handles.cc index 58c5d17d1c..bd7c4ad53c 100644 --- a/runtime/method_handles.cc +++ b/runtime/method_handles.cc @@ -49,10 +49,19 @@ namespace { bool GetUnboxedPrimitiveType(ObjPtr<mirror::Class> klass, Primitive::Type* type) REQUIRES_SHARED(Locks::mutator_lock_) { ScopedAssertNoThreadSuspension ants(__FUNCTION__); -#define LOOKUP_PRIMITIVE(primitive, _, __, ___) \ - if (klass->DescriptorEquals(Primitive::BoxedDescriptor(primitive))) { \ - *type = primitive; \ - return true; \ + std::string storage; + const char* descriptor = klass->GetDescriptor(&storage); + static const char kJavaLangPrefix[] = "Ljava/lang/"; + static const size_t kJavaLangPrefixSize = sizeof(kJavaLangPrefix) - 1; + if (strncmp(descriptor, kJavaLangPrefix, kJavaLangPrefixSize) != 0) { + return false; + } + + descriptor += kJavaLangPrefixSize; +#define LOOKUP_PRIMITIVE(primitive, _, java_name, ___) \ + if (strcmp(descriptor, #java_name ";") == 0) { \ + *type = primitive; \ + return true; \ } PRIMITIVES_LIST(LOOKUP_PRIMITIVE); @@ -141,21 +150,23 @@ bool IsParameterTypeConvertible(ObjPtr<mirror::Class> from, ObjPtr<mirror::Class if (from->DescriptorEquals("Ljava/lang/Object;")) { // Object might be converted into a primitive during unboxing. return true; - } else if (Primitive::IsNumericType(to_primitive) && - from->DescriptorEquals("Ljava/lang/Number;")) { + } + + if (Primitive::IsNumericType(to_primitive) && from->DescriptorEquals("Ljava/lang/Number;")) { // Number might be unboxed into any of the number primitive types. return true; } + Primitive::Type unboxed_type; if (GetUnboxedPrimitiveType(from, &unboxed_type)) { if (unboxed_type == to_primitive) { // Straightforward unboxing conversion such as Boolean => boolean. return true; - } else { - // Check if widening operations for numeric primitives would work, - // such as Byte => byte => long. - return Primitive::IsWidenable(unboxed_type, to_primitive); } + + // Check if widening operations for numeric primitives would work, + // such as Byte => byte => long. + return Primitive::IsWidenable(unboxed_type, to_primitive); } } @@ -372,25 +383,18 @@ inline bool IsFieldAccess(mirror::MethodHandle::Kind handle_kind) { static inline size_t GetInsForProxyOrNativeMethod(ArtMethod* method) REQUIRES_SHARED(Locks::mutator_lock_) { DCHECK(method->IsNative() || method->IsProxyMethod()); - method = method->GetInterfaceMethodIfProxy(kRuntimePointerSize); - size_t num_ins = 0; - // Separate accounting for the receiver, which isn't a part of the - // shorty. - if (!method->IsStatic()) { - ++num_ins; - } - - uint32_t shorty_len = 0; - const char* shorty = method->GetShorty(&shorty_len); - for (size_t i = 1; i < shorty_len; ++i) { - const char c = shorty[i]; - ++num_ins; - if (c == 'J' || c == 'D') { + uint32_t shorty_length = 0; + const char* shorty = method->GetShorty(&shorty_length); + + // Static methods do not include the receiver. The receiver isn't included + // in the shorty_length though the return value is. + size_t num_ins = method->IsStatic() ? shorty_length - 1 : shorty_length; + for (const char* c = shorty + 1; *c != '\0'; ++c) { + if (*c == 'J' || *c == 'D') { ++num_ins; } } - return num_ins; } @@ -402,7 +406,10 @@ static inline bool IsCallerTransformer(Handle<mirror::MethodType> callsite_type) ObjPtr<mirror::ObjectArray<mirror::Class>> param_types(callsite_type->GetPTypes()); if (param_types->GetLength() == 1) { ObjPtr<mirror::Class> param(param_types->GetWithoutChecks(0)); - return param == WellKnownClasses::ToClass(WellKnownClasses::dalvik_system_EmulatedStackFrame); + // NB Comparing descriptor here as it appears faster in cycle simulation than using: + // param == WellKnownClasses::ToClass(WellKnownClasses::dalvik_system_EmulatedStackFrame) + // Costs are 98 vs 173 cycles per invocation. + return param->DescriptorEquals("Ldalvik/system/EmulatedStackFrame;"); } return false; @@ -416,35 +423,8 @@ static inline bool DoCallPolymorphic(ArtMethod* called_method, ShadowFrame& shadow_frame, const uint32_t (&args)[Instruction::kMaxVarArgRegs], uint32_t first_arg, - JValue* result, - const mirror::MethodHandle::Kind handle_kind) + JValue* result) REQUIRES_SHARED(Locks::mutator_lock_) { - // For virtual and interface methods ensure called_method points to - // the actual method to invoke. - if (handle_kind == mirror::MethodHandle::Kind::kInvokeVirtual || - handle_kind == mirror::MethodHandle::Kind::kInvokeInterface) { - uint32_t receiver_reg = is_range ? first_arg : args[0]; - ObjPtr<mirror::Object> receiver(shadow_frame.GetVRegReference(receiver_reg)); - if (IsCallerTransformer(callsite_type)) { - // The current receiver is an emulated stack frame, the method's - // receiver needs to be fetched from there as the emulated frame - // will be unpacked into a new frame. - receiver = ObjPtr<mirror::EmulatedStackFrame>::DownCast(receiver)->GetReceiver(); - } - - ObjPtr<mirror::Class> declaring_class(called_method->GetDeclaringClass()); - if (receiver == nullptr || receiver->GetClass() != declaring_class) { - // Verify that _vRegC is an object reference and of the type expected by - // the receiver. - if (!VerifyObjectIsClass(receiver, declaring_class)) { - DCHECK(self->IsExceptionPending()); - return false; - } - called_method = receiver->GetClass()->FindVirtualMethodForVirtualOrInterface( - called_method, kRuntimePointerSize); - } - } - // Compute method information. const DexFile::CodeItem* code_item = called_method->GetCodeItem(); @@ -513,17 +493,23 @@ static inline bool DoCallPolymorphic(ArtMethod* called_method, result->SetL(0); return false; } - } else if (!ConvertAndCopyArgumentsFromCallerFrame<is_range>(self, - callsite_type, - target_type, - shadow_frame, - args, - first_arg, - first_dest_reg, - new_shadow_frame)) { - DCHECK(self->IsExceptionPending()); - result->SetL(0); - return false; + } else { + if (!callsite_type->IsConvertible(target_type.Get())) { + ThrowWrongMethodTypeException(target_type.Get(), callsite_type.Get()); + return false; + } + if (!ConvertAndCopyArgumentsFromCallerFrame<is_range>(self, + callsite_type, + target_type, + shadow_frame, + args, + first_arg, + first_dest_reg, + new_shadow_frame)) { + DCHECK(self->IsExceptionPending()); + result->SetL(0); + return false; + } } } } @@ -548,13 +534,13 @@ static inline bool DoCallPolymorphic(ArtMethod* called_method, if (ConvertReturnValue(emulated_stack_type, target_type, &local_result)) { emulated_stack_frame->SetReturnValue(self, local_result); return true; - } else { - DCHECK(self->IsExceptionPending()); - return false; } - } else { - return ConvertReturnValue(callsite_type, target_type, result); + + DCHECK(self->IsExceptionPending()); + return false; } + + return ConvertReturnValue(callsite_type, target_type, result); } template <bool is_range> @@ -650,98 +636,130 @@ inline static ObjPtr<mirror::Class> GetAndInitializeDeclaringClass(Thread* self, return klass; } +ArtMethod* RefineTargetMethod(Thread* self, + ShadowFrame& shadow_frame, + const mirror::MethodHandle::Kind& handle_kind, + Handle<mirror::MethodType> handle_type, + Handle<mirror::MethodType> callsite_type, + const uint32_t receiver_reg, + ArtMethod* target_method) + REQUIRES_SHARED(Locks::mutator_lock_) { + if (handle_kind == mirror::MethodHandle::Kind::kInvokeVirtual || + handle_kind == mirror::MethodHandle::Kind::kInvokeInterface) { + // For virtual and interface methods ensure target_method points to + // the actual method to invoke. + ObjPtr<mirror::Object> receiver(shadow_frame.GetVRegReference(receiver_reg)); + if (IsCallerTransformer(callsite_type)) { + // The current receiver is an emulated stack frame, the method's + // receiver needs to be fetched from there as the emulated frame + // will be unpacked into a new frame. + receiver = ObjPtr<mirror::EmulatedStackFrame>::DownCast(receiver)->GetReceiver(); + } + + ObjPtr<mirror::Class> declaring_class(target_method->GetDeclaringClass()); + if (receiver == nullptr || receiver->GetClass() != declaring_class) { + // Verify that _vRegC is an object reference and of the type expected by + // the receiver. + if (!VerifyObjectIsClass(receiver, declaring_class)) { + DCHECK(self->IsExceptionPending()); + return nullptr; + } + return receiver->GetClass()->FindVirtualMethodForVirtualOrInterface( + target_method, kRuntimePointerSize); + } + } else if (handle_kind == mirror::MethodHandle::Kind::kInvokeDirect) { + // String constructors are a special case, they are replaced with + // StringFactory methods. + if (target_method->IsConstructor() && target_method->GetDeclaringClass()->IsStringClass()) { + DCHECK(handle_type->GetRType()->IsStringClass()); + return WellKnownClasses::StringInitToStringFactory(target_method); + } + } else if (handle_kind == mirror::MethodHandle::Kind::kInvokeSuper) { + ObjPtr<mirror::Class> declaring_class = target_method->GetDeclaringClass(); + + // Note that we're not dynamically dispatching on the type of the receiver + // here. We use the static type of the "receiver" object that we've + // recorded in the method handle's type, which will be the same as the + // special caller that was specified at the point of lookup. + ObjPtr<mirror::Class> referrer_class = handle_type->GetPTypes()->Get(0); + if (!declaring_class->IsInterface()) { + ObjPtr<mirror::Class> super_class = referrer_class->GetSuperClass(); + uint16_t vtable_index = target_method->GetMethodIndex(); + DCHECK(super_class != nullptr); + DCHECK(super_class->HasVTable()); + // Note that super_class is a super of referrer_class and target_method + // will always be declared by super_class (or one of its super classes). + DCHECK_LT(vtable_index, super_class->GetVTableLength()); + return super_class->GetVTableEntry(vtable_index, kRuntimePointerSize); + } else { + return referrer_class->FindVirtualMethodForInterfaceSuper(target_method, kRuntimePointerSize); + } + } + return target_method; +} + template <bool is_range> -bool DoInvokePolymorphicUnchecked(Thread* self, - ShadowFrame& shadow_frame, - Handle<mirror::MethodHandle> method_handle, - Handle<mirror::MethodType> callsite_type, - const uint32_t (&args)[Instruction::kMaxVarArgRegs], - uint32_t first_arg, - JValue* result) +bool DoInvokePolymorphicMethod(Thread* self, + ShadowFrame& shadow_frame, + Handle<mirror::MethodHandle> method_handle, + Handle<mirror::MethodType> callsite_type, + const uint32_t (&args)[Instruction::kMaxVarArgRegs], + uint32_t first_arg, + JValue* result) REQUIRES_SHARED(Locks::mutator_lock_) { StackHandleScope<1> hs(self); Handle<mirror::MethodType> handle_type(hs.NewHandle(method_handle->GetMethodType())); const mirror::MethodHandle::Kind handle_kind = method_handle->GetHandleKind(); - if (IsInvoke(handle_kind)) { - // Get the method we're actually invoking along with the kind of - // invoke that is desired. We don't need to perform access checks at this - // point because they would have been performed on our behalf at the point - // of creation of the method handle. - ArtMethod* called_method = method_handle->GetTargetMethod(); - CHECK(called_method != nullptr); - - if (handle_kind == mirror::MethodHandle::Kind::kInvokeVirtual || - handle_kind == mirror::MethodHandle::Kind::kInvokeInterface) { - // TODO: Unfortunately, we have to postpone dynamic receiver based checks - // because the receiver might be cast or might come from an emulated stack - // frame, which means that it is unknown at this point. We perform these - // checks inside DoCallPolymorphic right before we do the actual invoke. - } else if (handle_kind == mirror::MethodHandle::Kind::kInvokeDirect) { - // String constructors are a special case, they are replaced with StringFactory - // methods. - if (called_method->IsConstructor() && called_method->GetDeclaringClass()->IsStringClass()) { - DCHECK(handle_type->GetRType()->IsStringClass()); - called_method = WellKnownClasses::StringInitToStringFactory(called_method); - } - } else if (handle_kind == mirror::MethodHandle::Kind::kInvokeSuper) { - ObjPtr<mirror::Class> declaring_class = called_method->GetDeclaringClass(); - - // Note that we're not dynamically dispatching on the type of the receiver - // here. We use the static type of the "receiver" object that we've - // recorded in the method handle's type, which will be the same as the - // special caller that was specified at the point of lookup. - ObjPtr<mirror::Class> referrer_class = handle_type->GetPTypes()->Get(0); - if (!declaring_class->IsInterface()) { - ObjPtr<mirror::Class> super_class = referrer_class->GetSuperClass(); - uint16_t vtable_index = called_method->GetMethodIndex(); - DCHECK(super_class != nullptr); - DCHECK(super_class->HasVTable()); - // Note that super_class is a super of referrer_class and called_method - // will always be declared by super_class (or one of its super classes). - DCHECK_LT(vtable_index, super_class->GetVTableLength()); - called_method = super_class->GetVTableEntry(vtable_index, kRuntimePointerSize); - } else { - called_method = referrer_class->FindVirtualMethodForInterfaceSuper( - called_method, kRuntimePointerSize); - } - CHECK(called_method != nullptr); - } - if (IsInvokeTransform(handle_kind)) { - // There are two cases here - method handles representing regular - // transforms and those representing call site transforms. Method - // handles for call site transforms adapt their MethodType to match - // the call site. For these, the |callee_type| is the same as the - // |callsite_type|. The VarargsCollector is such a tranform, its - // method type depends on the call site, ie. x(a) or x(a, b), or - // x(a, b, c). The VarargsCollector invokes a variable arity method - // with the arity arguments in an array. - Handle<mirror::MethodType> callee_type = - (handle_kind == mirror::MethodHandle::Kind::kInvokeCallSiteTransform) ? callsite_type - : handle_type; - return DoCallTransform<is_range>(called_method, + DCHECK(IsInvoke(handle_kind)); + + // Get the method we're actually invoking along with the kind of + // invoke that is desired. We don't need to perform access checks at this + // point because they would have been performed on our behalf at the point + // of creation of the method handle. + ArtMethod* target_method = method_handle->GetTargetMethod(); + uint32_t receiver_reg = is_range ? first_arg: args[0]; + ArtMethod* called_method = RefineTargetMethod(self, + shadow_frame, + handle_kind, + handle_type, + callsite_type, + receiver_reg, + target_method); + if (called_method == nullptr) { + DCHECK(self->IsExceptionPending()); + return false; + } + + if (IsInvokeTransform(handle_kind)) { + // There are two cases here - method handles representing regular + // transforms and those representing call site transforms. Method + // handles for call site transforms adapt their MethodType to match + // the call site. For these, the |callee_type| is the same as the + // |callsite_type|. The VarargsCollector is such a tranform, its + // method type depends on the call site, ie. x(a) or x(a, b), or + // x(a, b, c). The VarargsCollector invokes a variable arity method + // with the arity arguments in an array. + Handle<mirror::MethodType> callee_type = + (handle_kind == mirror::MethodHandle::Kind::kInvokeCallSiteTransform) ? callsite_type + : handle_type; + return DoCallTransform<is_range>(called_method, + callsite_type, + callee_type, + self, + shadow_frame, + method_handle /* receiver */, + args, + first_arg, + result); + } else { + return DoCallPolymorphic<is_range>(called_method, callsite_type, - callee_type, + handle_type, self, shadow_frame, - method_handle /* receiver */, args, first_arg, result); - - } else { - return DoCallPolymorphic<is_range>(called_method, - callsite_type, - handle_type, - self, - shadow_frame, - args, - first_arg, - result, - handle_kind); - } - } else { - LOG(FATAL) << "Unreachable: " << handle_kind; - UNREACHABLE(); } } @@ -948,55 +966,30 @@ static inline bool DoInvokePolymorphicNonExact(Thread* self, ObjPtr<mirror::MethodType> handle_type(method_handle->GetMethodType()); CHECK(handle_type != nullptr); - if (!IsInvokeTransform(handle_kind)) { - if (UNLIKELY(!IsCallerTransformer(callsite_type) && - !callsite_type->IsConvertible(handle_type.Ptr()))) { + if (IsFieldAccess(handle_kind)) { + DCHECK(!callsite_type->IsExactMatch(handle_type.Ptr())); + if (!callsite_type->IsConvertible(handle_type.Ptr())) { ThrowWrongMethodTypeException(handle_type.Ptr(), callsite_type.Get()); return false; } + const bool do_convert = true; + return DoInvokePolymorphicFieldAccess<is_range, do_convert>( + self, + shadow_frame, + method_handle, + callsite_type, + args, + first_arg, + result); } - if (IsFieldAccess(handle_kind)) { - if (UNLIKELY(callsite_type->IsExactMatch(handle_type.Ptr()))) { - const bool do_convert = false; - return DoInvokePolymorphicFieldAccess<is_range, do_convert>( - self, - shadow_frame, - method_handle, - callsite_type, - args, - first_arg, - result); - } else { - const bool do_convert = true; - return DoInvokePolymorphicFieldAccess<is_range, do_convert>( - self, - shadow_frame, - method_handle, - callsite_type, - args, - first_arg, - result); - } - } - - if (UNLIKELY(callsite_type->IsExactMatch(handle_type.Ptr()))) { - return DoInvokePolymorphicUnchecked<is_range>(self, - shadow_frame, - method_handle, - callsite_type, - args, - first_arg, - result); - } else { - return DoInvokePolymorphicUnchecked<is_range>(self, - shadow_frame, - method_handle, - callsite_type, - args, - first_arg, - result); - } + return DoInvokePolymorphicMethod<is_range>(self, + shadow_frame, + method_handle, + callsite_type, + args, + first_arg, + result); } template <bool is_range> @@ -1008,32 +1001,9 @@ bool DoInvokePolymorphicExact(Thread* self, uint32_t first_arg, JValue* result) REQUIRES_SHARED(Locks::mutator_lock_) { - // We need to check the nominal type of the handle in addition to the - // real type. The "nominal" type is present when MethodHandle.asType is - // called any handle, and results in the declared type of the handle - // changing. - ObjPtr<mirror::MethodType> nominal_type(method_handle->GetNominalType()); - if (UNLIKELY(nominal_type != nullptr)) { - if (UNLIKELY(!callsite_type->IsExactMatch(nominal_type.Ptr()))) { - ThrowWrongMethodTypeException(nominal_type.Ptr(), callsite_type.Get()); - return false; - } - return DoInvokePolymorphicNonExact<is_range>(self, - shadow_frame, - method_handle, - callsite_type, - args, - first_arg, - result); - } - - ObjPtr<mirror::MethodType> handle_type(method_handle->GetMethodType()); - if (UNLIKELY(!callsite_type->IsExactMatch(handle_type.Ptr()))) { - ThrowWrongMethodTypeException(handle_type.Ptr(), callsite_type.Get()); - return false; - } - + StackHandleScope<1> hs(self); const mirror::MethodHandle::Kind handle_kind = method_handle->GetHandleKind(); + Handle<mirror::MethodType> method_handle_type(hs.NewHandle(method_handle->GetMethodType())); if (IsFieldAccess(handle_kind)) { const bool do_convert = false; return DoInvokePolymorphicFieldAccess<is_range, do_convert>( @@ -1046,13 +1016,68 @@ bool DoInvokePolymorphicExact(Thread* self, result); } - return DoInvokePolymorphicUnchecked<is_range>(self, + // Slow-path check. + if (IsInvokeTransform(handle_kind) || IsCallerTransformer(callsite_type)) { + return DoInvokePolymorphicMethod<is_range>(self, + shadow_frame, + method_handle, + callsite_type, + args, + first_arg, + result); + } + + // On the fast-path. This is equivalent to DoCallPolymoprhic without the conversion paths. + ArtMethod* target_method = method_handle->GetTargetMethod(); + uint32_t receiver_reg = is_range ? first_arg : args[0]; + ArtMethod* called_method = RefineTargetMethod(self, shadow_frame, - method_handle, + handle_kind, + method_handle_type, callsite_type, - args, - first_arg, - result); + receiver_reg, + target_method); + if (called_method == nullptr) { + DCHECK(self->IsExceptionPending()); + return false; + } + + // Compute method information. + const DexFile::CodeItem* code_item = called_method->GetCodeItem(); + uint16_t num_regs; + size_t num_input_regs; + size_t first_dest_reg; + if (LIKELY(code_item != nullptr)) { + num_regs = code_item->registers_size_; + first_dest_reg = num_regs - code_item->ins_size_; + num_input_regs = code_item->ins_size_; + // Parameter registers go at the end of the shadow frame. + DCHECK_NE(first_dest_reg, (size_t)-1); + } else { + // No local regs for proxy and native methods. + DCHECK(called_method->IsNative() || called_method->IsProxyMethod()); + num_regs = num_input_regs = GetInsForProxyOrNativeMethod(called_method); + first_dest_reg = 0; + } + + // Allocate shadow frame on the stack. + const char* old_cause = self->StartAssertNoThreadSuspension("DoCallCommon"); + ShadowFrameAllocaUniquePtr shadow_frame_unique_ptr = + CREATE_SHADOW_FRAME(num_regs, &shadow_frame, called_method, /* dex pc */ 0); + ShadowFrame* new_shadow_frame = shadow_frame_unique_ptr.get(); + CopyArgumentsFromCallerFrame<is_range>(shadow_frame, + new_shadow_frame, + args, + first_arg, + first_dest_reg, + num_input_regs); + self->EndAssertNoThreadSuspension(old_cause); + + PerformCall(self, code_item, shadow_frame.GetMethod(), first_dest_reg, new_shadow_frame, result); + if (self->IsExceptionPending()) { + return false; + } + return true; } } // namespace @@ -1067,7 +1092,35 @@ bool DoInvokePolymorphic(Thread* self, uint32_t first_arg, JValue* result) REQUIRES_SHARED(Locks::mutator_lock_) { + ObjPtr<mirror::MethodType> method_handle_type = method_handle->GetMethodType(); if (IsMethodHandleInvokeExact(invoke_method)) { + // We need to check the nominal type of the handle in addition to the + // real type. The "nominal" type is present when MethodHandle.asType is + // called any handle, and results in the declared type of the handle + // changing. + ObjPtr<mirror::MethodType> nominal_type(method_handle->GetNominalType()); + if (UNLIKELY(nominal_type != nullptr)) { + if (UNLIKELY(!callsite_type->IsExactMatch(nominal_type.Ptr()))) { + ThrowWrongMethodTypeException(nominal_type.Ptr(), callsite_type.Get()); + return false; + } + + if (LIKELY(!nominal_type->IsExactMatch(method_handle_type.Ptr()))) { + // Different nominal type means we have to treat as non-exact. + return DoInvokePolymorphicNonExact<is_range>(self, + shadow_frame, + method_handle, + callsite_type, + args, + first_arg, + result); + } + } + + if (!callsite_type->IsExactMatch(method_handle_type.Ptr())) { + ThrowWrongMethodTypeException(method_handle_type.Ptr(), callsite_type.Get()); + return false; + } return DoInvokePolymorphicExact<is_range>(self, shadow_frame, method_handle, @@ -1076,6 +1129,16 @@ bool DoInvokePolymorphic(Thread* self, first_arg, result); } else { + if (UNLIKELY(callsite_type->IsExactMatch(method_handle_type.Ptr()))) { + // A non-exact invoke that can be invoked exactly. + return DoInvokePolymorphicExact<is_range>(self, + shadow_frame, + method_handle, + callsite_type, + args, + first_arg, + result); + } return DoInvokePolymorphicNonExact<is_range>(self, shadow_frame, method_handle, diff --git a/runtime/mirror/array.h b/runtime/mirror/array.h index 16cf30f1e2..51d9d24619 100644 --- a/runtime/mirror/array.h +++ b/runtime/mirror/array.h @@ -198,6 +198,13 @@ class PointerArray : public Array { T GetElementPtrSize(uint32_t idx, PointerSize ptr_size) REQUIRES_SHARED(Locks::mutator_lock_); + void** ElementAddress(size_t index, PointerSize ptr_size) REQUIRES_SHARED(Locks::mutator_lock_) { + DCHECK_LT(index, static_cast<size_t>(GetLength())); + return reinterpret_cast<void**>(reinterpret_cast<uint8_t*>(this) + + Array::DataOffset(static_cast<size_t>(ptr_size)).Uint32Value() + + static_cast<size_t>(ptr_size) * index); + } + template<bool kTransactionActive = false, bool kUnchecked = false> void SetElementPtrSize(uint32_t idx, uint64_t element, PointerSize ptr_size) REQUIRES_SHARED(Locks::mutator_lock_); diff --git a/runtime/mirror/class-inl.h b/runtime/mirror/class-inl.h index 003b03b2f9..be3b937f3e 100644 --- a/runtime/mirror/class-inl.h +++ b/runtime/mirror/class-inl.h @@ -1097,7 +1097,9 @@ inline void Class::FixupNativePointers(Class* dest, if (!IsTemp() && ShouldHaveEmbeddedVTable<kVerifyNone, kReadBarrierOption>()) { for (int32_t i = 0, count = GetEmbeddedVTableLength(); i < count; ++i) { ArtMethod* method = GetEmbeddedVTableEntry(i, pointer_size); - ArtMethod* new_method = visitor(method); + void** dest_addr = reinterpret_cast<void**>(reinterpret_cast<uintptr_t>(dest) + + EmbeddedVTableEntryOffset(i, pointer_size).Uint32Value()); + ArtMethod* new_method = visitor(method, dest_addr); if (method != new_method) { dest->SetEmbeddedVTableEntryUnchecked(i, new_method, pointer_size); } diff --git a/runtime/mirror/dex_cache.h b/runtime/mirror/dex_cache.h index 78b2e15f5a..cf570b8be0 100644 --- a/runtime/mirror/dex_cache.h +++ b/runtime/mirror/dex_cache.h @@ -65,7 +65,7 @@ template <typename T> struct PACKED(8) DexCachePair { DexCachePair(ObjPtr<T> object, uint32_t index) : object(object), index(index) {} - DexCachePair() = default; + DexCachePair() : index(0) {} DexCachePair(const DexCachePair<T>&) = default; DexCachePair& operator=(const DexCachePair<T>&) = default; diff --git a/runtime/mirror/object-inl.h b/runtime/mirror/object-inl.h index 8e591e4434..811f1ea726 100644 --- a/runtime/mirror/object-inl.h +++ b/runtime/mirror/object-inl.h @@ -187,10 +187,12 @@ inline uint32_t Object::GetReadBarrierState(uintptr_t* fake_address_dependency) uint32_t rb_state = lw.ReadBarrierState(); return rb_state; #else - // mips/mips64 - LOG(FATAL) << "Unreachable"; - UNREACHABLE(); - UNUSED(fake_address_dependency); + // MIPS32/MIPS64: use a memory barrier to prevent load-load reordering. + LockWord lw = GetLockWord(false); + *fake_address_dependency = 0; + std::atomic_thread_fence(std::memory_order_acquire); + uint32_t rb_state = lw.ReadBarrierState(); + return rb_state; #endif } diff --git a/runtime/monitor.h b/runtime/monitor.h index 1fa46826eb..e80d31cdd5 100644 --- a/runtime/monitor.h +++ b/runtime/monitor.h @@ -354,7 +354,7 @@ class MonitorList { // For use only by the JDWP implementation. class MonitorInfo { public: - MonitorInfo() = default; + MonitorInfo() : owner_(nullptr), entry_count_(0) {} MonitorInfo(const MonitorInfo&) = default; MonitorInfo& operator=(const MonitorInfo&) = default; explicit MonitorInfo(mirror::Object* o) REQUIRES(Locks::mutator_lock_); diff --git a/runtime/native/dalvik_system_VMRuntime.cc b/runtime/native/dalvik_system_VMRuntime.cc index d81c13df11..6ca951fd53 100644 --- a/runtime/native/dalvik_system_VMRuntime.cc +++ b/runtime/native/dalvik_system_VMRuntime.cc @@ -17,6 +17,8 @@ #include "dalvik_system_VMRuntime.h" #ifdef ART_TARGET_ANDROID +#include <sys/time.h> +#include <sys/resource.h> extern "C" void android_set_application_target_sdk_version(uint32_t version); #endif #include <limits.h> @@ -625,6 +627,23 @@ static jboolean VMRuntime_didPruneDalvikCache(JNIEnv* env ATTRIBUTE_UNUSED, return Runtime::Current()->GetPrunedDalvikCache() ? JNI_TRUE : JNI_FALSE; } +static void VMRuntime_setSystemDaemonThreadPriority(JNIEnv* env ATTRIBUTE_UNUSED, + jclass klass ATTRIBUTE_UNUSED) { +#ifdef ART_TARGET_ANDROID + Thread* self = Thread::Current(); + DCHECK(self != nullptr); + pid_t tid = self->GetTid(); + // We use a priority lower than the default for the system daemon threads (eg HeapTaskDaemon) to + // avoid jank due to CPU contentions between GC and other UI-related threads. b/36631902. + // We may use a native priority that doesn't have a corresponding java.lang.Thread-level priority. + static constexpr int kSystemDaemonNiceValue = 4; // priority 124 + if (setpriority(PRIO_PROCESS, tid, kSystemDaemonNiceValue) != 0) { + PLOG(INFO) << *self << " setpriority(PRIO_PROCESS, " << tid << ", " + << kSystemDaemonNiceValue << ") failed"; + } +#endif +} + static JNINativeMethod gMethods[] = { FAST_NATIVE_METHOD(VMRuntime, addressOf, "(Ljava/lang/Object;)J"), NATIVE_METHOD(VMRuntime, bootClassPath, "()Ljava/lang/String;"), @@ -662,6 +681,7 @@ static JNINativeMethod gMethods[] = { NATIVE_METHOD(VMRuntime, isBootClassPathOnDisk, "(Ljava/lang/String;)Z"), NATIVE_METHOD(VMRuntime, getCurrentInstructionSet, "()Ljava/lang/String;"), NATIVE_METHOD(VMRuntime, didPruneDalvikCache, "()Z"), + NATIVE_METHOD(VMRuntime, setSystemDaemonThreadPriority, "()V"), }; void register_dalvik_system_VMRuntime(JNIEnv* env) { diff --git a/runtime/native_bridge_art_interface.cc b/runtime/native_bridge_art_interface.cc index c58854b13e..d77cfa1d35 100644 --- a/runtime/native_bridge_art_interface.cc +++ b/runtime/native_bridge_art_interface.cc @@ -118,7 +118,7 @@ void InitializeNativeBridge(JNIEnv* env, const char* instruction_set) { for (int signal = 0; signal < _NSIG; ++signal) { android::NativeBridgeSignalHandlerFn fn = android::NativeBridgeGetSignalHandler(signal); if (fn != nullptr) { - SetSpecialSignalHandlerFn(signal, fn); + AddSpecialSignalHandlerFn(signal, fn); } } #endif diff --git a/runtime/native_stack_dump.cc b/runtime/native_stack_dump.cc index 7460d622b5..cbc502487f 100644 --- a/runtime/native_stack_dump.cc +++ b/runtime/native_stack_dump.cc @@ -105,7 +105,7 @@ static std::unique_ptr<Addr2linePipe> Connect(const std::string& name, const cha if (pid == -1) { close(caller_to_addr2line[0]); close(caller_to_addr2line[1]); - close(addr2line_to_caller[1]); + close(addr2line_to_caller[0]); close(addr2line_to_caller[1]); return nullptr; } diff --git a/runtime/nth_caller_visitor.h b/runtime/nth_caller_visitor.h index f72a853393..71c6a82dc5 100644 --- a/runtime/nth_caller_visitor.h +++ b/runtime/nth_caller_visitor.h @@ -31,7 +31,8 @@ struct NthCallerVisitor : public StackVisitor { n(n_in), include_runtime_and_upcalls_(include_runtime_and_upcalls), count(0), - caller(nullptr) {} + caller(nullptr), + caller_pc(0) {} bool VisitFrame() REQUIRES_SHARED(Locks::mutator_lock_) { ArtMethod* m = GetMethod(); diff --git a/runtime/oat.h b/runtime/oat.h index 190d533eff..faa0129d6b 100644 --- a/runtime/oat.h +++ b/runtime/oat.h @@ -32,7 +32,7 @@ class InstructionSetFeatures; class PACKED(4) OatHeader { public: static constexpr uint8_t kOatMagic[] = { 'o', 'a', 't', '\n' }; - static constexpr uint8_t kOatVersion[] = { '1', '1', '6', '\0' }; // Add method infos. + static constexpr uint8_t kOatVersion[] = { '1', '1', '7', '\0' }; // Read barriers on MIPS. static constexpr const char* kImageLocationKey = "image-location"; static constexpr const char* kDex2OatCmdLineKey = "dex2oat-cmdline"; diff --git a/runtime/oat_file_assistant.cc b/runtime/oat_file_assistant.cc index 3396ce0b57..db6f8ee488 100644 --- a/runtime/oat_file_assistant.cc +++ b/runtime/oat_file_assistant.cc @@ -640,15 +640,8 @@ bool OatFileAssistant::DexLocationToOdexFilename(const std::string& location, std::string dir = location.substr(0, pos+1); dir += "oat/" + std::string(GetInstructionSetString(isa)); - // Find the file portion of the dex location. - std::string file; - if (pos == std::string::npos) { - file = location; - } else { - file = location.substr(pos+1); - } - // Get the base part of the file without the extension. + std::string file = location.substr(pos+1); pos = file.rfind('.'); if (pos == std::string::npos) { *error_msg = "Dex location " + location + " has no extension."; diff --git a/runtime/oat_file_assistant.h b/runtime/oat_file_assistant.h index d61e9949b6..b84e711daa 100644 --- a/runtime/oat_file_assistant.h +++ b/runtime/oat_file_assistant.h @@ -369,7 +369,7 @@ class OatFileAssistant { std::unique_ptr<OatFile> file_; bool status_attempted_ = false; - OatStatus status_; + OatStatus status_ = OatStatus::kOatCannotOpen; // For debugging only. // If this flag is set, the file has been released to the user and the diff --git a/runtime/openjdkjvmti/Android.bp b/runtime/openjdkjvmti/Android.bp index dd49ad0cfb..e38f265c5a 100644 --- a/runtime/openjdkjvmti/Android.bp +++ b/runtime/openjdkjvmti/Android.bp @@ -24,6 +24,7 @@ cc_defaults { defaults: ["art_defaults"], host_supported: true, srcs: ["events.cc", + "fixed_up_dex_file.cc", "object_tagging.cc", "OpenjdkJvmTi.cc", "ti_class.cc", @@ -56,7 +57,10 @@ cc_defaults { art_cc_library { name: "libopenjdkjvmti", defaults: ["libopenjdkjvmti_defaults"], - shared_libs: ["libart"], + shared_libs: [ + "libart", + "libart-compiler", + ], } art_cc_library { @@ -65,5 +69,8 @@ art_cc_library { "art_debug_defaults", "libopenjdkjvmti_defaults", ], - shared_libs: ["libartd"], + shared_libs: [ + "libartd", + "libartd-compiler", + ], } diff --git a/runtime/openjdkjvmti/OpenjdkJvmTi.cc b/runtime/openjdkjvmti/OpenjdkJvmTi.cc index 5401e5cdf8..39e603e1e7 100644 --- a/runtime/openjdkjvmti/OpenjdkJvmTi.cc +++ b/runtime/openjdkjvmti/OpenjdkJvmTi.cc @@ -79,20 +79,26 @@ EventHandler gEventHandler; class JvmtiFunctions { private: - static bool IsValidEnv(jvmtiEnv* env) { - return env != nullptr; + static jvmtiError getEnvironmentError(jvmtiEnv* env) { + if (env == nullptr) { + return ERR(INVALID_ENVIRONMENT); + } else if (art::Thread::Current() == nullptr) { + return ERR(UNATTACHED_THREAD); + } else { + return OK; + } } -#define ENSURE_VALID_ENV(env) \ - do { \ - if (!IsValidEnv(env)) { \ - return ERR(INVALID_ENVIRONMENT); \ - } \ +#define ENSURE_VALID_ENV(env) \ + do { \ + jvmtiError ensure_valid_env_ ## __LINE__ = getEnvironmentError(env); \ + if (ensure_valid_env_ ## __LINE__ != OK) { \ + return ensure_valid_env_ ## __LINE__ ; \ + } \ } while (false) #define ENSURE_HAS_CAP(env, cap) \ do { \ - ENSURE_VALID_ENV(env); \ if (ArtJvmTiEnv::AsArtJvmTiEnv(env)->capabilities.cap != 1) { \ return ERR(MUST_POSSESS_CAPABILITY); \ } \ @@ -121,18 +127,22 @@ class JvmtiFunctions { } static jvmtiError GetThreadState(jvmtiEnv* env, jthread thread, jint* thread_state_ptr) { + ENSURE_VALID_ENV(env); return ThreadUtil::GetThreadState(env, thread, thread_state_ptr); } static jvmtiError GetCurrentThread(jvmtiEnv* env, jthread* thread_ptr) { + ENSURE_VALID_ENV(env); return ThreadUtil::GetCurrentThread(env, thread_ptr); } static jvmtiError GetAllThreads(jvmtiEnv* env, jint* threads_count_ptr, jthread** threads_ptr) { + ENSURE_VALID_ENV(env); return ThreadUtil::GetAllThreads(env, threads_count_ptr, threads_ptr); } static jvmtiError SuspendThread(jvmtiEnv* env, jthread thread ATTRIBUTE_UNUSED) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_suspend); return ERR(NOT_IMPLEMENTED); } @@ -141,11 +151,13 @@ class JvmtiFunctions { jint request_count ATTRIBUTE_UNUSED, const jthread* request_list ATTRIBUTE_UNUSED, jvmtiError* results ATTRIBUTE_UNUSED) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_suspend); return ERR(NOT_IMPLEMENTED); } static jvmtiError ResumeThread(jvmtiEnv* env, jthread thread ATTRIBUTE_UNUSED) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_suspend); return ERR(NOT_IMPLEMENTED); } @@ -154,6 +166,7 @@ class JvmtiFunctions { jint request_count ATTRIBUTE_UNUSED, const jthread* request_list ATTRIBUTE_UNUSED, jvmtiError* results ATTRIBUTE_UNUSED) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_suspend); return ERR(NOT_IMPLEMENTED); } @@ -161,16 +174,19 @@ class JvmtiFunctions { static jvmtiError StopThread(jvmtiEnv* env, jthread thread ATTRIBUTE_UNUSED, jobject exception ATTRIBUTE_UNUSED) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_signal_thread); return ERR(NOT_IMPLEMENTED); } static jvmtiError InterruptThread(jvmtiEnv* env, jthread thread ATTRIBUTE_UNUSED) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_signal_thread); return ERR(NOT_IMPLEMENTED); } static jvmtiError GetThreadInfo(jvmtiEnv* env, jthread thread, jvmtiThreadInfo* info_ptr) { + ENSURE_VALID_ENV(env); return ThreadUtil::GetThreadInfo(env, thread, info_ptr); } @@ -178,6 +194,7 @@ class JvmtiFunctions { jthread thread ATTRIBUTE_UNUSED, jint* owned_monitor_count_ptr ATTRIBUTE_UNUSED, jobject** owned_monitors_ptr ATTRIBUTE_UNUSED) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_get_owned_monitor_info); return ERR(NOT_IMPLEMENTED); } @@ -187,6 +204,7 @@ class JvmtiFunctions { jthread thread ATTRIBUTE_UNUSED, jint* monitor_info_count_ptr ATTRIBUTE_UNUSED, jvmtiMonitorStackDepthInfo** monitor_info_ptr ATTRIBUTE_UNUSED) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_get_owned_monitor_stack_depth_info); return ERR(NOT_IMPLEMENTED); } @@ -194,6 +212,7 @@ class JvmtiFunctions { static jvmtiError GetCurrentContendedMonitor(jvmtiEnv* env, jthread thread ATTRIBUTE_UNUSED, jobject* monitor_ptr ATTRIBUTE_UNUSED) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_get_current_contended_monitor); return ERR(NOT_IMPLEMENTED); } @@ -203,26 +222,31 @@ class JvmtiFunctions { jvmtiStartFunction proc, const void* arg, jint priority) { + ENSURE_VALID_ENV(env); return ThreadUtil::RunAgentThread(env, thread, proc, arg, priority); } static jvmtiError SetThreadLocalStorage(jvmtiEnv* env, jthread thread, const void* data) { + ENSURE_VALID_ENV(env); return ThreadUtil::SetThreadLocalStorage(env, thread, data); } static jvmtiError GetThreadLocalStorage(jvmtiEnv* env, jthread thread, void** data_ptr) { + ENSURE_VALID_ENV(env); return ThreadUtil::GetThreadLocalStorage(env, thread, data_ptr); } static jvmtiError GetTopThreadGroups(jvmtiEnv* env, jint* group_count_ptr, jthreadGroup** groups_ptr) { + ENSURE_VALID_ENV(env); return ThreadGroupUtil::GetTopThreadGroups(env, group_count_ptr, groups_ptr); } static jvmtiError GetThreadGroupInfo(jvmtiEnv* env, jthreadGroup group, jvmtiThreadGroupInfo* info_ptr) { + ENSURE_VALID_ENV(env); return ThreadGroupUtil::GetThreadGroupInfo(env, group, info_ptr); } @@ -232,6 +256,7 @@ class JvmtiFunctions { jthread** threads_ptr, jint* group_count_ptr, jthreadGroup** groups_ptr) { + ENSURE_VALID_ENV(env); return ThreadGroupUtil::GetThreadGroupChildren(env, group, thread_count_ptr, @@ -246,6 +271,7 @@ class JvmtiFunctions { jint max_frame_count, jvmtiFrameInfo* frame_buffer, jint* count_ptr) { + ENSURE_VALID_ENV(env); return StackUtil::GetStackTrace(env, thread, start_depth, @@ -258,6 +284,7 @@ class JvmtiFunctions { jint max_frame_count, jvmtiStackInfo** stack_info_ptr, jint* thread_count_ptr) { + ENSURE_VALID_ENV(env); return StackUtil::GetAllStackTraces(env, max_frame_count, stack_info_ptr, thread_count_ptr); } @@ -266,6 +293,7 @@ class JvmtiFunctions { const jthread* thread_list, jint max_frame_count, jvmtiStackInfo** stack_info_ptr) { + ENSURE_VALID_ENV(env); return StackUtil::GetThreadListStackTraces(env, thread_count, thread_list, @@ -274,10 +302,12 @@ class JvmtiFunctions { } static jvmtiError GetFrameCount(jvmtiEnv* env, jthread thread, jint* count_ptr) { + ENSURE_VALID_ENV(env); return StackUtil::GetFrameCount(env, thread, count_ptr); } static jvmtiError PopFrame(jvmtiEnv* env, jthread thread ATTRIBUTE_UNUSED) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_pop_frame); return ERR(NOT_IMPLEMENTED); } @@ -287,12 +317,14 @@ class JvmtiFunctions { jint depth, jmethodID* method_ptr, jlocation* location_ptr) { + ENSURE_VALID_ENV(env); return StackUtil::GetFrameLocation(env, thread, depth, method_ptr, location_ptr); } static jvmtiError NotifyFramePop(jvmtiEnv* env, jthread thread ATTRIBUTE_UNUSED, jint depth ATTRIBUTE_UNUSED) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_generate_frame_pop_events); return ERR(NOT_IMPLEMENTED); } @@ -300,6 +332,7 @@ class JvmtiFunctions { static jvmtiError ForceEarlyReturnObject(jvmtiEnv* env, jthread thread ATTRIBUTE_UNUSED, jobject value ATTRIBUTE_UNUSED) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_force_early_return); return ERR(NOT_IMPLEMENTED); } @@ -307,6 +340,7 @@ class JvmtiFunctions { static jvmtiError ForceEarlyReturnInt(jvmtiEnv* env, jthread thread ATTRIBUTE_UNUSED, jint value ATTRIBUTE_UNUSED) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_force_early_return); return ERR(NOT_IMPLEMENTED); } @@ -314,6 +348,7 @@ class JvmtiFunctions { static jvmtiError ForceEarlyReturnLong(jvmtiEnv* env, jthread thread ATTRIBUTE_UNUSED, jlong value ATTRIBUTE_UNUSED) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_force_early_return); return ERR(NOT_IMPLEMENTED); } @@ -321,6 +356,7 @@ class JvmtiFunctions { static jvmtiError ForceEarlyReturnFloat(jvmtiEnv* env, jthread thread ATTRIBUTE_UNUSED, jfloat value ATTRIBUTE_UNUSED) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_force_early_return); return ERR(NOT_IMPLEMENTED); } @@ -328,11 +364,13 @@ class JvmtiFunctions { static jvmtiError ForceEarlyReturnDouble(jvmtiEnv* env, jthread thread ATTRIBUTE_UNUSED, jdouble value ATTRIBUTE_UNUSED) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_force_early_return); return ERR(NOT_IMPLEMENTED); } static jvmtiError ForceEarlyReturnVoid(jvmtiEnv* env, jthread thread ATTRIBUTE_UNUSED) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_force_early_return); return ERR(NOT_IMPLEMENTED); } @@ -343,6 +381,7 @@ class JvmtiFunctions { jobject initial_object, const jvmtiHeapCallbacks* callbacks, const void* user_data) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_tag_objects); HeapUtil heap_util(ArtJvmTiEnv::AsArtJvmTiEnv(env)->object_tag_table.get()); return heap_util.FollowReferences(env, @@ -358,12 +397,14 @@ class JvmtiFunctions { jclass klass, const jvmtiHeapCallbacks* callbacks, const void* user_data) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_tag_objects); HeapUtil heap_util(ArtJvmTiEnv::AsArtJvmTiEnv(env)->object_tag_table.get()); return heap_util.IterateThroughHeap(env, heap_filter, klass, callbacks, user_data); } static jvmtiError GetTag(jvmtiEnv* env, jobject object, jlong* tag_ptr) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_tag_objects); JNIEnv* jni_env = GetJniEnv(env); @@ -381,6 +422,7 @@ class JvmtiFunctions { } static jvmtiError SetTag(jvmtiEnv* env, jobject object, jlong tag) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_tag_objects); if (object == nullptr) { @@ -405,6 +447,7 @@ class JvmtiFunctions { jint* count_ptr, jobject** object_result_ptr, jlong** tag_result_ptr) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_tag_objects); JNIEnv* jni_env = GetJniEnv(env); @@ -422,6 +465,7 @@ class JvmtiFunctions { } static jvmtiError ForceGarbageCollection(jvmtiEnv* env) { + ENSURE_VALID_ENV(env); return HeapUtil::ForceGarbageCollection(env); } @@ -430,6 +474,7 @@ class JvmtiFunctions { jobject object ATTRIBUTE_UNUSED, jvmtiObjectReferenceCallback object_reference_callback ATTRIBUTE_UNUSED, const void* user_data ATTRIBUTE_UNUSED) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_tag_objects); return ERR(NOT_IMPLEMENTED); } @@ -440,6 +485,7 @@ class JvmtiFunctions { jvmtiStackReferenceCallback stack_ref_callback ATTRIBUTE_UNUSED, jvmtiObjectReferenceCallback object_ref_callback ATTRIBUTE_UNUSED, const void* user_data ATTRIBUTE_UNUSED) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_tag_objects); return ERR(NOT_IMPLEMENTED); } @@ -448,6 +494,7 @@ class JvmtiFunctions { jvmtiHeapObjectFilter object_filter ATTRIBUTE_UNUSED, jvmtiHeapObjectCallback heap_object_callback ATTRIBUTE_UNUSED, const void* user_data ATTRIBUTE_UNUSED) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_tag_objects); return ERR(NOT_IMPLEMENTED); } @@ -458,6 +505,7 @@ class JvmtiFunctions { jvmtiHeapObjectFilter object_filter ATTRIBUTE_UNUSED, jvmtiHeapObjectCallback heap_object_callback ATTRIBUTE_UNUSED, const void* user_data ATTRIBUTE_UNUSED) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_tag_objects); return ERR(NOT_IMPLEMENTED); } @@ -467,6 +515,7 @@ class JvmtiFunctions { jint depth ATTRIBUTE_UNUSED, jint slot ATTRIBUTE_UNUSED, jobject* value_ptr ATTRIBUTE_UNUSED) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_access_local_variables); return ERR(NOT_IMPLEMENTED); } @@ -475,6 +524,7 @@ class JvmtiFunctions { jthread thread ATTRIBUTE_UNUSED, jint depth ATTRIBUTE_UNUSED, jobject* value_ptr ATTRIBUTE_UNUSED) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_access_local_variables); return ERR(NOT_IMPLEMENTED); } @@ -484,6 +534,7 @@ class JvmtiFunctions { jint depth ATTRIBUTE_UNUSED, jint slot ATTRIBUTE_UNUSED, jint* value_ptr ATTRIBUTE_UNUSED) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_access_local_variables); return ERR(NOT_IMPLEMENTED); } @@ -493,6 +544,7 @@ class JvmtiFunctions { jint depth ATTRIBUTE_UNUSED, jint slot ATTRIBUTE_UNUSED, jlong* value_ptr ATTRIBUTE_UNUSED) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_access_local_variables); return ERR(NOT_IMPLEMENTED); } @@ -502,6 +554,7 @@ class JvmtiFunctions { jint depth ATTRIBUTE_UNUSED, jint slot ATTRIBUTE_UNUSED, jfloat* value_ptr ATTRIBUTE_UNUSED) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_access_local_variables); return ERR(NOT_IMPLEMENTED); } @@ -511,6 +564,7 @@ class JvmtiFunctions { jint depth ATTRIBUTE_UNUSED, jint slot ATTRIBUTE_UNUSED, jdouble* value_ptr ATTRIBUTE_UNUSED) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_access_local_variables); return ERR(NOT_IMPLEMENTED); } @@ -520,6 +574,7 @@ class JvmtiFunctions { jint depth ATTRIBUTE_UNUSED, jint slot ATTRIBUTE_UNUSED, jobject value ATTRIBUTE_UNUSED) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_access_local_variables); return ERR(NOT_IMPLEMENTED); } @@ -529,6 +584,7 @@ class JvmtiFunctions { jint depth ATTRIBUTE_UNUSED, jint slot ATTRIBUTE_UNUSED, jint value ATTRIBUTE_UNUSED) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_access_local_variables); return ERR(NOT_IMPLEMENTED); } @@ -538,6 +594,7 @@ class JvmtiFunctions { jint depth ATTRIBUTE_UNUSED, jint slot ATTRIBUTE_UNUSED, jlong value ATTRIBUTE_UNUSED) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_access_local_variables); return ERR(NOT_IMPLEMENTED); } @@ -547,6 +604,7 @@ class JvmtiFunctions { jint depth ATTRIBUTE_UNUSED, jint slot ATTRIBUTE_UNUSED, jfloat value ATTRIBUTE_UNUSED) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_access_local_variables); return ERR(NOT_IMPLEMENTED); } @@ -556,6 +614,7 @@ class JvmtiFunctions { jint depth ATTRIBUTE_UNUSED, jint slot ATTRIBUTE_UNUSED, jdouble value ATTRIBUTE_UNUSED) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_access_local_variables); return ERR(NOT_IMPLEMENTED); } @@ -563,6 +622,7 @@ class JvmtiFunctions { static jvmtiError SetBreakpoint(jvmtiEnv* env, jmethodID method ATTRIBUTE_UNUSED, jlocation location ATTRIBUTE_UNUSED) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_generate_breakpoint_events); return ERR(NOT_IMPLEMENTED); } @@ -570,6 +630,7 @@ class JvmtiFunctions { static jvmtiError ClearBreakpoint(jvmtiEnv* env, jmethodID method ATTRIBUTE_UNUSED, jlocation location ATTRIBUTE_UNUSED) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_generate_breakpoint_events); return ERR(NOT_IMPLEMENTED); } @@ -577,6 +638,7 @@ class JvmtiFunctions { static jvmtiError SetFieldAccessWatch(jvmtiEnv* env, jclass klass ATTRIBUTE_UNUSED, jfieldID field ATTRIBUTE_UNUSED) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_generate_field_access_events); return ERR(NOT_IMPLEMENTED); } @@ -584,6 +646,7 @@ class JvmtiFunctions { static jvmtiError ClearFieldAccessWatch(jvmtiEnv* env, jclass klass ATTRIBUTE_UNUSED, jfieldID field ATTRIBUTE_UNUSED) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_generate_field_access_events); return ERR(NOT_IMPLEMENTED); } @@ -591,6 +654,7 @@ class JvmtiFunctions { static jvmtiError SetFieldModificationWatch(jvmtiEnv* env, jclass klass ATTRIBUTE_UNUSED, jfieldID field ATTRIBUTE_UNUSED) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_generate_field_modification_events); return ERR(NOT_IMPLEMENTED); } @@ -598,11 +662,13 @@ class JvmtiFunctions { static jvmtiError ClearFieldModificationWatch(jvmtiEnv* env, jclass klass ATTRIBUTE_UNUSED, jfieldID field ATTRIBUTE_UNUSED) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_generate_field_modification_events); return ERR(NOT_IMPLEMENTED); } static jvmtiError GetLoadedClasses(jvmtiEnv* env, jint* class_count_ptr, jclass** classes_ptr) { + ENSURE_VALID_ENV(env); HeapUtil heap_util(ArtJvmTiEnv::AsArtJvmTiEnv(env)->object_tag_table.get()); return heap_util.GetLoadedClasses(env, class_count_ptr, classes_ptr); } @@ -611,6 +677,7 @@ class JvmtiFunctions { jobject initiating_loader, jint* class_count_ptr, jclass** classes_ptr) { + ENSURE_VALID_ENV(env); return ClassUtil::GetClassLoaderClasses(env, initiating_loader, class_count_ptr, classes_ptr); } @@ -618,21 +685,25 @@ class JvmtiFunctions { jclass klass, char** signature_ptr, char** generic_ptr) { + ENSURE_VALID_ENV(env); return ClassUtil::GetClassSignature(env, klass, signature_ptr, generic_ptr); } static jvmtiError GetClassStatus(jvmtiEnv* env, jclass klass, jint* status_ptr) { + ENSURE_VALID_ENV(env); return ClassUtil::GetClassStatus(env, klass, status_ptr); } static jvmtiError GetSourceFileName(jvmtiEnv* env, jclass klass ATTRIBUTE_UNUSED, char** source_name_ptr ATTRIBUTE_UNUSED) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_get_source_file_name); return ERR(NOT_IMPLEMENTED); } static jvmtiError GetClassModifiers(jvmtiEnv* env, jclass klass, jint* modifiers_ptr) { + ENSURE_VALID_ENV(env); return ClassUtil::GetClassModifiers(env, klass, modifiers_ptr); } @@ -640,6 +711,7 @@ class JvmtiFunctions { jclass klass, jint* method_count_ptr, jmethodID** methods_ptr) { + ENSURE_VALID_ENV(env); return ClassUtil::GetClassMethods(env, klass, method_count_ptr, methods_ptr); } @@ -647,6 +719,7 @@ class JvmtiFunctions { jclass klass, jint* field_count_ptr, jfieldID** fields_ptr) { + ENSURE_VALID_ENV(env); return ClassUtil::GetClassFields(env, klass, field_count_ptr, fields_ptr); } @@ -654,6 +727,7 @@ class JvmtiFunctions { jclass klass, jint* interface_count_ptr, jclass** interfaces_ptr) { + ENSURE_VALID_ENV(env); return ClassUtil::GetImplementedInterfaces(env, klass, interface_count_ptr, interfaces_ptr); } @@ -661,6 +735,7 @@ class JvmtiFunctions { jclass klass, jint* minor_version_ptr, jint* major_version_ptr) { + ENSURE_VALID_ENV(env); return ClassUtil::GetClassVersionNumbers(env, klass, minor_version_ptr, major_version_ptr); } @@ -669,38 +744,45 @@ class JvmtiFunctions { jint* constant_pool_count_ptr ATTRIBUTE_UNUSED, jint* constant_pool_byte_count_ptr ATTRIBUTE_UNUSED, unsigned char** constant_pool_bytes_ptr ATTRIBUTE_UNUSED) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_get_constant_pool); return ERR(NOT_IMPLEMENTED); } static jvmtiError IsInterface(jvmtiEnv* env, jclass klass, jboolean* is_interface_ptr) { + ENSURE_VALID_ENV(env); return ClassUtil::IsInterface(env, klass, is_interface_ptr); } static jvmtiError IsArrayClass(jvmtiEnv* env, jclass klass, jboolean* is_array_class_ptr) { + ENSURE_VALID_ENV(env); return ClassUtil::IsArrayClass(env, klass, is_array_class_ptr); } static jvmtiError IsModifiableClass(jvmtiEnv* env, jclass klass, jboolean* is_modifiable_class_ptr) { + ENSURE_VALID_ENV(env); return Redefiner::IsModifiableClass(env, klass, is_modifiable_class_ptr); } static jvmtiError GetClassLoader(jvmtiEnv* env, jclass klass, jobject* classloader_ptr) { + ENSURE_VALID_ENV(env); return ClassUtil::GetClassLoader(env, klass, classloader_ptr); } static jvmtiError GetSourceDebugExtension(jvmtiEnv* env, jclass klass ATTRIBUTE_UNUSED, char** source_debug_extension_ptr ATTRIBUTE_UNUSED) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_get_source_debug_extension); return ERR(NOT_IMPLEMENTED); } static jvmtiError RetransformClasses(jvmtiEnv* env, jint class_count, const jclass* classes) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_retransform_classes); std::string error_msg; jvmtiError res = Transformer::RetransformClasses(ArtJvmTiEnv::AsArtJvmTiEnv(env), @@ -719,6 +801,7 @@ class JvmtiFunctions { static jvmtiError RedefineClasses(jvmtiEnv* env, jint class_count, const jvmtiClassDefinition* class_definitions) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_redefine_classes); std::string error_msg; jvmtiError res = Redefiner::RedefineClasses(ArtJvmTiEnv::AsArtJvmTiEnv(env), @@ -735,16 +818,19 @@ class JvmtiFunctions { } static jvmtiError GetObjectSize(jvmtiEnv* env, jobject object, jlong* size_ptr) { + ENSURE_VALID_ENV(env); return ObjectUtil::GetObjectSize(env, object, size_ptr); } static jvmtiError GetObjectHashCode(jvmtiEnv* env, jobject object, jint* hash_code_ptr) { + ENSURE_VALID_ENV(env); return ObjectUtil::GetObjectHashCode(env, object, hash_code_ptr); } static jvmtiError GetObjectMonitorUsage(jvmtiEnv* env, jobject object ATTRIBUTE_UNUSED, jvmtiMonitorUsage* info_ptr ATTRIBUTE_UNUSED) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_get_monitor_info); return ERR(NOT_IMPLEMENTED); } @@ -755,6 +841,7 @@ class JvmtiFunctions { char** name_ptr, char** signature_ptr, char** generic_ptr) { + ENSURE_VALID_ENV(env); return FieldUtil::GetFieldName(env, klass, field, name_ptr, signature_ptr, generic_ptr); } @@ -762,6 +849,7 @@ class JvmtiFunctions { jclass klass, jfieldID field, jclass* declaring_class_ptr) { + ENSURE_VALID_ENV(env); return FieldUtil::GetFieldDeclaringClass(env, klass, field, declaring_class_ptr); } @@ -769,6 +857,7 @@ class JvmtiFunctions { jclass klass, jfieldID field, jint* modifiers_ptr) { + ENSURE_VALID_ENV(env); return FieldUtil::GetFieldModifiers(env, klass, field, modifiers_ptr); } @@ -776,6 +865,7 @@ class JvmtiFunctions { jclass klass, jfieldID field, jboolean* is_synthetic_ptr) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_get_synthetic_attribute); return FieldUtil::IsFieldSynthetic(env, klass, field, is_synthetic_ptr); } @@ -785,30 +875,35 @@ class JvmtiFunctions { char** name_ptr, char** signature_ptr, char** generic_ptr) { + ENSURE_VALID_ENV(env); return MethodUtil::GetMethodName(env, method, name_ptr, signature_ptr, generic_ptr); } static jvmtiError GetMethodDeclaringClass(jvmtiEnv* env, jmethodID method, jclass* declaring_class_ptr) { + ENSURE_VALID_ENV(env); return MethodUtil::GetMethodDeclaringClass(env, method, declaring_class_ptr); } static jvmtiError GetMethodModifiers(jvmtiEnv* env, jmethodID method, jint* modifiers_ptr) { + ENSURE_VALID_ENV(env); return MethodUtil::GetMethodModifiers(env, method, modifiers_ptr); } static jvmtiError GetMaxLocals(jvmtiEnv* env, jmethodID method, jint* max_ptr) { + ENSURE_VALID_ENV(env); return MethodUtil::GetMaxLocals(env, method, max_ptr); } static jvmtiError GetArgumentsSize(jvmtiEnv* env, jmethodID method, jint* size_ptr) { + ENSURE_VALID_ENV(env); return MethodUtil::GetArgumentsSize(env, method, size_ptr); } @@ -816,6 +911,7 @@ class JvmtiFunctions { jmethodID method, jint* entry_count_ptr, jvmtiLineNumberEntry** table_ptr) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_get_line_numbers); return MethodUtil::GetLineNumberTable(env, method, entry_count_ptr, table_ptr); } @@ -824,6 +920,7 @@ class JvmtiFunctions { jmethodID method, jlocation* start_location_ptr, jlocation* end_location_ptr) { + ENSURE_VALID_ENV(env); return MethodUtil::GetMethodLocation(env, method, start_location_ptr, end_location_ptr); } @@ -831,6 +928,7 @@ class JvmtiFunctions { jmethodID method ATTRIBUTE_UNUSED, jint* entry_count_ptr ATTRIBUTE_UNUSED, jvmtiLocalVariableEntry** table_ptr ATTRIBUTE_UNUSED) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_access_local_variables); return ERR(NOT_IMPLEMENTED); } @@ -839,24 +937,29 @@ class JvmtiFunctions { jmethodID method ATTRIBUTE_UNUSED, jint* bytecode_count_ptr ATTRIBUTE_UNUSED, unsigned char** bytecodes_ptr ATTRIBUTE_UNUSED) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_get_bytecodes); return ERR(NOT_IMPLEMENTED); } static jvmtiError IsMethodNative(jvmtiEnv* env, jmethodID method, jboolean* is_native_ptr) { + ENSURE_VALID_ENV(env); return MethodUtil::IsMethodNative(env, method, is_native_ptr); } static jvmtiError IsMethodSynthetic(jvmtiEnv* env, jmethodID method, jboolean* is_synthetic_ptr) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_get_synthetic_attribute); return MethodUtil::IsMethodSynthetic(env, method, is_synthetic_ptr); } static jvmtiError IsMethodObsolete(jvmtiEnv* env, jmethodID method, jboolean* is_obsolete_ptr) { + ENSURE_VALID_ENV(env); return MethodUtil::IsMethodObsolete(env, method, is_obsolete_ptr); } static jvmtiError SetNativeMethodPrefix(jvmtiEnv* env, const char* prefix ATTRIBUTE_UNUSED) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_set_native_method_prefix); return ERR(NOT_IMPLEMENTED); } @@ -864,43 +967,53 @@ class JvmtiFunctions { static jvmtiError SetNativeMethodPrefixes(jvmtiEnv* env, jint prefix_count ATTRIBUTE_UNUSED, char** prefixes ATTRIBUTE_UNUSED) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_set_native_method_prefix); return ERR(NOT_IMPLEMENTED); } static jvmtiError CreateRawMonitor(jvmtiEnv* env, const char* name, jrawMonitorID* monitor_ptr) { + ENSURE_VALID_ENV(env); return MonitorUtil::CreateRawMonitor(env, name, monitor_ptr); } static jvmtiError DestroyRawMonitor(jvmtiEnv* env, jrawMonitorID monitor) { + ENSURE_VALID_ENV(env); return MonitorUtil::DestroyRawMonitor(env, monitor); } static jvmtiError RawMonitorEnter(jvmtiEnv* env, jrawMonitorID monitor) { + ENSURE_VALID_ENV(env); return MonitorUtil::RawMonitorEnter(env, monitor); } static jvmtiError RawMonitorExit(jvmtiEnv* env, jrawMonitorID monitor) { + ENSURE_VALID_ENV(env); return MonitorUtil::RawMonitorExit(env, monitor); } static jvmtiError RawMonitorWait(jvmtiEnv* env, jrawMonitorID monitor, jlong millis) { + ENSURE_VALID_ENV(env); return MonitorUtil::RawMonitorWait(env, monitor, millis); } static jvmtiError RawMonitorNotify(jvmtiEnv* env, jrawMonitorID monitor) { + ENSURE_VALID_ENV(env); return MonitorUtil::RawMonitorNotify(env, monitor); } static jvmtiError RawMonitorNotifyAll(jvmtiEnv* env, jrawMonitorID monitor) { + ENSURE_VALID_ENV(env); return MonitorUtil::RawMonitorNotifyAll(env, monitor); } static jvmtiError SetJNIFunctionTable(jvmtiEnv* env, const jniNativeInterface* function_table) { + ENSURE_VALID_ENV(env); return JNIUtil::SetJNIFunctionTable(env, function_table); } static jvmtiError GetJNIFunctionTable(jvmtiEnv* env, jniNativeInterface** function_table) { + ENSURE_VALID_ENV(env); return JNIUtil::GetJNIFunctionTable(env, function_table); } @@ -955,14 +1068,16 @@ class JvmtiFunctions { return gEventHandler.SetEvent(art_env, art_thread, GetArtJvmtiEvent(art_env, event_type), mode); } - static jvmtiError GenerateEvents(jvmtiEnv* env ATTRIBUTE_UNUSED, + static jvmtiError GenerateEvents(jvmtiEnv* env, jvmtiEvent event_type ATTRIBUTE_UNUSED) { + ENSURE_VALID_ENV(env); return OK; } - static jvmtiError GetExtensionFunctions(jvmtiEnv* env ATTRIBUTE_UNUSED, + static jvmtiError GetExtensionFunctions(jvmtiEnv* env, jint* extension_count_ptr, jvmtiExtensionFunctionInfo** extensions) { + ENSURE_VALID_ENV(env); // We do not have any extension functions. *extension_count_ptr = 0; *extensions = nullptr; @@ -970,9 +1085,10 @@ class JvmtiFunctions { return ERR(NONE); } - static jvmtiError GetExtensionEvents(jvmtiEnv* env ATTRIBUTE_UNUSED, + static jvmtiError GetExtensionEvents(jvmtiEnv* env, jint* extension_count_ptr, jvmtiExtensionEventInfo** extensions) { + ENSURE_VALID_ENV(env); // We do not have any extension events. *extension_count_ptr = 0; *extensions = nullptr; @@ -980,9 +1096,10 @@ class JvmtiFunctions { return ERR(NONE); } - static jvmtiError SetExtensionEventCallback(jvmtiEnv* env ATTRIBUTE_UNUSED, + static jvmtiError SetExtensionEventCallback(jvmtiEnv* env, jint extension_event_index ATTRIBUTE_UNUSED, jvmtiExtensionEvent callback ATTRIBUTE_UNUSED) { + ENSURE_VALID_ENV(env); // We do not have any extension events, so any call is illegal. return ERR(ILLEGAL_ARGUMENT); } @@ -999,8 +1116,8 @@ class JvmtiFunctions { ENSURE_NON_NULL(capabilities_ptr); ArtJvmTiEnv* art_env = static_cast<ArtJvmTiEnv*>(env); jvmtiError ret = OK; - jvmtiCapabilities changed; - jvmtiCapabilities potential_capabilities; + jvmtiCapabilities changed = {}; + jvmtiCapabilities potential_capabilities = {}; ret = env->GetPotentialCapabilities(&potential_capabilities); if (ret != OK) { return ret; @@ -1072,7 +1189,7 @@ class JvmtiFunctions { ENSURE_VALID_ENV(env); ENSURE_NON_NULL(capabilities_ptr); ArtJvmTiEnv* art_env = reinterpret_cast<ArtJvmTiEnv*>(env); - jvmtiCapabilities changed; + jvmtiCapabilities changed = {}; #define DEL_CAPABILITY(e) \ do { \ if (capabilities_ptr->e == 1) { \ @@ -1141,17 +1258,20 @@ class JvmtiFunctions { static jvmtiError GetCurrentThreadCpuTimerInfo(jvmtiEnv* env, jvmtiTimerInfo* info_ptr ATTRIBUTE_UNUSED) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_get_current_thread_cpu_time); return ERR(NOT_IMPLEMENTED); } static jvmtiError GetCurrentThreadCpuTime(jvmtiEnv* env, jlong* nanos_ptr ATTRIBUTE_UNUSED) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_get_current_thread_cpu_time); return ERR(NOT_IMPLEMENTED); } static jvmtiError GetThreadCpuTimerInfo(jvmtiEnv* env, jvmtiTimerInfo* info_ptr ATTRIBUTE_UNUSED) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_get_thread_cpu_time); return ERR(NOT_IMPLEMENTED); } @@ -1159,43 +1279,53 @@ class JvmtiFunctions { static jvmtiError GetThreadCpuTime(jvmtiEnv* env, jthread thread ATTRIBUTE_UNUSED, jlong* nanos_ptr ATTRIBUTE_UNUSED) { + ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_get_thread_cpu_time); return ERR(NOT_IMPLEMENTED); } static jvmtiError GetTimerInfo(jvmtiEnv* env, jvmtiTimerInfo* info_ptr) { + ENSURE_VALID_ENV(env); return TimerUtil::GetTimerInfo(env, info_ptr); } static jvmtiError GetTime(jvmtiEnv* env, jlong* nanos_ptr) { + ENSURE_VALID_ENV(env); return TimerUtil::GetTime(env, nanos_ptr); } static jvmtiError GetAvailableProcessors(jvmtiEnv* env, jint* processor_count_ptr) { + ENSURE_VALID_ENV(env); return TimerUtil::GetAvailableProcessors(env, processor_count_ptr); } static jvmtiError AddToBootstrapClassLoaderSearch(jvmtiEnv* env, const char* segment) { + ENSURE_VALID_ENV(env); return SearchUtil::AddToBootstrapClassLoaderSearch(env, segment); } static jvmtiError AddToSystemClassLoaderSearch(jvmtiEnv* env, const char* segment) { + ENSURE_VALID_ENV(env); return SearchUtil::AddToSystemClassLoaderSearch(env, segment); } static jvmtiError GetSystemProperties(jvmtiEnv* env, jint* count_ptr, char*** property_ptr) { + ENSURE_VALID_ENV(env); return PropertiesUtil::GetSystemProperties(env, count_ptr, property_ptr); } static jvmtiError GetSystemProperty(jvmtiEnv* env, const char* property, char** value_ptr) { + ENSURE_VALID_ENV(env); return PropertiesUtil::GetSystemProperty(env, property, value_ptr); } static jvmtiError SetSystemProperty(jvmtiEnv* env, const char* property, const char* value) { + ENSURE_VALID_ENV(env); return PropertiesUtil::SetSystemProperty(env, property, value); } static jvmtiError GetPhase(jvmtiEnv* env, jvmtiPhase* phase_ptr) { + ENSURE_VALID_ENV(env); return PhaseUtil::GetPhase(env, phase_ptr); } @@ -1303,9 +1433,10 @@ class JvmtiFunctions { } } - static jvmtiError SetVerboseFlag(jvmtiEnv* env ATTRIBUTE_UNUSED, + static jvmtiError SetVerboseFlag(jvmtiEnv* env, jvmtiVerboseFlag flag, jboolean value) { + ENSURE_VALID_ENV(env); if (flag == jvmtiVerboseFlag::JVMTI_VERBOSE_OTHER) { // OTHER is special, as it's 0, so can't do a bit check. bool val = (value == JNI_TRUE) ? true : false; @@ -1359,8 +1490,8 @@ class JvmtiFunctions { return ERR(NONE); } - static jvmtiError GetJLocationFormat(jvmtiEnv* env ATTRIBUTE_UNUSED, - jvmtiJlocationFormat* format_ptr) { + static jvmtiError GetJLocationFormat(jvmtiEnv* env, jvmtiJlocationFormat* format_ptr) { + ENSURE_VALID_ENV(env); // Report BCI as jlocation format. We report dex bytecode indices. if (format_ptr == nullptr) { return ERR(NULL_POINTER); @@ -1382,8 +1513,8 @@ extern const jvmtiInterface_1 gJvmtiInterface; ArtJvmTiEnv::ArtJvmTiEnv(art::JavaVMExt* runtime, EventHandler* event_handler) : art_vm(runtime), local_data(nullptr), - capabilities(), - object_tag_table(new ObjectTagTable(event_handler)) { + capabilities() { + object_tag_table = std::unique_ptr<ObjectTagTable>(new ObjectTagTable(event_handler, this)); functions = &gJvmtiInterface; } diff --git a/runtime/openjdkjvmti/events-inl.h b/runtime/openjdkjvmti/events-inl.h index 4f5eb0c33f..1ddbb869f9 100644 --- a/runtime/openjdkjvmti/events-inl.h +++ b/runtime/openjdkjvmti/events-inl.h @@ -126,6 +126,7 @@ inline void EventHandler::DispatchClassFileLoadHookEvent(art::Thread* thread, unsigned char** new_class_data) const { static_assert(kEvent == ArtJvmtiEvent::kClassFileLoadHookRetransformable || kEvent == ArtJvmtiEvent::kClassFileLoadHookNonRetransformable, "Unsupported event"); + DCHECK(*new_class_data == nullptr); jint current_len = class_data_len; unsigned char* current_class_data = const_cast<unsigned char*>(class_data); ArtJvmTiEnv* last_env = nullptr; @@ -168,15 +169,19 @@ inline void EventHandler::DispatchClassFileLoadHookEvent(art::Thread* thread, // exactly the argument types of the corresponding Jvmti kEvent function pointer. template <ArtJvmtiEvent kEvent, typename ...Args> -inline void EventHandler::DispatchEvent(art::Thread* thread, - Args... args) const { - using FnType = void(jvmtiEnv*, Args...); +inline void EventHandler::DispatchEvent(art::Thread* thread, Args... args) const { for (ArtJvmTiEnv* env : envs) { - if (ShouldDispatch<kEvent>(env, thread)) { - FnType* callback = impl::GetCallback<kEvent>(env); - if (callback != nullptr) { - (*callback)(env, args...); - } + DispatchEvent<kEvent, Args...>(env, thread, args...); + } +} + +template <ArtJvmtiEvent kEvent, typename ...Args> +inline void EventHandler::DispatchEvent(ArtJvmTiEnv* env, art::Thread* thread, Args... args) const { + using FnType = void(jvmtiEnv*, Args...); + if (ShouldDispatch<kEvent>(env, thread)) { + FnType* callback = impl::GetCallback<kEvent>(env); + if (callback != nullptr) { + (*callback)(env, args...); } } } diff --git a/runtime/openjdkjvmti/events.h b/runtime/openjdkjvmti/events.h index 4e20d1776b..ae8bf0f803 100644 --- a/runtime/openjdkjvmti/events.h +++ b/runtime/openjdkjvmti/events.h @@ -156,9 +156,14 @@ class EventHandler { ArtJvmtiEvent event, jvmtiEventMode mode); + // Dispatch event to all registered environments. template <ArtJvmtiEvent kEvent, typename ...Args> ALWAYS_INLINE inline void DispatchEvent(art::Thread* thread, Args... args) const; + // Dispatch event to the given environment, only. + template <ArtJvmtiEvent kEvent, typename ...Args> + ALWAYS_INLINE + inline void DispatchEvent(ArtJvmTiEnv* env, art::Thread* thread, Args... args) const; // Tell the event handler capabilities were added/lost so it can adjust the sent events.If // caps_added is true then caps is all the newly set capabilities of the jvmtiEnv. If it is false diff --git a/runtime/openjdkjvmti/fixed_up_dex_file.cc b/runtime/openjdkjvmti/fixed_up_dex_file.cc new file mode 100644 index 0000000000..3338358796 --- /dev/null +++ b/runtime/openjdkjvmti/fixed_up_dex_file.cc @@ -0,0 +1,145 @@ +/* Copyright (C) 2017 The Android Open Source Project + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This file implements interfaces from the file jvmti.h. This implementation + * is licensed under the same terms as the file jvmti.h. The + * copyright and license information for the file jvmti.h follows. + * + * Copyright (c) 2003, 2011, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "fixed_up_dex_file.h" +#include "dex_file-inl.h" + +// Compiler includes. +#include "dex/dex_to_dex_decompiler.h" + +// Runtime includes. +#include "oat_file.h" +#include "vdex_file.h" + +namespace openjdkjvmti { + +static void RecomputeDexChecksum(art::DexFile* dex_file) + REQUIRES_SHARED(art::Locks::mutator_lock_) { + reinterpret_cast<art::DexFile::Header*>(const_cast<uint8_t*>(dex_file->Begin()))->checksum_ = + dex_file->CalculateChecksum(); +} + +// TODO This is more complicated then it seems like it should be. +// The fact we don't keep around the data of where in the flat binary log of dex-quickening changes +// each dex file starts means we need to search for it. Since JVMTI is the exception though we are +// not going to put in the effort to optimize for it. +static void DoDexUnquicken(const art::DexFile& new_dex_file, + const art::DexFile& original_dex_file) + REQUIRES_SHARED(art::Locks::mutator_lock_) { + const art::OatDexFile* oat_dex = original_dex_file.GetOatDexFile(); + if (oat_dex == nullptr) { + return; + } + const art::OatFile* oat_file = oat_dex->GetOatFile(); + if (oat_file == nullptr) { + return; + } + const art::VdexFile* vdex = oat_file->GetVdexFile(); + if (vdex == nullptr || vdex->GetQuickeningInfo().size() == 0) { + return; + } + const art::ArrayRef<const uint8_t> quickening_info(vdex->GetQuickeningInfo()); + const uint8_t* quickening_info_ptr = quickening_info.data(); + for (const art::OatDexFile* cur_oat_dex : oat_file->GetOatDexFiles()) { + std::string error; + std::unique_ptr<const art::DexFile> cur_dex_file(cur_oat_dex->OpenDexFile(&error)); + DCHECK(cur_dex_file.get() != nullptr); + // Is this the dex file we are looking for? + if (UNLIKELY(cur_dex_file->Begin() == original_dex_file.Begin())) { + // Simple sanity check. + CHECK_EQ(new_dex_file.NumClassDefs(), original_dex_file.NumClassDefs()); + for (uint32_t i = 0; i < new_dex_file.NumClassDefs(); ++i) { + const art::DexFile::ClassDef& class_def = new_dex_file.GetClassDef(i); + const uint8_t* class_data = new_dex_file.GetClassData(class_def); + if (class_data == nullptr) { + continue; + } + for (art::ClassDataItemIterator it(new_dex_file, class_data); it.HasNext(); it.Next()) { + if (it.IsAtMethod() && it.GetMethodCodeItem() != nullptr) { + uint32_t quickening_size = *reinterpret_cast<const uint32_t*>(quickening_info_ptr); + quickening_info_ptr += sizeof(uint32_t); + art::optimizer::ArtDecompileDEX( + *it.GetMethodCodeItem(), + art::ArrayRef<const uint8_t>(quickening_info_ptr, quickening_size), + /*decompile_return_instruction*/true); + quickening_info_ptr += quickening_size; + } + } + } + // We don't need to bother looking through the rest of the dex-files. + break; + } else { + // Not the dex file we want. Skip over all the quickening info for all its classes. + for (uint32_t i = 0; i < cur_dex_file->NumClassDefs(); ++i) { + const art::DexFile::ClassDef& class_def = cur_dex_file->GetClassDef(i); + const uint8_t* class_data = cur_dex_file->GetClassData(class_def); + if (class_data == nullptr) { + continue; + } + for (art::ClassDataItemIterator it(*cur_dex_file, class_data); it.HasNext(); it.Next()) { + if (it.IsAtMethod() && it.GetMethodCodeItem() != nullptr) { + uint32_t quickening_size = *reinterpret_cast<const uint32_t*>(quickening_info_ptr); + quickening_info_ptr += sizeof(uint32_t); + quickening_info_ptr += quickening_size; + } + } + } + } + } +} + +std::unique_ptr<FixedUpDexFile> FixedUpDexFile::Create(const art::DexFile& original) { + // Copy the data into mutable memory. + std::vector<unsigned char> data; + data.resize(original.Size()); + memcpy(data.data(), original.Begin(), original.Size()); + std::string error; + std::unique_ptr<const art::DexFile> new_dex_file(art::DexFile::Open( + data.data(), + data.size(), + /*location*/"Unquickening_dexfile.dex", + /*location_checksum*/0, + /*oat_dex_file*/nullptr, + /*verify*/false, + /*verify_checksum*/false, + &error)); + if (new_dex_file.get() == nullptr) { + LOG(ERROR) << "Unable to open dex file from memory for unquickening! error: " << error; + return nullptr; + } + + DoDexUnquicken(*new_dex_file, original); + RecomputeDexChecksum(const_cast<art::DexFile*>(new_dex_file.get())); + std::unique_ptr<FixedUpDexFile> ret(new FixedUpDexFile(std::move(new_dex_file), std::move(data))); + return ret; +} + +} // namespace openjdkjvmti diff --git a/runtime/openjdkjvmti/fixed_up_dex_file.h b/runtime/openjdkjvmti/fixed_up_dex_file.h new file mode 100644 index 0000000000..db12f489e9 --- /dev/null +++ b/runtime/openjdkjvmti/fixed_up_dex_file.h @@ -0,0 +1,82 @@ +/* Copyright (C) 2017 The Android Open Source Project + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This file implements interfaces from the file jvmti.h. This implementation + * is licensed under the same terms as the file jvmti.h. The + * copyright and license information for the file jvmti.h follows. + * + * Copyright (c) 2003, 2011, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef ART_RUNTIME_OPENJDKJVMTI_FIXED_UP_DEX_FILE_H_ +#define ART_RUNTIME_OPENJDKJVMTI_FIXED_UP_DEX_FILE_H_ + +#include <memory> +#include <vector> + +#include "jni.h" +#include "jvmti.h" +#include "base/mutex.h" +#include "dex_file.h" + +namespace openjdkjvmti { + +// A holder for a DexFile that has been 'fixed up' to ensure it is fully compliant with the +// published standard (no internal/quick opcodes, all fields are the defined values, etc). This is +// used to ensure that agents get a consistent dex file regardless of what version of android they +// are running on. +class FixedUpDexFile { + public: + static std::unique_ptr<FixedUpDexFile> Create(const art::DexFile& original) + REQUIRES_SHARED(art::Locks::mutator_lock_); + + const art::DexFile& GetDexFile() { + return *dex_file_; + } + + const unsigned char* Begin() { + return data_.data(); + } + + size_t Size() { + return data_.size(); + } + + private: + explicit FixedUpDexFile(std::unique_ptr<const art::DexFile> fixed_up_dex_file, + std::vector<unsigned char> data) + : dex_file_(std::move(fixed_up_dex_file)), + data_(std::move(data)) {} + + // the fixed up DexFile + std::unique_ptr<const art::DexFile> dex_file_; + // The backing data for dex_file_. + const std::vector<unsigned char> data_; + + DISALLOW_COPY_AND_ASSIGN(FixedUpDexFile); +}; + +} // namespace openjdkjvmti + +#endif // ART_RUNTIME_OPENJDKJVMTI_FIXED_UP_DEX_FILE_H_ diff --git a/runtime/openjdkjvmti/jvmti_weak_table.h b/runtime/openjdkjvmti/jvmti_weak_table.h index ae3612208c..eeea75aa9d 100644 --- a/runtime/openjdkjvmti/jvmti_weak_table.h +++ b/runtime/openjdkjvmti/jvmti_weak_table.h @@ -53,7 +53,7 @@ template <typename T> class JvmtiWeakTable : public art::gc::SystemWeakHolder { public: JvmtiWeakTable() - : art::gc::SystemWeakHolder(kTaggingLockLevel), + : art::gc::SystemWeakHolder(art::kTaggingLockLevel), update_since_last_sweep_(false) { } @@ -200,10 +200,6 @@ class JvmtiWeakTable : public art::gc::SystemWeakHolder { } }; - // The tag table is used when visiting roots. So it needs to have a low lock level. - static constexpr art::LockLevel kTaggingLockLevel = - static_cast<art::LockLevel>(art::LockLevel::kAbortLock + 1); - std::unordered_map<art::GcRoot<art::mirror::Object>, T, HashGcRoot, diff --git a/runtime/openjdkjvmti/object_tagging.cc b/runtime/openjdkjvmti/object_tagging.cc index 4215588a09..dcdd3ede13 100644 --- a/runtime/openjdkjvmti/object_tagging.cc +++ b/runtime/openjdkjvmti/object_tagging.cc @@ -33,6 +33,7 @@ #include <limits> +#include "art_jvmti.h" #include "events-inl.h" #include "jvmti_weak_table-inl.h" @@ -60,7 +61,7 @@ bool ObjectTagTable::DoesHandleNullOnSweep() { return event_handler_->IsEventEnabledAnywhere(ArtJvmtiEvent::kObjectFree); } void ObjectTagTable::HandleNullSweep(jlong tag) { - event_handler_->DispatchEvent<ArtJvmtiEvent::kObjectFree>(nullptr, tag); + event_handler_->DispatchEvent<ArtJvmtiEvent::kObjectFree>(jvmti_env_, nullptr, tag); } } // namespace openjdkjvmti diff --git a/runtime/openjdkjvmti/object_tagging.h b/runtime/openjdkjvmti/object_tagging.h index b5a601cac9..ca84e442dc 100644 --- a/runtime/openjdkjvmti/object_tagging.h +++ b/runtime/openjdkjvmti/object_tagging.h @@ -42,12 +42,13 @@ namespace openjdkjvmti { +struct ArtJvmTiEnv; class EventHandler; class ObjectTagTable FINAL : public JvmtiWeakTable<jlong> { public: - explicit ObjectTagTable(EventHandler* event_handler) : event_handler_(event_handler) { - } + ObjectTagTable(EventHandler* event_handler, ArtJvmTiEnv* env) + : event_handler_(event_handler), jvmti_env_(env) {} bool Set(art::mirror::Object* obj, jlong tag) OVERRIDE REQUIRES_SHARED(art::Locks::mutator_lock_) @@ -77,6 +78,7 @@ class ObjectTagTable FINAL : public JvmtiWeakTable<jlong> { private: EventHandler* event_handler_; + ArtJvmTiEnv* jvmti_env_; }; } // namespace openjdkjvmti diff --git a/runtime/openjdkjvmti/ti_class.cc b/runtime/openjdkjvmti/ti_class.cc index 38fd1d4af3..e94c4e6112 100644 --- a/runtime/openjdkjvmti/ti_class.cc +++ b/runtime/openjdkjvmti/ti_class.cc @@ -43,6 +43,7 @@ #include "common_throws.h" #include "dex_file_annotations.h" #include "events-inl.h" +#include "fixed_up_dex_file.h" #include "gc/heap.h" #include "gc_root.h" #include "handle.h" @@ -55,6 +56,8 @@ #include "mirror/object_reference.h" #include "mirror/object-inl.h" #include "mirror/reference.h" +#include "primitive.h" +#include "reflection.h" #include "runtime.h" #include "runtime_callbacks.h" #include "ScopedLocalRef.h" @@ -78,9 +81,9 @@ static std::unique_ptr<const art::DexFile> MakeSingleDexFile(art::Thread* self, REQUIRES_SHARED(art::Locks::mutator_lock_) { // Make the mmap std::string error_msg; + art::ArraySlice<const unsigned char> final_data(final_dex_data, final_len); std::unique_ptr<art::MemMap> map(Redefiner::MoveDataToMemMap(orig_location, - final_len, - final_dex_data, + final_data, &error_msg)); if (map.get() == nullptr) { LOG(WARNING) << "Unable to allocate mmap for redefined dex file! Error was: " << error_msg; @@ -161,6 +164,8 @@ struct ClassCallback : public art::ClassLoadCallback { art::JNIEnvExt* env = self->GetJniEnv(); ScopedLocalRef<jobject> loader( env, class_loader.IsNull() ? nullptr : env->AddLocalReference<jobject>(class_loader.Get())); + std::unique_ptr<FixedUpDexFile> dex_file_copy(FixedUpDexFile::Create(initial_dex_file)); + // Go back to native. art::ScopedThreadSuspension sts(self, art::ThreadState::kNative); // Call all Non-retransformable agents. @@ -174,14 +179,14 @@ struct ClassCallback : public art::ClassLoadCallback { loader.get(), name.c_str(), static_cast<jobject>(nullptr), // Android doesn't seem to have protection domains - static_cast<jint>(initial_dex_file.Size()), - static_cast<const unsigned char*>(initial_dex_file.Begin()), + static_cast<jint>(dex_file_copy->Size()), + static_cast<const unsigned char*>(dex_file_copy->Begin()), static_cast<jint*>(&post_no_redefine_len), static_cast<unsigned char**>(&post_no_redefine_dex_data)); if (post_no_redefine_dex_data == nullptr) { DCHECK_EQ(post_no_redefine_len, 0); - post_no_redefine_dex_data = const_cast<unsigned char*>(initial_dex_file.Begin()); - post_no_redefine_len = initial_dex_file.Size(); + post_no_redefine_dex_data = const_cast<unsigned char*>(dex_file_copy->Begin()); + post_no_redefine_len = dex_file_copy->Size(); } else { post_no_redefine_unique_ptr = std::unique_ptr<const unsigned char>(post_no_redefine_dex_data); DCHECK_GT(post_no_redefine_len, 0); @@ -210,7 +215,7 @@ struct ClassCallback : public art::ClassLoadCallback { DCHECK_GT(final_len, 0); } - if (final_dex_data != initial_dex_file.Begin()) { + if (final_dex_data != dex_file_copy->Begin()) { LOG(WARNING) << "Changing class " << descriptor; art::ScopedObjectAccess soa(self); art::StackHandleScope<2> hs(self); @@ -228,14 +233,22 @@ struct ClassCallback : public art::ClassLoadCallback { } // Allocate the byte array to store the dex file bytes in. - art::Handle<art::mirror::ByteArray> arr(hs.NewHandle( - art::mirror::ByteArray::AllocateAndFill( - self, - reinterpret_cast<const signed char*>(post_no_redefine_dex_data), - post_no_redefine_len))); + art::MutableHandle<art::mirror::Object> arr(hs.NewHandle<art::mirror::Object>(nullptr)); + if (post_no_redefine_dex_data == dex_file_copy->Begin() && name != "java/lang/Long") { + // we didn't have any non-retransformable agents. We can just cache a pointer to the + // initial_dex_file. It will be kept live by the class_loader. + jlong dex_ptr = reinterpret_cast<uintptr_t>(&initial_dex_file); + art::JValue val; + val.SetJ(dex_ptr); + arr.Assign(art::BoxPrimitive(art::Primitive::kPrimLong, val)); + } else { + arr.Assign(art::mirror::ByteArray::AllocateAndFill( + self, + reinterpret_cast<const signed char*>(post_no_redefine_dex_data), + post_no_redefine_len)); + } if (arr.IsNull()) { - LOG(WARNING) << "Unable to allocate byte array for initial dex-file bytes. Aborting " - << "transformation"; + LOG(WARNING) << "Unable to allocate memory for initial dex-file. Aborting transformation"; self->AssertPendingOOMException(); return; } diff --git a/runtime/openjdkjvmti/ti_class_definition.cc b/runtime/openjdkjvmti/ti_class_definition.cc index 2c2a79bc58..153692b2fe 100644 --- a/runtime/openjdkjvmti/ti_class_definition.cc +++ b/runtime/openjdkjvmti/ti_class_definition.cc @@ -31,25 +31,145 @@ #include "ti_class_definition.h" +#include "base/array_slice.h" #include "dex_file.h" +#include "fixed_up_dex_file.h" #include "handle_scope-inl.h" #include "handle.h" #include "mirror/class-inl.h" #include "mirror/object-inl.h" +#include "reflection.h" #include "thread.h" namespace openjdkjvmti { -bool ArtClassDefinition::IsModified(art::Thread* self) const { - if (modified) { +bool ArtClassDefinition::IsModified() const { + // RedefineClasses calls always are 'modified' since they need to change the original_dex_file of + // the class. + if (redefined_) { return true; } // Check if the dex file we want to set is the same as the current one. + // Unfortunately we need to do this check even if no modifications have been done since it could + // be that agents were removed in the mean-time so we still have a different dex file. The dex + // checksum means this is likely to be fairly fast. + return static_cast<jint>(original_dex_file_.size()) != dex_len_ || + memcmp(&original_dex_file_.At(0), dex_data_.get(), dex_len_) != 0; +} + +jvmtiError ArtClassDefinition::InitCommon(ArtJvmTiEnv* env, jclass klass) { + JNIEnv* jni_env = GetJniEnv(env); + if (jni_env == nullptr) { + return ERR(INTERNAL); + } + art::ScopedObjectAccess soa(jni_env); + art::ObjPtr<art::mirror::Class> m_klass(soa.Decode<art::mirror::Class>(klass)); + if (m_klass.IsNull()) { + return ERR(INVALID_CLASS); + } + klass_ = klass; + loader_ = soa.AddLocalReference<jobject>(m_klass->GetClassLoader()); + std::string descriptor_store; + std::string descriptor(m_klass->GetDescriptor(&descriptor_store)); + name_ = descriptor.substr(1, descriptor.size() - 2); + // Android doesn't really have protection domains. + protection_domain_ = nullptr; + return OK; +} + +// Gets the data surrounding the given class. +static jvmtiError GetDexDataForRetransformation(ArtJvmTiEnv* env, + art::Handle<art::mirror::Class> klass, + /*out*/jint* dex_data_len, + /*out*/unsigned char** dex_data) + REQUIRES_SHARED(art::Locks::mutator_lock_) { + art::StackHandleScope<3> hs(art::Thread::Current()); + art::Handle<art::mirror::ClassExt> ext(hs.NewHandle(klass->GetExtData())); + const art::DexFile* dex_file = nullptr; + if (!ext.IsNull()) { + art::Handle<art::mirror::Object> orig_dex(hs.NewHandle(ext->GetOriginalDexFile())); + if (!orig_dex.IsNull()) { + if (orig_dex->IsArrayInstance()) { + DCHECK(orig_dex->GetClass()->GetComponentType()->IsPrimitiveByte()); + art::Handle<art::mirror::ByteArray> orig_dex_bytes( + hs.NewHandle(art::down_cast<art::mirror::ByteArray*>(orig_dex->AsArray()))); + *dex_data_len = static_cast<jint>(orig_dex_bytes->GetLength()); + return CopyDataIntoJvmtiBuffer( + env, + reinterpret_cast<const unsigned char*>(orig_dex_bytes->GetData()), + *dex_data_len, + /*out*/dex_data); + } else if (orig_dex->IsDexCache()) { + dex_file = orig_dex->AsDexCache()->GetDexFile(); + } else { + DCHECK_EQ(orig_dex->GetClass()->GetPrimitiveType(), art::Primitive::kPrimLong); + art::ObjPtr<art::mirror::Class> prim_long_class( + art::Runtime::Current()->GetClassLinker()->GetClassRoot( + art::ClassLinker::kPrimitiveLong)); + art::JValue val; + if (!art::UnboxPrimitiveForResult(orig_dex.Get(), prim_long_class, &val)) { + // This should never happen. + return ERR(INTERNAL); + } + dex_file = reinterpret_cast<const art::DexFile*>(static_cast<uintptr_t>(val.GetJ())); + } + } + } + if (dex_file == nullptr) { + dex_file = &klass->GetDexFile(); + } + std::unique_ptr<FixedUpDexFile> fixed_dex_file(FixedUpDexFile::Create(*dex_file)); + *dex_data_len = static_cast<jint>(fixed_dex_file->Size()); + return CopyDataIntoJvmtiBuffer(env, + fixed_dex_file->Begin(), + fixed_dex_file->Size(), + /*out*/dex_data); +} + +jvmtiError ArtClassDefinition::Init(ArtJvmTiEnv* env, jclass klass) { + jvmtiError res = InitCommon(env, klass); + if (res != OK) { + return res; + } + unsigned char* new_data = nullptr; + art::Thread* self = art::Thread::Current(); + art::ScopedObjectAccess soa(self); art::StackHandleScope<1> hs(self); - art::Handle<art::mirror::Class> h_klass(hs.NewHandle(self->DecodeJObject(klass)->AsClass())); - const art::DexFile& cur_dex_file = h_klass->GetDexFile(); - return static_cast<jint>(cur_dex_file.Size()) != dex_len || - memcmp(cur_dex_file.Begin(), dex_data.get(), dex_len) != 0; + art::Handle<art::mirror::Class> m_klass(hs.NewHandle(self->DecodeJObject(klass)->AsClass())); + res = GetDexDataForRetransformation(env, m_klass, &dex_len_, &new_data); + if (res != OK) { + return res; + } + dex_data_ = MakeJvmtiUniquePtr(env, new_data); + if (m_klass->GetExtData() == nullptr || m_klass->GetExtData()->GetOriginalDexFile() == nullptr) { + // We have never redefined class this yet. Keep track of what the (de-quickened) dex file looks + // like so we can tell if anything has changed. Really we would like to just always do the + // 'else' block but the fact that we de-quickened stuff screws us over. + unsigned char* original_data_memory = nullptr; + res = CopyDataIntoJvmtiBuffer(env, dex_data_.get(), dex_len_, &original_data_memory); + original_dex_file_memory_ = MakeJvmtiUniquePtr(env, original_data_memory); + original_dex_file_ = art::ArraySlice<const unsigned char>(original_data_memory, dex_len_); + } else { + // We know that we have been redefined at least once (there is an original_dex_file set in + // the class) so we can just use the current dex file directly. + const art::DexFile& dex_file = m_klass->GetDexFile(); + original_dex_file_ = art::ArraySlice<const unsigned char>(dex_file.Begin(), dex_file.Size()); + } + return res; +} + +jvmtiError ArtClassDefinition::Init(ArtJvmTiEnv* env, const jvmtiClassDefinition& def) { + jvmtiError res = InitCommon(env, def.klass); + if (res != OK) { + return res; + } + unsigned char* new_data = nullptr; + original_dex_file_ = art::ArraySlice<const unsigned char>(def.class_bytes, def.class_byte_count); + redefined_ = true; + dex_len_ = def.class_byte_count; + res = CopyDataIntoJvmtiBuffer(env, def.class_bytes, def.class_byte_count, /*out*/ &new_data); + dex_data_ = MakeJvmtiUniquePtr(env, new_data); + return res; } } // namespace openjdkjvmti diff --git a/runtime/openjdkjvmti/ti_class_definition.h b/runtime/openjdkjvmti/ti_class_definition.h index 3c251d4b44..43d0c3fc62 100644 --- a/runtime/openjdkjvmti/ti_class_definition.h +++ b/runtime/openjdkjvmti/ti_class_definition.h @@ -39,37 +39,89 @@ namespace openjdkjvmti { // A struct that stores data needed for redefining/transforming classes. This structure should only // even be accessed from a single thread and must not survive past the completion of the // redefinition/retransformation function that created it. -struct ArtClassDefinition { +class ArtClassDefinition { public: - jclass klass; - jobject loader; - std::string name; - jobject protection_domain; - jint dex_len; - JvmtiUniquePtr<unsigned char> dex_data; - art::ArraySlice<const unsigned char> original_dex_file; - - ArtClassDefinition() = default; + ArtClassDefinition() + : klass_(nullptr), + loader_(nullptr), + name_(), + protection_domain_(nullptr), + dex_len_(0), + dex_data_(nullptr), + original_dex_file_memory_(nullptr), + original_dex_file_(), + redefined_(false) {} + + jvmtiError Init(ArtJvmTiEnv* env, jclass klass); + jvmtiError Init(ArtJvmTiEnv* env, const jvmtiClassDefinition& def); + ArtClassDefinition(ArtClassDefinition&& o) = default; + ArtClassDefinition& operator=(ArtClassDefinition&& o) = default; void SetNewDexData(ArtJvmTiEnv* env, jint new_dex_len, unsigned char* new_dex_data) { + DCHECK(IsInitialized()); if (new_dex_data == nullptr) { return; - } else if (new_dex_data != dex_data.get() || new_dex_len != dex_len) { - SetModified(); - dex_len = new_dex_len; - dex_data = MakeJvmtiUniquePtr(env, new_dex_data); + } else if (new_dex_data != dex_data_.get() || new_dex_len != dex_len_) { + dex_len_ = new_dex_len; + dex_data_ = MakeJvmtiUniquePtr(env, new_dex_data); } } - void SetModified() { - modified = true; + art::ArraySlice<const unsigned char> GetNewOriginalDexFile() const { + DCHECK(IsInitialized()); + if (redefined_) { + return original_dex_file_; + } else { + return art::ArraySlice<const unsigned char>(); + } } - bool IsModified(art::Thread* self) const REQUIRES_SHARED(art::Locks::mutator_lock_); + bool IsModified() const; + + bool IsInitialized() const { + return klass_ != nullptr; + } + + jclass GetClass() const { + DCHECK(IsInitialized()); + return klass_; + } + + jobject GetLoader() const { + DCHECK(IsInitialized()); + return loader_; + } + + const std::string& GetName() const { + DCHECK(IsInitialized()); + return name_; + } + + jobject GetProtectionDomain() const { + DCHECK(IsInitialized()); + return protection_domain_; + } + + art::ArraySlice<const unsigned char> GetDexData() const { + DCHECK(IsInitialized()); + return art::ArraySlice<const unsigned char>(dex_data_.get(), dex_len_); + } private: - bool modified; + jvmtiError InitCommon(ArtJvmTiEnv* env, jclass klass); + + jclass klass_; + jobject loader_; + std::string name_; + jobject protection_domain_; + jint dex_len_; + JvmtiUniquePtr<unsigned char> dex_data_; + JvmtiUniquePtr<unsigned char> original_dex_file_memory_; + art::ArraySlice<const unsigned char> original_dex_file_; + bool redefined_; + + DISALLOW_COPY_AND_ASSIGN(ArtClassDefinition); }; } // namespace openjdkjvmti diff --git a/runtime/openjdkjvmti/ti_heap.cc b/runtime/openjdkjvmti/ti_heap.cc index c2495e3a6b..49d9aca46e 100644 --- a/runtime/openjdkjvmti/ti_heap.cc +++ b/runtime/openjdkjvmti/ti_heap.cc @@ -882,12 +882,17 @@ class FollowReferencesHelper FINAL { void AddRoot(art::mirror::Object* root_obj, const art::RootInfo& info) REQUIRES_SHARED(art::Locks::mutator_lock_) REQUIRES(!*tag_table_->GetAllowDisallowLock()) { + if (stop_reports_) { + return; + } + bool add_to_worklist = ReportRoot(root_obj, info); // We use visited_ to mark roots already so we do not need another set. if (visited_->find(root_obj) == visited_->end()) { visited_->insert(root_obj); - worklist_->push_back(root_obj); + if (add_to_worklist) { + worklist_->push_back(root_obj); + } } - ReportRoot(root_obj, info); } // Remove NO_THREAD_SAFETY_ANALYSIS once ASSERT_CAPABILITY works correctly. @@ -993,7 +998,7 @@ class FollowReferencesHelper FINAL { UNREACHABLE(); } - void ReportRoot(art::mirror::Object* root_obj, const art::RootInfo& info) + bool ReportRoot(art::mirror::Object* root_obj, const art::RootInfo& info) REQUIRES_SHARED(art::Locks::mutator_lock_) REQUIRES(!*tag_table_->GetAllowDisallowLock()) { jvmtiHeapReferenceInfo ref_info; @@ -1002,6 +1007,7 @@ class FollowReferencesHelper FINAL { if ((result & JVMTI_VISIT_ABORT) != 0) { stop_reports_ = true; } + return (result & JVMTI_VISIT_OBJECTS) != 0; } private: diff --git a/runtime/openjdkjvmti/ti_phase.cc b/runtime/openjdkjvmti/ti_phase.cc index e494cb6530..941cf7b73b 100644 --- a/runtime/openjdkjvmti/ti_phase.cc +++ b/runtime/openjdkjvmti/ti_phase.cc @@ -40,6 +40,7 @@ #include "scoped_thread_state_change-inl.h" #include "thread-inl.h" #include "thread_list.h" +#include "ti_thread.h" namespace openjdkjvmti { @@ -69,6 +70,7 @@ struct PhaseUtil::PhaseCallback : public art::RuntimePhaseCallback { break; case RuntimePhase::kInit: { + ThreadUtil::CacheData(); ScopedLocalRef<jthread> thread(GetJniEnv(), GetCurrentJThread()); art::ScopedThreadSuspension sts(art::Thread::Current(), art::ThreadState::kNative); event_handler->DispatchEvent<ArtJvmtiEvent::kVmInit>(nullptr, GetJniEnv(), thread.get()); @@ -105,6 +107,16 @@ jvmtiError PhaseUtil::GetPhase(jvmtiEnv* env ATTRIBUTE_UNUSED, jvmtiPhase* phase return ERR(NONE); } +bool PhaseUtil::IsLivePhase() { + jvmtiPhase now = PhaseUtil::current_phase_; + DCHECK(now == JVMTI_PHASE_ONLOAD || + now == JVMTI_PHASE_PRIMORDIAL || + now == JVMTI_PHASE_START || + now == JVMTI_PHASE_LIVE || + now == JVMTI_PHASE_DEAD); + return now == JVMTI_PHASE_LIVE; +} + void PhaseUtil::SetToOnLoad() { DCHECK_EQ(0u, static_cast<size_t>(PhaseUtil::current_phase_)); PhaseUtil::current_phase_ = JVMTI_PHASE_ONLOAD; @@ -117,6 +129,7 @@ void PhaseUtil::SetToPrimordial() { void PhaseUtil::SetToLive() { DCHECK_EQ(static_cast<size_t>(0), static_cast<size_t>(PhaseUtil::current_phase_)); + ThreadUtil::CacheData(); PhaseUtil::current_phase_ = JVMTI_PHASE_LIVE; } diff --git a/runtime/openjdkjvmti/ti_phase.h b/runtime/openjdkjvmti/ti_phase.h index 851fc27de5..a2c0d114ef 100644 --- a/runtime/openjdkjvmti/ti_phase.h +++ b/runtime/openjdkjvmti/ti_phase.h @@ -42,6 +42,7 @@ class EventHandler; class PhaseUtil { public: static jvmtiError GetPhase(jvmtiEnv* env, jvmtiPhase* phase_ptr); + static bool IsLivePhase(); static void Register(EventHandler* event_handler); static void Unregister(); diff --git a/runtime/openjdkjvmti/ti_redefine.cc b/runtime/openjdkjvmti/ti_redefine.cc index 7faddfb0f9..6e0d9a0be8 100644 --- a/runtime/openjdkjvmti/ti_redefine.cc +++ b/runtime/openjdkjvmti/ti_redefine.cc @@ -261,13 +261,12 @@ jvmtiError Redefiner::GetClassRedefinitionError(art::Handle<art::mirror::Class> // Moves dex data to an anonymous, read-only mmap'd region. std::unique_ptr<art::MemMap> Redefiner::MoveDataToMemMap(const std::string& original_location, - jint data_len, - const unsigned char* dex_data, + art::ArraySlice<const unsigned char> data, std::string* error_msg) { std::unique_ptr<art::MemMap> map(art::MemMap::MapAnonymous( StringPrintf("%s-transformed", original_location.c_str()).c_str(), nullptr, - data_len, + data.size(), PROT_READ|PROT_WRITE, /*low_4gb*/false, /*reuse*/false, @@ -275,7 +274,7 @@ std::unique_ptr<art::MemMap> Redefiner::MoveDataToMemMap(const std::string& orig if (map == nullptr) { return map; } - memcpy(map->Begin(), dex_data, data_len); + memcpy(map->Begin(), &data.At(0), data.size()); // Make the dex files mmap read only. This matches how other DexFiles are mmaped and prevents // programs from corrupting it. map->Protect(PROT_READ); @@ -344,13 +343,7 @@ jvmtiError Redefiner::RedefineClasses(ArtJvmTiEnv* env, memcpy(class_bytes_copy, definitions[i].class_bytes, definitions[i].class_byte_count); ArtClassDefinition def; - def.dex_len = definitions[i].class_byte_count; - def.dex_data = MakeJvmtiUniquePtr(env, class_bytes_copy); - // We are definitely modified. - def.SetModified(); - def.original_dex_file = art::ArraySlice<const unsigned char>(definitions[i].class_bytes, - definitions[i].class_byte_count); - res = Transformer::FillInTransformationData(env, definitions[i].klass, &def); + res = def.Init(env, definitions[i]); if (res != OK) { return res; } @@ -386,7 +379,7 @@ jvmtiError Redefiner::RedefineClassesDirect(ArtJvmTiEnv* env, Redefiner r(runtime, self, error_msg); for (const ArtClassDefinition& def : definitions) { // Only try to transform classes that have been modified. - if (def.IsModified(self)) { + if (def.IsModified()) { jvmtiError res = r.AddRedefinition(env, def); if (res != OK) { return res; @@ -399,25 +392,24 @@ jvmtiError Redefiner::RedefineClassesDirect(ArtJvmTiEnv* env, jvmtiError Redefiner::AddRedefinition(ArtJvmTiEnv* env, const ArtClassDefinition& def) { std::string original_dex_location; jvmtiError ret = OK; - if ((ret = GetClassLocation(env, def.klass, &original_dex_location))) { + if ((ret = GetClassLocation(env, def.GetClass(), &original_dex_location))) { *error_msg_ = "Unable to get original dex file location!"; return ret; } char* generic_ptr_unused = nullptr; char* signature_ptr = nullptr; - if ((ret = env->GetClassSignature(def.klass, &signature_ptr, &generic_ptr_unused)) != OK) { + if ((ret = env->GetClassSignature(def.GetClass(), &signature_ptr, &generic_ptr_unused)) != OK) { *error_msg_ = "Unable to get class signature!"; return ret; } JvmtiUniquePtr<char> generic_unique_ptr(MakeJvmtiUniquePtr(env, generic_ptr_unused)); JvmtiUniquePtr<char> signature_unique_ptr(MakeJvmtiUniquePtr(env, signature_ptr)); std::unique_ptr<art::MemMap> map(MoveDataToMemMap(original_dex_location, - def.dex_len, - def.dex_data.get(), + def.GetDexData(), error_msg_)); std::ostringstream os; if (map.get() == nullptr) { - os << "Failed to create anonymous mmap for modified dex file of class " << def.name + os << "Failed to create anonymous mmap for modified dex file of class " << def.GetName() << "in dex file " << original_dex_location << " because: " << *error_msg_; *error_msg_ = os.str(); return ERR(OUT_OF_MEMORY); @@ -434,16 +426,16 @@ jvmtiError Redefiner::AddRedefinition(ArtJvmTiEnv* env, const ArtClassDefinition /*verify_checksum*/true, error_msg_)); if (dex_file.get() == nullptr) { - os << "Unable to load modified dex file for " << def.name << ": " << *error_msg_; + os << "Unable to load modified dex file for " << def.GetName() << ": " << *error_msg_; *error_msg_ = os.str(); return ERR(INVALID_CLASS_FORMAT); } redefinitions_.push_back( Redefiner::ClassRedefinition(this, - def.klass, + def.GetClass(), dex_file.release(), signature_ptr, - def.original_dex_file)); + def.GetNewOriginalDexFile())); return OK; } diff --git a/runtime/openjdkjvmti/ti_redefine.h b/runtime/openjdkjvmti/ti_redefine.h index 6c09d46e89..809a681902 100644 --- a/runtime/openjdkjvmti/ti_redefine.h +++ b/runtime/openjdkjvmti/ti_redefine.h @@ -99,8 +99,7 @@ class Redefiner { static jvmtiError IsModifiableClass(jvmtiEnv* env, jclass klass, jboolean* is_redefinable); static std::unique_ptr<art::MemMap> MoveDataToMemMap(const std::string& original_location, - jint data_len, - const unsigned char* dex_data, + art::ArraySlice<const unsigned char> data, std::string* error_msg); private: diff --git a/runtime/openjdkjvmti/ti_thread.cc b/runtime/openjdkjvmti/ti_thread.cc index 788ac30873..e5ff090ddf 100644 --- a/runtime/openjdkjvmti/ti_thread.cc +++ b/runtime/openjdkjvmti/ti_thread.cc @@ -44,6 +44,7 @@ #include "mirror/object-inl.h" #include "mirror/string.h" #include "obj_ptr.h" +#include "ti_phase.h" #include "runtime.h" #include "runtime_callbacks.h" #include "ScopedLocalRef.h" @@ -54,6 +55,8 @@ namespace openjdkjvmti { +art::ArtField* ThreadUtil::context_class_loader_ = nullptr; + struct ThreadCallback : public art::ThreadLifecycleCallback, public art::RuntimePhaseCallback { jthread GetThreadObject(art::Thread* self) REQUIRES_SHARED(art::Locks::mutator_lock_) { if (self->GetPeer() == nullptr) { @@ -121,6 +124,16 @@ void ThreadUtil::Register(EventHandler* handler) { runtime->GetRuntimeCallbacks()->AddRuntimePhaseCallback(&gThreadCallback); } +void ThreadUtil::CacheData() { + art::ScopedObjectAccess soa(art::Thread::Current()); + art::ObjPtr<art::mirror::Class> thread_class = + soa.Decode<art::mirror::Class>(art::WellKnownClasses::java_lang_Thread); + CHECK(thread_class != nullptr); + context_class_loader_ = thread_class->FindDeclaredInstanceField("contextClassLoader", + "Ljava/lang/ClassLoader;"); + CHECK(context_class_loader_ != nullptr); +} + void ThreadUtil::Unregister() { art::ScopedThreadStateChange stsc(art::Thread::Current(), art::ThreadState::kWaitingForDebuggerToAttach); @@ -146,22 +159,6 @@ jvmtiError ThreadUtil::GetCurrentThread(jvmtiEnv* env ATTRIBUTE_UNUSED, jthread* return ERR(NONE); } -// Read the context classloader from a Java thread object. This is a lazy implementation -// that assumes GetThreadInfo isn't called too often. If we instead cache the ArtField, -// we will have to add synchronization as this can't be cached on startup (which is -// potentially runtime startup). -static art::ObjPtr<art::mirror::Object> GetContextClassLoader(art::ObjPtr<art::mirror::Object> peer) - REQUIRES_SHARED(art::Locks::mutator_lock_) { - if (peer == nullptr) { - return nullptr; - } - art::ObjPtr<art::mirror::Class> klass = peer->GetClass(); - art::ArtField* cc_field = klass->FindDeclaredInstanceField("contextClassLoader", - "Ljava/lang/ClassLoader;"); - CHECK(cc_field != nullptr); - return cc_field->GetObject(peer); -} - // Get the native thread. The spec says a null object denotes the current thread. static art::Thread* GetNativeThread(jthread thread, const art::ScopedObjectAccessAlreadyRunnable& soa) @@ -178,6 +175,9 @@ jvmtiError ThreadUtil::GetThreadInfo(jvmtiEnv* env, jthread thread, jvmtiThreadI if (info_ptr == nullptr) { return ERR(NULL_POINTER); } + if (!PhaseUtil::IsLivePhase()) { + return JVMTI_ERROR_WRONG_PHASE; + } art::ScopedObjectAccess soa(art::Thread::Current()); @@ -217,7 +217,10 @@ jvmtiError ThreadUtil::GetThreadInfo(jvmtiEnv* env, jthread thread, jvmtiThreadI } // Context classloader. - art::ObjPtr<art::mirror::Object> ccl = GetContextClassLoader(peer); + DCHECK(context_class_loader_ != nullptr); + art::ObjPtr<art::mirror::Object> ccl = peer != nullptr + ? context_class_loader_->GetObject(peer) + : nullptr; info_ptr->context_class_loader = ccl == nullptr ? nullptr : soa.AddLocalReference<jobject>(ccl); @@ -272,7 +275,10 @@ jvmtiError ThreadUtil::GetThreadInfo(jvmtiEnv* env, jthread thread, jvmtiThreadI } // Context classloader. - art::ObjPtr<art::mirror::Object> ccl = GetContextClassLoader(peer); + DCHECK(context_class_loader_ != nullptr); + art::ObjPtr<art::mirror::Object> ccl = peer != nullptr + ? context_class_loader_->GetObject(peer) + : nullptr; info_ptr->context_class_loader = ccl == nullptr ? nullptr : soa.AddLocalReference<jobject>(ccl); diff --git a/runtime/openjdkjvmti/ti_thread.h b/runtime/openjdkjvmti/ti_thread.h index f6f93ee91a..c7f75d8aec 100644 --- a/runtime/openjdkjvmti/ti_thread.h +++ b/runtime/openjdkjvmti/ti_thread.h @@ -35,6 +35,10 @@ #include "jni.h" #include "jvmti.h" +namespace art { +class ArtField; +} + namespace openjdkjvmti { class EventHandler; @@ -44,6 +48,9 @@ class ThreadUtil { static void Register(EventHandler* event_handler); static void Unregister(); + // To be called when it is safe to cache data. + static void CacheData(); + static jvmtiError GetAllThreads(jvmtiEnv* env, jint* threads_count_ptr, jthread** threads_ptr); static jvmtiError GetCurrentThread(jvmtiEnv* env, jthread* thread_ptr); @@ -60,6 +67,9 @@ class ThreadUtil { jvmtiStartFunction proc, const void* arg, jint priority); + + private: + static art::ArtField* context_class_loader_; }; } // namespace openjdkjvmti diff --git a/runtime/openjdkjvmti/transform.cc b/runtime/openjdkjvmti/transform.cc index 06aecbaee3..15d8dd0fc2 100644 --- a/runtime/openjdkjvmti/transform.cc +++ b/runtime/openjdkjvmti/transform.cc @@ -42,6 +42,7 @@ #include "gc_root-inl.h" #include "globals.h" #include "jni_env_ext-inl.h" +#include "jvalue.h" #include "jvmti.h" #include "linear_alloc.h" #include "mem_map.h" @@ -69,17 +70,18 @@ jvmtiError Transformer::RetransformClassesDirect( for (ArtClassDefinition& def : *definitions) { jint new_len = -1; unsigned char* new_data = nullptr; + art::ArraySlice<const unsigned char> dex_data = def.GetDexData(); event_handler->DispatchEvent<ArtJvmtiEvent::kClassFileLoadHookRetransformable>( self, GetJniEnv(env), - def.klass, - def.loader, - def.name.c_str(), - def.protection_domain, - def.dex_len, - static_cast<const unsigned char*>(def.dex_data.get()), - &new_len, - &new_data); + def.GetClass(), + def.GetLoader(), + def.GetName().c_str(), + def.GetProtectionDomain(), + static_cast<jint>(dex_data.size()), + &dex_data.At(0), + /*out*/&new_len, + /*out*/&new_data); def.SetNewDexData(env, new_len, new_data); } return OK; @@ -117,7 +119,7 @@ jvmtiError Transformer::RetransformClasses(ArtJvmTiEnv* env, return ERR(UNMODIFIABLE_CLASS); } ArtClassDefinition def; - res = FillInTransformationData(env, classes[i], &def); + res = def.Init(env, classes[i]); if (res != OK) { return res; } @@ -146,74 +148,4 @@ jvmtiError GetClassLocation(ArtJvmTiEnv* env, jclass klass, /*out*/std::string* return OK; } -jvmtiError Transformer::GetDexDataForRetransformation(ArtJvmTiEnv* env, - art::Handle<art::mirror::Class> klass, - /*out*/jint* dex_data_len, - /*out*/unsigned char** dex_data) { - art::StackHandleScope<3> hs(art::Thread::Current()); - art::Handle<art::mirror::ClassExt> ext(hs.NewHandle(klass->GetExtData())); - if (!ext.IsNull()) { - art::Handle<art::mirror::Object> orig_dex(hs.NewHandle(ext->GetOriginalDexFile())); - if (!orig_dex.IsNull()) { - if (orig_dex->IsArrayInstance()) { - DCHECK(orig_dex->GetClass()->GetComponentType()->IsPrimitiveByte()); - art::Handle<art::mirror::ByteArray> orig_dex_bytes( - hs.NewHandle(art::down_cast<art::mirror::ByteArray*>(orig_dex->AsArray()))); - *dex_data_len = static_cast<jint>(orig_dex_bytes->GetLength()); - return CopyDataIntoJvmtiBuffer( - env, - reinterpret_cast<const unsigned char*>(orig_dex_bytes->GetData()), - *dex_data_len, - /*out*/dex_data); - } else { - DCHECK(orig_dex->IsDexCache()); - const art::DexFile* dex_file = orig_dex->AsDexCache()->GetDexFile(); - *dex_data_len = static_cast<jint>(dex_file->Size()); - return CopyDataIntoJvmtiBuffer(env, dex_file->Begin(), dex_file->Size(), /*out*/dex_data); - } - } - } - // TODO De-quicken the dex file before passing it to the agents. - LOG(WARNING) << "Dex file is not de-quickened yet! Quickened dex instructions might be present"; - const art::DexFile& dex = klass->GetDexFile(); - *dex_data_len = static_cast<jint>(dex.Size()); - return CopyDataIntoJvmtiBuffer(env, dex.Begin(), *dex_data_len, /*out*/dex_data); -} - -// TODO Move this function somewhere more appropriate. -// Gets the data surrounding the given class. -// TODO Make this less magical. -jvmtiError Transformer::FillInTransformationData(ArtJvmTiEnv* env, - jclass klass, - ArtClassDefinition* def) { - JNIEnv* jni_env = GetJniEnv(env); - if (jni_env == nullptr) { - // TODO Different error might be better? - return ERR(INTERNAL); - } - art::ScopedObjectAccess soa(jni_env); - art::StackHandleScope<3> hs(art::Thread::Current()); - art::Handle<art::mirror::Class> hs_klass(hs.NewHandle(soa.Decode<art::mirror::Class>(klass))); - if (hs_klass.IsNull()) { - return ERR(INVALID_CLASS); - } - def->klass = klass; - def->loader = soa.AddLocalReference<jobject>(hs_klass->GetClassLoader()); - std::string descriptor_store; - std::string descriptor(hs_klass->GetDescriptor(&descriptor_store)); - def->name = descriptor.substr(1, descriptor.size() - 2); - // TODO is this always null? - def->protection_domain = nullptr; - if (def->dex_data.get() == nullptr) { - unsigned char* new_data; - jvmtiError res = GetDexDataForRetransformation(env, hs_klass, &def->dex_len, &new_data); - if (res == OK) { - def->dex_data = MakeJvmtiUniquePtr(env, new_data); - } else { - return res; - } - } - return OK; -} - } // namespace openjdkjvmti diff --git a/runtime/openjdkjvmti/transform.h b/runtime/openjdkjvmti/transform.h index c6a36e8e20..ba40e04b44 100644 --- a/runtime/openjdkjvmti/transform.h +++ b/runtime/openjdkjvmti/transform.h @@ -61,18 +61,6 @@ class Transformer { jint class_count, const jclass* classes, /*out*/std::string* error_msg); - - // Gets the data surrounding the given class. - static jvmtiError FillInTransformationData(ArtJvmTiEnv* env, - jclass klass, - ArtClassDefinition* def); - - private: - static jvmtiError GetDexDataForRetransformation(ArtJvmTiEnv* env, - art::Handle<art::mirror::Class> klass, - /*out*/jint* dex_data_length, - /*out*/unsigned char** dex_data) - REQUIRES_SHARED(art::Locks::mutator_lock_); }; } // namespace openjdkjvmti diff --git a/runtime/parsed_options.cc b/runtime/parsed_options.cc index 4d787db5ac..0784e599b5 100644 --- a/runtime/parsed_options.cc +++ b/runtime/parsed_options.cc @@ -96,7 +96,7 @@ std::unique_ptr<RuntimeParser> ParsedOptions::MakeParser(bool ignore_unrecognize // .WithType<std::vector<ti::Agent>>().AppendValues() // .IntoKey(M::AgentLib) .Define("-agentpath:_") - .WithType<std::vector<ti::Agent>>().AppendValues() + .WithType<std::list<ti::Agent>>().AppendValues() .IntoKey(M::AgentPath) .Define("-Xms_") .WithType<MemoryKiB>() diff --git a/runtime/primitive.cc b/runtime/primitive.cc index 2380284535..1ec345a359 100644 --- a/runtime/primitive.cc +++ b/runtime/primitive.cc @@ -44,7 +44,7 @@ static const char* kBoxedDescriptors[] = { "Ljava/lang/Void;", }; -#define COUNT_OF(x) (sizeof(x) / sizeof(x[0])) +#define COUNT_OF(x) (sizeof(x) / sizeof((x)[0])) const char* Primitive::PrettyDescriptor(Primitive::Type type) { static_assert(COUNT_OF(kTypeNames) == static_cast<size_t>(Primitive::kPrimLast) + 1, diff --git a/runtime/runtime.h b/runtime/runtime.h index d244a9b618..92feabb459 100644 --- a/runtime/runtime.h +++ b/runtime/runtime.h @@ -733,7 +733,7 @@ class Runtime { std::string class_path_string_; std::vector<std::string> properties_; - std::vector<ti::Agent> agents_; + std::list<ti::Agent> agents_; std::vector<Plugin> plugins_; // The default stack size for managed threads created by the runtime. diff --git a/runtime/runtime_options.def b/runtime/runtime_options.def index e68a1b2681..16190cd3c4 100644 --- a/runtime/runtime_options.def +++ b/runtime/runtime_options.def @@ -120,8 +120,8 @@ RUNTIME_OPTIONS_KEY (Unit, NoDexFileFallback) RUNTIME_OPTIONS_KEY (std::string, CpuAbiList) RUNTIME_OPTIONS_KEY (std::string, Fingerprint) RUNTIME_OPTIONS_KEY (ExperimentalFlags, Experimental, ExperimentalFlags::kNone) // -Xexperimental:{...} -RUNTIME_OPTIONS_KEY (std::vector<ti::Agent>, AgentLib) // -agentlib:<libname>=<options> -RUNTIME_OPTIONS_KEY (std::vector<ti::Agent>, AgentPath) // -agentpath:<libname>=<options> +RUNTIME_OPTIONS_KEY (std::list<ti::Agent>, AgentLib) // -agentlib:<libname>=<options> +RUNTIME_OPTIONS_KEY (std::list<ti::Agent>, AgentPath) // -agentpath:<libname>=<options> RUNTIME_OPTIONS_KEY (std::vector<Plugin>, Plugins) // -Xplugin:<library> // Not parse-able from command line, but can be provided explicitly. diff --git a/runtime/stack.h b/runtime/stack.h index 5c9614aba4..bdaa4c3ca2 100644 --- a/runtime/stack.h +++ b/runtime/stack.h @@ -430,8 +430,15 @@ class ShadowFrame { private: ShadowFrame(uint32_t num_vregs, ShadowFrame* link, ArtMethod* method, uint32_t dex_pc, bool has_reference_array) - : link_(link), method_(method), result_register_(nullptr), dex_pc_ptr_(nullptr), - code_item_(nullptr), number_of_vregs_(num_vregs), dex_pc_(dex_pc) { + : link_(link), + method_(method), + result_register_(nullptr), + dex_pc_ptr_(nullptr), + code_item_(nullptr), + number_of_vregs_(num_vregs), + dex_pc_(dex_pc), + cached_hotness_countdown_(0), + hotness_countdown_(0) { // TODO(iam): Remove this parameter, it's an an artifact of portable removal DCHECK(has_reference_array); if (has_reference_array) { diff --git a/runtime/stack_map.h b/runtime/stack_map.h index ffa17c9543..a22498661e 100644 --- a/runtime/stack_map.h +++ b/runtime/stack_map.h @@ -688,7 +688,13 @@ struct FieldEncoding { class StackMapEncoding { public: - StackMapEncoding() {} + StackMapEncoding() + : dex_pc_bit_offset_(0), + dex_register_map_bit_offset_(0), + inline_info_bit_offset_(0), + register_mask_index_bit_offset_(0), + stack_mask_index_bit_offset_(0), + total_bit_size_(0) {} // Set stack map bit layout based on given sizes. // Returns the size of stack map in bits. diff --git a/runtime/thread_list.cc b/runtime/thread_list.cc index caed36936a..8d72fe80c2 100644 --- a/runtime/thread_list.cc +++ b/runtime/thread_list.cc @@ -736,7 +736,7 @@ void ThreadList::SuspendAllInternal(Thread* self, // EAGAIN and EINTR both indicate a spurious failure, try again from the beginning. if ((errno != EAGAIN) && (errno != EINTR)) { if (errno == ETIMEDOUT) { - LOG(kIsDebugBuild ? ::android::base::FATAL : ::android::base::ERROR) + LOG(::android::base::FATAL) << "Timed out waiting for threads to suspend, waited for " << PrettyDuration(NanoTime() - start_time); } else { diff --git a/runtime/transaction.h b/runtime/transaction.h index 7aa98cd33d..0333fe8db4 100644 --- a/runtime/transaction.h +++ b/runtime/transaction.h @@ -162,7 +162,7 @@ class Transaction FINAL { FieldValueKind kind; bool is_volatile; - FieldValue() = default; + FieldValue() : value(0), kind(FieldValueKind::kBoolean), is_volatile(false) {} FieldValue(FieldValue&& log) = default; private: diff --git a/sigchainlib/sigchain.cc b/sigchainlib/sigchain.cc index c1efecd19d..1727f88515 100644 --- a/sigchainlib/sigchain.cc +++ b/sigchainlib/sigchain.cc @@ -26,187 +26,232 @@ #include <stdio.h> #include <stdlib.h> +#include <utility> + #include "sigchain.h" #if defined(__APPLE__) #define _NSIG NSIG #define sighandler_t sig_t + +// Darwin has an #error when ucontext.h is included without _XOPEN_SOURCE defined. +#define _XOPEN_SOURCE #endif +#include <ucontext.h> + +// libsigchain provides an interception layer for signal handlers, to allow ART and others to give +// their signal handlers the first stab at handling signals before passing them on to user code. +// +// It implements wrapper functions for signal, sigaction, and sigprocmask, and a handler that +// forwards signals appropriately. +// +// In our handler, we start off with all signals blocked, fetch the original signal mask from the +// passed in ucontext, and then adjust our signal mask appropriately for the user handler. +// +// It's somewhat tricky for us to properly handle some flag cases: +// SA_NOCLDSTOP and SA_NOCLDWAIT: shouldn't matter, we don't have special handlers for SIGCHLD. +// SA_NODEFER: unimplemented, we can manually change the signal mask appropriately. +// ~SA_ONSTACK: always silently enable this +// SA_RESETHAND: unimplemented, but we can probably do this? +// ~SA_RESTART: unimplemented, maybe we can reserve an RT signal, register an empty handler that +// doesn't have SA_RESTART, and raise the signal to avoid restarting syscalls that are +// expected to be interrupted? + + +static void log(const char* format, ...) { + char buf[256]; + va_list ap; + va_start(ap, format); + vsnprintf(buf, sizeof(buf), format, ap); +#ifdef ART_TARGET_ANDROID + __android_log_write(ANDROID_LOG_ERROR, "libsigchain", buf); +#else + std::cout << buf << "\n"; +#endif + va_end(ap); +} + +#define fatal(...) log(__VA_ARGS__); abort() + +static int sigorset(sigset_t* dest, sigset_t* left, sigset_t* right) { + sigemptyset(dest); + for (size_t i = 0; i < sizeof(sigset_t) * CHAR_BIT; ++i) { + if (sigismember(left, i) == 1 || sigismember(right, i) == 1) { + sigaddset(dest, i); + } + } + return 0; +} + namespace art { -typedef int (*SigActionFnPtr)(int, const struct sigaction*, struct sigaction*); +static decltype(&sigaction) linked_sigaction; +static decltype(&sigprocmask) linked_sigprocmask; +__thread bool handling_signal; -class SignalAction { +class SignalChain { public: - SignalAction() : claimed_(false), uses_old_style_(false), special_handler_(nullptr) { + SignalChain() : claimed_(false) { } - // Claim the signal and keep the action specified. - void Claim(const struct sigaction& action) { - action_ = action; - claimed_ = true; + bool IsClaimed() { + return claimed_; } - // Unclaim the signal and restore the old action. - void Unclaim(int signal) { - claimed_ = false; - sigaction(signal, &action_, nullptr); // Restore old action. + void Claim(int signo) { + if (!claimed_) { + Register(signo); + claimed_ = true; + } } - // Get the action associated with this signal. - const struct sigaction& GetAction() const { - return action_; + // Register the signal chain with the kernel if needed. + void Register(int signo) { + struct sigaction handler_action = {}; + handler_action.sa_sigaction = SignalChain::Handler; + handler_action.sa_flags = SA_RESTART | SA_SIGINFO | SA_ONSTACK; + sigfillset(&handler_action.sa_mask); + linked_sigaction(signo, &handler_action, &action_); } - // Is the signal claimed? - bool IsClaimed() const { - return claimed_; + void SetAction(const struct sigaction* action) { + action_ = *action; } - // Change the recorded action to that specified. - // If oldstyle is true then this action is from an older style signal() - // call as opposed to sigaction(). In this case the sa_handler is - // used when invoking the user's handler. - void SetAction(const struct sigaction& action, bool oldstyle) { - action_ = action; - uses_old_style_ = oldstyle; + struct sigaction GetAction() { + return action_; } - bool OldStyle() const { - return uses_old_style_; - } + void AddSpecialHandler(SpecialSignalHandlerFn fn) { + for (SpecialSignalHandlerFn& slot : special_handlers_) { + if (slot == nullptr) { + slot = fn; + return; + } + } - void SetSpecialHandler(SpecialSignalHandlerFn fn) { - special_handler_ = fn; + fatal("too many special signal handlers"); } - SpecialSignalHandlerFn GetSpecialHandler() { - return special_handler_; + void RemoveSpecialHandler(SpecialSignalHandlerFn fn) { + // This isn't thread safe, but it's unlikely to be a real problem. + size_t len = sizeof(special_handlers_)/sizeof(*special_handlers_); + for (size_t i = 0; i < len; ++i) { + if (special_handlers_[i] == fn) { + for (size_t j = i; j < len - 1; ++j) { + special_handlers_[j] = special_handlers_[j + 1]; + } + special_handlers_[len - 1] = nullptr; + return; + } + } + + fatal("failed to find special handler to remove"); } + + static void Handler(int signo, siginfo_t* siginfo, void*); + private: - struct sigaction action_; // Action to be performed. - bool claimed_; // Whether signal is claimed or not. - bool uses_old_style_; // Action is created using signal(). Use sa_handler. - SpecialSignalHandlerFn special_handler_; // A special handler executed before user handlers. + bool claimed_; + struct sigaction action_; + SpecialSignalHandlerFn special_handlers_[2]; }; -// User's signal handlers -static SignalAction user_sigactions[_NSIG]; -static bool initialized; -static void* linked_sigaction_sym; -static void* linked_sigprocmask_sym; +static SignalChain chains[_NSIG]; -static void log(const char* format, ...) { - char buf[256]; - va_list ap; - va_start(ap, format); - vsnprintf(buf, sizeof(buf), format, ap); -#ifdef ART_TARGET_ANDROID - __android_log_write(ANDROID_LOG_ERROR, "libsigchain", buf); -#else - std::cout << buf << "\n"; -#endif - va_end(ap); -} +class ScopedFlagRestorer { + public: + explicit ScopedFlagRestorer(bool* flag) : flag_(flag), original_value_(*flag) { + } -static void CheckSignalValid(int signal) { - if (signal <= 0 || signal >= _NSIG) { - log("Invalid signal %d", signal); - abort(); + ~ScopedFlagRestorer() { + *flag_ = original_value_; } -} -// Sigchainlib's own handler so we can ensure a managed handler is called first even if nobody -// claimed a chain. Simply forward to InvokeUserSignalHandler. -static void sigchainlib_managed_handler_sigaction(int sig, siginfo_t* info, void* context) { - InvokeUserSignalHandler(sig, info, context); -} + private: + bool* flag_; + bool original_value_; +}; -// Claim a signal chain for a particular signal. -extern "C" void ClaimSignalChain(int signal, struct sigaction* oldaction) { - CheckSignalValid(signal); +class ScopedSignalUnblocker { + public: + explicit ScopedSignalUnblocker(const std::initializer_list<int>& signals) { + sigset_t new_mask; + sigemptyset(&new_mask); + for (int signal : signals) { + sigaddset(&new_mask, signal); + } + if (sigprocmask(SIG_UNBLOCK, &new_mask, &previous_mask_) != 0) { + fatal("failed to unblock signals: %s", strerror(errno)); + } + } - user_sigactions[signal].Claim(*oldaction); -} + ~ScopedSignalUnblocker() { + if (sigprocmask(SIG_SETMASK, &previous_mask_, nullptr) != 0) { + fatal("failed to unblock signals: %s", strerror(errno)); + } + } -extern "C" void UnclaimSignalChain(int signal) { - CheckSignalValid(signal); + private: + sigset_t previous_mask_; +}; - user_sigactions[signal].Unclaim(signal); -} +void SignalChain::Handler(int signo, siginfo_t* siginfo, void* ucontext_raw) { + ScopedFlagRestorer flag(&handling_signal); -// Invoke the user's signal handler. -extern "C" void InvokeUserSignalHandler(int sig, siginfo_t* info, void* context) { - // Check the arguments. - CheckSignalValid(sig); + // Try the special handlers first. + // If one of them crashes, we'll reenter this handler and pass that crash onto the user handler. + if (!handling_signal) { + ScopedSignalUnblocker unblocked { SIGABRT, SIGBUS, SIGFPE, SIGILL, SIGSEGV }; // NOLINT + handling_signal = true; - // The signal must have been claimed in order to get here. Check it. - if (!user_sigactions[sig].IsClaimed()) { - abort(); + for (const auto& handler : chains[signo].special_handlers_) { + if (handler != nullptr && handler(signo, siginfo, ucontext_raw)) { + return; + } + } } - // Do we have a managed handler? If so, run it first. - SpecialSignalHandlerFn managed = user_sigactions[sig].GetSpecialHandler(); - if (managed != nullptr) { - sigset_t mask, old_mask; - sigfillset(&mask); - sigprocmask(SIG_BLOCK, &mask, &old_mask); - // Call the handler. If it succeeds, we're done. - if (managed(sig, info, context)) { - sigprocmask(SIG_SETMASK, &old_mask, nullptr); - return; - } - sigprocmask(SIG_SETMASK, &old_mask, nullptr); + // Forward to the user's signal handler. + int handler_flags = chains[signo].action_.sa_flags; + ucontext_t* ucontext = static_cast<ucontext_t*>(ucontext_raw); + sigset_t mask; + sigorset(&mask, &ucontext->uc_sigmask, &chains[signo].action_.sa_mask); + if ((handler_flags & SA_NODEFER)) { + sigdelset(&mask, signo); } + sigprocmask(SIG_SETMASK, &mask, nullptr); - const struct sigaction& action = user_sigactions[sig].GetAction(); - if (user_sigactions[sig].OldStyle()) { - if (action.sa_handler != nullptr) { - action.sa_handler(sig); - } else { - signal(sig, SIG_DFL); - raise(sig); - } + if ((handler_flags & SA_SIGINFO)) { + chains[signo].action_.sa_sigaction(signo, siginfo, ucontext_raw); } else { - if (action.sa_sigaction != nullptr) { - sigset_t old_mask; - sigprocmask(SIG_BLOCK, &action.sa_mask, &old_mask); - action.sa_sigaction(sig, info, context); - sigprocmask(SIG_SETMASK, &old_mask, nullptr); + auto handler = chains[signo].action_.sa_handler; + if (handler == SIG_IGN) { + return; + } else if (handler == SIG_DFL) { + fatal("exiting due to SIG_DFL handler for signal %d", signo); } else { - signal(sig, SIG_DFL); - raise(sig); + handler(signo); } } } -extern "C" void EnsureFrontOfChain(int signal, struct sigaction* expected_action) { - CheckSignalValid(signal); - // Read the current action without looking at the chain, it should be the expected action. - SigActionFnPtr linked_sigaction = reinterpret_cast<SigActionFnPtr>(linked_sigaction_sym); - struct sigaction current_action; - linked_sigaction(signal, nullptr, ¤t_action); - // If the sigactions don't match then we put the current action on the chain and make ourself as - // the main action. - if (current_action.sa_sigaction != expected_action->sa_sigaction) { - log("Warning: Unexpected sigaction action found %p\n", current_action.sa_sigaction); - user_sigactions[signal].Claim(current_action); - linked_sigaction(signal, expected_action, nullptr); - } -} - extern "C" int sigaction(int signal, const struct sigaction* new_action, struct sigaction* old_action) { // If this signal has been claimed as a signal chain, record the user's // action but don't pass it on to the kernel. // Note that we check that the signal number is in range here. An out of range signal // number should behave exactly as the libc sigaction. - if (signal > 0 && signal < _NSIG && user_sigactions[signal].IsClaimed() && - (new_action == nullptr || new_action->sa_handler != SIG_DFL)) { - struct sigaction saved_action = user_sigactions[signal].GetAction(); + if (signal < 0 || signal >= _NSIG) { + errno = EINVAL; + return -1; + } + + if (chains[signal].IsClaimed()) { + struct sigaction saved_action = chains[signal].GetAction(); if (new_action != nullptr) { - user_sigactions[signal].SetAction(*new_action, false); + chains[signal].SetAction(new_action); } if (old_action != nullptr) { *old_action = saved_action; @@ -216,73 +261,52 @@ extern "C" int sigaction(int signal, const struct sigaction* new_action, struct // Will only get here if the signal chain has not been claimed. We want // to pass the sigaction on to the kernel via the real sigaction in libc. - - if (linked_sigaction_sym == nullptr) { - // Perform lazy initialization. - // This will only occur outside of a signal context since we have - // not been initialized and therefore cannot be within the ART - // runtime. - InitializeSignalChain(); - } - - if (linked_sigaction_sym == nullptr) { - log("Unable to find next sigaction in signal chain"); - abort(); - } - SigActionFnPtr linked_sigaction = reinterpret_cast<SigActionFnPtr>(linked_sigaction_sym); + InitializeSignalChain(); return linked_sigaction(signal, new_action, old_action); } -static sighandler_t signal_impl(int signal, sighandler_t handler) { - struct sigaction sa; +extern "C" sighandler_t signal(int signo, sighandler_t handler) { + if (signo < 0 || signo > _NSIG) { + errno = EINVAL; + return SIG_ERR; + } + + struct sigaction sa = {}; sigemptyset(&sa.sa_mask); sa.sa_handler = handler; - sa.sa_flags = SA_RESTART; + sa.sa_flags = SA_RESTART | SA_ONSTACK; sighandler_t oldhandler; // If this signal has been claimed as a signal chain, record the user's // action but don't pass it on to the kernel. - // Note that we check that the signal number is in range here. An out of range signal - // number should behave exactly as the libc sigaction. - if (signal > 0 && signal < _NSIG && user_sigactions[signal].IsClaimed() && handler != SIG_DFL) { - oldhandler = reinterpret_cast<sighandler_t>(user_sigactions[signal].GetAction().sa_handler); - user_sigactions[signal].SetAction(sa, true); + if (chains[signo].IsClaimed()) { + oldhandler = reinterpret_cast<sighandler_t>(chains[signo].GetAction().sa_handler); + chains[signo].SetAction(&sa); return oldhandler; } // Will only get here if the signal chain has not been claimed. We want // to pass the sigaction on to the kernel via the real sigaction in libc. - - if (linked_sigaction_sym == nullptr) { - // Perform lazy initialization. - InitializeSignalChain(); - } - - if (linked_sigaction_sym == nullptr) { - log("Unable to find next sigaction in signal chain"); - abort(); - } - - typedef int (*SigAction)(int, const struct sigaction*, struct sigaction*); - SigAction linked_sigaction = reinterpret_cast<SigAction>(linked_sigaction_sym); - if (linked_sigaction(signal, &sa, &sa) == -1) { + InitializeSignalChain(); + if (linked_sigaction(signo, &sa, &sa) == -1) { return SIG_ERR; } return reinterpret_cast<sighandler_t>(sa.sa_handler); } -extern "C" sighandler_t signal(int signal, sighandler_t handler) { - return signal_impl(signal, handler); -} - #if !defined(__LP64__) -extern "C" sighandler_t bsd_signal(int signal, sighandler_t handler) { - return signal_impl(signal, handler); +extern "C" sighandler_t bsd_signal(int signo, sighandler_t handler) { + return signal(signo, handler); } #endif extern "C" int sigprocmask(int how, const sigset_t* bionic_new_set, sigset_t* bionic_old_set) { + // When inside a signal handler, forward directly to the actual sigprocmask. + if (handling_signal) { + return linked_sigprocmask(how, bionic_new_set, bionic_old_set); + } + const sigset_t* new_set_ptr = bionic_new_set; sigset_t tmpset; if (bionic_new_set != nullptr) { @@ -292,7 +316,7 @@ extern "C" int sigprocmask(int how, const sigset_t* bionic_new_set, sigset_t* bi // Don't allow claimed signals in the mask. If a signal chain has been claimed // we can't allow the user to block that signal. for (int i = 0 ; i < _NSIG; ++i) { - if (user_sigactions[i].IsClaimed() && sigismember(&tmpset, i)) { + if (chains[i].IsClaimed() && sigismember(&tmpset, i)) { sigdelset(&tmpset, i); } } @@ -300,18 +324,7 @@ extern "C" int sigprocmask(int how, const sigset_t* bionic_new_set, sigset_t* bi new_set_ptr = &tmpset; } - if (linked_sigprocmask_sym == nullptr) { - // Perform lazy initialization. - InitializeSignalChain(); - } - - if (linked_sigprocmask_sym == nullptr) { - log("Unable to find next sigprocmask in signal chain"); - abort(); - } - - typedef int (*SigProcMask)(int how, const sigset_t*, sigset_t*); - SigProcMask linked_sigprocmask= reinterpret_cast<SigProcMask>(linked_sigprocmask_sym); + InitializeSignalChain(); return linked_sigprocmask(how, new_set_ptr, bionic_old_set); } @@ -322,49 +335,67 @@ extern "C" void InitializeSignalChain() { // taken and if it so happens that a signal occurs while one of these // locks is already taken, dlsym will block trying to reenter a // mutex and we will never get out of it. + static bool initialized = false; if (initialized) { // Don't initialize twice. return; } - linked_sigaction_sym = dlsym(RTLD_NEXT, "sigaction"); + + void* linked_sigaction_sym = dlsym(RTLD_NEXT, "sigaction"); if (linked_sigaction_sym == nullptr) { linked_sigaction_sym = dlsym(RTLD_DEFAULT, "sigaction"); if (linked_sigaction_sym == nullptr || linked_sigaction_sym == reinterpret_cast<void*>(sigaction)) { - linked_sigaction_sym = nullptr; + fatal("Unable to find next sigaction in signal chain"); } } - linked_sigprocmask_sym = dlsym(RTLD_NEXT, "sigprocmask"); + void* linked_sigprocmask_sym = dlsym(RTLD_NEXT, "sigprocmask"); if (linked_sigprocmask_sym == nullptr) { linked_sigprocmask_sym = dlsym(RTLD_DEFAULT, "sigprocmask"); if (linked_sigprocmask_sym == nullptr || linked_sigprocmask_sym == reinterpret_cast<void*>(sigprocmask)) { - linked_sigprocmask_sym = nullptr; + fatal("Unable to find next sigprocmask in signal chain"); } } + + linked_sigaction = reinterpret_cast<decltype(linked_sigaction)>(linked_sigaction_sym); + linked_sigprocmask = reinterpret_cast<decltype(linked_sigprocmask)>(linked_sigprocmask_sym); initialized = true; } -extern "C" void SetSpecialSignalHandlerFn(int signal, SpecialSignalHandlerFn fn) { - CheckSignalValid(signal); +extern "C" void AddSpecialSignalHandlerFn(int signal, SpecialSignalHandlerFn fn) { + if (signal <= 0 || signal >= _NSIG) { + fatal("Invalid signal %d", signal); + } // Set the managed_handler. - user_sigactions[signal].SetSpecialHandler(fn); - - // In case the chain isn't claimed, claim it for ourself so we can ensure the managed handler - // goes first. - if (!user_sigactions[signal].IsClaimed()) { - struct sigaction act, old_act; - act.sa_sigaction = sigchainlib_managed_handler_sigaction; - sigemptyset(&act.sa_mask); - act.sa_flags = SA_SIGINFO | SA_ONSTACK; -#if !defined(__APPLE__) && !defined(__mips__) - act.sa_restorer = nullptr; -#endif - if (sigaction(signal, &act, &old_act) != -1) { - user_sigactions[signal].Claim(old_act); - } + chains[signal].AddSpecialHandler(fn); + chains[signal].Claim(signal); +} + +extern "C" void RemoveSpecialSignalHandlerFn(int signal, SpecialSignalHandlerFn fn) { + if (signal <= 0 || signal >= _NSIG) { + fatal("Invalid signal %d", signal); + } + + chains[signal].RemoveSpecialHandler(fn); +} + +extern "C" void EnsureFrontOfChain(int signal) { + if (signal <= 0 || signal >= _NSIG) { + fatal("Invalid signal %d", signal); + } + + // Read the current action without looking at the chain, it should be the expected action. + struct sigaction current_action; + InitializeSignalChain(); + linked_sigaction(signal, nullptr, ¤t_action); + // If the sigactions don't match then we put the current action on the chain and make ourself as + // the main action. + if (current_action.sa_sigaction != SignalChain::Handler) { + log("Warning: Unexpected sigaction action found %p\n", current_action.sa_sigaction); + chains[signal].Register(signal); } } diff --git a/sigchainlib/sigchain.h b/sigchainlib/sigchain.h index 01ccedf498..960d221177 100644 --- a/sigchainlib/sigchain.h +++ b/sigchainlib/sigchain.h @@ -23,16 +23,11 @@ namespace art { extern "C" void InitializeSignalChain(); -extern "C" void ClaimSignalChain(int signal, struct sigaction* oldaction); - -extern "C" void UnclaimSignalChain(int signal); - typedef bool (*SpecialSignalHandlerFn)(int, siginfo_t*, void*); -extern "C" void SetSpecialSignalHandlerFn(int signal, SpecialSignalHandlerFn fn); - -extern "C" void InvokeUserSignalHandler(int sig, siginfo_t* info, void* context); +extern "C" void AddSpecialSignalHandlerFn(int signal, SpecialSignalHandlerFn fn); +extern "C" void RemoveSpecialSignalHandlerFn(int signal, SpecialSignalHandlerFn fn); -extern "C" void EnsureFrontOfChain(int signal, struct sigaction* expected_action); +extern "C" void EnsureFrontOfChain(int signal); } // namespace art diff --git a/sigchainlib/sigchain_dummy.cc b/sigchainlib/sigchain_dummy.cc index aa3c360b3a..d6a5e12827 100644 --- a/sigchainlib/sigchain_dummy.cc +++ b/sigchainlib/sigchain_dummy.cc @@ -48,37 +48,23 @@ static void log(const char* format, ...) { namespace art { - -extern "C" void ClaimSignalChain(int signal ATTRIBUTE_UNUSED, - struct sigaction* oldaction ATTRIBUTE_UNUSED) { - log("ClaimSignalChain is not exported by the main executable."); - abort(); -} - -extern "C" void UnclaimSignalChain(int signal ATTRIBUTE_UNUSED) { - log("UnclaimSignalChain is not exported by the main executable."); - abort(); -} - -extern "C" void InvokeUserSignalHandler(int sig ATTRIBUTE_UNUSED, - siginfo_t* info ATTRIBUTE_UNUSED, - void* context ATTRIBUTE_UNUSED) { - log("InvokeUserSignalHandler is not exported by the main executable."); - abort(); -} - extern "C" void InitializeSignalChain() { log("InitializeSignalChain is not exported by the main executable."); abort(); } -extern "C" void EnsureFrontOfChain(int signal ATTRIBUTE_UNUSED, - struct sigaction* expected_action ATTRIBUTE_UNUSED) { +extern "C" void EnsureFrontOfChain(int signal ATTRIBUTE_UNUSED) { log("EnsureFrontOfChain is not exported by the main executable."); abort(); } -extern "C" void SetSpecialSignalHandlerFn(int signal ATTRIBUTE_UNUSED, +extern "C" void AddSpecialSignalHandlerFn(int signal ATTRIBUTE_UNUSED, + SpecialSignalHandlerFn fn ATTRIBUTE_UNUSED) { + log("SetSpecialSignalHandlerFn is not exported by the main executable."); + abort(); +} + +extern "C" void RemoveSpecialSignalHandlerFn(int signal ATTRIBUTE_UNUSED, SpecialSignalHandlerFn fn ATTRIBUTE_UNUSED) { log("SetSpecialSignalHandlerFn is not exported by the main executable."); abort(); diff --git a/sigchainlib/version-script32.txt b/sigchainlib/version-script32.txt index eec9103d3f..f360efafcf 100644 --- a/sigchainlib/version-script32.txt +++ b/sigchainlib/version-script32.txt @@ -1,11 +1,9 @@ { global: - ClaimSignalChain; - UnclaimSignalChain; - InvokeUserSignalHandler; InitializeSignalChain; EnsureFrontOfChain; - SetSpecialSignalHandlerFn; + AddSpecialSignalHandlerFn; + RemoveSpecialSignalHandlerFn; bsd_signal; sigaction; signal; diff --git a/sigchainlib/version-script64.txt b/sigchainlib/version-script64.txt index 08c312e7e2..319d1c645a 100644 --- a/sigchainlib/version-script64.txt +++ b/sigchainlib/version-script64.txt @@ -1,11 +1,9 @@ { global: - ClaimSignalChain; - UnclaimSignalChain; - InvokeUserSignalHandler; InitializeSignalChain; EnsureFrontOfChain; - SetSpecialSignalHandlerFn; + AddSpecialSignalHandlerFn; + RemoveSpecialSignalHandlerFn; sigaction; signal; sigprocmask; diff --git a/test/004-SignalTest/expected.txt b/test/004-SignalTest/expected.txt index b3a0e1cbe0..847b56f823 100644 --- a/test/004-SignalTest/expected.txt +++ b/test/004-SignalTest/expected.txt @@ -3,4 +3,8 @@ init signal test Caught NullPointerException Caught StackOverflowError signal caught +unblocked signal received +unblocking blocked signal +blocked signal received +signal handler done Signal test OK diff --git a/test/004-SignalTest/signaltest.cc b/test/004-SignalTest/signaltest.cc index 6dd63551fd..a58a0752c9 100644 --- a/test/004-SignalTest/signaltest.cc +++ b/test/004-SignalTest/signaltest.cc @@ -18,13 +18,14 @@ #include <signal.h> #include <stdio.h> #include <stdlib.h> +#include <string.h> #include <sys/ucontext.h> #include <unistd.h> #include "base/macros.h" static int signal_count; -static const int kMaxSignal = 2; +static const int kMaxSignal = 1; #if defined(__i386__) || defined(__x86_64__) #if defined(__APPLE__) @@ -47,6 +48,17 @@ static const int kMaxSignal = 2; #endif #endif +#define BLOCKED_SIGNAL SIGUSR1 +#define UNBLOCKED_SIGNAL SIGUSR2 + +static void blocked_signal(int sig ATTRIBUTE_UNUSED) { + printf("blocked signal received\n"); +} + +static void unblocked_signal(int sig ATTRIBUTE_UNUSED) { + printf("unblocked signal received\n"); +} + static void signalhandler(int sig ATTRIBUTE_UNUSED, siginfo_t* info ATTRIBUTE_UNUSED, void* context) { printf("signal caught\n"); @@ -54,6 +66,16 @@ static void signalhandler(int sig ATTRIBUTE_UNUSED, siginfo_t* info ATTRIBUTE_UN if (signal_count > kMaxSignal) { abort(); } + + raise(UNBLOCKED_SIGNAL); + raise(BLOCKED_SIGNAL); + printf("unblocking blocked signal\n"); + + sigset_t mask; + sigemptyset(&mask); + sigaddset(&mask, BLOCKED_SIGNAL); + sigprocmask(SIG_UNBLOCK, &mask, nullptr); + #if defined(__arm__) struct ucontext *uc = reinterpret_cast<struct ucontext*>(context); struct sigcontext *sc = reinterpret_cast<struct sigcontext*>(&uc->uc_mcontext); @@ -71,6 +93,8 @@ static void signalhandler(int sig ATTRIBUTE_UNUSED, siginfo_t* info ATTRIBUTE_UN #else UNUSED(context); #endif + + printf("signal handler done\n"); } static struct sigaction oldaction; @@ -78,13 +102,21 @@ static struct sigaction oldaction; extern "C" JNIEXPORT void JNICALL Java_Main_initSignalTest(JNIEnv*, jclass) { struct sigaction action; action.sa_sigaction = signalhandler; - sigemptyset(&action.sa_mask); + sigfillset(&action.sa_mask); + sigdelset(&action.sa_mask, UNBLOCKED_SIGNAL); action.sa_flags = SA_SIGINFO | SA_ONSTACK; #if !defined(__APPLE__) && !defined(__mips__) action.sa_restorer = nullptr; #endif sigaction(SIGSEGV, &action, &oldaction); + struct sigaction check; + sigaction(SIGSEGV, nullptr, &check); + if (memcmp(&action, &check, sizeof(action)) != 0) { + printf("sigaction returned different value\n"); + } + signal(BLOCKED_SIGNAL, blocked_signal); + signal(UNBLOCKED_SIGNAL, unblocked_signal); } extern "C" JNIEXPORT void JNICALL Java_Main_terminateSignalTest(JNIEnv*, jclass) { @@ -96,6 +128,12 @@ extern "C" JNIEXPORT void JNICALL Java_Main_terminateSignalTest(JNIEnv*, jclass) char *go_away_compiler = nullptr; extern "C" JNIEXPORT jint JNICALL Java_Main_testSignal(JNIEnv*, jclass) { + // Unblock UNBLOCKED_SIGNAL. + sigset_t mask; + memset(&mask, 0, sizeof(mask)); + sigaddset(&mask, UNBLOCKED_SIGNAL); + sigprocmask(SIG_UNBLOCK, &mask, nullptr); + #if defined(__arm__) || defined(__i386__) || defined(__aarch64__) // On supported architectures we cause a real SEGV. *go_away_compiler = 'a'; diff --git a/test/115-native-bridge/nativebridge.cc b/test/115-native-bridge/nativebridge.cc index 41329af138..f913cf69d0 100644 --- a/test/115-native-bridge/nativebridge.cc +++ b/test/115-native-bridge/nativebridge.cc @@ -395,20 +395,6 @@ extern "C" bool native_bridge_isCompatibleWith(uint32_t bridge_version ATTRIBUTE #endif #endif -static bool cannot_be_blocked(int signum) { - // These two sigs cannot be blocked anywhere. - if ((signum == SIGKILL) || (signum == SIGSTOP)) { - return true; - } - - // The invalid rt_sig cannot be blocked. - if (((signum >= 32) && (signum < SIGRTMIN)) || (signum > SIGRTMAX)) { - return true; - } - - return false; -} - // A dummy special handler, continueing after the faulting location. This code comes from // 004-SignalTest. static bool nb_signalhandler(int sig, siginfo_t* info ATTRIBUTE_UNUSED, void* context) { @@ -433,22 +419,6 @@ static bool nb_signalhandler(int sig, siginfo_t* info ATTRIBUTE_UNUSED, void* co #endif } - // Before invoking this handler, all other unclaimed signals must be blocked. - // We're trying to check the signal mask to verify its status here. - sigset_t tmpset; - sigemptyset(&tmpset); - sigprocmask(SIG_SETMASK, nullptr, &tmpset); - int other_claimed = (sig == SIGSEGV) ? SIGILL : SIGSEGV; - for (int signum = 0; signum < NSIG; ++signum) { - if (cannot_be_blocked(signum)) { - continue; - } else if ((sigismember(&tmpset, signum)) && (signum == other_claimed)) { - printf("ERROR: The claimed signal %d is blocked\n", signum); - } else if ((!sigismember(&tmpset, signum)) && (signum != other_claimed)) { - printf("ERROR: The unclaimed signal %d is not blocked\n", signum); - } - } - // We handled this... return true; } diff --git a/test/141-class-unload/src/Main.java b/test/141-class-unload/src/Main.java index 595c70d0f6..7e8431fb52 100644 --- a/test/141-class-unload/src/Main.java +++ b/test/141-class-unload/src/Main.java @@ -59,7 +59,7 @@ public class Main { // Stop the JIT to ensure its threads and work queue are not keeping classes // artifically alive. stopJit(); - Runtime.getRuntime().gc(); + doUnloading(); System.runFinalization(); BufferedReader reader = new BufferedReader(new FileReader ("/proc/" + pid + "/maps")); String line; @@ -83,12 +83,20 @@ public class Main { } } + private static void doUnloading() { + // Do multiple GCs to prevent rare flakiness if some other thread is keeping the + // classloader live. + for (int i = 0; i < 5; ++i) { + Runtime.getRuntime().gc(); + } + } + private static void testUnloadClass(Constructor<?> constructor) throws Exception { WeakReference<Class> klass = setUpUnloadClassWeak(constructor); // No strong references to class loader, should get unloaded. - Runtime.getRuntime().gc(); + doUnloading(); WeakReference<Class> klass2 = setUpUnloadClassWeak(constructor); - Runtime.getRuntime().gc(); + doUnloading(); // If the weak reference is cleared, then it was unloaded. System.out.println(klass.get()); System.out.println(klass2.get()); @@ -98,7 +106,7 @@ public class Main { throws Exception { WeakReference<ClassLoader> loader = setUpUnloadLoader(constructor, true); // No strong references to class loader, should get unloaded. - Runtime.getRuntime().gc(); + doUnloading(); // If the weak reference is cleared, then it was unloaded. System.out.println(loader.get()); } @@ -110,7 +118,7 @@ public class Main { Throwable throwable = (Throwable) stackTraceMethod.invoke(klass); stackTraceMethod = null; klass = null; - Runtime.getRuntime().gc(); + doUnloading(); boolean isNull = weak_klass.get() == null; System.out.println("class null " + isNull + " " + throwable.getMessage()); } @@ -118,7 +126,7 @@ public class Main { private static void testLoadAndUnloadLibrary(Constructor<?> constructor) throws Exception { WeakReference<ClassLoader> loader = setUpLoadLibrary(constructor); // No strong references to class loader, should get unloaded. - Runtime.getRuntime().gc(); + doUnloading(); // If the weak reference is cleared, then it was unloaded. System.out.println(loader.get()); } @@ -147,7 +155,7 @@ public class Main { private static void testNoUnloadInstance(Constructor<?> constructor) throws Exception { Pair p = testNoUnloadInstanceHelper(constructor); - Runtime.getRuntime().gc(); + doUnloading(); // If the class loader was unloded too early due to races, just pass the test. boolean isNull = p.classLoader.get() == null; System.out.println("loader null " + isNull); diff --git a/test/638-checker-inline-caches/src/Main.java b/test/638-checker-inline-caches/src/Main.java index 2cee47e240..680bd14dbc 100644 --- a/test/638-checker-inline-caches/src/Main.java +++ b/test/638-checker-inline-caches/src/Main.java @@ -40,10 +40,11 @@ public class Main { /// CHECK-START: int Main.inlineMonomorphicSubA(Super) inliner (after) /// CHECK: <<SubARet:i\d+>> IntConstant 42 - /// CHECK: <<ObjClass:l\d+>> InstanceFieldGet field_name:java.lang.Object.shadow$_klass_ + /// CHECK: <<Obj:l\d+>> NullCheck + /// CHECK: <<ObjClass:l\d+>> InstanceFieldGet [<<Obj>>] field_name:java.lang.Object.shadow$_klass_ /// CHECK: <<InlineClass:l\d+>> LoadClass class_name:SubA /// CHECK: <<Test:z\d+>> NotEqual [<<InlineClass>>,<<ObjClass>>] - /// CHECK: Deoptimize [<<Test>>] + /// CHECK: Deoptimize [<<Test>>,<<Obj>>] /// CHECK: Return [<<SubARet>>] public static int inlineMonomorphicSubA(Super a) { return a.getValue(); @@ -60,7 +61,8 @@ public class Main { /// CHECK-START: int Main.inlinePolymophicSubASubB(Super) inliner (after) /// CHECK-DAG: <<SubARet:i\d+>> IntConstant 42 /// CHECK-DAG: <<SubBRet:i\d+>> IntConstant 38 - /// CHECK: <<ObjClassSubA:l\d+>> InstanceFieldGet field_name:java.lang.Object.shadow$_klass_ + /// CHECK: <<Obj:l\d+>> NullCheck + /// CHECK: <<ObjClassSubA:l\d+>> InstanceFieldGet [<<Obj>>] field_name:java.lang.Object.shadow$_klass_ /// CHECK: <<InlineClassSubA:l\d+>> LoadClass class_name:SubA /// CHECK: <<TestSubA:z\d+>> NotEqual [<<InlineClassSubA>>,<<ObjClassSubA>>] /// CHECK: If [<<TestSubA>>] @@ -68,7 +70,7 @@ public class Main { /// CHECK: <<ObjClassSubB:l\d+>> InstanceFieldGet field_name:java.lang.Object.shadow$_klass_ /// CHECK: <<InlineClassSubB:l\d+>> LoadClass class_name:SubB /// CHECK: <<TestSubB:z\d+>> NotEqual [<<InlineClassSubB>>,<<ObjClassSubB>>] - /// CHECK: Deoptimize [<<TestSubB>>] + /// CHECK: Deoptimize [<<TestSubB>>,<<Obj>>] /// CHECK: <<Ret:i\d+>> Phi [<<SubARet>>,<<SubBRet>>] /// CHECK: Return [<<Ret>>] @@ -87,7 +89,8 @@ public class Main { /// CHECK-START: int Main.inlinePolymophicCrossDexSubASubC(Super) inliner (after) /// CHECK-DAG: <<SubARet:i\d+>> IntConstant 42 /// CHECK-DAG: <<SubCRet:i\d+>> IntConstant 24 - /// CHECK: <<ObjClassSubA:l\d+>> InstanceFieldGet field_name:java.lang.Object.shadow$_klass_ + /// CHECK: <<Obj:l\d+>> NullCheck + /// CHECK: <<ObjClassSubA:l\d+>> InstanceFieldGet [<<Obj>>] field_name:java.lang.Object.shadow$_klass_ /// CHECK: <<InlineClassSubA:l\d+>> LoadClass class_name:SubA /// CHECK: <<TestSubA:z\d+>> NotEqual [<<InlineClassSubA>>,<<ObjClassSubA>>] /// CHECK: If [<<TestSubA>>] @@ -95,7 +98,7 @@ public class Main { /// CHECK: <<ObjClassSubC:l\d+>> InstanceFieldGet field_name:java.lang.Object.shadow$_klass_ /// CHECK: <<InlineClassSubC:l\d+>> LoadClass class_name:SubC /// CHECK: <<TestSubC:z\d+>> NotEqual [<<InlineClassSubC>>,<<ObjClassSubC>>] - /// CHECK: Deoptimize [<<TestSubC>>] + /// CHECK: Deoptimize [<<TestSubC>>,<<Obj>>] /// CHECK: <<Ret:i\d+>> Phi [<<SubARet>>,<<SubCRet>>] /// CHECK: Return [<<Ret>>] diff --git a/test/640-checker-boolean-simd/expected.txt b/test/640-checker-boolean-simd/expected.txt new file mode 100644 index 0000000000..b0aad4deb5 --- /dev/null +++ b/test/640-checker-boolean-simd/expected.txt @@ -0,0 +1 @@ +passed diff --git a/test/640-checker-boolean-simd/info.txt b/test/640-checker-boolean-simd/info.txt new file mode 100644 index 0000000000..c9c6d5ed9f --- /dev/null +++ b/test/640-checker-boolean-simd/info.txt @@ -0,0 +1 @@ +Functional tests on SIMD vectorization. diff --git a/test/640-checker-boolean-simd/src/Main.java b/test/640-checker-boolean-simd/src/Main.java new file mode 100644 index 0000000000..f8239faaf3 --- /dev/null +++ b/test/640-checker-boolean-simd/src/Main.java @@ -0,0 +1,136 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Functional tests for SIMD vectorization. + */ +public class Main { + + static boolean[] a; + + // + // Arithmetic operations. + // + + /// CHECK-START: void Main.and(boolean) loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.and(boolean) loop_optimization (after) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: VecLoad loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecAnd loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore loop:<<Loop>> outer_loop:none + static void and(boolean x) { + for (int i = 0; i < 128; i++) + a[i] &= x; // NOTE: bitwise and, not the common && + } + + /// CHECK-START: void Main.or(boolean) loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.or(boolean) loop_optimization (after) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: VecLoad loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecOr loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore loop:<<Loop>> outer_loop:none + static void or(boolean x) { + for (int i = 0; i < 128; i++) + a[i] |= x; // NOTE: bitwise or, not the common || + } + + /// CHECK-START: void Main.xor(boolean) loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.xor(boolean) loop_optimization (after) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: VecLoad loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecXor loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore loop:<<Loop>> outer_loop:none + static void xor(boolean x) { + for (int i = 0; i < 128; i++) + a[i] ^= x; // NOTE: bitwise xor + } + + /// CHECK-START: void Main.not() loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.not() loop_optimization (after) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: VecLoad loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecNot loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore loop:<<Loop>> outer_loop:none + static void not() { + for (int i = 0; i < 128; i++) + a[i] = !a[i]; + } + + // + // Test Driver. + // + + public static void main(String[] args) { + // Set up. + a = new boolean[128]; + for (int i = 0; i < 128; i++) { + a[i] = (i & 1) == 0; + } + // Arithmetic operations. + and(true); + for (int i = 0; i < 128; i++) { + expectEquals((i & 1) == 0, a[i], "and-true"); + } + xor(true); + for (int i = 0; i < 128; i++) { + expectEquals((i & 1) != 0, a[i], "xor-true"); + } + xor(false); + for (int i = 0; i < 128; i++) { + expectEquals((i & 1) != 0, a[i], "xor-false"); + } + not(); + for (int i = 0; i < 128; i++) { + expectEquals((i & 1) == 0, a[i], "not"); + } + or(true); + for (int i = 0; i < 128; i++) { + expectEquals(true, a[i], "or-true"); + } + and(false); + for (int i = 0; i < 128; i++) { + expectEquals(false, a[i], "and-false"); + } + or(false); + for (int i = 0; i < 128; i++) { + expectEquals(false, a[i], "or-false"); + } + // Done. + System.out.println("passed"); + } + + private static void expectEquals(boolean expected, boolean result, String action) { + if (expected != result) { + throw new Error("Expected: " + expected + ", found: " + result + " for " + action); + } + } +} diff --git a/test/640-checker-byte-simd/expected.txt b/test/640-checker-byte-simd/expected.txt new file mode 100644 index 0000000000..b0aad4deb5 --- /dev/null +++ b/test/640-checker-byte-simd/expected.txt @@ -0,0 +1 @@ +passed diff --git a/test/640-checker-byte-simd/info.txt b/test/640-checker-byte-simd/info.txt new file mode 100644 index 0000000000..c9c6d5ed9f --- /dev/null +++ b/test/640-checker-byte-simd/info.txt @@ -0,0 +1 @@ +Functional tests on SIMD vectorization. diff --git a/test/640-checker-byte-simd/src/Main.java b/test/640-checker-byte-simd/src/Main.java new file mode 100644 index 0000000000..0f7452b045 --- /dev/null +++ b/test/640-checker-byte-simd/src/Main.java @@ -0,0 +1,277 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Functional tests for SIMD vectorization. + */ +public class Main { + + static byte[] a; + + // + // Arithmetic operations. + // + + /// CHECK-START: void Main.add(int) loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.add(int) loop_optimization (after) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: VecLoad loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecAdd loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore loop:<<Loop>> outer_loop:none + static void add(int x) { + for (int i = 0; i < 128; i++) + a[i] += x; + } + + /// CHECK-START: void Main.sub(int) loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.sub(int) loop_optimization (after) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: VecLoad loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecSub loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore loop:<<Loop>> outer_loop:none + static void sub(int x) { + for (int i = 0; i < 128; i++) + a[i] -= x; + } + + /// CHECK-START: void Main.mul(int) loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.mul(int) loop_optimization (after) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: VecLoad loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecMul loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore loop:<<Loop>> outer_loop:none + static void mul(int x) { + for (int i = 0; i < 128; i++) + a[i] *= x; + } + + /// CHECK-START: void Main.div(int) loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START: void Main.div(int) loop_optimization (after) + // + // Not supported on any architecture. + // + static void div(int x) { + for (int i = 0; i < 128; i++) + a[i] /= x; + } + + /// CHECK-START: void Main.neg() loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.neg() loop_optimization (after) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: VecLoad loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecNeg loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore loop:<<Loop>> outer_loop:none + static void neg() { + for (int i = 0; i < 128; i++) + a[i] = (byte) -a[i]; + } + + /// CHECK-START: void Main.not() loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.not() loop_optimization (after) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: VecLoad loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecNot loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore loop:<<Loop>> outer_loop:none + static void not() { + for (int i = 0; i < 128; i++) + a[i] = (byte) ~a[i]; + } + + /// CHECK-START: void Main.shl4() loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.shl4() loop_optimization (after) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: VecLoad loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecShl loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore loop:<<Loop>> outer_loop:none + static void shl4() { + for (int i = 0; i < 128; i++) + a[i] <<= 4; + } + + /// CHECK-START: void Main.sar2() loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.sar2() loop_optimization (after) + // + // TODO: fill in when supported + static void sar2() { + for (int i = 0; i < 128; i++) + a[i] >>= 2; + } + + /// CHECK-START: void Main.shr2() loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.shr2() loop_optimization (after) + // + // TODO: fill in when supported + static void shr2() { + for (int i = 0; i < 128; i++) + a[i] >>>= 2; + } + + // + // Shift sanity. + // + + static void sar31() { + for (int i = 0; i < 128; i++) + a[i] >>= 31; + } + + static void shr31() { + for (int i = 0; i < 128; i++) + a[i] >>>= 31; + } + + static void shr32() { + for (int i = 0; i < 128; i++) + a[i] >>>= 32; // 0, since & 31 + } + + static void shr33() { + for (int i = 0; i < 128; i++) + a[i] >>>= 33; // 1, since & 31 + } + + // + // Loop bounds. + // + + static void bounds() { + for (int i = 1; i < 127; i++) + a[i] += 11; + } + + // + // Test Driver. + // + + public static void main(String[] args) { + // Set up. + a = new byte[128]; + for (int i = 0; i < 128; i++) { + a[i] = (byte) i; + } + // Arithmetic operations. + add(2); + for (int i = 0; i < 128; i++) { + expectEquals((byte)(i + 2), a[i], "add"); + } + sub(2); + for (int i = 0; i < 128; i++) { + expectEquals(i, a[i], "sub"); + } + mul(2); + for (int i = 0; i < 128; i++) { + expectEquals((byte)(i + i), a[i], "mul"); + } + div(2); + for (int i = 0; i < 128; i++) { + expectEquals(((byte)(i + i)) >> 1, a[i], "div"); + a[i] = (byte) i; // undo arithmetic wrap-around effects + } + neg(); + for (int i = 0; i < 128; i++) { + expectEquals(-i, a[i], "neg"); + } + // Loop bounds. + bounds(); + expectEquals(0, a[0], "bounds0"); + for (int i = 1; i < 127; i++) { + expectEquals(11 - i, a[i], "bounds"); + } + expectEquals(-127, a[127], "bounds127"); + // Shifts. + for (int i = 0; i < 128; i++) { + a[i] = (byte) 0xff; + } + shl4(); + for (int i = 0; i < 128; i++) { + expectEquals((byte) 0xf0, a[i], "shl4"); + } + sar2(); + for (int i = 0; i < 128; i++) { + expectEquals((byte) 0xfc, a[i], "sar2"); + } + shr2(); + for (int i = 0; i < 128; i++) { + expectEquals((byte) 0xff, a[i], "shr2"); // sic! + } + sar31(); + for (int i = 0; i < 128; i++) { + expectEquals((byte) 0xff, a[i], "sar31"); + } + shr31(); + for (int i = 0; i < 128; i++) { + expectEquals(0x01, a[i], "shr31"); + a[i] = (byte) 0x12; // reset + } + shr32(); + for (int i = 0; i < 128; i++) { + expectEquals((byte) 0x12, a[i], "shr32"); + } + shr33(); + for (int i = 0; i < 128; i++) { + expectEquals((byte) 0x09, a[i], "shr33"); + a[i] = (byte) 0xf0; // reset + } + not(); + for (int i = 0; i < 128; i++) { + expectEquals((byte) 0x0f, a[i], "not"); + } + // Done. + System.out.println("passed"); + } + + private static void expectEquals(int expected, int result, String action) { + if (expected != result) { + throw new Error("Expected: " + expected + ", found: " + result + " for " + action); + } + } +} diff --git a/test/640-checker-char-simd/expected.txt b/test/640-checker-char-simd/expected.txt new file mode 100644 index 0000000000..b0aad4deb5 --- /dev/null +++ b/test/640-checker-char-simd/expected.txt @@ -0,0 +1 @@ +passed diff --git a/test/640-checker-char-simd/info.txt b/test/640-checker-char-simd/info.txt new file mode 100644 index 0000000000..c9c6d5ed9f --- /dev/null +++ b/test/640-checker-char-simd/info.txt @@ -0,0 +1 @@ +Functional tests on SIMD vectorization. diff --git a/test/640-checker-char-simd/src/Main.java b/test/640-checker-char-simd/src/Main.java new file mode 100644 index 0000000000..0628b36003 --- /dev/null +++ b/test/640-checker-char-simd/src/Main.java @@ -0,0 +1,278 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Functional tests for SIMD vectorization. + */ +public class Main { + + static char[] a; + + // + // Arithmetic operations. + // + + /// CHECK-START: void Main.add(int) loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.add(int) loop_optimization (after) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: VecLoad loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecAdd loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore loop:<<Loop>> outer_loop:none + static void add(int x) { + for (int i = 0; i < 128; i++) + a[i] += x; + } + + /// CHECK-START: void Main.sub(int) loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.sub(int) loop_optimization (after) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: VecLoad loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecSub loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore loop:<<Loop>> outer_loop:none + static void sub(int x) { + for (int i = 0; i < 128; i++) + a[i] -= x; + } + + /// CHECK-START: void Main.mul(int) loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.mul(int) loop_optimization (after) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: VecLoad loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecMul loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore loop:<<Loop>> outer_loop:none + static void mul(int x) { + for (int i = 0; i < 128; i++) + a[i] *= x; + } + + /// CHECK-START: void Main.div(int) loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START: void Main.div(int) loop_optimization (after) + // + // Not supported on any architecture. + // + static void div(int x) { + for (int i = 0; i < 128; i++) + a[i] /= x; + } + + /// CHECK-START: void Main.neg() loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.neg() loop_optimization (after) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: VecLoad loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecNeg loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore loop:<<Loop>> outer_loop:none + static void neg() { + for (int i = 0; i < 128; i++) + a[i] = (char) -a[i]; + } + + /// CHECK-START: void Main.not() loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.not() loop_optimization (after) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: VecLoad loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecNot loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore loop:<<Loop>> outer_loop:none + static void not() { + for (int i = 0; i < 128; i++) + a[i] = (char) ~a[i]; + } + + /// CHECK-START: void Main.shl4() loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.shl4() loop_optimization (after) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: VecLoad loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecShl loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore loop:<<Loop>> outer_loop:none + static void shl4() { + for (int i = 0; i < 128; i++) + a[i] <<= 4; + } + + /// CHECK-START: void Main.sar2() loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.sar2() loop_optimization (after) + // + // TODO: fill in when supported + static void sar2() { + for (int i = 0; i < 128; i++) + a[i] >>= 2; + } + + /// CHECK-START: void Main.shr2() loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.shr2() loop_optimization (after) + // + // TODO: fill in when supported + static void shr2() { + for (int i = 0; i < 128; i++) + a[i] >>>= 2; + } + + // + // Shift sanity. + // + + static void sar31() { + for (int i = 0; i < 128; i++) + a[i] >>= 31; + } + + static void shr31() { + for (int i = 0; i < 128; i++) + a[i] >>>= 31; + } + + static void shr32() { + for (int i = 0; i < 128; i++) + a[i] >>>= 32; // 0, since & 31 + } + + static void shr33() { + for (int i = 0; i < 128; i++) + a[i] >>>= 33; // 1, since & 31 + } + + // + // Loop bounds. + // + + static void bounds() { + for (int i = 1; i < 127; i++) + a[i] += 11; + } + + // + // Test Driver. + // + + public static void main(String[] args) { + // Set up. + a = new char[128]; + for (int i = 0; i < 128; i++) { + a[i] = (char) i; + } + // Arithmetic operations. + add(2); + for (int i = 0; i < 128; i++) { + expectEquals(i + 2, a[i], "add"); + } + sub(2); + for (int i = 0; i < 128; i++) { + expectEquals(i, a[i], "sub"); + } + mul(2); + for (int i = 0; i < 128; i++) { + expectEquals(i + i, a[i], "mul"); + } + div(2); + for (int i = 0; i < 128; i++) { + expectEquals(i, a[i], "div"); + } + neg(); + for (int i = 0; i < 128; i++) { + expectEquals((char)-i, a[i], "neg"); + } + // Loop bounds. + bounds(); + expectEquals(0, a[0], "bounds0"); + for (int i = 1; i < 127; i++) { + expectEquals((char)(11 - i), a[i], "bounds"); + } + expectEquals((char)-127, a[127], "bounds127"); + // Shifts. + for (int i = 0; i < 128; i++) { + a[i] = (char) 0xffff; + } + shl4(); + for (int i = 0; i < 128; i++) { + expectEquals((char) 0xfff0, a[i], "shl4"); + } + sar2(); + for (int i = 0; i < 128; i++) { + expectEquals((char) 0x3ffc, a[i], "sar2"); + } + shr2(); + for (int i = 0; i < 128; i++) { + expectEquals((char) 0x0fff, a[i], "shr2"); + a[i] = (char) 0xffff; // reset + } + sar31(); + for (int i = 0; i < 128; i++) { + expectEquals(0, a[i], "sar31"); + a[i] = (char) 0xffff; // reset + } + shr31(); + for (int i = 0; i < 128; i++) { + expectEquals(0, a[i], "shr31"); + a[i] = (char) 0x1200; // reset + } + shr32(); + for (int i = 0; i < 128; i++) { + expectEquals((char) 0x1200, a[i], "shr32"); + } + shr33(); + for (int i = 0; i < 128; i++) { + expectEquals((char) 0x0900, a[i], "shr33"); + a[i] = (char) 0xf1f0; // reset + } + not(); + for (int i = 0; i < 128; i++) { + expectEquals((char) 0x0e0f, a[i], "not"); + } + // Done. + System.out.println("passed"); + } + + private static void expectEquals(int expected, int result, String action) { + if (expected != result) { + throw new Error("Expected: " + expected + ", found: " + result + " for " + action); + } + } +} diff --git a/test/640-checker-double-simd/expected.txt b/test/640-checker-double-simd/expected.txt new file mode 100644 index 0000000000..b0aad4deb5 --- /dev/null +++ b/test/640-checker-double-simd/expected.txt @@ -0,0 +1 @@ +passed diff --git a/test/640-checker-double-simd/info.txt b/test/640-checker-double-simd/info.txt new file mode 100644 index 0000000000..c9c6d5ed9f --- /dev/null +++ b/test/640-checker-double-simd/info.txt @@ -0,0 +1 @@ +Functional tests on SIMD vectorization. diff --git a/test/640-checker-double-simd/src/Main.java b/test/640-checker-double-simd/src/Main.java new file mode 100644 index 0000000000..43f65f1792 --- /dev/null +++ b/test/640-checker-double-simd/src/Main.java @@ -0,0 +1,195 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Functional tests for SIMD vectorization. Note that this class provides a mere + * functional test, not a precise numerical verifier. + */ +public class Main { + + static double[] a; + + // + // Arithmetic operations. + // + + /// CHECK-START: void Main.add(double) loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.add(double) loop_optimization (after) + // + // TODO: fill in when supported + static void add(double x) { + for (int i = 0; i < 128; i++) + a[i] += x; + } + + /// CHECK-START: void Main.sub(double) loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.sub(double) loop_optimization (after) + // + // TODO: fill in when supported + static void sub(double x) { + for (int i = 0; i < 128; i++) + a[i] -= x; + } + + /// CHECK-START: void Main.mul(double) loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.mul(double) loop_optimization (after) + // + // TODO: fill in when supported + static void mul(double x) { + for (int i = 0; i < 128; i++) + a[i] *= x; + } + + /// CHECK-START: void Main.div(double) loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.div(double) loop_optimization (after) + // + // TODO: fill in when supported + static void div(double x) { + for (int i = 0; i < 128; i++) + a[i] /= x; + } + + /// CHECK-START: void Main.neg() loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.neg() loop_optimization (after) + // + // TODO: fill in when supported + static void neg() { + for (int i = 0; i < 128; i++) + a[i] = -a[i]; + } + + /// CHECK-START: void Main.abs() loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.abs() loop_optimization (after) + // + // TODO: fill in when supported + static void abs() { + for (int i = 0; i < 128; i++) + a[i] = Math.abs(a[i]); + } + + /// CHECK-START: void Main.conv(long[]) loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.conv(long[]) loop_optimization (after) + // + // TODO: fill in when supported + static void conv(long[] b) { + for (int i = 0; i < 128; i++) + a[i] = b[i]; + } + + // + // Loop bounds. + // + + static void bounds() { + for (int i = 1; i < 127; i++) + a[i] += 11; + } + + // + // Test Driver. + // + + public static void main(String[] args) { + // Set up. + a = new double[128]; + for (int i = 0; i < 128; i++) { + a[i] = i; + } + // Arithmetic operations. + add(2.0); + for (int i = 0; i < 128; i++) { + expectEquals(i + 2, a[i], "add"); + } + sub(2.0); + for (int i = 0; i < 128; i++) { + expectEquals(i, a[i], "sub"); + } + mul(2.0); + for (int i = 0; i < 128; i++) { + expectEquals(i + i, a[i], "mul"); + } + div(2.0); + for (int i = 0; i < 128; i++) { + expectEquals(i, a[i], "div"); + } + neg(); + for (int i = 0; i < 128; i++) { + expectEquals(-i, a[i], "neg"); + } + // Loop bounds. + bounds(); + expectEquals(0, a[0], "bounds0"); + for (int i = 1; i < 127; i++) { + expectEquals(11 - i, a[i], "bounds"); + } + expectEquals(-127, a[127], "bounds127"); + // Abs. + abs(); + expectEquals(0, a[0], "abs0"); + for (int i = 1; i <= 11; i++) { + expectEquals(11 - i, a[i], "abs_lo"); + } + for (int i = 12; i < 127; i++) { + expectEquals(i - 11, a[i], "abs_hi"); + } + expectEquals(127, a[127], "abs127"); + // Conversion. + long[] b = new long[128]; + for (int i = 0; i < 128; i++) { + b[i] = 1000 * i; + } + conv(b); + for (int i = 1; i < 127; i++) { + expectEquals(1000.0 * i, a[i], "conv"); + } + // Done. + System.out.println("passed"); + } + + private static void expectEquals(double expected, double result, String action) { + if (expected != result) { + throw new Error("Expected: " + expected + ", found: " + result + " for " + action); + } + } +} diff --git a/test/640-checker-float-simd/expected.txt b/test/640-checker-float-simd/expected.txt new file mode 100644 index 0000000000..b0aad4deb5 --- /dev/null +++ b/test/640-checker-float-simd/expected.txt @@ -0,0 +1 @@ +passed diff --git a/test/640-checker-float-simd/info.txt b/test/640-checker-float-simd/info.txt new file mode 100644 index 0000000000..c9c6d5ed9f --- /dev/null +++ b/test/640-checker-float-simd/info.txt @@ -0,0 +1 @@ +Functional tests on SIMD vectorization. diff --git a/test/640-checker-float-simd/src/Main.java b/test/640-checker-float-simd/src/Main.java new file mode 100644 index 0000000000..80c3112b6a --- /dev/null +++ b/test/640-checker-float-simd/src/Main.java @@ -0,0 +1,207 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Functional tests for SIMD vectorization. Note that this class provides a mere + * functional test, not a precise numerical verifier. + */ +public class Main { + + static float[] a; + + // + // Arithmetic operations. + // + + /// CHECK-START: void Main.add(float) loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.add(float) loop_optimization (after) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: VecLoad loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecAdd loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore loop:<<Loop>> outer_loop:none + static void add(float x) { + for (int i = 0; i < 128; i++) + a[i] += x; + } + + /// CHECK-START: void Main.sub(float) loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.sub(float) loop_optimization (after) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: VecLoad loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecSub loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore loop:<<Loop>> outer_loop:none + static void sub(float x) { + for (int i = 0; i < 128; i++) + a[i] -= x; + } + + /// CHECK-START: void Main.mul(float) loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.mul(float) loop_optimization (after) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: VecLoad loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecMul loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore loop:<<Loop>> outer_loop:none + static void mul(float x) { + for (int i = 0; i < 128; i++) + a[i] *= x; + } + + /// CHECK-START: void Main.div(float) loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.div(float) loop_optimization (after) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: VecLoad loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecDiv loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore loop:<<Loop>> outer_loop:none + static void div(float x) { + for (int i = 0; i < 128; i++) + a[i] /= x; + } + + /// CHECK-START: void Main.neg() loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.neg() loop_optimization (after) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: VecLoad loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecNeg loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore loop:<<Loop>> outer_loop:none + static void neg() { + for (int i = 0; i < 128; i++) + a[i] = -a[i]; + } + + /// CHECK-START: void Main.abs() loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.abs() loop_optimization (after) + // + // TODO: fill in when supported + static void abs() { + for (int i = 0; i < 128; i++) + a[i] = Math.abs(a[i]); + } + + /// CHECK-START: void Main.conv(int[]) loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.conv(int[]) loop_optimization (after) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: VecLoad loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecCnv loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore loop:<<Loop>> outer_loop:none + static void conv(int[] b) { + for (int i = 0; i < 128; i++) + a[i] = b[i]; + } + + // + // Loop bounds. + // + + static void bounds() { + for (int i = 1; i < 127; i++) + a[i] += 11; + } + + // + // Test Driver. + // + + public static void main(String[] args) { + // Set up. + a = new float[128]; + for (int i = 0; i < 128; i++) { + a[i] = i; + } + // Arithmetic operations. + add(2.0f); + for (int i = 0; i < 128; i++) { + expectEquals(i + 2, a[i], "add"); + } + sub(2.0f); + for (int i = 0; i < 128; i++) { + expectEquals(i, a[i], "sub"); + } + mul(2.0f); + for (int i = 0; i < 128; i++) { + expectEquals(i + i, a[i], "mul"); + } + div(2.0f); + for (int i = 0; i < 128; i++) { + expectEquals(i, a[i], "div"); + } + neg(); + for (int i = 0; i < 128; i++) { + expectEquals(-i, a[i], "neg"); + } + // Loop bounds. + bounds(); + expectEquals(0, a[0], "bounds0"); + for (int i = 1; i < 127; i++) { + expectEquals(11 - i, a[i], "bounds"); + } + expectEquals(-127, a[127], "bounds127"); + // Abs. + abs(); + expectEquals(0, a[0], "abs0"); + for (int i = 1; i <= 11; i++) { + expectEquals(11 - i, a[i], "abs_lo"); + } + for (int i = 12; i < 127; i++) { + expectEquals(i - 11, a[i], "abs_hi"); + } + expectEquals(127, a[127], "abs127"); + // Conversion. + int[] b = new int[128]; + for (int i = 0; i < 128; i++) { + b[i] = 1000 * i; + } + conv(b); + for (int i = 1; i < 127; i++) { + expectEquals(1000.0f * i, a[i], "conv"); + } + // Done. + System.out.println("passed"); + } + + private static void expectEquals(float expected, float result, String action) { + if (expected != result) { + throw new Error("Expected: " + expected + ", found: " + result + " for " + action); + } + } +} diff --git a/test/640-checker-int-simd/expected.txt b/test/640-checker-int-simd/expected.txt new file mode 100644 index 0000000000..b0aad4deb5 --- /dev/null +++ b/test/640-checker-int-simd/expected.txt @@ -0,0 +1 @@ +passed diff --git a/test/640-checker-int-simd/info.txt b/test/640-checker-int-simd/info.txt new file mode 100644 index 0000000000..c9c6d5ed9f --- /dev/null +++ b/test/640-checker-int-simd/info.txt @@ -0,0 +1 @@ +Functional tests on SIMD vectorization. diff --git a/test/640-checker-int-simd/src/Main.java b/test/640-checker-int-simd/src/Main.java new file mode 100644 index 0000000000..ba1e142668 --- /dev/null +++ b/test/640-checker-int-simd/src/Main.java @@ -0,0 +1,256 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Functional tests for SIMD vectorization. + */ +public class Main { + + static int[] a; + + // + // Arithmetic operations. + // + + /// CHECK-START: void Main.add(int) loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.add(int) loop_optimization (after) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: VecLoad loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecAdd loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore loop:<<Loop>> outer_loop:none + static void add(int x) { + for (int i = 0; i < 128; i++) + a[i] += x; + } + + /// CHECK-START: void Main.sub(int) loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.sub(int) loop_optimization (after) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: VecLoad loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecSub loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore loop:<<Loop>> outer_loop:none + static void sub(int x) { + for (int i = 0; i < 128; i++) + a[i] -= x; + } + + /// CHECK-START: void Main.mul(int) loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.mul(int) loop_optimization (after) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: VecLoad loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecMul loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore loop:<<Loop>> outer_loop:none + static void mul(int x) { + for (int i = 0; i < 128; i++) + a[i] *= x; + } + + /// CHECK-START: void Main.div(int) loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START: void Main.div(int) loop_optimization (after) + // + // Not supported on any architecture. + // + static void div(int x) { + for (int i = 0; i < 128; i++) + a[i] /= x; + } + + /// CHECK-START: void Main.neg() loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.neg() loop_optimization (after) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: VecLoad loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecNeg loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore loop:<<Loop>> outer_loop:none + static void neg() { + for (int i = 0; i < 128; i++) + a[i] = -a[i]; + } + + /// CHECK-START: void Main.not() loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.not() loop_optimization (after) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: VecLoad loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecNot loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore loop:<<Loop>> outer_loop:none + static void not() { + for (int i = 0; i < 128; i++) + a[i] = ~a[i]; + } + + /// CHECK-START: void Main.shl4() loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.shl4() loop_optimization (after) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: VecLoad loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecShl loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore loop:<<Loop>> outer_loop:none + static void shl4() { + for (int i = 0; i < 128; i++) + a[i] <<= 4; + } + + /// CHECK-START: void Main.sar2() loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.sar2() loop_optimization (after) + // + // TODO: fill in when supported + static void sar2() { + for (int i = 0; i < 128; i++) + a[i] >>= 2; + } + + /// CHECK-START: void Main.shr2() loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.shr2() loop_optimization (after) + // + // TODO: fill in when supported + static void shr2() { + for (int i = 0; i < 128; i++) + a[i] >>>= 2; + } + + // + // Shift sanity. + // + + static void shr32() { + for (int i = 0; i < 128; i++) + a[i] >>>= 32; // 0, since & 31 + } + + static void shr33() { + for (int i = 0; i < 128; i++) + a[i] >>>= 33; // 1, since & 31 + } + + // + // Loop bounds. + // + + static void bounds() { + for (int i = 1; i < 127; i++) + a[i] += 11; + } + + // + // Test Driver. + // + + public static void main(String[] args) { + // Set up. + a = new int[128]; + for (int i = 0; i < 128; i++) { + a[i] = i; + } + // Arithmetic operations. + add(2); + for (int i = 0; i < 128; i++) { + expectEquals(i + 2, a[i], "add"); + } + sub(2); + for (int i = 0; i < 128; i++) { + expectEquals(i, a[i], "sub"); + } + mul(2); + for (int i = 0; i < 128; i++) { + expectEquals(i + i, a[i], "mul"); + } + div(2); + for (int i = 0; i < 128; i++) { + expectEquals(i, a[i], "div"); + } + neg(); + for (int i = 0; i < 128; i++) { + expectEquals(-i, a[i], "neg"); + } + // Loop bounds. + bounds(); + expectEquals(0, a[0], "bounds0"); + for (int i = 1; i < 127; i++) { + expectEquals(11 - i, a[i], "bounds"); + } + expectEquals(-127, a[127], "bounds127"); + // Shifts. + for (int i = 0; i < 128; i++) { + a[i] = 0xffffffff; + } + shl4(); + for (int i = 0; i < 128; i++) { + expectEquals(0xfffffff0, a[i], "shl4"); + } + sar2(); + for (int i = 0; i < 128; i++) { + expectEquals(0xfffffffc, a[i], "sar2"); + } + shr2(); + for (int i = 0; i < 128; i++) { + expectEquals(0x3fffffff, a[i], "shr2"); + } + shr32(); + for (int i = 0; i < 128; i++) { + expectEquals(0x3fffffff, a[i], "shr32"); + } + shr33(); + for (int i = 0; i < 128; i++) { + expectEquals(0x1fffffff, a[i], "shr33"); + } + not(); + for (int i = 0; i < 128; i++) { + expectEquals(0xe0000000, a[i], "not"); + } + // Done. + System.out.println("passed"); + } + + private static void expectEquals(int expected, int result, String action) { + if (expected != result) { + throw new Error("Expected: " + expected + ", found: " + result + " for " + action); + } + } +} diff --git a/test/640-checker-long-simd/expected.txt b/test/640-checker-long-simd/expected.txt new file mode 100644 index 0000000000..b0aad4deb5 --- /dev/null +++ b/test/640-checker-long-simd/expected.txt @@ -0,0 +1 @@ +passed diff --git a/test/640-checker-long-simd/info.txt b/test/640-checker-long-simd/info.txt new file mode 100644 index 0000000000..c9c6d5ed9f --- /dev/null +++ b/test/640-checker-long-simd/info.txt @@ -0,0 +1 @@ +Functional tests on SIMD vectorization. diff --git a/test/640-checker-long-simd/src/Main.java b/test/640-checker-long-simd/src/Main.java new file mode 100644 index 0000000000..90a2e76538 --- /dev/null +++ b/test/640-checker-long-simd/src/Main.java @@ -0,0 +1,244 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Functional tests for SIMD vectorization. + */ +public class Main { + + static long[] a; + + // + // Arithmetic operations. + // + + /// CHECK-START: void Main.add(long) loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.add(long) loop_optimization (after) + // + // TODO: fill in when supported + static void add(long x) { + for (int i = 0; i < 128; i++) + a[i] += x; + } + + /// CHECK-START: void Main.sub(long) loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.sub(long) loop_optimization (after) + // + // TODO: fill in when supported + static void sub(long x) { + for (int i = 0; i < 128; i++) + a[i] -= x; + } + + /// CHECK-START: void Main.mul(long) loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.mul(long) loop_optimization (after) + // + // TODO: fill in when supported + static void mul(long x) { + for (int i = 0; i < 128; i++) + a[i] *= x; + } + + /// CHECK-START: void Main.div(long) loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START: void Main.div(long) loop_optimization (after) + // + // Not supported on any architecture. + // + static void div(long x) { + for (int i = 0; i < 128; i++) + a[i] /= x; + } + + /// CHECK-START: void Main.neg() loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.neg() loop_optimization (after) + // + // TODO: fill in when supported + static void neg() { + for (int i = 0; i < 128; i++) + a[i] = -a[i]; + } + + /// CHECK-START: void Main.not() loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.not() loop_optimization (after) + // + // TODO: fill in when supported + static void not() { + for (int i = 0; i < 128; i++) + a[i] = ~a[i]; + } + + /// CHECK-START: void Main.shl4() loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.shl4() loop_optimization (after) + // + // TODO: fill in when supported + static void shl4() { + for (int i = 0; i < 128; i++) + a[i] <<= 4; + } + + /// CHECK-START: void Main.sar2() loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.sar2() loop_optimization (after) + // + // TODO: fill in when supported + static void sar2() { + for (int i = 0; i < 128; i++) + a[i] >>= 2; + } + + /// CHECK-START: void Main.shr2() loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.shr2() loop_optimization (after) + // + // TODO: fill in when supported + static void shr2() { + for (int i = 0; i < 128; i++) + a[i] >>>= 2; + } + + // + // Shift sanity. + // + + static void shr64() { + for (int i = 0; i < 128; i++) + a[i] >>>= 64; // 0, since & 63 + } + + static void shr65() { + for (int i = 0; i < 128; i++) + a[i] >>>= 65; // 1, since & 63 + } + + // + // Loop bounds. + // + + static void bounds() { + for (int i = 1; i < 127; i++) + a[i] += 11; + } + + // + // Test Driver. + // + + public static void main(String[] args) { + // Set up. + a = new long[128]; + for (int i = 0; i < 128; i++) { + a[i] = i; + } + // Arithmetic operations. + add(2L); + for (int i = 0; i < 128; i++) { + expectEquals(i + 2, a[i], "add"); + } + sub(2L); + for (int i = 0; i < 128; i++) { + expectEquals(i, a[i], "sub"); + } + mul(2L); + for (int i = 0; i < 128; i++) { + expectEquals(i + i, a[i], "mul"); + } + div(2L); + for (int i = 0; i < 128; i++) { + expectEquals(i, a[i], "div"); + } + neg(); + for (int i = 0; i < 128; i++) { + expectEquals(-i, a[i], "neg"); + } + // Loop bounds. + bounds(); + expectEquals(0, a[0], "bounds0"); + for (int i = 1; i < 127; i++) { + expectEquals(11 - i, a[i], "bounds"); + } + expectEquals(-127, a[127], "bounds127"); + // Shifts. + for (int i = 0; i < 128; i++) { + a[i] = 0xffffffffffffffffL; + } + shl4(); + for (int i = 0; i < 128; i++) { + expectEquals(0xfffffffffffffff0L, a[i], "shl4"); + } + sar2(); + for (int i = 0; i < 128; i++) { + expectEquals(0xfffffffffffffffcL, a[i], "sar2"); + } + shr2(); + for (int i = 0; i < 128; i++) { + expectEquals(0x3fffffffffffffffL, a[i], "shr2"); + } + shr64(); + for (int i = 0; i < 128; i++) { + expectEquals(0x3fffffffffffffffL, a[i], "shr64"); + } + shr65(); + for (int i = 0; i < 128; i++) { + expectEquals(0x1fffffffffffffffL, a[i], "shr65"); + } + not(); + for (int i = 0; i < 128; i++) { + expectEquals(0xe000000000000000L, a[i], "not"); + } + // Done. + System.out.println("passed"); + } + + private static void expectEquals(long expected, long result, String action) { + if (expected != result) { + throw new Error("Expected: " + expected + ", found: " + result + " for " + action); + } + } +} diff --git a/test/640-checker-short-simd/expected.txt b/test/640-checker-short-simd/expected.txt new file mode 100644 index 0000000000..b0aad4deb5 --- /dev/null +++ b/test/640-checker-short-simd/expected.txt @@ -0,0 +1 @@ +passed diff --git a/test/640-checker-short-simd/info.txt b/test/640-checker-short-simd/info.txt new file mode 100644 index 0000000000..c9c6d5ed9f --- /dev/null +++ b/test/640-checker-short-simd/info.txt @@ -0,0 +1 @@ +Functional tests on SIMD vectorization. diff --git a/test/640-checker-short-simd/src/Main.java b/test/640-checker-short-simd/src/Main.java new file mode 100644 index 0000000000..241f8e6eea --- /dev/null +++ b/test/640-checker-short-simd/src/Main.java @@ -0,0 +1,277 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Functional tests for SIMD vectorization. + */ +public class Main { + + static short[] a; + + // + // Arithmetic operations. + // + + /// CHECK-START: void Main.add(int) loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.add(int) loop_optimization (after) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: VecLoad loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecAdd loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore loop:<<Loop>> outer_loop:none + static void add(int x) { + for (int i = 0; i < 128; i++) + a[i] += x; + } + + /// CHECK-START: void Main.sub(int) loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.sub(int) loop_optimization (after) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: VecLoad loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecSub loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore loop:<<Loop>> outer_loop:none + static void sub(int x) { + for (int i = 0; i < 128; i++) + a[i] -= x; + } + + /// CHECK-START: void Main.mul(int) loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.mul(int) loop_optimization (after) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: VecLoad loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecMul loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore loop:<<Loop>> outer_loop:none + static void mul(int x) { + for (int i = 0; i < 128; i++) + a[i] *= x; + } + + /// CHECK-START: void Main.div(int) loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START: void Main.div(int) loop_optimization (after) + // + // Not supported on any architecture. + // + static void div(int x) { + for (int i = 0; i < 128; i++) + a[i] /= x; + } + + /// CHECK-START: void Main.neg() loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.neg() loop_optimization (after) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: VecLoad loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecNeg loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore loop:<<Loop>> outer_loop:none + static void neg() { + for (int i = 0; i < 128; i++) + a[i] = (short) -a[i]; + } + + /// CHECK-START: void Main.not() loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.not() loop_optimization (after) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: VecLoad loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecNot loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore loop:<<Loop>> outer_loop:none + static void not() { + for (int i = 0; i < 128; i++) + a[i] = (short) ~a[i]; + } + + /// CHECK-START: void Main.shl4() loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.shl4() loop_optimization (after) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: VecLoad loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecShl loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore loop:<<Loop>> outer_loop:none + static void shl4() { + for (int i = 0; i < 128; i++) + a[i] <<= 4; + } + + /// CHECK-START: void Main.sar2() loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.sar2() loop_optimization (after) + // + // TODO: fill in when supported + static void sar2() { + for (int i = 0; i < 128; i++) + a[i] >>= 2; + } + + /// CHECK-START: void Main.shr2() loop_optimization (before) + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + // + /// CHECK-START-ARM64: void Main.shr2() loop_optimization (after) + // + // TODO: fill in when supported + static void shr2() { + for (int i = 0; i < 128; i++) + a[i] >>>= 2; + } + + // + // Shift sanity. + // + + static void sar31() { + for (int i = 0; i < 128; i++) + a[i] >>= 31; + } + + static void shr31() { + for (int i = 0; i < 128; i++) + a[i] >>>= 31; + } + + static void shr32() { + for (int i = 0; i < 128; i++) + a[i] >>>= 32; // 0, since & 31 + } + + + static void shr33() { + for (int i = 0; i < 128; i++) + a[i] >>>= 33; // 1, since & 31 + } + + // + // Loop bounds. + // + + static void add() { + for (int i = 1; i < 127; i++) + a[i] += 11; + } + + // + // Test Driver. + // + + public static void main(String[] args) { + // Set up. + a = new short[128]; + for (int i = 0; i < 128; i++) { + a[i] = (short) i; + } + // Arithmetic operations. + add(2); + for (int i = 0; i < 128; i++) { + expectEquals(i + 2, a[i], "add"); + } + sub(2); + for (int i = 0; i < 128; i++) { + expectEquals(i, a[i], "sub"); + } + mul(2); + for (int i = 0; i < 128; i++) { + expectEquals(i + i, a[i], "mul"); + } + div(2); + for (int i = 0; i < 128; i++) { + expectEquals(i, a[i], "div"); + } + neg(); + for (int i = 0; i < 128; i++) { + expectEquals(-i, a[i], "neg"); + } + // Loop bounds. + add(); + expectEquals(0, a[0], "bounds0"); + for (int i = 1; i < 127; i++) { + expectEquals(11 - i, a[i], "bounds"); + } + expectEquals(-127, a[127], "bounds127"); + // Shifts. + for (int i = 0; i < 128; i++) { + a[i] = (short) 0xffff; + } + shl4(); + for (int i = 0; i < 128; i++) { + expectEquals((short) 0xfff0, a[i], "shl4"); + } + sar2(); + for (int i = 0; i < 128; i++) { + expectEquals((short) 0xfffc, a[i], "sar2"); + } + shr2(); + for (int i = 0; i < 128; i++) { + expectEquals((short) 0xffff, a[i], "shr2"); // sic! + } + sar31(); + for (int i = 0; i < 128; i++) { + expectEquals((short) 0xffff, a[i], "sar31"); + } + shr31(); + for (int i = 0; i < 128; i++) { + expectEquals(0x0001, a[i], "shr31"); + a[i] = (short) 0x1200; // reset + } + shr32(); + for (int i = 0; i < 128; i++) { + expectEquals((short) 0x1200, a[i], "shr32"); + } + shr33(); + for (int i = 0; i < 128; i++) { + expectEquals((short) 0x0900, a[i], "shr33"); + a[i] = (short) 0xf0f1; // reset + } + not(); + for (int i = 0; i < 128; i++) { + expectEquals((short) 0x0f0e, a[i], "not"); + } + // Done. + System.out.println("passed"); + } + + private static void expectEquals(int expected, int result, String action) { + if (expected != result) { + throw new Error("Expected: " + expected + ", found: " + result + " for " + action); + } + } +} diff --git a/test/643-checker-bogus-ic/expected.txt b/test/643-checker-bogus-ic/expected.txt new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/test/643-checker-bogus-ic/expected.txt diff --git a/test/643-checker-bogus-ic/info.txt b/test/643-checker-bogus-ic/info.txt new file mode 100644 index 0000000000..d5dfff48e4 --- /dev/null +++ b/test/643-checker-bogus-ic/info.txt @@ -0,0 +1 @@ +Verify the compiler can handle a bogus inline cache in a profile. diff --git a/test/643-checker-bogus-ic/profile b/test/643-checker-bogus-ic/profile new file mode 100644 index 0000000000..cbf77961c7 --- /dev/null +++ b/test/643-checker-bogus-ic/profile @@ -0,0 +1,2 @@ +LMain;->inlineMonomorphic(LMain;)I+LUnrelated; +LMain;->inlinePolymorphic(LMain;)I+LUnrelated;,LMain; diff --git a/test/643-checker-bogus-ic/run b/test/643-checker-bogus-ic/run new file mode 100644 index 0000000000..146e180000 --- /dev/null +++ b/test/643-checker-bogus-ic/run @@ -0,0 +1,17 @@ +#!/bin/bash +# +# Copyright (C) 2017 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +exec ${RUN} $@ --profile -Xcompiler-option --compiler-filter=speed-profile diff --git a/test/643-checker-bogus-ic/src/Main.java b/test/643-checker-bogus-ic/src/Main.java new file mode 100644 index 0000000000..0aa84772a8 --- /dev/null +++ b/test/643-checker-bogus-ic/src/Main.java @@ -0,0 +1,49 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +class Unrelated { +} + +public class Main { + + /// CHECK-START: int Main.inlineMonomorphic(Main) inliner (before) + /// CHECK: InvokeVirtual method_name:Main.getValue + + /// CHECK-START: int Main.inlineMonomorphic(Main) inliner (after) + /// CHECK: InvokeVirtual method_name:Main.getValue + + public static int inlineMonomorphic(Main a) { + return a.getValue(); + } + + /// CHECK-START: int Main.inlinePolymorphic(Main) inliner (before) + /// CHECK: InvokeVirtual method_name:Main.getValue + + /// CHECK-START: int Main.inlinePolymorphic(Main) inliner (after) + /// CHECK: InvokeVirtual method_name:Main.getValue + public static int inlinePolymorphic(Main a) { + return a.getValue(); + } + + public int getValue() { + return 42; + } + + public static void main(String[] args) { + inlineMonomorphic(new Main()); + } + +} diff --git a/test/644-checker-deopt/expected.txt b/test/644-checker-deopt/expected.txt new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/test/644-checker-deopt/expected.txt diff --git a/test/644-checker-deopt/info.txt b/test/644-checker-deopt/info.txt new file mode 100644 index 0000000000..c5fb12c570 --- /dev/null +++ b/test/644-checker-deopt/info.txt @@ -0,0 +1,2 @@ +Regression test for making sure HDeoptimize is executed before +the code it should have prevented executing. diff --git a/test/644-checker-deopt/profile b/test/644-checker-deopt/profile new file mode 100644 index 0000000000..cb261cc694 --- /dev/null +++ b/test/644-checker-deopt/profile @@ -0,0 +1,2 @@ +LMain;->inlineMonomorphic(LMain;)I+LMain; +LMain;->inlinePolymorphic(LMain;)I+LMain;,LSubMain; diff --git a/test/644-checker-deopt/run b/test/644-checker-deopt/run new file mode 100644 index 0000000000..146e180000 --- /dev/null +++ b/test/644-checker-deopt/run @@ -0,0 +1,17 @@ +#!/bin/bash +# +# Copyright (C) 2017 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +exec ${RUN} $@ --profile -Xcompiler-option --compiler-filter=speed-profile diff --git a/test/644-checker-deopt/src/Main.java b/test/644-checker-deopt/src/Main.java new file mode 100644 index 0000000000..17c80a6057 --- /dev/null +++ b/test/644-checker-deopt/src/Main.java @@ -0,0 +1,74 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +public class Main { + + /// CHECK-START: int Main.inlineMonomorphic(Main) inliner (before) + /// CHECK: InvokeVirtual method_name:Main.getValue + + /// CHECK-START: int Main.inlineMonomorphic(Main) inliner (after) + /// CHECK-NOT: InvokeVirtual method_name:Main.getValue + + /// CHECK-START: int Main.inlineMonomorphic(Main) licm (before) + /// CHECK: <<Deopt:l\d+>> Deoptimize + /// CHECK: InstanceFieldGet [<<Deopt>>] field_name:Main.value + + /// CHECK-START: int Main.inlineMonomorphic(Main) licm (after) + /// CHECK: <<Deopt:l\d+>> Deoptimize + /// CHECK: InstanceFieldGet [<<Deopt>>] field_name:Main.value + + public static int inlineMonomorphic(Main a) { + if (a == null) { + return 42; + } + int i = 0; + while (i < 100) { + i += a.getValue(); + } + return i; + } + + /// CHECK-START: int Main.inlinePolymorphic(Main) inliner (before) + /// CHECK: InvokeVirtual method_name:Main.getValue + + /// CHECK-START: int Main.inlinePolymorphic(Main) inliner (after) + /// CHECK-NOT: InvokeVirtual method_name:Main.getValue + + /// CHECK-START: int Main.inlineMonomorphic(Main) licm (before) + /// CHECK: <<Deopt:l\d+>> Deoptimize + /// CHECK: InstanceFieldGet [<<Deopt>>] field_name:Main.value + + /// CHECK-START: int Main.inlineMonomorphic(Main) licm (after) + /// CHECK: <<Deopt:l\d+>> Deoptimize + /// CHECK: InstanceFieldGet [<<Deopt>>] field_name:Main.value + public static int inlinePolymorphic(Main a) { + return a.getValue(); + } + + public int getValue() { + return value; + } + + public static void main(String[] args) { + inlineMonomorphic(new Main()); + } + + int value = 1; +} + +// Add a subclass of 'Main' to write the polymorphic inline cache in the profile. +class SubMain extends Main { +} diff --git a/test/900-hello-plugin/expected.txt b/test/900-hello-plugin/expected.txt index 43db31c722..c160f65d31 100644 --- a/test/900-hello-plugin/expected.txt +++ b/test/900-hello-plugin/expected.txt @@ -3,6 +3,8 @@ Agent_OnLoad called with options "test_900" GetEnvHandler called in test 900 GetEnvHandler called with version 0x900fffff GetEnv returned '900' environment! +Agent_OnLoad called with options "test_900_round_2" Hello, world! Agent_OnUnload called +Agent_OnUnload called ArtPlugin_Deinitialize called in test 900 diff --git a/test/900-hello-plugin/load_unload.cc b/test/900-hello-plugin/load_unload.cc index a38cc3d6ac..290997aa08 100644 --- a/test/900-hello-plugin/load_unload.cc +++ b/test/900-hello-plugin/load_unload.cc @@ -52,6 +52,9 @@ extern "C" JNIEXPORT jint JNICALL Agent_OnLoad(JavaVM* vm, char* options, void* reserved ATTRIBUTE_UNUSED) { printf("Agent_OnLoad called with options \"%s\"\n", options); + if (strcmp("test_900_round_2", options) == 0) { + return 0; + } uintptr_t env = 0; jint res = vm->GetEnv(reinterpret_cast<void**>(&env), TEST_900_ENV_VERSION_NUMBER); if (res != JNI_OK) { diff --git a/test/900-hello-plugin/run b/test/900-hello-plugin/run index 50835f89af..c633f6df5d 100755 --- a/test/900-hello-plugin/run +++ b/test/900-hello-plugin/run @@ -19,4 +19,5 @@ if [[ "$@" == *"-O"* ]]; then plugin=libartagent.so fi ./default-run "$@" --runtime-option -agentpath:${plugin}=test_900 \ + --runtime-option -agentpath:${plugin}=test_900_round_2 \ --android-runtime-option -Xplugin:${plugin} diff --git a/test/901-hello-ti-agent/basics.cc b/test/901-hello-ti-agent/basics.cc index 91662770be..20b227ab24 100644 --- a/test/901-hello-ti-agent/basics.cc +++ b/test/901-hello-ti-agent/basics.cc @@ -16,14 +16,17 @@ #include "901-hello-ti-agent/basics.h" +#include <thread> + #include <jni.h> #include <stdio.h> #include <string.h> -#include "base/macros.h" +#include "android-base/macros.h" #include "jvmti.h" -#include "ti-agent/common_helper.h" -#include "ti-agent/common_load.h" +// Test infrastructure +#include "jvmti_helper.h" +#include "test_env.h" namespace art { namespace Test901HelloTi { @@ -146,18 +149,32 @@ extern "C" JNIEXPORT void JNICALL Java_Main_setVerboseFlag( JNIEnv* env, jclass Main_klass ATTRIBUTE_UNUSED, jint iflag, jboolean val) { jvmtiVerboseFlag flag = static_cast<jvmtiVerboseFlag>(iflag); jvmtiError result = jvmti_env->SetVerboseFlag(flag, val); - JvmtiErrorToException(env, result); + JvmtiErrorToException(env, jvmti_env, result); } extern "C" JNIEXPORT jboolean JNICALL Java_Main_checkLivePhase( JNIEnv* env, jclass Main_klass ATTRIBUTE_UNUSED) { jvmtiPhase current_phase; jvmtiError phase_result = jvmti_env->GetPhase(¤t_phase); - if (JvmtiErrorToException(env, phase_result)) { + if (JvmtiErrorToException(env, jvmti_env, phase_result)) { return JNI_FALSE; } return (current_phase == JVMTI_PHASE_LIVE) ? JNI_TRUE : JNI_FALSE; } +static void CallJvmtiFunction(jvmtiEnv* env, jclass klass, jvmtiError* err) { + jint n; + jmethodID* methods = nullptr; + *err = env->GetClassMethods(klass, &n, &methods); +} + +extern "C" JNIEXPORT jboolean JNICALL Java_Main_checkUnattached( + JNIEnv* env ATTRIBUTE_UNUSED, jclass Main_klass) { + jvmtiError res = JVMTI_ERROR_NONE; + std::thread t1(CallJvmtiFunction, jvmti_env, Main_klass, &res); + t1.join(); + return res == JVMTI_ERROR_UNATTACHED_THREAD; +} + } // namespace Test901HelloTi } // namespace art diff --git a/test/901-hello-ti-agent/expected.txt b/test/901-hello-ti-agent/expected.txt index c4b24cba90..eb5b6a2f93 100644 --- a/test/901-hello-ti-agent/expected.txt +++ b/test/901-hello-ti-agent/expected.txt @@ -3,6 +3,7 @@ VMStart VMInit Hello, world! Agent in live phase. +Received expected error for unattached JVMTI calls 0 1 2 diff --git a/test/901-hello-ti-agent/src/Main.java b/test/901-hello-ti-agent/src/Main.java index 4d62ed3f5d..556e05b5d0 100644 --- a/test/901-hello-ti-agent/src/Main.java +++ b/test/901-hello-ti-agent/src/Main.java @@ -21,6 +21,9 @@ public class Main { if (checkLivePhase()) { System.out.println("Agent in live phase."); } + if (checkUnattached()) { + System.out.println("Received expected error for unattached JVMTI calls"); + } set(0); // OTHER set(1); // GC @@ -41,4 +44,5 @@ public class Main { private static native boolean checkLivePhase(); private static native void setVerboseFlag(int flag, boolean value); + private static native boolean checkUnattached(); } diff --git a/test/903-hello-tagging/tagging.cc b/test/903-hello-tagging/tagging.cc index b85ed48930..701b0c3817 100644 --- a/test/903-hello-tagging/tagging.cc +++ b/test/903-hello-tagging/tagging.cc @@ -19,43 +19,30 @@ #include <stdio.h> #include <vector> +#include "android-base/logging.h" #include "jni.h" -#include "ScopedLocalRef.h" -#include "ScopedPrimitiveArray.h" +#include "scoped_local_ref.h" +#include "scoped_primitive_array.h" -#include "art_method-inl.h" -#include "base/logging.h" #include "jvmti.h" -#include "ti-agent/common_helper.h" -#include "ti-agent/common_load.h" -#include "utils.h" + +// Test infrastructure +#include "jvmti_helper.h" +#include "test_env.h" namespace art { namespace Test903HelloTagging { -extern "C" JNIEXPORT void JNICALL Java_Main_setTag(JNIEnv* env ATTRIBUTE_UNUSED, - jclass, - jobject obj, - jlong tag) { +extern "C" JNIEXPORT void JNICALL Java_Main_setTag(JNIEnv* env, jclass, jobject obj, jlong tag) { jvmtiError ret = jvmti_env->SetTag(obj, tag); - if (ret != JVMTI_ERROR_NONE) { - char* err; - jvmti_env->GetErrorName(ret, &err); - printf("Error setting tag: %s\n", err); - jvmti_env->Deallocate(reinterpret_cast<unsigned char*>(err)); - } + JvmtiErrorToException(env, jvmti_env, ret); } -extern "C" JNIEXPORT jlong JNICALL Java_Main_getTag(JNIEnv* env ATTRIBUTE_UNUSED, - jclass, - jobject obj) { +extern "C" JNIEXPORT jlong JNICALL Java_Main_getTag(JNIEnv* env, jclass, jobject obj) { jlong tag = 0; jvmtiError ret = jvmti_env->GetTag(obj, &tag); - if (ret != JVMTI_ERROR_NONE) { - char* err; - jvmti_env->GetErrorName(ret, &err); - printf("Error getting tag: %s\n", err); - jvmti_env->Deallocate(reinterpret_cast<unsigned char*>(err)); + if (JvmtiErrorToException(env, jvmti_env, ret)) { + return 0; } return tag; } @@ -86,11 +73,7 @@ extern "C" JNIEXPORT jobjectArray JNICALL Java_Main_getTaggedObjects(JNIEnv* env &result_count, result_object_array_ptr, result_tag_array_ptr); - if (ret != JVMTI_ERROR_NONE) { - char* err; - jvmti_env->GetErrorName(ret, &err); - printf("Failure running GetLoadedClasses: %s\n", err); - jvmti_env->Deallocate(reinterpret_cast<unsigned char*>(err)); + if (JvmtiErrorToException(env, jvmti_env, ret)) { return nullptr; } @@ -197,4 +180,3 @@ extern "C" JNIEXPORT jlongArray JNICALL Java_Main_testTagsInDifferentEnvs( } // namespace Test903HelloTagging } // namespace art - diff --git a/test/904-object-allocation/tracking.cc b/test/904-object-allocation/tracking.cc index cc6f681d79..c829496975 100644 --- a/test/904-object-allocation/tracking.cc +++ b/test/904-object-allocation/tracking.cc @@ -19,14 +19,15 @@ #include <stdio.h> #include <vector> -#include "base/logging.h" +#include "android-base/logging.h" #include "jni.h" #include "jvmti.h" -#include "ScopedLocalRef.h" -#include "ScopedUtfChars.h" -#include "ti-agent/common_helper.h" -#include "ti-agent/common_load.h" -#include "utils.h" +#include "scoped_local_ref.h" +#include "scoped_utf_chars.h" + +// Test infrastructure +#include "jvmti_helper.h" +#include "test_env.h" namespace art { namespace Test904ObjectAllocation { @@ -57,21 +58,16 @@ static void JNICALL ObjectAllocated(jvmtiEnv* ti_env ATTRIBUTE_UNUSED, } extern "C" JNIEXPORT void JNICALL Java_Main_setupObjectAllocCallback( - JNIEnv* env ATTRIBUTE_UNUSED, jclass klass ATTRIBUTE_UNUSED, jboolean enable) { + JNIEnv* env, jclass klass ATTRIBUTE_UNUSED, jboolean enable) { jvmtiEventCallbacks callbacks; memset(&callbacks, 0, sizeof(jvmtiEventCallbacks)); callbacks.VMObjectAlloc = enable ? ObjectAllocated : nullptr; jvmtiError ret = jvmti_env->SetEventCallbacks(&callbacks, sizeof(callbacks)); - if (ret != JVMTI_ERROR_NONE) { - char* err; - jvmti_env->GetErrorName(ret, &err); - printf("Error setting callbacks: %s\n", err); - jvmti_env->Deallocate(reinterpret_cast<unsigned char*>(err)); - } + JvmtiErrorToException(env, jvmti_env, ret); } -extern "C" JNIEXPORT void JNICALL Java_Main_enableAllocationTracking(JNIEnv* env ATTRIBUTE_UNUSED, +extern "C" JNIEXPORT void JNICALL Java_Main_enableAllocationTracking(JNIEnv* env, jclass, jthread thread, jboolean enable) { @@ -79,14 +75,8 @@ extern "C" JNIEXPORT void JNICALL Java_Main_enableAllocationTracking(JNIEnv* env enable ? JVMTI_ENABLE : JVMTI_DISABLE, JVMTI_EVENT_VM_OBJECT_ALLOC, thread); - if (ret != JVMTI_ERROR_NONE) { - char* err; - jvmti_env->GetErrorName(ret, &err); - printf("Error enabling/disabling allocation tracking: %s\n", err); - jvmti_env->Deallocate(reinterpret_cast<unsigned char*>(err)); - } + JvmtiErrorToException(env, jvmti_env, ret); } } // namespace Test904ObjectAllocation } // namespace art - diff --git a/test/905-object-free/expected.txt b/test/905-object-free/expected.txt index 436ca115b1..c226df7b5a 100644 --- a/test/905-object-free/expected.txt +++ b/test/905-object-free/expected.txt @@ -10,3 +10,4 @@ --- [] --- +Free counts 100000 100000 diff --git a/test/905-object-free/src/Main.java b/test/905-object-free/src/Main.java index e41e378c19..0d576291cb 100644 --- a/test/905-object-free/src/Main.java +++ b/test/905-object-free/src/Main.java @@ -33,6 +33,9 @@ public class Main { enableFreeTracking(false); run(l); + + enableFreeTracking(true); + stress(); } private static void run(ArrayList<Object> l) { @@ -62,6 +65,30 @@ public class Main { System.out.println("---"); } + private static void stressAllocate(int i) { + Object obj = new Object(); + setTag(obj, i); + setTag2(obj, i + 1); + } + + private static void stress() { + getCollectedTags(0); + getCollectedTags(1); + // Allocate objects. + for (int i = 1; i <= 100000; ++i) { + stressAllocate(i); + } + Runtime.getRuntime().gc(); + long[] freedTags1 = getCollectedTags(0); + long[] freedTags2 = getCollectedTags(1); + System.out.println("Free counts " + freedTags1.length + " " + freedTags2.length); + for (int i = 0; i < freedTags1.length; ++i) { + if (freedTags1[i] + 1 != freedTags2[i]) { + System.out.println("Mismatched tags " + freedTags1[i] + " " + freedTags2[i]); + } + } + } + private static void allocate(ArrayList<Object> l, long tag) { Object obj = new Object(); l.add(obj); @@ -69,7 +96,7 @@ public class Main { } private static void getAndPrintTags() { - long[] freedTags = getCollectedTags(); + long[] freedTags = getCollectedTags(0); Arrays.sort(freedTags); System.out.println(Arrays.toString(freedTags)); } @@ -77,5 +104,6 @@ public class Main { private static native void setupObjectFreeCallback(); private static native void enableFreeTracking(boolean enable); private static native void setTag(Object o, long tag); - private static native long[] getCollectedTags(); + private static native long[] getCollectedTags(int index); + private static native void setTag2(Object o, long tag); } diff --git a/test/905-object-free/tracking_free.cc b/test/905-object-free/tracking_free.cc index 5eed4729af..59b429ca40 100644 --- a/test/905-object-free/tracking_free.cc +++ b/test/905-object-free/tracking_free.cc @@ -19,66 +19,91 @@ #include <stdio.h> #include <vector> -#include "base/logging.h" +#include "android-base/logging.h" #include "jni.h" #include "jvmti.h" -#include "ScopedLocalRef.h" -#include "ScopedUtfChars.h" -#include "ti-agent/common_helper.h" -#include "ti-agent/common_load.h" -#include "utils.h" +#include "scoped_local_ref.h" +#include "scoped_utf_chars.h" + +// Test infrastructure +#include "jvmti_helper.h" +#include "test_env.h" namespace art { namespace Test905ObjectFree { -static std::vector<jlong> collected_tags; +static std::vector<jlong> collected_tags1; +static std::vector<jlong> collected_tags2; + +jvmtiEnv* jvmti_env2; -static void JNICALL ObjectFree(jvmtiEnv* ti_env ATTRIBUTE_UNUSED, jlong tag) { - collected_tags.push_back(tag); +static void JNICALL ObjectFree1(jvmtiEnv* ti_env, jlong tag) { + CHECK_EQ(ti_env, jvmti_env); + collected_tags1.push_back(tag); } -extern "C" JNIEXPORT void JNICALL Java_Main_setupObjectFreeCallback( - JNIEnv* env ATTRIBUTE_UNUSED, jclass klass ATTRIBUTE_UNUSED) { +static void JNICALL ObjectFree2(jvmtiEnv* ti_env, jlong tag) { + CHECK_EQ(ti_env, jvmti_env2); + collected_tags2.push_back(tag); +} + +static void setupObjectFreeCallback(JNIEnv* env, jvmtiEnv* jenv, jvmtiEventObjectFree callback) { jvmtiEventCallbacks callbacks; memset(&callbacks, 0, sizeof(jvmtiEventCallbacks)); - callbacks.ObjectFree = ObjectFree; - - jvmtiError ret = jvmti_env->SetEventCallbacks(&callbacks, sizeof(callbacks)); - if (ret != JVMTI_ERROR_NONE) { - char* err; - jvmti_env->GetErrorName(ret, &err); - printf("Error setting callbacks: %s\n", err); - jvmti_env->Deallocate(reinterpret_cast<unsigned char*>(err)); - } + callbacks.ObjectFree = callback; + jvmtiError ret = jenv->SetEventCallbacks(&callbacks, sizeof(callbacks)); + JvmtiErrorToException(env, jenv, ret); +} + +extern "C" JNIEXPORT void JNICALL Java_Main_setupObjectFreeCallback( + JNIEnv* env, jclass klass ATTRIBUTE_UNUSED) { + setupObjectFreeCallback(env, jvmti_env, ObjectFree1); + JavaVM* jvm = nullptr; + env->GetJavaVM(&jvm); + CHECK_EQ(jvm->GetEnv(reinterpret_cast<void**>(&jvmti_env2), JVMTI_VERSION_1_2), 0); + SetAllCapabilities(jvmti_env2); + setupObjectFreeCallback(env, jvmti_env2, ObjectFree2); } -extern "C" JNIEXPORT void JNICALL Java_Main_enableFreeTracking(JNIEnv* env ATTRIBUTE_UNUSED, +extern "C" JNIEXPORT void JNICALL Java_Main_enableFreeTracking(JNIEnv* env, jclass klass ATTRIBUTE_UNUSED, jboolean enable) { jvmtiError ret = jvmti_env->SetEventNotificationMode( enable ? JVMTI_ENABLE : JVMTI_DISABLE, JVMTI_EVENT_OBJECT_FREE, nullptr); - if (ret != JVMTI_ERROR_NONE) { - char* err; - jvmti_env->GetErrorName(ret, &err); - printf("Error enabling/disabling object-free callbacks: %s\n", err); - jvmti_env->Deallocate(reinterpret_cast<unsigned char*>(err)); + if (JvmtiErrorToException(env, jvmti_env, ret)) { + return; } + ret = jvmti_env2->SetEventNotificationMode( + enable ? JVMTI_ENABLE : JVMTI_DISABLE, + JVMTI_EVENT_OBJECT_FREE, + nullptr); + JvmtiErrorToException(env, jvmti_env, ret); } extern "C" JNIEXPORT jlongArray JNICALL Java_Main_getCollectedTags(JNIEnv* env, - jclass klass ATTRIBUTE_UNUSED) { - jlongArray ret = env->NewLongArray(collected_tags.size()); + jclass klass ATTRIBUTE_UNUSED, + jint index) { + std::vector<jlong>& tags = (index == 0) ? collected_tags1 : collected_tags2; + jlongArray ret = env->NewLongArray(tags.size()); if (ret == nullptr) { return ret; } - env->SetLongArrayRegion(ret, 0, collected_tags.size(), collected_tags.data()); - collected_tags.clear(); + env->SetLongArrayRegion(ret, 0, tags.size(), tags.data()); + tags.clear(); return ret; } +extern "C" JNIEXPORT void JNICALL Java_Main_setTag2(JNIEnv* env, + jclass klass ATTRIBUTE_UNUSED, + jobject obj, + jlong tag) { + jvmtiError ret = jvmti_env2->SetTag(obj, tag); + JvmtiErrorToException(env, jvmti_env, ret); +} + } // namespace Test905ObjectFree } // namespace art diff --git a/test/906-iterate-heap/iterate_heap.cc b/test/906-iterate-heap/iterate_heap.cc index f2532debfb..bb30074958 100644 --- a/test/906-iterate-heap/iterate_heap.cc +++ b/test/906-iterate-heap/iterate_heap.cc @@ -23,14 +23,18 @@ #include <stdio.h> #include <vector> +#include "android-base/logging.h" #include "android-base/stringprintf.h" -#include "base/logging.h" + #include "jni.h" #include "jvmti.h" -#include "ScopedPrimitiveArray.h" -#include "ti-agent/common_helper.h" -#include "ti-agent/common_load.h" -#include "utf.h" +#include "scoped_primitive_array.h" + +// Test infrastructure +#include "jvmti_helper.h" +#include "test_env.h" +#include "ti_macros.h" +#include "ti_utf.h" namespace art { namespace Test906IterateHeap { @@ -52,7 +56,7 @@ static jint JNICALL HeapIterationCallback(jlong class_tag, return config->Handle(class_tag, size, tag_ptr, length); } -static bool Run(jint heap_filter, jclass klass_filter, IterationConfig* config) { +static bool Run(JNIEnv* env, jint heap_filter, jclass klass_filter, IterationConfig* config) { jvmtiHeapCallbacks callbacks; memset(&callbacks, 0, sizeof(jvmtiHeapCallbacks)); callbacks.heap_iteration_callback = HeapIterationCallback; @@ -61,17 +65,13 @@ static bool Run(jint heap_filter, jclass klass_filter, IterationConfig* config) klass_filter, &callbacks, config); - if (ret != JVMTI_ERROR_NONE) { - char* err; - jvmti_env->GetErrorName(ret, &err); - printf("Failure running IterateThroughHeap: %s\n", err); - jvmti_env->Deallocate(reinterpret_cast<unsigned char*>(err)); + if (JvmtiErrorToException(env, jvmti_env, ret)) { return false; } return true; } -extern "C" JNIEXPORT jint JNICALL Java_Main_iterateThroughHeapCount(JNIEnv* env ATTRIBUTE_UNUSED, +extern "C" JNIEXPORT jint JNICALL Java_Main_iterateThroughHeapCount(JNIEnv* env, jclass klass ATTRIBUTE_UNUSED, jint heap_filter, jclass klass_filter, @@ -99,7 +99,7 @@ extern "C" JNIEXPORT jint JNICALL Java_Main_iterateThroughHeapCount(JNIEnv* env }; CountIterationConfig config(0, stop_after); - Run(heap_filter, klass_filter, &config); + Run(env, heap_filter, klass_filter, &config); if (config.counter > config.stop_after) { printf("Error: more objects visited than signaled."); @@ -135,7 +135,7 @@ extern "C" JNIEXPORT jint JNICALL Java_Main_iterateThroughHeapData(JNIEnv* env, }; DataIterationConfig config; - if (!Run(heap_filter, klass_filter, &config)) { + if (!Run(env, heap_filter, klass_filter, &config)) { return -1; } @@ -154,7 +154,7 @@ extern "C" JNIEXPORT jint JNICALL Java_Main_iterateThroughHeapData(JNIEnv* env, return static_cast<jint>(config.class_tags_.size()); } -extern "C" JNIEXPORT void JNICALL Java_Main_iterateThroughHeapAdd(JNIEnv* env ATTRIBUTE_UNUSED, +extern "C" JNIEXPORT void JNICALL Java_Main_iterateThroughHeapAdd(JNIEnv* env, jclass klass ATTRIBUTE_UNUSED, jint heap_filter, jclass klass_filter) { @@ -175,7 +175,7 @@ extern "C" JNIEXPORT void JNICALL Java_Main_iterateThroughHeapAdd(JNIEnv* env AT }; AddIterationConfig config; - Run(heap_filter, klass_filter, &config); + Run(env, heap_filter, klass_filter, &config); } extern "C" JNIEXPORT jstring JNICALL Java_Main_iterateThroughHeapString( @@ -199,10 +199,10 @@ extern "C" JNIEXPORT jstring JNICALL Java_Main_iterateThroughHeapString( void* user_data) { FindStringCallbacks* p = reinterpret_cast<FindStringCallbacks*>(user_data); if (*tag_ptr == p->tag_to_find) { - size_t utf_byte_count = CountUtf8Bytes(value, value_length); + size_t utf_byte_count = ti::CountUtf8Bytes(value, value_length); std::unique_ptr<char[]> mod_utf(new char[utf_byte_count + 1]); memset(mod_utf.get(), 0, utf_byte_count + 1); - ConvertUtf16ToModifiedUtf8(mod_utf.get(), utf_byte_count, value, value_length); + ti::ConvertUtf16ToModifiedUtf8(mod_utf.get(), utf_byte_count, value, value_length); if (!p->data.empty()) { p->data += "\n"; } @@ -228,7 +228,7 @@ extern "C" JNIEXPORT jstring JNICALL Java_Main_iterateThroughHeapString( FindStringCallbacks fsc(tag); jvmtiError ret = jvmti_env->IterateThroughHeap(0, nullptr, &callbacks, &fsc); - if (JvmtiErrorToException(env, ret)) { + if (JvmtiErrorToException(env, jvmti_env, ret)) { return nullptr; } return env->NewStringUTF(fsc.data.c_str()); @@ -316,7 +316,7 @@ extern "C" JNIEXPORT jstring JNICALL Java_Main_iterateThroughHeapPrimitiveArray( FindArrayCallbacks fac(tag); jvmtiError ret = jvmti_env->IterateThroughHeap(0, nullptr, &callbacks, &fac); - if (JvmtiErrorToException(env, ret)) { + if (JvmtiErrorToException(env, jvmti_env, ret)) { return nullptr; } return env->NewStringUTF(fac.data.c_str()); @@ -403,7 +403,7 @@ extern "C" JNIEXPORT jstring JNICALL Java_Main_iterateThroughHeapPrimitiveFields FindFieldCallbacks ffc(tag); jvmtiError ret = jvmti_env->IterateThroughHeap(0, nullptr, &callbacks, &ffc); - if (JvmtiErrorToException(env, ret)) { + if (JvmtiErrorToException(env, jvmti_env, ret)) { return nullptr; } return env->NewStringUTF(ffc.data.c_str()); diff --git a/test/907-get-loaded-classes/get_loaded_classes.cc b/test/907-get-loaded-classes/get_loaded_classes.cc index 48ce2e2de1..5ec56c4997 100644 --- a/test/907-get-loaded-classes/get_loaded_classes.cc +++ b/test/907-get-loaded-classes/get_loaded_classes.cc @@ -19,14 +19,16 @@ #include <stdio.h> #include <vector> -#include "base/macros.h" +#include "android-base/macros.h" + #include "jni.h" #include "jvmti.h" -#include "ScopedLocalRef.h" -#include "ScopedUtfChars.h" +#include "scoped_local_ref.h" +#include "scoped_utf_chars.h" -#include "ti-agent/common_helper.h" -#include "ti-agent/common_load.h" +// Test infrastructure +#include "jni_helper.h" +#include "test_env.h" namespace art { namespace Test907GetLoadedClasses { diff --git a/test/908-gc-start-finish/gc_callbacks.cc b/test/908-gc-start-finish/gc_callbacks.cc index 45148f87cd..f186895f84 100644 --- a/test/908-gc-start-finish/gc_callbacks.cc +++ b/test/908-gc-start-finish/gc_callbacks.cc @@ -17,11 +17,14 @@ #include <stdio.h> #include <string.h> -#include "base/macros.h" +#include "android-base/macros.h" + #include "jni.h" #include "jvmti.h" -#include "ti-agent/common_helper.h" -#include "ti-agent/common_load.h" + +// Test infrastructure +#include "jvmti_helper.h" +#include "test_env.h" namespace art { namespace Test908GcStartFinish { @@ -45,7 +48,7 @@ extern "C" JNIEXPORT void JNICALL Java_Main_setupGcCallback( callbacks.GarbageCollectionStart = GarbageCollectionStart; jvmtiError ret = jvmti_env->SetEventCallbacks(&callbacks, sizeof(callbacks)); - JvmtiErrorToException(env, ret); + JvmtiErrorToException(env, jvmti_env, ret); } extern "C" JNIEXPORT void JNICALL Java_Main_enableGcTracking(JNIEnv* env, @@ -55,14 +58,14 @@ extern "C" JNIEXPORT void JNICALL Java_Main_enableGcTracking(JNIEnv* env, enable ? JVMTI_ENABLE : JVMTI_DISABLE, JVMTI_EVENT_GARBAGE_COLLECTION_START, nullptr); - if (JvmtiErrorToException(env, ret)) { + if (JvmtiErrorToException(env, jvmti_env, ret)) { return; } ret = jvmti_env->SetEventNotificationMode( enable ? JVMTI_ENABLE : JVMTI_DISABLE, JVMTI_EVENT_GARBAGE_COLLECTION_FINISH, nullptr); - if (JvmtiErrorToException(env, ret)) { + if (JvmtiErrorToException(env, jvmti_env, ret)) { return; } } diff --git a/test/909-attach-agent/attach.cc b/test/909-attach-agent/attach.cc index 67c756745f..0150e0962f 100644 --- a/test/909-attach-agent/attach.cc +++ b/test/909-attach-agent/attach.cc @@ -19,7 +19,9 @@ #include <jni.h> #include <stdio.h> #include <string.h> -#include "base/macros.h" + +#include "android-base/macros.h" + #include "jvmti.h" namespace art { diff --git a/test/910-methods/methods.cc b/test/910-methods/methods.cc index fdc4cdbe04..ded4f09df7 100644 --- a/test/910-methods/methods.cc +++ b/test/910-methods/methods.cc @@ -16,13 +16,16 @@ #include <stdio.h> -#include "base/macros.h" +#include "android-base/macros.h" + #include "jni.h" #include "jvmti.h" -#include "ScopedLocalRef.h" +#include "scoped_local_ref.h" -#include "ti-agent/common_helper.h" -#include "ti-agent/common_load.h" +// Test infrastructure +#include "jni_helper.h" +#include "jvmti_helper.h" +#include "test_env.h" namespace art { namespace Test910Methods { @@ -35,11 +38,7 @@ extern "C" JNIEXPORT jobjectArray JNICALL Java_Main_getMethodName( char* sig; char* gen; jvmtiError result = jvmti_env->GetMethodName(id, &name, &sig, &gen); - if (result != JVMTI_ERROR_NONE) { - char* err; - jvmti_env->GetErrorName(result, &err); - printf("Failure running GetMethodName: %s\n", err); - jvmti_env->Deallocate(reinterpret_cast<unsigned char*>(err)); + if (JvmtiErrorToException(env, jvmti_env, result)) { return nullptr; } @@ -67,11 +66,7 @@ extern "C" JNIEXPORT jobjectArray JNICALL Java_Main_getMethodName( // Also run GetMethodName with all parameter pointers null to check for segfaults. jvmtiError result2 = jvmti_env->GetMethodName(id, nullptr, nullptr, nullptr); - if (result2 != JVMTI_ERROR_NONE) { - char* err; - jvmti_env->GetErrorName(result2, &err); - printf("Failure running GetMethodName(null, null, null): %s\n", err); - jvmti_env->Deallocate(reinterpret_cast<unsigned char*>(err)); + if (JvmtiErrorToException(env, jvmti_env, result2)) { return nullptr; } @@ -84,11 +79,7 @@ extern "C" JNIEXPORT jclass JNICALL Java_Main_getMethodDeclaringClass( jclass declaring_class; jvmtiError result = jvmti_env->GetMethodDeclaringClass(id, &declaring_class); - if (result != JVMTI_ERROR_NONE) { - char* err; - jvmti_env->GetErrorName(result, &err); - printf("Failure running GetMethodDeclaringClass: %s\n", err); - jvmti_env->Deallocate(reinterpret_cast<unsigned char*>(err)); + if (JvmtiErrorToException(env, jvmti_env, result)) { return nullptr; } @@ -101,11 +92,7 @@ extern "C" JNIEXPORT jint JNICALL Java_Main_getMethodModifiers( jint modifiers; jvmtiError result = jvmti_env->GetMethodModifiers(id, &modifiers); - if (result != JVMTI_ERROR_NONE) { - char* err; - jvmti_env->GetErrorName(result, &err); - printf("Failure running GetMethodModifiers: %s\n", err); - jvmti_env->Deallocate(reinterpret_cast<unsigned char*>(err)); + if (JvmtiErrorToException(env, jvmti_env, result)) { return 0; } @@ -118,7 +105,7 @@ extern "C" JNIEXPORT jint JNICALL Java_Main_getMaxLocals( jint max_locals; jvmtiError result = jvmti_env->GetMaxLocals(id, &max_locals); - if (JvmtiErrorToException(env, result)) { + if (JvmtiErrorToException(env, jvmti_env, result)) { return -1; } @@ -131,7 +118,7 @@ extern "C" JNIEXPORT jint JNICALL Java_Main_getArgumentsSize( jint arguments; jvmtiError result = jvmti_env->GetArgumentsSize(id, &arguments); - if (JvmtiErrorToException(env, result)) { + if (JvmtiErrorToException(env, jvmti_env, result)) { return -1; } @@ -145,7 +132,7 @@ extern "C" JNIEXPORT jlong JNICALL Java_Main_getMethodLocationStart( jlong start; jlong end; jvmtiError result = jvmti_env->GetMethodLocation(id, &start, &end); - if (JvmtiErrorToException(env, result)) { + if (JvmtiErrorToException(env, jvmti_env, result)) { return -1; } @@ -159,7 +146,7 @@ extern "C" JNIEXPORT jlong JNICALL Java_Main_getMethodLocationEnd( jlong start; jlong end; jvmtiError result = jvmti_env->GetMethodLocation(id, &start, &end); - if (JvmtiErrorToException(env, result)) { + if (JvmtiErrorToException(env, jvmti_env, result)) { return -1; } @@ -172,7 +159,7 @@ extern "C" JNIEXPORT jboolean JNICALL Java_Main_isMethodNative( jboolean is_native; jvmtiError result = jvmti_env->IsMethodNative(id, &is_native); - if (JvmtiErrorToException(env, result)) { + if (JvmtiErrorToException(env, jvmti_env, result)) { return JNI_FALSE; } @@ -185,7 +172,7 @@ extern "C" JNIEXPORT jboolean JNICALL Java_Main_isMethodObsolete( jboolean is_obsolete; jvmtiError result = jvmti_env->IsMethodObsolete(id, &is_obsolete); - if (JvmtiErrorToException(env, result)) { + if (JvmtiErrorToException(env, jvmti_env, result)) { return JNI_FALSE; } @@ -198,7 +185,7 @@ extern "C" JNIEXPORT jboolean JNICALL Java_Main_isMethodSynthetic( jboolean is_synthetic; jvmtiError result = jvmti_env->IsMethodSynthetic(id, &is_synthetic); - if (JvmtiErrorToException(env, result)) { + if (JvmtiErrorToException(env, jvmti_env, result)) { return JNI_FALSE; } diff --git a/test/911-get-stack-trace/stack_trace.cc b/test/911-get-stack-trace/stack_trace.cc index 5a3a311255..a499e90ad9 100644 --- a/test/911-get-stack-trace/stack_trace.cc +++ b/test/911-get-stack-trace/stack_trace.cc @@ -18,16 +18,19 @@ #include <memory> #include <stdio.h> +#include "android-base/logging.h" #include "android-base/stringprintf.h" -#include "android-base/stringprintf.h" -#include "base/logging.h" -#include "base/macros.h" #include "jni.h" #include "jvmti.h" -#include "ScopedLocalRef.h" -#include "ti-agent/common_helper.h" -#include "ti-agent/common_load.h" +#include "scoped_local_ref.h" + +// Test infrastructure +#include "jni_binder.h" +#include "jni_helper.h" +#include "jvmti_helper.h" +#include "test_env.h" +#include "ti_macros.h" namespace art { namespace Test911GetStackTrace { @@ -68,7 +71,7 @@ static jobjectArray TranslateJvmtiFrameInfoArray(JNIEnv* env, char* gen; { jvmtiError result2 = jvmti_env->GetMethodName(frames[method_index].method, &name, &sig, &gen); - if (JvmtiErrorToException(env, result2)) { + if (JvmtiErrorToException(env, jvmti_env, result2)) { return nullptr; } } @@ -83,10 +86,7 @@ static jobjectArray TranslateJvmtiFrameInfoArray(JNIEnv* env, // Accept absent info and native method errors. if (line_result != JVMTI_ERROR_ABSENT_INFORMATION && line_result != JVMTI_ERROR_NATIVE_METHOD) { - char* err; - jvmti_env->GetErrorName(line_result, &err); - printf("Failure running GetLineNumberTable: %s\n", err); - jvmti_env->Deallocate(reinterpret_cast<unsigned char*>(err)); + JvmtiErrorToException(env, jvmti_env, line_result); return nullptr; } line_number_table = nullptr; @@ -139,7 +139,7 @@ extern "C" JNIEXPORT jobjectArray JNICALL Java_PrintThread_getStackTrace( jint count; { jvmtiError result = jvmti_env->GetStackTrace(thread, start, max, frames.get(), &count); - if (JvmtiErrorToException(env, result)) { + if (JvmtiErrorToException(env, jvmti_env, result)) { return nullptr; } } @@ -153,7 +153,7 @@ extern "C" JNIEXPORT jobjectArray JNICALL Java_AllTraces_getAllStackTraces( jvmtiStackInfo* stack_infos; { jvmtiError result = jvmti_env->GetAllStackTraces(max, &stack_infos, &thread_count); - if (JvmtiErrorToException(env, result)) { + if (JvmtiErrorToException(env, jvmti_env, result)) { return nullptr; } } @@ -189,7 +189,7 @@ extern "C" JNIEXPORT jobjectArray JNICALL Java_ThreadListTraces_getThreadListSta threads.get(), max, &stack_infos); - if (JvmtiErrorToException(env, result)) { + if (JvmtiErrorToException(env, jvmti_env, result)) { return nullptr; } } @@ -215,7 +215,7 @@ extern "C" JNIEXPORT jint JNICALL Java_Frames_getFrameCount( JNIEnv* env, jclass klass ATTRIBUTE_UNUSED, jthread thread) { jint count; jvmtiError result = jvmti_env->GetFrameCount(thread, &count); - if (JvmtiErrorToException(env, result)) { + if (JvmtiErrorToException(env, jvmti_env, result)) { return -1; } return count; @@ -227,7 +227,7 @@ extern "C" JNIEXPORT jobjectArray JNICALL Java_Frames_getFrameLocation( jlocation location; jvmtiError result = jvmti_env->GetFrameLocation(thread, depth, &method, &location); - if (JvmtiErrorToException(env, result)) { + if (JvmtiErrorToException(env, jvmti_env, result)) { return nullptr; } @@ -237,12 +237,12 @@ extern "C" JNIEXPORT jobjectArray JNICALL Java_Frames_getFrameLocation( { jclass decl_class; jvmtiError class_result = jvmti_env->GetMethodDeclaringClass(method, &decl_class); - if (JvmtiErrorToException(env, class_result)) { + if (JvmtiErrorToException(env, jvmti_env, class_result)) { return nullptr; } jint modifiers; jvmtiError mod_result = jvmti_env->GetMethodModifiers(method, &modifiers); - if (JvmtiErrorToException(env, mod_result)) { + if (JvmtiErrorToException(env, jvmti_env, mod_result)) { return nullptr; } constexpr jint kStatic = 0x8; diff --git a/test/912-classes/classes.cc b/test/912-classes/classes.cc index 5bd34f6be8..2636367548 100644 --- a/test/912-classes/classes.cc +++ b/test/912-classes/classes.cc @@ -16,19 +16,22 @@ #include <stdio.h> -#include "base/macros.h" +#include "android-base/macros.h" + #include "class_linker.h" #include "jni.h" #include "mirror/class_loader.h" #include "jvmti.h" #include "runtime.h" -#include "ScopedLocalRef.h" -#include "ScopedUtfChars.h" +#include "scoped_local_ref.h" +#include "scoped_utf_chars.h" #include "scoped_thread_state_change-inl.h" #include "thread-inl.h" -#include "ti-agent/common_helper.h" -#include "ti-agent/common_load.h" +// Test infrastructure +#include "jni_helper.h" +#include "jvmti_helper.h" +#include "test_env.h" namespace art { namespace Test912Classes { @@ -233,7 +236,7 @@ extern "C" JNIEXPORT jobjectArray JNICALL Java_Main_getClassLoaderClasses( jint count = 0; jclass* classes = nullptr; jvmtiError result = jvmti_env->GetClassLoaderClasses(jclassloader, &count, &classes); - if (JvmtiErrorToException(env, result)) { + if (JvmtiErrorToException(env, jvmti_env, result)) { return nullptr; } @@ -251,7 +254,7 @@ extern "C" JNIEXPORT jintArray JNICALL Java_Main_getClassVersion( JNIEnv* env, jclass Main_klass ATTRIBUTE_UNUSED, jclass klass) { jint major, minor; jvmtiError result = jvmti_env->GetClassVersionNumbers(klass, &minor, &major); - if (JvmtiErrorToException(env, result)) { + if (JvmtiErrorToException(env, jvmti_env, result)) { return nullptr; } @@ -270,7 +273,7 @@ static std::string GetClassName(jvmtiEnv* jenv, JNIEnv* jni_env, jclass klass) { jvmtiError result = jenv->GetClassSignature(klass, &name, nullptr); if (result != JVMTI_ERROR_NONE) { if (jni_env != nullptr) { - JvmtiErrorToException(jni_env, result); + JvmtiErrorToException(jni_env, jenv, result); } else { printf("Failed to get class signature.\n"); } @@ -291,13 +294,13 @@ static void EnableEvents(JNIEnv* env, jvmtiError ret = jvmti_env->SetEventNotificationMode(JVMTI_DISABLE, JVMTI_EVENT_CLASS_LOAD, nullptr); - if (JvmtiErrorToException(env, ret)) { + if (JvmtiErrorToException(env, jvmti_env, ret)) { return; } ret = jvmti_env->SetEventNotificationMode(JVMTI_DISABLE, JVMTI_EVENT_CLASS_PREPARE, nullptr); - JvmtiErrorToException(env, ret); + JvmtiErrorToException(env, jvmti_env, ret); return; } @@ -306,20 +309,20 @@ static void EnableEvents(JNIEnv* env, callbacks.ClassLoad = class_load; callbacks.ClassPrepare = class_prepare; jvmtiError ret = jvmti_env->SetEventCallbacks(&callbacks, sizeof(callbacks)); - if (JvmtiErrorToException(env, ret)) { + if (JvmtiErrorToException(env, jvmti_env, ret)) { return; } ret = jvmti_env->SetEventNotificationMode(JVMTI_ENABLE, JVMTI_EVENT_CLASS_LOAD, nullptr); - if (JvmtiErrorToException(env, ret)) { + if (JvmtiErrorToException(env, jvmti_env, ret)) { return; } ret = jvmti_env->SetEventNotificationMode(JVMTI_ENABLE, JVMTI_EVENT_CLASS_PREPARE, nullptr); - JvmtiErrorToException(env, ret); + JvmtiErrorToException(env, jvmti_env, ret); } class ClassLoadPreparePrinter { @@ -364,7 +367,7 @@ class ClassLoadPreparePrinter { jvmtiError result = jenv->GetThreadInfo(thread, &info); if (result != JVMTI_ERROR_NONE) { if (jni_env != nullptr) { - JvmtiErrorToException(jni_env, result); + JvmtiErrorToException(jni_env, jenv, result); } else { printf("Failed to get thread name.\n"); } diff --git a/test/913-heaps/expected.txt b/test/913-heaps/expected.txt index fc2761e800..2a183ee06b 100644 --- a/test/913-heaps/expected.txt +++ b/test/913-heaps/expected.txt @@ -79,6 +79,37 @@ root@root --(thread)--> 3000@0 [size=132, length=-1] 5@1002 --(field@9)--> 1@1000 [size=16, length=-1] 6@1000 --(class)--> 1000@0 [size=123, length=-1] --- +root@root --(thread)--> 3000@0 [size=132, length=-1] +--- +3@1001 --(class)--> 1001@0 [size=123, length=-1] +--- +root@root --(thread)--> 3000@0 [size=132, length=-1] +--- +3@1001 --(class)--> 1001@0 [size=123, length=-1] +--- +root@root --(stack-local[id=1,tag=3000,depth=2,method=doFollowReferencesTestNonRoot,vreg=13,location= 32])--> 1@1000 [size=16, length=-1] +root@root --(stack-local[id=1,tag=3000,depth=3,method=doFollowReferencesTest,vreg=1,location= 28])--> 3000@0 [size=132, length=-1] +root@root --(thread)--> 3000@0 [size=132, length=-1] +0@0 --(array-element@0)--> 1@1000 [size=16, length=-1] +--- +1001@0 --(superclass)--> 1000@0 [size=123, length=-1] +3@1001 --(class)--> 1001@0 [size=123, length=-1] +3@1001 --(field@4)--> 4@1000 [size=16, length=-1] +3@1001 --(field@5)--> 5@1002 [size=32, length=-1] +--- +root@root --(jni-global)--> 1@1000 [size=16, length=-1] +root@root --(jni-local[id=1,tag=3000,depth=0,method=followReferences])--> 1@1000 [size=16, length=-1] +root@root --(stack-local[id=1,tag=3000,depth=1,method=doFollowReferencesTestImpl,vreg=13,location= 10])--> 1@1000 [size=16, length=-1] +root@root --(stack-local[id=1,tag=3000,depth=1,method=doFollowReferencesTestImpl,vreg=5,location= 10])--> 1@1000 [size=16, length=-1] +root@root --(stack-local[id=1,tag=3000,depth=2,method=doFollowReferencesTestRoot,vreg=4,location= 19])--> 1@1000 [size=16, length=-1] +root@root --(thread)--> 1@1000 [size=16, length=-1] +root@root --(thread)--> 3000@0 [size=132, length=-1] +--- +1001@0 --(superclass)--> 1000@0 [size=123, length=-1] +3@1001 --(class)--> 1001@0 [size=123, length=-1] +3@1001 --(field@4)--> 4@1000 [size=16, length=-1] +3@1001 --(field@5)--> 5@1002 [size=32, length=-1] +--- [1@0 (32, 'HelloWorld'), 2@0 (16, '')] 2 3 diff --git a/test/913-heaps/heaps.cc b/test/913-heaps/heaps.cc index 66fc7bef9a..6a47ca139c 100644 --- a/test/913-heaps/heaps.cc +++ b/test/913-heaps/heaps.cc @@ -21,10 +21,10 @@ #include <iostream> #include <vector> +#include "android-base/macros.h" +#include "android-base/logging.h" #include "android-base/stringprintf.h" -#include "base/logging.h" -#include "base/macros.h" #include "jit/jit.h" #include "jni.h" #include "native_stack_dump.h" @@ -34,8 +34,10 @@ #include "thread-inl.h" #include "thread_list.h" -#include "ti-agent/common_helper.h" -#include "ti-agent/common_load.h" +// Test infrastructure +#include "jni_helper.h" +#include "jvmti_helper.h" +#include "test_env.h" namespace art { namespace Test913Heaps { @@ -550,7 +552,7 @@ extern "C" JNIEXPORT jobjectArray JNICALL Java_Main_followReferencesString( FindStringCallbacks fsc; jvmtiError ret = jvmti_env->FollowReferences(0, nullptr, initial_object, &callbacks, &fsc); - if (JvmtiErrorToException(env, ret)) { + if (JvmtiErrorToException(env, jvmti_env, ret)) { return nullptr; } @@ -648,7 +650,7 @@ extern "C" JNIEXPORT jstring JNICALL Java_Main_followReferencesPrimitiveArray( FindArrayCallbacks fac; jvmtiError ret = jvmti_env->FollowReferences(0, nullptr, initial_object, &callbacks, &fac); - if (JvmtiErrorToException(env, ret)) { + if (JvmtiErrorToException(env, jvmti_env, ret)) { return nullptr; } return env->NewStringUTF(fac.data.c_str()); @@ -738,7 +740,7 @@ extern "C" JNIEXPORT jstring JNICALL Java_Main_followReferencesPrimitiveFields( FindFieldCallbacks ffc; jvmtiError ret = jvmti_env->FollowReferences(0, nullptr, initial_object, &callbacks, &ffc); - if (JvmtiErrorToException(env, ret)) { + if (JvmtiErrorToException(env, jvmti_env, ret)) { return nullptr; } return env->NewStringUTF(ffc.data.c_str()); diff --git a/test/913-heaps/src/Main.java b/test/913-heaps/src/Main.java index 66f68834a1..10778ff3dd 100644 --- a/test/913-heaps/src/Main.java +++ b/test/913-heaps/src/Main.java @@ -28,6 +28,16 @@ public class Main { Runtime.getRuntime().gc(); Runtime.getRuntime().gc(); + new TestConfig(null, 0, 1, -1).doFollowReferencesTest(); + + Runtime.getRuntime().gc(); + Runtime.getRuntime().gc(); + + new TestConfig(null, 0, Integer.MAX_VALUE, 1).doFollowReferencesTest(); + + Runtime.getRuntime().gc(); + Runtime.getRuntime().gc(); + doStringTest(); Runtime.getRuntime().gc(); @@ -202,6 +212,8 @@ public class Main { private static class TestConfig { private Class<?> klass = null; private int heapFilter = 0; + private int stopAfter = Integer.MAX_VALUE; + private int followSet = -1; public TestConfig() { } @@ -209,6 +221,12 @@ public class Main { this.klass = klass; this.heapFilter = heapFilter; } + public TestConfig(Class<?> klass, int heapFilter, int stopAfter, int followSet) { + this.klass = klass; + this.heapFilter = heapFilter; + this.stopAfter = stopAfter; + this.followSet = followSet; + } public void doFollowReferencesTest() throws Exception { // Force GCs to clean up dirt. @@ -241,8 +259,8 @@ public class Main { tmpStorage.add(a); v.add("0@0", "1@1000"); // tmpStorage[0] --(array-element)--> a. - doFollowReferencesTestImpl(null, Integer.MAX_VALUE, -1, null, v, null); - doFollowReferencesTestImpl(a.foo2, Integer.MAX_VALUE, -1, null, v, "3@1001"); + doFollowReferencesTestImpl(null, stopAfter, followSet, null, v, null); + doFollowReferencesTestImpl(a.foo2, stopAfter, followSet, null, v, "3@1001"); tmpStorage.clear(); } @@ -252,8 +270,8 @@ public class Main { tagClasses(v); A a = createTree(v); - doFollowReferencesTestImpl(null, Integer.MAX_VALUE, -1, a, v, null); - doFollowReferencesTestImpl(a.foo2, Integer.MAX_VALUE, -1, a, v, "3@1001"); + doFollowReferencesTestImpl(null, stopAfter, followSet, a, v, null); + doFollowReferencesTestImpl(a.foo2, stopAfter, followSet, a, v, "3@1001"); } private void doFollowReferencesTestImpl(A root, int stopAfter, int followSet, diff --git a/test/918-fields/fields.cc b/test/918-fields/fields.cc index c659126aae..726c5cf161 100644 --- a/test/918-fields/fields.cc +++ b/test/918-fields/fields.cc @@ -16,13 +16,14 @@ #include <stdio.h> -#include "base/macros.h" +#include "android-base/macros.h" #include "jni.h" #include "jvmti.h" -#include "ScopedLocalRef.h" +#include "scoped_local_ref.h" -#include "ti-agent/common_helper.h" -#include "ti-agent/common_load.h" +// Test infrastructure +#include "jni_helper.h" +#include "test_env.h" namespace art { namespace Test918Fields { diff --git a/test/920-objects/objects.cc b/test/920-objects/objects.cc index ad1431ed00..5263e753ed 100644 --- a/test/920-objects/objects.cc +++ b/test/920-objects/objects.cc @@ -16,13 +16,13 @@ #include <stdio.h> -#include "base/macros.h" +#include "android-base/macros.h" #include "jni.h" #include "jvmti.h" -#include "ScopedLocalRef.h" +#include "scoped_local_ref.h" -#include "ti-agent/common_helper.h" -#include "ti-agent/common_load.h" +// Test infrastructure +#include "test_env.h" namespace art { namespace Test920Objects { diff --git a/test/922-properties/properties.cc b/test/922-properties/properties.cc index 3fd274e9d6..896e4c30af 100644 --- a/test/922-properties/properties.cc +++ b/test/922-properties/properties.cc @@ -16,13 +16,15 @@ #include <stdio.h> -#include "base/macros.h" +#include "android-base/macros.h" #include "jni.h" #include "jvmti.h" -#include "ScopedUtfChars.h" +#include "scoped_utf_chars.h" -#include "ti-agent/common_helper.h" -#include "ti-agent/common_load.h" +// Test infrastructure +#include "jni_helper.h" +#include "jvmti_helper.h" +#include "test_env.h" namespace art { namespace Test922Properties { @@ -32,7 +34,7 @@ extern "C" JNIEXPORT jobjectArray JNICALL Java_Main_getSystemProperties( jint count; char** properties; jvmtiError result = jvmti_env->GetSystemProperties(&count, &properties); - if (JvmtiErrorToException(env, result)) { + if (JvmtiErrorToException(env, jvmti_env, result)) { return nullptr; } @@ -61,7 +63,7 @@ extern "C" JNIEXPORT jstring JNICALL Java_Main_getSystemProperty( char* value = nullptr; jvmtiError result = jvmti_env->GetSystemProperty(string.c_str(), &value); - if (JvmtiErrorToException(env, result)) { + if (JvmtiErrorToException(env, jvmti_env, result)) { return nullptr; } @@ -84,7 +86,7 @@ extern "C" JNIEXPORT void JNICALL Java_Main_setSystemProperty( } jvmtiError result = jvmti_env->SetSystemProperty(key_string.c_str(), value_string.c_str()); - if (JvmtiErrorToException(env, result)) { + if (JvmtiErrorToException(env, jvmti_env, result)) { return; } } diff --git a/test/923-monitors/monitors.cc b/test/923-monitors/monitors.cc index 131fc6a4d4..6369a740e1 100644 --- a/test/923-monitors/monitors.cc +++ b/test/923-monitors/monitors.cc @@ -16,13 +16,14 @@ #include <stdio.h> -#include "base/macros.h" +#include "android-base/macros.h" #include "jni.h" #include "jvmti.h" -#include "ScopedUtfChars.h" +#include "scoped_utf_chars.h" -#include "ti-agent/common_helper.h" -#include "ti-agent/common_load.h" +// Test infrastructure +#include "jvmti_helper.h" +#include "test_env.h" namespace art { namespace Test923Monitors { @@ -40,7 +41,7 @@ extern "C" JNIEXPORT jlong JNICALL Java_Main_createRawMonitor( JNIEnv* env, jclass Main_klass ATTRIBUTE_UNUSED) { jrawMonitorID id; jvmtiError result = jvmti_env->CreateRawMonitor("dummy", &id); - if (JvmtiErrorToException(env, result)) { + if (JvmtiErrorToException(env, jvmti_env, result)) { return 0; } return MonitorToLong(id); @@ -49,37 +50,37 @@ extern "C" JNIEXPORT jlong JNICALL Java_Main_createRawMonitor( extern "C" JNIEXPORT void JNICALL Java_Main_destroyRawMonitor( JNIEnv* env, jclass Main_klass ATTRIBUTE_UNUSED, jlong l) { jvmtiError result = jvmti_env->DestroyRawMonitor(LongToMonitor(l)); - JvmtiErrorToException(env, result); + JvmtiErrorToException(env, jvmti_env, result); } extern "C" JNIEXPORT void JNICALL Java_Main_rawMonitorEnter( JNIEnv* env, jclass Main_klass ATTRIBUTE_UNUSED, jlong l) { jvmtiError result = jvmti_env->RawMonitorEnter(LongToMonitor(l)); - JvmtiErrorToException(env, result); + JvmtiErrorToException(env, jvmti_env, result); } extern "C" JNIEXPORT void JNICALL Java_Main_rawMonitorExit( JNIEnv* env, jclass Main_klass ATTRIBUTE_UNUSED, jlong l) { jvmtiError result = jvmti_env->RawMonitorExit(LongToMonitor(l)); - JvmtiErrorToException(env, result); + JvmtiErrorToException(env, jvmti_env, result); } extern "C" JNIEXPORT void JNICALL Java_Main_rawMonitorWait( JNIEnv* env, jclass Main_klass ATTRIBUTE_UNUSED, jlong l, jlong millis) { jvmtiError result = jvmti_env->RawMonitorWait(LongToMonitor(l), millis); - JvmtiErrorToException(env, result); + JvmtiErrorToException(env, jvmti_env, result); } extern "C" JNIEXPORT void JNICALL Java_Main_rawMonitorNotify( JNIEnv* env, jclass Main_klass ATTRIBUTE_UNUSED, jlong l) { jvmtiError result = jvmti_env->RawMonitorNotify(LongToMonitor(l)); - JvmtiErrorToException(env, result); + JvmtiErrorToException(env, jvmti_env, result); } extern "C" JNIEXPORT void JNICALL Java_Main_rawMonitorNotifyAll( JNIEnv* env, jclass Main_klass ATTRIBUTE_UNUSED, jlong l) { jvmtiError result = jvmti_env->RawMonitorNotifyAll(LongToMonitor(l)); - JvmtiErrorToException(env, result); + JvmtiErrorToException(env, jvmti_env, result); } } // namespace Test923Monitors diff --git a/test/924-threads/expected.txt b/test/924-threads/expected.txt index 67d20eb750..4c0f4eaa5b 100644 --- a/test/924-threads/expected.txt +++ b/test/924-threads/expected.txt @@ -19,6 +19,11 @@ Daemon Thread true java.lang.ThreadGroup[name=main,maxpri=10] class dalvik.system.PathClassLoader +Subclass +5 +false +java.lang.ThreadGroup[name=main,maxpri=10] +class dalvik.system.PathClassLoader 5 5 0 = NEW diff --git a/test/924-threads/src/Main.java b/test/924-threads/src/Main.java index 716f59e3af..7328560084 100644 --- a/test/924-threads/src/Main.java +++ b/test/924-threads/src/Main.java @@ -52,6 +52,11 @@ public class Main { // Thread has died, check that we can still get info. printThreadInfo(t3); + // Try a subclass of thread. + Thread t4 = new Thread("Subclass") { + }; + printThreadInfo(t4); + doStateTests(); doAllThreadsTests(); diff --git a/test/924-threads/threads.cc b/test/924-threads/threads.cc index 14ea5af60e..a8b37ecd37 100644 --- a/test/924-threads/threads.cc +++ b/test/924-threads/threads.cc @@ -16,15 +16,17 @@ #include <stdio.h> +#include "android-base/logging.h" #include "android-base/stringprintf.h" -#include "base/macros.h" -#include "base/logging.h" #include "jni.h" #include "jvmti.h" -#include "ScopedLocalRef.h" +#include "scoped_local_ref.h" -#include "ti-agent/common_helper.h" -#include "ti-agent/common_load.h" +// Test infrastructure +#include "jni_helper.h" +#include "jvmti_helper.h" +#include "test_env.h" +#include "ti_macros.h" namespace art { namespace Test924Threads { @@ -36,7 +38,7 @@ extern "C" JNIEXPORT jthread JNICALL Java_Main_getCurrentThread( JNIEnv* env, jclass Main_klass ATTRIBUTE_UNUSED) { jthread thread = nullptr; jvmtiError result = jvmti_env->GetCurrentThread(&thread); - if (JvmtiErrorToException(env, result)) { + if (JvmtiErrorToException(env, jvmti_env, result)) { return nullptr; } return thread; @@ -48,7 +50,7 @@ extern "C" JNIEXPORT jobjectArray JNICALL Java_Main_getThreadInfo( memset(&info, 0, sizeof(jvmtiThreadInfo)); jvmtiError result = jvmti_env->GetThreadInfo(thread, &info); - if (JvmtiErrorToException(env, result)) { + if (JvmtiErrorToException(env, jvmti_env, result)) { return nullptr; } @@ -94,7 +96,7 @@ extern "C" JNIEXPORT jint JNICALL Java_Main_getThreadState( JNIEnv* env, jclass Main_klass ATTRIBUTE_UNUSED, jthread thread) { jint state; jvmtiError result = jvmti_env->GetThreadState(thread, &state); - if (JvmtiErrorToException(env, result)) { + if (JvmtiErrorToException(env, jvmti_env, result)) { return 0; } return state; @@ -106,7 +108,7 @@ extern "C" JNIEXPORT jobjectArray JNICALL Java_Main_getAllThreads( jthread* threads; jvmtiError result = jvmti_env->GetAllThreads(&thread_count, &threads); - if (JvmtiErrorToException(env, result)) { + if (JvmtiErrorToException(env, jvmti_env, result)) { return nullptr; } @@ -124,7 +126,7 @@ extern "C" JNIEXPORT jlong JNICALL Java_Main_getTLS( JNIEnv* env, jclass Main_klass ATTRIBUTE_UNUSED, jthread thread) { void* tls; jvmtiError result = jvmti_env->GetThreadLocalStorage(thread, &tls); - if (JvmtiErrorToException(env, result)) { + if (JvmtiErrorToException(env, jvmti_env, result)) { return 0; } return static_cast<jlong>(reinterpret_cast<uintptr_t>(tls)); @@ -134,7 +136,7 @@ extern "C" JNIEXPORT void JNICALL Java_Main_setTLS( JNIEnv* env, jclass Main_klass ATTRIBUTE_UNUSED, jthread thread, jlong val) { const void* tls = reinterpret_cast<void*>(static_cast<uintptr_t>(val)); jvmtiError result = jvmti_env->SetThreadLocalStorage(thread, tls); - JvmtiErrorToException(env, result); + JvmtiErrorToException(env, jvmti_env, result); } static void JNICALL ThreadEvent(jvmtiEnv* jvmti_env, @@ -172,13 +174,13 @@ extern "C" JNIEXPORT void JNICALL Java_Main_enableThreadEvents( jvmtiError ret = jvmti_env->SetEventNotificationMode(JVMTI_DISABLE, JVMTI_EVENT_THREAD_START, nullptr); - if (JvmtiErrorToException(env, ret)) { + if (JvmtiErrorToException(env, jvmti_env, ret)) { return; } ret = jvmti_env->SetEventNotificationMode(JVMTI_DISABLE, JVMTI_EVENT_THREAD_END, nullptr); - JvmtiErrorToException(env, ret); + JvmtiErrorToException(env, jvmti_env, ret); return; } @@ -187,20 +189,20 @@ extern "C" JNIEXPORT void JNICALL Java_Main_enableThreadEvents( callbacks.ThreadStart = ThreadStart; callbacks.ThreadEnd = ThreadEnd; jvmtiError ret = jvmti_env->SetEventCallbacks(&callbacks, sizeof(callbacks)); - if (JvmtiErrorToException(env, ret)) { + if (JvmtiErrorToException(env, jvmti_env, ret)) { return; } ret = jvmti_env->SetEventNotificationMode(JVMTI_ENABLE, JVMTI_EVENT_THREAD_START, nullptr); - if (JvmtiErrorToException(env, ret)) { + if (JvmtiErrorToException(env, jvmti_env, ret)) { return; } ret = jvmti_env->SetEventNotificationMode(JVMTI_ENABLE, JVMTI_EVENT_THREAD_END, nullptr); - JvmtiErrorToException(env, ret); + JvmtiErrorToException(env, jvmti_env, ret); } } // namespace Test924Threads diff --git a/test/925-threadgroups/threadgroups.cc b/test/925-threadgroups/threadgroups.cc index 2feaab079b..d55555355c 100644 --- a/test/925-threadgroups/threadgroups.cc +++ b/test/925-threadgroups/threadgroups.cc @@ -16,15 +16,17 @@ #include <stdio.h> +#include "android-base/logging.h" #include "android-base/stringprintf.h" -#include "base/macros.h" -#include "base/logging.h" #include "jni.h" #include "jvmti.h" -#include "ScopedLocalRef.h" +#include "scoped_local_ref.h" -#include "ti-agent/common_helper.h" -#include "ti-agent/common_load.h" +// Test infrastructure +#include "jni_helper.h" +#include "jvmti_helper.h" +#include "test_env.h" +#include "ti_macros.h" namespace art { namespace Test925ThreadGroups { @@ -38,7 +40,7 @@ extern "C" JNIEXPORT jobjectArray JNICALL Java_Main_getTopThreadGroups( jthreadGroup* groups; jint group_count; jvmtiError result = jvmti_env->GetTopThreadGroups(&group_count, &groups); - if (JvmtiErrorToException(env, result)) { + if (JvmtiErrorToException(env, jvmti_env, result)) { return nullptr; } @@ -56,7 +58,7 @@ extern "C" JNIEXPORT jobjectArray JNICALL Java_Main_getThreadGroupInfo( JNIEnv* env, jclass Main_klass ATTRIBUTE_UNUSED, jthreadGroup group) { jvmtiThreadGroupInfo info; jvmtiError result = jvmti_env->GetThreadGroupInfo(group, &info); - if (JvmtiErrorToException(env, result)) { + if (JvmtiErrorToException(env, jvmti_env, result)) { return nullptr; } @@ -96,7 +98,7 @@ extern "C" JNIEXPORT jobjectArray JNICALL Java_Main_getThreadGroupChildren( &threads, &threadgroup_count, &groups); - if (JvmtiErrorToException(env, result)) { + if (JvmtiErrorToException(env, jvmti_env, result)) { return nullptr; } diff --git a/test/927-timers/timers.cc b/test/927-timers/timers.cc index 7b1d5c3f52..55d3921cb6 100644 --- a/test/927-timers/timers.cc +++ b/test/927-timers/timers.cc @@ -16,14 +16,17 @@ #include <inttypes.h> +#include "android-base/logging.h" #include "android-base/stringprintf.h" -#include "base/logging.h" -#include "base/macros.h" + #include "jni.h" #include "jvmti.h" -#include "ti-agent/common_helper.h" -#include "ti-agent/common_load.h" +// Test infrastructure +#include "jni_helper.h" +#include "jvmti_helper.h" +#include "test_env.h" +#include "ti_macros.h" namespace art { namespace Test926Timers { @@ -32,7 +35,7 @@ extern "C" JNIEXPORT jint JNICALL Java_Main_getAvailableProcessors( JNIEnv* env, jclass Main_klass ATTRIBUTE_UNUSED) { jint count; jvmtiError result = jvmti_env->GetAvailableProcessors(&count); - if (JvmtiErrorToException(env, result)) { + if (JvmtiErrorToException(env, jvmti_env, result)) { return -1; } return count; @@ -42,7 +45,7 @@ extern "C" JNIEXPORT jlong JNICALL Java_Main_getTime( JNIEnv* env, jclass Main_klass ATTRIBUTE_UNUSED) { jlong time; jvmtiError result = jvmti_env->GetTime(&time); - if (JvmtiErrorToException(env, result)) { + if (JvmtiErrorToException(env, jvmti_env, result)) { return -1; } return time; @@ -52,7 +55,7 @@ extern "C" JNIEXPORT jobjectArray JNICALL Java_Main_getTimerInfo( JNIEnv* env, jclass Main_klass ATTRIBUTE_UNUSED) { jvmtiTimerInfo info; jvmtiError result = jvmti_env->GetTimerInfo(&info); - if (JvmtiErrorToException(env, result)) { + if (JvmtiErrorToException(env, jvmti_env, result)) { return nullptr; } diff --git a/test/928-jni-table/jni_table.cc b/test/928-jni-table/jni_table.cc index b5c0efdd95..26a6707f91 100644 --- a/test/928-jni-table/jni_table.cc +++ b/test/928-jni-table/jni_table.cc @@ -19,11 +19,12 @@ #include "jni.h" #include "jvmti.h" -#include "base/logging.h" -#include "base/macros.h" +#include "android-base/logging.h" +#include "android-base/macros.h" -#include "ti-agent/common_helper.h" -#include "ti-agent/common_load.h" +// Test infrastructure +#include "jvmti_helper.h" +#include "test_env.h" namespace art { namespace Test927JNITable { @@ -42,14 +43,14 @@ extern "C" JNIEXPORT void JNICALL Java_Main_doJNITableTest( JNIEnv* env, jclass klass) { // Get the current table, as the delegate. jvmtiError getorig_result = jvmti_env->GetJNIFunctionTable(&gOriginalEnv); - if (JvmtiErrorToException(env, getorig_result)) { + if (JvmtiErrorToException(env, jvmti_env, getorig_result)) { return; } // Get the current table, as the override we'll install. JNINativeInterface* env_override; jvmtiError getoverride_result = jvmti_env->GetJNIFunctionTable(&env_override); - if (JvmtiErrorToException(env, getoverride_result)) { + if (JvmtiErrorToException(env, jvmti_env, getoverride_result)) { return; } @@ -58,7 +59,7 @@ extern "C" JNIEXPORT void JNICALL Java_Main_doJNITableTest( // Install the override. jvmtiError setoverride_result = jvmti_env->SetJNIFunctionTable(env_override); - if (JvmtiErrorToException(env, setoverride_result)) { + if (JvmtiErrorToException(env, jvmti_env, setoverride_result)) { return; } @@ -68,7 +69,7 @@ extern "C" JNIEXPORT void JNICALL Java_Main_doJNITableTest( // Install the "original." There is no real reset. jvmtiError setoverride2_result = jvmti_env->SetJNIFunctionTable(gOriginalEnv); - if (JvmtiErrorToException(env, setoverride2_result)) { + if (JvmtiErrorToException(env, jvmti_env, setoverride2_result)) { return; } diff --git a/test/929-search/search.cc b/test/929-search/search.cc index ad7a05323b..5516105abe 100644 --- a/test/929-search/search.cc +++ b/test/929-search/search.cc @@ -16,15 +16,16 @@ #include <inttypes.h> +#include "android-base/logging.h" +#include "android-base/macros.h" #include "android-base/stringprintf.h" -#include "base/logging.h" -#include "base/macros.h" #include "jni.h" #include "jvmti.h" -#include "ScopedUtfChars.h" +#include "scoped_utf_chars.h" -#include "ti-agent/common_helper.h" -#include "ti-agent/common_load.h" +// Test infrastructure +#include "jvmti_helper.h" +#include "test_env.h" namespace art { namespace Test929Search { @@ -36,7 +37,7 @@ extern "C" JNIEXPORT void JNICALL Java_Main_addToBootClassLoader( return; } jvmtiError result = jvmti_env->AddToBootstrapClassLoaderSearch(utf.c_str()); - JvmtiErrorToException(env, result); + JvmtiErrorToException(env, jvmti_env, result); } extern "C" JNIEXPORT void JNICALL Java_Main_addToSystemClassLoader( @@ -46,7 +47,7 @@ extern "C" JNIEXPORT void JNICALL Java_Main_addToSystemClassLoader( return; } jvmtiError result = jvmti_env->AddToSystemClassLoaderSearch(utf.c_str()); - JvmtiErrorToException(env, result); + JvmtiErrorToException(env, jvmti_env, result); } } // namespace Test929Search diff --git a/test/931-agent-thread/agent_thread.cc b/test/931-agent-thread/agent_thread.cc index f8f9e48657..f9af8cfe71 100644 --- a/test/931-agent-thread/agent_thread.cc +++ b/test/931-agent-thread/agent_thread.cc @@ -15,20 +15,18 @@ */ #include <inttypes.h> +#include <pthread.h> #include <sched.h> -#include "barrier.h" -#include "base/logging.h" -#include "base/macros.h" +#include "android-base/logging.h" +#include "android-base/macros.h" #include "jni.h" #include "jvmti.h" -#include "runtime.h" -#include "ScopedLocalRef.h" -#include "thread-inl.h" -#include "well_known_classes.h" +#include "scoped_local_ref.h" -#include "ti-agent/common_helper.h" -#include "ti-agent/common_load.h" +// Test infrastructure +#include "jvmti_helper.h" +#include "test_env.h" namespace art { namespace Test930AgentThread { @@ -36,12 +34,12 @@ namespace Test930AgentThread { struct AgentData { AgentData() : main_thread(nullptr), jvmti_env(nullptr), - b(2) { + priority(0) { } jthread main_thread; jvmtiEnv* jvmti_env; - Barrier b; + pthread_barrier_t b; jint priority; }; @@ -52,14 +50,21 @@ static void AgentMain(jvmtiEnv* jenv, JNIEnv* env, void* arg) { // This thread is not the main thread. jthread this_thread; jvmtiError this_thread_result = jenv->GetCurrentThread(&this_thread); - CHECK(!JvmtiErrorToException(env, this_thread_result)); + CheckJvmtiError(jenv, this_thread_result); CHECK(!env->IsSameObject(this_thread, data->main_thread)); // The thread is a daemon. jvmtiThreadInfo info; jvmtiError info_result = jenv->GetThreadInfo(this_thread, &info); - CHECK(!JvmtiErrorToException(env, info_result)); + CheckJvmtiError(jenv, info_result); CHECK(info.is_daemon); + CheckJvmtiError(jenv, jenv->Deallocate(reinterpret_cast<unsigned char*>(info.name))); + if (info.thread_group != nullptr) { + env->DeleteLocalRef(info.thread_group); + } + if (info.context_class_loader != nullptr) { + env->DeleteLocalRef(info.context_class_loader); + } // The thread has the requested priority. // TODO: Our thread priorities do not work on the host. @@ -69,7 +74,7 @@ static void AgentMain(jvmtiEnv* jenv, JNIEnv* env, void* arg) { jint thread_count; jthread* threads; jvmtiError threads_result = jenv->GetAllThreads(&thread_count, &threads); - CHECK(!JvmtiErrorToException(env, threads_result)); + CheckJvmtiError(jenv, threads_result); bool found = false; for (jint i = 0; i != thread_count; ++i) { if (env->IsSameObject(threads[i], this_thread)) { @@ -80,29 +85,53 @@ static void AgentMain(jvmtiEnv* jenv, JNIEnv* env, void* arg) { CHECK(found); // Done, let the main thread progress. - data->b.Pass(Thread::Current()); + int wait_result = pthread_barrier_wait(&data->b); + CHECK(wait_result == PTHREAD_BARRIER_SERIAL_THREAD || wait_result == 0); } extern "C" JNIEXPORT void JNICALL Java_Main_testAgentThread( JNIEnv* env, jclass Main_klass ATTRIBUTE_UNUSED) { // Create a Thread object. - ScopedLocalRef<jobject> thread_name(env, - env->NewStringUTF("Agent Thread")); + ScopedLocalRef<jobject> thread_name(env, env->NewStringUTF("Agent Thread")); if (thread_name.get() == nullptr) { return; } - ScopedLocalRef<jobject> thread(env, env->AllocObject(WellKnownClasses::java_lang_Thread)); + ScopedLocalRef<jclass> thread_klass(env, env->FindClass("java/lang/Thread")); + if (thread_klass.get() == nullptr) { + return; + } + ScopedLocalRef<jobject> thread(env, env->AllocObject(thread_klass.get())); if (thread.get() == nullptr) { return; } + // Get a ThreadGroup from the current thread. We need a non-null one as we're gonna call a + // runtime-only constructor (so we can set priority and daemon state). + jvmtiThreadInfo cur_thread_info; + jvmtiError info_result = jvmti_env->GetThreadInfo(nullptr, &cur_thread_info); + if (JvmtiErrorToException(env, jvmti_env, info_result)) { + return; + } + CheckJvmtiError(jvmti_env, + jvmti_env->Deallocate(reinterpret_cast<unsigned char*>(cur_thread_info.name))); + ScopedLocalRef<jobject> thread_group(env, cur_thread_info.thread_group); + if (cur_thread_info.context_class_loader != nullptr) { + env->DeleteLocalRef(cur_thread_info.context_class_loader); + } + + jmethodID initID = env->GetMethodID(thread_klass.get(), + "<init>", + "(Ljava/lang/ThreadGroup;Ljava/lang/String;IZ)V"); + if (initID == nullptr) { + return; + } env->CallNonvirtualVoidMethod(thread.get(), - WellKnownClasses::java_lang_Thread, - WellKnownClasses::java_lang_Thread_init, - Runtime::Current()->GetMainThreadGroup(), + thread_klass.get(), + initID, + thread_group.get(), thread_name.get(), - kMinThreadPriority, + 0, JNI_FALSE); if (env->ExceptionCheck()) { return; @@ -110,7 +139,7 @@ extern "C" JNIEXPORT void JNICALL Java_Main_testAgentThread( jthread main_thread; jvmtiError main_thread_result = jvmti_env->GetCurrentThread(&main_thread); - if (JvmtiErrorToException(env, main_thread_result)) { + if (JvmtiErrorToException(env, jvmti_env, main_thread_result)) { return; } @@ -118,21 +147,23 @@ extern "C" JNIEXPORT void JNICALL Java_Main_testAgentThread( data.main_thread = env->NewGlobalRef(main_thread); data.jvmti_env = jvmti_env; data.priority = JVMTI_THREAD_MIN_PRIORITY; + CHECK_EQ(0, pthread_barrier_init(&data.b, nullptr, 2)); jvmtiError result = jvmti_env->RunAgentThread(thread.get(), AgentMain, &data, data.priority); - if (JvmtiErrorToException(env, result)) { + if (JvmtiErrorToException(env, jvmti_env, result)) { return; } - data.b.Wait(Thread::Current()); + int wait_result = pthread_barrier_wait(&data.b); + CHECK(wait_result == PTHREAD_BARRIER_SERIAL_THREAD || wait_result == 0); // Scheduling may mean that the agent thread is put to sleep. Wait until it's dead in an effort // to not unload the plugin and crash. for (;;) { - NanoSleep(1000 * 1000); + sleep(1); jint thread_state; jvmtiError state_result = jvmti_env->GetThreadState(thread.get(), &thread_state); - if (JvmtiErrorToException(env, state_result)) { + if (JvmtiErrorToException(env, jvmti_env, state_result)) { return; } if (thread_state == 0 || // Was never alive. @@ -142,9 +173,11 @@ extern "C" JNIEXPORT void JNICALL Java_Main_testAgentThread( } // Yield and sleep a bit more, to give the plugin time to tear down the native thread structure. sched_yield(); - NanoSleep(100 * 1000 * 1000); + sleep(1); env->DeleteGlobalRef(data.main_thread); + + pthread_barrier_destroy(&data.b); } } // namespace Test930AgentThread diff --git a/test/932-transform-saves/src/Transform.java b/test/932-transform-saves/src/Transform.java index 8e8af355da..83f7aa4b4d 100644 --- a/test/932-transform-saves/src/Transform.java +++ b/test/932-transform-saves/src/Transform.java @@ -23,6 +23,6 @@ class Transform { // of the two different strings were the same). // We know the string ids will be different because lexicographically: // "Goodbye" < "LTransform;" < "hello". - System.out.println("hello"); + System.out.println("foobar"); } } diff --git a/test/933-misc-events/misc_events.cc b/test/933-misc-events/misc_events.cc index 7043350b5a..2b74c407e0 100644 --- a/test/933-misc-events/misc_events.cc +++ b/test/933-misc-events/misc_events.cc @@ -18,13 +18,14 @@ #include <signal.h> #include <sys/types.h> -#include "base/logging.h" -#include "base/macros.h" +#include "android-base/logging.h" +#include "android-base/macros.h" #include "jni.h" #include "jvmti.h" -#include "ti-agent/common_helper.h" -#include "ti-agent/common_load.h" +// Test infrastructure +#include "jvmti_helper.h" +#include "test_env.h" namespace art { namespace Test933MiscEvents { @@ -42,14 +43,14 @@ extern "C" JNIEXPORT void JNICALL Java_Main_testSigQuit( memset(&callbacks, 0, sizeof(jvmtiEventCallbacks)); callbacks.DataDumpRequest = DumpRequestCallback; jvmtiError ret = jvmti_env->SetEventCallbacks(&callbacks, sizeof(callbacks)); - if (JvmtiErrorToException(env, ret)) { + if (JvmtiErrorToException(env, jvmti_env, ret)) { return; } ret = jvmti_env->SetEventNotificationMode(JVMTI_ENABLE, JVMTI_EVENT_DATA_DUMP_REQUEST, nullptr); - if (JvmtiErrorToException(env, ret)) { + if (JvmtiErrorToException(env, jvmti_env, ret)) { return; } @@ -65,7 +66,7 @@ extern "C" JNIEXPORT void JNICALL Java_Main_testSigQuit( } ret = jvmti_env->SetEventNotificationMode(JVMTI_DISABLE, JVMTI_EVENT_DATA_DUMP_REQUEST, nullptr); - JvmtiErrorToException(env, ret); + JvmtiErrorToException(env, jvmti_env, ret); } } // namespace Test933MiscEvents diff --git a/test/936-search-onload/search_onload.cc b/test/936-search-onload/search_onload.cc index 3b19ca591d..b2ef05690f 100644 --- a/test/936-search-onload/search_onload.cc +++ b/test/936-search-onload/search_onload.cc @@ -23,10 +23,11 @@ #include "base/macros.h" #include "jni.h" #include "jvmti.h" -#include "ScopedUtfChars.h" +#include "scoped_utf_chars.h" -#include "ti-agent/common_helper.h" -#include "ti-agent/common_load.h" +// Test infrastructure +#include "jvmti_helper.h" +#include "test_env.h" namespace art { namespace Test936SearchOnload { diff --git a/test/944-transform-classloaders/classloader.cc b/test/944-transform-classloaders/classloader.cc index 7cb3c08dc3..698e023771 100644 --- a/test/944-transform-classloaders/classloader.cc +++ b/test/944-transform-classloaders/classloader.cc @@ -14,19 +14,18 @@ * limitations under the License. */ -#include "base/macros.h" +#include "android-base/macros.h" #include "jni.h" #include "jvmti.h" #include "mirror/class-inl.h" -#include "ScopedLocalRef.h" +#include "scoped_local_ref.h" -#include "ti-agent/common_helper.h" -#include "ti-agent/common_load.h" +// Test infrastructure +#include "test_env.h" namespace art { namespace Test944TransformClassloaders { - extern "C" JNIEXPORT jlong JNICALL Java_Main_getDexFilePointer(JNIEnv* env, jclass, jclass klass) { if (Runtime::Current() == nullptr) { env->ThrowNew(env->FindClass("java/lang/Exception"), diff --git a/test/945-obsolete-native/obsolete_native.cc b/test/945-obsolete-native/obsolete_native.cc index 442836b7ff..ee653a4a12 100644 --- a/test/945-obsolete-native/obsolete_native.cc +++ b/test/945-obsolete-native/obsolete_native.cc @@ -25,9 +25,11 @@ #include "base/macros.h" #include "jni.h" #include "jvmti.h" -#include "ScopedLocalRef.h" -#include "ti-agent/common_helper.h" -#include "ti-agent/common_load.h" +#include "scoped_local_ref.h" + +// Test infrastructure +#include "jni_binder.h" +#include "test_env.h" namespace art { namespace Test945ObsoleteNative { diff --git a/test/980-redefine-object/redefine_object.cc b/test/980-redefine-object/redefine_object.cc index daae08792a..1faf1a16a7 100644 --- a/test/980-redefine-object/redefine_object.cc +++ b/test/980-redefine-object/redefine_object.cc @@ -22,10 +22,12 @@ #include "base/macros.h" #include "jni.h" #include "jvmti.h" -#include "ScopedUtfChars.h" +#include "scoped_utf_chars.h" -#include "ti-agent/common_helper.h" -#include "ti-agent/common_load.h" +// Test infrastructure +#include "jni_binder.h" +#include "jvmti_helper.h" +#include "test_env.h" namespace art { namespace Test980RedefineObjects { @@ -39,9 +41,11 @@ extern "C" JNIEXPORT void JNICALL Java_art_test_TestWatcher_NotifyConstructed( JNIEnv* env, jclass TestWatcherClass ATTRIBUTE_UNUSED, jobject constructed) { char* sig = nullptr; char* generic_sig = nullptr; - if (JvmtiErrorToException(env, jvmti_env->GetClassSignature(env->GetObjectClass(constructed), - &sig, - &generic_sig))) { + if (JvmtiErrorToException(env, + jvmti_env, + jvmti_env->GetClassSignature(env->GetObjectClass(constructed), + &sig, + &generic_sig))) { // Exception. return; } diff --git a/test/981-dedup-original-dex/src/Main.java b/test/981-dedup-original-dex/src/Main.java index cd3f007532..1e063cf626 100644 --- a/test/981-dedup-original-dex/src/Main.java +++ b/test/981-dedup-original-dex/src/Main.java @@ -16,8 +16,10 @@ import java.lang.reflect.Field; import java.util.Base64; +import java.nio.ByteBuffer; import dalvik.system.ClassExt; +import dalvik.system.InMemoryDexClassLoader; public class Main { @@ -67,6 +69,53 @@ public class Main { "AAQAAADgAAAABgAAAAEAAAAAAQAAASAAAAIAAAAgAQAAARAAAAEAAABcAQAAAiAAAA4AAABiAQAA" + "AyAAAAIAAAAWAgAAACAAAAEAAAAhAgAAABAAAAEAAAAwAgAA"); + + /** + * base64 encoded class/dex file for + * class Transform3 { + * public void sayHi() { + * System.out.println("hello3"); + * } + * } + */ + private static final byte[] DEX_BYTES_3_INITIAL = Base64.getDecoder().decode( + "ZGV4CjAzNQC2W2fBsAeLNAwWYlG8FVigzfsV7nBWITzQAgAAcAAAAHhWNBIAAAAAAAAAADACAAAO" + + "AAAAcAAAAAYAAACoAAAAAgAAAMAAAAABAAAA2AAAAAQAAADgAAAAAQAAAAABAACwAQAAIAEAAGIB" + + "AABqAQAAeAEAAI8BAACjAQAAtwEAAMsBAADcAQAA3wEAAOMBAAD3AQAA/wEAAAQCAAANAgAAAQAA" + + "AAIAAAADAAAABAAAAAUAAAAHAAAABwAAAAUAAAAAAAAACAAAAAUAAABcAQAABAABAAsAAAAAAAAA" + + "AAAAAAAAAAANAAAAAQABAAwAAAACAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAGAAAAAAAAAB8CAAAA" + + "AAAAAQABAAEAAAAUAgAABAAAAHAQAwAAAA4AAwABAAIAAAAZAgAACQAAAGIAAAAbAQoAAABuIAIA" + + "EAAOAAAAAQAAAAMABjxpbml0PgAMTFRyYW5zZm9ybTM7ABVMamF2YS9pby9QcmludFN0cmVhbTsA" + + "EkxqYXZhL2xhbmcvT2JqZWN0OwASTGphdmEvbGFuZy9TdHJpbmc7ABJMamF2YS9sYW5nL1N5c3Rl" + + "bTsAD1RyYW5zZm9ybTMuamF2YQABVgACVkwAEmVtaXR0ZXI6IGphY2stNC4zMAAGaGVsbG8zAANv" + + "dXQAB3ByaW50bG4ABXNheUhpAAIABw4ABAAHDocAAAABAQCAgASgAgEBuAIAAAANAAAAAAAAAAEA" + + "AAAAAAAAAQAAAA4AAABwAAAAAgAAAAYAAACoAAAAAwAAAAIAAADAAAAABAAAAAEAAADYAAAABQAA" + + "AAQAAADgAAAABgAAAAEAAAAAAQAAASAAAAIAAAAgAQAAARAAAAEAAABcAQAAAiAAAA4AAABiAQAA" + + "AyAAAAIAAAAUAgAAACAAAAEAAAAfAgAAABAAAAEAAAAwAgAA"); + + /** + * base64 encoded class/dex file for + * class Transform3 { + * public void sayHi() { + * System.out.println("Goodbye3"); + * } + * } + */ + private static final byte[] DEX_BYTES_3_FINAL = Base64.getDecoder().decode( + "ZGV4CjAzNQBAXE5GthgMydaFBuinf+ZBfXcBYIw2UlXQAgAAcAAAAHhWNBIAAAAAAAAAADACAAAO" + + "AAAAcAAAAAYAAACoAAAAAgAAAMAAAAABAAAA2AAAAAQAAADgAAAAAQAAAAABAACwAQAAIAEAAGIB" + + "AABqAQAAdAEAAIIBAACZAQAArQEAAMEBAADVAQAA5gEAAOkBAADtAQAAAQIAAAYCAAAPAgAAAgAA" + + "AAMAAAAEAAAABQAAAAYAAAAIAAAACAAAAAUAAAAAAAAACQAAAAUAAABcAQAABAABAAsAAAAAAAAA" + + "AAAAAAAAAAANAAAAAQABAAwAAAACAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAHAAAAAAAAACECAAAA" + + "AAAAAQABAAEAAAAWAgAABAAAAHAQAwAAAA4AAwABAAIAAAAbAgAACQAAAGIAAAAbAQEAAABuIAIA" + + "EAAOAAAAAQAAAAMABjxpbml0PgAIR29vZGJ5ZTMADExUcmFuc2Zvcm0zOwAVTGphdmEvaW8vUHJp" + + "bnRTdHJlYW07ABJMamF2YS9sYW5nL09iamVjdDsAEkxqYXZhL2xhbmcvU3RyaW5nOwASTGphdmEv" + + "bGFuZy9TeXN0ZW07AA9UcmFuc2Zvcm0zLmphdmEAAVYAAlZMABJlbWl0dGVyOiBqYWNrLTQuMzAA" + + "A291dAAHcHJpbnRsbgAFc2F5SGkAAgAHDgAEAAcOhwAAAAEBAICABKACAQG4AgANAAAAAAAAAAEA" + + "AAAAAAAAAQAAAA4AAABwAAAAAgAAAAYAAACoAAAAAwAAAAIAAADAAAAABAAAAAEAAADYAAAABQAA" + + "AAQAAADgAAAABgAAAAEAAAAAAQAAASAAAAIAAAAgAQAAARAAAAEAAABcAQAAAiAAAA4AAABiAQAA" + + "AyAAAAIAAAAWAgAAACAAAAEAAAAhAgAAABAAAAEAAAAwAgAA"); + public static void main(String[] args) { try { doTest(); @@ -125,6 +174,20 @@ public class Main { enableCommonRetransformation(false); doCommonClassRedefinition(Transform.class, new byte[0], DEX_BYTES_1); assertSame((new byte[0]).getClass(), getOriginalDexFile(t1.getClass()).getClass()); + + // Check we don't have anything if we don't have any originalDexFile if the onload + // transformation doesn't do anything. + enableCommonRetransformation(true); + Class<?> transform3Class = new InMemoryDexClassLoader( + ByteBuffer.wrap(DEX_BYTES_3_INITIAL), Main.class.getClassLoader()).loadClass("Transform3"); + assertSame(null, getOriginalDexFile(transform3Class)); + + // Check that we end up with a java.lang.Long pointer if we do an 'on-load' redefinition. + addCommonTransformationResult("Transform3", new byte[0], DEX_BYTES_3_FINAL); + enableCommonRetransformation(true); + Class<?> transform3ClassTransformed = new InMemoryDexClassLoader( + ByteBuffer.wrap(DEX_BYTES_3_INITIAL), Main.class.getClassLoader()).loadClass("Transform3"); + assertSame(Long.class, getOriginalDexFile(transform3ClassTransformed).getClass()); } // Transforms the class diff --git a/test/982-ok-no-retransform/expected.txt b/test/982-ok-no-retransform/expected.txt new file mode 100644 index 0000000000..317e9677c3 --- /dev/null +++ b/test/982-ok-no-retransform/expected.txt @@ -0,0 +1,2 @@ +hello +hello diff --git a/test/982-ok-no-retransform/info.txt b/test/982-ok-no-retransform/info.txt new file mode 100644 index 0000000000..875a5f6ec1 --- /dev/null +++ b/test/982-ok-no-retransform/info.txt @@ -0,0 +1 @@ +Tests basic functions in the jvmti plugin. diff --git a/test/982-ok-no-retransform/run b/test/982-ok-no-retransform/run new file mode 100755 index 0000000000..c6e62ae6cd --- /dev/null +++ b/test/982-ok-no-retransform/run @@ -0,0 +1,17 @@ +#!/bin/bash +# +# Copyright 2016 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +./default-run "$@" --jvmti diff --git a/test/982-ok-no-retransform/src/Main.java b/test/982-ok-no-retransform/src/Main.java new file mode 100644 index 0000000000..7bb4a46155 --- /dev/null +++ b/test/982-ok-no-retransform/src/Main.java @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.Base64; +public class Main { + + public static void main(String[] args) { + doTest(new Transform()); + } + + public static void doTest(Transform t) { + t.sayHi(); + enableCommonRetransformation(true); + doCommonClassRetransformation(Transform.class); + t.sayHi(); + } + + // Transforms the class + private static native void doCommonClassRetransformation(Class<?>... target); + private static native void enableCommonRetransformation(boolean enable); +} diff --git a/test/982-ok-no-retransform/src/Transform.java b/test/982-ok-no-retransform/src/Transform.java new file mode 100644 index 0000000000..8e8af355da --- /dev/null +++ b/test/982-ok-no-retransform/src/Transform.java @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +class Transform { + public void sayHi() { + // Use lower 'h' to make sure the string will have a different string id + // than the transformation (the transformation code is the same except + // the actual printed String, which was making the test inacurately passing + // in JIT mode when loading the string from the dex cache, as the string ids + // of the two different strings were the same). + // We know the string ids will be different because lexicographically: + // "Goodbye" < "LTransform;" < "hello". + System.out.println("hello"); + } +} diff --git a/test/983-source-transform-verify/expected.txt b/test/983-source-transform-verify/expected.txt new file mode 100644 index 0000000000..0a94212ad2 --- /dev/null +++ b/test/983-source-transform-verify/expected.txt @@ -0,0 +1,2 @@ +Dex file hook for Transform +Dex file hook for java/lang/Object diff --git a/test/983-source-transform-verify/info.txt b/test/983-source-transform-verify/info.txt new file mode 100644 index 0000000000..875a5f6ec1 --- /dev/null +++ b/test/983-source-transform-verify/info.txt @@ -0,0 +1 @@ +Tests basic functions in the jvmti plugin. diff --git a/test/983-source-transform-verify/run b/test/983-source-transform-verify/run new file mode 100755 index 0000000000..c6e62ae6cd --- /dev/null +++ b/test/983-source-transform-verify/run @@ -0,0 +1,17 @@ +#!/bin/bash +# +# Copyright 2016 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +./default-run "$@" --jvmti diff --git a/test/983-source-transform-verify/source_transform.cc b/test/983-source-transform-verify/source_transform.cc new file mode 100644 index 0000000000..3ef3c7cb45 --- /dev/null +++ b/test/983-source-transform-verify/source_transform.cc @@ -0,0 +1,126 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <inttypes.h> +#include <stdio.h> +#include <string.h> + +#include <iostream> +#include <vector> + +#include "android-base/stringprintf.h" + +#include "base/logging.h" +#include "base/macros.h" +#include "bytecode_utils.h" +#include "dex_file.h" +#include "dex_instruction.h" +#include "jit/jit.h" +#include "jni.h" +#include "native_stack_dump.h" +#include "jvmti.h" +#include "runtime.h" +#include "scoped_thread_state_change-inl.h" +#include "thread-inl.h" +#include "thread_list.h" + +// Test infrastructure +#include "jvmti_helper.h" +#include "test_env.h" + +namespace art { +namespace Test983SourceTransformVerify { + +constexpr bool kSkipInitialLoad = true; + +// The hook we are using. +void JNICALL CheckDexFileHook(jvmtiEnv* jvmti_env ATTRIBUTE_UNUSED, + JNIEnv* jni_env ATTRIBUTE_UNUSED, + jclass class_being_redefined, + jobject loader ATTRIBUTE_UNUSED, + const char* name, + jobject protection_domain ATTRIBUTE_UNUSED, + jint class_data_len, + const unsigned char* class_data, + jint* new_class_data_len ATTRIBUTE_UNUSED, + unsigned char** new_class_data ATTRIBUTE_UNUSED) { + if (kSkipInitialLoad && class_being_redefined == nullptr) { + // Something got loaded concurrently. Just ignore it for now. + return; + } + std::cout << "Dex file hook for " << name << std::endl; + if (IsJVM()) { + return; + } + std::string error; + std::unique_ptr<const DexFile> dex(DexFile::Open(class_data, + class_data_len, + "fake_location.dex", + /*location_checksum*/ 0, + /*oat_dex_file*/ nullptr, + /*verify*/ true, + /*verify_checksum*/ true, + &error)); + if (dex.get() == nullptr) { + std::cout << "Failed to verify dex file for " << name << " because " << error << std::endl; + return; + } + for (uint32_t i = 0; i < dex->NumClassDefs(); i++) { + const DexFile::ClassDef& def = dex->GetClassDef(i); + const uint8_t* data_item = dex->GetClassData(def); + if (data_item == nullptr) { + continue; + } + for (ClassDataItemIterator it(*dex, data_item); it.HasNext(); it.Next()) { + if (!it.IsAtMethod() || it.GetMethodCodeItem() == nullptr) { + continue; + } + for (CodeItemIterator code_it(*it.GetMethodCodeItem()); !code_it.Done(); code_it.Advance()) { + const Instruction& inst = code_it.CurrentInstruction(); + int forbiden_flags = (Instruction::kVerifyError | Instruction::kVerifyRuntimeOnly); + if (inst.Opcode() == Instruction::RETURN_VOID_NO_BARRIER || + (inst.GetVerifyExtraFlags() & forbiden_flags) != 0) { + std::cout << "Unexpected instruction found in " << dex->PrettyMethod(it.GetMemberIndex()) + << " [Dex PC: 0x" << std::hex << code_it.CurrentDexPc() << std::dec << "] : " + << inst.DumpString(dex.get()) << std::endl; + continue; + } + } + } + } +} + +// Get all capabilities except those related to retransformation. +jint OnLoad(JavaVM* vm, + char* options ATTRIBUTE_UNUSED, + void* reserved ATTRIBUTE_UNUSED) { + if (vm->GetEnv(reinterpret_cast<void**>(&jvmti_env), JVMTI_VERSION_1_0)) { + printf("Unable to get jvmti env!\n"); + return 1; + } + SetAllCapabilities(jvmti_env); + jvmtiEventCallbacks cb; + memset(&cb, 0, sizeof(cb)); + cb.ClassFileLoadHook = CheckDexFileHook; + if (jvmti_env->SetEventCallbacks(&cb, sizeof(cb)) != JVMTI_ERROR_NONE) { + printf("Unable to set class file load hook cb!\n"); + return 1; + } + return 0; +} + +} // namespace Test983SourceTransformVerify +} // namespace art diff --git a/test/983-source-transform-verify/source_transform.h b/test/983-source-transform-verify/source_transform.h new file mode 100644 index 0000000000..db9415aec1 --- /dev/null +++ b/test/983-source-transform-verify/source_transform.h @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_TEST_983_SOURCE_TRANSFORM_VERIFY_SOURCE_TRANSFORM_H_ +#define ART_TEST_983_SOURCE_TRANSFORM_VERIFY_SOURCE_TRANSFORM_H_ + +#include <jni.h> + +namespace art { +namespace Test983SourceTransformVerify { + +jint OnLoad(JavaVM* vm, char* options, void* reserved); + +} // namespace Test983SourceTransformVerify +} // namespace art + +#endif // ART_TEST_983_SOURCE_TRANSFORM_VERIFY_SOURCE_TRANSFORM_H_ diff --git a/test/983-source-transform-verify/src/Main.java b/test/983-source-transform-verify/src/Main.java new file mode 100644 index 0000000000..5f42d29abe --- /dev/null +++ b/test/983-source-transform-verify/src/Main.java @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.Base64; +public class Main { + + public static void main(String[] args) { + doTest(); + } + + public static void doTest() { + Transform abc = new Transform(); + enableCommonRetransformation(true); + doCommonClassRetransformation(Transform.class); + doCommonClassRetransformation(Object.class); + enableCommonRetransformation(false); + } + + // Transforms the class + private static native void doCommonClassRetransformation(Class<?>... target); + private static native void enableCommonRetransformation(boolean enable); +} diff --git a/test/983-source-transform-verify/src/Transform.java b/test/983-source-transform-verify/src/Transform.java new file mode 100644 index 0000000000..8e8af355da --- /dev/null +++ b/test/983-source-transform-verify/src/Transform.java @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +class Transform { + public void sayHi() { + // Use lower 'h' to make sure the string will have a different string id + // than the transformation (the transformation code is the same except + // the actual printed String, which was making the test inacurately passing + // in JIT mode when loading the string from the dex cache, as the string ids + // of the two different strings were the same). + // We know the string ids will be different because lexicographically: + // "Goodbye" < "LTransform;" < "hello". + System.out.println("hello"); + } +} diff --git a/test/Android.bp b/test/Android.bp index 4ebfd7429a..40f7edd1d2 100644 --- a/test/Android.bp +++ b/test/Android.bp @@ -241,26 +241,24 @@ art_cc_test_library { } art_cc_defaults { - name: "libtiagent-defaults", + name: "libtiagent-base-defaults", defaults: ["libartagent-defaults"], srcs: [ - // This is to get the IsInterpreted native method. - "common/stack_inspect.cc", - "common/runtime_state.cc", - "ti-agent/common_load.cc", - "ti-agent/common_helper.cc", - "901-hello-ti-agent/basics.cc", + // These are the ART-independent parts. + "ti-agent/agent_startup.cc", + "ti-agent/jni_binder.cc", + "ti-agent/jvmti_helper.cc", + "ti-agent/test_env.cc", + // This is the list of non-special OnLoad things and excludes BCI and anything that depends + // on ART internals. "903-hello-tagging/tagging.cc", "904-object-allocation/tracking.cc", "905-object-free/tracking_free.cc", "906-iterate-heap/iterate_heap.cc", "907-get-loaded-classes/get_loaded_classes.cc", "908-gc-start-finish/gc_callbacks.cc", - "909-attach-agent/attach.cc", "910-methods/methods.cc", "911-get-stack-trace/stack_trace.cc", - "912-classes/classes.cc", - "913-heaps/heaps.cc", "918-fields/fields.cc", "920-objects/objects.cc", "922-properties/properties.cc", @@ -272,15 +270,35 @@ art_cc_defaults { "929-search/search.cc", "931-agent-thread/agent_thread.cc", "933-misc-events/misc_events.cc", - "936-search-onload/search_onload.cc", - "944-transform-classloaders/classloader.cc", - "945-obsolete-native/obsolete_native.cc", - "980-redefine-object/redefine_object.cc", ], shared_libs: [ "libbase", ], header_libs: ["libopenjdkjvmti_headers"], + include_dirs: ["art/test/ti-agent"], +} + +art_cc_defaults { + name: "libtiagent-defaults", + defaults: ["libtiagent-base-defaults"], + srcs: [ + // This is to get the IsInterpreted native method. + "common/stack_inspect.cc", + "common/runtime_state.cc", + // This includes the remaining test functions. We should try to refactor things to + // make this list smaller. + "ti-agent/common_helper.cc", + "ti-agent/common_load.cc", + "901-hello-ti-agent/basics.cc", + "909-attach-agent/attach.cc", + "912-classes/classes.cc", + "913-heaps/heaps.cc", + "936-search-onload/search_onload.cc", + "944-transform-classloaders/classloader.cc", + "945-obsolete-native/obsolete_native.cc", + "980-redefine-object/redefine_object.cc", + "983-source-transform-verify/source_transform.cc", + ], } art_cc_test_library { @@ -298,6 +316,12 @@ art_cc_test_library { shared_libs: ["libartd"], } +art_cc_test_library { + name: "libctstiagent", + defaults: ["libtiagent-base-defaults"], + export_include_dirs: ["ti-agent"], +} + cc_defaults { name: "libarttest-defaults", defaults: [ diff --git a/test/Android.run-test.mk b/test/Android.run-test.mk index cc015b031d..187b383b14 100644 --- a/test/Android.run-test.mk +++ b/test/Android.run-test.mk @@ -208,7 +208,7 @@ define core-image-dependencies endef TARGET_TYPES := host target -COMPILER_TYPES := jit interpreter optimizing regalloc_gc jit interp-ac +COMPILER_TYPES := jit interpreter optimizing regalloc_gc jit interp-ac speed-profile IMAGE_TYPES := picimage no-image multipicimage ALL_ADDRESS_SIZES := 64 32 diff --git a/test/etc/run-test-jar b/test/etc/run-test-jar index 808e58a7bd..f1b6132e94 100755 --- a/test/etc/run-test-jar +++ b/test/etc/run-test-jar @@ -64,6 +64,7 @@ TEST_IS_NDEBUG="n" APP_IMAGE="y" VDEX_FILTER="" PROFILE="n" +RANDOM_PROFILE="n" # if "y", run 'sync' before dalvikvm to make sure all files from # build step (e.g. dex2oat) were finished writing. @@ -273,6 +274,9 @@ while true; do elif [ "x$1" = "x--profile" ]; then PROFILE="y" shift + elif [ "x$1" = "x--random-profile" ]; then + RANDOM_PROFILE="y" + shift elif expr "x$1" : "x--" >/dev/null 2>&1; then echo "unknown $0 option: $1" 1>&2 exit 1 @@ -506,17 +510,23 @@ mkdir_locations="${DEX_LOCATION}/dalvik-cache/$ISA" strip_cmdline="true" sync_cmdline="true" -if [ "$PROFILE" = "y" ]; then +# PROFILE takes precedence over RANDOM_PROFILE, since PROFILE tests require a +# specific profile to run properly. +if [ "$PROFILE" = "y" ] || [ "$RANDOM_PROFILE" = "y" ]; then profman_cmdline="${ANDROID_ROOT}/bin/profman \ --apk=$DEX_LOCATION/$TEST_NAME.jar \ - --dex-location=$DEX_LOCATION/$TEST_NAME.jar \ - --create-profile-from=$DEX_LOCATION/profile \ - --reference-profile-file=$DEX_LOCATION/$TEST_NAME.prof" + --dex-location=$DEX_LOCATION/$TEST_NAME.jar" COMPILE_FLAGS="${COMPILE_FLAGS} --profile-file=$DEX_LOCATION/$TEST_NAME.prof" FLAGS="${FLAGS} -Xcompiler-option --profile-file=$DEX_LOCATION/$TEST_NAME.prof" + if [ "$PROFILE" = "y" ]; then + profman_cmdline="${profman_cmdline} --create-profile-from=$DEX_LOCATION/profile \ + --reference-profile-file=$DEX_LOCATION/$TEST_NAME.prof" + else + profman_cmdline="${profman_cmdline} --generate-test-profile=$DEX_LOCATION/$TEST_NAME.prof \ + --generate-test-profile-seed=0" + fi fi - if [ "$PREBUILD" = "y" ]; then mkdir_locations="${mkdir_locations} ${DEX_LOCATION}/oat/$ISA" if [ "$APP_IMAGE" = "y" ]; then @@ -603,7 +613,7 @@ if [ "$HOST" = "n" ]; then adb shell mkdir -p $DEX_LOCATION adb push $TEST_NAME.jar $DEX_LOCATION adb push $TEST_NAME-ex.jar $DEX_LOCATION - if [ "$PROFILE" = "y" ]; then + if [ "$PROFILE" = "y" ] || [ "$RANDOM_PROFILE" = "y" ]; then adb push profile $DEX_LOCATION fi else @@ -611,7 +621,7 @@ if [ "$HOST" = "n" ]; then adb shell mkdir -p $DEX_LOCATION >/dev/null 2>&1 adb push $TEST_NAME.jar $DEX_LOCATION >/dev/null 2>&1 adb push $TEST_NAME-ex.jar $DEX_LOCATION >/dev/null 2>&1 - if [ "$PROFILE" = "y" ]; then + if [ "$PROFILE" = "y" ] || [ "$RANDOM_PROFILE" = "y" ]; then adb push profile $DEX_LOCATION >/dev/null 2>&1 fi diff --git a/test/knownfailures.json b/test/knownfailures.json index abbdbb1f95..7891d4c19f 100644 --- a/test/knownfailures.json +++ b/test/knownfailures.json @@ -259,7 +259,7 @@ "602-deoptimizeable"], "description": ["Tests that should fail when the optimizing compiler ", "compiles them non-debuggable."], - "variant": "optimizing & ndebuggable | regalloc_gc & ndebuggable" + "variant": "optimizing & ndebuggable | regalloc_gc & ndebuggable | speed-profile & ndebuggable" }, { "tests": "596-app-images", @@ -334,6 +334,7 @@ "616-cha", "616-cha-abstract", "616-cha-interface", + "616-cha-interface-default", "616-cha-miranda", "616-cha-proxy-method-inline"], "bug": "http://b/36344364 http://b/36344221", @@ -361,9 +362,9 @@ "variant": "interp-ac" }, { - "tests": "638-checker-inline-caches", - "description": ["Disable 638-checker-inline-caches temporarily until a fix", - "arrives."], - "bug": "http://b/36371709" + "tests": ["629-vdex-speed", + "634-vdex-duplicate"], + "description": ["Profile driven dexlayout does not work with vdex or dex verifier."], + "variant": "speed-profile" } ] diff --git a/test/run-all-tests b/test/run-all-tests index 402c299b55..a0d2f23aa4 100755 --- a/test/run-all-tests +++ b/test/run-all-tests @@ -155,6 +155,9 @@ while true; do elif [ "x$1" = "x--strace" ]; then run_args="${run_args} --strace" shift + elif [ "x$1" = "x--random-profile" ]; then + run_args="${run_args} --random-profile" + shift elif expr "x$1" : "x--" >/dev/null 2>&1; then echo "unknown $0 option: $1" 1>&2 usage="yes" diff --git a/test/run-test b/test/run-test index a6903ff0d4..91ffdfa2cc 100755 --- a/test/run-test +++ b/test/run-test @@ -382,6 +382,9 @@ while true; do filter=$1 run_args="${run_args} --vdex-filter $filter" shift + elif [ "x$1" = "x--random-profile" ]; then + run_args="${run_args} --random-profile" + shift elif expr "x$1" : "x--" >/dev/null 2>&1; then echo "unknown $0 option: $1" 1>&2 usage="yes" diff --git a/test/testrunner/env.py b/test/testrunner/env.py index e93fb3afa8..46244a4766 100644 --- a/test/testrunner/env.py +++ b/test/testrunner/env.py @@ -105,6 +105,9 @@ ART_TEST_OPTIMIZING = getEnvBoolean('ART_TEST_OPTIMIZING', ART_TEST_FULL) # Do you want to test the optimizing compiler with graph coloring register allocation? ART_TEST_OPTIMIZING_GRAPH_COLOR = getEnvBoolean('ART_TEST_OPTIMIZING_GRAPH_COLOR', ART_TEST_FULL) +# Do you want to do run-tests with profiles? +ART_TEST_SPEED_PROFILE = getEnvBoolean('ART_TEST_SPEED_PROFILE', ART_TEST_FULL) + # Do we want to test PIC-compiled tests ("apps")? ART_TEST_PIC_TEST = getEnvBoolean('ART_TEST_PIC_TEST', ART_TEST_FULL) # Do you want tracing tests run? diff --git a/test/testrunner/run_build_test_target.py b/test/testrunner/run_build_test_target.py index e105da33d4..282ac484b6 100755 --- a/test/testrunner/run_build_test_target.py +++ b/test/testrunner/run_build_test_target.py @@ -53,7 +53,8 @@ if target.get('target'): build_command += ' ' + target.get('target') # Add 'dist' to avoid Jack issues b/36169180. build_command += ' dist' - print build_command.split() + sys.stdout.write(str(build_command)) + sys.stdout.flush() if subprocess.call(build_command.split()): sys.exit(1) @@ -66,7 +67,8 @@ if target.get('run-tests'): run_test_command += ['--host'] run_test_command += ['--verbose'] - print run_test_command + sys.stdout.write(str(run_test_command)) + sys.stdout.flush() if subprocess.call(run_test_command): sys.exit(1) diff --git a/test/testrunner/testrunner.py b/test/testrunner/testrunner.py index 3203f7ad84..49dc657020 100755 --- a/test/testrunner/testrunner.py +++ b/test/testrunner/testrunner.py @@ -53,6 +53,7 @@ import os import re import subprocess import sys +import tempfile import threading import time @@ -146,7 +147,7 @@ def gather_test_info(): VARIANT_TYPE_DICT['jni'] = {'jni', 'forcecopy', 'checkjni'} VARIANT_TYPE_DICT['address_sizes'] = {'64', '32'} VARIANT_TYPE_DICT['compiler'] = {'interp-ac', 'interpreter', 'jit', 'optimizing', - 'regalloc_gc'} + 'regalloc_gc', 'speed-profile'} for v_type in VARIANT_TYPE_DICT: TOTAL_VARIANTS_SET = TOTAL_VARIANTS_SET.union(VARIANT_TYPE_DICT.get(v_type)) @@ -191,6 +192,8 @@ def setup_test_env(): if env.ART_TEST_OPTIMIZING: COMPILER_TYPES.add('optimizing') OPTIMIZING_COMPILER_TYPES.add('optimizing') + if env.ART_TEST_SPEED_PROFILE: + COMPILER_TYPES.add('speed-profile') # By default we run all 'compiler' variants. if not COMPILER_TYPES: @@ -198,6 +201,7 @@ def setup_test_env(): COMPILER_TYPES.add('jit') COMPILER_TYPES.add('interpreter') COMPILER_TYPES.add('interp-ac') + COMPILER_TYPES.add('speed-profile') OPTIMIZING_COMPILER_TYPES.add('optimizing') if env.ART_TEST_RUN_TEST_RELOCATE: @@ -388,6 +392,8 @@ def run_tests(tests): options_test += ' --interpreter --verify-soft-fail' elif compiler == 'jit': options_test += ' --jit' + elif compiler == 'speed-profile': + options_test += ' --random-profile' if relocate == 'relocate': options_test += ' --relocate' @@ -433,8 +439,10 @@ def run_tests(tests): options_test += ' --instruction-set-features ' + \ env.HOST_2ND_ARCH_PREFIX_DEX2OAT_HOST_INSTRUCTION_SET_FEATURES - options_test = (' --output-path %s/run-test-output/%s') % ( - env.ART_HOST_TEST_DIR, test_name) + options_test + # TODO(http://36039166): This is a temporary solution to + # fix build breakages. + options_test = (' --output-path %s') % ( + tempfile.mkdtemp(dir=env.ART_HOST_TEST_DIR)) + options_test run_test_sh = env.ANDROID_BUILD_TOP + '/art/test/run-test' command = run_test_sh + ' ' + options_test + ' ' + test @@ -480,7 +488,7 @@ def run_test(command, test, test_variant, test_name): if test_passed: print_test_info(test_name, 'PASS') else: - failed_tests.append(test_name) + failed_tests.append((test_name, script_output)) if not env.ART_TEST_KEEP_GOING: stop_testrunner = True print_test_info(test_name, 'FAIL', ('%s\n%s') % ( @@ -491,13 +499,13 @@ def run_test(command, test, test_variant, test_name): else: print_test_info(test_name, '') except subprocess.TimeoutExpired as e: - failed_tests.append(test_name) - print_test_info(test_name, 'TIMEOUT', 'timed out in %d\n%s' % ( + failed_tests.append((test_name, 'Timed out in %d seconds')) + print_test_info(test_name, 'TIMEOUT', 'Timed out in %d seconds\n%s' % ( timeout, command)) except Exception as e: - failed_tests.append(test_name) - print_test_info(test_name, 'FAIL') - print_text(('%s\n%s\n\n') % (command, str(e))) + failed_tests.append((test_name, str(e))) + print_test_info(test_name, 'FAIL', + ('%s\n%s\n\n') % (command, str(e))) finally: semaphore.release() @@ -711,16 +719,16 @@ def print_analysis(): # Prints the list of skipped tests, if any. if skipped_tests: - print_text(COLOR_SKIP + 'SKIPPED TESTS' + COLOR_NORMAL + '\n') + print_text(COLOR_SKIP + 'SKIPPED TESTS: ' + COLOR_NORMAL + '\n') for test in skipped_tests: print_text(test + '\n') print_text('\n') # Prints the list of failed tests, if any. if failed_tests: - print_text(COLOR_ERROR + 'FAILED TESTS' + COLOR_NORMAL + '\n') - for test in failed_tests: - print_text(test + '\n') + print_text(COLOR_ERROR + 'FAILED: ' + COLOR_NORMAL + '\n') + for test_info in failed_tests: + print_text(('%s\n%s\n' % (test_info[0], test_info[1]))) def parse_test_name(test_name): @@ -878,6 +886,8 @@ def parse_option(): IMAGE_TYPES.add('no-image') if options['optimizing']: COMPILER_TYPES.add('optimizing') + if options['speed_profile']: + COMPILER_TYPES.add('speed-profile') if options['trace']: TRACE_TYPES.add('trace') if options['gcstress']: diff --git a/test/ti-agent/agent_startup.cc b/test/ti-agent/agent_startup.cc new file mode 100644 index 0000000000..b55db7b3af --- /dev/null +++ b/test/ti-agent/agent_startup.cc @@ -0,0 +1,95 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "agent_startup.h" + +#include "android-base/logging.h" +#include "android-base/macros.h" + +#include "jni_binder.h" +#include "jvmti_helper.h" + +namespace art { + +static constexpr const char* kMainClass = "Main"; + +static StartCallback gCallback = nullptr; + +// TODO: Check this. This may not work on device. The classloader containing the app's classes +// may not have been created at this point (i.e., if it's not the system classloader). +static void JNICALL VMInitCallback(jvmtiEnv* jvmti_env, + JNIEnv* jni_env, + jthread thread ATTRIBUTE_UNUSED) { + // Bind kMainClass native methods. + BindFunctions(jvmti_env, jni_env, kMainClass); + + if (gCallback != nullptr) { + gCallback(jvmti_env, jni_env); + gCallback = nullptr; + } + + // And delete the jvmtiEnv. + jvmti_env->DisposeEnvironment(); +} + +// Install a phase callback that will bind JNI functions on VMInit. +void BindOnLoad(JavaVM* vm, StartCallback callback) { + // Use a new jvmtiEnv. Otherwise we might collide with table changes. + jvmtiEnv* install_env; + if (vm->GetEnv(reinterpret_cast<void**>(&install_env), JVMTI_VERSION_1_0) != 0) { + LOG(FATAL) << "Could not get jvmtiEnv"; + } + SetAllCapabilities(install_env); + + { + jvmtiEventCallbacks callbacks; + memset(&callbacks, 0, sizeof(jvmtiEventCallbacks)); + callbacks.VMInit = VMInitCallback; + + CheckJvmtiError(install_env, install_env->SetEventCallbacks(&callbacks, sizeof(callbacks))); + } + + CheckJvmtiError(install_env, install_env->SetEventNotificationMode(JVMTI_ENABLE, + JVMTI_EVENT_VM_INIT, + nullptr)); + + gCallback = callback; +} + +// Ensure binding of the Main class when the agent is started through OnAttach. +void BindOnAttach(JavaVM* vm, StartCallback callback) { + // Get a JNIEnv. As the thread is attached, we must not destroy it. + JNIEnv* env; + CHECK_EQ(0, vm->GetEnv(reinterpret_cast<void**>(&env), JNI_VERSION_1_6)) + << "Could not get JNIEnv"; + + jvmtiEnv* jvmti_env; + CHECK_EQ(0, vm->GetEnv(reinterpret_cast<void**>(&jvmti_env), JVMTI_VERSION_1_0)) + << "Could not get jvmtiEnv"; + SetAllCapabilities(jvmti_env); + + BindFunctions(jvmti_env, env, kMainClass); + + if (callback != nullptr) { + callback(jvmti_env, env); + } + + if (jvmti_env->DisposeEnvironment() != JVMTI_ERROR_NONE) { + LOG(FATAL) << "Could not dispose temporary jvmtiEnv"; + } +} + +} // namespace art diff --git a/test/ti-agent/agent_startup.h b/test/ti-agent/agent_startup.h new file mode 100644 index 0000000000..49633202ab --- /dev/null +++ b/test/ti-agent/agent_startup.h @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_TEST_TI_AGENT_AGENT_STARTUP_H_ +#define ART_TEST_TI_AGENT_AGENT_STARTUP_H_ + +#include <functional> + +#include "jni.h" +#include "jvmti.h" + +namespace art { + +using StartCallback = void(*)(jvmtiEnv*, JNIEnv*); + +// Ensure binding of the Main class when the agent is started through OnLoad. +void BindOnLoad(JavaVM* vm, StartCallback callback); + +// Ensure binding of the Main class when the agent is started through OnAttach. +void BindOnAttach(JavaVM* vm, StartCallback callback); + +} // namespace art + +#endif // ART_TEST_TI_AGENT_AGENT_STARTUP_H_ diff --git a/test/ti-agent/common_helper.cc b/test/ti-agent/common_helper.cc index 6316a9c368..ab5dbcc0db 100644 --- a/test/ti-agent/common_helper.cc +++ b/test/ti-agent/common_helper.cc @@ -14,7 +14,7 @@ * limitations under the License. */ -#include "ti-agent/common_helper.h" +#include "common_helper.h" #include <dlfcn.h> #include <stdio.h> @@ -27,44 +27,15 @@ #include "jni_internal.h" #include "jvmti.h" #include "scoped_thread_state_change-inl.h" -#include "ScopedLocalRef.h" #include "stack.h" -#include "ti-agent/common_load.h" #include "utils.h" -namespace art { -bool RuntimeIsJVM; - -bool IsJVM() { - return RuntimeIsJVM; -} - -void SetAllCapabilities(jvmtiEnv* env) { - jvmtiCapabilities caps; - env->GetPotentialCapabilities(&caps); - env->AddCapabilities(&caps); -} - -bool JvmtiErrorToException(JNIEnv* env, jvmtiError error) { - if (error == JVMTI_ERROR_NONE) { - return false; - } - - ScopedLocalRef<jclass> rt_exception(env, env->FindClass("java/lang/RuntimeException")); - if (rt_exception.get() == nullptr) { - // CNFE should be pending. - return true; - } - - char* err; - jvmti_env->GetErrorName(error, &err); - - env->ThrowNew(rt_exception.get(), err); - - jvmti_env->Deallocate(reinterpret_cast<unsigned char*>(err)); - return true; -} +#include "jni_binder.h" +#include "jvmti_helper.h" +#include "scoped_local_ref.h" +#include "test_env.h" +namespace art { template <bool is_redefine> static void throwCommonRedefinitionError(jvmtiEnv* jvmti, @@ -303,7 +274,7 @@ extern "C" JNIEXPORT void Java_Main_enableCommonRetransformation(JNIEnv* env, JVMTI_EVENT_CLASS_FILE_LOAD_HOOK, nullptr); if (res != JVMTI_ERROR_NONE) { - JvmtiErrorToException(env, res); + JvmtiErrorToException(env, jvmti_env, res); } } @@ -412,140 +383,4 @@ jint OnLoad(JavaVM* vm, } // namespace common_transform -static void BindMethod(jvmtiEnv* jenv, - JNIEnv* env, - jclass klass, - jmethodID method) { - char* name; - char* signature; - jvmtiError name_result = jenv->GetMethodName(method, &name, &signature, nullptr); - if (name_result != JVMTI_ERROR_NONE) { - LOG(FATAL) << "Could not get methods"; - } - - std::string names[2]; - if (IsJVM()) { - // TODO Get the JNI long name - char* klass_name; - jvmtiError klass_result = jenv->GetClassSignature(klass, &klass_name, nullptr); - if (klass_result == JVMTI_ERROR_NONE) { - std::string name_str(name); - std::string klass_str(klass_name); - names[0] = GetJniShortName(klass_str, name_str); - jenv->Deallocate(reinterpret_cast<unsigned char*>(klass_name)); - } else { - LOG(FATAL) << "Could not get class name!"; - } - } else { - ScopedObjectAccess soa(Thread::Current()); - ArtMethod* m = jni::DecodeArtMethod(method); - names[0] = m->JniShortName(); - names[1] = m->JniLongName(); - } - for (const std::string& mangled_name : names) { - if (mangled_name == "") { - continue; - } - void* sym = dlsym(RTLD_DEFAULT, mangled_name.c_str()); - if (sym == nullptr) { - continue; - } - - JNINativeMethod native_method; - native_method.fnPtr = sym; - native_method.name = name; - native_method.signature = signature; - - env->RegisterNatives(klass, &native_method, 1); - - jenv->Deallocate(reinterpret_cast<unsigned char*>(name)); - jenv->Deallocate(reinterpret_cast<unsigned char*>(signature)); - return; - } - - LOG(FATAL) << "Could not find " << names[0]; -} - -static jclass FindClassWithSystemClassLoader(JNIEnv* env, const char* class_name) { - // Find the system classloader. - ScopedLocalRef<jclass> cl_klass(env, env->FindClass("java/lang/ClassLoader")); - if (cl_klass.get() == nullptr) { - return nullptr; - } - jmethodID getsystemclassloader_method = env->GetStaticMethodID(cl_klass.get(), - "getSystemClassLoader", - "()Ljava/lang/ClassLoader;"); - if (getsystemclassloader_method == nullptr) { - return nullptr; - } - ScopedLocalRef<jobject> cl(env, env->CallStaticObjectMethod(cl_klass.get(), - getsystemclassloader_method)); - if (cl.get() == nullptr) { - return nullptr; - } - - // Create a String of the name. - std::string descriptor = android::base::StringPrintf("L%s;", class_name); - std::string dot_name = DescriptorToDot(descriptor.c_str()); - ScopedLocalRef<jstring> name_str(env, env->NewStringUTF(dot_name.c_str())); - - // Call Class.forName with it. - ScopedLocalRef<jclass> c_klass(env, env->FindClass("java/lang/Class")); - if (c_klass.get() == nullptr) { - return nullptr; - } - jmethodID forname_method = env->GetStaticMethodID( - c_klass.get(), - "forName", - "(Ljava/lang/String;ZLjava/lang/ClassLoader;)Ljava/lang/Class;"); - if (forname_method == nullptr) { - return nullptr; - } - - return reinterpret_cast<jclass>(env->CallStaticObjectMethod(c_klass.get(), - forname_method, - name_str.get(), - JNI_FALSE, - cl.get())); -} - -void BindFunctions(jvmtiEnv* jenv, JNIEnv* env, const char* class_name) { - // Use JNI to load the class. - ScopedLocalRef<jclass> klass(env, env->FindClass(class_name)); - if (klass.get() == nullptr) { - // We may be called with the wrong classloader. Try explicitly using the system classloader. - env->ExceptionClear(); - klass.reset(FindClassWithSystemClassLoader(env, class_name)); - if (klass.get() == nullptr) { - LOG(FATAL) << "Could not load " << class_name; - } - } - BindFunctionsOnClass(jenv, env, klass.get()); -} - -void BindFunctionsOnClass(jvmtiEnv* jenv, JNIEnv* env, jclass klass) { - // Use JVMTI to get the methods. - jint method_count; - jmethodID* methods; - jvmtiError methods_result = jenv->GetClassMethods(klass, &method_count, &methods); - if (methods_result != JVMTI_ERROR_NONE) { - LOG(FATAL) << "Could not get methods"; - } - - // Check each method. - for (jint i = 0; i < method_count; ++i) { - jint modifiers; - jvmtiError mod_result = jenv->GetMethodModifiers(methods[i], &modifiers); - if (mod_result != JVMTI_ERROR_NONE) { - LOG(FATAL) << "Could not get methods"; - } - constexpr jint kNative = static_cast<jint>(kAccNative); - if ((modifiers & kNative) != 0) { - BindMethod(jenv, env, klass, methods[i]); - } - } - - jenv->Deallocate(reinterpret_cast<unsigned char*>(methods)); -} - } // namespace art diff --git a/test/ti-agent/common_helper.h b/test/ti-agent/common_helper.h index f10356dcbb..610019e4d2 100644 --- a/test/ti-agent/common_helper.h +++ b/test/ti-agent/common_helper.h @@ -19,13 +19,11 @@ #include "jni.h" #include "jvmti.h" -#include "ScopedLocalRef.h" namespace art { -namespace common_redefine { +namespace common_redefine { jint OnLoad(JavaVM* vm, char* options, void* reserved); - } // namespace common_redefine namespace common_retransform { @@ -36,53 +34,6 @@ namespace common_transform { jint OnLoad(JavaVM* vm, char* options, void* reserved); } // namespace common_transform - -extern bool RuntimeIsJVM; - -bool IsJVM(); - -template <typename T> -static jobjectArray CreateObjectArray(JNIEnv* env, - jint length, - const char* component_type_descriptor, - T src) { - if (length < 0) { - return nullptr; - } - - ScopedLocalRef<jclass> obj_class(env, env->FindClass(component_type_descriptor)); - if (obj_class.get() == nullptr) { - return nullptr; - } - - ScopedLocalRef<jobjectArray> ret(env, env->NewObjectArray(length, obj_class.get(), nullptr)); - if (ret.get() == nullptr) { - return nullptr; - } - - for (jint i = 0; i < length; ++i) { - jobject element = src(i); - env->SetObjectArrayElement(ret.get(), static_cast<jint>(i), element); - env->DeleteLocalRef(element); - if (env->ExceptionCheck()) { - return nullptr; - } - } - - return ret.release(); -} - -void SetAllCapabilities(jvmtiEnv* env); - -bool JvmtiErrorToException(JNIEnv* env, jvmtiError error); - -// Load the class through JNI. Inspect it, find all native methods. Construct the corresponding -// mangled name, run dlsym and bind the method. -// -// This will abort on failure. -void BindFunctions(jvmtiEnv* jvmti_env, JNIEnv* env, const char* class_name); -void BindFunctionsOnClass(jvmtiEnv* jvmti_env, JNIEnv* env, jclass klass); - } // namespace art #endif // ART_TEST_TI_AGENT_COMMON_HELPER_H_ diff --git a/test/ti-agent/common_load.cc b/test/ti-agent/common_load.cc index 8cb14bdfc5..9e7b75daf2 100644 --- a/test/ti-agent/common_load.cc +++ b/test/ti-agent/common_load.cc @@ -14,24 +14,25 @@ * limitations under the License. */ -#include "common_load.h" - #include <jni.h> #include <stdio.h> -#include "art_method-inl.h" #include "base/logging.h" #include "base/macros.h" + +#include "agent_startup.h" #include "common_helper.h" +#include "jni_binder.h" +#include "jvmti_helper.h" +#include "test_env.h" #include "901-hello-ti-agent/basics.h" #include "909-attach-agent/attach.h" #include "936-search-onload/search_onload.h" +#include "983-source-transform-verify/source_transform.h" namespace art { -jvmtiEnv* jvmti_env; - namespace { using OnLoad = jint (*)(JavaVM* vm, char* options, void* reserved); @@ -43,45 +44,6 @@ struct AgentLib { OnAttach attach; }; -static void JNICALL VMInitCallback(jvmtiEnv *jvmti_env, - JNIEnv* jni_env, - jthread thread ATTRIBUTE_UNUSED) { - // Bind Main native methods. - BindFunctions(jvmti_env, jni_env, "Main"); -} - -// Install a phase callback that will bind JNI functions on VMInit. -bool InstallBindCallback(JavaVM* vm) { - // Use a new jvmtiEnv. Otherwise we might collide with table changes. - jvmtiEnv* install_env; - if (vm->GetEnv(reinterpret_cast<void**>(&install_env), JVMTI_VERSION_1_0) != 0) { - return false; - } - SetAllCapabilities(install_env); - - { - jvmtiEventCallbacks callbacks; - memset(&callbacks, 0, sizeof(jvmtiEventCallbacks)); - callbacks.VMInit = VMInitCallback; - - jvmtiError install_error = install_env->SetEventCallbacks(&callbacks, sizeof(callbacks)); - if (install_error != JVMTI_ERROR_NONE) { - return false; - } - } - - { - jvmtiError enable_error = install_env->SetEventNotificationMode(JVMTI_ENABLE, - JVMTI_EVENT_VM_INIT, - nullptr); - if (enable_error != JVMTI_ERROR_NONE) { - return false; - } - } - - return true; -} - // A trivial OnLoad implementation that only initializes the global jvmti_env. static jint MinimalOnLoad(JavaVM* vm, char* options ATTRIBUTE_UNUSED, @@ -122,6 +84,8 @@ static AgentLib agents[] = { { "944-transform-classloaders", common_redefine::OnLoad, nullptr }, { "945-obsolete-native", common_redefine::OnLoad, nullptr }, { "981-dedup-original-dex", common_retransform::OnLoad, nullptr }, + { "982-ok-no-retransform", common_retransform::OnLoad, nullptr }, + { "983-source-transform-verify", Test983SourceTransformVerify::OnLoad, nullptr }, }; static AgentLib* FindAgent(char* name) { @@ -151,28 +115,8 @@ static bool FindAgentNameAndOptions(char* options, return true; } -static void SetIsJVM(char* options) { - RuntimeIsJVM = strncmp(options, "jvm", 3) == 0; -} - -static bool BindFunctionsAttached(JavaVM* vm, const char* class_name) { - // Get a JNIEnv. As the thread is attached, we must not destroy it. - JNIEnv* env; - if (vm->GetEnv(reinterpret_cast<void**>(&env), JNI_VERSION_1_6) != 0) { - printf("Unable to get JNI env!\n"); - return false; - } - - jvmtiEnv* jenv; - if (vm->GetEnv(reinterpret_cast<void**>(&jenv), JVMTI_VERSION_1_0) != 0) { - printf("Unable to get jvmti env!\n"); - return false; - } - SetAllCapabilities(jenv); - - BindFunctions(jenv, env, class_name); - - return true; +static void SetIsJVM(const char* options) { + SetJVM(strncmp(options, "jvm", 3) == 0); } } // namespace @@ -187,9 +131,7 @@ extern "C" JNIEXPORT jint JNICALL Agent_OnLoad(JavaVM* vm, char* options, void* SetIsJVM(remaining_options); - if (!InstallBindCallback(vm)) { - return 1; - } + BindOnLoad(vm, nullptr); AgentLib* lib = FindAgent(name_option); OnLoad fn = nullptr; @@ -213,7 +155,7 @@ extern "C" JNIEXPORT jint JNICALL Agent_OnAttach(JavaVM* vm, char* options, void return -1; } - BindFunctionsAttached(vm, "Main"); + BindOnAttach(vm, nullptr); AgentLib* lib = FindAgent(name_option); if (lib == nullptr) { diff --git a/test/ti-agent/jni_binder.cc b/test/ti-agent/jni_binder.cc new file mode 100644 index 0000000000..b66c2c76e9 --- /dev/null +++ b/test/ti-agent/jni_binder.cc @@ -0,0 +1,264 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "jni_binder.h" + +#include <dlfcn.h> +#include <inttypes.h> +#include <stdio.h> + +#include "android-base/logging.h" +#include "android-base/stringprintf.h" + +#include "jvmti_helper.h" +#include "scoped_local_ref.h" +#include "scoped_utf_chars.h" +#include "ti_utf.h" + +namespace art { + +static std::string MangleForJni(const std::string& s) { + std::string result; + size_t char_count = ti::CountModifiedUtf8Chars(s.c_str(), s.length()); + const char* cp = &s[0]; + for (size_t i = 0; i < char_count; ++i) { + uint32_t ch = ti::GetUtf16FromUtf8(&cp); + if ((ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z') || (ch >= '0' && ch <= '9')) { + result.push_back(ch); + } else if (ch == '.' || ch == '/') { + result += "_"; + } else if (ch == '_') { + result += "_1"; + } else if (ch == ';') { + result += "_2"; + } else if (ch == '[') { + result += "_3"; + } else { + const uint16_t leading = ti::GetLeadingUtf16Char(ch); + const uint32_t trailing = ti::GetTrailingUtf16Char(ch); + + android::base::StringAppendF(&result, "_0%04x", leading); + if (trailing != 0) { + android::base::StringAppendF(&result, "_0%04x", trailing); + } + } + } + return result; +} + +static std::string GetJniShortName(const std::string& class_descriptor, const std::string& method) { + // Remove the leading 'L' and trailing ';'... + std::string class_name(class_descriptor); + CHECK_EQ(class_name[0], 'L') << class_name; + CHECK_EQ(class_name[class_name.size() - 1], ';') << class_name; + class_name.erase(0, 1); + class_name.erase(class_name.size() - 1, 1); + + std::string short_name; + short_name += "Java_"; + short_name += MangleForJni(class_name); + short_name += "_"; + short_name += MangleForJni(method); + return short_name; +} + +static void BindMethod(jvmtiEnv* jvmti_env, JNIEnv* env, jclass klass, jmethodID method) { + std::string name; + std::string signature; + std::string mangled_names[2]; + { + char* name_cstr; + char* sig_cstr; + jvmtiError name_result = jvmti_env->GetMethodName(method, &name_cstr, &sig_cstr, nullptr); + CheckJvmtiError(jvmti_env, name_result); + CHECK(name_cstr != nullptr); + CHECK(sig_cstr != nullptr); + name = name_cstr; + signature = sig_cstr; + + char* klass_name; + jvmtiError klass_result = jvmti_env->GetClassSignature(klass, &klass_name, nullptr); + CheckJvmtiError(jvmti_env, klass_result); + + mangled_names[0] = GetJniShortName(klass_name, name); + // TODO: Long JNI name. + + CheckJvmtiError(jvmti_env, Deallocate(jvmti_env, name_cstr)); + CheckJvmtiError(jvmti_env, Deallocate(jvmti_env, sig_cstr)); + CheckJvmtiError(jvmti_env, Deallocate(jvmti_env, klass_name)); + } + + for (const std::string& mangled_name : mangled_names) { + if (mangled_name.empty()) { + continue; + } + void* sym = dlsym(RTLD_DEFAULT, mangled_name.c_str()); + if (sym == nullptr) { + continue; + } + + JNINativeMethod native_method; + native_method.fnPtr = sym; + native_method.name = name.c_str(); + native_method.signature = signature.c_str(); + + env->RegisterNatives(klass, &native_method, 1); + + return; + } + + LOG(FATAL) << "Could not find " << mangled_names[0]; +} + +static std::string DescriptorToDot(const char* descriptor) { + size_t length = strlen(descriptor); + if (length > 1) { + if (descriptor[0] == 'L' && descriptor[length - 1] == ';') { + // Descriptors have the leading 'L' and trailing ';' stripped. + std::string result(descriptor + 1, length - 2); + std::replace(result.begin(), result.end(), '/', '.'); + return result; + } else { + // For arrays the 'L' and ';' remain intact. + std::string result(descriptor); + std::replace(result.begin(), result.end(), '/', '.'); + return result; + } + } + // Do nothing for non-class/array descriptors. + return descriptor; +} + +static jobject GetSystemClassLoader(JNIEnv* env) { + ScopedLocalRef<jclass> cl_klass(env, env->FindClass("java/lang/ClassLoader")); + CHECK(cl_klass.get() != nullptr); + jmethodID getsystemclassloader_method = env->GetStaticMethodID(cl_klass.get(), + "getSystemClassLoader", + "()Ljava/lang/ClassLoader;"); + CHECK(getsystemclassloader_method != nullptr); + return env->CallStaticObjectMethod(cl_klass.get(), getsystemclassloader_method); +} + +static jclass FindClassWithClassLoader(JNIEnv* env, const char* class_name, jobject class_loader) { + // Create a String of the name. + std::string descriptor = android::base::StringPrintf("L%s;", class_name); + std::string dot_name = DescriptorToDot(descriptor.c_str()); + ScopedLocalRef<jstring> name_str(env, env->NewStringUTF(dot_name.c_str())); + + // Call Class.forName with it. + ScopedLocalRef<jclass> c_klass(env, env->FindClass("java/lang/Class")); + CHECK(c_klass.get() != nullptr); + jmethodID forname_method = env->GetStaticMethodID( + c_klass.get(), + "forName", + "(Ljava/lang/String;ZLjava/lang/ClassLoader;)Ljava/lang/Class;"); + CHECK(forname_method != nullptr); + + return static_cast<jclass>(env->CallStaticObjectMethod(c_klass.get(), + forname_method, + name_str.get(), + JNI_FALSE, + class_loader)); +} + +// Find the given classname. First try the implied classloader, then the system classloader, +// then use JVMTI to find all classloaders. +static jclass FindClass(jvmtiEnv* jvmti_env, + JNIEnv* env, + const char* class_name, + jobject class_loader) { + if (class_loader != nullptr) { + return FindClassWithClassLoader(env, class_name, class_loader); + } + + jclass from_implied = env->FindClass(class_name); + if (from_implied != nullptr) { + return from_implied; + } + env->ExceptionClear(); + + ScopedLocalRef<jobject> system_class_loader(env, GetSystemClassLoader(env)); + CHECK(system_class_loader.get() != nullptr); + jclass from_system = FindClassWithClassLoader(env, class_name, system_class_loader.get()); + if (from_system != nullptr) { + return from_system; + } + env->ExceptionClear(); + + // Look at the context classloaders of all threads. + jint thread_count; + jthread* threads; + CheckJvmtiError(jvmti_env, jvmti_env->GetAllThreads(&thread_count, &threads)); + JvmtiUniquePtr threads_uptr = MakeJvmtiUniquePtr(jvmti_env, threads); + + jclass result = nullptr; + for (jint t = 0; t != thread_count; ++t) { + // Always loop over all elements, as we need to free the local references. + if (result == nullptr) { + jvmtiThreadInfo info; + CheckJvmtiError(jvmti_env, jvmti_env->GetThreadInfo(threads[t], &info)); + CheckJvmtiError(jvmti_env, Deallocate(jvmti_env, info.name)); + if (info.thread_group != nullptr) { + env->DeleteLocalRef(info.thread_group); + } + if (info.context_class_loader != nullptr) { + result = FindClassWithClassLoader(env, class_name, info.context_class_loader); + env->ExceptionClear(); + env->DeleteLocalRef(info.context_class_loader); + } + } + env->DeleteLocalRef(threads[t]); + } + + if (result != nullptr) { + return result; + } + + // TODO: Implement scanning *all* classloaders. + LOG(FATAL) << "Unimplemented"; + + return nullptr; +} + +void BindFunctionsOnClass(jvmtiEnv* jvmti_env, JNIEnv* env, jclass klass) { + // Use JVMTI to get the methods. + jint method_count; + jmethodID* methods; + jvmtiError methods_result = jvmti_env->GetClassMethods(klass, &method_count, &methods); + CheckJvmtiError(jvmti_env, methods_result); + + // Check each method. + for (jint i = 0; i < method_count; ++i) { + jint modifiers; + jvmtiError mod_result = jvmti_env->GetMethodModifiers(methods[i], &modifiers); + CheckJvmtiError(jvmti_env, mod_result); + constexpr jint kNative = static_cast<jint>(0x0100); + if ((modifiers & kNative) != 0) { + BindMethod(jvmti_env, env, klass, methods[i]); + } + } + + CheckJvmtiError(jvmti_env, Deallocate(jvmti_env, methods)); +} + +void BindFunctions(jvmtiEnv* jvmti_env, JNIEnv* env, const char* class_name, jobject class_loader) { + // Use JNI to load the class. + ScopedLocalRef<jclass> klass(env, FindClass(jvmti_env, env, class_name, class_loader)); + CHECK(klass.get() != nullptr) << class_name; + BindFunctionsOnClass(jvmti_env, env, klass.get()); +} + +} // namespace art diff --git a/test/ti-agent/jni_binder.h b/test/ti-agent/jni_binder.h new file mode 100644 index 0000000000..6f96257c75 --- /dev/null +++ b/test/ti-agent/jni_binder.h @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_TEST_TI_AGENT_JNI_BINDER_H_ +#define ART_TEST_TI_AGENT_JNI_BINDER_H_ + +#include "jni.h" +#include "jvmti.h" + +namespace art { + +// Load the class through JNI. Inspect it, find all native methods. Construct the corresponding +// mangled name, run dlsym and bind the method. +// +// This will abort on failure. +void BindFunctions(jvmtiEnv* jvmti_env, + JNIEnv* env, + const char* class_name, + jobject class_loader = nullptr); + +void BindFunctionsOnClass(jvmtiEnv* jvmti_env, JNIEnv* env, jclass klass); + +} // namespace art + +#endif // ART_TEST_TI_AGENT_JNI_BINDER_H_ diff --git a/test/ti-agent/jni_helper.h b/test/ti-agent/jni_helper.h new file mode 100644 index 0000000000..0cbc6341fc --- /dev/null +++ b/test/ti-agent/jni_helper.h @@ -0,0 +1,72 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_TEST_TI_AGENT_JNI_HELPER_H_ +#define ART_TEST_TI_AGENT_JNI_HELPER_H_ + +#include "jni.h" +#include "scoped_local_ref.h" + +namespace art { + +// Create an object array using a lambda that returns a local ref for each element. +template <typename T> +static inline jobjectArray CreateObjectArray(JNIEnv* env, + jint length, + const char* component_type_descriptor, + T src) { + if (length < 0) { + return nullptr; + } + + ScopedLocalRef<jclass> obj_class(env, env->FindClass(component_type_descriptor)); + if (obj_class.get() == nullptr) { + return nullptr; + } + + ScopedLocalRef<jobjectArray> ret(env, env->NewObjectArray(length, obj_class.get(), nullptr)); + if (ret.get() == nullptr) { + return nullptr; + } + + for (jint i = 0; i < length; ++i) { + jobject element = src(i); + env->SetObjectArrayElement(ret.get(), static_cast<jint>(i), element); + env->DeleteLocalRef(element); + if (env->ExceptionCheck()) { + return nullptr; + } + } + + return ret.release(); +} + +inline bool JniThrowNullPointerException(JNIEnv* env, const char* msg) { + if (env->ExceptionCheck()) { + env->ExceptionClear(); + } + + ScopedLocalRef<jclass> exc_class(env, env->FindClass("java/lang/NullPointerException")); + if (exc_class.get() == nullptr) { + return -1; + } + + return env->ThrowNew(exc_class.get(), msg) == JNI_OK; +} + +} // namespace art + +#endif // ART_TEST_TI_AGENT_JNI_HELPER_H_ diff --git a/test/ti-agent/jvmti_helper.cc b/test/ti-agent/jvmti_helper.cc new file mode 100644 index 0000000000..598a30f940 --- /dev/null +++ b/test/ti-agent/jvmti_helper.cc @@ -0,0 +1,174 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "jvmti_helper.h" + +#include <algorithm> +#include <dlfcn.h> +#include <stdio.h> +#include <sstream> +#include <string.h> + +#include "android-base/logging.h" +#include "scoped_local_ref.h" + +namespace art { + +void CheckJvmtiError(jvmtiEnv* env, jvmtiError error) { + if (error != JVMTI_ERROR_NONE) { + char* error_name; + jvmtiError name_error = env->GetErrorName(error, &error_name); + if (name_error != JVMTI_ERROR_NONE) { + LOG(FATAL) << "Unable to get error name for " << error; + } + LOG(FATAL) << "Unexpected error: " << error_name; + } +} + +void SetAllCapabilities(jvmtiEnv* env) { + jvmtiCapabilities caps; + jvmtiError error1 = env->GetPotentialCapabilities(&caps); + CheckJvmtiError(env, error1); + jvmtiError error2 = env->AddCapabilities(&caps); + CheckJvmtiError(env, error2); +} + +bool JvmtiErrorToException(JNIEnv* env, jvmtiEnv* jvmti_env, jvmtiError error) { + if (error == JVMTI_ERROR_NONE) { + return false; + } + + ScopedLocalRef<jclass> rt_exception(env, env->FindClass("java/lang/RuntimeException")); + if (rt_exception.get() == nullptr) { + // CNFE should be pending. + return true; + } + + char* err; + CheckJvmtiError(jvmti_env, jvmti_env->GetErrorName(error, &err)); + + env->ThrowNew(rt_exception.get(), err); + + Deallocate(jvmti_env, err); + return true; +} + +std::ostream& operator<<(std::ostream& os, const jvmtiError& rhs) { + switch (rhs) { + case JVMTI_ERROR_NONE: + return os << "NONE"; + case JVMTI_ERROR_INVALID_THREAD: + return os << "INVALID_THREAD"; + case JVMTI_ERROR_INVALID_THREAD_GROUP: + return os << "INVALID_THREAD_GROUP"; + case JVMTI_ERROR_INVALID_PRIORITY: + return os << "INVALID_PRIORITY"; + case JVMTI_ERROR_THREAD_NOT_SUSPENDED: + return os << "THREAD_NOT_SUSPENDED"; + case JVMTI_ERROR_THREAD_SUSPENDED: + return os << "THREAD_SUSPENDED"; + case JVMTI_ERROR_THREAD_NOT_ALIVE: + return os << "THREAD_NOT_ALIVE"; + case JVMTI_ERROR_INVALID_OBJECT: + return os << "INVALID_OBJECT"; + case JVMTI_ERROR_INVALID_CLASS: + return os << "INVALID_CLASS"; + case JVMTI_ERROR_CLASS_NOT_PREPARED: + return os << "CLASS_NOT_PREPARED"; + case JVMTI_ERROR_INVALID_METHODID: + return os << "INVALID_METHODID"; + case JVMTI_ERROR_INVALID_LOCATION: + return os << "INVALID_LOCATION"; + case JVMTI_ERROR_INVALID_FIELDID: + return os << "INVALID_FIELDID"; + case JVMTI_ERROR_NO_MORE_FRAMES: + return os << "NO_MORE_FRAMES"; + case JVMTI_ERROR_OPAQUE_FRAME: + return os << "OPAQUE_FRAME"; + case JVMTI_ERROR_TYPE_MISMATCH: + return os << "TYPE_MISMATCH"; + case JVMTI_ERROR_INVALID_SLOT: + return os << "INVALID_SLOT"; + case JVMTI_ERROR_DUPLICATE: + return os << "DUPLICATE"; + case JVMTI_ERROR_NOT_FOUND: + return os << "NOT_FOUND"; + case JVMTI_ERROR_INVALID_MONITOR: + return os << "INVALID_MONITOR"; + case JVMTI_ERROR_NOT_MONITOR_OWNER: + return os << "NOT_MONITOR_OWNER"; + case JVMTI_ERROR_INTERRUPT: + return os << "INTERRUPT"; + case JVMTI_ERROR_INVALID_CLASS_FORMAT: + return os << "INVALID_CLASS_FORMAT"; + case JVMTI_ERROR_CIRCULAR_CLASS_DEFINITION: + return os << "CIRCULAR_CLASS_DEFINITION"; + case JVMTI_ERROR_FAILS_VERIFICATION: + return os << "FAILS_VERIFICATION"; + case JVMTI_ERROR_UNSUPPORTED_REDEFINITION_METHOD_ADDED: + return os << "UNSUPPORTED_REDEFINITION_METHOD_ADDED"; + case JVMTI_ERROR_UNSUPPORTED_REDEFINITION_SCHEMA_CHANGED: + return os << "UNSUPPORTED_REDEFINITION_SCHEMA_CHANGED"; + case JVMTI_ERROR_INVALID_TYPESTATE: + return os << "INVALID_TYPESTATE"; + case JVMTI_ERROR_UNSUPPORTED_REDEFINITION_HIERARCHY_CHANGED: + return os << "UNSUPPORTED_REDEFINITION_HIERARCHY_CHANGED"; + case JVMTI_ERROR_UNSUPPORTED_REDEFINITION_METHOD_DELETED: + return os << "UNSUPPORTED_REDEFINITION_METHOD_DELETED"; + case JVMTI_ERROR_UNSUPPORTED_VERSION: + return os << "UNSUPPORTED_VERSION"; + case JVMTI_ERROR_NAMES_DONT_MATCH: + return os << "NAMES_DONT_MATCH"; + case JVMTI_ERROR_UNSUPPORTED_REDEFINITION_CLASS_MODIFIERS_CHANGED: + return os << "UNSUPPORTED_REDEFINITION_CLASS_MODIFIERS_CHANGED"; + case JVMTI_ERROR_UNSUPPORTED_REDEFINITION_METHOD_MODIFIERS_CHANGED: + return os << "UNSUPPORTED_REDEFINITION_METHOD_MODIFIERS_CHANGED"; + case JVMTI_ERROR_UNMODIFIABLE_CLASS: + return os << "JVMTI_ERROR_UNMODIFIABLE_CLASS"; + case JVMTI_ERROR_NOT_AVAILABLE: + return os << "NOT_AVAILABLE"; + case JVMTI_ERROR_MUST_POSSESS_CAPABILITY: + return os << "MUST_POSSESS_CAPABILITY"; + case JVMTI_ERROR_NULL_POINTER: + return os << "NULL_POINTER"; + case JVMTI_ERROR_ABSENT_INFORMATION: + return os << "ABSENT_INFORMATION"; + case JVMTI_ERROR_INVALID_EVENT_TYPE: + return os << "INVALID_EVENT_TYPE"; + case JVMTI_ERROR_ILLEGAL_ARGUMENT: + return os << "ILLEGAL_ARGUMENT"; + case JVMTI_ERROR_NATIVE_METHOD: + return os << "NATIVE_METHOD"; + case JVMTI_ERROR_CLASS_LOADER_UNSUPPORTED: + return os << "CLASS_LOADER_UNSUPPORTED"; + case JVMTI_ERROR_OUT_OF_MEMORY: + return os << "OUT_OF_MEMORY"; + case JVMTI_ERROR_ACCESS_DENIED: + return os << "ACCESS_DENIED"; + case JVMTI_ERROR_WRONG_PHASE: + return os << "WRONG_PHASE"; + case JVMTI_ERROR_INTERNAL: + return os << "INTERNAL"; + case JVMTI_ERROR_UNATTACHED_THREAD: + return os << "UNATTACHED_THREAD"; + case JVMTI_ERROR_INVALID_ENVIRONMENT: + return os << "INVALID_ENVIRONMENT"; + } + LOG(FATAL) << "Unexpected error type " << static_cast<int>(rhs); + __builtin_unreachable(); +} + +} // namespace art diff --git a/test/ti-agent/jvmti_helper.h b/test/ti-agent/jvmti_helper.h new file mode 100644 index 0000000000..66d88d0752 --- /dev/null +++ b/test/ti-agent/jvmti_helper.h @@ -0,0 +1,76 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_TEST_TI_AGENT_JVMTI_HELPER_H_ +#define ART_TEST_TI_AGENT_JVMTI_HELPER_H_ + +#include "jni.h" +#include "jvmti.h" +#include <memory> +#include <ostream> + +#include "android-base/logging.h" + +namespace art { + +// Add all capabilities to the given env. +void SetAllCapabilities(jvmtiEnv* env); + +// Check whether the given error is NONE. If not, print out the corresponding error message +// and abort. +void CheckJvmtiError(jvmtiEnv* env, jvmtiError error); + +// Convert the given error to a RuntimeException with a message derived from the error. Returns +// true on error, false if error is JVMTI_ERROR_NONE. +bool JvmtiErrorToException(JNIEnv* env, jvmtiEnv* jvmti_env, jvmtiError error); + +class JvmtiDeleter { + public: + JvmtiDeleter() : env_(nullptr) {} + explicit JvmtiDeleter(jvmtiEnv* env) : env_(env) {} + + JvmtiDeleter(JvmtiDeleter&) = default; + JvmtiDeleter(JvmtiDeleter&&) = default; + JvmtiDeleter& operator=(const JvmtiDeleter&) = default; + + void operator()(unsigned char* ptr) const { + CHECK(env_ != nullptr); + jvmtiError ret = env_->Deallocate(ptr); + CheckJvmtiError(env_, ret); + } + + private: + mutable jvmtiEnv* env_; +}; + +using JvmtiUniquePtr = std::unique_ptr<unsigned char, JvmtiDeleter>; + +template <typename T> +static inline JvmtiUniquePtr MakeJvmtiUniquePtr(jvmtiEnv* env, T* mem) { + return JvmtiUniquePtr(reinterpret_cast<unsigned char*>(mem), JvmtiDeleter(env)); +} + +template <typename T> +static inline jvmtiError Deallocate(jvmtiEnv* env, T* mem) { + return env->Deallocate(reinterpret_cast<unsigned char*>(mem)); +} + +// To print jvmtiError. Does not rely on GetErrorName, so is an approximation. +std::ostream& operator<<(std::ostream& os, const jvmtiError& rhs); + +} // namespace art + +#endif // ART_TEST_TI_AGENT_JVMTI_HELPER_H_ diff --git a/test/ti-agent/scoped_local_ref.h b/test/ti-agent/scoped_local_ref.h new file mode 100644 index 0000000000..daa1583457 --- /dev/null +++ b/test/ti-agent/scoped_local_ref.h @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2010 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_TEST_TI_AGENT_SCOPED_LOCAL_REF_H_ +#define ART_TEST_TI_AGENT_SCOPED_LOCAL_REF_H_ + +#include "jni.h" + +#include <stddef.h> + +#include "android-base/macros.h" + +namespace art { + +template<typename T> +class ScopedLocalRef { + public: + ScopedLocalRef(JNIEnv* env, T localRef) : mEnv(env), mLocalRef(localRef) { + } + + ~ScopedLocalRef() { + reset(); + } + + void reset(T ptr = nullptr) { + if (ptr != mLocalRef) { + if (mLocalRef != nullptr) { + mEnv->DeleteLocalRef(mLocalRef); + } + mLocalRef = ptr; + } + } + + T release() __attribute__((warn_unused_result)) { + T localRef = mLocalRef; + mLocalRef = nullptr; + return localRef; + } + + T get() const { + return mLocalRef; + } + + private: + JNIEnv* const mEnv; + T mLocalRef; + + DISALLOW_COPY_AND_ASSIGN(ScopedLocalRef); +}; + +} // namespace art + +#endif // ART_TEST_TI_AGENT_SCOPED_LOCAL_REF_H_ diff --git a/test/ti-agent/scoped_primitive_array.h b/test/ti-agent/scoped_primitive_array.h new file mode 100644 index 0000000000..1649ed997a --- /dev/null +++ b/test/ti-agent/scoped_primitive_array.h @@ -0,0 +1,154 @@ +/* + * Copyright (C) 2010 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_TEST_TI_AGENT_SCOPED_PRIMITIVE_ARRAY_H_ +#define ART_TEST_TI_AGENT_SCOPED_PRIMITIVE_ARRAY_H_ + +#include "jni.h" + +#include "android-base/macros.h" + +#include "jni_helper.h" + +namespace art { + +#ifdef POINTER_TYPE +#error POINTER_TYPE is defined. +#else +#define POINTER_TYPE(T) T* /* NOLINT */ +#endif + +#ifdef REFERENCE_TYPE +#error REFERENCE_TYPE is defined. +#else +#define REFERENCE_TYPE(T) T& /* NOLINT */ +#endif + +// ScopedBooleanArrayRO, ScopedByteArrayRO, ScopedCharArrayRO, ScopedDoubleArrayRO, +// ScopedFloatArrayRO, ScopedIntArrayRO, ScopedLongArrayRO, and ScopedShortArrayRO provide +// convenient read-only access to Java arrays from JNI code. This is cheaper than read-write +// access and should be used by default. +#define INSTANTIATE_SCOPED_PRIMITIVE_ARRAY_RO(PRIMITIVE_TYPE, NAME) \ + class Scoped ## NAME ## ArrayRO { \ + public: \ + explicit Scoped ## NAME ## ArrayRO(JNIEnv* env) \ + : mEnv(env), mJavaArray(nullptr), mRawArray(nullptr), mSize(0) {} \ + Scoped ## NAME ## ArrayRO(JNIEnv* env, PRIMITIVE_TYPE ## Array javaArray) \ + : mEnv(env) { \ + if (javaArray == nullptr) { \ + mJavaArray = nullptr; \ + mSize = 0; \ + mRawArray = nullptr; \ + JniThrowNullPointerException(env, nullptr); \ + } else { \ + reset(javaArray); \ + } \ + } \ + ~Scoped ## NAME ## ArrayRO() { \ + if (mRawArray != nullptr && mRawArray != mBuffer) { \ + mEnv->Release ## NAME ## ArrayElements(mJavaArray, mRawArray, JNI_ABORT); \ + } \ + } \ + void reset(PRIMITIVE_TYPE ## Array javaArray) { \ + mJavaArray = javaArray; \ + mSize = mEnv->GetArrayLength(mJavaArray); \ + if (mSize <= kBufferSize) { \ + mEnv->Get ## NAME ## ArrayRegion(mJavaArray, 0, mSize, mBuffer); \ + mRawArray = mBuffer; \ + } else { \ + mRawArray = mEnv->Get ## NAME ## ArrayElements(mJavaArray, nullptr); \ + } \ + } \ + const PRIMITIVE_TYPE* get() const { return mRawArray; } \ + PRIMITIVE_TYPE ## Array getJavaArray() const { return mJavaArray; } \ + const PRIMITIVE_TYPE& operator[](size_t n) const { return mRawArray[n]; } \ + size_t size() const { return mSize; } \ + private: \ + static constexpr jsize kBufferSize = 1024; \ + JNIEnv* const mEnv; \ + PRIMITIVE_TYPE ## Array mJavaArray; \ + POINTER_TYPE(PRIMITIVE_TYPE) mRawArray; \ + jsize mSize; \ + PRIMITIVE_TYPE mBuffer[kBufferSize]; \ + DISALLOW_COPY_AND_ASSIGN(Scoped ## NAME ## ArrayRO); \ + } + +INSTANTIATE_SCOPED_PRIMITIVE_ARRAY_RO(jboolean, Boolean); +INSTANTIATE_SCOPED_PRIMITIVE_ARRAY_RO(jbyte, Byte); +INSTANTIATE_SCOPED_PRIMITIVE_ARRAY_RO(jchar, Char); +INSTANTIATE_SCOPED_PRIMITIVE_ARRAY_RO(jdouble, Double); +INSTANTIATE_SCOPED_PRIMITIVE_ARRAY_RO(jfloat, Float); +INSTANTIATE_SCOPED_PRIMITIVE_ARRAY_RO(jint, Int); +INSTANTIATE_SCOPED_PRIMITIVE_ARRAY_RO(jlong, Long); +INSTANTIATE_SCOPED_PRIMITIVE_ARRAY_RO(jshort, Short); + +#undef INSTANTIATE_SCOPED_PRIMITIVE_ARRAY_RO + +// ScopedBooleanArrayRW, ScopedByteArrayRW, ScopedCharArrayRW, ScopedDoubleArrayRW, +// ScopedFloatArrayRW, ScopedIntArrayRW, ScopedLongArrayRW, and ScopedShortArrayRW provide +// convenient read-write access to Java arrays from JNI code. These are more expensive, +// since they entail a copy back onto the Java heap, and should only be used when necessary. +#define INSTANTIATE_SCOPED_PRIMITIVE_ARRAY_RW(PRIMITIVE_TYPE, NAME) \ + class Scoped ## NAME ## ArrayRW { \ + public: \ + explicit Scoped ## NAME ## ArrayRW(JNIEnv* env) \ + : mEnv(env), mJavaArray(nullptr), mRawArray(nullptr) {} \ + Scoped ## NAME ## ArrayRW(JNIEnv* env, PRIMITIVE_TYPE ## Array javaArray) \ + : mEnv(env), mJavaArray(javaArray), mRawArray(nullptr) { \ + if (mJavaArray == nullptr) { \ + JniThrowNullPointerException(env, nullptr); \ + } else { \ + mRawArray = mEnv->Get ## NAME ## ArrayElements(mJavaArray, nullptr); \ + } \ + } \ + ~Scoped ## NAME ## ArrayRW() { \ + if (mRawArray) { \ + mEnv->Release ## NAME ## ArrayElements(mJavaArray, mRawArray, 0); \ + } \ + } \ + void reset(PRIMITIVE_TYPE ## Array javaArray) { \ + mJavaArray = javaArray; \ + mRawArray = mEnv->Get ## NAME ## ArrayElements(mJavaArray, nullptr); \ + } \ + const PRIMITIVE_TYPE* get() const { return mRawArray; } \ + PRIMITIVE_TYPE ## Array getJavaArray() const { return mJavaArray; } \ + const PRIMITIVE_TYPE& operator[](size_t n) const { return mRawArray[n]; } \ + POINTER_TYPE(PRIMITIVE_TYPE) get() { return mRawArray; } \ + REFERENCE_TYPE(PRIMITIVE_TYPE) operator[](size_t n) { return mRawArray[n]; } \ + size_t size() const { return mEnv->GetArrayLength(mJavaArray); } \ + private: \ + JNIEnv* const mEnv; \ + PRIMITIVE_TYPE ## Array mJavaArray; \ + POINTER_TYPE(PRIMITIVE_TYPE) mRawArray; \ + DISALLOW_COPY_AND_ASSIGN(Scoped ## NAME ## ArrayRW); \ + } + +INSTANTIATE_SCOPED_PRIMITIVE_ARRAY_RW(jboolean, Boolean); +INSTANTIATE_SCOPED_PRIMITIVE_ARRAY_RW(jbyte, Byte); +INSTANTIATE_SCOPED_PRIMITIVE_ARRAY_RW(jchar, Char); +INSTANTIATE_SCOPED_PRIMITIVE_ARRAY_RW(jdouble, Double); +INSTANTIATE_SCOPED_PRIMITIVE_ARRAY_RW(jfloat, Float); +INSTANTIATE_SCOPED_PRIMITIVE_ARRAY_RW(jint, Int); +INSTANTIATE_SCOPED_PRIMITIVE_ARRAY_RW(jlong, Long); +INSTANTIATE_SCOPED_PRIMITIVE_ARRAY_RW(jshort, Short); + +#undef INSTANTIATE_SCOPED_PRIMITIVE_ARRAY_RW +#undef POINTER_TYPE +#undef REFERENCE_TYPE + +} // namespace art + +#endif // ART_TEST_TI_AGENT_SCOPED_PRIMITIVE_ARRAY_H_ diff --git a/test/ti-agent/scoped_utf_chars.h b/test/ti-agent/scoped_utf_chars.h new file mode 100644 index 0000000000..422caaf84e --- /dev/null +++ b/test/ti-agent/scoped_utf_chars.h @@ -0,0 +1,92 @@ +/* + * Copyright (C) 2010 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_TEST_TI_AGENT_SCOPED_UTF_CHARS_H_ +#define ART_TEST_TI_AGENT_SCOPED_UTF_CHARS_H_ + +#include "jni.h" + +#include <string.h> + +#include "android-base/macros.h" + +#include "jni_helper.h" + +namespace art { + +class ScopedUtfChars { + public: + ScopedUtfChars(JNIEnv* env, jstring s) : env_(env), string_(s) { + if (s == nullptr) { + utf_chars_ = nullptr; + JniThrowNullPointerException(env, nullptr); + } else { + utf_chars_ = env->GetStringUTFChars(s, nullptr); + } + } + + ScopedUtfChars(ScopedUtfChars&& rhs) : + env_(rhs.env_), string_(rhs.string_), utf_chars_(rhs.utf_chars_) { + rhs.env_ = nullptr; + rhs.string_ = nullptr; + rhs.utf_chars_ = nullptr; + } + + ~ScopedUtfChars() { + if (utf_chars_) { + env_->ReleaseStringUTFChars(string_, utf_chars_); + } + } + + ScopedUtfChars& operator=(ScopedUtfChars&& rhs) { + if (this != &rhs) { + // Delete the currently owned UTF chars. + this->~ScopedUtfChars(); + + // Move the rhs ScopedUtfChars and zero it out. + env_ = rhs.env_; + string_ = rhs.string_; + utf_chars_ = rhs.utf_chars_; + rhs.env_ = nullptr; + rhs.string_ = nullptr; + rhs.utf_chars_ = nullptr; + } + return *this; + } + + const char* c_str() const { + return utf_chars_; + } + + size_t size() const { + return strlen(utf_chars_); + } + + const char& operator[](size_t n) const { + return utf_chars_[n]; + } + + private: + JNIEnv* env_; + jstring string_; + const char* utf_chars_; + + DISALLOW_COPY_AND_ASSIGN(ScopedUtfChars); +}; + +} // namespace art + +#endif // ART_TEST_TI_AGENT_SCOPED_UTF_CHARS_H_ diff --git a/test/ti-agent/test_env.cc b/test/ti-agent/test_env.cc new file mode 100644 index 0000000000..cf47f22b03 --- /dev/null +++ b/test/ti-agent/test_env.cc @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "test_env.h" + +namespace art { + +jvmtiEnv* jvmti_env = nullptr; + +static bool gRuntimeIsJVM = false; + +bool IsJVM() { + return gRuntimeIsJVM; +} + +void SetJVM(bool b) { + gRuntimeIsJVM = b; +} + +} // namespace art diff --git a/test/ti-agent/common_load.h b/test/ti-agent/test_env.h index e79a0067b0..2eb631c36c 100644 --- a/test/ti-agent/common_load.h +++ b/test/ti-agent/test_env.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef ART_TEST_TI_AGENT_COMMON_LOAD_H_ -#define ART_TEST_TI_AGENT_COMMON_LOAD_H_ +#ifndef ART_TEST_TI_AGENT_TEST_ENV_H_ +#define ART_TEST_TI_AGENT_TEST_ENV_H_ #include "jvmti.h" @@ -23,6 +23,9 @@ namespace art { extern jvmtiEnv* jvmti_env; +bool IsJVM(); +void SetJVM(bool b); + } // namespace art -#endif // ART_TEST_TI_AGENT_COMMON_LOAD_H_ +#endif // ART_TEST_TI_AGENT_TEST_ENV_H_ diff --git a/test/ti-agent/ti_macros.h b/test/ti-agent/ti_macros.h new file mode 100644 index 0000000000..d91338324f --- /dev/null +++ b/test/ti-agent/ti_macros.h @@ -0,0 +1,26 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_TEST_TI_AGENT_TI_MACROS_H_ +#define ART_TEST_TI_AGENT_TI_MACROS_H_ + +#include "android-base/macros.h" + +#define FINAL final +#define OVERRIDE override +#define UNREACHABLE __builtin_unreachable + +#endif // ART_TEST_TI_AGENT_TI_MACROS_H_ diff --git a/test/ti-agent/ti_utf.h b/test/ti-agent/ti_utf.h new file mode 100644 index 0000000000..341e1066c3 --- /dev/null +++ b/test/ti-agent/ti_utf.h @@ -0,0 +1,196 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_TEST_TI_AGENT_TI_UTF_H_ +#define ART_TEST_TI_AGENT_TI_UTF_H_ + +#include <inttypes.h> +#include <string.h> + +#include "android-base/logging.h" + +namespace art { +namespace ti { + +inline size_t CountModifiedUtf8Chars(const char* utf8, size_t byte_count) { + DCHECK_LE(byte_count, strlen(utf8)); + size_t len = 0; + const char* end = utf8 + byte_count; + for (; utf8 < end; ++utf8) { + int ic = *utf8; + len++; + if (LIKELY((ic & 0x80) == 0)) { + // One-byte encoding. + continue; + } + // Two- or three-byte encoding. + utf8++; + if ((ic & 0x20) == 0) { + // Two-byte encoding. + continue; + } + utf8++; + if ((ic & 0x10) == 0) { + // Three-byte encoding. + continue; + } + + // Four-byte encoding: needs to be converted into a surrogate + // pair. + utf8++; + len++; + } + return len; +} + +inline uint16_t GetTrailingUtf16Char(uint32_t maybe_pair) { + return static_cast<uint16_t>(maybe_pair >> 16); +} + +inline uint16_t GetLeadingUtf16Char(uint32_t maybe_pair) { + return static_cast<uint16_t>(maybe_pair & 0x0000FFFF); +} + +inline uint32_t GetUtf16FromUtf8(const char** utf8_data_in) { + const uint8_t one = *(*utf8_data_in)++; + if ((one & 0x80) == 0) { + // one-byte encoding + return one; + } + + const uint8_t two = *(*utf8_data_in)++; + if ((one & 0x20) == 0) { + // two-byte encoding + return ((one & 0x1f) << 6) | (two & 0x3f); + } + + const uint8_t three = *(*utf8_data_in)++; + if ((one & 0x10) == 0) { + return ((one & 0x0f) << 12) | ((two & 0x3f) << 6) | (three & 0x3f); + } + + // Four byte encodings need special handling. We'll have + // to convert them into a surrogate pair. + const uint8_t four = *(*utf8_data_in)++; + + // Since this is a 4 byte UTF-8 sequence, it will lie between + // U+10000 and U+1FFFFF. + // + // TODO: What do we do about values in (U+10FFFF, U+1FFFFF) ? The + // spec says they're invalid but nobody appears to check for them. + const uint32_t code_point = ((one & 0x0f) << 18) | ((two & 0x3f) << 12) + | ((three & 0x3f) << 6) | (four & 0x3f); + + uint32_t surrogate_pair = 0; + // Step two: Write out the high (leading) surrogate to the bottom 16 bits + // of the of the 32 bit type. + surrogate_pair |= ((code_point >> 10) + 0xd7c0) & 0xffff; + // Step three : Write out the low (trailing) surrogate to the top 16 bits. + surrogate_pair |= ((code_point & 0x03ff) + 0xdc00) << 16; + + return surrogate_pair; +} + +inline void ConvertUtf16ToModifiedUtf8(char* utf8_out, + size_t byte_count, + const uint16_t* utf16_in, + size_t char_count) { + if (LIKELY(byte_count == char_count)) { + // Common case where all characters are ASCII. + const uint16_t *utf16_end = utf16_in + char_count; + for (const uint16_t *p = utf16_in; p < utf16_end;) { + *utf8_out++ = static_cast<char>(*p++); + } + return; + } + + // String contains non-ASCII characters. + while (char_count--) { + const uint16_t ch = *utf16_in++; + if (ch > 0 && ch <= 0x7f) { + *utf8_out++ = ch; + } else { + // Char_count == 0 here implies we've encountered an unpaired + // surrogate and we have no choice but to encode it as 3-byte UTF + // sequence. Note that unpaired surrogates can occur as a part of + // "normal" operation. + if ((ch >= 0xd800 && ch <= 0xdbff) && (char_count > 0)) { + const uint16_t ch2 = *utf16_in; + + // Check if the other half of the pair is within the expected + // range. If it isn't, we will have to emit both "halves" as + // separate 3 byte sequences. + if (ch2 >= 0xdc00 && ch2 <= 0xdfff) { + utf16_in++; + char_count--; + const uint32_t code_point = (ch << 10) + ch2 - 0x035fdc00; + *utf8_out++ = (code_point >> 18) | 0xf0; + *utf8_out++ = ((code_point >> 12) & 0x3f) | 0x80; + *utf8_out++ = ((code_point >> 6) & 0x3f) | 0x80; + *utf8_out++ = (code_point & 0x3f) | 0x80; + continue; + } + } + + if (ch > 0x07ff) { + // Three byte encoding. + *utf8_out++ = (ch >> 12) | 0xe0; + *utf8_out++ = ((ch >> 6) & 0x3f) | 0x80; + *utf8_out++ = (ch & 0x3f) | 0x80; + } else /*(ch > 0x7f || ch == 0)*/ { + // Two byte encoding. + *utf8_out++ = (ch >> 6) | 0xc0; + *utf8_out++ = (ch & 0x3f) | 0x80; + } + } + } +} + +inline size_t CountUtf8Bytes(const uint16_t* chars, size_t char_count) { + size_t result = 0; + const uint16_t *end = chars + char_count; + while (chars < end) { + const uint16_t ch = *chars++; + if (LIKELY(ch != 0 && ch < 0x80)) { + result++; + continue; + } + if (ch < 0x800) { + result += 2; + continue; + } + if (ch >= 0xd800 && ch < 0xdc00) { + if (chars < end) { + const uint16_t ch2 = *chars; + // If we find a properly paired surrogate, we emit it as a 4 byte + // UTF sequence. If we find an unpaired leading or trailing surrogate, + // we emit it as a 3 byte sequence like would have done earlier. + if (ch2 >= 0xdc00 && ch2 < 0xe000) { + chars++; + result += 4; + continue; + } + } + } + result += 3; + } + return result; +} + +} // namespace ti +} // namespace art + +#endif // ART_TEST_TI_AGENT_TI_UTF_H_ @@ -16,106 +16,261 @@ # shell dialect that should work on the host (e.g. bash), and # Android (e.g. mksh). -function follow_links() { - if [ z"$BASH_SOURCE" != z ]; then - file="$BASH_SOURCE" - else - file="$0" - fi - while [ -h "$file" ]; do - # On Mac OS, readlink -f doesn't work. - file="$(readlink "$file")" - done - echo "$file" -} +# Globals +ARCHS={arm,arm64,mips,mips64,x86,x86_64} +ART_BINARY=dalvikvm +DELETE_ANDROID_DATA="no" +LAUNCH_WRAPPER= +LIBART=libart.so +JIT_PROFILE="no" +VERBOSE="no" + +# Follow all sym links to get the program name. +if [ z"$BASH_SOURCE" != z ]; then + PROG_NAME="$BASH_SOURCE" +else + PROG_NAME="$0" +fi +while [ -h "$PROG_NAME" ]; do + # On Mac OS, readlink -f doesn't work. + PROG_NAME="$(readlink "$PROG_NAME")" +done function find_libdir() { - # Get the actual file, $DALVIKVM may be a symbolic link. + # Get the actual file, $1 is the ART_BINARY_PATH and may be a symbolic link. # Use realpath instead of readlink because Android does not have a readlink. - if [[ "$(realpath "$ANDROID_ROOT/bin/$DALVIKVM")" == *dalvikvm64 ]]; then + if [[ "$(realpath "$1")" == *dalvikvm64 ]]; then echo "lib64" else echo "lib" fi } -invoke_with= -DALVIKVM=dalvikvm -LIBART=libart.so +function replace_compiler_filter_with_interepret_only() { + ARGS_WITH_INTERPRET_ONLY=("$@") + + found="false" + ((index=0)) + while ((index <= $#)); do + what="${ARGS_WITH_INTERPRET_ONLY[$index]}" -while true; do - if [ "$1" = "--invoke-with" ]; then + case "$what" in + --compiler-filter=*) + ARGS_WITH_INTERPRET_ONLY[$index]="--compiler-filter=interpret-only" + found="true" + ;; + esac + + ((index++)) shift - invoke_with="$invoke_with $1" + done + if [ "$found" != "true" ]; then + ARGS_WITH_INTERPRET_ONLY=(-Xcompiler-option --compiler-filter=interpret-only "${ARGS_WITH_INTERPRET_ONLY[@]}") + fi +} + +function usage() { + cat 1>&2 <<EOF +Usage: art [OPTIONS] [--] [ART_OPTIONS] CLASS + +Supported OPTIONS include: + --32 Use the 32-bit Android Runtime. + --64 Use the 64-bit Android Runtime. + --callgrind Launch the Android Runtime in callgrind. + -d Use the debug ART library (libartd.so). + --debug Equivalent to -d. + --gdb Launch the Android Runtime in gdb. + --help Display usage message. + --invoke-with <program> Launch the Android Runtime in <program>. + --perf Launch the Android Runtime with perf recording. + --perf-report Launch the Android Runtime with perf recording with + report upon completion. + --profile Run with profiling, then run using profile data. + --verbose Run script verbosely. + +The ART_OPTIONS are passed directly to the Android Runtime. + +Example: + art --32 -cp my_classes.dex MainClass + +Common errors: + 1) Not having core.art available (see $ANDROID_BUILD_TOP/art/Android.mk). + eg m -j32 build-art-host + 2) Not having boot.art available (see $ANDROID_BUILD_TOP/build/make/core/dex_preopt_libart_boot.mk) + eg m -j32 out/target/product/generic_x86_64/dex_bootjars/system/framework/x86_64/boot.art +EOF +} + +function clean_android_data() { + if [ "$DELETE_ANDROID_DATA" = "yes" ]; then + rm -rf $ANDROID_DATA + fi +} + +function verbose_run() { + if [ "$VERBOSE" = "yes" ]; then + echo "$@" + fi + eval "$@" +} + +function run_art() { + verbose_run ANDROID_DATA=$ANDROID_DATA \ + ANDROID_ROOT=$ANDROID_ROOT \ + LD_LIBRARY_PATH=$LD_LIBRARY_PATH \ + PATH=$ANDROID_ROOT/bin:$PATH \ + LD_USE_LOAD_BIAS=1 \ + $LAUNCH_WRAPPER $ART_BINARY_PATH $lib \ + -XXlib:$LIBART \ + -Xnorelocate \ + -Ximage:$ANDROID_ROOT/framework/core.art \ + "$@" +} + +while [[ "$1" = "-"* ]]; do + case $1 in + --) + # No more arguments for this script. shift - elif [ "$1" = "-d" ]; then + break + ;; + --32) + ART_BINARY=dalvikvm32 + ;; + --64) + ART_BINARY=dalvikvm64 + ;; + --callgrind) + LAUNCH_WRAPPER="valgrind --tool=callgrind" + ;; + -d) + ;& # Fallthrough + --debug) LIBART="libartd.so" + ;; + --gdb) + LIBART="libartd.so" + LAUNCH_WRAPPER="gdb --args" + ;; + --help) + usage + exit 0 + ;; + --invoke-with) + LAUNCH_WRAPPER=$2 shift - elif [ "$1" = "--32" ]; then - DALVIKVM=dalvikvm32 - shift - elif [ "$1" = "--64" ]; then - DALVIKVM=dalvikvm64 - shift - elif [ "$1" = "--perf" ]; then + ;; + --perf) PERF="record" - shift - elif [ "$1" = "--perf-report" ]; then + ;; + --perf-report) PERF="report" - shift - elif expr "$1" : "--" >/dev/null 2>&1; then + ;; + --profile) + JIT_PROFILE="yes" + ;; + --verbose) + VERBOSE="yes" + ;; + --*) echo "unknown option: $1" 1>&2 + usage exit 1 - else + ;; + *) break - fi + ;; + esac + shift done -PROG_NAME="$(follow_links)" +if [ $# -eq 0 ]; then + usage + exit 1 +fi + PROG_DIR="$(cd "${PROG_NAME%/*}" ; pwd -P)" ANDROID_ROOT=$PROG_DIR/.. -LIBDIR=$(find_libdir) +ART_BINARY_PATH=$ANDROID_ROOT/bin/$ART_BINARY + +if [ ! -x "$ART_BINARY_PATH" ]; then + cat 1>&2 <<EOF +Android Runtime not found: $ART_BINARY_PATH +This script should be in the same directory as the Android Runtime ($ART_BINARY). +EOF + exit 1 +fi + +LIBDIR="$(find_libdir $ART_BINARY_PATH)" LD_LIBRARY_PATH=$ANDROID_ROOT/$LIBDIR -DEBUG_OPTION="" +EXTRA_OPTIONS="" -DELETE_ANDROID_DATA=false # If ANDROID_DATA is the system ANDROID_DATA or is not set, use our own, # and ensure we delete it at the end. if [ "$ANDROID_DATA" = "/data" ] || [ "$ANDROID_DATA" = "" ]; then ANDROID_DATA=$PWD/android-data$$ - mkdir -p $ANDROID_DATA/dalvik-cache/{arm,arm64,x86,x86_64} - DELETE_ANDROID_DATA=true + mkdir -p $ANDROID_DATA/dalvik-cache/$ARCHS + DELETE_ANDROID_DATA="yes" fi -if [ z"$PERF" != z ]; then - invoke_with="perf record -g -o $ANDROID_DATA/perf.data -e cycles:u $invoke_with" - DEBUG_OPTION="-Xcompiler-option --generate-debug-info" +if [ "$PERF" != "" ]; then + LAUNCH_WRAPPER="perf record -g -o $ANDROID_DATA/perf.data -e cycles:u $LAUNCH_WRAPPER" + EXTRA_OPTIONS="-Xcompiler-option --generate-debug-info" fi -# We use the PIC core image to work with perf. -ANDROID_DATA=$ANDROID_DATA \ - ANDROID_ROOT=$ANDROID_ROOT \ - LD_LIBRARY_PATH=$LD_LIBRARY_PATH \ - PATH=$ANDROID_ROOT/bin:$PATH \ - LD_USE_LOAD_BIAS=1 \ - $invoke_with $ANDROID_ROOT/bin/$DALVIKVM $lib \ - -XXlib:$LIBART \ - -Xnorelocate \ - -Ximage:$ANDROID_ROOT/framework/core.art \ - $DEBUG_OPTION \ - "$@" +if [ "$JIT_PROFILE" = "yes" ]; then + # Create the profile. The runtime expects profiles to be created before + # execution. + PROFILE_PATH="$ANDROID_DATA/primary.prof" + touch $PROFILE_PATH + + # Replace the compiler filter with interpret-only so that we + # can capture the profile. + ARGS_WITH_INTERPRET_ONLY= + replace_compiler_filter_with_interepret_only "$@" + + run_art -Xjitsaveprofilinginfo \ + -Xps-min-methods-to-save:0 \ + -Xps-min-classes-to-save:0 \ + -Xps-min-notification-before-wake:10 \ + -Xps-profile-path:$PROFILE_PATH \ + -Xusejit:true \ + "${ARGS_WITH_INTERPRET_ONLY[@]}" \ + "&>" "$ANDROID_DATA/profile_gen.log" + EXIT_STATUS=$? + + if [ $EXIT_STATUS != 0 ]; then + cat "$ANDROID_DATA/profile_gen.log" + clean_android_data + exit $EXIT_STATUS + fi + + # Wipe dalvik-cache to prepare it for the next invocation. + rm -rf $ANDROID_DATA/dalvik-cache/$ARCHS/* + # Append arguments so next invocation of run_art uses the profile. + EXTRA_OPTIONS="$EXTRA_OPTIONS -Xcompiler-option --profile-file=$PROFILE_PATH" +fi + +# Protect additional arguments in quotes to preserve whitespaces when evaluated. +# This is for run-jdwp-test.sh which uses this script and has arguments with +# whitespaces when running on device. +while [ $# -gt 0 ]; do + EXTRA_OPTIONS="$EXTRA_OPTIONS \"$1\"" + shift +done + +run_art $EXTRA_OPTIONS EXIT_STATUS=$? -if [ z"$PERF" != z ]; then - if [ z"$PERF" = zreport ]; then +if [ "$PERF" != "" ]; then + if [ "$PERF" = report ]; then perf report -i $ANDROID_DATA/perf.data fi echo "Perf data saved in: $ANDROID_DATA/perf.data" else - if [ "$DELETE_ANDROID_DATA" = "true" ]; then - rm -rf $ANDROID_DATA - fi + # Perf output is placed under $ANDROID_DATA so not cleaned when perf options used. + clean_android_data fi exit $EXIT_STATUS diff --git a/tools/cpplint_presubmit.py b/tools/cpplint_presubmit.py new file mode 100755 index 0000000000..478151736f --- /dev/null +++ b/tools/cpplint_presubmit.py @@ -0,0 +1,67 @@ +#!/usr/bin/python3 +# +# Copyright 2017, The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# TODO We should unify this with build/Android.cpplint.mk. + +import os +import pathlib +import subprocess +import sys + +IGNORED_FILES = {"runtime/elf.h", "runtime/openjdkjvmti/include/jvmti.h"} + +INTERESTING_SUFFIXES = {".h", ".cc"} + +CPPLINT_FLAGS = [ + '--filter=-whitespace/line_length,-build/include,-readability/function,-readability/streams,-readability/todo,-runtime/references,-runtime/sizeof,-runtime/threadsafe_fn,-runtime/printf', + '--quiet', +] + +def is_interesting(f): + """ + Returns true if this is a file we want to run through cpplint before uploading. False otherwise. + """ + path = pathlib.Path(f) + return f not in IGNORED_FILES and path.suffix in INTERESTING_SUFFIXES and path.exists() + +def get_changed_files(commit): + """ + Gets the files changed in the given commit. + """ + return subprocess.check_output( + ["git", 'diff-tree', '--no-commit-id', '--name-only', '-r', commit], + stderr=subprocess.STDOUT, + universal_newlines=True).split() + +def run_cpplint(files): + """ + Runs cpplint on the given files. + """ + if len(files) == 0: + return + sys.exit(subprocess.call(['tools/cpplint.py'] + CPPLINT_FLAGS + files)) + +def main(): + if 'PREUPLOAD_COMMIT' in os.environ: + commit = os.environ['PREUPLOAD_COMMIT'] + else: + print("WARNING: Not running as a pre-upload hook. Assuming commit to check = 'HEAD'") + commit = "HEAD" + files_to_check = [f for f in get_changed_files(commit) if is_interesting(f)] + run_cpplint(files_to_check) + +if __name__ == '__main__': + main() diff --git a/tools/setup-buildbot-device.sh b/tools/setup-buildbot-device.sh index 7eaaaf9cbd..6c2c07213f 100755 --- a/tools/setup-buildbot-device.sh +++ b/tools/setup-buildbot-device.sh @@ -61,6 +61,9 @@ adb shell uptime echo -e "${green}Battery info${nc}" adb shell dumpsys battery +echo -e "${green}Killing logd, seen leaking on fugu/N${nc}" +adb shell killall -9 /system/bin/logd + echo -e "${green}Setting adb buffer size to 32MB${nc}" adb logcat -G 32M adb logcat -g |