diff options
220 files changed, 7124 insertions, 15505 deletions
diff --git a/Android.mk b/Android.mk index 7eb0bf926e..2df1b13abd 100644 --- a/Android.mk +++ b/Android.mk @@ -388,6 +388,7 @@ ifeq (true,$(art_target_include_debug_build)) LOCAL_REQUIRED_MODULES += \ libartd \ libartd-compiler \ + libopenjdkd \ libopenjdkjvmd \ libopenjdkjvmtid \ diff --git a/PREUPLOAD.cfg b/PREUPLOAD.cfg index 0ed230c408..8a8df36101 100644 --- a/PREUPLOAD.cfg +++ b/PREUPLOAD.cfg @@ -1,3 +1,4 @@ [Hook Scripts] check_generated_files_up_to_date = tools/cpp-define-generator/presubmit-check-files-up-to-date +check_generated_tests_up_to_date = tools/test_presubmit.py check_cpplint_on_changed_files = tools/cpplint_presubmit.py diff --git a/build/Android.cpplint.mk b/build/Android.cpplint.mk index f924a855b7..66ac897f76 100644 --- a/build/Android.cpplint.mk +++ b/build/Android.cpplint.mk @@ -18,7 +18,8 @@ include art/build/Android.common_build.mk ART_CPPLINT := $(LOCAL_PATH)/tools/cpplint.py ART_CPPLINT_FILTER := --filter=-whitespace/line_length,-build/include,-readability/function,-readability/streams,-readability/todo,-runtime/references,-runtime/sizeof,-runtime/threadsafe_fn,-runtime/printf -ART_CPPLINT_FLAGS := --quiet --root=$(ANDROID_BUILD_TOP) +ART_CPPLINT_FLAGS := --root=$(TOP) +ART_CPPLINT_QUIET := --quiet ART_CPPLINT_INGORED := \ runtime/elf.h \ runtime/openjdkjvmti/include/jvmti.h @@ -32,12 +33,12 @@ ART_CPPLINT_SRC := $(filter-out $(patsubst %,$(LOCAL_PATH)/%,$(ART_CPPLINT_INGOR # "mm cpplint-art" to verify we aren't regressing .PHONY: cpplint-art cpplint-art: - $(ART_CPPLINT) $(ART_CPPLINT_FILTER) $(ART_CPPLINT_SRC) + $(ART_CPPLINT) $(ART_CPPLINT_FLAGS) $(ART_CPPLINT_FILTER) $(ART_CPPLINT_SRC) # "mm cpplint-art-all" to see all warnings .PHONY: cpplint-art-all cpplint-art-all: - $(ART_CPPLINT) $(ART_CPPLINT_SRC) + $(ART_CPPLINT) $(ART_CPPLINT_FLAGS) $(ART_CPPLINT_SRC) OUT_CPPLINT := $(TARGET_COMMON_OUT_ROOT)/cpplint @@ -48,7 +49,7 @@ art_cpplint_file := $(1) art_cpplint_touch := $$(OUT_CPPLINT)/$$(subst /,__,$$(art_cpplint_file)) $$(art_cpplint_touch): $$(art_cpplint_file) $(ART_CPPLINT) art/build/Android.cpplint.mk - $(hide) $(ART_CPPLINT) $(ART_CPPLINT_FLAGS) $(ART_CPPLINT_FILTER) $$< + $(hide) $(ART_CPPLINT) $(ART_CPPLINT_QUIET) $(ART_CPPLINT_FLAGS) $(ART_CPPLINT_FILTER) $$< $(hide) mkdir -p $$(dir $$@) $(hide) touch $$@ diff --git a/build/Android.gtest.mk b/build/Android.gtest.mk index b6ffcc54f7..ff4f9016c0 100644 --- a/build/Android.gtest.mk +++ b/build/Android.gtest.mk @@ -104,6 +104,7 @@ ART_GTEST_dex2oat_environment_tests_DEX_DEPS := Main MainStripped MultiDex Multi ART_GTEST_atomic_method_ref_map_test_DEX_DEPS := Interfaces ART_GTEST_class_linker_test_DEX_DEPS := AllFields ErroneousA ErroneousB ErroneousInit ForClassLoaderA ForClassLoaderB ForClassLoaderC ForClassLoaderD Interfaces MethodTypes MultiDex MyClass Nested Statics StaticsFromCode +ART_GTEST_class_loader_context_test_DEX_DEPS := Main MultiDex MyClass ART_GTEST_class_table_test_DEX_DEPS := XandY ART_GTEST_compiler_driver_test_DEX_DEPS := AbstractMethod StaticLeafMethods ProfileTestMultiDex ART_GTEST_dex_cache_test_DEX_DEPS := Main Packages MethodTypes diff --git a/build/art.go b/build/art.go index 1b9c646433..6c9aa89f8f 100644 --- a/build/art.go +++ b/build/art.go @@ -68,12 +68,6 @@ func globalFlags(ctx android.BaseContext) ([]string, []string) { "-DART_READ_BARRIER_TYPE_IS_"+barrierType+"=1") } - if envTrue(ctx, "ART_USE_OLD_ARM_BACKEND") { - // Used to enable the old, pre-VIXL ARM code generator. - cflags = append(cflags, "-DART_USE_OLD_ARM_BACKEND=1") - asflags = append(asflags, "-DART_USE_OLD_ARM_BACKEND=1") - } - // We need larger stack overflow guards for ASAN, as the compiled code will have // larger frame sizes. For simplicity, just use global not-target-specific cflags. // Note: We increase this for both debug and non-debug, as the overflow gap will @@ -97,6 +91,12 @@ func globalFlags(ctx android.BaseContext) ([]string, []string) { "-DART_STACK_OVERFLOW_GAP_x86_64=8192") } + if envTrue(ctx, "ART_ENABLE_ADDRESS_SANITIZER") { + // Used to enable full sanitization, i.e., user poisoning, under ASAN. + cflags = append(cflags, "-DART_ENABLE_ADDRESS_SANITIZER=1") + asflags = append(asflags, "-DART_ENABLE_ADDRESS_SANITIZER=1") + } + return cflags, asflags } diff --git a/compiler/Android.bp b/compiler/Android.bp index e590fb745c..cacb8cbc2a 100644 --- a/compiler/Android.bp +++ b/compiler/Android.bp @@ -106,20 +106,16 @@ art_cc_defaults { "jni/quick/arm/calling_convention_arm.cc", "linker/arm/relative_patcher_arm_base.cc", "linker/arm/relative_patcher_thumb2.cc", - "optimizing/code_generator_arm.cc", - "optimizing/code_generator_vector_arm.cc", "optimizing/code_generator_arm_vixl.cc", "optimizing/code_generator_vector_arm_vixl.cc", "optimizing/instruction_simplifier_arm.cc", "optimizing/instruction_simplifier_shared.cc", - "optimizing/intrinsics_arm.cc", "optimizing/intrinsics_arm_vixl.cc", "optimizing/nodes_shared.cc", "optimizing/scheduler_arm.cc", "utils/arm/assembler_arm.cc", "utils/arm/assembler_arm_vixl.cc", "utils/arm/assembler_thumb2.cc", - "utils/arm/jni_macro_assembler_arm.cc", "utils/arm/jni_macro_assembler_arm_vixl.cc", "utils/arm/managed_register_arm.cc", ], @@ -356,6 +352,7 @@ art_cc_test { "optimizing/live_interval_test.cc", "optimizing/loop_optimization_test.cc", "optimizing/nodes_test.cc", + "optimizing/nodes_vector_test.cc", "optimizing/parallel_move_test.cc", "optimizing/pretty_printer_test.cc", "optimizing/reference_type_propagation_test.cc", diff --git a/compiler/dex/dex_to_dex_compiler.cc b/compiler/dex/dex_to_dex_compiler.cc index 2db99cda3e..fba1136d19 100644 --- a/compiler/dex/dex_to_dex_compiler.cc +++ b/compiler/dex/dex_to_dex_compiler.cc @@ -22,12 +22,14 @@ #include "art_method-inl.h" #include "base/logging.h" #include "base/mutex.h" +#include "bytecode_utils.h" #include "compiled_method.h" #include "dex_file-inl.h" #include "dex_instruction-inl.h" #include "driver/compiler_driver.h" #include "driver/dex_compilation_unit.h" #include "mirror/dex_cache.h" +#include "quicken_info.h" #include "thread-current-inl.h" namespace art { @@ -110,13 +112,9 @@ class DexCompiler { void DexCompiler::Compile() { DCHECK_EQ(dex_to_dex_compilation_level_, DexToDexCompilationLevel::kOptimize); - const DexFile::CodeItem* code_item = unit_.GetCodeItem(); - const uint16_t* insns = code_item->insns_; - const uint32_t insns_size = code_item->insns_size_in_code_units_; - Instruction* inst = const_cast<Instruction*>(Instruction::At(insns)); - - for (uint32_t dex_pc = 0; dex_pc < insns_size; - inst = const_cast<Instruction*>(inst->Next()), dex_pc = inst->GetDexPc(insns)) { + for (CodeItemIterator it(*unit_.GetCodeItem()); !it.Done(); it.Advance()) { + Instruction* inst = const_cast<Instruction*>(&it.CurrentInstruction()); + const uint32_t dex_pc = it.CurrentDexPc(); switch (inst->Opcode()) { case Instruction::RETURN_VOID: CompileReturnVoid(inst, dex_pc); @@ -124,6 +122,11 @@ void DexCompiler::Compile() { case Instruction::CHECK_CAST: inst = CompileCheckCast(inst, dex_pc); + if (inst->Opcode() == Instruction::NOP) { + // We turned the CHECK_CAST into two NOPs, avoid visiting the second NOP twice since this + // would add 2 quickening info entries. + it.Advance(); + } break; case Instruction::IGET: @@ -190,7 +193,14 @@ void DexCompiler::Compile() { CompileInvokeVirtual(inst, dex_pc, Instruction::INVOKE_VIRTUAL_RANGE_QUICK, true); break; + case Instruction::NOP: + // We need to differentiate between check cast inserted NOP and normal NOP, put an invalid + // index in the map for normal nops. This should be rare in real code. + quickened_info_.push_back(QuickenedInfo(dex_pc, DexFile::kDexNoIndex16)); + break; + default: + DCHECK(!inst->IsQuickened()); // Nothing to do. break; } @@ -348,10 +358,26 @@ CompiledMethod* ArtCompileDEX( } // Create a `CompiledMethod`, with the quickened information in the vmap table. - Leb128EncodingVector<> builder; + if (kIsDebugBuild) { + // Double check that the counts line up with the size of the quicken info. + size_t quicken_count = 0; + for (CodeItemIterator it(*code_item); !it.Done(); it.Advance()) { + if (QuickenInfoTable::NeedsIndexForInstruction(&it.CurrentInstruction())) { + ++quicken_count; + } + } + CHECK_EQ(quicken_count, dex_compiler.GetQuickenedInfo().size()); + } + std::vector<uint8_t> quicken_data; for (QuickenedInfo info : dex_compiler.GetQuickenedInfo()) { - builder.PushBackUnsigned(info.dex_pc); - builder.PushBackUnsigned(info.dex_member_index); + // Dex pc is not serialized, only used for checking the instructions. Since we access the + // array based on the index of the quickened instruction, the indexes must line up perfectly. + // The reader side uses the NeedsIndexForInstruction function too. + const Instruction* inst = Instruction::At(code_item->insns_ + info.dex_pc); + CHECK(QuickenInfoTable::NeedsIndexForInstruction(inst)) << inst->Opcode(); + // Add the index. + quicken_data.push_back(static_cast<uint8_t>(info.dex_member_index >> 0)); + quicken_data.push_back(static_cast<uint8_t>(info.dex_member_index >> 8)); } InstructionSet instruction_set = driver->GetInstructionSet(); if (instruction_set == kThumb2) { @@ -366,7 +392,7 @@ CompiledMethod* ArtCompileDEX( 0, 0, ArrayRef<const uint8_t>(), // method_info - ArrayRef<const uint8_t>(builder.GetData()), // vmap_table + ArrayRef<const uint8_t>(quicken_data), // vmap_table ArrayRef<const uint8_t>(), // cfi data ArrayRef<const LinkerPatch>()); } diff --git a/compiler/image_writer.cc b/compiler/image_writer.cc index 1c73dfab37..4f1fef9f58 100644 --- a/compiler/image_writer.cc +++ b/compiler/image_writer.cc @@ -737,16 +737,82 @@ bool ImageWriter::IsBootClassLoaderNonImageClass(mirror::Class* klass) { return IsBootClassLoaderClass(klass) && !IsInBootImage(klass); } +// This visitor follows the references of an instance, recursively then prune this class +// if a type of any field is pruned. +class ImageWriter::PruneObjectReferenceVisitor { + public: + PruneObjectReferenceVisitor(ImageWriter* image_writer, + bool* early_exit, + std::unordered_set<mirror::Object*>* visited, + bool* result) + : image_writer_(image_writer), early_exit_(early_exit), visited_(visited), result_(result) {} + + ALWAYS_INLINE void VisitRootIfNonNull( + mirror::CompressedReference<mirror::Object>* root ATTRIBUTE_UNUSED) const + REQUIRES_SHARED(Locks::mutator_lock_) { } + + ALWAYS_INLINE void VisitRoot( + mirror::CompressedReference<mirror::Object>* root ATTRIBUTE_UNUSED) const + REQUIRES_SHARED(Locks::mutator_lock_) { } + + ALWAYS_INLINE void operator() (ObjPtr<mirror::Object> obj, + MemberOffset offset, + bool is_static ATTRIBUTE_UNUSED) const + REQUIRES_SHARED(Locks::mutator_lock_) { + mirror::Object* ref = + obj->GetFieldObject<mirror::Object, kVerifyNone, kWithoutReadBarrier>(offset); + if (ref == nullptr || visited_->find(ref) != visited_->end()) { + return; + } + + ObjPtr<mirror::Class> klass = ref->IsClass() ? ref->AsClass() : ref->GetClass(); + if (klass == mirror::Method::StaticClass() || klass == mirror::Constructor::StaticClass()) { + // Prune all classes using reflection because the content they held will not be fixup. + *result_ = true; + } + + // Record the object visited in case of circular reference. + visited_->emplace(ref); + if (ref->IsClass()) { + *result_ = *result_ || + image_writer_->PruneAppImageClassInternal(ref->AsClass(), early_exit_, visited_); + } else { + *result_ = *result_ || + image_writer_->PruneAppImageClassInternal(klass, early_exit_, visited_); + ref->VisitReferences(*this, *this); + } + // Clean up before exit for next call of this function. + visited_->erase(ref); + } + + ALWAYS_INLINE void operator() (ObjPtr<mirror::Class> klass ATTRIBUTE_UNUSED, + ObjPtr<mirror::Reference> ref) const + REQUIRES_SHARED(Locks::mutator_lock_) { + operator()(ref, mirror::Reference::ReferentOffset(), /* is_static */ false); + } + + ALWAYS_INLINE bool GetResult() const { + return result_; + } + + private: + ImageWriter* image_writer_; + bool* early_exit_; + std::unordered_set<mirror::Object*>* visited_; + bool* const result_; +}; + + bool ImageWriter::PruneAppImageClass(ObjPtr<mirror::Class> klass) { bool early_exit = false; - std::unordered_set<mirror::Class*> visited; + std::unordered_set<mirror::Object*> visited; return PruneAppImageClassInternal(klass, &early_exit, &visited); } bool ImageWriter::PruneAppImageClassInternal( ObjPtr<mirror::Class> klass, bool* early_exit, - std::unordered_set<mirror::Class*>* visited) { + std::unordered_set<mirror::Object*>* visited) { DCHECK(early_exit != nullptr); DCHECK(visited != nullptr); DCHECK(compile_app_image_); @@ -807,9 +873,18 @@ bool ImageWriter::PruneAppImageClassInternal( &my_early_exit, visited); } else { - result = result || PruneAppImageClassInternal(ref->GetClass(), + mirror::Class* type = ref->GetClass(); + result = result || PruneAppImageClassInternal(type, &my_early_exit, visited); + if (!result) { + // For non-class case, also go through all the types mentioned by it's fields' + // references recursively to decide whether to keep this class. + bool tmp = false; + PruneObjectReferenceVisitor visitor(this, &my_early_exit, visited, &tmp); + ref->VisitReferences(visitor, visitor); + result = result || tmp; + } } } field_offset = MemberOffset(field_offset.Uint32Value() + diff --git a/compiler/image_writer.h b/compiler/image_writer.h index 5e2db7d8f7..c42523b783 100644 --- a/compiler/image_writer.h +++ b/compiler/image_writer.h @@ -484,7 +484,7 @@ class ImageWriter FINAL { // early_exit is true if we had a cyclic dependency anywhere down the chain. bool PruneAppImageClassInternal(ObjPtr<mirror::Class> klass, bool* early_exit, - std::unordered_set<mirror::Class*>* visited) + std::unordered_set<mirror::Object*>* visited) REQUIRES_SHARED(Locks::mutator_lock_); bool IsMultiImage() const { @@ -621,6 +621,7 @@ class ImageWriter FINAL { class PruneClassLoaderClassesVisitor; class RegisterBootClassPathClassesVisitor; class VisitReferencesVisitor; + class PruneObjectReferenceVisitor; DISALLOW_COPY_AND_ASSIGN(ImageWriter); }; diff --git a/compiler/jni/jni_cfi_test.cc b/compiler/jni/jni_cfi_test.cc index 28b7290bef..b552a6e531 100644 --- a/compiler/jni/jni_cfi_test.cc +++ b/compiler/jni/jni_cfi_test.cc @@ -110,20 +110,35 @@ class JNICFITest : public CFITest { } #ifdef ART_ENABLE_CODEGEN_arm +// Run the tests for ARM only with Baker read barriers, as the +// expected generated code contains a Marking Register refresh +// instruction. +#if defined(USE_READ_BARRIER) && defined(USE_BAKER_READ_BARRIER) TEST_ISA(kThumb2) #endif +#endif + #ifdef ART_ENABLE_CODEGEN_arm64 +// Run the tests for ARM64 only with Baker read barriers, as the +// expected generated code contains a Marking Register refresh +// instruction. +#if defined(USE_READ_BARRIER) && defined(USE_BAKER_READ_BARRIER) TEST_ISA(kArm64) #endif +#endif + #ifdef ART_ENABLE_CODEGEN_x86 TEST_ISA(kX86) #endif + #ifdef ART_ENABLE_CODEGEN_x86_64 TEST_ISA(kX86_64) #endif + #ifdef ART_ENABLE_CODEGEN_mips TEST_ISA(kMips) #endif + #ifdef ART_ENABLE_CODEGEN_mips64 TEST_ISA(kMips64) #endif diff --git a/compiler/jni/jni_cfi_test_expected.inc b/compiler/jni/jni_cfi_test_expected.inc index 2710ae9b53..d641fe4251 100644 --- a/compiler/jni/jni_cfi_test_expected.inc +++ b/compiler/jni/jni_cfi_test_expected.inc @@ -1,7 +1,8 @@ static constexpr uint8_t expected_asm_kThumb2[] = { 0x2D, 0xE9, 0xE0, 0x4D, 0x2D, 0xED, 0x10, 0x8A, 0x89, 0xB0, 0x00, 0x90, 0x21, 0x91, 0x8D, 0xED, 0x22, 0x0A, 0x23, 0x92, 0x24, 0x93, 0x88, 0xB0, - 0x08, 0xB0, 0x09, 0xB0, 0xBD, 0xEC, 0x10, 0x8A, 0xBD, 0xE8, 0xE0, 0x8D, + 0x08, 0xB0, 0x09, 0xB0, 0xBD, 0xEC, 0x10, 0x8A, 0xBD, 0xE8, 0xE0, 0x4D, + 0xD9, 0xF8, 0x34, 0x80, 0x70, 0x47, }; static constexpr uint8_t expected_cfi_kThumb2[] = { 0x44, 0x0E, 0x1C, 0x85, 0x07, 0x86, 0x06, 0x87, 0x05, 0x88, 0x04, 0x8A, @@ -13,10 +14,10 @@ static constexpr uint8_t expected_cfi_kThumb2[] = { 0x4E, 0x0E, 0xA0, 0x01, 0x42, 0x0E, 0x80, 0x01, 0x0A, 0x42, 0x0E, 0x5C, 0x44, 0x0E, 0x1C, 0x06, 0x50, 0x06, 0x51, 0x06, 0x52, 0x06, 0x53, 0x06, 0x54, 0x06, 0x55, 0x06, 0x56, 0x06, 0x57, 0x06, 0x58, 0x06, 0x59, 0x06, - 0x5A, 0x06, 0x5B, 0x06, 0x5C, 0x06, 0x5D, 0x06, 0x5E, 0x06, 0x5F, 0x44, + 0x5A, 0x06, 0x5B, 0x06, 0x5C, 0x06, 0x5D, 0x06, 0x5E, 0x06, 0x5F, 0x4A, 0x0B, 0x0E, 0x80, 0x01, }; -// 0x00000000: push {r5, r6, r7, r8, r10, r11, lr} +// 0x00000000: push {r5,r6,r7,r8,r10,r11,lr} // 0x00000004: .cfi_def_cfa_offset: 28 // 0x00000004: .cfi_offset: r5 at cfa-28 // 0x00000004: .cfi_offset: r6 at cfa-24 @@ -25,7 +26,7 @@ static constexpr uint8_t expected_cfi_kThumb2[] = { // 0x00000004: .cfi_offset: r10 at cfa-12 // 0x00000004: .cfi_offset: r11 at cfa-8 // 0x00000004: .cfi_offset: r14 at cfa-4 -// 0x00000004: vpush.f32 {s16-s31} +// 0x00000004: vpush {s16-s31} // 0x00000008: .cfi_def_cfa_offset: 92 // 0x00000008: .cfi_offset_extended: r80 at cfa-92 // 0x00000008: .cfi_offset_extended: r81 at cfa-88 @@ -43,21 +44,21 @@ static constexpr uint8_t expected_cfi_kThumb2[] = { // 0x00000008: .cfi_offset_extended: r93 at cfa-40 // 0x00000008: .cfi_offset_extended: r94 at cfa-36 // 0x00000008: .cfi_offset_extended: r95 at cfa-32 -// 0x00000008: sub sp, sp, #36 +// 0x00000008: sub sp, #36 // 0x0000000a: .cfi_def_cfa_offset: 128 -// 0x0000000a: str r0, [sp, #0] +// 0x0000000a: str r0, [sp] // 0x0000000c: str r1, [sp, #132] -// 0x0000000e: vstr.f32 s0, [sp, #136] +// 0x0000000e: vstr s0, [sp, #136] // 0x00000012: str r2, [sp, #140] // 0x00000014: str r3, [sp, #144] -// 0x00000016: sub sp, sp, #32 +// 0x00000016: sub sp, #32 // 0x00000018: .cfi_def_cfa_offset: 160 -// 0x00000018: add sp, sp, #32 +// 0x00000018: add sp, #32 // 0x0000001a: .cfi_def_cfa_offset: 128 // 0x0000001a: .cfi_remember_state -// 0x0000001a: add sp, sp, #36 +// 0x0000001a: add sp, #36 // 0x0000001c: .cfi_def_cfa_offset: 92 -// 0x0000001c: vpop.f32 {s16-s31} +// 0x0000001c: vpop {s16-s31} // 0x00000020: .cfi_def_cfa_offset: 28 // 0x00000020: .cfi_restore_extended: r80 // 0x00000020: .cfi_restore_extended: r81 @@ -75,9 +76,11 @@ static constexpr uint8_t expected_cfi_kThumb2[] = { // 0x00000020: .cfi_restore_extended: r93 // 0x00000020: .cfi_restore_extended: r94 // 0x00000020: .cfi_restore_extended: r95 -// 0x00000020: pop {r5, r6, r7, r8, r10, r11, pc} -// 0x00000024: .cfi_restore_state -// 0x00000024: .cfi_def_cfa_offset: 128 +// 0x00000020: pop {r5,r6,r7,r8,r10,r11,lr} +// 0x00000024: ldr r8, [tr, #52] ; is_gc_marking +// 0x00000028: bx lr +// 0x0000002a: .cfi_restore_state +// 0x0000002a: .cfi_def_cfa_offset: 128 static constexpr uint8_t expected_asm_kArm64[] = { 0xFF, 0x03, 0x03, 0xD1, 0xF3, 0x53, 0x06, 0xA9, 0xF5, 0x5B, 0x07, 0xA9, @@ -89,7 +92,8 @@ static constexpr uint8_t expected_asm_kArm64[] = { 0xF3, 0x53, 0x46, 0xA9, 0xF5, 0x5B, 0x47, 0xA9, 0xF7, 0x63, 0x48, 0xA9, 0xF9, 0x6B, 0x49, 0xA9, 0xFB, 0x73, 0x4A, 0xA9, 0xFD, 0x7B, 0x4B, 0xA9, 0xE8, 0x27, 0x42, 0x6D, 0xEA, 0x2F, 0x43, 0x6D, 0xEC, 0x37, 0x44, 0x6D, - 0xEE, 0x3F, 0x45, 0x6D, 0xFF, 0x03, 0x03, 0x91, 0xC0, 0x03, 0x5F, 0xD6, + 0xEE, 0x3F, 0x45, 0x6D, 0x74, 0x36, 0x40, 0xB9, 0xFF, 0x03, 0x03, 0x91, + 0xC0, 0x03, 0x5F, 0xD6, }; static constexpr uint8_t expected_cfi_kArm64[] = { 0x44, 0x0E, 0xC0, 0x01, 0x44, 0x93, 0x18, 0x94, 0x16, 0x44, 0x95, 0x14, @@ -101,7 +105,7 @@ static constexpr uint8_t expected_cfi_kArm64[] = { 0xD3, 0xD4, 0x44, 0xD5, 0xD6, 0x44, 0xD7, 0xD8, 0x44, 0xD9, 0xDA, 0x44, 0xDB, 0xDC, 0x44, 0xDD, 0xDE, 0x44, 0x06, 0x48, 0x06, 0x49, 0x44, 0x06, 0x4A, 0x06, 0x4B, 0x44, 0x06, 0x4C, 0x06, 0x4D, 0x44, 0x06, 0x4E, 0x06, - 0x4F, 0x44, 0x0E, 0x00, 0x44, 0x0B, 0x0E, 0xC0, 0x01, + 0x4F, 0x48, 0x0E, 0x00, 0x44, 0x0B, 0x0E, 0xC0, 0x01, }; // 0x00000000: sub sp, sp, #0xc0 (192) // 0x00000004: .cfi_def_cfa_offset: 192 @@ -175,11 +179,12 @@ static constexpr uint8_t expected_cfi_kArm64[] = { // 0x0000006c: ldp d14, d15, [sp, #80] // 0x00000070: .cfi_restore_extended: r78 // 0x00000070: .cfi_restore_extended: r79 -// 0x00000070: add sp, sp, #0xc0 (192) -// 0x00000074: .cfi_def_cfa_offset: 0 -// 0x00000074: ret -// 0x00000078: .cfi_restore_state -// 0x00000078: .cfi_def_cfa_offset: 192 +// 0x00000070: ldr w20, [tr, #52] ; is_gc_marking +// 0x00000074: add sp, sp, #0xc0 (192) +// 0x00000078: .cfi_def_cfa_offset: 0 +// 0x00000078: ret +// 0x0000007c: .cfi_restore_state +// 0x0000007c: .cfi_def_cfa_offset: 192 static constexpr uint8_t expected_asm_kX86[] = { 0x57, 0x56, 0x55, 0x83, 0xC4, 0xE4, 0x50, 0x89, 0x4C, 0x24, 0x34, 0xF3, diff --git a/compiler/jni/jni_compiler_test.cc b/compiler/jni/jni_compiler_test.cc index b34d9385c8..6ce7d75da6 100644 --- a/compiler/jni/jni_compiler_test.cc +++ b/compiler/jni/jni_compiler_test.cc @@ -49,6 +49,9 @@ extern "C" JNIEXPORT jint JNICALL Java_MyClassNatives_sbar(JNIEnv*, jclass, jint return count + 1; } +// TODO: In the Baker read barrier configuration, add checks to ensure +// the Marking Register's value is correct. + namespace art { enum class JniKind { diff --git a/compiler/jni/quick/arm64/calling_convention_arm64.cc b/compiler/jni/quick/arm64/calling_convention_arm64.cc index 33f4d77fc2..e086455620 100644 --- a/compiler/jni/quick/arm64/calling_convention_arm64.cc +++ b/compiler/jni/quick/arm64/calling_convention_arm64.cc @@ -108,11 +108,25 @@ static constexpr uint32_t kFpCalleeSpillMask = CalculateFpCalleeSpillMask(); // Calling convention ManagedRegister Arm64ManagedRuntimeCallingConvention::InterproceduralScratchRegister() { - return Arm64ManagedRegister::FromXRegister(X20); // saved on entry restored on exit + // X20 is safe to use as a scratch register: + // - with Baker read barriers, it is reserved as Marking Register, + // and thus does not actually need to be saved/restored; it is + // refreshed on exit (see Arm64JNIMacroAssembler::RemoveFrame); + // - in other cases, it is saved on entry (in + // Arm64JNIMacroAssembler::BuildFrame) and restored on exit (in + // Arm64JNIMacroAssembler::RemoveFrame). + return Arm64ManagedRegister::FromXRegister(X20); } ManagedRegister Arm64JniCallingConvention::InterproceduralScratchRegister() { - return Arm64ManagedRegister::FromXRegister(X20); // saved on entry restored on exit + // X20 is safe to use as a scratch register: + // - with Baker read barriers, it is reserved as Marking Register, + // and thus does not actually need to be saved/restored; it is + // refreshed on exit (see Arm64JNIMacroAssembler::RemoveFrame); + // - in other cases, it is saved on entry (in + // Arm64JNIMacroAssembler::BuildFrame) and restored on exit (in + // Arm64JNIMacroAssembler::RemoveFrame). + return Arm64ManagedRegister::FromXRegister(X20); } static ManagedRegister ReturnRegisterForShorty(const char* shorty) { diff --git a/compiler/linker/arm/relative_patcher_thumb2.cc b/compiler/linker/arm/relative_patcher_thumb2.cc index aa5a9457b2..18d6b9ad03 100644 --- a/compiler/linker/arm/relative_patcher_thumb2.cc +++ b/compiler/linker/arm/relative_patcher_thumb2.cc @@ -199,6 +199,24 @@ static void EmitGrayCheckAndFastPath(arm::ArmVIXLAssembler& assembler, // Note: The fake dependency is unnecessary for the slow path. } +// Load the read barrier introspection entrypoint in register `entrypoint` +static void LoadReadBarrierMarkIntrospectionEntrypoint(arm::ArmVIXLAssembler& assembler, + vixl::aarch32::Register entrypoint) { + using vixl::aarch32::MemOperand; + using vixl::aarch32::ip; + // Thread Register. + const vixl::aarch32::Register tr = vixl::aarch32::r9; + + // The register where the read barrier introspection entrypoint is loaded + // is fixed: `Thumb2RelativePatcher::kBakerCcEntrypointRegister` (R4). + DCHECK_EQ(entrypoint.GetCode(), Thumb2RelativePatcher::kBakerCcEntrypointRegister); + // entrypoint = Thread::Current()->pReadBarrierMarkReg12, i.e. pReadBarrierMarkIntrospection. + DCHECK_EQ(ip.GetCode(), 12u); + const int32_t entry_point_offset = + Thread::ReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ip.GetCode()); + __ Ldr(entrypoint, MemOperand(tr, entry_point_offset)); +} + void Thumb2RelativePatcher::CompileBakerReadBarrierThunk(arm::ArmVIXLAssembler& assembler, uint32_t encoded_data) { using namespace vixl::aarch32; // NOLINT(build/namespaces) @@ -233,6 +251,7 @@ void Thumb2RelativePatcher::CompileBakerReadBarrierThunk(arm::ArmVIXLAssembler& const int32_t ldr_offset = /* Thumb state adjustment (LR contains Thumb state). */ -1 + raw_ldr_offset; Register ep_reg(kBakerCcEntrypointRegister); + LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ep_reg); if (width == BakerReadBarrierWidth::kWide) { MemOperand ldr_half_address(lr, ldr_offset + 2); __ Ldrh(ip, ldr_half_address); // Load the LDR immediate half-word with "Rt | imm12". @@ -278,8 +297,10 @@ void Thumb2RelativePatcher::CompileBakerReadBarrierThunk(arm::ArmVIXLAssembler& MemOperand ldr_address(lr, ldr_offset + 2); __ Ldrb(ip, ldr_address); // Load the LDR (register) byte with "00 | imm2 | Rm", // i.e. Rm+32 because the scale in imm2 is 2. - Register ep_reg(kBakerCcEntrypointRegister); // Insert ip to the entrypoint address to create - __ Bfi(ep_reg, ip, 3, 6); // a switch case target based on the index register. + Register ep_reg(kBakerCcEntrypointRegister); + LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ep_reg); + __ Bfi(ep_reg, ip, 3, 6); // Insert ip to the entrypoint address to create + // a switch case target based on the index register. __ Mov(ip, base_reg); // Move the base register to ip0. __ Bx(ep_reg); // Jump to the entrypoint's array switch case. break; @@ -309,9 +330,10 @@ void Thumb2RelativePatcher::CompileBakerReadBarrierThunk(arm::ArmVIXLAssembler& " the highest bits and the 'forwarding address' state to have all bits set"); __ Cmp(ip, Operand(0xc0000000)); __ B(hs, &forwarding_address); + Register ep_reg(kBakerCcEntrypointRegister); + LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ep_reg); // Adjust the art_quick_read_barrier_mark_introspection address in kBakerCcEntrypointRegister // to art_quick_read_barrier_mark_introspection_gc_roots. - Register ep_reg(kBakerCcEntrypointRegister); int32_t entrypoint_offset = (width == BakerReadBarrierWidth::kWide) ? BAKER_MARK_INTROSPECTION_GC_ROOT_LDR_WIDE_ENTRYPOINT_OFFSET : BAKER_MARK_INTROSPECTION_GC_ROOT_LDR_NARROW_ENTRYPOINT_OFFSET; diff --git a/compiler/linker/arm64/relative_patcher_arm64.cc b/compiler/linker/arm64/relative_patcher_arm64.cc index bc21607c5b..38c732b8ba 100644 --- a/compiler/linker/arm64/relative_patcher_arm64.cc +++ b/compiler/linker/arm64/relative_patcher_arm64.cc @@ -381,6 +381,21 @@ static void EmitGrayCheckAndFastPath(arm64::Arm64Assembler& assembler, // Note: The fake dependency is unnecessary for the slow path. } +// Load the read barrier introspection entrypoint in register `entrypoint`. +static void LoadReadBarrierMarkIntrospectionEntrypoint(arm64::Arm64Assembler& assembler, + vixl::aarch64::Register entrypoint) { + using vixl::aarch64::MemOperand; + using vixl::aarch64::ip0; + // Thread Register. + const vixl::aarch64::Register tr = vixl::aarch64::x19; + + // entrypoint = Thread::Current()->pReadBarrierMarkReg16, i.e. pReadBarrierMarkIntrospection. + DCHECK_EQ(ip0.GetCode(), 16u); + const int32_t entry_point_offset = + Thread::ReadBarrierMarkEntryPointsOffset<kArm64PointerSize>(ip0.GetCode()); + __ Ldr(entrypoint, MemOperand(tr, entry_point_offset)); +} + void Arm64RelativePatcher::CompileBakerReadBarrierThunk(arm64::Arm64Assembler& assembler, uint32_t encoded_data) { using namespace vixl::aarch64; // NOLINT(build/namespaces) @@ -412,6 +427,7 @@ void Arm64RelativePatcher::CompileBakerReadBarrierThunk(arm64::Arm64Assembler& a __ Bind(&slow_path); MemOperand ldr_address(lr, BAKER_MARK_INTROSPECTION_FIELD_LDR_OFFSET); __ Ldr(ip0.W(), ldr_address); // Load the LDR (immediate) unsigned offset. + LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ip1); __ Ubfx(ip0.W(), ip0.W(), 10, 12); // Extract the offset. __ Ldr(ip0.W(), MemOperand(base_reg, ip0, LSL, 2)); // Load the reference. // Do not unpoison. With heap poisoning enabled, the entrypoint expects a poisoned reference. @@ -441,6 +457,7 @@ void Arm64RelativePatcher::CompileBakerReadBarrierThunk(arm64::Arm64Assembler& a __ Bind(&slow_path); MemOperand ldr_address(lr, BAKER_MARK_INTROSPECTION_ARRAY_LDR_OFFSET); __ Ldr(ip0.W(), ldr_address); // Load the LDR (register) unsigned offset. + LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ip1); __ Ubfx(ip0, ip0, 16, 6); // Extract the index register, plus 32 (bit 21 is set). __ Bfi(ip1, ip0, 3, 6); // Insert ip0 to the entrypoint address to create // a switch case target based on the index register. @@ -469,6 +486,7 @@ void Arm64RelativePatcher::CompileBakerReadBarrierThunk(arm64::Arm64Assembler& a __ Bind(¬_marked); __ Tst(ip0.W(), Operand(ip0.W(), LSL, 1)); __ B(&forwarding_address, mi); + LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ip1); // Adjust the art_quick_read_barrier_mark_introspection address in IP1 to // art_quick_read_barrier_mark_introspection_gc_roots. __ Add(ip1, ip1, Operand(BAKER_MARK_INTROSPECTION_GC_ROOT_ENTRYPOINT_OFFSET)); diff --git a/compiler/oat_writer.cc b/compiler/oat_writer.cc index f7465c0d5f..7cb3166cde 100644 --- a/compiler/oat_writer.cc +++ b/compiler/oat_writer.cc @@ -473,8 +473,8 @@ bool OatWriter::AddRawDexFileSource(const ArrayRef<const uint8_t>& data, return true; } -dchecked_vector<const char*> OatWriter::GetSourceLocations() const { - dchecked_vector<const char*> locations; +dchecked_vector<std::string> OatWriter::GetSourceLocations() const { + dchecked_vector<std::string> locations; locations.reserve(oat_dex_files_.size()); for (const OatDexFile& oat_dex_file : oat_dex_files_) { locations.push_back(oat_dex_file.GetLocation()); diff --git a/compiler/oat_writer.h b/compiler/oat_writer.h index 9217701bc5..024a3e80ca 100644 --- a/compiler/oat_writer.h +++ b/compiler/oat_writer.h @@ -153,7 +153,7 @@ class OatWriter { const VdexFile& vdex_file, const char* location, CreateTypeLookupTable create_type_lookup_table = CreateTypeLookupTable::kDefault); - dchecked_vector<const char*> GetSourceLocations() const; + dchecked_vector<std::string> GetSourceLocations() const; // Write raw dex files to the vdex file, mmap the file and open the dex files from it. // Supporting data structures are written into the .rodata section of the oat file. diff --git a/compiler/optimizing/block_builder.cc b/compiler/optimizing/block_builder.cc index 1e75f10ebe..fe7ecd1ae1 100644 --- a/compiler/optimizing/block_builder.cc +++ b/compiler/optimizing/block_builder.cc @@ -17,6 +17,7 @@ #include "block_builder.h" #include "bytecode_utils.h" +#include "quicken_info.h" namespace art { @@ -121,13 +122,18 @@ void HBasicBlockBuilder::ConnectBasicBlocks() { HBasicBlock* block = graph_->GetEntryBlock(); graph_->AddBlock(block); + size_t quicken_index = 0; bool is_throwing_block = false; + // Calculate the qucikening index here instead of CreateBranchTargets since it's easier to + // calculate in dex_pc order. for (CodeItemIterator it(code_item_); !it.Done(); it.Advance()) { uint32_t dex_pc = it.CurrentDexPc(); // Check if this dex_pc address starts a new basic block. HBasicBlock* next_block = GetBlockAt(dex_pc); if (next_block != nullptr) { + // We only need quicken index entries for basic block boundaries. + quicken_index_for_dex_pc_.Put(dex_pc, quicken_index); if (block != nullptr) { // Last instruction did not end its basic block but a new one starts here. // It must have been a block falling through into the next one. @@ -137,6 +143,10 @@ void HBasicBlockBuilder::ConnectBasicBlocks() { is_throwing_block = false; graph_->AddBlock(block); } + // Make sure to increment this before the continues. + if (QuickenInfoTable::NeedsIndexForInstruction(&it.CurrentInstruction())) { + ++quicken_index; + } if (block == nullptr) { // Ignore dead code. @@ -371,4 +381,8 @@ bool HBasicBlockBuilder::Build() { return true; } +size_t HBasicBlockBuilder::GetQuickenIndex(uint32_t dex_pc) const { + return quicken_index_for_dex_pc_.Get(dex_pc); +} + } // namespace art diff --git a/compiler/optimizing/block_builder.h b/compiler/optimizing/block_builder.h index 1be0b4ce98..6adce815f4 100644 --- a/compiler/optimizing/block_builder.h +++ b/compiler/optimizing/block_builder.h @@ -37,7 +37,8 @@ class HBasicBlockBuilder : public ValueObject { nullptr, arena_->Adapter(kArenaAllocGraphBuilder)), throwing_blocks_(kDefaultNumberOfThrowingBlocks, arena_->Adapter(kArenaAllocGraphBuilder)), - number_of_branches_(0u) {} + number_of_branches_(0u), + quicken_index_for_dex_pc_(std::less<uint32_t>(), arena_->Adapter()) {} // Creates basic blocks in `graph_` at branch target dex_pc positions of the // `code_item_`. Blocks are connected but left unpopulated with instructions. @@ -48,6 +49,8 @@ class HBasicBlockBuilder : public ValueObject { size_t GetNumberOfBranches() const { return number_of_branches_; } HBasicBlock* GetBlockAt(uint32_t dex_pc) const { return branch_targets_[dex_pc]; } + size_t GetQuickenIndex(uint32_t dex_pc) const; + private: // Creates a basic block starting at given `dex_pc`. HBasicBlock* MaybeCreateBlockAt(uint32_t dex_pc); @@ -78,6 +81,9 @@ class HBasicBlockBuilder : public ValueObject { ArenaVector<HBasicBlock*> throwing_blocks_; size_t number_of_branches_; + // A table to quickly find the quicken index for the first instruction of a basic block. + ArenaSafeMap<uint32_t, uint32_t> quicken_index_for_dex_pc_; + static constexpr size_t kDefaultNumberOfThrowingBlocks = 2u; DISALLOW_COPY_AND_ASSIGN(HBasicBlockBuilder); diff --git a/compiler/optimizing/code_generator.cc b/compiler/optimizing/code_generator.cc index 93234f9630..2872cf7458 100644 --- a/compiler/optimizing/code_generator.cc +++ b/compiler/optimizing/code_generator.cc @@ -17,7 +17,6 @@ #include "code_generator.h" #ifdef ART_ENABLE_CODEGEN_arm -#include "code_generator_arm.h" #include "code_generator_arm_vixl.h" #endif @@ -627,19 +626,11 @@ std::unique_ptr<CodeGenerator> CodeGenerator::Create(HGraph* graph, #ifdef ART_ENABLE_CODEGEN_arm case kArm: case kThumb2: { - if (kArmUseVIXL32) { - return std::unique_ptr<CodeGenerator>( - new (arena) arm::CodeGeneratorARMVIXL(graph, - *isa_features.AsArmInstructionSetFeatures(), - compiler_options, - stats)); - } else { - return std::unique_ptr<CodeGenerator>( - new (arena) arm::CodeGeneratorARM(graph, - *isa_features.AsArmInstructionSetFeatures(), - compiler_options, - stats)); - } + return std::unique_ptr<CodeGenerator>( + new (arena) arm::CodeGeneratorARMVIXL(graph, + *isa_features.AsArmInstructionSetFeatures(), + compiler_options, + stats)); } #endif #ifdef ART_ENABLE_CODEGEN_arm64 diff --git a/compiler/optimizing/code_generator.h b/compiler/optimizing/code_generator.h index 7bf43f7971..73202b4fd1 100644 --- a/compiler/optimizing/code_generator.h +++ b/compiler/optimizing/code_generator.h @@ -404,17 +404,6 @@ class CodeGenerator : public DeletableArenaObject<kArenaAllocCodeGenerator> { // accessing the String's `value` field in String intrinsics. static uint32_t GetArrayDataOffset(HArrayGet* array_get); - // Return the entry point offset for ReadBarrierMarkRegX, where X is `reg`. - template <PointerSize pointer_size> - static int32_t GetReadBarrierMarkEntryPointsOffset(size_t reg) { - // The entry point list defines 30 ReadBarrierMarkRegX entry points. - DCHECK_LT(reg, 30u); - // The ReadBarrierMarkRegX entry points are ordered by increasing - // register number in Thread::tls_Ptr_.quick_entrypoints. - return QUICK_ENTRYPOINT_OFFSET(pointer_size, pReadBarrierMarkReg00).Int32Value() - + static_cast<size_t>(pointer_size) * reg; - } - void EmitParallelMoves(Location from1, Location to1, Primitive::Type type1, diff --git a/compiler/optimizing/code_generator_arm.cc b/compiler/optimizing/code_generator_arm.cc deleted file mode 100644 index 6b9f232e8f..0000000000 --- a/compiler/optimizing/code_generator_arm.cc +++ /dev/null @@ -1,9400 +0,0 @@ -/* - * Copyright (C) 2014 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "code_generator_arm.h" - -#include "arch/arm/asm_support_arm.h" -#include "arch/arm/instruction_set_features_arm.h" -#include "art_method.h" -#include "base/bit_utils.h" -#include "base/bit_utils_iterator.h" -#include "code_generator_utils.h" -#include "common_arm.h" -#include "compiled_method.h" -#include "entrypoints/quick/quick_entrypoints.h" -#include "gc/accounting/card_table.h" -#include "intrinsics.h" -#include "intrinsics_arm.h" -#include "linker/arm/relative_patcher_thumb2.h" -#include "mirror/array-inl.h" -#include "mirror/class-inl.h" -#include "thread.h" -#include "utils/arm/assembler_arm.h" -#include "utils/arm/managed_register_arm.h" -#include "utils/assembler.h" -#include "utils/stack_checks.h" - -namespace art { - -template<class MirrorType> -class GcRoot; - -namespace arm { - -static bool ExpectedPairLayout(Location location) { - // We expected this for both core and fpu register pairs. - return ((location.low() & 1) == 0) && (location.low() + 1 == location.high()); -} - -static constexpr Register kMethodRegisterArgument = R0; - -static constexpr Register kCoreAlwaysSpillRegister = R5; -static constexpr Register kCoreCalleeSaves[] = - { R5, R6, R7, R8, R10, R11, LR }; -static constexpr SRegister kFpuCalleeSaves[] = - { S16, S17, S18, S19, S20, S21, S22, S23, S24, S25, S26, S27, S28, S29, S30, S31 }; - -// D31 cannot be split into two S registers, and the register allocator only works on -// S registers. Therefore there is no need to block it. -static constexpr DRegister DTMP = D31; - -static constexpr uint32_t kPackedSwitchCompareJumpThreshold = 7; - -// Reference load (except object array loads) is using LDR Rt, [Rn, #offset] which can handle -// offset < 4KiB. For offsets >= 4KiB, the load shall be emitted as two or more instructions. -// For the Baker read barrier implementation using link-generated thunks we need to split -// the offset explicitly. -constexpr uint32_t kReferenceLoadMinFarOffset = 4 * KB; - -// Flags controlling the use of link-time generated thunks for Baker read barriers. -constexpr bool kBakerReadBarrierLinkTimeThunksEnableForFields = true; -constexpr bool kBakerReadBarrierLinkTimeThunksEnableForArrays = true; -constexpr bool kBakerReadBarrierLinkTimeThunksEnableForGcRoots = true; - -// The reserved entrypoint register for link-time generated thunks. -const Register kBakerCcEntrypointRegister = R4; - -// NOLINT on __ macro to suppress wrong warning/fix (misc-macro-parentheses) from clang-tidy. -#define __ down_cast<ArmAssembler*>(codegen->GetAssembler())-> // NOLINT -#define QUICK_ENTRY_POINT(x) QUICK_ENTRYPOINT_OFFSET(kArmPointerSize, x).Int32Value() - -static inline void CheckLastTempIsBakerCcEntrypointRegister(HInstruction* instruction) { - DCHECK_EQ(static_cast<uint32_t>(kBakerCcEntrypointRegister), - linker::Thumb2RelativePatcher::kBakerCcEntrypointRegister); - DCHECK_NE(instruction->GetLocations()->GetTempCount(), 0u); - DCHECK_EQ(kBakerCcEntrypointRegister, - instruction->GetLocations()->GetTemp( - instruction->GetLocations()->GetTempCount() - 1u).AsRegister<Register>()); -} - -static inline void EmitPlaceholderBne(CodeGeneratorARM* codegen, Label* bne_label) { - ScopedForce32Bit force_32bit(down_cast<Thumb2Assembler*>(codegen->GetAssembler())); - __ BindTrackedLabel(bne_label); - Label placeholder_label; - __ b(&placeholder_label, NE); // Placeholder, patched at link-time. - __ Bind(&placeholder_label); -} - -static inline bool CanEmitNarrowLdr(Register rt, Register rn, uint32_t offset) { - return ArmAssembler::IsLowRegister(rt) && ArmAssembler::IsLowRegister(rn) && offset < 32u; -} - -static constexpr int kRegListThreshold = 4; - -// SaveLiveRegisters and RestoreLiveRegisters from SlowPathCodeARM operate on sets of S registers, -// for each live D registers they treat two corresponding S registers as live ones. -// -// Two following functions (SaveContiguousSRegisterList, RestoreContiguousSRegisterList) build -// from a list of contiguous S registers a list of contiguous D registers (processing first/last -// S registers corner cases) and save/restore this new list treating them as D registers. -// - decreasing code size -// - avoiding hazards on Cortex-A57, when a pair of S registers for an actual live D register is -// restored and then used in regular non SlowPath code as D register. -// -// For the following example (v means the S register is live): -// D names: | D0 | D1 | D2 | D4 | ... -// S names: | S0 | S1 | S2 | S3 | S4 | S5 | S6 | S7 | ... -// Live? | | v | v | v | v | v | v | | ... -// -// S1 and S6 will be saved/restored independently; D registers list (D1, D2) will be processed -// as D registers. -static size_t SaveContiguousSRegisterList(size_t first, - size_t last, - CodeGenerator* codegen, - size_t stack_offset) { - DCHECK_LE(first, last); - if ((first == last) && (first == 0)) { - stack_offset += codegen->SaveFloatingPointRegister(stack_offset, first); - return stack_offset; - } - if (first % 2 == 1) { - stack_offset += codegen->SaveFloatingPointRegister(stack_offset, first++); - } - - bool save_last = false; - if (last % 2 == 0) { - save_last = true; - --last; - } - - if (first < last) { - DRegister d_reg = static_cast<DRegister>(first / 2); - DCHECK_EQ((last - first + 1) % 2, 0u); - size_t number_of_d_regs = (last - first + 1) / 2; - - if (number_of_d_regs == 1) { - __ StoreDToOffset(d_reg, SP, stack_offset); - } else if (number_of_d_regs > 1) { - __ add(IP, SP, ShifterOperand(stack_offset)); - __ vstmiad(IP, d_reg, number_of_d_regs); - } - stack_offset += number_of_d_regs * kArmWordSize * 2; - } - - if (save_last) { - stack_offset += codegen->SaveFloatingPointRegister(stack_offset, last + 1); - } - - return stack_offset; -} - -static size_t RestoreContiguousSRegisterList(size_t first, - size_t last, - CodeGenerator* codegen, - size_t stack_offset) { - DCHECK_LE(first, last); - if ((first == last) && (first == 0)) { - stack_offset += codegen->RestoreFloatingPointRegister(stack_offset, first); - return stack_offset; - } - if (first % 2 == 1) { - stack_offset += codegen->RestoreFloatingPointRegister(stack_offset, first++); - } - - bool restore_last = false; - if (last % 2 == 0) { - restore_last = true; - --last; - } - - if (first < last) { - DRegister d_reg = static_cast<DRegister>(first / 2); - DCHECK_EQ((last - first + 1) % 2, 0u); - size_t number_of_d_regs = (last - first + 1) / 2; - if (number_of_d_regs == 1) { - __ LoadDFromOffset(d_reg, SP, stack_offset); - } else if (number_of_d_regs > 1) { - __ add(IP, SP, ShifterOperand(stack_offset)); - __ vldmiad(IP, d_reg, number_of_d_regs); - } - stack_offset += number_of_d_regs * kArmWordSize * 2; - } - - if (restore_last) { - stack_offset += codegen->RestoreFloatingPointRegister(stack_offset, last + 1); - } - - return stack_offset; -} - -void SlowPathCodeARM::SaveLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) { - size_t stack_offset = codegen->GetFirstRegisterSlotInSlowPath(); - size_t orig_offset = stack_offset; - - const uint32_t core_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ true); - for (uint32_t i : LowToHighBits(core_spills)) { - // If the register holds an object, update the stack mask. - if (locations->RegisterContainsObject(i)) { - locations->SetStackBit(stack_offset / kVRegSize); - } - DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); - DCHECK_LT(i, kMaximumNumberOfExpectedRegisters); - saved_core_stack_offsets_[i] = stack_offset; - stack_offset += kArmWordSize; - } - - int reg_num = POPCOUNT(core_spills); - if (reg_num != 0) { - if (reg_num > kRegListThreshold) { - __ StoreList(RegList(core_spills), orig_offset); - } else { - stack_offset = orig_offset; - for (uint32_t i : LowToHighBits(core_spills)) { - stack_offset += codegen->SaveCoreRegister(stack_offset, i); - } - } - } - - uint32_t fp_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ false); - orig_offset = stack_offset; - for (uint32_t i : LowToHighBits(fp_spills)) { - DCHECK_LT(i, kMaximumNumberOfExpectedRegisters); - saved_fpu_stack_offsets_[i] = stack_offset; - stack_offset += kArmWordSize; - } - - stack_offset = orig_offset; - while (fp_spills != 0u) { - uint32_t begin = CTZ(fp_spills); - uint32_t tmp = fp_spills + (1u << begin); - fp_spills &= tmp; // Clear the contiguous range of 1s. - uint32_t end = (tmp == 0u) ? 32u : CTZ(tmp); // CTZ(0) is undefined. - stack_offset = SaveContiguousSRegisterList(begin, end - 1, codegen, stack_offset); - } - DCHECK_LE(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); -} - -void SlowPathCodeARM::RestoreLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) { - size_t stack_offset = codegen->GetFirstRegisterSlotInSlowPath(); - size_t orig_offset = stack_offset; - - const uint32_t core_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ true); - for (uint32_t i : LowToHighBits(core_spills)) { - DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); - DCHECK_LT(i, kMaximumNumberOfExpectedRegisters); - stack_offset += kArmWordSize; - } - - int reg_num = POPCOUNT(core_spills); - if (reg_num != 0) { - if (reg_num > kRegListThreshold) { - __ LoadList(RegList(core_spills), orig_offset); - } else { - stack_offset = orig_offset; - for (uint32_t i : LowToHighBits(core_spills)) { - stack_offset += codegen->RestoreCoreRegister(stack_offset, i); - } - } - } - - uint32_t fp_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ false); - while (fp_spills != 0u) { - uint32_t begin = CTZ(fp_spills); - uint32_t tmp = fp_spills + (1u << begin); - fp_spills &= tmp; // Clear the contiguous range of 1s. - uint32_t end = (tmp == 0u) ? 32u : CTZ(tmp); // CTZ(0) is undefined. - stack_offset = RestoreContiguousSRegisterList(begin, end - 1, codegen, stack_offset); - } - DCHECK_LE(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); -} - -class NullCheckSlowPathARM : public SlowPathCodeARM { - public: - explicit NullCheckSlowPathARM(HNullCheck* instruction) : SlowPathCodeARM(instruction) {} - - void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { - CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); - __ Bind(GetEntryLabel()); - if (instruction_->CanThrowIntoCatchBlock()) { - // Live registers will be restored in the catch block if caught. - SaveLiveRegisters(codegen, instruction_->GetLocations()); - } - arm_codegen->InvokeRuntime(kQuickThrowNullPointer, - instruction_, - instruction_->GetDexPc(), - this); - CheckEntrypointTypes<kQuickThrowNullPointer, void, void>(); - } - - bool IsFatal() const OVERRIDE { return true; } - - const char* GetDescription() const OVERRIDE { return "NullCheckSlowPathARM"; } - - private: - DISALLOW_COPY_AND_ASSIGN(NullCheckSlowPathARM); -}; - -class DivZeroCheckSlowPathARM : public SlowPathCodeARM { - public: - explicit DivZeroCheckSlowPathARM(HDivZeroCheck* instruction) : SlowPathCodeARM(instruction) {} - - void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { - CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); - __ Bind(GetEntryLabel()); - arm_codegen->InvokeRuntime(kQuickThrowDivZero, instruction_, instruction_->GetDexPc(), this); - CheckEntrypointTypes<kQuickThrowDivZero, void, void>(); - } - - bool IsFatal() const OVERRIDE { return true; } - - const char* GetDescription() const OVERRIDE { return "DivZeroCheckSlowPathARM"; } - - private: - DISALLOW_COPY_AND_ASSIGN(DivZeroCheckSlowPathARM); -}; - -class SuspendCheckSlowPathARM : public SlowPathCodeARM { - public: - SuspendCheckSlowPathARM(HSuspendCheck* instruction, HBasicBlock* successor) - : SlowPathCodeARM(instruction), successor_(successor) {} - - void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { - CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); - __ Bind(GetEntryLabel()); - arm_codegen->InvokeRuntime(kQuickTestSuspend, instruction_, instruction_->GetDexPc(), this); - CheckEntrypointTypes<kQuickTestSuspend, void, void>(); - if (successor_ == nullptr) { - __ b(GetReturnLabel()); - } else { - __ b(arm_codegen->GetLabelOf(successor_)); - } - } - - Label* GetReturnLabel() { - DCHECK(successor_ == nullptr); - return &return_label_; - } - - HBasicBlock* GetSuccessor() const { - return successor_; - } - - const char* GetDescription() const OVERRIDE { return "SuspendCheckSlowPathARM"; } - - private: - // If not null, the block to branch to after the suspend check. - HBasicBlock* const successor_; - - // If `successor_` is null, the label to branch to after the suspend check. - Label return_label_; - - DISALLOW_COPY_AND_ASSIGN(SuspendCheckSlowPathARM); -}; - -class BoundsCheckSlowPathARM : public SlowPathCodeARM { - public: - explicit BoundsCheckSlowPathARM(HBoundsCheck* instruction) - : SlowPathCodeARM(instruction) {} - - void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { - CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); - LocationSummary* locations = instruction_->GetLocations(); - - __ Bind(GetEntryLabel()); - if (instruction_->CanThrowIntoCatchBlock()) { - // Live registers will be restored in the catch block if caught. - SaveLiveRegisters(codegen, instruction_->GetLocations()); - } - // We're moving two locations to locations that could overlap, so we need a parallel - // move resolver. - InvokeRuntimeCallingConvention calling_convention; - codegen->EmitParallelMoves( - locations->InAt(0), - Location::RegisterLocation(calling_convention.GetRegisterAt(0)), - Primitive::kPrimInt, - locations->InAt(1), - Location::RegisterLocation(calling_convention.GetRegisterAt(1)), - Primitive::kPrimInt); - QuickEntrypointEnum entrypoint = instruction_->AsBoundsCheck()->IsStringCharAt() - ? kQuickThrowStringBounds - : kQuickThrowArrayBounds; - arm_codegen->InvokeRuntime(entrypoint, instruction_, instruction_->GetDexPc(), this); - CheckEntrypointTypes<kQuickThrowStringBounds, void, int32_t, int32_t>(); - CheckEntrypointTypes<kQuickThrowArrayBounds, void, int32_t, int32_t>(); - } - - bool IsFatal() const OVERRIDE { return true; } - - const char* GetDescription() const OVERRIDE { return "BoundsCheckSlowPathARM"; } - - private: - DISALLOW_COPY_AND_ASSIGN(BoundsCheckSlowPathARM); -}; - -class LoadClassSlowPathARM : public SlowPathCodeARM { - public: - LoadClassSlowPathARM(HLoadClass* cls, HInstruction* at, uint32_t dex_pc, bool do_clinit) - : SlowPathCodeARM(at), cls_(cls), dex_pc_(dex_pc), do_clinit_(do_clinit) { - DCHECK(at->IsLoadClass() || at->IsClinitCheck()); - } - - void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { - LocationSummary* locations = instruction_->GetLocations(); - Location out = locations->Out(); - constexpr bool call_saves_everything_except_r0 = (!kUseReadBarrier || kUseBakerReadBarrier); - - CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); - __ Bind(GetEntryLabel()); - SaveLiveRegisters(codegen, locations); - - InvokeRuntimeCallingConvention calling_convention; - // For HLoadClass/kBssEntry/kSaveEverything, make sure we preserve the address of the entry. - DCHECK_EQ(instruction_->IsLoadClass(), cls_ == instruction_); - bool is_load_class_bss_entry = - (cls_ == instruction_) && (cls_->GetLoadKind() == HLoadClass::LoadKind::kBssEntry); - Register entry_address = kNoRegister; - if (is_load_class_bss_entry && call_saves_everything_except_r0) { - Register temp = locations->GetTemp(0).AsRegister<Register>(); - // In the unlucky case that the `temp` is R0, we preserve the address in `out` across - // the kSaveEverything call. - bool temp_is_r0 = (temp == calling_convention.GetRegisterAt(0)); - entry_address = temp_is_r0 ? out.AsRegister<Register>() : temp; - DCHECK_NE(entry_address, calling_convention.GetRegisterAt(0)); - if (temp_is_r0) { - __ mov(entry_address, ShifterOperand(temp)); - } - } - dex::TypeIndex type_index = cls_->GetTypeIndex(); - __ LoadImmediate(calling_convention.GetRegisterAt(0), type_index.index_); - QuickEntrypointEnum entrypoint = do_clinit_ ? kQuickInitializeStaticStorage - : kQuickInitializeType; - arm_codegen->InvokeRuntime(entrypoint, instruction_, dex_pc_, this); - if (do_clinit_) { - CheckEntrypointTypes<kQuickInitializeStaticStorage, void*, uint32_t>(); - } else { - CheckEntrypointTypes<kQuickInitializeType, void*, uint32_t>(); - } - - // For HLoadClass/kBssEntry, store the resolved Class to the BSS entry. - if (is_load_class_bss_entry) { - if (call_saves_everything_except_r0) { - // The class entry address was preserved in `entry_address` thanks to kSaveEverything. - __ str(R0, Address(entry_address)); - } else { - // For non-Baker read barrier, we need to re-calculate the address of the string entry. - Register temp = IP; - CodeGeneratorARM::PcRelativePatchInfo* labels = - arm_codegen->NewTypeBssEntryPatch(cls_->GetDexFile(), type_index); - __ BindTrackedLabel(&labels->movw_label); - __ movw(temp, /* placeholder */ 0u); - __ BindTrackedLabel(&labels->movt_label); - __ movt(temp, /* placeholder */ 0u); - __ BindTrackedLabel(&labels->add_pc_label); - __ add(temp, temp, ShifterOperand(PC)); - __ str(R0, Address(temp)); - } - } - // Move the class to the desired location. - if (out.IsValid()) { - DCHECK(out.IsRegister() && !locations->GetLiveRegisters()->ContainsCoreRegister(out.reg())); - arm_codegen->Move32(locations->Out(), Location::RegisterLocation(R0)); - } - RestoreLiveRegisters(codegen, locations); - __ b(GetExitLabel()); - } - - const char* GetDescription() const OVERRIDE { return "LoadClassSlowPathARM"; } - - private: - // The class this slow path will load. - HLoadClass* const cls_; - - // The dex PC of `at_`. - const uint32_t dex_pc_; - - // Whether to initialize the class. - const bool do_clinit_; - - DISALLOW_COPY_AND_ASSIGN(LoadClassSlowPathARM); -}; - -class LoadStringSlowPathARM : public SlowPathCodeARM { - public: - explicit LoadStringSlowPathARM(HLoadString* instruction) : SlowPathCodeARM(instruction) {} - - void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { - DCHECK(instruction_->IsLoadString()); - DCHECK_EQ(instruction_->AsLoadString()->GetLoadKind(), HLoadString::LoadKind::kBssEntry); - LocationSummary* locations = instruction_->GetLocations(); - DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg())); - HLoadString* load = instruction_->AsLoadString(); - const dex::StringIndex string_index = load->GetStringIndex(); - Register out = locations->Out().AsRegister<Register>(); - constexpr bool call_saves_everything_except_r0 = (!kUseReadBarrier || kUseBakerReadBarrier); - - CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); - __ Bind(GetEntryLabel()); - SaveLiveRegisters(codegen, locations); - - InvokeRuntimeCallingConvention calling_convention; - // In the unlucky case that the `temp` is R0, we preserve the address in `out` across - // the kSaveEverything call. - Register entry_address = kNoRegister; - if (call_saves_everything_except_r0) { - Register temp = locations->GetTemp(0).AsRegister<Register>(); - bool temp_is_r0 = (temp == calling_convention.GetRegisterAt(0)); - entry_address = temp_is_r0 ? out : temp; - DCHECK_NE(entry_address, calling_convention.GetRegisterAt(0)); - if (temp_is_r0) { - __ mov(entry_address, ShifterOperand(temp)); - } - } - - __ LoadImmediate(calling_convention.GetRegisterAt(0), string_index.index_); - arm_codegen->InvokeRuntime(kQuickResolveString, instruction_, instruction_->GetDexPc(), this); - CheckEntrypointTypes<kQuickResolveString, void*, uint32_t>(); - - // Store the resolved String to the .bss entry. - if (call_saves_everything_except_r0) { - // The string entry address was preserved in `entry_address` thanks to kSaveEverything. - __ str(R0, Address(entry_address)); - } else { - // For non-Baker read barrier, we need to re-calculate the address of the string entry. - Register temp = IP; - CodeGeneratorARM::PcRelativePatchInfo* labels = - arm_codegen->NewPcRelativeStringPatch(load->GetDexFile(), string_index); - __ BindTrackedLabel(&labels->movw_label); - __ movw(temp, /* placeholder */ 0u); - __ BindTrackedLabel(&labels->movt_label); - __ movt(temp, /* placeholder */ 0u); - __ BindTrackedLabel(&labels->add_pc_label); - __ add(temp, temp, ShifterOperand(PC)); - __ str(R0, Address(temp)); - } - - arm_codegen->Move32(locations->Out(), Location::RegisterLocation(R0)); - RestoreLiveRegisters(codegen, locations); - - __ b(GetExitLabel()); - } - - const char* GetDescription() const OVERRIDE { return "LoadStringSlowPathARM"; } - - private: - DISALLOW_COPY_AND_ASSIGN(LoadStringSlowPathARM); -}; - -class TypeCheckSlowPathARM : public SlowPathCodeARM { - public: - TypeCheckSlowPathARM(HInstruction* instruction, bool is_fatal) - : SlowPathCodeARM(instruction), is_fatal_(is_fatal) {} - - void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { - LocationSummary* locations = instruction_->GetLocations(); - DCHECK(instruction_->IsCheckCast() - || !locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg())); - - CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); - __ Bind(GetEntryLabel()); - - if (!is_fatal_) { - SaveLiveRegisters(codegen, locations); - } - - // We're moving two locations to locations that could overlap, so we need a parallel - // move resolver. - InvokeRuntimeCallingConvention calling_convention; - codegen->EmitParallelMoves(locations->InAt(0), - Location::RegisterLocation(calling_convention.GetRegisterAt(0)), - Primitive::kPrimNot, - locations->InAt(1), - Location::RegisterLocation(calling_convention.GetRegisterAt(1)), - Primitive::kPrimNot); - if (instruction_->IsInstanceOf()) { - arm_codegen->InvokeRuntime(kQuickInstanceofNonTrivial, - instruction_, - instruction_->GetDexPc(), - this); - CheckEntrypointTypes<kQuickInstanceofNonTrivial, size_t, mirror::Object*, mirror::Class*>(); - arm_codegen->Move32(locations->Out(), Location::RegisterLocation(R0)); - } else { - DCHECK(instruction_->IsCheckCast()); - arm_codegen->InvokeRuntime(kQuickCheckInstanceOf, - instruction_, - instruction_->GetDexPc(), - this); - CheckEntrypointTypes<kQuickCheckInstanceOf, void, mirror::Object*, mirror::Class*>(); - } - - if (!is_fatal_) { - RestoreLiveRegisters(codegen, locations); - __ b(GetExitLabel()); - } - } - - const char* GetDescription() const OVERRIDE { return "TypeCheckSlowPathARM"; } - - bool IsFatal() const OVERRIDE { return is_fatal_; } - - private: - const bool is_fatal_; - - DISALLOW_COPY_AND_ASSIGN(TypeCheckSlowPathARM); -}; - -class DeoptimizationSlowPathARM : public SlowPathCodeARM { - public: - explicit DeoptimizationSlowPathARM(HDeoptimize* instruction) - : SlowPathCodeARM(instruction) {} - - void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { - CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); - __ Bind(GetEntryLabel()); - LocationSummary* locations = instruction_->GetLocations(); - SaveLiveRegisters(codegen, locations); - InvokeRuntimeCallingConvention calling_convention; - __ LoadImmediate(calling_convention.GetRegisterAt(0), - static_cast<uint32_t>(instruction_->AsDeoptimize()->GetDeoptimizationKind())); - arm_codegen->InvokeRuntime(kQuickDeoptimize, instruction_, instruction_->GetDexPc(), this); - CheckEntrypointTypes<kQuickDeoptimize, void, DeoptimizationKind>(); - } - - const char* GetDescription() const OVERRIDE { return "DeoptimizationSlowPathARM"; } - - private: - DISALLOW_COPY_AND_ASSIGN(DeoptimizationSlowPathARM); -}; - -class ArraySetSlowPathARM : public SlowPathCodeARM { - public: - explicit ArraySetSlowPathARM(HInstruction* instruction) : SlowPathCodeARM(instruction) {} - - void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { - LocationSummary* locations = instruction_->GetLocations(); - __ Bind(GetEntryLabel()); - SaveLiveRegisters(codegen, locations); - - InvokeRuntimeCallingConvention calling_convention; - HParallelMove parallel_move(codegen->GetGraph()->GetArena()); - parallel_move.AddMove( - locations->InAt(0), - Location::RegisterLocation(calling_convention.GetRegisterAt(0)), - Primitive::kPrimNot, - nullptr); - parallel_move.AddMove( - locations->InAt(1), - Location::RegisterLocation(calling_convention.GetRegisterAt(1)), - Primitive::kPrimInt, - nullptr); - parallel_move.AddMove( - locations->InAt(2), - Location::RegisterLocation(calling_convention.GetRegisterAt(2)), - Primitive::kPrimNot, - nullptr); - codegen->GetMoveResolver()->EmitNativeCode(¶llel_move); - - CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); - arm_codegen->InvokeRuntime(kQuickAputObject, instruction_, instruction_->GetDexPc(), this); - CheckEntrypointTypes<kQuickAputObject, void, mirror::Array*, int32_t, mirror::Object*>(); - RestoreLiveRegisters(codegen, locations); - __ b(GetExitLabel()); - } - - const char* GetDescription() const OVERRIDE { return "ArraySetSlowPathARM"; } - - private: - DISALLOW_COPY_AND_ASSIGN(ArraySetSlowPathARM); -}; - -// Abstract base class for read barrier slow paths marking a reference -// `ref`. -// -// Argument `entrypoint` must be a register location holding the read -// barrier marking runtime entry point to be invoked. -class ReadBarrierMarkSlowPathBaseARM : public SlowPathCodeARM { - protected: - ReadBarrierMarkSlowPathBaseARM(HInstruction* instruction, Location ref, Location entrypoint) - : SlowPathCodeARM(instruction), ref_(ref), entrypoint_(entrypoint) { - DCHECK(kEmitCompilerReadBarrier); - } - - const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkSlowPathBaseARM"; } - - // Generate assembly code calling the read barrier marking runtime - // entry point (ReadBarrierMarkRegX). - void GenerateReadBarrierMarkRuntimeCall(CodeGenerator* codegen) { - Register ref_reg = ref_.AsRegister<Register>(); - - // No need to save live registers; it's taken care of by the - // entrypoint. Also, there is no need to update the stack mask, - // as this runtime call will not trigger a garbage collection. - CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); - DCHECK_NE(ref_reg, SP); - DCHECK_NE(ref_reg, LR); - DCHECK_NE(ref_reg, PC); - // IP is used internally by the ReadBarrierMarkRegX entry point - // as a temporary, it cannot be the entry point's input/output. - DCHECK_NE(ref_reg, IP); - DCHECK(0 <= ref_reg && ref_reg < kNumberOfCoreRegisters) << ref_reg; - // "Compact" slow path, saving two moves. - // - // Instead of using the standard runtime calling convention (input - // and output in R0): - // - // R0 <- ref - // R0 <- ReadBarrierMark(R0) - // ref <- R0 - // - // we just use rX (the register containing `ref`) as input and output - // of a dedicated entrypoint: - // - // rX <- ReadBarrierMarkRegX(rX) - // - if (entrypoint_.IsValid()) { - arm_codegen->ValidateInvokeRuntimeWithoutRecordingPcInfo(instruction_, this); - __ blx(entrypoint_.AsRegister<Register>()); - } else { - // Entrypoint is not already loaded, load from the thread. - int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref_reg); - // This runtime call does not require a stack map. - arm_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this); - } - } - - // The location (register) of the marked object reference. - const Location ref_; - - // The location of the entrypoint if it is already loaded. - const Location entrypoint_; - - private: - DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkSlowPathBaseARM); -}; - -// Slow path marking an object reference `ref` during a read -// barrier. The field `obj.field` in the object `obj` holding this -// reference does not get updated by this slow path after marking. -// -// This means that after the execution of this slow path, `ref` will -// always be up-to-date, but `obj.field` may not; i.e., after the -// flip, `ref` will be a to-space reference, but `obj.field` will -// probably still be a from-space reference (unless it gets updated by -// another thread, or if another thread installed another object -// reference (different from `ref`) in `obj.field`). -// -// If `entrypoint` is a valid location it is assumed to already be -// holding the entrypoint. The case where the entrypoint is passed in -// is when the decision to mark is based on whether the GC is marking. -class ReadBarrierMarkSlowPathARM : public ReadBarrierMarkSlowPathBaseARM { - public: - ReadBarrierMarkSlowPathARM(HInstruction* instruction, - Location ref, - Location entrypoint = Location::NoLocation()) - : ReadBarrierMarkSlowPathBaseARM(instruction, ref, entrypoint) { - DCHECK(kEmitCompilerReadBarrier); - } - - const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkSlowPathARM"; } - - void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { - LocationSummary* locations = instruction_->GetLocations(); - DCHECK(locations->CanCall()); - if (kIsDebugBuild) { - Register ref_reg = ref_.AsRegister<Register>(); - DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg)) << ref_reg; - } - DCHECK(instruction_->IsLoadClass() || instruction_->IsLoadString()) - << "Unexpected instruction in read barrier marking slow path: " - << instruction_->DebugName(); - - __ Bind(GetEntryLabel()); - GenerateReadBarrierMarkRuntimeCall(codegen); - __ b(GetExitLabel()); - } - - private: - DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkSlowPathARM); -}; - -// Slow path loading `obj`'s lock word, loading a reference from -// object `*(obj + offset + (index << scale_factor))` into `ref`, and -// marking `ref` if `obj` is gray according to the lock word (Baker -// read barrier). The field `obj.field` in the object `obj` holding -// this reference does not get updated by this slow path after marking -// (see LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM -// below for that). -// -// This means that after the execution of this slow path, `ref` will -// always be up-to-date, but `obj.field` may not; i.e., after the -// flip, `ref` will be a to-space reference, but `obj.field` will -// probably still be a from-space reference (unless it gets updated by -// another thread, or if another thread installed another object -// reference (different from `ref`) in `obj.field`). -// -// Argument `entrypoint` must be a register location holding the read -// barrier marking runtime entry point to be invoked. -class LoadReferenceWithBakerReadBarrierSlowPathARM : public ReadBarrierMarkSlowPathBaseARM { - public: - LoadReferenceWithBakerReadBarrierSlowPathARM(HInstruction* instruction, - Location ref, - Register obj, - uint32_t offset, - Location index, - ScaleFactor scale_factor, - bool needs_null_check, - Register temp, - Location entrypoint) - : ReadBarrierMarkSlowPathBaseARM(instruction, ref, entrypoint), - obj_(obj), - offset_(offset), - index_(index), - scale_factor_(scale_factor), - needs_null_check_(needs_null_check), - temp_(temp) { - DCHECK(kEmitCompilerReadBarrier); - DCHECK(kUseBakerReadBarrier); - } - - const char* GetDescription() const OVERRIDE { - return "LoadReferenceWithBakerReadBarrierSlowPathARM"; - } - - void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { - LocationSummary* locations = instruction_->GetLocations(); - Register ref_reg = ref_.AsRegister<Register>(); - DCHECK(locations->CanCall()); - DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg)) << ref_reg; - DCHECK_NE(ref_reg, temp_); - DCHECK(instruction_->IsInstanceFieldGet() || - instruction_->IsStaticFieldGet() || - instruction_->IsArrayGet() || - instruction_->IsArraySet() || - instruction_->IsInstanceOf() || - instruction_->IsCheckCast() || - (instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified()) || - (instruction_->IsInvokeStaticOrDirect() && instruction_->GetLocations()->Intrinsified())) - << "Unexpected instruction in read barrier marking slow path: " - << instruction_->DebugName(); - // The read barrier instrumentation of object ArrayGet - // instructions does not support the HIntermediateAddress - // instruction. - DCHECK(!(instruction_->IsArrayGet() && - instruction_->AsArrayGet()->GetArray()->IsIntermediateAddress())); - - __ Bind(GetEntryLabel()); - - // When using MaybeGenerateReadBarrierSlow, the read barrier call is - // inserted after the original load. However, in fast path based - // Baker's read barriers, we need to perform the load of - // mirror::Object::monitor_ *before* the original reference load. - // This load-load ordering is required by the read barrier. - // The slow path (for Baker's algorithm) should look like: - // - // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState(); - // lfence; // Load fence or artificial data dependency to prevent load-load reordering - // HeapReference<mirror::Object> ref = *src; // Original reference load. - // bool is_gray = (rb_state == ReadBarrier::GrayState()); - // if (is_gray) { - // ref = entrypoint(ref); // ref = ReadBarrier::Mark(ref); // Runtime entry point call. - // } - // - // Note: the original implementation in ReadBarrier::Barrier is - // slightly more complex as it performs additional checks that we do - // not do here for performance reasons. - - // /* int32_t */ monitor = obj->monitor_ - uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value(); - __ LoadFromOffset(kLoadWord, temp_, obj_, monitor_offset); - if (needs_null_check_) { - codegen->MaybeRecordImplicitNullCheck(instruction_); - } - // /* LockWord */ lock_word = LockWord(monitor) - static_assert(sizeof(LockWord) == sizeof(int32_t), - "art::LockWord and int32_t have different sizes."); - - // Introduce a dependency on the lock_word including the rb_state, - // which shall prevent load-load reordering without using - // a memory barrier (which would be more expensive). - // `obj` is unchanged by this operation, but its value now depends - // on `temp`. - __ add(obj_, obj_, ShifterOperand(temp_, LSR, 32)); - - // The actual reference load. - // A possible implicit null check has already been handled above. - CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); - arm_codegen->GenerateRawReferenceLoad( - instruction_, ref_, obj_, offset_, index_, scale_factor_, /* needs_null_check */ false); - - // Mark the object `ref` when `obj` is gray. - // - // if (rb_state == ReadBarrier::GrayState()) - // ref = ReadBarrier::Mark(ref); - // - // Given the numeric representation, it's enough to check the low bit of the - // rb_state. We do that by shifting the bit out of the lock word with LSRS - // which can be a 16-bit instruction unlike the TST immediate. - static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0"); - static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1"); - __ Lsrs(temp_, temp_, LockWord::kReadBarrierStateShift + 1); - __ b(GetExitLabel(), CC); // Carry flag is the last bit shifted out by LSRS. - GenerateReadBarrierMarkRuntimeCall(codegen); - - __ b(GetExitLabel()); - } - - private: - // The register containing the object holding the marked object reference field. - Register obj_; - // The offset, index and scale factor to access the reference in `obj_`. - uint32_t offset_; - Location index_; - ScaleFactor scale_factor_; - // Is a null check required? - bool needs_null_check_; - // A temporary register used to hold the lock word of `obj_`. - Register temp_; - - DISALLOW_COPY_AND_ASSIGN(LoadReferenceWithBakerReadBarrierSlowPathARM); -}; - -// Slow path loading `obj`'s lock word, loading a reference from -// object `*(obj + offset + (index << scale_factor))` into `ref`, and -// marking `ref` if `obj` is gray according to the lock word (Baker -// read barrier). If needed, this slow path also atomically updates -// the field `obj.field` in the object `obj` holding this reference -// after marking (contrary to -// LoadReferenceWithBakerReadBarrierSlowPathARM above, which never -// tries to update `obj.field`). -// -// This means that after the execution of this slow path, both `ref` -// and `obj.field` will be up-to-date; i.e., after the flip, both will -// hold the same to-space reference (unless another thread installed -// another object reference (different from `ref`) in `obj.field`). -// -// Argument `entrypoint` must be a register location holding the read -// barrier marking runtime entry point to be invoked. -class LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM - : public ReadBarrierMarkSlowPathBaseARM { - public: - LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM(HInstruction* instruction, - Location ref, - Register obj, - uint32_t offset, - Location index, - ScaleFactor scale_factor, - bool needs_null_check, - Register temp1, - Register temp2, - Location entrypoint) - : ReadBarrierMarkSlowPathBaseARM(instruction, ref, entrypoint), - obj_(obj), - offset_(offset), - index_(index), - scale_factor_(scale_factor), - needs_null_check_(needs_null_check), - temp1_(temp1), - temp2_(temp2) { - DCHECK(kEmitCompilerReadBarrier); - DCHECK(kUseBakerReadBarrier); - } - - const char* GetDescription() const OVERRIDE { - return "LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM"; - } - - void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { - LocationSummary* locations = instruction_->GetLocations(); - Register ref_reg = ref_.AsRegister<Register>(); - DCHECK(locations->CanCall()); - DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg)) << ref_reg; - DCHECK_NE(ref_reg, temp1_); - - // This slow path is only used by the UnsafeCASObject intrinsic at the moment. - DCHECK((instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified())) - << "Unexpected instruction in read barrier marking and field updating slow path: " - << instruction_->DebugName(); - DCHECK(instruction_->GetLocations()->Intrinsified()); - DCHECK_EQ(instruction_->AsInvoke()->GetIntrinsic(), Intrinsics::kUnsafeCASObject); - DCHECK_EQ(offset_, 0u); - DCHECK_EQ(scale_factor_, ScaleFactor::TIMES_1); - // The location of the offset of the marked reference field within `obj_`. - Location field_offset = index_; - DCHECK(field_offset.IsRegisterPair()) << field_offset; - - __ Bind(GetEntryLabel()); - - // The implementation is similar to LoadReferenceWithBakerReadBarrierSlowPathARM's: - // - // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState(); - // lfence; // Load fence or artificial data dependency to prevent load-load reordering - // HeapReference<mirror::Object> ref = *src; // Original reference load. - // bool is_gray = (rb_state == ReadBarrier::GrayState()); - // if (is_gray) { - // old_ref = ref; - // ref = entrypoint(ref); // ref = ReadBarrier::Mark(ref); // Runtime entry point call. - // compareAndSwapObject(obj, field_offset, old_ref, ref); - // } - - // /* int32_t */ monitor = obj->monitor_ - uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value(); - __ LoadFromOffset(kLoadWord, temp1_, obj_, monitor_offset); - if (needs_null_check_) { - codegen->MaybeRecordImplicitNullCheck(instruction_); - } - // /* LockWord */ lock_word = LockWord(monitor) - static_assert(sizeof(LockWord) == sizeof(int32_t), - "art::LockWord and int32_t have different sizes."); - - // Introduce a dependency on the lock_word including the rb_state, - // which shall prevent load-load reordering without using - // a memory barrier (which would be more expensive). - // `obj` is unchanged by this operation, but its value now depends - // on `temp1`. - __ add(obj_, obj_, ShifterOperand(temp1_, LSR, 32)); - - // The actual reference load. - // A possible implicit null check has already been handled above. - CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); - arm_codegen->GenerateRawReferenceLoad( - instruction_, ref_, obj_, offset_, index_, scale_factor_, /* needs_null_check */ false); - - // Mark the object `ref` when `obj` is gray. - // - // if (rb_state == ReadBarrier::GrayState()) - // ref = ReadBarrier::Mark(ref); - // - // Given the numeric representation, it's enough to check the low bit of the - // rb_state. We do that by shifting the bit out of the lock word with LSRS - // which can be a 16-bit instruction unlike the TST immediate. - static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0"); - static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1"); - __ Lsrs(temp1_, temp1_, LockWord::kReadBarrierStateShift + 1); - __ b(GetExitLabel(), CC); // Carry flag is the last bit shifted out by LSRS. - - // Save the old value of the reference before marking it. - // Note that we cannot use IP to save the old reference, as IP is - // used internally by the ReadBarrierMarkRegX entry point, and we - // need the old reference after the call to that entry point. - DCHECK_NE(temp1_, IP); - __ Mov(temp1_, ref_reg); - - GenerateReadBarrierMarkRuntimeCall(codegen); - - // If the new reference is different from the old reference, - // update the field in the holder (`*(obj_ + field_offset)`). - // - // Note that this field could also hold a different object, if - // another thread had concurrently changed it. In that case, the - // LDREX/SUBS/ITNE sequence of instructions in the compare-and-set - // (CAS) operation below would abort the CAS, leaving the field - // as-is. - __ cmp(temp1_, ShifterOperand(ref_reg)); - __ b(GetExitLabel(), EQ); - - // Update the the holder's field atomically. This may fail if - // mutator updates before us, but it's OK. This is achieved - // using a strong compare-and-set (CAS) operation with relaxed - // memory synchronization ordering, where the expected value is - // the old reference and the desired value is the new reference. - - // Convenience aliases. - Register base = obj_; - // The UnsafeCASObject intrinsic uses a register pair as field - // offset ("long offset"), of which only the low part contains - // data. - Register offset = field_offset.AsRegisterPairLow<Register>(); - Register expected = temp1_; - Register value = ref_reg; - Register tmp_ptr = IP; // Pointer to actual memory. - Register tmp = temp2_; // Value in memory. - - __ add(tmp_ptr, base, ShifterOperand(offset)); - - if (kPoisonHeapReferences) { - __ PoisonHeapReference(expected); - if (value == expected) { - // Do not poison `value`, as it is the same register as - // `expected`, which has just been poisoned. - } else { - __ PoisonHeapReference(value); - } - } - - // do { - // tmp = [r_ptr] - expected; - // } while (tmp == 0 && failure([r_ptr] <- r_new_value)); - - Label loop_head, exit_loop; - __ Bind(&loop_head); - - __ ldrex(tmp, tmp_ptr); - - __ subs(tmp, tmp, ShifterOperand(expected)); - - __ it(NE); - __ clrex(NE); - - __ b(&exit_loop, NE); - - __ strex(tmp, value, tmp_ptr); - __ cmp(tmp, ShifterOperand(1)); - __ b(&loop_head, EQ); - - __ Bind(&exit_loop); - - if (kPoisonHeapReferences) { - __ UnpoisonHeapReference(expected); - if (value == expected) { - // Do not unpoison `value`, as it is the same register as - // `expected`, which has just been unpoisoned. - } else { - __ UnpoisonHeapReference(value); - } - } - - __ b(GetExitLabel()); - } - - private: - // The register containing the object holding the marked object reference field. - const Register obj_; - // The offset, index and scale factor to access the reference in `obj_`. - uint32_t offset_; - Location index_; - ScaleFactor scale_factor_; - // Is a null check required? - bool needs_null_check_; - // A temporary register used to hold the lock word of `obj_`; and - // also to hold the original reference value, when the reference is - // marked. - const Register temp1_; - // A temporary register used in the implementation of the CAS, to - // update the object's reference field. - const Register temp2_; - - DISALLOW_COPY_AND_ASSIGN(LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM); -}; - -// Slow path generating a read barrier for a heap reference. -class ReadBarrierForHeapReferenceSlowPathARM : public SlowPathCodeARM { - public: - ReadBarrierForHeapReferenceSlowPathARM(HInstruction* instruction, - Location out, - Location ref, - Location obj, - uint32_t offset, - Location index) - : SlowPathCodeARM(instruction), - out_(out), - ref_(ref), - obj_(obj), - offset_(offset), - index_(index) { - DCHECK(kEmitCompilerReadBarrier); - // If `obj` is equal to `out` or `ref`, it means the initial object - // has been overwritten by (or after) the heap object reference load - // to be instrumented, e.g.: - // - // __ LoadFromOffset(kLoadWord, out, out, offset); - // codegen_->GenerateReadBarrierSlow(instruction, out_loc, out_loc, out_loc, offset); - // - // In that case, we have lost the information about the original - // object, and the emitted read barrier cannot work properly. - DCHECK(!obj.Equals(out)) << "obj=" << obj << " out=" << out; - DCHECK(!obj.Equals(ref)) << "obj=" << obj << " ref=" << ref; - } - - void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { - CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); - LocationSummary* locations = instruction_->GetLocations(); - Register reg_out = out_.AsRegister<Register>(); - DCHECK(locations->CanCall()); - DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out)); - DCHECK(instruction_->IsInstanceFieldGet() || - instruction_->IsStaticFieldGet() || - instruction_->IsArrayGet() || - instruction_->IsInstanceOf() || - instruction_->IsCheckCast() || - (instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified())) - << "Unexpected instruction in read barrier for heap reference slow path: " - << instruction_->DebugName(); - // The read barrier instrumentation of object ArrayGet - // instructions does not support the HIntermediateAddress - // instruction. - DCHECK(!(instruction_->IsArrayGet() && - instruction_->AsArrayGet()->GetArray()->IsIntermediateAddress())); - - __ Bind(GetEntryLabel()); - SaveLiveRegisters(codegen, locations); - - // We may have to change the index's value, but as `index_` is a - // constant member (like other "inputs" of this slow path), - // introduce a copy of it, `index`. - Location index = index_; - if (index_.IsValid()) { - // Handle `index_` for HArrayGet and UnsafeGetObject/UnsafeGetObjectVolatile intrinsics. - if (instruction_->IsArrayGet()) { - // Compute the actual memory offset and store it in `index`. - Register index_reg = index_.AsRegister<Register>(); - DCHECK(locations->GetLiveRegisters()->ContainsCoreRegister(index_reg)); - if (codegen->IsCoreCalleeSaveRegister(index_reg)) { - // We are about to change the value of `index_reg` (see the - // calls to art::arm::Thumb2Assembler::Lsl and - // art::arm::Thumb2Assembler::AddConstant below), but it has - // not been saved by the previous call to - // art::SlowPathCode::SaveLiveRegisters, as it is a - // callee-save register -- - // art::SlowPathCode::SaveLiveRegisters does not consider - // callee-save registers, as it has been designed with the - // assumption that callee-save registers are supposed to be - // handled by the called function. So, as a callee-save - // register, `index_reg` _would_ eventually be saved onto - // the stack, but it would be too late: we would have - // changed its value earlier. Therefore, we manually save - // it here into another freely available register, - // `free_reg`, chosen of course among the caller-save - // registers (as a callee-save `free_reg` register would - // exhibit the same problem). - // - // Note we could have requested a temporary register from - // the register allocator instead; but we prefer not to, as - // this is a slow path, and we know we can find a - // caller-save register that is available. - Register free_reg = FindAvailableCallerSaveRegister(codegen); - __ Mov(free_reg, index_reg); - index_reg = free_reg; - index = Location::RegisterLocation(index_reg); - } else { - // The initial register stored in `index_` has already been - // saved in the call to art::SlowPathCode::SaveLiveRegisters - // (as it is not a callee-save register), so we can freely - // use it. - } - // Shifting the index value contained in `index_reg` by the scale - // factor (2) cannot overflow in practice, as the runtime is - // unable to allocate object arrays with a size larger than - // 2^26 - 1 (that is, 2^28 - 4 bytes). - __ Lsl(index_reg, index_reg, TIMES_4); - static_assert( - sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), - "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); - __ AddConstant(index_reg, index_reg, offset_); - } else { - // In the case of the UnsafeGetObject/UnsafeGetObjectVolatile - // intrinsics, `index_` is not shifted by a scale factor of 2 - // (as in the case of ArrayGet), as it is actually an offset - // to an object field within an object. - DCHECK(instruction_->IsInvoke()) << instruction_->DebugName(); - DCHECK(instruction_->GetLocations()->Intrinsified()); - DCHECK((instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObject) || - (instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile)) - << instruction_->AsInvoke()->GetIntrinsic(); - DCHECK_EQ(offset_, 0U); - DCHECK(index_.IsRegisterPair()); - // UnsafeGet's offset location is a register pair, the low - // part contains the correct offset. - index = index_.ToLow(); - } - } - - // We're moving two or three locations to locations that could - // overlap, so we need a parallel move resolver. - InvokeRuntimeCallingConvention calling_convention; - HParallelMove parallel_move(codegen->GetGraph()->GetArena()); - parallel_move.AddMove(ref_, - Location::RegisterLocation(calling_convention.GetRegisterAt(0)), - Primitive::kPrimNot, - nullptr); - parallel_move.AddMove(obj_, - Location::RegisterLocation(calling_convention.GetRegisterAt(1)), - Primitive::kPrimNot, - nullptr); - if (index.IsValid()) { - parallel_move.AddMove(index, - Location::RegisterLocation(calling_convention.GetRegisterAt(2)), - Primitive::kPrimInt, - nullptr); - codegen->GetMoveResolver()->EmitNativeCode(¶llel_move); - } else { - codegen->GetMoveResolver()->EmitNativeCode(¶llel_move); - __ LoadImmediate(calling_convention.GetRegisterAt(2), offset_); - } - arm_codegen->InvokeRuntime(kQuickReadBarrierSlow, instruction_, instruction_->GetDexPc(), this); - CheckEntrypointTypes< - kQuickReadBarrierSlow, mirror::Object*, mirror::Object*, mirror::Object*, uint32_t>(); - arm_codegen->Move32(out_, Location::RegisterLocation(R0)); - - RestoreLiveRegisters(codegen, locations); - __ b(GetExitLabel()); - } - - const char* GetDescription() const OVERRIDE { return "ReadBarrierForHeapReferenceSlowPathARM"; } - - private: - Register FindAvailableCallerSaveRegister(CodeGenerator* codegen) { - size_t ref = static_cast<int>(ref_.AsRegister<Register>()); - size_t obj = static_cast<int>(obj_.AsRegister<Register>()); - for (size_t i = 0, e = codegen->GetNumberOfCoreRegisters(); i < e; ++i) { - if (i != ref && i != obj && !codegen->IsCoreCalleeSaveRegister(i)) { - return static_cast<Register>(i); - } - } - // We shall never fail to find a free caller-save register, as - // there are more than two core caller-save registers on ARM - // (meaning it is possible to find one which is different from - // `ref` and `obj`). - DCHECK_GT(codegen->GetNumberOfCoreCallerSaveRegisters(), 2u); - LOG(FATAL) << "Could not find a free caller-save register"; - UNREACHABLE(); - } - - const Location out_; - const Location ref_; - const Location obj_; - const uint32_t offset_; - // An additional location containing an index to an array. - // Only used for HArrayGet and the UnsafeGetObject & - // UnsafeGetObjectVolatile intrinsics. - const Location index_; - - DISALLOW_COPY_AND_ASSIGN(ReadBarrierForHeapReferenceSlowPathARM); -}; - -// Slow path generating a read barrier for a GC root. -class ReadBarrierForRootSlowPathARM : public SlowPathCodeARM { - public: - ReadBarrierForRootSlowPathARM(HInstruction* instruction, Location out, Location root) - : SlowPathCodeARM(instruction), out_(out), root_(root) { - DCHECK(kEmitCompilerReadBarrier); - } - - void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { - LocationSummary* locations = instruction_->GetLocations(); - Register reg_out = out_.AsRegister<Register>(); - DCHECK(locations->CanCall()); - DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out)); - DCHECK(instruction_->IsLoadClass() || instruction_->IsLoadString()) - << "Unexpected instruction in read barrier for GC root slow path: " - << instruction_->DebugName(); - - __ Bind(GetEntryLabel()); - SaveLiveRegisters(codegen, locations); - - InvokeRuntimeCallingConvention calling_convention; - CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); - arm_codegen->Move32(Location::RegisterLocation(calling_convention.GetRegisterAt(0)), root_); - arm_codegen->InvokeRuntime(kQuickReadBarrierForRootSlow, - instruction_, - instruction_->GetDexPc(), - this); - CheckEntrypointTypes<kQuickReadBarrierForRootSlow, mirror::Object*, GcRoot<mirror::Object>*>(); - arm_codegen->Move32(out_, Location::RegisterLocation(R0)); - - RestoreLiveRegisters(codegen, locations); - __ b(GetExitLabel()); - } - - const char* GetDescription() const OVERRIDE { return "ReadBarrierForRootSlowPathARM"; } - - private: - const Location out_; - const Location root_; - - DISALLOW_COPY_AND_ASSIGN(ReadBarrierForRootSlowPathARM); -}; - -inline Condition ARMCondition(IfCondition cond) { - switch (cond) { - case kCondEQ: return EQ; - case kCondNE: return NE; - case kCondLT: return LT; - case kCondLE: return LE; - case kCondGT: return GT; - case kCondGE: return GE; - case kCondB: return LO; - case kCondBE: return LS; - case kCondA: return HI; - case kCondAE: return HS; - } - LOG(FATAL) << "Unreachable"; - UNREACHABLE(); -} - -// Maps signed condition to unsigned condition. -inline Condition ARMUnsignedCondition(IfCondition cond) { - switch (cond) { - case kCondEQ: return EQ; - case kCondNE: return NE; - // Signed to unsigned. - case kCondLT: return LO; - case kCondLE: return LS; - case kCondGT: return HI; - case kCondGE: return HS; - // Unsigned remain unchanged. - case kCondB: return LO; - case kCondBE: return LS; - case kCondA: return HI; - case kCondAE: return HS; - } - LOG(FATAL) << "Unreachable"; - UNREACHABLE(); -} - -inline Condition ARMFPCondition(IfCondition cond, bool gt_bias) { - // The ARM condition codes can express all the necessary branches, see the - // "Meaning (floating-point)" column in the table A8-1 of the ARMv7 reference manual. - // There is no dex instruction or HIR that would need the missing conditions - // "equal or unordered" or "not equal". - switch (cond) { - case kCondEQ: return EQ; - case kCondNE: return NE /* unordered */; - case kCondLT: return gt_bias ? CC : LT /* unordered */; - case kCondLE: return gt_bias ? LS : LE /* unordered */; - case kCondGT: return gt_bias ? HI /* unordered */ : GT; - case kCondGE: return gt_bias ? CS /* unordered */ : GE; - default: - LOG(FATAL) << "UNREACHABLE"; - UNREACHABLE(); - } -} - -inline Shift ShiftFromOpKind(HDataProcWithShifterOp::OpKind op_kind) { - switch (op_kind) { - case HDataProcWithShifterOp::kASR: return ASR; - case HDataProcWithShifterOp::kLSL: return LSL; - case HDataProcWithShifterOp::kLSR: return LSR; - default: - LOG(FATAL) << "Unexpected op kind " << op_kind; - UNREACHABLE(); - } -} - -static void GenerateDataProcInstruction(HInstruction::InstructionKind kind, - Register out, - Register first, - const ShifterOperand& second, - CodeGeneratorARM* codegen) { - if (second.IsImmediate() && second.GetImmediate() == 0) { - const ShifterOperand in = kind == HInstruction::kAnd - ? ShifterOperand(0) - : ShifterOperand(first); - - __ mov(out, in); - } else { - switch (kind) { - case HInstruction::kAdd: - __ add(out, first, second); - break; - case HInstruction::kAnd: - __ and_(out, first, second); - break; - case HInstruction::kOr: - __ orr(out, first, second); - break; - case HInstruction::kSub: - __ sub(out, first, second); - break; - case HInstruction::kXor: - __ eor(out, first, second); - break; - default: - LOG(FATAL) << "Unexpected instruction kind: " << kind; - UNREACHABLE(); - } - } -} - -static void GenerateDataProc(HInstruction::InstructionKind kind, - const Location& out, - const Location& first, - const ShifterOperand& second_lo, - const ShifterOperand& second_hi, - CodeGeneratorARM* codegen) { - const Register first_hi = first.AsRegisterPairHigh<Register>(); - const Register first_lo = first.AsRegisterPairLow<Register>(); - const Register out_hi = out.AsRegisterPairHigh<Register>(); - const Register out_lo = out.AsRegisterPairLow<Register>(); - - if (kind == HInstruction::kAdd) { - __ adds(out_lo, first_lo, second_lo); - __ adc(out_hi, first_hi, second_hi); - } else if (kind == HInstruction::kSub) { - __ subs(out_lo, first_lo, second_lo); - __ sbc(out_hi, first_hi, second_hi); - } else { - GenerateDataProcInstruction(kind, out_lo, first_lo, second_lo, codegen); - GenerateDataProcInstruction(kind, out_hi, first_hi, second_hi, codegen); - } -} - -static ShifterOperand GetShifterOperand(Register rm, Shift shift, uint32_t shift_imm) { - return shift_imm == 0 ? ShifterOperand(rm) : ShifterOperand(rm, shift, shift_imm); -} - -static void GenerateLongDataProc(HDataProcWithShifterOp* instruction, CodeGeneratorARM* codegen) { - DCHECK_EQ(instruction->GetType(), Primitive::kPrimLong); - DCHECK(HDataProcWithShifterOp::IsShiftOp(instruction->GetOpKind())); - - const LocationSummary* const locations = instruction->GetLocations(); - const uint32_t shift_value = instruction->GetShiftAmount(); - const HInstruction::InstructionKind kind = instruction->GetInstrKind(); - const Location first = locations->InAt(0); - const Location second = locations->InAt(1); - const Location out = locations->Out(); - const Register first_hi = first.AsRegisterPairHigh<Register>(); - const Register first_lo = first.AsRegisterPairLow<Register>(); - const Register out_hi = out.AsRegisterPairHigh<Register>(); - const Register out_lo = out.AsRegisterPairLow<Register>(); - const Register second_hi = second.AsRegisterPairHigh<Register>(); - const Register second_lo = second.AsRegisterPairLow<Register>(); - const Shift shift = ShiftFromOpKind(instruction->GetOpKind()); - - if (shift_value >= 32) { - if (shift == LSL) { - GenerateDataProcInstruction(kind, - out_hi, - first_hi, - ShifterOperand(second_lo, LSL, shift_value - 32), - codegen); - GenerateDataProcInstruction(kind, - out_lo, - first_lo, - ShifterOperand(0), - codegen); - } else if (shift == ASR) { - GenerateDataProc(kind, - out, - first, - GetShifterOperand(second_hi, ASR, shift_value - 32), - ShifterOperand(second_hi, ASR, 31), - codegen); - } else { - DCHECK_EQ(shift, LSR); - GenerateDataProc(kind, - out, - first, - GetShifterOperand(second_hi, LSR, shift_value - 32), - ShifterOperand(0), - codegen); - } - } else { - DCHECK_GT(shift_value, 1U); - DCHECK_LT(shift_value, 32U); - - if (shift == LSL) { - // We are not doing this for HInstruction::kAdd because the output will require - // Location::kOutputOverlap; not applicable to other cases. - if (kind == HInstruction::kOr || kind == HInstruction::kXor) { - GenerateDataProcInstruction(kind, - out_hi, - first_hi, - ShifterOperand(second_hi, LSL, shift_value), - codegen); - GenerateDataProcInstruction(kind, - out_hi, - out_hi, - ShifterOperand(second_lo, LSR, 32 - shift_value), - codegen); - GenerateDataProcInstruction(kind, - out_lo, - first_lo, - ShifterOperand(second_lo, LSL, shift_value), - codegen); - } else { - __ Lsl(IP, second_hi, shift_value); - __ orr(IP, IP, ShifterOperand(second_lo, LSR, 32 - shift_value)); - GenerateDataProc(kind, - out, - first, - ShifterOperand(second_lo, LSL, shift_value), - ShifterOperand(IP), - codegen); - } - } else { - DCHECK(shift == ASR || shift == LSR); - - // We are not doing this for HInstruction::kAdd because the output will require - // Location::kOutputOverlap; not applicable to other cases. - if (kind == HInstruction::kOr || kind == HInstruction::kXor) { - GenerateDataProcInstruction(kind, - out_lo, - first_lo, - ShifterOperand(second_lo, LSR, shift_value), - codegen); - GenerateDataProcInstruction(kind, - out_lo, - out_lo, - ShifterOperand(second_hi, LSL, 32 - shift_value), - codegen); - GenerateDataProcInstruction(kind, - out_hi, - first_hi, - ShifterOperand(second_hi, shift, shift_value), - codegen); - } else { - __ Lsr(IP, second_lo, shift_value); - __ orr(IP, IP, ShifterOperand(second_hi, LSL, 32 - shift_value)); - GenerateDataProc(kind, - out, - first, - ShifterOperand(IP), - ShifterOperand(second_hi, shift, shift_value), - codegen); - } - } - } -} - -static void GenerateVcmp(HInstruction* instruction, CodeGeneratorARM* codegen) { - Primitive::Type type = instruction->InputAt(0)->GetType(); - Location lhs_loc = instruction->GetLocations()->InAt(0); - Location rhs_loc = instruction->GetLocations()->InAt(1); - if (rhs_loc.IsConstant()) { - // 0.0 is the only immediate that can be encoded directly in - // a VCMP instruction. - // - // Both the JLS (section 15.20.1) and the JVMS (section 6.5) - // specify that in a floating-point comparison, positive zero - // and negative zero are considered equal, so we can use the - // literal 0.0 for both cases here. - // - // Note however that some methods (Float.equal, Float.compare, - // Float.compareTo, Double.equal, Double.compare, - // Double.compareTo, Math.max, Math.min, StrictMath.max, - // StrictMath.min) consider 0.0 to be (strictly) greater than - // -0.0. So if we ever translate calls to these methods into a - // HCompare instruction, we must handle the -0.0 case with - // care here. - DCHECK(rhs_loc.GetConstant()->IsArithmeticZero()); - if (type == Primitive::kPrimFloat) { - __ vcmpsz(lhs_loc.AsFpuRegister<SRegister>()); - } else { - DCHECK_EQ(type, Primitive::kPrimDouble); - __ vcmpdz(FromLowSToD(lhs_loc.AsFpuRegisterPairLow<SRegister>())); - } - } else { - if (type == Primitive::kPrimFloat) { - __ vcmps(lhs_loc.AsFpuRegister<SRegister>(), rhs_loc.AsFpuRegister<SRegister>()); - } else { - DCHECK_EQ(type, Primitive::kPrimDouble); - __ vcmpd(FromLowSToD(lhs_loc.AsFpuRegisterPairLow<SRegister>()), - FromLowSToD(rhs_loc.AsFpuRegisterPairLow<SRegister>())); - } - } -} - -static int64_t AdjustConstantForCondition(int64_t value, - IfCondition* condition, - IfCondition* opposite) { - if (value == 1) { - if (*condition == kCondB) { - value = 0; - *condition = kCondEQ; - *opposite = kCondNE; - } else if (*condition == kCondAE) { - value = 0; - *condition = kCondNE; - *opposite = kCondEQ; - } - } else if (value == -1) { - if (*condition == kCondGT) { - value = 0; - *condition = kCondGE; - *opposite = kCondLT; - } else if (*condition == kCondLE) { - value = 0; - *condition = kCondLT; - *opposite = kCondGE; - } - } - - return value; -} - -static std::pair<Condition, Condition> GenerateLongTestConstant(HCondition* condition, - bool invert, - CodeGeneratorARM* codegen) { - DCHECK_EQ(condition->GetLeft()->GetType(), Primitive::kPrimLong); - - const LocationSummary* const locations = condition->GetLocations(); - IfCondition cond = condition->GetCondition(); - IfCondition opposite = condition->GetOppositeCondition(); - - if (invert) { - std::swap(cond, opposite); - } - - std::pair<Condition, Condition> ret(EQ, NE); - const Location left = locations->InAt(0); - const Location right = locations->InAt(1); - - DCHECK(right.IsConstant()); - - const Register left_high = left.AsRegisterPairHigh<Register>(); - const Register left_low = left.AsRegisterPairLow<Register>(); - int64_t value = AdjustConstantForCondition(right.GetConstant()->AsLongConstant()->GetValue(), - &cond, - &opposite); - - // Comparisons against 0 are common enough to deserve special attention. - if (value == 0) { - switch (cond) { - case kCondNE: - // x > 0 iff x != 0 when the comparison is unsigned. - case kCondA: - ret = std::make_pair(NE, EQ); - FALLTHROUGH_INTENDED; - case kCondEQ: - // x <= 0 iff x == 0 when the comparison is unsigned. - case kCondBE: - __ orrs(IP, left_low, ShifterOperand(left_high)); - return ret; - case kCondLT: - case kCondGE: - __ cmp(left_high, ShifterOperand(0)); - return std::make_pair(ARMCondition(cond), ARMCondition(opposite)); - // Trivially true or false. - case kCondB: - ret = std::make_pair(NE, EQ); - FALLTHROUGH_INTENDED; - case kCondAE: - __ cmp(left_low, ShifterOperand(left_low)); - return ret; - default: - break; - } - } - - switch (cond) { - case kCondEQ: - case kCondNE: - case kCondB: - case kCondBE: - case kCondA: - case kCondAE: - __ CmpConstant(left_high, High32Bits(value)); - __ it(EQ); - __ cmp(left_low, ShifterOperand(Low32Bits(value)), EQ); - ret = std::make_pair(ARMUnsignedCondition(cond), ARMUnsignedCondition(opposite)); - break; - case kCondLE: - case kCondGT: - // Trivially true or false. - if (value == std::numeric_limits<int64_t>::max()) { - __ cmp(left_low, ShifterOperand(left_low)); - ret = cond == kCondLE ? std::make_pair(EQ, NE) : std::make_pair(NE, EQ); - break; - } - - if (cond == kCondLE) { - DCHECK_EQ(opposite, kCondGT); - cond = kCondLT; - opposite = kCondGE; - } else { - DCHECK_EQ(cond, kCondGT); - DCHECK_EQ(opposite, kCondLE); - cond = kCondGE; - opposite = kCondLT; - } - - value++; - FALLTHROUGH_INTENDED; - case kCondGE: - case kCondLT: - __ CmpConstant(left_low, Low32Bits(value)); - __ sbcs(IP, left_high, ShifterOperand(High32Bits(value))); - ret = std::make_pair(ARMCondition(cond), ARMCondition(opposite)); - break; - default: - LOG(FATAL) << "Unreachable"; - UNREACHABLE(); - } - - return ret; -} - -static std::pair<Condition, Condition> GenerateLongTest(HCondition* condition, - bool invert, - CodeGeneratorARM* codegen) { - DCHECK_EQ(condition->GetLeft()->GetType(), Primitive::kPrimLong); - - const LocationSummary* const locations = condition->GetLocations(); - IfCondition cond = condition->GetCondition(); - IfCondition opposite = condition->GetOppositeCondition(); - - if (invert) { - std::swap(cond, opposite); - } - - std::pair<Condition, Condition> ret; - Location left = locations->InAt(0); - Location right = locations->InAt(1); - - DCHECK(right.IsRegisterPair()); - - switch (cond) { - case kCondEQ: - case kCondNE: - case kCondB: - case kCondBE: - case kCondA: - case kCondAE: - __ cmp(left.AsRegisterPairHigh<Register>(), - ShifterOperand(right.AsRegisterPairHigh<Register>())); - __ it(EQ); - __ cmp(left.AsRegisterPairLow<Register>(), - ShifterOperand(right.AsRegisterPairLow<Register>()), - EQ); - ret = std::make_pair(ARMUnsignedCondition(cond), ARMUnsignedCondition(opposite)); - break; - case kCondLE: - case kCondGT: - if (cond == kCondLE) { - DCHECK_EQ(opposite, kCondGT); - cond = kCondGE; - opposite = kCondLT; - } else { - DCHECK_EQ(cond, kCondGT); - DCHECK_EQ(opposite, kCondLE); - cond = kCondLT; - opposite = kCondGE; - } - - std::swap(left, right); - FALLTHROUGH_INTENDED; - case kCondGE: - case kCondLT: - __ cmp(left.AsRegisterPairLow<Register>(), - ShifterOperand(right.AsRegisterPairLow<Register>())); - __ sbcs(IP, - left.AsRegisterPairHigh<Register>(), - ShifterOperand(right.AsRegisterPairHigh<Register>())); - ret = std::make_pair(ARMCondition(cond), ARMCondition(opposite)); - break; - default: - LOG(FATAL) << "Unreachable"; - UNREACHABLE(); - } - - return ret; -} - -static std::pair<Condition, Condition> GenerateTest(HCondition* condition, - bool invert, - CodeGeneratorARM* codegen) { - const LocationSummary* const locations = condition->GetLocations(); - const Primitive::Type type = condition->GetLeft()->GetType(); - IfCondition cond = condition->GetCondition(); - IfCondition opposite = condition->GetOppositeCondition(); - std::pair<Condition, Condition> ret; - const Location right = locations->InAt(1); - - if (invert) { - std::swap(cond, opposite); - } - - if (type == Primitive::kPrimLong) { - ret = locations->InAt(1).IsConstant() - ? GenerateLongTestConstant(condition, invert, codegen) - : GenerateLongTest(condition, invert, codegen); - } else if (Primitive::IsFloatingPointType(type)) { - GenerateVcmp(condition, codegen); - __ vmstat(); - ret = std::make_pair(ARMFPCondition(cond, condition->IsGtBias()), - ARMFPCondition(opposite, condition->IsGtBias())); - } else { - DCHECK(Primitive::IsIntegralType(type) || type == Primitive::kPrimNot) << type; - - const Register left = locations->InAt(0).AsRegister<Register>(); - - if (right.IsRegister()) { - __ cmp(left, ShifterOperand(right.AsRegister<Register>())); - } else { - DCHECK(right.IsConstant()); - __ CmpConstant(left, CodeGenerator::GetInt32ValueOf(right.GetConstant())); - } - - ret = std::make_pair(ARMCondition(cond), ARMCondition(opposite)); - } - - return ret; -} - -static bool CanGenerateTest(HCondition* condition, ArmAssembler* assembler) { - if (condition->GetLeft()->GetType() == Primitive::kPrimLong) { - const LocationSummary* const locations = condition->GetLocations(); - - if (locations->InAt(1).IsConstant()) { - IfCondition c = condition->GetCondition(); - IfCondition opposite = condition->GetOppositeCondition(); - const int64_t value = AdjustConstantForCondition( - Int64FromConstant(locations->InAt(1).GetConstant()), - &c, - &opposite); - ShifterOperand so; - - if (c < kCondLT || c > kCondGE) { - // Since IT blocks longer than a 16-bit instruction are deprecated by ARMv8, - // we check that the least significant half of the first input to be compared - // is in a low register (the other half is read outside an IT block), and - // the constant fits in an 8-bit unsigned integer, so that a 16-bit CMP - // encoding can be used; 0 is always handled, no matter what registers are - // used by the first input. - if (value != 0 && - (!ArmAssembler::IsLowRegister(locations->InAt(0).AsRegisterPairLow<Register>()) || - !IsUint<8>(Low32Bits(value)))) { - return false; - } - } else if (c == kCondLE || c == kCondGT) { - if (value < std::numeric_limits<int64_t>::max() && - !assembler->ShifterOperandCanHold(kNoRegister, - kNoRegister, - SBC, - High32Bits(value + 1), - kCcSet, - &so)) { - return false; - } - } else if (!assembler->ShifterOperandCanHold(kNoRegister, - kNoRegister, - SBC, - High32Bits(value), - kCcSet, - &so)) { - return false; - } - } - } - - return true; -} - -static void GenerateConditionGeneric(HCondition* cond, CodeGeneratorARM* codegen) { - DCHECK(CanGenerateTest(cond, codegen->GetAssembler())); - - const Register out = cond->GetLocations()->Out().AsRegister<Register>(); - const auto condition = GenerateTest(cond, false, codegen); - - __ mov(out, ShifterOperand(0), AL, kCcKeep); - - if (ArmAssembler::IsLowRegister(out)) { - __ it(condition.first); - __ mov(out, ShifterOperand(1), condition.first); - } else { - Label done_label; - Label* const final_label = codegen->GetFinalLabel(cond, &done_label); - - __ b(final_label, condition.second); - __ LoadImmediate(out, 1); - - if (done_label.IsLinked()) { - __ Bind(&done_label); - } - } -} - -static void GenerateEqualLong(HCondition* cond, CodeGeneratorARM* codegen) { - DCHECK_EQ(cond->GetLeft()->GetType(), Primitive::kPrimLong); - - const LocationSummary* const locations = cond->GetLocations(); - IfCondition condition = cond->GetCondition(); - const Register out = locations->Out().AsRegister<Register>(); - const Location left = locations->InAt(0); - const Location right = locations->InAt(1); - Register left_high = left.AsRegisterPairHigh<Register>(); - Register left_low = left.AsRegisterPairLow<Register>(); - - if (right.IsConstant()) { - IfCondition opposite = cond->GetOppositeCondition(); - const int64_t value = AdjustConstantForCondition(Int64FromConstant(right.GetConstant()), - &condition, - &opposite); - int32_t value_high = -High32Bits(value); - int32_t value_low = -Low32Bits(value); - - // The output uses Location::kNoOutputOverlap. - if (out == left_high) { - std::swap(left_low, left_high); - std::swap(value_low, value_high); - } - - __ AddConstant(out, left_low, value_low); - __ AddConstant(IP, left_high, value_high); - } else { - DCHECK(right.IsRegisterPair()); - __ sub(IP, left_high, ShifterOperand(right.AsRegisterPairHigh<Register>())); - __ sub(out, left_low, ShifterOperand(right.AsRegisterPairLow<Register>())); - } - - // Need to check after calling AdjustConstantForCondition(). - DCHECK(condition == kCondEQ || condition == kCondNE) << condition; - - if (condition == kCondNE && ArmAssembler::IsLowRegister(out)) { - __ orrs(out, out, ShifterOperand(IP)); - __ it(NE); - __ mov(out, ShifterOperand(1), NE); - } else { - __ orr(out, out, ShifterOperand(IP)); - codegen->GenerateConditionWithZero(condition, out, out, IP); - } -} - -static void GenerateLongComparesAndJumps(HCondition* cond, - Label* true_label, - Label* false_label, - CodeGeneratorARM* codegen) { - LocationSummary* locations = cond->GetLocations(); - Location left = locations->InAt(0); - Location right = locations->InAt(1); - IfCondition if_cond = cond->GetCondition(); - - Register left_high = left.AsRegisterPairHigh<Register>(); - Register left_low = left.AsRegisterPairLow<Register>(); - IfCondition true_high_cond = if_cond; - IfCondition false_high_cond = cond->GetOppositeCondition(); - Condition final_condition = ARMUnsignedCondition(if_cond); // unsigned on lower part - - // Set the conditions for the test, remembering that == needs to be - // decided using the low words. - switch (if_cond) { - case kCondEQ: - case kCondNE: - // Nothing to do. - break; - case kCondLT: - false_high_cond = kCondGT; - break; - case kCondLE: - true_high_cond = kCondLT; - break; - case kCondGT: - false_high_cond = kCondLT; - break; - case kCondGE: - true_high_cond = kCondGT; - break; - case kCondB: - false_high_cond = kCondA; - break; - case kCondBE: - true_high_cond = kCondB; - break; - case kCondA: - false_high_cond = kCondB; - break; - case kCondAE: - true_high_cond = kCondA; - break; - } - if (right.IsConstant()) { - int64_t value = right.GetConstant()->AsLongConstant()->GetValue(); - int32_t val_low = Low32Bits(value); - int32_t val_high = High32Bits(value); - - __ CmpConstant(left_high, val_high); - if (if_cond == kCondNE) { - __ b(true_label, ARMCondition(true_high_cond)); - } else if (if_cond == kCondEQ) { - __ b(false_label, ARMCondition(false_high_cond)); - } else { - __ b(true_label, ARMCondition(true_high_cond)); - __ b(false_label, ARMCondition(false_high_cond)); - } - // Must be equal high, so compare the lows. - __ CmpConstant(left_low, val_low); - } else { - Register right_high = right.AsRegisterPairHigh<Register>(); - Register right_low = right.AsRegisterPairLow<Register>(); - - __ cmp(left_high, ShifterOperand(right_high)); - if (if_cond == kCondNE) { - __ b(true_label, ARMCondition(true_high_cond)); - } else if (if_cond == kCondEQ) { - __ b(false_label, ARMCondition(false_high_cond)); - } else { - __ b(true_label, ARMCondition(true_high_cond)); - __ b(false_label, ARMCondition(false_high_cond)); - } - // Must be equal high, so compare the lows. - __ cmp(left_low, ShifterOperand(right_low)); - } - // The last comparison might be unsigned. - // TODO: optimize cases where this is always true/false - __ b(true_label, final_condition); -} - -static void GenerateConditionLong(HCondition* cond, CodeGeneratorARM* codegen) { - DCHECK_EQ(cond->GetLeft()->GetType(), Primitive::kPrimLong); - - const LocationSummary* const locations = cond->GetLocations(); - IfCondition condition = cond->GetCondition(); - const Register out = locations->Out().AsRegister<Register>(); - const Location left = locations->InAt(0); - const Location right = locations->InAt(1); - - if (right.IsConstant()) { - IfCondition opposite = cond->GetOppositeCondition(); - - // Comparisons against 0 are common enough to deserve special attention. - if (AdjustConstantForCondition(Int64FromConstant(right.GetConstant()), - &condition, - &opposite) == 0) { - switch (condition) { - case kCondNE: - case kCondA: - if (ArmAssembler::IsLowRegister(out)) { - // We only care if both input registers are 0 or not. - __ orrs(out, - left.AsRegisterPairLow<Register>(), - ShifterOperand(left.AsRegisterPairHigh<Register>())); - __ it(NE); - __ mov(out, ShifterOperand(1), NE); - return; - } - - FALLTHROUGH_INTENDED; - case kCondEQ: - case kCondBE: - // We only care if both input registers are 0 or not. - __ orr(out, - left.AsRegisterPairLow<Register>(), - ShifterOperand(left.AsRegisterPairHigh<Register>())); - codegen->GenerateConditionWithZero(condition, out, out); - return; - case kCondLT: - case kCondGE: - // We only care about the sign bit. - FALLTHROUGH_INTENDED; - case kCondAE: - case kCondB: - codegen->GenerateConditionWithZero(condition, out, left.AsRegisterPairHigh<Register>()); - return; - case kCondLE: - case kCondGT: - default: - break; - } - } - } - - if ((condition == kCondEQ || condition == kCondNE) && - // If `out` is a low register, then the GenerateConditionGeneric() - // function generates a shorter code sequence that is still branchless. - (!ArmAssembler::IsLowRegister(out) || !CanGenerateTest(cond, codegen->GetAssembler()))) { - GenerateEqualLong(cond, codegen); - return; - } - - if (CanGenerateTest(cond, codegen->GetAssembler())) { - GenerateConditionGeneric(cond, codegen); - return; - } - - // Convert the jumps into the result. - Label done_label; - Label* const final_label = codegen->GetFinalLabel(cond, &done_label); - Label true_label, false_label; - - GenerateLongComparesAndJumps(cond, &true_label, &false_label, codegen); - - // False case: result = 0. - __ Bind(&false_label); - __ mov(out, ShifterOperand(0)); - __ b(final_label); - - // True case: result = 1. - __ Bind(&true_label); - __ mov(out, ShifterOperand(1)); - - if (done_label.IsLinked()) { - __ Bind(&done_label); - } -} - -static void GenerateConditionIntegralOrNonPrimitive(HCondition* cond, CodeGeneratorARM* codegen) { - const Primitive::Type type = cond->GetLeft()->GetType(); - - DCHECK(Primitive::IsIntegralType(type) || type == Primitive::kPrimNot) << type; - - if (type == Primitive::kPrimLong) { - GenerateConditionLong(cond, codegen); - return; - } - - const LocationSummary* const locations = cond->GetLocations(); - IfCondition condition = cond->GetCondition(); - Register in = locations->InAt(0).AsRegister<Register>(); - const Register out = locations->Out().AsRegister<Register>(); - const Location right = cond->GetLocations()->InAt(1); - int64_t value; - - if (right.IsConstant()) { - IfCondition opposite = cond->GetOppositeCondition(); - - value = AdjustConstantForCondition(Int64FromConstant(right.GetConstant()), - &condition, - &opposite); - - // Comparisons against 0 are common enough to deserve special attention. - if (value == 0) { - switch (condition) { - case kCondNE: - case kCondA: - if (ArmAssembler::IsLowRegister(out) && out == in) { - __ cmp(out, ShifterOperand(0)); - __ it(NE); - __ mov(out, ShifterOperand(1), NE); - return; - } - - FALLTHROUGH_INTENDED; - case kCondEQ: - case kCondBE: - case kCondLT: - case kCondGE: - case kCondAE: - case kCondB: - codegen->GenerateConditionWithZero(condition, out, in); - return; - case kCondLE: - case kCondGT: - default: - break; - } - } - } - - if (condition == kCondEQ || condition == kCondNE) { - ShifterOperand operand; - - if (right.IsConstant()) { - operand = ShifterOperand(value); - } else if (out == right.AsRegister<Register>()) { - // Avoid 32-bit instructions if possible. - operand = ShifterOperand(in); - in = right.AsRegister<Register>(); - } else { - operand = ShifterOperand(right.AsRegister<Register>()); - } - - if (condition == kCondNE && ArmAssembler::IsLowRegister(out)) { - __ subs(out, in, operand); - __ it(NE); - __ mov(out, ShifterOperand(1), NE); - } else { - __ sub(out, in, operand); - codegen->GenerateConditionWithZero(condition, out, out); - } - - return; - } - - GenerateConditionGeneric(cond, codegen); -} - -static bool CanEncodeConstantAs8BitImmediate(HConstant* constant) { - const Primitive::Type type = constant->GetType(); - bool ret = false; - - DCHECK(Primitive::IsIntegralType(type) || type == Primitive::kPrimNot) << type; - - if (type == Primitive::kPrimLong) { - const uint64_t value = constant->AsLongConstant()->GetValueAsUint64(); - - ret = IsUint<8>(Low32Bits(value)) && IsUint<8>(High32Bits(value)); - } else { - ret = IsUint<8>(CodeGenerator::GetInt32ValueOf(constant)); - } - - return ret; -} - -static Location Arm8BitEncodableConstantOrRegister(HInstruction* constant) { - DCHECK(!Primitive::IsFloatingPointType(constant->GetType())); - - if (constant->IsConstant() && CanEncodeConstantAs8BitImmediate(constant->AsConstant())) { - return Location::ConstantLocation(constant->AsConstant()); - } - - return Location::RequiresRegister(); -} - -static bool CanGenerateConditionalMove(const Location& out, const Location& src) { - // Since IT blocks longer than a 16-bit instruction are deprecated by ARMv8, - // we check that we are not dealing with floating-point output (there is no - // 16-bit VMOV encoding). - if (!out.IsRegister() && !out.IsRegisterPair()) { - return false; - } - - // For constants, we also check that the output is in one or two low registers, - // and that the constants fit in an 8-bit unsigned integer, so that a 16-bit - // MOV encoding can be used. - if (src.IsConstant()) { - if (!CanEncodeConstantAs8BitImmediate(src.GetConstant())) { - return false; - } - - if (out.IsRegister()) { - if (!ArmAssembler::IsLowRegister(out.AsRegister<Register>())) { - return false; - } - } else { - DCHECK(out.IsRegisterPair()); - - if (!ArmAssembler::IsLowRegister(out.AsRegisterPairHigh<Register>())) { - return false; - } - } - } - - return true; -} - -#undef __ -// NOLINT on __ macro to suppress wrong warning/fix (misc-macro-parentheses) from clang-tidy. -#define __ down_cast<ArmAssembler*>(GetAssembler())-> // NOLINT - -Label* CodeGeneratorARM::GetFinalLabel(HInstruction* instruction, Label* final_label) { - DCHECK(!instruction->IsControlFlow() && !instruction->IsSuspendCheck()); - DCHECK(!instruction->IsInvoke() || !instruction->GetLocations()->CanCall()); - - const HBasicBlock* const block = instruction->GetBlock(); - const HLoopInformation* const info = block->GetLoopInformation(); - HInstruction* const next = instruction->GetNext(); - - // Avoid a branch to a branch. - if (next->IsGoto() && (info == nullptr || - !info->IsBackEdge(*block) || - !info->HasSuspendCheck())) { - final_label = GetLabelOf(next->AsGoto()->GetSuccessor()); - } - - return final_label; -} - -void CodeGeneratorARM::DumpCoreRegister(std::ostream& stream, int reg) const { - stream << Register(reg); -} - -void CodeGeneratorARM::DumpFloatingPointRegister(std::ostream& stream, int reg) const { - stream << SRegister(reg); -} - -size_t CodeGeneratorARM::SaveCoreRegister(size_t stack_index, uint32_t reg_id) { - __ StoreToOffset(kStoreWord, static_cast<Register>(reg_id), SP, stack_index); - return kArmWordSize; -} - -size_t CodeGeneratorARM::RestoreCoreRegister(size_t stack_index, uint32_t reg_id) { - __ LoadFromOffset(kLoadWord, static_cast<Register>(reg_id), SP, stack_index); - return kArmWordSize; -} - -size_t CodeGeneratorARM::SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) { - __ StoreSToOffset(static_cast<SRegister>(reg_id), SP, stack_index); - return kArmWordSize; -} - -size_t CodeGeneratorARM::RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) { - __ LoadSFromOffset(static_cast<SRegister>(reg_id), SP, stack_index); - return kArmWordSize; -} - -CodeGeneratorARM::CodeGeneratorARM(HGraph* graph, - const ArmInstructionSetFeatures& isa_features, - const CompilerOptions& compiler_options, - OptimizingCompilerStats* stats) - : CodeGenerator(graph, - kNumberOfCoreRegisters, - kNumberOfSRegisters, - kNumberOfRegisterPairs, - ComputeRegisterMask(reinterpret_cast<const int*>(kCoreCalleeSaves), - arraysize(kCoreCalleeSaves)), - ComputeRegisterMask(reinterpret_cast<const int*>(kFpuCalleeSaves), - arraysize(kFpuCalleeSaves)), - compiler_options, - stats), - block_labels_(nullptr), - location_builder_(graph, this), - instruction_visitor_(graph, this), - move_resolver_(graph->GetArena(), this), - assembler_(graph->GetArena()), - isa_features_(isa_features), - uint32_literals_(std::less<uint32_t>(), - graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), - pc_relative_method_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), - method_bss_entry_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), - pc_relative_type_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), - type_bss_entry_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), - pc_relative_string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), - baker_read_barrier_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), - jit_string_patches_(StringReferenceValueComparator(), - graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), - jit_class_patches_(TypeReferenceValueComparator(), - graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) { - // Always save the LR register to mimic Quick. - AddAllocatedRegister(Location::RegisterLocation(LR)); -} - -void CodeGeneratorARM::Finalize(CodeAllocator* allocator) { - // Ensure that we fix up branches and literal loads and emit the literal pool. - __ FinalizeCode(); - - // Adjust native pc offsets in stack maps. - for (size_t i = 0, num = stack_map_stream_.GetNumberOfStackMaps(); i != num; ++i) { - uint32_t old_position = - stack_map_stream_.GetStackMap(i).native_pc_code_offset.Uint32Value(kThumb2); - uint32_t new_position = __ GetAdjustedPosition(old_position); - stack_map_stream_.SetStackMapNativePcOffset(i, new_position); - } - // Adjust pc offsets for the disassembly information. - if (disasm_info_ != nullptr) { - GeneratedCodeInterval* frame_entry_interval = disasm_info_->GetFrameEntryInterval(); - frame_entry_interval->start = __ GetAdjustedPosition(frame_entry_interval->start); - frame_entry_interval->end = __ GetAdjustedPosition(frame_entry_interval->end); - for (auto& it : *disasm_info_->GetInstructionIntervals()) { - it.second.start = __ GetAdjustedPosition(it.second.start); - it.second.end = __ GetAdjustedPosition(it.second.end); - } - for (auto& it : *disasm_info_->GetSlowPathIntervals()) { - it.code_interval.start = __ GetAdjustedPosition(it.code_interval.start); - it.code_interval.end = __ GetAdjustedPosition(it.code_interval.end); - } - } - - CodeGenerator::Finalize(allocator); -} - -void CodeGeneratorARM::SetupBlockedRegisters() const { - // Stack register, LR and PC are always reserved. - blocked_core_registers_[SP] = true; - blocked_core_registers_[LR] = true; - blocked_core_registers_[PC] = true; - - // Reserve thread register. - blocked_core_registers_[TR] = true; - - // Reserve temp register. - blocked_core_registers_[IP] = true; - - if (GetGraph()->IsDebuggable()) { - // Stubs do not save callee-save floating point registers. If the graph - // is debuggable, we need to deal with these registers differently. For - // now, just block them. - for (size_t i = 0; i < arraysize(kFpuCalleeSaves); ++i) { - blocked_fpu_registers_[kFpuCalleeSaves[i]] = true; - } - } -} - -InstructionCodeGeneratorARM::InstructionCodeGeneratorARM(HGraph* graph, CodeGeneratorARM* codegen) - : InstructionCodeGenerator(graph, codegen), - assembler_(codegen->GetAssembler()), - codegen_(codegen) {} - -void CodeGeneratorARM::ComputeSpillMask() { - core_spill_mask_ = allocated_registers_.GetCoreRegisters() & core_callee_save_mask_; - DCHECK_NE(core_spill_mask_, 0u) << "At least the return address register must be saved"; - // There is no easy instruction to restore just the PC on thumb2. We spill and - // restore another arbitrary register. - core_spill_mask_ |= (1 << kCoreAlwaysSpillRegister); - fpu_spill_mask_ = allocated_registers_.GetFloatingPointRegisters() & fpu_callee_save_mask_; - // We use vpush and vpop for saving and restoring floating point registers, which take - // a SRegister and the number of registers to save/restore after that SRegister. We - // therefore update the `fpu_spill_mask_` to also contain those registers not allocated, - // but in the range. - if (fpu_spill_mask_ != 0) { - uint32_t least_significant_bit = LeastSignificantBit(fpu_spill_mask_); - uint32_t most_significant_bit = MostSignificantBit(fpu_spill_mask_); - for (uint32_t i = least_significant_bit + 1 ; i < most_significant_bit; ++i) { - fpu_spill_mask_ |= (1 << i); - } - } -} - -static dwarf::Reg DWARFReg(Register reg) { - return dwarf::Reg::ArmCore(static_cast<int>(reg)); -} - -static dwarf::Reg DWARFReg(SRegister reg) { - return dwarf::Reg::ArmFp(static_cast<int>(reg)); -} - -void CodeGeneratorARM::GenerateFrameEntry() { - bool skip_overflow_check = - IsLeafMethod() && !FrameNeedsStackCheck(GetFrameSize(), InstructionSet::kArm); - DCHECK(GetCompilerOptions().GetImplicitStackOverflowChecks()); - __ Bind(&frame_entry_label_); - - if (HasEmptyFrame()) { - return; - } - - if (!skip_overflow_check) { - __ AddConstant(IP, SP, -static_cast<int32_t>(GetStackOverflowReservedBytes(kArm))); - __ LoadFromOffset(kLoadWord, IP, IP, 0); - RecordPcInfo(nullptr, 0); - } - - __ PushList(core_spill_mask_); - __ cfi().AdjustCFAOffset(kArmWordSize * POPCOUNT(core_spill_mask_)); - __ cfi().RelOffsetForMany(DWARFReg(kMethodRegisterArgument), 0, core_spill_mask_, kArmWordSize); - if (fpu_spill_mask_ != 0) { - SRegister start_register = SRegister(LeastSignificantBit(fpu_spill_mask_)); - __ vpushs(start_register, POPCOUNT(fpu_spill_mask_)); - __ cfi().AdjustCFAOffset(kArmWordSize * POPCOUNT(fpu_spill_mask_)); - __ cfi().RelOffsetForMany(DWARFReg(S0), 0, fpu_spill_mask_, kArmWordSize); - } - - int adjust = GetFrameSize() - FrameEntrySpillSize(); - __ AddConstant(SP, -adjust); - __ cfi().AdjustCFAOffset(adjust); - - // Save the current method if we need it. Note that we do not - // do this in HCurrentMethod, as the instruction might have been removed - // in the SSA graph. - if (RequiresCurrentMethod()) { - __ StoreToOffset(kStoreWord, kMethodRegisterArgument, SP, 0); - } - - if (GetGraph()->HasShouldDeoptimizeFlag()) { - // Initialize should_deoptimize flag to 0. - __ mov(IP, ShifterOperand(0)); - __ StoreToOffset(kStoreWord, IP, SP, GetStackOffsetOfShouldDeoptimizeFlag()); - } -} - -void CodeGeneratorARM::GenerateFrameExit() { - if (HasEmptyFrame()) { - __ bx(LR); - return; - } - __ cfi().RememberState(); - int adjust = GetFrameSize() - FrameEntrySpillSize(); - __ AddConstant(SP, adjust); - __ cfi().AdjustCFAOffset(-adjust); - if (fpu_spill_mask_ != 0) { - SRegister start_register = SRegister(LeastSignificantBit(fpu_spill_mask_)); - __ vpops(start_register, POPCOUNT(fpu_spill_mask_)); - __ cfi().AdjustCFAOffset(-static_cast<int>(kArmPointerSize) * POPCOUNT(fpu_spill_mask_)); - __ cfi().RestoreMany(DWARFReg(SRegister(0)), fpu_spill_mask_); - } - // Pop LR into PC to return. - DCHECK_NE(core_spill_mask_ & (1 << LR), 0U); - uint32_t pop_mask = (core_spill_mask_ & (~(1 << LR))) | 1 << PC; - __ PopList(pop_mask); - __ cfi().RestoreState(); - __ cfi().DefCFAOffset(GetFrameSize()); -} - -void CodeGeneratorARM::Bind(HBasicBlock* block) { - Label* label = GetLabelOf(block); - __ BindTrackedLabel(label); -} - -Location InvokeDexCallingConventionVisitorARM::GetNextLocation(Primitive::Type type) { - switch (type) { - case Primitive::kPrimBoolean: - case Primitive::kPrimByte: - case Primitive::kPrimChar: - case Primitive::kPrimShort: - case Primitive::kPrimInt: - case Primitive::kPrimNot: { - uint32_t index = gp_index_++; - uint32_t stack_index = stack_index_++; - if (index < calling_convention.GetNumberOfRegisters()) { - return Location::RegisterLocation(calling_convention.GetRegisterAt(index)); - } else { - return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index)); - } - } - - case Primitive::kPrimLong: { - uint32_t index = gp_index_; - uint32_t stack_index = stack_index_; - gp_index_ += 2; - stack_index_ += 2; - if (index + 1 < calling_convention.GetNumberOfRegisters()) { - if (calling_convention.GetRegisterAt(index) == R1) { - // Skip R1, and use R2_R3 instead. - gp_index_++; - index++; - } - } - if (index + 1 < calling_convention.GetNumberOfRegisters()) { - DCHECK_EQ(calling_convention.GetRegisterAt(index) + 1, - calling_convention.GetRegisterAt(index + 1)); - - return Location::RegisterPairLocation(calling_convention.GetRegisterAt(index), - calling_convention.GetRegisterAt(index + 1)); - } else { - return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index)); - } - } - - case Primitive::kPrimFloat: { - uint32_t stack_index = stack_index_++; - if (float_index_ % 2 == 0) { - float_index_ = std::max(double_index_, float_index_); - } - if (float_index_ < calling_convention.GetNumberOfFpuRegisters()) { - return Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(float_index_++)); - } else { - return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index)); - } - } - - case Primitive::kPrimDouble: { - double_index_ = std::max(double_index_, RoundUp(float_index_, 2)); - uint32_t stack_index = stack_index_; - stack_index_ += 2; - if (double_index_ + 1 < calling_convention.GetNumberOfFpuRegisters()) { - uint32_t index = double_index_; - double_index_ += 2; - Location result = Location::FpuRegisterPairLocation( - calling_convention.GetFpuRegisterAt(index), - calling_convention.GetFpuRegisterAt(index + 1)); - DCHECK(ExpectedPairLayout(result)); - return result; - } else { - return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index)); - } - } - - case Primitive::kPrimVoid: - LOG(FATAL) << "Unexpected parameter type " << type; - break; - } - return Location::NoLocation(); -} - -Location InvokeDexCallingConventionVisitorARM::GetReturnLocation(Primitive::Type type) const { - switch (type) { - case Primitive::kPrimBoolean: - case Primitive::kPrimByte: - case Primitive::kPrimChar: - case Primitive::kPrimShort: - case Primitive::kPrimInt: - case Primitive::kPrimNot: { - return Location::RegisterLocation(R0); - } - - case Primitive::kPrimFloat: { - return Location::FpuRegisterLocation(S0); - } - - case Primitive::kPrimLong: { - return Location::RegisterPairLocation(R0, R1); - } - - case Primitive::kPrimDouble: { - return Location::FpuRegisterPairLocation(S0, S1); - } - - case Primitive::kPrimVoid: - return Location::NoLocation(); - } - - UNREACHABLE(); -} - -Location InvokeDexCallingConventionVisitorARM::GetMethodLocation() const { - return Location::RegisterLocation(kMethodRegisterArgument); -} - -void CodeGeneratorARM::Move32(Location destination, Location source) { - if (source.Equals(destination)) { - return; - } - if (destination.IsRegister()) { - if (source.IsRegister()) { - __ Mov(destination.AsRegister<Register>(), source.AsRegister<Register>()); - } else if (source.IsFpuRegister()) { - __ vmovrs(destination.AsRegister<Register>(), source.AsFpuRegister<SRegister>()); - } else { - __ LoadFromOffset(kLoadWord, destination.AsRegister<Register>(), SP, source.GetStackIndex()); - } - } else if (destination.IsFpuRegister()) { - if (source.IsRegister()) { - __ vmovsr(destination.AsFpuRegister<SRegister>(), source.AsRegister<Register>()); - } else if (source.IsFpuRegister()) { - __ vmovs(destination.AsFpuRegister<SRegister>(), source.AsFpuRegister<SRegister>()); - } else { - __ LoadSFromOffset(destination.AsFpuRegister<SRegister>(), SP, source.GetStackIndex()); - } - } else { - DCHECK(destination.IsStackSlot()) << destination; - if (source.IsRegister()) { - __ StoreToOffset(kStoreWord, source.AsRegister<Register>(), SP, destination.GetStackIndex()); - } else if (source.IsFpuRegister()) { - __ StoreSToOffset(source.AsFpuRegister<SRegister>(), SP, destination.GetStackIndex()); - } else { - DCHECK(source.IsStackSlot()) << source; - __ LoadFromOffset(kLoadWord, IP, SP, source.GetStackIndex()); - __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); - } - } -} - -void CodeGeneratorARM::Move64(Location destination, Location source) { - if (source.Equals(destination)) { - return; - } - if (destination.IsRegisterPair()) { - if (source.IsRegisterPair()) { - EmitParallelMoves( - Location::RegisterLocation(source.AsRegisterPairHigh<Register>()), - Location::RegisterLocation(destination.AsRegisterPairHigh<Register>()), - Primitive::kPrimInt, - Location::RegisterLocation(source.AsRegisterPairLow<Register>()), - Location::RegisterLocation(destination.AsRegisterPairLow<Register>()), - Primitive::kPrimInt); - } else if (source.IsFpuRegister()) { - UNIMPLEMENTED(FATAL); - } else if (source.IsFpuRegisterPair()) { - __ vmovrrd(destination.AsRegisterPairLow<Register>(), - destination.AsRegisterPairHigh<Register>(), - FromLowSToD(source.AsFpuRegisterPairLow<SRegister>())); - } else { - DCHECK(source.IsDoubleStackSlot()); - DCHECK(ExpectedPairLayout(destination)); - __ LoadFromOffset(kLoadWordPair, destination.AsRegisterPairLow<Register>(), - SP, source.GetStackIndex()); - } - } else if (destination.IsFpuRegisterPair()) { - if (source.IsDoubleStackSlot()) { - __ LoadDFromOffset(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), - SP, - source.GetStackIndex()); - } else if (source.IsRegisterPair()) { - __ vmovdrr(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), - source.AsRegisterPairLow<Register>(), - source.AsRegisterPairHigh<Register>()); - } else { - UNIMPLEMENTED(FATAL); - } - } else { - DCHECK(destination.IsDoubleStackSlot()); - if (source.IsRegisterPair()) { - // No conflict possible, so just do the moves. - if (source.AsRegisterPairLow<Register>() == R1) { - DCHECK_EQ(source.AsRegisterPairHigh<Register>(), R2); - __ StoreToOffset(kStoreWord, R1, SP, destination.GetStackIndex()); - __ StoreToOffset(kStoreWord, R2, SP, destination.GetHighStackIndex(kArmWordSize)); - } else { - __ StoreToOffset(kStoreWordPair, source.AsRegisterPairLow<Register>(), - SP, destination.GetStackIndex()); - } - } else if (source.IsFpuRegisterPair()) { - __ StoreDToOffset(FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()), - SP, - destination.GetStackIndex()); - } else { - DCHECK(source.IsDoubleStackSlot()); - EmitParallelMoves( - Location::StackSlot(source.GetStackIndex()), - Location::StackSlot(destination.GetStackIndex()), - Primitive::kPrimInt, - Location::StackSlot(source.GetHighStackIndex(kArmWordSize)), - Location::StackSlot(destination.GetHighStackIndex(kArmWordSize)), - Primitive::kPrimInt); - } - } -} - -void CodeGeneratorARM::MoveConstant(Location location, int32_t value) { - DCHECK(location.IsRegister()); - __ LoadImmediate(location.AsRegister<Register>(), value); -} - -void CodeGeneratorARM::MoveLocation(Location dst, Location src, Primitive::Type dst_type) { - HParallelMove move(GetGraph()->GetArena()); - move.AddMove(src, dst, dst_type, nullptr); - GetMoveResolver()->EmitNativeCode(&move); -} - -void CodeGeneratorARM::AddLocationAsTemp(Location location, LocationSummary* locations) { - if (location.IsRegister()) { - locations->AddTemp(location); - } else if (location.IsRegisterPair()) { - locations->AddTemp(Location::RegisterLocation(location.AsRegisterPairLow<Register>())); - locations->AddTemp(Location::RegisterLocation(location.AsRegisterPairHigh<Register>())); - } else { - UNIMPLEMENTED(FATAL) << "AddLocationAsTemp not implemented for location " << location; - } -} - -void CodeGeneratorARM::InvokeRuntime(QuickEntrypointEnum entrypoint, - HInstruction* instruction, - uint32_t dex_pc, - SlowPathCode* slow_path) { - ValidateInvokeRuntime(entrypoint, instruction, slow_path); - GenerateInvokeRuntime(GetThreadOffset<kArmPointerSize>(entrypoint).Int32Value()); - if (EntrypointRequiresStackMap(entrypoint)) { - RecordPcInfo(instruction, dex_pc, slow_path); - } -} - -void CodeGeneratorARM::InvokeRuntimeWithoutRecordingPcInfo(int32_t entry_point_offset, - HInstruction* instruction, - SlowPathCode* slow_path) { - ValidateInvokeRuntimeWithoutRecordingPcInfo(instruction, slow_path); - GenerateInvokeRuntime(entry_point_offset); -} - -void CodeGeneratorARM::GenerateInvokeRuntime(int32_t entry_point_offset) { - __ LoadFromOffset(kLoadWord, LR, TR, entry_point_offset); - __ blx(LR); -} - -void InstructionCodeGeneratorARM::HandleGoto(HInstruction* got, HBasicBlock* successor) { - DCHECK(!successor->IsExitBlock()); - - HBasicBlock* block = got->GetBlock(); - HInstruction* previous = got->GetPrevious(); - - HLoopInformation* info = block->GetLoopInformation(); - if (info != nullptr && info->IsBackEdge(*block) && info->HasSuspendCheck()) { - codegen_->ClearSpillSlotsFromLoopPhisInStackMap(info->GetSuspendCheck()); - GenerateSuspendCheck(info->GetSuspendCheck(), successor); - return; - } - - if (block->IsEntryBlock() && (previous != nullptr) && previous->IsSuspendCheck()) { - GenerateSuspendCheck(previous->AsSuspendCheck(), nullptr); - } - if (!codegen_->GoesToNextBlock(got->GetBlock(), successor)) { - __ b(codegen_->GetLabelOf(successor)); - } -} - -void LocationsBuilderARM::VisitGoto(HGoto* got) { - got->SetLocations(nullptr); -} - -void InstructionCodeGeneratorARM::VisitGoto(HGoto* got) { - HandleGoto(got, got->GetSuccessor()); -} - -void LocationsBuilderARM::VisitTryBoundary(HTryBoundary* try_boundary) { - try_boundary->SetLocations(nullptr); -} - -void InstructionCodeGeneratorARM::VisitTryBoundary(HTryBoundary* try_boundary) { - HBasicBlock* successor = try_boundary->GetNormalFlowSuccessor(); - if (!successor->IsExitBlock()) { - HandleGoto(try_boundary, successor); - } -} - -void LocationsBuilderARM::VisitExit(HExit* exit) { - exit->SetLocations(nullptr); -} - -void InstructionCodeGeneratorARM::VisitExit(HExit* exit ATTRIBUTE_UNUSED) { -} - -void InstructionCodeGeneratorARM::GenerateCompareTestAndBranch(HCondition* condition, - Label* true_target_in, - Label* false_target_in) { - if (CanGenerateTest(condition, codegen_->GetAssembler())) { - Label* non_fallthrough_target; - bool invert; - bool emit_both_branches; - - if (true_target_in == nullptr) { - // The true target is fallthrough. - DCHECK(false_target_in != nullptr); - non_fallthrough_target = false_target_in; - invert = true; - emit_both_branches = false; - } else { - // Either the false target is fallthrough, or there is no fallthrough - // and both branches must be emitted. - non_fallthrough_target = true_target_in; - invert = false; - emit_both_branches = (false_target_in != nullptr); - } - - const auto cond = GenerateTest(condition, invert, codegen_); - - __ b(non_fallthrough_target, cond.first); - - if (emit_both_branches) { - // No target falls through, we need to branch. - __ b(false_target_in); - } - - return; - } - - // Generated branching requires both targets to be explicit. If either of the - // targets is nullptr (fallthrough) use and bind `fallthrough_target` instead. - Label fallthrough_target; - Label* true_target = true_target_in == nullptr ? &fallthrough_target : true_target_in; - Label* false_target = false_target_in == nullptr ? &fallthrough_target : false_target_in; - - DCHECK_EQ(condition->InputAt(0)->GetType(), Primitive::kPrimLong); - GenerateLongComparesAndJumps(condition, true_target, false_target, codegen_); - - if (false_target != &fallthrough_target) { - __ b(false_target); - } - - if (fallthrough_target.IsLinked()) { - __ Bind(&fallthrough_target); - } -} - -void InstructionCodeGeneratorARM::GenerateTestAndBranch(HInstruction* instruction, - size_t condition_input_index, - Label* true_target, - Label* false_target) { - HInstruction* cond = instruction->InputAt(condition_input_index); - - if (true_target == nullptr && false_target == nullptr) { - // Nothing to do. The code always falls through. - return; - } else if (cond->IsIntConstant()) { - // Constant condition, statically compared against "true" (integer value 1). - if (cond->AsIntConstant()->IsTrue()) { - if (true_target != nullptr) { - __ b(true_target); - } - } else { - DCHECK(cond->AsIntConstant()->IsFalse()) << cond->AsIntConstant()->GetValue(); - if (false_target != nullptr) { - __ b(false_target); - } - } - return; - } - - // The following code generates these patterns: - // (1) true_target == nullptr && false_target != nullptr - // - opposite condition true => branch to false_target - // (2) true_target != nullptr && false_target == nullptr - // - condition true => branch to true_target - // (3) true_target != nullptr && false_target != nullptr - // - condition true => branch to true_target - // - branch to false_target - if (IsBooleanValueOrMaterializedCondition(cond)) { - // Condition has been materialized, compare the output to 0. - Location cond_val = instruction->GetLocations()->InAt(condition_input_index); - DCHECK(cond_val.IsRegister()); - if (true_target == nullptr) { - __ CompareAndBranchIfZero(cond_val.AsRegister<Register>(), false_target); - } else { - __ CompareAndBranchIfNonZero(cond_val.AsRegister<Register>(), true_target); - } - } else { - // Condition has not been materialized. Use its inputs as the comparison and - // its condition as the branch condition. - HCondition* condition = cond->AsCondition(); - - // If this is a long or FP comparison that has been folded into - // the HCondition, generate the comparison directly. - Primitive::Type type = condition->InputAt(0)->GetType(); - if (type == Primitive::kPrimLong || Primitive::IsFloatingPointType(type)) { - GenerateCompareTestAndBranch(condition, true_target, false_target); - return; - } - - Label* non_fallthrough_target; - Condition arm_cond; - LocationSummary* locations = cond->GetLocations(); - DCHECK(locations->InAt(0).IsRegister()); - Register left = locations->InAt(0).AsRegister<Register>(); - Location right = locations->InAt(1); - - if (true_target == nullptr) { - arm_cond = ARMCondition(condition->GetOppositeCondition()); - non_fallthrough_target = false_target; - } else { - arm_cond = ARMCondition(condition->GetCondition()); - non_fallthrough_target = true_target; - } - - if (right.IsConstant() && (arm_cond == NE || arm_cond == EQ) && - CodeGenerator::GetInt32ValueOf(right.GetConstant()) == 0) { - if (arm_cond == EQ) { - __ CompareAndBranchIfZero(left, non_fallthrough_target); - } else { - DCHECK_EQ(arm_cond, NE); - __ CompareAndBranchIfNonZero(left, non_fallthrough_target); - } - } else { - if (right.IsRegister()) { - __ cmp(left, ShifterOperand(right.AsRegister<Register>())); - } else { - DCHECK(right.IsConstant()); - __ CmpConstant(left, CodeGenerator::GetInt32ValueOf(right.GetConstant())); - } - - __ b(non_fallthrough_target, arm_cond); - } - } - - // If neither branch falls through (case 3), the conditional branch to `true_target` - // was already emitted (case 2) and we need to emit a jump to `false_target`. - if (true_target != nullptr && false_target != nullptr) { - __ b(false_target); - } -} - -void LocationsBuilderARM::VisitIf(HIf* if_instr) { - LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(if_instr); - if (IsBooleanValueOrMaterializedCondition(if_instr->InputAt(0))) { - locations->SetInAt(0, Location::RequiresRegister()); - } -} - -void InstructionCodeGeneratorARM::VisitIf(HIf* if_instr) { - HBasicBlock* true_successor = if_instr->IfTrueSuccessor(); - HBasicBlock* false_successor = if_instr->IfFalseSuccessor(); - Label* true_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), true_successor) ? - nullptr : codegen_->GetLabelOf(true_successor); - Label* false_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), false_successor) ? - nullptr : codegen_->GetLabelOf(false_successor); - GenerateTestAndBranch(if_instr, /* condition_input_index */ 0, true_target, false_target); -} - -void LocationsBuilderARM::VisitDeoptimize(HDeoptimize* deoptimize) { - LocationSummary* locations = new (GetGraph()->GetArena()) - LocationSummary(deoptimize, LocationSummary::kCallOnSlowPath); - InvokeRuntimeCallingConvention calling_convention; - RegisterSet caller_saves = RegisterSet::Empty(); - caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); - locations->SetCustomSlowPathCallerSaves(caller_saves); - if (IsBooleanValueOrMaterializedCondition(deoptimize->InputAt(0))) { - locations->SetInAt(0, Location::RequiresRegister()); - } -} - -void InstructionCodeGeneratorARM::VisitDeoptimize(HDeoptimize* deoptimize) { - SlowPathCodeARM* slow_path = deopt_slow_paths_.NewSlowPath<DeoptimizationSlowPathARM>(deoptimize); - GenerateTestAndBranch(deoptimize, - /* condition_input_index */ 0, - slow_path->GetEntryLabel(), - /* false_target */ nullptr); -} - -void LocationsBuilderARM::VisitShouldDeoptimizeFlag(HShouldDeoptimizeFlag* flag) { - LocationSummary* locations = new (GetGraph()->GetArena()) - LocationSummary(flag, LocationSummary::kNoCall); - locations->SetOut(Location::RequiresRegister()); -} - -void InstructionCodeGeneratorARM::VisitShouldDeoptimizeFlag(HShouldDeoptimizeFlag* flag) { - __ LoadFromOffset(kLoadWord, - flag->GetLocations()->Out().AsRegister<Register>(), - SP, - codegen_->GetStackOffsetOfShouldDeoptimizeFlag()); -} - -void LocationsBuilderARM::VisitSelect(HSelect* select) { - LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(select); - const bool is_floating_point = Primitive::IsFloatingPointType(select->GetType()); - - if (is_floating_point) { - locations->SetInAt(0, Location::RequiresFpuRegister()); - locations->SetInAt(1, Location::FpuRegisterOrConstant(select->GetTrueValue())); - } else { - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Arm8BitEncodableConstantOrRegister(select->GetTrueValue())); - } - - if (IsBooleanValueOrMaterializedCondition(select->GetCondition())) { - locations->SetInAt(2, Location::RegisterOrConstant(select->GetCondition())); - // The code generator handles overlap with the values, but not with the condition. - locations->SetOut(Location::SameAsFirstInput()); - } else if (is_floating_point) { - locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); - } else { - if (!locations->InAt(1).IsConstant()) { - locations->SetInAt(0, Arm8BitEncodableConstantOrRegister(select->GetFalseValue())); - } - - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); - } -} - -void InstructionCodeGeneratorARM::VisitSelect(HSelect* select) { - HInstruction* const condition = select->GetCondition(); - const LocationSummary* const locations = select->GetLocations(); - const Primitive::Type type = select->GetType(); - const Location first = locations->InAt(0); - const Location out = locations->Out(); - const Location second = locations->InAt(1); - Location src; - - if (condition->IsIntConstant()) { - if (condition->AsIntConstant()->IsFalse()) { - src = first; - } else { - src = second; - } - - codegen_->MoveLocation(out, src, type); - return; - } - - if (!Primitive::IsFloatingPointType(type) && - (IsBooleanValueOrMaterializedCondition(condition) || - CanGenerateTest(condition->AsCondition(), codegen_->GetAssembler()))) { - bool invert = false; - - if (out.Equals(second)) { - src = first; - invert = true; - } else if (out.Equals(first)) { - src = second; - } else if (second.IsConstant()) { - DCHECK(CanEncodeConstantAs8BitImmediate(second.GetConstant())); - src = second; - } else if (first.IsConstant()) { - DCHECK(CanEncodeConstantAs8BitImmediate(first.GetConstant())); - src = first; - invert = true; - } else { - src = second; - } - - if (CanGenerateConditionalMove(out, src)) { - if (!out.Equals(first) && !out.Equals(second)) { - codegen_->MoveLocation(out, src.Equals(first) ? second : first, type); - } - - std::pair<Condition, Condition> cond; - - if (IsBooleanValueOrMaterializedCondition(condition)) { - __ CmpConstant(locations->InAt(2).AsRegister<Register>(), 0); - cond = invert ? std::make_pair(EQ, NE) : std::make_pair(NE, EQ); - } else { - cond = GenerateTest(condition->AsCondition(), invert, codegen_); - } - - if (out.IsRegister()) { - ShifterOperand operand; - - if (src.IsConstant()) { - operand = ShifterOperand(CodeGenerator::GetInt32ValueOf(src.GetConstant())); - } else { - DCHECK(src.IsRegister()); - operand = ShifterOperand(src.AsRegister<Register>()); - } - - __ it(cond.first); - __ mov(out.AsRegister<Register>(), operand, cond.first); - } else { - DCHECK(out.IsRegisterPair()); - - ShifterOperand operand_high; - ShifterOperand operand_low; - - if (src.IsConstant()) { - const int64_t value = src.GetConstant()->AsLongConstant()->GetValue(); - - operand_high = ShifterOperand(High32Bits(value)); - operand_low = ShifterOperand(Low32Bits(value)); - } else { - DCHECK(src.IsRegisterPair()); - operand_high = ShifterOperand(src.AsRegisterPairHigh<Register>()); - operand_low = ShifterOperand(src.AsRegisterPairLow<Register>()); - } - - __ it(cond.first); - __ mov(out.AsRegisterPairLow<Register>(), operand_low, cond.first); - __ it(cond.first); - __ mov(out.AsRegisterPairHigh<Register>(), operand_high, cond.first); - } - - return; - } - } - - Label* false_target = nullptr; - Label* true_target = nullptr; - Label select_end; - Label* target = codegen_->GetFinalLabel(select, &select_end); - - if (out.Equals(second)) { - true_target = target; - src = first; - } else { - false_target = target; - src = second; - - if (!out.Equals(first)) { - codegen_->MoveLocation(out, first, type); - } - } - - GenerateTestAndBranch(select, 2, true_target, false_target); - codegen_->MoveLocation(out, src, type); - - if (select_end.IsLinked()) { - __ Bind(&select_end); - } -} - -void LocationsBuilderARM::VisitNativeDebugInfo(HNativeDebugInfo* info) { - new (GetGraph()->GetArena()) LocationSummary(info); -} - -void InstructionCodeGeneratorARM::VisitNativeDebugInfo(HNativeDebugInfo*) { - // MaybeRecordNativeDebugInfo is already called implicitly in CodeGenerator::Compile. -} - -void CodeGeneratorARM::GenerateNop() { - __ nop(); -} - -// `temp` is an extra temporary register that is used for some conditions; -// callers may not specify it, in which case the method will use a scratch -// register instead. -void CodeGeneratorARM::GenerateConditionWithZero(IfCondition condition, - Register out, - Register in, - Register temp) { - switch (condition) { - case kCondEQ: - // x <= 0 iff x == 0 when the comparison is unsigned. - case kCondBE: - if (temp == kNoRegister || (ArmAssembler::IsLowRegister(out) && out != in)) { - temp = out; - } - - // Avoid 32-bit instructions if possible; note that `in` and `temp` must be - // different as well. - if (ArmAssembler::IsLowRegister(in) && ArmAssembler::IsLowRegister(temp) && in != temp) { - // temp = - in; only 0 sets the carry flag. - __ rsbs(temp, in, ShifterOperand(0)); - - if (out == in) { - std::swap(in, temp); - } - - // out = - in + in + carry = carry - __ adc(out, temp, ShifterOperand(in)); - } else { - // If `in` is 0, then it has 32 leading zeros, and less than that otherwise. - __ clz(out, in); - // Any number less than 32 logically shifted right by 5 bits results in 0; - // the same operation on 32 yields 1. - __ Lsr(out, out, 5); - } - - break; - case kCondNE: - // x > 0 iff x != 0 when the comparison is unsigned. - case kCondA: - if (out == in) { - if (temp == kNoRegister || in == temp) { - temp = IP; - } - } else if (temp == kNoRegister || !ArmAssembler::IsLowRegister(temp)) { - temp = out; - } - - // temp = in - 1; only 0 does not set the carry flag. - __ subs(temp, in, ShifterOperand(1)); - // out = in + ~temp + carry = in + (-(in - 1) - 1) + carry = in - in + 1 - 1 + carry = carry - __ sbc(out, in, ShifterOperand(temp)); - break; - case kCondGE: - __ mvn(out, ShifterOperand(in)); - in = out; - FALLTHROUGH_INTENDED; - case kCondLT: - // We only care about the sign bit. - __ Lsr(out, in, 31); - break; - case kCondAE: - // Trivially true. - __ mov(out, ShifterOperand(1)); - break; - case kCondB: - // Trivially false. - __ mov(out, ShifterOperand(0)); - break; - default: - LOG(FATAL) << "Unexpected condition " << condition; - UNREACHABLE(); - } -} - -void LocationsBuilderARM::HandleCondition(HCondition* cond) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(cond, LocationSummary::kNoCall); - // Handle the long/FP comparisons made in instruction simplification. - switch (cond->InputAt(0)->GetType()) { - case Primitive::kPrimLong: - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RegisterOrConstant(cond->InputAt(1))); - if (!cond->IsEmittedAtUseSite()) { - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); - } - break; - - case Primitive::kPrimFloat: - case Primitive::kPrimDouble: - locations->SetInAt(0, Location::RequiresFpuRegister()); - locations->SetInAt(1, ArithmeticZeroOrFpuRegister(cond->InputAt(1))); - if (!cond->IsEmittedAtUseSite()) { - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); - } - break; - - default: - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RegisterOrConstant(cond->InputAt(1))); - if (!cond->IsEmittedAtUseSite()) { - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); - } - } -} - -void InstructionCodeGeneratorARM::HandleCondition(HCondition* cond) { - if (cond->IsEmittedAtUseSite()) { - return; - } - - const Primitive::Type type = cond->GetLeft()->GetType(); - - if (Primitive::IsFloatingPointType(type)) { - GenerateConditionGeneric(cond, codegen_); - return; - } - - DCHECK(Primitive::IsIntegralType(type) || type == Primitive::kPrimNot) << type; - - const IfCondition condition = cond->GetCondition(); - - // A condition with only one boolean input, or two boolean inputs without being equality or - // inequality results from transformations done by the instruction simplifier, and is handled - // as a regular condition with integral inputs. - if (type == Primitive::kPrimBoolean && - cond->GetRight()->GetType() == Primitive::kPrimBoolean && - (condition == kCondEQ || condition == kCondNE)) { - const LocationSummary* const locations = cond->GetLocations(); - Register left = locations->InAt(0).AsRegister<Register>(); - const Register out = locations->Out().AsRegister<Register>(); - const Location right_loc = locations->InAt(1); - - // The constant case is handled by the instruction simplifier. - DCHECK(!right_loc.IsConstant()); - - Register right = right_loc.AsRegister<Register>(); - - // Avoid 32-bit instructions if possible. - if (out == right) { - std::swap(left, right); - } - - __ eor(out, left, ShifterOperand(right)); - - if (condition == kCondEQ) { - __ eor(out, out, ShifterOperand(1)); - } - - return; - } - - GenerateConditionIntegralOrNonPrimitive(cond, codegen_); -} - -void LocationsBuilderARM::VisitEqual(HEqual* comp) { - HandleCondition(comp); -} - -void InstructionCodeGeneratorARM::VisitEqual(HEqual* comp) { - HandleCondition(comp); -} - -void LocationsBuilderARM::VisitNotEqual(HNotEqual* comp) { - HandleCondition(comp); -} - -void InstructionCodeGeneratorARM::VisitNotEqual(HNotEqual* comp) { - HandleCondition(comp); -} - -void LocationsBuilderARM::VisitLessThan(HLessThan* comp) { - HandleCondition(comp); -} - -void InstructionCodeGeneratorARM::VisitLessThan(HLessThan* comp) { - HandleCondition(comp); -} - -void LocationsBuilderARM::VisitLessThanOrEqual(HLessThanOrEqual* comp) { - HandleCondition(comp); -} - -void InstructionCodeGeneratorARM::VisitLessThanOrEqual(HLessThanOrEqual* comp) { - HandleCondition(comp); -} - -void LocationsBuilderARM::VisitGreaterThan(HGreaterThan* comp) { - HandleCondition(comp); -} - -void InstructionCodeGeneratorARM::VisitGreaterThan(HGreaterThan* comp) { - HandleCondition(comp); -} - -void LocationsBuilderARM::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) { - HandleCondition(comp); -} - -void InstructionCodeGeneratorARM::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) { - HandleCondition(comp); -} - -void LocationsBuilderARM::VisitBelow(HBelow* comp) { - HandleCondition(comp); -} - -void InstructionCodeGeneratorARM::VisitBelow(HBelow* comp) { - HandleCondition(comp); -} - -void LocationsBuilderARM::VisitBelowOrEqual(HBelowOrEqual* comp) { - HandleCondition(comp); -} - -void InstructionCodeGeneratorARM::VisitBelowOrEqual(HBelowOrEqual* comp) { - HandleCondition(comp); -} - -void LocationsBuilderARM::VisitAbove(HAbove* comp) { - HandleCondition(comp); -} - -void InstructionCodeGeneratorARM::VisitAbove(HAbove* comp) { - HandleCondition(comp); -} - -void LocationsBuilderARM::VisitAboveOrEqual(HAboveOrEqual* comp) { - HandleCondition(comp); -} - -void InstructionCodeGeneratorARM::VisitAboveOrEqual(HAboveOrEqual* comp) { - HandleCondition(comp); -} - -void LocationsBuilderARM::VisitIntConstant(HIntConstant* constant) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); - locations->SetOut(Location::ConstantLocation(constant)); -} - -void InstructionCodeGeneratorARM::VisitIntConstant(HIntConstant* constant ATTRIBUTE_UNUSED) { - // Will be generated at use site. -} - -void LocationsBuilderARM::VisitNullConstant(HNullConstant* constant) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); - locations->SetOut(Location::ConstantLocation(constant)); -} - -void InstructionCodeGeneratorARM::VisitNullConstant(HNullConstant* constant ATTRIBUTE_UNUSED) { - // Will be generated at use site. -} - -void LocationsBuilderARM::VisitLongConstant(HLongConstant* constant) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); - locations->SetOut(Location::ConstantLocation(constant)); -} - -void InstructionCodeGeneratorARM::VisitLongConstant(HLongConstant* constant ATTRIBUTE_UNUSED) { - // Will be generated at use site. -} - -void LocationsBuilderARM::VisitFloatConstant(HFloatConstant* constant) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); - locations->SetOut(Location::ConstantLocation(constant)); -} - -void InstructionCodeGeneratorARM::VisitFloatConstant(HFloatConstant* constant ATTRIBUTE_UNUSED) { - // Will be generated at use site. -} - -void LocationsBuilderARM::VisitDoubleConstant(HDoubleConstant* constant) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); - locations->SetOut(Location::ConstantLocation(constant)); -} - -void InstructionCodeGeneratorARM::VisitDoubleConstant(HDoubleConstant* constant ATTRIBUTE_UNUSED) { - // Will be generated at use site. -} - -void LocationsBuilderARM::VisitConstructorFence(HConstructorFence* constructor_fence) { - constructor_fence->SetLocations(nullptr); -} - -void InstructionCodeGeneratorARM::VisitConstructorFence( - HConstructorFence* constructor_fence ATTRIBUTE_UNUSED) { - codegen_->GenerateMemoryBarrier(MemBarrierKind::kStoreStore); -} - -void LocationsBuilderARM::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) { - memory_barrier->SetLocations(nullptr); -} - -void InstructionCodeGeneratorARM::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) { - codegen_->GenerateMemoryBarrier(memory_barrier->GetBarrierKind()); -} - -void LocationsBuilderARM::VisitReturnVoid(HReturnVoid* ret) { - ret->SetLocations(nullptr); -} - -void InstructionCodeGeneratorARM::VisitReturnVoid(HReturnVoid* ret ATTRIBUTE_UNUSED) { - codegen_->GenerateFrameExit(); -} - -void LocationsBuilderARM::VisitReturn(HReturn* ret) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(ret, LocationSummary::kNoCall); - locations->SetInAt(0, parameter_visitor_.GetReturnLocation(ret->InputAt(0)->GetType())); -} - -void InstructionCodeGeneratorARM::VisitReturn(HReturn* ret ATTRIBUTE_UNUSED) { - codegen_->GenerateFrameExit(); -} - -void LocationsBuilderARM::VisitInvokeUnresolved(HInvokeUnresolved* invoke) { - // The trampoline uses the same calling convention as dex calling conventions, - // except instead of loading arg0/r0 with the target Method*, arg0/r0 will contain - // the method_idx. - HandleInvoke(invoke); -} - -void InstructionCodeGeneratorARM::VisitInvokeUnresolved(HInvokeUnresolved* invoke) { - codegen_->GenerateInvokeUnresolvedRuntimeCall(invoke); -} - -void LocationsBuilderARM::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) { - // Explicit clinit checks triggered by static invokes must have been pruned by - // art::PrepareForRegisterAllocation. - DCHECK(!invoke->IsStaticWithExplicitClinitCheck()); - - IntrinsicLocationsBuilderARM intrinsic(codegen_); - if (intrinsic.TryDispatch(invoke)) { - return; - } - - HandleInvoke(invoke); -} - -static bool TryGenerateIntrinsicCode(HInvoke* invoke, CodeGeneratorARM* codegen) { - if (invoke->GetLocations()->Intrinsified()) { - IntrinsicCodeGeneratorARM intrinsic(codegen); - intrinsic.Dispatch(invoke); - return true; - } - return false; -} - -void InstructionCodeGeneratorARM::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) { - // Explicit clinit checks triggered by static invokes must have been pruned by - // art::PrepareForRegisterAllocation. - DCHECK(!invoke->IsStaticWithExplicitClinitCheck()); - - if (TryGenerateIntrinsicCode(invoke, codegen_)) { - return; - } - - LocationSummary* locations = invoke->GetLocations(); - codegen_->GenerateStaticOrDirectCall( - invoke, locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation()); -} - -void LocationsBuilderARM::HandleInvoke(HInvoke* invoke) { - InvokeDexCallingConventionVisitorARM calling_convention_visitor; - CodeGenerator::CreateCommonInvokeLocationSummary(invoke, &calling_convention_visitor); -} - -void LocationsBuilderARM::VisitInvokeVirtual(HInvokeVirtual* invoke) { - IntrinsicLocationsBuilderARM intrinsic(codegen_); - if (intrinsic.TryDispatch(invoke)) { - return; - } - - HandleInvoke(invoke); -} - -void InstructionCodeGeneratorARM::VisitInvokeVirtual(HInvokeVirtual* invoke) { - if (TryGenerateIntrinsicCode(invoke, codegen_)) { - return; - } - - codegen_->GenerateVirtualCall(invoke, invoke->GetLocations()->GetTemp(0)); - DCHECK(!codegen_->IsLeafMethod()); -} - -void LocationsBuilderARM::VisitInvokeInterface(HInvokeInterface* invoke) { - HandleInvoke(invoke); - // Add the hidden argument. - invoke->GetLocations()->AddTemp(Location::RegisterLocation(R12)); -} - -void InstructionCodeGeneratorARM::VisitInvokeInterface(HInvokeInterface* invoke) { - // TODO: b/18116999, our IMTs can miss an IncompatibleClassChangeError. - LocationSummary* locations = invoke->GetLocations(); - Register temp = locations->GetTemp(0).AsRegister<Register>(); - Register hidden_reg = locations->GetTemp(1).AsRegister<Register>(); - Location receiver = locations->InAt(0); - uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); - - // Set the hidden argument. This is safe to do this here, as R12 - // won't be modified thereafter, before the `blx` (call) instruction. - DCHECK_EQ(R12, hidden_reg); - __ LoadImmediate(hidden_reg, invoke->GetDexMethodIndex()); - - if (receiver.IsStackSlot()) { - __ LoadFromOffset(kLoadWord, temp, SP, receiver.GetStackIndex()); - // /* HeapReference<Class> */ temp = temp->klass_ - __ LoadFromOffset(kLoadWord, temp, temp, class_offset); - } else { - // /* HeapReference<Class> */ temp = receiver->klass_ - __ LoadFromOffset(kLoadWord, temp, receiver.AsRegister<Register>(), class_offset); - } - codegen_->MaybeRecordImplicitNullCheck(invoke); - // Instead of simply (possibly) unpoisoning `temp` here, we should - // emit a read barrier for the previous class reference load. - // However this is not required in practice, as this is an - // intermediate/temporary reference and because the current - // concurrent copying collector keeps the from-space memory - // intact/accessible until the end of the marking phase (the - // concurrent copying collector may not in the future). - __ MaybeUnpoisonHeapReference(temp); - __ LoadFromOffset(kLoadWord, temp, temp, - mirror::Class::ImtPtrOffset(kArmPointerSize).Uint32Value()); - uint32_t method_offset = static_cast<uint32_t>(ImTable::OffsetOfElement( - invoke->GetImtIndex(), kArmPointerSize)); - // temp = temp->GetImtEntryAt(method_offset); - __ LoadFromOffset(kLoadWord, temp, temp, method_offset); - uint32_t entry_point = - ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmPointerSize).Int32Value(); - // LR = temp->GetEntryPoint(); - __ LoadFromOffset(kLoadWord, LR, temp, entry_point); - // LR(); - __ blx(LR); - DCHECK(!codegen_->IsLeafMethod()); - codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); -} - -void LocationsBuilderARM::VisitInvokePolymorphic(HInvokePolymorphic* invoke) { - HandleInvoke(invoke); -} - -void InstructionCodeGeneratorARM::VisitInvokePolymorphic(HInvokePolymorphic* invoke) { - codegen_->GenerateInvokePolymorphicCall(invoke); -} - -void LocationsBuilderARM::VisitNeg(HNeg* neg) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(neg, LocationSummary::kNoCall); - switch (neg->GetResultType()) { - case Primitive::kPrimInt: { - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); - break; - } - case Primitive::kPrimLong: { - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); - break; - } - - case Primitive::kPrimFloat: - case Primitive::kPrimDouble: - locations->SetInAt(0, Location::RequiresFpuRegister()); - locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); - break; - - default: - LOG(FATAL) << "Unexpected neg type " << neg->GetResultType(); - } -} - -void InstructionCodeGeneratorARM::VisitNeg(HNeg* neg) { - LocationSummary* locations = neg->GetLocations(); - Location out = locations->Out(); - Location in = locations->InAt(0); - switch (neg->GetResultType()) { - case Primitive::kPrimInt: - DCHECK(in.IsRegister()); - __ rsb(out.AsRegister<Register>(), in.AsRegister<Register>(), ShifterOperand(0)); - break; - - case Primitive::kPrimLong: - DCHECK(in.IsRegisterPair()); - // out.lo = 0 - in.lo (and update the carry/borrow (C) flag) - __ rsbs(out.AsRegisterPairLow<Register>(), - in.AsRegisterPairLow<Register>(), - ShifterOperand(0)); - // We cannot emit an RSC (Reverse Subtract with Carry) - // instruction here, as it does not exist in the Thumb-2 - // instruction set. We use the following approach - // using SBC and SUB instead. - // - // out.hi = -C - __ sbc(out.AsRegisterPairHigh<Register>(), - out.AsRegisterPairHigh<Register>(), - ShifterOperand(out.AsRegisterPairHigh<Register>())); - // out.hi = out.hi - in.hi - __ sub(out.AsRegisterPairHigh<Register>(), - out.AsRegisterPairHigh<Register>(), - ShifterOperand(in.AsRegisterPairHigh<Register>())); - break; - - case Primitive::kPrimFloat: - DCHECK(in.IsFpuRegister()); - __ vnegs(out.AsFpuRegister<SRegister>(), in.AsFpuRegister<SRegister>()); - break; - - case Primitive::kPrimDouble: - DCHECK(in.IsFpuRegisterPair()); - __ vnegd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), - FromLowSToD(in.AsFpuRegisterPairLow<SRegister>())); - break; - - default: - LOG(FATAL) << "Unexpected neg type " << neg->GetResultType(); - } -} - -void LocationsBuilderARM::VisitTypeConversion(HTypeConversion* conversion) { - Primitive::Type result_type = conversion->GetResultType(); - Primitive::Type input_type = conversion->GetInputType(); - DCHECK_NE(result_type, input_type); - - // The float-to-long, double-to-long and long-to-float type conversions - // rely on a call to the runtime. - LocationSummary::CallKind call_kind = - (((input_type == Primitive::kPrimFloat || input_type == Primitive::kPrimDouble) - && result_type == Primitive::kPrimLong) - || (input_type == Primitive::kPrimLong && result_type == Primitive::kPrimFloat)) - ? LocationSummary::kCallOnMainOnly - : LocationSummary::kNoCall; - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(conversion, call_kind); - - // The Java language does not allow treating boolean as an integral type but - // our bit representation makes it safe. - - switch (result_type) { - case Primitive::kPrimByte: - switch (input_type) { - case Primitive::kPrimLong: - // Type conversion from long to byte is a result of code transformations. - case Primitive::kPrimBoolean: - // Boolean input is a result of code transformations. - case Primitive::kPrimShort: - case Primitive::kPrimInt: - case Primitive::kPrimChar: - // Processing a Dex `int-to-byte' instruction. - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); - break; - - default: - LOG(FATAL) << "Unexpected type conversion from " << input_type - << " to " << result_type; - } - break; - - case Primitive::kPrimShort: - switch (input_type) { - case Primitive::kPrimLong: - // Type conversion from long to short is a result of code transformations. - case Primitive::kPrimBoolean: - // Boolean input is a result of code transformations. - case Primitive::kPrimByte: - case Primitive::kPrimInt: - case Primitive::kPrimChar: - // Processing a Dex `int-to-short' instruction. - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); - break; - - default: - LOG(FATAL) << "Unexpected type conversion from " << input_type - << " to " << result_type; - } - break; - - case Primitive::kPrimInt: - switch (input_type) { - case Primitive::kPrimLong: - // Processing a Dex `long-to-int' instruction. - locations->SetInAt(0, Location::Any()); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); - break; - - case Primitive::kPrimFloat: - // Processing a Dex `float-to-int' instruction. - locations->SetInAt(0, Location::RequiresFpuRegister()); - locations->SetOut(Location::RequiresRegister()); - locations->AddTemp(Location::RequiresFpuRegister()); - break; - - case Primitive::kPrimDouble: - // Processing a Dex `double-to-int' instruction. - locations->SetInAt(0, Location::RequiresFpuRegister()); - locations->SetOut(Location::RequiresRegister()); - locations->AddTemp(Location::RequiresFpuRegister()); - break; - - default: - LOG(FATAL) << "Unexpected type conversion from " << input_type - << " to " << result_type; - } - break; - - case Primitive::kPrimLong: - switch (input_type) { - case Primitive::kPrimBoolean: - // Boolean input is a result of code transformations. - case Primitive::kPrimByte: - case Primitive::kPrimShort: - case Primitive::kPrimInt: - case Primitive::kPrimChar: - // Processing a Dex `int-to-long' instruction. - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); - break; - - case Primitive::kPrimFloat: { - // Processing a Dex `float-to-long' instruction. - InvokeRuntimeCallingConvention calling_convention; - locations->SetInAt(0, Location::FpuRegisterLocation( - calling_convention.GetFpuRegisterAt(0))); - locations->SetOut(Location::RegisterPairLocation(R0, R1)); - break; - } - - case Primitive::kPrimDouble: { - // Processing a Dex `double-to-long' instruction. - InvokeRuntimeCallingConvention calling_convention; - locations->SetInAt(0, Location::FpuRegisterPairLocation( - calling_convention.GetFpuRegisterAt(0), - calling_convention.GetFpuRegisterAt(1))); - locations->SetOut(Location::RegisterPairLocation(R0, R1)); - break; - } - - default: - LOG(FATAL) << "Unexpected type conversion from " << input_type - << " to " << result_type; - } - break; - - case Primitive::kPrimChar: - switch (input_type) { - case Primitive::kPrimLong: - // Type conversion from long to char is a result of code transformations. - case Primitive::kPrimBoolean: - // Boolean input is a result of code transformations. - case Primitive::kPrimByte: - case Primitive::kPrimShort: - case Primitive::kPrimInt: - // Processing a Dex `int-to-char' instruction. - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); - break; - - default: - LOG(FATAL) << "Unexpected type conversion from " << input_type - << " to " << result_type; - } - break; - - case Primitive::kPrimFloat: - switch (input_type) { - case Primitive::kPrimBoolean: - // Boolean input is a result of code transformations. - case Primitive::kPrimByte: - case Primitive::kPrimShort: - case Primitive::kPrimInt: - case Primitive::kPrimChar: - // Processing a Dex `int-to-float' instruction. - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetOut(Location::RequiresFpuRegister()); - break; - - case Primitive::kPrimLong: { - // Processing a Dex `long-to-float' instruction. - InvokeRuntimeCallingConvention calling_convention; - locations->SetInAt(0, Location::RegisterPairLocation( - calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1))); - locations->SetOut(Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(0))); - break; - } - - case Primitive::kPrimDouble: - // Processing a Dex `double-to-float' instruction. - locations->SetInAt(0, Location::RequiresFpuRegister()); - locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); - break; - - default: - LOG(FATAL) << "Unexpected type conversion from " << input_type - << " to " << result_type; - }; - break; - - case Primitive::kPrimDouble: - switch (input_type) { - case Primitive::kPrimBoolean: - // Boolean input is a result of code transformations. - case Primitive::kPrimByte: - case Primitive::kPrimShort: - case Primitive::kPrimInt: - case Primitive::kPrimChar: - // Processing a Dex `int-to-double' instruction. - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetOut(Location::RequiresFpuRegister()); - break; - - case Primitive::kPrimLong: - // Processing a Dex `long-to-double' instruction. - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetOut(Location::RequiresFpuRegister()); - locations->AddTemp(Location::RequiresFpuRegister()); - locations->AddTemp(Location::RequiresFpuRegister()); - break; - - case Primitive::kPrimFloat: - // Processing a Dex `float-to-double' instruction. - locations->SetInAt(0, Location::RequiresFpuRegister()); - locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); - break; - - default: - LOG(FATAL) << "Unexpected type conversion from " << input_type - << " to " << result_type; - }; - break; - - default: - LOG(FATAL) << "Unexpected type conversion from " << input_type - << " to " << result_type; - } -} - -void InstructionCodeGeneratorARM::VisitTypeConversion(HTypeConversion* conversion) { - LocationSummary* locations = conversion->GetLocations(); - Location out = locations->Out(); - Location in = locations->InAt(0); - Primitive::Type result_type = conversion->GetResultType(); - Primitive::Type input_type = conversion->GetInputType(); - DCHECK_NE(result_type, input_type); - switch (result_type) { - case Primitive::kPrimByte: - switch (input_type) { - case Primitive::kPrimLong: - // Type conversion from long to byte is a result of code transformations. - __ sbfx(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>(), 0, 8); - break; - case Primitive::kPrimBoolean: - // Boolean input is a result of code transformations. - case Primitive::kPrimShort: - case Primitive::kPrimInt: - case Primitive::kPrimChar: - // Processing a Dex `int-to-byte' instruction. - __ sbfx(out.AsRegister<Register>(), in.AsRegister<Register>(), 0, 8); - break; - - default: - LOG(FATAL) << "Unexpected type conversion from " << input_type - << " to " << result_type; - } - break; - - case Primitive::kPrimShort: - switch (input_type) { - case Primitive::kPrimLong: - // Type conversion from long to short is a result of code transformations. - __ sbfx(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>(), 0, 16); - break; - case Primitive::kPrimBoolean: - // Boolean input is a result of code transformations. - case Primitive::kPrimByte: - case Primitive::kPrimInt: - case Primitive::kPrimChar: - // Processing a Dex `int-to-short' instruction. - __ sbfx(out.AsRegister<Register>(), in.AsRegister<Register>(), 0, 16); - break; - - default: - LOG(FATAL) << "Unexpected type conversion from " << input_type - << " to " << result_type; - } - break; - - case Primitive::kPrimInt: - switch (input_type) { - case Primitive::kPrimLong: - // Processing a Dex `long-to-int' instruction. - DCHECK(out.IsRegister()); - if (in.IsRegisterPair()) { - __ Mov(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>()); - } else if (in.IsDoubleStackSlot()) { - __ LoadFromOffset(kLoadWord, out.AsRegister<Register>(), SP, in.GetStackIndex()); - } else { - DCHECK(in.IsConstant()); - DCHECK(in.GetConstant()->IsLongConstant()); - int64_t value = in.GetConstant()->AsLongConstant()->GetValue(); - __ LoadImmediate(out.AsRegister<Register>(), static_cast<int32_t>(value)); - } - break; - - case Primitive::kPrimFloat: { - // Processing a Dex `float-to-int' instruction. - SRegister temp = locations->GetTemp(0).AsFpuRegisterPairLow<SRegister>(); - __ vcvtis(temp, in.AsFpuRegister<SRegister>()); - __ vmovrs(out.AsRegister<Register>(), temp); - break; - } - - case Primitive::kPrimDouble: { - // Processing a Dex `double-to-int' instruction. - SRegister temp_s = locations->GetTemp(0).AsFpuRegisterPairLow<SRegister>(); - __ vcvtid(temp_s, FromLowSToD(in.AsFpuRegisterPairLow<SRegister>())); - __ vmovrs(out.AsRegister<Register>(), temp_s); - break; - } - - default: - LOG(FATAL) << "Unexpected type conversion from " << input_type - << " to " << result_type; - } - break; - - case Primitive::kPrimLong: - switch (input_type) { - case Primitive::kPrimBoolean: - // Boolean input is a result of code transformations. - case Primitive::kPrimByte: - case Primitive::kPrimShort: - case Primitive::kPrimInt: - case Primitive::kPrimChar: - // Processing a Dex `int-to-long' instruction. - DCHECK(out.IsRegisterPair()); - DCHECK(in.IsRegister()); - __ Mov(out.AsRegisterPairLow<Register>(), in.AsRegister<Register>()); - // Sign extension. - __ Asr(out.AsRegisterPairHigh<Register>(), - out.AsRegisterPairLow<Register>(), - 31); - break; - - case Primitive::kPrimFloat: - // Processing a Dex `float-to-long' instruction. - codegen_->InvokeRuntime(kQuickF2l, conversion, conversion->GetDexPc()); - CheckEntrypointTypes<kQuickF2l, int64_t, float>(); - break; - - case Primitive::kPrimDouble: - // Processing a Dex `double-to-long' instruction. - codegen_->InvokeRuntime(kQuickD2l, conversion, conversion->GetDexPc()); - CheckEntrypointTypes<kQuickD2l, int64_t, double>(); - break; - - default: - LOG(FATAL) << "Unexpected type conversion from " << input_type - << " to " << result_type; - } - break; - - case Primitive::kPrimChar: - switch (input_type) { - case Primitive::kPrimLong: - // Type conversion from long to char is a result of code transformations. - __ ubfx(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>(), 0, 16); - break; - case Primitive::kPrimBoolean: - // Boolean input is a result of code transformations. - case Primitive::kPrimByte: - case Primitive::kPrimShort: - case Primitive::kPrimInt: - // Processing a Dex `int-to-char' instruction. - __ ubfx(out.AsRegister<Register>(), in.AsRegister<Register>(), 0, 16); - break; - - default: - LOG(FATAL) << "Unexpected type conversion from " << input_type - << " to " << result_type; - } - break; - - case Primitive::kPrimFloat: - switch (input_type) { - case Primitive::kPrimBoolean: - // Boolean input is a result of code transformations. - case Primitive::kPrimByte: - case Primitive::kPrimShort: - case Primitive::kPrimInt: - case Primitive::kPrimChar: { - // Processing a Dex `int-to-float' instruction. - __ vmovsr(out.AsFpuRegister<SRegister>(), in.AsRegister<Register>()); - __ vcvtsi(out.AsFpuRegister<SRegister>(), out.AsFpuRegister<SRegister>()); - break; - } - - case Primitive::kPrimLong: - // Processing a Dex `long-to-float' instruction. - codegen_->InvokeRuntime(kQuickL2f, conversion, conversion->GetDexPc()); - CheckEntrypointTypes<kQuickL2f, float, int64_t>(); - break; - - case Primitive::kPrimDouble: - // Processing a Dex `double-to-float' instruction. - __ vcvtsd(out.AsFpuRegister<SRegister>(), - FromLowSToD(in.AsFpuRegisterPairLow<SRegister>())); - break; - - default: - LOG(FATAL) << "Unexpected type conversion from " << input_type - << " to " << result_type; - }; - break; - - case Primitive::kPrimDouble: - switch (input_type) { - case Primitive::kPrimBoolean: - // Boolean input is a result of code transformations. - case Primitive::kPrimByte: - case Primitive::kPrimShort: - case Primitive::kPrimInt: - case Primitive::kPrimChar: { - // Processing a Dex `int-to-double' instruction. - __ vmovsr(out.AsFpuRegisterPairLow<SRegister>(), in.AsRegister<Register>()); - __ vcvtdi(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), - out.AsFpuRegisterPairLow<SRegister>()); - break; - } - - case Primitive::kPrimLong: { - // Processing a Dex `long-to-double' instruction. - Register low = in.AsRegisterPairLow<Register>(); - Register high = in.AsRegisterPairHigh<Register>(); - SRegister out_s = out.AsFpuRegisterPairLow<SRegister>(); - DRegister out_d = FromLowSToD(out_s); - SRegister temp_s = locations->GetTemp(0).AsFpuRegisterPairLow<SRegister>(); - DRegister temp_d = FromLowSToD(temp_s); - SRegister constant_s = locations->GetTemp(1).AsFpuRegisterPairLow<SRegister>(); - DRegister constant_d = FromLowSToD(constant_s); - - // temp_d = int-to-double(high) - __ vmovsr(temp_s, high); - __ vcvtdi(temp_d, temp_s); - // constant_d = k2Pow32EncodingForDouble - __ LoadDImmediate(constant_d, bit_cast<double, int64_t>(k2Pow32EncodingForDouble)); - // out_d = unsigned-to-double(low) - __ vmovsr(out_s, low); - __ vcvtdu(out_d, out_s); - // out_d += temp_d * constant_d - __ vmlad(out_d, temp_d, constant_d); - break; - } - - case Primitive::kPrimFloat: - // Processing a Dex `float-to-double' instruction. - __ vcvtds(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), - in.AsFpuRegister<SRegister>()); - break; - - default: - LOG(FATAL) << "Unexpected type conversion from " << input_type - << " to " << result_type; - }; - break; - - default: - LOG(FATAL) << "Unexpected type conversion from " << input_type - << " to " << result_type; - } -} - -void LocationsBuilderARM::VisitAdd(HAdd* add) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(add, LocationSummary::kNoCall); - switch (add->GetResultType()) { - case Primitive::kPrimInt: { - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RegisterOrConstant(add->InputAt(1))); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); - break; - } - - case Primitive::kPrimLong: { - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, ArmEncodableConstantOrRegister(add->InputAt(1), ADD)); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); - break; - } - - case Primitive::kPrimFloat: - case Primitive::kPrimDouble: { - locations->SetInAt(0, Location::RequiresFpuRegister()); - locations->SetInAt(1, Location::RequiresFpuRegister()); - locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); - break; - } - - default: - LOG(FATAL) << "Unexpected add type " << add->GetResultType(); - } -} - -void InstructionCodeGeneratorARM::VisitAdd(HAdd* add) { - LocationSummary* locations = add->GetLocations(); - Location out = locations->Out(); - Location first = locations->InAt(0); - Location second = locations->InAt(1); - switch (add->GetResultType()) { - case Primitive::kPrimInt: - if (second.IsRegister()) { - __ add(out.AsRegister<Register>(), - first.AsRegister<Register>(), - ShifterOperand(second.AsRegister<Register>())); - } else { - __ AddConstant(out.AsRegister<Register>(), - first.AsRegister<Register>(), - second.GetConstant()->AsIntConstant()->GetValue()); - } - break; - - case Primitive::kPrimLong: { - if (second.IsConstant()) { - uint64_t value = static_cast<uint64_t>(Int64FromConstant(second.GetConstant())); - GenerateAddLongConst(out, first, value); - } else { - DCHECK(second.IsRegisterPair()); - __ adds(out.AsRegisterPairLow<Register>(), - first.AsRegisterPairLow<Register>(), - ShifterOperand(second.AsRegisterPairLow<Register>())); - __ adc(out.AsRegisterPairHigh<Register>(), - first.AsRegisterPairHigh<Register>(), - ShifterOperand(second.AsRegisterPairHigh<Register>())); - } - break; - } - - case Primitive::kPrimFloat: - __ vadds(out.AsFpuRegister<SRegister>(), - first.AsFpuRegister<SRegister>(), - second.AsFpuRegister<SRegister>()); - break; - - case Primitive::kPrimDouble: - __ vaddd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), - FromLowSToD(first.AsFpuRegisterPairLow<SRegister>()), - FromLowSToD(second.AsFpuRegisterPairLow<SRegister>())); - break; - - default: - LOG(FATAL) << "Unexpected add type " << add->GetResultType(); - } -} - -void LocationsBuilderARM::VisitSub(HSub* sub) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(sub, LocationSummary::kNoCall); - switch (sub->GetResultType()) { - case Primitive::kPrimInt: { - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RegisterOrConstant(sub->InputAt(1))); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); - break; - } - - case Primitive::kPrimLong: { - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, ArmEncodableConstantOrRegister(sub->InputAt(1), SUB)); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); - break; - } - case Primitive::kPrimFloat: - case Primitive::kPrimDouble: { - locations->SetInAt(0, Location::RequiresFpuRegister()); - locations->SetInAt(1, Location::RequiresFpuRegister()); - locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); - break; - } - default: - LOG(FATAL) << "Unexpected sub type " << sub->GetResultType(); - } -} - -void InstructionCodeGeneratorARM::VisitSub(HSub* sub) { - LocationSummary* locations = sub->GetLocations(); - Location out = locations->Out(); - Location first = locations->InAt(0); - Location second = locations->InAt(1); - switch (sub->GetResultType()) { - case Primitive::kPrimInt: { - if (second.IsRegister()) { - __ sub(out.AsRegister<Register>(), - first.AsRegister<Register>(), - ShifterOperand(second.AsRegister<Register>())); - } else { - __ AddConstant(out.AsRegister<Register>(), - first.AsRegister<Register>(), - -second.GetConstant()->AsIntConstant()->GetValue()); - } - break; - } - - case Primitive::kPrimLong: { - if (second.IsConstant()) { - uint64_t value = static_cast<uint64_t>(Int64FromConstant(second.GetConstant())); - GenerateAddLongConst(out, first, -value); - } else { - DCHECK(second.IsRegisterPair()); - __ subs(out.AsRegisterPairLow<Register>(), - first.AsRegisterPairLow<Register>(), - ShifterOperand(second.AsRegisterPairLow<Register>())); - __ sbc(out.AsRegisterPairHigh<Register>(), - first.AsRegisterPairHigh<Register>(), - ShifterOperand(second.AsRegisterPairHigh<Register>())); - } - break; - } - - case Primitive::kPrimFloat: { - __ vsubs(out.AsFpuRegister<SRegister>(), - first.AsFpuRegister<SRegister>(), - second.AsFpuRegister<SRegister>()); - break; - } - - case Primitive::kPrimDouble: { - __ vsubd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), - FromLowSToD(first.AsFpuRegisterPairLow<SRegister>()), - FromLowSToD(second.AsFpuRegisterPairLow<SRegister>())); - break; - } - - - default: - LOG(FATAL) << "Unexpected sub type " << sub->GetResultType(); - } -} - -void LocationsBuilderARM::VisitMul(HMul* mul) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(mul, LocationSummary::kNoCall); - switch (mul->GetResultType()) { - case Primitive::kPrimInt: - case Primitive::kPrimLong: { - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); - break; - } - - case Primitive::kPrimFloat: - case Primitive::kPrimDouble: { - locations->SetInAt(0, Location::RequiresFpuRegister()); - locations->SetInAt(1, Location::RequiresFpuRegister()); - locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); - break; - } - - default: - LOG(FATAL) << "Unexpected mul type " << mul->GetResultType(); - } -} - -void InstructionCodeGeneratorARM::VisitMul(HMul* mul) { - LocationSummary* locations = mul->GetLocations(); - Location out = locations->Out(); - Location first = locations->InAt(0); - Location second = locations->InAt(1); - switch (mul->GetResultType()) { - case Primitive::kPrimInt: { - __ mul(out.AsRegister<Register>(), - first.AsRegister<Register>(), - second.AsRegister<Register>()); - break; - } - case Primitive::kPrimLong: { - Register out_hi = out.AsRegisterPairHigh<Register>(); - Register out_lo = out.AsRegisterPairLow<Register>(); - Register in1_hi = first.AsRegisterPairHigh<Register>(); - Register in1_lo = first.AsRegisterPairLow<Register>(); - Register in2_hi = second.AsRegisterPairHigh<Register>(); - Register in2_lo = second.AsRegisterPairLow<Register>(); - - // Extra checks to protect caused by the existence of R1_R2. - // The algorithm is wrong if out.hi is either in1.lo or in2.lo: - // (e.g. in1=r0_r1, in2=r2_r3 and out=r1_r2); - DCHECK_NE(out_hi, in1_lo); - DCHECK_NE(out_hi, in2_lo); - - // input: in1 - 64 bits, in2 - 64 bits - // output: out - // formula: out.hi : out.lo = (in1.lo * in2.hi + in1.hi * in2.lo)* 2^32 + in1.lo * in2.lo - // parts: out.hi = in1.lo * in2.hi + in1.hi * in2.lo + (in1.lo * in2.lo)[63:32] - // parts: out.lo = (in1.lo * in2.lo)[31:0] - - // IP <- in1.lo * in2.hi - __ mul(IP, in1_lo, in2_hi); - // out.hi <- in1.lo * in2.hi + in1.hi * in2.lo - __ mla(out_hi, in1_hi, in2_lo, IP); - // out.lo <- (in1.lo * in2.lo)[31:0]; - __ umull(out_lo, IP, in1_lo, in2_lo); - // out.hi <- in2.hi * in1.lo + in2.lo * in1.hi + (in1.lo * in2.lo)[63:32] - __ add(out_hi, out_hi, ShifterOperand(IP)); - break; - } - - case Primitive::kPrimFloat: { - __ vmuls(out.AsFpuRegister<SRegister>(), - first.AsFpuRegister<SRegister>(), - second.AsFpuRegister<SRegister>()); - break; - } - - case Primitive::kPrimDouble: { - __ vmuld(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), - FromLowSToD(first.AsFpuRegisterPairLow<SRegister>()), - FromLowSToD(second.AsFpuRegisterPairLow<SRegister>())); - break; - } - - default: - LOG(FATAL) << "Unexpected mul type " << mul->GetResultType(); - } -} - -void InstructionCodeGeneratorARM::DivRemOneOrMinusOne(HBinaryOperation* instruction) { - DCHECK(instruction->IsDiv() || instruction->IsRem()); - DCHECK(instruction->GetResultType() == Primitive::kPrimInt); - - LocationSummary* locations = instruction->GetLocations(); - Location second = locations->InAt(1); - DCHECK(second.IsConstant()); - - Register out = locations->Out().AsRegister<Register>(); - Register dividend = locations->InAt(0).AsRegister<Register>(); - int32_t imm = second.GetConstant()->AsIntConstant()->GetValue(); - DCHECK(imm == 1 || imm == -1); - - if (instruction->IsRem()) { - __ LoadImmediate(out, 0); - } else { - if (imm == 1) { - __ Mov(out, dividend); - } else { - __ rsb(out, dividend, ShifterOperand(0)); - } - } -} - -void InstructionCodeGeneratorARM::DivRemByPowerOfTwo(HBinaryOperation* instruction) { - DCHECK(instruction->IsDiv() || instruction->IsRem()); - DCHECK(instruction->GetResultType() == Primitive::kPrimInt); - - LocationSummary* locations = instruction->GetLocations(); - Location second = locations->InAt(1); - DCHECK(second.IsConstant()); - - Register out = locations->Out().AsRegister<Register>(); - Register dividend = locations->InAt(0).AsRegister<Register>(); - Register temp = locations->GetTemp(0).AsRegister<Register>(); - int32_t imm = second.GetConstant()->AsIntConstant()->GetValue(); - uint32_t abs_imm = static_cast<uint32_t>(AbsOrMin(imm)); - int ctz_imm = CTZ(abs_imm); - - if (ctz_imm == 1) { - __ Lsr(temp, dividend, 32 - ctz_imm); - } else { - __ Asr(temp, dividend, 31); - __ Lsr(temp, temp, 32 - ctz_imm); - } - __ add(out, temp, ShifterOperand(dividend)); - - if (instruction->IsDiv()) { - __ Asr(out, out, ctz_imm); - if (imm < 0) { - __ rsb(out, out, ShifterOperand(0)); - } - } else { - __ ubfx(out, out, 0, ctz_imm); - __ sub(out, out, ShifterOperand(temp)); - } -} - -void InstructionCodeGeneratorARM::GenerateDivRemWithAnyConstant(HBinaryOperation* instruction) { - DCHECK(instruction->IsDiv() || instruction->IsRem()); - DCHECK(instruction->GetResultType() == Primitive::kPrimInt); - - LocationSummary* locations = instruction->GetLocations(); - Location second = locations->InAt(1); - DCHECK(second.IsConstant()); - - Register out = locations->Out().AsRegister<Register>(); - Register dividend = locations->InAt(0).AsRegister<Register>(); - Register temp1 = locations->GetTemp(0).AsRegister<Register>(); - Register temp2 = locations->GetTemp(1).AsRegister<Register>(); - int64_t imm = second.GetConstant()->AsIntConstant()->GetValue(); - - int64_t magic; - int shift; - CalculateMagicAndShiftForDivRem(imm, false /* is_long */, &magic, &shift); - - __ LoadImmediate(temp1, magic); - __ smull(temp2, temp1, dividend, temp1); - - if (imm > 0 && magic < 0) { - __ add(temp1, temp1, ShifterOperand(dividend)); - } else if (imm < 0 && magic > 0) { - __ sub(temp1, temp1, ShifterOperand(dividend)); - } - - if (shift != 0) { - __ Asr(temp1, temp1, shift); - } - - if (instruction->IsDiv()) { - __ sub(out, temp1, ShifterOperand(temp1, ASR, 31)); - } else { - __ sub(temp1, temp1, ShifterOperand(temp1, ASR, 31)); - // TODO: Strength reduction for mls. - __ LoadImmediate(temp2, imm); - __ mls(out, temp1, temp2, dividend); - } -} - -void InstructionCodeGeneratorARM::GenerateDivRemConstantIntegral(HBinaryOperation* instruction) { - DCHECK(instruction->IsDiv() || instruction->IsRem()); - DCHECK(instruction->GetResultType() == Primitive::kPrimInt); - - LocationSummary* locations = instruction->GetLocations(); - Location second = locations->InAt(1); - DCHECK(second.IsConstant()); - - int32_t imm = second.GetConstant()->AsIntConstant()->GetValue(); - if (imm == 0) { - // Do not generate anything. DivZeroCheck would prevent any code to be executed. - } else if (imm == 1 || imm == -1) { - DivRemOneOrMinusOne(instruction); - } else if (IsPowerOfTwo(AbsOrMin(imm))) { - DivRemByPowerOfTwo(instruction); - } else { - DCHECK(imm <= -2 || imm >= 2); - GenerateDivRemWithAnyConstant(instruction); - } -} - -void LocationsBuilderARM::VisitDiv(HDiv* div) { - LocationSummary::CallKind call_kind = LocationSummary::kNoCall; - if (div->GetResultType() == Primitive::kPrimLong) { - // pLdiv runtime call. - call_kind = LocationSummary::kCallOnMainOnly; - } else if (div->GetResultType() == Primitive::kPrimInt && div->InputAt(1)->IsConstant()) { - // sdiv will be replaced by other instruction sequence. - } else if (div->GetResultType() == Primitive::kPrimInt && - !codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { - // pIdivmod runtime call. - call_kind = LocationSummary::kCallOnMainOnly; - } - - LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(div, call_kind); - - switch (div->GetResultType()) { - case Primitive::kPrimInt: { - if (div->InputAt(1)->IsConstant()) { - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::ConstantLocation(div->InputAt(1)->AsConstant())); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); - int32_t value = div->InputAt(1)->AsIntConstant()->GetValue(); - if (value == 1 || value == 0 || value == -1) { - // No temp register required. - } else { - locations->AddTemp(Location::RequiresRegister()); - if (!IsPowerOfTwo(AbsOrMin(value))) { - locations->AddTemp(Location::RequiresRegister()); - } - } - } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); - } else { - InvokeRuntimeCallingConvention calling_convention; - locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); - locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); - // Note: divmod will compute both the quotient and the remainder as the pair R0 and R1, but - // we only need the former. - locations->SetOut(Location::RegisterLocation(R0)); - } - break; - } - case Primitive::kPrimLong: { - InvokeRuntimeCallingConvention calling_convention; - locations->SetInAt(0, Location::RegisterPairLocation( - calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1))); - locations->SetInAt(1, Location::RegisterPairLocation( - calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3))); - locations->SetOut(Location::RegisterPairLocation(R0, R1)); - break; - } - case Primitive::kPrimFloat: - case Primitive::kPrimDouble: { - locations->SetInAt(0, Location::RequiresFpuRegister()); - locations->SetInAt(1, Location::RequiresFpuRegister()); - locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); - break; - } - - default: - LOG(FATAL) << "Unexpected div type " << div->GetResultType(); - } -} - -void InstructionCodeGeneratorARM::VisitDiv(HDiv* div) { - LocationSummary* locations = div->GetLocations(); - Location out = locations->Out(); - Location first = locations->InAt(0); - Location second = locations->InAt(1); - - switch (div->GetResultType()) { - case Primitive::kPrimInt: { - if (second.IsConstant()) { - GenerateDivRemConstantIntegral(div); - } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { - __ sdiv(out.AsRegister<Register>(), - first.AsRegister<Register>(), - second.AsRegister<Register>()); - } else { - InvokeRuntimeCallingConvention calling_convention; - DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegister<Register>()); - DCHECK_EQ(calling_convention.GetRegisterAt(1), second.AsRegister<Register>()); - DCHECK_EQ(R0, out.AsRegister<Register>()); - - codegen_->InvokeRuntime(kQuickIdivmod, div, div->GetDexPc()); - CheckEntrypointTypes<kQuickIdivmod, int32_t, int32_t, int32_t>(); - } - break; - } - - case Primitive::kPrimLong: { - InvokeRuntimeCallingConvention calling_convention; - DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegisterPairLow<Register>()); - DCHECK_EQ(calling_convention.GetRegisterAt(1), first.AsRegisterPairHigh<Register>()); - DCHECK_EQ(calling_convention.GetRegisterAt(2), second.AsRegisterPairLow<Register>()); - DCHECK_EQ(calling_convention.GetRegisterAt(3), second.AsRegisterPairHigh<Register>()); - DCHECK_EQ(R0, out.AsRegisterPairLow<Register>()); - DCHECK_EQ(R1, out.AsRegisterPairHigh<Register>()); - - codegen_->InvokeRuntime(kQuickLdiv, div, div->GetDexPc()); - CheckEntrypointTypes<kQuickLdiv, int64_t, int64_t, int64_t>(); - break; - } - - case Primitive::kPrimFloat: { - __ vdivs(out.AsFpuRegister<SRegister>(), - first.AsFpuRegister<SRegister>(), - second.AsFpuRegister<SRegister>()); - break; - } - - case Primitive::kPrimDouble: { - __ vdivd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), - FromLowSToD(first.AsFpuRegisterPairLow<SRegister>()), - FromLowSToD(second.AsFpuRegisterPairLow<SRegister>())); - break; - } - - default: - LOG(FATAL) << "Unexpected div type " << div->GetResultType(); - } -} - -void LocationsBuilderARM::VisitRem(HRem* rem) { - Primitive::Type type = rem->GetResultType(); - - // Most remainders are implemented in the runtime. - LocationSummary::CallKind call_kind = LocationSummary::kCallOnMainOnly; - if (rem->GetResultType() == Primitive::kPrimInt && rem->InputAt(1)->IsConstant()) { - // sdiv will be replaced by other instruction sequence. - call_kind = LocationSummary::kNoCall; - } else if ((rem->GetResultType() == Primitive::kPrimInt) - && codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { - // Have hardware divide instruction for int, do it with three instructions. - call_kind = LocationSummary::kNoCall; - } - - LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(rem, call_kind); - - switch (type) { - case Primitive::kPrimInt: { - if (rem->InputAt(1)->IsConstant()) { - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::ConstantLocation(rem->InputAt(1)->AsConstant())); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); - int32_t value = rem->InputAt(1)->AsIntConstant()->GetValue(); - if (value == 1 || value == 0 || value == -1) { - // No temp register required. - } else { - locations->AddTemp(Location::RequiresRegister()); - if (!IsPowerOfTwo(AbsOrMin(value))) { - locations->AddTemp(Location::RequiresRegister()); - } - } - } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); - locations->AddTemp(Location::RequiresRegister()); - } else { - InvokeRuntimeCallingConvention calling_convention; - locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); - locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); - // Note: divmod will compute both the quotient and the remainder as the pair R0 and R1, but - // we only need the latter. - locations->SetOut(Location::RegisterLocation(R1)); - } - break; - } - case Primitive::kPrimLong: { - InvokeRuntimeCallingConvention calling_convention; - locations->SetInAt(0, Location::RegisterPairLocation( - calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1))); - locations->SetInAt(1, Location::RegisterPairLocation( - calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3))); - // The runtime helper puts the output in R2,R3. - locations->SetOut(Location::RegisterPairLocation(R2, R3)); - break; - } - case Primitive::kPrimFloat: { - InvokeRuntimeCallingConvention calling_convention; - locations->SetInAt(0, Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(0))); - locations->SetInAt(1, Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(1))); - locations->SetOut(Location::FpuRegisterLocation(S0)); - break; - } - - case Primitive::kPrimDouble: { - InvokeRuntimeCallingConvention calling_convention; - locations->SetInAt(0, Location::FpuRegisterPairLocation( - calling_convention.GetFpuRegisterAt(0), calling_convention.GetFpuRegisterAt(1))); - locations->SetInAt(1, Location::FpuRegisterPairLocation( - calling_convention.GetFpuRegisterAt(2), calling_convention.GetFpuRegisterAt(3))); - locations->SetOut(Location::Location::FpuRegisterPairLocation(S0, S1)); - break; - } - - default: - LOG(FATAL) << "Unexpected rem type " << type; - } -} - -void InstructionCodeGeneratorARM::VisitRem(HRem* rem) { - LocationSummary* locations = rem->GetLocations(); - Location out = locations->Out(); - Location first = locations->InAt(0); - Location second = locations->InAt(1); - - Primitive::Type type = rem->GetResultType(); - switch (type) { - case Primitive::kPrimInt: { - if (second.IsConstant()) { - GenerateDivRemConstantIntegral(rem); - } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { - Register reg1 = first.AsRegister<Register>(); - Register reg2 = second.AsRegister<Register>(); - Register temp = locations->GetTemp(0).AsRegister<Register>(); - - // temp = reg1 / reg2 (integer division) - // dest = reg1 - temp * reg2 - __ sdiv(temp, reg1, reg2); - __ mls(out.AsRegister<Register>(), temp, reg2, reg1); - } else { - InvokeRuntimeCallingConvention calling_convention; - DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegister<Register>()); - DCHECK_EQ(calling_convention.GetRegisterAt(1), second.AsRegister<Register>()); - DCHECK_EQ(R1, out.AsRegister<Register>()); - - codegen_->InvokeRuntime(kQuickIdivmod, rem, rem->GetDexPc()); - CheckEntrypointTypes<kQuickIdivmod, int32_t, int32_t, int32_t>(); - } - break; - } - - case Primitive::kPrimLong: { - codegen_->InvokeRuntime(kQuickLmod, rem, rem->GetDexPc()); - CheckEntrypointTypes<kQuickLmod, int64_t, int64_t, int64_t>(); - break; - } - - case Primitive::kPrimFloat: { - codegen_->InvokeRuntime(kQuickFmodf, rem, rem->GetDexPc()); - CheckEntrypointTypes<kQuickFmodf, float, float, float>(); - break; - } - - case Primitive::kPrimDouble: { - codegen_->InvokeRuntime(kQuickFmod, rem, rem->GetDexPc()); - CheckEntrypointTypes<kQuickFmod, double, double, double>(); - break; - } - - default: - LOG(FATAL) << "Unexpected rem type " << type; - } -} - -void LocationsBuilderARM::VisitDivZeroCheck(HDivZeroCheck* instruction) { - LocationSummary* locations = codegen_->CreateThrowingSlowPathLocations(instruction); - locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0))); -} - -void InstructionCodeGeneratorARM::VisitDivZeroCheck(HDivZeroCheck* instruction) { - SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) DivZeroCheckSlowPathARM(instruction); - codegen_->AddSlowPath(slow_path); - - LocationSummary* locations = instruction->GetLocations(); - Location value = locations->InAt(0); - - switch (instruction->GetType()) { - case Primitive::kPrimBoolean: - case Primitive::kPrimByte: - case Primitive::kPrimChar: - case Primitive::kPrimShort: - case Primitive::kPrimInt: { - if (value.IsRegister()) { - __ CompareAndBranchIfZero(value.AsRegister<Register>(), slow_path->GetEntryLabel()); - } else { - DCHECK(value.IsConstant()) << value; - if (value.GetConstant()->AsIntConstant()->GetValue() == 0) { - __ b(slow_path->GetEntryLabel()); - } - } - break; - } - case Primitive::kPrimLong: { - if (value.IsRegisterPair()) { - __ orrs(IP, - value.AsRegisterPairLow<Register>(), - ShifterOperand(value.AsRegisterPairHigh<Register>())); - __ b(slow_path->GetEntryLabel(), EQ); - } else { - DCHECK(value.IsConstant()) << value; - if (value.GetConstant()->AsLongConstant()->GetValue() == 0) { - __ b(slow_path->GetEntryLabel()); - } - } - break; - default: - LOG(FATAL) << "Unexpected type for HDivZeroCheck " << instruction->GetType(); - } - } -} - -void InstructionCodeGeneratorARM::HandleIntegerRotate(LocationSummary* locations) { - Register in = locations->InAt(0).AsRegister<Register>(); - Location rhs = locations->InAt(1); - Register out = locations->Out().AsRegister<Register>(); - - if (rhs.IsConstant()) { - // Arm32 and Thumb2 assemblers require a rotation on the interval [1,31], - // so map all rotations to a +ve. equivalent in that range. - // (e.g. left *or* right by -2 bits == 30 bits in the same direction.) - uint32_t rot = CodeGenerator::GetInt32ValueOf(rhs.GetConstant()) & 0x1F; - if (rot) { - // Rotate, mapping left rotations to right equivalents if necessary. - // (e.g. left by 2 bits == right by 30.) - __ Ror(out, in, rot); - } else if (out != in) { - __ Mov(out, in); - } - } else { - __ Ror(out, in, rhs.AsRegister<Register>()); - } -} - -// Gain some speed by mapping all Long rotates onto equivalent pairs of Integer -// rotates by swapping input regs (effectively rotating by the first 32-bits of -// a larger rotation) or flipping direction (thus treating larger right/left -// rotations as sub-word sized rotations in the other direction) as appropriate. -void InstructionCodeGeneratorARM::HandleLongRotate(HRor* ror) { - LocationSummary* locations = ror->GetLocations(); - Register in_reg_lo = locations->InAt(0).AsRegisterPairLow<Register>(); - Register in_reg_hi = locations->InAt(0).AsRegisterPairHigh<Register>(); - Location rhs = locations->InAt(1); - Register out_reg_lo = locations->Out().AsRegisterPairLow<Register>(); - Register out_reg_hi = locations->Out().AsRegisterPairHigh<Register>(); - - if (rhs.IsConstant()) { - uint64_t rot = CodeGenerator::GetInt64ValueOf(rhs.GetConstant()); - // Map all rotations to +ve. equivalents on the interval [0,63]. - rot &= kMaxLongShiftDistance; - // For rotates over a word in size, 'pre-rotate' by 32-bits to keep rotate - // logic below to a simple pair of binary orr. - // (e.g. 34 bits == in_reg swap + 2 bits right.) - if (rot >= kArmBitsPerWord) { - rot -= kArmBitsPerWord; - std::swap(in_reg_hi, in_reg_lo); - } - // Rotate, or mov to out for zero or word size rotations. - if (rot != 0u) { - __ Lsr(out_reg_hi, in_reg_hi, rot); - __ orr(out_reg_hi, out_reg_hi, ShifterOperand(in_reg_lo, arm::LSL, kArmBitsPerWord - rot)); - __ Lsr(out_reg_lo, in_reg_lo, rot); - __ orr(out_reg_lo, out_reg_lo, ShifterOperand(in_reg_hi, arm::LSL, kArmBitsPerWord - rot)); - } else { - __ Mov(out_reg_lo, in_reg_lo); - __ Mov(out_reg_hi, in_reg_hi); - } - } else { - Register shift_right = locations->GetTemp(0).AsRegister<Register>(); - Register shift_left = locations->GetTemp(1).AsRegister<Register>(); - Label end; - Label shift_by_32_plus_shift_right; - Label* final_label = codegen_->GetFinalLabel(ror, &end); - - __ and_(shift_right, rhs.AsRegister<Register>(), ShifterOperand(0x1F)); - __ Lsrs(shift_left, rhs.AsRegister<Register>(), 6); - __ rsb(shift_left, shift_right, ShifterOperand(kArmBitsPerWord), AL, kCcKeep); - __ b(&shift_by_32_plus_shift_right, CC); - - // out_reg_hi = (reg_hi << shift_left) | (reg_lo >> shift_right). - // out_reg_lo = (reg_lo << shift_left) | (reg_hi >> shift_right). - __ Lsl(out_reg_hi, in_reg_hi, shift_left); - __ Lsr(out_reg_lo, in_reg_lo, shift_right); - __ add(out_reg_hi, out_reg_hi, ShifterOperand(out_reg_lo)); - __ Lsl(out_reg_lo, in_reg_lo, shift_left); - __ Lsr(shift_left, in_reg_hi, shift_right); - __ add(out_reg_lo, out_reg_lo, ShifterOperand(shift_left)); - __ b(final_label); - - __ Bind(&shift_by_32_plus_shift_right); // Shift by 32+shift_right. - // out_reg_hi = (reg_hi >> shift_right) | (reg_lo << shift_left). - // out_reg_lo = (reg_lo >> shift_right) | (reg_hi << shift_left). - __ Lsr(out_reg_hi, in_reg_hi, shift_right); - __ Lsl(out_reg_lo, in_reg_lo, shift_left); - __ add(out_reg_hi, out_reg_hi, ShifterOperand(out_reg_lo)); - __ Lsr(out_reg_lo, in_reg_lo, shift_right); - __ Lsl(shift_right, in_reg_hi, shift_left); - __ add(out_reg_lo, out_reg_lo, ShifterOperand(shift_right)); - - if (end.IsLinked()) { - __ Bind(&end); - } - } -} - -void LocationsBuilderARM::VisitRor(HRor* ror) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(ror, LocationSummary::kNoCall); - switch (ror->GetResultType()) { - case Primitive::kPrimInt: { - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RegisterOrConstant(ror->InputAt(1))); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); - break; - } - case Primitive::kPrimLong: { - locations->SetInAt(0, Location::RequiresRegister()); - if (ror->InputAt(1)->IsConstant()) { - locations->SetInAt(1, Location::ConstantLocation(ror->InputAt(1)->AsConstant())); - } else { - locations->SetInAt(1, Location::RequiresRegister()); - locations->AddTemp(Location::RequiresRegister()); - locations->AddTemp(Location::RequiresRegister()); - } - locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); - break; - } - default: - LOG(FATAL) << "Unexpected operation type " << ror->GetResultType(); - } -} - -void InstructionCodeGeneratorARM::VisitRor(HRor* ror) { - LocationSummary* locations = ror->GetLocations(); - Primitive::Type type = ror->GetResultType(); - switch (type) { - case Primitive::kPrimInt: { - HandleIntegerRotate(locations); - break; - } - case Primitive::kPrimLong: { - HandleLongRotate(ror); - break; - } - default: - LOG(FATAL) << "Unexpected operation type " << type; - UNREACHABLE(); - } -} - -void LocationsBuilderARM::HandleShift(HBinaryOperation* op) { - DCHECK(op->IsShl() || op->IsShr() || op->IsUShr()); - - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(op, LocationSummary::kNoCall); - - switch (op->GetResultType()) { - case Primitive::kPrimInt: { - locations->SetInAt(0, Location::RequiresRegister()); - if (op->InputAt(1)->IsConstant()) { - locations->SetInAt(1, Location::ConstantLocation(op->InputAt(1)->AsConstant())); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); - } else { - locations->SetInAt(1, Location::RequiresRegister()); - // Make the output overlap, as it will be used to hold the masked - // second input. - locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); - } - break; - } - case Primitive::kPrimLong: { - locations->SetInAt(0, Location::RequiresRegister()); - if (op->InputAt(1)->IsConstant()) { - locations->SetInAt(1, Location::ConstantLocation(op->InputAt(1)->AsConstant())); - // For simplicity, use kOutputOverlap even though we only require that low registers - // don't clash with high registers which the register allocator currently guarantees. - locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); - } else { - locations->SetInAt(1, Location::RequiresRegister()); - locations->AddTemp(Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); - } - break; - } - default: - LOG(FATAL) << "Unexpected operation type " << op->GetResultType(); - } -} - -void InstructionCodeGeneratorARM::HandleShift(HBinaryOperation* op) { - DCHECK(op->IsShl() || op->IsShr() || op->IsUShr()); - - LocationSummary* locations = op->GetLocations(); - Location out = locations->Out(); - Location first = locations->InAt(0); - Location second = locations->InAt(1); - - Primitive::Type type = op->GetResultType(); - switch (type) { - case Primitive::kPrimInt: { - Register out_reg = out.AsRegister<Register>(); - Register first_reg = first.AsRegister<Register>(); - if (second.IsRegister()) { - Register second_reg = second.AsRegister<Register>(); - // ARM doesn't mask the shift count so we need to do it ourselves. - __ and_(out_reg, second_reg, ShifterOperand(kMaxIntShiftDistance)); - if (op->IsShl()) { - __ Lsl(out_reg, first_reg, out_reg); - } else if (op->IsShr()) { - __ Asr(out_reg, first_reg, out_reg); - } else { - __ Lsr(out_reg, first_reg, out_reg); - } - } else { - int32_t cst = second.GetConstant()->AsIntConstant()->GetValue(); - uint32_t shift_value = cst & kMaxIntShiftDistance; - if (shift_value == 0) { // ARM does not support shifting with 0 immediate. - __ Mov(out_reg, first_reg); - } else if (op->IsShl()) { - __ Lsl(out_reg, first_reg, shift_value); - } else if (op->IsShr()) { - __ Asr(out_reg, first_reg, shift_value); - } else { - __ Lsr(out_reg, first_reg, shift_value); - } - } - break; - } - case Primitive::kPrimLong: { - Register o_h = out.AsRegisterPairHigh<Register>(); - Register o_l = out.AsRegisterPairLow<Register>(); - - Register high = first.AsRegisterPairHigh<Register>(); - Register low = first.AsRegisterPairLow<Register>(); - - if (second.IsRegister()) { - Register temp = locations->GetTemp(0).AsRegister<Register>(); - - Register second_reg = second.AsRegister<Register>(); - - if (op->IsShl()) { - __ and_(o_l, second_reg, ShifterOperand(kMaxLongShiftDistance)); - // Shift the high part - __ Lsl(o_h, high, o_l); - // Shift the low part and `or` what overflew on the high part - __ rsb(temp, o_l, ShifterOperand(kArmBitsPerWord)); - __ Lsr(temp, low, temp); - __ orr(o_h, o_h, ShifterOperand(temp)); - // If the shift is > 32 bits, override the high part - __ subs(temp, o_l, ShifterOperand(kArmBitsPerWord)); - __ it(PL); - __ Lsl(o_h, low, temp, PL); - // Shift the low part - __ Lsl(o_l, low, o_l); - } else if (op->IsShr()) { - __ and_(o_h, second_reg, ShifterOperand(kMaxLongShiftDistance)); - // Shift the low part - __ Lsr(o_l, low, o_h); - // Shift the high part and `or` what underflew on the low part - __ rsb(temp, o_h, ShifterOperand(kArmBitsPerWord)); - __ Lsl(temp, high, temp); - __ orr(o_l, o_l, ShifterOperand(temp)); - // If the shift is > 32 bits, override the low part - __ subs(temp, o_h, ShifterOperand(kArmBitsPerWord)); - __ it(PL); - __ Asr(o_l, high, temp, PL); - // Shift the high part - __ Asr(o_h, high, o_h); - } else { - __ and_(o_h, second_reg, ShifterOperand(kMaxLongShiftDistance)); - // same as Shr except we use `Lsr`s and not `Asr`s - __ Lsr(o_l, low, o_h); - __ rsb(temp, o_h, ShifterOperand(kArmBitsPerWord)); - __ Lsl(temp, high, temp); - __ orr(o_l, o_l, ShifterOperand(temp)); - __ subs(temp, o_h, ShifterOperand(kArmBitsPerWord)); - __ it(PL); - __ Lsr(o_l, high, temp, PL); - __ Lsr(o_h, high, o_h); - } - } else { - // Register allocator doesn't create partial overlap. - DCHECK_NE(o_l, high); - DCHECK_NE(o_h, low); - int32_t cst = second.GetConstant()->AsIntConstant()->GetValue(); - uint32_t shift_value = cst & kMaxLongShiftDistance; - if (shift_value > 32) { - if (op->IsShl()) { - __ Lsl(o_h, low, shift_value - 32); - __ LoadImmediate(o_l, 0); - } else if (op->IsShr()) { - __ Asr(o_l, high, shift_value - 32); - __ Asr(o_h, high, 31); - } else { - __ Lsr(o_l, high, shift_value - 32); - __ LoadImmediate(o_h, 0); - } - } else if (shift_value == 32) { - if (op->IsShl()) { - __ mov(o_h, ShifterOperand(low)); - __ LoadImmediate(o_l, 0); - } else if (op->IsShr()) { - __ mov(o_l, ShifterOperand(high)); - __ Asr(o_h, high, 31); - } else { - __ mov(o_l, ShifterOperand(high)); - __ LoadImmediate(o_h, 0); - } - } else if (shift_value == 1) { - if (op->IsShl()) { - __ Lsls(o_l, low, 1); - __ adc(o_h, high, ShifterOperand(high)); - } else if (op->IsShr()) { - __ Asrs(o_h, high, 1); - __ Rrx(o_l, low); - } else { - __ Lsrs(o_h, high, 1); - __ Rrx(o_l, low); - } - } else { - DCHECK(2 <= shift_value && shift_value < 32) << shift_value; - if (op->IsShl()) { - __ Lsl(o_h, high, shift_value); - __ orr(o_h, o_h, ShifterOperand(low, LSR, 32 - shift_value)); - __ Lsl(o_l, low, shift_value); - } else if (op->IsShr()) { - __ Lsr(o_l, low, shift_value); - __ orr(o_l, o_l, ShifterOperand(high, LSL, 32 - shift_value)); - __ Asr(o_h, high, shift_value); - } else { - __ Lsr(o_l, low, shift_value); - __ orr(o_l, o_l, ShifterOperand(high, LSL, 32 - shift_value)); - __ Lsr(o_h, high, shift_value); - } - } - } - break; - } - default: - LOG(FATAL) << "Unexpected operation type " << type; - UNREACHABLE(); - } -} - -void LocationsBuilderARM::VisitShl(HShl* shl) { - HandleShift(shl); -} - -void InstructionCodeGeneratorARM::VisitShl(HShl* shl) { - HandleShift(shl); -} - -void LocationsBuilderARM::VisitShr(HShr* shr) { - HandleShift(shr); -} - -void InstructionCodeGeneratorARM::VisitShr(HShr* shr) { - HandleShift(shr); -} - -void LocationsBuilderARM::VisitUShr(HUShr* ushr) { - HandleShift(ushr); -} - -void InstructionCodeGeneratorARM::VisitUShr(HUShr* ushr) { - HandleShift(ushr); -} - -void LocationsBuilderARM::VisitNewInstance(HNewInstance* instruction) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly); - if (instruction->IsStringAlloc()) { - locations->AddTemp(Location::RegisterLocation(kMethodRegisterArgument)); - } else { - InvokeRuntimeCallingConvention calling_convention; - locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); - } - locations->SetOut(Location::RegisterLocation(R0)); -} - -void InstructionCodeGeneratorARM::VisitNewInstance(HNewInstance* instruction) { - // Note: if heap poisoning is enabled, the entry point takes cares - // of poisoning the reference. - if (instruction->IsStringAlloc()) { - // String is allocated through StringFactory. Call NewEmptyString entry point. - Register temp = instruction->GetLocations()->GetTemp(0).AsRegister<Register>(); - MemberOffset code_offset = ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmPointerSize); - __ LoadFromOffset(kLoadWord, temp, TR, QUICK_ENTRY_POINT(pNewEmptyString)); - __ LoadFromOffset(kLoadWord, LR, temp, code_offset.Int32Value()); - __ blx(LR); - codegen_->RecordPcInfo(instruction, instruction->GetDexPc()); - } else { - codegen_->InvokeRuntime(instruction->GetEntrypoint(), instruction, instruction->GetDexPc()); - CheckEntrypointTypes<kQuickAllocObjectWithChecks, void*, mirror::Class*>(); - } -} - -void LocationsBuilderARM::VisitNewArray(HNewArray* instruction) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly); - InvokeRuntimeCallingConvention calling_convention; - locations->SetOut(Location::RegisterLocation(R0)); - locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); - locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); -} - -void InstructionCodeGeneratorARM::VisitNewArray(HNewArray* instruction) { - // Note: if heap poisoning is enabled, the entry point takes cares - // of poisoning the reference. - QuickEntrypointEnum entrypoint = - CodeGenerator::GetArrayAllocationEntrypoint(instruction->GetLoadClass()->GetClass()); - codegen_->InvokeRuntime(entrypoint, instruction, instruction->GetDexPc()); - CheckEntrypointTypes<kQuickAllocArrayResolved, void*, mirror::Class*, int32_t>(); - DCHECK(!codegen_->IsLeafMethod()); -} - -void LocationsBuilderARM::VisitParameterValue(HParameterValue* instruction) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); - Location location = parameter_visitor_.GetNextLocation(instruction->GetType()); - if (location.IsStackSlot()) { - location = Location::StackSlot(location.GetStackIndex() + codegen_->GetFrameSize()); - } else if (location.IsDoubleStackSlot()) { - location = Location::DoubleStackSlot(location.GetStackIndex() + codegen_->GetFrameSize()); - } - locations->SetOut(location); -} - -void InstructionCodeGeneratorARM::VisitParameterValue( - HParameterValue* instruction ATTRIBUTE_UNUSED) { - // Nothing to do, the parameter is already at its location. -} - -void LocationsBuilderARM::VisitCurrentMethod(HCurrentMethod* instruction) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); - locations->SetOut(Location::RegisterLocation(kMethodRegisterArgument)); -} - -void InstructionCodeGeneratorARM::VisitCurrentMethod(HCurrentMethod* instruction ATTRIBUTE_UNUSED) { - // Nothing to do, the method is already at its location. -} - -void LocationsBuilderARM::VisitNot(HNot* not_) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(not_, LocationSummary::kNoCall); - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); -} - -void InstructionCodeGeneratorARM::VisitNot(HNot* not_) { - LocationSummary* locations = not_->GetLocations(); - Location out = locations->Out(); - Location in = locations->InAt(0); - switch (not_->GetResultType()) { - case Primitive::kPrimInt: - __ mvn(out.AsRegister<Register>(), ShifterOperand(in.AsRegister<Register>())); - break; - - case Primitive::kPrimLong: - __ mvn(out.AsRegisterPairLow<Register>(), - ShifterOperand(in.AsRegisterPairLow<Register>())); - __ mvn(out.AsRegisterPairHigh<Register>(), - ShifterOperand(in.AsRegisterPairHigh<Register>())); - break; - - default: - LOG(FATAL) << "Unimplemented type for not operation " << not_->GetResultType(); - } -} - -void LocationsBuilderARM::VisitBooleanNot(HBooleanNot* bool_not) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(bool_not, LocationSummary::kNoCall); - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); -} - -void InstructionCodeGeneratorARM::VisitBooleanNot(HBooleanNot* bool_not) { - LocationSummary* locations = bool_not->GetLocations(); - Location out = locations->Out(); - Location in = locations->InAt(0); - __ eor(out.AsRegister<Register>(), in.AsRegister<Register>(), ShifterOperand(1)); -} - -void LocationsBuilderARM::VisitCompare(HCompare* compare) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(compare, LocationSummary::kNoCall); - switch (compare->InputAt(0)->GetType()) { - case Primitive::kPrimBoolean: - case Primitive::kPrimByte: - case Primitive::kPrimShort: - case Primitive::kPrimChar: - case Primitive::kPrimInt: - case Primitive::kPrimLong: { - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RequiresRegister()); - // Output overlaps because it is written before doing the low comparison. - locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); - break; - } - case Primitive::kPrimFloat: - case Primitive::kPrimDouble: { - locations->SetInAt(0, Location::RequiresFpuRegister()); - locations->SetInAt(1, ArithmeticZeroOrFpuRegister(compare->InputAt(1))); - locations->SetOut(Location::RequiresRegister()); - break; - } - default: - LOG(FATAL) << "Unexpected type for compare operation " << compare->InputAt(0)->GetType(); - } -} - -void InstructionCodeGeneratorARM::VisitCompare(HCompare* compare) { - LocationSummary* locations = compare->GetLocations(); - Register out = locations->Out().AsRegister<Register>(); - Location left = locations->InAt(0); - Location right = locations->InAt(1); - - Label less, greater, done; - Label* final_label = codegen_->GetFinalLabel(compare, &done); - Primitive::Type type = compare->InputAt(0)->GetType(); - Condition less_cond; - switch (type) { - case Primitive::kPrimBoolean: - case Primitive::kPrimByte: - case Primitive::kPrimShort: - case Primitive::kPrimChar: - case Primitive::kPrimInt: { - __ LoadImmediate(out, 0); - __ cmp(left.AsRegister<Register>(), - ShifterOperand(right.AsRegister<Register>())); // Signed compare. - less_cond = LT; - break; - } - case Primitive::kPrimLong: { - __ cmp(left.AsRegisterPairHigh<Register>(), - ShifterOperand(right.AsRegisterPairHigh<Register>())); // Signed compare. - __ b(&less, LT); - __ b(&greater, GT); - // Do LoadImmediate before the last `cmp`, as LoadImmediate might affect the status flags. - __ LoadImmediate(out, 0); - __ cmp(left.AsRegisterPairLow<Register>(), - ShifterOperand(right.AsRegisterPairLow<Register>())); // Unsigned compare. - less_cond = LO; - break; - } - case Primitive::kPrimFloat: - case Primitive::kPrimDouble: { - __ LoadImmediate(out, 0); - GenerateVcmp(compare, codegen_); - __ vmstat(); // transfer FP status register to ARM APSR. - less_cond = ARMFPCondition(kCondLT, compare->IsGtBias()); - break; - } - default: - LOG(FATAL) << "Unexpected compare type " << type; - UNREACHABLE(); - } - - __ b(final_label, EQ); - __ b(&less, less_cond); - - __ Bind(&greater); - __ LoadImmediate(out, 1); - __ b(final_label); - - __ Bind(&less); - __ LoadImmediate(out, -1); - - if (done.IsLinked()) { - __ Bind(&done); - } -} - -void LocationsBuilderARM::VisitPhi(HPhi* instruction) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); - for (size_t i = 0, e = locations->GetInputCount(); i < e; ++i) { - locations->SetInAt(i, Location::Any()); - } - locations->SetOut(Location::Any()); -} - -void InstructionCodeGeneratorARM::VisitPhi(HPhi* instruction ATTRIBUTE_UNUSED) { - LOG(FATAL) << "Unreachable"; -} - -void CodeGeneratorARM::GenerateMemoryBarrier(MemBarrierKind kind) { - // TODO (ported from quick): revisit ARM barrier kinds. - DmbOptions flavor = DmbOptions::ISH; // Quiet C++ warnings. - switch (kind) { - case MemBarrierKind::kAnyStore: - case MemBarrierKind::kLoadAny: - case MemBarrierKind::kAnyAny: { - flavor = DmbOptions::ISH; - break; - } - case MemBarrierKind::kStoreStore: { - flavor = DmbOptions::ISHST; - break; - } - default: - LOG(FATAL) << "Unexpected memory barrier " << kind; - } - __ dmb(flavor); -} - -void InstructionCodeGeneratorARM::GenerateWideAtomicLoad(Register addr, - uint32_t offset, - Register out_lo, - Register out_hi) { - if (offset != 0) { - // Ensure `out_lo` is different from `addr`, so that loading - // `offset` into `out_lo` does not clutter `addr`. - DCHECK_NE(out_lo, addr); - __ LoadImmediate(out_lo, offset); - __ add(IP, addr, ShifterOperand(out_lo)); - addr = IP; - } - __ ldrexd(out_lo, out_hi, addr); -} - -void InstructionCodeGeneratorARM::GenerateWideAtomicStore(Register addr, - uint32_t offset, - Register value_lo, - Register value_hi, - Register temp1, - Register temp2, - HInstruction* instruction) { - Label fail; - if (offset != 0) { - __ LoadImmediate(temp1, offset); - __ add(IP, addr, ShifterOperand(temp1)); - addr = IP; - } - __ Bind(&fail); - // We need a load followed by store. (The address used in a STREX instruction must - // be the same as the address in the most recently executed LDREX instruction.) - __ ldrexd(temp1, temp2, addr); - codegen_->MaybeRecordImplicitNullCheck(instruction); - __ strexd(temp1, value_lo, value_hi, addr); - __ CompareAndBranchIfNonZero(temp1, &fail); -} - -void LocationsBuilderARM::HandleFieldSet(HInstruction* instruction, const FieldInfo& field_info) { - DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet()); - - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); - locations->SetInAt(0, Location::RequiresRegister()); - - Primitive::Type field_type = field_info.GetFieldType(); - if (Primitive::IsFloatingPointType(field_type)) { - locations->SetInAt(1, Location::RequiresFpuRegister()); - } else { - locations->SetInAt(1, Location::RequiresRegister()); - } - - bool is_wide = field_type == Primitive::kPrimLong || field_type == Primitive::kPrimDouble; - bool generate_volatile = field_info.IsVolatile() - && is_wide - && !codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd(); - bool needs_write_barrier = - CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1)); - // Temporary registers for the write barrier. - // TODO: consider renaming StoreNeedsWriteBarrier to StoreNeedsGCMark. - if (needs_write_barrier) { - locations->AddTemp(Location::RequiresRegister()); // Possibly used for reference poisoning too. - locations->AddTemp(Location::RequiresRegister()); - } else if (generate_volatile) { - // ARM encoding have some additional constraints for ldrexd/strexd: - // - registers need to be consecutive - // - the first register should be even but not R14. - // We don't test for ARM yet, and the assertion makes sure that we - // revisit this if we ever enable ARM encoding. - DCHECK_EQ(InstructionSet::kThumb2, codegen_->GetInstructionSet()); - - locations->AddTemp(Location::RequiresRegister()); - locations->AddTemp(Location::RequiresRegister()); - if (field_type == Primitive::kPrimDouble) { - // For doubles we need two more registers to copy the value. - locations->AddTemp(Location::RegisterLocation(R2)); - locations->AddTemp(Location::RegisterLocation(R3)); - } - } -} - -void InstructionCodeGeneratorARM::HandleFieldSet(HInstruction* instruction, - const FieldInfo& field_info, - bool value_can_be_null) { - DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet()); - - LocationSummary* locations = instruction->GetLocations(); - Register base = locations->InAt(0).AsRegister<Register>(); - Location value = locations->InAt(1); - - bool is_volatile = field_info.IsVolatile(); - bool atomic_ldrd_strd = codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd(); - Primitive::Type field_type = field_info.GetFieldType(); - uint32_t offset = field_info.GetFieldOffset().Uint32Value(); - bool needs_write_barrier = - CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1)); - - if (is_volatile) { - codegen_->GenerateMemoryBarrier(MemBarrierKind::kAnyStore); - } - - switch (field_type) { - case Primitive::kPrimBoolean: - case Primitive::kPrimByte: { - __ StoreToOffset(kStoreByte, value.AsRegister<Register>(), base, offset); - break; - } - - case Primitive::kPrimShort: - case Primitive::kPrimChar: { - __ StoreToOffset(kStoreHalfword, value.AsRegister<Register>(), base, offset); - break; - } - - case Primitive::kPrimInt: - case Primitive::kPrimNot: { - if (kPoisonHeapReferences && needs_write_barrier) { - // Note that in the case where `value` is a null reference, - // we do not enter this block, as a null reference does not - // need poisoning. - DCHECK_EQ(field_type, Primitive::kPrimNot); - Register temp = locations->GetTemp(0).AsRegister<Register>(); - __ Mov(temp, value.AsRegister<Register>()); - __ PoisonHeapReference(temp); - __ StoreToOffset(kStoreWord, temp, base, offset); - } else { - __ StoreToOffset(kStoreWord, value.AsRegister<Register>(), base, offset); - } - break; - } - - case Primitive::kPrimLong: { - if (is_volatile && !atomic_ldrd_strd) { - GenerateWideAtomicStore(base, offset, - value.AsRegisterPairLow<Register>(), - value.AsRegisterPairHigh<Register>(), - locations->GetTemp(0).AsRegister<Register>(), - locations->GetTemp(1).AsRegister<Register>(), - instruction); - } else { - __ StoreToOffset(kStoreWordPair, value.AsRegisterPairLow<Register>(), base, offset); - codegen_->MaybeRecordImplicitNullCheck(instruction); - } - break; - } - - case Primitive::kPrimFloat: { - __ StoreSToOffset(value.AsFpuRegister<SRegister>(), base, offset); - break; - } - - case Primitive::kPrimDouble: { - DRegister value_reg = FromLowSToD(value.AsFpuRegisterPairLow<SRegister>()); - if (is_volatile && !atomic_ldrd_strd) { - Register value_reg_lo = locations->GetTemp(0).AsRegister<Register>(); - Register value_reg_hi = locations->GetTemp(1).AsRegister<Register>(); - - __ vmovrrd(value_reg_lo, value_reg_hi, value_reg); - - GenerateWideAtomicStore(base, offset, - value_reg_lo, - value_reg_hi, - locations->GetTemp(2).AsRegister<Register>(), - locations->GetTemp(3).AsRegister<Register>(), - instruction); - } else { - __ StoreDToOffset(value_reg, base, offset); - codegen_->MaybeRecordImplicitNullCheck(instruction); - } - break; - } - - case Primitive::kPrimVoid: - LOG(FATAL) << "Unreachable type " << field_type; - UNREACHABLE(); - } - - // Longs and doubles are handled in the switch. - if (field_type != Primitive::kPrimLong && field_type != Primitive::kPrimDouble) { - codegen_->MaybeRecordImplicitNullCheck(instruction); - } - - if (CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1))) { - Register temp = locations->GetTemp(0).AsRegister<Register>(); - Register card = locations->GetTemp(1).AsRegister<Register>(); - codegen_->MarkGCCard( - temp, card, base, value.AsRegister<Register>(), value_can_be_null); - } - - if (is_volatile) { - codegen_->GenerateMemoryBarrier(MemBarrierKind::kAnyAny); - } -} - -void LocationsBuilderARM::HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info) { - DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet()); - - bool object_field_get_with_read_barrier = - kEmitCompilerReadBarrier && (field_info.GetFieldType() == Primitive::kPrimNot); - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(instruction, - object_field_get_with_read_barrier ? - LocationSummary::kCallOnSlowPath : - LocationSummary::kNoCall); - if (object_field_get_with_read_barrier && kUseBakerReadBarrier) { - locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. - } - locations->SetInAt(0, Location::RequiresRegister()); - - bool volatile_for_double = field_info.IsVolatile() - && (field_info.GetFieldType() == Primitive::kPrimDouble) - && !codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd(); - // The output overlaps in case of volatile long: we don't want the - // code generated by GenerateWideAtomicLoad to overwrite the - // object's location. Likewise, in the case of an object field get - // with read barriers enabled, we do not want the load to overwrite - // the object's location, as we need it to emit the read barrier. - bool overlap = (field_info.IsVolatile() && (field_info.GetFieldType() == Primitive::kPrimLong)) || - object_field_get_with_read_barrier; - - if (Primitive::IsFloatingPointType(instruction->GetType())) { - locations->SetOut(Location::RequiresFpuRegister()); - } else { - locations->SetOut(Location::RequiresRegister(), - (overlap ? Location::kOutputOverlap : Location::kNoOutputOverlap)); - } - if (volatile_for_double) { - // ARM encoding have some additional constraints for ldrexd/strexd: - // - registers need to be consecutive - // - the first register should be even but not R14. - // We don't test for ARM yet, and the assertion makes sure that we - // revisit this if we ever enable ARM encoding. - DCHECK_EQ(InstructionSet::kThumb2, codegen_->GetInstructionSet()); - locations->AddTemp(Location::RequiresRegister()); - locations->AddTemp(Location::RequiresRegister()); - } else if (object_field_get_with_read_barrier && kUseBakerReadBarrier) { - // We need a temporary register for the read barrier marking slow - // path in CodeGeneratorARM::GenerateFieldLoadWithBakerReadBarrier. - if (kBakerReadBarrierLinkTimeThunksEnableForFields && - !Runtime::Current()->UseJitCompilation()) { - // If link-time thunks for the Baker read barrier are enabled, for AOT - // loads we need a temporary only if the offset is too big. - if (field_info.GetFieldOffset().Uint32Value() >= kReferenceLoadMinFarOffset) { - locations->AddTemp(Location::RequiresRegister()); - } - // And we always need the reserved entrypoint register. - locations->AddTemp(Location::RegisterLocation(kBakerCcEntrypointRegister)); - } else { - locations->AddTemp(Location::RequiresRegister()); - } - } -} - -Location LocationsBuilderARM::ArithmeticZeroOrFpuRegister(HInstruction* input) { - DCHECK(input->GetType() == Primitive::kPrimDouble || input->GetType() == Primitive::kPrimFloat) - << input->GetType(); - if ((input->IsFloatConstant() && (input->AsFloatConstant()->IsArithmeticZero())) || - (input->IsDoubleConstant() && (input->AsDoubleConstant()->IsArithmeticZero()))) { - return Location::ConstantLocation(input->AsConstant()); - } else { - return Location::RequiresFpuRegister(); - } -} - -Location LocationsBuilderARM::ArmEncodableConstantOrRegister(HInstruction* constant, - Opcode opcode) { - DCHECK(!Primitive::IsFloatingPointType(constant->GetType())); - if (constant->IsConstant() && - CanEncodeConstantAsImmediate(constant->AsConstant(), opcode)) { - return Location::ConstantLocation(constant->AsConstant()); - } - return Location::RequiresRegister(); -} - -bool LocationsBuilderARM::CanEncodeConstantAsImmediate(HConstant* input_cst, - Opcode opcode) { - uint64_t value = static_cast<uint64_t>(Int64FromConstant(input_cst)); - if (Primitive::Is64BitType(input_cst->GetType())) { - Opcode high_opcode = opcode; - SetCc low_set_cc = kCcDontCare; - switch (opcode) { - case SUB: - // Flip the operation to an ADD. - value = -value; - opcode = ADD; - FALLTHROUGH_INTENDED; - case ADD: - if (Low32Bits(value) == 0u) { - return CanEncodeConstantAsImmediate(High32Bits(value), opcode, kCcDontCare); - } - high_opcode = ADC; - low_set_cc = kCcSet; - break; - default: - break; - } - return CanEncodeConstantAsImmediate(Low32Bits(value), opcode, low_set_cc) && - CanEncodeConstantAsImmediate(High32Bits(value), high_opcode, kCcDontCare); - } else { - return CanEncodeConstantAsImmediate(Low32Bits(value), opcode); - } -} - -bool LocationsBuilderARM::CanEncodeConstantAsImmediate(uint32_t value, - Opcode opcode, - SetCc set_cc) { - ShifterOperand so; - ArmAssembler* assembler = codegen_->GetAssembler(); - if (assembler->ShifterOperandCanHold(kNoRegister, kNoRegister, opcode, value, set_cc, &so)) { - return true; - } - Opcode neg_opcode = kNoOperand; - uint32_t neg_value = 0; - switch (opcode) { - case AND: neg_opcode = BIC; neg_value = ~value; break; - case ORR: neg_opcode = ORN; neg_value = ~value; break; - case ADD: neg_opcode = SUB; neg_value = -value; break; - case ADC: neg_opcode = SBC; neg_value = ~value; break; - case SUB: neg_opcode = ADD; neg_value = -value; break; - case SBC: neg_opcode = ADC; neg_value = ~value; break; - case MOV: neg_opcode = MVN; neg_value = ~value; break; - default: - return false; - } - - if (assembler->ShifterOperandCanHold(kNoRegister, - kNoRegister, - neg_opcode, - neg_value, - set_cc, - &so)) { - return true; - } - - return opcode == AND && IsPowerOfTwo(value + 1); -} - -void InstructionCodeGeneratorARM::HandleFieldGet(HInstruction* instruction, - const FieldInfo& field_info) { - DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet()); - - LocationSummary* locations = instruction->GetLocations(); - Location base_loc = locations->InAt(0); - Register base = base_loc.AsRegister<Register>(); - Location out = locations->Out(); - bool is_volatile = field_info.IsVolatile(); - bool atomic_ldrd_strd = codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd(); - Primitive::Type field_type = field_info.GetFieldType(); - uint32_t offset = field_info.GetFieldOffset().Uint32Value(); - - switch (field_type) { - case Primitive::kPrimBoolean: - __ LoadFromOffset(kLoadUnsignedByte, out.AsRegister<Register>(), base, offset); - break; - - case Primitive::kPrimByte: - __ LoadFromOffset(kLoadSignedByte, out.AsRegister<Register>(), base, offset); - break; - - case Primitive::kPrimShort: - __ LoadFromOffset(kLoadSignedHalfword, out.AsRegister<Register>(), base, offset); - break; - - case Primitive::kPrimChar: - __ LoadFromOffset(kLoadUnsignedHalfword, out.AsRegister<Register>(), base, offset); - break; - - case Primitive::kPrimInt: - __ LoadFromOffset(kLoadWord, out.AsRegister<Register>(), base, offset); - break; - - case Primitive::kPrimNot: { - // /* HeapReference<Object> */ out = *(base + offset) - if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { - Location temp_loc = locations->GetTemp(0); - // Note that a potential implicit null check is handled in this - // CodeGeneratorARM::GenerateFieldLoadWithBakerReadBarrier call. - codegen_->GenerateFieldLoadWithBakerReadBarrier( - instruction, out, base, offset, temp_loc, /* needs_null_check */ true); - if (is_volatile) { - codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny); - } - } else { - __ LoadFromOffset(kLoadWord, out.AsRegister<Register>(), base, offset); - codegen_->MaybeRecordImplicitNullCheck(instruction); - if (is_volatile) { - codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny); - } - // If read barriers are enabled, emit read barriers other than - // Baker's using a slow path (and also unpoison the loaded - // reference, if heap poisoning is enabled). - codegen_->MaybeGenerateReadBarrierSlow(instruction, out, out, base_loc, offset); - } - break; - } - - case Primitive::kPrimLong: - if (is_volatile && !atomic_ldrd_strd) { - GenerateWideAtomicLoad(base, offset, - out.AsRegisterPairLow<Register>(), - out.AsRegisterPairHigh<Register>()); - } else { - __ LoadFromOffset(kLoadWordPair, out.AsRegisterPairLow<Register>(), base, offset); - } - break; - - case Primitive::kPrimFloat: - __ LoadSFromOffset(out.AsFpuRegister<SRegister>(), base, offset); - break; - - case Primitive::kPrimDouble: { - DRegister out_reg = FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()); - if (is_volatile && !atomic_ldrd_strd) { - Register lo = locations->GetTemp(0).AsRegister<Register>(); - Register hi = locations->GetTemp(1).AsRegister<Register>(); - GenerateWideAtomicLoad(base, offset, lo, hi); - codegen_->MaybeRecordImplicitNullCheck(instruction); - __ vmovdrr(out_reg, lo, hi); - } else { - __ LoadDFromOffset(out_reg, base, offset); - codegen_->MaybeRecordImplicitNullCheck(instruction); - } - break; - } - - case Primitive::kPrimVoid: - LOG(FATAL) << "Unreachable type " << field_type; - UNREACHABLE(); - } - - if (field_type == Primitive::kPrimNot || field_type == Primitive::kPrimDouble) { - // Potential implicit null checks, in the case of reference or - // double fields, are handled in the previous switch statement. - } else { - codegen_->MaybeRecordImplicitNullCheck(instruction); - } - - if (is_volatile) { - if (field_type == Primitive::kPrimNot) { - // Memory barriers, in the case of references, are also handled - // in the previous switch statement. - } else { - codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny); - } - } -} - -void LocationsBuilderARM::VisitInstanceFieldSet(HInstanceFieldSet* instruction) { - HandleFieldSet(instruction, instruction->GetFieldInfo()); -} - -void InstructionCodeGeneratorARM::VisitInstanceFieldSet(HInstanceFieldSet* instruction) { - HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull()); -} - -void LocationsBuilderARM::VisitInstanceFieldGet(HInstanceFieldGet* instruction) { - HandleFieldGet(instruction, instruction->GetFieldInfo()); -} - -void InstructionCodeGeneratorARM::VisitInstanceFieldGet(HInstanceFieldGet* instruction) { - HandleFieldGet(instruction, instruction->GetFieldInfo()); -} - -void LocationsBuilderARM::VisitStaticFieldGet(HStaticFieldGet* instruction) { - HandleFieldGet(instruction, instruction->GetFieldInfo()); -} - -void InstructionCodeGeneratorARM::VisitStaticFieldGet(HStaticFieldGet* instruction) { - HandleFieldGet(instruction, instruction->GetFieldInfo()); -} - -void LocationsBuilderARM::VisitStaticFieldSet(HStaticFieldSet* instruction) { - HandleFieldSet(instruction, instruction->GetFieldInfo()); -} - -void InstructionCodeGeneratorARM::VisitStaticFieldSet(HStaticFieldSet* instruction) { - HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull()); -} - -void LocationsBuilderARM::VisitUnresolvedInstanceFieldGet( - HUnresolvedInstanceFieldGet* instruction) { - FieldAccessCallingConventionARM calling_convention; - codegen_->CreateUnresolvedFieldLocationSummary( - instruction, instruction->GetFieldType(), calling_convention); -} - -void InstructionCodeGeneratorARM::VisitUnresolvedInstanceFieldGet( - HUnresolvedInstanceFieldGet* instruction) { - FieldAccessCallingConventionARM calling_convention; - codegen_->GenerateUnresolvedFieldAccess(instruction, - instruction->GetFieldType(), - instruction->GetFieldIndex(), - instruction->GetDexPc(), - calling_convention); -} - -void LocationsBuilderARM::VisitUnresolvedInstanceFieldSet( - HUnresolvedInstanceFieldSet* instruction) { - FieldAccessCallingConventionARM calling_convention; - codegen_->CreateUnresolvedFieldLocationSummary( - instruction, instruction->GetFieldType(), calling_convention); -} - -void InstructionCodeGeneratorARM::VisitUnresolvedInstanceFieldSet( - HUnresolvedInstanceFieldSet* instruction) { - FieldAccessCallingConventionARM calling_convention; - codegen_->GenerateUnresolvedFieldAccess(instruction, - instruction->GetFieldType(), - instruction->GetFieldIndex(), - instruction->GetDexPc(), - calling_convention); -} - -void LocationsBuilderARM::VisitUnresolvedStaticFieldGet( - HUnresolvedStaticFieldGet* instruction) { - FieldAccessCallingConventionARM calling_convention; - codegen_->CreateUnresolvedFieldLocationSummary( - instruction, instruction->GetFieldType(), calling_convention); -} - -void InstructionCodeGeneratorARM::VisitUnresolvedStaticFieldGet( - HUnresolvedStaticFieldGet* instruction) { - FieldAccessCallingConventionARM calling_convention; - codegen_->GenerateUnresolvedFieldAccess(instruction, - instruction->GetFieldType(), - instruction->GetFieldIndex(), - instruction->GetDexPc(), - calling_convention); -} - -void LocationsBuilderARM::VisitUnresolvedStaticFieldSet( - HUnresolvedStaticFieldSet* instruction) { - FieldAccessCallingConventionARM calling_convention; - codegen_->CreateUnresolvedFieldLocationSummary( - instruction, instruction->GetFieldType(), calling_convention); -} - -void InstructionCodeGeneratorARM::VisitUnresolvedStaticFieldSet( - HUnresolvedStaticFieldSet* instruction) { - FieldAccessCallingConventionARM calling_convention; - codegen_->GenerateUnresolvedFieldAccess(instruction, - instruction->GetFieldType(), - instruction->GetFieldIndex(), - instruction->GetDexPc(), - calling_convention); -} - -void LocationsBuilderARM::VisitNullCheck(HNullCheck* instruction) { - LocationSummary* locations = codegen_->CreateThrowingSlowPathLocations(instruction); - locations->SetInAt(0, Location::RequiresRegister()); -} - -void CodeGeneratorARM::GenerateImplicitNullCheck(HNullCheck* instruction) { - if (CanMoveNullCheckToUser(instruction)) { - return; - } - Location obj = instruction->GetLocations()->InAt(0); - - __ LoadFromOffset(kLoadWord, IP, obj.AsRegister<Register>(), 0); - RecordPcInfo(instruction, instruction->GetDexPc()); -} - -void CodeGeneratorARM::GenerateExplicitNullCheck(HNullCheck* instruction) { - SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) NullCheckSlowPathARM(instruction); - AddSlowPath(slow_path); - - LocationSummary* locations = instruction->GetLocations(); - Location obj = locations->InAt(0); - - __ CompareAndBranchIfZero(obj.AsRegister<Register>(), slow_path->GetEntryLabel()); -} - -void InstructionCodeGeneratorARM::VisitNullCheck(HNullCheck* instruction) { - codegen_->GenerateNullCheck(instruction); -} - -static LoadOperandType GetLoadOperandType(Primitive::Type type) { - switch (type) { - case Primitive::kPrimNot: - return kLoadWord; - case Primitive::kPrimBoolean: - return kLoadUnsignedByte; - case Primitive::kPrimByte: - return kLoadSignedByte; - case Primitive::kPrimChar: - return kLoadUnsignedHalfword; - case Primitive::kPrimShort: - return kLoadSignedHalfword; - case Primitive::kPrimInt: - return kLoadWord; - case Primitive::kPrimLong: - return kLoadWordPair; - case Primitive::kPrimFloat: - return kLoadSWord; - case Primitive::kPrimDouble: - return kLoadDWord; - default: - LOG(FATAL) << "Unreachable type " << type; - UNREACHABLE(); - } -} - -static StoreOperandType GetStoreOperandType(Primitive::Type type) { - switch (type) { - case Primitive::kPrimNot: - return kStoreWord; - case Primitive::kPrimBoolean: - case Primitive::kPrimByte: - return kStoreByte; - case Primitive::kPrimChar: - case Primitive::kPrimShort: - return kStoreHalfword; - case Primitive::kPrimInt: - return kStoreWord; - case Primitive::kPrimLong: - return kStoreWordPair; - case Primitive::kPrimFloat: - return kStoreSWord; - case Primitive::kPrimDouble: - return kStoreDWord; - default: - LOG(FATAL) << "Unreachable type " << type; - UNREACHABLE(); - } -} - -void CodeGeneratorARM::LoadFromShiftedRegOffset(Primitive::Type type, - Location out_loc, - Register base, - Register reg_offset, - Condition cond) { - uint32_t shift_count = Primitive::ComponentSizeShift(type); - Address mem_address(base, reg_offset, Shift::LSL, shift_count); - - switch (type) { - case Primitive::kPrimByte: - __ ldrsb(out_loc.AsRegister<Register>(), mem_address, cond); - break; - case Primitive::kPrimBoolean: - __ ldrb(out_loc.AsRegister<Register>(), mem_address, cond); - break; - case Primitive::kPrimShort: - __ ldrsh(out_loc.AsRegister<Register>(), mem_address, cond); - break; - case Primitive::kPrimChar: - __ ldrh(out_loc.AsRegister<Register>(), mem_address, cond); - break; - case Primitive::kPrimNot: - case Primitive::kPrimInt: - __ ldr(out_loc.AsRegister<Register>(), mem_address, cond); - break; - // T32 doesn't support LoadFromShiftedRegOffset mem address mode for these types. - case Primitive::kPrimLong: - case Primitive::kPrimFloat: - case Primitive::kPrimDouble: - default: - LOG(FATAL) << "Unreachable type " << type; - UNREACHABLE(); - } -} - -void CodeGeneratorARM::StoreToShiftedRegOffset(Primitive::Type type, - Location loc, - Register base, - Register reg_offset, - Condition cond) { - uint32_t shift_count = Primitive::ComponentSizeShift(type); - Address mem_address(base, reg_offset, Shift::LSL, shift_count); - - switch (type) { - case Primitive::kPrimByte: - case Primitive::kPrimBoolean: - __ strb(loc.AsRegister<Register>(), mem_address, cond); - break; - case Primitive::kPrimShort: - case Primitive::kPrimChar: - __ strh(loc.AsRegister<Register>(), mem_address, cond); - break; - case Primitive::kPrimNot: - case Primitive::kPrimInt: - __ str(loc.AsRegister<Register>(), mem_address, cond); - break; - // T32 doesn't support StoreToShiftedRegOffset mem address mode for these types. - case Primitive::kPrimLong: - case Primitive::kPrimFloat: - case Primitive::kPrimDouble: - default: - LOG(FATAL) << "Unreachable type " << type; - UNREACHABLE(); - } -} - -void LocationsBuilderARM::VisitArrayGet(HArrayGet* instruction) { - bool object_array_get_with_read_barrier = - kEmitCompilerReadBarrier && (instruction->GetType() == Primitive::kPrimNot); - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(instruction, - object_array_get_with_read_barrier ? - LocationSummary::kCallOnSlowPath : - LocationSummary::kNoCall); - if (object_array_get_with_read_barrier && kUseBakerReadBarrier) { - locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. - } - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); - if (Primitive::IsFloatingPointType(instruction->GetType())) { - locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); - } else { - // The output overlaps in the case of an object array get with - // read barriers enabled: we do not want the move to overwrite the - // array's location, as we need it to emit the read barrier. - locations->SetOut( - Location::RequiresRegister(), - object_array_get_with_read_barrier ? Location::kOutputOverlap : Location::kNoOutputOverlap); - } - if (object_array_get_with_read_barrier && kUseBakerReadBarrier) { - // We need a temporary register for the read barrier marking slow - // path in CodeGeneratorARM::GenerateArrayLoadWithBakerReadBarrier. - if (kBakerReadBarrierLinkTimeThunksEnableForFields && - !Runtime::Current()->UseJitCompilation() && - instruction->GetIndex()->IsConstant()) { - // Array loads with constant index are treated as field loads. - // If link-time thunks for the Baker read barrier are enabled, for AOT - // constant index loads we need a temporary only if the offset is too big. - uint32_t offset = CodeGenerator::GetArrayDataOffset(instruction); - uint32_t index = instruction->GetIndex()->AsIntConstant()->GetValue(); - offset += index << Primitive::ComponentSizeShift(Primitive::kPrimNot); - if (offset >= kReferenceLoadMinFarOffset) { - locations->AddTemp(Location::RequiresRegister()); - } - // And we always need the reserved entrypoint register. - locations->AddTemp(Location::RegisterLocation(kBakerCcEntrypointRegister)); - } else if (kBakerReadBarrierLinkTimeThunksEnableForArrays && - !Runtime::Current()->UseJitCompilation() && - !instruction->GetIndex()->IsConstant()) { - // We need a non-scratch temporary for the array data pointer. - locations->AddTemp(Location::RequiresRegister()); - // And we always need the reserved entrypoint register. - locations->AddTemp(Location::RegisterLocation(kBakerCcEntrypointRegister)); - } else { - locations->AddTemp(Location::RequiresRegister()); - } - } else if (mirror::kUseStringCompression && instruction->IsStringCharAt()) { - // Also need a temporary for String compression feature. - locations->AddTemp(Location::RequiresRegister()); - } -} - -void InstructionCodeGeneratorARM::VisitArrayGet(HArrayGet* instruction) { - LocationSummary* locations = instruction->GetLocations(); - Location obj_loc = locations->InAt(0); - Register obj = obj_loc.AsRegister<Register>(); - Location index = locations->InAt(1); - Location out_loc = locations->Out(); - uint32_t data_offset = CodeGenerator::GetArrayDataOffset(instruction); - Primitive::Type type = instruction->GetType(); - const bool maybe_compressed_char_at = mirror::kUseStringCompression && - instruction->IsStringCharAt(); - HInstruction* array_instr = instruction->GetArray(); - bool has_intermediate_address = array_instr->IsIntermediateAddress(); - - switch (type) { - case Primitive::kPrimBoolean: - case Primitive::kPrimByte: - case Primitive::kPrimShort: - case Primitive::kPrimChar: - case Primitive::kPrimInt: { - Register length; - if (maybe_compressed_char_at) { - length = locations->GetTemp(0).AsRegister<Register>(); - uint32_t count_offset = mirror::String::CountOffset().Uint32Value(); - __ LoadFromOffset(kLoadWord, length, obj, count_offset); - codegen_->MaybeRecordImplicitNullCheck(instruction); - } - if (index.IsConstant()) { - int32_t const_index = index.GetConstant()->AsIntConstant()->GetValue(); - if (maybe_compressed_char_at) { - Label uncompressed_load, done; - Label* final_label = codegen_->GetFinalLabel(instruction, &done); - __ Lsrs(length, length, 1u); // LSRS has a 16-bit encoding, TST (immediate) does not. - static_assert(static_cast<uint32_t>(mirror::StringCompressionFlag::kCompressed) == 0u, - "Expecting 0=compressed, 1=uncompressed"); - __ b(&uncompressed_load, CS); - __ LoadFromOffset(kLoadUnsignedByte, - out_loc.AsRegister<Register>(), - obj, - data_offset + const_index); - __ b(final_label); - __ Bind(&uncompressed_load); - __ LoadFromOffset(GetLoadOperandType(Primitive::kPrimChar), - out_loc.AsRegister<Register>(), - obj, - data_offset + (const_index << 1)); - if (done.IsLinked()) { - __ Bind(&done); - } - } else { - uint32_t full_offset = data_offset + (const_index << Primitive::ComponentSizeShift(type)); - - LoadOperandType load_type = GetLoadOperandType(type); - __ LoadFromOffset(load_type, out_loc.AsRegister<Register>(), obj, full_offset); - } - } else { - Register temp = IP; - - if (has_intermediate_address) { - // We do not need to compute the intermediate address from the array: the - // input instruction has done it already. See the comment in - // `TryExtractArrayAccessAddress()`. - if (kIsDebugBuild) { - HIntermediateAddress* tmp = array_instr->AsIntermediateAddress(); - DCHECK_EQ(tmp->GetOffset()->AsIntConstant()->GetValueAsUint64(), data_offset); - } - temp = obj; - } else { - __ add(temp, obj, ShifterOperand(data_offset)); - } - if (maybe_compressed_char_at) { - Label uncompressed_load, done; - Label* final_label = codegen_->GetFinalLabel(instruction, &done); - __ Lsrs(length, length, 1u); // LSRS has a 16-bit encoding, TST (immediate) does not. - static_assert(static_cast<uint32_t>(mirror::StringCompressionFlag::kCompressed) == 0u, - "Expecting 0=compressed, 1=uncompressed"); - __ b(&uncompressed_load, CS); - __ ldrb(out_loc.AsRegister<Register>(), - Address(temp, index.AsRegister<Register>(), Shift::LSL, 0)); - __ b(final_label); - __ Bind(&uncompressed_load); - __ ldrh(out_loc.AsRegister<Register>(), - Address(temp, index.AsRegister<Register>(), Shift::LSL, 1)); - if (done.IsLinked()) { - __ Bind(&done); - } - } else { - codegen_->LoadFromShiftedRegOffset(type, out_loc, temp, index.AsRegister<Register>()); - } - } - break; - } - - case Primitive::kPrimNot: { - // The read barrier instrumentation of object ArrayGet - // instructions does not support the HIntermediateAddress - // instruction. - DCHECK(!(has_intermediate_address && kEmitCompilerReadBarrier)); - - static_assert( - sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), - "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); - // /* HeapReference<Object> */ out = - // *(obj + data_offset + index * sizeof(HeapReference<Object>)) - if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { - Location temp = locations->GetTemp(0); - // Note that a potential implicit null check is handled in this - // CodeGeneratorARM::GenerateArrayLoadWithBakerReadBarrier call. - DCHECK(!instruction->CanDoImplicitNullCheckOn(instruction->InputAt(0))); - if (index.IsConstant()) { - // Array load with a constant index can be treated as a field load. - data_offset += helpers::Int32ConstantFrom(index) << Primitive::ComponentSizeShift(type); - codegen_->GenerateFieldLoadWithBakerReadBarrier(instruction, - out_loc, - obj, - data_offset, - locations->GetTemp(0), - /* needs_null_check */ false); - } else { - codegen_->GenerateArrayLoadWithBakerReadBarrier( - instruction, out_loc, obj, data_offset, index, temp, /* needs_null_check */ false); - } - } else { - Register out = out_loc.AsRegister<Register>(); - if (index.IsConstant()) { - size_t offset = - (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; - __ LoadFromOffset(kLoadWord, out, obj, offset); - codegen_->MaybeRecordImplicitNullCheck(instruction); - // If read barriers are enabled, emit read barriers other than - // Baker's using a slow path (and also unpoison the loaded - // reference, if heap poisoning is enabled). - codegen_->MaybeGenerateReadBarrierSlow(instruction, out_loc, out_loc, obj_loc, offset); - } else { - Register temp = IP; - - if (has_intermediate_address) { - // We do not need to compute the intermediate address from the array: the - // input instruction has done it already. See the comment in - // `TryExtractArrayAccessAddress()`. - if (kIsDebugBuild) { - HIntermediateAddress* tmp = array_instr->AsIntermediateAddress(); - DCHECK_EQ(tmp->GetOffset()->AsIntConstant()->GetValueAsUint64(), data_offset); - } - temp = obj; - } else { - __ add(temp, obj, ShifterOperand(data_offset)); - } - codegen_->LoadFromShiftedRegOffset(type, out_loc, temp, index.AsRegister<Register>()); - - codegen_->MaybeRecordImplicitNullCheck(instruction); - // If read barriers are enabled, emit read barriers other than - // Baker's using a slow path (and also unpoison the loaded - // reference, if heap poisoning is enabled). - codegen_->MaybeGenerateReadBarrierSlow( - instruction, out_loc, out_loc, obj_loc, data_offset, index); - } - } - break; - } - - case Primitive::kPrimLong: { - if (index.IsConstant()) { - size_t offset = - (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; - __ LoadFromOffset(kLoadWordPair, out_loc.AsRegisterPairLow<Register>(), obj, offset); - } else { - __ add(IP, obj, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_8)); - __ LoadFromOffset(kLoadWordPair, out_loc.AsRegisterPairLow<Register>(), IP, data_offset); - } - break; - } - - case Primitive::kPrimFloat: { - SRegister out = out_loc.AsFpuRegister<SRegister>(); - if (index.IsConstant()) { - size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; - __ LoadSFromOffset(out, obj, offset); - } else { - __ add(IP, obj, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_4)); - __ LoadSFromOffset(out, IP, data_offset); - } - break; - } - - case Primitive::kPrimDouble: { - SRegister out = out_loc.AsFpuRegisterPairLow<SRegister>(); - if (index.IsConstant()) { - size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; - __ LoadDFromOffset(FromLowSToD(out), obj, offset); - } else { - __ add(IP, obj, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_8)); - __ LoadDFromOffset(FromLowSToD(out), IP, data_offset); - } - break; - } - - case Primitive::kPrimVoid: - LOG(FATAL) << "Unreachable type " << type; - UNREACHABLE(); - } - - if (type == Primitive::kPrimNot) { - // Potential implicit null checks, in the case of reference - // arrays, are handled in the previous switch statement. - } else if (!maybe_compressed_char_at) { - codegen_->MaybeRecordImplicitNullCheck(instruction); - } -} - -void LocationsBuilderARM::VisitArraySet(HArraySet* instruction) { - Primitive::Type value_type = instruction->GetComponentType(); - - bool needs_write_barrier = - CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue()); - bool may_need_runtime_call_for_type_check = instruction->NeedsTypeCheck(); - - LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary( - instruction, - may_need_runtime_call_for_type_check ? - LocationSummary::kCallOnSlowPath : - LocationSummary::kNoCall); - - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); - if (Primitive::IsFloatingPointType(value_type)) { - locations->SetInAt(2, Location::RequiresFpuRegister()); - } else { - locations->SetInAt(2, Location::RequiresRegister()); - } - if (needs_write_barrier) { - // Temporary registers for the write barrier. - locations->AddTemp(Location::RequiresRegister()); // Possibly used for ref. poisoning too. - locations->AddTemp(Location::RequiresRegister()); - } -} - -void InstructionCodeGeneratorARM::VisitArraySet(HArraySet* instruction) { - LocationSummary* locations = instruction->GetLocations(); - Location array_loc = locations->InAt(0); - Register array = array_loc.AsRegister<Register>(); - Location index = locations->InAt(1); - Primitive::Type value_type = instruction->GetComponentType(); - bool may_need_runtime_call_for_type_check = instruction->NeedsTypeCheck(); - bool needs_write_barrier = - CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue()); - uint32_t data_offset = - mirror::Array::DataOffset(Primitive::ComponentSize(value_type)).Uint32Value(); - Location value_loc = locations->InAt(2); - HInstruction* array_instr = instruction->GetArray(); - bool has_intermediate_address = array_instr->IsIntermediateAddress(); - - switch (value_type) { - case Primitive::kPrimBoolean: - case Primitive::kPrimByte: - case Primitive::kPrimShort: - case Primitive::kPrimChar: - case Primitive::kPrimInt: { - if (index.IsConstant()) { - int32_t const_index = index.GetConstant()->AsIntConstant()->GetValue(); - uint32_t full_offset = - data_offset + (const_index << Primitive::ComponentSizeShift(value_type)); - StoreOperandType store_type = GetStoreOperandType(value_type); - __ StoreToOffset(store_type, value_loc.AsRegister<Register>(), array, full_offset); - } else { - Register temp = IP; - - if (has_intermediate_address) { - // We do not need to compute the intermediate address from the array: the - // input instruction has done it already. See the comment in - // `TryExtractArrayAccessAddress()`. - if (kIsDebugBuild) { - HIntermediateAddress* tmp = array_instr->AsIntermediateAddress(); - DCHECK(tmp->GetOffset()->AsIntConstant()->GetValueAsUint64() == data_offset); - } - temp = array; - } else { - __ add(temp, array, ShifterOperand(data_offset)); - } - codegen_->StoreToShiftedRegOffset(value_type, - value_loc, - temp, - index.AsRegister<Register>()); - } - break; - } - - case Primitive::kPrimNot: { - Register value = value_loc.AsRegister<Register>(); - // TryExtractArrayAccessAddress optimization is never applied for non-primitive ArraySet. - // See the comment in instruction_simplifier_shared.cc. - DCHECK(!has_intermediate_address); - - if (instruction->InputAt(2)->IsNullConstant()) { - // Just setting null. - if (index.IsConstant()) { - size_t offset = - (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; - __ StoreToOffset(kStoreWord, value, array, offset); - } else { - DCHECK(index.IsRegister()) << index; - __ add(IP, array, ShifterOperand(data_offset)); - codegen_->StoreToShiftedRegOffset(value_type, - value_loc, - IP, - index.AsRegister<Register>()); - } - codegen_->MaybeRecordImplicitNullCheck(instruction); - DCHECK(!needs_write_barrier); - DCHECK(!may_need_runtime_call_for_type_check); - break; - } - - DCHECK(needs_write_barrier); - Location temp1_loc = locations->GetTemp(0); - Register temp1 = temp1_loc.AsRegister<Register>(); - Location temp2_loc = locations->GetTemp(1); - Register temp2 = temp2_loc.AsRegister<Register>(); - uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); - uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); - uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); - Label done; - Label* final_label = codegen_->GetFinalLabel(instruction, &done); - SlowPathCodeARM* slow_path = nullptr; - - if (may_need_runtime_call_for_type_check) { - slow_path = new (GetGraph()->GetArena()) ArraySetSlowPathARM(instruction); - codegen_->AddSlowPath(slow_path); - if (instruction->GetValueCanBeNull()) { - Label non_zero; - __ CompareAndBranchIfNonZero(value, &non_zero); - if (index.IsConstant()) { - size_t offset = - (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; - __ StoreToOffset(kStoreWord, value, array, offset); - } else { - DCHECK(index.IsRegister()) << index; - __ add(IP, array, ShifterOperand(data_offset)); - codegen_->StoreToShiftedRegOffset(value_type, - value_loc, - IP, - index.AsRegister<Register>()); - } - codegen_->MaybeRecordImplicitNullCheck(instruction); - __ b(final_label); - __ Bind(&non_zero); - } - - // Note that when read barriers are enabled, the type checks - // are performed without read barriers. This is fine, even in - // the case where a class object is in the from-space after - // the flip, as a comparison involving such a type would not - // produce a false positive; it may of course produce a false - // negative, in which case we would take the ArraySet slow - // path. - - // /* HeapReference<Class> */ temp1 = array->klass_ - __ LoadFromOffset(kLoadWord, temp1, array, class_offset); - codegen_->MaybeRecordImplicitNullCheck(instruction); - __ MaybeUnpoisonHeapReference(temp1); - - // /* HeapReference<Class> */ temp1 = temp1->component_type_ - __ LoadFromOffset(kLoadWord, temp1, temp1, component_offset); - // /* HeapReference<Class> */ temp2 = value->klass_ - __ LoadFromOffset(kLoadWord, temp2, value, class_offset); - // If heap poisoning is enabled, no need to unpoison `temp1` - // nor `temp2`, as we are comparing two poisoned references. - __ cmp(temp1, ShifterOperand(temp2)); - - if (instruction->StaticTypeOfArrayIsObjectArray()) { - Label do_put; - __ b(&do_put, EQ); - // If heap poisoning is enabled, the `temp1` reference has - // not been unpoisoned yet; unpoison it now. - __ MaybeUnpoisonHeapReference(temp1); - - // /* HeapReference<Class> */ temp1 = temp1->super_class_ - __ LoadFromOffset(kLoadWord, temp1, temp1, super_offset); - // If heap poisoning is enabled, no need to unpoison - // `temp1`, as we are comparing against null below. - __ CompareAndBranchIfNonZero(temp1, slow_path->GetEntryLabel()); - __ Bind(&do_put); - } else { - __ b(slow_path->GetEntryLabel(), NE); - } - } - - Register source = value; - if (kPoisonHeapReferences) { - // Note that in the case where `value` is a null reference, - // we do not enter this block, as a null reference does not - // need poisoning. - DCHECK_EQ(value_type, Primitive::kPrimNot); - __ Mov(temp1, value); - __ PoisonHeapReference(temp1); - source = temp1; - } - - if (index.IsConstant()) { - size_t offset = - (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; - __ StoreToOffset(kStoreWord, source, array, offset); - } else { - DCHECK(index.IsRegister()) << index; - - __ add(IP, array, ShifterOperand(data_offset)); - codegen_->StoreToShiftedRegOffset(value_type, - Location::RegisterLocation(source), - IP, - index.AsRegister<Register>()); - } - - if (!may_need_runtime_call_for_type_check) { - codegen_->MaybeRecordImplicitNullCheck(instruction); - } - - codegen_->MarkGCCard(temp1, temp2, array, value, instruction->GetValueCanBeNull()); - - if (done.IsLinked()) { - __ Bind(&done); - } - - if (slow_path != nullptr) { - __ Bind(slow_path->GetExitLabel()); - } - - break; - } - - case Primitive::kPrimLong: { - Location value = locations->InAt(2); - if (index.IsConstant()) { - size_t offset = - (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; - __ StoreToOffset(kStoreWordPair, value.AsRegisterPairLow<Register>(), array, offset); - } else { - __ add(IP, array, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_8)); - __ StoreToOffset(kStoreWordPair, value.AsRegisterPairLow<Register>(), IP, data_offset); - } - break; - } - - case Primitive::kPrimFloat: { - Location value = locations->InAt(2); - DCHECK(value.IsFpuRegister()); - if (index.IsConstant()) { - size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; - __ StoreSToOffset(value.AsFpuRegister<SRegister>(), array, offset); - } else { - __ add(IP, array, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_4)); - __ StoreSToOffset(value.AsFpuRegister<SRegister>(), IP, data_offset); - } - break; - } - - case Primitive::kPrimDouble: { - Location value = locations->InAt(2); - DCHECK(value.IsFpuRegisterPair()); - if (index.IsConstant()) { - size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; - __ StoreDToOffset(FromLowSToD(value.AsFpuRegisterPairLow<SRegister>()), array, offset); - } else { - __ add(IP, array, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_8)); - __ StoreDToOffset(FromLowSToD(value.AsFpuRegisterPairLow<SRegister>()), IP, data_offset); - } - - break; - } - - case Primitive::kPrimVoid: - LOG(FATAL) << "Unreachable type " << value_type; - UNREACHABLE(); - } - - // Objects are handled in the switch. - if (value_type != Primitive::kPrimNot) { - codegen_->MaybeRecordImplicitNullCheck(instruction); - } -} - -void LocationsBuilderARM::VisitArrayLength(HArrayLength* instruction) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); -} - -void InstructionCodeGeneratorARM::VisitArrayLength(HArrayLength* instruction) { - LocationSummary* locations = instruction->GetLocations(); - uint32_t offset = CodeGenerator::GetArrayLengthOffset(instruction); - Register obj = locations->InAt(0).AsRegister<Register>(); - Register out = locations->Out().AsRegister<Register>(); - __ LoadFromOffset(kLoadWord, out, obj, offset); - codegen_->MaybeRecordImplicitNullCheck(instruction); - // Mask out compression flag from String's array length. - if (mirror::kUseStringCompression && instruction->IsStringLength()) { - __ Lsr(out, out, 1u); - } -} - -void LocationsBuilderARM::VisitIntermediateAddress(HIntermediateAddress* instruction) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); - - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RegisterOrConstant(instruction->GetOffset())); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); -} - -void InstructionCodeGeneratorARM::VisitIntermediateAddress(HIntermediateAddress* instruction) { - LocationSummary* locations = instruction->GetLocations(); - Location out = locations->Out(); - Location first = locations->InAt(0); - Location second = locations->InAt(1); - - if (second.IsRegister()) { - __ add(out.AsRegister<Register>(), - first.AsRegister<Register>(), - ShifterOperand(second.AsRegister<Register>())); - } else { - __ AddConstant(out.AsRegister<Register>(), - first.AsRegister<Register>(), - second.GetConstant()->AsIntConstant()->GetValue()); - } -} - -void LocationsBuilderARM::VisitIntermediateAddressIndex(HIntermediateAddressIndex* instruction) { - LOG(FATAL) << "Unreachable " << instruction->GetId(); -} - -void InstructionCodeGeneratorARM::VisitIntermediateAddressIndex( - HIntermediateAddressIndex* instruction) { - LOG(FATAL) << "Unreachable " << instruction->GetId(); -} - -void LocationsBuilderARM::VisitBoundsCheck(HBoundsCheck* instruction) { - RegisterSet caller_saves = RegisterSet::Empty(); - InvokeRuntimeCallingConvention calling_convention; - caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); - caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(1))); - LocationSummary* locations = codegen_->CreateThrowingSlowPathLocations(instruction, caller_saves); - - HInstruction* index = instruction->InputAt(0); - HInstruction* length = instruction->InputAt(1); - // If both index and length are constants we can statically check the bounds. But if at least one - // of them is not encodable ArmEncodableConstantOrRegister will create - // Location::RequiresRegister() which is not desired to happen. Instead we create constant - // locations. - bool both_const = index->IsConstant() && length->IsConstant(); - locations->SetInAt(0, both_const - ? Location::ConstantLocation(index->AsConstant()) - : ArmEncodableConstantOrRegister(index, CMP)); - locations->SetInAt(1, both_const - ? Location::ConstantLocation(length->AsConstant()) - : ArmEncodableConstantOrRegister(length, CMP)); -} - -void InstructionCodeGeneratorARM::VisitBoundsCheck(HBoundsCheck* instruction) { - LocationSummary* locations = instruction->GetLocations(); - Location index_loc = locations->InAt(0); - Location length_loc = locations->InAt(1); - - if (length_loc.IsConstant()) { - int32_t length = helpers::Int32ConstantFrom(length_loc); - if (index_loc.IsConstant()) { - // BCE will remove the bounds check if we are guaranteed to pass. - int32_t index = helpers::Int32ConstantFrom(index_loc); - if (index < 0 || index >= length) { - SlowPathCodeARM* slow_path = - new (GetGraph()->GetArena()) BoundsCheckSlowPathARM(instruction); - codegen_->AddSlowPath(slow_path); - __ b(slow_path->GetEntryLabel()); - } else { - // Some optimization after BCE may have generated this, and we should not - // generate a bounds check if it is a valid range. - } - return; - } - - SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) BoundsCheckSlowPathARM(instruction); - __ cmp(index_loc.AsRegister<Register>(), ShifterOperand(length)); - codegen_->AddSlowPath(slow_path); - __ b(slow_path->GetEntryLabel(), HS); - } else { - SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) BoundsCheckSlowPathARM(instruction); - if (index_loc.IsConstant()) { - int32_t index = helpers::Int32ConstantFrom(index_loc); - __ cmp(length_loc.AsRegister<Register>(), ShifterOperand(index)); - } else { - __ cmp(length_loc.AsRegister<Register>(), ShifterOperand(index_loc.AsRegister<Register>())); - } - codegen_->AddSlowPath(slow_path); - __ b(slow_path->GetEntryLabel(), LS); - } -} - -void CodeGeneratorARM::MarkGCCard(Register temp, - Register card, - Register object, - Register value, - bool can_be_null) { - Label is_null; - if (can_be_null) { - __ CompareAndBranchIfZero(value, &is_null); - } - __ LoadFromOffset(kLoadWord, card, TR, Thread::CardTableOffset<kArmPointerSize>().Int32Value()); - __ Lsr(temp, object, gc::accounting::CardTable::kCardShift); - __ strb(card, Address(card, temp)); - if (can_be_null) { - __ Bind(&is_null); - } -} - -void LocationsBuilderARM::VisitParallelMove(HParallelMove* instruction ATTRIBUTE_UNUSED) { - LOG(FATAL) << "Unreachable"; -} - -void InstructionCodeGeneratorARM::VisitParallelMove(HParallelMove* instruction) { - codegen_->GetMoveResolver()->EmitNativeCode(instruction); -} - -void LocationsBuilderARM::VisitSuspendCheck(HSuspendCheck* instruction) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnSlowPath); - locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. -} - -void InstructionCodeGeneratorARM::VisitSuspendCheck(HSuspendCheck* instruction) { - HBasicBlock* block = instruction->GetBlock(); - if (block->GetLoopInformation() != nullptr) { - DCHECK(block->GetLoopInformation()->GetSuspendCheck() == instruction); - // The back edge will generate the suspend check. - return; - } - if (block->IsEntryBlock() && instruction->GetNext()->IsGoto()) { - // The goto will generate the suspend check. - return; - } - GenerateSuspendCheck(instruction, nullptr); -} - -void InstructionCodeGeneratorARM::GenerateSuspendCheck(HSuspendCheck* instruction, - HBasicBlock* successor) { - SuspendCheckSlowPathARM* slow_path = - down_cast<SuspendCheckSlowPathARM*>(instruction->GetSlowPath()); - if (slow_path == nullptr) { - slow_path = new (GetGraph()->GetArena()) SuspendCheckSlowPathARM(instruction, successor); - instruction->SetSlowPath(slow_path); - codegen_->AddSlowPath(slow_path); - if (successor != nullptr) { - DCHECK(successor->IsLoopHeader()); - codegen_->ClearSpillSlotsFromLoopPhisInStackMap(instruction); - } - } else { - DCHECK_EQ(slow_path->GetSuccessor(), successor); - } - - __ LoadFromOffset( - kLoadUnsignedHalfword, IP, TR, Thread::ThreadFlagsOffset<kArmPointerSize>().Int32Value()); - if (successor == nullptr) { - __ CompareAndBranchIfNonZero(IP, slow_path->GetEntryLabel()); - __ Bind(slow_path->GetReturnLabel()); - } else { - __ CompareAndBranchIfZero(IP, codegen_->GetLabelOf(successor)); - __ b(slow_path->GetEntryLabel()); - } -} - -ArmAssembler* ParallelMoveResolverARM::GetAssembler() const { - return codegen_->GetAssembler(); -} - -void ParallelMoveResolverARM::EmitMove(size_t index) { - MoveOperands* move = moves_[index]; - Location source = move->GetSource(); - Location destination = move->GetDestination(); - - if (source.IsRegister()) { - if (destination.IsRegister()) { - __ Mov(destination.AsRegister<Register>(), source.AsRegister<Register>()); - } else if (destination.IsFpuRegister()) { - __ vmovsr(destination.AsFpuRegister<SRegister>(), source.AsRegister<Register>()); - } else { - DCHECK(destination.IsStackSlot()); - __ StoreToOffset(kStoreWord, source.AsRegister<Register>(), - SP, destination.GetStackIndex()); - } - } else if (source.IsStackSlot()) { - if (destination.IsRegister()) { - __ LoadFromOffset(kLoadWord, destination.AsRegister<Register>(), - SP, source.GetStackIndex()); - } else if (destination.IsFpuRegister()) { - __ LoadSFromOffset(destination.AsFpuRegister<SRegister>(), SP, source.GetStackIndex()); - } else { - DCHECK(destination.IsStackSlot()); - __ LoadFromOffset(kLoadWord, IP, SP, source.GetStackIndex()); - __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); - } - } else if (source.IsFpuRegister()) { - if (destination.IsRegister()) { - __ vmovrs(destination.AsRegister<Register>(), source.AsFpuRegister<SRegister>()); - } else if (destination.IsFpuRegister()) { - __ vmovs(destination.AsFpuRegister<SRegister>(), source.AsFpuRegister<SRegister>()); - } else { - DCHECK(destination.IsStackSlot()); - __ StoreSToOffset(source.AsFpuRegister<SRegister>(), SP, destination.GetStackIndex()); - } - } else if (source.IsDoubleStackSlot()) { - if (destination.IsDoubleStackSlot()) { - __ LoadDFromOffset(DTMP, SP, source.GetStackIndex()); - __ StoreDToOffset(DTMP, SP, destination.GetStackIndex()); - } else if (destination.IsRegisterPair()) { - DCHECK(ExpectedPairLayout(destination)); - __ LoadFromOffset( - kLoadWordPair, destination.AsRegisterPairLow<Register>(), SP, source.GetStackIndex()); - } else { - DCHECK(destination.IsFpuRegisterPair()) << destination; - __ LoadDFromOffset(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), - SP, - source.GetStackIndex()); - } - } else if (source.IsRegisterPair()) { - if (destination.IsRegisterPair()) { - __ Mov(destination.AsRegisterPairLow<Register>(), source.AsRegisterPairLow<Register>()); - __ Mov(destination.AsRegisterPairHigh<Register>(), source.AsRegisterPairHigh<Register>()); - } else if (destination.IsFpuRegisterPair()) { - __ vmovdrr(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), - source.AsRegisterPairLow<Register>(), - source.AsRegisterPairHigh<Register>()); - } else { - DCHECK(destination.IsDoubleStackSlot()) << destination; - DCHECK(ExpectedPairLayout(source)); - __ StoreToOffset( - kStoreWordPair, source.AsRegisterPairLow<Register>(), SP, destination.GetStackIndex()); - } - } else if (source.IsFpuRegisterPair()) { - if (destination.IsRegisterPair()) { - __ vmovrrd(destination.AsRegisterPairLow<Register>(), - destination.AsRegisterPairHigh<Register>(), - FromLowSToD(source.AsFpuRegisterPairLow<SRegister>())); - } else if (destination.IsFpuRegisterPair()) { - __ vmovd(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), - FromLowSToD(source.AsFpuRegisterPairLow<SRegister>())); - } else { - DCHECK(destination.IsDoubleStackSlot()) << destination; - __ StoreDToOffset(FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()), - SP, - destination.GetStackIndex()); - } - } else { - DCHECK(source.IsConstant()) << source; - HConstant* constant = source.GetConstant(); - if (constant->IsIntConstant() || constant->IsNullConstant()) { - int32_t value = CodeGenerator::GetInt32ValueOf(constant); - if (destination.IsRegister()) { - __ LoadImmediate(destination.AsRegister<Register>(), value); - } else { - DCHECK(destination.IsStackSlot()); - __ LoadImmediate(IP, value); - __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); - } - } else if (constant->IsLongConstant()) { - int64_t value = constant->AsLongConstant()->GetValue(); - if (destination.IsRegisterPair()) { - __ LoadImmediate(destination.AsRegisterPairLow<Register>(), Low32Bits(value)); - __ LoadImmediate(destination.AsRegisterPairHigh<Register>(), High32Bits(value)); - } else { - DCHECK(destination.IsDoubleStackSlot()) << destination; - __ LoadImmediate(IP, Low32Bits(value)); - __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); - __ LoadImmediate(IP, High32Bits(value)); - __ StoreToOffset(kStoreWord, IP, SP, destination.GetHighStackIndex(kArmWordSize)); - } - } else if (constant->IsDoubleConstant()) { - double value = constant->AsDoubleConstant()->GetValue(); - if (destination.IsFpuRegisterPair()) { - __ LoadDImmediate(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), value); - } else { - DCHECK(destination.IsDoubleStackSlot()) << destination; - uint64_t int_value = bit_cast<uint64_t, double>(value); - __ LoadImmediate(IP, Low32Bits(int_value)); - __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); - __ LoadImmediate(IP, High32Bits(int_value)); - __ StoreToOffset(kStoreWord, IP, SP, destination.GetHighStackIndex(kArmWordSize)); - } - } else { - DCHECK(constant->IsFloatConstant()) << constant->DebugName(); - float value = constant->AsFloatConstant()->GetValue(); - if (destination.IsFpuRegister()) { - __ LoadSImmediate(destination.AsFpuRegister<SRegister>(), value); - } else { - DCHECK(destination.IsStackSlot()); - __ LoadImmediate(IP, bit_cast<int32_t, float>(value)); - __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); - } - } - } -} - -void ParallelMoveResolverARM::Exchange(Register reg, int mem) { - __ Mov(IP, reg); - __ LoadFromOffset(kLoadWord, reg, SP, mem); - __ StoreToOffset(kStoreWord, IP, SP, mem); -} - -void ParallelMoveResolverARM::Exchange(int mem1, int mem2) { - ScratchRegisterScope ensure_scratch(this, IP, R0, codegen_->GetNumberOfCoreRegisters()); - int stack_offset = ensure_scratch.IsSpilled() ? kArmWordSize : 0; - __ LoadFromOffset(kLoadWord, static_cast<Register>(ensure_scratch.GetRegister()), - SP, mem1 + stack_offset); - __ LoadFromOffset(kLoadWord, IP, SP, mem2 + stack_offset); - __ StoreToOffset(kStoreWord, static_cast<Register>(ensure_scratch.GetRegister()), - SP, mem2 + stack_offset); - __ StoreToOffset(kStoreWord, IP, SP, mem1 + stack_offset); -} - -void ParallelMoveResolverARM::EmitSwap(size_t index) { - MoveOperands* move = moves_[index]; - Location source = move->GetSource(); - Location destination = move->GetDestination(); - - if (source.IsRegister() && destination.IsRegister()) { - DCHECK_NE(source.AsRegister<Register>(), IP); - DCHECK_NE(destination.AsRegister<Register>(), IP); - __ Mov(IP, source.AsRegister<Register>()); - __ Mov(source.AsRegister<Register>(), destination.AsRegister<Register>()); - __ Mov(destination.AsRegister<Register>(), IP); - } else if (source.IsRegister() && destination.IsStackSlot()) { - Exchange(source.AsRegister<Register>(), destination.GetStackIndex()); - } else if (source.IsStackSlot() && destination.IsRegister()) { - Exchange(destination.AsRegister<Register>(), source.GetStackIndex()); - } else if (source.IsStackSlot() && destination.IsStackSlot()) { - Exchange(source.GetStackIndex(), destination.GetStackIndex()); - } else if (source.IsFpuRegister() && destination.IsFpuRegister()) { - __ vmovrs(IP, source.AsFpuRegister<SRegister>()); - __ vmovs(source.AsFpuRegister<SRegister>(), destination.AsFpuRegister<SRegister>()); - __ vmovsr(destination.AsFpuRegister<SRegister>(), IP); - } else if (source.IsRegisterPair() && destination.IsRegisterPair()) { - __ vmovdrr(DTMP, source.AsRegisterPairLow<Register>(), source.AsRegisterPairHigh<Register>()); - __ Mov(source.AsRegisterPairLow<Register>(), destination.AsRegisterPairLow<Register>()); - __ Mov(source.AsRegisterPairHigh<Register>(), destination.AsRegisterPairHigh<Register>()); - __ vmovrrd(destination.AsRegisterPairLow<Register>(), - destination.AsRegisterPairHigh<Register>(), - DTMP); - } else if (source.IsRegisterPair() || destination.IsRegisterPair()) { - Register low_reg = source.IsRegisterPair() - ? source.AsRegisterPairLow<Register>() - : destination.AsRegisterPairLow<Register>(); - int mem = source.IsRegisterPair() - ? destination.GetStackIndex() - : source.GetStackIndex(); - DCHECK(ExpectedPairLayout(source.IsRegisterPair() ? source : destination)); - __ vmovdrr(DTMP, low_reg, static_cast<Register>(low_reg + 1)); - __ LoadFromOffset(kLoadWordPair, low_reg, SP, mem); - __ StoreDToOffset(DTMP, SP, mem); - } else if (source.IsFpuRegisterPair() && destination.IsFpuRegisterPair()) { - DRegister first = FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()); - DRegister second = FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()); - __ vmovd(DTMP, first); - __ vmovd(first, second); - __ vmovd(second, DTMP); - } else if (source.IsFpuRegisterPair() || destination.IsFpuRegisterPair()) { - DRegister reg = source.IsFpuRegisterPair() - ? FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()) - : FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()); - int mem = source.IsFpuRegisterPair() - ? destination.GetStackIndex() - : source.GetStackIndex(); - __ vmovd(DTMP, reg); - __ LoadDFromOffset(reg, SP, mem); - __ StoreDToOffset(DTMP, SP, mem); - } else if (source.IsFpuRegister() || destination.IsFpuRegister()) { - SRegister reg = source.IsFpuRegister() ? source.AsFpuRegister<SRegister>() - : destination.AsFpuRegister<SRegister>(); - int mem = source.IsFpuRegister() - ? destination.GetStackIndex() - : source.GetStackIndex(); - - __ vmovrs(IP, reg); - __ LoadSFromOffset(reg, SP, mem); - __ StoreToOffset(kStoreWord, IP, SP, mem); - } else if (source.IsDoubleStackSlot() && destination.IsDoubleStackSlot()) { - Exchange(source.GetStackIndex(), destination.GetStackIndex()); - Exchange(source.GetHighStackIndex(kArmWordSize), destination.GetHighStackIndex(kArmWordSize)); - } else { - LOG(FATAL) << "Unimplemented" << source << " <-> " << destination; - } -} - -void ParallelMoveResolverARM::SpillScratch(int reg) { - __ Push(static_cast<Register>(reg)); -} - -void ParallelMoveResolverARM::RestoreScratch(int reg) { - __ Pop(static_cast<Register>(reg)); -} - -HLoadClass::LoadKind CodeGeneratorARM::GetSupportedLoadClassKind( - HLoadClass::LoadKind desired_class_load_kind) { - switch (desired_class_load_kind) { - case HLoadClass::LoadKind::kInvalid: - LOG(FATAL) << "UNREACHABLE"; - UNREACHABLE(); - case HLoadClass::LoadKind::kReferrersClass: - break; - case HLoadClass::LoadKind::kBootImageLinkTimePcRelative: - case HLoadClass::LoadKind::kBssEntry: - DCHECK(!Runtime::Current()->UseJitCompilation()); - break; - case HLoadClass::LoadKind::kJitTableAddress: - DCHECK(Runtime::Current()->UseJitCompilation()); - break; - case HLoadClass::LoadKind::kBootImageAddress: - case HLoadClass::LoadKind::kRuntimeCall: - break; - } - return desired_class_load_kind; -} - -void LocationsBuilderARM::VisitLoadClass(HLoadClass* cls) { - HLoadClass::LoadKind load_kind = cls->GetLoadKind(); - if (load_kind == HLoadClass::LoadKind::kRuntimeCall) { - InvokeRuntimeCallingConvention calling_convention; - CodeGenerator::CreateLoadClassRuntimeCallLocationSummary( - cls, - Location::RegisterLocation(calling_convention.GetRegisterAt(0)), - Location::RegisterLocation(R0)); - DCHECK_EQ(calling_convention.GetRegisterAt(0), R0); - return; - } - DCHECK(!cls->NeedsAccessCheck()); - - const bool requires_read_barrier = kEmitCompilerReadBarrier && !cls->IsInBootImage(); - LocationSummary::CallKind call_kind = (cls->NeedsEnvironment() || requires_read_barrier) - ? LocationSummary::kCallOnSlowPath - : LocationSummary::kNoCall; - LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(cls, call_kind); - if (kUseBakerReadBarrier && requires_read_barrier && !cls->NeedsEnvironment()) { - locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. - } - - if (load_kind == HLoadClass::LoadKind::kReferrersClass) { - locations->SetInAt(0, Location::RequiresRegister()); - } - locations->SetOut(Location::RequiresRegister()); - if (load_kind == HLoadClass::LoadKind::kBssEntry) { - if (!kUseReadBarrier || kUseBakerReadBarrier) { - // Rely on the type resolution or initialization and marking to save everything we need. - // Note that IP may be clobbered by saving/restoring the live register (only one thanks - // to the custom calling convention) or by marking, so we request a different temp. - locations->AddTemp(Location::RequiresRegister()); - RegisterSet caller_saves = RegisterSet::Empty(); - InvokeRuntimeCallingConvention calling_convention; - caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); - // TODO: Add GetReturnLocation() to the calling convention so that we can DCHECK() - // that the the kPrimNot result register is the same as the first argument register. - locations->SetCustomSlowPathCallerSaves(caller_saves); - } else { - // For non-Baker read barrier we have a temp-clobbering call. - } - } - if (kUseBakerReadBarrier && kBakerReadBarrierLinkTimeThunksEnableForGcRoots) { - if (load_kind == HLoadClass::LoadKind::kBssEntry || - (load_kind == HLoadClass::LoadKind::kReferrersClass && - !Runtime::Current()->UseJitCompilation())) { - locations->AddTemp(Location::RegisterLocation(kBakerCcEntrypointRegister)); - } - } -} - -// NO_THREAD_SAFETY_ANALYSIS as we manipulate handles whose internal object we know does not -// move. -void InstructionCodeGeneratorARM::VisitLoadClass(HLoadClass* cls) NO_THREAD_SAFETY_ANALYSIS { - HLoadClass::LoadKind load_kind = cls->GetLoadKind(); - if (load_kind == HLoadClass::LoadKind::kRuntimeCall) { - codegen_->GenerateLoadClassRuntimeCall(cls); - return; - } - DCHECK(!cls->NeedsAccessCheck()); - - LocationSummary* locations = cls->GetLocations(); - Location out_loc = locations->Out(); - Register out = out_loc.AsRegister<Register>(); - - const ReadBarrierOption read_barrier_option = cls->IsInBootImage() - ? kWithoutReadBarrier - : kCompilerReadBarrierOption; - bool generate_null_check = false; - switch (load_kind) { - case HLoadClass::LoadKind::kReferrersClass: { - DCHECK(!cls->CanCallRuntime()); - DCHECK(!cls->MustGenerateClinitCheck()); - // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_ - Register current_method = locations->InAt(0).AsRegister<Register>(); - GenerateGcRootFieldLoad(cls, - out_loc, - current_method, - ArtMethod::DeclaringClassOffset().Int32Value(), - read_barrier_option); - break; - } - case HLoadClass::LoadKind::kBootImageLinkTimePcRelative: { - DCHECK(codegen_->GetCompilerOptions().IsBootImage()); - DCHECK_EQ(read_barrier_option, kWithoutReadBarrier); - CodeGeneratorARM::PcRelativePatchInfo* labels = - codegen_->NewPcRelativeTypePatch(cls->GetDexFile(), cls->GetTypeIndex()); - __ BindTrackedLabel(&labels->movw_label); - __ movw(out, /* placeholder */ 0u); - __ BindTrackedLabel(&labels->movt_label); - __ movt(out, /* placeholder */ 0u); - __ BindTrackedLabel(&labels->add_pc_label); - __ add(out, out, ShifterOperand(PC)); - break; - } - case HLoadClass::LoadKind::kBootImageAddress: { - DCHECK_EQ(read_barrier_option, kWithoutReadBarrier); - uint32_t address = dchecked_integral_cast<uint32_t>( - reinterpret_cast<uintptr_t>(cls->GetClass().Get())); - DCHECK_NE(address, 0u); - __ LoadLiteral(out, codegen_->DeduplicateBootImageAddressLiteral(address)); - break; - } - case HLoadClass::LoadKind::kBssEntry: { - Register temp = (!kUseReadBarrier || kUseBakerReadBarrier) - ? locations->GetTemp(0).AsRegister<Register>() - : out; - CodeGeneratorARM::PcRelativePatchInfo* labels = - codegen_->NewTypeBssEntryPatch(cls->GetDexFile(), cls->GetTypeIndex()); - __ BindTrackedLabel(&labels->movw_label); - __ movw(temp, /* placeholder */ 0u); - __ BindTrackedLabel(&labels->movt_label); - __ movt(temp, /* placeholder */ 0u); - __ BindTrackedLabel(&labels->add_pc_label); - __ add(temp, temp, ShifterOperand(PC)); - GenerateGcRootFieldLoad(cls, out_loc, temp, /* offset */ 0, read_barrier_option); - generate_null_check = true; - break; - } - case HLoadClass::LoadKind::kJitTableAddress: { - __ LoadLiteral(out, codegen_->DeduplicateJitClassLiteral(cls->GetDexFile(), - cls->GetTypeIndex(), - cls->GetClass())); - // /* GcRoot<mirror::Class> */ out = *out - GenerateGcRootFieldLoad(cls, out_loc, out, /* offset */ 0, read_barrier_option); - break; - } - case HLoadClass::LoadKind::kRuntimeCall: - case HLoadClass::LoadKind::kInvalid: - LOG(FATAL) << "UNREACHABLE"; - UNREACHABLE(); - } - - if (generate_null_check || cls->MustGenerateClinitCheck()) { - DCHECK(cls->CanCallRuntime()); - SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathARM( - cls, cls, cls->GetDexPc(), cls->MustGenerateClinitCheck()); - codegen_->AddSlowPath(slow_path); - if (generate_null_check) { - __ CompareAndBranchIfZero(out, slow_path->GetEntryLabel()); - } - if (cls->MustGenerateClinitCheck()) { - GenerateClassInitializationCheck(slow_path, out); - } else { - __ Bind(slow_path->GetExitLabel()); - } - } -} - -void LocationsBuilderARM::VisitClinitCheck(HClinitCheck* check) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(check, LocationSummary::kCallOnSlowPath); - locations->SetInAt(0, Location::RequiresRegister()); - if (check->HasUses()) { - locations->SetOut(Location::SameAsFirstInput()); - } -} - -void InstructionCodeGeneratorARM::VisitClinitCheck(HClinitCheck* check) { - // We assume the class is not null. - SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathARM( - check->GetLoadClass(), check, check->GetDexPc(), true); - codegen_->AddSlowPath(slow_path); - GenerateClassInitializationCheck(slow_path, - check->GetLocations()->InAt(0).AsRegister<Register>()); -} - -void InstructionCodeGeneratorARM::GenerateClassInitializationCheck( - SlowPathCodeARM* slow_path, Register class_reg) { - __ LoadFromOffset(kLoadWord, IP, class_reg, mirror::Class::StatusOffset().Int32Value()); - __ cmp(IP, ShifterOperand(mirror::Class::kStatusInitialized)); - __ b(slow_path->GetEntryLabel(), LT); - // Even if the initialized flag is set, we may be in a situation where caches are not synced - // properly. Therefore, we do a memory fence. - __ dmb(ISH); - __ Bind(slow_path->GetExitLabel()); -} - -HLoadString::LoadKind CodeGeneratorARM::GetSupportedLoadStringKind( - HLoadString::LoadKind desired_string_load_kind) { - switch (desired_string_load_kind) { - case HLoadString::LoadKind::kBootImageLinkTimePcRelative: - case HLoadString::LoadKind::kBssEntry: - DCHECK(!Runtime::Current()->UseJitCompilation()); - break; - case HLoadString::LoadKind::kJitTableAddress: - DCHECK(Runtime::Current()->UseJitCompilation()); - break; - case HLoadString::LoadKind::kBootImageAddress: - case HLoadString::LoadKind::kRuntimeCall: - break; - } - return desired_string_load_kind; -} - -void LocationsBuilderARM::VisitLoadString(HLoadString* load) { - LocationSummary::CallKind call_kind = CodeGenerator::GetLoadStringCallKind(load); - LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(load, call_kind); - HLoadString::LoadKind load_kind = load->GetLoadKind(); - if (load_kind == HLoadString::LoadKind::kRuntimeCall) { - locations->SetOut(Location::RegisterLocation(R0)); - } else { - locations->SetOut(Location::RequiresRegister()); - if (load_kind == HLoadString::LoadKind::kBssEntry) { - if (!kUseReadBarrier || kUseBakerReadBarrier) { - // Rely on the pResolveString and marking to save everything we need, including temps. - // Note that IP may be clobbered by saving/restoring the live register (only one thanks - // to the custom calling convention) or by marking, so we request a different temp. - locations->AddTemp(Location::RequiresRegister()); - RegisterSet caller_saves = RegisterSet::Empty(); - InvokeRuntimeCallingConvention calling_convention; - caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); - // TODO: Add GetReturnLocation() to the calling convention so that we can DCHECK() - // that the the kPrimNot result register is the same as the first argument register. - locations->SetCustomSlowPathCallerSaves(caller_saves); - if (kUseBakerReadBarrier && kBakerReadBarrierLinkTimeThunksEnableForGcRoots) { - locations->AddTemp(Location::RegisterLocation(kBakerCcEntrypointRegister)); - } - } else { - // For non-Baker read barrier we have a temp-clobbering call. - } - } - } -} - -// NO_THREAD_SAFETY_ANALYSIS as we manipulate handles whose internal object we know does not -// move. -void InstructionCodeGeneratorARM::VisitLoadString(HLoadString* load) NO_THREAD_SAFETY_ANALYSIS { - LocationSummary* locations = load->GetLocations(); - Location out_loc = locations->Out(); - Register out = out_loc.AsRegister<Register>(); - HLoadString::LoadKind load_kind = load->GetLoadKind(); - - switch (load_kind) { - case HLoadString::LoadKind::kBootImageLinkTimePcRelative: { - DCHECK(codegen_->GetCompilerOptions().IsBootImage()); - CodeGeneratorARM::PcRelativePatchInfo* labels = - codegen_->NewPcRelativeStringPatch(load->GetDexFile(), load->GetStringIndex()); - __ BindTrackedLabel(&labels->movw_label); - __ movw(out, /* placeholder */ 0u); - __ BindTrackedLabel(&labels->movt_label); - __ movt(out, /* placeholder */ 0u); - __ BindTrackedLabel(&labels->add_pc_label); - __ add(out, out, ShifterOperand(PC)); - return; // No dex cache slow path. - } - case HLoadString::LoadKind::kBootImageAddress: { - uint32_t address = dchecked_integral_cast<uint32_t>( - reinterpret_cast<uintptr_t>(load->GetString().Get())); - DCHECK_NE(address, 0u); - __ LoadLiteral(out, codegen_->DeduplicateBootImageAddressLiteral(address)); - return; // No dex cache slow path. - } - case HLoadString::LoadKind::kBssEntry: { - DCHECK(!codegen_->GetCompilerOptions().IsBootImage()); - Register temp = (!kUseReadBarrier || kUseBakerReadBarrier) - ? locations->GetTemp(0).AsRegister<Register>() - : out; - CodeGeneratorARM::PcRelativePatchInfo* labels = - codegen_->NewPcRelativeStringPatch(load->GetDexFile(), load->GetStringIndex()); - __ BindTrackedLabel(&labels->movw_label); - __ movw(temp, /* placeholder */ 0u); - __ BindTrackedLabel(&labels->movt_label); - __ movt(temp, /* placeholder */ 0u); - __ BindTrackedLabel(&labels->add_pc_label); - __ add(temp, temp, ShifterOperand(PC)); - GenerateGcRootFieldLoad(load, out_loc, temp, /* offset */ 0, kCompilerReadBarrierOption); - SlowPathCode* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathARM(load); - codegen_->AddSlowPath(slow_path); - __ CompareAndBranchIfZero(out, slow_path->GetEntryLabel()); - __ Bind(slow_path->GetExitLabel()); - return; - } - case HLoadString::LoadKind::kJitTableAddress: { - __ LoadLiteral(out, codegen_->DeduplicateJitStringLiteral(load->GetDexFile(), - load->GetStringIndex(), - load->GetString())); - // /* GcRoot<mirror::String> */ out = *out - GenerateGcRootFieldLoad(load, out_loc, out, /* offset */ 0, kCompilerReadBarrierOption); - return; - } - default: - break; - } - - // TODO: Consider re-adding the compiler code to do string dex cache lookup again. - DCHECK(load_kind == HLoadString::LoadKind::kRuntimeCall); - InvokeRuntimeCallingConvention calling_convention; - DCHECK_EQ(calling_convention.GetRegisterAt(0), out); - __ LoadImmediate(calling_convention.GetRegisterAt(0), load->GetStringIndex().index_); - codegen_->InvokeRuntime(kQuickResolveString, load, load->GetDexPc()); - CheckEntrypointTypes<kQuickResolveString, void*, uint32_t>(); -} - -static int32_t GetExceptionTlsOffset() { - return Thread::ExceptionOffset<kArmPointerSize>().Int32Value(); -} - -void LocationsBuilderARM::VisitLoadException(HLoadException* load) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kNoCall); - locations->SetOut(Location::RequiresRegister()); -} - -void InstructionCodeGeneratorARM::VisitLoadException(HLoadException* load) { - Register out = load->GetLocations()->Out().AsRegister<Register>(); - __ LoadFromOffset(kLoadWord, out, TR, GetExceptionTlsOffset()); -} - -void LocationsBuilderARM::VisitClearException(HClearException* clear) { - new (GetGraph()->GetArena()) LocationSummary(clear, LocationSummary::kNoCall); -} - -void InstructionCodeGeneratorARM::VisitClearException(HClearException* clear ATTRIBUTE_UNUSED) { - __ LoadImmediate(IP, 0); - __ StoreToOffset(kStoreWord, IP, TR, GetExceptionTlsOffset()); -} - -void LocationsBuilderARM::VisitThrow(HThrow* instruction) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly); - InvokeRuntimeCallingConvention calling_convention; - locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); -} - -void InstructionCodeGeneratorARM::VisitThrow(HThrow* instruction) { - codegen_->InvokeRuntime(kQuickDeliverException, instruction, instruction->GetDexPc()); - CheckEntrypointTypes<kQuickDeliverException, void, mirror::Object*>(); -} - -// Temp is used for read barrier. -static size_t NumberOfInstanceOfTemps(TypeCheckKind type_check_kind) { - if (kEmitCompilerReadBarrier && - (kUseBakerReadBarrier || - type_check_kind == TypeCheckKind::kAbstractClassCheck || - type_check_kind == TypeCheckKind::kClassHierarchyCheck || - type_check_kind == TypeCheckKind::kArrayObjectCheck)) { - return 1; - } - return 0; -} - -// Interface case has 3 temps, one for holding the number of interfaces, one for the current -// interface pointer, one for loading the current interface. -// The other checks have one temp for loading the object's class. -static size_t NumberOfCheckCastTemps(TypeCheckKind type_check_kind) { - if (type_check_kind == TypeCheckKind::kInterfaceCheck) { - return 3; - } - return 1 + NumberOfInstanceOfTemps(type_check_kind); -} - -void LocationsBuilderARM::VisitInstanceOf(HInstanceOf* instruction) { - LocationSummary::CallKind call_kind = LocationSummary::kNoCall; - TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); - bool baker_read_barrier_slow_path = false; - switch (type_check_kind) { - case TypeCheckKind::kExactCheck: - case TypeCheckKind::kAbstractClassCheck: - case TypeCheckKind::kClassHierarchyCheck: - case TypeCheckKind::kArrayObjectCheck: - call_kind = - kEmitCompilerReadBarrier ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall; - baker_read_barrier_slow_path = kUseBakerReadBarrier; - break; - case TypeCheckKind::kArrayCheck: - case TypeCheckKind::kUnresolvedCheck: - case TypeCheckKind::kInterfaceCheck: - call_kind = LocationSummary::kCallOnSlowPath; - break; - } - - LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind); - if (baker_read_barrier_slow_path) { - locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. - } - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RequiresRegister()); - // The "out" register is used as a temporary, so it overlaps with the inputs. - // Note that TypeCheckSlowPathARM uses this register too. - locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); - locations->AddRegisterTemps(NumberOfInstanceOfTemps(type_check_kind)); - if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { - codegen_->MaybeAddBakerCcEntrypointTempForFields(locations); - } -} - -void InstructionCodeGeneratorARM::VisitInstanceOf(HInstanceOf* instruction) { - TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); - LocationSummary* locations = instruction->GetLocations(); - Location obj_loc = locations->InAt(0); - Register obj = obj_loc.AsRegister<Register>(); - Register cls = locations->InAt(1).AsRegister<Register>(); - Location out_loc = locations->Out(); - Register out = out_loc.AsRegister<Register>(); - const size_t num_temps = NumberOfInstanceOfTemps(type_check_kind); - DCHECK_LE(num_temps, 1u); - Location maybe_temp_loc = (num_temps >= 1) ? locations->GetTemp(0) : Location::NoLocation(); - uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); - uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); - uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); - uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value(); - Label done; - Label* const final_label = codegen_->GetFinalLabel(instruction, &done); - SlowPathCodeARM* slow_path = nullptr; - - // Return 0 if `obj` is null. - // avoid null check if we know obj is not null. - if (instruction->MustDoNullCheck()) { - DCHECK_NE(out, obj); - __ LoadImmediate(out, 0); - __ CompareAndBranchIfZero(obj, final_label); - } - - switch (type_check_kind) { - case TypeCheckKind::kExactCheck: { - // /* HeapReference<Class> */ out = obj->klass_ - GenerateReferenceLoadTwoRegisters(instruction, - out_loc, - obj_loc, - class_offset, - maybe_temp_loc, - kCompilerReadBarrierOption); - // Classes must be equal for the instanceof to succeed. - __ cmp(out, ShifterOperand(cls)); - // We speculatively set the result to false without changing the condition - // flags, which allows us to avoid some branching later. - __ mov(out, ShifterOperand(0), AL, kCcKeep); - - // Since IT blocks longer than a 16-bit instruction are deprecated by ARMv8, - // we check that the output is in a low register, so that a 16-bit MOV - // encoding can be used. - if (ArmAssembler::IsLowRegister(out)) { - __ it(EQ); - __ mov(out, ShifterOperand(1), EQ); - } else { - __ b(final_label, NE); - __ LoadImmediate(out, 1); - } - - break; - } - - case TypeCheckKind::kAbstractClassCheck: { - // /* HeapReference<Class> */ out = obj->klass_ - GenerateReferenceLoadTwoRegisters(instruction, - out_loc, - obj_loc, - class_offset, - maybe_temp_loc, - kCompilerReadBarrierOption); - // If the class is abstract, we eagerly fetch the super class of the - // object to avoid doing a comparison we know will fail. - Label loop; - __ Bind(&loop); - // /* HeapReference<Class> */ out = out->super_class_ - GenerateReferenceLoadOneRegister(instruction, - out_loc, - super_offset, - maybe_temp_loc, - kCompilerReadBarrierOption); - // If `out` is null, we use it for the result, and jump to the final label. - __ CompareAndBranchIfZero(out, final_label); - __ cmp(out, ShifterOperand(cls)); - __ b(&loop, NE); - __ LoadImmediate(out, 1); - break; - } - - case TypeCheckKind::kClassHierarchyCheck: { - // /* HeapReference<Class> */ out = obj->klass_ - GenerateReferenceLoadTwoRegisters(instruction, - out_loc, - obj_loc, - class_offset, - maybe_temp_loc, - kCompilerReadBarrierOption); - // Walk over the class hierarchy to find a match. - Label loop, success; - __ Bind(&loop); - __ cmp(out, ShifterOperand(cls)); - __ b(&success, EQ); - // /* HeapReference<Class> */ out = out->super_class_ - GenerateReferenceLoadOneRegister(instruction, - out_loc, - super_offset, - maybe_temp_loc, - kCompilerReadBarrierOption); - // This is essentially a null check, but it sets the condition flags to the - // proper value for the code that follows the loop, i.e. not `EQ`. - __ cmp(out, ShifterOperand(1)); - __ b(&loop, HS); - - // Since IT blocks longer than a 16-bit instruction are deprecated by ARMv8, - // we check that the output is in a low register, so that a 16-bit MOV - // encoding can be used. - if (ArmAssembler::IsLowRegister(out)) { - // If `out` is null, we use it for the result, and the condition flags - // have already been set to `NE`, so the IT block that comes afterwards - // (and which handles the successful case) turns into a NOP (instead of - // overwriting `out`). - __ Bind(&success); - // There is only one branch to the `success` label (which is bound to this - // IT block), and it has the same condition, `EQ`, so in that case the MOV - // is executed. - __ it(EQ); - __ mov(out, ShifterOperand(1), EQ); - } else { - // If `out` is null, we use it for the result, and jump to the final label. - __ b(final_label); - __ Bind(&success); - __ LoadImmediate(out, 1); - } - - break; - } - - case TypeCheckKind::kArrayObjectCheck: { - // /* HeapReference<Class> */ out = obj->klass_ - GenerateReferenceLoadTwoRegisters(instruction, - out_loc, - obj_loc, - class_offset, - maybe_temp_loc, - kCompilerReadBarrierOption); - // Do an exact check. - Label exact_check; - __ cmp(out, ShifterOperand(cls)); - __ b(&exact_check, EQ); - // Otherwise, we need to check that the object's class is a non-primitive array. - // /* HeapReference<Class> */ out = out->component_type_ - GenerateReferenceLoadOneRegister(instruction, - out_loc, - component_offset, - maybe_temp_loc, - kCompilerReadBarrierOption); - // If `out` is null, we use it for the result, and jump to the final label. - __ CompareAndBranchIfZero(out, final_label); - __ LoadFromOffset(kLoadUnsignedHalfword, out, out, primitive_offset); - static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot"); - __ cmp(out, ShifterOperand(0)); - // We speculatively set the result to false without changing the condition - // flags, which allows us to avoid some branching later. - __ mov(out, ShifterOperand(0), AL, kCcKeep); - - // Since IT blocks longer than a 16-bit instruction are deprecated by ARMv8, - // we check that the output is in a low register, so that a 16-bit MOV - // encoding can be used. - if (ArmAssembler::IsLowRegister(out)) { - __ Bind(&exact_check); - __ it(EQ); - __ mov(out, ShifterOperand(1), EQ); - } else { - __ b(final_label, NE); - __ Bind(&exact_check); - __ LoadImmediate(out, 1); - } - - break; - } - - case TypeCheckKind::kArrayCheck: { - // No read barrier since the slow path will retry upon failure. - // /* HeapReference<Class> */ out = obj->klass_ - GenerateReferenceLoadTwoRegisters(instruction, - out_loc, - obj_loc, - class_offset, - maybe_temp_loc, - kWithoutReadBarrier); - __ cmp(out, ShifterOperand(cls)); - DCHECK(locations->OnlyCallsOnSlowPath()); - slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathARM(instruction, - /* is_fatal */ false); - codegen_->AddSlowPath(slow_path); - __ b(slow_path->GetEntryLabel(), NE); - __ LoadImmediate(out, 1); - break; - } - - case TypeCheckKind::kUnresolvedCheck: - case TypeCheckKind::kInterfaceCheck: { - // Note that we indeed only call on slow path, but we always go - // into the slow path for the unresolved and interface check - // cases. - // - // We cannot directly call the InstanceofNonTrivial runtime - // entry point without resorting to a type checking slow path - // here (i.e. by calling InvokeRuntime directly), as it would - // require to assign fixed registers for the inputs of this - // HInstanceOf instruction (following the runtime calling - // convention), which might be cluttered by the potential first - // read barrier emission at the beginning of this method. - // - // TODO: Introduce a new runtime entry point taking the object - // to test (instead of its class) as argument, and let it deal - // with the read barrier issues. This will let us refactor this - // case of the `switch` code as it was previously (with a direct - // call to the runtime not using a type checking slow path). - // This should also be beneficial for the other cases above. - DCHECK(locations->OnlyCallsOnSlowPath()); - slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathARM(instruction, - /* is_fatal */ false); - codegen_->AddSlowPath(slow_path); - __ b(slow_path->GetEntryLabel()); - break; - } - } - - if (done.IsLinked()) { - __ Bind(&done); - } - - if (slow_path != nullptr) { - __ Bind(slow_path->GetExitLabel()); - } -} - -void LocationsBuilderARM::VisitCheckCast(HCheckCast* instruction) { - LocationSummary::CallKind call_kind = LocationSummary::kNoCall; - bool throws_into_catch = instruction->CanThrowIntoCatchBlock(); - - TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); - switch (type_check_kind) { - case TypeCheckKind::kExactCheck: - case TypeCheckKind::kAbstractClassCheck: - case TypeCheckKind::kClassHierarchyCheck: - case TypeCheckKind::kArrayObjectCheck: - call_kind = (throws_into_catch || kEmitCompilerReadBarrier) ? - LocationSummary::kCallOnSlowPath : - LocationSummary::kNoCall; // In fact, call on a fatal (non-returning) slow path. - break; - case TypeCheckKind::kArrayCheck: - case TypeCheckKind::kUnresolvedCheck: - case TypeCheckKind::kInterfaceCheck: - call_kind = LocationSummary::kCallOnSlowPath; - break; - } - - LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind); - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RequiresRegister()); - locations->AddRegisterTemps(NumberOfCheckCastTemps(type_check_kind)); -} - -void InstructionCodeGeneratorARM::VisitCheckCast(HCheckCast* instruction) { - TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); - LocationSummary* locations = instruction->GetLocations(); - Location obj_loc = locations->InAt(0); - Register obj = obj_loc.AsRegister<Register>(); - Register cls = locations->InAt(1).AsRegister<Register>(); - Location temp_loc = locations->GetTemp(0); - Register temp = temp_loc.AsRegister<Register>(); - const size_t num_temps = NumberOfCheckCastTemps(type_check_kind); - DCHECK_LE(num_temps, 3u); - Location maybe_temp2_loc = (num_temps >= 2) ? locations->GetTemp(1) : Location::NoLocation(); - Location maybe_temp3_loc = (num_temps >= 3) ? locations->GetTemp(2) : Location::NoLocation(); - const uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); - const uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); - const uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); - const uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value(); - const uint32_t iftable_offset = mirror::Class::IfTableOffset().Uint32Value(); - const uint32_t array_length_offset = mirror::Array::LengthOffset().Uint32Value(); - const uint32_t object_array_data_offset = - mirror::Array::DataOffset(kHeapReferenceSize).Uint32Value(); - - // Always false for read barriers since we may need to go to the entrypoint for non-fatal cases - // from false negatives. The false negatives may come from avoiding read barriers below. Avoiding - // read barriers is done for performance and code size reasons. - bool is_type_check_slow_path_fatal = false; - if (!kEmitCompilerReadBarrier) { - is_type_check_slow_path_fatal = - (type_check_kind == TypeCheckKind::kExactCheck || - type_check_kind == TypeCheckKind::kAbstractClassCheck || - type_check_kind == TypeCheckKind::kClassHierarchyCheck || - type_check_kind == TypeCheckKind::kArrayObjectCheck) && - !instruction->CanThrowIntoCatchBlock(); - } - SlowPathCodeARM* type_check_slow_path = - new (GetGraph()->GetArena()) TypeCheckSlowPathARM(instruction, - is_type_check_slow_path_fatal); - codegen_->AddSlowPath(type_check_slow_path); - - Label done; - Label* final_label = codegen_->GetFinalLabel(instruction, &done); - // Avoid null check if we know obj is not null. - if (instruction->MustDoNullCheck()) { - __ CompareAndBranchIfZero(obj, final_label); - } - - switch (type_check_kind) { - case TypeCheckKind::kExactCheck: - case TypeCheckKind::kArrayCheck: { - // /* HeapReference<Class> */ temp = obj->klass_ - GenerateReferenceLoadTwoRegisters(instruction, - temp_loc, - obj_loc, - class_offset, - maybe_temp2_loc, - kWithoutReadBarrier); - - __ cmp(temp, ShifterOperand(cls)); - // Jump to slow path for throwing the exception or doing a - // more involved array check. - __ b(type_check_slow_path->GetEntryLabel(), NE); - break; - } - - case TypeCheckKind::kAbstractClassCheck: { - // /* HeapReference<Class> */ temp = obj->klass_ - GenerateReferenceLoadTwoRegisters(instruction, - temp_loc, - obj_loc, - class_offset, - maybe_temp2_loc, - kWithoutReadBarrier); - - // If the class is abstract, we eagerly fetch the super class of the - // object to avoid doing a comparison we know will fail. - Label loop; - __ Bind(&loop); - // /* HeapReference<Class> */ temp = temp->super_class_ - GenerateReferenceLoadOneRegister(instruction, - temp_loc, - super_offset, - maybe_temp2_loc, - kWithoutReadBarrier); - - // If the class reference currently in `temp` is null, jump to the slow path to throw the - // exception. - __ CompareAndBranchIfZero(temp, type_check_slow_path->GetEntryLabel()); - - // Otherwise, compare the classes. - __ cmp(temp, ShifterOperand(cls)); - __ b(&loop, NE); - break; - } - - case TypeCheckKind::kClassHierarchyCheck: { - // /* HeapReference<Class> */ temp = obj->klass_ - GenerateReferenceLoadTwoRegisters(instruction, - temp_loc, - obj_loc, - class_offset, - maybe_temp2_loc, - kWithoutReadBarrier); - - // Walk over the class hierarchy to find a match. - Label loop; - __ Bind(&loop); - __ cmp(temp, ShifterOperand(cls)); - __ b(final_label, EQ); - - // /* HeapReference<Class> */ temp = temp->super_class_ - GenerateReferenceLoadOneRegister(instruction, - temp_loc, - super_offset, - maybe_temp2_loc, - kWithoutReadBarrier); - - // If the class reference currently in `temp` is null, jump to the slow path to throw the - // exception. - __ CompareAndBranchIfZero(temp, type_check_slow_path->GetEntryLabel()); - // Otherwise, jump to the beginning of the loop. - __ b(&loop); - break; - } - - case TypeCheckKind::kArrayObjectCheck: { - // /* HeapReference<Class> */ temp = obj->klass_ - GenerateReferenceLoadTwoRegisters(instruction, - temp_loc, - obj_loc, - class_offset, - maybe_temp2_loc, - kWithoutReadBarrier); - - // Do an exact check. - __ cmp(temp, ShifterOperand(cls)); - __ b(final_label, EQ); - - // Otherwise, we need to check that the object's class is a non-primitive array. - // /* HeapReference<Class> */ temp = temp->component_type_ - GenerateReferenceLoadOneRegister(instruction, - temp_loc, - component_offset, - maybe_temp2_loc, - kWithoutReadBarrier); - // If the component type is null, jump to the slow path to throw the exception. - __ CompareAndBranchIfZero(temp, type_check_slow_path->GetEntryLabel()); - // Otherwise,the object is indeed an array, jump to label `check_non_primitive_component_type` - // to further check that this component type is not a primitive type. - __ LoadFromOffset(kLoadUnsignedHalfword, temp, temp, primitive_offset); - static_assert(Primitive::kPrimNot == 0, "Expected 0 for art::Primitive::kPrimNot"); - __ CompareAndBranchIfNonZero(temp, type_check_slow_path->GetEntryLabel()); - break; - } - - case TypeCheckKind::kUnresolvedCheck: - // We always go into the type check slow path for the unresolved check case. - // We cannot directly call the CheckCast runtime entry point - // without resorting to a type checking slow path here (i.e. by - // calling InvokeRuntime directly), as it would require to - // assign fixed registers for the inputs of this HInstanceOf - // instruction (following the runtime calling convention), which - // might be cluttered by the potential first read barrier - // emission at the beginning of this method. - - __ b(type_check_slow_path->GetEntryLabel()); - break; - - case TypeCheckKind::kInterfaceCheck: { - // Avoid read barriers to improve performance of the fast path. We can not get false - // positives by doing this. - // /* HeapReference<Class> */ temp = obj->klass_ - GenerateReferenceLoadTwoRegisters(instruction, - temp_loc, - obj_loc, - class_offset, - maybe_temp2_loc, - kWithoutReadBarrier); - - // /* HeapReference<Class> */ temp = temp->iftable_ - GenerateReferenceLoadTwoRegisters(instruction, - temp_loc, - temp_loc, - iftable_offset, - maybe_temp2_loc, - kWithoutReadBarrier); - // Iftable is never null. - __ ldr(maybe_temp2_loc.AsRegister<Register>(), Address(temp, array_length_offset)); - // Loop through the iftable and check if any class matches. - Label start_loop; - __ Bind(&start_loop); - __ CompareAndBranchIfZero(maybe_temp2_loc.AsRegister<Register>(), - type_check_slow_path->GetEntryLabel()); - __ ldr(maybe_temp3_loc.AsRegister<Register>(), Address(temp, object_array_data_offset)); - __ MaybeUnpoisonHeapReference(maybe_temp3_loc.AsRegister<Register>()); - // Go to next interface. - __ add(temp, temp, ShifterOperand(2 * kHeapReferenceSize)); - __ sub(maybe_temp2_loc.AsRegister<Register>(), - maybe_temp2_loc.AsRegister<Register>(), - ShifterOperand(2)); - // Compare the classes and continue the loop if they do not match. - __ cmp(cls, ShifterOperand(maybe_temp3_loc.AsRegister<Register>())); - __ b(&start_loop, NE); - break; - } - } - - if (done.IsLinked()) { - __ Bind(&done); - } - - __ Bind(type_check_slow_path->GetExitLabel()); -} - -void LocationsBuilderARM::VisitMonitorOperation(HMonitorOperation* instruction) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly); - InvokeRuntimeCallingConvention calling_convention; - locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); -} - -void InstructionCodeGeneratorARM::VisitMonitorOperation(HMonitorOperation* instruction) { - codegen_->InvokeRuntime(instruction->IsEnter() ? kQuickLockObject : kQuickUnlockObject, - instruction, - instruction->GetDexPc()); - if (instruction->IsEnter()) { - CheckEntrypointTypes<kQuickLockObject, void, mirror::Object*>(); - } else { - CheckEntrypointTypes<kQuickUnlockObject, void, mirror::Object*>(); - } -} - -void LocationsBuilderARM::VisitAnd(HAnd* instruction) { HandleBitwiseOperation(instruction, AND); } -void LocationsBuilderARM::VisitOr(HOr* instruction) { HandleBitwiseOperation(instruction, ORR); } -void LocationsBuilderARM::VisitXor(HXor* instruction) { HandleBitwiseOperation(instruction, EOR); } - -void LocationsBuilderARM::HandleBitwiseOperation(HBinaryOperation* instruction, Opcode opcode) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); - DCHECK(instruction->GetResultType() == Primitive::kPrimInt - || instruction->GetResultType() == Primitive::kPrimLong); - // Note: GVN reorders commutative operations to have the constant on the right hand side. - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, ArmEncodableConstantOrRegister(instruction->InputAt(1), opcode)); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); -} - -void InstructionCodeGeneratorARM::VisitAnd(HAnd* instruction) { - HandleBitwiseOperation(instruction); -} - -void InstructionCodeGeneratorARM::VisitOr(HOr* instruction) { - HandleBitwiseOperation(instruction); -} - -void InstructionCodeGeneratorARM::VisitXor(HXor* instruction) { - HandleBitwiseOperation(instruction); -} - - -void LocationsBuilderARM::VisitBitwiseNegatedRight(HBitwiseNegatedRight* instruction) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); - DCHECK(instruction->GetResultType() == Primitive::kPrimInt - || instruction->GetResultType() == Primitive::kPrimLong); - - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); -} - -void InstructionCodeGeneratorARM::VisitBitwiseNegatedRight(HBitwiseNegatedRight* instruction) { - LocationSummary* locations = instruction->GetLocations(); - Location first = locations->InAt(0); - Location second = locations->InAt(1); - Location out = locations->Out(); - - if (instruction->GetResultType() == Primitive::kPrimInt) { - Register first_reg = first.AsRegister<Register>(); - ShifterOperand second_reg(second.AsRegister<Register>()); - Register out_reg = out.AsRegister<Register>(); - - switch (instruction->GetOpKind()) { - case HInstruction::kAnd: - __ bic(out_reg, first_reg, second_reg); - break; - case HInstruction::kOr: - __ orn(out_reg, first_reg, second_reg); - break; - // There is no EON on arm. - case HInstruction::kXor: - default: - LOG(FATAL) << "Unexpected instruction " << instruction->DebugName(); - UNREACHABLE(); - } - return; - - } else { - DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong); - Register first_low = first.AsRegisterPairLow<Register>(); - Register first_high = first.AsRegisterPairHigh<Register>(); - ShifterOperand second_low(second.AsRegisterPairLow<Register>()); - ShifterOperand second_high(second.AsRegisterPairHigh<Register>()); - Register out_low = out.AsRegisterPairLow<Register>(); - Register out_high = out.AsRegisterPairHigh<Register>(); - - switch (instruction->GetOpKind()) { - case HInstruction::kAnd: - __ bic(out_low, first_low, second_low); - __ bic(out_high, first_high, second_high); - break; - case HInstruction::kOr: - __ orn(out_low, first_low, second_low); - __ orn(out_high, first_high, second_high); - break; - // There is no EON on arm. - case HInstruction::kXor: - default: - LOG(FATAL) << "Unexpected instruction " << instruction->DebugName(); - UNREACHABLE(); - } - } -} - -void LocationsBuilderARM::VisitDataProcWithShifterOp( - HDataProcWithShifterOp* instruction) { - DCHECK(instruction->GetType() == Primitive::kPrimInt || - instruction->GetType() == Primitive::kPrimLong); - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); - const bool overlap = instruction->GetType() == Primitive::kPrimLong && - HDataProcWithShifterOp::IsExtensionOp(instruction->GetOpKind()); - - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), - overlap ? Location::kOutputOverlap : Location::kNoOutputOverlap); -} - -void InstructionCodeGeneratorARM::VisitDataProcWithShifterOp( - HDataProcWithShifterOp* instruction) { - const LocationSummary* const locations = instruction->GetLocations(); - const HInstruction::InstructionKind kind = instruction->GetInstrKind(); - const HDataProcWithShifterOp::OpKind op_kind = instruction->GetOpKind(); - const Location left = locations->InAt(0); - const Location right = locations->InAt(1); - const Location out = locations->Out(); - - if (instruction->GetType() == Primitive::kPrimInt) { - DCHECK(!HDataProcWithShifterOp::IsExtensionOp(op_kind)); - - const Register second = instruction->InputAt(1)->GetType() == Primitive::kPrimLong - ? right.AsRegisterPairLow<Register>() - : right.AsRegister<Register>(); - - GenerateDataProcInstruction(kind, - out.AsRegister<Register>(), - left.AsRegister<Register>(), - ShifterOperand(second, - ShiftFromOpKind(op_kind), - instruction->GetShiftAmount()), - codegen_); - } else { - DCHECK_EQ(instruction->GetType(), Primitive::kPrimLong); - - if (HDataProcWithShifterOp::IsExtensionOp(op_kind)) { - const Register second = right.AsRegister<Register>(); - - DCHECK_NE(out.AsRegisterPairLow<Register>(), second); - GenerateDataProc(kind, - out, - left, - ShifterOperand(second), - ShifterOperand(second, ASR, 31), - codegen_); - } else { - GenerateLongDataProc(instruction, codegen_); - } - } -} - -void InstructionCodeGeneratorARM::GenerateAndConst(Register out, Register first, uint32_t value) { - // Optimize special cases for individual halfs of `and-long` (`and` is simplified earlier). - if (value == 0xffffffffu) { - if (out != first) { - __ mov(out, ShifterOperand(first)); - } - return; - } - if (value == 0u) { - __ mov(out, ShifterOperand(0)); - return; - } - ShifterOperand so; - if (__ ShifterOperandCanHold(kNoRegister, kNoRegister, AND, value, &so)) { - __ and_(out, first, so); - } else if (__ ShifterOperandCanHold(kNoRegister, kNoRegister, BIC, ~value, &so)) { - __ bic(out, first, ShifterOperand(~value)); - } else { - DCHECK(IsPowerOfTwo(value + 1)); - __ ubfx(out, first, 0, WhichPowerOf2(value + 1)); - } -} - -void InstructionCodeGeneratorARM::GenerateOrrConst(Register out, Register first, uint32_t value) { - // Optimize special cases for individual halfs of `or-long` (`or` is simplified earlier). - if (value == 0u) { - if (out != first) { - __ mov(out, ShifterOperand(first)); - } - return; - } - if (value == 0xffffffffu) { - __ mvn(out, ShifterOperand(0)); - return; - } - ShifterOperand so; - if (__ ShifterOperandCanHold(kNoRegister, kNoRegister, ORR, value, &so)) { - __ orr(out, first, so); - } else { - DCHECK(__ ShifterOperandCanHold(kNoRegister, kNoRegister, ORN, ~value, &so)); - __ orn(out, first, ShifterOperand(~value)); - } -} - -void InstructionCodeGeneratorARM::GenerateEorConst(Register out, Register first, uint32_t value) { - // Optimize special case for individual halfs of `xor-long` (`xor` is simplified earlier). - if (value == 0u) { - if (out != first) { - __ mov(out, ShifterOperand(first)); - } - return; - } - __ eor(out, first, ShifterOperand(value)); -} - -void InstructionCodeGeneratorARM::GenerateAddLongConst(Location out, - Location first, - uint64_t value) { - Register out_low = out.AsRegisterPairLow<Register>(); - Register out_high = out.AsRegisterPairHigh<Register>(); - Register first_low = first.AsRegisterPairLow<Register>(); - Register first_high = first.AsRegisterPairHigh<Register>(); - uint32_t value_low = Low32Bits(value); - uint32_t value_high = High32Bits(value); - if (value_low == 0u) { - if (out_low != first_low) { - __ mov(out_low, ShifterOperand(first_low)); - } - __ AddConstant(out_high, first_high, value_high); - return; - } - __ AddConstantSetFlags(out_low, first_low, value_low); - ShifterOperand so; - if (__ ShifterOperandCanHold(out_high, first_high, ADC, value_high, kCcDontCare, &so)) { - __ adc(out_high, first_high, so); - } else if (__ ShifterOperandCanHold(out_low, first_low, SBC, ~value_high, kCcDontCare, &so)) { - __ sbc(out_high, first_high, so); - } else { - LOG(FATAL) << "Unexpected constant " << value_high; - UNREACHABLE(); - } -} - -void InstructionCodeGeneratorARM::HandleBitwiseOperation(HBinaryOperation* instruction) { - LocationSummary* locations = instruction->GetLocations(); - Location first = locations->InAt(0); - Location second = locations->InAt(1); - Location out = locations->Out(); - - if (second.IsConstant()) { - uint64_t value = static_cast<uint64_t>(Int64FromConstant(second.GetConstant())); - uint32_t value_low = Low32Bits(value); - if (instruction->GetResultType() == Primitive::kPrimInt) { - Register first_reg = first.AsRegister<Register>(); - Register out_reg = out.AsRegister<Register>(); - if (instruction->IsAnd()) { - GenerateAndConst(out_reg, first_reg, value_low); - } else if (instruction->IsOr()) { - GenerateOrrConst(out_reg, first_reg, value_low); - } else { - DCHECK(instruction->IsXor()); - GenerateEorConst(out_reg, first_reg, value_low); - } - } else { - DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong); - uint32_t value_high = High32Bits(value); - Register first_low = first.AsRegisterPairLow<Register>(); - Register first_high = first.AsRegisterPairHigh<Register>(); - Register out_low = out.AsRegisterPairLow<Register>(); - Register out_high = out.AsRegisterPairHigh<Register>(); - if (instruction->IsAnd()) { - GenerateAndConst(out_low, first_low, value_low); - GenerateAndConst(out_high, first_high, value_high); - } else if (instruction->IsOr()) { - GenerateOrrConst(out_low, first_low, value_low); - GenerateOrrConst(out_high, first_high, value_high); - } else { - DCHECK(instruction->IsXor()); - GenerateEorConst(out_low, first_low, value_low); - GenerateEorConst(out_high, first_high, value_high); - } - } - return; - } - - if (instruction->GetResultType() == Primitive::kPrimInt) { - Register first_reg = first.AsRegister<Register>(); - ShifterOperand second_reg(second.AsRegister<Register>()); - Register out_reg = out.AsRegister<Register>(); - if (instruction->IsAnd()) { - __ and_(out_reg, first_reg, second_reg); - } else if (instruction->IsOr()) { - __ orr(out_reg, first_reg, second_reg); - } else { - DCHECK(instruction->IsXor()); - __ eor(out_reg, first_reg, second_reg); - } - } else { - DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong); - Register first_low = first.AsRegisterPairLow<Register>(); - Register first_high = first.AsRegisterPairHigh<Register>(); - ShifterOperand second_low(second.AsRegisterPairLow<Register>()); - ShifterOperand second_high(second.AsRegisterPairHigh<Register>()); - Register out_low = out.AsRegisterPairLow<Register>(); - Register out_high = out.AsRegisterPairHigh<Register>(); - if (instruction->IsAnd()) { - __ and_(out_low, first_low, second_low); - __ and_(out_high, first_high, second_high); - } else if (instruction->IsOr()) { - __ orr(out_low, first_low, second_low); - __ orr(out_high, first_high, second_high); - } else { - DCHECK(instruction->IsXor()); - __ eor(out_low, first_low, second_low); - __ eor(out_high, first_high, second_high); - } - } -} - -void InstructionCodeGeneratorARM::GenerateReferenceLoadOneRegister( - HInstruction* instruction, - Location out, - uint32_t offset, - Location maybe_temp, - ReadBarrierOption read_barrier_option) { - Register out_reg = out.AsRegister<Register>(); - if (read_barrier_option == kWithReadBarrier) { - CHECK(kEmitCompilerReadBarrier); - DCHECK(maybe_temp.IsRegister()) << maybe_temp; - if (kUseBakerReadBarrier) { - // Load with fast path based Baker's read barrier. - // /* HeapReference<Object> */ out = *(out + offset) - codegen_->GenerateFieldLoadWithBakerReadBarrier( - instruction, out, out_reg, offset, maybe_temp, /* needs_null_check */ false); - } else { - // Load with slow path based read barrier. - // Save the value of `out` into `maybe_temp` before overwriting it - // in the following move operation, as we will need it for the - // read barrier below. - __ Mov(maybe_temp.AsRegister<Register>(), out_reg); - // /* HeapReference<Object> */ out = *(out + offset) - __ LoadFromOffset(kLoadWord, out_reg, out_reg, offset); - codegen_->GenerateReadBarrierSlow(instruction, out, out, maybe_temp, offset); - } - } else { - // Plain load with no read barrier. - // /* HeapReference<Object> */ out = *(out + offset) - __ LoadFromOffset(kLoadWord, out_reg, out_reg, offset); - __ MaybeUnpoisonHeapReference(out_reg); - } -} - -void InstructionCodeGeneratorARM::GenerateReferenceLoadTwoRegisters( - HInstruction* instruction, - Location out, - Location obj, - uint32_t offset, - Location maybe_temp, - ReadBarrierOption read_barrier_option) { - Register out_reg = out.AsRegister<Register>(); - Register obj_reg = obj.AsRegister<Register>(); - if (read_barrier_option == kWithReadBarrier) { - CHECK(kEmitCompilerReadBarrier); - if (kUseBakerReadBarrier) { - DCHECK(maybe_temp.IsRegister()) << maybe_temp; - // Load with fast path based Baker's read barrier. - // /* HeapReference<Object> */ out = *(obj + offset) - codegen_->GenerateFieldLoadWithBakerReadBarrier( - instruction, out, obj_reg, offset, maybe_temp, /* needs_null_check */ false); - } else { - // Load with slow path based read barrier. - // /* HeapReference<Object> */ out = *(obj + offset) - __ LoadFromOffset(kLoadWord, out_reg, obj_reg, offset); - codegen_->GenerateReadBarrierSlow(instruction, out, out, obj, offset); - } - } else { - // Plain load with no read barrier. - // /* HeapReference<Object> */ out = *(obj + offset) - __ LoadFromOffset(kLoadWord, out_reg, obj_reg, offset); - __ MaybeUnpoisonHeapReference(out_reg); - } -} - -void InstructionCodeGeneratorARM::GenerateGcRootFieldLoad(HInstruction* instruction, - Location root, - Register obj, - uint32_t offset, - ReadBarrierOption read_barrier_option) { - Register root_reg = root.AsRegister<Register>(); - if (read_barrier_option == kWithReadBarrier) { - DCHECK(kEmitCompilerReadBarrier); - if (kUseBakerReadBarrier) { - // Fast path implementation of art::ReadBarrier::BarrierForRoot when - // Baker's read barrier are used. - if (kBakerReadBarrierLinkTimeThunksEnableForGcRoots && - !Runtime::Current()->UseJitCompilation()) { - // Note that we do not actually check the value of `GetIsGcMarking()` - // to decide whether to mark the loaded GC root or not. Instead, we - // load into `temp` (actually kBakerCcEntrypointRegister) the read - // barrier mark introspection entrypoint. If `temp` is null, it means - // that `GetIsGcMarking()` is false, and vice versa. - // - // We use link-time generated thunks for the slow path. That thunk - // checks the reference and jumps to the entrypoint if needed. - // - // temp = Thread::Current()->pReadBarrierMarkIntrospection - // lr = &return_address; - // GcRoot<mirror::Object> root = *(obj+offset); // Original reference load. - // if (temp != nullptr) { - // goto gc_root_thunk<root_reg>(lr) - // } - // return_address: - - CheckLastTempIsBakerCcEntrypointRegister(instruction); - bool narrow = CanEmitNarrowLdr(root_reg, obj, offset); - uint32_t custom_data = - linker::Thumb2RelativePatcher::EncodeBakerReadBarrierGcRootData(root_reg, narrow); - Label* bne_label = codegen_->NewBakerReadBarrierPatch(custom_data); - - // entrypoint_reg = - // Thread::Current()->pReadBarrierMarkReg12, i.e. pReadBarrierMarkIntrospection. - DCHECK_EQ(IP, 12); - const int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(IP); - __ LoadFromOffset(kLoadWord, kBakerCcEntrypointRegister, TR, entry_point_offset); - - Label return_address; - __ AdrCode(LR, &return_address); - __ CmpConstant(kBakerCcEntrypointRegister, 0); - // Currently the offset is always within range. If that changes, - // we shall have to split the load the same way as for fields. - DCHECK_LT(offset, kReferenceLoadMinFarOffset); - DCHECK(!down_cast<Thumb2Assembler*>(GetAssembler())->IsForced32Bit()); - ScopedForce32Bit maybe_force_32bit(down_cast<Thumb2Assembler*>(GetAssembler()), !narrow); - int old_position = GetAssembler()->GetBuffer()->GetPosition(); - __ LoadFromOffset(kLoadWord, root_reg, obj, offset); - EmitPlaceholderBne(codegen_, bne_label); - __ Bind(&return_address); - DCHECK_EQ(old_position - GetAssembler()->GetBuffer()->GetPosition(), - narrow ? BAKER_MARK_INTROSPECTION_GC_ROOT_LDR_NARROW_OFFSET - : BAKER_MARK_INTROSPECTION_GC_ROOT_LDR_WIDE_OFFSET); - } else { - // Note that we do not actually check the value of - // `GetIsGcMarking()` to decide whether to mark the loaded GC - // root or not. Instead, we load into `temp` the read barrier - // mark entry point corresponding to register `root`. If `temp` - // is null, it means that `GetIsGcMarking()` is false, and vice - // versa. - // - // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg() - // GcRoot<mirror::Object> root = *(obj+offset); // Original reference load. - // if (temp != nullptr) { // <=> Thread::Current()->GetIsGcMarking() - // // Slow path. - // root = temp(root); // root = ReadBarrier::Mark(root); // Runtime entry point call. - // } - - // Slow path marking the GC root `root`. The entrypoint will already be loaded in `temp`. - Location temp = Location::RegisterLocation(LR); - SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM( - instruction, root, /* entrypoint */ temp); - codegen_->AddSlowPath(slow_path); - - // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg() - const int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(root.reg()); - // Loading the entrypoint does not require a load acquire since it is only changed when - // threads are suspended or running a checkpoint. - __ LoadFromOffset(kLoadWord, temp.AsRegister<Register>(), TR, entry_point_offset); - - // /* GcRoot<mirror::Object> */ root = *(obj + offset) - __ LoadFromOffset(kLoadWord, root_reg, obj, offset); - static_assert( - sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(GcRoot<mirror::Object>), - "art::mirror::CompressedReference<mirror::Object> and art::GcRoot<mirror::Object> " - "have different sizes."); - static_assert(sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(int32_t), - "art::mirror::CompressedReference<mirror::Object> and int32_t " - "have different sizes."); - - // The entrypoint is null when the GC is not marking, this prevents one load compared to - // checking GetIsGcMarking. - __ CompareAndBranchIfNonZero(temp.AsRegister<Register>(), slow_path->GetEntryLabel()); - __ Bind(slow_path->GetExitLabel()); - } - } else { - // GC root loaded through a slow path for read barriers other - // than Baker's. - // /* GcRoot<mirror::Object>* */ root = obj + offset - __ AddConstant(root_reg, obj, offset); - // /* mirror::Object* */ root = root->Read() - codegen_->GenerateReadBarrierForRootSlow(instruction, root, root); - } - } else { - // Plain GC root load with no read barrier. - // /* GcRoot<mirror::Object> */ root = *(obj + offset) - __ LoadFromOffset(kLoadWord, root_reg, obj, offset); - // Note that GC roots are not affected by heap poisoning, thus we - // do not have to unpoison `root_reg` here. - } -} - -void CodeGeneratorARM::MaybeAddBakerCcEntrypointTempForFields(LocationSummary* locations) { - DCHECK(kEmitCompilerReadBarrier); - DCHECK(kUseBakerReadBarrier); - if (kBakerReadBarrierLinkTimeThunksEnableForFields) { - if (!Runtime::Current()->UseJitCompilation()) { - locations->AddTemp(Location::RegisterLocation(kBakerCcEntrypointRegister)); - } - } -} - -void CodeGeneratorARM::GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction, - Location ref, - Register obj, - uint32_t offset, - Location temp, - bool needs_null_check) { - DCHECK(kEmitCompilerReadBarrier); - DCHECK(kUseBakerReadBarrier); - - if (kBakerReadBarrierLinkTimeThunksEnableForFields && - !Runtime::Current()->UseJitCompilation()) { - // Note that we do not actually check the value of `GetIsGcMarking()` - // to decide whether to mark the loaded reference or not. Instead, we - // load into `temp` (actually kBakerCcEntrypointRegister) the read - // barrier mark introspection entrypoint. If `temp` is null, it means - // that `GetIsGcMarking()` is false, and vice versa. - // - // We use link-time generated thunks for the slow path. That thunk checks - // the holder and jumps to the entrypoint if needed. If the holder is not - // gray, it creates a fake dependency and returns to the LDR instruction. - // - // temp = Thread::Current()->pReadBarrierMarkIntrospection - // lr = &gray_return_address; - // if (temp != nullptr) { - // goto field_thunk<holder_reg, base_reg>(lr) - // } - // not_gray_return_address: - // // Original reference load. If the offset is too large to fit - // // into LDR, we use an adjusted base register here. - // HeapReference<mirror::Object> reference = *(obj+offset); - // gray_return_address: - - DCHECK_ALIGNED(offset, sizeof(mirror::HeapReference<mirror::Object>)); - Register ref_reg = ref.AsRegister<Register>(); - bool narrow = CanEmitNarrowLdr(ref_reg, obj, offset); - Register base = obj; - if (offset >= kReferenceLoadMinFarOffset) { - base = temp.AsRegister<Register>(); - DCHECK_NE(base, kBakerCcEntrypointRegister); - static_assert(IsPowerOfTwo(kReferenceLoadMinFarOffset), "Expecting a power of 2."); - __ AddConstant(base, obj, offset & ~(kReferenceLoadMinFarOffset - 1u)); - offset &= (kReferenceLoadMinFarOffset - 1u); - // Use narrow LDR only for small offsets. Generating narrow encoding LDR for the large - // offsets with `(offset & (kReferenceLoadMinFarOffset - 1u)) < 32u` would most likely - // increase the overall code size when taking the generated thunks into account. - DCHECK(!narrow); - } - CheckLastTempIsBakerCcEntrypointRegister(instruction); - uint32_t custom_data = - linker::Thumb2RelativePatcher::EncodeBakerReadBarrierFieldData(base, obj, narrow); - Label* bne_label = NewBakerReadBarrierPatch(custom_data); - - // entrypoint_reg = - // Thread::Current()->pReadBarrierMarkReg12, i.e. pReadBarrierMarkIntrospection. - DCHECK_EQ(IP, 12); - const int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(IP); - __ LoadFromOffset(kLoadWord, kBakerCcEntrypointRegister, TR, entry_point_offset); - - Label return_address; - __ AdrCode(LR, &return_address); - __ CmpConstant(kBakerCcEntrypointRegister, 0); - EmitPlaceholderBne(this, bne_label); - DCHECK_LT(offset, kReferenceLoadMinFarOffset); - DCHECK(!down_cast<Thumb2Assembler*>(GetAssembler())->IsForced32Bit()); - ScopedForce32Bit maybe_force_32bit(down_cast<Thumb2Assembler*>(GetAssembler()), !narrow); - int old_position = GetAssembler()->GetBuffer()->GetPosition(); - __ LoadFromOffset(kLoadWord, ref_reg, base, offset); - if (needs_null_check) { - MaybeRecordImplicitNullCheck(instruction); - } - GetAssembler()->MaybeUnpoisonHeapReference(ref_reg); - __ Bind(&return_address); - DCHECK_EQ(old_position - GetAssembler()->GetBuffer()->GetPosition(), - narrow ? BAKER_MARK_INTROSPECTION_FIELD_LDR_NARROW_OFFSET - : BAKER_MARK_INTROSPECTION_FIELD_LDR_WIDE_OFFSET); - return; - } - - // /* HeapReference<Object> */ ref = *(obj + offset) - Location no_index = Location::NoLocation(); - ScaleFactor no_scale_factor = TIMES_1; - GenerateReferenceLoadWithBakerReadBarrier( - instruction, ref, obj, offset, no_index, no_scale_factor, temp, needs_null_check); -} - -void CodeGeneratorARM::GenerateArrayLoadWithBakerReadBarrier(HInstruction* instruction, - Location ref, - Register obj, - uint32_t data_offset, - Location index, - Location temp, - bool needs_null_check) { - DCHECK(kEmitCompilerReadBarrier); - DCHECK(kUseBakerReadBarrier); - - static_assert( - sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), - "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); - ScaleFactor scale_factor = TIMES_4; - - if (kBakerReadBarrierLinkTimeThunksEnableForArrays && - !Runtime::Current()->UseJitCompilation()) { - // Note that we do not actually check the value of `GetIsGcMarking()` - // to decide whether to mark the loaded reference or not. Instead, we - // load into `temp` (actually kBakerCcEntrypointRegister) the read - // barrier mark introspection entrypoint. If `temp` is null, it means - // that `GetIsGcMarking()` is false, and vice versa. - // - // We use link-time generated thunks for the slow path. That thunk checks - // the holder and jumps to the entrypoint if needed. If the holder is not - // gray, it creates a fake dependency and returns to the LDR instruction. - // - // temp = Thread::Current()->pReadBarrierMarkIntrospection - // lr = &gray_return_address; - // if (temp != nullptr) { - // goto field_thunk<holder_reg, base_reg>(lr) - // } - // not_gray_return_address: - // // Original reference load. If the offset is too large to fit - // // into LDR, we use an adjusted base register here. - // HeapReference<mirror::Object> reference = data[index]; - // gray_return_address: - - DCHECK(index.IsValid()); - Register index_reg = index.AsRegister<Register>(); - Register ref_reg = ref.AsRegister<Register>(); - Register data_reg = temp.AsRegister<Register>(); - DCHECK_NE(data_reg, kBakerCcEntrypointRegister); - - CheckLastTempIsBakerCcEntrypointRegister(instruction); - uint32_t custom_data = - linker::Thumb2RelativePatcher::EncodeBakerReadBarrierArrayData(data_reg); - Label* bne_label = NewBakerReadBarrierPatch(custom_data); - - // entrypoint_reg = - // Thread::Current()->pReadBarrierMarkReg16, i.e. pReadBarrierMarkIntrospection. - DCHECK_EQ(IP, 12); - const int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(IP); - __ LoadFromOffset(kLoadWord, kBakerCcEntrypointRegister, TR, entry_point_offset); - __ AddConstant(data_reg, obj, data_offset); - - Label return_address; - __ AdrCode(LR, &return_address); - __ CmpConstant(kBakerCcEntrypointRegister, 0); - EmitPlaceholderBne(this, bne_label); - ScopedForce32Bit maybe_force_32bit(down_cast<Thumb2Assembler*>(GetAssembler())); - int old_position = GetAssembler()->GetBuffer()->GetPosition(); - __ ldr(ref_reg, Address(data_reg, index_reg, LSL, scale_factor)); - DCHECK(!needs_null_check); // The thunk cannot handle the null check. - GetAssembler()->MaybeUnpoisonHeapReference(ref_reg); - __ Bind(&return_address); - DCHECK_EQ(old_position - GetAssembler()->GetBuffer()->GetPosition(), - BAKER_MARK_INTROSPECTION_ARRAY_LDR_OFFSET); - return; - } - - // /* HeapReference<Object> */ ref = - // *(obj + data_offset + index * sizeof(HeapReference<Object>)) - GenerateReferenceLoadWithBakerReadBarrier( - instruction, ref, obj, data_offset, index, scale_factor, temp, needs_null_check); -} - -void CodeGeneratorARM::GenerateReferenceLoadWithBakerReadBarrier(HInstruction* instruction, - Location ref, - Register obj, - uint32_t offset, - Location index, - ScaleFactor scale_factor, - Location temp, - bool needs_null_check) { - DCHECK(kEmitCompilerReadBarrier); - DCHECK(kUseBakerReadBarrier); - - // Query `art::Thread::Current()->GetIsGcMarking()` to decide - // whether we need to enter the slow path to mark the reference. - // Then, in the slow path, check the gray bit in the lock word of - // the reference's holder (`obj`) to decide whether to mark `ref` or - // not. - // - // Note that we do not actually check the value of `GetIsGcMarking()`; - // instead, we load into `temp2` the read barrier mark entry point - // corresponding to register `ref`. If `temp2` is null, it means - // that `GetIsGcMarking()` is false, and vice versa. - // - // temp2 = Thread::Current()->pReadBarrierMarkReg ## root.reg() - // if (temp2 != nullptr) { // <=> Thread::Current()->GetIsGcMarking() - // // Slow path. - // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState(); - // lfence; // Load fence or artificial data dependency to prevent load-load reordering - // HeapReference<mirror::Object> ref = *src; // Original reference load. - // bool is_gray = (rb_state == ReadBarrier::GrayState()); - // if (is_gray) { - // ref = temp2(ref); // ref = ReadBarrier::Mark(ref); // Runtime entry point call. - // } - // } else { - // HeapReference<mirror::Object> ref = *src; // Original reference load. - // } - - Register temp_reg = temp.AsRegister<Register>(); - - // Slow path marking the object `ref` when the GC is marking. The - // entrypoint will already be loaded in `temp2`. - Location temp2 = Location::RegisterLocation(LR); - SlowPathCodeARM* slow_path = - new (GetGraph()->GetArena()) LoadReferenceWithBakerReadBarrierSlowPathARM( - instruction, - ref, - obj, - offset, - index, - scale_factor, - needs_null_check, - temp_reg, - /* entrypoint */ temp2); - AddSlowPath(slow_path); - - // temp2 = Thread::Current()->pReadBarrierMarkReg ## ref.reg() - const int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref.reg()); - // Loading the entrypoint does not require a load acquire since it is only changed when - // threads are suspended or running a checkpoint. - __ LoadFromOffset(kLoadWord, temp2.AsRegister<Register>(), TR, entry_point_offset); - // The entrypoint is null when the GC is not marking, this prevents one load compared to - // checking GetIsGcMarking. - __ CompareAndBranchIfNonZero(temp2.AsRegister<Register>(), slow_path->GetEntryLabel()); - // Fast path: the GC is not marking: just load the reference. - GenerateRawReferenceLoad(instruction, ref, obj, offset, index, scale_factor, needs_null_check); - __ Bind(slow_path->GetExitLabel()); -} - -void CodeGeneratorARM::UpdateReferenceFieldWithBakerReadBarrier(HInstruction* instruction, - Location ref, - Register obj, - Location field_offset, - Location temp, - bool needs_null_check, - Register temp2) { - DCHECK(kEmitCompilerReadBarrier); - DCHECK(kUseBakerReadBarrier); - - // Query `art::Thread::Current()->GetIsGcMarking()` to decide - // whether we need to enter the slow path to update the reference - // field within `obj`. Then, in the slow path, check the gray bit - // in the lock word of the reference's holder (`obj`) to decide - // whether to mark `ref` and update the field or not. - // - // Note that we do not actually check the value of `GetIsGcMarking()`; - // instead, we load into `temp3` the read barrier mark entry point - // corresponding to register `ref`. If `temp3` is null, it means - // that `GetIsGcMarking()` is false, and vice versa. - // - // temp3 = Thread::Current()->pReadBarrierMarkReg ## root.reg() - // if (temp3 != nullptr) { // <=> Thread::Current()->GetIsGcMarking() - // // Slow path. - // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState(); - // lfence; // Load fence or artificial data dependency to prevent load-load reordering - // HeapReference<mirror::Object> ref = *src; // Original reference load. - // bool is_gray = (rb_state == ReadBarrier::GrayState()); - // if (is_gray) { - // old_ref = ref; - // ref = temp3(ref); // ref = ReadBarrier::Mark(ref); // Runtime entry point call. - // compareAndSwapObject(obj, field_offset, old_ref, ref); - // } - // } - - Register temp_reg = temp.AsRegister<Register>(); - - // Slow path updating the object reference at address `obj + - // field_offset` when the GC is marking. The entrypoint will already - // be loaded in `temp3`. - Location temp3 = Location::RegisterLocation(LR); - SlowPathCodeARM* slow_path = - new (GetGraph()->GetArena()) LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM( - instruction, - ref, - obj, - /* offset */ 0u, - /* index */ field_offset, - /* scale_factor */ ScaleFactor::TIMES_1, - needs_null_check, - temp_reg, - temp2, - /* entrypoint */ temp3); - AddSlowPath(slow_path); - - // temp3 = Thread::Current()->pReadBarrierMarkReg ## ref.reg() - const int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref.reg()); - // Loading the entrypoint does not require a load acquire since it is only changed when - // threads are suspended or running a checkpoint. - __ LoadFromOffset(kLoadWord, temp3.AsRegister<Register>(), TR, entry_point_offset); - // The entrypoint is null when the GC is not marking, this prevents one load compared to - // checking GetIsGcMarking. - __ CompareAndBranchIfNonZero(temp3.AsRegister<Register>(), slow_path->GetEntryLabel()); - // Fast path: the GC is not marking: nothing to do (the field is - // up-to-date, and we don't need to load the reference). - __ Bind(slow_path->GetExitLabel()); -} - -void CodeGeneratorARM::GenerateRawReferenceLoad(HInstruction* instruction, - Location ref, - Register obj, - uint32_t offset, - Location index, - ScaleFactor scale_factor, - bool needs_null_check) { - Register ref_reg = ref.AsRegister<Register>(); - - if (index.IsValid()) { - // Load types involving an "index": ArrayGet, - // UnsafeGetObject/UnsafeGetObjectVolatile and UnsafeCASObject - // intrinsics. - // /* HeapReference<mirror::Object> */ ref = *(obj + offset + (index << scale_factor)) - if (index.IsConstant()) { - size_t computed_offset = - (index.GetConstant()->AsIntConstant()->GetValue() << scale_factor) + offset; - __ LoadFromOffset(kLoadWord, ref_reg, obj, computed_offset); - } else { - // Handle the special case of the - // UnsafeGetObject/UnsafeGetObjectVolatile and UnsafeCASObject - // intrinsics, which use a register pair as index ("long - // offset"), of which only the low part contains data. - Register index_reg = index.IsRegisterPair() - ? index.AsRegisterPairLow<Register>() - : index.AsRegister<Register>(); - __ add(IP, obj, ShifterOperand(index_reg, LSL, scale_factor)); - __ LoadFromOffset(kLoadWord, ref_reg, IP, offset); - } - } else { - // /* HeapReference<mirror::Object> */ ref = *(obj + offset) - __ LoadFromOffset(kLoadWord, ref_reg, obj, offset); - } - - if (needs_null_check) { - MaybeRecordImplicitNullCheck(instruction); - } - - // Object* ref = ref_addr->AsMirrorPtr() - __ MaybeUnpoisonHeapReference(ref_reg); -} - -void CodeGeneratorARM::GenerateReadBarrierSlow(HInstruction* instruction, - Location out, - Location ref, - Location obj, - uint32_t offset, - Location index) { - DCHECK(kEmitCompilerReadBarrier); - - // Insert a slow path based read barrier *after* the reference load. - // - // If heap poisoning is enabled, the unpoisoning of the loaded - // reference will be carried out by the runtime within the slow - // path. - // - // Note that `ref` currently does not get unpoisoned (when heap - // poisoning is enabled), which is alright as the `ref` argument is - // not used by the artReadBarrierSlow entry point. - // - // TODO: Unpoison `ref` when it is used by artReadBarrierSlow. - SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) - ReadBarrierForHeapReferenceSlowPathARM(instruction, out, ref, obj, offset, index); - AddSlowPath(slow_path); - - __ b(slow_path->GetEntryLabel()); - __ Bind(slow_path->GetExitLabel()); -} - -void CodeGeneratorARM::MaybeGenerateReadBarrierSlow(HInstruction* instruction, - Location out, - Location ref, - Location obj, - uint32_t offset, - Location index) { - if (kEmitCompilerReadBarrier) { - // Baker's read barriers shall be handled by the fast path - // (CodeGeneratorARM::GenerateReferenceLoadWithBakerReadBarrier). - DCHECK(!kUseBakerReadBarrier); - // If heap poisoning is enabled, unpoisoning will be taken care of - // by the runtime within the slow path. - GenerateReadBarrierSlow(instruction, out, ref, obj, offset, index); - } else if (kPoisonHeapReferences) { - __ UnpoisonHeapReference(out.AsRegister<Register>()); - } -} - -void CodeGeneratorARM::GenerateReadBarrierForRootSlow(HInstruction* instruction, - Location out, - Location root) { - DCHECK(kEmitCompilerReadBarrier); - - // Insert a slow path based read barrier *after* the GC root load. - // - // Note that GC roots are not affected by heap poisoning, so we do - // not need to do anything special for this here. - SlowPathCodeARM* slow_path = - new (GetGraph()->GetArena()) ReadBarrierForRootSlowPathARM(instruction, out, root); - AddSlowPath(slow_path); - - __ b(slow_path->GetEntryLabel()); - __ Bind(slow_path->GetExitLabel()); -} - -HInvokeStaticOrDirect::DispatchInfo CodeGeneratorARM::GetSupportedInvokeStaticOrDirectDispatch( - const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info, - HInvokeStaticOrDirect* invoke ATTRIBUTE_UNUSED) { - return desired_dispatch_info; -} - -Register CodeGeneratorARM::GetInvokeStaticOrDirectExtraParameter(HInvokeStaticOrDirect* invoke, - Register temp) { - DCHECK_EQ(invoke->InputCount(), invoke->GetNumberOfArguments() + 1u); - Location location = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); - if (!invoke->GetLocations()->Intrinsified()) { - return location.AsRegister<Register>(); - } - // For intrinsics we allow any location, so it may be on the stack. - if (!location.IsRegister()) { - __ LoadFromOffset(kLoadWord, temp, SP, location.GetStackIndex()); - return temp; - } - // For register locations, check if the register was saved. If so, get it from the stack. - // Note: There is a chance that the register was saved but not overwritten, so we could - // save one load. However, since this is just an intrinsic slow path we prefer this - // simple and more robust approach rather that trying to determine if that's the case. - SlowPathCode* slow_path = GetCurrentSlowPath(); - DCHECK(slow_path != nullptr); // For intrinsified invokes the call is emitted on the slow path. - if (slow_path->IsCoreRegisterSaved(location.AsRegister<Register>())) { - int stack_offset = slow_path->GetStackOffsetOfCoreRegister(location.AsRegister<Register>()); - __ LoadFromOffset(kLoadWord, temp, SP, stack_offset); - return temp; - } - return location.AsRegister<Register>(); -} - -void CodeGeneratorARM::GenerateStaticOrDirectCall( - HInvokeStaticOrDirect* invoke, Location temp, SlowPathCode* slow_path) { - Location callee_method = temp; // For all kinds except kRecursive, callee will be in temp. - switch (invoke->GetMethodLoadKind()) { - case HInvokeStaticOrDirect::MethodLoadKind::kStringInit: { - uint32_t offset = - GetThreadOffset<kArmPointerSize>(invoke->GetStringInitEntryPoint()).Int32Value(); - // temp = thread->string_init_entrypoint - __ LoadFromOffset(kLoadWord, temp.AsRegister<Register>(), TR, offset); - break; - } - case HInvokeStaticOrDirect::MethodLoadKind::kRecursive: - callee_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); - break; - case HInvokeStaticOrDirect::MethodLoadKind::kBootImageLinkTimePcRelative: { - DCHECK(GetCompilerOptions().IsBootImage()); - Register temp_reg = temp.AsRegister<Register>(); - PcRelativePatchInfo* labels = NewPcRelativeMethodPatch(invoke->GetTargetMethod()); - __ BindTrackedLabel(&labels->movw_label); - __ movw(temp_reg, /* placeholder */ 0u); - __ BindTrackedLabel(&labels->movt_label); - __ movt(temp_reg, /* placeholder */ 0u); - __ BindTrackedLabel(&labels->add_pc_label); - __ add(temp_reg, temp_reg, ShifterOperand(PC)); - break; - } - case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress: - __ LoadImmediate(temp.AsRegister<Register>(), invoke->GetMethodAddress()); - break; - case HInvokeStaticOrDirect::MethodLoadKind::kBssEntry: { - Register temp_reg = temp.AsRegister<Register>(); - PcRelativePatchInfo* labels = NewMethodBssEntryPatch( - MethodReference(&GetGraph()->GetDexFile(), invoke->GetDexMethodIndex())); - __ BindTrackedLabel(&labels->movw_label); - __ movw(temp_reg, /* placeholder */ 0u); - __ BindTrackedLabel(&labels->movt_label); - __ movt(temp_reg, /* placeholder */ 0u); - __ BindTrackedLabel(&labels->add_pc_label); - __ add(temp_reg, temp_reg, ShifterOperand(PC)); - __ LoadFromOffset(kLoadWord, temp_reg, temp_reg, /* offset */ 0); - break; - } - case HInvokeStaticOrDirect::MethodLoadKind::kRuntimeCall: { - GenerateInvokeStaticOrDirectRuntimeCall(invoke, temp, slow_path); - return; // No code pointer retrieval; the runtime performs the call directly. - } - } - - switch (invoke->GetCodePtrLocation()) { - case HInvokeStaticOrDirect::CodePtrLocation::kCallSelf: - __ bl(GetFrameEntryLabel()); - break; - case HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod: - // LR = callee_method->entry_point_from_quick_compiled_code_ - __ LoadFromOffset( - kLoadWord, LR, callee_method.AsRegister<Register>(), - ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmPointerSize).Int32Value()); - // LR() - __ blx(LR); - break; - } - RecordPcInfo(invoke, invoke->GetDexPc(), slow_path); - - DCHECK(!IsLeafMethod()); -} - -void CodeGeneratorARM::GenerateVirtualCall( - HInvokeVirtual* invoke, Location temp_location, SlowPathCode* slow_path) { - Register temp = temp_location.AsRegister<Register>(); - uint32_t method_offset = mirror::Class::EmbeddedVTableEntryOffset( - invoke->GetVTableIndex(), kArmPointerSize).Uint32Value(); - - // Use the calling convention instead of the location of the receiver, as - // intrinsics may have put the receiver in a different register. In the intrinsics - // slow path, the arguments have been moved to the right place, so here we are - // guaranteed that the receiver is the first register of the calling convention. - InvokeDexCallingConvention calling_convention; - Register receiver = calling_convention.GetRegisterAt(0); - uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); - // /* HeapReference<Class> */ temp = receiver->klass_ - __ LoadFromOffset(kLoadWord, temp, receiver, class_offset); - MaybeRecordImplicitNullCheck(invoke); - // Instead of simply (possibly) unpoisoning `temp` here, we should - // emit a read barrier for the previous class reference load. - // However this is not required in practice, as this is an - // intermediate/temporary reference and because the current - // concurrent copying collector keeps the from-space memory - // intact/accessible until the end of the marking phase (the - // concurrent copying collector may not in the future). - __ MaybeUnpoisonHeapReference(temp); - // temp = temp->GetMethodAt(method_offset); - uint32_t entry_point = ArtMethod::EntryPointFromQuickCompiledCodeOffset( - kArmPointerSize).Int32Value(); - __ LoadFromOffset(kLoadWord, temp, temp, method_offset); - // LR = temp->GetEntryPoint(); - __ LoadFromOffset(kLoadWord, LR, temp, entry_point); - // LR(); - __ blx(LR); - RecordPcInfo(invoke, invoke->GetDexPc(), slow_path); -} - -CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativeMethodPatch( - MethodReference target_method) { - return NewPcRelativePatch(*target_method.dex_file, - target_method.dex_method_index, - &pc_relative_method_patches_); -} - -CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewMethodBssEntryPatch( - MethodReference target_method) { - return NewPcRelativePatch(*target_method.dex_file, - target_method.dex_method_index, - &method_bss_entry_patches_); -} - -CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativeTypePatch( - const DexFile& dex_file, dex::TypeIndex type_index) { - return NewPcRelativePatch(dex_file, type_index.index_, &pc_relative_type_patches_); -} - -CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewTypeBssEntryPatch( - const DexFile& dex_file, dex::TypeIndex type_index) { - return NewPcRelativePatch(dex_file, type_index.index_, &type_bss_entry_patches_); -} - -CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativeStringPatch( - const DexFile& dex_file, dex::StringIndex string_index) { - return NewPcRelativePatch(dex_file, string_index.index_, &pc_relative_string_patches_); -} - -CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativePatch( - const DexFile& dex_file, uint32_t offset_or_index, ArenaDeque<PcRelativePatchInfo>* patches) { - patches->emplace_back(dex_file, offset_or_index); - return &patches->back(); -} - -Label* CodeGeneratorARM::NewBakerReadBarrierPatch(uint32_t custom_data) { - baker_read_barrier_patches_.emplace_back(custom_data); - return &baker_read_barrier_patches_.back().label; -} - -Literal* CodeGeneratorARM::DeduplicateBootImageAddressLiteral(uint32_t address) { - return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address), &uint32_literals_); -} - -Literal* CodeGeneratorARM::DeduplicateJitStringLiteral(const DexFile& dex_file, - dex::StringIndex string_index, - Handle<mirror::String> handle) { - jit_string_roots_.Overwrite(StringReference(&dex_file, string_index), - reinterpret_cast64<uint64_t>(handle.GetReference())); - return jit_string_patches_.GetOrCreate( - StringReference(&dex_file, string_index), - [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); -} - -Literal* CodeGeneratorARM::DeduplicateJitClassLiteral(const DexFile& dex_file, - dex::TypeIndex type_index, - Handle<mirror::Class> handle) { - jit_class_roots_.Overwrite(TypeReference(&dex_file, type_index), - reinterpret_cast64<uint64_t>(handle.GetReference())); - return jit_class_patches_.GetOrCreate( - TypeReference(&dex_file, type_index), - [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); -} - -template <LinkerPatch (*Factory)(size_t, const DexFile*, uint32_t, uint32_t)> -inline void CodeGeneratorARM::EmitPcRelativeLinkerPatches( - const ArenaDeque<PcRelativePatchInfo>& infos, - ArenaVector<LinkerPatch>* linker_patches) { - for (const PcRelativePatchInfo& info : infos) { - const DexFile& dex_file = info.target_dex_file; - size_t offset_or_index = info.offset_or_index; - DCHECK(info.add_pc_label.IsBound()); - uint32_t add_pc_offset = dchecked_integral_cast<uint32_t>(info.add_pc_label.Position()); - // Add MOVW patch. - DCHECK(info.movw_label.IsBound()); - uint32_t movw_offset = dchecked_integral_cast<uint32_t>(info.movw_label.Position()); - linker_patches->push_back(Factory(movw_offset, &dex_file, add_pc_offset, offset_or_index)); - // Add MOVT patch. - DCHECK(info.movt_label.IsBound()); - uint32_t movt_offset = dchecked_integral_cast<uint32_t>(info.movt_label.Position()); - linker_patches->push_back(Factory(movt_offset, &dex_file, add_pc_offset, offset_or_index)); - } -} - -void CodeGeneratorARM::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) { - DCHECK(linker_patches->empty()); - size_t size = - /* MOVW+MOVT for each entry */ 2u * pc_relative_method_patches_.size() + - /* MOVW+MOVT for each entry */ 2u * method_bss_entry_patches_.size() + - /* MOVW+MOVT for each entry */ 2u * pc_relative_type_patches_.size() + - /* MOVW+MOVT for each entry */ 2u * type_bss_entry_patches_.size() + - /* MOVW+MOVT for each entry */ 2u * pc_relative_string_patches_.size() + - baker_read_barrier_patches_.size(); - linker_patches->reserve(size); - if (GetCompilerOptions().IsBootImage()) { - EmitPcRelativeLinkerPatches<LinkerPatch::RelativeMethodPatch>(pc_relative_method_patches_, - linker_patches); - EmitPcRelativeLinkerPatches<LinkerPatch::RelativeTypePatch>(pc_relative_type_patches_, - linker_patches); - EmitPcRelativeLinkerPatches<LinkerPatch::RelativeStringPatch>(pc_relative_string_patches_, - linker_patches); - } else { - DCHECK(pc_relative_method_patches_.empty()); - DCHECK(pc_relative_type_patches_.empty()); - EmitPcRelativeLinkerPatches<LinkerPatch::StringBssEntryPatch>(pc_relative_string_patches_, - linker_patches); - } - EmitPcRelativeLinkerPatches<LinkerPatch::MethodBssEntryPatch>(method_bss_entry_patches_, - linker_patches); - EmitPcRelativeLinkerPatches<LinkerPatch::TypeBssEntryPatch>(type_bss_entry_patches_, - linker_patches); - for (const BakerReadBarrierPatchInfo& info : baker_read_barrier_patches_) { - linker_patches->push_back(LinkerPatch::BakerReadBarrierBranchPatch(info.label.Position(), - info.custom_data)); - } - DCHECK_EQ(size, linker_patches->size()); -} - -Literal* CodeGeneratorARM::DeduplicateUint32Literal(uint32_t value, Uint32ToLiteralMap* map) { - return map->GetOrCreate( - value, - [this, value]() { return __ NewLiteral<uint32_t>(value); }); -} - -void LocationsBuilderARM::VisitMultiplyAccumulate(HMultiplyAccumulate* instr) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(instr, LocationSummary::kNoCall); - locations->SetInAt(HMultiplyAccumulate::kInputAccumulatorIndex, - Location::RequiresRegister()); - locations->SetInAt(HMultiplyAccumulate::kInputMulLeftIndex, Location::RequiresRegister()); - locations->SetInAt(HMultiplyAccumulate::kInputMulRightIndex, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); -} - -void InstructionCodeGeneratorARM::VisitMultiplyAccumulate(HMultiplyAccumulate* instr) { - LocationSummary* locations = instr->GetLocations(); - Register res = locations->Out().AsRegister<Register>(); - Register accumulator = - locations->InAt(HMultiplyAccumulate::kInputAccumulatorIndex).AsRegister<Register>(); - Register mul_left = - locations->InAt(HMultiplyAccumulate::kInputMulLeftIndex).AsRegister<Register>(); - Register mul_right = - locations->InAt(HMultiplyAccumulate::kInputMulRightIndex).AsRegister<Register>(); - - if (instr->GetOpKind() == HInstruction::kAdd) { - __ mla(res, mul_left, mul_right, accumulator); - } else { - __ mls(res, mul_left, mul_right, accumulator); - } -} - -void LocationsBuilderARM::VisitBoundType(HBoundType* instruction ATTRIBUTE_UNUSED) { - // Nothing to do, this should be removed during prepare for register allocator. - LOG(FATAL) << "Unreachable"; -} - -void InstructionCodeGeneratorARM::VisitBoundType(HBoundType* instruction ATTRIBUTE_UNUSED) { - // Nothing to do, this should be removed during prepare for register allocator. - LOG(FATAL) << "Unreachable"; -} - -// Simple implementation of packed switch - generate cascaded compare/jumps. -void LocationsBuilderARM::VisitPackedSwitch(HPackedSwitch* switch_instr) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(switch_instr, LocationSummary::kNoCall); - locations->SetInAt(0, Location::RequiresRegister()); - if (switch_instr->GetNumEntries() > kPackedSwitchCompareJumpThreshold && - codegen_->GetAssembler()->IsThumb()) { - locations->AddTemp(Location::RequiresRegister()); // We need a temp for the table base. - if (switch_instr->GetStartValue() != 0) { - locations->AddTemp(Location::RequiresRegister()); // We need a temp for the bias. - } - } -} - -void InstructionCodeGeneratorARM::VisitPackedSwitch(HPackedSwitch* switch_instr) { - int32_t lower_bound = switch_instr->GetStartValue(); - uint32_t num_entries = switch_instr->GetNumEntries(); - LocationSummary* locations = switch_instr->GetLocations(); - Register value_reg = locations->InAt(0).AsRegister<Register>(); - HBasicBlock* default_block = switch_instr->GetDefaultBlock(); - - if (num_entries <= kPackedSwitchCompareJumpThreshold || !codegen_->GetAssembler()->IsThumb()) { - // Create a series of compare/jumps. - Register temp_reg = IP; - // Note: It is fine for the below AddConstantSetFlags() using IP register to temporarily store - // the immediate, because IP is used as the destination register. For the other - // AddConstantSetFlags() and GenerateCompareWithImmediate(), the immediate values are constant, - // and they can be encoded in the instruction without making use of IP register. - __ AddConstantSetFlags(temp_reg, value_reg, -lower_bound); - - const ArenaVector<HBasicBlock*>& successors = switch_instr->GetBlock()->GetSuccessors(); - // Jump to successors[0] if value == lower_bound. - __ b(codegen_->GetLabelOf(successors[0]), EQ); - int32_t last_index = 0; - for (; num_entries - last_index > 2; last_index += 2) { - __ AddConstantSetFlags(temp_reg, temp_reg, -2); - // Jump to successors[last_index + 1] if value < case_value[last_index + 2]. - __ b(codegen_->GetLabelOf(successors[last_index + 1]), LO); - // Jump to successors[last_index + 2] if value == case_value[last_index + 2]. - __ b(codegen_->GetLabelOf(successors[last_index + 2]), EQ); - } - if (num_entries - last_index == 2) { - // The last missing case_value. - __ CmpConstant(temp_reg, 1); - __ b(codegen_->GetLabelOf(successors[last_index + 1]), EQ); - } - - // And the default for any other value. - if (!codegen_->GoesToNextBlock(switch_instr->GetBlock(), default_block)) { - __ b(codegen_->GetLabelOf(default_block)); - } - } else { - // Create a table lookup. - Register temp_reg = locations->GetTemp(0).AsRegister<Register>(); - - // Materialize a pointer to the switch table - std::vector<Label*> labels(num_entries); - const ArenaVector<HBasicBlock*>& successors = switch_instr->GetBlock()->GetSuccessors(); - for (uint32_t i = 0; i < num_entries; i++) { - labels[i] = codegen_->GetLabelOf(successors[i]); - } - JumpTable* table = __ CreateJumpTable(std::move(labels), temp_reg); - - // Remove the bias. - Register key_reg; - if (lower_bound != 0) { - key_reg = locations->GetTemp(1).AsRegister<Register>(); - __ AddConstant(key_reg, value_reg, -lower_bound); - } else { - key_reg = value_reg; - } - - // Check whether the value is in the table, jump to default block if not. - __ CmpConstant(key_reg, num_entries - 1); - __ b(codegen_->GetLabelOf(default_block), Condition::HI); - - // Load the displacement from the table. - __ ldr(temp_reg, Address(temp_reg, key_reg, Shift::LSL, 2)); - - // Dispatch is a direct add to the PC (for Thumb2). - __ EmitJumpTableDispatch(table, temp_reg); - } -} - -void CodeGeneratorARM::MoveFromReturnRegister(Location trg, Primitive::Type type) { - if (!trg.IsValid()) { - DCHECK_EQ(type, Primitive::kPrimVoid); - return; - } - - DCHECK_NE(type, Primitive::kPrimVoid); - - Location return_loc = InvokeDexCallingConventionVisitorARM().GetReturnLocation(type); - if (return_loc.Equals(trg)) { - return; - } - - // TODO: Consider pairs in the parallel move resolver, then this could be nicely merged - // with the last branch. - if (type == Primitive::kPrimLong) { - HParallelMove parallel_move(GetGraph()->GetArena()); - parallel_move.AddMove(return_loc.ToLow(), trg.ToLow(), Primitive::kPrimInt, nullptr); - parallel_move.AddMove(return_loc.ToHigh(), trg.ToHigh(), Primitive::kPrimInt, nullptr); - GetMoveResolver()->EmitNativeCode(¶llel_move); - } else if (type == Primitive::kPrimDouble) { - HParallelMove parallel_move(GetGraph()->GetArena()); - parallel_move.AddMove(return_loc.ToLow(), trg.ToLow(), Primitive::kPrimFloat, nullptr); - parallel_move.AddMove(return_loc.ToHigh(), trg.ToHigh(), Primitive::kPrimFloat, nullptr); - GetMoveResolver()->EmitNativeCode(¶llel_move); - } else { - // Let the parallel move resolver take care of all of this. - HParallelMove parallel_move(GetGraph()->GetArena()); - parallel_move.AddMove(return_loc, trg, type, nullptr); - GetMoveResolver()->EmitNativeCode(¶llel_move); - } -} - -void LocationsBuilderARM::VisitClassTableGet(HClassTableGet* instruction) { - LocationSummary* locations = - new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister()); -} - -void InstructionCodeGeneratorARM::VisitClassTableGet(HClassTableGet* instruction) { - LocationSummary* locations = instruction->GetLocations(); - if (instruction->GetTableKind() == HClassTableGet::TableKind::kVTable) { - uint32_t method_offset = mirror::Class::EmbeddedVTableEntryOffset( - instruction->GetIndex(), kArmPointerSize).SizeValue(); - __ LoadFromOffset(kLoadWord, - locations->Out().AsRegister<Register>(), - locations->InAt(0).AsRegister<Register>(), - method_offset); - } else { - uint32_t method_offset = static_cast<uint32_t>(ImTable::OffsetOfElement( - instruction->GetIndex(), kArmPointerSize)); - __ LoadFromOffset(kLoadWord, - locations->Out().AsRegister<Register>(), - locations->InAt(0).AsRegister<Register>(), - mirror::Class::ImtPtrOffset(kArmPointerSize).Uint32Value()); - __ LoadFromOffset(kLoadWord, - locations->Out().AsRegister<Register>(), - locations->Out().AsRegister<Register>(), - method_offset); - } -} - -static void PatchJitRootUse(uint8_t* code, - const uint8_t* roots_data, - Literal* literal, - uint64_t index_in_table) { - DCHECK(literal->GetLabel()->IsBound()); - uint32_t literal_offset = literal->GetLabel()->Position(); - uintptr_t address = - reinterpret_cast<uintptr_t>(roots_data) + index_in_table * sizeof(GcRoot<mirror::Object>); - uint8_t* data = code + literal_offset; - reinterpret_cast<uint32_t*>(data)[0] = dchecked_integral_cast<uint32_t>(address); -} - -void CodeGeneratorARM::EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) { - for (const auto& entry : jit_string_patches_) { - const StringReference& string_reference = entry.first; - Literal* table_entry_literal = entry.second; - const auto it = jit_string_roots_.find(string_reference); - DCHECK(it != jit_string_roots_.end()); - uint64_t index_in_table = it->second; - PatchJitRootUse(code, roots_data, table_entry_literal, index_in_table); - } - for (const auto& entry : jit_class_patches_) { - const TypeReference& type_reference = entry.first; - Literal* table_entry_literal = entry.second; - const auto it = jit_class_roots_.find(type_reference); - DCHECK(it != jit_class_roots_.end()); - uint64_t index_in_table = it->second; - PatchJitRootUse(code, roots_data, table_entry_literal, index_in_table); - } -} - -#undef __ -#undef QUICK_ENTRY_POINT - -} // namespace arm -} // namespace art diff --git a/compiler/optimizing/code_generator_arm.h b/compiler/optimizing/code_generator_arm.h deleted file mode 100644 index 9280e6377c..0000000000 --- a/compiler/optimizing/code_generator_arm.h +++ /dev/null @@ -1,695 +0,0 @@ -/* - * Copyright (C) 2014 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef ART_COMPILER_OPTIMIZING_CODE_GENERATOR_ARM_H_ -#define ART_COMPILER_OPTIMIZING_CODE_GENERATOR_ARM_H_ - -#include "base/enums.h" -#include "code_generator.h" -#include "dex_file_types.h" -#include "driver/compiler_options.h" -#include "nodes.h" -#include "string_reference.h" -#include "parallel_move_resolver.h" -#include "type_reference.h" -#include "utils/arm/assembler_thumb2.h" - -namespace art { -namespace arm { - -class CodeGeneratorARM; - -// Use a local definition to prevent copying mistakes. -static constexpr size_t kArmWordSize = static_cast<size_t>(kArmPointerSize); -static constexpr size_t kArmBitsPerWord = kArmWordSize * kBitsPerByte; - -static constexpr Register kParameterCoreRegisters[] = { R1, R2, R3 }; -static constexpr size_t kParameterCoreRegistersLength = arraysize(kParameterCoreRegisters); -static constexpr SRegister kParameterFpuRegisters[] = - { S0, S1, S2, S3, S4, S5, S6, S7, S8, S9, S10, S11, S12, S13, S14, S15 }; -static constexpr size_t kParameterFpuRegistersLength = arraysize(kParameterFpuRegisters); - -static constexpr Register kArtMethodRegister = R0; - -static constexpr Register kRuntimeParameterCoreRegisters[] = { R0, R1, R2, R3 }; -static constexpr size_t kRuntimeParameterCoreRegistersLength = - arraysize(kRuntimeParameterCoreRegisters); -static constexpr SRegister kRuntimeParameterFpuRegisters[] = { S0, S1, S2, S3 }; -static constexpr size_t kRuntimeParameterFpuRegistersLength = - arraysize(kRuntimeParameterFpuRegisters); - -class SlowPathCodeARM : public SlowPathCode { - public: - explicit SlowPathCodeARM(HInstruction* instruction) : SlowPathCode(instruction) {} - - void SaveLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) FINAL; - void RestoreLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) FINAL; - - private: - DISALLOW_COPY_AND_ASSIGN(SlowPathCodeARM); -}; - - -class InvokeRuntimeCallingConvention : public CallingConvention<Register, SRegister> { - public: - InvokeRuntimeCallingConvention() - : CallingConvention(kRuntimeParameterCoreRegisters, - kRuntimeParameterCoreRegistersLength, - kRuntimeParameterFpuRegisters, - kRuntimeParameterFpuRegistersLength, - kArmPointerSize) {} - - private: - DISALLOW_COPY_AND_ASSIGN(InvokeRuntimeCallingConvention); -}; - -constexpr DRegister FromLowSToD(SRegister reg) { - DCHECK_EQ(reg % 2, 0); - return static_cast<DRegister>(reg / 2); -} - - -class InvokeDexCallingConvention : public CallingConvention<Register, SRegister> { - public: - InvokeDexCallingConvention() - : CallingConvention(kParameterCoreRegisters, - kParameterCoreRegistersLength, - kParameterFpuRegisters, - kParameterFpuRegistersLength, - kArmPointerSize) {} - - private: - DISALLOW_COPY_AND_ASSIGN(InvokeDexCallingConvention); -}; - -class InvokeDexCallingConventionVisitorARM : public InvokeDexCallingConventionVisitor { - public: - InvokeDexCallingConventionVisitorARM() {} - virtual ~InvokeDexCallingConventionVisitorARM() {} - - Location GetNextLocation(Primitive::Type type) OVERRIDE; - Location GetReturnLocation(Primitive::Type type) const OVERRIDE; - Location GetMethodLocation() const OVERRIDE; - - private: - InvokeDexCallingConvention calling_convention; - uint32_t double_index_ = 0; - - DISALLOW_COPY_AND_ASSIGN(InvokeDexCallingConventionVisitorARM); -}; - -class FieldAccessCallingConventionARM : public FieldAccessCallingConvention { - public: - FieldAccessCallingConventionARM() {} - - Location GetObjectLocation() const OVERRIDE { - return Location::RegisterLocation(R1); - } - Location GetFieldIndexLocation() const OVERRIDE { - return Location::RegisterLocation(R0); - } - Location GetReturnLocation(Primitive::Type type) const OVERRIDE { - return Primitive::Is64BitType(type) - ? Location::RegisterPairLocation(R0, R1) - : Location::RegisterLocation(R0); - } - Location GetSetValueLocation(Primitive::Type type, bool is_instance) const OVERRIDE { - return Primitive::Is64BitType(type) - ? Location::RegisterPairLocation(R2, R3) - : (is_instance - ? Location::RegisterLocation(R2) - : Location::RegisterLocation(R1)); - } - Location GetFpuLocation(Primitive::Type type) const OVERRIDE { - return Primitive::Is64BitType(type) - ? Location::FpuRegisterPairLocation(S0, S1) - : Location::FpuRegisterLocation(S0); - } - - private: - DISALLOW_COPY_AND_ASSIGN(FieldAccessCallingConventionARM); -}; - -class ParallelMoveResolverARM : public ParallelMoveResolverWithSwap { - public: - ParallelMoveResolverARM(ArenaAllocator* allocator, CodeGeneratorARM* codegen) - : ParallelMoveResolverWithSwap(allocator), codegen_(codegen) {} - - void EmitMove(size_t index) OVERRIDE; - void EmitSwap(size_t index) OVERRIDE; - void SpillScratch(int reg) OVERRIDE; - void RestoreScratch(int reg) OVERRIDE; - - ArmAssembler* GetAssembler() const; - - private: - void Exchange(Register reg, int mem); - void Exchange(int mem1, int mem2); - - CodeGeneratorARM* const codegen_; - - DISALLOW_COPY_AND_ASSIGN(ParallelMoveResolverARM); -}; - -class LocationsBuilderARM : public HGraphVisitor { - public: - LocationsBuilderARM(HGraph* graph, CodeGeneratorARM* codegen) - : HGraphVisitor(graph), codegen_(codegen) {} - -#define DECLARE_VISIT_INSTRUCTION(name, super) \ - void Visit##name(H##name* instr) OVERRIDE; - - FOR_EACH_CONCRETE_INSTRUCTION_COMMON(DECLARE_VISIT_INSTRUCTION) - FOR_EACH_CONCRETE_INSTRUCTION_ARM(DECLARE_VISIT_INSTRUCTION) - FOR_EACH_CONCRETE_INSTRUCTION_SHARED(DECLARE_VISIT_INSTRUCTION) - -#undef DECLARE_VISIT_INSTRUCTION - - void VisitInstruction(HInstruction* instruction) OVERRIDE { - LOG(FATAL) << "Unreachable instruction " << instruction->DebugName() - << " (id " << instruction->GetId() << ")"; - } - - private: - void HandleInvoke(HInvoke* invoke); - void HandleBitwiseOperation(HBinaryOperation* operation, Opcode opcode); - void HandleCondition(HCondition* condition); - void HandleIntegerRotate(LocationSummary* locations); - void HandleLongRotate(LocationSummary* locations); - void HandleShift(HBinaryOperation* operation); - void HandleFieldSet(HInstruction* instruction, const FieldInfo& field_info); - void HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info); - - Location ArithmeticZeroOrFpuRegister(HInstruction* input); - Location ArmEncodableConstantOrRegister(HInstruction* constant, Opcode opcode); - bool CanEncodeConstantAsImmediate(HConstant* input_cst, Opcode opcode); - bool CanEncodeConstantAsImmediate(uint32_t value, Opcode opcode, SetCc set_cc = kCcDontCare); - - CodeGeneratorARM* const codegen_; - InvokeDexCallingConventionVisitorARM parameter_visitor_; - - DISALLOW_COPY_AND_ASSIGN(LocationsBuilderARM); -}; - -class InstructionCodeGeneratorARM : public InstructionCodeGenerator { - public: - InstructionCodeGeneratorARM(HGraph* graph, CodeGeneratorARM* codegen); - -#define DECLARE_VISIT_INSTRUCTION(name, super) \ - void Visit##name(H##name* instr) OVERRIDE; - - FOR_EACH_CONCRETE_INSTRUCTION_COMMON(DECLARE_VISIT_INSTRUCTION) - FOR_EACH_CONCRETE_INSTRUCTION_ARM(DECLARE_VISIT_INSTRUCTION) - FOR_EACH_CONCRETE_INSTRUCTION_SHARED(DECLARE_VISIT_INSTRUCTION) - -#undef DECLARE_VISIT_INSTRUCTION - - void VisitInstruction(HInstruction* instruction) OVERRIDE { - LOG(FATAL) << "Unreachable instruction " << instruction->DebugName() - << " (id " << instruction->GetId() << ")"; - } - - ArmAssembler* GetAssembler() const { return assembler_; } - - private: - // Generate code for the given suspend check. If not null, `successor` - // is the block to branch to if the suspend check is not needed, and after - // the suspend call. - void GenerateSuspendCheck(HSuspendCheck* check, HBasicBlock* successor); - void GenerateClassInitializationCheck(SlowPathCodeARM* slow_path, Register class_reg); - void GenerateAndConst(Register out, Register first, uint32_t value); - void GenerateOrrConst(Register out, Register first, uint32_t value); - void GenerateEorConst(Register out, Register first, uint32_t value); - void GenerateAddLongConst(Location out, Location first, uint64_t value); - void HandleBitwiseOperation(HBinaryOperation* operation); - void HandleCondition(HCondition* condition); - void HandleIntegerRotate(LocationSummary* locations); - void HandleLongRotate(HRor* ror); - void HandleShift(HBinaryOperation* operation); - - void GenerateWideAtomicStore(Register addr, uint32_t offset, - Register value_lo, Register value_hi, - Register temp1, Register temp2, - HInstruction* instruction); - void GenerateWideAtomicLoad(Register addr, uint32_t offset, - Register out_lo, Register out_hi); - - void HandleFieldSet(HInstruction* instruction, - const FieldInfo& field_info, - bool value_can_be_null); - void HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info); - - // Generate a heap reference load using one register `out`: - // - // out <- *(out + offset) - // - // while honoring heap poisoning and/or read barriers (if any). - // - // Location `maybe_temp` is used when generating a read barrier and - // shall be a register in that case; it may be an invalid location - // otherwise. - void GenerateReferenceLoadOneRegister(HInstruction* instruction, - Location out, - uint32_t offset, - Location maybe_temp, - ReadBarrierOption read_barrier_option); - // Generate a heap reference load using two different registers - // `out` and `obj`: - // - // out <- *(obj + offset) - // - // while honoring heap poisoning and/or read barriers (if any). - // - // Location `maybe_temp` is used when generating a Baker's (fast - // path) read barrier and shall be a register in that case; it may - // be an invalid location otherwise. - void GenerateReferenceLoadTwoRegisters(HInstruction* instruction, - Location out, - Location obj, - uint32_t offset, - Location maybe_temp, - ReadBarrierOption read_barrier_option); - // Generate a GC root reference load: - // - // root <- *(obj + offset) - // - // while honoring read barriers based on read_barrier_option. - void GenerateGcRootFieldLoad(HInstruction* instruction, - Location root, - Register obj, - uint32_t offset, - ReadBarrierOption read_barrier_option); - void GenerateTestAndBranch(HInstruction* instruction, - size_t condition_input_index, - Label* true_target, - Label* false_target); - void GenerateCompareTestAndBranch(HCondition* condition, - Label* true_target, - Label* false_target); - void DivRemOneOrMinusOne(HBinaryOperation* instruction); - void DivRemByPowerOfTwo(HBinaryOperation* instruction); - void GenerateDivRemWithAnyConstant(HBinaryOperation* instruction); - void GenerateDivRemConstantIntegral(HBinaryOperation* instruction); - void HandleGoto(HInstruction* got, HBasicBlock* successor); - - ArmAssembler* const assembler_; - CodeGeneratorARM* const codegen_; - - DISALLOW_COPY_AND_ASSIGN(InstructionCodeGeneratorARM); -}; - -class CodeGeneratorARM : public CodeGenerator { - public: - CodeGeneratorARM(HGraph* graph, - const ArmInstructionSetFeatures& isa_features, - const CompilerOptions& compiler_options, - OptimizingCompilerStats* stats = nullptr); - virtual ~CodeGeneratorARM() {} - - void GenerateFrameEntry() OVERRIDE; - void GenerateFrameExit() OVERRIDE; - void Bind(HBasicBlock* block) OVERRIDE; - void MoveConstant(Location destination, int32_t value) OVERRIDE; - void MoveLocation(Location dst, Location src, Primitive::Type dst_type) OVERRIDE; - void AddLocationAsTemp(Location location, LocationSummary* locations) OVERRIDE; - - size_t SaveCoreRegister(size_t stack_index, uint32_t reg_id) OVERRIDE; - size_t RestoreCoreRegister(size_t stack_index, uint32_t reg_id) OVERRIDE; - size_t SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) OVERRIDE; - size_t RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) OVERRIDE; - - size_t GetWordSize() const OVERRIDE { - return kArmWordSize; - } - - size_t GetFloatingPointSpillSlotSize() const OVERRIDE { - // Allocated in S registers, which are word sized. - return kArmWordSize; - } - - HGraphVisitor* GetLocationBuilder() OVERRIDE { - return &location_builder_; - } - - HGraphVisitor* GetInstructionVisitor() OVERRIDE { - return &instruction_visitor_; - } - - ArmAssembler* GetAssembler() OVERRIDE { - return &assembler_; - } - - const ArmAssembler& GetAssembler() const OVERRIDE { - return assembler_; - } - - uintptr_t GetAddressOf(HBasicBlock* block) OVERRIDE { - return GetLabelOf(block)->Position(); - } - - void SetupBlockedRegisters() const OVERRIDE; - - void DumpCoreRegister(std::ostream& stream, int reg) const OVERRIDE; - void DumpFloatingPointRegister(std::ostream& stream, int reg) const OVERRIDE; - - ParallelMoveResolverARM* GetMoveResolver() OVERRIDE { - return &move_resolver_; - } - - InstructionSet GetInstructionSet() const OVERRIDE { - return InstructionSet::kThumb2; - } - - // Helper method to move a 32bits value between two locations. - void Move32(Location destination, Location source); - // Helper method to move a 64bits value between two locations. - void Move64(Location destination, Location source); - - void LoadOrStoreToOffset(Primitive::Type type, - Location loc, - Register base, - int32_t offset, - bool is_load, - Condition cond = AL); - - void LoadFromShiftedRegOffset(Primitive::Type type, - Location out_loc, - Register base, - Register reg_offset, - Condition cond = AL); - void StoreToShiftedRegOffset(Primitive::Type type, - Location out_loc, - Register base, - Register reg_offset, - Condition cond = AL); - - // Generate code to invoke a runtime entry point. - void InvokeRuntime(QuickEntrypointEnum entrypoint, - HInstruction* instruction, - uint32_t dex_pc, - SlowPathCode* slow_path = nullptr) OVERRIDE; - - // Generate code to invoke a runtime entry point, but do not record - // PC-related information in a stack map. - void InvokeRuntimeWithoutRecordingPcInfo(int32_t entry_point_offset, - HInstruction* instruction, - SlowPathCode* slow_path); - - void GenerateInvokeRuntime(int32_t entry_point_offset); - - // Emit a write barrier. - void MarkGCCard(Register temp, Register card, Register object, Register value, bool can_be_null); - - void GenerateMemoryBarrier(MemBarrierKind kind); - - Label* GetLabelOf(HBasicBlock* block) const { - return CommonGetLabelOf<Label>(block_labels_, block); - } - - Label* GetFinalLabel(HInstruction* instruction, Label* final_label); - - void Initialize() OVERRIDE { - block_labels_ = CommonInitializeLabels<Label>(); - } - - void Finalize(CodeAllocator* allocator) OVERRIDE; - - const ArmInstructionSetFeatures& GetInstructionSetFeatures() const { - return isa_features_; - } - - bool NeedsTwoRegisters(Primitive::Type type) const OVERRIDE { - return type == Primitive::kPrimDouble || type == Primitive::kPrimLong; - } - - void ComputeSpillMask() OVERRIDE; - - Label* GetFrameEntryLabel() { return &frame_entry_label_; } - - // Check if the desired_string_load_kind is supported. If it is, return it, - // otherwise return a fall-back kind that should be used instead. - HLoadString::LoadKind GetSupportedLoadStringKind( - HLoadString::LoadKind desired_string_load_kind) OVERRIDE; - - // Check if the desired_class_load_kind is supported. If it is, return it, - // otherwise return a fall-back kind that should be used instead. - HLoadClass::LoadKind GetSupportedLoadClassKind( - HLoadClass::LoadKind desired_class_load_kind) OVERRIDE; - - // Check if the desired_dispatch_info is supported. If it is, return it, - // otherwise return a fall-back info that should be used instead. - HInvokeStaticOrDirect::DispatchInfo GetSupportedInvokeStaticOrDirectDispatch( - const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info, - HInvokeStaticOrDirect* invoke) OVERRIDE; - - void GenerateStaticOrDirectCall( - HInvokeStaticOrDirect* invoke, Location temp, SlowPathCode* slow_path = nullptr) OVERRIDE; - void GenerateVirtualCall( - HInvokeVirtual* invoke, Location temp, SlowPathCode* slow_path = nullptr) OVERRIDE; - - void MoveFromReturnRegister(Location trg, Primitive::Type type) OVERRIDE; - - // The PcRelativePatchInfo is used for PC-relative addressing of dex cache arrays - // and boot image strings/types. The only difference is the interpretation of the - // offset_or_index. The PC-relative address is loaded with three instructions, - // MOVW+MOVT to load the offset to base_reg and then ADD base_reg, PC. The offset - // is calculated from the ADD's effective PC, i.e. PC+4 on Thumb2. Though we - // currently emit these 3 instructions together, instruction scheduling could - // split this sequence apart, so we keep separate labels for each of them. - struct PcRelativePatchInfo { - PcRelativePatchInfo(const DexFile& dex_file, uint32_t off_or_idx) - : target_dex_file(dex_file), offset_or_index(off_or_idx) { } - PcRelativePatchInfo(PcRelativePatchInfo&& other) = default; - - const DexFile& target_dex_file; - // Either the dex cache array element offset or the string/type index. - uint32_t offset_or_index; - Label movw_label; - Label movt_label; - Label add_pc_label; - }; - - PcRelativePatchInfo* NewPcRelativeMethodPatch(MethodReference target_method); - PcRelativePatchInfo* NewMethodBssEntryPatch(MethodReference target_method); - PcRelativePatchInfo* NewPcRelativeTypePatch(const DexFile& dex_file, dex::TypeIndex type_index); - PcRelativePatchInfo* NewTypeBssEntryPatch(const DexFile& dex_file, dex::TypeIndex type_index); - PcRelativePatchInfo* NewPcRelativeStringPatch(const DexFile& dex_file, - dex::StringIndex string_index); - - // Add a new baker read barrier patch and return the label to be bound - // before the BNE instruction. - Label* NewBakerReadBarrierPatch(uint32_t custom_data); - - Literal* DeduplicateBootImageAddressLiteral(uint32_t address); - Literal* DeduplicateJitStringLiteral(const DexFile& dex_file, - dex::StringIndex string_index, - Handle<mirror::String> handle); - Literal* DeduplicateJitClassLiteral(const DexFile& dex_file, - dex::TypeIndex type_index, - Handle<mirror::Class> handle); - - void EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) OVERRIDE; - - void EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) OVERRIDE; - - // Maybe add the reserved entrypoint register as a temporary for field load. This temp - // is added only for AOT compilation if link-time generated thunks for fields are enabled. - void MaybeAddBakerCcEntrypointTempForFields(LocationSummary* locations); - - // Fast path implementation of ReadBarrier::Barrier for a heap - // reference field load when Baker's read barriers are used. - void GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction, - Location ref, - Register obj, - uint32_t offset, - Location temp, - bool needs_null_check); - // Fast path implementation of ReadBarrier::Barrier for a heap - // reference array load when Baker's read barriers are used. - void GenerateArrayLoadWithBakerReadBarrier(HInstruction* instruction, - Location ref, - Register obj, - uint32_t data_offset, - Location index, - Location temp, - bool needs_null_check); - // Factored implementation, used by GenerateFieldLoadWithBakerReadBarrier, - // GenerateArrayLoadWithBakerReadBarrier and some intrinsics. - // - // Load the object reference located at the address - // `obj + offset + (index << scale_factor)`, held by object `obj`, into - // `ref`, and mark it if needed. - void GenerateReferenceLoadWithBakerReadBarrier(HInstruction* instruction, - Location ref, - Register obj, - uint32_t offset, - Location index, - ScaleFactor scale_factor, - Location temp, - bool needs_null_check); - - // Generate code checking whether the the reference field at the - // address `obj + field_offset`, held by object `obj`, needs to be - // marked, and if so, marking it and updating the field within `obj` - // with the marked value. - // - // This routine is used for the implementation of the - // UnsafeCASObject intrinsic with Baker read barriers. - // - // This method has a structure similar to - // GenerateReferenceLoadWithBakerReadBarrier, but note that argument - // `ref` is only as a temporary here, and thus its value should not - // be used afterwards. - void UpdateReferenceFieldWithBakerReadBarrier(HInstruction* instruction, - Location ref, - Register obj, - Location field_offset, - Location temp, - bool needs_null_check, - Register temp2); - - // Generate a heap reference load (with no read barrier). - void GenerateRawReferenceLoad(HInstruction* instruction, - Location ref, - Register obj, - uint32_t offset, - Location index, - ScaleFactor scale_factor, - bool needs_null_check); - - // Generate a read barrier for a heap reference within `instruction` - // using a slow path. - // - // A read barrier for an object reference read from the heap is - // implemented as a call to the artReadBarrierSlow runtime entry - // point, which is passed the values in locations `ref`, `obj`, and - // `offset`: - // - // mirror::Object* artReadBarrierSlow(mirror::Object* ref, - // mirror::Object* obj, - // uint32_t offset); - // - // The `out` location contains the value returned by - // artReadBarrierSlow. - // - // When `index` is provided (i.e. for array accesses), the offset - // value passed to artReadBarrierSlow is adjusted to take `index` - // into account. - void GenerateReadBarrierSlow(HInstruction* instruction, - Location out, - Location ref, - Location obj, - uint32_t offset, - Location index = Location::NoLocation()); - - // If read barriers are enabled, generate a read barrier for a heap - // reference using a slow path. If heap poisoning is enabled, also - // unpoison the reference in `out`. - void MaybeGenerateReadBarrierSlow(HInstruction* instruction, - Location out, - Location ref, - Location obj, - uint32_t offset, - Location index = Location::NoLocation()); - - // Generate a read barrier for a GC root within `instruction` using - // a slow path. - // - // A read barrier for an object reference GC root is implemented as - // a call to the artReadBarrierForRootSlow runtime entry point, - // which is passed the value in location `root`: - // - // mirror::Object* artReadBarrierForRootSlow(GcRoot<mirror::Object>* root); - // - // The `out` location contains the value returned by - // artReadBarrierForRootSlow. - void GenerateReadBarrierForRootSlow(HInstruction* instruction, Location out, Location root); - - void GenerateNop() OVERRIDE; - - void GenerateImplicitNullCheck(HNullCheck* instruction) OVERRIDE; - void GenerateExplicitNullCheck(HNullCheck* instruction) OVERRIDE; - - // `temp` is an extra temporary register that is used for some conditions; - // callers may not specify it, in which case the method will use a scratch - // register instead. - void GenerateConditionWithZero(IfCondition condition, - Register out, - Register in, - Register temp = kNoRegister); - - private: - Register GetInvokeStaticOrDirectExtraParameter(HInvokeStaticOrDirect* invoke, Register temp); - - using Uint32ToLiteralMap = ArenaSafeMap<uint32_t, Literal*>; - using StringToLiteralMap = ArenaSafeMap<StringReference, - Literal*, - StringReferenceValueComparator>; - using TypeToLiteralMap = ArenaSafeMap<TypeReference, - Literal*, - TypeReferenceValueComparator>; - - struct BakerReadBarrierPatchInfo { - explicit BakerReadBarrierPatchInfo(uint32_t data) : label(), custom_data(data) { } - - Label label; - uint32_t custom_data; - }; - - Literal* DeduplicateUint32Literal(uint32_t value, Uint32ToLiteralMap* map); - PcRelativePatchInfo* NewPcRelativePatch(const DexFile& dex_file, - uint32_t offset_or_index, - ArenaDeque<PcRelativePatchInfo>* patches); - template <LinkerPatch (*Factory)(size_t, const DexFile*, uint32_t, uint32_t)> - static void EmitPcRelativeLinkerPatches(const ArenaDeque<PcRelativePatchInfo>& infos, - ArenaVector<LinkerPatch>* linker_patches); - - // Labels for each block that will be compiled. - Label* block_labels_; // Indexed by block id. - Label frame_entry_label_; - LocationsBuilderARM location_builder_; - InstructionCodeGeneratorARM instruction_visitor_; - ParallelMoveResolverARM move_resolver_; - Thumb2Assembler assembler_; - const ArmInstructionSetFeatures& isa_features_; - - // Deduplication map for 32-bit literals, used for non-patchable boot image addresses. - Uint32ToLiteralMap uint32_literals_; - // PC-relative method patch info for kBootImageLinkTimePcRelative. - ArenaDeque<PcRelativePatchInfo> pc_relative_method_patches_; - // PC-relative method patch info for kBssEntry. - ArenaDeque<PcRelativePatchInfo> method_bss_entry_patches_; - // PC-relative type patch info for kBootImageLinkTimePcRelative. - ArenaDeque<PcRelativePatchInfo> pc_relative_type_patches_; - // PC-relative type patch info for kBssEntry. - ArenaDeque<PcRelativePatchInfo> type_bss_entry_patches_; - // PC-relative String patch info; type depends on configuration (app .bss or boot image PIC). - ArenaDeque<PcRelativePatchInfo> pc_relative_string_patches_; - // Baker read barrier patch info. - ArenaDeque<BakerReadBarrierPatchInfo> baker_read_barrier_patches_; - - // Patches for string literals in JIT compiled code. - StringToLiteralMap jit_string_patches_; - // Patches for class literals in JIT compiled code. - TypeToLiteralMap jit_class_patches_; - - DISALLOW_COPY_AND_ASSIGN(CodeGeneratorARM); -}; - -} // namespace arm -} // namespace art - -#endif // ART_COMPILER_OPTIMIZING_CODE_GENERATOR_ARM_H_ diff --git a/compiler/optimizing/code_generator_arm64.cc b/compiler/optimizing/code_generator_arm64.cc index 2561ed0762..7e5b1a0fd1 100644 --- a/compiler/optimizing/code_generator_arm64.cc +++ b/compiler/optimizing/code_generator_arm64.cc @@ -672,7 +672,9 @@ void JumpTableARM64::EmitTable(CodeGeneratorARM64* codegen) { // `ref`. // // Argument `entrypoint` must be a register location holding the read -// barrier marking runtime entry point to be invoked. +// barrier marking runtime entry point to be invoked or an empty +// location; in the latter case, the read barrier marking runtime +// entry point will be loaded by the slow path code itself. class ReadBarrierMarkSlowPathBaseARM64 : public SlowPathCodeARM64 { protected: ReadBarrierMarkSlowPathBaseARM64(HInstruction* instruction, Location ref, Location entrypoint) @@ -716,7 +718,7 @@ class ReadBarrierMarkSlowPathBaseARM64 : public SlowPathCodeARM64 { } else { // Entrypoint is not already loaded, load from the thread. int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArm64PointerSize>(ref_.reg()); + Thread::ReadBarrierMarkEntryPointsOffset<kArm64PointerSize>(ref_.reg()); // This runtime call does not require a stack map. arm64_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this); } @@ -743,9 +745,10 @@ class ReadBarrierMarkSlowPathBaseARM64 : public SlowPathCodeARM64 { // another thread, or if another thread installed another object // reference (different from `ref`) in `obj.field`). // -// If `entrypoint` is a valid location it is assumed to already be -// holding the entrypoint. The case where the entrypoint is passed in -// is when the decision to mark is based on whether the GC is marking. +// Argument `entrypoint` must be a register location holding the read +// barrier marking runtime entry point to be invoked or an empty +// location; in the latter case, the read barrier marking runtime +// entry point will be loaded by the slow path code itself. class ReadBarrierMarkSlowPathARM64 : public ReadBarrierMarkSlowPathBaseARM64 { public: ReadBarrierMarkSlowPathARM64(HInstruction* instruction, @@ -791,7 +794,9 @@ class ReadBarrierMarkSlowPathARM64 : public ReadBarrierMarkSlowPathBaseARM64 { // reference (different from `ref`) in `obj.field`). // // Argument `entrypoint` must be a register location holding the read -// barrier marking runtime entry point to be invoked. +// barrier marking runtime entry point to be invoked or an empty +// location; in the latter case, the read barrier marking runtime +// entry point will be loaded by the slow path code itself. class LoadReferenceWithBakerReadBarrierSlowPathARM64 : public ReadBarrierMarkSlowPathBaseARM64 { public: LoadReferenceWithBakerReadBarrierSlowPathARM64(HInstruction* instruction, @@ -803,7 +808,7 @@ class LoadReferenceWithBakerReadBarrierSlowPathARM64 : public ReadBarrierMarkSlo bool needs_null_check, bool use_load_acquire, Register temp, - Location entrypoint) + Location entrypoint = Location::NoLocation()) : ReadBarrierMarkSlowPathBaseARM64(instruction, ref, entrypoint), obj_(obj), offset_(offset), @@ -947,20 +952,23 @@ class LoadReferenceWithBakerReadBarrierSlowPathARM64 : public ReadBarrierMarkSlo // another object reference (different from `ref`) in `obj.field`). // // Argument `entrypoint` must be a register location holding the read -// barrier marking runtime entry point to be invoked. +// barrier marking runtime entry point to be invoked or an empty +// location; in the latter case, the read barrier marking runtime +// entry point will be loaded by the slow path code itself. class LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM64 : public ReadBarrierMarkSlowPathBaseARM64 { public: - LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM64(HInstruction* instruction, - Location ref, - Register obj, - uint32_t offset, - Location index, - size_t scale_factor, - bool needs_null_check, - bool use_load_acquire, - Register temp, - Location entrypoint) + LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM64( + HInstruction* instruction, + Location ref, + Register obj, + uint32_t offset, + Location index, + size_t scale_factor, + bool needs_null_check, + bool use_load_acquire, + Register temp, + Location entrypoint = Location::NoLocation()) : ReadBarrierMarkSlowPathBaseARM64(instruction, ref, entrypoint), obj_(obj), offset_(offset), @@ -1655,7 +1663,7 @@ void CodeGeneratorARM64::SetupBlockedRegisters() const { // Blocked core registers: // lr : Runtime reserved. // tr : Runtime reserved. - // xSuspend : Runtime reserved. TODO: Unblock this when the runtime stops using it. + // mr : Runtime reserved. // ip1 : VIXL core temp. // ip0 : VIXL core temp. // @@ -5921,20 +5929,17 @@ void InstructionCodeGeneratorARM64::GenerateGcRootFieldLoad( // Baker's read barrier are used. if (kBakerReadBarrierLinkTimeThunksEnableForGcRoots && !Runtime::Current()->UseJitCompilation()) { - // Note that we do not actually check the value of `GetIsGcMarking()` - // to decide whether to mark the loaded GC root or not. Instead, we - // load into `temp` (actually IP1) the read barrier mark introspection - // entrypoint. If `temp` is null, it means that `GetIsGcMarking()` is - // false, and vice versa. + // Query `art::Thread::Current()->GetIsGcMarking()` (stored in + // the Marking Register) to decide whether we need to enter + // the slow path to mark the GC root. // // We use link-time generated thunks for the slow path. That thunk // checks the reference and jumps to the entrypoint if needed. // - // temp = Thread::Current()->pReadBarrierMarkIntrospection // lr = &return_address; // GcRoot<mirror::Object> root = *(obj+offset); // Original reference load. - // if (temp != nullptr) { - // goto gc_root_thunk<root_reg>(lr) + // if (mr) { // Thread::Current()->GetIsGcMarking() + // goto gc_root_thunk<root_reg>(lr) // } // return_address: @@ -5946,11 +5951,6 @@ void InstructionCodeGeneratorARM64::GenerateGcRootFieldLoad( linker::Arm64RelativePatcher::EncodeBakerReadBarrierGcRootData(root_reg.GetCode()); vixl::aarch64::Label* cbnz_label = codegen_->NewBakerReadBarrierPatch(custom_data); - // ip1 = Thread::Current()->pReadBarrierMarkReg16, i.e. pReadBarrierMarkIntrospection. - DCHECK_EQ(ip0.GetCode(), 16u); - const int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArm64PointerSize>(ip0.GetCode()); - __ Ldr(ip1, MemOperand(tr, entry_point_offset)); EmissionCheckScope guard(GetVIXLAssembler(), 3 * vixl::aarch64::kInstructionSize); vixl::aarch64::Label return_address; __ adr(lr, &return_address); @@ -5961,36 +5961,26 @@ void InstructionCodeGeneratorARM64::GenerateGcRootFieldLoad( "GC root LDR must be 2 instruction (8B) before the return address label."); __ ldr(root_reg, MemOperand(obj.X(), offset)); __ Bind(cbnz_label); - __ cbnz(ip1, static_cast<int64_t>(0)); // Placeholder, patched at link-time. + __ cbnz(mr, static_cast<int64_t>(0)); // Placeholder, patched at link-time. __ Bind(&return_address); } else { - // Note that we do not actually check the value of - // `GetIsGcMarking()` to decide whether to mark the loaded GC - // root or not. Instead, we load into `temp` the read barrier - // mark entry point corresponding to register `root`. If `temp` - // is null, it means that `GetIsGcMarking()` is false, and vice - // versa. + // Query `art::Thread::Current()->GetIsGcMarking()` (stored in + // the Marking Register) to decide whether we need to enter + // the slow path to mark the GC root. // - // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg() // GcRoot<mirror::Object> root = *(obj+offset); // Original reference load. - // if (temp != nullptr) { // <=> Thread::Current()->GetIsGcMarking() + // if (mr) { // Thread::Current()->GetIsGcMarking() // // Slow path. - // root = temp(root); // root = ReadBarrier::Mark(root); // Runtime entry point call. + // entrypoint = Thread::Current()->pReadBarrierMarkReg ## root.reg() + // root = entrypoint(root); // root = ReadBarrier::Mark(root); // Entry point call. // } - // Slow path marking the GC root `root`. The entrypoint will already be loaded in `temp`. - Register temp = lr; - SlowPathCodeARM64* slow_path = new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM64( - instruction, root, /* entrypoint */ LocationFrom(temp)); + // Slow path marking the GC root `root`. The entrypoint will + // be loaded by the slow path code. + SlowPathCodeARM64* slow_path = + new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM64(instruction, root); codegen_->AddSlowPath(slow_path); - // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg() - const int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArm64PointerSize>(root.reg()); - // Loading the entrypoint does not require a load acquire since it is only changed when - // threads are suspended or running a checkpoint. - __ Ldr(temp, MemOperand(tr, entry_point_offset)); - // /* GcRoot<mirror::Object> */ root = *(obj + offset) if (fixup_label == nullptr) { __ Ldr(root_reg, MemOperand(obj, offset)); @@ -6005,9 +5995,7 @@ void InstructionCodeGeneratorARM64::GenerateGcRootFieldLoad( "art::mirror::CompressedReference<mirror::Object> and int32_t " "have different sizes."); - // The entrypoint is null when the GC is not marking, this prevents one load compared to - // checking GetIsGcMarking. - __ Cbnz(temp, slow_path->GetEntryLabel()); + __ Cbnz(mr, slow_path->GetEntryLabel()); __ Bind(slow_path->GetExitLabel()); } } else { @@ -6048,20 +6036,19 @@ void CodeGeneratorARM64::GenerateFieldLoadWithBakerReadBarrier(HInstruction* ins if (kBakerReadBarrierLinkTimeThunksEnableForFields && !use_load_acquire && !Runtime::Current()->UseJitCompilation()) { - // Note that we do not actually check the value of `GetIsGcMarking()` - // to decide whether to mark the loaded reference or not. Instead, we - // load into `temp` (actually IP1) the read barrier mark introspection - // entrypoint. If `temp` is null, it means that `GetIsGcMarking()` is - // false, and vice versa. + // Query `art::Thread::Current()->GetIsGcMarking()` (stored in the + // Marking Register) to decide whether we need to enter the slow + // path to mark the reference. Then, in the slow path, check the + // gray bit in the lock word of the reference's holder (`obj`) to + // decide whether to mark `ref` or not. // // We use link-time generated thunks for the slow path. That thunk checks // the holder and jumps to the entrypoint if needed. If the holder is not // gray, it creates a fake dependency and returns to the LDR instruction. // - // temp = Thread::Current()->pReadBarrierMarkIntrospection // lr = &gray_return_address; - // if (temp != nullptr) { - // goto field_thunk<holder_reg, base_reg>(lr) + // if (mr) { // Thread::Current()->GetIsGcMarking() + // goto field_thunk<holder_reg, base_reg>(lr) // } // not_gray_return_address: // // Original reference load. If the offset is too large to fit @@ -6087,17 +6074,12 @@ void CodeGeneratorARM64::GenerateFieldLoadWithBakerReadBarrier(HInstruction* ins obj.GetCode()); vixl::aarch64::Label* cbnz_label = NewBakerReadBarrierPatch(custom_data); - // ip1 = Thread::Current()->pReadBarrierMarkReg16, i.e. pReadBarrierMarkIntrospection. - DCHECK_EQ(ip0.GetCode(), 16u); - const int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArm64PointerSize>(ip0.GetCode()); - __ Ldr(ip1, MemOperand(tr, entry_point_offset)); EmissionCheckScope guard(GetVIXLAssembler(), (kPoisonHeapReferences ? 4u : 3u) * vixl::aarch64::kInstructionSize); vixl::aarch64::Label return_address; __ adr(lr, &return_address); __ Bind(cbnz_label); - __ cbnz(ip1, static_cast<int64_t>(0)); // Placeholder, patched at link-time. + __ cbnz(mr, static_cast<int64_t>(0)); // Placeholder, patched at link-time. static_assert(BAKER_MARK_INTROSPECTION_FIELD_LDR_OFFSET == (kPoisonHeapReferences ? -8 : -4), "Field LDR must be 1 instruction (4B) before the return address label; " " 2 instructions (8B) for heap poisoning."); @@ -6143,20 +6125,19 @@ void CodeGeneratorARM64::GenerateArrayLoadWithBakerReadBarrier(HInstruction* ins if (kBakerReadBarrierLinkTimeThunksEnableForArrays && !Runtime::Current()->UseJitCompilation()) { - // Note that we do not actually check the value of `GetIsGcMarking()` - // to decide whether to mark the loaded reference or not. Instead, we - // load into `temp` (actually IP1) the read barrier mark introspection - // entrypoint. If `temp` is null, it means that `GetIsGcMarking()` is - // false, and vice versa. + // Query `art::Thread::Current()->GetIsGcMarking()` (stored in the + // Marking Register) to decide whether we need to enter the slow + // path to mark the reference. Then, in the slow path, check the + // gray bit in the lock word of the reference's holder (`obj`) to + // decide whether to mark `ref` or not. // // We use link-time generated thunks for the slow path. That thunk checks // the holder and jumps to the entrypoint if needed. If the holder is not // gray, it creates a fake dependency and returns to the LDR instruction. // - // temp = Thread::Current()->pReadBarrierMarkIntrospection // lr = &gray_return_address; - // if (temp != nullptr) { - // goto field_thunk<holder_reg, base_reg>(lr) + // if (mr) { // Thread::Current()->GetIsGcMarking() + // goto array_thunk<base_reg>(lr) // } // not_gray_return_address: // // Original reference load. If the offset is too large to fit @@ -6176,18 +6157,13 @@ void CodeGeneratorARM64::GenerateArrayLoadWithBakerReadBarrier(HInstruction* ins linker::Arm64RelativePatcher::EncodeBakerReadBarrierArrayData(temp.GetCode()); vixl::aarch64::Label* cbnz_label = NewBakerReadBarrierPatch(custom_data); - // ip1 = Thread::Current()->pReadBarrierMarkReg16, i.e. pReadBarrierMarkIntrospection. - DCHECK_EQ(ip0.GetCode(), 16u); - const int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArm64PointerSize>(ip0.GetCode()); - __ Ldr(ip1, MemOperand(tr, entry_point_offset)); __ Add(temp.X(), obj.X(), Operand(data_offset)); EmissionCheckScope guard(GetVIXLAssembler(), (kPoisonHeapReferences ? 4u : 3u) * vixl::aarch64::kInstructionSize); vixl::aarch64::Label return_address; __ adr(lr, &return_address); __ Bind(cbnz_label); - __ cbnz(ip1, static_cast<int64_t>(0)); // Placeholder, patched at link-time. + __ cbnz(mr, static_cast<int64_t>(0)); // Placeholder, patched at link-time. static_assert(BAKER_MARK_INTROSPECTION_ARRAY_LDR_OFFSET == (kPoisonHeapReferences ? -8 : -4), "Array LDR must be 1 instruction (4B) before the return address label; " " 2 instructions (8B) for heap poisoning."); @@ -6231,35 +6207,28 @@ void CodeGeneratorARM64::GenerateReferenceLoadWithBakerReadBarrier(HInstruction* // `instruction->IsArrayGet()` => `!use_load_acquire`. DCHECK(!instruction->IsArrayGet() || !use_load_acquire); - // Query `art::Thread::Current()->GetIsGcMarking()` to decide - // whether we need to enter the slow path to mark the reference. - // Then, in the slow path, check the gray bit in the lock word of - // the reference's holder (`obj`) to decide whether to mark `ref` or - // not. + // Query `art::Thread::Current()->GetIsGcMarking()` (stored in the + // Marking Register) to decide whether we need to enter the slow + // path to mark the reference. Then, in the slow path, check the + // gray bit in the lock word of the reference's holder (`obj`) to + // decide whether to mark `ref` or not. // - // Note that we do not actually check the value of `GetIsGcMarking()`; - // instead, we load into `temp2` the read barrier mark entry point - // corresponding to register `ref`. If `temp2` is null, it means - // that `GetIsGcMarking()` is false, and vice versa. - // - // temp2 = Thread::Current()->pReadBarrierMarkReg ## root.reg() - // if (temp2 != nullptr) { // <=> Thread::Current()->GetIsGcMarking() + // if (mr) { // Thread::Current()->GetIsGcMarking() // // Slow path. // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState(); // lfence; // Load fence or artificial data dependency to prevent load-load reordering // HeapReference<mirror::Object> ref = *src; // Original reference load. // bool is_gray = (rb_state == ReadBarrier::GrayState()); // if (is_gray) { - // ref = temp2(ref); // ref = ReadBarrier::Mark(ref); // Runtime entry point call. + // entrypoint = Thread::Current()->pReadBarrierMarkReg ## root.reg() + // ref = entrypoint(ref); // ref = ReadBarrier::Mark(ref); // Runtime entry point call. // } // } else { // HeapReference<mirror::Object> ref = *src; // Original reference load. // } // Slow path marking the object `ref` when the GC is marking. The - // entrypoint will already be loaded in `temp2`. - Register temp2 = lr; - Location temp2_loc = LocationFrom(temp2); + // entrypoint will be loaded by the slow path code. SlowPathCodeARM64* slow_path = new (GetGraph()->GetArena()) LoadReferenceWithBakerReadBarrierSlowPathARM64( instruction, @@ -6270,19 +6239,10 @@ void CodeGeneratorARM64::GenerateReferenceLoadWithBakerReadBarrier(HInstruction* scale_factor, needs_null_check, use_load_acquire, - temp, - /* entrypoint */ temp2_loc); + temp); AddSlowPath(slow_path); - // temp2 = Thread::Current()->pReadBarrierMarkReg ## ref.reg() - const int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArm64PointerSize>(ref.reg()); - // Loading the entrypoint does not require a load acquire since it is only changed when - // threads are suspended or running a checkpoint. - __ Ldr(temp2, MemOperand(tr, entry_point_offset)); - // The entrypoint is null when the GC is not marking, this prevents one load compared to - // checking GetIsGcMarking. - __ Cbnz(temp2, slow_path->GetEntryLabel()); + __ Cbnz(mr, slow_path->GetEntryLabel()); // Fast path: the GC is not marking: just load the reference. GenerateRawReferenceLoad( instruction, ref, obj, offset, index, scale_factor, needs_null_check, use_load_acquire); @@ -6303,19 +6263,14 @@ void CodeGeneratorARM64::UpdateReferenceFieldWithBakerReadBarrier(HInstruction* // `instruction->IsArrayGet()` => `!use_load_acquire`. DCHECK(!instruction->IsArrayGet() || !use_load_acquire); - // Query `art::Thread::Current()->GetIsGcMarking()` to decide - // whether we need to enter the slow path to update the reference - // field within `obj`. Then, in the slow path, check the gray bit - // in the lock word of the reference's holder (`obj`) to decide - // whether to mark `ref` and update the field or not. - // - // Note that we do not actually check the value of `GetIsGcMarking()`; - // instead, we load into `temp2` the read barrier mark entry point - // corresponding to register `ref`. If `temp2` is null, it means - // that `GetIsGcMarking()` is false, and vice versa. + // Query `art::Thread::Current()->GetIsGcMarking()` (stored in the + // Marking Register) to decide whether we need to enter the slow + // path to update the reference field within `obj`. Then, in the + // slow path, check the gray bit in the lock word of the reference's + // holder (`obj`) to decide whether to mark `ref` and update the + // field or not. // - // temp2 = Thread::Current()->pReadBarrierMarkReg ## root.reg() - // if (temp2 != nullptr) { // <=> Thread::Current()->GetIsGcMarking() + // if (mr) { // Thread::Current()->GetIsGcMarking() // // Slow path. // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState(); // lfence; // Load fence or artificial data dependency to prevent load-load reordering @@ -6323,15 +6278,14 @@ void CodeGeneratorARM64::UpdateReferenceFieldWithBakerReadBarrier(HInstruction* // bool is_gray = (rb_state == ReadBarrier::GrayState()); // if (is_gray) { // old_ref = ref; - // ref = temp2(ref); // ref = ReadBarrier::Mark(ref); // Runtime entry point call. + // entrypoint = Thread::Current()->pReadBarrierMarkReg ## root.reg() + // ref = entrypoint(ref); // ref = ReadBarrier::Mark(ref); // Runtime entry point call. // compareAndSwapObject(obj, field_offset, old_ref, ref); // } // } // Slow path updating the object reference at address `obj + field_offset` - // when the GC is marking. The entrypoint will already be loaded in `temp2`. - Register temp2 = lr; - Location temp2_loc = LocationFrom(temp2); + // when the GC is marking. The entrypoint will be loaded by the slow path code. SlowPathCodeARM64* slow_path = new (GetGraph()->GetArena()) LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM64( instruction, @@ -6342,19 +6296,10 @@ void CodeGeneratorARM64::UpdateReferenceFieldWithBakerReadBarrier(HInstruction* /* scale_factor */ 0u /* "times 1" */, needs_null_check, use_load_acquire, - temp, - /* entrypoint */ temp2_loc); + temp); AddSlowPath(slow_path); - // temp2 = Thread::Current()->pReadBarrierMarkReg ## ref.reg() - const int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArm64PointerSize>(ref.reg()); - // Loading the entrypoint does not require a load acquire since it is only changed when - // threads are suspended or running a checkpoint. - __ Ldr(temp2, MemOperand(tr, entry_point_offset)); - // The entrypoint is null when the GC is not marking, this prevents one load compared to - // checking GetIsGcMarking. - __ Cbnz(temp2, slow_path->GetEntryLabel()); + __ Cbnz(mr, slow_path->GetEntryLabel()); // Fast path: the GC is not marking: nothing to do (the field is // up-to-date, and we don't need to load the reference). __ Bind(slow_path->GetExitLabel()); diff --git a/compiler/optimizing/code_generator_arm64.h b/compiler/optimizing/code_generator_arm64.h index d9c49d19bb..584eead81b 100644 --- a/compiler/optimizing/code_generator_arm64.h +++ b/compiler/optimizing/code_generator_arm64.h @@ -70,21 +70,32 @@ static const vixl::aarch64::FPRegister kParameterFPRegisters[] = { }; static constexpr size_t kParameterFPRegistersLength = arraysize(kParameterFPRegisters); -// Thread Register +// Thread Register. const vixl::aarch64::Register tr = vixl::aarch64::x19; +// Marking Register. +const vixl::aarch64::Register mr = vixl::aarch64::x20; // Method register on invoke. static const vixl::aarch64::Register kArtMethodRegister = vixl::aarch64::x0; const vixl::aarch64::CPURegList vixl_reserved_core_registers(vixl::aarch64::ip0, vixl::aarch64::ip1); const vixl::aarch64::CPURegList vixl_reserved_fp_registers(vixl::aarch64::d31); -const vixl::aarch64::CPURegList runtime_reserved_core_registers(tr, vixl::aarch64::lr); - -// Callee-saved registers AAPCS64 (without x19 - Thread Register) -const vixl::aarch64::CPURegList callee_saved_core_registers(vixl::aarch64::CPURegister::kRegister, - vixl::aarch64::kXRegSize, - vixl::aarch64::x20.GetCode(), - vixl::aarch64::x30.GetCode()); +const vixl::aarch64::CPURegList runtime_reserved_core_registers = + vixl::aarch64::CPURegList( + tr, + // Reserve X20 as Marking Register when emitting Baker read barriers. + ((kEmitCompilerReadBarrier && kUseBakerReadBarrier) ? mr : vixl::aarch64::NoCPUReg), + vixl::aarch64::lr); + +// Callee-save registers AAPCS64, without x19 (Thread Register) (nor +// x20 (Marking Register) when emitting Baker read barriers). +const vixl::aarch64::CPURegList callee_saved_core_registers( + vixl::aarch64::CPURegister::kRegister, + vixl::aarch64::kXRegSize, + ((kEmitCompilerReadBarrier && kUseBakerReadBarrier) + ? vixl::aarch64::x21.GetCode() + : vixl::aarch64::x20.GetCode()), + vixl::aarch64::x30.GetCode()); const vixl::aarch64::CPURegList callee_saved_fp_registers(vixl::aarch64::CPURegister::kFPRegister, vixl::aarch64::kDRegSize, vixl::aarch64::d8.GetCode(), diff --git a/compiler/optimizing/code_generator_arm_vixl.cc b/compiler/optimizing/code_generator_arm_vixl.cc index 9a2402be04..b9d4700511 100644 --- a/compiler/optimizing/code_generator_arm_vixl.cc +++ b/compiler/optimizing/code_generator_arm_vixl.cc @@ -740,7 +740,9 @@ class ArraySetSlowPathARMVIXL : public SlowPathCodeARMVIXL { // `ref`. // // Argument `entrypoint` must be a register location holding the read -// barrier marking runtime entry point to be invoked. +// barrier marking runtime entry point to be invoked or an empty +// location; in the latter case, the read barrier marking runtime +// entry point will be loaded by the slow path code itself. class ReadBarrierMarkSlowPathBaseARMVIXL : public SlowPathCodeARMVIXL { protected: ReadBarrierMarkSlowPathBaseARMVIXL(HInstruction* instruction, Location ref, Location entrypoint) @@ -786,7 +788,7 @@ class ReadBarrierMarkSlowPathBaseARMVIXL : public SlowPathCodeARMVIXL { } else { // Entrypoint is not already loaded, load from the thread. int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref_reg.GetCode()); + Thread::ReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref_reg.GetCode()); // This runtime call does not require a stack map. arm_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this); } @@ -813,9 +815,10 @@ class ReadBarrierMarkSlowPathBaseARMVIXL : public SlowPathCodeARMVIXL { // another thread, or if another thread installed another object // reference (different from `ref`) in `obj.field`). // -// If `entrypoint` is a valid location it is assumed to already be -// holding the entrypoint. The case where the entrypoint is passed in -// is when the decision to mark is based on whether the GC is marking. +// Argument `entrypoint` must be a register location holding the read +// barrier marking runtime entry point to be invoked or an empty +// location; in the latter case, the read barrier marking runtime +// entry point will be loaded by the slow path code itself. class ReadBarrierMarkSlowPathARMVIXL : public ReadBarrierMarkSlowPathBaseARMVIXL { public: ReadBarrierMarkSlowPathARMVIXL(HInstruction* instruction, @@ -861,7 +864,9 @@ class ReadBarrierMarkSlowPathARMVIXL : public ReadBarrierMarkSlowPathBaseARMVIXL // reference (different from `ref`) in `obj.field`). // // Argument `entrypoint` must be a register location holding the read -// barrier marking runtime entry point to be invoked. +// barrier marking runtime entry point to be invoked or an empty +// location; in the latter case, the read barrier marking runtime +// entry point will be loaded by the slow path code itself. class LoadReferenceWithBakerReadBarrierSlowPathARMVIXL : public ReadBarrierMarkSlowPathBaseARMVIXL { public: LoadReferenceWithBakerReadBarrierSlowPathARMVIXL(HInstruction* instruction, @@ -872,7 +877,7 @@ class LoadReferenceWithBakerReadBarrierSlowPathARMVIXL : public ReadBarrierMarkS ScaleFactor scale_factor, bool needs_null_check, vixl32::Register temp, - Location entrypoint) + Location entrypoint = Location::NoLocation()) : ReadBarrierMarkSlowPathBaseARMVIXL(instruction, ref, entrypoint), obj_(obj), offset_(offset), @@ -1006,22 +1011,24 @@ class LoadReferenceWithBakerReadBarrierSlowPathARMVIXL : public ReadBarrierMarkS // hold the same to-space reference (unless another thread installed // another object reference (different from `ref`) in `obj.field`). // -// // Argument `entrypoint` must be a register location holding the read -// barrier marking runtime entry point to be invoked. +// barrier marking runtime entry point to be invoked or an empty +// location; in the latter case, the read barrier marking runtime +// entry point will be loaded by the slow path code itself. class LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARMVIXL : public ReadBarrierMarkSlowPathBaseARMVIXL { public: - LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARMVIXL(HInstruction* instruction, - Location ref, - vixl32::Register obj, - uint32_t offset, - Location index, - ScaleFactor scale_factor, - bool needs_null_check, - vixl32::Register temp1, - vixl32::Register temp2, - Location entrypoint) + LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARMVIXL( + HInstruction* instruction, + Location ref, + vixl32::Register obj, + uint32_t offset, + Location index, + ScaleFactor scale_factor, + bool needs_null_check, + vixl32::Register temp1, + vixl32::Register temp2, + Location entrypoint = Location::NoLocation()) : ReadBarrierMarkSlowPathBaseARMVIXL(instruction, ref, entrypoint), obj_(obj), offset_(offset), @@ -1288,8 +1295,8 @@ class ReadBarrierForHeapReferenceSlowPathARMVIXL : public SlowPathCodeARMVIXL { DCHECK(locations->GetLiveRegisters()->ContainsCoreRegister(index_reg.GetCode())); if (codegen->IsCoreCalleeSaveRegister(index_reg.GetCode())) { // We are about to change the value of `index_reg` (see the - // calls to art::arm::Thumb2Assembler::Lsl and - // art::arm::Thumb2Assembler::AddConstant below), but it has + // calls to art::arm::ArmVIXLMacroAssembler::Lsl and + // art::arm::ArmVIXLMacroAssembler::Add below), but it has // not been saved by the previous call to // art::SlowPathCode::SaveLiveRegisters, as it is a // callee-save register -- @@ -2310,7 +2317,8 @@ static void GenerateConditionLong(HCondition* cond, CodeGeneratorARMVIXL* codege } } -static void GenerateConditionIntegralOrNonPrimitive(HCondition* cond, CodeGeneratorARMVIXL* codegen) { +static void GenerateConditionIntegralOrNonPrimitive(HCondition* cond, + CodeGeneratorARMVIXL* codegen) { const Primitive::Type type = cond->GetLeft()->GetType(); DCHECK(Primitive::IsIntegralType(type) || type == Primitive::kPrimNot) << type; @@ -2576,6 +2584,11 @@ void CodeGeneratorARMVIXL::SetupBlockedRegisters() const { blocked_core_registers_[LR] = true; blocked_core_registers_[PC] = true; + if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { + // Reserve marking register. + blocked_core_registers_[MR] = true; + } + // Reserve thread register. blocked_core_registers_[TR] = true; @@ -8531,20 +8544,17 @@ void InstructionCodeGeneratorARMVIXL::GenerateGcRootFieldLoad( // Baker's read barrier are used. if (kBakerReadBarrierLinkTimeThunksEnableForGcRoots && !Runtime::Current()->UseJitCompilation()) { - // Note that we do not actually check the value of `GetIsGcMarking()` - // to decide whether to mark the loaded GC root or not. Instead, we - // load into `temp` (actually kBakerCcEntrypointRegister) the read - // barrier mark introspection entrypoint. If `temp` is null, it means - // that `GetIsGcMarking()` is false, and vice versa. + // Query `art::Thread::Current()->GetIsGcMarking()` (stored in + // the Marking Register) to decide whether we need to enter + // the slow path to mark the GC root. // // We use link-time generated thunks for the slow path. That thunk // checks the reference and jumps to the entrypoint if needed. // - // temp = Thread::Current()->pReadBarrierMarkIntrospection // lr = &return_address; // GcRoot<mirror::Object> root = *(obj+offset); // Original reference load. - // if (temp != nullptr) { - // goto gc_root_thunk<root_reg>(lr) + // if (mr) { // Thread::Current()->GetIsGcMarking() + // goto gc_root_thunk<root_reg>(lr) // } // return_address: @@ -8555,18 +8565,10 @@ void InstructionCodeGeneratorARMVIXL::GenerateGcRootFieldLoad( root_reg.GetCode(), narrow); vixl32::Label* bne_label = codegen_->NewBakerReadBarrierPatch(custom_data); - // entrypoint_reg = - // Thread::Current()->pReadBarrierMarkReg12, i.e. pReadBarrierMarkIntrospection. - DCHECK_EQ(ip.GetCode(), 12u); - const int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ip.GetCode()); - __ Ldr(kBakerCcEntrypointRegister, MemOperand(tr, entry_point_offset)); - - vixl::EmissionCheckScope guard(GetVIXLAssembler(), - 4 * vixl32::kMaxInstructionSizeInBytes); + vixl::EmissionCheckScope guard(GetVIXLAssembler(), 4 * vixl32::kMaxInstructionSizeInBytes); vixl32::Label return_address; EmitAdrCode adr(GetVIXLAssembler(), lr, &return_address); - __ cmp(kBakerCcEntrypointRegister, Operand(0)); + __ cmp(mr, Operand(0)); // Currently the offset is always within range. If that changes, // we shall have to split the load the same way as for fields. DCHECK_LT(offset, kReferenceLoadMinFarOffset); @@ -8578,34 +8580,23 @@ void InstructionCodeGeneratorARMVIXL::GenerateGcRootFieldLoad( narrow ? BAKER_MARK_INTROSPECTION_GC_ROOT_LDR_NARROW_OFFSET : BAKER_MARK_INTROSPECTION_GC_ROOT_LDR_WIDE_OFFSET); } else { - // Note that we do not actually check the value of - // `GetIsGcMarking()` to decide whether to mark the loaded GC - // root or not. Instead, we load into `temp` the read barrier - // mark entry point corresponding to register `root`. If `temp` - // is null, it means that `GetIsGcMarking()` is false, and vice - // versa. + // Query `art::Thread::Current()->GetIsGcMarking()` (stored in + // the Marking Register) to decide whether we need to enter + // the slow path to mark the GC root. // - // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg() // GcRoot<mirror::Object> root = *(obj+offset); // Original reference load. - // if (temp != nullptr) { // <=> Thread::Current()->GetIsGcMarking() + // if (mr) { // Thread::Current()->GetIsGcMarking() // // Slow path. - // root = temp(root); // root = ReadBarrier::Mark(root); // Runtime entry point call. + // entrypoint = Thread::Current()->pReadBarrierMarkReg ## root.reg() + // root = entrypoint(root); // root = ReadBarrier::Mark(root); // Entry point call. // } - // Slow path marking the GC root `root`. The entrypoint will already be loaded in `temp`. - Location temp = LocationFrom(lr); + // Slow path marking the GC root `root`. The entrypoint will + // be loaded by the slow path code. SlowPathCodeARMVIXL* slow_path = - new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARMVIXL( - instruction, root, /* entrypoint */ temp); + new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARMVIXL(instruction, root); codegen_->AddSlowPath(slow_path); - // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg() - const int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(root.reg()); - // Loading the entrypoint does not require a load acquire since it is only changed when - // threads are suspended or running a checkpoint. - GetAssembler()->LoadFromOffset(kLoadWord, RegisterFrom(temp), tr, entry_point_offset); - // /* GcRoot<mirror::Object> */ root = *(obj + offset) GetAssembler()->LoadFromOffset(kLoadWord, root_reg, obj, offset); static_assert( @@ -8616,9 +8607,7 @@ void InstructionCodeGeneratorARMVIXL::GenerateGcRootFieldLoad( "art::mirror::CompressedReference<mirror::Object> and int32_t " "have different sizes."); - // The entrypoint is null when the GC is not marking, this prevents one load compared to - // checking GetIsGcMarking. - __ CompareAndBranchIfNonZero(RegisterFrom(temp), slow_path->GetEntryLabel()); + __ CompareAndBranchIfNonZero(mr, slow_path->GetEntryLabel()); __ Bind(slow_path->GetExitLabel()); } } else { @@ -8659,20 +8648,19 @@ void CodeGeneratorARMVIXL::GenerateFieldLoadWithBakerReadBarrier(HInstruction* i if (kBakerReadBarrierLinkTimeThunksEnableForFields && !Runtime::Current()->UseJitCompilation()) { - // Note that we do not actually check the value of `GetIsGcMarking()` - // to decide whether to mark the loaded reference or not. Instead, we - // load into `temp` (actually kBakerCcEntrypointRegister) the read - // barrier mark introspection entrypoint. If `temp` is null, it means - // that `GetIsGcMarking()` is false, and vice versa. + // Query `art::Thread::Current()->GetIsGcMarking()` (stored in the + // Marking Register) to decide whether we need to enter the slow + // path to mark the reference. Then, in the slow path, check the + // gray bit in the lock word of the reference's holder (`obj`) to + // decide whether to mark `ref` or not. // // We use link-time generated thunks for the slow path. That thunk checks // the holder and jumps to the entrypoint if needed. If the holder is not // gray, it creates a fake dependency and returns to the LDR instruction. // - // temp = Thread::Current()->pReadBarrierMarkIntrospection // lr = &gray_return_address; - // if (temp != nullptr) { - // goto field_thunk<holder_reg, base_reg>(lr) + // if (mr) { // Thread::Current()->GetIsGcMarking() + // goto field_thunk<holder_reg, base_reg>(lr) // } // not_gray_return_address: // // Original reference load. If the offset is too large to fit @@ -8701,19 +8689,12 @@ void CodeGeneratorARMVIXL::GenerateFieldLoadWithBakerReadBarrier(HInstruction* i base.GetCode(), obj.GetCode(), narrow); vixl32::Label* bne_label = NewBakerReadBarrierPatch(custom_data); - // entrypoint_reg = - // Thread::Current()->pReadBarrierMarkReg12, i.e. pReadBarrierMarkIntrospection. - DCHECK_EQ(ip.GetCode(), 12u); - const int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ip.GetCode()); - __ Ldr(kBakerCcEntrypointRegister, MemOperand(tr, entry_point_offset)); - vixl::EmissionCheckScope guard( GetVIXLAssembler(), (kPoisonHeapReferences ? 5u : 4u) * vixl32::kMaxInstructionSizeInBytes); vixl32::Label return_address; EmitAdrCode adr(GetVIXLAssembler(), lr, &return_address); - __ cmp(kBakerCcEntrypointRegister, Operand(0)); + __ cmp(mr, Operand(0)); EmitPlaceholderBne(this, bne_label); ptrdiff_t old_offset = GetVIXLAssembler()->GetBuffer()->GetCursorOffset(); __ ldr(EncodingSize(narrow ? Narrow : Wide), ref_reg, MemOperand(base, offset)); @@ -8760,20 +8741,19 @@ void CodeGeneratorARMVIXL::GenerateArrayLoadWithBakerReadBarrier(HInstruction* i if (kBakerReadBarrierLinkTimeThunksEnableForArrays && !Runtime::Current()->UseJitCompilation()) { - // Note that we do not actually check the value of `GetIsGcMarking()` - // to decide whether to mark the loaded reference or not. Instead, we - // load into `temp` (actually kBakerCcEntrypointRegister) the read - // barrier mark introspection entrypoint. If `temp` is null, it means - // that `GetIsGcMarking()` is false, and vice versa. + // Query `art::Thread::Current()->GetIsGcMarking()` (stored in the + // Marking Register) to decide whether we need to enter the slow + // path to mark the reference. Then, in the slow path, check the + // gray bit in the lock word of the reference's holder (`obj`) to + // decide whether to mark `ref` or not. // // We use link-time generated thunks for the slow path. That thunk checks // the holder and jumps to the entrypoint if needed. If the holder is not // gray, it creates a fake dependency and returns to the LDR instruction. // - // temp = Thread::Current()->pReadBarrierMarkIntrospection // lr = &gray_return_address; - // if (temp != nullptr) { - // goto field_thunk<holder_reg, base_reg>(lr) + // if (mr) { // Thread::Current()->GetIsGcMarking() + // goto array_thunk<base_reg>(lr) // } // not_gray_return_address: // // Original reference load. If the offset is too large to fit @@ -8793,20 +8773,13 @@ void CodeGeneratorARMVIXL::GenerateArrayLoadWithBakerReadBarrier(HInstruction* i linker::Thumb2RelativePatcher::EncodeBakerReadBarrierArrayData(data_reg.GetCode()); vixl32::Label* bne_label = NewBakerReadBarrierPatch(custom_data); - // entrypoint_reg = - // Thread::Current()->pReadBarrierMarkReg16, i.e. pReadBarrierMarkIntrospection. - DCHECK_EQ(ip.GetCode(), 12u); - const int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ip.GetCode()); - __ Ldr(kBakerCcEntrypointRegister, MemOperand(tr, entry_point_offset)); __ Add(data_reg, obj, Operand(data_offset)); - vixl::EmissionCheckScope guard( GetVIXLAssembler(), (kPoisonHeapReferences ? 5u : 4u) * vixl32::kMaxInstructionSizeInBytes); vixl32::Label return_address; EmitAdrCode adr(GetVIXLAssembler(), lr, &return_address); - __ cmp(kBakerCcEntrypointRegister, Operand(0)); + __ cmp(mr, Operand(0)); EmitPlaceholderBne(this, bne_label); ptrdiff_t old_offset = GetVIXLAssembler()->GetBuffer()->GetCursorOffset(); __ ldr(ref_reg, MemOperand(data_reg, index_reg, vixl32::LSL, scale_factor)); @@ -8838,26 +8811,21 @@ void CodeGeneratorARMVIXL::GenerateReferenceLoadWithBakerReadBarrier(HInstructio DCHECK(kEmitCompilerReadBarrier); DCHECK(kUseBakerReadBarrier); - // Query `art::Thread::Current()->GetIsGcMarking()` to decide - // whether we need to enter the slow path to mark the reference. - // Then, in the slow path, check the gray bit in the lock word of - // the reference's holder (`obj`) to decide whether to mark `ref` or - // not. + // Query `art::Thread::Current()->GetIsGcMarking()` (stored in the + // Marking Register) to decide whether we need to enter the slow + // path to mark the reference. Then, in the slow path, check the + // gray bit in the lock word of the reference's holder (`obj`) to + // decide whether to mark `ref` or not. // - // Note that we do not actually check the value of `GetIsGcMarking()`; - // instead, we load into `temp2` the read barrier mark entry point - // corresponding to register `ref`. If `temp2` is null, it means - // that `GetIsGcMarking()` is false, and vice versa. - // - // temp2 = Thread::Current()->pReadBarrierMarkReg ## root.reg() - // if (temp2 != nullptr) { // <=> Thread::Current()->GetIsGcMarking() + // if (mr) { // Thread::Current()->GetIsGcMarking() // // Slow path. // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState(); // lfence; // Load fence or artificial data dependency to prevent load-load reordering // HeapReference<mirror::Object> ref = *src; // Original reference load. // bool is_gray = (rb_state == ReadBarrier::GrayState()); // if (is_gray) { - // ref = temp2(ref); // ref = ReadBarrier::Mark(ref); // Runtime entry point call. + // entrypoint = Thread::Current()->pReadBarrierMarkReg ## root.reg() + // ref = entrypoint(ref); // ref = ReadBarrier::Mark(ref); // Runtime entry point call. // } // } else { // HeapReference<mirror::Object> ref = *src; // Original reference load. @@ -8866,30 +8834,13 @@ void CodeGeneratorARMVIXL::GenerateReferenceLoadWithBakerReadBarrier(HInstructio vixl32::Register temp_reg = RegisterFrom(temp); // Slow path marking the object `ref` when the GC is marking. The - // entrypoint will already be loaded in `temp2`. - Location temp2 = LocationFrom(lr); + // entrypoint will be loaded by the slow path code. SlowPathCodeARMVIXL* slow_path = new (GetGraph()->GetArena()) LoadReferenceWithBakerReadBarrierSlowPathARMVIXL( - instruction, - ref, - obj, - offset, - index, - scale_factor, - needs_null_check, - temp_reg, - /* entrypoint */ temp2); + instruction, ref, obj, offset, index, scale_factor, needs_null_check, temp_reg); AddSlowPath(slow_path); - // temp2 = Thread::Current()->pReadBarrierMarkReg ## ref.reg() - const int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref.reg()); - // Loading the entrypoint does not require a load acquire since it is only changed when - // threads are suspended or running a checkpoint. - GetAssembler()->LoadFromOffset(kLoadWord, RegisterFrom(temp2), tr, entry_point_offset); - // The entrypoint is null when the GC is not marking, this prevents one load compared to - // checking GetIsGcMarking. - __ CompareAndBranchIfNonZero(RegisterFrom(temp2), slow_path->GetEntryLabel()); + __ CompareAndBranchIfNonZero(mr, slow_path->GetEntryLabel()); // Fast path: the GC is not marking: just load the reference. GenerateRawReferenceLoad(instruction, ref, obj, offset, index, scale_factor, needs_null_check); __ Bind(slow_path->GetExitLabel()); @@ -8905,19 +8856,14 @@ void CodeGeneratorARMVIXL::UpdateReferenceFieldWithBakerReadBarrier(HInstruction DCHECK(kEmitCompilerReadBarrier); DCHECK(kUseBakerReadBarrier); - // Query `art::Thread::Current()->GetIsGcMarking()` to decide - // whether we need to enter the slow path to update the reference - // field within `obj`. Then, in the slow path, check the gray bit - // in the lock word of the reference's holder (`obj`) to decide - // whether to mark `ref` and update the field or not. - // - // Note that we do not actually check the value of `GetIsGcMarking()`; - // instead, we load into `temp3` the read barrier mark entry point - // corresponding to register `ref`. If `temp3` is null, it means - // that `GetIsGcMarking()` is false, and vice versa. + // Query `art::Thread::Current()->GetIsGcMarking()` (stored in the + // Marking Register) to decide whether we need to enter the slow + // path to update the reference field within `obj`. Then, in the + // slow path, check the gray bit in the lock word of the reference's + // holder (`obj`) to decide whether to mark `ref` and update the + // field or not. // - // temp3 = Thread::Current()->pReadBarrierMarkReg ## root.reg() - // if (temp3 != nullptr) { // <=> Thread::Current()->GetIsGcMarking() + // if (mr) { // Thread::Current()->GetIsGcMarking() // // Slow path. // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState(); // lfence; // Load fence or artificial data dependency to prevent load-load reordering @@ -8925,7 +8871,8 @@ void CodeGeneratorARMVIXL::UpdateReferenceFieldWithBakerReadBarrier(HInstruction // bool is_gray = (rb_state == ReadBarrier::GrayState()); // if (is_gray) { // old_ref = ref; - // ref = temp3(ref); // ref = ReadBarrier::Mark(ref); // Runtime entry point call. + // entrypoint = Thread::Current()->pReadBarrierMarkReg ## root.reg() + // ref = entrypoint(ref); // ref = ReadBarrier::Mark(ref); // Runtime entry point call. // compareAndSwapObject(obj, field_offset, old_ref, ref); // } // } @@ -8933,8 +8880,7 @@ void CodeGeneratorARMVIXL::UpdateReferenceFieldWithBakerReadBarrier(HInstruction vixl32::Register temp_reg = RegisterFrom(temp); // Slow path updating the object reference at address `obj + field_offset` - // when the GC is marking. The entrypoint will already be loaded in `temp3`. - Location temp3 = LocationFrom(lr); + // when the GC is marking. The entrypoint will be loaded by the slow path code. SlowPathCodeARMVIXL* slow_path = new (GetGraph()->GetArena()) LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARMVIXL( instruction, @@ -8945,19 +8891,10 @@ void CodeGeneratorARMVIXL::UpdateReferenceFieldWithBakerReadBarrier(HInstruction /* scale_factor */ ScaleFactor::TIMES_1, needs_null_check, temp_reg, - temp2, - /* entrypoint */ temp3); + temp2); AddSlowPath(slow_path); - // temp3 = Thread::Current()->pReadBarrierMarkReg ## ref.reg() - const int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref.reg()); - // Loading the entrypoint does not require a load acquire since it is only changed when - // threads are suspended or running a checkpoint. - GetAssembler()->LoadFromOffset(kLoadWord, RegisterFrom(temp3), tr, entry_point_offset); - // The entrypoint is null when the GC is not marking, this prevents one load compared to - // checking GetIsGcMarking. - __ CompareAndBranchIfNonZero(RegisterFrom(temp3), slow_path->GetEntryLabel()); + __ CompareAndBranchIfNonZero(mr, slow_path->GetEntryLabel()); // Fast path: the GC is not marking: nothing to do (the field is // up-to-date, and we don't need to load the reference). __ Bind(slow_path->GetExitLabel()); @@ -9057,7 +8994,7 @@ void CodeGeneratorARMVIXL::MaybeGenerateReadBarrierSlow(HInstruction* instructio Location index) { if (kEmitCompilerReadBarrier) { // Baker's read barriers shall be handled by the fast path - // (CodeGeneratorARM::GenerateReferenceLoadWithBakerReadBarrier). + // (CodeGeneratorARMVIXL::GenerateReferenceLoadWithBakerReadBarrier). DCHECK(!kUseBakerReadBarrier); // If heap poisoning is enabled, unpoisoning will be taken care of // by the runtime within the slow path. diff --git a/compiler/optimizing/code_generator_arm_vixl.h b/compiler/optimizing/code_generator_arm_vixl.h index ad3283ad4f..01cf287f29 100644 --- a/compiler/optimizing/code_generator_arm_vixl.h +++ b/compiler/optimizing/code_generator_arm_vixl.h @@ -35,13 +35,6 @@ #include "aarch32/macro-assembler-aarch32.h" #pragma GCC diagnostic pop -// Default to use the VIXL-based backend on ARM. -#ifdef ART_USE_OLD_ARM_BACKEND -static constexpr bool kArmUseVIXL32 = false; -#else -static constexpr bool kArmUseVIXL32 = true; -#endif - namespace art { namespace arm { @@ -80,12 +73,16 @@ static const vixl::aarch32::Register kMethodRegister = vixl::aarch32::r0; static const vixl::aarch32::Register kCoreAlwaysSpillRegister = vixl::aarch32::r5; -// Callee saves core registers r5, r6, r7, r8, r10, r11, and lr. +// Callee saves core registers r5, r6, r7, r8 (except when emitting Baker +// read barriers, where it is used as Marking Register), r10, r11, and lr. static const vixl::aarch32::RegisterList kCoreCalleeSaves = vixl::aarch32::RegisterList::Union( vixl::aarch32::RegisterList(vixl::aarch32::r5, vixl::aarch32::r6, - vixl::aarch32::r7, - vixl::aarch32::r8), + vixl::aarch32::r7), + // Do not consider r8 as a callee-save register with Baker read barriers. + ((kEmitCompilerReadBarrier && kUseBakerReadBarrier) + ? vixl::aarch32::RegisterList() + : vixl::aarch32::RegisterList(vixl::aarch32::r8)), vixl::aarch32::RegisterList(vixl::aarch32::r10, vixl::aarch32::r11, vixl::aarch32::lr)); diff --git a/compiler/optimizing/code_generator_mips.cc b/compiler/optimizing/code_generator_mips.cc index 4c4d97bc8d..be8f9e9cf8 100644 --- a/compiler/optimizing/code_generator_mips.cc +++ b/compiler/optimizing/code_generator_mips.cc @@ -434,10 +434,13 @@ class SuspendCheckSlowPathMIPS : public SlowPathCodeMIPS { : SlowPathCodeMIPS(instruction), successor_(successor) {} void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { + LocationSummary* locations = instruction_->GetLocations(); CodeGeneratorMIPS* mips_codegen = down_cast<CodeGeneratorMIPS*>(codegen); __ Bind(GetEntryLabel()); + SaveLiveRegisters(codegen, locations); // Only saves live vector registers for SIMD. mips_codegen->InvokeRuntime(kQuickTestSuspend, instruction_, instruction_->GetDexPc(), this); CheckEntrypointTypes<kQuickTestSuspend, void, void>(); + RestoreLiveRegisters(codegen, locations); // Only restores live vector registers for SIMD. if (successor_ == nullptr) { __ B(GetReturnLabel()); } else { @@ -653,7 +656,7 @@ class ReadBarrierMarkSlowPathMIPS : public SlowPathCodeMIPS { __ NopIfNoReordering(); } else { int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kMipsPointerSize>(ref_reg - 1); + Thread::ReadBarrierMarkEntryPointsOffset<kMipsPointerSize>(ref_reg - 1); // This runtime call does not require a stack map. mips_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, @@ -747,7 +750,7 @@ class ReadBarrierMarkAndUpdateFieldSlowPathMIPS : public SlowPathCodeMIPS { // rX <- ReadBarrierMarkRegX(rX) // int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kMipsPointerSize>(ref_reg - 1); + Thread::ReadBarrierMarkEntryPointsOffset<kMipsPointerSize>(ref_reg - 1); // This runtime call does not require a stack map. mips_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, @@ -1448,6 +1451,11 @@ void CodeGeneratorMIPS::Bind(HBasicBlock* block) { __ Bind(GetLabelOf(block)); } +VectorRegister VectorRegisterFrom(Location location) { + DCHECK(location.IsFpuRegister()); + return static_cast<VectorRegister>(location.AsFpuRegister<FRegister>()); +} + void CodeGeneratorMIPS::MoveLocation(Location destination, Location source, Primitive::Type dst_type) { @@ -1495,12 +1503,19 @@ void CodeGeneratorMIPS::MoveLocation(Location destination, __ Mtc1(src_low, dst); __ MoveToFpuHigh(src_high, dst); } else if (source.IsFpuRegister()) { - if (Primitive::Is64BitType(dst_type)) { - __ MovD(destination.AsFpuRegister<FRegister>(), source.AsFpuRegister<FRegister>()); + if (GetGraph()->HasSIMD()) { + __ MoveV(VectorRegisterFrom(destination), + VectorRegisterFrom(source)); } else { - DCHECK_EQ(dst_type, Primitive::kPrimFloat); - __ MovS(destination.AsFpuRegister<FRegister>(), source.AsFpuRegister<FRegister>()); + if (Primitive::Is64BitType(dst_type)) { + __ MovD(destination.AsFpuRegister<FRegister>(), source.AsFpuRegister<FRegister>()); + } else { + DCHECK_EQ(dst_type, Primitive::kPrimFloat); + __ MovS(destination.AsFpuRegister<FRegister>(), source.AsFpuRegister<FRegister>()); + } } + } else if (source.IsSIMDStackSlot()) { + __ LoadQFromOffset(destination.AsFpuRegister<FRegister>(), SP, source.GetStackIndex()); } else if (source.IsDoubleStackSlot()) { DCHECK(Primitive::Is64BitType(dst_type)); __ LoadDFromOffset(destination.AsFpuRegister<FRegister>(), SP, source.GetStackIndex()); @@ -1509,6 +1524,14 @@ void CodeGeneratorMIPS::MoveLocation(Location destination, DCHECK(source.IsStackSlot()) << "Cannot move from " << source << " to " << destination; __ LoadSFromOffset(destination.AsFpuRegister<FRegister>(), SP, source.GetStackIndex()); } + } else if (destination.IsSIMDStackSlot()) { + if (source.IsFpuRegister()) { + __ StoreQToOffset(source.AsFpuRegister<FRegister>(), SP, destination.GetStackIndex()); + } else { + DCHECK(source.IsSIMDStackSlot()); + __ LoadQFromOffset(FTMP, SP, source.GetStackIndex()); + __ StoreQToOffset(FTMP, SP, destination.GetStackIndex()); + } } else if (destination.IsDoubleStackSlot()) { int32_t dst_offset = destination.GetStackIndex(); if (source.IsRegisterPair()) { @@ -1875,13 +1898,21 @@ size_t CodeGeneratorMIPS::RestoreCoreRegister(size_t stack_index, uint32_t reg_i } size_t CodeGeneratorMIPS::SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) { - __ StoreDToOffset(FRegister(reg_id), SP, stack_index); - return kMipsDoublewordSize; + if (GetGraph()->HasSIMD()) { + __ StoreQToOffset(FRegister(reg_id), SP, stack_index); + } else { + __ StoreDToOffset(FRegister(reg_id), SP, stack_index); + } + return GetFloatingPointSpillSlotSize(); } size_t CodeGeneratorMIPS::RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) { - __ LoadDFromOffset(FRegister(reg_id), SP, stack_index); - return kMipsDoublewordSize; + if (GetGraph()->HasSIMD()) { + __ LoadQFromOffset(FRegister(reg_id), SP, stack_index); + } else { + __ LoadDFromOffset(FRegister(reg_id), SP, stack_index); + } + return GetFloatingPointSpillSlotSize(); } void CodeGeneratorMIPS::DumpCoreRegister(std::ostream& stream, int reg) const { @@ -6466,7 +6497,7 @@ void InstructionCodeGeneratorMIPS::GenerateGcRootFieldLoad(HInstruction* instruc // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg() const int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kMipsPointerSize>(root.reg() - 1); + Thread::ReadBarrierMarkEntryPointsOffset<kMipsPointerSize>(root.reg() - 1); // Loading the entrypoint does not require a load acquire since it is only changed when // threads are suspended or running a checkpoint. __ LoadFromOffset(kLoadWord, temp.AsRegister<Register>(), TR, entry_point_offset); @@ -8216,7 +8247,11 @@ void InstructionCodeGeneratorMIPS::VisitUnresolvedStaticFieldSet( void LocationsBuilderMIPS::VisitSuspendCheck(HSuspendCheck* instruction) { LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnSlowPath); - locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. + // In suspend check slow path, usually there are no caller-save registers at all. + // If SIMD instructions are present, however, we force spilling all live SIMD + // registers in full width (since the runtime only saves/restores lower part). + locations->SetCustomSlowPathCallerSaves( + GetGraph()->HasSIMD() ? RegisterSet::AllFpu() : RegisterSet::Empty()); } void InstructionCodeGeneratorMIPS::VisitSuspendCheck(HSuspendCheck* instruction) { diff --git a/compiler/optimizing/code_generator_mips.h b/compiler/optimizing/code_generator_mips.h index c259ea3e48..52ee852269 100644 --- a/compiler/optimizing/code_generator_mips.h +++ b/compiler/optimizing/code_generator_mips.h @@ -61,6 +61,8 @@ static constexpr FRegister kFpuCalleeSaves[] = class CodeGeneratorMIPS; +VectorRegister VectorRegisterFrom(Location location); + class InvokeDexCallingConvention : public CallingConvention<Register, FRegister> { public: InvokeDexCallingConvention() @@ -344,6 +346,10 @@ class InstructionCodeGeneratorMIPS : public InstructionCodeGenerator { uint32_t num_entries, HBasicBlock* switch_block, HBasicBlock* default_block); + + int32_t VecAddress(LocationSummary* locations, + size_t size, + /* out */ Register* adjusted_base); void GenConditionalMoveR2(HSelect* select); void GenConditionalMoveR6(HSelect* select); @@ -372,7 +378,11 @@ class CodeGeneratorMIPS : public CodeGenerator { size_t GetWordSize() const OVERRIDE { return kMipsWordSize; } - size_t GetFloatingPointSpillSlotSize() const OVERRIDE { return kMipsDoublewordSize; } + size_t GetFloatingPointSpillSlotSize() const OVERRIDE { + return GetGraph()->HasSIMD() + ? 2 * kMipsDoublewordSize // 16 bytes for each spill. + : 1 * kMipsDoublewordSize; // 8 bytes for each spill. + } uintptr_t GetAddressOf(HBasicBlock* block) OVERRIDE { return assembler_.GetLabelLocation(GetLabelOf(block)); diff --git a/compiler/optimizing/code_generator_mips64.cc b/compiler/optimizing/code_generator_mips64.cc index 5fb8755086..cf6b3d5805 100644 --- a/compiler/optimizing/code_generator_mips64.cc +++ b/compiler/optimizing/code_generator_mips64.cc @@ -606,7 +606,7 @@ class ReadBarrierMarkSlowPathMIPS64 : public SlowPathCodeMIPS64 { __ Nop(); } else { int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kMips64PointerSize>(ref_reg - 1); + Thread::ReadBarrierMarkEntryPointsOffset<kMips64PointerSize>(ref_reg - 1); // This runtime call does not require a stack map. mips64_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, @@ -699,7 +699,7 @@ class ReadBarrierMarkAndUpdateFieldSlowPathMIPS64 : public SlowPathCodeMIPS64 { // rX <- ReadBarrierMarkRegX(rX) // int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kMips64PointerSize>(ref_reg - 1); + Thread::ReadBarrierMarkEntryPointsOffset<kMips64PointerSize>(ref_reg - 1); // This runtime call does not require a stack map. mips64_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, @@ -1289,6 +1289,11 @@ void CodeGeneratorMIPS64::MoveLocation(Location destination, SP, source.GetStackIndex()); } + } else if (source.IsSIMDStackSlot()) { + __ LoadFpuFromOffset(kLoadQuadword, + destination.AsFpuRegister<FpuRegister>(), + SP, + source.GetStackIndex()); } else if (source.IsConstant()) { // Move to GPR/FPR from constant GpuRegister gpr = AT; @@ -1329,12 +1334,17 @@ void CodeGeneratorMIPS64::MoveLocation(Location destination, } } else if (source.IsFpuRegister()) { if (destination.IsFpuRegister()) { - // Move to FPR from FPR - if (dst_type == Primitive::kPrimFloat) { - __ MovS(destination.AsFpuRegister<FpuRegister>(), source.AsFpuRegister<FpuRegister>()); + if (GetGraph()->HasSIMD()) { + __ MoveV(VectorRegisterFrom(destination), + VectorRegisterFrom(source)); } else { - DCHECK_EQ(dst_type, Primitive::kPrimDouble); - __ MovD(destination.AsFpuRegister<FpuRegister>(), source.AsFpuRegister<FpuRegister>()); + // Move to FPR from FPR + if (dst_type == Primitive::kPrimFloat) { + __ MovS(destination.AsFpuRegister<FpuRegister>(), source.AsFpuRegister<FpuRegister>()); + } else { + DCHECK_EQ(dst_type, Primitive::kPrimDouble); + __ MovD(destination.AsFpuRegister<FpuRegister>(), source.AsFpuRegister<FpuRegister>()); + } } } else { DCHECK(destination.IsRegister()); @@ -1345,6 +1355,23 @@ void CodeGeneratorMIPS64::MoveLocation(Location destination, } } } + } else if (destination.IsSIMDStackSlot()) { + if (source.IsFpuRegister()) { + __ StoreFpuToOffset(kStoreQuadword, + source.AsFpuRegister<FpuRegister>(), + SP, + destination.GetStackIndex()); + } else { + DCHECK(source.IsSIMDStackSlot()); + __ LoadFpuFromOffset(kLoadQuadword, + FTMP, + SP, + source.GetStackIndex()); + __ StoreFpuToOffset(kStoreQuadword, + FTMP, + SP, + destination.GetStackIndex()); + } } else { // The destination is not a register. It must be a stack slot. DCHECK(destination.IsStackSlot() || destination.IsDoubleStackSlot()); if (source.IsRegister() || source.IsFpuRegister()) { @@ -4394,7 +4421,7 @@ void InstructionCodeGeneratorMIPS64::GenerateGcRootFieldLoad( // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg() const int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kMips64PointerSize>(root.reg() - 1); + Thread::ReadBarrierMarkEntryPointsOffset<kMips64PointerSize>(root.reg() - 1); // Loading the entrypoint does not require a load acquire since it is only changed when // threads are suspended or running a checkpoint. __ LoadFromOffset(kLoadDoubleword, temp.AsRegister<GpuRegister>(), TR, entry_point_offset); diff --git a/compiler/optimizing/code_generator_mips64.h b/compiler/optimizing/code_generator_mips64.h index b6209735b5..c94cc93dad 100644 --- a/compiler/optimizing/code_generator_mips64.h +++ b/compiler/optimizing/code_generator_mips64.h @@ -59,6 +59,8 @@ static constexpr FpuRegister kFpuCalleeSaves[] = class CodeGeneratorMIPS64; +VectorRegister VectorRegisterFrom(Location location); + class InvokeDexCallingConvention : public CallingConvention<GpuRegister, FpuRegister> { public: InvokeDexCallingConvention() diff --git a/compiler/optimizing/code_generator_vector_arm.cc b/compiler/optimizing/code_generator_vector_arm.cc deleted file mode 100644 index f8552dcfc9..0000000000 --- a/compiler/optimizing/code_generator_vector_arm.cc +++ /dev/null @@ -1,275 +0,0 @@ -/* - * Copyright (C) 2017 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "code_generator_arm.h" - -namespace art { -namespace arm { - -// NOLINT on __ macro to suppress wrong warning/fix (misc-macro-parentheses) from clang-tidy. -#define __ down_cast<ArmAssembler*>(GetAssembler())-> // NOLINT - -void LocationsBuilderARM::VisitVecReplicateScalar(HVecReplicateScalar* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); -} - -void InstructionCodeGeneratorARM::VisitVecReplicateScalar(HVecReplicateScalar* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); -} - -void LocationsBuilderARM::VisitVecSetScalars(HVecSetScalars* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); -} - -void InstructionCodeGeneratorARM::VisitVecSetScalars(HVecSetScalars* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); -} - -void LocationsBuilderARM::VisitVecSumReduce(HVecSumReduce* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); -} - -void InstructionCodeGeneratorARM::VisitVecSumReduce(HVecSumReduce* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); -} - -// Helper to set up locations for vector unary operations. -static void CreateVecUnOpLocations(ArenaAllocator* arena, HVecUnaryOperation* instruction) { - LocationSummary* locations = new (arena) LocationSummary(instruction); - switch (instruction->GetPackedType()) { - case Primitive::kPrimBoolean: - case Primitive::kPrimByte: - case Primitive::kPrimChar: - case Primitive::kPrimShort: - case Primitive::kPrimInt: - case Primitive::kPrimFloat: - case Primitive::kPrimDouble: - DCHECK(locations); - break; - default: - LOG(FATAL) << "Unsupported SIMD type"; - UNREACHABLE(); - } -} - -void LocationsBuilderARM::VisitVecCnv(HVecCnv* instruction) { - CreateVecUnOpLocations(GetGraph()->GetArena(), instruction); -} - -void InstructionCodeGeneratorARM::VisitVecCnv(HVecCnv* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); -} - -void LocationsBuilderARM::VisitVecNeg(HVecNeg* instruction) { - CreateVecUnOpLocations(GetGraph()->GetArena(), instruction); -} - -void InstructionCodeGeneratorARM::VisitVecNeg(HVecNeg* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); -} - -void LocationsBuilderARM::VisitVecAbs(HVecAbs* instruction) { - CreateVecUnOpLocations(GetGraph()->GetArena(), instruction); -} - -void InstructionCodeGeneratorARM::VisitVecAbs(HVecAbs* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); -} - -void LocationsBuilderARM::VisitVecNot(HVecNot* instruction) { - CreateVecUnOpLocations(GetGraph()->GetArena(), instruction); -} - -void InstructionCodeGeneratorARM::VisitVecNot(HVecNot* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); -} - -// Helper to set up locations for vector binary operations. -static void CreateVecBinOpLocations(ArenaAllocator* arena, HVecBinaryOperation* instruction) { - LocationSummary* locations = new (arena) LocationSummary(instruction); - switch (instruction->GetPackedType()) { - case Primitive::kPrimBoolean: - case Primitive::kPrimByte: - case Primitive::kPrimChar: - case Primitive::kPrimShort: - case Primitive::kPrimInt: - case Primitive::kPrimFloat: - case Primitive::kPrimDouble: - DCHECK(locations); - break; - default: - LOG(FATAL) << "Unsupported SIMD type"; - UNREACHABLE(); - } -} - -void LocationsBuilderARM::VisitVecAdd(HVecAdd* instruction) { - CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); -} - -void InstructionCodeGeneratorARM::VisitVecAdd(HVecAdd* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); -} - -void LocationsBuilderARM::VisitVecHalvingAdd(HVecHalvingAdd* instruction) { - CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); -} - -void InstructionCodeGeneratorARM::VisitVecHalvingAdd(HVecHalvingAdd* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); -} - -void LocationsBuilderARM::VisitVecSub(HVecSub* instruction) { - CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); -} - -void InstructionCodeGeneratorARM::VisitVecSub(HVecSub* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); -} - -void LocationsBuilderARM::VisitVecMul(HVecMul* instruction) { - CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); -} - -void InstructionCodeGeneratorARM::VisitVecMul(HVecMul* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); -} - -void LocationsBuilderARM::VisitVecDiv(HVecDiv* instruction) { - CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); -} - -void InstructionCodeGeneratorARM::VisitVecDiv(HVecDiv* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); -} - -void LocationsBuilderARM::VisitVecMin(HVecMin* instruction) { - CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); -} - -void InstructionCodeGeneratorARM::VisitVecMin(HVecMin* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); -} - -void LocationsBuilderARM::VisitVecMax(HVecMax* instruction) { - CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); -} - -void InstructionCodeGeneratorARM::VisitVecMax(HVecMax* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); -} - -void LocationsBuilderARM::VisitVecAnd(HVecAnd* instruction) { - CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); -} - -void InstructionCodeGeneratorARM::VisitVecAnd(HVecAnd* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); -} - -void LocationsBuilderARM::VisitVecAndNot(HVecAndNot* instruction) { - CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); -} - -void InstructionCodeGeneratorARM::VisitVecAndNot(HVecAndNot* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); -} - -void LocationsBuilderARM::VisitVecOr(HVecOr* instruction) { - CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); -} - -void InstructionCodeGeneratorARM::VisitVecOr(HVecOr* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); -} - -void LocationsBuilderARM::VisitVecXor(HVecXor* instruction) { - CreateVecBinOpLocations(GetGraph()->GetArena(), instruction); -} - -void InstructionCodeGeneratorARM::VisitVecXor(HVecXor* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); -} - -// Helper to set up locations for vector shift operations. -static void CreateVecShiftLocations(ArenaAllocator* arena, HVecBinaryOperation* instruction) { - LocationSummary* locations = new (arena) LocationSummary(instruction); - switch (instruction->GetPackedType()) { - case Primitive::kPrimByte: - case Primitive::kPrimChar: - case Primitive::kPrimShort: - case Primitive::kPrimInt: - case Primitive::kPrimLong: - DCHECK(locations); - break; - default: - LOG(FATAL) << "Unsupported SIMD type"; - UNREACHABLE(); - } -} - -void LocationsBuilderARM::VisitVecShl(HVecShl* instruction) { - CreateVecShiftLocations(GetGraph()->GetArena(), instruction); -} - -void InstructionCodeGeneratorARM::VisitVecShl(HVecShl* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); -} - -void LocationsBuilderARM::VisitVecShr(HVecShr* instruction) { - CreateVecShiftLocations(GetGraph()->GetArena(), instruction); -} - -void InstructionCodeGeneratorARM::VisitVecShr(HVecShr* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); -} - -void LocationsBuilderARM::VisitVecUShr(HVecUShr* instruction) { - CreateVecShiftLocations(GetGraph()->GetArena(), instruction); -} - -void InstructionCodeGeneratorARM::VisitVecUShr(HVecUShr* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); -} - -void LocationsBuilderARM::VisitVecMultiplyAccumulate(HVecMultiplyAccumulate* instr) { - LOG(FATAL) << "No SIMD for " << instr->GetId(); -} - -void InstructionCodeGeneratorARM::VisitVecMultiplyAccumulate(HVecMultiplyAccumulate* instr) { - LOG(FATAL) << "No SIMD for " << instr->GetId(); -} - -void LocationsBuilderARM::VisitVecLoad(HVecLoad* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); -} - -void InstructionCodeGeneratorARM::VisitVecLoad(HVecLoad* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); -} - -void LocationsBuilderARM::VisitVecStore(HVecStore* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); -} - -void InstructionCodeGeneratorARM::VisitVecStore(HVecStore* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); -} - -#undef __ - -} // namespace arm -} // namespace art diff --git a/compiler/optimizing/code_generator_vector_mips.cc b/compiler/optimizing/code_generator_vector_mips.cc index c4a32252d9..ea36e90112 100644 --- a/compiler/optimizing/code_generator_vector_mips.cc +++ b/compiler/optimizing/code_generator_vector_mips.cc @@ -15,6 +15,7 @@ */ #include "code_generator_mips.h" +#include "mirror/array-inl.h" namespace art { namespace mips { @@ -23,11 +24,68 @@ namespace mips { #define __ down_cast<MipsAssembler*>(GetAssembler())-> // NOLINT void LocationsBuilderMIPS::VisitVecReplicateScalar(HVecReplicateScalar* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + case Primitive::kPrimLong: + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetOut(Location::RequiresFpuRegister()); + break; + case Primitive::kPrimFloat: + case Primitive::kPrimDouble: + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } void InstructionCodeGeneratorMIPS::VisitVecReplicateScalar(HVecReplicateScalar* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + VectorRegister dst = VectorRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + DCHECK_EQ(16u, instruction->GetVectorLength()); + __ FillB(dst, locations->InAt(0).AsRegister<Register>()); + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ FillH(dst, locations->InAt(0).AsRegister<Register>()); + break; + case Primitive::kPrimInt: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ FillW(dst, locations->InAt(0).AsRegister<Register>()); + break; + case Primitive::kPrimLong: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ Mtc1(locations->InAt(0).AsRegisterPairLow<Register>(), FTMP); + __ MoveToFpuHigh(locations->InAt(0).AsRegisterPairHigh<Register>(), FTMP); + __ ReplicateFPToVectorRegister(dst, FTMP, /* is_double */ true); + break; + case Primitive::kPrimFloat: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ ReplicateFPToVectorRegister(dst, + locations->InAt(0).AsFpuRegister<FRegister>(), + /* is_double */ false); + break; + case Primitive::kPrimDouble: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ ReplicateFPToVectorRegister(dst, + locations->InAt(0).AsFpuRegister<FRegister>(), + /* is_double */ true); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } void LocationsBuilderMIPS::VisitVecSetScalars(HVecSetScalars* instruction) { @@ -51,13 +109,23 @@ static void CreateVecUnOpLocations(ArenaAllocator* arena, HVecUnaryOperation* in LocationSummary* locations = new (arena) LocationSummary(instruction); switch (instruction->GetPackedType()) { case Primitive::kPrimBoolean: + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetOut(Location::RequiresFpuRegister(), + instruction->IsVecNot() ? Location::kOutputOverlap + : Location::kNoOutputOverlap); + break; case Primitive::kPrimByte: case Primitive::kPrimChar: case Primitive::kPrimShort: case Primitive::kPrimInt: + case Primitive::kPrimLong: case Primitive::kPrimFloat: case Primitive::kPrimDouble: - DCHECK(locations); + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetOut(Location::RequiresFpuRegister(), + (instruction->IsVecNeg() || instruction->IsVecAbs()) + ? Location::kOutputOverlap + : Location::kNoOutputOverlap); break; default: LOG(FATAL) << "Unsupported SIMD type"; @@ -70,7 +138,17 @@ void LocationsBuilderMIPS::VisitVecCnv(HVecCnv* instruction) { } void InstructionCodeGeneratorMIPS::VisitVecCnv(HVecCnv* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + VectorRegister src = VectorRegisterFrom(locations->InAt(0)); + VectorRegister dst = VectorRegisterFrom(locations->Out()); + Primitive::Type from = instruction->GetInputType(); + Primitive::Type to = instruction->GetResultType(); + if (from == Primitive::kPrimInt && to == Primitive::kPrimFloat) { + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ Ffint_sW(dst, src); + } else { + LOG(FATAL) << "Unsupported SIMD type"; + } } void LocationsBuilderMIPS::VisitVecNeg(HVecNeg* instruction) { @@ -78,7 +156,45 @@ void LocationsBuilderMIPS::VisitVecNeg(HVecNeg* instruction) { } void InstructionCodeGeneratorMIPS::VisitVecNeg(HVecNeg* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + VectorRegister src = VectorRegisterFrom(locations->InAt(0)); + VectorRegister dst = VectorRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case Primitive::kPrimByte: + DCHECK_EQ(16u, instruction->GetVectorLength()); + __ FillB(dst, ZERO); + __ SubvB(dst, dst, src); + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ FillH(dst, ZERO); + __ SubvH(dst, dst, src); + break; + case Primitive::kPrimInt: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ FillW(dst, ZERO); + __ SubvW(dst, dst, src); + break; + case Primitive::kPrimLong: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ FillW(dst, ZERO); + __ SubvD(dst, dst, src); + break; + case Primitive::kPrimFloat: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ FillW(dst, ZERO); + __ FsubW(dst, dst, src); + break; + case Primitive::kPrimDouble: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ FillW(dst, ZERO); + __ FsubD(dst, dst, src); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } void LocationsBuilderMIPS::VisitVecAbs(HVecAbs* instruction) { @@ -86,7 +202,47 @@ void LocationsBuilderMIPS::VisitVecAbs(HVecAbs* instruction) { } void InstructionCodeGeneratorMIPS::VisitVecAbs(HVecAbs* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + VectorRegister src = VectorRegisterFrom(locations->InAt(0)); + VectorRegister dst = VectorRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case Primitive::kPrimByte: + DCHECK_EQ(16u, instruction->GetVectorLength()); + __ FillB(dst, ZERO); // all zeroes + __ Add_aB(dst, dst, src); // dst = abs(0) + abs(src) + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ FillH(dst, ZERO); // all zeroes + __ Add_aH(dst, dst, src); // dst = abs(0) + abs(src) + break; + case Primitive::kPrimInt: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ FillW(dst, ZERO); // all zeroes + __ Add_aW(dst, dst, src); // dst = abs(0) + abs(src) + break; + case Primitive::kPrimLong: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ FillW(dst, ZERO); // all zeroes + __ Add_aD(dst, dst, src); // dst = abs(0) + abs(src) + break; + case Primitive::kPrimFloat: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ LdiW(dst, -1); // all ones + __ SrliW(dst, dst, 1); + __ AndV(dst, dst, src); + break; + case Primitive::kPrimDouble: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ LdiD(dst, -1); // all ones + __ SrliD(dst, dst, 1); + __ AndV(dst, dst, src); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } void LocationsBuilderMIPS::VisitVecNot(HVecNot* instruction) { @@ -94,7 +250,30 @@ void LocationsBuilderMIPS::VisitVecNot(HVecNot* instruction) { } void InstructionCodeGeneratorMIPS::VisitVecNot(HVecNot* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + VectorRegister src = VectorRegisterFrom(locations->InAt(0)); + VectorRegister dst = VectorRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: // special case boolean-not + DCHECK_EQ(16u, instruction->GetVectorLength()); + __ LdiB(dst, 1); + __ XorV(dst, dst, src); + break; + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + case Primitive::kPrimLong: + case Primitive::kPrimFloat: + case Primitive::kPrimDouble: + DCHECK_LE(2u, instruction->GetVectorLength()); + DCHECK_LE(instruction->GetVectorLength(), 16u); + __ NorV(dst, src, src); // lanes do not matter + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } // Helper to set up locations for vector binary operations. @@ -106,9 +285,12 @@ static void CreateVecBinOpLocations(ArenaAllocator* arena, HVecBinaryOperation* case Primitive::kPrimChar: case Primitive::kPrimShort: case Primitive::kPrimInt: + case Primitive::kPrimLong: case Primitive::kPrimFloat: case Primitive::kPrimDouble: - DCHECK(locations); + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetInAt(1, Location::RequiresFpuRegister()); + locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); break; default: LOG(FATAL) << "Unsupported SIMD type"; @@ -121,7 +303,40 @@ void LocationsBuilderMIPS::VisitVecAdd(HVecAdd* instruction) { } void InstructionCodeGeneratorMIPS::VisitVecAdd(HVecAdd* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + VectorRegister lhs = VectorRegisterFrom(locations->InAt(0)); + VectorRegister rhs = VectorRegisterFrom(locations->InAt(1)); + VectorRegister dst = VectorRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case Primitive::kPrimByte: + DCHECK_EQ(16u, instruction->GetVectorLength()); + __ AddvB(dst, lhs, rhs); + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ AddvH(dst, lhs, rhs); + break; + case Primitive::kPrimInt: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ AddvW(dst, lhs, rhs); + break; + case Primitive::kPrimLong: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ AddvD(dst, lhs, rhs); + break; + case Primitive::kPrimFloat: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ FaddW(dst, lhs, rhs); + break; + case Primitive::kPrimDouble: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ FaddD(dst, lhs, rhs); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } void LocationsBuilderMIPS::VisitVecHalvingAdd(HVecHalvingAdd* instruction) { @@ -129,7 +344,40 @@ void LocationsBuilderMIPS::VisitVecHalvingAdd(HVecHalvingAdd* instruction) { } void InstructionCodeGeneratorMIPS::VisitVecHalvingAdd(HVecHalvingAdd* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + VectorRegister lhs = VectorRegisterFrom(locations->InAt(0)); + VectorRegister rhs = VectorRegisterFrom(locations->InAt(1)); + VectorRegister dst = VectorRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case Primitive::kPrimByte: + DCHECK_EQ(16u, instruction->GetVectorLength()); + if (instruction->IsUnsigned()) { + instruction->IsRounded() + ? __ Aver_uB(dst, lhs, rhs) + : __ Ave_uB(dst, lhs, rhs); + } else { + instruction->IsRounded() + ? __ Aver_sB(dst, lhs, rhs) + : __ Ave_sB(dst, lhs, rhs); + } + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(8u, instruction->GetVectorLength()); + if (instruction->IsUnsigned()) { + instruction->IsRounded() + ? __ Aver_uH(dst, lhs, rhs) + : __ Ave_uH(dst, lhs, rhs); + } else { + instruction->IsRounded() + ? __ Aver_sH(dst, lhs, rhs) + : __ Ave_sH(dst, lhs, rhs); + } + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } void LocationsBuilderMIPS::VisitVecSub(HVecSub* instruction) { @@ -137,7 +385,40 @@ void LocationsBuilderMIPS::VisitVecSub(HVecSub* instruction) { } void InstructionCodeGeneratorMIPS::VisitVecSub(HVecSub* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + VectorRegister lhs = VectorRegisterFrom(locations->InAt(0)); + VectorRegister rhs = VectorRegisterFrom(locations->InAt(1)); + VectorRegister dst = VectorRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case Primitive::kPrimByte: + DCHECK_EQ(16u, instruction->GetVectorLength()); + __ SubvB(dst, lhs, rhs); + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ SubvH(dst, lhs, rhs); + break; + case Primitive::kPrimInt: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ SubvW(dst, lhs, rhs); + break; + case Primitive::kPrimLong: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ SubvD(dst, lhs, rhs); + break; + case Primitive::kPrimFloat: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ FsubW(dst, lhs, rhs); + break; + case Primitive::kPrimDouble: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ FsubD(dst, lhs, rhs); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } void LocationsBuilderMIPS::VisitVecMul(HVecMul* instruction) { @@ -145,7 +426,40 @@ void LocationsBuilderMIPS::VisitVecMul(HVecMul* instruction) { } void InstructionCodeGeneratorMIPS::VisitVecMul(HVecMul* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + VectorRegister lhs = VectorRegisterFrom(locations->InAt(0)); + VectorRegister rhs = VectorRegisterFrom(locations->InAt(1)); + VectorRegister dst = VectorRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case Primitive::kPrimByte: + DCHECK_EQ(16u, instruction->GetVectorLength()); + __ MulvB(dst, lhs, rhs); + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ MulvH(dst, lhs, rhs); + break; + case Primitive::kPrimInt: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ MulvW(dst, lhs, rhs); + break; + case Primitive::kPrimLong: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ MulvD(dst, lhs, rhs); + break; + case Primitive::kPrimFloat: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ FmulW(dst, lhs, rhs); + break; + case Primitive::kPrimDouble: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ FmulD(dst, lhs, rhs); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } void LocationsBuilderMIPS::VisitVecDiv(HVecDiv* instruction) { @@ -153,7 +467,23 @@ void LocationsBuilderMIPS::VisitVecDiv(HVecDiv* instruction) { } void InstructionCodeGeneratorMIPS::VisitVecDiv(HVecDiv* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + VectorRegister lhs = VectorRegisterFrom(locations->InAt(0)); + VectorRegister rhs = VectorRegisterFrom(locations->InAt(1)); + VectorRegister dst = VectorRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case Primitive::kPrimFloat: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ FdivW(dst, lhs, rhs); + break; + case Primitive::kPrimDouble: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ FdivD(dst, lhs, rhs); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } void LocationsBuilderMIPS::VisitVecMin(HVecMin* instruction) { @@ -161,7 +491,60 @@ void LocationsBuilderMIPS::VisitVecMin(HVecMin* instruction) { } void InstructionCodeGeneratorMIPS::VisitVecMin(HVecMin* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + VectorRegister lhs = VectorRegisterFrom(locations->InAt(0)); + VectorRegister rhs = VectorRegisterFrom(locations->InAt(1)); + VectorRegister dst = VectorRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case Primitive::kPrimByte: + DCHECK_EQ(16u, instruction->GetVectorLength()); + if (instruction->IsUnsigned()) { + __ Min_uB(dst, lhs, rhs); + } else { + __ Min_sB(dst, lhs, rhs); + } + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(8u, instruction->GetVectorLength()); + if (instruction->IsUnsigned()) { + __ Min_uH(dst, lhs, rhs); + } else { + __ Min_sH(dst, lhs, rhs); + } + break; + case Primitive::kPrimInt: + DCHECK_EQ(4u, instruction->GetVectorLength()); + if (instruction->IsUnsigned()) { + __ Min_uW(dst, lhs, rhs); + } else { + __ Min_sW(dst, lhs, rhs); + } + break; + case Primitive::kPrimLong: + DCHECK_EQ(2u, instruction->GetVectorLength()); + if (instruction->IsUnsigned()) { + __ Min_uD(dst, lhs, rhs); + } else { + __ Min_sD(dst, lhs, rhs); + } + break; + // When one of arguments is NaN, fmin.df returns other argument, but Java expects a NaN value. + // TODO: Fix min(x, NaN) cases for float and double. + case Primitive::kPrimFloat: + DCHECK_EQ(4u, instruction->GetVectorLength()); + DCHECK(!instruction->IsUnsigned()); + __ FminW(dst, lhs, rhs); + break; + case Primitive::kPrimDouble: + DCHECK_EQ(2u, instruction->GetVectorLength()); + DCHECK(!instruction->IsUnsigned()); + __ FminD(dst, lhs, rhs); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } void LocationsBuilderMIPS::VisitVecMax(HVecMax* instruction) { @@ -169,7 +552,60 @@ void LocationsBuilderMIPS::VisitVecMax(HVecMax* instruction) { } void InstructionCodeGeneratorMIPS::VisitVecMax(HVecMax* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + VectorRegister lhs = VectorRegisterFrom(locations->InAt(0)); + VectorRegister rhs = VectorRegisterFrom(locations->InAt(1)); + VectorRegister dst = VectorRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case Primitive::kPrimByte: + DCHECK_EQ(16u, instruction->GetVectorLength()); + if (instruction->IsUnsigned()) { + __ Max_uB(dst, lhs, rhs); + } else { + __ Max_sB(dst, lhs, rhs); + } + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(8u, instruction->GetVectorLength()); + if (instruction->IsUnsigned()) { + __ Max_uH(dst, lhs, rhs); + } else { + __ Max_sH(dst, lhs, rhs); + } + break; + case Primitive::kPrimInt: + DCHECK_EQ(4u, instruction->GetVectorLength()); + if (instruction->IsUnsigned()) { + __ Max_uW(dst, lhs, rhs); + } else { + __ Max_sW(dst, lhs, rhs); + } + break; + case Primitive::kPrimLong: + DCHECK_EQ(2u, instruction->GetVectorLength()); + if (instruction->IsUnsigned()) { + __ Max_uD(dst, lhs, rhs); + } else { + __ Max_sD(dst, lhs, rhs); + } + break; + // When one of arguments is NaN, fmax.df returns other argument, but Java expects a NaN value. + // TODO: Fix max(x, NaN) cases for float and double. + case Primitive::kPrimFloat: + DCHECK_EQ(4u, instruction->GetVectorLength()); + DCHECK(!instruction->IsUnsigned()); + __ FmaxW(dst, lhs, rhs); + break; + case Primitive::kPrimDouble: + DCHECK_EQ(2u, instruction->GetVectorLength()); + DCHECK(!instruction->IsUnsigned()); + __ FmaxD(dst, lhs, rhs); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } void LocationsBuilderMIPS::VisitVecAnd(HVecAnd* instruction) { @@ -177,7 +613,27 @@ void LocationsBuilderMIPS::VisitVecAnd(HVecAnd* instruction) { } void InstructionCodeGeneratorMIPS::VisitVecAnd(HVecAnd* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + VectorRegister lhs = VectorRegisterFrom(locations->InAt(0)); + VectorRegister rhs = VectorRegisterFrom(locations->InAt(1)); + VectorRegister dst = VectorRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + case Primitive::kPrimLong: + case Primitive::kPrimFloat: + case Primitive::kPrimDouble: + DCHECK_LE(2u, instruction->GetVectorLength()); + DCHECK_LE(instruction->GetVectorLength(), 16u); + __ AndV(dst, lhs, rhs); // lanes do not matter + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } void LocationsBuilderMIPS::VisitVecAndNot(HVecAndNot* instruction) { @@ -193,7 +649,27 @@ void LocationsBuilderMIPS::VisitVecOr(HVecOr* instruction) { } void InstructionCodeGeneratorMIPS::VisitVecOr(HVecOr* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + VectorRegister lhs = VectorRegisterFrom(locations->InAt(0)); + VectorRegister rhs = VectorRegisterFrom(locations->InAt(1)); + VectorRegister dst = VectorRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + case Primitive::kPrimLong: + case Primitive::kPrimFloat: + case Primitive::kPrimDouble: + DCHECK_LE(2u, instruction->GetVectorLength()); + DCHECK_LE(instruction->GetVectorLength(), 16u); + __ OrV(dst, lhs, rhs); // lanes do not matter + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } void LocationsBuilderMIPS::VisitVecXor(HVecXor* instruction) { @@ -201,7 +677,27 @@ void LocationsBuilderMIPS::VisitVecXor(HVecXor* instruction) { } void InstructionCodeGeneratorMIPS::VisitVecXor(HVecXor* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + VectorRegister lhs = VectorRegisterFrom(locations->InAt(0)); + VectorRegister rhs = VectorRegisterFrom(locations->InAt(1)); + VectorRegister dst = VectorRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + case Primitive::kPrimLong: + case Primitive::kPrimFloat: + case Primitive::kPrimDouble: + DCHECK_LE(2u, instruction->GetVectorLength()); + DCHECK_LE(instruction->GetVectorLength(), 16u); + __ XorV(dst, lhs, rhs); // lanes do not matter + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } // Helper to set up locations for vector shift operations. @@ -213,7 +709,9 @@ static void CreateVecShiftLocations(ArenaAllocator* arena, HVecBinaryOperation* case Primitive::kPrimShort: case Primitive::kPrimInt: case Primitive::kPrimLong: - DCHECK(locations); + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetInAt(1, Location::ConstantLocation(instruction->InputAt(1)->AsConstant())); + locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); break; default: LOG(FATAL) << "Unsupported SIMD type"; @@ -226,7 +724,32 @@ void LocationsBuilderMIPS::VisitVecShl(HVecShl* instruction) { } void InstructionCodeGeneratorMIPS::VisitVecShl(HVecShl* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + VectorRegister lhs = VectorRegisterFrom(locations->InAt(0)); + VectorRegister dst = VectorRegisterFrom(locations->Out()); + int32_t value = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue(); + switch (instruction->GetPackedType()) { + case Primitive::kPrimByte: + DCHECK_EQ(16u, instruction->GetVectorLength()); + __ SlliB(dst, lhs, value); + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ SlliH(dst, lhs, value); + break; + case Primitive::kPrimInt: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ SlliW(dst, lhs, value); + break; + case Primitive::kPrimLong: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ SlliD(dst, lhs, value); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } void LocationsBuilderMIPS::VisitVecShr(HVecShr* instruction) { @@ -234,7 +757,32 @@ void LocationsBuilderMIPS::VisitVecShr(HVecShr* instruction) { } void InstructionCodeGeneratorMIPS::VisitVecShr(HVecShr* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + VectorRegister lhs = VectorRegisterFrom(locations->InAt(0)); + VectorRegister dst = VectorRegisterFrom(locations->Out()); + int32_t value = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue(); + switch (instruction->GetPackedType()) { + case Primitive::kPrimByte: + DCHECK_EQ(16u, instruction->GetVectorLength()); + __ SraiB(dst, lhs, value); + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ SraiH(dst, lhs, value); + break; + case Primitive::kPrimInt: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ SraiW(dst, lhs, value); + break; + case Primitive::kPrimLong: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ SraiD(dst, lhs, value); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } void LocationsBuilderMIPS::VisitVecUShr(HVecUShr* instruction) { @@ -242,7 +790,32 @@ void LocationsBuilderMIPS::VisitVecUShr(HVecUShr* instruction) { } void InstructionCodeGeneratorMIPS::VisitVecUShr(HVecUShr* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + VectorRegister lhs = VectorRegisterFrom(locations->InAt(0)); + VectorRegister dst = VectorRegisterFrom(locations->Out()); + int32_t value = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue(); + switch (instruction->GetPackedType()) { + case Primitive::kPrimByte: + DCHECK_EQ(16u, instruction->GetVectorLength()); + __ SrliB(dst, lhs, value); + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ SrliH(dst, lhs, value); + break; + case Primitive::kPrimInt: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ SrliW(dst, lhs, value); + break; + case Primitive::kPrimLong: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ SrliD(dst, lhs, value); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } void LocationsBuilderMIPS::VisitVecMultiplyAccumulate(HVecMultiplyAccumulate* instr) { @@ -253,20 +826,143 @@ void InstructionCodeGeneratorMIPS::VisitVecMultiplyAccumulate(HVecMultiplyAccumu LOG(FATAL) << "No SIMD for " << instr->GetId(); } +// Helper to set up locations for vector memory operations. +static void CreateVecMemLocations(ArenaAllocator* arena, + HVecMemoryOperation* instruction, + bool is_load) { + LocationSummary* locations = new (arena) LocationSummary(instruction); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + case Primitive::kPrimChar: + case Primitive::kPrimShort: + case Primitive::kPrimInt: + case Primitive::kPrimLong: + case Primitive::kPrimFloat: + case Primitive::kPrimDouble: + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); + if (is_load) { + locations->SetOut(Location::RequiresFpuRegister()); + } else { + locations->SetInAt(2, Location::RequiresFpuRegister()); + } + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } +} + +// Helper to prepare register and offset for vector memory operations. Returns the offset and sets +// the output parameter adjusted_base to the original base or to a reserved temporary register (AT). +int32_t InstructionCodeGeneratorMIPS::VecAddress(LocationSummary* locations, + size_t size, + /* out */ Register* adjusted_base) { + Register base = locations->InAt(0).AsRegister<Register>(); + Location index = locations->InAt(1); + int scale = TIMES_1; + switch (size) { + case 2: scale = TIMES_2; break; + case 4: scale = TIMES_4; break; + case 8: scale = TIMES_8; break; + default: break; + } + int32_t offset = mirror::Array::DataOffset(size).Int32Value(); + + if (index.IsConstant()) { + offset += index.GetConstant()->AsIntConstant()->GetValue() << scale; + __ AdjustBaseOffsetAndElementSizeShift(base, offset, scale); + *adjusted_base = base; + } else { + Register index_reg = index.AsRegister<Register>(); + if (scale != TIMES_1) { + __ Lsa(AT, index_reg, base, scale); + } else { + __ Addu(AT, base, index_reg); + } + *adjusted_base = AT; + } + return offset; +} + void LocationsBuilderMIPS::VisitVecLoad(HVecLoad* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + CreateVecMemLocations(GetGraph()->GetArena(), instruction, /* is_load */ true); } void InstructionCodeGeneratorMIPS::VisitVecLoad(HVecLoad* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + size_t size = Primitive::ComponentSize(instruction->GetPackedType()); + VectorRegister reg = VectorRegisterFrom(locations->Out()); + Register base; + int32_t offset = VecAddress(locations, size, &base); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + DCHECK_EQ(16u, instruction->GetVectorLength()); + __ LdB(reg, base, offset); + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + // Loading 8-bytes (needed if dealing with compressed strings in StringCharAt) from unaligned + // memory address may cause a trap to the kernel if the CPU doesn't directly support unaligned + // loads and stores. + // TODO: Implement support for StringCharAt. + DCHECK(!instruction->IsStringCharAt()); + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ LdH(reg, base, offset); + break; + case Primitive::kPrimInt: + case Primitive::kPrimFloat: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ LdW(reg, base, offset); + break; + case Primitive::kPrimLong: + case Primitive::kPrimDouble: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ LdD(reg, base, offset); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } void LocationsBuilderMIPS::VisitVecStore(HVecStore* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + CreateVecMemLocations(GetGraph()->GetArena(), instruction, /* is_load */ false); } void InstructionCodeGeneratorMIPS::VisitVecStore(HVecStore* instruction) { - LOG(FATAL) << "No SIMD for " << instruction->GetId(); + LocationSummary* locations = instruction->GetLocations(); + size_t size = Primitive::ComponentSize(instruction->GetPackedType()); + VectorRegister reg = VectorRegisterFrom(locations->InAt(2)); + Register base; + int32_t offset = VecAddress(locations, size, &base); + switch (instruction->GetPackedType()) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + DCHECK_EQ(16u, instruction->GetVectorLength()); + __ StB(reg, base, offset); + break; + case Primitive::kPrimChar: + case Primitive::kPrimShort: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ StH(reg, base, offset); + break; + case Primitive::kPrimInt: + case Primitive::kPrimFloat: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ StW(reg, base, offset); + break; + case Primitive::kPrimLong: + case Primitive::kPrimDouble: + DCHECK_EQ(2u, instruction->GetVectorLength()); + __ StD(reg, base, offset); + break; + default: + LOG(FATAL) << "Unsupported SIMD type"; + UNREACHABLE(); + } } #undef __ diff --git a/compiler/optimizing/code_generator_x86.cc b/compiler/optimizing/code_generator_x86.cc index 79fccfeaef..af0e6462a2 100644 --- a/compiler/optimizing/code_generator_x86.cc +++ b/compiler/optimizing/code_generator_x86.cc @@ -509,8 +509,7 @@ class ReadBarrierMarkSlowPathX86 : public SlowPathCode { // // rX <- ReadBarrierMarkRegX(rX) // - int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kX86PointerSize>(ref_reg); + int32_t entry_point_offset = Thread::ReadBarrierMarkEntryPointsOffset<kX86PointerSize>(ref_reg); // This runtime call does not require a stack map. x86_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this); __ jmp(GetExitLabel()); @@ -595,8 +594,7 @@ class ReadBarrierMarkAndUpdateFieldSlowPathX86 : public SlowPathCode { // // rX <- ReadBarrierMarkRegX(rX) // - int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kX86PointerSize>(ref_reg); + int32_t entry_point_offset = Thread::ReadBarrierMarkEntryPointsOffset<kX86PointerSize>(ref_reg); // This runtime call does not require a stack map. x86_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this); @@ -7153,7 +7151,7 @@ void InstructionCodeGeneratorX86::GenerateGcRootFieldLoad( // Test the entrypoint (`Thread::Current()->pReadBarrierMarkReg ## root.reg()`). const int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kX86PointerSize>(root.reg()); + Thread::ReadBarrierMarkEntryPointsOffset<kX86PointerSize>(root.reg()); __ fs()->cmpl(Address::Absolute(entry_point_offset), Immediate(0)); // The entrypoint is null when the GC is not marking. __ j(kNotEqual, slow_path->GetEntryLabel()); diff --git a/compiler/optimizing/code_generator_x86_64.cc b/compiler/optimizing/code_generator_x86_64.cc index 57319ce735..86f6d51734 100644 --- a/compiler/optimizing/code_generator_x86_64.cc +++ b/compiler/optimizing/code_generator_x86_64.cc @@ -524,7 +524,7 @@ class ReadBarrierMarkSlowPathX86_64 : public SlowPathCode { // rX <- ReadBarrierMarkRegX(rX) // int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kX86_64PointerSize>(ref_reg); + Thread::ReadBarrierMarkEntryPointsOffset<kX86_64PointerSize>(ref_reg); // This runtime call does not require a stack map. x86_64_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this); __ jmp(GetExitLabel()); @@ -615,7 +615,7 @@ class ReadBarrierMarkAndUpdateFieldSlowPathX86_64 : public SlowPathCode { // rX <- ReadBarrierMarkRegX(rX) // int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kX86_64PointerSize>(ref_reg); + Thread::ReadBarrierMarkEntryPointsOffset<kX86_64PointerSize>(ref_reg); // This runtime call does not require a stack map. x86_64_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this); @@ -6540,7 +6540,7 @@ void InstructionCodeGeneratorX86_64::GenerateGcRootFieldLoad( // Test the `Thread::Current()->pReadBarrierMarkReg ## root.reg()` entrypoint. const int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kX86_64PointerSize>(root.reg()); + Thread::ReadBarrierMarkEntryPointsOffset<kX86_64PointerSize>(root.reg()); __ gs()->cmpl(Address::Absolute(entry_point_offset, /* no_rip */ true), Immediate(0)); // The entrypoint is null when the GC is not marking. __ j(kNotEqual, slow_path->GetEntryLabel()); diff --git a/compiler/optimizing/codegen_test.cc b/compiler/optimizing/codegen_test.cc index fe25b7690d..0a8e97cf0d 100644 --- a/compiler/optimizing/codegen_test.cc +++ b/compiler/optimizing/codegen_test.cc @@ -43,8 +43,7 @@ static ::std::vector<CodegenTargetConfig> GetTargetConfigs() { ::std::vector<CodegenTargetConfig> v; ::std::vector<CodegenTargetConfig> test_config_candidates = { #ifdef ART_ENABLE_CODEGEN_arm - CodegenTargetConfig(kArm, create_codegen_arm), - CodegenTargetConfig(kThumb2, create_codegen_arm), + // TODO: Should't this be `kThumb2` instead of `kArm` here? CodegenTargetConfig(kArm, create_codegen_arm_vixl32), #endif #ifdef ART_ENABLE_CODEGEN_arm64 diff --git a/compiler/optimizing/codegen_test_utils.h b/compiler/optimizing/codegen_test_utils.h index 00a16fe849..1b38acd8b0 100644 --- a/compiler/optimizing/codegen_test_utils.h +++ b/compiler/optimizing/codegen_test_utils.h @@ -35,7 +35,6 @@ #include "ssa_liveness_analysis.h" #ifdef ART_ENABLE_CODEGEN_arm -#include "code_generator_arm.h" #include "code_generator_arm_vixl.h" #endif @@ -84,26 +83,6 @@ class CodegenTargetConfig { // in ART, and callee-save in C. Alternatively, we could use or write // the stub that saves and restores all registers, but it is easier // to just overwrite the code generator. -class TestCodeGeneratorARM : public arm::CodeGeneratorARM { - public: - TestCodeGeneratorARM(HGraph* graph, - const ArmInstructionSetFeatures& isa_features, - const CompilerOptions& compiler_options) - : arm::CodeGeneratorARM(graph, isa_features, compiler_options) { - AddAllocatedRegister(Location::RegisterLocation(arm::R6)); - AddAllocatedRegister(Location::RegisterLocation(arm::R7)); - } - - void SetupBlockedRegisters() const OVERRIDE { - arm::CodeGeneratorARM::SetupBlockedRegisters(); - blocked_core_registers_[arm::R4] = true; - blocked_core_registers_[arm::R6] = false; - blocked_core_registers_[arm::R7] = false; - } -}; - -// A way to test the VIXL32-based code generator on ARM. This will replace -// TestCodeGeneratorARM when the VIXL32-based backend replaces the existing one. class TestCodeGeneratorARMVIXL : public arm::CodeGeneratorARMVIXL { public: TestCodeGeneratorARMVIXL(HGraph* graph, @@ -288,14 +267,6 @@ static void RunCode(CodegenTargetConfig target_config, } #ifdef ART_ENABLE_CODEGEN_arm -CodeGenerator* create_codegen_arm(HGraph* graph, const CompilerOptions& compiler_options) { - std::unique_ptr<const ArmInstructionSetFeatures> features_arm( - ArmInstructionSetFeatures::FromCppDefines()); - return new (graph->GetArena()) TestCodeGeneratorARM(graph, - *features_arm.get(), - compiler_options); -} - CodeGenerator* create_codegen_arm_vixl32(HGraph* graph, const CompilerOptions& compiler_options) { std::unique_ptr<const ArmInstructionSetFeatures> features_arm( ArmInstructionSetFeatures::FromCppDefines()); diff --git a/compiler/optimizing/instruction_builder.cc b/compiler/optimizing/instruction_builder.cc index a73b1246d8..839f328a4f 100644 --- a/compiler/optimizing/instruction_builder.cc +++ b/compiler/optimizing/instruction_builder.cc @@ -22,6 +22,7 @@ #include "dex_instruction-inl.h" #include "driver/compiler_options.h" #include "imtable-inl.h" +#include "quicken_info.h" #include "sharpening.h" #include "scoped_thread_state_change-inl.h" @@ -312,6 +313,11 @@ bool HInstructionBuilder::Build() { DCHECK(!IsBlockPopulated(current_block_)); + uint32_t quicken_index = 0; + if (CanDecodeQuickenedInfo()) { + quicken_index = block_builder_->GetQuickenIndex(block_dex_pc); + } + for (CodeItemIterator it(code_item_, block_dex_pc); !it.Done(); it.Advance()) { if (current_block_ == nullptr) { // The previous instruction ended this block. @@ -332,9 +338,13 @@ bool HInstructionBuilder::Build() { AppendInstruction(new (arena_) HNativeDebugInfo(dex_pc)); } - if (!ProcessDexInstruction(it.CurrentInstruction(), dex_pc)) { + if (!ProcessDexInstruction(it.CurrentInstruction(), dex_pc, quicken_index)) { return false; } + + if (QuickenInfoTable::NeedsIndexForInstruction(&it.CurrentInstruction())) { + ++quicken_index; + } } if (current_block_ != nullptr) { @@ -1261,7 +1271,8 @@ static Primitive::Type GetFieldAccessType(const DexFile& dex_file, uint16_t fiel bool HInstructionBuilder::BuildInstanceFieldAccess(const Instruction& instruction, uint32_t dex_pc, - bool is_put) { + bool is_put, + size_t quicken_index) { uint32_t source_or_dest_reg = instruction.VRegA_22c(); uint32_t obj_reg = instruction.VRegB_22c(); uint16_t field_index; @@ -1269,7 +1280,7 @@ bool HInstructionBuilder::BuildInstanceFieldAccess(const Instruction& instructio if (!CanDecodeQuickenedInfo()) { return false; } - field_index = LookupQuickenedInfo(dex_pc); + field_index = LookupQuickenedInfo(quicken_index); } else { field_index = instruction.VRegC_22c(); } @@ -1805,40 +1816,17 @@ bool HInstructionBuilder::NeedsAccessCheck(dex::TypeIndex type_index, bool* fina } bool HInstructionBuilder::CanDecodeQuickenedInfo() const { - return interpreter_metadata_ != nullptr; + return !quicken_info_.IsNull(); } -uint16_t HInstructionBuilder::LookupQuickenedInfo(uint32_t dex_pc) { - DCHECK(interpreter_metadata_ != nullptr); - - // First check if the info has already been decoded from `interpreter_metadata_`. - auto it = skipped_interpreter_metadata_.find(dex_pc); - if (it != skipped_interpreter_metadata_.end()) { - // Remove the entry from the map and return the parsed info. - uint16_t value_in_map = it->second; - skipped_interpreter_metadata_.erase(it); - return value_in_map; - } - - // Otherwise start parsing `interpreter_metadata_` until the slot for `dex_pc` - // is found. Store skipped values in the `skipped_interpreter_metadata_` map. - while (true) { - uint32_t dex_pc_in_map = DecodeUnsignedLeb128(&interpreter_metadata_); - uint16_t value_in_map = DecodeUnsignedLeb128(&interpreter_metadata_); - DCHECK_LE(dex_pc_in_map, dex_pc); - - if (dex_pc_in_map == dex_pc) { - return value_in_map; - } else { - // Overwrite and not Put, as quickened CHECK-CAST has two entries with - // the same dex_pc. This is OK, because the compiler does not care about those - // entries. - skipped_interpreter_metadata_.Overwrite(dex_pc_in_map, value_in_map); - } - } +uint16_t HInstructionBuilder::LookupQuickenedInfo(uint32_t quicken_index) { + DCHECK(CanDecodeQuickenedInfo()); + return quicken_info_.GetData(quicken_index); } -bool HInstructionBuilder::ProcessDexInstruction(const Instruction& instruction, uint32_t dex_pc) { +bool HInstructionBuilder::ProcessDexInstruction(const Instruction& instruction, + uint32_t dex_pc, + size_t quicken_index) { switch (instruction.Opcode()) { case Instruction::CONST_4: { int32_t register_index = instruction.VRegA(); @@ -1995,7 +1983,7 @@ bool HInstructionBuilder::ProcessDexInstruction(const Instruction& instruction, if (!CanDecodeQuickenedInfo()) { return false; } - method_idx = LookupQuickenedInfo(dex_pc); + method_idx = LookupQuickenedInfo(quicken_index); } else { method_idx = instruction.VRegB_35c(); } @@ -2020,7 +2008,7 @@ bool HInstructionBuilder::ProcessDexInstruction(const Instruction& instruction, if (!CanDecodeQuickenedInfo()) { return false; } - method_idx = LookupQuickenedInfo(dex_pc); + method_idx = LookupQuickenedInfo(quicken_index); } else { method_idx = instruction.VRegB_3rc(); } @@ -2693,7 +2681,7 @@ bool HInstructionBuilder::ProcessDexInstruction(const Instruction& instruction, case Instruction::IGET_CHAR_QUICK: case Instruction::IGET_SHORT: case Instruction::IGET_SHORT_QUICK: { - if (!BuildInstanceFieldAccess(instruction, dex_pc, false)) { + if (!BuildInstanceFieldAccess(instruction, dex_pc, false, quicken_index)) { return false; } break; @@ -2713,7 +2701,7 @@ bool HInstructionBuilder::ProcessDexInstruction(const Instruction& instruction, case Instruction::IPUT_CHAR_QUICK: case Instruction::IPUT_SHORT: case Instruction::IPUT_SHORT_QUICK: { - if (!BuildInstanceFieldAccess(instruction, dex_pc, true)) { + if (!BuildInstanceFieldAccess(instruction, dex_pc, true, quicken_index)) { return false; } break; diff --git a/compiler/optimizing/instruction_builder.h b/compiler/optimizing/instruction_builder.h index e968760d84..5a83df3813 100644 --- a/compiler/optimizing/instruction_builder.h +++ b/compiler/optimizing/instruction_builder.h @@ -27,6 +27,7 @@ #include "mirror/dex_cache.h" #include "nodes.h" #include "optimizing_compiler_stats.h" +#include "quicken_info.h" #include "ssa_builder.h" namespace art { @@ -67,9 +68,7 @@ class HInstructionBuilder : public ValueObject { code_generator_(code_generator), dex_compilation_unit_(dex_compilation_unit), outer_compilation_unit_(outer_compilation_unit), - interpreter_metadata_(interpreter_metadata), - skipped_interpreter_metadata_(std::less<uint32_t>(), - arena_->Adapter(kArenaAllocGraphBuilder)), + quicken_info_(interpreter_metadata), compilation_stats_(compiler_stats), dex_cache_(dex_cache), loop_headers_(graph->GetArena()->Adapter(kArenaAllocGraphBuilder)) { @@ -85,11 +84,11 @@ class HInstructionBuilder : public ValueObject { void PropagateLocalsToCatchBlocks(); void SetLoopHeaderPhiInputs(); - bool ProcessDexInstruction(const Instruction& instruction, uint32_t dex_pc); + bool ProcessDexInstruction(const Instruction& instruction, uint32_t dex_pc, size_t quicken_index); void FindNativeDebugInfoLocations(ArenaBitVector* locations); bool CanDecodeQuickenedInfo() const; - uint16_t LookupQuickenedInfo(uint32_t dex_pc); + uint16_t LookupQuickenedInfo(uint32_t quicken_index); HBasicBlock* FindBlockStartingAt(uint32_t dex_pc) const; @@ -159,7 +158,10 @@ class HInstructionBuilder : public ValueObject { void BuildReturn(const Instruction& instruction, Primitive::Type type, uint32_t dex_pc); // Builds an instance field access node and returns whether the instruction is supported. - bool BuildInstanceFieldAccess(const Instruction& instruction, uint32_t dex_pc, bool is_put); + bool BuildInstanceFieldAccess(const Instruction& instruction, + uint32_t dex_pc, + bool is_put, + size_t quicken_index); void BuildUnresolvedStaticFieldAccess(const Instruction& instruction, uint32_t dex_pc, @@ -349,14 +351,8 @@ class HInstructionBuilder : public ValueObject { // methods. const DexCompilationUnit* const outer_compilation_unit_; - // Original values kept after instruction quickening. This is a data buffer - // of Leb128-encoded (dex_pc, value) pairs sorted by dex_pc. - const uint8_t* interpreter_metadata_; - - // InstructionBuilder does not parse instructions in dex_pc order. Quickening - // info for out-of-order dex_pcs is stored in a map until the positions - // are eventually visited. - ArenaSafeMap<uint32_t, uint16_t> skipped_interpreter_metadata_; + // Original values kept after instruction quickening. + QuickenInfoTable quicken_info_; OptimizingCompilerStats* compilation_stats_; Handle<mirror::DexCache> dex_cache_; diff --git a/compiler/optimizing/instruction_simplifier_shared.cc b/compiler/optimizing/instruction_simplifier_shared.cc index e5a8499ff4..d1bc4dadeb 100644 --- a/compiler/optimizing/instruction_simplifier_shared.cc +++ b/compiler/optimizing/instruction_simplifier_shared.cc @@ -274,8 +274,8 @@ bool TryExtractArrayAccessAddress(HInstruction* access, // `HArm64Load` and `HArm64Store`,`HArmLoad` and `HArmStore`). We defer these changes // because these new instructions would not bring any advantages yet. // Also see the comments in - // `InstructionCodeGeneratorARM::VisitArrayGet()` - // `InstructionCodeGeneratorARM::VisitArraySet()` + // `InstructionCodeGeneratorARMVIXL::VisitArrayGet()` + // `InstructionCodeGeneratorARMVIXL::VisitArraySet()` // `InstructionCodeGeneratorARM64::VisitArrayGet()` // `InstructionCodeGeneratorARM64::VisitArraySet()`. return true; diff --git a/compiler/optimizing/intrinsics_arm.cc b/compiler/optimizing/intrinsics_arm.cc deleted file mode 100644 index ae5f8d1760..0000000000 --- a/compiler/optimizing/intrinsics_arm.cc +++ /dev/null @@ -1,2760 +0,0 @@ -/* - * Copyright (C) 2015 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "intrinsics_arm.h" - -#include "arch/arm/instruction_set_features_arm.h" -#include "art_method.h" -#include "code_generator_arm.h" -#include "entrypoints/quick/quick_entrypoints.h" -#include "intrinsics.h" -#include "intrinsics_utils.h" -#include "lock_word.h" -#include "mirror/array-inl.h" -#include "mirror/object_array-inl.h" -#include "mirror/reference.h" -#include "mirror/string.h" -#include "scoped_thread_state_change-inl.h" -#include "thread-current-inl.h" -#include "utils/arm/assembler_arm.h" - -namespace art { - -namespace arm { - -ArmAssembler* IntrinsicCodeGeneratorARM::GetAssembler() { - return codegen_->GetAssembler(); -} - -ArenaAllocator* IntrinsicCodeGeneratorARM::GetAllocator() { - return codegen_->GetGraph()->GetArena(); -} - -using IntrinsicSlowPathARM = IntrinsicSlowPath<InvokeDexCallingConventionVisitorARM>; - -#define __ assembler-> - -// Compute base address for the System.arraycopy intrinsic in `base`. -static void GenSystemArrayCopyBaseAddress(ArmAssembler* assembler, - Primitive::Type type, - const Register& array, - const Location& pos, - const Register& base) { - // This routine is only used by the SystemArrayCopy intrinsic at the - // moment. We can allow Primitive::kPrimNot as `type` to implement - // the SystemArrayCopyChar intrinsic. - DCHECK_EQ(type, Primitive::kPrimNot); - const int32_t element_size = Primitive::ComponentSize(type); - const uint32_t element_size_shift = Primitive::ComponentSizeShift(type); - const uint32_t data_offset = mirror::Array::DataOffset(element_size).Uint32Value(); - - if (pos.IsConstant()) { - int32_t constant = pos.GetConstant()->AsIntConstant()->GetValue(); - __ AddConstant(base, array, element_size * constant + data_offset); - } else { - __ add(base, array, ShifterOperand(pos.AsRegister<Register>(), LSL, element_size_shift)); - __ AddConstant(base, data_offset); - } -} - -// Compute end address for the System.arraycopy intrinsic in `end`. -static void GenSystemArrayCopyEndAddress(ArmAssembler* assembler, - Primitive::Type type, - const Location& copy_length, - const Register& base, - const Register& end) { - // This routine is only used by the SystemArrayCopy intrinsic at the - // moment. We can allow Primitive::kPrimNot as `type` to implement - // the SystemArrayCopyChar intrinsic. - DCHECK_EQ(type, Primitive::kPrimNot); - const int32_t element_size = Primitive::ComponentSize(type); - const uint32_t element_size_shift = Primitive::ComponentSizeShift(type); - - if (copy_length.IsConstant()) { - int32_t constant = copy_length.GetConstant()->AsIntConstant()->GetValue(); - __ AddConstant(end, base, element_size * constant); - } else { - __ add(end, base, ShifterOperand(copy_length.AsRegister<Register>(), LSL, element_size_shift)); - } -} - -#undef __ - -// NOLINT on __ macro to suppress wrong warning/fix (misc-macro-parentheses) from clang-tidy. -#define __ down_cast<ArmAssembler*>(codegen->GetAssembler())-> // NOLINT - -// Slow path implementing the SystemArrayCopy intrinsic copy loop with read barriers. -class ReadBarrierSystemArrayCopySlowPathARM : public SlowPathCode { - public: - explicit ReadBarrierSystemArrayCopySlowPathARM(HInstruction* instruction) - : SlowPathCode(instruction) { - DCHECK(kEmitCompilerReadBarrier); - DCHECK(kUseBakerReadBarrier); - } - - void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { - CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); - ArmAssembler* assembler = arm_codegen->GetAssembler(); - LocationSummary* locations = instruction_->GetLocations(); - DCHECK(locations->CanCall()); - DCHECK(instruction_->IsInvokeStaticOrDirect()) - << "Unexpected instruction in read barrier arraycopy slow path: " - << instruction_->DebugName(); - DCHECK(instruction_->GetLocations()->Intrinsified()); - DCHECK_EQ(instruction_->AsInvoke()->GetIntrinsic(), Intrinsics::kSystemArrayCopy); - - Primitive::Type type = Primitive::kPrimNot; - const int32_t element_size = Primitive::ComponentSize(type); - - Register dest = locations->InAt(2).AsRegister<Register>(); - Location dest_pos = locations->InAt(3); - Register src_curr_addr = locations->GetTemp(0).AsRegister<Register>(); - Register dst_curr_addr = locations->GetTemp(1).AsRegister<Register>(); - Register src_stop_addr = locations->GetTemp(2).AsRegister<Register>(); - Register tmp = locations->GetTemp(3).AsRegister<Register>(); - - __ Bind(GetEntryLabel()); - // Compute the base destination address in `dst_curr_addr`. - GenSystemArrayCopyBaseAddress(assembler, type, dest, dest_pos, dst_curr_addr); - - Label loop; - __ Bind(&loop); - __ ldr(tmp, Address(src_curr_addr, element_size, Address::PostIndex)); - __ MaybeUnpoisonHeapReference(tmp); - // TODO: Inline the mark bit check before calling the runtime? - // tmp = ReadBarrier::Mark(tmp); - // No need to save live registers; it's taken care of by the - // entrypoint. Also, there is no need to update the stack mask, - // as this runtime call will not trigger a garbage collection. - // (See ReadBarrierMarkSlowPathARM::EmitNativeCode for more - // explanations.) - DCHECK_NE(tmp, SP); - DCHECK_NE(tmp, LR); - DCHECK_NE(tmp, PC); - // IP is used internally by the ReadBarrierMarkRegX entry point - // as a temporary (and not preserved). It thus cannot be used by - // any live register in this slow path. - DCHECK_NE(src_curr_addr, IP); - DCHECK_NE(dst_curr_addr, IP); - DCHECK_NE(src_stop_addr, IP); - DCHECK_NE(tmp, IP); - DCHECK(0 <= tmp && tmp < kNumberOfCoreRegisters) << tmp; - // TODO: Load the entrypoint once before the loop, instead of - // loading it at every iteration. - int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(tmp); - // This runtime call does not require a stack map. - arm_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this); - __ MaybePoisonHeapReference(tmp); - __ str(tmp, Address(dst_curr_addr, element_size, Address::PostIndex)); - __ cmp(src_curr_addr, ShifterOperand(src_stop_addr)); - __ b(&loop, NE); - __ b(GetExitLabel()); - } - - const char* GetDescription() const OVERRIDE { return "ReadBarrierSystemArrayCopySlowPathARM"; } - - private: - DISALLOW_COPY_AND_ASSIGN(ReadBarrierSystemArrayCopySlowPathARM); -}; - -#undef __ - -IntrinsicLocationsBuilderARM::IntrinsicLocationsBuilderARM(CodeGeneratorARM* codegen) - : arena_(codegen->GetGraph()->GetArena()), - codegen_(codegen), - assembler_(codegen->GetAssembler()), - features_(codegen->GetInstructionSetFeatures()) {} - -bool IntrinsicLocationsBuilderARM::TryDispatch(HInvoke* invoke) { - Dispatch(invoke); - LocationSummary* res = invoke->GetLocations(); - if (res == nullptr) { - return false; - } - return res->Intrinsified(); -} - -#define __ assembler-> - -static void CreateFPToIntLocations(ArenaAllocator* arena, HInvoke* invoke) { - LocationSummary* locations = new (arena) LocationSummary(invoke, - LocationSummary::kNoCall, - kIntrinsified); - locations->SetInAt(0, Location::RequiresFpuRegister()); - locations->SetOut(Location::RequiresRegister()); -} - -static void CreateIntToFPLocations(ArenaAllocator* arena, HInvoke* invoke) { - LocationSummary* locations = new (arena) LocationSummary(invoke, - LocationSummary::kNoCall, - kIntrinsified); - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetOut(Location::RequiresFpuRegister()); -} - -static void MoveFPToInt(LocationSummary* locations, bool is64bit, ArmAssembler* assembler) { - Location input = locations->InAt(0); - Location output = locations->Out(); - if (is64bit) { - __ vmovrrd(output.AsRegisterPairLow<Register>(), - output.AsRegisterPairHigh<Register>(), - FromLowSToD(input.AsFpuRegisterPairLow<SRegister>())); - } else { - __ vmovrs(output.AsRegister<Register>(), input.AsFpuRegister<SRegister>()); - } -} - -static void MoveIntToFP(LocationSummary* locations, bool is64bit, ArmAssembler* assembler) { - Location input = locations->InAt(0); - Location output = locations->Out(); - if (is64bit) { - __ vmovdrr(FromLowSToD(output.AsFpuRegisterPairLow<SRegister>()), - input.AsRegisterPairLow<Register>(), - input.AsRegisterPairHigh<Register>()); - } else { - __ vmovsr(output.AsFpuRegister<SRegister>(), input.AsRegister<Register>()); - } -} - -void IntrinsicLocationsBuilderARM::VisitDoubleDoubleToRawLongBits(HInvoke* invoke) { - CreateFPToIntLocations(arena_, invoke); -} -void IntrinsicLocationsBuilderARM::VisitDoubleLongBitsToDouble(HInvoke* invoke) { - CreateIntToFPLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitDoubleDoubleToRawLongBits(HInvoke* invoke) { - MoveFPToInt(invoke->GetLocations(), /* is64bit */ true, GetAssembler()); -} -void IntrinsicCodeGeneratorARM::VisitDoubleLongBitsToDouble(HInvoke* invoke) { - MoveIntToFP(invoke->GetLocations(), /* is64bit */ true, GetAssembler()); -} - -void IntrinsicLocationsBuilderARM::VisitFloatFloatToRawIntBits(HInvoke* invoke) { - CreateFPToIntLocations(arena_, invoke); -} -void IntrinsicLocationsBuilderARM::VisitFloatIntBitsToFloat(HInvoke* invoke) { - CreateIntToFPLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitFloatFloatToRawIntBits(HInvoke* invoke) { - MoveFPToInt(invoke->GetLocations(), /* is64bit */ false, GetAssembler()); -} -void IntrinsicCodeGeneratorARM::VisitFloatIntBitsToFloat(HInvoke* invoke) { - MoveIntToFP(invoke->GetLocations(), /* is64bit */ false, GetAssembler()); -} - -static void CreateIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) { - LocationSummary* locations = new (arena) LocationSummary(invoke, - LocationSummary::kNoCall, - kIntrinsified); - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); -} - -static void CreateFPToFPLocations(ArenaAllocator* arena, HInvoke* invoke) { - LocationSummary* locations = new (arena) LocationSummary(invoke, - LocationSummary::kNoCall, - kIntrinsified); - locations->SetInAt(0, Location::RequiresFpuRegister()); - locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); -} - -static void GenNumberOfLeadingZeros(HInvoke* invoke, - Primitive::Type type, - CodeGeneratorARM* codegen) { - ArmAssembler* assembler = codegen->GetAssembler(); - LocationSummary* locations = invoke->GetLocations(); - Location in = locations->InAt(0); - Register out = locations->Out().AsRegister<Register>(); - - DCHECK((type == Primitive::kPrimInt) || (type == Primitive::kPrimLong)); - - if (type == Primitive::kPrimLong) { - Register in_reg_lo = in.AsRegisterPairLow<Register>(); - Register in_reg_hi = in.AsRegisterPairHigh<Register>(); - Label end; - Label* final_label = codegen->GetFinalLabel(invoke, &end); - __ clz(out, in_reg_hi); - __ CompareAndBranchIfNonZero(in_reg_hi, final_label); - __ clz(out, in_reg_lo); - __ AddConstant(out, 32); - if (end.IsLinked()) { - __ Bind(&end); - } - } else { - __ clz(out, in.AsRegister<Register>()); - } -} - -void IntrinsicLocationsBuilderARM::VisitIntegerNumberOfLeadingZeros(HInvoke* invoke) { - CreateIntToIntLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitIntegerNumberOfLeadingZeros(HInvoke* invoke) { - GenNumberOfLeadingZeros(invoke, Primitive::kPrimInt, codegen_); -} - -void IntrinsicLocationsBuilderARM::VisitLongNumberOfLeadingZeros(HInvoke* invoke) { - LocationSummary* locations = new (arena_) LocationSummary(invoke, - LocationSummary::kNoCall, - kIntrinsified); - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); -} - -void IntrinsicCodeGeneratorARM::VisitLongNumberOfLeadingZeros(HInvoke* invoke) { - GenNumberOfLeadingZeros(invoke, Primitive::kPrimLong, codegen_); -} - -static void GenNumberOfTrailingZeros(HInvoke* invoke, - Primitive::Type type, - CodeGeneratorARM* codegen) { - DCHECK((type == Primitive::kPrimInt) || (type == Primitive::kPrimLong)); - - ArmAssembler* assembler = codegen->GetAssembler(); - LocationSummary* locations = invoke->GetLocations(); - Register out = locations->Out().AsRegister<Register>(); - - if (type == Primitive::kPrimLong) { - Register in_reg_lo = locations->InAt(0).AsRegisterPairLow<Register>(); - Register in_reg_hi = locations->InAt(0).AsRegisterPairHigh<Register>(); - Label end; - Label* final_label = codegen->GetFinalLabel(invoke, &end); - __ rbit(out, in_reg_lo); - __ clz(out, out); - __ CompareAndBranchIfNonZero(in_reg_lo, final_label); - __ rbit(out, in_reg_hi); - __ clz(out, out); - __ AddConstant(out, 32); - if (end.IsLinked()) { - __ Bind(&end); - } - } else { - Register in = locations->InAt(0).AsRegister<Register>(); - __ rbit(out, in); - __ clz(out, out); - } -} - -void IntrinsicLocationsBuilderARM::VisitIntegerNumberOfTrailingZeros(HInvoke* invoke) { - LocationSummary* locations = new (arena_) LocationSummary(invoke, - LocationSummary::kNoCall, - kIntrinsified); - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); -} - -void IntrinsicCodeGeneratorARM::VisitIntegerNumberOfTrailingZeros(HInvoke* invoke) { - GenNumberOfTrailingZeros(invoke, Primitive::kPrimInt, codegen_); -} - -void IntrinsicLocationsBuilderARM::VisitLongNumberOfTrailingZeros(HInvoke* invoke) { - LocationSummary* locations = new (arena_) LocationSummary(invoke, - LocationSummary::kNoCall, - kIntrinsified); - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); -} - -void IntrinsicCodeGeneratorARM::VisitLongNumberOfTrailingZeros(HInvoke* invoke) { - GenNumberOfTrailingZeros(invoke, Primitive::kPrimLong, codegen_); -} - -static void MathAbsFP(LocationSummary* locations, bool is64bit, ArmAssembler* assembler) { - Location in = locations->InAt(0); - Location out = locations->Out(); - - if (is64bit) { - __ vabsd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), - FromLowSToD(in.AsFpuRegisterPairLow<SRegister>())); - } else { - __ vabss(out.AsFpuRegister<SRegister>(), in.AsFpuRegister<SRegister>()); - } -} - -void IntrinsicLocationsBuilderARM::VisitMathAbsDouble(HInvoke* invoke) { - CreateFPToFPLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMathAbsDouble(HInvoke* invoke) { - MathAbsFP(invoke->GetLocations(), /* is64bit */ true, GetAssembler()); -} - -void IntrinsicLocationsBuilderARM::VisitMathAbsFloat(HInvoke* invoke) { - CreateFPToFPLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMathAbsFloat(HInvoke* invoke) { - MathAbsFP(invoke->GetLocations(), /* is64bit */ false, GetAssembler()); -} - -static void CreateIntToIntPlusTemp(ArenaAllocator* arena, HInvoke* invoke) { - LocationSummary* locations = new (arena) LocationSummary(invoke, - LocationSummary::kNoCall, - kIntrinsified); - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); - - locations->AddTemp(Location::RequiresRegister()); -} - -static void GenAbsInteger(LocationSummary* locations, - bool is64bit, - ArmAssembler* assembler) { - Location in = locations->InAt(0); - Location output = locations->Out(); - - Register mask = locations->GetTemp(0).AsRegister<Register>(); - - if (is64bit) { - Register in_reg_lo = in.AsRegisterPairLow<Register>(); - Register in_reg_hi = in.AsRegisterPairHigh<Register>(); - Register out_reg_lo = output.AsRegisterPairLow<Register>(); - Register out_reg_hi = output.AsRegisterPairHigh<Register>(); - - DCHECK_NE(out_reg_lo, in_reg_hi) << "Diagonal overlap unexpected."; - - __ Asr(mask, in_reg_hi, 31); - __ adds(out_reg_lo, in_reg_lo, ShifterOperand(mask)); - __ adc(out_reg_hi, in_reg_hi, ShifterOperand(mask)); - __ eor(out_reg_lo, mask, ShifterOperand(out_reg_lo)); - __ eor(out_reg_hi, mask, ShifterOperand(out_reg_hi)); - } else { - Register in_reg = in.AsRegister<Register>(); - Register out_reg = output.AsRegister<Register>(); - - __ Asr(mask, in_reg, 31); - __ add(out_reg, in_reg, ShifterOperand(mask)); - __ eor(out_reg, mask, ShifterOperand(out_reg)); - } -} - -void IntrinsicLocationsBuilderARM::VisitMathAbsInt(HInvoke* invoke) { - CreateIntToIntPlusTemp(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMathAbsInt(HInvoke* invoke) { - GenAbsInteger(invoke->GetLocations(), /* is64bit */ false, GetAssembler()); -} - - -void IntrinsicLocationsBuilderARM::VisitMathAbsLong(HInvoke* invoke) { - CreateIntToIntPlusTemp(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMathAbsLong(HInvoke* invoke) { - GenAbsInteger(invoke->GetLocations(), /* is64bit */ true, GetAssembler()); -} - -static void GenMinMax(LocationSummary* locations, - bool is_min, - ArmAssembler* assembler) { - Register op1 = locations->InAt(0).AsRegister<Register>(); - Register op2 = locations->InAt(1).AsRegister<Register>(); - Register out = locations->Out().AsRegister<Register>(); - - __ cmp(op1, ShifterOperand(op2)); - - __ it((is_min) ? Condition::LT : Condition::GT, kItElse); - __ mov(out, ShifterOperand(op1), is_min ? Condition::LT : Condition::GT); - __ mov(out, ShifterOperand(op2), is_min ? Condition::GE : Condition::LE); -} - -static void CreateIntIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) { - LocationSummary* locations = new (arena) LocationSummary(invoke, - LocationSummary::kNoCall, - kIntrinsified); - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); -} - -void IntrinsicLocationsBuilderARM::VisitMathMinIntInt(HInvoke* invoke) { - CreateIntIntToIntLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMathMinIntInt(HInvoke* invoke) { - GenMinMax(invoke->GetLocations(), /* is_min */ true, GetAssembler()); -} - -void IntrinsicLocationsBuilderARM::VisitMathMaxIntInt(HInvoke* invoke) { - CreateIntIntToIntLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMathMaxIntInt(HInvoke* invoke) { - GenMinMax(invoke->GetLocations(), /* is_min */ false, GetAssembler()); -} - -void IntrinsicLocationsBuilderARM::VisitMathSqrt(HInvoke* invoke) { - CreateFPToFPLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMathSqrt(HInvoke* invoke) { - LocationSummary* locations = invoke->GetLocations(); - ArmAssembler* assembler = GetAssembler(); - __ vsqrtd(FromLowSToD(locations->Out().AsFpuRegisterPairLow<SRegister>()), - FromLowSToD(locations->InAt(0).AsFpuRegisterPairLow<SRegister>())); -} - -void IntrinsicLocationsBuilderARM::VisitMemoryPeekByte(HInvoke* invoke) { - CreateIntToIntLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMemoryPeekByte(HInvoke* invoke) { - ArmAssembler* assembler = GetAssembler(); - // Ignore upper 4B of long address. - __ ldrsb(invoke->GetLocations()->Out().AsRegister<Register>(), - Address(invoke->GetLocations()->InAt(0).AsRegisterPairLow<Register>())); -} - -void IntrinsicLocationsBuilderARM::VisitMemoryPeekIntNative(HInvoke* invoke) { - CreateIntToIntLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMemoryPeekIntNative(HInvoke* invoke) { - ArmAssembler* assembler = GetAssembler(); - // Ignore upper 4B of long address. - __ ldr(invoke->GetLocations()->Out().AsRegister<Register>(), - Address(invoke->GetLocations()->InAt(0).AsRegisterPairLow<Register>())); -} - -void IntrinsicLocationsBuilderARM::VisitMemoryPeekLongNative(HInvoke* invoke) { - CreateIntToIntLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMemoryPeekLongNative(HInvoke* invoke) { - ArmAssembler* assembler = GetAssembler(); - // Ignore upper 4B of long address. - Register addr = invoke->GetLocations()->InAt(0).AsRegisterPairLow<Register>(); - // Worst case: Control register bit SCTLR.A = 0. Then unaligned accesses throw a processor - // exception. So we can't use ldrd as addr may be unaligned. - Register lo = invoke->GetLocations()->Out().AsRegisterPairLow<Register>(); - Register hi = invoke->GetLocations()->Out().AsRegisterPairHigh<Register>(); - if (addr == lo) { - __ ldr(hi, Address(addr, 4)); - __ ldr(lo, Address(addr, 0)); - } else { - __ ldr(lo, Address(addr, 0)); - __ ldr(hi, Address(addr, 4)); - } -} - -void IntrinsicLocationsBuilderARM::VisitMemoryPeekShortNative(HInvoke* invoke) { - CreateIntToIntLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMemoryPeekShortNative(HInvoke* invoke) { - ArmAssembler* assembler = GetAssembler(); - // Ignore upper 4B of long address. - __ ldrsh(invoke->GetLocations()->Out().AsRegister<Register>(), - Address(invoke->GetLocations()->InAt(0).AsRegisterPairLow<Register>())); -} - -static void CreateIntIntToVoidLocations(ArenaAllocator* arena, HInvoke* invoke) { - LocationSummary* locations = new (arena) LocationSummary(invoke, - LocationSummary::kNoCall, - kIntrinsified); - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RequiresRegister()); -} - -void IntrinsicLocationsBuilderARM::VisitMemoryPokeByte(HInvoke* invoke) { - CreateIntIntToVoidLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMemoryPokeByte(HInvoke* invoke) { - ArmAssembler* assembler = GetAssembler(); - __ strb(invoke->GetLocations()->InAt(1).AsRegister<Register>(), - Address(invoke->GetLocations()->InAt(0).AsRegisterPairLow<Register>())); -} - -void IntrinsicLocationsBuilderARM::VisitMemoryPokeIntNative(HInvoke* invoke) { - CreateIntIntToVoidLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMemoryPokeIntNative(HInvoke* invoke) { - ArmAssembler* assembler = GetAssembler(); - __ str(invoke->GetLocations()->InAt(1).AsRegister<Register>(), - Address(invoke->GetLocations()->InAt(0).AsRegisterPairLow<Register>())); -} - -void IntrinsicLocationsBuilderARM::VisitMemoryPokeLongNative(HInvoke* invoke) { - CreateIntIntToVoidLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMemoryPokeLongNative(HInvoke* invoke) { - ArmAssembler* assembler = GetAssembler(); - // Ignore upper 4B of long address. - Register addr = invoke->GetLocations()->InAt(0).AsRegisterPairLow<Register>(); - // Worst case: Control register bit SCTLR.A = 0. Then unaligned accesses throw a processor - // exception. So we can't use ldrd as addr may be unaligned. - __ str(invoke->GetLocations()->InAt(1).AsRegisterPairLow<Register>(), Address(addr, 0)); - __ str(invoke->GetLocations()->InAt(1).AsRegisterPairHigh<Register>(), Address(addr, 4)); -} - -void IntrinsicLocationsBuilderARM::VisitMemoryPokeShortNative(HInvoke* invoke) { - CreateIntIntToVoidLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMemoryPokeShortNative(HInvoke* invoke) { - ArmAssembler* assembler = GetAssembler(); - __ strh(invoke->GetLocations()->InAt(1).AsRegister<Register>(), - Address(invoke->GetLocations()->InAt(0).AsRegisterPairLow<Register>())); -} - -void IntrinsicLocationsBuilderARM::VisitThreadCurrentThread(HInvoke* invoke) { - LocationSummary* locations = new (arena_) LocationSummary(invoke, - LocationSummary::kNoCall, - kIntrinsified); - locations->SetOut(Location::RequiresRegister()); -} - -void IntrinsicCodeGeneratorARM::VisitThreadCurrentThread(HInvoke* invoke) { - ArmAssembler* assembler = GetAssembler(); - __ LoadFromOffset(kLoadWord, - invoke->GetLocations()->Out().AsRegister<Register>(), - TR, - Thread::PeerOffset<kArmPointerSize>().Int32Value()); -} - -static void GenUnsafeGet(HInvoke* invoke, - Primitive::Type type, - bool is_volatile, - CodeGeneratorARM* codegen) { - LocationSummary* locations = invoke->GetLocations(); - ArmAssembler* assembler = codegen->GetAssembler(); - Location base_loc = locations->InAt(1); - Register base = base_loc.AsRegister<Register>(); // Object pointer. - Location offset_loc = locations->InAt(2); - Register offset = offset_loc.AsRegisterPairLow<Register>(); // Long offset, lo part only. - Location trg_loc = locations->Out(); - - switch (type) { - case Primitive::kPrimInt: { - Register trg = trg_loc.AsRegister<Register>(); - __ ldr(trg, Address(base, offset)); - if (is_volatile) { - __ dmb(ISH); - } - break; - } - - case Primitive::kPrimNot: { - Register trg = trg_loc.AsRegister<Register>(); - if (kEmitCompilerReadBarrier) { - if (kUseBakerReadBarrier) { - Location temp = locations->GetTemp(0); - codegen->GenerateReferenceLoadWithBakerReadBarrier( - invoke, trg_loc, base, 0U, offset_loc, TIMES_1, temp, /* needs_null_check */ false); - if (is_volatile) { - __ dmb(ISH); - } - } else { - __ ldr(trg, Address(base, offset)); - if (is_volatile) { - __ dmb(ISH); - } - codegen->GenerateReadBarrierSlow(invoke, trg_loc, trg_loc, base_loc, 0U, offset_loc); - } - } else { - __ ldr(trg, Address(base, offset)); - if (is_volatile) { - __ dmb(ISH); - } - __ MaybeUnpoisonHeapReference(trg); - } - break; - } - - case Primitive::kPrimLong: { - Register trg_lo = trg_loc.AsRegisterPairLow<Register>(); - __ add(IP, base, ShifterOperand(offset)); - if (is_volatile && !codegen->GetInstructionSetFeatures().HasAtomicLdrdAndStrd()) { - Register trg_hi = trg_loc.AsRegisterPairHigh<Register>(); - __ ldrexd(trg_lo, trg_hi, IP); - } else { - __ ldrd(trg_lo, Address(IP)); - } - if (is_volatile) { - __ dmb(ISH); - } - break; - } - - default: - LOG(FATAL) << "Unexpected type " << type; - UNREACHABLE(); - } -} - -static void CreateIntIntIntToIntLocations(ArenaAllocator* arena, - HInvoke* invoke, - Primitive::Type type) { - bool can_call = kEmitCompilerReadBarrier && - (invoke->GetIntrinsic() == Intrinsics::kUnsafeGetObject || - invoke->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile); - LocationSummary* locations = new (arena) LocationSummary(invoke, - (can_call - ? LocationSummary::kCallOnSlowPath - : LocationSummary::kNoCall), - kIntrinsified); - if (can_call && kUseBakerReadBarrier) { - locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. - } - locations->SetInAt(0, Location::NoLocation()); // Unused receiver. - locations->SetInAt(1, Location::RequiresRegister()); - locations->SetInAt(2, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), - (can_call ? Location::kOutputOverlap : Location::kNoOutputOverlap)); - if (type == Primitive::kPrimNot && kEmitCompilerReadBarrier && kUseBakerReadBarrier) { - // We need a temporary register for the read barrier marking slow - // path in InstructionCodeGeneratorARM::GenerateReferenceLoadWithBakerReadBarrier. - locations->AddTemp(Location::RequiresRegister()); - } -} - -void IntrinsicLocationsBuilderARM::VisitUnsafeGet(HInvoke* invoke) { - CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimInt); -} -void IntrinsicLocationsBuilderARM::VisitUnsafeGetVolatile(HInvoke* invoke) { - CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimInt); -} -void IntrinsicLocationsBuilderARM::VisitUnsafeGetLong(HInvoke* invoke) { - CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimLong); -} -void IntrinsicLocationsBuilderARM::VisitUnsafeGetLongVolatile(HInvoke* invoke) { - CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimLong); -} -void IntrinsicLocationsBuilderARM::VisitUnsafeGetObject(HInvoke* invoke) { - CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimNot); -} -void IntrinsicLocationsBuilderARM::VisitUnsafeGetObjectVolatile(HInvoke* invoke) { - CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimNot); -} - -void IntrinsicCodeGeneratorARM::VisitUnsafeGet(HInvoke* invoke) { - GenUnsafeGet(invoke, Primitive::kPrimInt, /* is_volatile */ false, codegen_); -} -void IntrinsicCodeGeneratorARM::VisitUnsafeGetVolatile(HInvoke* invoke) { - GenUnsafeGet(invoke, Primitive::kPrimInt, /* is_volatile */ true, codegen_); -} -void IntrinsicCodeGeneratorARM::VisitUnsafeGetLong(HInvoke* invoke) { - GenUnsafeGet(invoke, Primitive::kPrimLong, /* is_volatile */ false, codegen_); -} -void IntrinsicCodeGeneratorARM::VisitUnsafeGetLongVolatile(HInvoke* invoke) { - GenUnsafeGet(invoke, Primitive::kPrimLong, /* is_volatile */ true, codegen_); -} -void IntrinsicCodeGeneratorARM::VisitUnsafeGetObject(HInvoke* invoke) { - GenUnsafeGet(invoke, Primitive::kPrimNot, /* is_volatile */ false, codegen_); -} -void IntrinsicCodeGeneratorARM::VisitUnsafeGetObjectVolatile(HInvoke* invoke) { - GenUnsafeGet(invoke, Primitive::kPrimNot, /* is_volatile */ true, codegen_); -} - -static void CreateIntIntIntIntToVoid(ArenaAllocator* arena, - const ArmInstructionSetFeatures& features, - Primitive::Type type, - bool is_volatile, - HInvoke* invoke) { - LocationSummary* locations = new (arena) LocationSummary(invoke, - LocationSummary::kNoCall, - kIntrinsified); - locations->SetInAt(0, Location::NoLocation()); // Unused receiver. - locations->SetInAt(1, Location::RequiresRegister()); - locations->SetInAt(2, Location::RequiresRegister()); - locations->SetInAt(3, Location::RequiresRegister()); - - if (type == Primitive::kPrimLong) { - // Potentially need temps for ldrexd-strexd loop. - if (is_volatile && !features.HasAtomicLdrdAndStrd()) { - locations->AddTemp(Location::RequiresRegister()); // Temp_lo. - locations->AddTemp(Location::RequiresRegister()); // Temp_hi. - } - } else if (type == Primitive::kPrimNot) { - // Temps for card-marking. - locations->AddTemp(Location::RequiresRegister()); // Temp. - locations->AddTemp(Location::RequiresRegister()); // Card. - } -} - -void IntrinsicLocationsBuilderARM::VisitUnsafePut(HInvoke* invoke) { - CreateIntIntIntIntToVoid(arena_, features_, Primitive::kPrimInt, /* is_volatile */ false, invoke); -} -void IntrinsicLocationsBuilderARM::VisitUnsafePutOrdered(HInvoke* invoke) { - CreateIntIntIntIntToVoid(arena_, features_, Primitive::kPrimInt, /* is_volatile */ false, invoke); -} -void IntrinsicLocationsBuilderARM::VisitUnsafePutVolatile(HInvoke* invoke) { - CreateIntIntIntIntToVoid(arena_, features_, Primitive::kPrimInt, /* is_volatile */ true, invoke); -} -void IntrinsicLocationsBuilderARM::VisitUnsafePutObject(HInvoke* invoke) { - CreateIntIntIntIntToVoid(arena_, features_, Primitive::kPrimNot, /* is_volatile */ false, invoke); -} -void IntrinsicLocationsBuilderARM::VisitUnsafePutObjectOrdered(HInvoke* invoke) { - CreateIntIntIntIntToVoid(arena_, features_, Primitive::kPrimNot, /* is_volatile */ false, invoke); -} -void IntrinsicLocationsBuilderARM::VisitUnsafePutObjectVolatile(HInvoke* invoke) { - CreateIntIntIntIntToVoid(arena_, features_, Primitive::kPrimNot, /* is_volatile */ true, invoke); -} -void IntrinsicLocationsBuilderARM::VisitUnsafePutLong(HInvoke* invoke) { - CreateIntIntIntIntToVoid( - arena_, features_, Primitive::kPrimLong, /* is_volatile */ false, invoke); -} -void IntrinsicLocationsBuilderARM::VisitUnsafePutLongOrdered(HInvoke* invoke) { - CreateIntIntIntIntToVoid( - arena_, features_, Primitive::kPrimLong, /* is_volatile */ false, invoke); -} -void IntrinsicLocationsBuilderARM::VisitUnsafePutLongVolatile(HInvoke* invoke) { - CreateIntIntIntIntToVoid( - arena_, features_, Primitive::kPrimLong, /* is_volatile */ true, invoke); -} - -static void GenUnsafePut(LocationSummary* locations, - Primitive::Type type, - bool is_volatile, - bool is_ordered, - CodeGeneratorARM* codegen) { - ArmAssembler* assembler = codegen->GetAssembler(); - - Register base = locations->InAt(1).AsRegister<Register>(); // Object pointer. - Register offset = locations->InAt(2).AsRegisterPairLow<Register>(); // Long offset, lo part only. - Register value; - - if (is_volatile || is_ordered) { - __ dmb(ISH); - } - - if (type == Primitive::kPrimLong) { - Register value_lo = locations->InAt(3).AsRegisterPairLow<Register>(); - value = value_lo; - if (is_volatile && !codegen->GetInstructionSetFeatures().HasAtomicLdrdAndStrd()) { - Register temp_lo = locations->GetTemp(0).AsRegister<Register>(); - Register temp_hi = locations->GetTemp(1).AsRegister<Register>(); - Register value_hi = locations->InAt(3).AsRegisterPairHigh<Register>(); - - __ add(IP, base, ShifterOperand(offset)); - Label loop_head; - __ Bind(&loop_head); - __ ldrexd(temp_lo, temp_hi, IP); - __ strexd(temp_lo, value_lo, value_hi, IP); - __ cmp(temp_lo, ShifterOperand(0)); - __ b(&loop_head, NE); - } else { - __ add(IP, base, ShifterOperand(offset)); - __ strd(value_lo, Address(IP)); - } - } else { - value = locations->InAt(3).AsRegister<Register>(); - Register source = value; - if (kPoisonHeapReferences && type == Primitive::kPrimNot) { - Register temp = locations->GetTemp(0).AsRegister<Register>(); - __ Mov(temp, value); - __ PoisonHeapReference(temp); - source = temp; - } - __ str(source, Address(base, offset)); - } - - if (is_volatile) { - __ dmb(ISH); - } - - if (type == Primitive::kPrimNot) { - Register temp = locations->GetTemp(0).AsRegister<Register>(); - Register card = locations->GetTemp(1).AsRegister<Register>(); - bool value_can_be_null = true; // TODO: Worth finding out this information? - codegen->MarkGCCard(temp, card, base, value, value_can_be_null); - } -} - -void IntrinsicCodeGeneratorARM::VisitUnsafePut(HInvoke* invoke) { - GenUnsafePut(invoke->GetLocations(), - Primitive::kPrimInt, - /* is_volatile */ false, - /* is_ordered */ false, - codegen_); -} -void IntrinsicCodeGeneratorARM::VisitUnsafePutOrdered(HInvoke* invoke) { - GenUnsafePut(invoke->GetLocations(), - Primitive::kPrimInt, - /* is_volatile */ false, - /* is_ordered */ true, - codegen_); -} -void IntrinsicCodeGeneratorARM::VisitUnsafePutVolatile(HInvoke* invoke) { - GenUnsafePut(invoke->GetLocations(), - Primitive::kPrimInt, - /* is_volatile */ true, - /* is_ordered */ false, - codegen_); -} -void IntrinsicCodeGeneratorARM::VisitUnsafePutObject(HInvoke* invoke) { - GenUnsafePut(invoke->GetLocations(), - Primitive::kPrimNot, - /* is_volatile */ false, - /* is_ordered */ false, - codegen_); -} -void IntrinsicCodeGeneratorARM::VisitUnsafePutObjectOrdered(HInvoke* invoke) { - GenUnsafePut(invoke->GetLocations(), - Primitive::kPrimNot, - /* is_volatile */ false, - /* is_ordered */ true, - codegen_); -} -void IntrinsicCodeGeneratorARM::VisitUnsafePutObjectVolatile(HInvoke* invoke) { - GenUnsafePut(invoke->GetLocations(), - Primitive::kPrimNot, - /* is_volatile */ true, - /* is_ordered */ false, - codegen_); -} -void IntrinsicCodeGeneratorARM::VisitUnsafePutLong(HInvoke* invoke) { - GenUnsafePut(invoke->GetLocations(), - Primitive::kPrimLong, - /* is_volatile */ false, - /* is_ordered */ false, - codegen_); -} -void IntrinsicCodeGeneratorARM::VisitUnsafePutLongOrdered(HInvoke* invoke) { - GenUnsafePut(invoke->GetLocations(), - Primitive::kPrimLong, - /* is_volatile */ false, - /* is_ordered */ true, - codegen_); -} -void IntrinsicCodeGeneratorARM::VisitUnsafePutLongVolatile(HInvoke* invoke) { - GenUnsafePut(invoke->GetLocations(), - Primitive::kPrimLong, - /* is_volatile */ true, - /* is_ordered */ false, - codegen_); -} - -static void CreateIntIntIntIntIntToIntPlusTemps(ArenaAllocator* arena, - HInvoke* invoke, - Primitive::Type type) { - bool can_call = kEmitCompilerReadBarrier && - kUseBakerReadBarrier && - (invoke->GetIntrinsic() == Intrinsics::kUnsafeCASObject); - LocationSummary* locations = new (arena) LocationSummary(invoke, - (can_call - ? LocationSummary::kCallOnSlowPath - : LocationSummary::kNoCall), - kIntrinsified); - locations->SetInAt(0, Location::NoLocation()); // Unused receiver. - locations->SetInAt(1, Location::RequiresRegister()); - locations->SetInAt(2, Location::RequiresRegister()); - locations->SetInAt(3, Location::RequiresRegister()); - locations->SetInAt(4, Location::RequiresRegister()); - - // If heap poisoning is enabled, we don't want the unpoisoning - // operations to potentially clobber the output. Likewise when - // emitting a (Baker) read barrier, which may call. - Location::OutputOverlap overlaps = - ((kPoisonHeapReferences && type == Primitive::kPrimNot) || can_call) - ? Location::kOutputOverlap - : Location::kNoOutputOverlap; - locations->SetOut(Location::RequiresRegister(), overlaps); - - // Temporary registers used in CAS. In the object case - // (UnsafeCASObject intrinsic), these are also used for - // card-marking, and possibly for (Baker) read barrier. - locations->AddTemp(Location::RequiresRegister()); // Pointer. - locations->AddTemp(Location::RequiresRegister()); // Temp 1. -} - -static void GenCas(HInvoke* invoke, Primitive::Type type, CodeGeneratorARM* codegen) { - DCHECK_NE(type, Primitive::kPrimLong); - - ArmAssembler* assembler = codegen->GetAssembler(); - LocationSummary* locations = invoke->GetLocations(); - - Location out_loc = locations->Out(); - Register out = out_loc.AsRegister<Register>(); // Boolean result. - - Register base = locations->InAt(1).AsRegister<Register>(); // Object pointer. - Location offset_loc = locations->InAt(2); - Register offset = offset_loc.AsRegisterPairLow<Register>(); // Offset (discard high 4B). - Register expected = locations->InAt(3).AsRegister<Register>(); // Expected. - Register value = locations->InAt(4).AsRegister<Register>(); // Value. - - Location tmp_ptr_loc = locations->GetTemp(0); - Register tmp_ptr = tmp_ptr_loc.AsRegister<Register>(); // Pointer to actual memory. - Register tmp = locations->GetTemp(1).AsRegister<Register>(); // Value in memory. - - if (type == Primitive::kPrimNot) { - // The only read barrier implementation supporting the - // UnsafeCASObject intrinsic is the Baker-style read barriers. - DCHECK(!kEmitCompilerReadBarrier || kUseBakerReadBarrier); - - // Mark card for object assuming new value is stored. Worst case we will mark an unchanged - // object and scan the receiver at the next GC for nothing. - bool value_can_be_null = true; // TODO: Worth finding out this information? - codegen->MarkGCCard(tmp_ptr, tmp, base, value, value_can_be_null); - - if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { - // Need to make sure the reference stored in the field is a to-space - // one before attempting the CAS or the CAS could fail incorrectly. - codegen->UpdateReferenceFieldWithBakerReadBarrier( - invoke, - out_loc, // Unused, used only as a "temporary" within the read barrier. - base, - /* field_offset */ offset_loc, - tmp_ptr_loc, - /* needs_null_check */ false, - tmp); - } - } - - // Prevent reordering with prior memory operations. - // Emit a DMB ISH instruction instead of an DMB ISHST one, as the - // latter allows a preceding load to be delayed past the STXR - // instruction below. - __ dmb(ISH); - - __ add(tmp_ptr, base, ShifterOperand(offset)); - - if (kPoisonHeapReferences && type == Primitive::kPrimNot) { - __ PoisonHeapReference(expected); - if (value == expected) { - // Do not poison `value`, as it is the same register as - // `expected`, which has just been poisoned. - } else { - __ PoisonHeapReference(value); - } - } - - // do { - // tmp = [r_ptr] - expected; - // } while (tmp == 0 && failure([r_ptr] <- r_new_value)); - // result = tmp != 0; - - Label loop_head; - __ Bind(&loop_head); - - __ ldrex(tmp, tmp_ptr); - - __ subs(tmp, tmp, ShifterOperand(expected)); - - __ it(EQ, ItState::kItT); - __ strex(tmp, value, tmp_ptr, EQ); - __ cmp(tmp, ShifterOperand(1), EQ); - - __ b(&loop_head, EQ); - - __ dmb(ISH); - - __ rsbs(out, tmp, ShifterOperand(1)); - __ it(CC); - __ mov(out, ShifterOperand(0), CC); - - if (kPoisonHeapReferences && type == Primitive::kPrimNot) { - __ UnpoisonHeapReference(expected); - if (value == expected) { - // Do not unpoison `value`, as it is the same register as - // `expected`, which has just been unpoisoned. - } else { - __ UnpoisonHeapReference(value); - } - } -} - -void IntrinsicLocationsBuilderARM::VisitUnsafeCASInt(HInvoke* invoke) { - CreateIntIntIntIntIntToIntPlusTemps(arena_, invoke, Primitive::kPrimInt); -} -void IntrinsicLocationsBuilderARM::VisitUnsafeCASObject(HInvoke* invoke) { - // The only read barrier implementation supporting the - // UnsafeCASObject intrinsic is the Baker-style read barriers. - if (kEmitCompilerReadBarrier && !kUseBakerReadBarrier) { - return; - } - - CreateIntIntIntIntIntToIntPlusTemps(arena_, invoke, Primitive::kPrimNot); -} -void IntrinsicCodeGeneratorARM::VisitUnsafeCASInt(HInvoke* invoke) { - GenCas(invoke, Primitive::kPrimInt, codegen_); -} -void IntrinsicCodeGeneratorARM::VisitUnsafeCASObject(HInvoke* invoke) { - // The only read barrier implementation supporting the - // UnsafeCASObject intrinsic is the Baker-style read barriers. - DCHECK(!kEmitCompilerReadBarrier || kUseBakerReadBarrier); - - GenCas(invoke, Primitive::kPrimNot, codegen_); -} - -void IntrinsicLocationsBuilderARM::VisitStringCompareTo(HInvoke* invoke) { - // The inputs plus one temp. - LocationSummary* locations = new (arena_) LocationSummary(invoke, - invoke->InputAt(1)->CanBeNull() - ? LocationSummary::kCallOnSlowPath - : LocationSummary::kNoCall, - kIntrinsified); - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RequiresRegister()); - locations->AddTemp(Location::RequiresRegister()); - locations->AddTemp(Location::RequiresRegister()); - locations->AddTemp(Location::RequiresRegister()); - // Need temporary registers for String compression's feature. - if (mirror::kUseStringCompression) { - locations->AddTemp(Location::RequiresRegister()); - } - locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); -} - -void IntrinsicCodeGeneratorARM::VisitStringCompareTo(HInvoke* invoke) { - ArmAssembler* assembler = GetAssembler(); - LocationSummary* locations = invoke->GetLocations(); - - Register str = locations->InAt(0).AsRegister<Register>(); - Register arg = locations->InAt(1).AsRegister<Register>(); - Register out = locations->Out().AsRegister<Register>(); - - Register temp0 = locations->GetTemp(0).AsRegister<Register>(); - Register temp1 = locations->GetTemp(1).AsRegister<Register>(); - Register temp2 = locations->GetTemp(2).AsRegister<Register>(); - Register temp3; - if (mirror::kUseStringCompression) { - temp3 = locations->GetTemp(3).AsRegister<Register>(); - } - - Label loop; - Label find_char_diff; - Label end; - Label different_compression; - - // Get offsets of count and value fields within a string object. - const int32_t count_offset = mirror::String::CountOffset().Int32Value(); - const int32_t value_offset = mirror::String::ValueOffset().Int32Value(); - - // Note that the null check must have been done earlier. - DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0))); - - // Take slow path and throw if input can be and is null. - SlowPathCode* slow_path = nullptr; - const bool can_slow_path = invoke->InputAt(1)->CanBeNull(); - if (can_slow_path) { - slow_path = new (GetAllocator()) IntrinsicSlowPathARM(invoke); - codegen_->AddSlowPath(slow_path); - __ CompareAndBranchIfZero(arg, slow_path->GetEntryLabel()); - } - - // Reference equality check, return 0 if same reference. - __ subs(out, str, ShifterOperand(arg)); - __ b(&end, EQ); - - if (mirror::kUseStringCompression) { - // Load `count` fields of this and argument strings. - __ ldr(temp3, Address(str, count_offset)); - __ ldr(temp2, Address(arg, count_offset)); - // Extract lengths from the `count` fields. - __ Lsr(temp0, temp3, 1u); - __ Lsr(temp1, temp2, 1u); - } else { - // Load lengths of this and argument strings. - __ ldr(temp0, Address(str, count_offset)); - __ ldr(temp1, Address(arg, count_offset)); - } - // out = length diff. - __ subs(out, temp0, ShifterOperand(temp1)); - // temp0 = min(len(str), len(arg)). - __ it(GT); - __ mov(temp0, ShifterOperand(temp1), GT); - // Shorter string is empty? - __ CompareAndBranchIfZero(temp0, &end); - - if (mirror::kUseStringCompression) { - // Check if both strings using same compression style to use this comparison loop. - __ eor(temp2, temp2, ShifterOperand(temp3)); - __ Lsrs(temp2, temp2, 1u); - __ b(&different_compression, CS); - // For string compression, calculate the number of bytes to compare (not chars). - // This could in theory exceed INT32_MAX, so treat temp0 as unsigned. - __ Lsls(temp3, temp3, 31u); // Extract purely the compression flag. - __ it(NE); - __ add(temp0, temp0, ShifterOperand(temp0), NE); - } - - // Store offset of string value in preparation for comparison loop. - __ mov(temp1, ShifterOperand(value_offset)); - - // Assertions that must hold in order to compare multiple characters at a time. - CHECK_ALIGNED(value_offset, 8); - static_assert(IsAligned<8>(kObjectAlignment), - "String data must be 8-byte aligned for unrolled CompareTo loop."); - - const size_t char_size = Primitive::ComponentSize(Primitive::kPrimChar); - DCHECK_EQ(char_size, 2u); - - Label find_char_diff_2nd_cmp; - // Unrolled loop comparing 4x16-bit chars per iteration (ok because of string data alignment). - __ Bind(&loop); - __ ldr(IP, Address(str, temp1)); - __ ldr(temp2, Address(arg, temp1)); - __ cmp(IP, ShifterOperand(temp2)); - __ b(&find_char_diff, NE); - __ add(temp1, temp1, ShifterOperand(char_size * 2)); - - __ ldr(IP, Address(str, temp1)); - __ ldr(temp2, Address(arg, temp1)); - __ cmp(IP, ShifterOperand(temp2)); - __ b(&find_char_diff_2nd_cmp, NE); - __ add(temp1, temp1, ShifterOperand(char_size * 2)); - // With string compression, we have compared 8 bytes, otherwise 4 chars. - __ subs(temp0, temp0, ShifterOperand(mirror::kUseStringCompression ? 8 : 4)); - __ b(&loop, HI); - __ b(&end); - - __ Bind(&find_char_diff_2nd_cmp); - if (mirror::kUseStringCompression) { - __ subs(temp0, temp0, ShifterOperand(4)); // 4 bytes previously compared. - __ b(&end, LS); // Was the second comparison fully beyond the end? - } else { - // Without string compression, we can start treating temp0 as signed - // and rely on the signed comparison below. - __ sub(temp0, temp0, ShifterOperand(2)); - } - - // Find the single character difference. - __ Bind(&find_char_diff); - // Get the bit position of the first character that differs. - __ eor(temp1, temp2, ShifterOperand(IP)); - __ rbit(temp1, temp1); - __ clz(temp1, temp1); - - // temp0 = number of characters remaining to compare. - // (Without string compression, it could be < 1 if a difference is found by the second CMP - // in the comparison loop, and after the end of the shorter string data). - - // Without string compression (temp1 >> 4) = character where difference occurs between the last - // two words compared, in the interval [0,1]. - // (0 for low half-word different, 1 for high half-word different). - // With string compression, (temp1 << 3) = byte where the difference occurs, - // in the interval [0,3]. - - // If temp0 <= (temp1 >> (kUseStringCompression ? 3 : 4)), the difference occurs outside - // the remaining string data, so just return length diff (out). - // The comparison is unsigned for string compression, otherwise signed. - __ cmp(temp0, ShifterOperand(temp1, LSR, mirror::kUseStringCompression ? 3 : 4)); - __ b(&end, mirror::kUseStringCompression ? LS : LE); - - // Extract the characters and calculate the difference. - if (mirror::kUseStringCompression) { - // For compressed strings we need to clear 0x7 from temp1, for uncompressed we need to clear - // 0xf. We also need to prepare the character extraction mask `uncompressed ? 0xffffu : 0xffu`. - // The compression flag is now in the highest bit of temp3, so let's play some tricks. - __ orr(temp3, temp3, ShifterOperand(0xffu << 23)); // uncompressed ? 0xff800000u : 0x7ff80000u - __ bic(temp1, temp1, ShifterOperand(temp3, LSR, 31 - 3)); // &= ~(uncompressed ? 0xfu : 0x7u) - __ Asr(temp3, temp3, 7u); // uncompressed ? 0xffff0000u : 0xff0000u. - __ Lsr(temp2, temp2, temp1); // Extract second character. - __ Lsr(temp3, temp3, 16u); // uncompressed ? 0xffffu : 0xffu - __ Lsr(out, IP, temp1); // Extract first character. - __ and_(temp2, temp2, ShifterOperand(temp3)); - __ and_(out, out, ShifterOperand(temp3)); - } else { - __ bic(temp1, temp1, ShifterOperand(0xf)); - __ Lsr(temp2, temp2, temp1); - __ Lsr(out, IP, temp1); - __ movt(temp2, 0); - __ movt(out, 0); - } - - __ sub(out, out, ShifterOperand(temp2)); - - if (mirror::kUseStringCompression) { - __ b(&end); - __ Bind(&different_compression); - - // Comparison for different compression style. - const size_t c_char_size = Primitive::ComponentSize(Primitive::kPrimByte); - DCHECK_EQ(c_char_size, 1u); - - // We want to free up the temp3, currently holding `str.count`, for comparison. - // So, we move it to the bottom bit of the iteration count `temp0` which we tnen - // need to treat as unsigned. Start by freeing the bit with an ADD and continue - // further down by a LSRS+SBC which will flip the meaning of the flag but allow - // `subs temp0, #2; bhi different_compression_loop` to serve as the loop condition. - __ add(temp0, temp0, ShifterOperand(temp0)); // Unlike LSL, this ADD is always 16-bit. - // `temp1` will hold the compressed data pointer, `temp2` the uncompressed data pointer. - __ mov(temp1, ShifterOperand(str)); - __ mov(temp2, ShifterOperand(arg)); - __ Lsrs(temp3, temp3, 1u); // Continue the move of the compression flag. - __ it(CS, kItThen); // Interleave with selection of temp1 and temp2. - __ mov(temp1, ShifterOperand(arg), CS); // Preserves flags. - __ mov(temp2, ShifterOperand(str), CS); // Preserves flags. - __ sbc(temp0, temp0, ShifterOperand(0)); // Complete the move of the compression flag. - - // Adjust temp1 and temp2 from string pointers to data pointers. - __ add(temp1, temp1, ShifterOperand(value_offset)); - __ add(temp2, temp2, ShifterOperand(value_offset)); - - Label different_compression_loop; - Label different_compression_diff; - - // Main loop for different compression. - __ Bind(&different_compression_loop); - __ ldrb(IP, Address(temp1, c_char_size, Address::PostIndex)); - __ ldrh(temp3, Address(temp2, char_size, Address::PostIndex)); - __ cmp(IP, ShifterOperand(temp3)); - __ b(&different_compression_diff, NE); - __ subs(temp0, temp0, ShifterOperand(2)); - __ b(&different_compression_loop, HI); - __ b(&end); - - // Calculate the difference. - __ Bind(&different_compression_diff); - __ sub(out, IP, ShifterOperand(temp3)); - // Flip the difference if the `arg` is compressed. - // `temp0` contains inverted `str` compression flag, i.e the same as `arg` compression flag. - __ Lsrs(temp0, temp0, 1u); - static_assert(static_cast<uint32_t>(mirror::StringCompressionFlag::kCompressed) == 0u, - "Expecting 0=compressed, 1=uncompressed"); - __ it(CC); - __ rsb(out, out, ShifterOperand(0), CC); - } - - __ Bind(&end); - - if (can_slow_path) { - __ Bind(slow_path->GetExitLabel()); - } -} - -void IntrinsicLocationsBuilderARM::VisitStringEquals(HInvoke* invoke) { - LocationSummary* locations = new (arena_) LocationSummary(invoke, - LocationSummary::kNoCall, - kIntrinsified); - InvokeRuntimeCallingConvention calling_convention; - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RequiresRegister()); - // Temporary registers to store lengths of strings and for calculations. - // Using instruction cbz requires a low register, so explicitly set a temp to be R0. - locations->AddTemp(Location::RegisterLocation(R0)); - locations->AddTemp(Location::RequiresRegister()); - locations->AddTemp(Location::RequiresRegister()); - - locations->SetOut(Location::RequiresRegister()); -} - -void IntrinsicCodeGeneratorARM::VisitStringEquals(HInvoke* invoke) { - ArmAssembler* assembler = GetAssembler(); - LocationSummary* locations = invoke->GetLocations(); - - Register str = locations->InAt(0).AsRegister<Register>(); - Register arg = locations->InAt(1).AsRegister<Register>(); - Register out = locations->Out().AsRegister<Register>(); - - Register temp = locations->GetTemp(0).AsRegister<Register>(); - Register temp1 = locations->GetTemp(1).AsRegister<Register>(); - Register temp2 = locations->GetTemp(2).AsRegister<Register>(); - - Label loop; - Label end; - Label return_true; - Label return_false; - Label* final_label = codegen_->GetFinalLabel(invoke, &end); - - // Get offsets of count, value, and class fields within a string object. - const uint32_t count_offset = mirror::String::CountOffset().Uint32Value(); - const uint32_t value_offset = mirror::String::ValueOffset().Uint32Value(); - const uint32_t class_offset = mirror::Object::ClassOffset().Uint32Value(); - - // Note that the null check must have been done earlier. - DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0))); - - StringEqualsOptimizations optimizations(invoke); - if (!optimizations.GetArgumentNotNull()) { - // Check if input is null, return false if it is. - __ CompareAndBranchIfZero(arg, &return_false); - } - - // Reference equality check, return true if same reference. - __ cmp(str, ShifterOperand(arg)); - __ b(&return_true, EQ); - - if (!optimizations.GetArgumentIsString()) { - // Instanceof check for the argument by comparing class fields. - // All string objects must have the same type since String cannot be subclassed. - // Receiver must be a string object, so its class field is equal to all strings' class fields. - // If the argument is a string object, its class field must be equal to receiver's class field. - __ ldr(temp, Address(str, class_offset)); - __ ldr(temp1, Address(arg, class_offset)); - __ cmp(temp, ShifterOperand(temp1)); - __ b(&return_false, NE); - } - - // Load `count` fields of this and argument strings. - __ ldr(temp, Address(str, count_offset)); - __ ldr(temp1, Address(arg, count_offset)); - // Check if `count` fields are equal, return false if they're not. - // Also compares the compression style, if differs return false. - __ cmp(temp, ShifterOperand(temp1)); - __ b(&return_false, NE); - // Return true if both strings are empty. Even with string compression `count == 0` means empty. - static_assert(static_cast<uint32_t>(mirror::StringCompressionFlag::kCompressed) == 0u, - "Expecting 0=compressed, 1=uncompressed"); - __ cbz(temp, &return_true); - - // Assertions that must hold in order to compare strings 4 bytes at a time. - DCHECK_ALIGNED(value_offset, 4); - static_assert(IsAligned<4>(kObjectAlignment), "String data must be aligned for fast compare."); - - if (mirror::kUseStringCompression) { - // For string compression, calculate the number of bytes to compare (not chars). - // This could in theory exceed INT32_MAX, so treat temp as unsigned. - __ Lsrs(temp, temp, 1u); // Extract length and check compression flag. - __ it(CS); // If uncompressed, - __ add(temp, temp, ShifterOperand(temp), CS); // double the byte count. - } - - // Store offset of string value in preparation for comparison loop. - __ LoadImmediate(temp1, value_offset); - - // Loop to compare strings 4 bytes at a time starting at the front of the string. - // Ok to do this because strings are zero-padded to kObjectAlignment. - __ Bind(&loop); - __ ldr(out, Address(str, temp1)); - __ ldr(temp2, Address(arg, temp1)); - __ add(temp1, temp1, ShifterOperand(sizeof(uint32_t))); - __ cmp(out, ShifterOperand(temp2)); - __ b(&return_false, NE); - // With string compression, we have compared 4 bytes, otherwise 2 chars. - __ subs(temp, temp, ShifterOperand(mirror::kUseStringCompression ? 4 : 2)); - __ b(&loop, HI); - - // Return true and exit the function. - // If loop does not result in returning false, we return true. - __ Bind(&return_true); - __ LoadImmediate(out, 1); - __ b(final_label); - - // Return false and exit the function. - __ Bind(&return_false); - __ LoadImmediate(out, 0); - - if (end.IsLinked()) { - __ Bind(&end); - } -} - -static void GenerateVisitStringIndexOf(HInvoke* invoke, - ArmAssembler* assembler, - CodeGeneratorARM* codegen, - ArenaAllocator* allocator, - bool start_at_zero) { - LocationSummary* locations = invoke->GetLocations(); - - // Note that the null check must have been done earlier. - DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0))); - - // Check for code points > 0xFFFF. Either a slow-path check when we don't know statically, - // or directly dispatch for a large constant, or omit slow-path for a small constant or a char. - SlowPathCode* slow_path = nullptr; - HInstruction* code_point = invoke->InputAt(1); - if (code_point->IsIntConstant()) { - if (static_cast<uint32_t>(code_point->AsIntConstant()->GetValue()) > - std::numeric_limits<uint16_t>::max()) { - // Always needs the slow-path. We could directly dispatch to it, but this case should be - // rare, so for simplicity just put the full slow-path down and branch unconditionally. - slow_path = new (allocator) IntrinsicSlowPathARM(invoke); - codegen->AddSlowPath(slow_path); - __ b(slow_path->GetEntryLabel()); - __ Bind(slow_path->GetExitLabel()); - return; - } - } else if (code_point->GetType() != Primitive::kPrimChar) { - Register char_reg = locations->InAt(1).AsRegister<Register>(); - // 0xffff is not modified immediate but 0x10000 is, so use `>= 0x10000` instead of `> 0xffff`. - __ cmp(char_reg, - ShifterOperand(static_cast<uint32_t>(std::numeric_limits<uint16_t>::max()) + 1)); - slow_path = new (allocator) IntrinsicSlowPathARM(invoke); - codegen->AddSlowPath(slow_path); - __ b(slow_path->GetEntryLabel(), HS); - } - - if (start_at_zero) { - Register tmp_reg = locations->GetTemp(0).AsRegister<Register>(); - DCHECK_EQ(tmp_reg, R2); - // Start-index = 0. - __ LoadImmediate(tmp_reg, 0); - } - - codegen->InvokeRuntime(kQuickIndexOf, invoke, invoke->GetDexPc(), slow_path); - CheckEntrypointTypes<kQuickIndexOf, int32_t, void*, uint32_t, uint32_t>(); - - if (slow_path != nullptr) { - __ Bind(slow_path->GetExitLabel()); - } -} - -void IntrinsicLocationsBuilderARM::VisitStringIndexOf(HInvoke* invoke) { - LocationSummary* locations = new (arena_) LocationSummary(invoke, - LocationSummary::kCallOnMainAndSlowPath, - kIntrinsified); - // We have a hand-crafted assembly stub that follows the runtime calling convention. So it's - // best to align the inputs accordingly. - InvokeRuntimeCallingConvention calling_convention; - locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); - locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); - locations->SetOut(Location::RegisterLocation(R0)); - - // Need to send start-index=0. - locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(2))); -} - -void IntrinsicCodeGeneratorARM::VisitStringIndexOf(HInvoke* invoke) { - GenerateVisitStringIndexOf( - invoke, GetAssembler(), codegen_, GetAllocator(), /* start_at_zero */ true); -} - -void IntrinsicLocationsBuilderARM::VisitStringIndexOfAfter(HInvoke* invoke) { - LocationSummary* locations = new (arena_) LocationSummary(invoke, - LocationSummary::kCallOnMainAndSlowPath, - kIntrinsified); - // We have a hand-crafted assembly stub that follows the runtime calling convention. So it's - // best to align the inputs accordingly. - InvokeRuntimeCallingConvention calling_convention; - locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); - locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); - locations->SetInAt(2, Location::RegisterLocation(calling_convention.GetRegisterAt(2))); - locations->SetOut(Location::RegisterLocation(R0)); -} - -void IntrinsicCodeGeneratorARM::VisitStringIndexOfAfter(HInvoke* invoke) { - GenerateVisitStringIndexOf( - invoke, GetAssembler(), codegen_, GetAllocator(), /* start_at_zero */ false); -} - -void IntrinsicLocationsBuilderARM::VisitStringNewStringFromBytes(HInvoke* invoke) { - LocationSummary* locations = new (arena_) LocationSummary(invoke, - LocationSummary::kCallOnMainAndSlowPath, - kIntrinsified); - InvokeRuntimeCallingConvention calling_convention; - locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); - locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); - locations->SetInAt(2, Location::RegisterLocation(calling_convention.GetRegisterAt(2))); - locations->SetInAt(3, Location::RegisterLocation(calling_convention.GetRegisterAt(3))); - locations->SetOut(Location::RegisterLocation(R0)); -} - -void IntrinsicCodeGeneratorARM::VisitStringNewStringFromBytes(HInvoke* invoke) { - ArmAssembler* assembler = GetAssembler(); - LocationSummary* locations = invoke->GetLocations(); - - Register byte_array = locations->InAt(0).AsRegister<Register>(); - __ cmp(byte_array, ShifterOperand(0)); - SlowPathCode* slow_path = new (GetAllocator()) IntrinsicSlowPathARM(invoke); - codegen_->AddSlowPath(slow_path); - __ b(slow_path->GetEntryLabel(), EQ); - - codegen_->InvokeRuntime(kQuickAllocStringFromBytes, invoke, invoke->GetDexPc(), slow_path); - CheckEntrypointTypes<kQuickAllocStringFromBytes, void*, void*, int32_t, int32_t, int32_t>(); - __ Bind(slow_path->GetExitLabel()); -} - -void IntrinsicLocationsBuilderARM::VisitStringNewStringFromChars(HInvoke* invoke) { - LocationSummary* locations = new (arena_) LocationSummary(invoke, - LocationSummary::kCallOnMainOnly, - kIntrinsified); - InvokeRuntimeCallingConvention calling_convention; - locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); - locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); - locations->SetInAt(2, Location::RegisterLocation(calling_convention.GetRegisterAt(2))); - locations->SetOut(Location::RegisterLocation(R0)); -} - -void IntrinsicCodeGeneratorARM::VisitStringNewStringFromChars(HInvoke* invoke) { - // No need to emit code checking whether `locations->InAt(2)` is a null - // pointer, as callers of the native method - // - // java.lang.StringFactory.newStringFromChars(int offset, int charCount, char[] data) - // - // all include a null check on `data` before calling that method. - codegen_->InvokeRuntime(kQuickAllocStringFromChars, invoke, invoke->GetDexPc()); - CheckEntrypointTypes<kQuickAllocStringFromChars, void*, int32_t, int32_t, void*>(); -} - -void IntrinsicLocationsBuilderARM::VisitStringNewStringFromString(HInvoke* invoke) { - LocationSummary* locations = new (arena_) LocationSummary(invoke, - LocationSummary::kCallOnMainAndSlowPath, - kIntrinsified); - InvokeRuntimeCallingConvention calling_convention; - locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); - locations->SetOut(Location::RegisterLocation(R0)); -} - -void IntrinsicCodeGeneratorARM::VisitStringNewStringFromString(HInvoke* invoke) { - ArmAssembler* assembler = GetAssembler(); - LocationSummary* locations = invoke->GetLocations(); - - Register string_to_copy = locations->InAt(0).AsRegister<Register>(); - __ cmp(string_to_copy, ShifterOperand(0)); - SlowPathCode* slow_path = new (GetAllocator()) IntrinsicSlowPathARM(invoke); - codegen_->AddSlowPath(slow_path); - __ b(slow_path->GetEntryLabel(), EQ); - - codegen_->InvokeRuntime(kQuickAllocStringFromString, invoke, invoke->GetDexPc(), slow_path); - CheckEntrypointTypes<kQuickAllocStringFromString, void*, void*>(); - - __ Bind(slow_path->GetExitLabel()); -} - -void IntrinsicLocationsBuilderARM::VisitSystemArrayCopy(HInvoke* invoke) { - // The only read barrier implementation supporting the - // SystemArrayCopy intrinsic is the Baker-style read barriers. - if (kEmitCompilerReadBarrier && !kUseBakerReadBarrier) { - return; - } - - CodeGenerator::CreateSystemArrayCopyLocationSummary(invoke); - LocationSummary* locations = invoke->GetLocations(); - if (locations == nullptr) { - return; - } - - HIntConstant* src_pos = invoke->InputAt(1)->AsIntConstant(); - HIntConstant* dest_pos = invoke->InputAt(3)->AsIntConstant(); - HIntConstant* length = invoke->InputAt(4)->AsIntConstant(); - - if (src_pos != nullptr && !assembler_->ShifterOperandCanAlwaysHold(src_pos->GetValue())) { - locations->SetInAt(1, Location::RequiresRegister()); - } - if (dest_pos != nullptr && !assembler_->ShifterOperandCanAlwaysHold(dest_pos->GetValue())) { - locations->SetInAt(3, Location::RequiresRegister()); - } - if (length != nullptr && !assembler_->ShifterOperandCanAlwaysHold(length->GetValue())) { - locations->SetInAt(4, Location::RequiresRegister()); - } - if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { - // Temporary register IP cannot be used in - // ReadBarrierSystemArrayCopySlowPathARM (because that register - // is clobbered by ReadBarrierMarkRegX entry points). Get an extra - // temporary register from the register allocator. - locations->AddTemp(Location::RequiresRegister()); - CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen_); - arm_codegen->MaybeAddBakerCcEntrypointTempForFields(locations); - } -} - -static void CheckPosition(ArmAssembler* assembler, - Location pos, - Register input, - Location length, - SlowPathCode* slow_path, - Register temp, - bool length_is_input_length = false) { - // Where is the length in the Array? - const uint32_t length_offset = mirror::Array::LengthOffset().Uint32Value(); - - if (pos.IsConstant()) { - int32_t pos_const = pos.GetConstant()->AsIntConstant()->GetValue(); - if (pos_const == 0) { - if (!length_is_input_length) { - // Check that length(input) >= length. - __ LoadFromOffset(kLoadWord, temp, input, length_offset); - if (length.IsConstant()) { - __ cmp(temp, ShifterOperand(length.GetConstant()->AsIntConstant()->GetValue())); - } else { - __ cmp(temp, ShifterOperand(length.AsRegister<Register>())); - } - __ b(slow_path->GetEntryLabel(), LT); - } - } else { - // Check that length(input) >= pos. - __ LoadFromOffset(kLoadWord, temp, input, length_offset); - __ subs(temp, temp, ShifterOperand(pos_const)); - __ b(slow_path->GetEntryLabel(), LT); - - // Check that (length(input) - pos) >= length. - if (length.IsConstant()) { - __ cmp(temp, ShifterOperand(length.GetConstant()->AsIntConstant()->GetValue())); - } else { - __ cmp(temp, ShifterOperand(length.AsRegister<Register>())); - } - __ b(slow_path->GetEntryLabel(), LT); - } - } else if (length_is_input_length) { - // The only way the copy can succeed is if pos is zero. - Register pos_reg = pos.AsRegister<Register>(); - __ CompareAndBranchIfNonZero(pos_reg, slow_path->GetEntryLabel()); - } else { - // Check that pos >= 0. - Register pos_reg = pos.AsRegister<Register>(); - __ cmp(pos_reg, ShifterOperand(0)); - __ b(slow_path->GetEntryLabel(), LT); - - // Check that pos <= length(input). - __ LoadFromOffset(kLoadWord, temp, input, length_offset); - __ subs(temp, temp, ShifterOperand(pos_reg)); - __ b(slow_path->GetEntryLabel(), LT); - - // Check that (length(input) - pos) >= length. - if (length.IsConstant()) { - __ cmp(temp, ShifterOperand(length.GetConstant()->AsIntConstant()->GetValue())); - } else { - __ cmp(temp, ShifterOperand(length.AsRegister<Register>())); - } - __ b(slow_path->GetEntryLabel(), LT); - } -} - -void IntrinsicCodeGeneratorARM::VisitSystemArrayCopy(HInvoke* invoke) { - // The only read barrier implementation supporting the - // SystemArrayCopy intrinsic is the Baker-style read barriers. - DCHECK(!kEmitCompilerReadBarrier || kUseBakerReadBarrier); - - ArmAssembler* assembler = GetAssembler(); - LocationSummary* locations = invoke->GetLocations(); - - uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); - uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); - uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); - uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value(); - uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value(); - - Register src = locations->InAt(0).AsRegister<Register>(); - Location src_pos = locations->InAt(1); - Register dest = locations->InAt(2).AsRegister<Register>(); - Location dest_pos = locations->InAt(3); - Location length = locations->InAt(4); - Location temp1_loc = locations->GetTemp(0); - Register temp1 = temp1_loc.AsRegister<Register>(); - Location temp2_loc = locations->GetTemp(1); - Register temp2 = temp2_loc.AsRegister<Register>(); - Location temp3_loc = locations->GetTemp(2); - Register temp3 = temp3_loc.AsRegister<Register>(); - - SlowPathCode* intrinsic_slow_path = new (GetAllocator()) IntrinsicSlowPathARM(invoke); - codegen_->AddSlowPath(intrinsic_slow_path); - - Label conditions_on_positions_validated; - SystemArrayCopyOptimizations optimizations(invoke); - - // If source and destination are the same, we go to slow path if we need to do - // forward copying. - if (src_pos.IsConstant()) { - int32_t src_pos_constant = src_pos.GetConstant()->AsIntConstant()->GetValue(); - if (dest_pos.IsConstant()) { - int32_t dest_pos_constant = dest_pos.GetConstant()->AsIntConstant()->GetValue(); - if (optimizations.GetDestinationIsSource()) { - // Checked when building locations. - DCHECK_GE(src_pos_constant, dest_pos_constant); - } else if (src_pos_constant < dest_pos_constant) { - __ cmp(src, ShifterOperand(dest)); - __ b(intrinsic_slow_path->GetEntryLabel(), EQ); - } - - // Checked when building locations. - DCHECK(!optimizations.GetDestinationIsSource() - || (src_pos_constant >= dest_pos.GetConstant()->AsIntConstant()->GetValue())); - } else { - if (!optimizations.GetDestinationIsSource()) { - __ cmp(src, ShifterOperand(dest)); - __ b(&conditions_on_positions_validated, NE); - } - __ cmp(dest_pos.AsRegister<Register>(), ShifterOperand(src_pos_constant)); - __ b(intrinsic_slow_path->GetEntryLabel(), GT); - } - } else { - if (!optimizations.GetDestinationIsSource()) { - __ cmp(src, ShifterOperand(dest)); - __ b(&conditions_on_positions_validated, NE); - } - if (dest_pos.IsConstant()) { - int32_t dest_pos_constant = dest_pos.GetConstant()->AsIntConstant()->GetValue(); - __ cmp(src_pos.AsRegister<Register>(), ShifterOperand(dest_pos_constant)); - } else { - __ cmp(src_pos.AsRegister<Register>(), ShifterOperand(dest_pos.AsRegister<Register>())); - } - __ b(intrinsic_slow_path->GetEntryLabel(), LT); - } - - __ Bind(&conditions_on_positions_validated); - - if (!optimizations.GetSourceIsNotNull()) { - // Bail out if the source is null. - __ CompareAndBranchIfZero(src, intrinsic_slow_path->GetEntryLabel()); - } - - if (!optimizations.GetDestinationIsNotNull() && !optimizations.GetDestinationIsSource()) { - // Bail out if the destination is null. - __ CompareAndBranchIfZero(dest, intrinsic_slow_path->GetEntryLabel()); - } - - // If the length is negative, bail out. - // We have already checked in the LocationsBuilder for the constant case. - if (!length.IsConstant() && - !optimizations.GetCountIsSourceLength() && - !optimizations.GetCountIsDestinationLength()) { - __ cmp(length.AsRegister<Register>(), ShifterOperand(0)); - __ b(intrinsic_slow_path->GetEntryLabel(), LT); - } - - // Validity checks: source. - CheckPosition(assembler, - src_pos, - src, - length, - intrinsic_slow_path, - temp1, - optimizations.GetCountIsSourceLength()); - - // Validity checks: dest. - CheckPosition(assembler, - dest_pos, - dest, - length, - intrinsic_slow_path, - temp1, - optimizations.GetCountIsDestinationLength()); - - if (!optimizations.GetDoesNotNeedTypeCheck()) { - // Check whether all elements of the source array are assignable to the component - // type of the destination array. We do two checks: the classes are the same, - // or the destination is Object[]. If none of these checks succeed, we go to the - // slow path. - - if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { - if (!optimizations.GetSourceIsNonPrimitiveArray()) { - // /* HeapReference<Class> */ temp1 = src->klass_ - codegen_->GenerateFieldLoadWithBakerReadBarrier( - invoke, temp1_loc, src, class_offset, temp2_loc, /* needs_null_check */ false); - // Bail out if the source is not a non primitive array. - // /* HeapReference<Class> */ temp1 = temp1->component_type_ - codegen_->GenerateFieldLoadWithBakerReadBarrier( - invoke, temp1_loc, temp1, component_offset, temp2_loc, /* needs_null_check */ false); - __ CompareAndBranchIfZero(temp1, intrinsic_slow_path->GetEntryLabel()); - // If heap poisoning is enabled, `temp1` has been unpoisoned - // by the the previous call to GenerateFieldLoadWithBakerReadBarrier. - // /* uint16_t */ temp1 = static_cast<uint16>(temp1->primitive_type_); - __ LoadFromOffset(kLoadUnsignedHalfword, temp1, temp1, primitive_offset); - static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot"); - __ CompareAndBranchIfNonZero(temp1, intrinsic_slow_path->GetEntryLabel()); - } - - // /* HeapReference<Class> */ temp1 = dest->klass_ - codegen_->GenerateFieldLoadWithBakerReadBarrier( - invoke, temp1_loc, dest, class_offset, temp2_loc, /* needs_null_check */ false); - - if (!optimizations.GetDestinationIsNonPrimitiveArray()) { - // Bail out if the destination is not a non primitive array. - // - // Register `temp1` is not trashed by the read barrier emitted - // by GenerateFieldLoadWithBakerReadBarrier below, as that - // method produces a call to a ReadBarrierMarkRegX entry point, - // which saves all potentially live registers, including - // temporaries such a `temp1`. - // /* HeapReference<Class> */ temp2 = temp1->component_type_ - codegen_->GenerateFieldLoadWithBakerReadBarrier( - invoke, temp2_loc, temp1, component_offset, temp3_loc, /* needs_null_check */ false); - __ CompareAndBranchIfZero(temp2, intrinsic_slow_path->GetEntryLabel()); - // If heap poisoning is enabled, `temp2` has been unpoisoned - // by the the previous call to GenerateFieldLoadWithBakerReadBarrier. - // /* uint16_t */ temp2 = static_cast<uint16>(temp2->primitive_type_); - __ LoadFromOffset(kLoadUnsignedHalfword, temp2, temp2, primitive_offset); - static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot"); - __ CompareAndBranchIfNonZero(temp2, intrinsic_slow_path->GetEntryLabel()); - } - - // For the same reason given earlier, `temp1` is not trashed by the - // read barrier emitted by GenerateFieldLoadWithBakerReadBarrier below. - // /* HeapReference<Class> */ temp2 = src->klass_ - codegen_->GenerateFieldLoadWithBakerReadBarrier( - invoke, temp2_loc, src, class_offset, temp3_loc, /* needs_null_check */ false); - // Note: if heap poisoning is on, we are comparing two unpoisoned references here. - __ cmp(temp1, ShifterOperand(temp2)); - - if (optimizations.GetDestinationIsTypedObjectArray()) { - Label do_copy; - __ b(&do_copy, EQ); - // /* HeapReference<Class> */ temp1 = temp1->component_type_ - codegen_->GenerateFieldLoadWithBakerReadBarrier( - invoke, temp1_loc, temp1, component_offset, temp2_loc, /* needs_null_check */ false); - // /* HeapReference<Class> */ temp1 = temp1->super_class_ - // We do not need to emit a read barrier for the following - // heap reference load, as `temp1` is only used in a - // comparison with null below, and this reference is not - // kept afterwards. - __ LoadFromOffset(kLoadWord, temp1, temp1, super_offset); - __ CompareAndBranchIfNonZero(temp1, intrinsic_slow_path->GetEntryLabel()); - __ Bind(&do_copy); - } else { - __ b(intrinsic_slow_path->GetEntryLabel(), NE); - } - } else { - // Non read barrier code. - - // /* HeapReference<Class> */ temp1 = dest->klass_ - __ LoadFromOffset(kLoadWord, temp1, dest, class_offset); - // /* HeapReference<Class> */ temp2 = src->klass_ - __ LoadFromOffset(kLoadWord, temp2, src, class_offset); - bool did_unpoison = false; - if (!optimizations.GetDestinationIsNonPrimitiveArray() || - !optimizations.GetSourceIsNonPrimitiveArray()) { - // One or two of the references need to be unpoisoned. Unpoison them - // both to make the identity check valid. - __ MaybeUnpoisonHeapReference(temp1); - __ MaybeUnpoisonHeapReference(temp2); - did_unpoison = true; - } - - if (!optimizations.GetDestinationIsNonPrimitiveArray()) { - // Bail out if the destination is not a non primitive array. - // /* HeapReference<Class> */ temp3 = temp1->component_type_ - __ LoadFromOffset(kLoadWord, temp3, temp1, component_offset); - __ CompareAndBranchIfZero(temp3, intrinsic_slow_path->GetEntryLabel()); - __ MaybeUnpoisonHeapReference(temp3); - // /* uint16_t */ temp3 = static_cast<uint16>(temp3->primitive_type_); - __ LoadFromOffset(kLoadUnsignedHalfword, temp3, temp3, primitive_offset); - static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot"); - __ CompareAndBranchIfNonZero(temp3, intrinsic_slow_path->GetEntryLabel()); - } - - if (!optimizations.GetSourceIsNonPrimitiveArray()) { - // Bail out if the source is not a non primitive array. - // /* HeapReference<Class> */ temp3 = temp2->component_type_ - __ LoadFromOffset(kLoadWord, temp3, temp2, component_offset); - __ CompareAndBranchIfZero(temp3, intrinsic_slow_path->GetEntryLabel()); - __ MaybeUnpoisonHeapReference(temp3); - // /* uint16_t */ temp3 = static_cast<uint16>(temp3->primitive_type_); - __ LoadFromOffset(kLoadUnsignedHalfword, temp3, temp3, primitive_offset); - static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot"); - __ CompareAndBranchIfNonZero(temp3, intrinsic_slow_path->GetEntryLabel()); - } - - __ cmp(temp1, ShifterOperand(temp2)); - - if (optimizations.GetDestinationIsTypedObjectArray()) { - Label do_copy; - __ b(&do_copy, EQ); - if (!did_unpoison) { - __ MaybeUnpoisonHeapReference(temp1); - } - // /* HeapReference<Class> */ temp1 = temp1->component_type_ - __ LoadFromOffset(kLoadWord, temp1, temp1, component_offset); - __ MaybeUnpoisonHeapReference(temp1); - // /* HeapReference<Class> */ temp1 = temp1->super_class_ - __ LoadFromOffset(kLoadWord, temp1, temp1, super_offset); - // No need to unpoison the result, we're comparing against null. - __ CompareAndBranchIfNonZero(temp1, intrinsic_slow_path->GetEntryLabel()); - __ Bind(&do_copy); - } else { - __ b(intrinsic_slow_path->GetEntryLabel(), NE); - } - } - } else if (!optimizations.GetSourceIsNonPrimitiveArray()) { - DCHECK(optimizations.GetDestinationIsNonPrimitiveArray()); - // Bail out if the source is not a non primitive array. - if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { - // /* HeapReference<Class> */ temp1 = src->klass_ - codegen_->GenerateFieldLoadWithBakerReadBarrier( - invoke, temp1_loc, src, class_offset, temp2_loc, /* needs_null_check */ false); - // /* HeapReference<Class> */ temp3 = temp1->component_type_ - codegen_->GenerateFieldLoadWithBakerReadBarrier( - invoke, temp3_loc, temp1, component_offset, temp2_loc, /* needs_null_check */ false); - __ CompareAndBranchIfZero(temp3, intrinsic_slow_path->GetEntryLabel()); - // If heap poisoning is enabled, `temp3` has been unpoisoned - // by the the previous call to GenerateFieldLoadWithBakerReadBarrier. - } else { - // /* HeapReference<Class> */ temp1 = src->klass_ - __ LoadFromOffset(kLoadWord, temp1, src, class_offset); - __ MaybeUnpoisonHeapReference(temp1); - // /* HeapReference<Class> */ temp3 = temp1->component_type_ - __ LoadFromOffset(kLoadWord, temp3, temp1, component_offset); - __ CompareAndBranchIfZero(temp3, intrinsic_slow_path->GetEntryLabel()); - __ MaybeUnpoisonHeapReference(temp3); - } - // /* uint16_t */ temp3 = static_cast<uint16>(temp3->primitive_type_); - __ LoadFromOffset(kLoadUnsignedHalfword, temp3, temp3, primitive_offset); - static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot"); - __ CompareAndBranchIfNonZero(temp3, intrinsic_slow_path->GetEntryLabel()); - } - - if (length.IsConstant() && length.GetConstant()->AsIntConstant()->GetValue() == 0) { - // Null constant length: not need to emit the loop code at all. - } else { - Label done; - const Primitive::Type type = Primitive::kPrimNot; - const int32_t element_size = Primitive::ComponentSize(type); - - if (length.IsRegister()) { - // Don't enter the copy loop if the length is null. - __ CompareAndBranchIfZero(length.AsRegister<Register>(), &done); - } - - if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { - // TODO: Also convert this intrinsic to the IsGcMarking strategy? - - // SystemArrayCopy implementation for Baker read barriers (see - // also CodeGeneratorARM::GenerateReferenceLoadWithBakerReadBarrier): - // - // uint32_t rb_state = Lockword(src->monitor_).ReadBarrierState(); - // lfence; // Load fence or artificial data dependency to prevent load-load reordering - // bool is_gray = (rb_state == ReadBarrier::GrayState()); - // if (is_gray) { - // // Slow-path copy. - // do { - // *dest_ptr++ = MaybePoison(ReadBarrier::Mark(MaybeUnpoison(*src_ptr++))); - // } while (src_ptr != end_ptr) - // } else { - // // Fast-path copy. - // do { - // *dest_ptr++ = *src_ptr++; - // } while (src_ptr != end_ptr) - // } - - // /* int32_t */ monitor = src->monitor_ - __ LoadFromOffset(kLoadWord, temp2, src, monitor_offset); - // /* LockWord */ lock_word = LockWord(monitor) - static_assert(sizeof(LockWord) == sizeof(int32_t), - "art::LockWord and int32_t have different sizes."); - - // Introduce a dependency on the lock_word including the rb_state, - // which shall prevent load-load reordering without using - // a memory barrier (which would be more expensive). - // `src` is unchanged by this operation, but its value now depends - // on `temp2`. - __ add(src, src, ShifterOperand(temp2, LSR, 32)); - - // Compute the base source address in `temp1`. - // Note that `temp1` (the base source address) is computed from - // `src` (and `src_pos`) here, and thus honors the artificial - // dependency of `src` on `temp2`. - GenSystemArrayCopyBaseAddress(GetAssembler(), type, src, src_pos, temp1); - // Compute the end source address in `temp3`. - GenSystemArrayCopyEndAddress(GetAssembler(), type, length, temp1, temp3); - // The base destination address is computed later, as `temp2` is - // used for intermediate computations. - - // Slow path used to copy array when `src` is gray. - // Note that the base destination address is computed in `temp2` - // by the slow path code. - SlowPathCode* read_barrier_slow_path = - new (GetAllocator()) ReadBarrierSystemArrayCopySlowPathARM(invoke); - codegen_->AddSlowPath(read_barrier_slow_path); - - // Given the numeric representation, it's enough to check the low bit of the - // rb_state. We do that by shifting the bit out of the lock word with LSRS - // which can be a 16-bit instruction unlike the TST immediate. - static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0"); - static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1"); - __ Lsrs(temp2, temp2, LockWord::kReadBarrierStateShift + 1); - // Carry flag is the last bit shifted out by LSRS. - __ b(read_barrier_slow_path->GetEntryLabel(), CS); - - // Fast-path copy. - // Compute the base destination address in `temp2`. - GenSystemArrayCopyBaseAddress(GetAssembler(), type, dest, dest_pos, temp2); - // Iterate over the arrays and do a raw copy of the objects. We don't need to - // poison/unpoison. - Label loop; - __ Bind(&loop); - __ ldr(IP, Address(temp1, element_size, Address::PostIndex)); - __ str(IP, Address(temp2, element_size, Address::PostIndex)); - __ cmp(temp1, ShifterOperand(temp3)); - __ b(&loop, NE); - - __ Bind(read_barrier_slow_path->GetExitLabel()); - } else { - // Non read barrier code. - // Compute the base source address in `temp1`. - GenSystemArrayCopyBaseAddress(GetAssembler(), type, src, src_pos, temp1); - // Compute the base destination address in `temp2`. - GenSystemArrayCopyBaseAddress(GetAssembler(), type, dest, dest_pos, temp2); - // Compute the end source address in `temp3`. - GenSystemArrayCopyEndAddress(GetAssembler(), type, length, temp1, temp3); - // Iterate over the arrays and do a raw copy of the objects. We don't need to - // poison/unpoison. - Label loop; - __ Bind(&loop); - __ ldr(IP, Address(temp1, element_size, Address::PostIndex)); - __ str(IP, Address(temp2, element_size, Address::PostIndex)); - __ cmp(temp1, ShifterOperand(temp3)); - __ b(&loop, NE); - } - __ Bind(&done); - } - - // We only need one card marking on the destination array. - codegen_->MarkGCCard(temp1, temp2, dest, Register(kNoRegister), /* value_can_be_null */ false); - - __ Bind(intrinsic_slow_path->GetExitLabel()); -} - -static void CreateFPToFPCallLocations(ArenaAllocator* arena, HInvoke* invoke) { - // If the graph is debuggable, all callee-saved floating-point registers are blocked by - // the code generator. Furthermore, the register allocator creates fixed live intervals - // for all caller-saved registers because we are doing a function call. As a result, if - // the input and output locations are unallocated, the register allocator runs out of - // registers and fails; however, a debuggable graph is not the common case. - if (invoke->GetBlock()->GetGraph()->IsDebuggable()) { - return; - } - - DCHECK_EQ(invoke->GetNumberOfArguments(), 1U); - DCHECK_EQ(invoke->InputAt(0)->GetType(), Primitive::kPrimDouble); - DCHECK_EQ(invoke->GetType(), Primitive::kPrimDouble); - - LocationSummary* const locations = new (arena) LocationSummary(invoke, - LocationSummary::kCallOnMainOnly, - kIntrinsified); - const InvokeRuntimeCallingConvention calling_convention; - - locations->SetInAt(0, Location::RequiresFpuRegister()); - locations->SetOut(Location::RequiresFpuRegister()); - // Native code uses the soft float ABI. - locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); - locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(1))); -} - -static void CreateFPFPToFPCallLocations(ArenaAllocator* arena, HInvoke* invoke) { - // If the graph is debuggable, all callee-saved floating-point registers are blocked by - // the code generator. Furthermore, the register allocator creates fixed live intervals - // for all caller-saved registers because we are doing a function call. As a result, if - // the input and output locations are unallocated, the register allocator runs out of - // registers and fails; however, a debuggable graph is not the common case. - if (invoke->GetBlock()->GetGraph()->IsDebuggable()) { - return; - } - - DCHECK_EQ(invoke->GetNumberOfArguments(), 2U); - DCHECK_EQ(invoke->InputAt(0)->GetType(), Primitive::kPrimDouble); - DCHECK_EQ(invoke->InputAt(1)->GetType(), Primitive::kPrimDouble); - DCHECK_EQ(invoke->GetType(), Primitive::kPrimDouble); - - LocationSummary* const locations = new (arena) LocationSummary(invoke, - LocationSummary::kCallOnMainOnly, - kIntrinsified); - const InvokeRuntimeCallingConvention calling_convention; - - locations->SetInAt(0, Location::RequiresFpuRegister()); - locations->SetInAt(1, Location::RequiresFpuRegister()); - locations->SetOut(Location::RequiresFpuRegister()); - // Native code uses the soft float ABI. - locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); - locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(1))); - locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(2))); - locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(3))); -} - -static void GenFPToFPCall(HInvoke* invoke, - ArmAssembler* assembler, - CodeGeneratorARM* codegen, - QuickEntrypointEnum entry) { - LocationSummary* const locations = invoke->GetLocations(); - const InvokeRuntimeCallingConvention calling_convention; - - DCHECK_EQ(invoke->GetNumberOfArguments(), 1U); - DCHECK(locations->WillCall() && locations->Intrinsified()); - DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(calling_convention.GetRegisterAt(0))); - DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(calling_convention.GetRegisterAt(1))); - - // Native code uses the soft float ABI. - __ vmovrrd(calling_convention.GetRegisterAt(0), - calling_convention.GetRegisterAt(1), - FromLowSToD(locations->InAt(0).AsFpuRegisterPairLow<SRegister>())); - codegen->InvokeRuntime(entry, invoke, invoke->GetDexPc()); - __ vmovdrr(FromLowSToD(locations->Out().AsFpuRegisterPairLow<SRegister>()), - calling_convention.GetRegisterAt(0), - calling_convention.GetRegisterAt(1)); -} - -static void GenFPFPToFPCall(HInvoke* invoke, - ArmAssembler* assembler, - CodeGeneratorARM* codegen, - QuickEntrypointEnum entry) { - LocationSummary* const locations = invoke->GetLocations(); - const InvokeRuntimeCallingConvention calling_convention; - - DCHECK_EQ(invoke->GetNumberOfArguments(), 2U); - DCHECK(locations->WillCall() && locations->Intrinsified()); - DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(calling_convention.GetRegisterAt(0))); - DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(calling_convention.GetRegisterAt(1))); - DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(calling_convention.GetRegisterAt(2))); - DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(calling_convention.GetRegisterAt(3))); - - // Native code uses the soft float ABI. - __ vmovrrd(calling_convention.GetRegisterAt(0), - calling_convention.GetRegisterAt(1), - FromLowSToD(locations->InAt(0).AsFpuRegisterPairLow<SRegister>())); - __ vmovrrd(calling_convention.GetRegisterAt(2), - calling_convention.GetRegisterAt(3), - FromLowSToD(locations->InAt(1).AsFpuRegisterPairLow<SRegister>())); - codegen->InvokeRuntime(entry, invoke, invoke->GetDexPc()); - __ vmovdrr(FromLowSToD(locations->Out().AsFpuRegisterPairLow<SRegister>()), - calling_convention.GetRegisterAt(0), - calling_convention.GetRegisterAt(1)); -} - -void IntrinsicLocationsBuilderARM::VisitMathCos(HInvoke* invoke) { - CreateFPToFPCallLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMathCos(HInvoke* invoke) { - GenFPToFPCall(invoke, GetAssembler(), codegen_, kQuickCos); -} - -void IntrinsicLocationsBuilderARM::VisitMathSin(HInvoke* invoke) { - CreateFPToFPCallLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMathSin(HInvoke* invoke) { - GenFPToFPCall(invoke, GetAssembler(), codegen_, kQuickSin); -} - -void IntrinsicLocationsBuilderARM::VisitMathAcos(HInvoke* invoke) { - CreateFPToFPCallLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMathAcos(HInvoke* invoke) { - GenFPToFPCall(invoke, GetAssembler(), codegen_, kQuickAcos); -} - -void IntrinsicLocationsBuilderARM::VisitMathAsin(HInvoke* invoke) { - CreateFPToFPCallLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMathAsin(HInvoke* invoke) { - GenFPToFPCall(invoke, GetAssembler(), codegen_, kQuickAsin); -} - -void IntrinsicLocationsBuilderARM::VisitMathAtan(HInvoke* invoke) { - CreateFPToFPCallLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMathAtan(HInvoke* invoke) { - GenFPToFPCall(invoke, GetAssembler(), codegen_, kQuickAtan); -} - -void IntrinsicLocationsBuilderARM::VisitMathCbrt(HInvoke* invoke) { - CreateFPToFPCallLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMathCbrt(HInvoke* invoke) { - GenFPToFPCall(invoke, GetAssembler(), codegen_, kQuickCbrt); -} - -void IntrinsicLocationsBuilderARM::VisitMathCosh(HInvoke* invoke) { - CreateFPToFPCallLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMathCosh(HInvoke* invoke) { - GenFPToFPCall(invoke, GetAssembler(), codegen_, kQuickCosh); -} - -void IntrinsicLocationsBuilderARM::VisitMathExp(HInvoke* invoke) { - CreateFPToFPCallLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMathExp(HInvoke* invoke) { - GenFPToFPCall(invoke, GetAssembler(), codegen_, kQuickExp); -} - -void IntrinsicLocationsBuilderARM::VisitMathExpm1(HInvoke* invoke) { - CreateFPToFPCallLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMathExpm1(HInvoke* invoke) { - GenFPToFPCall(invoke, GetAssembler(), codegen_, kQuickExpm1); -} - -void IntrinsicLocationsBuilderARM::VisitMathLog(HInvoke* invoke) { - CreateFPToFPCallLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMathLog(HInvoke* invoke) { - GenFPToFPCall(invoke, GetAssembler(), codegen_, kQuickLog); -} - -void IntrinsicLocationsBuilderARM::VisitMathLog10(HInvoke* invoke) { - CreateFPToFPCallLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMathLog10(HInvoke* invoke) { - GenFPToFPCall(invoke, GetAssembler(), codegen_, kQuickLog10); -} - -void IntrinsicLocationsBuilderARM::VisitMathSinh(HInvoke* invoke) { - CreateFPToFPCallLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMathSinh(HInvoke* invoke) { - GenFPToFPCall(invoke, GetAssembler(), codegen_, kQuickSinh); -} - -void IntrinsicLocationsBuilderARM::VisitMathTan(HInvoke* invoke) { - CreateFPToFPCallLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMathTan(HInvoke* invoke) { - GenFPToFPCall(invoke, GetAssembler(), codegen_, kQuickTan); -} - -void IntrinsicLocationsBuilderARM::VisitMathTanh(HInvoke* invoke) { - CreateFPToFPCallLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMathTanh(HInvoke* invoke) { - GenFPToFPCall(invoke, GetAssembler(), codegen_, kQuickTanh); -} - -void IntrinsicLocationsBuilderARM::VisitMathAtan2(HInvoke* invoke) { - CreateFPFPToFPCallLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMathAtan2(HInvoke* invoke) { - GenFPFPToFPCall(invoke, GetAssembler(), codegen_, kQuickAtan2); -} - -void IntrinsicLocationsBuilderARM::VisitMathHypot(HInvoke* invoke) { - CreateFPFPToFPCallLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMathHypot(HInvoke* invoke) { - GenFPFPToFPCall(invoke, GetAssembler(), codegen_, kQuickHypot); -} - -void IntrinsicLocationsBuilderARM::VisitMathNextAfter(HInvoke* invoke) { - CreateFPFPToFPCallLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitMathNextAfter(HInvoke* invoke) { - GenFPFPToFPCall(invoke, GetAssembler(), codegen_, kQuickNextAfter); -} - -void IntrinsicLocationsBuilderARM::VisitIntegerReverse(HInvoke* invoke) { - CreateIntToIntLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitIntegerReverse(HInvoke* invoke) { - ArmAssembler* assembler = GetAssembler(); - LocationSummary* locations = invoke->GetLocations(); - - Register out = locations->Out().AsRegister<Register>(); - Register in = locations->InAt(0).AsRegister<Register>(); - - __ rbit(out, in); -} - -void IntrinsicLocationsBuilderARM::VisitLongReverse(HInvoke* invoke) { - LocationSummary* locations = new (arena_) LocationSummary(invoke, - LocationSummary::kNoCall, - kIntrinsified); - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); -} - -void IntrinsicCodeGeneratorARM::VisitLongReverse(HInvoke* invoke) { - ArmAssembler* assembler = GetAssembler(); - LocationSummary* locations = invoke->GetLocations(); - - Register in_reg_lo = locations->InAt(0).AsRegisterPairLow<Register>(); - Register in_reg_hi = locations->InAt(0).AsRegisterPairHigh<Register>(); - Register out_reg_lo = locations->Out().AsRegisterPairLow<Register>(); - Register out_reg_hi = locations->Out().AsRegisterPairHigh<Register>(); - - __ rbit(out_reg_lo, in_reg_hi); - __ rbit(out_reg_hi, in_reg_lo); -} - -void IntrinsicLocationsBuilderARM::VisitIntegerReverseBytes(HInvoke* invoke) { - CreateIntToIntLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitIntegerReverseBytes(HInvoke* invoke) { - ArmAssembler* assembler = GetAssembler(); - LocationSummary* locations = invoke->GetLocations(); - - Register out = locations->Out().AsRegister<Register>(); - Register in = locations->InAt(0).AsRegister<Register>(); - - __ rev(out, in); -} - -void IntrinsicLocationsBuilderARM::VisitLongReverseBytes(HInvoke* invoke) { - LocationSummary* locations = new (arena_) LocationSummary(invoke, - LocationSummary::kNoCall, - kIntrinsified); - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); -} - -void IntrinsicCodeGeneratorARM::VisitLongReverseBytes(HInvoke* invoke) { - ArmAssembler* assembler = GetAssembler(); - LocationSummary* locations = invoke->GetLocations(); - - Register in_reg_lo = locations->InAt(0).AsRegisterPairLow<Register>(); - Register in_reg_hi = locations->InAt(0).AsRegisterPairHigh<Register>(); - Register out_reg_lo = locations->Out().AsRegisterPairLow<Register>(); - Register out_reg_hi = locations->Out().AsRegisterPairHigh<Register>(); - - __ rev(out_reg_lo, in_reg_hi); - __ rev(out_reg_hi, in_reg_lo); -} - -void IntrinsicLocationsBuilderARM::VisitShortReverseBytes(HInvoke* invoke) { - CreateIntToIntLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitShortReverseBytes(HInvoke* invoke) { - ArmAssembler* assembler = GetAssembler(); - LocationSummary* locations = invoke->GetLocations(); - - Register out = locations->Out().AsRegister<Register>(); - Register in = locations->InAt(0).AsRegister<Register>(); - - __ revsh(out, in); -} - -static void GenBitCount(HInvoke* instr, Primitive::Type type, ArmAssembler* assembler) { - DCHECK(Primitive::IsIntOrLongType(type)) << type; - DCHECK_EQ(instr->GetType(), Primitive::kPrimInt); - DCHECK_EQ(Primitive::PrimitiveKind(instr->InputAt(0)->GetType()), type); - - bool is_long = type == Primitive::kPrimLong; - LocationSummary* locations = instr->GetLocations(); - Location in = locations->InAt(0); - Register src_0 = is_long ? in.AsRegisterPairLow<Register>() : in.AsRegister<Register>(); - Register src_1 = is_long ? in.AsRegisterPairHigh<Register>() : src_0; - SRegister tmp_s = locations->GetTemp(0).AsFpuRegisterPairLow<SRegister>(); - DRegister tmp_d = FromLowSToD(tmp_s); - Register out_r = locations->Out().AsRegister<Register>(); - - // Move data from core register(s) to temp D-reg for bit count calculation, then move back. - // According to Cortex A57 and A72 optimization guides, compared to transferring to full D-reg, - // transferring data from core reg to upper or lower half of vfp D-reg requires extra latency, - // That's why for integer bit count, we use 'vmov d0, r0, r0' instead of 'vmov d0[0], r0'. - __ vmovdrr(tmp_d, src_1, src_0); // Temp DReg |--src_1|--src_0| - __ vcntd(tmp_d, tmp_d); // Temp DReg |c|c|c|c|c|c|c|c| - __ vpaddld(tmp_d, tmp_d, 8, /* is_unsigned */ true); // Temp DReg |--c|--c|--c|--c| - __ vpaddld(tmp_d, tmp_d, 16, /* is_unsigned */ true); // Temp DReg |------c|------c| - if (is_long) { - __ vpaddld(tmp_d, tmp_d, 32, /* is_unsigned */ true); // Temp DReg |--------------c| - } - __ vmovrs(out_r, tmp_s); -} - -void IntrinsicLocationsBuilderARM::VisitIntegerBitCount(HInvoke* invoke) { - CreateIntToIntLocations(arena_, invoke); - invoke->GetLocations()->AddTemp(Location::RequiresFpuRegister()); -} - -void IntrinsicCodeGeneratorARM::VisitIntegerBitCount(HInvoke* invoke) { - GenBitCount(invoke, Primitive::kPrimInt, GetAssembler()); -} - -void IntrinsicLocationsBuilderARM::VisitLongBitCount(HInvoke* invoke) { - VisitIntegerBitCount(invoke); -} - -void IntrinsicCodeGeneratorARM::VisitLongBitCount(HInvoke* invoke) { - GenBitCount(invoke, Primitive::kPrimLong, GetAssembler()); -} - -void IntrinsicLocationsBuilderARM::VisitStringGetCharsNoCheck(HInvoke* invoke) { - LocationSummary* locations = new (arena_) LocationSummary(invoke, - LocationSummary::kNoCall, - kIntrinsified); - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RequiresRegister()); - locations->SetInAt(2, Location::RequiresRegister()); - locations->SetInAt(3, Location::RequiresRegister()); - locations->SetInAt(4, Location::RequiresRegister()); - - // Temporary registers to store lengths of strings and for calculations. - locations->AddTemp(Location::RequiresRegister()); - locations->AddTemp(Location::RequiresRegister()); - locations->AddTemp(Location::RequiresRegister()); -} - -void IntrinsicCodeGeneratorARM::VisitStringGetCharsNoCheck(HInvoke* invoke) { - ArmAssembler* assembler = GetAssembler(); - LocationSummary* locations = invoke->GetLocations(); - - // Check assumption that sizeof(Char) is 2 (used in scaling below). - const size_t char_size = Primitive::ComponentSize(Primitive::kPrimChar); - DCHECK_EQ(char_size, 2u); - - // Location of data in char array buffer. - const uint32_t data_offset = mirror::Array::DataOffset(char_size).Uint32Value(); - - // Location of char array data in string. - const uint32_t value_offset = mirror::String::ValueOffset().Uint32Value(); - - // void getCharsNoCheck(int srcBegin, int srcEnd, char[] dst, int dstBegin); - // Since getChars() calls getCharsNoCheck() - we use registers rather than constants. - Register srcObj = locations->InAt(0).AsRegister<Register>(); - Register srcBegin = locations->InAt(1).AsRegister<Register>(); - Register srcEnd = locations->InAt(2).AsRegister<Register>(); - Register dstObj = locations->InAt(3).AsRegister<Register>(); - Register dstBegin = locations->InAt(4).AsRegister<Register>(); - - Register num_chr = locations->GetTemp(0).AsRegister<Register>(); - Register src_ptr = locations->GetTemp(1).AsRegister<Register>(); - Register dst_ptr = locations->GetTemp(2).AsRegister<Register>(); - - Label done, compressed_string_loop; - Label* final_label = codegen_->GetFinalLabel(invoke, &done); - // dst to be copied. - __ add(dst_ptr, dstObj, ShifterOperand(data_offset)); - __ add(dst_ptr, dst_ptr, ShifterOperand(dstBegin, LSL, 1)); - - __ subs(num_chr, srcEnd, ShifterOperand(srcBegin)); - // Early out for valid zero-length retrievals. - __ b(final_label, EQ); - - // src range to copy. - __ add(src_ptr, srcObj, ShifterOperand(value_offset)); - Label compressed_string_preloop; - if (mirror::kUseStringCompression) { - // Location of count in string. - const uint32_t count_offset = mirror::String::CountOffset().Uint32Value(); - // String's length. - __ ldr(IP, Address(srcObj, count_offset)); - __ tst(IP, ShifterOperand(1)); - __ b(&compressed_string_preloop, EQ); - } - __ add(src_ptr, src_ptr, ShifterOperand(srcBegin, LSL, 1)); - - // Do the copy. - Label loop, remainder; - - // Save repairing the value of num_chr on the < 4 character path. - __ subs(IP, num_chr, ShifterOperand(4)); - __ b(&remainder, LT); - - // Keep the result of the earlier subs, we are going to fetch at least 4 characters. - __ mov(num_chr, ShifterOperand(IP)); - - // Main loop used for longer fetches loads and stores 4x16-bit characters at a time. - // (LDRD/STRD fault on unaligned addresses and it's not worth inlining extra code - // to rectify these everywhere this intrinsic applies.) - __ Bind(&loop); - __ ldr(IP, Address(src_ptr, char_size * 2)); - __ subs(num_chr, num_chr, ShifterOperand(4)); - __ str(IP, Address(dst_ptr, char_size * 2)); - __ ldr(IP, Address(src_ptr, char_size * 4, Address::PostIndex)); - __ str(IP, Address(dst_ptr, char_size * 4, Address::PostIndex)); - __ b(&loop, GE); - - __ adds(num_chr, num_chr, ShifterOperand(4)); - __ b(final_label, EQ); - - // Main loop for < 4 character case and remainder handling. Loads and stores one - // 16-bit Java character at a time. - __ Bind(&remainder); - __ ldrh(IP, Address(src_ptr, char_size, Address::PostIndex)); - __ subs(num_chr, num_chr, ShifterOperand(1)); - __ strh(IP, Address(dst_ptr, char_size, Address::PostIndex)); - __ b(&remainder, GT); - - if (mirror::kUseStringCompression) { - __ b(final_label); - - const size_t c_char_size = Primitive::ComponentSize(Primitive::kPrimByte); - DCHECK_EQ(c_char_size, 1u); - // Copy loop for compressed src, copying 1 character (8-bit) to (16-bit) at a time. - __ Bind(&compressed_string_preloop); - __ add(src_ptr, src_ptr, ShifterOperand(srcBegin)); - __ Bind(&compressed_string_loop); - __ ldrb(IP, Address(src_ptr, c_char_size, Address::PostIndex)); - __ strh(IP, Address(dst_ptr, char_size, Address::PostIndex)); - __ subs(num_chr, num_chr, ShifterOperand(1)); - __ b(&compressed_string_loop, GT); - } - - if (done.IsLinked()) { - __ Bind(&done); - } -} - -void IntrinsicLocationsBuilderARM::VisitFloatIsInfinite(HInvoke* invoke) { - CreateFPToIntLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitFloatIsInfinite(HInvoke* invoke) { - ArmAssembler* const assembler = GetAssembler(); - LocationSummary* const locations = invoke->GetLocations(); - const Register out = locations->Out().AsRegister<Register>(); - // Shifting left by 1 bit makes the value encodable as an immediate operand; - // we don't care about the sign bit anyway. - constexpr uint32_t infinity = kPositiveInfinityFloat << 1U; - - __ vmovrs(out, locations->InAt(0).AsFpuRegister<SRegister>()); - // We don't care about the sign bit, so shift left. - __ Lsl(out, out, 1); - __ eor(out, out, ShifterOperand(infinity)); - codegen_->GenerateConditionWithZero(kCondEQ, out, out); -} - -void IntrinsicLocationsBuilderARM::VisitDoubleIsInfinite(HInvoke* invoke) { - CreateFPToIntLocations(arena_, invoke); -} - -void IntrinsicCodeGeneratorARM::VisitDoubleIsInfinite(HInvoke* invoke) { - ArmAssembler* const assembler = GetAssembler(); - LocationSummary* const locations = invoke->GetLocations(); - const Register out = locations->Out().AsRegister<Register>(); - // The highest 32 bits of double precision positive infinity separated into - // two constants encodable as immediate operands. - constexpr uint32_t infinity_high = 0x7f000000U; - constexpr uint32_t infinity_high2 = 0x00f00000U; - - static_assert((infinity_high | infinity_high2) == static_cast<uint32_t>(kPositiveInfinityDouble >> 32U), - "The constants do not add up to the high 32 bits of double precision positive infinity."); - __ vmovrrd(IP, out, FromLowSToD(locations->InAt(0).AsFpuRegisterPairLow<SRegister>())); - __ eor(out, out, ShifterOperand(infinity_high)); - __ eor(out, out, ShifterOperand(infinity_high2)); - // We don't care about the sign bit, so shift left. - __ orr(out, IP, ShifterOperand(out, LSL, 1)); - codegen_->GenerateConditionWithZero(kCondEQ, out, out); -} - -void IntrinsicLocationsBuilderARM::VisitIntegerValueOf(HInvoke* invoke) { - InvokeRuntimeCallingConvention calling_convention; - IntrinsicVisitor::ComputeIntegerValueOfLocations( - invoke, - codegen_, - Location::RegisterLocation(R0), - Location::RegisterLocation(calling_convention.GetRegisterAt(0))); -} - -void IntrinsicCodeGeneratorARM::VisitIntegerValueOf(HInvoke* invoke) { - IntrinsicVisitor::IntegerValueOfInfo info = IntrinsicVisitor::ComputeIntegerValueOfInfo(); - LocationSummary* locations = invoke->GetLocations(); - ArmAssembler* const assembler = GetAssembler(); - - Register out = locations->Out().AsRegister<Register>(); - InvokeRuntimeCallingConvention calling_convention; - Register argument = calling_convention.GetRegisterAt(0); - if (invoke->InputAt(0)->IsConstant()) { - int32_t value = invoke->InputAt(0)->AsIntConstant()->GetValue(); - if (value >= info.low && value <= info.high) { - // Just embed the j.l.Integer in the code. - ScopedObjectAccess soa(Thread::Current()); - mirror::Object* boxed = info.cache->Get(value + (-info.low)); - DCHECK(boxed != nullptr && Runtime::Current()->GetHeap()->ObjectIsInBootImageSpace(boxed)); - uint32_t address = dchecked_integral_cast<uint32_t>(reinterpret_cast<uintptr_t>(boxed)); - __ LoadLiteral(out, codegen_->DeduplicateBootImageAddressLiteral(address)); - } else { - // Allocate and initialize a new j.l.Integer. - // TODO: If we JIT, we could allocate the j.l.Integer now, and store it in the - // JIT object table. - uint32_t address = - dchecked_integral_cast<uint32_t>(reinterpret_cast<uintptr_t>(info.integer)); - __ LoadLiteral(argument, codegen_->DeduplicateBootImageAddressLiteral(address)); - codegen_->InvokeRuntime(kQuickAllocObjectInitialized, invoke, invoke->GetDexPc()); - CheckEntrypointTypes<kQuickAllocObjectWithChecks, void*, mirror::Class*>(); - __ LoadImmediate(IP, value); - __ StoreToOffset(kStoreWord, IP, out, info.value_offset); - // `value` is a final field :-( Ideally, we'd merge this memory barrier with the allocation - // one. - codegen_->GenerateMemoryBarrier(MemBarrierKind::kStoreStore); - } - } else { - Register in = locations->InAt(0).AsRegister<Register>(); - // Check bounds of our cache. - __ AddConstant(out, in, -info.low); - __ CmpConstant(out, info.high - info.low + 1); - Label allocate, done; - __ b(&allocate, HS); - // If the value is within the bounds, load the j.l.Integer directly from the array. - uint32_t data_offset = mirror::Array::DataOffset(kHeapReferenceSize).Uint32Value(); - uint32_t address = dchecked_integral_cast<uint32_t>(reinterpret_cast<uintptr_t>(info.cache)); - __ LoadLiteral(IP, codegen_->DeduplicateBootImageAddressLiteral(data_offset + address)); - codegen_->LoadFromShiftedRegOffset(Primitive::kPrimNot, locations->Out(), IP, out); - __ MaybeUnpoisonHeapReference(out); - __ b(&done); - __ Bind(&allocate); - // Otherwise allocate and initialize a new j.l.Integer. - address = dchecked_integral_cast<uint32_t>(reinterpret_cast<uintptr_t>(info.integer)); - __ LoadLiteral(argument, codegen_->DeduplicateBootImageAddressLiteral(address)); - codegen_->InvokeRuntime(kQuickAllocObjectInitialized, invoke, invoke->GetDexPc()); - CheckEntrypointTypes<kQuickAllocObjectWithChecks, void*, mirror::Class*>(); - __ StoreToOffset(kStoreWord, in, out, info.value_offset); - // `value` is a final field :-( Ideally, we'd merge this memory barrier with the allocation - // one. - codegen_->GenerateMemoryBarrier(MemBarrierKind::kStoreStore); - __ Bind(&done); - } -} - -void IntrinsicLocationsBuilderARM::VisitThreadInterrupted(HInvoke* invoke) { - LocationSummary* locations = new (arena_) LocationSummary(invoke, - LocationSummary::kNoCall, - kIntrinsified); - locations->SetOut(Location::RequiresRegister()); -} - -void IntrinsicCodeGeneratorARM::VisitThreadInterrupted(HInvoke* invoke) { - ArmAssembler* assembler = GetAssembler(); - Register out = invoke->GetLocations()->Out().AsRegister<Register>(); - int32_t offset = Thread::InterruptedOffset<kArmPointerSize>().Int32Value(); - __ LoadFromOffset(kLoadWord, out, TR, offset); - Label done; - Label* const final_label = codegen_->GetFinalLabel(invoke, &done); - __ CompareAndBranchIfZero(out, final_label); - __ dmb(ISH); - __ LoadImmediate(IP, 0); - __ StoreToOffset(kStoreWord, IP, TR, offset); - __ dmb(ISH); - if (done.IsLinked()) { - __ Bind(&done); - } -} - -UNIMPLEMENTED_INTRINSIC(ARM, MathMinDoubleDouble) -UNIMPLEMENTED_INTRINSIC(ARM, MathMinFloatFloat) -UNIMPLEMENTED_INTRINSIC(ARM, MathMaxDoubleDouble) -UNIMPLEMENTED_INTRINSIC(ARM, MathMaxFloatFloat) -UNIMPLEMENTED_INTRINSIC(ARM, MathMinLongLong) -UNIMPLEMENTED_INTRINSIC(ARM, MathMaxLongLong) -UNIMPLEMENTED_INTRINSIC(ARM, MathCeil) // Could be done by changing rounding mode, maybe? -UNIMPLEMENTED_INTRINSIC(ARM, MathFloor) // Could be done by changing rounding mode, maybe? -UNIMPLEMENTED_INTRINSIC(ARM, MathRint) -UNIMPLEMENTED_INTRINSIC(ARM, MathRoundDouble) // Could be done by changing rounding mode, maybe? -UNIMPLEMENTED_INTRINSIC(ARM, MathRoundFloat) // Could be done by changing rounding mode, maybe? -UNIMPLEMENTED_INTRINSIC(ARM, UnsafeCASLong) // High register pressure. -UNIMPLEMENTED_INTRINSIC(ARM, SystemArrayCopyChar) -UNIMPLEMENTED_INTRINSIC(ARM, ReferenceGetReferent) -UNIMPLEMENTED_INTRINSIC(ARM, IntegerHighestOneBit) -UNIMPLEMENTED_INTRINSIC(ARM, LongHighestOneBit) -UNIMPLEMENTED_INTRINSIC(ARM, IntegerLowestOneBit) -UNIMPLEMENTED_INTRINSIC(ARM, LongLowestOneBit) - -UNIMPLEMENTED_INTRINSIC(ARM, StringStringIndexOf); -UNIMPLEMENTED_INTRINSIC(ARM, StringStringIndexOfAfter); -UNIMPLEMENTED_INTRINSIC(ARM, StringBufferAppend); -UNIMPLEMENTED_INTRINSIC(ARM, StringBufferLength); -UNIMPLEMENTED_INTRINSIC(ARM, StringBufferToString); -UNIMPLEMENTED_INTRINSIC(ARM, StringBuilderAppend); -UNIMPLEMENTED_INTRINSIC(ARM, StringBuilderLength); -UNIMPLEMENTED_INTRINSIC(ARM, StringBuilderToString); - -// 1.8. -UNIMPLEMENTED_INTRINSIC(ARM, UnsafeGetAndAddInt) -UNIMPLEMENTED_INTRINSIC(ARM, UnsafeGetAndAddLong) -UNIMPLEMENTED_INTRINSIC(ARM, UnsafeGetAndSetInt) -UNIMPLEMENTED_INTRINSIC(ARM, UnsafeGetAndSetLong) -UNIMPLEMENTED_INTRINSIC(ARM, UnsafeGetAndSetObject) - -UNREACHABLE_INTRINSICS(ARM) - -#undef __ - -} // namespace arm -} // namespace art diff --git a/compiler/optimizing/intrinsics_arm.h b/compiler/optimizing/intrinsics_arm.h deleted file mode 100644 index 2840863632..0000000000 --- a/compiler/optimizing/intrinsics_arm.h +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright (C) 2015 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef ART_COMPILER_OPTIMIZING_INTRINSICS_ARM_H_ -#define ART_COMPILER_OPTIMIZING_INTRINSICS_ARM_H_ - -#include "intrinsics.h" - -namespace art { - -class ArenaAllocator; -class ArmInstructionSetFeatures; -class HInvokeStaticOrDirect; -class HInvokeVirtual; - -namespace arm { - -class ArmAssembler; -class CodeGeneratorARM; - -class IntrinsicLocationsBuilderARM FINAL : public IntrinsicVisitor { - public: - explicit IntrinsicLocationsBuilderARM(CodeGeneratorARM* codegen); - - // Define visitor methods. - -#define OPTIMIZING_INTRINSICS(Name, IsStatic, NeedsEnvironmentOrCache, SideEffects, Exceptions, ...) \ - void Visit ## Name(HInvoke* invoke) OVERRIDE; -#include "intrinsics_list.h" -INTRINSICS_LIST(OPTIMIZING_INTRINSICS) -#undef INTRINSICS_LIST -#undef OPTIMIZING_INTRINSICS - - // Check whether an invoke is an intrinsic, and if so, create a location summary. Returns whether - // a corresponding LocationSummary with the intrinsified_ flag set was generated and attached to - // the invoke. - bool TryDispatch(HInvoke* invoke); - - private: - ArenaAllocator* arena_; - CodeGenerator* codegen_; - ArmAssembler* assembler_; - - const ArmInstructionSetFeatures& features_; - - DISALLOW_COPY_AND_ASSIGN(IntrinsicLocationsBuilderARM); -}; - -class IntrinsicCodeGeneratorARM FINAL : public IntrinsicVisitor { - public: - explicit IntrinsicCodeGeneratorARM(CodeGeneratorARM* codegen) : codegen_(codegen) {} - - // Define visitor methods. - -#define OPTIMIZING_INTRINSICS(Name, IsStatic, NeedsEnvironmentOrCache, SideEffects, Exceptions, ...) \ - void Visit ## Name(HInvoke* invoke) OVERRIDE; -#include "intrinsics_list.h" -INTRINSICS_LIST(OPTIMIZING_INTRINSICS) -#undef INTRINSICS_LIST -#undef OPTIMIZING_INTRINSICS - - private: - ArmAssembler* GetAssembler(); - - ArenaAllocator* GetAllocator(); - - CodeGeneratorARM* codegen_; - - DISALLOW_COPY_AND_ASSIGN(IntrinsicCodeGeneratorARM); -}; - -} // namespace arm -} // namespace art - -#endif // ART_COMPILER_OPTIMIZING_INTRINSICS_ARM_H_ diff --git a/compiler/optimizing/intrinsics_arm64.cc b/compiler/optimizing/intrinsics_arm64.cc index 37d79814be..5691dd0d4a 100644 --- a/compiler/optimizing/intrinsics_arm64.cc +++ b/compiler/optimizing/intrinsics_arm64.cc @@ -205,7 +205,7 @@ class ReadBarrierSystemArrayCopySlowPathARM64 : public SlowPathCodeARM64 { // TODO: Load the entrypoint once before the loop, instead of // loading it at every iteration. int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArm64PointerSize>(tmp_.reg()); + Thread::ReadBarrierMarkEntryPointsOffset<kArm64PointerSize>(tmp_.reg()); // This runtime call does not require a stack map. codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this); codegen->GetAssembler()->MaybePoisonHeapReference(tmp_reg); @@ -2738,7 +2738,7 @@ void IntrinsicCodeGeneratorARM64::VisitSystemArrayCopy(HInvoke* invoke) { // TODO: Also convert this intrinsic to the IsGcMarking strategy? // SystemArrayCopy implementation for Baker read barriers (see - // also CodeGeneratorARM::GenerateReferenceLoadWithBakerReadBarrier): + // also CodeGeneratorARM64::GenerateReferenceLoadWithBakerReadBarrier): // // uint32_t rb_state = Lockword(src->monitor_).ReadBarrierState(); // lfence; // Load fence or artificial data dependency to prevent load-load reordering diff --git a/compiler/optimizing/intrinsics_arm_vixl.cc b/compiler/optimizing/intrinsics_arm_vixl.cc index 3c9b613803..8b4044d69b 100644 --- a/compiler/optimizing/intrinsics_arm_vixl.cc +++ b/compiler/optimizing/intrinsics_arm_vixl.cc @@ -226,7 +226,7 @@ class ReadBarrierSystemArrayCopySlowPathARMVIXL : public SlowPathCodeARMVIXL { // TODO: Load the entrypoint once before the loop, instead of // loading it at every iteration. int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(tmp.GetCode()); + Thread::ReadBarrierMarkEntryPointsOffset<kArmPointerSize>(tmp.GetCode()); // This runtime call does not require a stack map. arm_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this); assembler->MaybePoisonHeapReference(tmp); @@ -1058,7 +1058,7 @@ static void CreateIntIntIntToIntLocations(ArenaAllocator* arena, (can_call ? Location::kOutputOverlap : Location::kNoOutputOverlap)); if (type == Primitive::kPrimNot && kEmitCompilerReadBarrier && kUseBakerReadBarrier) { // We need a temporary register for the read barrier marking slow - // path in InstructionCodeGeneratorARM::GenerateReferenceLoadWithBakerReadBarrier. + // path in CodeGeneratorARMVIXL::GenerateReferenceLoadWithBakerReadBarrier. locations->AddTemp(Location::RequiresRegister()); } } @@ -2377,7 +2377,7 @@ void IntrinsicCodeGeneratorARMVIXL::VisitSystemArrayCopy(HInvoke* invoke) { // TODO: Also convert this intrinsic to the IsGcMarking strategy? // SystemArrayCopy implementation for Baker read barriers (see - // also CodeGeneratorARM::GenerateReferenceLoadWithBakerReadBarrier): + // also CodeGeneratorARMVIXL::GenerateReferenceLoadWithBakerReadBarrier): // // uint32_t rb_state = Lockword(src->monitor_).ReadBarrierState(); // lfence; // Load fence or artificial data dependency to prevent load-load reordering diff --git a/compiler/optimizing/intrinsics_x86.cc b/compiler/optimizing/intrinsics_x86.cc index 6b4851d541..a18b0cc400 100644 --- a/compiler/optimizing/intrinsics_x86.cc +++ b/compiler/optimizing/intrinsics_x86.cc @@ -143,8 +143,7 @@ class ReadBarrierSystemArrayCopySlowPathX86 : public SlowPathCode { // explanations.) DCHECK_NE(temp2, ESP); DCHECK(0 <= temp2 && temp2 < kNumberOfCpuRegisters) << temp2; - int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kX86PointerSize>(temp2); + int32_t entry_point_offset = Thread::ReadBarrierMarkEntryPointsOffset<kX86PointerSize>(temp2); // This runtime call does not require a stack map. x86_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this); __ MaybePoisonHeapReference(temp2); diff --git a/compiler/optimizing/intrinsics_x86_64.cc b/compiler/optimizing/intrinsics_x86_64.cc index ef98b7be30..5abdb1d1bd 100644 --- a/compiler/optimizing/intrinsics_x86_64.cc +++ b/compiler/optimizing/intrinsics_x86_64.cc @@ -105,8 +105,7 @@ class ReadBarrierSystemArrayCopySlowPathX86_64 : public SlowPathCode { // No need to save live registers; it's taken care of by the // entrypoint. Also, there is no need to update the stack mask, // as this runtime call will not trigger a garbage collection. - int32_t entry_point_offset = - CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kX86_64PointerSize>(TMP); + int32_t entry_point_offset = Thread::ReadBarrierMarkEntryPointsOffset<kX86_64PointerSize>(TMP); // This runtime call does not require a stack map. x86_64_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this); __ MaybePoisonHeapReference(CpuRegister(TMP)); diff --git a/compiler/optimizing/loop_optimization.cc b/compiler/optimizing/loop_optimization.cc index 83f31c77d3..422e58debb 100644 --- a/compiler/optimizing/loop_optimization.cc +++ b/compiler/optimizing/loop_optimization.cc @@ -1171,7 +1171,32 @@ bool HLoopOptimization::TrySetVectorType(Primitive::Type type, uint64_t* restric } return false; case kMips: - // TODO: implement MIPS SIMD. + if (features->AsMipsInstructionSetFeatures()->HasMsa()) { + switch (type) { + case Primitive::kPrimBoolean: + case Primitive::kPrimByte: + *restrictions |= kNoDiv; + return TrySetVectorLength(16); + case Primitive::kPrimChar: + case Primitive::kPrimShort: + *restrictions |= kNoDiv | kNoStringCharAt; + return TrySetVectorLength(8); + case Primitive::kPrimInt: + *restrictions |= kNoDiv; + return TrySetVectorLength(4); + case Primitive::kPrimLong: + *restrictions |= kNoDiv; + return TrySetVectorLength(2); + case Primitive::kPrimFloat: + *restrictions |= kNoMinMax; // min/max(x, NaN) + return TrySetVectorLength(4); + case Primitive::kPrimDouble: + *restrictions |= kNoMinMax; // min/max(x, NaN) + return TrySetVectorLength(2); + default: + break; + } // switch type + } return false; case kMips64: if (features->AsMips64InstructionSetFeatures()->HasMsa()) { diff --git a/compiler/optimizing/nodes_vector.h b/compiler/optimizing/nodes_vector.h index 5dbe29b4fa..6261171a00 100644 --- a/compiler/optimizing/nodes_vector.h +++ b/compiler/optimizing/nodes_vector.h @@ -46,6 +46,10 @@ class Alignment { return "ALIGN(" + std::to_string(base_) + "," + std::to_string(offset_) + ")"; } + bool operator==(const Alignment& other) const { + return base_ == other.base_ && offset_ == other.offset_; + } + private: size_t base_; size_t offset_; @@ -96,6 +100,19 @@ class HVecOperation : public HVariableInputSizeInstruction { return GetPackedField<TypeField>(); } + // Assumes vector nodes cannot be moved by default. Each concrete implementation + // that can be moved should override this method and return true. + bool CanBeMoved() const OVERRIDE { return false; } + + // Tests if all data of a vector node (vector length and packed type) is equal. + // Each concrete implementation that adds more fields should test equality of + // those fields in its own method *and* call all super methods. + bool InstructionDataEquals(const HInstruction* other) const OVERRIDE { + DCHECK(other->IsVecOperation()); + const HVecOperation* o = other->AsVecOperation(); + return GetVectorLength() == o->GetVectorLength() && GetPackedType() == o->GetPackedType(); + } + DECLARE_ABSTRACT_INSTRUCTION(VecOperation); protected: @@ -189,6 +206,12 @@ class HVecMemoryOperation : public HVecOperation { HInstruction* GetArray() const { return InputAt(0); } HInstruction* GetIndex() const { return InputAt(1); } + bool InstructionDataEquals(const HInstruction* other) const OVERRIDE { + DCHECK(other->IsVecMemoryOperation()); + const HVecMemoryOperation* o = other->AsVecMemoryOperation(); + return HVecOperation::InstructionDataEquals(o) && GetAlignment() == o->GetAlignment(); + } + DECLARE_ABSTRACT_INSTRUCTION(VecMemoryOperation); private: @@ -231,7 +254,13 @@ class HVecReplicateScalar FINAL : public HVecUnaryOperation { : HVecUnaryOperation(arena, scalar, packed_type, vector_length, dex_pc) { DCHECK(!scalar->IsVecOperation()); } + + // A replicate needs to stay in place, since SIMD registers are not + // kept alive across vector loop boundaries (yet). + bool CanBeMoved() const OVERRIDE { return false; } + DECLARE_INSTRUCTION(VecReplicateScalar); + private: DISALLOW_COPY_AND_ASSIGN(HVecReplicateScalar); }; @@ -251,7 +280,10 @@ class HVecSumReduce FINAL : public HVecUnaryOperation { // TODO: probably integral promotion Primitive::Type GetType() const OVERRIDE { return GetPackedType(); } + bool CanBeMoved() const OVERRIDE { return true; } + DECLARE_INSTRUCTION(VecSumReduce); + private: DISALLOW_COPY_AND_ASSIGN(HVecSumReduce); }; @@ -273,6 +305,8 @@ class HVecCnv FINAL : public HVecUnaryOperation { Primitive::Type GetInputType() const { return InputAt(0)->AsVecOperation()->GetPackedType(); } Primitive::Type GetResultType() const { return GetPackedType(); } + bool CanBeMoved() const OVERRIDE { return true; } + DECLARE_INSTRUCTION(VecCnv); private: @@ -291,7 +325,11 @@ class HVecNeg FINAL : public HVecUnaryOperation { : HVecUnaryOperation(arena, input, packed_type, vector_length, dex_pc) { DCHECK(HasConsistentPackedTypes(input, packed_type)); } + + bool CanBeMoved() const OVERRIDE { return true; } + DECLARE_INSTRUCTION(VecNeg); + private: DISALLOW_COPY_AND_ASSIGN(HVecNeg); }; @@ -308,7 +346,11 @@ class HVecAbs FINAL : public HVecUnaryOperation { : HVecUnaryOperation(arena, input, packed_type, vector_length, dex_pc) { DCHECK(HasConsistentPackedTypes(input, packed_type)); } + + bool CanBeMoved() const OVERRIDE { return true; } + DECLARE_INSTRUCTION(VecAbs); + private: DISALLOW_COPY_AND_ASSIGN(HVecAbs); }; @@ -326,7 +368,11 @@ class HVecNot FINAL : public HVecUnaryOperation { : HVecUnaryOperation(arena, input, packed_type, vector_length, dex_pc) { DCHECK(input->IsVecOperation()); } + + bool CanBeMoved() const OVERRIDE { return true; } + DECLARE_INSTRUCTION(VecNot); + private: DISALLOW_COPY_AND_ASSIGN(HVecNot); }; @@ -349,7 +395,11 @@ class HVecAdd FINAL : public HVecBinaryOperation { DCHECK(HasConsistentPackedTypes(left, packed_type)); DCHECK(HasConsistentPackedTypes(right, packed_type)); } + + bool CanBeMoved() const OVERRIDE { return true; } + DECLARE_INSTRUCTION(VecAdd); + private: DISALLOW_COPY_AND_ASSIGN(HVecAdd); }; @@ -378,6 +428,16 @@ class HVecHalvingAdd FINAL : public HVecBinaryOperation { bool IsUnsigned() const { return GetPackedFlag<kFieldHAddIsUnsigned>(); } bool IsRounded() const { return GetPackedFlag<kFieldHAddIsRounded>(); } + bool CanBeMoved() const OVERRIDE { return true; } + + bool InstructionDataEquals(const HInstruction* other) const OVERRIDE { + DCHECK(other->IsVecHalvingAdd()); + const HVecHalvingAdd* o = other->AsVecHalvingAdd(); + return HVecOperation::InstructionDataEquals(o) && + IsUnsigned() == o->IsUnsigned() && + IsRounded() == o->IsRounded(); + } + DECLARE_INSTRUCTION(VecHalvingAdd); private: @@ -404,7 +464,11 @@ class HVecSub FINAL : public HVecBinaryOperation { DCHECK(HasConsistentPackedTypes(left, packed_type)); DCHECK(HasConsistentPackedTypes(right, packed_type)); } + + bool CanBeMoved() const OVERRIDE { return true; } + DECLARE_INSTRUCTION(VecSub); + private: DISALLOW_COPY_AND_ASSIGN(HVecSub); }; @@ -423,7 +487,11 @@ class HVecMul FINAL : public HVecBinaryOperation { DCHECK(HasConsistentPackedTypes(left, packed_type)); DCHECK(HasConsistentPackedTypes(right, packed_type)); } + + bool CanBeMoved() const OVERRIDE { return true; } + DECLARE_INSTRUCTION(VecMul); + private: DISALLOW_COPY_AND_ASSIGN(HVecMul); }; @@ -442,7 +510,11 @@ class HVecDiv FINAL : public HVecBinaryOperation { DCHECK(HasConsistentPackedTypes(left, packed_type)); DCHECK(HasConsistentPackedTypes(right, packed_type)); } + + bool CanBeMoved() const OVERRIDE { return true; } + DECLARE_INSTRUCTION(VecDiv); + private: DISALLOW_COPY_AND_ASSIGN(HVecDiv); }; @@ -466,6 +538,14 @@ class HVecMin FINAL : public HVecBinaryOperation { bool IsUnsigned() const { return GetPackedFlag<kFieldMinOpIsUnsigned>(); } + bool CanBeMoved() const OVERRIDE { return true; } + + bool InstructionDataEquals(const HInstruction* other) const OVERRIDE { + DCHECK(other->IsVecMin()); + const HVecMin* o = other->AsVecMin(); + return HVecOperation::InstructionDataEquals(o) && IsUnsigned() == o->IsUnsigned(); + } + DECLARE_INSTRUCTION(VecMin); private: @@ -496,6 +576,14 @@ class HVecMax FINAL : public HVecBinaryOperation { bool IsUnsigned() const { return GetPackedFlag<kFieldMaxOpIsUnsigned>(); } + bool CanBeMoved() const OVERRIDE { return true; } + + bool InstructionDataEquals(const HInstruction* other) const OVERRIDE { + DCHECK(other->IsVecMax()); + const HVecMax* o = other->AsVecMax(); + return HVecOperation::InstructionDataEquals(o) && IsUnsigned() == o->IsUnsigned(); + } + DECLARE_INSTRUCTION(VecMax); private: @@ -520,7 +608,11 @@ class HVecAnd FINAL : public HVecBinaryOperation { : HVecBinaryOperation(arena, left, right, packed_type, vector_length, dex_pc) { DCHECK(left->IsVecOperation() && right->IsVecOperation()); } + + bool CanBeMoved() const OVERRIDE { return true; } + DECLARE_INSTRUCTION(VecAnd); + private: DISALLOW_COPY_AND_ASSIGN(HVecAnd); }; @@ -538,7 +630,11 @@ class HVecAndNot FINAL : public HVecBinaryOperation { : HVecBinaryOperation(arena, left, right, packed_type, vector_length, dex_pc) { DCHECK(left->IsVecOperation() && right->IsVecOperation()); } + + bool CanBeMoved() const OVERRIDE { return true; } + DECLARE_INSTRUCTION(VecAndNot); + private: DISALLOW_COPY_AND_ASSIGN(HVecAndNot); }; @@ -556,7 +652,11 @@ class HVecOr FINAL : public HVecBinaryOperation { : HVecBinaryOperation(arena, left, right, packed_type, vector_length, dex_pc) { DCHECK(left->IsVecOperation() && right->IsVecOperation()); } + + bool CanBeMoved() const OVERRIDE { return true; } + DECLARE_INSTRUCTION(VecOr); + private: DISALLOW_COPY_AND_ASSIGN(HVecOr); }; @@ -574,7 +674,11 @@ class HVecXor FINAL : public HVecBinaryOperation { : HVecBinaryOperation(arena, left, right, packed_type, vector_length, dex_pc) { DCHECK(left->IsVecOperation() && right->IsVecOperation()); } + + bool CanBeMoved() const OVERRIDE { return true; } + DECLARE_INSTRUCTION(VecXor); + private: DISALLOW_COPY_AND_ASSIGN(HVecXor); }; @@ -592,7 +696,11 @@ class HVecShl FINAL : public HVecBinaryOperation { : HVecBinaryOperation(arena, left, right, packed_type, vector_length, dex_pc) { DCHECK(HasConsistentPackedTypes(left, packed_type)); } + + bool CanBeMoved() const OVERRIDE { return true; } + DECLARE_INSTRUCTION(VecShl); + private: DISALLOW_COPY_AND_ASSIGN(HVecShl); }; @@ -610,7 +718,11 @@ class HVecShr FINAL : public HVecBinaryOperation { : HVecBinaryOperation(arena, left, right, packed_type, vector_length, dex_pc) { DCHECK(HasConsistentPackedTypes(left, packed_type)); } + + bool CanBeMoved() const OVERRIDE { return true; } + DECLARE_INSTRUCTION(VecShr); + private: DISALLOW_COPY_AND_ASSIGN(HVecShr); }; @@ -628,7 +740,11 @@ class HVecUShr FINAL : public HVecBinaryOperation { : HVecBinaryOperation(arena, left, right, packed_type, vector_length, dex_pc) { DCHECK(HasConsistentPackedTypes(left, packed_type)); } + + bool CanBeMoved() const OVERRIDE { return true; } + DECLARE_INSTRUCTION(VecUShr); + private: DISALLOW_COPY_AND_ASSIGN(HVecUShr); }; @@ -656,7 +772,13 @@ class HVecSetScalars FINAL : public HVecOperation { SetRawInputAt(0, scalars[i]); } } + + // Setting scalars needs to stay in place, since SIMD registers are not + // kept alive across vector loop boundaries (yet). + bool CanBeMoved() const OVERRIDE { return false; } + DECLARE_INSTRUCTION(VecSetScalars); + private: DISALLOW_COPY_AND_ASSIGN(HVecSetScalars); }; @@ -697,7 +819,9 @@ class HVecMultiplyAccumulate FINAL : public HVecOperation { bool CanBeMoved() const OVERRIDE { return true; } bool InstructionDataEquals(const HInstruction* other) const OVERRIDE { - return op_kind_ == other->AsVecMultiplyAccumulate()->op_kind_; + DCHECK(other->IsVecMultiplyAccumulate()); + const HVecMultiplyAccumulate* o = other->AsVecMultiplyAccumulate(); + return HVecOperation::InstructionDataEquals(o) && GetOpKind() == o->GetOpKind(); } InstructionKind GetOpKind() const { return op_kind_; } @@ -732,10 +856,19 @@ class HVecLoad FINAL : public HVecMemoryOperation { SetRawInputAt(1, index); SetPackedFlag<kFieldIsStringCharAt>(is_string_char_at); } - DECLARE_INSTRUCTION(VecLoad); bool IsStringCharAt() const { return GetPackedFlag<kFieldIsStringCharAt>(); } + bool CanBeMoved() const OVERRIDE { return true; } + + bool InstructionDataEquals(const HInstruction* other) const OVERRIDE { + DCHECK(other->IsVecLoad()); + const HVecLoad* o = other->AsVecLoad(); + return HVecMemoryOperation::InstructionDataEquals(o) && IsStringCharAt() == o->IsStringCharAt(); + } + + DECLARE_INSTRUCTION(VecLoad); + private: // Additional packed bits. static constexpr size_t kFieldIsStringCharAt = HVecOperation::kNumberOfVectorOpPackedBits; @@ -767,7 +900,12 @@ class HVecStore FINAL : public HVecMemoryOperation { SetRawInputAt(1, index); SetRawInputAt(2, value); } + + // A store needs to stay in place. + bool CanBeMoved() const OVERRIDE { return false; } + DECLARE_INSTRUCTION(VecStore); + private: DISALLOW_COPY_AND_ASSIGN(HVecStore); }; diff --git a/compiler/optimizing/nodes_vector_test.cc b/compiler/optimizing/nodes_vector_test.cc new file mode 100644 index 0000000000..0238ea4602 --- /dev/null +++ b/compiler/optimizing/nodes_vector_test.cc @@ -0,0 +1,335 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "base/arena_allocator.h" +#include "nodes.h" +#include "optimizing_unit_test.h" + +namespace art { + +/** + * Fixture class for testing vector nodes. + */ +class NodesVectorTest : public CommonCompilerTest { + public: + NodesVectorTest() + : pool_(), + allocator_(&pool_), + graph_(CreateGraph(&allocator_)) { + BuildGraph(); + } + + ~NodesVectorTest() { } + + void BuildGraph() { + graph_->SetNumberOfVRegs(1); + entry_block_ = new (&allocator_) HBasicBlock(graph_); + exit_block_ = new (&allocator_) HBasicBlock(graph_); + graph_->AddBlock(entry_block_); + graph_->AddBlock(exit_block_); + graph_->SetEntryBlock(entry_block_); + graph_->SetExitBlock(exit_block_); + parameter_ = new (&allocator_) HParameterValue(graph_->GetDexFile(), + dex::TypeIndex(0), + 0, + Primitive::kPrimInt); + entry_block_->AddInstruction(parameter_); + } + + // General building fields. + ArenaPool pool_; + ArenaAllocator allocator_; + HGraph* graph_; + + HBasicBlock* entry_block_; + HBasicBlock* exit_block_; + + HInstruction* parameter_; +}; + +// +// The actual vector nodes tests. +// + +TEST(NodesVector, Alignment) { + EXPECT_TRUE(Alignment(1, 0).IsAlignedAt(1)); + EXPECT_FALSE(Alignment(1, 0).IsAlignedAt(2)); + + EXPECT_TRUE(Alignment(2, 0).IsAlignedAt(1)); + EXPECT_TRUE(Alignment(2, 1).IsAlignedAt(1)); + EXPECT_TRUE(Alignment(2, 0).IsAlignedAt(2)); + EXPECT_FALSE(Alignment(2, 1).IsAlignedAt(2)); + EXPECT_FALSE(Alignment(2, 0).IsAlignedAt(4)); + EXPECT_FALSE(Alignment(2, 1).IsAlignedAt(4)); + + EXPECT_TRUE(Alignment(4, 0).IsAlignedAt(1)); + EXPECT_TRUE(Alignment(4, 2).IsAlignedAt(1)); + EXPECT_TRUE(Alignment(4, 0).IsAlignedAt(2)); + EXPECT_TRUE(Alignment(4, 2).IsAlignedAt(2)); + EXPECT_TRUE(Alignment(4, 0).IsAlignedAt(4)); + EXPECT_FALSE(Alignment(4, 2).IsAlignedAt(4)); + EXPECT_FALSE(Alignment(4, 0).IsAlignedAt(8)); + EXPECT_FALSE(Alignment(4, 2).IsAlignedAt(8)); + + EXPECT_TRUE(Alignment(16, 0).IsAlignedAt(1)); + EXPECT_TRUE(Alignment(16, 0).IsAlignedAt(2)); + EXPECT_TRUE(Alignment(16, 0).IsAlignedAt(4)); + EXPECT_TRUE(Alignment(16, 8).IsAlignedAt(8)); + EXPECT_TRUE(Alignment(16, 0).IsAlignedAt(16)); + EXPECT_FALSE(Alignment(16, 1).IsAlignedAt(16)); + EXPECT_FALSE(Alignment(16, 7).IsAlignedAt(16)); + EXPECT_FALSE(Alignment(16, 0).IsAlignedAt(32)); +} + +TEST(NodesVector, AlignmentEQ) { + EXPECT_TRUE(Alignment(2, 0) == Alignment(2, 0)); + EXPECT_TRUE(Alignment(2, 1) == Alignment(2, 1)); + EXPECT_TRUE(Alignment(4, 0) == Alignment(4, 0)); + EXPECT_TRUE(Alignment(4, 2) == Alignment(4, 2)); + + EXPECT_FALSE(Alignment(4, 0) == Alignment(2, 0)); + EXPECT_FALSE(Alignment(4, 0) == Alignment(4, 1)); + EXPECT_FALSE(Alignment(4, 0) == Alignment(8, 0)); +} + +TEST(NodesVector, AlignmentString) { + EXPECT_STREQ("ALIGN(1,0)", Alignment(1, 0).ToString().c_str()); + + EXPECT_STREQ("ALIGN(2,0)", Alignment(2, 0).ToString().c_str()); + EXPECT_STREQ("ALIGN(2,1)", Alignment(2, 1).ToString().c_str()); + + EXPECT_STREQ("ALIGN(16,0)", Alignment(16, 0).ToString().c_str()); + EXPECT_STREQ("ALIGN(16,1)", Alignment(16, 1).ToString().c_str()); + EXPECT_STREQ("ALIGN(16,8)", Alignment(16, 8).ToString().c_str()); + EXPECT_STREQ("ALIGN(16,9)", Alignment(16, 9).ToString().c_str()); +} + +TEST_F(NodesVectorTest, VectorOperationProperties) { + HVecOperation* v0 = new (&allocator_) + HVecReplicateScalar(&allocator_, parameter_, Primitive::kPrimInt, 4); + HVecOperation* v1 = new (&allocator_) + HVecReplicateScalar(&allocator_, parameter_, Primitive::kPrimInt, 4); + HVecOperation* v2 = new (&allocator_) + HVecReplicateScalar(&allocator_, parameter_, Primitive::kPrimInt, 2); + HVecOperation* v3 = new (&allocator_) + HVecReplicateScalar(&allocator_, parameter_, Primitive::kPrimShort, 4); + HVecOperation* v4 = new (&allocator_) + HVecStore(&allocator_, parameter_, parameter_, v0, Primitive::kPrimInt, 4); + + EXPECT_TRUE(v0->Equals(v0)); + EXPECT_TRUE(v1->Equals(v1)); + EXPECT_TRUE(v2->Equals(v2)); + EXPECT_TRUE(v3->Equals(v3)); + EXPECT_TRUE(v4->Equals(v4)); + + EXPECT_TRUE(v0->Equals(v1)); + EXPECT_FALSE(v0->Equals(v2)); // different vector lengths + EXPECT_FALSE(v0->Equals(v3)); // different packed types + EXPECT_FALSE(v0->Equals(v4)); // different kinds + + EXPECT_TRUE(v1->Equals(v0)); // switch operands + EXPECT_FALSE(v4->Equals(v0)); + + EXPECT_EQ(4u, v0->GetVectorLength()); + EXPECT_EQ(4u, v1->GetVectorLength()); + EXPECT_EQ(2u, v2->GetVectorLength()); + EXPECT_EQ(4u, v3->GetVectorLength()); + EXPECT_EQ(4u, v4->GetVectorLength()); + + EXPECT_EQ(Primitive::kPrimDouble, v0->GetType()); + EXPECT_EQ(Primitive::kPrimDouble, v1->GetType()); + EXPECT_EQ(Primitive::kPrimDouble, v2->GetType()); + EXPECT_EQ(Primitive::kPrimDouble, v3->GetType()); + EXPECT_EQ(Primitive::kPrimDouble, v4->GetType()); + + EXPECT_EQ(Primitive::kPrimInt, v0->GetPackedType()); + EXPECT_EQ(Primitive::kPrimInt, v1->GetPackedType()); + EXPECT_EQ(Primitive::kPrimInt, v2->GetPackedType()); + EXPECT_EQ(Primitive::kPrimShort, v3->GetPackedType()); + EXPECT_EQ(Primitive::kPrimInt, v4->GetPackedType()); + + EXPECT_EQ(16u, v0->GetVectorNumberOfBytes()); + EXPECT_EQ(16u, v1->GetVectorNumberOfBytes()); + EXPECT_EQ(8u, v2->GetVectorNumberOfBytes()); + EXPECT_EQ(8u, v3->GetVectorNumberOfBytes()); + EXPECT_EQ(16u, v4->GetVectorNumberOfBytes()); + + EXPECT_FALSE(v0->CanBeMoved()); + EXPECT_FALSE(v1->CanBeMoved()); + EXPECT_FALSE(v2->CanBeMoved()); + EXPECT_FALSE(v3->CanBeMoved()); + EXPECT_FALSE(v4->CanBeMoved()); +} + +TEST_F(NodesVectorTest, VectorAlignmentAndStringCharAtMatterOnLoad) { + HVecLoad* v0 = new (&allocator_) + HVecLoad(&allocator_, parameter_, parameter_, Primitive::kPrimInt, 4, /*is_string_char_at*/ false); + HVecLoad* v1 = new (&allocator_) + HVecLoad(&allocator_, parameter_, parameter_, Primitive::kPrimInt, 4, /*is_string_char_at*/ false); + HVecLoad* v2 = new (&allocator_) + HVecLoad(&allocator_, parameter_, parameter_, Primitive::kPrimInt, 4, /*is_string_char_at*/ true); + + EXPECT_TRUE(v0->CanBeMoved()); + EXPECT_TRUE(v1->CanBeMoved()); + EXPECT_TRUE(v2->CanBeMoved()); + + EXPECT_FALSE(v0->IsStringCharAt()); + EXPECT_FALSE(v1->IsStringCharAt()); + EXPECT_TRUE(v2->IsStringCharAt()); + + EXPECT_TRUE(v0->Equals(v0)); + EXPECT_TRUE(v1->Equals(v1)); + EXPECT_TRUE(v2->Equals(v2)); + + EXPECT_TRUE(v0->Equals(v1)); + EXPECT_FALSE(v0->Equals(v2)); + + EXPECT_TRUE(v0->GetAlignment() == Alignment(4, 0)); + EXPECT_TRUE(v1->GetAlignment() == Alignment(4, 0)); + EXPECT_TRUE(v2->GetAlignment() == Alignment(4, 0)); + + v1->SetAlignment(Alignment(8, 0)); + + EXPECT_TRUE(v1->GetAlignment() == Alignment(8, 0)); + + EXPECT_FALSE(v0->Equals(v1)); // no longer equal +} + +TEST_F(NodesVectorTest, VectorSignMattersOnMin) { + HVecOperation* v0 = new (&allocator_) + HVecReplicateScalar(&allocator_, parameter_, Primitive::kPrimInt, 4); + + HVecMin* v1 = new (&allocator_) + HVecMin(&allocator_, v0, v0, Primitive::kPrimInt, 4, /*is_unsigned*/ true); + HVecMin* v2 = new (&allocator_) + HVecMin(&allocator_, v0, v0, Primitive::kPrimInt, 4, /*is_unsigned*/ false); + HVecMin* v3 = new (&allocator_) + HVecMin(&allocator_, v0, v0, Primitive::kPrimInt, 2, /*is_unsigned*/ true); + + EXPECT_FALSE(v0->CanBeMoved()); + EXPECT_TRUE(v1->CanBeMoved()); + EXPECT_TRUE(v2->CanBeMoved()); + EXPECT_TRUE(v3->CanBeMoved()); + + EXPECT_TRUE(v1->IsUnsigned()); + EXPECT_FALSE(v2->IsUnsigned()); + EXPECT_TRUE(v3->IsUnsigned()); + + EXPECT_TRUE(v1->Equals(v1)); + EXPECT_TRUE(v2->Equals(v2)); + EXPECT_TRUE(v3->Equals(v3)); + + EXPECT_FALSE(v1->Equals(v2)); // different signs + EXPECT_FALSE(v1->Equals(v3)); // different vector lengths +} + +TEST_F(NodesVectorTest, VectorSignMattersOnMax) { + HVecOperation* v0 = new (&allocator_) + HVecReplicateScalar(&allocator_, parameter_, Primitive::kPrimInt, 4); + + HVecMax* v1 = new (&allocator_) + HVecMax(&allocator_, v0, v0, Primitive::kPrimInt, 4, /*is_unsigned*/ true); + HVecMax* v2 = new (&allocator_) + HVecMax(&allocator_, v0, v0, Primitive::kPrimInt, 4, /*is_unsigned*/ false); + HVecMax* v3 = new (&allocator_) + HVecMax(&allocator_, v0, v0, Primitive::kPrimInt, 2, /*is_unsigned*/ true); + + EXPECT_FALSE(v0->CanBeMoved()); + EXPECT_TRUE(v1->CanBeMoved()); + EXPECT_TRUE(v2->CanBeMoved()); + EXPECT_TRUE(v3->CanBeMoved()); + + EXPECT_TRUE(v1->IsUnsigned()); + EXPECT_FALSE(v2->IsUnsigned()); + EXPECT_TRUE(v3->IsUnsigned()); + + EXPECT_TRUE(v1->Equals(v1)); + EXPECT_TRUE(v2->Equals(v2)); + EXPECT_TRUE(v3->Equals(v3)); + + EXPECT_FALSE(v1->Equals(v2)); // different signs + EXPECT_FALSE(v1->Equals(v3)); // different vector lengths +} + +TEST_F(NodesVectorTest, VectorAttributesMatterOnHalvingAdd) { + HVecOperation* v0 = new (&allocator_) + HVecReplicateScalar(&allocator_, parameter_, Primitive::kPrimInt, 4); + + HVecHalvingAdd* v1 = new (&allocator_) HVecHalvingAdd( + &allocator_, v0, v0, Primitive::kPrimInt, 4, /*is_unsigned*/ true, /*is_rounded*/ true); + HVecHalvingAdd* v2 = new (&allocator_) HVecHalvingAdd( + &allocator_, v0, v0, Primitive::kPrimInt, 4, /*is_unsigned*/ true, /*is_rounded*/ false); + HVecHalvingAdd* v3 = new (&allocator_) HVecHalvingAdd( + &allocator_, v0, v0, Primitive::kPrimInt, 4, /*is_unsigned*/ false, /*is_rounded*/ true); + HVecHalvingAdd* v4 = new (&allocator_) HVecHalvingAdd( + &allocator_, v0, v0, Primitive::kPrimInt, 4, /*is_unsigned*/ false, /*is_rounded*/ false); + HVecHalvingAdd* v5 = new (&allocator_) HVecHalvingAdd( + &allocator_, v0, v0, Primitive::kPrimInt, 2, /*is_unsigned*/ true, /*is_rounded*/ true); + + EXPECT_FALSE(v0->CanBeMoved()); + EXPECT_TRUE(v1->CanBeMoved()); + EXPECT_TRUE(v2->CanBeMoved()); + EXPECT_TRUE(v3->CanBeMoved()); + EXPECT_TRUE(v4->CanBeMoved()); + EXPECT_TRUE(v5->CanBeMoved()); + + EXPECT_TRUE(v1->Equals(v1)); + EXPECT_TRUE(v2->Equals(v2)); + EXPECT_TRUE(v3->Equals(v3)); + EXPECT_TRUE(v4->Equals(v4)); + EXPECT_TRUE(v5->Equals(v5)); + + EXPECT_TRUE(v1->IsUnsigned() && v1->IsRounded()); + EXPECT_TRUE(v2->IsUnsigned() && !v2->IsRounded()); + EXPECT_TRUE(!v3->IsUnsigned() && v3->IsRounded()); + EXPECT_TRUE(!v4->IsUnsigned() && !v4->IsRounded()); + EXPECT_TRUE(v5->IsUnsigned() && v5->IsRounded()); + + EXPECT_FALSE(v1->Equals(v2)); // different attributes + EXPECT_FALSE(v1->Equals(v3)); // different attributes + EXPECT_FALSE(v1->Equals(v4)); // different attributes + EXPECT_FALSE(v1->Equals(v5)); // different vector lengths +} + +TEST_F(NodesVectorTest, VectorOperationMattersOnMultiplyAccumulate) { + HVecOperation* v0 = new (&allocator_) + HVecReplicateScalar(&allocator_, parameter_, Primitive::kPrimInt, 4); + + HVecMultiplyAccumulate* v1 = new (&allocator_) + HVecMultiplyAccumulate(&allocator_, HInstruction::kAdd, v0, v0, v0, Primitive::kPrimInt, 4); + HVecMultiplyAccumulate* v2 = new (&allocator_) + HVecMultiplyAccumulate(&allocator_, HInstruction::kSub, v0, v0, v0, Primitive::kPrimInt, 4); + HVecMultiplyAccumulate* v3 = new (&allocator_) + HVecMultiplyAccumulate(&allocator_, HInstruction::kAdd, v0, v0, v0, Primitive::kPrimInt, 2); + + EXPECT_FALSE(v0->CanBeMoved()); + EXPECT_TRUE(v1->CanBeMoved()); + EXPECT_TRUE(v2->CanBeMoved()); + EXPECT_TRUE(v3->CanBeMoved()); + + EXPECT_EQ(HInstruction::kAdd, v1->GetOpKind()); + EXPECT_EQ(HInstruction::kSub, v2->GetOpKind()); + EXPECT_EQ(HInstruction::kAdd, v3->GetOpKind()); + + EXPECT_TRUE(v1->Equals(v1)); + EXPECT_TRUE(v2->Equals(v2)); + EXPECT_TRUE(v3->Equals(v3)); + + EXPECT_FALSE(v1->Equals(v2)); // different operators + EXPECT_FALSE(v1->Equals(v3)); // different vector lengths +} + +} // namespace art diff --git a/compiler/optimizing/optimizing_cfi_test.cc b/compiler/optimizing/optimizing_cfi_test.cc index 490e50cb77..6cb27b3b1b 100644 --- a/compiler/optimizing/optimizing_cfi_test.cc +++ b/compiler/optimizing/optimizing_cfi_test.cc @@ -24,21 +24,15 @@ #include "optimizing/code_generator.h" #include "optimizing/optimizing_unit_test.h" #include "utils/assembler.h" -#ifdef ART_USE_OLD_ARM_BACKEND -#include "utils/arm/assembler_thumb2.h" -#else #include "utils/arm/assembler_arm_vixl.h" -#endif #include "utils/mips/assembler_mips.h" #include "utils/mips64/assembler_mips64.h" #include "optimizing/optimizing_cfi_test_expected.inc" -#ifndef ART_USE_OLD_ARM_BACKEND namespace vixl32 = vixl::aarch32; using vixl32::r0; -#endif namespace art { @@ -171,18 +165,31 @@ class OptimizingCFITest : public CFITest { #ifdef ART_ENABLE_CODEGEN_arm TEST_ISA(kThumb2) #endif + #ifdef ART_ENABLE_CODEGEN_arm64 +// Run the tests for ARM64 only with Baker read barriers, as the +// expected generated code saves and restore X21 and X22 (instead of +// X20 and X21), as X20 is used as Marking Register in the Baker read +// barrier configuration, and as such is removed from the set of +// callee-save registers in the ARM64 code generator of the Optimizing +// compiler. +#if defined(USE_READ_BARRIER) && defined(USE_BAKER_READ_BARRIER) TEST_ISA(kArm64) #endif +#endif + #ifdef ART_ENABLE_CODEGEN_x86 TEST_ISA(kX86) #endif + #ifdef ART_ENABLE_CODEGEN_x86_64 TEST_ISA(kX86_64) #endif + #ifdef ART_ENABLE_CODEGEN_mips TEST_ISA(kMips) #endif + #ifdef ART_ENABLE_CODEGEN_mips64 TEST_ISA(kMips64) #endif @@ -196,15 +203,6 @@ TEST_F(OptimizingCFITest, kThumb2Adjust) { expected_cfi_kThumb2_adjust, expected_cfi_kThumb2_adjust + arraysize(expected_cfi_kThumb2_adjust)); SetUpFrame(kThumb2); -#ifdef ART_USE_OLD_ARM_BACKEND -#define __ down_cast<arm::Thumb2Assembler*>(GetCodeGenerator()->GetAssembler())-> - Label target; - __ CompareAndBranchIfZero(arm::R0, &target); - // Push the target out of range of CBZ. - for (size_t i = 0; i != 65; ++i) { - __ ldr(arm::R0, arm::Address(arm::R0)); - } -#else #define __ down_cast<arm::ArmVIXLAssembler*>(GetCodeGenerator() \ ->GetAssembler())->GetVIXLAssembler()-> vixl32::Label target; @@ -213,7 +211,6 @@ TEST_F(OptimizingCFITest, kThumb2Adjust) { for (size_t i = 0; i != 65; ++i) { __ Ldr(r0, vixl32::MemOperand(r0)); } -#endif __ Bind(&target); #undef __ Finish(); diff --git a/compiler/optimizing/optimizing_cfi_test_expected.inc b/compiler/optimizing/optimizing_cfi_test_expected.inc index 60af2b4201..77a63acd18 100644 --- a/compiler/optimizing/optimizing_cfi_test_expected.inc +++ b/compiler/optimizing/optimizing_cfi_test_expected.inc @@ -31,21 +31,21 @@ static constexpr uint8_t expected_cfi_kThumb2[] = { // 0x00000010: .cfi_def_cfa_offset: 64 static constexpr uint8_t expected_asm_kArm64[] = { - 0xFF, 0x03, 0x01, 0xD1, 0xF4, 0x17, 0x00, 0xF9, 0xF5, 0x7B, 0x03, 0xA9, - 0xE8, 0xA7, 0x01, 0x6D, 0xE8, 0xA7, 0x41, 0x6D, 0xF4, 0x17, 0x40, 0xF9, - 0xF5, 0x7B, 0x43, 0xA9, 0xFF, 0x03, 0x01, 0x91, 0xC0, 0x03, 0x5F, 0xD6, + 0xFF, 0x03, 0x01, 0xD1, 0xF5, 0x17, 0x00, 0xF9, 0xF6, 0x7B, 0x03, 0xA9, + 0xE8, 0xA7, 0x01, 0x6D, 0xE8, 0xA7, 0x41, 0x6D, 0xF5, 0x17, 0x40, 0xF9, + 0xF6, 0x7B, 0x43, 0xA9, 0xFF, 0x03, 0x01, 0x91, 0xC0, 0x03, 0x5F, 0xD6, }; static constexpr uint8_t expected_cfi_kArm64[] = { - 0x44, 0x0E, 0x40, 0x44, 0x94, 0x06, 0x44, 0x95, 0x04, 0x9E, 0x02, 0x44, + 0x44, 0x0E, 0x40, 0x44, 0x95, 0x06, 0x44, 0x96, 0x04, 0x9E, 0x02, 0x44, 0x05, 0x48, 0x0A, 0x05, 0x49, 0x08, 0x0A, 0x44, 0x06, 0x48, 0x06, 0x49, - 0x44, 0xD4, 0x44, 0xD5, 0xDE, 0x44, 0x0E, 0x00, 0x44, 0x0B, 0x0E, 0x40, + 0x44, 0xD5, 0x44, 0xD6, 0xDE, 0x44, 0x0E, 0x00, 0x44, 0x0B, 0x0E, 0x40, }; // 0x00000000: sub sp, sp, #0x40 (64) // 0x00000004: .cfi_def_cfa_offset: 64 -// 0x00000004: str x20, [sp, #40] -// 0x00000008: .cfi_offset: r20 at cfa-24 -// 0x00000008: stp x21, lr, [sp, #48] -// 0x0000000c: .cfi_offset: r21 at cfa-16 +// 0x00000004: str x21, [sp, #40] +// 0x00000008: .cfi_offset: r21 at cfa-24 +// 0x00000008: stp x22, lr, [sp, #48] +// 0x0000000c: .cfi_offset: r22 at cfa-16 // 0x0000000c: .cfi_offset: r30 at cfa-8 // 0x0000000c: stp d8, d9, [sp, #24] // 0x00000010: .cfi_offset_extended: r72 at cfa-40 @@ -54,10 +54,10 @@ static constexpr uint8_t expected_cfi_kArm64[] = { // 0x00000010: ldp d8, d9, [sp, #24] // 0x00000014: .cfi_restore_extended: r72 // 0x00000014: .cfi_restore_extended: r73 -// 0x00000014: ldr x20, [sp, #40] -// 0x00000018: .cfi_restore: r20 -// 0x00000018: ldp x21, lr, [sp, #48] -// 0x0000001c: .cfi_restore: r21 +// 0x00000014: ldr x21, [sp, #40] +// 0x00000018: .cfi_restore: r21 +// 0x00000018: ldp x22, lr, [sp, #48] +// 0x0000001c: .cfi_restore: r22 // 0x0000001c: .cfi_restore: r30 // 0x0000001c: add sp, sp, #0x40 (64) // 0x00000020: .cfi_def_cfa_offset: 0 @@ -215,16 +215,11 @@ static constexpr uint8_t expected_cfi_kMips64[] = { // 0x00000034: .cfi_def_cfa_offset: 64 static constexpr uint8_t expected_asm_kThumb2_adjust[] = { -#ifdef ART_USE_OLD_ARM_BACKEND - 0x60, 0xB5, 0x2D, 0xED, 0x02, 0x8A, 0x8B, 0xB0, 0x00, 0x28, - 0x40, 0xD0, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, -#else // VIXL emits an extra 2 bytes here for a 32-bit beq as there is no // optimistic 16-bit emit and subsequent fixup for out of reach targets // as with the old assembler. 0x60, 0xB5, 0x2D, 0xED, 0x02, 0x8A, 0x8B, 0xB0, 0x00, 0x28, 0x00, 0xF0, 0x41, 0x80, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, -#endif 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, 0x00, 0x68, @@ -239,11 +234,7 @@ static constexpr uint8_t expected_asm_kThumb2_adjust[] = { }; static constexpr uint8_t expected_cfi_kThumb2_adjust[] = { 0x42, 0x0E, 0x0C, 0x85, 0x03, 0x86, 0x02, 0x8E, 0x01, 0x44, 0x0E, 0x14, -#ifdef ART_USE_OLD_ARM_BACKEND - 0x05, 0x50, 0x05, 0x05, 0x51, 0x04, 0x42, 0x0E, 0x40, 0x02, 0x86, 0x0A, -#else 0x05, 0x50, 0x05, 0x05, 0x51, 0x04, 0x42, 0x0E, 0x40, 0x02, 0x88, 0x0A, -#endif 0x42, 0x0E, 0x14, 0x44, 0x0E, 0x0C, 0x06, 0x50, 0x06, 0x51, 0x42, 0x0B, 0x0E, 0x40, }; diff --git a/compiler/optimizing/scheduler_arm.h b/compiler/optimizing/scheduler_arm.h index 897e97da49..a9f2295c35 100644 --- a/compiler/optimizing/scheduler_arm.h +++ b/compiler/optimizing/scheduler_arm.h @@ -17,20 +17,13 @@ #ifndef ART_COMPILER_OPTIMIZING_SCHEDULER_ARM_H_ #define ART_COMPILER_OPTIMIZING_SCHEDULER_ARM_H_ -#ifdef ART_USE_OLD_ARM_BACKEND -#include "code_generator_arm.h" -#else #include "code_generator_arm_vixl.h" -#endif #include "scheduler.h" namespace art { namespace arm { -#ifdef ART_USE_OLD_ARM_BACKEND -typedef CodeGeneratorARM CodeGeneratorARMType; -#else +// TODO: Replace CodeGeneratorARMType with CodeGeneratorARMVIXL everywhere? typedef CodeGeneratorARMVIXL CodeGeneratorARMType; -#endif // AArch32 instruction latencies. // We currently assume that all ARM CPUs share the same instruction latency list. diff --git a/compiler/optimizing/scheduler_test.cc b/compiler/optimizing/scheduler_test.cc index cc7222dd45..10c3cd7535 100644 --- a/compiler/optimizing/scheduler_test.cc +++ b/compiler/optimizing/scheduler_test.cc @@ -41,8 +41,8 @@ static ::std::vector<CodegenTargetConfig> GetTargetConfigs() { ::std::vector<CodegenTargetConfig> v; ::std::vector<CodegenTargetConfig> test_config_candidates = { #ifdef ART_ENABLE_CODEGEN_arm - CodegenTargetConfig(kArm, create_codegen_arm), - CodegenTargetConfig(kThumb2, create_codegen_arm), + // TODO: Should't this be `kThumb2` instead of `kArm` here? + CodegenTargetConfig(kArm, create_codegen_arm_vixl32), #endif #ifdef ART_ENABLE_CODEGEN_arm64 CodegenTargetConfig(kArm64, create_codegen_arm64), diff --git a/compiler/utils/arm/assembler_arm_vixl.cc b/compiler/utils/arm/assembler_arm_vixl.cc index eb3f870432..af3b4474e3 100644 --- a/compiler/utils/arm/assembler_arm_vixl.cc +++ b/compiler/utils/arm/assembler_arm_vixl.cc @@ -37,7 +37,10 @@ namespace arm { #define ___ vixl_masm_. #endif +// Thread register definition. extern const vixl32::Register tr(TR); +// Marking register definition. +extern const vixl32::Register mr(MR); void ArmVIXLAssembler::FinalizeCode() { vixl_masm_.FinalizeCode(); diff --git a/compiler/utils/arm/assembler_arm_vixl.h b/compiler/utils/arm/assembler_arm_vixl.h index e81e767575..66b22ea87c 100644 --- a/compiler/utils/arm/assembler_arm_vixl.h +++ b/compiler/utils/arm/assembler_arm_vixl.h @@ -241,6 +241,8 @@ class ArmVIXLAssembler FINAL : public Assembler { // Thread register declaration. extern const vixl32::Register tr; +// Marking register declaration. +extern const vixl32::Register mr; } // namespace arm } // namespace art diff --git a/compiler/utils/arm/jni_macro_assembler_arm.cc b/compiler/utils/arm/jni_macro_assembler_arm.cc deleted file mode 100644 index 3f425dfaf5..0000000000 --- a/compiler/utils/arm/jni_macro_assembler_arm.cc +++ /dev/null @@ -1,659 +0,0 @@ -/* - * Copyright (C) 2016 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "jni_macro_assembler_arm.h" - -#include <algorithm> - -#include "assembler_thumb2.h" -#include "base/arena_allocator.h" -#include "base/bit_utils.h" -#include "base/logging.h" -#include "entrypoints/quick/quick_entrypoints.h" -#include "offsets.h" -#include "thread.h" - -namespace art { -namespace arm { - -constexpr size_t kFramePointerSize = static_cast<size_t>(kArmPointerSize); - -// Slowpath entered when Thread::Current()->_exception is non-null -class ArmExceptionSlowPath FINAL : public SlowPath { - public: - ArmExceptionSlowPath(ArmManagedRegister scratch, size_t stack_adjust) - : scratch_(scratch), stack_adjust_(stack_adjust) { - } - void Emit(Assembler *sp_asm) OVERRIDE; - private: - const ArmManagedRegister scratch_; - const size_t stack_adjust_; -}; - -ArmJNIMacroAssembler::ArmJNIMacroAssembler(ArenaAllocator* arena, InstructionSet isa) { - switch (isa) { - case kArm: - case kThumb2: - asm_.reset(new (arena) Thumb2Assembler(arena)); - break; - - default: - LOG(FATAL) << isa; - UNREACHABLE(); - } -} - -ArmJNIMacroAssembler::~ArmJNIMacroAssembler() { -} - -size_t ArmJNIMacroAssembler::CodeSize() const { - return asm_->CodeSize(); -} - -DebugFrameOpCodeWriterForAssembler& ArmJNIMacroAssembler::cfi() { - return asm_->cfi(); -} - -void ArmJNIMacroAssembler::FinalizeCode() { - asm_->FinalizeCode(); -} - -void ArmJNIMacroAssembler::FinalizeInstructions(const MemoryRegion& region) { - asm_->FinalizeInstructions(region); -} - -static dwarf::Reg DWARFReg(Register reg) { - return dwarf::Reg::ArmCore(static_cast<int>(reg)); -} - -static dwarf::Reg DWARFReg(SRegister reg) { - return dwarf::Reg::ArmFp(static_cast<int>(reg)); -} - -#define __ asm_-> - -void ArmJNIMacroAssembler::BuildFrame(size_t frame_size, - ManagedRegister method_reg, - ArrayRef<const ManagedRegister> callee_save_regs, - const ManagedRegisterEntrySpills& entry_spills) { - CHECK_EQ(CodeSize(), 0U); // Nothing emitted yet - CHECK_ALIGNED(frame_size, kStackAlignment); - CHECK_EQ(R0, method_reg.AsArm().AsCoreRegister()); - - // Push callee saves and link register. - RegList core_spill_mask = 1 << LR; - uint32_t fp_spill_mask = 0; - for (const ManagedRegister& reg : callee_save_regs) { - if (reg.AsArm().IsCoreRegister()) { - core_spill_mask |= 1 << reg.AsArm().AsCoreRegister(); - } else { - fp_spill_mask |= 1 << reg.AsArm().AsSRegister(); - } - } - __ PushList(core_spill_mask); - cfi().AdjustCFAOffset(POPCOUNT(core_spill_mask) * kFramePointerSize); - cfi().RelOffsetForMany(DWARFReg(Register(0)), 0, core_spill_mask, kFramePointerSize); - if (fp_spill_mask != 0) { - __ vpushs(SRegister(CTZ(fp_spill_mask)), POPCOUNT(fp_spill_mask)); - cfi().AdjustCFAOffset(POPCOUNT(fp_spill_mask) * kFramePointerSize); - cfi().RelOffsetForMany(DWARFReg(SRegister(0)), 0, fp_spill_mask, kFramePointerSize); - } - - // Increase frame to required size. - int pushed_values = POPCOUNT(core_spill_mask) + POPCOUNT(fp_spill_mask); - CHECK_GT(frame_size, pushed_values * kFramePointerSize); // Must at least have space for Method*. - IncreaseFrameSize(frame_size - pushed_values * kFramePointerSize); // handles CFI as well. - - // Write out Method*. - __ StoreToOffset(kStoreWord, R0, SP, 0); - - // Write out entry spills. - int32_t offset = frame_size + kFramePointerSize; - for (size_t i = 0; i < entry_spills.size(); ++i) { - ArmManagedRegister reg = entry_spills.at(i).AsArm(); - if (reg.IsNoRegister()) { - // only increment stack offset. - ManagedRegisterSpill spill = entry_spills.at(i); - offset += spill.getSize(); - } else if (reg.IsCoreRegister()) { - __ StoreToOffset(kStoreWord, reg.AsCoreRegister(), SP, offset); - offset += 4; - } else if (reg.IsSRegister()) { - __ StoreSToOffset(reg.AsSRegister(), SP, offset); - offset += 4; - } else if (reg.IsDRegister()) { - __ StoreDToOffset(reg.AsDRegister(), SP, offset); - offset += 8; - } - } -} - -void ArmJNIMacroAssembler::RemoveFrame(size_t frame_size, - ArrayRef<const ManagedRegister> callee_save_regs) { - CHECK_ALIGNED(frame_size, kStackAlignment); - cfi().RememberState(); - - // Compute callee saves to pop and PC. - RegList core_spill_mask = 1 << PC; - uint32_t fp_spill_mask = 0; - for (const ManagedRegister& reg : callee_save_regs) { - if (reg.AsArm().IsCoreRegister()) { - core_spill_mask |= 1 << reg.AsArm().AsCoreRegister(); - } else { - fp_spill_mask |= 1 << reg.AsArm().AsSRegister(); - } - } - - // Decrease frame to start of callee saves. - int pop_values = POPCOUNT(core_spill_mask) + POPCOUNT(fp_spill_mask); - CHECK_GT(frame_size, pop_values * kFramePointerSize); - DecreaseFrameSize(frame_size - (pop_values * kFramePointerSize)); // handles CFI as well. - - if (fp_spill_mask != 0) { - __ vpops(SRegister(CTZ(fp_spill_mask)), POPCOUNT(fp_spill_mask)); - cfi().AdjustCFAOffset(-kFramePointerSize * POPCOUNT(fp_spill_mask)); - cfi().RestoreMany(DWARFReg(SRegister(0)), fp_spill_mask); - } - - // Pop callee saves and PC. - __ PopList(core_spill_mask); - - // The CFI should be restored for any code that follows the exit block. - cfi().RestoreState(); - cfi().DefCFAOffset(frame_size); -} - -void ArmJNIMacroAssembler::IncreaseFrameSize(size_t adjust) { - __ AddConstant(SP, -adjust); - cfi().AdjustCFAOffset(adjust); -} - -static void DecreaseFrameSizeImpl(ArmAssembler* assembler, size_t adjust) { - assembler->AddConstant(SP, adjust); - assembler->cfi().AdjustCFAOffset(-adjust); -} - -void ArmJNIMacroAssembler::DecreaseFrameSize(size_t adjust) { - DecreaseFrameSizeImpl(asm_.get(), adjust); -} - -void ArmJNIMacroAssembler::Store(FrameOffset dest, ManagedRegister msrc, size_t size) { - ArmManagedRegister src = msrc.AsArm(); - if (src.IsNoRegister()) { - CHECK_EQ(0u, size); - } else if (src.IsCoreRegister()) { - CHECK_EQ(4u, size); - __ StoreToOffset(kStoreWord, src.AsCoreRegister(), SP, dest.Int32Value()); - } else if (src.IsRegisterPair()) { - CHECK_EQ(8u, size); - __ StoreToOffset(kStoreWord, src.AsRegisterPairLow(), SP, dest.Int32Value()); - __ StoreToOffset(kStoreWord, src.AsRegisterPairHigh(), SP, dest.Int32Value() + 4); - } else if (src.IsSRegister()) { - __ StoreSToOffset(src.AsSRegister(), SP, dest.Int32Value()); - } else { - CHECK(src.IsDRegister()) << src; - __ StoreDToOffset(src.AsDRegister(), SP, dest.Int32Value()); - } -} - -void ArmJNIMacroAssembler::StoreRef(FrameOffset dest, ManagedRegister msrc) { - ArmManagedRegister src = msrc.AsArm(); - CHECK(src.IsCoreRegister()) << src; - __ StoreToOffset(kStoreWord, src.AsCoreRegister(), SP, dest.Int32Value()); -} - -void ArmJNIMacroAssembler::StoreRawPtr(FrameOffset dest, ManagedRegister msrc) { - ArmManagedRegister src = msrc.AsArm(); - CHECK(src.IsCoreRegister()) << src; - __ StoreToOffset(kStoreWord, src.AsCoreRegister(), SP, dest.Int32Value()); -} - -void ArmJNIMacroAssembler::StoreSpanning(FrameOffset dest, - ManagedRegister msrc, - FrameOffset in_off, - ManagedRegister mscratch) { - ArmManagedRegister src = msrc.AsArm(); - ArmManagedRegister scratch = mscratch.AsArm(); - __ StoreToOffset(kStoreWord, src.AsCoreRegister(), SP, dest.Int32Value()); - __ LoadFromOffset(kLoadWord, scratch.AsCoreRegister(), SP, in_off.Int32Value()); - __ StoreToOffset(kStoreWord, scratch.AsCoreRegister(), SP, dest.Int32Value() + sizeof(uint32_t)); -} - -void ArmJNIMacroAssembler::CopyRef(FrameOffset dest, FrameOffset src, ManagedRegister mscratch) { - ArmManagedRegister scratch = mscratch.AsArm(); - __ LoadFromOffset(kLoadWord, scratch.AsCoreRegister(), SP, src.Int32Value()); - __ StoreToOffset(kStoreWord, scratch.AsCoreRegister(), SP, dest.Int32Value()); -} - -void ArmJNIMacroAssembler::LoadRef(ManagedRegister mdest, - ManagedRegister mbase, - MemberOffset offs, - bool unpoison_reference) { - ArmManagedRegister base = mbase.AsArm(); - ArmManagedRegister dst = mdest.AsArm(); - CHECK(base.IsCoreRegister()) << base; - CHECK(dst.IsCoreRegister()) << dst; - __ LoadFromOffset(kLoadWord, - dst.AsCoreRegister(), - base.AsCoreRegister(), - offs.Int32Value()); - if (unpoison_reference) { - __ MaybeUnpoisonHeapReference(dst.AsCoreRegister()); - } -} - -void ArmJNIMacroAssembler::LoadRef(ManagedRegister mdest, FrameOffset src) { - ArmManagedRegister dst = mdest.AsArm(); - CHECK(dst.IsCoreRegister()) << dst; - __ LoadFromOffset(kLoadWord, dst.AsCoreRegister(), SP, src.Int32Value()); -} - -void ArmJNIMacroAssembler::LoadRawPtr(ManagedRegister mdest, - ManagedRegister mbase, - Offset offs) { - ArmManagedRegister base = mbase.AsArm(); - ArmManagedRegister dst = mdest.AsArm(); - CHECK(base.IsCoreRegister()) << base; - CHECK(dst.IsCoreRegister()) << dst; - __ LoadFromOffset(kLoadWord, - dst.AsCoreRegister(), - base.AsCoreRegister(), - offs.Int32Value()); -} - -void ArmJNIMacroAssembler::StoreImmediateToFrame(FrameOffset dest, - uint32_t imm, - ManagedRegister mscratch) { - ArmManagedRegister scratch = mscratch.AsArm(); - CHECK(scratch.IsCoreRegister()) << scratch; - __ LoadImmediate(scratch.AsCoreRegister(), imm); - __ StoreToOffset(kStoreWord, scratch.AsCoreRegister(), SP, dest.Int32Value()); -} - -static void EmitLoad(ArmAssembler* assembler, - ManagedRegister m_dst, - Register src_register, - int32_t src_offset, - size_t size) { - ArmManagedRegister dst = m_dst.AsArm(); - if (dst.IsNoRegister()) { - CHECK_EQ(0u, size) << dst; - } else if (dst.IsCoreRegister()) { - CHECK_EQ(4u, size) << dst; - assembler->LoadFromOffset(kLoadWord, dst.AsCoreRegister(), src_register, src_offset); - } else if (dst.IsRegisterPair()) { - CHECK_EQ(8u, size) << dst; - assembler->LoadFromOffset(kLoadWord, dst.AsRegisterPairLow(), src_register, src_offset); - assembler->LoadFromOffset(kLoadWord, dst.AsRegisterPairHigh(), src_register, src_offset + 4); - } else if (dst.IsSRegister()) { - assembler->LoadSFromOffset(dst.AsSRegister(), src_register, src_offset); - } else { - CHECK(dst.IsDRegister()) << dst; - assembler->LoadDFromOffset(dst.AsDRegister(), src_register, src_offset); - } -} - -void ArmJNIMacroAssembler::Load(ManagedRegister m_dst, FrameOffset src, size_t size) { - EmitLoad(asm_.get(), m_dst, SP, src.Int32Value(), size); -} - -void ArmJNIMacroAssembler::LoadFromThread(ManagedRegister m_dst, ThreadOffset32 src, size_t size) { - EmitLoad(asm_.get(), m_dst, TR, src.Int32Value(), size); -} - -void ArmJNIMacroAssembler::LoadRawPtrFromThread(ManagedRegister m_dst, ThreadOffset32 offs) { - ArmManagedRegister dst = m_dst.AsArm(); - CHECK(dst.IsCoreRegister()) << dst; - __ LoadFromOffset(kLoadWord, dst.AsCoreRegister(), TR, offs.Int32Value()); -} - -void ArmJNIMacroAssembler::CopyRawPtrFromThread(FrameOffset fr_offs, - ThreadOffset32 thr_offs, - ManagedRegister mscratch) { - ArmManagedRegister scratch = mscratch.AsArm(); - CHECK(scratch.IsCoreRegister()) << scratch; - __ LoadFromOffset(kLoadWord, scratch.AsCoreRegister(), TR, thr_offs.Int32Value()); - __ StoreToOffset(kStoreWord, scratch.AsCoreRegister(), SP, fr_offs.Int32Value()); -} - -void ArmJNIMacroAssembler::CopyRawPtrToThread(ThreadOffset32 thr_offs, - FrameOffset fr_offs, - ManagedRegister mscratch) { - ArmManagedRegister scratch = mscratch.AsArm(); - CHECK(scratch.IsCoreRegister()) << scratch; - __ LoadFromOffset(kLoadWord, scratch.AsCoreRegister(), SP, fr_offs.Int32Value()); - __ StoreToOffset(kStoreWord, scratch.AsCoreRegister(), TR, thr_offs.Int32Value()); -} - -void ArmJNIMacroAssembler::StoreStackOffsetToThread(ThreadOffset32 thr_offs, - FrameOffset fr_offs, - ManagedRegister mscratch) { - ArmManagedRegister scratch = mscratch.AsArm(); - CHECK(scratch.IsCoreRegister()) << scratch; - __ AddConstant(scratch.AsCoreRegister(), SP, fr_offs.Int32Value(), AL); - __ StoreToOffset(kStoreWord, scratch.AsCoreRegister(), TR, thr_offs.Int32Value()); -} - -void ArmJNIMacroAssembler::StoreStackPointerToThread(ThreadOffset32 thr_offs) { - __ StoreToOffset(kStoreWord, SP, TR, thr_offs.Int32Value()); -} - -void ArmJNIMacroAssembler::SignExtend(ManagedRegister /*mreg*/, size_t /*size*/) { - UNIMPLEMENTED(FATAL) << "no sign extension necessary for arm"; -} - -void ArmJNIMacroAssembler::ZeroExtend(ManagedRegister /*mreg*/, size_t /*size*/) { - UNIMPLEMENTED(FATAL) << "no zero extension necessary for arm"; -} - -void ArmJNIMacroAssembler::Move(ManagedRegister m_dst, ManagedRegister m_src, size_t /*size*/) { - ArmManagedRegister dst = m_dst.AsArm(); - ArmManagedRegister src = m_src.AsArm(); - if (!dst.Equals(src)) { - if (dst.IsCoreRegister()) { - CHECK(src.IsCoreRegister()) << src; - __ mov(dst.AsCoreRegister(), ShifterOperand(src.AsCoreRegister())); - } else if (dst.IsDRegister()) { - if (src.IsDRegister()) { - __ vmovd(dst.AsDRegister(), src.AsDRegister()); - } else { - // VMOV Dn, Rlo, Rhi (Dn = {Rlo, Rhi}) - CHECK(src.IsRegisterPair()) << src; - __ vmovdrr(dst.AsDRegister(), src.AsRegisterPairLow(), src.AsRegisterPairHigh()); - } - } else if (dst.IsSRegister()) { - if (src.IsSRegister()) { - __ vmovs(dst.AsSRegister(), src.AsSRegister()); - } else { - // VMOV Sn, Rn (Sn = Rn) - CHECK(src.IsCoreRegister()) << src; - __ vmovsr(dst.AsSRegister(), src.AsCoreRegister()); - } - } else { - CHECK(dst.IsRegisterPair()) << dst; - CHECK(src.IsRegisterPair()) << src; - // Ensure that the first move doesn't clobber the input of the second. - if (src.AsRegisterPairHigh() != dst.AsRegisterPairLow()) { - __ mov(dst.AsRegisterPairLow(), ShifterOperand(src.AsRegisterPairLow())); - __ mov(dst.AsRegisterPairHigh(), ShifterOperand(src.AsRegisterPairHigh())); - } else { - __ mov(dst.AsRegisterPairHigh(), ShifterOperand(src.AsRegisterPairHigh())); - __ mov(dst.AsRegisterPairLow(), ShifterOperand(src.AsRegisterPairLow())); - } - } - } -} - -void ArmJNIMacroAssembler::Copy(FrameOffset dest, - FrameOffset src, - ManagedRegister mscratch, - size_t size) { - ArmManagedRegister scratch = mscratch.AsArm(); - CHECK(scratch.IsCoreRegister()) << scratch; - CHECK(size == 4 || size == 8) << size; - if (size == 4) { - __ LoadFromOffset(kLoadWord, scratch.AsCoreRegister(), SP, src.Int32Value()); - __ StoreToOffset(kStoreWord, scratch.AsCoreRegister(), SP, dest.Int32Value()); - } else if (size == 8) { - __ LoadFromOffset(kLoadWord, scratch.AsCoreRegister(), SP, src.Int32Value()); - __ StoreToOffset(kStoreWord, scratch.AsCoreRegister(), SP, dest.Int32Value()); - __ LoadFromOffset(kLoadWord, scratch.AsCoreRegister(), SP, src.Int32Value() + 4); - __ StoreToOffset(kStoreWord, scratch.AsCoreRegister(), SP, dest.Int32Value() + 4); - } -} - -void ArmJNIMacroAssembler::Copy(FrameOffset dest, - ManagedRegister src_base, - Offset src_offset, - ManagedRegister mscratch, - size_t size) { - Register scratch = mscratch.AsArm().AsCoreRegister(); - CHECK_EQ(size, 4u); - __ LoadFromOffset(kLoadWord, scratch, src_base.AsArm().AsCoreRegister(), src_offset.Int32Value()); - __ StoreToOffset(kStoreWord, scratch, SP, dest.Int32Value()); -} - -void ArmJNIMacroAssembler::Copy(ManagedRegister dest_base, - Offset dest_offset, - FrameOffset src, - ManagedRegister mscratch, - size_t size) { - Register scratch = mscratch.AsArm().AsCoreRegister(); - CHECK_EQ(size, 4u); - __ LoadFromOffset(kLoadWord, scratch, SP, src.Int32Value()); - __ StoreToOffset(kStoreWord, - scratch, - dest_base.AsArm().AsCoreRegister(), - dest_offset.Int32Value()); -} - -void ArmJNIMacroAssembler::Copy(FrameOffset /*dst*/, - FrameOffset /*src_base*/, - Offset /*src_offset*/, - ManagedRegister /*mscratch*/, - size_t /*size*/) { - UNIMPLEMENTED(FATAL); -} - -void ArmJNIMacroAssembler::Copy(ManagedRegister dest, - Offset dest_offset, - ManagedRegister src, - Offset src_offset, - ManagedRegister mscratch, - size_t size) { - CHECK_EQ(size, 4u); - Register scratch = mscratch.AsArm().AsCoreRegister(); - __ LoadFromOffset(kLoadWord, scratch, src.AsArm().AsCoreRegister(), src_offset.Int32Value()); - __ StoreToOffset(kStoreWord, scratch, dest.AsArm().AsCoreRegister(), dest_offset.Int32Value()); -} - -void ArmJNIMacroAssembler::Copy(FrameOffset /*dst*/, - Offset /*dest_offset*/, - FrameOffset /*src*/, - Offset /*src_offset*/, - ManagedRegister /*scratch*/, - size_t /*size*/) { - UNIMPLEMENTED(FATAL); -} - -void ArmJNIMacroAssembler::CreateHandleScopeEntry(ManagedRegister mout_reg, - FrameOffset handle_scope_offset, - ManagedRegister min_reg, - bool null_allowed) { - ArmManagedRegister out_reg = mout_reg.AsArm(); - ArmManagedRegister in_reg = min_reg.AsArm(); - CHECK(in_reg.IsNoRegister() || in_reg.IsCoreRegister()) << in_reg; - CHECK(out_reg.IsCoreRegister()) << out_reg; - if (null_allowed) { - // Null values get a handle scope entry value of 0. Otherwise, the handle scope entry is - // the address in the handle scope holding the reference. - // e.g. out_reg = (handle == 0) ? 0 : (SP+handle_offset) - if (in_reg.IsNoRegister()) { - __ LoadFromOffset(kLoadWord, out_reg.AsCoreRegister(), SP, handle_scope_offset.Int32Value()); - in_reg = out_reg; - } - __ cmp(in_reg.AsCoreRegister(), ShifterOperand(0)); - if (!out_reg.Equals(in_reg)) { - __ it(EQ, kItElse); - __ LoadImmediate(out_reg.AsCoreRegister(), 0, EQ); - } else { - __ it(NE); - } - __ AddConstant(out_reg.AsCoreRegister(), SP, handle_scope_offset.Int32Value(), NE); - } else { - __ AddConstant(out_reg.AsCoreRegister(), SP, handle_scope_offset.Int32Value(), AL); - } -} - -void ArmJNIMacroAssembler::CreateHandleScopeEntry(FrameOffset out_off, - FrameOffset handle_scope_offset, - ManagedRegister mscratch, - bool null_allowed) { - ArmManagedRegister scratch = mscratch.AsArm(); - CHECK(scratch.IsCoreRegister()) << scratch; - if (null_allowed) { - __ LoadFromOffset(kLoadWord, scratch.AsCoreRegister(), SP, handle_scope_offset.Int32Value()); - // Null values get a handle scope entry value of 0. Otherwise, the handle scope entry is - // the address in the handle scope holding the reference. - // e.g. scratch = (scratch == 0) ? 0 : (SP+handle_scope_offset) - __ cmp(scratch.AsCoreRegister(), ShifterOperand(0)); - __ it(NE); - __ AddConstant(scratch.AsCoreRegister(), SP, handle_scope_offset.Int32Value(), NE); - } else { - __ AddConstant(scratch.AsCoreRegister(), SP, handle_scope_offset.Int32Value(), AL); - } - __ StoreToOffset(kStoreWord, scratch.AsCoreRegister(), SP, out_off.Int32Value()); -} - -void ArmJNIMacroAssembler::LoadReferenceFromHandleScope(ManagedRegister mout_reg, - ManagedRegister min_reg) { - ArmManagedRegister out_reg = mout_reg.AsArm(); - ArmManagedRegister in_reg = min_reg.AsArm(); - CHECK(out_reg.IsCoreRegister()) << out_reg; - CHECK(in_reg.IsCoreRegister()) << in_reg; - Label null_arg; - if (!out_reg.Equals(in_reg)) { - __ LoadImmediate(out_reg.AsCoreRegister(), 0, EQ); // TODO: why EQ? - } - __ cmp(in_reg.AsCoreRegister(), ShifterOperand(0)); - __ it(NE); - __ LoadFromOffset(kLoadWord, out_reg.AsCoreRegister(), in_reg.AsCoreRegister(), 0, NE); -} - -void ArmJNIMacroAssembler::VerifyObject(ManagedRegister /*src*/, bool /*could_be_null*/) { - // TODO: not validating references. -} - -void ArmJNIMacroAssembler::VerifyObject(FrameOffset /*src*/, bool /*could_be_null*/) { - // TODO: not validating references. -} - -void ArmJNIMacroAssembler::Call(ManagedRegister mbase, - Offset offset, - ManagedRegister mscratch) { - ArmManagedRegister base = mbase.AsArm(); - ArmManagedRegister scratch = mscratch.AsArm(); - CHECK(base.IsCoreRegister()) << base; - CHECK(scratch.IsCoreRegister()) << scratch; - __ LoadFromOffset(kLoadWord, - scratch.AsCoreRegister(), - base.AsCoreRegister(), - offset.Int32Value()); - __ blx(scratch.AsCoreRegister()); - // TODO: place reference map on call. -} - -void ArmJNIMacroAssembler::Call(FrameOffset base, Offset offset, ManagedRegister mscratch) { - ArmManagedRegister scratch = mscratch.AsArm(); - CHECK(scratch.IsCoreRegister()) << scratch; - // Call *(*(SP + base) + offset) - __ LoadFromOffset(kLoadWord, scratch.AsCoreRegister(), SP, base.Int32Value()); - __ LoadFromOffset(kLoadWord, - scratch.AsCoreRegister(), - scratch.AsCoreRegister(), - offset.Int32Value()); - __ blx(scratch.AsCoreRegister()); - // TODO: place reference map on call -} - -void ArmJNIMacroAssembler::CallFromThread(ThreadOffset32 offset ATTRIBUTE_UNUSED, - ManagedRegister scratch ATTRIBUTE_UNUSED) { - UNIMPLEMENTED(FATAL); -} - -void ArmJNIMacroAssembler::GetCurrentThread(ManagedRegister tr) { - __ mov(tr.AsArm().AsCoreRegister(), ShifterOperand(TR)); -} - -void ArmJNIMacroAssembler::GetCurrentThread(FrameOffset offset, ManagedRegister /*scratch*/) { - __ StoreToOffset(kStoreWord, TR, SP, offset.Int32Value(), AL); -} - -void ArmJNIMacroAssembler::ExceptionPoll(ManagedRegister mscratch, size_t stack_adjust) { - ArmManagedRegister scratch = mscratch.AsArm(); - ArmExceptionSlowPath* slow = new (__ GetArena()) ArmExceptionSlowPath(scratch, stack_adjust); - __ GetBuffer()->EnqueueSlowPath(slow); - __ LoadFromOffset(kLoadWord, - scratch.AsCoreRegister(), - TR, - Thread::ExceptionOffset<kArmPointerSize>().Int32Value()); - __ cmp(scratch.AsCoreRegister(), ShifterOperand(0)); - __ b(slow->Entry(), NE); -} - -std::unique_ptr<JNIMacroLabel> ArmJNIMacroAssembler::CreateLabel() { - return std::unique_ptr<JNIMacroLabel>(new ArmJNIMacroLabel()); -} - -void ArmJNIMacroAssembler::Jump(JNIMacroLabel* label) { - CHECK(label != nullptr); - __ b(ArmJNIMacroLabel::Cast(label)->AsArm()); -} - -void ArmJNIMacroAssembler::Jump(JNIMacroLabel* label, - JNIMacroUnaryCondition condition, - ManagedRegister test) { - CHECK(label != nullptr); - - arm::Condition arm_cond; - switch (condition) { - case JNIMacroUnaryCondition::kZero: - arm_cond = EQ; - break; - case JNIMacroUnaryCondition::kNotZero: - arm_cond = NE; - break; - default: - LOG(FATAL) << "Not implemented condition: " << static_cast<int>(condition); - UNREACHABLE(); - } - __ cmp(test.AsArm().AsCoreRegister(), ShifterOperand(0)); - __ b(ArmJNIMacroLabel::Cast(label)->AsArm(), arm_cond); -} - -void ArmJNIMacroAssembler::Bind(JNIMacroLabel* label) { - CHECK(label != nullptr); - __ Bind(ArmJNIMacroLabel::Cast(label)->AsArm()); -} - -#undef __ - -void ArmExceptionSlowPath::Emit(Assembler* sasm) { - ArmAssembler* sp_asm = down_cast<ArmAssembler*>(sasm); -#define __ sp_asm-> - __ Bind(&entry_); - if (stack_adjust_ != 0) { // Fix up the frame. - DecreaseFrameSizeImpl(sp_asm, stack_adjust_); - } - // Pass exception object as argument. - // Don't care about preserving R0 as this call won't return. - __ mov(R0, ShifterOperand(scratch_.AsCoreRegister())); - // Set up call to Thread::Current()->pDeliverException. - __ LoadFromOffset(kLoadWord, - R12, - TR, - QUICK_ENTRYPOINT_OFFSET(kArmPointerSize, pDeliverException).Int32Value()); - __ blx(R12); -#undef __ -} - -void ArmJNIMacroAssembler::MemoryBarrier(ManagedRegister mscratch) { - CHECK_EQ(mscratch.AsArm().AsCoreRegister(), R12); - asm_->dmb(SY); -} - -} // namespace arm -} // namespace art diff --git a/compiler/utils/arm/jni_macro_assembler_arm.h b/compiler/utils/arm/jni_macro_assembler_arm.h deleted file mode 100644 index 809ac8be94..0000000000 --- a/compiler/utils/arm/jni_macro_assembler_arm.h +++ /dev/null @@ -1,186 +0,0 @@ -/* - * Copyright (C) 2016 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef ART_COMPILER_UTILS_ARM_JNI_MACRO_ASSEMBLER_ARM_H_ -#define ART_COMPILER_UTILS_ARM_JNI_MACRO_ASSEMBLER_ARM_H_ - -#include <memory> -#include <type_traits> -#include <vector> - -#include "arch/instruction_set.h" -#include "base/enums.h" -#include "base/macros.h" -#include "utils/jni_macro_assembler.h" -#include "utils/label.h" -#include "offsets.h" - -namespace art { -namespace arm { - -class ArmAssembler; - -class ArmJNIMacroAssembler : public JNIMacroAssembler<PointerSize::k32> { - public: - ArmJNIMacroAssembler(ArenaAllocator* arena, InstructionSet isa); - virtual ~ArmJNIMacroAssembler(); - - size_t CodeSize() const OVERRIDE; - DebugFrameOpCodeWriterForAssembler& cfi() OVERRIDE; - void FinalizeCode() OVERRIDE; - void FinalizeInstructions(const MemoryRegion& region) OVERRIDE; - - // - // Overridden common assembler high-level functionality - // - - // Emit code that will create an activation on the stack - void BuildFrame(size_t frame_size, - ManagedRegister method_reg, - ArrayRef<const ManagedRegister> callee_save_regs, - const ManagedRegisterEntrySpills& entry_spills) OVERRIDE; - - // Emit code that will remove an activation from the stack - void RemoveFrame(size_t frame_size, ArrayRef<const ManagedRegister> callee_save_regs) - OVERRIDE; - - void IncreaseFrameSize(size_t adjust) OVERRIDE; - void DecreaseFrameSize(size_t adjust) OVERRIDE; - - // Store routines - void Store(FrameOffset offs, ManagedRegister src, size_t size) OVERRIDE; - void StoreRef(FrameOffset dest, ManagedRegister src) OVERRIDE; - void StoreRawPtr(FrameOffset dest, ManagedRegister src) OVERRIDE; - - void StoreImmediateToFrame(FrameOffset dest, uint32_t imm, ManagedRegister scratch) OVERRIDE; - - void StoreStackOffsetToThread(ThreadOffset32 thr_offs, - FrameOffset fr_offs, - ManagedRegister scratch) OVERRIDE; - - void StoreStackPointerToThread(ThreadOffset32 thr_offs) OVERRIDE; - - void StoreSpanning(FrameOffset dest, ManagedRegister src, FrameOffset in_off, - ManagedRegister scratch) OVERRIDE; - - // Load routines - void Load(ManagedRegister dest, FrameOffset src, size_t size) OVERRIDE; - - void LoadFromThread(ManagedRegister dest, ThreadOffset32 src, size_t size) OVERRIDE; - - void LoadRef(ManagedRegister dest, FrameOffset src) OVERRIDE; - - void LoadRef(ManagedRegister dest, ManagedRegister base, MemberOffset offs, - bool unpoison_reference) OVERRIDE; - - void LoadRawPtr(ManagedRegister dest, ManagedRegister base, Offset offs) OVERRIDE; - - void LoadRawPtrFromThread(ManagedRegister dest, ThreadOffset32 offs) OVERRIDE; - - // Copying routines - void Move(ManagedRegister dest, ManagedRegister src, size_t size) OVERRIDE; - - void CopyRawPtrFromThread(FrameOffset fr_offs, - ThreadOffset32 thr_offs, - ManagedRegister scratch) OVERRIDE; - - void CopyRawPtrToThread(ThreadOffset32 thr_offs, FrameOffset fr_offs, ManagedRegister scratch) - OVERRIDE; - - void CopyRef(FrameOffset dest, FrameOffset src, ManagedRegister scratch) OVERRIDE; - - void Copy(FrameOffset dest, FrameOffset src, ManagedRegister scratch, size_t size) OVERRIDE; - - void Copy(FrameOffset dest, ManagedRegister src_base, Offset src_offset, ManagedRegister scratch, - size_t size) OVERRIDE; - - void Copy(ManagedRegister dest_base, Offset dest_offset, FrameOffset src, ManagedRegister scratch, - size_t size) OVERRIDE; - - void Copy(FrameOffset dest, FrameOffset src_base, Offset src_offset, ManagedRegister scratch, - size_t size) OVERRIDE; - - void Copy(ManagedRegister dest, Offset dest_offset, ManagedRegister src, Offset src_offset, - ManagedRegister scratch, size_t size) OVERRIDE; - - void Copy(FrameOffset dest, Offset dest_offset, FrameOffset src, Offset src_offset, - ManagedRegister scratch, size_t size) OVERRIDE; - - // Sign extension - void SignExtend(ManagedRegister mreg, size_t size) OVERRIDE; - - // Zero extension - void ZeroExtend(ManagedRegister mreg, size_t size) OVERRIDE; - - // Exploit fast access in managed code to Thread::Current() - void GetCurrentThread(ManagedRegister tr) OVERRIDE; - void GetCurrentThread(FrameOffset dest_offset, ManagedRegister scratch) OVERRIDE; - - // Set up out_reg to hold a Object** into the handle scope, or to be null if the - // value is null and null_allowed. in_reg holds a possibly stale reference - // that can be used to avoid loading the handle scope entry to see if the value is - // null. - void CreateHandleScopeEntry(ManagedRegister out_reg, FrameOffset handlescope_offset, - ManagedRegister in_reg, bool null_allowed) OVERRIDE; - - // Set up out_off to hold a Object** into the handle scope, or to be null if the - // value is null and null_allowed. - void CreateHandleScopeEntry(FrameOffset out_off, FrameOffset handlescope_offset, - ManagedRegister scratch, bool null_allowed) OVERRIDE; - - // src holds a handle scope entry (Object**) load this into dst - void LoadReferenceFromHandleScope(ManagedRegister dst, ManagedRegister src) OVERRIDE; - - // Heap::VerifyObject on src. In some cases (such as a reference to this) we - // know that src may not be null. - void VerifyObject(ManagedRegister src, bool could_be_null) OVERRIDE; - void VerifyObject(FrameOffset src, bool could_be_null) OVERRIDE; - - // Call to address held at [base+offset] - void Call(ManagedRegister base, Offset offset, ManagedRegister scratch) OVERRIDE; - void Call(FrameOffset base, Offset offset, ManagedRegister scratch) OVERRIDE; - void CallFromThread(ThreadOffset32 offset, ManagedRegister scratch) OVERRIDE; - - // Generate code to check if Thread::Current()->exception_ is non-null - // and branch to a ExceptionSlowPath if it is. - void ExceptionPoll(ManagedRegister scratch, size_t stack_adjust) OVERRIDE; - - void MemoryBarrier(ManagedRegister scratch) OVERRIDE; - - // Create a new label that can be used with Jump/Bind calls. - std::unique_ptr<JNIMacroLabel> CreateLabel() OVERRIDE; - // Emit an unconditional jump to the label. - void Jump(JNIMacroLabel* label) OVERRIDE; - // Emit a conditional jump to the label by applying a unary condition test to the register. - void Jump(JNIMacroLabel* label, JNIMacroUnaryCondition cond, ManagedRegister test) OVERRIDE; - // Code at this offset will serve as the target for the Jump call. - void Bind(JNIMacroLabel* label) OVERRIDE; - - private: - std::unique_ptr<ArmAssembler> asm_; -}; - -class ArmJNIMacroLabel FINAL : public JNIMacroLabelCommon<ArmJNIMacroLabel, art::Label, kArm> { - public: - art::Label* AsArm() { - return AsPlatformLabel(); - } -}; - -} // namespace arm -} // namespace art - -#endif // ART_COMPILER_UTILS_ARM_JNI_MACRO_ASSEMBLER_ARM_H_ diff --git a/compiler/utils/arm/jni_macro_assembler_arm_vixl.cc b/compiler/utils/arm/jni_macro_assembler_arm_vixl.cc index d07c047253..bebe64c2b9 100644 --- a/compiler/utils/arm/jni_macro_assembler_arm_vixl.cc +++ b/compiler/utils/arm/jni_macro_assembler_arm_vixl.cc @@ -120,8 +120,8 @@ void ArmVIXLJNIMacroAssembler::RemoveFrame(size_t frame_size, CHECK_ALIGNED(frame_size, kStackAlignment); cfi().RememberState(); - // Compute callee saves to pop and PC. - RegList core_spill_mask = 1 << PC; + // Compute callee saves to pop and LR. + RegList core_spill_mask = 1 << LR; uint32_t fp_spill_mask = 0; for (const ManagedRegister& reg : callee_save_regs) { if (reg.AsArm().IsCoreRegister()) { @@ -136,6 +136,7 @@ void ArmVIXLJNIMacroAssembler::RemoveFrame(size_t frame_size, CHECK_GT(frame_size, pop_values * kFramePointerSize); DecreaseFrameSize(frame_size - (pop_values * kFramePointerSize)); // handles CFI as well. + // Pop FP callee saves. if (fp_spill_mask != 0) { uint32_t first = CTZ(fp_spill_mask); // Check that list is contiguous. @@ -146,9 +147,18 @@ void ArmVIXLJNIMacroAssembler::RemoveFrame(size_t frame_size, cfi().RestoreMany(DWARFReg(s0), fp_spill_mask); } - // Pop callee saves and PC. + // Pop core callee saves and LR. ___ Pop(RegisterList(core_spill_mask)); + if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { + // Refresh Mark Register. + // TODO: Refresh MR only if suspend is taken. + ___ Ldr(mr, MemOperand(tr, Thread::IsGcMarkingOffset<kArmPointerSize>().Int32Value())); + } + + // Return to LR. + ___ Bx(vixl32::lr); + // The CFI should be restored for any code that follows the exit block. cfi().RestoreState(); cfi().DefCFAOffset(frame_size); diff --git a/compiler/utils/arm64/jni_macro_assembler_arm64.cc b/compiler/utils/arm64/jni_macro_assembler_arm64.cc index 9cd6884cbe..bab84bea4c 100644 --- a/compiler/utils/arm64/jni_macro_assembler_arm64.cc +++ b/compiler/utils/arm64/jni_macro_assembler_arm64.cc @@ -772,10 +772,17 @@ void Arm64JNIMacroAssembler::RemoveFrame(size_t frame_size, asm_.UnspillRegisters(core_reg_list, frame_size - core_reg_size); asm_.UnspillRegisters(fp_reg_list, frame_size - core_reg_size - fp_reg_size); + if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { + // Refresh Mark Register. + // TODO: Refresh MR only if suspend is taken. + ___ Ldr(reg_w(MR), + MemOperand(reg_x(TR), Thread::IsGcMarkingOffset<kArm64PointerSize>().Int32Value())); + } + // Decrease frame size to start of callee saved regs. DecreaseFrameSize(frame_size); - // Pop callee saved and return to LR. + // Return to LR. ___ Ret(); // The CFI should be restored for any code that follows the exit block. diff --git a/compiler/utils/assembler_thumb_test.cc b/compiler/utils/assembler_thumb_test.cc index 4e9b619979..759ed38601 100644 --- a/compiler/utils/assembler_thumb_test.cc +++ b/compiler/utils/assembler_thumb_test.cc @@ -1643,6 +1643,10 @@ void EmitAndCheck(JniAssemblerType* assembler, const char* testname) { #define __ assembler. TEST_F(ArmVIXLAssemblerTest, VixlJniHelpers) { + // Run the test only with Baker read barriers, as the expected + // generated code contains a Marking Register refresh instruction. + TEST_DISABLED_WITHOUT_BAKER_READ_BARRIERS(); + const bool is_static = true; const bool is_synchronized = false; const bool is_critical_native = false; diff --git a/compiler/utils/assembler_thumb_test_expected.cc.inc b/compiler/utils/assembler_thumb_test_expected.cc.inc index eaaf81518a..563d1351e4 100644 --- a/compiler/utils/assembler_thumb_test_expected.cc.inc +++ b/compiler/utils/assembler_thumb_test_expected.cc.inc @@ -5595,7 +5595,7 @@ const char* const VixlJniHelpersResults[] = { " 1dc: f8cd c7ff str.w ip, [sp, #2047] ; 0x7ff\n", " 1e0: f8cd c7ff str.w ip, [sp, #2047] ; 0x7ff\n", " 1e4: f000 b802 b.w 1ec <VixlJniHelpers+0x1ec>\n", - " 1e8: f000 b818 b.w 21c <VixlJniHelpers+0x21c>\n", + " 1e8: f000 b81b b.w 222 <VixlJniHelpers+0x222>\n", " 1ec: f8cd c7ff str.w ip, [sp, #2047] ; 0x7ff\n", " 1f0: f8cd c7ff str.w ip, [sp, #2047] ; 0x7ff\n", " 1f4: f8cd c7ff str.w ip, [sp, #2047] ; 0x7ff\n", @@ -5608,10 +5608,12 @@ const char* const VixlJniHelpersResults[] = { " 210: b008 add sp, #32\n", " 212: b009 add sp, #36 ; 0x24\n", " 214: ecbd 8a10 vpop {s16-s31}\n", - " 218: e8bd 8de0 ldmia.w sp!, {r5, r6, r7, r8, sl, fp, pc}\n", - " 21c: 4660 mov r0, ip\n", - " 21e: f8d9 c2c0 ldr.w ip, [r9, #704] ; 0x2c0\n", - " 222: 47e0 blx ip\n", + " 218: e8bd 4de0 ldmia.w sp!, {r5, r6, r7, r8, sl, fp, lr}\n", + " 21c: f8d9 8034 ldr.w r8, [r9, #52] ; 0x34\n", + " 220: 4770 bx lr\n", + " 222: 4660 mov r0, ip\n", + " 224: f8d9 c2c0 ldr.w ip, [r9, #704] ; 0x2c0\n", + " 228: 47e0 blx ip\n", nullptr }; diff --git a/compiler/utils/mips/assembler_mips.cc b/compiler/utils/mips/assembler_mips.cc index 44b9bb4eb9..c581f1c58f 100644 --- a/compiler/utils/mips/assembler_mips.cc +++ b/compiler/utils/mips/assembler_mips.cc @@ -2904,6 +2904,17 @@ void MipsAssembler::IlvrD(VectorRegister wd, VectorRegister ws, VectorRegister w static_cast<FRegister>(wt)); } +void MipsAssembler::ReplicateFPToVectorRegister(VectorRegister dst, + FRegister src, + bool is_double) { + // Float or double in FPU register Fx can be considered as 0th element in vector register Wx. + if (is_double) { + SplatiD(dst, static_cast<VectorRegister>(src), 0); + } else { + SplatiW(dst, static_cast<VectorRegister>(src), 0); + } +} + void MipsAssembler::LoadConst32(Register rd, int32_t value) { if (IsUint<16>(value)) { // Use OR with (unsigned) immediate to encode 16b unsigned int. diff --git a/compiler/utils/mips/assembler_mips.h b/compiler/utils/mips/assembler_mips.h index a229882d18..33803bb576 100644 --- a/compiler/utils/mips/assembler_mips.h +++ b/compiler/utils/mips/assembler_mips.h @@ -612,6 +612,9 @@ class MipsAssembler FINAL : public Assembler, public JNIMacroAssembler<PointerSi void IlvrW(VectorRegister wd, VectorRegister ws, VectorRegister wt); void IlvrD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + // Helper for replicating floating point value in all destination elements. + void ReplicateFPToVectorRegister(VectorRegister dst, FRegister src, bool is_double); + // Higher level composite instructions. void LoadConst32(Register rd, int32_t value); void LoadConst64(Register reg_hi, Register reg_lo, int64_t value); diff --git a/dex2oat/dex2oat.cc b/dex2oat/dex2oat.cc index ca0bae1c8e..746a9407ca 100644 --- a/dex2oat/dex2oat.cc +++ b/dex2oat/dex2oat.cc @@ -49,6 +49,7 @@ #include "base/timing_logger.h" #include "base/unix_file/fd_file.h" #include "class_linker.h" +#include "class_loader_context.h" #include "compiler.h" #include "compiler_callbacks.h" #include "debug/elf_debug_writer.h" @@ -97,6 +98,9 @@ using android::base::StringPrintf; static constexpr size_t kDefaultMinDexFilesForSwap = 2; static constexpr size_t kDefaultMinDexFileCumulativeSizeForSwap = 20 * MB; +// Compiler filter override for very large apps. +static constexpr CompilerFilter::Filter kLargeAppFilter = CompilerFilter::kVerify; + static int original_argc; static char** original_argv; @@ -376,7 +380,7 @@ NO_RETURN static void Usage(const char* fmt, ...) { UsageError(" Default: %zu", kDefaultMinDexFilesForSwap); UsageError(""); UsageError(" --very-large-app-threshold=<size>: specifies the minimum total dex file size in"); - UsageError(" bytes to consider the input \"very large\" and punt on the compilation."); + UsageError(" bytes to consider the input \"very large\" and reduce compilation done."); UsageError(" Example: --very-large-app-threshold=100000000"); UsageError(""); UsageError(" --app-image-fd=<file-descriptor>: specify output file descriptor for app image."); @@ -400,6 +404,27 @@ NO_RETURN static void Usage(const char* fmt, ...) { UsageError(""); UsageError(" --classpath-dir=<directory-path>: directory used to resolve relative class paths."); UsageError(""); + UsageError(" --class-loader-context=<string spec>: a string specifying the intended"); + UsageError(" runtime loading context for the compiled dex files."); + UsageError(" "); + UsageError(" It describes how the class loader chain should be built in order to ensure"); + UsageError(" classes are resolved during dex2aot as they would be resolved at runtime."); + UsageError(" This spec will be encoded in the oat file. If at runtime the dex file is"); + UsageError(" loaded in a different context, the oat file will be rejected."); + UsageError(" "); + UsageError(" The chain is interpreted in the natural 'parent order', meaning that class"); + UsageError(" loader 'i+1' will be the parent of class loader 'i'."); + UsageError(" The compilation sources will be added to the classpath of the last class"); + UsageError(" loader. This allows the compiled dex files to be loaded at runtime in"); + UsageError(" a class loader that contains other dex files as well (e.g. shared libraries)."); + UsageError(" "); + UsageError(" Note that the compiler will be tolerant if the source dex files specified"); + UsageError(" with --dex-file are found in the classpath. The source dex files will be"); + UsageError(" removed from any class loader's classpath possibly resulting in empty"); + UsageError(" class loaders."); + UsageError(" "); + UsageError(" Example: --classloader-spec=PCL[lib1.dex:lib2.dex];DLC[lib3.dex]"); + UsageError(""); std::cerr << "See log for usage error information\n"; exit(EXIT_FAILURE); } @@ -1271,6 +1296,12 @@ class Dex2Oat FINAL { force_determinism_ = true; } else if (option.starts_with("--classpath-dir=")) { classpath_dir_ = option.substr(strlen("--classpath-dir=")).data(); + } else if (option.starts_with("--class-loader-context=")) { + class_loader_context_ = ClassLoaderContext::Create( + option.substr(strlen("--class-loader-context=")).data()); + if (class_loader_context_== nullptr) { + Usage("Option --class-loader-context has an incorrect format: %s", option.data()); + } } else if (!compiler_options_->ParseCompilerOption(option, Usage)) { Usage("Unknown argument %s", option.data()); } @@ -1542,25 +1573,45 @@ class Dex2Oat FINAL { } // Open dex files for class path. - std::vector<std::string> class_path_locations = - GetClassPathLocations(runtime_->GetClassPathString()); - OpenClassPathFiles(class_path_locations, - &class_path_files_, - &opened_oat_files_, - runtime_->GetInstructionSet(), - classpath_dir_); - - // Store the classpath we have right now. - std::vector<const DexFile*> class_path_files = MakeNonOwningPointerVector(class_path_files_); - std::string encoded_class_path; - if (class_path_locations.size() == 1 && - class_path_locations[0] == OatFile::kSpecialSharedLibrary) { - // When passing the special shared library as the classpath, it is the only path. - encoded_class_path = OatFile::kSpecialSharedLibrary; - } else { - encoded_class_path = OatFile::EncodeDexFileDependencies(class_path_files, classpath_dir_); + if (class_loader_context_ == nullptr) { + // TODO(calin): Temporary workaround while we transition to use + // --class-loader-context instead of --runtime-arg -cp + if (runtime_->GetClassPathString().empty()) { + class_loader_context_ = std::unique_ptr<ClassLoaderContext>( + new ClassLoaderContext()); + } else { + std::string spec = runtime_->GetClassPathString() == OatFile::kSpecialSharedLibrary + ? OatFile::kSpecialSharedLibrary + : "PCL[" + runtime_->GetClassPathString() + "]"; + class_loader_context_ = ClassLoaderContext::Create(spec); + } } - key_value_store_->Put(OatHeader::kClassPathKey, encoded_class_path); + CHECK(class_loader_context_ != nullptr); + DCHECK_EQ(oat_writers_.size(), 1u); + + // Note: Ideally we would reject context where the source dex files are also + // specified in the classpath (as it doesn't make sense). However this is currently + // needed for non-prebuild tests and benchmarks which expects on the fly compilation. + // Also, for secondary dex files we do not have control on the actual classpath. + // Instead of aborting, remove all the source location from the context classpaths. + if (class_loader_context_->RemoveLocationsFromClassPaths( + oat_writers_[0]->GetSourceLocations())) { + LOG(WARNING) << "The source files to be compiled are also in the classpath."; + } + + // We need to open the dex files before encoding the context in the oat file. + // (because the encoding adds the dex checksum...) + // TODO(calin): consider redesigning this so we don't have to open the dex files before + // creating the actual class loader. + if (!class_loader_context_->OpenDexFiles(runtime_->GetInstructionSet(), classpath_dir_)) { + // Do not abort if we couldn't open files from the classpath. They might be + // apks without dex files and right now are opening flow will fail them. + LOG(WARNING) << "Failed to open classpath dex files"; + } + + // Store the class loader context in the oat header. + key_value_store_->Put(OatHeader::kClassPathKey, + class_loader_context_->EncodeContextForOatFile(classpath_dir_)); } // Now that we have finalized key_value_store_, start writing the oat file. @@ -1623,14 +1674,13 @@ class Dex2Oat FINAL { // If we need to downgrade the compiler-filter for size reasons, do that check now. if (!IsBootImage() && IsVeryLarge(dex_files_)) { - if (!CompilerFilter::IsAsGoodAs(CompilerFilter::kExtract, - compiler_options_->GetCompilerFilter())) { - LOG(INFO) << "Very large app, downgrading to extract."; + if (!CompilerFilter::IsAsGoodAs(kLargeAppFilter, compiler_options_->GetCompilerFilter())) { + LOG(INFO) << "Very large app, downgrading to verify."; // Note: this change won't be reflected in the key-value store, as that had to be // finalized before loading the dex files. This setup is currently required // to get the size from the DexFile objects. // TODO: refactor. b/29790079 - compiler_options_->SetCompilerFilter(CompilerFilter::kExtract); + compiler_options_->SetCompilerFilter(kLargeAppFilter); } } @@ -1656,17 +1706,7 @@ class Dex2Oat FINAL { if (kSaveDexInput) { SaveDexInput(); } - - // Handle and ClassLoader creation needs to come after Runtime::Create. - ScopedObjectAccess soa(self); - - // Classpath: first the class-path given. - std::vector<const DexFile*> class_path_files = MakeNonOwningPointerVector(class_path_files_); - - // Then the dex files we'll compile. Thus we'll resolve the class-path first. - class_path_files.insert(class_path_files.end(), dex_files_.begin(), dex_files_.end()); - - class_loader_ = class_linker->CreatePathClassLoader(self, class_path_files); + class_loader_ = class_loader_context_->CreateClassLoader(dex_files_); } // Ensure opened dex files are writable for dex-to-dex transformations. @@ -1721,7 +1761,12 @@ class Dex2Oat FINAL { if (!no_inline_filters.empty()) { ClassLinker* class_linker = Runtime::Current()->GetClassLinker(); - std::vector<const DexFile*> class_path_files = MakeNonOwningPointerVector(class_path_files_); + std::vector<const DexFile*> class_path_files; + if (!IsBootImage()) { + // The class loader context is used only for apps. + class_path_files = class_loader_context_->FlattenOpenedDexFiles(); + } + std::vector<const std::vector<const DexFile*>*> dex_file_vectors = { &class_linker->GetBootClassPath(), &class_path_files, @@ -2224,8 +2269,8 @@ class Dex2Oat FINAL { DCHECK(!IsBootImage()); DCHECK_EQ(oat_writers_.size(), 1u); std::vector<std::string> dex_files_canonical_locations; - for (const char* location : oat_writers_[0]->GetSourceLocations()) { - dex_files_canonical_locations.push_back(DexFile::GetDexCanonicalLocation(location)); + for (const std::string& location : oat_writers_[0]->GetSourceLocations()) { + dex_files_canonical_locations.push_back(DexFile::GetDexCanonicalLocation(location.c_str())); } std::vector<std::string> parsed; @@ -2240,48 +2285,6 @@ class Dex2Oat FINAL { return parsed; } - // Opens requested class path files and appends them to opened_dex_files. If the dex files have - // been stripped, this opens them from their oat files and appends them to opened_oat_files. - static void OpenClassPathFiles(std::vector<std::string>& class_path_locations, - std::vector<std::unique_ptr<const DexFile>>* opened_dex_files, - std::vector<std::unique_ptr<OatFile>>* opened_oat_files, - InstructionSet isa, - std::string& classpath_dir) { - DCHECK(opened_dex_files != nullptr) << "OpenClassPathFiles dex out-param is nullptr"; - DCHECK(opened_oat_files != nullptr) << "OpenClassPathFiles oat out-param is nullptr"; - for (std::string& location : class_path_locations) { - // Stop early if we detect the special shared library, which may be passed as the classpath - // for dex2oat when we want to skip the shared libraries check. - if (location == OatFile::kSpecialSharedLibrary) { - break; - } - // If path is relative, append it to the provided base directory. - if (!classpath_dir.empty() && location[0] != '/') { - location = classpath_dir + '/' + location; - } - static constexpr bool kVerifyChecksum = true; - std::string error_msg; - if (!DexFile::Open( - location.c_str(), location.c_str(), kVerifyChecksum, &error_msg, opened_dex_files)) { - // If we fail to open the dex file because it's been stripped, try to open the dex file - // from its corresponding oat file. - OatFileAssistant oat_file_assistant(location.c_str(), isa, false); - std::unique_ptr<OatFile> oat_file(oat_file_assistant.GetBestOatFile()); - if (oat_file == nullptr) { - LOG(WARNING) << "Failed to open dex file and associated oat file for '" << location - << "': " << error_msg; - } else { - std::vector<std::unique_ptr<const DexFile>> oat_dex_files = - oat_file_assistant.LoadDexFiles(*oat_file, location.c_str()); - opened_oat_files->push_back(std::move(oat_file)); - opened_dex_files->insert(opened_dex_files->end(), - std::make_move_iterator(oat_dex_files.begin()), - std::make_move_iterator(oat_dex_files.end())); - } - } - } - } - bool PrepareImageClasses() { // If --image-classes was specified, calculate the full list of classes to include in the image. if (image_classes_filename_ != nullptr) { @@ -2737,8 +2740,8 @@ class Dex2Oat FINAL { std::unique_ptr<Runtime> runtime_; - // Ownership for the class path files. - std::vector<std::unique_ptr<const DexFile>> class_path_files_; + // The spec describing how the class loader should be setup for compilation. + std::unique_ptr<ClassLoaderContext> class_loader_context_; size_t thread_count_; uint64_t start_ns_; @@ -2792,9 +2795,9 @@ class Dex2Oat FINAL { std::unique_ptr<CompilerDriver> driver_; std::vector<std::unique_ptr<MemMap>> opened_dex_files_maps_; - std::vector<std::unique_ptr<OatFile>> opened_oat_files_; std::vector<std::unique_ptr<const DexFile>> opened_dex_files_; + // Note that this might contain pointers owned by class_loader_context_. std::vector<const DexFile*> no_inline_from_dex_files_; bool dump_stats_; diff --git a/dex2oat/dex2oat_test.cc b/dex2oat/dex2oat_test.cc index b604e8b5f1..d9f44d7a85 100644 --- a/dex2oat/dex2oat_test.cc +++ b/dex2oat/dex2oat_test.cc @@ -89,7 +89,8 @@ class Dex2oatTest : public Dex2oatEnvironmentTest { CompilerFilter::Filter filter, const std::vector<std::string>& extra_args = {}, bool expect_success = true, - bool use_fd = false) { + bool use_fd = false, + std::function<void(const OatFile&)> check_oat = [](const OatFile&) {}) { std::string error_msg; int status = GenerateOdexForTestWithStatus(dex_location, odex_location, @@ -113,6 +114,7 @@ class Dex2oatTest : public Dex2oatEnvironmentTest { ASSERT_TRUE(odex_file.get() != nullptr) << error_msg; CheckFilter(filter, odex_file->GetCompilerFilter()); + check_oat(*(odex_file.get())); } else { ASSERT_FALSE(success) << output_; @@ -542,11 +544,11 @@ class Dex2oatVeryLargeTest : public Dex2oatTest { void CheckHostResult(bool expect_large) { if (!kIsTargetBuild) { if (expect_large) { - EXPECT_NE(output_.find("Very large app, downgrading to extract."), + EXPECT_NE(output_.find("Very large app, downgrading to"), std::string::npos) << output_; } else { - EXPECT_EQ(output_.find("Very large app, downgrading to extract."), + EXPECT_EQ(output_.find("Very large app, downgrading to"), std::string::npos) << output_; } @@ -895,4 +897,130 @@ TEST_F(Dex2oatReturnCodeTest, TestCreateRuntime) { EXPECT_EQ(static_cast<int>(dex2oat::ReturnCode::kCreateRuntime), WEXITSTATUS(status)) << output_; } +class Dex2oatClassLoaderContextTest : public Dex2oatTest { + protected: + void RunTest(const char* class_loader_context, + const char* expected_classpath_key, + bool expected_success, + bool use_second_source = false) { + std::string dex_location = GetUsedDexLocation(); + std::string odex_location = GetUsedOatLocation(); + + Copy(use_second_source ? GetDexSrc2() : GetDexSrc1(), dex_location); + + std::string error_msg; + std::vector<std::string> extra_args; + if (class_loader_context != nullptr) { + extra_args.push_back(std::string("--class-loader-context=") + class_loader_context); + } + auto check_oat = [expected_classpath_key](const OatFile& oat_file) { + ASSERT_TRUE(expected_classpath_key != nullptr); + const char* classpath = oat_file.GetOatHeader().GetStoreValueByKey(OatHeader::kClassPathKey); + ASSERT_TRUE(classpath != nullptr); + ASSERT_STREQ(expected_classpath_key, classpath); + }; + + GenerateOdexForTest(dex_location, + odex_location, + CompilerFilter::kQuicken, + extra_args, + expected_success, + /*use_fd*/ false, + check_oat); + } + + std::string GetUsedDexLocation() { + return GetScratchDir() + "/Context.jar"; + } + + std::string GetUsedOatLocation() { + return GetOdexDir() + "/Context.odex"; + } + + const char* kEmptyClassPathKey = "PCL[]"; +}; + +TEST_F(Dex2oatClassLoaderContextTest, InvalidContext) { + RunTest("Invalid[]", /*expected_classpath_key*/ nullptr, /*expected_success*/ false); +} + +TEST_F(Dex2oatClassLoaderContextTest, EmptyContext) { + RunTest("PCL[]", kEmptyClassPathKey, /*expected_success*/ true); +} + +TEST_F(Dex2oatClassLoaderContextTest, SpecialContext) { + RunTest(OatFile::kSpecialSharedLibrary, + OatFile::kSpecialSharedLibrary, + /*expected_success*/ true); +} + +TEST_F(Dex2oatClassLoaderContextTest, ContextWithTheSourceDexFiles) { + std::string context = "PCL[" + GetUsedDexLocation() + "]"; + RunTest(context.c_str(), kEmptyClassPathKey, /*expected_success*/ true); +} + +TEST_F(Dex2oatClassLoaderContextTest, ContextWithOtherDexFiles) { + std::vector<std::unique_ptr<const DexFile>> dex_files = OpenTestDexFiles("Nested"); + + std::string context = "PCL[" + dex_files[0]->GetLocation() + "]"; + std::string expected_classpath_key = "PCL[" + + dex_files[0]->GetLocation() + "*" + std::to_string(dex_files[0]->GetLocationChecksum()) + "]"; + RunTest(context.c_str(), expected_classpath_key.c_str(), true); +} + +TEST_F(Dex2oatClassLoaderContextTest, ContextWithStrippedDexFiles) { + std::string stripped_classpath = GetScratchDir() + "/stripped_classpath.jar"; + Copy(GetStrippedDexSrc1(), stripped_classpath); + + std::string context = "PCL[" + stripped_classpath + "]"; + // Expect an empty context because stripped dex files cannot be open. + RunTest(context.c_str(), kEmptyClassPathKey , /*expected_success*/ true); +} + +TEST_F(Dex2oatClassLoaderContextTest, ContextWithStrippedDexFilesBackedByOdex) { + std::string stripped_classpath = GetScratchDir() + "/stripped_classpath.jar"; + std::string odex_for_classpath = GetOdexDir() + "/stripped_classpath.odex"; + + Copy(GetDexSrc1(), stripped_classpath); + + GenerateOdexForTest(stripped_classpath, + odex_for_classpath, + CompilerFilter::kQuicken, + {}, + true); + + // Strip the dex file + Copy(GetStrippedDexSrc1(), stripped_classpath); + + std::string context = "PCL[" + stripped_classpath + "]"; + std::string expected_classpath_key; + { + // Open the oat file to get the expected classpath. + OatFileAssistant oat_file_assistant(stripped_classpath.c_str(), kRuntimeISA, false); + std::unique_ptr<OatFile> oat_file(oat_file_assistant.GetBestOatFile()); + std::vector<std::unique_ptr<const DexFile>> oat_dex_files = + OatFileAssistant::LoadDexFiles(*oat_file, stripped_classpath.c_str()); + expected_classpath_key = "PCL["; + for (size_t i = 0; i < oat_dex_files.size(); i++) { + if (i > 0) { + expected_classpath_key + ":"; + } + expected_classpath_key += oat_dex_files[i]->GetLocation() + "*" + + std::to_string(oat_dex_files[i]->GetLocationChecksum()); + } + expected_classpath_key += "]"; + } + + RunTest(context.c_str(), + expected_classpath_key.c_str(), + /*expected_success*/ true, + /*use_second_source*/ true); +} + +TEST_F(Dex2oatClassLoaderContextTest, ContextWithNotExistentDexFiles) { + std::string context = "PCL[does_not_exists.dex]"; + // Expect an empty context because stripped dex files cannot be open. + RunTest(context.c_str(), kEmptyClassPathKey, /*expected_success*/ true); +} + } // namespace art diff --git a/disassembler/disassembler.cc b/disassembler/disassembler.cc index 8eecc62cd5..5af51c1355 100644 --- a/disassembler/disassembler.cc +++ b/disassembler/disassembler.cc @@ -40,8 +40,10 @@ Disassembler* Disassembler::Create(InstructionSet instruction_set, DisassemblerO return new arm::DisassemblerArm(options); } else if (instruction_set == kArm64) { return new arm64::DisassemblerArm64(options); - } else if (instruction_set == kMips || instruction_set == kMips64) { - return new mips::DisassemblerMips(options); + } else if (instruction_set == kMips) { + return new mips::DisassemblerMips(options, /* is_o32_abi */ true); + } else if (instruction_set == kMips64) { + return new mips::DisassemblerMips(options, /* is_o32_abi */ false); } else if (instruction_set == kX86) { return new x86::DisassemblerX86(options, false); } else if (instruction_set == kX86_64) { diff --git a/disassembler/disassembler_mips.cc b/disassembler/disassembler_mips.cc index 91203cb9f9..7cb216e766 100644 --- a/disassembler/disassembler_mips.cc +++ b/disassembler/disassembler_mips.cc @@ -40,6 +40,20 @@ struct MipsInstruction { } }; +static const char* gO32AbiRegNames[] = { + "zero", "at", "v0", "v1", "a0", "a1", "a2", "a3", + "t0", "t1", "t2", "t3", "t4", "t5", "t6", "t7", + "s0", "s1", "s2", "s3", "s4", "s5", "s6", "s7", + "t8", "t9", "k0", "k1", "gp", "sp", "s8", "ra" +}; + +static const char* gN64AbiRegNames[] = { + "zero", "at", "v0", "v1", "a0", "a1", "a2", "a3", + "a4", "a5", "a6", "a7", "t0", "t1", "t2", "t3", + "s0", "s1", "s2", "s3", "s4", "s5", "s6", "s7", + "t8", "t9", "k0", "k1", "gp", "sp", "s8", "ra" +}; + static const uint32_t kOpcodeShift = 26; static const uint32_t kCop1 = (17 << kOpcodeShift); @@ -470,6 +484,14 @@ static uint32_t ReadU32(const uint8_t* ptr) { return ptr[0] | (ptr[1] << 8) | (ptr[2] << 16) | (ptr[3] << 24); } +const char* DisassemblerMips::RegName(uint32_t reg) { + if (is_o32_abi_) { + return gO32AbiRegNames[reg]; + } else { + return gN64AbiRegNames[reg]; + } +} + size_t DisassemblerMips::Dump(std::ostream& os, const uint8_t* instr_ptr) { uint32_t instruction = ReadU32(instr_ptr); @@ -518,7 +540,7 @@ size_t DisassemblerMips::Dump(std::ostream& os, const uint8_t* instr_ptr) { case 'c': // Floating-point condition code flag in bc1f/bc1t and movf/movt. args << "cc" << (rt >> 2); break; - case 'D': args << 'r' << rd; break; + case 'D': args << RegName(rd); break; case 'd': args << 'f' << rd; break; case 'a': args << 'f' << sa; break; case 'F': args << (sa + 32); break; // dinsu position. @@ -553,13 +575,13 @@ size_t DisassemblerMips::Dump(std::ostream& os, const uint8_t* instr_ptr) { case 'l': // 9-bit signed offset { int32_t offset = static_cast<int16_t>(instruction) >> 7; - args << StringPrintf("%+d(r%d)", offset, rs); + args << StringPrintf("%+d(%s)", offset, RegName(rs)); } break; case 'O': // +x(rs) { int32_t offset = static_cast<int16_t>(instruction & 0xffff); - args << StringPrintf("%+d(r%d)", offset, rs); + args << StringPrintf("%+d(%s)", offset, RegName(rs)); if (rs == 17) { args << " ; "; GetDisassemblerOptions()->thread_offset_name_function_(args, offset); @@ -595,13 +617,13 @@ size_t DisassemblerMips::Dump(std::ostream& os, const uint8_t* instr_ptr) { case 'p': // 19-bit offset in addiupc. { int32_t offset = (instruction & 0x7ffff) - ((instruction & 0x40000) << 1); - args << offset << " ; move r" << rs << ", "; + args << offset << " ; move " << RegName(rs) << ", "; args << FormatInstructionPointer(instr_ptr + (offset << 2)); } break; - case 'S': args << 'r' << rs; break; + case 'S': args << RegName(rs); break; case 's': args << 'f' << rs; break; - case 'T': args << 'r' << rt; break; + case 'T': args << RegName(rt); break; case 't': args << 'f' << rt; break; case 'Z': args << (rd + 1); break; // sz ([d]ext size). case 'z': args << (rd - sa + 1); break; // sz ([d]ins, dinsu size). @@ -683,7 +705,7 @@ size_t DisassemblerMips::Dump(std::ostream& os, const uint8_t* instr_ptr) { case 2: opcode += ".w"; break; case 3: opcode += ".d"; break; } - args << StringPrintf("%+d(r%d)", s10 << df, rd); + args << StringPrintf("%+d(%s)", s10 << df, RegName(rd)); break; } case 'X': // MSA df/n - ws[x]. diff --git a/disassembler/disassembler_mips.h b/disassembler/disassembler_mips.h index 6342f22962..afa6af366f 100644 --- a/disassembler/disassembler_mips.h +++ b/disassembler/disassembler_mips.h @@ -26,11 +26,13 @@ namespace mips { class DisassemblerMips FINAL : public Disassembler { public: - explicit DisassemblerMips(DisassemblerOptions* options) + explicit DisassemblerMips(DisassemblerOptions* options, bool is_o32_abi) : Disassembler(options), last_ptr_(nullptr), - last_instr_(0) {} + last_instr_(0), + is_o32_abi_(is_o32_abi) {} + const char* RegName(uint32_t reg); size_t Dump(std::ostream& os, const uint8_t* begin) OVERRIDE; void Dump(std::ostream& os, const uint8_t* begin, const uint8_t* end) OVERRIDE; @@ -39,6 +41,7 @@ class DisassemblerMips FINAL : public Disassembler { // Needed to produce more readable disassembly of certain 2-instruction sequences. const uint8_t* last_ptr_; uint32_t last_instr_; + const bool is_o32_abi_; DISALLOW_COPY_AND_ASSIGN(DisassemblerMips); }; diff --git a/profman/profile_assistant_test.cc b/profman/profile_assistant_test.cc index 75f8ec9e27..c78d34ebb3 100644 --- a/profman/profile_assistant_test.cc +++ b/profman/profile_assistant_test.cc @@ -639,10 +639,13 @@ TEST_F(ProfileAssistantTest, TestBootImageProfile) { // Method that doesn't add the class since its only in one profile. Should still show up in the // boot profile. const std::string kOtherMethod = "Ljava/util/HashMap;-><init>()V"; + // Method that gets marked as hot since it's in multiple profiles. + const std::string kMultiMethod = "Ljava/util/ArrayList;->clear()V"; // Thresholds for this test. static const size_t kDirtyThreshold = 3; static const size_t kCleanThreshold = 2; + static const size_t kMethodThreshold = 2; // Create a bunch of boot profiles. std::string dex1 = @@ -659,6 +662,7 @@ TEST_F(ProfileAssistantTest, TestBootImageProfile) { kCleanClass + "\n" + kDirtyClass + "\n" + "P" + kHotMethod + "\n" + + "P" + kMultiMethod + "\n" + kUncommonDirtyClass; profiles.emplace_back(ScratchFile()); EXPECT_TRUE(CreateProfile(dex2, profiles.back().GetFilename(), core_dex)); @@ -667,6 +671,7 @@ TEST_F(ProfileAssistantTest, TestBootImageProfile) { std::string dex3 = "S" + kHotMethod + "\n" + "P" + kOtherMethod + "\n" + + "P" + kMultiMethod + "\n" + kDirtyClass + "\n"; profiles.emplace_back(ScratchFile()); EXPECT_TRUE(CreateProfile(dex3, profiles.back().GetFilename(), core_dex)); @@ -678,6 +683,7 @@ TEST_F(ProfileAssistantTest, TestBootImageProfile) { args.push_back("--generate-boot-image-profile"); args.push_back("--boot-image-class-threshold=" + std::to_string(kDirtyThreshold)); args.push_back("--boot-image-clean-class-threshold=" + std::to_string(kCleanThreshold)); + args.push_back("--boot-image-sampled-method-threshold=" + std::to_string(kMethodThreshold)); args.push_back("--reference-profile-file=" + out_profile.GetFilename()); args.push_back("--apk=" + core_dex); args.push_back("--dex-location=" + core_dex); @@ -708,11 +714,18 @@ TEST_F(ProfileAssistantTest, TestBootImageProfile) { // Aggregated methods hotness information. EXPECT_NE(output_file_contents.find("HSP" + kHotMethod), std::string::npos) << output_file_contents; - EXPECT_NE(output_file_contents.find(kOtherMethod), std::string::npos) + EXPECT_NE(output_file_contents.find("P" + kOtherMethod), std::string::npos) << output_file_contents; // Not inferred class, method is only in one profile. EXPECT_EQ(output_file_contents.find("Ljava/util/HashMap;\n"), std::string::npos) << output_file_contents; + // Test the sampled methods that became hot. + // Other method is in only one profile, it should not become hot. + EXPECT_EQ(output_file_contents.find("HP" + kOtherMethod), std::string::npos) + << output_file_contents; + // Multi method is in at least two profiles, it should become hot. + EXPECT_NE(output_file_contents.find("HP" + kMultiMethod), std::string::npos) + << output_file_contents; } TEST_F(ProfileAssistantTest, TestProfileCreationOneNotMatched) { diff --git a/profman/profman.cc b/profman/profman.cc index 94e81c76c1..6c8ca56408 100644 --- a/profman/profman.cc +++ b/profman/profman.cc @@ -142,6 +142,9 @@ NO_RETURN static void Usage(const char *fmt, ...) { UsageError(" occurrences to include a class in the boot image profile. A clean class is a"); UsageError(" class that doesn't have any static fields or native methods and is likely to"); UsageError(" remain clean in the image. Default is 3."); + UsageError(" --boot-image-sampled-method-threshold=<value>: minimum number of profiles a"); + UsageError(" non-hot method needs to be in order to be hot in the output profile. The"); + UsageError(" default is max int."); UsageError(""); exit(EXIT_FAILURE); @@ -225,6 +228,11 @@ class ProfMan FINAL { "--boot-image-clean-class-threshold", &boot_image_options_.image_class_clean_theshold, Usage); + } else if (option.starts_with("--boot-image-sampled-method-threshold=")) { + ParseUintOption(option, + "--boot-image-sampled-method-threshold", + &boot_image_options_.compiled_method_threshold, + Usage); } else if (option.starts_with("--profile-file=")) { profile_files_.push_back(option.substr(strlen("--profile-file=")).ToString()); } else if (option.starts_with("--profile-file-fd=")) { diff --git a/runtime/Android.bp b/runtime/Android.bp index 0dfc60d88a..8d15c349dd 100644 --- a/runtime/Android.bp +++ b/runtime/Android.bp @@ -48,6 +48,7 @@ cc_defaults { "cha.cc", "check_jni.cc", "class_linker.cc", + "class_loader_context.cc", "class_table.cc", "code_simulator_container.cc", "common_throws.cc", @@ -542,6 +543,7 @@ art_cc_test { "base/unix_file/fd_file_test.cc", "cha_test.cc", "class_linker_test.cc", + "class_loader_context_test.cc", "class_table_test.cc", "compiler_filter_test.cc", "dex_file_test.cc", diff --git a/runtime/arch/arm/asm_support_arm.S b/runtime/arch/arm/asm_support_arm.S index 9eca86232d..eeac743df2 100644 --- a/runtime/arch/arm/asm_support_arm.S +++ b/runtime/arch/arm/asm_support_arm.S @@ -26,6 +26,13 @@ // Register holding Thread::Current(). #define rSELF r9 +#if defined(USE_READ_BARRIER) && defined(USE_BAKER_READ_BARRIER) +// Marking Register, holding Thread::Current()->GetIsGcMarking(). +// Only used with the Concurrent Copying (CC) garbage +// collector, with the Baker read barrier configuration. +#define rMR r8 +#endif + .syntax unified .arch armv7-a .thumb @@ -121,14 +128,14 @@ END \name .endm -// Macros to poison (negate) the reference for heap poisoning. +// Macro to poison (negate) the reference for heap poisoning. .macro POISON_HEAP_REF rRef #ifdef USE_HEAP_POISONING rsb \rRef, \rRef, #0 #endif // USE_HEAP_POISONING .endm -// Macros to unpoison (negate) the reference for heap poisoning. +// Macro to unpoison (negate) the reference for heap poisoning. .macro UNPOISON_HEAP_REF rRef #ifdef USE_HEAP_POISONING rsb \rRef, \rRef, #0 diff --git a/runtime/arch/arm/context_arm.cc b/runtime/arch/arm/context_arm.cc index 0db14fb8a5..711452cffb 100644 --- a/runtime/arch/arm/context_arm.cc +++ b/runtime/arch/arm/context_arm.cc @@ -108,7 +108,9 @@ void ArmContext::DoLongJump() { for (size_t i = 0; i < kNumberOfSRegisters; ++i) { fprs[i] = fprs_[i] != nullptr ? *fprs_[i] : ArmContext::kBadFprBase + i; } + // Ensure the Thread Register contains the address of the current thread. DCHECK_EQ(reinterpret_cast<uintptr_t>(Thread::Current()), gprs[TR]); + // The Marking Register will be updated by art_quick_do_long_jump. art_quick_do_long_jump(gprs, fprs); } diff --git a/runtime/arch/arm/quick_entrypoints_arm.S b/runtime/arch/arm/quick_entrypoints_arm.S index 676efc4a77..0de59053af 100644 --- a/runtime/arch/arm/quick_entrypoints_arm.S +++ b/runtime/arch/arm/quick_entrypoints_arm.S @@ -67,6 +67,9 @@ * Runtime::CreateCalleeSaveMethod(kSaveRefsOnly). */ .macro SETUP_SAVE_REFS_ONLY_FRAME rTemp + // Note: We could avoid saving R8 in the case of Baker read + // barriers, as it is overwritten by REFRESH_MARKING_REGISTER + // later; but it's not worth handling this special case. push {r5-r8, r10-r11, lr} @ 7 words of callee saves .cfi_adjust_cfa_offset 28 .cfi_rel_offset r5, 0 @@ -93,6 +96,9 @@ .macro RESTORE_SAVE_REFS_ONLY_FRAME add sp, #4 @ bottom word holds Method* .cfi_adjust_cfa_offset -4 + // Note: Likewise, we could avoid restoring R8 in the case of Baker + // read barriers, as it is overwritten by REFRESH_MARKING_REGISTER + // later; but it's not worth handling this special case. pop {r5-r8, r10-r11, lr} @ 7 words of callee saves .cfi_restore r5 .cfi_restore r6 @@ -104,16 +110,14 @@ .cfi_adjust_cfa_offset -28 .endm -.macro RESTORE_SAVE_REFS_ONLY_FRAME_AND_RETURN - RESTORE_SAVE_REFS_ONLY_FRAME - bx lr @ return -.endm - /* * Macro that sets up the callee save frame to conform with * Runtime::CreateCalleeSaveMethod(kSaveRefsAndArgs). */ .macro SETUP_SAVE_REFS_AND_ARGS_FRAME_REGISTERS_ONLY + // Note: We could avoid saving R8 in the case of Baker read + // barriers, as it is overwritten by REFRESH_MARKING_REGISTER + // later; but it's not worth handling this special case. push {r1-r3, r5-r8, r10-r11, lr} @ 10 words of callee saves and args. .cfi_adjust_cfa_offset 40 .cfi_rel_offset r1, 0 @@ -156,6 +160,9 @@ .cfi_adjust_cfa_offset -8 vpop {s0-s15} .cfi_adjust_cfa_offset -64 + // Note: Likewise, we could avoid restoring X20 in the case of Baker + // read barriers, as it is overwritten by REFRESH_MARKING_REGISTER + // later; but it's not worth handling this special case. pop {r1-r3, r5-r8, r10-r11, lr} @ 10 words of callee saves .cfi_restore r1 .cfi_restore r2 @@ -227,6 +234,7 @@ .cfi_restore r1 .cfi_restore r2 .cfi_restore r3 + .cfi_restore r4 .cfi_restore r5 .cfi_restore r6 .cfi_restore r7 @@ -251,6 +259,7 @@ .cfi_restore r1 .cfi_restore r2 .cfi_restore r3 + .cfi_restore r4 .cfi_restore r5 .cfi_restore r6 .cfi_restore r7 @@ -263,6 +272,17 @@ .cfi_adjust_cfa_offset -52 .endm +// Macro to refresh the Marking Register (R8). +// +// This macro must be called at the end of functions implementing +// entrypoints that possibly (directly or indirectly) perform a +// suspend check (before they return). +.macro REFRESH_MARKING_REGISTER +#if defined(USE_READ_BARRIER) && defined(USE_BAKER_READ_BARRIER) + ldr rMR, [rSELF, #THREAD_IS_GC_MARKING_OFFSET] +#endif +.endm + .macro RETURN_IF_RESULT_IS_ZERO cbnz r0, 1f @ result non-zero branch over bx lr @ return @@ -359,6 +379,7 @@ ENTRY \name mov r1, r9 @ pass Thread::Current bl \entrypoint @ (uint32_t field_idx, Thread*) RESTORE_SAVE_REFS_ONLY_FRAME + REFRESH_MARKING_REGISTER \return END \name .endm @@ -370,6 +391,7 @@ ENTRY \name mov r2, r9 @ pass Thread::Current bl \entrypoint @ (field_idx, Object*, Thread*) RESTORE_SAVE_REFS_ONLY_FRAME + REFRESH_MARKING_REGISTER \return END \name .endm @@ -381,6 +403,7 @@ ENTRY \name mov r3, r9 @ pass Thread::Current bl \entrypoint @ (field_idx, Object*, new_val, Thread*) RESTORE_SAVE_REFS_ONLY_FRAME @ TODO: we can clearly save an add here + REFRESH_MARKING_REGISTER \return END \name .endm @@ -464,6 +487,8 @@ NO_ARG_RUNTIME_EXCEPTION art_quick_throw_stack_overflow, artThrowStackOverflowFr * * On success this wrapper will restore arguments and *jump* to the target, leaving the lr * pointing back to the original caller. + * + * Clobbers IP (R12). */ .macro INVOKE_TRAMPOLINE_BODY cxx_name .extern \cxx_name @@ -473,6 +498,7 @@ NO_ARG_RUNTIME_EXCEPTION art_quick_throw_stack_overflow, artThrowStackOverflowFr bl \cxx_name @ (method_idx, this, Thread*, SP) mov r12, r1 @ save Method*->code_ RESTORE_SAVE_REFS_AND_ARGS_FRAME + REFRESH_MARKING_REGISTER cbz r0, 1f @ did we find the target? if not go to exception delivery bx r12 @ tail call to target 1: @@ -549,6 +575,8 @@ ENTRY art_quick_invoke_stub_internal mov r4, #SUSPEND_CHECK_INTERVAL @ reset r4 to suspend check interval #endif + REFRESH_MARKING_REGISTER + ldr ip, [r0, #ART_METHOD_QUICK_CODE_OFFSET_32] @ get pointer to the code blx ip @ call the method @@ -580,7 +608,8 @@ ENTRY art_quick_osr_stub mov r11, sp @ Save the stack pointer mov r10, r1 @ Save size of stack ldr r9, [r11, #40] @ Move managed thread pointer into r9 - mov r8, r2 @ Save the pc to call + REFRESH_MARKING_REGISTER + mov r6, r2 @ Save the pc to call sub r7, sp, #12 @ Reserve space for stack pointer, @ JValue* result, and ArtMethod* slot. and r7, #0xFFFFFFF0 @ Align stack pointer @@ -612,7 +641,7 @@ ENTRY art_quick_osr_stub .Losr_entry: sub r10, r10, #4 str lr, [sp, r10] @ Store link register per the compiler ABI - bx r8 + bx r6 END art_quick_osr_stub /* @@ -624,6 +653,7 @@ ARM_ENTRY art_quick_do_long_jump ldr r14, [r0, #56] @ (LR from gprs_ 56=4*14) add r0, r0, #12 @ increment r0 to skip gprs_[0..2] 12=4*3 ldm r0, {r3-r13} @ load remaining gprs from argument gprs_ + REFRESH_MARKING_REGISTER ldr r0, [r0, #-12] @ load r0 value mov r1, #0 @ clear result register r1 bx r2 @ do long jump @@ -677,6 +707,7 @@ ENTRY art_quick_lock_object mov r1, r9 @ pass Thread::Current bl artLockObjectFromCode @ (Object* obj, Thread*) RESTORE_SAVE_REFS_ONLY_FRAME + REFRESH_MARKING_REGISTER RETURN_IF_RESULT_IS_ZERO DELIVER_PENDING_EXCEPTION END art_quick_lock_object @@ -686,6 +717,7 @@ ENTRY art_quick_lock_object_no_inline mov r1, r9 @ pass Thread::Current bl artLockObjectFromCode @ (Object* obj, Thread*) RESTORE_SAVE_REFS_ONLY_FRAME + REFRESH_MARKING_REGISTER RETURN_IF_RESULT_IS_ZERO DELIVER_PENDING_EXCEPTION END art_quick_lock_object_no_inline @@ -743,6 +775,7 @@ ENTRY art_quick_unlock_object mov r1, r9 @ pass Thread::Current bl artUnlockObjectFromCode @ (Object* obj, Thread*) RESTORE_SAVE_REFS_ONLY_FRAME + REFRESH_MARKING_REGISTER RETURN_IF_RESULT_IS_ZERO DELIVER_PENDING_EXCEPTION END art_quick_unlock_object @@ -753,6 +786,7 @@ ENTRY art_quick_unlock_object_no_inline mov r1, r9 @ pass Thread::Current bl artUnlockObjectFromCode @ (Object* obj, Thread*) RESTORE_SAVE_REFS_ONLY_FRAME + REFRESH_MARKING_REGISTER RETURN_IF_RESULT_IS_ZERO DELIVER_PENDING_EXCEPTION END art_quick_unlock_object_no_inline @@ -921,6 +955,7 @@ ENTRY \name mov r1, r9 @ pass Thread::Current bl \entrypoint @ (uint32_t type_idx, Method* method, Thread*) RESTORE_SAVE_REFS_ONLY_FRAME + REFRESH_MARKING_REGISTER \return END \name .endm @@ -933,6 +968,7 @@ ENTRY \name mov r2, r9 @ pass Thread::Current bl \entrypoint @ (uint32_t type_idx, Method* method, Thread*) RESTORE_SAVE_REFS_ONLY_FRAME + REFRESH_MARKING_REGISTER \return END \name .endm @@ -946,6 +982,7 @@ ENTRY \name @ (uint32_t type_idx, Method* method, int32_t component_count, Thread*) bl \entrypoint RESTORE_SAVE_REFS_ONLY_FRAME + REFRESH_MARKING_REGISTER \return END \name .endm @@ -961,6 +998,7 @@ ENTRY \name add sp, #16 @ strip the extra frame .cfi_adjust_cfa_offset -16 RESTORE_SAVE_REFS_ONLY_FRAME + REFRESH_MARKING_REGISTER \return END \name .endm @@ -975,6 +1013,7 @@ ENTRY \name cbz r0, 1f @ If result is null, deliver the OOME. .cfi_remember_state RESTORE_SAVE_EVERYTHING_FRAME_KEEP_R0 + REFRESH_MARKING_REGISTER bx lr .cfi_restore_state 1: @@ -987,6 +1026,9 @@ ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_initialize_type, artInitializeTypeFro ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_initialize_type_and_verify_access, artInitializeTypeAndVerifyAccessFromCode ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_resolve_string, artResolveStringFromCode +// Note: Functions `art{Get,Set}<Kind>{Static,Instance>FromCompiledCode` are +// defined by macros in runtime/entrypoints/quick/quick_field_entrypoints.cc. + /* * Called by managed code to resolve a static field and load a non-wide value. */ @@ -1006,6 +1048,7 @@ ENTRY art_quick_get64_static bl artGet64StaticFromCompiledCode @ (uint32_t field_idx, Thread*) ldr r2, [r9, #THREAD_EXCEPTION_OFFSET] @ load Thread::Current()->exception_ RESTORE_SAVE_REFS_ONLY_FRAME + REFRESH_MARKING_REGISTER cbnz r2, 1f @ success if no exception pending bx lr @ return on success 1: @@ -1031,6 +1074,7 @@ ENTRY art_quick_get64_instance bl artGet64InstanceFromCompiledCode @ (field_idx, Object*, Thread*) ldr r2, [r9, #THREAD_EXCEPTION_OFFSET] @ load Thread::Current()->exception_ RESTORE_SAVE_REFS_ONLY_FRAME + REFRESH_MARKING_REGISTER cbnz r2, 1f @ success if no exception pending bx lr @ return on success 1: @@ -1066,6 +1110,7 @@ ENTRY art_quick_set64_instance add sp, #16 @ release out args .cfi_adjust_cfa_offset -16 RESTORE_SAVE_REFS_ONLY_FRAME @ TODO: we can clearly save an add here + REFRESH_MARKING_REGISTER RETURN_IF_RESULT_IS_ZERO DELIVER_PENDING_EXCEPTION END art_quick_set64_instance @@ -1080,6 +1125,7 @@ ENTRY art_quick_set64_static add sp, #16 @ release out args .cfi_adjust_cfa_offset -16 RESTORE_SAVE_REFS_ONLY_FRAME @ TODO: we can clearly save an add here + REFRESH_MARKING_REGISTER RETURN_IF_RESULT_IS_ZERO DELIVER_PENDING_EXCEPTION END art_quick_set64_static @@ -1223,6 +1269,7 @@ ENTRY \c_name mov r1, r9 @ pass Thread::Current bl \cxx_name @ (mirror::Class* cls, Thread*) RESTORE_SAVE_REFS_ONLY_FRAME + REFRESH_MARKING_REGISTER RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER END \c_name .endm @@ -1315,6 +1362,7 @@ ENTRY \name mov r1, r9 // Pass Thread::Current. bl \entrypoint // (mirror::Class* klass, Thread*) RESTORE_SAVE_REFS_ONLY_FRAME + REFRESH_MARKING_REGISTER RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER END \name .endm @@ -1331,7 +1379,7 @@ GENERATE_ALLOC_OBJECT_RESOLVED_TLAB art_quick_alloc_object_initialized_tlab, art // r0: type r1: component_count r2: total_size r9: Thread::Current, r3, r12: free. // Need to preserve r0 and r1 to the slow path. .macro ALLOC_ARRAY_TLAB_FAST_PATH_RESOLVED_WITH_SIZE slowPathLabel - and r2, r2, #OBJECT_ALIGNMENT_MASK_TOGGLED // Apply alignemnt mask + and r2, r2, #OBJECT_ALIGNMENT_MASK_TOGGLED // Apply alignment mask // (addr + 7) & ~7. // Load thread_local_pos (r3) and @@ -1386,6 +1434,7 @@ ENTRY \name mov r2, r9 // pass Thread::Current bl \entrypoint RESTORE_SAVE_REFS_ONLY_FRAME + REFRESH_MARKING_REGISTER RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER END \name .endm @@ -1462,8 +1511,8 @@ END \name add r2, r2, #(MIRROR_WIDE_ARRAY_DATA_OFFSET + OBJECT_ALIGNMENT_MASK) .endm -# TODO(ngeoffray): art_quick_alloc_array_resolved_region_tlab is not used for arm, remove -# the entrypoint once all backends have been updated to use the size variants. +// TODO(ngeoffray): art_quick_alloc_array_resolved_region_tlab is not used for arm, remove +// the entrypoint once all backends have been updated to use the size variants. GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved_region_tlab, artAllocArrayFromCodeResolvedRegionTLAB, COMPUTE_ARRAY_SIZE_UNKNOWN GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved8_region_tlab, artAllocArrayFromCodeResolvedRegionTLAB, COMPUTE_ARRAY_SIZE_8 GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved16_region_tlab, artAllocArrayFromCodeResolvedRegionTLAB, COMPUTE_ARRAY_SIZE_16 @@ -1492,6 +1541,7 @@ ENTRY art_quick_test_suspend mov r0, rSELF bl artTestSuspendFromCode @ (Thread*) RESTORE_SAVE_EVERYTHING_FRAME + REFRESH_MARKING_REGISTER bx lr END art_quick_test_suspend @@ -1499,7 +1549,9 @@ ENTRY art_quick_implicit_suspend mov r0, rSELF SETUP_SAVE_REFS_ONLY_FRAME r1 @ save callee saves for stack crawl bl artTestSuspendFromCode @ (Thread*) - RESTORE_SAVE_REFS_ONLY_FRAME_AND_RETURN + RESTORE_SAVE_REFS_ONLY_FRAME + REFRESH_MARKING_REGISTER + bx lr END art_quick_implicit_suspend /* @@ -1518,6 +1570,7 @@ ENTRY art_quick_proxy_invoke_handler add sp, #(FRAME_SIZE_SAVE_REFS_AND_ARGS - FRAME_SIZE_SAVE_REFS_ONLY) .cfi_adjust_cfa_offset -(FRAME_SIZE_SAVE_REFS_AND_ARGS - FRAME_SIZE_SAVE_REFS_ONLY) RESTORE_SAVE_REFS_ONLY_FRAME + REFRESH_MARKING_REGISTER cbnz r2, 1f @ success if no exception is pending vmov d0, r0, r1 @ store into fpr, for when it's a fpr return... bx lr @ return on success @@ -1567,8 +1620,9 @@ ENTRY art_quick_resolution_trampoline blx artQuickResolutionTrampoline @ (Method* called, receiver, Thread*, SP) cbz r0, 1f @ is code pointer null? goto exception mov r12, r0 - ldr r0, [sp, #0] @ load resolved method in r0 + ldr r0, [sp, #0] @ load resolved method in r0 RESTORE_SAVE_REFS_AND_ARGS_FRAME + REFRESH_MARKING_REGISTER bx r12 @ tail-call into actual code 1: RESTORE_SAVE_REFS_AND_ARGS_FRAME @@ -1649,6 +1703,7 @@ ENTRY art_quick_generic_jni_trampoline add sp, #FRAME_SIZE_SAVE_REFS_AND_ARGS-FRAME_SIZE_SAVE_REFS_ONLY .cfi_adjust_cfa_offset -(FRAME_SIZE_SAVE_REFS_AND_ARGS-FRAME_SIZE_SAVE_REFS_ONLY) RESTORE_SAVE_REFS_ONLY_FRAME + REFRESH_MARKING_REGISTER // store into fpr, for when it's a fpr return... vmov d0, r0, r1 @@ -1675,6 +1730,7 @@ ENTRY art_quick_to_interpreter_bridge add sp, #(FRAME_SIZE_SAVE_REFS_AND_ARGS - FRAME_SIZE_SAVE_REFS_ONLY) .cfi_adjust_cfa_offset -(FRAME_SIZE_SAVE_REFS_AND_ARGS - FRAME_SIZE_SAVE_REFS_ONLY) RESTORE_SAVE_REFS_ONLY_FRAME + REFRESH_MARKING_REGISTER cbnz r2, 1f @ success if no exception is pending vmov d0, r0, r1 @ store into fpr, for when it's a fpr return... bx lr @ return on success @@ -1705,6 +1761,7 @@ ENTRY art_quick_instrumentation_entry mov r12, r0 @ r12 holds reference to code ldr r0, [sp, #4] @ restore r0 RESTORE_SAVE_REFS_AND_ARGS_FRAME + REFRESH_MARKING_REGISTER blx r12 @ call method with lr set to art_quick_instrumentation_exit @ Deliberate fall-through into art_quick_instrumentation_exit. .type art_quick_instrumentation_exit, #function @@ -1734,6 +1791,7 @@ art_quick_instrumentation_exit: .cfi_restore r0 .cfi_restore r1 RESTORE_SAVE_REFS_ONLY_FRAME + REFRESH_MARKING_REGISTER cbz r2, .Ldo_deliver_instrumentation_exception @ Deliver exception if we got nullptr as function. bx r2 @ Otherwise, return @@ -1787,7 +1845,7 @@ END art_quick_deoptimize_from_compiled_code */ /* mul-long vAA, vBB, vCC */ ENTRY art_quick_mul_long - push {r9 - r10} + push {r9-r10} .cfi_adjust_cfa_offset 8 .cfi_rel_offset r9, 0 .cfi_rel_offset r10, 4 @@ -1797,7 +1855,7 @@ ENTRY art_quick_mul_long add r10, r2, r10 @ r10<- r10 + low(ZxW + (YxX)) mov r0,r9 mov r1,r10 - pop {r9 - r10} + pop {r9-r10} .cfi_adjust_cfa_offset -8 .cfi_restore r9 .cfi_restore r10 @@ -2544,6 +2602,7 @@ ENTRY art_quick_invoke_polymorphic add sp, #8 .cfi_adjust_cfa_offset -8 RESTORE_SAVE_REFS_AND_ARGS_FRAME + REFRESH_MARKING_REGISTER RETURN_OR_DELIVER_PENDING_EXCEPTION_REG r2 .macro HANDLER_TABLE_OFFSET handler_label diff --git a/runtime/arch/arm/registers_arm.h b/runtime/arch/arm/registers_arm.h index 932095d0c9..d39a2a274f 100644 --- a/runtime/arch/arm/registers_arm.h +++ b/runtime/arch/arm/registers_arm.h @@ -40,7 +40,8 @@ enum Register { R13 = 13, R14 = 14, R15 = 15, - TR = 9, // thread register + MR = 8, // ART Marking Register + TR = 9, // ART Thread Register FP = 11, IP = 12, SP = 13, diff --git a/runtime/arch/arm64/asm_support_arm64.S b/runtime/arch/arm64/asm_support_arm64.S index bcf55e339e..715fc35ff4 100644 --- a/runtime/arch/arm64/asm_support_arm64.S +++ b/runtime/arch/arm64/asm_support_arm64.S @@ -33,6 +33,12 @@ #define xIP1 x17 #define wIP1 w17 +#if defined(USE_READ_BARRIER) && defined(USE_BAKER_READ_BARRIER) +// Marking Register, holding Thread::Current()->GetIsGcMarking(). +// Only used with the Concurrent Copying (CC) garbage +// collector, with the Baker read barrier configuration. +#define wMR w20 +#endif .macro ENTRY name .type \name, #function @@ -55,14 +61,14 @@ END \name .endm -// Macros to poison (negate) the reference for heap poisoning. +// Macro to poison (negate) the reference for heap poisoning. .macro POISON_HEAP_REF rRef #ifdef USE_HEAP_POISONING neg \rRef, \rRef #endif // USE_HEAP_POISONING .endm -// Macros to unpoison (negate) the reference for heap poisoning. +// Macro to unpoison (negate) the reference for heap poisoning. .macro UNPOISON_HEAP_REF rRef #ifdef USE_HEAP_POISONING neg \rRef, \rRef diff --git a/runtime/arch/arm64/context_arm64.cc b/runtime/arch/arm64/context_arm64.cc index 0465c1e79d..0f0814a675 100644 --- a/runtime/arch/arm64/context_arm64.cc +++ b/runtime/arch/arm64/context_arm64.cc @@ -137,7 +137,9 @@ void Arm64Context::DoLongJump() { for (size_t i = 0; i < kNumberOfDRegisters; ++i) { fprs[i] = fprs_[i] != nullptr ? *fprs_[i] : Arm64Context::kBadFprBase + i; } + // Ensure the Thread Register contains the address of the current thread. DCHECK_EQ(reinterpret_cast<uintptr_t>(Thread::Current()), gprs[TR]); + // The Marking Register will be updated by art_quick_do_long_jump. art_quick_do_long_jump(gprs, fprs); } diff --git a/runtime/arch/arm64/quick_entrypoints_arm64.S b/runtime/arch/arm64/quick_entrypoints_arm64.S index 138dbf9495..e097a336d4 100644 --- a/runtime/arch/arm64/quick_entrypoints_arm64.S +++ b/runtime/arch/arm64/quick_entrypoints_arm64.S @@ -39,6 +39,18 @@ .cfi_restore \reg .endm +.macro SAVE_REG_INCREASE_FRAME reg, frame_adjustment + str \reg, [sp, #-(\frame_adjustment)]! + .cfi_adjust_cfa_offset (\frame_adjustment) + .cfi_rel_offset \reg, 0 +.endm + +.macro RESTORE_REG_DECREASE_FRAME reg, frame_adjustment + ldr \reg, [sp], #(\frame_adjustment) + .cfi_restore \reg + .cfi_adjust_cfa_offset -(\frame_adjustment) +.endm + .macro SAVE_TWO_REGS reg1, reg2, offset stp \reg1, \reg2, [sp, #(\offset)] .cfi_rel_offset \reg1, (\offset) @@ -140,6 +152,9 @@ SAVE_TWO_REGS x29, xLR, 80 // Store ArtMethod* Runtime::callee_save_methods_[kSaveRefsOnly]. + // Note: We could avoid saving X20 in the case of Baker read + // barriers, as it is overwritten by REFRESH_MARKING_REGISTER + // later; but it's not worth handling this special case. stp xIP0, x20, [sp] .cfi_rel_offset x20, 8 @@ -151,6 +166,9 @@ // TODO: Probably no need to restore registers preserved by aapcs64. .macro RESTORE_SAVE_REFS_ONLY_FRAME // Callee-saves. + // Note: Likewise, we could avoid restoring X20 in the case of Baker + // read barriers, as it is overwritten by REFRESH_MARKING_REGISTER + // later; but it's not worth handling this special case. RESTORE_REG x20, 8 RESTORE_TWO_REGS x21, x22, 16 RESTORE_TWO_REGS x23, x24, 32 @@ -165,11 +183,6 @@ DECREASE_FRAME 96 .endm -.macro RESTORE_SAVE_REFS_ONLY_FRAME_AND_RETURN - RESTORE_SAVE_REFS_ONLY_FRAME - ret -.endm - .macro SETUP_SAVE_REFS_AND_ARGS_FRAME_INTERNAL INCREASE_FRAME 224 @@ -192,6 +205,9 @@ SAVE_TWO_REGS x5, x6, 112 // x7, Callee-saves. + // Note: We could avoid saving X20 in the case of Baker read + // barriers, as it is overwritten by REFRESH_MARKING_REGISTER + // later; but it's not worth handling this special case. SAVE_TWO_REGS x7, x20, 128 SAVE_TWO_REGS x21, x22, 144 SAVE_TWO_REGS x23, x24, 160 @@ -250,6 +266,9 @@ RESTORE_TWO_REGS x5, x6, 112 // x7, Callee-saves. + // Note: Likewise, we could avoid restoring X20 in the case of Baker + // read barriers, as it is overwritten by REFRESH_MARKING_REGISTER + // later; but it's not worth handling this special case. RESTORE_TWO_REGS x7, x20, 128 RESTORE_TWO_REGS x21, x22, 144 RESTORE_TWO_REGS x23, x24, 160 @@ -358,7 +377,7 @@ ldp d29, d30, [sp, #240] ldr d31, [sp, #256] - // Restore core registers. + // Restore core registers, except x0. RESTORE_TWO_REGS x1, x2, 272 RESTORE_TWO_REGS x3, x4, 288 RESTORE_TWO_REGS x5, x6, 304 @@ -379,10 +398,21 @@ .endm .macro RESTORE_SAVE_EVERYTHING_FRAME - RESTORE_REG x0, 264 + RESTORE_REG x0, 264 RESTORE_SAVE_EVERYTHING_FRAME_KEEP_X0 .endm +// Macro to refresh the Marking Register (W20). +// +// This macro must be called at the end of functions implementing +// entrypoints that possibly (directly or indirectly) perform a +// suspend check (before they return). +.macro REFRESH_MARKING_REGISTER +#if defined(USE_READ_BARRIER) && defined(USE_BAKER_READ_BARRIER) + ldr wMR, [xSELF, #THREAD_IS_GC_MARKING_OFFSET] +#endif +.endm + .macro RETURN_IF_RESULT_IS_ZERO cbnz x0, 1f // result non-zero branch over ret // return @@ -562,6 +592,7 @@ NO_ARG_RUNTIME_EXCEPTION art_quick_throw_stack_overflow, artThrowStackOverflowFr bl \cxx_name // (method_idx, this, Thread*, SP) mov xIP0, x1 // save Method*->code_ RESTORE_SAVE_REFS_AND_ARGS_FRAME + REFRESH_MARKING_REGISTER cbz x0, 1f // did we find the target? if not go to exception delivery br xIP0 // tail call to target 1: @@ -661,13 +692,15 @@ SAVE_SIZE_AND_METHOD=SAVE_SIZE+8 .macro INVOKE_STUB_CALL_AND_RETURN + REFRESH_MARKING_REGISTER + // load method-> METHOD_QUICK_CODE_OFFSET ldr x9, [x0, #ART_METHOD_QUICK_CODE_OFFSET_64] // Branch to method. blr x9 // Restore return value address and shorty address. - ldp x4,x5, [xFP, #16] + ldp x4, x5, [xFP, #16] .cfi_restore x4 .cfi_restore x5 @@ -1046,6 +1079,7 @@ SAVE_SIZE=15*8 // x3, x4, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, SP stp x3, x4, [sp, #16] // Save result and shorty addresses. stp xFP, xLR, [sp] // Store LR & FP. mov xSELF, x5 // Move thread pointer into SELF register. + REFRESH_MARKING_REGISTER sub sp, sp, #16 str xzr, [sp] // Store null for ArtMethod* slot @@ -1152,7 +1186,7 @@ ENTRY art_quick_do_long_jump ldp x24, x25, [x0], #-16 ldp x22, x23, [x0], #-16 ldp x20, x21, [x0], #-16 - ldp x18, x19, [x0], #-16 + ldp x18, x19, [x0], #-16 // X18 & xSELF ldp x16, x17, [x0], #-16 ldp x14, x15, [x0], #-16 ldp x12, x13, [x0], #-16 @@ -1163,6 +1197,8 @@ ENTRY art_quick_do_long_jump ldp x2, x3, [x0], #-16 mov sp, x1 + REFRESH_MARKING_REGISTER + // Need to load PC, it's at the end (after the space for the unused XZR). Use x1. ldr x1, [x0, #33*8] // And the value of x0. @@ -1213,6 +1249,7 @@ ENTRY art_quick_lock_object mov x1, xSELF // pass Thread::Current bl artLockObjectFromCode // (Object* obj, Thread*) RESTORE_SAVE_REFS_ONLY_FRAME + REFRESH_MARKING_REGISTER RETURN_IF_W0_IS_ZERO_OR_DELIVER END art_quick_lock_object @@ -1221,6 +1258,7 @@ ENTRY art_quick_lock_object_no_inline mov x1, xSELF // pass Thread::Current bl artLockObjectFromCode // (Object* obj, Thread*) RESTORE_SAVE_REFS_ONLY_FRAME + REFRESH_MARKING_REGISTER RETURN_IF_W0_IS_ZERO_OR_DELIVER END art_quick_lock_object_no_inline @@ -1275,6 +1313,7 @@ ENTRY art_quick_unlock_object mov x1, xSELF // pass Thread::Current bl artUnlockObjectFromCode // (Object* obj, Thread*) RESTORE_SAVE_REFS_ONLY_FRAME + REFRESH_MARKING_REGISTER RETURN_IF_W0_IS_ZERO_OR_DELIVER END art_quick_unlock_object @@ -1283,6 +1322,7 @@ ENTRY art_quick_unlock_object_no_inline mov x1, xSELF // pass Thread::Current bl artUnlockObjectFromCode // (Object* obj, Thread*) RESTORE_SAVE_REFS_ONLY_FRAME + REFRESH_MARKING_REGISTER RETURN_IF_W0_IS_ZERO_OR_DELIVER END art_quick_unlock_object_no_inline @@ -1356,7 +1396,7 @@ END art_quick_check_instance_of */ .macro READ_BARRIER xDest, wDest, xObj, xTemp, wTemp, offset, number #ifdef USE_READ_BARRIER -#ifdef USE_BAKER_READ_BARRIER +# ifdef USE_BAKER_READ_BARRIER ldr \wTemp, [\xObj, #MIRROR_OBJECT_LOCK_WORD_OFFSET] tbnz \wTemp, #LOCK_WORD_READ_BARRIER_STATE_SHIFT, .Lrb_slowpath\number // False dependency to avoid needing load/load fence. @@ -1364,7 +1404,7 @@ END art_quick_check_instance_of ldr \wDest, [\xObj, #\offset] // Heap reference = 32b. This also zero-extends to \xDest. UNPOISON_HEAP_REF \wDest b .Lrb_exit\number -#endif +# endif // USE_BAKER_READ_BARRIER .Lrb_slowpath\number: // Store registers used in art_quick_aput_obj (x0-x4, LR), stack is 16B aligned. SAVE_TWO_REGS_INCREASE_FRAME x0, x1, 48 @@ -1471,6 +1511,7 @@ ENTRY \name mov x1, xSELF // pass Thread::Current bl \entrypoint // (uint32_t type_idx, Method* method, Thread*) RESTORE_SAVE_REFS_ONLY_FRAME + REFRESH_MARKING_REGISTER \return END \name .endm @@ -1483,6 +1524,7 @@ ENTRY \name mov x2, xSELF // pass Thread::Current bl \entrypoint // (uint32_t type_idx, Method* method, Thread*) RESTORE_SAVE_REFS_ONLY_FRAME + REFRESH_MARKING_REGISTER \return END \name .endm @@ -1495,6 +1537,7 @@ ENTRY \name mov x3, xSELF // pass Thread::Current bl \entrypoint RESTORE_SAVE_REFS_ONLY_FRAME + REFRESH_MARKING_REGISTER \return END \name .endm @@ -1507,8 +1550,8 @@ ENTRY \name mov x4, xSELF // pass Thread::Current bl \entrypoint // RESTORE_SAVE_REFS_ONLY_FRAME + REFRESH_MARKING_REGISTER \return - DELIVER_PENDING_EXCEPTION END \name .endm @@ -1520,6 +1563,7 @@ ENTRY \name mov x1, xSELF // pass Thread::Current bl \entrypoint // (uint32_t type_idx, Thread*) RESTORE_SAVE_REFS_ONLY_FRAME + REFRESH_MARKING_REGISTER \return END \name .endm @@ -1531,6 +1575,7 @@ ENTRY \name mov x2, xSELF // pass Thread::Current bl \entrypoint RESTORE_SAVE_REFS_ONLY_FRAME + REFRESH_MARKING_REGISTER \return END \name .endm @@ -1542,6 +1587,7 @@ ENTRY \name mov x3, xSELF // pass Thread::Current bl \entrypoint RESTORE_SAVE_REFS_ONLY_FRAME + REFRESH_MARKING_REGISTER \return END \name .endm @@ -1556,6 +1602,7 @@ ENTRY \name cbz w0, 1f // If result is null, deliver the OOME. .cfi_remember_state RESTORE_SAVE_EVERYTHING_FRAME_KEEP_X0 + REFRESH_MARKING_REGISTER ret // return .cfi_restore_state .cfi_def_cfa_offset FRAME_SIZE_SAVE_EVERYTHING // workaround for clang bug: 31975598 @@ -1588,6 +1635,9 @@ ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_initialize_type, artInitializeTypeFro ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_initialize_type_and_verify_access, artInitializeTypeAndVerifyAccessFromCode ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_resolve_string, artResolveStringFromCode +// Note: Functions `art{Get,Set}<Kind>{Static,Instance>FromCompiledCode` are +// defined by macros in runtime/entrypoints/quick/quick_field_entrypoints.cc. + ONE_ARG_REF_DOWNCALL art_quick_get_boolean_static, artGetBooleanStaticFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 ONE_ARG_REF_DOWNCALL art_quick_get_byte_static, artGetByteStaticFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 ONE_ARG_REF_DOWNCALL art_quick_get_char_static, artGetCharStaticFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 @@ -1752,6 +1802,7 @@ ENTRY \c_name mov x1, xSELF // pass Thread::Current bl \cxx_name RESTORE_SAVE_REFS_ONLY_FRAME + REFRESH_MARKING_REGISTER RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER END \c_name .endm @@ -1815,6 +1866,7 @@ ENTRY \name mov x1, xSELF // Pass Thread::Current. bl \entrypoint // (mirror::Class*, Thread*) RESTORE_SAVE_REFS_ONLY_FRAME + REFRESH_MARKING_REGISTER RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER END \name .endm @@ -1825,7 +1877,7 @@ GENERATE_ALLOC_OBJECT_RESOLVED_TLAB art_quick_alloc_object_resolved_tlab, artAll GENERATE_ALLOC_OBJECT_RESOLVED_TLAB art_quick_alloc_object_initialized_tlab, artAllocObjectFromCodeInitializedTLAB, /* isInitialized */ 1 .macro ALLOC_ARRAY_TLAB_FAST_PATH_RESOLVED_WITH_SIZE slowPathLabel, xClass, wClass, xCount, wCount, xTemp0, wTemp0, xTemp1, wTemp1, xTemp2, wTemp2 - and \xTemp1, \xTemp1, #OBJECT_ALIGNMENT_MASK_TOGGLED64 // Apply alignemnt mask + and \xTemp1, \xTemp1, #OBJECT_ALIGNMENT_MASK_TOGGLED64 // Apply alignment mask // (addr + 7) & ~7. The mask must // be 64 bits to keep high bits in // case of overflow. @@ -1887,6 +1939,7 @@ ENTRY \name mov x2, xSELF // pass Thread::Current bl \entrypoint RESTORE_SAVE_REFS_ONLY_FRAME + REFRESH_MARKING_REGISTER RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER END \name .endm @@ -1937,8 +1990,8 @@ END \name add \xTemp1, \xTemp1, #(MIRROR_WIDE_ARRAY_DATA_OFFSET + OBJECT_ALIGNMENT_MASK) .endm -# TODO(ngeoffray): art_quick_alloc_array_resolved_region_tlab is not used for arm64, remove -# the entrypoint once all backends have been updated to use the size variants. +// TODO(ngeoffray): art_quick_alloc_array_resolved_region_tlab is not used for arm64, remove +// the entrypoint once all backends have been updated to use the size variants. GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved_region_tlab, artAllocArrayFromCodeResolvedRegionTLAB, COMPUTE_ARRAY_SIZE_UNKNOWN GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved8_region_tlab, artAllocArrayFromCodeResolvedRegionTLAB, COMPUTE_ARRAY_SIZE_8 GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved16_region_tlab, artAllocArrayFromCodeResolvedRegionTLAB, COMPUTE_ARRAY_SIZE_16 @@ -1959,6 +2012,7 @@ ENTRY art_quick_test_suspend mov x0, xSELF bl artTestSuspendFromCode // (Thread*) RESTORE_SAVE_EVERYTHING_FRAME + REFRESH_MARKING_REGISTER ret END art_quick_test_suspend @@ -1966,7 +2020,9 @@ ENTRY art_quick_implicit_suspend mov x0, xSELF SETUP_SAVE_REFS_ONLY_FRAME // save callee saves for stack crawl bl artTestSuspendFromCode // (Thread*) - RESTORE_SAVE_REFS_ONLY_FRAME_AND_RETURN + RESTORE_SAVE_REFS_ONLY_FRAME + REFRESH_MARKING_REGISTER + ret END art_quick_implicit_suspend /* @@ -1983,6 +2039,7 @@ ENTRY art_quick_proxy_invoke_handler ldr x2, [xSELF, THREAD_EXCEPTION_OFFSET] cbnz x2, .Lexception_in_proxy // success if no exception is pending RESTORE_SAVE_REFS_AND_ARGS_FRAME // Restore frame + REFRESH_MARKING_REGISTER fmov d0, x0 // Store result in d0 in case it was float or double ret // return on success .Lexception_in_proxy: @@ -2035,6 +2092,7 @@ ENTRY art_quick_resolution_trampoline mov xIP0, x0 // Remember returned code pointer in xIP0. ldr x0, [sp, #0] // artQuickResolutionTrampoline puts called method in *SP. RESTORE_SAVE_REFS_AND_ARGS_FRAME + REFRESH_MARKING_REGISTER br xIP0 1: RESTORE_SAVE_REFS_AND_ARGS_FRAME @@ -2170,6 +2228,7 @@ ENTRY art_quick_generic_jni_trampoline // Tear down the callee-save frame. RESTORE_SAVE_REFS_AND_ARGS_FRAME + REFRESH_MARKING_REGISTER // store into fpr, for when it's a fpr return... fmov d0, x0 @@ -2202,6 +2261,7 @@ ENTRY art_quick_to_interpreter_bridge bl artQuickToInterpreterBridge RESTORE_SAVE_REFS_AND_ARGS_FRAME // TODO: no need to restore arguments in this case. + REFRESH_MARKING_REGISTER fmov d0, x0 @@ -2231,6 +2291,7 @@ ENTRY art_quick_instrumentation_entry mov x0, x20 // Reload method reference. RESTORE_SAVE_REFS_AND_ARGS_FRAME // Note: will restore xSELF + REFRESH_MARKING_REGISTER cbz xIP0, 1f // Deliver the pending exception if method is null. adr xLR, art_quick_instrumentation_exit br xIP0 // Tail-call method with lr set to art_quick_instrumentation_exit. @@ -2263,6 +2324,7 @@ ENTRY art_quick_instrumentation_exit .cfi_adjust_cfa_offset -16 RESTORE_SAVE_REFS_ONLY_FRAME + REFRESH_MARKING_REGISTER cbz xIP0, 1f // Handle error br xIP0 // Tail-call out. 1: @@ -2831,6 +2893,7 @@ ENTRY art_quick_invoke_polymorphic .Lcleanup_and_return: DECREASE_FRAME 16 RESTORE_SAVE_REFS_AND_ARGS_FRAME + REFRESH_MARKING_REGISTER RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 .section .rodata // Place handler table in read-only section away from text. diff --git a/runtime/arch/arm64/registers_arm64.h b/runtime/arch/arm64/registers_arm64.h index 4683fc3fdd..d4c919220d 100644 --- a/runtime/arch/arm64/registers_arm64.h +++ b/runtime/arch/arm64/registers_arm64.h @@ -61,6 +61,7 @@ enum XRegister { kNumberOfXRegisters = 33, // Aliases. TR = X19, // ART Thread Register - Managed Runtime (Callee Saved Reg) + MR = X20, // ART Marking Register - Managed Runtime (Callee Saved Reg) IP0 = X16, // Used as scratch by VIXL. IP1 = X17, // Used as scratch by ART JNI Assembler. FP = X29, diff --git a/runtime/arch/quick_alloc_entrypoints.S b/runtime/arch/quick_alloc_entrypoints.S index 2b3525b189..fbfa7564a7 100644 --- a/runtime/arch/quick_alloc_entrypoints.S +++ b/runtime/arch/quick_alloc_entrypoints.S @@ -53,7 +53,7 @@ GENERATE_ALLOC_ENTRYPOINTS _region_tlab_instrumented, RegionTLABInstrumented .endm // Generate the allocation entrypoints for each allocator. This is used as an alternative to -// GNERATE_ALL_ALLOC_ENTRYPOINTS for selectively implementing allocation fast paths in +// GENERATE_ALL_ALLOC_ENTRYPOINTS for selectively implementing allocation fast paths in // hand-written assembly. #define GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_RESOLVED(c_suffix, cxx_suffix) \ ONE_ARG_DOWNCALL art_quick_alloc_object_resolved ## c_suffix, artAllocObjectFromCodeResolved ## cxx_suffix, RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER diff --git a/runtime/base/arena_allocator.h b/runtime/base/arena_allocator.h index ebde82db55..a484c5c105 100644 --- a/runtime/base/arena_allocator.h +++ b/runtime/base/arena_allocator.h @@ -336,7 +336,8 @@ class ArenaAllocator auto* end = reinterpret_cast<uint8_t*>(ptr) + aligned_ptr_size; // If we haven't allocated anything else, we can safely extend. if (end == ptr_) { - DCHECK(!IsRunningOnMemoryTool()); // Red zone prevents end == ptr_. + // Red zone prevents end == ptr_ (unless input = allocator state = null). + DCHECK(!IsRunningOnMemoryTool() || ptr_ == nullptr); const size_t aligned_new_size = RoundUp(new_size, kAlignment); const size_t size_delta = aligned_new_size - aligned_ptr_size; // Check remain space. diff --git a/runtime/base/mutex.cc b/runtime/base/mutex.cc index b0394a5255..a472b67fcd 100644 --- a/runtime/base/mutex.cc +++ b/runtime/base/mutex.cc @@ -68,6 +68,7 @@ ConditionVariable* Locks::thread_exit_cond_ = nullptr; Mutex* Locks::thread_suspend_count_lock_ = nullptr; Mutex* Locks::trace_lock_ = nullptr; Mutex* Locks::unexpected_signal_lock_ = nullptr; +Mutex* Locks::user_code_suspension_lock_ = nullptr; Uninterruptible Roles::uninterruptible_; ReaderWriterMutex* Locks::jni_globals_lock_ = nullptr; Mutex* Locks::jni_weak_globals_lock_ = nullptr; @@ -1029,6 +1030,7 @@ void Locks::Init() { DCHECK(thread_suspend_count_lock_ != nullptr); DCHECK(trace_lock_ != nullptr); DCHECK(unexpected_signal_lock_ != nullptr); + DCHECK(user_code_suspension_lock_ != nullptr); DCHECK(dex_lock_ != nullptr); } else { // Create global locks in level order from highest lock level to lowest. @@ -1045,6 +1047,10 @@ void Locks::Init() { } \ current_lock_level = new_level; + UPDATE_CURRENT_LOCK_LEVEL(kUserCodeSuspensionLock); + DCHECK(user_code_suspension_lock_ == nullptr); + user_code_suspension_lock_ = new Mutex("user code suspension lock", current_lock_level); + UPDATE_CURRENT_LOCK_LEVEL(kMutatorLock); DCHECK(mutator_lock_ == nullptr); mutator_lock_ = new MutatorMutex("mutator lock", current_lock_level); diff --git a/runtime/base/mutex.h b/runtime/base/mutex.h index e77d8d749d..7a472e741b 100644 --- a/runtime/base/mutex.h +++ b/runtime/base/mutex.h @@ -116,6 +116,7 @@ enum LockLevel { kTraceLock, kHeapBitmapLock, kMutatorLock, + kUserCodeSuspensionLock, kInstrumentEntrypointsLock, kZygoteCreationLock, @@ -578,6 +579,11 @@ class Locks { // Guards allocation entrypoint instrumenting. static Mutex* instrument_entrypoints_lock_; + // Guards code that deals with user-code suspension. This mutex must be held when suspending or + // resuming threads with SuspendReason::kForUserCode. It may be held by a suspended thread, but + // only if the suspension is not due to SuspendReason::kForUserCode. + static Mutex* user_code_suspension_lock_ ACQUIRED_AFTER(instrument_entrypoints_lock_); + // A barrier is used to synchronize the GC/Debugger thread with mutator threads. When GC/Debugger // thread wants to suspend all mutator threads, it needs to wait for all mutator threads to pass // a barrier. Threads that are already suspended will get their barrier passed by the GC/Debugger @@ -613,7 +619,7 @@ class Locks { // state is changed | .. running .. // - if the CAS operation fails then goto x | .. running .. // .. running .. | .. running .. - static MutatorMutex* mutator_lock_ ACQUIRED_AFTER(instrument_entrypoints_lock_); + static MutatorMutex* mutator_lock_ ACQUIRED_AFTER(user_code_suspension_lock_); // Allow reader-writer mutual exclusion on the mark and live bitmaps of the heap. static ReaderWriterMutex* heap_bitmap_lock_ ACQUIRED_AFTER(mutator_lock_); diff --git a/runtime/class_linker.cc b/runtime/class_linker.cc index a19085f5b5..a5d4540f88 100644 --- a/runtime/class_linker.cc +++ b/runtime/class_linker.cc @@ -8369,150 +8369,244 @@ mirror::MethodType* ClassLinker::ResolveMethodType(const DexFile& dex_file, return type.Get(); } -mirror::MethodHandle* ClassLinker::ResolveMethodHandle(uint32_t method_handle_idx, - ArtMethod* referrer) - REQUIRES_SHARED(Locks::mutator_lock_) { - Thread* const self = Thread::Current(); - const DexFile* const dex_file = referrer->GetDexFile(); - const DexFile::MethodHandleItem& mh = dex_file->GetMethodHandle(method_handle_idx); - - ArtField* target_field = nullptr; - ArtMethod* target_method = nullptr; - +mirror::MethodHandle* ClassLinker::ResolveMethodHandleForField( + Thread* self, + const DexFile::MethodHandleItem& method_handle, + ArtMethod* referrer) { DexFile::MethodHandleType handle_type = - static_cast<DexFile::MethodHandleType>(mh.method_handle_type_); + static_cast<DexFile::MethodHandleType>(method_handle.method_handle_type_); + mirror::MethodHandle::Kind kind; + bool is_static; + int32_t num_params; switch (handle_type) { case DexFile::MethodHandleType::kStaticPut: { - target_field = ResolveField(mh.field_or_method_idx_, referrer, true /* is_static */); + kind = mirror::MethodHandle::Kind::kStaticPut; + is_static = true; + num_params = 1; break; } case DexFile::MethodHandleType::kStaticGet: { - target_field = ResolveField(mh.field_or_method_idx_, referrer, true /* is_static */); + kind = mirror::MethodHandle::Kind::kStaticGet; + is_static = true; + num_params = 0; break; } case DexFile::MethodHandleType::kInstancePut: { - target_field = ResolveField(mh.field_or_method_idx_, referrer, false /* is_static */); + kind = mirror::MethodHandle::Kind::kInstancePut; + is_static = false; + num_params = 2; break; } case DexFile::MethodHandleType::kInstanceGet: { - target_field = ResolveField(mh.field_or_method_idx_, referrer, false /* is_static */); - break; - } - case DexFile::MethodHandleType::kInvokeStatic: { - target_method = ResolveMethod<kNoICCECheckForCache>(self, - mh.field_or_method_idx_, - referrer, - InvokeType::kStatic); - break; - } - case DexFile::MethodHandleType::kInvokeInstance: { - target_method = ResolveMethod<kNoICCECheckForCache>(self, - mh.field_or_method_idx_, - referrer, - InvokeType::kVirtual); - break; - } - case DexFile::MethodHandleType::kInvokeConstructor: { - UNIMPLEMENTED(FATAL) << "Invoke constructor is implemented as a transform."; - break; - } - case DexFile::MethodHandleType::kInvokeDirect: { - target_method = ResolveMethod<kNoICCECheckForCache>(self, - mh.field_or_method_idx_, - referrer, - InvokeType::kDirect); - break; - } - case DexFile::MethodHandleType::kInvokeInterface: { - target_method = ResolveMethod<kNoICCECheckForCache>(self, - mh.field_or_method_idx_, - referrer, - InvokeType::kInterface); + kind = mirror::MethodHandle::Kind::kInstanceGet; + is_static = false; + num_params = 1; break; } + case DexFile::MethodHandleType::kInvokeStatic: + case DexFile::MethodHandleType::kInvokeInstance: + case DexFile::MethodHandleType::kInvokeConstructor: + case DexFile::MethodHandleType::kInvokeDirect: + case DexFile::MethodHandleType::kInvokeInterface: + UNREACHABLE(); } - if (target_field != nullptr) { + ArtField* target_field = + ResolveField(method_handle.field_or_method_idx_, referrer, is_static); + if (LIKELY(target_field != nullptr)) { ObjPtr<mirror::Class> target_class = target_field->GetDeclaringClass(); ObjPtr<mirror::Class> referring_class = referrer->GetDeclaringClass(); - if (!referring_class->CanAccessMember(target_class, target_field->GetAccessFlags())) { + if (UNLIKELY(!referring_class->CanAccessMember(target_class, target_field->GetAccessFlags()))) { ThrowIllegalAccessErrorField(referring_class, target_field); return nullptr; } - } else if (target_method != nullptr) { - ObjPtr<mirror::Class> target_class = target_method->GetDeclaringClass(); - ObjPtr<mirror::Class> referring_class = referrer->GetDeclaringClass(); - if (!referring_class->CanAccessMember(target_class, target_method->GetAccessFlags())) { - ThrowIllegalAccessErrorMethod(referring_class, target_method); - return nullptr; - } } else { - // Common check for resolution failure. DCHECK(Thread::Current()->IsExceptionPending()); return nullptr; } - // Determine the kind and number of parameters after it's safe to - // follow the field or method pointer. - mirror::MethodHandle::Kind kind; - uint32_t num_params; + StackHandleScope<4> hs(self); + ObjPtr<mirror::Class> class_type = mirror::Class::GetJavaLangClass(); + ObjPtr<mirror::Class> array_of_class = FindArrayClass(self, &class_type); + Handle<mirror::ObjectArray<mirror::Class>> method_params(hs.NewHandle( + mirror::ObjectArray<mirror::Class>::Alloc(self, array_of_class, num_params))); + if (UNLIKELY(method_params.Get() == nullptr)) { + DCHECK(self->IsExceptionPending()); + return nullptr; + } + + Handle<mirror::Class> constructor_class; + Handle<mirror::Class> return_type; switch (handle_type) { case DexFile::MethodHandleType::kStaticPut: { - kind = mirror::MethodHandle::Kind::kStaticPut; - num_params = 1; + method_params->Set(0, target_field->GetType<true>()); + return_type = hs.NewHandle(FindPrimitiveClass('V')); break; } case DexFile::MethodHandleType::kStaticGet: { - kind = mirror::MethodHandle::Kind::kStaticGet; - num_params = 0; + return_type = hs.NewHandle(target_field->GetType<true>()); break; } case DexFile::MethodHandleType::kInstancePut: { - kind = mirror::MethodHandle::Kind::kInstancePut; - num_params = 2; + method_params->Set(0, target_field->GetDeclaringClass()); + method_params->Set(1, target_field->GetType<true>()); + return_type = hs.NewHandle(FindPrimitiveClass('V')); break; } case DexFile::MethodHandleType::kInstanceGet: { - kind = mirror::MethodHandle::Kind::kInstanceGet; - num_params = 1; + method_params->Set(0, target_field->GetDeclaringClass()); + return_type = hs.NewHandle(target_field->GetType<true>()); break; } + case DexFile::MethodHandleType::kInvokeStatic: + case DexFile::MethodHandleType::kInvokeInstance: + case DexFile::MethodHandleType::kInvokeConstructor: + case DexFile::MethodHandleType::kInvokeDirect: + case DexFile::MethodHandleType::kInvokeInterface: + UNREACHABLE(); + } + + for (int32_t i = 0; i < num_params; ++i) { + if (UNLIKELY(method_params->Get(i) == nullptr)) { + DCHECK(self->IsExceptionPending()); + return nullptr; + } + } + + if (UNLIKELY(return_type.IsNull())) { + DCHECK(self->IsExceptionPending()); + return nullptr; + } + + Handle<mirror::MethodType> + method_type(hs.NewHandle(mirror::MethodType::Create(self, return_type, method_params))); + if (UNLIKELY(method_type.IsNull())) { + DCHECK(self->IsExceptionPending()); + return nullptr; + } + + uintptr_t target = reinterpret_cast<uintptr_t>(target_field); + return mirror::MethodHandleImpl::Create(self, target, kind, method_type); +} + +mirror::MethodHandle* ClassLinker::ResolveMethodHandleForMethod( + Thread* self, + const DexFile* const dex_file, + const DexFile::MethodHandleItem& method_handle, + ArtMethod* referrer) { + DexFile::MethodHandleType handle_type = + static_cast<DexFile::MethodHandleType>(method_handle.method_handle_type_); + mirror::MethodHandle::Kind kind; + uint32_t receiver_count = 0; + ArtMethod* target_method = nullptr; + switch (handle_type) { + case DexFile::MethodHandleType::kStaticPut: + case DexFile::MethodHandleType::kStaticGet: + case DexFile::MethodHandleType::kInstancePut: + case DexFile::MethodHandleType::kInstanceGet: + UNREACHABLE(); case DexFile::MethodHandleType::kInvokeStatic: { kind = mirror::MethodHandle::Kind::kInvokeStatic; - uint32_t shorty_length; - target_method->GetShorty(&shorty_length); - num_params = shorty_length - 1; // Remove 1 for the return value. + receiver_count = 0; + target_method = ResolveMethod<kNoICCECheckForCache>(self, + method_handle.field_or_method_idx_, + referrer, + InvokeType::kStatic); break; } case DexFile::MethodHandleType::kInvokeInstance: { kind = mirror::MethodHandle::Kind::kInvokeVirtual; - uint32_t shorty_length; - target_method->GetShorty(&shorty_length); - num_params = shorty_length; // Add 1 for the receiver, remove 1 for the return value. + receiver_count = 1; + target_method = ResolveMethod<kNoICCECheckForCache>(self, + method_handle.field_or_method_idx_, + referrer, + InvokeType::kVirtual); break; } case DexFile::MethodHandleType::kInvokeConstructor: { - UNIMPLEMENTED(FATAL) << "Invoke constructor is implemented as a transform."; - num_params = 0; + // Constructors are currently implemented as a transform. They + // are special cased later in this method. + kind = mirror::MethodHandle::Kind::kInvokeTransform; + receiver_count = 0; + target_method = ResolveMethod<kNoICCECheckForCache>(self, + method_handle.field_or_method_idx_, + referrer, + InvokeType::kDirect); break; } case DexFile::MethodHandleType::kInvokeDirect: { kind = mirror::MethodHandle::Kind::kInvokeDirect; - uint32_t shorty_length; - target_method->GetShorty(&shorty_length); - num_params = shorty_length; // Add 1 for the receiver, remove 1 for the return value. + receiver_count = 1; + StackHandleScope<2> hs(self); + // A constant method handle with type kInvokeDirect can refer to + // a method that is private or to a method in a super class. To + // disambiguate the two options, we resolve the method ignoring + // the invocation type to determine if the method is private. We + // then resolve again specifying the intended invocation type to + // force the appropriate checks. + target_method = ResolveMethodWithoutInvokeType(*dex_file, + method_handle.field_or_method_idx_, + hs.NewHandle(referrer->GetDexCache()), + hs.NewHandle(referrer->GetClassLoader())); + if (UNLIKELY(target_method == nullptr)) { + break; + } + + if (target_method->IsPrivate()) { + kind = mirror::MethodHandle::Kind::kInvokeDirect; + target_method = ResolveMethod<kNoICCECheckForCache>(self, + method_handle.field_or_method_idx_, + referrer, + InvokeType::kDirect); + } else { + kind = mirror::MethodHandle::Kind::kInvokeSuper; + target_method = ResolveMethod<kNoICCECheckForCache>(self, + method_handle.field_or_method_idx_, + referrer, + InvokeType::kSuper); + if (UNLIKELY(target_method == nullptr)) { + break; + } + // Find the method specified in the parent in referring class + // so invoke-super invokes the method in the parent of the + // referrer. + target_method = + referrer->GetDeclaringClass()->FindVirtualMethodForVirtual(target_method, + kRuntimePointerSize); + } break; } case DexFile::MethodHandleType::kInvokeInterface: { kind = mirror::MethodHandle::Kind::kInvokeInterface; - uint32_t shorty_length; - target_method->GetShorty(&shorty_length); - num_params = shorty_length; // Add 1 for the receiver, remove 1 for the return value. + receiver_count = 1; + target_method = ResolveMethod<kNoICCECheckForCache>(self, + method_handle.field_or_method_idx_, + referrer, + InvokeType::kInterface); break; } } - StackHandleScope<5> hs(self); + if (UNLIKELY(target_method == nullptr)) { + DCHECK(Thread::Current()->IsExceptionPending()); + return nullptr; + } + + ObjPtr<mirror::Class> target_class = target_method->GetDeclaringClass(); + ObjPtr<mirror::Class> referring_class = referrer->GetDeclaringClass(); + uint32_t access_flags = target_method->GetAccessFlags(); + if (UNLIKELY(!referring_class->CanAccessMember(target_class, access_flags))) { + ThrowIllegalAccessErrorMethod(referring_class, target_method); + return nullptr; + } + + // Calculate the number of parameters from the method shorty. We add the + // receiver count (0 or 1) and deduct one for the return value. + uint32_t shorty_length; + target_method->GetShorty(&shorty_length); + int32_t num_params = static_cast<int32_t>(shorty_length + receiver_count - 1); + + StackHandleScope<7> hs(self); ObjPtr<mirror::Class> class_type = mirror::Class::GetJavaLangClass(); ObjPtr<mirror::Class> array_of_class = FindArrayClass(self, &class_type); Handle<mirror::ObjectArray<mirror::Class>> method_params(hs.NewHandle( @@ -8522,81 +8616,70 @@ mirror::MethodHandle* ClassLinker::ResolveMethodHandle(uint32_t method_handle_id return nullptr; } - Handle<mirror::Class> return_type; - switch (handle_type) { - case DexFile::MethodHandleType::kStaticPut: { - method_params->Set(0, target_field->GetType<true>()); - return_type = hs.NewHandle(FindPrimitiveClass('V')); - break; - } - case DexFile::MethodHandleType::kStaticGet: { - return_type = hs.NewHandle(target_field->GetType<true>()); - break; - } - case DexFile::MethodHandleType::kInstancePut: { - method_params->Set(0, target_field->GetDeclaringClass()); - method_params->Set(1, target_field->GetType<true>()); - return_type = hs.NewHandle(FindPrimitiveClass('V')); - break; - } - case DexFile::MethodHandleType::kInstanceGet: { - method_params->Set(0, target_field->GetDeclaringClass()); - return_type = hs.NewHandle(target_field->GetType<true>()); - break; - } - case DexFile::MethodHandleType::kInvokeDirect: - case DexFile::MethodHandleType::kInvokeInstance: - case DexFile::MethodHandleType::kInvokeInterface: - case DexFile::MethodHandleType::kInvokeStatic: { - // TODO(oth): This will not work for varargs methods as this - // requires instantiating a Transformer. This resolution step - // would be best done in managed code rather than in the run - // time (b/35235705) - Handle<mirror::DexCache> dex_cache(hs.NewHandle(referrer->GetDexCache())); - Handle<mirror::ClassLoader> class_loader(hs.NewHandle(referrer->GetClassLoader())); - DexFileParameterIterator it(*dex_file, target_method->GetPrototype()); - int32_t index = 0; - if (handle_type != DexFile::MethodHandleType::kInvokeStatic) { - method_params->Set(index++, target_method->GetDeclaringClass()); - } - while (it.HasNext()) { - const dex::TypeIndex type_idx = it.GetTypeIdx(); - mirror::Class* klass = ResolveType(*dex_file, type_idx, dex_cache, class_loader); - if (nullptr == klass) { - DCHECK(self->IsExceptionPending()); - return nullptr; - } - method_params->Set(index++, klass); - it.Next(); - } - return_type = hs.NewHandle(target_method->GetReturnType(true)); - break; - } - case DexFile::MethodHandleType::kInvokeConstructor: { - // TODO(oth): b/35235705 - UNIMPLEMENTED(FATAL) << "Invoke constructor is implemented as a transform."; + Handle<mirror::DexCache> dex_cache(hs.NewHandle(referrer->GetDexCache())); + Handle<mirror::ClassLoader> class_loader(hs.NewHandle(referrer->GetClassLoader())); + int32_t index = 0; + + if (receiver_count != 0) { + // Insert receiver + method_params->Set(index++, target_method->GetDeclaringClass()); + } + + DexFileParameterIterator it(*dex_file, target_method->GetPrototype()); + while (it.HasNext()) { + const dex::TypeIndex type_idx = it.GetTypeIdx(); + mirror::Class* klass = ResolveType(*dex_file, type_idx, dex_cache, class_loader); + if (nullptr == klass) { + DCHECK(self->IsExceptionPending()); + return nullptr; } + method_params->Set(index++, klass); + it.Next(); } - if (return_type.IsNull()) { + Handle<mirror::Class> return_type = hs.NewHandle(target_method->GetReturnType(true)); + if (UNLIKELY(return_type.IsNull())) { DCHECK(self->IsExceptionPending()); return nullptr; } Handle<mirror::MethodType> - mt(hs.NewHandle(mirror::MethodType::Create(self, return_type, method_params))); - if (mt.IsNull()) { + method_type(hs.NewHandle(mirror::MethodType::Create(self, return_type, method_params))); + if (UNLIKELY(method_type.IsNull())) { DCHECK(self->IsExceptionPending()); return nullptr; } - uintptr_t target; - if (target_field != nullptr) { - target = reinterpret_cast<uintptr_t>(target_field); - } else { - target = reinterpret_cast<uintptr_t>(target_method); + if (UNLIKELY(handle_type == DexFile::MethodHandleType::kInvokeConstructor)) { + Handle<mirror::Class> constructor_class = hs.NewHandle(target_method->GetDeclaringClass()); + Handle<mirror::MethodHandlesLookup> lookup = + hs.NewHandle(mirror::MethodHandlesLookup::GetDefault(self)); + return lookup->FindConstructor(self, constructor_class, method_type); + } + + uintptr_t target = reinterpret_cast<uintptr_t>(target_method); + return mirror::MethodHandleImpl::Create(self, target, kind, method_type); +} + +mirror::MethodHandle* ClassLinker::ResolveMethodHandle(uint32_t method_handle_idx, + ArtMethod* referrer) + REQUIRES_SHARED(Locks::mutator_lock_) { + Thread* const self = Thread::Current(); + const DexFile* const dex_file = referrer->GetDexFile(); + const DexFile::MethodHandleItem& method_handle = dex_file->GetMethodHandle(method_handle_idx); + switch (static_cast<DexFile::MethodHandleType>(method_handle.method_handle_type_)) { + case DexFile::MethodHandleType::kStaticPut: + case DexFile::MethodHandleType::kStaticGet: + case DexFile::MethodHandleType::kInstancePut: + case DexFile::MethodHandleType::kInstanceGet: + return ResolveMethodHandleForField(self, method_handle, referrer); + case DexFile::MethodHandleType::kInvokeStatic: + case DexFile::MethodHandleType::kInvokeInstance: + case DexFile::MethodHandleType::kInvokeConstructor: + case DexFile::MethodHandleType::kInvokeDirect: + case DexFile::MethodHandleType::kInvokeInterface: + return ResolveMethodHandleForMethod(self, dex_file, method_handle, referrer); } - return mirror::MethodHandleImpl::Create(self, target, kind, mt); } bool ClassLinker::IsQuickResolutionStub(const void* entry_point) const { diff --git a/runtime/class_linker.h b/runtime/class_linker.h index de1fefd20e..8b8a6fd420 100644 --- a/runtime/class_linker.h +++ b/runtime/class_linker.h @@ -671,6 +671,10 @@ class ClassLinker { REQUIRES_SHARED(Locks::mutator_lock_) REQUIRES(!Locks::dex_lock_); + // Visit all of the class loaders in the class linker. + void VisitClassLoaders(ClassLoaderVisitor* visitor) const + REQUIRES_SHARED(Locks::classlinker_classes_lock_, Locks::mutator_lock_); + struct DexCacheData { // Construct an invalid data object. DexCacheData() @@ -720,9 +724,6 @@ class ClassLinker { static void DeleteClassLoader(Thread* self, const ClassLoaderData& data) REQUIRES_SHARED(Locks::mutator_lock_); - void VisitClassLoaders(ClassLoaderVisitor* visitor) const - REQUIRES_SHARED(Locks::classlinker_classes_lock_, Locks::mutator_lock_); - void VisitClassesInternal(ClassVisitor* visitor) REQUIRES_SHARED(Locks::classlinker_classes_lock_, Locks::mutator_lock_); @@ -943,6 +944,17 @@ class ClassLinker { ArtMethod** out_imt) REQUIRES_SHARED(Locks::mutator_lock_); + mirror::MethodHandle* ResolveMethodHandleForField(Thread* self, + const DexFile::MethodHandleItem& method_handle, + ArtMethod* referrer) + REQUIRES_SHARED(Locks::mutator_lock_); + + mirror::MethodHandle* ResolveMethodHandleForMethod(Thread* self, + const DexFile* const dex_file, + const DexFile::MethodHandleItem& method_handle, + ArtMethod* referrer) + REQUIRES_SHARED(Locks::mutator_lock_); + // A wrapper class representing the result of a method translation used for linking methods and // updating superclass default methods. For each method in a classes vtable there are 4 states it // could be in: diff --git a/runtime/class_loader_context.cc b/runtime/class_loader_context.cc new file mode 100644 index 0000000000..2bed1d5b2d --- /dev/null +++ b/runtime/class_loader_context.cc @@ -0,0 +1,360 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "class_loader_context.h" + +#include "base/dchecked_vector.h" +#include "base/stl_util.h" +#include "class_linker.h" +#include "dex_file.h" +#include "oat_file_assistant.h" +#include "runtime.h" +#include "scoped_thread_state_change-inl.h" +#include "thread.h" + +namespace art { + +static constexpr char kPathClassLoaderString[] = "PCL"; +static constexpr char kDelegateLastClassLoaderString[] = "DLC"; +static constexpr char kClassLoaderOpeningMark = '['; +static constexpr char kClassLoaderClosingMark = ']'; +static constexpr char kClassLoaderSeparator = ';'; +static constexpr char kClasspathSeparator = ':'; +static constexpr char kDexFileChecksumSeparator = '*'; + +ClassLoaderContext::ClassLoaderContext() + : special_shared_library_(false), + dex_files_open_attempted_(false), + dex_files_open_result_(false) {} + +std::unique_ptr<ClassLoaderContext> ClassLoaderContext::Create(const std::string& spec) { + std::unique_ptr<ClassLoaderContext> result(new ClassLoaderContext()); + if (result->Parse(spec)) { + return result; + } else { + return nullptr; + } +} + +// The expected format is: "ClassLoaderType1[ClasspathElem1*Checksum1:ClasspathElem2*Checksum2...]". +// The checksum part of the format is expected only if parse_cheksums is true. +bool ClassLoaderContext::ParseClassLoaderSpec(const std::string& class_loader_spec, + ClassLoaderType class_loader_type, + bool parse_checksums) { + const char* class_loader_type_str = GetClassLoaderTypeName(class_loader_type); + size_t type_str_size = strlen(class_loader_type_str); + + CHECK_EQ(0, class_loader_spec.compare(0, type_str_size, class_loader_type_str)); + + // Check the opening and closing markers. + if (class_loader_spec[type_str_size] != kClassLoaderOpeningMark) { + return false; + } + if (class_loader_spec[class_loader_spec.length() - 1] != kClassLoaderClosingMark) { + return false; + } + + // At this point we know the format is ok; continue and extract the classpath. + // Note that class loaders with an empty class path are allowed. + std::string classpath = class_loader_spec.substr(type_str_size + 1, + class_loader_spec.length() - type_str_size - 2); + + class_loader_chain_.push_back(ClassLoaderInfo(class_loader_type)); + + if (!parse_checksums) { + Split(classpath, kClasspathSeparator, &class_loader_chain_.back().classpath); + } else { + std::vector<std::string> classpath_elements; + Split(classpath, kClasspathSeparator, &classpath_elements); + for (const std::string& element : classpath_elements) { + std::vector<std::string> dex_file_with_checksum; + Split(element, kDexFileChecksumSeparator, &dex_file_with_checksum); + if (dex_file_with_checksum.size() != 2) { + return false; + } + uint32_t checksum = 0; + if (!ParseInt(dex_file_with_checksum[1].c_str(), &checksum)) { + return false; + } + class_loader_chain_.back().classpath.push_back(dex_file_with_checksum[0]); + class_loader_chain_.back().checksums.push_back(checksum); + } + } + + return true; +} + +// Extracts the class loader type from the given spec. +// Return ClassLoaderContext::kInvalidClassLoader if the class loader type is not +// recognized. +ClassLoaderContext::ClassLoaderType +ClassLoaderContext::ExtractClassLoaderType(const std::string& class_loader_spec) { + const ClassLoaderType kValidTypes[] = {kPathClassLoader, kDelegateLastClassLoader}; + for (const ClassLoaderType& type : kValidTypes) { + const char* type_str = GetClassLoaderTypeName(type); + if (class_loader_spec.compare(0, strlen(type_str), type_str) == 0) { + return type; + } + } + return kInvalidClassLoader; +} + +// The format: ClassLoaderType1[ClasspathElem1:ClasspathElem2...];ClassLoaderType2[...]... +// ClassLoaderType is either "PCL" (PathClassLoader) or "DLC" (DelegateLastClassLoader). +// ClasspathElem is the path of dex/jar/apk file. +bool ClassLoaderContext::Parse(const std::string& spec, bool parse_checksums) { + if (spec.empty()) { + return true; + } + + // Stop early if we detect the special shared library, which may be passed as the classpath + // for dex2oat when we want to skip the shared libraries check. + if (spec == OatFile::kSpecialSharedLibrary) { + LOG(INFO) << "The ClassLoaderContext is a special shared library."; + special_shared_library_ = true; + return true; + } + + std::vector<std::string> class_loaders; + Split(spec, kClassLoaderSeparator, &class_loaders); + + for (const std::string& class_loader : class_loaders) { + ClassLoaderType type = ExtractClassLoaderType(class_loader); + if (type == kInvalidClassLoader) { + LOG(ERROR) << "Invalid class loader type: " << class_loader; + return false; + } + if (!ParseClassLoaderSpec(class_loader, type, parse_checksums)) { + LOG(ERROR) << "Invalid class loader spec: " << class_loader; + return false; + } + } + return true; +} + +// Opens requested class path files and appends them to opened_dex_files. If the dex files have +// been stripped, this opens them from their oat files (which get added to opened_oat_files). +bool ClassLoaderContext::OpenDexFiles(InstructionSet isa, const std::string& classpath_dir) { + CHECK(!dex_files_open_attempted_) << "OpenDexFiles should not be called twice"; + + dex_files_open_attempted_ = true; + // Assume we can open all dex files. If not, we will set this to false as we go. + dex_files_open_result_ = true; + + if (special_shared_library_) { + // Nothing to open if the context is a special shared library. + return true; + } + + // Note that we try to open all dex files even if some fail. + // We may get resource-only apks which we cannot load. + // TODO(calin): Refine the dex opening interface to be able to tell if an archive contains + // no dex files. So that we can distinguish the real failures... + for (ClassLoaderInfo& info : class_loader_chain_) { + for (const std::string& cp_elem : info.classpath) { + // If path is relative, append it to the provided base directory. + std::string location = cp_elem; + if (location[0] != '/') { + location = classpath_dir + '/' + location; + } + std::string error_msg; + // When opening the dex files from the context we expect their checksum to match their + // contents. So pass true to verify_checksum. + if (!DexFile::Open(location.c_str(), + location.c_str(), + /*verify_checksum*/ true, + &error_msg, + &info.opened_dex_files)) { + // If we fail to open the dex file because it's been stripped, try to open the dex file + // from its corresponding oat file. + // This could happen when we need to recompile a pre-build whose dex code has been stripped. + // (for example, if the pre-build is only quicken and we want to re-compile it + // speed-profile). + // TODO(calin): Use the vdex directly instead of going through the oat file. + OatFileAssistant oat_file_assistant(location.c_str(), isa, false); + std::unique_ptr<OatFile> oat_file(oat_file_assistant.GetBestOatFile()); + std::vector<std::unique_ptr<const DexFile>> oat_dex_files; + if (oat_file != nullptr && + OatFileAssistant::LoadDexFiles(*oat_file, location, &oat_dex_files)) { + info.opened_oat_files.push_back(std::move(oat_file)); + info.opened_dex_files.insert(info.opened_dex_files.end(), + std::make_move_iterator(oat_dex_files.begin()), + std::make_move_iterator(oat_dex_files.end())); + } else { + LOG(WARNING) << "Could not open dex files from location: " << location; + dex_files_open_result_ = false; + } + } + } + } + + return dex_files_open_result_; +} + +bool ClassLoaderContext::RemoveLocationsFromClassPaths( + const dchecked_vector<std::string>& locations) { + CHECK(!dex_files_open_attempted_) + << "RemoveLocationsFromClasspaths cannot be call after OpenDexFiles"; + + std::set<std::string> canonical_locations; + for (const std::string& location : locations) { + canonical_locations.insert(DexFile::GetDexCanonicalLocation(location.c_str())); + } + bool removed_locations = false; + for (ClassLoaderInfo& info : class_loader_chain_) { + size_t initial_size = info.classpath.size(); + auto kept_it = std::remove_if( + info.classpath.begin(), + info.classpath.end(), + [canonical_locations](const std::string& location) { + return ContainsElement(canonical_locations, + DexFile::GetDexCanonicalLocation(location.c_str())); + }); + info.classpath.erase(kept_it, info.classpath.end()); + if (initial_size != info.classpath.size()) { + removed_locations = true; + } + } + return removed_locations; +} + +std::string ClassLoaderContext::EncodeContextForOatFile(const std::string& base_dir) const { + CheckDexFilesOpened("EncodeContextForOatFile"); + if (special_shared_library_) { + return OatFile::kSpecialSharedLibrary; + } + + if (class_loader_chain_.empty()) { + return ""; + } + + std::ostringstream out; + + for (size_t i = 0; i < class_loader_chain_.size(); i++) { + const ClassLoaderInfo& info = class_loader_chain_[i]; + if (i > 0) { + out << kClassLoaderSeparator; + } + out << GetClassLoaderTypeName(info.type); + out << kClassLoaderOpeningMark; + for (size_t k = 0; k < info.opened_dex_files.size(); k++) { + const std::unique_ptr<const DexFile>& dex_file = info.opened_dex_files[k]; + const std::string& location = dex_file->GetLocation(); + if (k > 0) { + out << kClasspathSeparator; + } + // Find paths that were relative and convert them back from absolute. + if (!base_dir.empty() && location.substr(0, base_dir.length()) == base_dir) { + out << location.substr(base_dir.length() + 1).c_str(); + } else { + out << dex_file->GetLocation().c_str(); + } + out << kDexFileChecksumSeparator; + out << dex_file->GetLocationChecksum(); + } + out << kClassLoaderClosingMark; + } + return out.str(); +} + +jobject ClassLoaderContext::CreateClassLoader( + const std::vector<const DexFile*>& compilation_sources) const { + CheckDexFilesOpened("CreateClassLoader"); + + Thread* self = Thread::Current(); + ScopedObjectAccess soa(self); + + std::vector<const DexFile*> class_path_files; + + // TODO(calin): Transition period: assume we only have a classloader until + // the oat file assistant implements the full class loader check. + if (!class_loader_chain_.empty()) { + CHECK_EQ(1u, class_loader_chain_.size()); + CHECK_EQ(kPathClassLoader, class_loader_chain_[0].type); + class_path_files = MakeNonOwningPointerVector(class_loader_chain_[0].opened_dex_files); + } + + // Classpath: first the class-path given; then the dex files we'll compile. + // Thus we'll resolve the class-path first. + class_path_files.insert(class_path_files.end(), + compilation_sources.begin(), + compilation_sources.end()); + + ClassLinker* const class_linker = Runtime::Current()->GetClassLinker(); + return class_linker->CreatePathClassLoader(self, class_path_files); +} + +std::vector<const DexFile*> ClassLoaderContext::FlattenOpenedDexFiles() const { + CheckDexFilesOpened("FlattenOpenedDexFiles"); + + std::vector<const DexFile*> result; + for (const ClassLoaderInfo& info : class_loader_chain_) { + for (const std::unique_ptr<const DexFile>& dex_file : info.opened_dex_files) { + result.push_back(dex_file.get()); + } + } + return result; +} + +const char* ClassLoaderContext::GetClassLoaderTypeName(ClassLoaderType type) { + switch (type) { + case kPathClassLoader: return kPathClassLoaderString; + case kDelegateLastClassLoader: return kDelegateLastClassLoaderString; + default: + LOG(FATAL) << "Invalid class loader type " << type; + UNREACHABLE(); + } +} + +void ClassLoaderContext::CheckDexFilesOpened(const std::string& calling_method) const { + CHECK(dex_files_open_attempted_) + << "Dex files were not successfully opened before the call to " << calling_method + << "attempt=" << dex_files_open_attempted_ << ", result=" << dex_files_open_result_; +} + +bool ClassLoaderContext::DecodePathClassLoaderContextFromOatFileKey( + const std::string& context_spec, + std::vector<std::string>* out_classpath, + std::vector<uint32_t>* out_checksums, + bool* out_is_special_shared_library) { + ClassLoaderContext context; + if (!context.Parse(context_spec, /*parse_checksums*/ true)) { + LOG(ERROR) << "Invalid class loader context: " << context_spec; + return false; + } + + *out_is_special_shared_library = context.special_shared_library_; + if (context.special_shared_library_) { + return true; + } + + if (context.class_loader_chain_.empty()) { + return true; + } + + // TODO(calin): assert that we only have a PathClassLoader until the logic for + // checking the context covers all case. + CHECK_EQ(1u, context.class_loader_chain_.size()); + const ClassLoaderInfo& info = context.class_loader_chain_[0]; + CHECK_EQ(kPathClassLoader, info.type); + DCHECK_EQ(info.classpath.size(), info.checksums.size()); + + *out_classpath = info.classpath; + *out_checksums = info.checksums; + return true; +} +} // namespace art + diff --git a/runtime/class_loader_context.h b/runtime/class_loader_context.h new file mode 100644 index 0000000000..9727a3b0dd --- /dev/null +++ b/runtime/class_loader_context.h @@ -0,0 +1,167 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_RUNTIME_CLASS_LOADER_CONTEXT_H_ +#define ART_RUNTIME_CLASS_LOADER_CONTEXT_H_ + +#include <string> +#include <vector> + +#include "arch/instruction_set.h" +#include "base/dchecked_vector.h" +#include "jni.h" + +namespace art { + +class DexFile; +class OatFile; + +// Utility class which holds the class loader context used during compilation/verification. +class ClassLoaderContext { + public: + // Creates an empty context (with no class loaders). + ClassLoaderContext(); + + // Opens requested class path files and appends them to ClassLoaderInfo::opened_dex_files. + // If the dex files have been stripped, the method opens them from their oat files which are added + // to ClassLoaderInfo::opened_oat_files. The 'classpath_dir' argument specifies the directory to + // use for the relative class paths. + // Returns true if all dex files where successfully opened. + // It may be called only once per ClassLoaderContext. The second call will abort. + // + // Note that a "false" return could mean that either an apk/jar contained no dex files or + // that we hit a I/O or checksum mismatch error. + // TODO(calin): Currently there's no easy way to tell the difference. + // + // TODO(calin): we're forced to complicate the flow in this class with a different + // OpenDexFiles step because the current dex2oat flow requires the dex files be opened before + // the class loader is created. Consider reworking the dex2oat part. + bool OpenDexFiles(InstructionSet isa, const std::string& classpath_dir); + + // Remove the specified compilation sources from all classpaths present in this context. + // Should only be called before the first call to OpenDexFiles(). + bool RemoveLocationsFromClassPaths(const dchecked_vector<std::string>& compilation_sources); + + // Creates the entire class loader hierarchy according to the current context. + // The compilation sources are appended to the classpath of the top class loader + // (i.e the class loader whose parent is the BootClassLoader). + // Should only be called if OpenDexFiles() returned true. + // If the context is empty, this method only creates a single PathClassLoader with the + // given compilation_sources. + jobject CreateClassLoader(const std::vector<const DexFile*>& compilation_sources) const; + + // Encodes the context as a string suitable to be added in oat files. + // (so that it can be read and verified at runtime against the actual class + // loader hierarchy). + // Should only be called if OpenDexFiles() returned true. + // E.g. if the context is PCL[a.dex:b.dex] this will return "a.dex*a_checksum*b.dex*a_checksum". + std::string EncodeContextForOatFile(const std::string& base_dir) const; + + // Flattens the opened dex files into the given vector. + // Should only be called if OpenDexFiles() returned true. + std::vector<const DexFile*> FlattenOpenedDexFiles() const; + + // Creates the class loader context from the given string. + // The format: ClassLoaderType1[ClasspathElem1:ClasspathElem2...];ClassLoaderType2[...]... + // ClassLoaderType is either "PCL" (PathClassLoader) or "DLC" (DelegateLastClassLoader). + // ClasspathElem is the path of dex/jar/apk file. + // Note that we allowed class loaders with an empty class path in order to support a custom + // class loader for the source dex files. + static std::unique_ptr<ClassLoaderContext> Create(const std::string& spec); + + // Decodes the class loader context stored in the oat file with EncodeContextForOatFile. + // Returns true if the format matches, or false otherwise. If the return is true, the out + // arguments will contain the classpath dex files, their checksums and whether or not the + // context is a special shared library. + // The method asserts that the context is made out of only one PathClassLoader. + static bool DecodePathClassLoaderContextFromOatFileKey( + const std::string& context_spec, + std::vector<std::string>* out_classpath, + std::vector<uint32_t>* out_checksums, + bool* out_is_special_shared_library); + + private: + enum ClassLoaderType { + kInvalidClassLoader = 0, + kPathClassLoader = 1, + kDelegateLastClassLoader = 2 + }; + + struct ClassLoaderInfo { + // The type of this class loader. + ClassLoaderType type; + // The list of class path elements that this loader loads. + // Note that this list may contain relative paths. + std::vector<std::string> classpath; + // The list of class path elements checksums. + // May be empty if the checksums are not given when the context is created. + std::vector<uint32_t> checksums; + // After OpenDexFiles is called this holds the opened dex files. + std::vector<std::unique_ptr<const DexFile>> opened_dex_files; + // After OpenDexFiles, in case some of the dex files were opened from their oat files + // this holds the list of opened oat files. + std::vector<std::unique_ptr<OatFile>> opened_oat_files; + + explicit ClassLoaderInfo(ClassLoaderType cl_type) : type(cl_type) {} + }; + + // Reads the class loader spec in place and returns true if the spec is valid and the + // compilation context was constructed. + bool Parse(const std::string& spec, bool parse_checksums = false); + + // Attempts to parse a single class loader spec for the given class_loader_type. + // If successful the class loader spec will be added to the chain. + // Returns whether or not the operation was successful. + bool ParseClassLoaderSpec(const std::string& class_loader_spec, + ClassLoaderType class_loader_type, + bool parse_checksums = false); + + // Extracts the class loader type from the given spec. + // Return ClassLoaderContext::kInvalidClassLoader if the class loader type is not + // recognized. + static ClassLoaderType ExtractClassLoaderType(const std::string& class_loader_spec); + + // Returns the string representation of the class loader type. + // The returned format can be used when parsing a context spec. + static const char* GetClassLoaderTypeName(ClassLoaderType type); + + // CHECKs that the dex files were opened (OpenDexFiles was called). Aborts if not. + void CheckDexFilesOpened(const std::string& calling_method) const; + + // The class loader chain represented as a vector. + // The parent of class_loader_chain_[i] is class_loader_chain_[i++]. + // The parent of the last element is assumed to be the boot class loader. + std::vector<ClassLoaderInfo> class_loader_chain_; + + // Whether or not the class loader context should be ignored at runtime when loading the oat + // files. When true, dex2oat will use OatFile::kSpecialSharedLibrary as the classpath key in + // the oat file. + // TODO(calin): Can we get rid of this and cover all relevant use cases? + // (e.g. packages using prebuild system packages as shared libraries b/36480683) + bool special_shared_library_; + + // Whether or not OpenDexFiles() was called. + bool dex_files_open_attempted_; + // The result of the last OpenDexFiles() operation. + bool dex_files_open_result_; + + friend class ClassLoaderContextTest; + + DISALLOW_COPY_AND_ASSIGN(ClassLoaderContext); +}; + +} // namespace art +#endif // ART_RUNTIME_CLASS_LOADER_CONTEXT_H_ diff --git a/runtime/class_loader_context_test.cc b/runtime/class_loader_context_test.cc new file mode 100644 index 0000000000..03eb0e42a0 --- /dev/null +++ b/runtime/class_loader_context_test.cc @@ -0,0 +1,337 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <gtest/gtest.h> + + +#include "class_loader_context.h" +#include "common_runtime_test.h" + +#include "base/dchecked_vector.h" +#include "base/stl_util.h" +#include "class_linker.h" +#include "dex_file.h" +#include "handle_scope-inl.h" +#include "mirror/class.h" +#include "mirror/class_loader.h" +#include "mirror/object-inl.h" +#include "oat_file_assistant.h" +#include "runtime.h" +#include "scoped_thread_state_change-inl.h" +#include "thread.h" +#include "well_known_classes.h" + +namespace art { + +class ClassLoaderContextTest : public CommonRuntimeTest { + public: + void VerifyContextSize(ClassLoaderContext* context, size_t expected_size) { + ASSERT_TRUE(context != nullptr); + ASSERT_EQ(expected_size, context->class_loader_chain_.size()); + } + + void VerifyClassLoaderPCL(ClassLoaderContext* context, + size_t index, + std::string classpath) { + VerifyClassLoaderInfo( + context, index, ClassLoaderContext::kPathClassLoader, classpath); + } + + void VerifyClassLoaderDLC(ClassLoaderContext* context, + size_t index, + std::string classpath) { + VerifyClassLoaderInfo( + context, index, ClassLoaderContext::kDelegateLastClassLoader, classpath); + } + + void VerifyOpenDexFiles( + ClassLoaderContext* context, + size_t index, + std::vector<std::vector<std::unique_ptr<const DexFile>>*>& all_dex_files) { + ASSERT_TRUE(context != nullptr); + ASSERT_TRUE(context->dex_files_open_attempted_); + ASSERT_TRUE(context->dex_files_open_result_); + ClassLoaderContext::ClassLoaderInfo& info = context->class_loader_chain_[index]; + ASSERT_EQ(all_dex_files.size(), info.classpath.size()); + size_t cur_open_dex_index = 0; + for (size_t k = 0; k < all_dex_files.size(); k++) { + std::vector<std::unique_ptr<const DexFile>>& dex_files_for_cp_elem = *(all_dex_files[k]); + for (size_t i = 0; i < dex_files_for_cp_elem.size(); i++) { + ASSERT_LT(cur_open_dex_index, info.opened_dex_files.size()); + + std::unique_ptr<const DexFile>& opened_dex_file = + info.opened_dex_files[cur_open_dex_index++]; + std::unique_ptr<const DexFile>& expected_dex_file = dex_files_for_cp_elem[i]; + + ASSERT_EQ(expected_dex_file->GetLocation(), opened_dex_file->GetLocation()); + ASSERT_EQ(expected_dex_file->GetLocationChecksum(), opened_dex_file->GetLocationChecksum()); + ASSERT_EQ(info.classpath[k], opened_dex_file->GetBaseLocation()); + } + } + } + + private: + void VerifyClassLoaderInfo(ClassLoaderContext* context, + size_t index, + ClassLoaderContext::ClassLoaderType type, + std::string classpath) { + ASSERT_TRUE(context != nullptr); + ASSERT_GT(context->class_loader_chain_.size(), index); + ClassLoaderContext::ClassLoaderInfo& info = context->class_loader_chain_[index]; + ASSERT_EQ(type, info.type); + std::vector<std::string> expected_classpath; + Split(classpath, ':', &expected_classpath); + ASSERT_EQ(expected_classpath, info.classpath); + } +}; + +TEST_F(ClassLoaderContextTest, ParseValidContextPCL) { + std::unique_ptr<ClassLoaderContext> context = + ClassLoaderContext::Create("PCL[a.dex]"); + VerifyContextSize(context.get(), 1); + VerifyClassLoaderPCL(context.get(), 0, "a.dex"); +} + +TEST_F(ClassLoaderContextTest, ParseValidContextDLC) { + std::unique_ptr<ClassLoaderContext> context = + ClassLoaderContext::Create("DLC[a.dex]"); + VerifyContextSize(context.get(), 1); + VerifyClassLoaderDLC(context.get(), 0, "a.dex"); +} + +TEST_F(ClassLoaderContextTest, ParseValidContextChain) { + std::unique_ptr<ClassLoaderContext> context = + ClassLoaderContext::Create("PCL[a.dex:b.dex];DLC[c.dex:d.dex];PCL[e.dex]"); + VerifyContextSize(context.get(), 3); + VerifyClassLoaderPCL(context.get(), 0, "a.dex:b.dex"); + VerifyClassLoaderDLC(context.get(), 1, "c.dex:d.dex"); + VerifyClassLoaderPCL(context.get(), 2, "e.dex"); +} + +TEST_F(ClassLoaderContextTest, ParseValidEmptyContextDLC) { + std::unique_ptr<ClassLoaderContext> context = + ClassLoaderContext::Create("DLC[]"); + VerifyContextSize(context.get(), 1); + VerifyClassLoaderDLC(context.get(), 0, ""); +} + +TEST_F(ClassLoaderContextTest, ParseValidContextSpecialSymbol) { + std::unique_ptr<ClassLoaderContext> context = + ClassLoaderContext::Create(OatFile::kSpecialSharedLibrary); + VerifyContextSize(context.get(), 0); +} + +TEST_F(ClassLoaderContextTest, ParseInvalidValidContexts) { + ASSERT_TRUE(nullptr == ClassLoaderContext::Create("ABC[a.dex]")); + ASSERT_TRUE(nullptr == ClassLoaderContext::Create("PCL")); + ASSERT_TRUE(nullptr == ClassLoaderContext::Create("PCL[a.dex")); + ASSERT_TRUE(nullptr == ClassLoaderContext::Create("PCLa.dex]")); + ASSERT_TRUE(nullptr == ClassLoaderContext::Create("PCL{a.dex}")); + ASSERT_TRUE(nullptr == ClassLoaderContext::Create("PCL[a.dex];DLC[b.dex")); +} + +TEST_F(ClassLoaderContextTest, OpenInvalidDexFiles) { + std::unique_ptr<ClassLoaderContext> context = + ClassLoaderContext::Create("PCL[does_not_exist.dex]"); + VerifyContextSize(context.get(), 1); + ASSERT_FALSE(context->OpenDexFiles(InstructionSet::kArm, ".")); +} + +TEST_F(ClassLoaderContextTest, OpenValidDexFiles) { + std::string multidex_name = GetTestDexFileName("MultiDex"); + std::vector<std::unique_ptr<const DexFile>> multidex_files = OpenTestDexFiles("MultiDex"); + std::string myclass_dex_name = GetTestDexFileName("MyClass"); + std::vector<std::unique_ptr<const DexFile>> myclass_dex_files = OpenTestDexFiles("MyClass"); + std::string dex_name = GetTestDexFileName("Main"); + std::vector<std::unique_ptr<const DexFile>> dex_files = OpenTestDexFiles("Main"); + + + std::unique_ptr<ClassLoaderContext> context = + ClassLoaderContext::Create( + "PCL[" + multidex_name + ":" + myclass_dex_name + "];" + + "DLC[" + dex_name + "]"); + + ASSERT_TRUE(context->OpenDexFiles(InstructionSet::kArm, /*classpath_dir*/ "")); + + VerifyContextSize(context.get(), 2); + std::vector<std::vector<std::unique_ptr<const DexFile>>*> all_dex_files0; + all_dex_files0.push_back(&multidex_files); + all_dex_files0.push_back(&myclass_dex_files); + std::vector<std::vector<std::unique_ptr<const DexFile>>*> all_dex_files1; + all_dex_files1.push_back(&dex_files); + + VerifyOpenDexFiles(context.get(), 0, all_dex_files0); + VerifyOpenDexFiles(context.get(), 1, all_dex_files1); +} + +TEST_F(ClassLoaderContextTest, OpenInvalidDexFilesMix) { + std::string dex_name = GetTestDexFileName("Main"); + std::unique_ptr<ClassLoaderContext> context = + ClassLoaderContext::Create("PCL[does_not_exist.dex];DLC[" + dex_name + "]"); + ASSERT_FALSE(context->OpenDexFiles(InstructionSet::kArm, "")); +} + +TEST_F(ClassLoaderContextTest, CreateClassLoader) { + std::string dex_name = GetTestDexFileName("Main"); + std::unique_ptr<ClassLoaderContext> context = + ClassLoaderContext::Create("PCL[" + dex_name + "]"); + ASSERT_TRUE(context->OpenDexFiles(InstructionSet::kArm, "")); + + std::vector<std::unique_ptr<const DexFile>> classpath_dex = OpenTestDexFiles("Main"); + std::vector<std::unique_ptr<const DexFile>> compilation_sources = OpenTestDexFiles("MultiDex"); + + std::vector<const DexFile*> compilation_sources_raw = + MakeNonOwningPointerVector(compilation_sources); + jobject jclass_loader = context->CreateClassLoader(compilation_sources_raw); + ASSERT_TRUE(jclass_loader != nullptr); + + ScopedObjectAccess soa(Thread::Current()); + + StackHandleScope<2> hs(soa.Self()); + Handle<mirror::ClassLoader> class_loader = hs.NewHandle( + soa.Decode<mirror::ClassLoader>(jclass_loader)); + + ASSERT_TRUE(class_loader->GetClass() == + soa.Decode<mirror::Class>(WellKnownClasses::dalvik_system_PathClassLoader)); + ASSERT_TRUE(class_loader->GetParent()->GetClass() == + soa.Decode<mirror::Class>(WellKnownClasses::java_lang_BootClassLoader)); + + + std::vector<const DexFile*> class_loader_dex_files = GetDexFiles(jclass_loader); + ASSERT_EQ(classpath_dex.size() + compilation_sources.size(), class_loader_dex_files.size()); + + // The classpath dex files must come first. + for (size_t i = 0; i < classpath_dex.size(); i++) { + ASSERT_EQ(classpath_dex[i]->GetLocation(), + class_loader_dex_files[i]->GetLocation()); + ASSERT_EQ(classpath_dex[i]->GetLocationChecksum(), + class_loader_dex_files[i]->GetLocationChecksum()); + } + + // The compilation dex files must come second. + for (size_t i = 0, k = classpath_dex.size(); i < compilation_sources.size(); i++, k++) { + ASSERT_EQ(compilation_sources[i]->GetLocation(), + class_loader_dex_files[k]->GetLocation()); + ASSERT_EQ(compilation_sources[i]->GetLocationChecksum(), + class_loader_dex_files[k]->GetLocationChecksum()); + } +} + +TEST_F(ClassLoaderContextTest, CreateClassLoaderWithEmptyContext) { + std::unique_ptr<ClassLoaderContext> context = + ClassLoaderContext::Create(""); + ASSERT_TRUE(context->OpenDexFiles(InstructionSet::kArm, "")); + + std::vector<std::unique_ptr<const DexFile>> compilation_sources = OpenTestDexFiles("MultiDex"); + + std::vector<const DexFile*> compilation_sources_raw = + MakeNonOwningPointerVector(compilation_sources); + jobject jclass_loader = context->CreateClassLoader(compilation_sources_raw); + ASSERT_TRUE(jclass_loader != nullptr); + + ScopedObjectAccess soa(Thread::Current()); + + StackHandleScope<2> hs(soa.Self()); + Handle<mirror::ClassLoader> class_loader = hs.NewHandle( + soa.Decode<mirror::ClassLoader>(jclass_loader)); + + ASSERT_TRUE(class_loader->GetClass() == + soa.Decode<mirror::Class>(WellKnownClasses::dalvik_system_PathClassLoader)); + ASSERT_TRUE(class_loader->GetParent()->GetClass() == + soa.Decode<mirror::Class>(WellKnownClasses::java_lang_BootClassLoader)); + + + std::vector<const DexFile*> class_loader_dex_files = GetDexFiles(jclass_loader); + + // The compilation sources should be the only files present in the class loader + ASSERT_EQ(compilation_sources.size(), class_loader_dex_files.size()); + for (size_t i = 0; i < compilation_sources.size(); i++) { + ASSERT_EQ(compilation_sources[i]->GetLocation(), + class_loader_dex_files[i]->GetLocation()); + ASSERT_EQ(compilation_sources[i]->GetLocationChecksum(), + class_loader_dex_files[i]->GetLocationChecksum()); + } +} + +TEST_F(ClassLoaderContextTest, RemoveSourceLocations) { + std::unique_ptr<ClassLoaderContext> context = + ClassLoaderContext::Create("PCL[a.dex]"); + dchecked_vector<std::string> classpath_dex; + classpath_dex.push_back("a.dex"); + dchecked_vector<std::string> compilation_sources; + compilation_sources.push_back("src.dex"); + + // Nothing should be removed. + ASSERT_FALSE(context->RemoveLocationsFromClassPaths(compilation_sources)); + VerifyClassLoaderPCL(context.get(), 0, "a.dex"); + // Classes should be removed. + ASSERT_TRUE(context->RemoveLocationsFromClassPaths(classpath_dex)); + VerifyClassLoaderPCL(context.get(), 0, ""); +} + +TEST_F(ClassLoaderContextTest, EncodeInOatFile) { + std::string dex1_name = GetTestDexFileName("Main"); + std::string dex2_name = GetTestDexFileName("MyClass"); + std::unique_ptr<ClassLoaderContext> context = + ClassLoaderContext::Create("PCL[" + dex1_name + ":" + dex2_name + "]"); + ASSERT_TRUE(context->OpenDexFiles(InstructionSet::kArm, "")); + + std::vector<std::unique_ptr<const DexFile>> dex1 = OpenTestDexFiles("Main"); + std::vector<std::unique_ptr<const DexFile>> dex2 = OpenTestDexFiles("MyClass"); + std::string encoding = context->EncodeContextForOatFile(""); + std::string expected_encoding = "PCL[" + + dex1[0]->GetLocation() + "*" + std::to_string(dex1[0]->GetLocationChecksum()) + ":" + + dex2[0]->GetLocation() + "*" + std::to_string(dex2[0]->GetLocationChecksum()) + "]"; + ASSERT_EQ(expected_encoding, context->EncodeContextForOatFile("")); +} + +TEST_F(ClassLoaderContextTest, DecodeOatFileKey) { + std::string oat_file_encoding = "PCL[a.dex*123:b.dex*456]"; + std::vector<std::string> classpath; + std::vector<uint32_t> checksums; + bool is_special_shared_library; + bool result = ClassLoaderContext::DecodePathClassLoaderContextFromOatFileKey( + oat_file_encoding, + &classpath, + &checksums, + &is_special_shared_library); + ASSERT_TRUE(result); + ASSERT_FALSE(is_special_shared_library); + ASSERT_EQ(2u, classpath.size()); + ASSERT_EQ(2u, checksums.size()); + ASSERT_EQ("a.dex", classpath[0]); + ASSERT_EQ(123u, checksums[0]); + ASSERT_EQ("b.dex", classpath[1]); + ASSERT_EQ(456u, checksums[1]); +} + +TEST_F(ClassLoaderContextTest, DecodeOatFileKeySpecialLibrary) { + std::string oat_file_encoding = "&"; + std::vector<std::string> classpath; + std::vector<uint32_t> checksums; + bool is_special_shared_library; + bool result = ClassLoaderContext::DecodePathClassLoaderContextFromOatFileKey( + oat_file_encoding, + &classpath, + &checksums, + &is_special_shared_library); + ASSERT_TRUE(result); + ASSERT_TRUE(is_special_shared_library); + ASSERT_TRUE(classpath.empty()); + ASSERT_TRUE(checksums.empty()); +} + +} // namespace art diff --git a/runtime/common_runtime_test.h b/runtime/common_runtime_test.h index 5893573bdd..fcf3a31fbc 100644 --- a/runtime/common_runtime_test.h +++ b/runtime/common_runtime_test.h @@ -247,6 +247,12 @@ class CheckJniAbortCatcher { return; \ } +#define TEST_DISABLED_WITHOUT_BAKER_READ_BARRIERS() \ + if (!kEmitCompilerReadBarrier || !kUseBakerReadBarrier) { \ + printf("WARNING: TEST DISABLED FOR GC WITHOUT BAKER READ BARRIER\n"); \ + return; \ + } + #define TEST_DISABLED_FOR_NON_STATIC_HOST_BUILDS() \ if (!kHostStaticBuildEnabled) { \ printf("WARNING: TEST DISABLED FOR NON-STATIC HOST BUILDS\n"); \ diff --git a/runtime/debugger.cc b/runtime/debugger.cc index 634930f67a..0f15e8b934 100644 --- a/runtime/debugger.cc +++ b/runtime/debugger.cc @@ -2480,7 +2480,8 @@ void Dbg::ResumeThread(JDWP::ObjectId thread_id) { needs_resume = thread->GetDebugSuspendCount() > 0; } if (needs_resume) { - Runtime::Current()->GetThreadList()->Resume(thread, SuspendReason::kForDebugger); + bool resumed = Runtime::Current()->GetThreadList()->Resume(thread, SuspendReason::kForDebugger); + DCHECK(resumed); } } @@ -3721,7 +3722,9 @@ class ScopedDebuggerThreadSuspension { ~ScopedDebuggerThreadSuspension() { if (other_suspend_) { - Runtime::Current()->GetThreadList()->Resume(thread_, SuspendReason::kForDebugger); + bool resumed = Runtime::Current()->GetThreadList()->Resume(thread_, + SuspendReason::kForDebugger); + DCHECK(resumed); } } @@ -4043,7 +4046,8 @@ JDWP::JdwpError Dbg::PrepareInvokeMethod(uint32_t request_id, JDWP::ObjectId thr thread_list->UndoDebuggerSuspensions(); } else { VLOG(jdwp) << " Resuming event thread only"; - thread_list->Resume(targetThread, SuspendReason::kForDebugger); + bool resumed = thread_list->Resume(targetThread, SuspendReason::kForDebugger); + DCHECK(resumed); } return JDWP::ERR_NONE; diff --git a/runtime/dex_to_dex_decompiler.cc b/runtime/dex_to_dex_decompiler.cc index c15c9ec448..908405b7ca 100644 --- a/runtime/dex_to_dex_decompiler.cc +++ b/runtime/dex_to_dex_decompiler.cc @@ -18,9 +18,10 @@ #include "base/logging.h" #include "base/mutex.h" +#include "bytecode_utils.h" #include "dex_file-inl.h" #include "dex_instruction-inl.h" -#include "bytecode_utils.h" +#include "quicken_info.h" namespace art { namespace optimizer { @@ -31,27 +32,21 @@ class DexDecompiler { const ArrayRef<const uint8_t>& quickened_info, bool decompile_return_instruction) : code_item_(code_item), - quickened_info_ptr_(quickened_info.data()), - quickened_info_start_(quickened_info.data()), - quickened_info_end_(quickened_info.data() + quickened_info.size()), + quicken_info_(quickened_info.data()), + quicken_info_number_of_indices_(QuickenInfoTable::NumberOfIndices(quickened_info.size())), decompile_return_instruction_(decompile_return_instruction) {} bool Decompile(); private: - void DecompileInstanceFieldAccess(Instruction* inst, - uint32_t dex_pc, - Instruction::Code new_opcode) { - uint16_t index = GetIndexAt(dex_pc); + void DecompileInstanceFieldAccess(Instruction* inst, Instruction::Code new_opcode) { + uint16_t index = NextIndex(); inst->SetOpcode(new_opcode); inst->SetVRegC_22c(index); } - void DecompileInvokeVirtual(Instruction* inst, - uint32_t dex_pc, - Instruction::Code new_opcode, - bool is_range) { - uint16_t index = GetIndexAt(dex_pc); + void DecompileInvokeVirtual(Instruction* inst, Instruction::Code new_opcode, bool is_range) { + const uint16_t index = NextIndex(); inst->SetOpcode(new_opcode); if (is_range) { inst->SetVRegB_3rc(index); @@ -60,40 +55,32 @@ class DexDecompiler { } } - void DecompileNop(Instruction* inst, uint32_t dex_pc) { - if (quickened_info_ptr_ == quickened_info_end_) { - return; - } - const uint8_t* temporary_pointer = quickened_info_ptr_; - uint32_t quickened_pc = DecodeUnsignedLeb128(&temporary_pointer); - if (quickened_pc != dex_pc) { + void DecompileNop(Instruction* inst) { + const uint16_t reference_index = NextIndex(); + if (reference_index == DexFile::kDexNoIndex16) { + // This means it was a normal nop and not a check-cast. return; } - uint16_t reference_index = GetIndexAt(dex_pc); - uint16_t type_index = GetIndexAt(dex_pc); + const uint16_t type_index = NextIndex(); inst->SetOpcode(Instruction::CHECK_CAST); inst->SetVRegA_21c(reference_index); inst->SetVRegB_21c(type_index); } - uint16_t GetIndexAt(uint32_t dex_pc) { - // Note that as a side effect, DecodeUnsignedLeb128 update the given pointer - // to the new position in the buffer. - DCHECK_LT(quickened_info_ptr_, quickened_info_end_); - uint32_t quickened_pc = DecodeUnsignedLeb128(&quickened_info_ptr_); - DCHECK_LT(quickened_info_ptr_, quickened_info_end_); - uint16_t index = DecodeUnsignedLeb128(&quickened_info_ptr_); - DCHECK_LE(quickened_info_ptr_, quickened_info_end_); - DCHECK_EQ(quickened_pc, dex_pc); - return index; + uint16_t NextIndex() { + DCHECK_LT(quicken_index_, quicken_info_number_of_indices_); + const uint16_t ret = quicken_info_.GetData(quicken_index_); + quicken_index_++; + return ret; } const DexFile::CodeItem& code_item_; - const uint8_t* quickened_info_ptr_; - const uint8_t* const quickened_info_start_; - const uint8_t* const quickened_info_end_; + const QuickenInfoTable quicken_info_; + const size_t quicken_info_number_of_indices_; const bool decompile_return_instruction_; + size_t quicken_index_ = 0u; + DISALLOW_COPY_AND_ASSIGN(DexDecompiler); }; @@ -103,7 +90,6 @@ bool DexDecompiler::Decompile() { // unquickening is a rare need and not performance sensitive, it is not worth the // added storage to also add the RETURN_VOID quickening in the quickened data. for (CodeItemIterator it(code_item_); !it.Done(); it.Advance()) { - uint32_t dex_pc = it.CurrentDexPc(); Instruction* inst = const_cast<Instruction*>(&it.CurrentInstruction()); switch (inst->Opcode()) { @@ -114,71 +100,76 @@ bool DexDecompiler::Decompile() { break; case Instruction::NOP: - DecompileNop(inst, dex_pc); + if (quicken_info_number_of_indices_ > 0) { + // Only try to decompile NOP if there are more than 0 indices. Not having + // any index happens when we unquicken a code item that only has + // RETURN_VOID_NO_BARRIER as quickened instruction. + DecompileNop(inst); + } break; case Instruction::IGET_QUICK: - DecompileInstanceFieldAccess(inst, dex_pc, Instruction::IGET); + DecompileInstanceFieldAccess(inst, Instruction::IGET); break; case Instruction::IGET_WIDE_QUICK: - DecompileInstanceFieldAccess(inst, dex_pc, Instruction::IGET_WIDE); + DecompileInstanceFieldAccess(inst, Instruction::IGET_WIDE); break; case Instruction::IGET_OBJECT_QUICK: - DecompileInstanceFieldAccess(inst, dex_pc, Instruction::IGET_OBJECT); + DecompileInstanceFieldAccess(inst, Instruction::IGET_OBJECT); break; case Instruction::IGET_BOOLEAN_QUICK: - DecompileInstanceFieldAccess(inst, dex_pc, Instruction::IGET_BOOLEAN); + DecompileInstanceFieldAccess(inst, Instruction::IGET_BOOLEAN); break; case Instruction::IGET_BYTE_QUICK: - DecompileInstanceFieldAccess(inst, dex_pc, Instruction::IGET_BYTE); + DecompileInstanceFieldAccess(inst, Instruction::IGET_BYTE); break; case Instruction::IGET_CHAR_QUICK: - DecompileInstanceFieldAccess(inst, dex_pc, Instruction::IGET_CHAR); + DecompileInstanceFieldAccess(inst, Instruction::IGET_CHAR); break; case Instruction::IGET_SHORT_QUICK: - DecompileInstanceFieldAccess(inst, dex_pc, Instruction::IGET_SHORT); + DecompileInstanceFieldAccess(inst, Instruction::IGET_SHORT); break; case Instruction::IPUT_QUICK: - DecompileInstanceFieldAccess(inst, dex_pc, Instruction::IPUT); + DecompileInstanceFieldAccess(inst, Instruction::IPUT); break; case Instruction::IPUT_BOOLEAN_QUICK: - DecompileInstanceFieldAccess(inst, dex_pc, Instruction::IPUT_BOOLEAN); + DecompileInstanceFieldAccess(inst, Instruction::IPUT_BOOLEAN); break; case Instruction::IPUT_BYTE_QUICK: - DecompileInstanceFieldAccess(inst, dex_pc, Instruction::IPUT_BYTE); + DecompileInstanceFieldAccess(inst, Instruction::IPUT_BYTE); break; case Instruction::IPUT_CHAR_QUICK: - DecompileInstanceFieldAccess(inst, dex_pc, Instruction::IPUT_CHAR); + DecompileInstanceFieldAccess(inst, Instruction::IPUT_CHAR); break; case Instruction::IPUT_SHORT_QUICK: - DecompileInstanceFieldAccess(inst, dex_pc, Instruction::IPUT_SHORT); + DecompileInstanceFieldAccess(inst, Instruction::IPUT_SHORT); break; case Instruction::IPUT_WIDE_QUICK: - DecompileInstanceFieldAccess(inst, dex_pc, Instruction::IPUT_WIDE); + DecompileInstanceFieldAccess(inst, Instruction::IPUT_WIDE); break; case Instruction::IPUT_OBJECT_QUICK: - DecompileInstanceFieldAccess(inst, dex_pc, Instruction::IPUT_OBJECT); + DecompileInstanceFieldAccess(inst, Instruction::IPUT_OBJECT); break; case Instruction::INVOKE_VIRTUAL_QUICK: - DecompileInvokeVirtual(inst, dex_pc, Instruction::INVOKE_VIRTUAL, false); + DecompileInvokeVirtual(inst, Instruction::INVOKE_VIRTUAL, false); break; case Instruction::INVOKE_VIRTUAL_RANGE_QUICK: - DecompileInvokeVirtual(inst, dex_pc, Instruction::INVOKE_VIRTUAL_RANGE, true); + DecompileInvokeVirtual(inst, Instruction::INVOKE_VIRTUAL_RANGE, true); break; default: @@ -186,14 +177,14 @@ bool DexDecompiler::Decompile() { } } - if (quickened_info_ptr_ != quickened_info_end_) { - if (quickened_info_start_ == quickened_info_ptr_) { + if (quicken_index_ != quicken_info_number_of_indices_) { + if (quicken_index_ == 0) { LOG(WARNING) << "Failed to use any value in quickening info," << " potentially due to duplicate methods."; } else { LOG(FATAL) << "Failed to use all values in quickening info." - << " Actual: " << std::hex << reinterpret_cast<uintptr_t>(quickened_info_ptr_) - << " Expected: " << reinterpret_cast<uintptr_t>(quickened_info_end_); + << " Actual: " << std::hex << quicken_index_ + << " Expected: " << quicken_info_number_of_indices_; return false; } } diff --git a/runtime/gc/collector/concurrent_copying.cc b/runtime/gc/collector/concurrent_copying.cc index 458e830eda..8d3c62f3d0 100644 --- a/runtime/gc/collector/concurrent_copying.cc +++ b/runtime/gc/collector/concurrent_copying.cc @@ -166,7 +166,7 @@ void ConcurrentCopying::RunPhases() { } if (kUseBakerReadBarrier && kGrayDirtyImmuneObjects) { // Switch to read barrier mark entrypoints before we gray the objects. This is required in case - // a mutator sees a gray bit and dispatches on the entrpoint. (b/37876887). + // a mutator sees a gray bit and dispatches on the entrypoint. (b/37876887). ActivateReadBarrierEntrypoints(); // Gray dirty immune objects concurrently to reduce GC pause times. We re-process gray cards in // the pause. diff --git a/runtime/gc/space/image_space.cc b/runtime/gc/space/image_space.cc index 1bf92851af..3ae382e36f 100644 --- a/runtime/gc/space/image_space.cc +++ b/runtime/gc/space/image_space.cc @@ -1008,6 +1008,20 @@ class ImageSpaceLoader { } } + if (obj->IsClass()) { + mirror::Class* klass = obj->AsClass<kVerifyNone, kWithoutReadBarrier>(); + // Fixup super class before visiting instance fields which require + // information from their super class to calculate offsets. + mirror::Class* super_class = klass->GetSuperClass<kVerifyNone, kWithoutReadBarrier>(); + if (super_class != nullptr) { + mirror::Class* new_super_class = down_cast<mirror::Class*>(ForwardObject(super_class)); + if (new_super_class != super_class && IsInAppImage(new_super_class)) { + // Recursively fix all dependencies. + operator()(new_super_class); + } + } + } + obj->VisitReferences</*visit native roots*/false, kVerifyNone, kWithoutReadBarrier>( *this, *this); diff --git a/runtime/gc/space/region_space.cc b/runtime/gc/space/region_space.cc index b8f1e8fc71..fe3c1c022c 100644 --- a/runtime/gc/space/region_space.cc +++ b/runtime/gc/space/region_space.cc @@ -290,6 +290,7 @@ void RegionSpace::ClearFromSpace(uint64_t* cleared_bytes, uint64_t* cleared_obje clear_region(r); } else if (r->IsInUnevacFromSpace()) { if (r->LiveBytes() == 0) { + DCHECK(!r->IsLargeTail()); // Special case for 0 live bytes, this means all of the objects in the region are dead and // we can clear it. This is important for large objects since we must not visit dead ones in // RegionSpace::Walk because they may contain dangling references to invalid objects. @@ -312,28 +313,29 @@ void RegionSpace::ClearFromSpace(uint64_t* cleared_bytes, uint64_t* cleared_obje reinterpret_cast<mirror::Object*>(r->Begin() + free_regions * kRegionSize)); continue; } - size_t full_count = 0; - while (r->IsInUnevacFromSpace()) { - Region* const cur = ®ions_[i + full_count]; - if (i + full_count >= num_regions_ || - cur->LiveBytes() != static_cast<size_t>(cur->Top() - cur->Begin())) { - break; - } - DCHECK(cur->IsInUnevacFromSpace()); - if (full_count != 0) { + r->SetUnevacFromSpaceAsToSpace(); + if (r->AllAllocatedBytesAreLive()) { + // Try to optimize the number of ClearRange calls by checking whether the next regions + // can also be cleared. + size_t regions_to_clear_bitmap = 1; + while (i + regions_to_clear_bitmap < num_regions_) { + Region* const cur = ®ions_[i + regions_to_clear_bitmap]; + if (!cur->AllAllocatedBytesAreLive()) { + DCHECK(!cur->IsLargeTail()); + break; + } + CHECK(cur->IsInUnevacFromSpace()); cur->SetUnevacFromSpaceAsToSpace(); + ++regions_to_clear_bitmap; } - ++full_count; - } - // Note that r is the full_count == 0 iteration since it is not handled by the loop. - r->SetUnevacFromSpaceAsToSpace(); - if (full_count >= 1) { + GetLiveBitmap()->ClearRange( reinterpret_cast<mirror::Object*>(r->Begin()), - reinterpret_cast<mirror::Object*>(r->Begin() + full_count * kRegionSize)); - // Skip over extra regions we cleared. + reinterpret_cast<mirror::Object*>(r->Begin() + regions_to_clear_bitmap * kRegionSize)); + // Skip over extra regions we cleared the bitmaps: we don't need to clear them, as they + // are unevac region sthat are live. // Subtract one for the for loop. - i += full_count - 1; + i += regions_to_clear_bitmap - 1; } } // Note r != last_checked_region if r->IsInUnevacFromSpace() was true above. diff --git a/runtime/gc/space/region_space.h b/runtime/gc/space/region_space.h index 7b16ffe23d..6412158a77 100644 --- a/runtime/gc/space/region_space.h +++ b/runtime/gc/space/region_space.h @@ -393,6 +393,10 @@ class RegionSpace FINAL : public ContinuousMemMapAllocSpace { DCHECK_LE(live_bytes_, BytesAllocated()); } + bool AllAllocatedBytesAreLive() const { + return LiveBytes() == static_cast<size_t>(Top() - Begin()); + } + size_t LiveBytes() const { return live_bytes_; } diff --git a/runtime/image.cc b/runtime/image.cc index 489a53b35c..ac36d7ca20 100644 --- a/runtime/image.cc +++ b/runtime/image.cc @@ -26,7 +26,7 @@ namespace art { const uint8_t ImageHeader::kImageMagic[] = { 'a', 'r', 't', '\n' }; -const uint8_t ImageHeader::kImageVersion[] = { '0', '4', '4', '\0' }; // Thread.interrupted +const uint8_t ImageHeader::kImageVersion[] = { '0', '4', '5', '\0' }; // Fix DexCache fields. ImageHeader::ImageHeader(uint32_t image_begin, uint32_t image_size, diff --git a/runtime/interpreter/interpreter_common.h b/runtime/interpreter/interpreter_common.h index 38edc7a9e7..74fec48342 100644 --- a/runtime/interpreter/interpreter_common.h +++ b/runtime/interpreter/interpreter_common.h @@ -241,7 +241,7 @@ static inline bool DoInvokeVirtualQuick(Thread* self, ShadowFrame& shadow_frame, } CHECK(receiver->GetClass()->ShouldHaveEmbeddedVTable()); ArtMethod* const called_method = receiver->GetClass()->GetEmbeddedVTableEntry( - vtable_idx, kRuntimePointerSize); + vtable_idx, Runtime::Current()->GetClassLinker()->GetImagePointerSize()); if (UNLIKELY(called_method == nullptr)) { CHECK(self->IsExceptionPending()); result->SetJ(0); diff --git a/runtime/interpreter/unstarted_runtime_test.cc b/runtime/interpreter/unstarted_runtime_test.cc index c314f3c35e..c2ef72460d 100644 --- a/runtime/interpreter/unstarted_runtime_test.cc +++ b/runtime/interpreter/unstarted_runtime_test.cc @@ -386,8 +386,9 @@ TEST_F(UnstartedRuntimeTest, StringInit) { Thread* self = Thread::Current(); ScopedObjectAccess soa(self); mirror::Class* klass = mirror::String::GetJavaLangString(); - ArtMethod* method = klass->FindDeclaredDirectMethod("<init>", "(Ljava/lang/String;)V", - kRuntimePointerSize); + ArtMethod* method = + klass->FindDeclaredDirectMethod("<init>", "(Ljava/lang/String;)V", + Runtime::Current()->GetClassLinker()->GetImagePointerSize()); // create instruction data for invoke-direct {v0, v1} of method with fake index uint16_t inst_data[3] = { 0x2070, 0x0000, 0x0010 }; @@ -1335,10 +1336,16 @@ TEST_F(UnstartedRuntimeTest, ConstructorNewInstance0) { ArtMethod* throw_cons = throw_class->FindDeclaredDirectMethod( "<init>", "(Ljava/lang/String;)V", class_linker->GetImagePointerSize()); ASSERT_TRUE(throw_cons != nullptr); - - Handle<mirror::Constructor> cons = hs.NewHandle( - mirror::Constructor::CreateFromArtMethod<kRuntimePointerSize, false>(self, throw_cons)); - ASSERT_TRUE(cons != nullptr); + Handle<mirror::Constructor> cons; + if (class_linker->GetImagePointerSize() == PointerSize::k64) { + cons = hs.NewHandle( + mirror::Constructor::CreateFromArtMethod<PointerSize::k64, false>(self, throw_cons)); + ASSERT_TRUE(cons != nullptr); + } else { + cons = hs.NewHandle( + mirror::Constructor::CreateFromArtMethod<PointerSize::k32, false>(self, throw_cons)); + ASSERT_TRUE(cons != nullptr); + } Handle<mirror::ObjectArray<mirror::Object>> args = hs.NewHandle( mirror::ObjectArray<mirror::Object>::Alloc( diff --git a/runtime/jit/profile_compilation_info.cc b/runtime/jit/profile_compilation_info.cc index 147173c5a3..a247b56587 100644 --- a/runtime/jit/profile_compilation_info.cc +++ b/runtime/jit/profile_compilation_info.cc @@ -336,7 +336,9 @@ bool ProfileCompilationInfo::Save(int fd) { methods_region_size + dex_data.bitmap_storage.size(); } - if (required_capacity > kProfileSizeErrorThresholdInBytes) { + // Allow large profiles for non target builds for the case where we are merging many profiles + // to generate a boot image profile. + if (kIsTargetBuild && required_capacity > kProfileSizeErrorThresholdInBytes) { LOG(ERROR) << "Profile data size exceeds " << std::to_string(kProfileSizeErrorThresholdInBytes) << " bytes. Profile will not be written to disk."; @@ -1030,8 +1032,9 @@ ProfileCompilationInfo::ProfileLoadSatus ProfileCompilationInfo::LoadInternal( if (status != kProfileLoadSuccess) { return status; } - - if (uncompressed_data_size > kProfileSizeErrorThresholdInBytes) { + // Allow large profiles for non target builds for the case where we are merging many profiles + // to generate a boot image profile. + if (kIsTargetBuild && uncompressed_data_size > kProfileSizeErrorThresholdInBytes) { LOG(ERROR) << "Profile data size exceeds " << std::to_string(kProfileSizeErrorThresholdInBytes) << " bytes"; diff --git a/runtime/jit/profile_saver.cc b/runtime/jit/profile_saver.cc index 10dddaefc8..61e5be34cb 100644 --- a/runtime/jit/profile_saver.cc +++ b/runtime/jit/profile_saver.cc @@ -29,6 +29,7 @@ #include "base/stl_util.h" #include "base/systrace.h" #include "base/time_utils.h" +#include "class_table-inl.h" #include "compiler_filter.h" #include "dex_reference_collection.h" #include "gc/collector_type.h" @@ -121,7 +122,7 @@ void ProfileSaver::Run() { } total_ms_of_sleep_ += options_.GetSaveResolvedClassesDelayMs(); } - FetchAndCacheResolvedClassesAndMethods(); + FetchAndCacheResolvedClassesAndMethods(/*startup*/ true); // Loop for the profiled methods. while (!ShuttingDown(self)) { @@ -210,24 +211,49 @@ void ProfileSaver::NotifyJitActivityInternal() { } } -using MethodReferenceCollection = DexReferenceCollection<uint16_t, ScopedArenaAllocatorAdapter>; -using TypeReferenceCollection = DexReferenceCollection<dex::TypeIndex, - ScopedArenaAllocatorAdapter>; +class ScopedDefaultPriority { + public: + explicit ScopedDefaultPriority(pthread_t thread) : thread_(thread) { + SetProfileSaverThreadPriority(thread_, GetDefaultThreadPriority()); + } -// Get resolved methods that have a profile info or more than kStartupMethodSamples samples. -// Excludes native methods and classes in the boot image. -class GetClassesAndMethodsVisitor : public ClassVisitor { + ~ScopedDefaultPriority() { + SetProfileSaverThreadPriority(thread_, kProfileSaverPthreadPriority); + } + + private: + const pthread_t thread_; +}; + +// GetClassLoadersVisitor takes a snapshot of the class loaders and stores them in the out +// class_loaders argument. Not affected by class unloading since there are no suspend points in +// the caller. +class GetClassLoadersVisitor : public ClassLoaderVisitor { public: - GetClassesAndMethodsVisitor(MethodReferenceCollection* hot_methods, - MethodReferenceCollection* sampled_methods, - TypeReferenceCollection* resolved_classes, - uint32_t hot_method_sample_threshold, - bool profile_boot_class_path) - : hot_methods_(hot_methods), - sampled_methods_(sampled_methods), - resolved_classes_(resolved_classes), - hot_method_sample_threshold_(hot_method_sample_threshold), - profile_boot_class_path_(profile_boot_class_path) {} + explicit GetClassLoadersVisitor(VariableSizedHandleScope* hs, + std::vector<Handle<mirror::ClassLoader>>* class_loaders) + : hs_(hs), + class_loaders_(class_loaders) {} + + void Visit(ObjPtr<mirror::ClassLoader> class_loader) + REQUIRES_SHARED(Locks::classlinker_classes_lock_, Locks::mutator_lock_) OVERRIDE { + class_loaders_->push_back(hs_->NewHandle(class_loader)); + } + + private: + VariableSizedHandleScope* const hs_; + std::vector<Handle<mirror::ClassLoader>>* const class_loaders_; +}; + +// GetClassesVisitor takes a snapshot of the loaded classes that we may want to visit and stores +// them in the out argument. Not affected by class unloading since there are no suspend points in +// the caller. +class GetClassesVisitor : public ClassVisitor { + public: + explicit GetClassesVisitor(bool profile_boot_class_path, + ScopedArenaVector<ObjPtr<mirror::Class>>* out) + : profile_boot_class_path_(profile_boot_class_path), + out_(out) {} virtual bool operator()(ObjPtr<mirror::Class> klass) REQUIRES_SHARED(Locks::mutator_lock_) { if (klass->IsProxyClass() || @@ -238,51 +264,107 @@ class GetClassesAndMethodsVisitor : public ClassVisitor { (!profile_boot_class_path_ && klass->GetClassLoader() == nullptr)) { return true; } - CHECK(klass->GetDexCache() != nullptr) << klass->PrettyClass(); - resolved_classes_->AddReference(&klass->GetDexFile(), klass->GetDexTypeIndex()); - for (ArtMethod& method : klass->GetMethods(kRuntimePointerSize)) { - if (!method.IsNative()) { - DCHECK(!method.IsProxyMethod()); - const uint16_t counter = method.GetCounter(); - // Mark startup methods as hot if they have more than hot_method_sample_threshold_ samples. - // This means they will get compiled by the compiler driver. - if (method.GetProfilingInfo(kRuntimePointerSize) != nullptr || - (method.GetAccessFlags() & kAccPreviouslyWarm) != 0 || - counter >= hot_method_sample_threshold_) { - hot_methods_->AddReference(method.GetDexFile(), method.GetDexMethodIndex()); - } else if (counter != 0) { - sampled_methods_->AddReference(method.GetDexFile(), method.GetDexMethodIndex()); - } - } else { - CHECK_EQ(method.GetCounter(), 0u); - } - } + out_->push_back(klass); return true; } private: - MethodReferenceCollection* const hot_methods_; - MethodReferenceCollection* const sampled_methods_; - TypeReferenceCollection* const resolved_classes_; - uint32_t hot_method_sample_threshold_; const bool profile_boot_class_path_; + ScopedArenaVector<ObjPtr<mirror::Class>>* const out_; }; -class ScopedDefaultPriority { - public: - explicit ScopedDefaultPriority(pthread_t thread) : thread_(thread) { - SetProfileSaverThreadPriority(thread_, GetDefaultThreadPriority()); - } +using MethodReferenceCollection = DexReferenceCollection<uint16_t, ScopedArenaAllocatorAdapter>; +using TypeReferenceCollection = DexReferenceCollection<dex::TypeIndex, + ScopedArenaAllocatorAdapter>; - ~ScopedDefaultPriority() { - SetProfileSaverThreadPriority(thread_, kProfileSaverPthreadPriority); +// Iterate over all of the loaded classes and visit each one. For each class, add it to the +// resolved_classes out argument if startup is true. +// Add methods to the hot_methods out argument if the number of samples is greater or equal to +// hot_method_sample_threshold, add it to sampled_methods if it has at least one sample. +static void SampleClassesAndExecutedMethods(pthread_t profiler_pthread, + bool profile_boot_class_path, + ScopedArenaAllocator* allocator, + uint32_t hot_method_sample_threshold, + bool startup, + TypeReferenceCollection* resolved_classes, + MethodReferenceCollection* hot_methods, + MethodReferenceCollection* sampled_methods) { + Thread* const self = Thread::Current(); + ClassLinker* const class_linker = Runtime::Current()->GetClassLinker(); + // Restore profile saver thread priority during the GC critical section. This helps prevent + // priority inversions blocking the GC for long periods of time. + std::unique_ptr<ScopedDefaultPriority> sdp; + // Only restore default priority if we are the profile saver thread. Other threads that call this + // are threads calling Stop and the signal catcher (for SIGUSR1). + if (pthread_self() == profiler_pthread) { + sdp.reset(new ScopedDefaultPriority(profiler_pthread)); + } + + // Do ScopedGCCriticalSection before acquiring mutator lock to prevent the GC running and + // blocking threads during thread root flipping. Since the GC is a background thread, blocking it + // is not a problem. + ScopedObjectAccess soa(self); + gc::ScopedGCCriticalSection sgcs(self, + gc::kGcCauseProfileSaver, + gc::kCollectorTypeCriticalSection); + VariableSizedHandleScope hs(soa.Self()); + std::vector<Handle<mirror::ClassLoader>> class_loaders; + if (profile_boot_class_path) { + // First add the boot class loader since visit classloaders doesn't visit it. + class_loaders.push_back(hs.NewHandle<mirror::ClassLoader>(nullptr)); + } + GetClassLoadersVisitor class_loader_visitor(&hs, &class_loaders); + { + // Read the class loaders into a temporary array to prevent contention problems on the + // class_linker_classes_lock. + ScopedTrace trace2("Get class loaders"); + ReaderMutexLock mu(soa.Self(), *Locks::classlinker_classes_lock_); + class_linker->VisitClassLoaders(&class_loader_visitor); + } + ScopedArenaVector<ObjPtr<mirror::Class>> classes(allocator->Adapter()); + for (Handle<mirror::ClassLoader> class_loader : class_loaders) { + ClassTable* table = class_linker->ClassTableForClassLoader(class_loader.Get()); + if (table == nullptr) { + // If the class loader has not loaded any classes, it may have a null table. + continue; + } + GetClassesVisitor get_classes_visitor(profile_boot_class_path, &classes); + { + // Collect the classes into a temporary array to prevent lock contention on the class + // table lock. We want to avoid blocking class loading in other threads as much as + // possible. + ScopedTrace trace3("Visiting class table"); + table->Visit(get_classes_visitor); + } + for (ObjPtr<mirror::Class> klass : classes) { + if (startup) { + // We only record classes for the startup case. This may change in the future. + resolved_classes->AddReference(&klass->GetDexFile(), klass->GetDexTypeIndex()); + } + // Visit all of the methods in the class to see which ones were executed. + for (ArtMethod& method : klass->GetMethods(kRuntimePointerSize)) { + if (!method.IsNative()) { + DCHECK(!method.IsProxyMethod()); + const uint16_t counter = method.GetCounter(); + // Mark startup methods as hot if they have more than hot_method_sample_threshold + // samples. This means they will get compiled by the compiler driver. + if (method.GetProfilingInfo(kRuntimePointerSize) != nullptr || + (method.GetAccessFlags() & kAccPreviouslyWarm) != 0 || + counter >= hot_method_sample_threshold) { + hot_methods->AddReference(method.GetDexFile(), method.GetDexMethodIndex()); + } else if (counter != 0) { + sampled_methods->AddReference(method.GetDexFile(), method.GetDexMethodIndex()); + } + } else { + CHECK_EQ(method.GetCounter(), 0u); + } + } + } + classes.clear(); } +} - private: - const pthread_t thread_; -}; - -void ProfileSaver::FetchAndCacheResolvedClassesAndMethods() { +void ProfileSaver::FetchAndCacheResolvedClassesAndMethods(bool startup) { ScopedTrace trace(__PRETTY_FUNCTION__); const uint64_t start_time = NanoTime(); @@ -294,34 +376,25 @@ void ProfileSaver::FetchAndCacheResolvedClassesAndMethods() { ArenaStack stack(runtime->GetArenaPool()); ScopedArenaAllocator allocator(&stack); MethodReferenceCollection hot_methods(allocator.Adapter(), allocator.Adapter()); - MethodReferenceCollection startup_methods(allocator.Adapter(), allocator.Adapter()); + MethodReferenceCollection sampled_methods(allocator.Adapter(), allocator.Adapter()); TypeReferenceCollection resolved_classes(allocator.Adapter(), allocator.Adapter()); const bool is_low_ram = Runtime::Current()->GetHeap()->IsLowMemoryMode(); - const size_t hot_threshold = options_.GetHotStartupMethodSamples(is_low_ram); pthread_t profiler_pthread; { MutexLock mu(self, *Locks::profiler_lock_); profiler_pthread = profiler_pthread_; } - { - // Restore profile saver thread priority during the GC critical section. This helps prevent - // priority inversions blocking the GC for long periods of time. - ScopedDefaultPriority sdp(profiler_pthread); - ScopedObjectAccess soa(self); - gc::ScopedGCCriticalSection sgcs(self, - gc::kGcCauseProfileSaver, - gc::kCollectorTypeCriticalSection); - { - ScopedTrace trace2("Get hot methods"); - GetClassesAndMethodsVisitor visitor(&hot_methods, - &startup_methods, - &resolved_classes, - hot_threshold, - options_.GetProfileBootClassPath()); - runtime->GetClassLinker()->VisitClasses(&visitor); - } - } - + const uint32_t hot_method_sample_threshold = startup ? + options_.GetHotStartupMethodSamples(is_low_ram) : + std::numeric_limits<uint32_t>::max(); + SampleClassesAndExecutedMethods(profiler_pthread, + options_.GetProfileBootClassPath(), + &allocator, + hot_method_sample_threshold, + startup, + &resolved_classes, + &hot_methods, + &sampled_methods); MutexLock mu(self, *Locks::profiler_lock_); uint64_t total_number_of_profile_entries_cached = 0; using Hotness = ProfileCompilationInfo::MethodHotness; @@ -329,9 +402,12 @@ void ProfileSaver::FetchAndCacheResolvedClassesAndMethods() { for (const auto& it : tracked_dex_base_locations_) { std::set<DexCacheResolvedClasses> resolved_classes_for_location; const std::string& filename = it.first; - auto info_it = profile_cache_.Put( - filename, - new ProfileCompilationInfo(Runtime::Current()->GetArenaPool())); + auto info_it = profile_cache_.find(filename); + if (info_it == profile_cache_.end()) { + info_it = profile_cache_.Put( + filename, + new ProfileCompilationInfo(Runtime::Current()->GetArenaPool())); + } ProfileCompilationInfo* cached_info = info_it->second; const std::set<std::string>& locations = it.second; @@ -339,18 +415,20 @@ void ProfileSaver::FetchAndCacheResolvedClassesAndMethods() { const DexFile* const dex_file = pair.first; if (locations.find(dex_file->GetBaseLocation()) != locations.end()) { const MethodReferenceCollection::IndexVector& indices = pair.second; + uint8_t flags = Hotness::kFlagHot; + flags |= startup ? Hotness::kFlagStartup : Hotness::kFlagPostStartup; cached_info->AddMethodsForDex( - static_cast<Hotness::Flag>(Hotness::kFlagHot | Hotness::kFlagStartup), + static_cast<Hotness::Flag>(flags), dex_file, indices.begin(), indices.end()); } } - for (const auto& pair : startup_methods.GetMap()) { + for (const auto& pair : sampled_methods.GetMap()) { const DexFile* const dex_file = pair.first; if (locations.find(dex_file->GetBaseLocation()) != locations.end()) { const MethodReferenceCollection::IndexVector& indices = pair.second; - cached_info->AddMethodsForDex(Hotness::kFlagStartup, + cached_info->AddMethodsForDex(startup ? Hotness::kFlagStartup : Hotness::kFlagPostStartup, dex_file, indices.begin(), indices.end()); @@ -375,8 +453,9 @@ void ProfileSaver::FetchAndCacheResolvedClassesAndMethods() { max_number_of_profile_entries_cached_, total_number_of_profile_entries_cached); VLOG(profiler) << "Profile saver recorded " << hot_methods.NumReferences() << " hot methods and " - << startup_methods.NumReferences() << " startup methods with threshold " - << hot_threshold << " in " << PrettyDuration(NanoTime() - start_time); + << sampled_methods.NumReferences() << " sampled methods with threshold " + << hot_method_sample_threshold << " in " + << PrettyDuration(NanoTime() - start_time); } bool ProfileSaver::ProcessProfilingInfo(bool force_save, /*out*/uint16_t* number_of_new_methods) { @@ -397,6 +476,10 @@ bool ProfileSaver::ProcessProfilingInfo(bool force_save, /*out*/uint16_t* number *number_of_new_methods = 0; } + // We only need to do this once, not once per dex location. + // TODO: Figure out a way to only do it when stuff has changed? It takes 30-50ms. + FetchAndCacheResolvedClassesAndMethods(/*startup*/ false); + for (const auto& it : tracked_locations) { if (!force_save && ShuttingDown(Thread::Current())) { // The ProfileSaver is in shutdown mode, meaning a stop request was made and @@ -442,6 +525,7 @@ bool ProfileSaver::ProcessProfilingInfo(bool force_save, /*out*/uint16_t* number total_number_of_skipped_writes_++; continue; } + if (number_of_new_methods != nullptr) { *number_of_new_methods = std::max(static_cast<uint16_t>(delta_number_of_methods), @@ -473,11 +557,12 @@ bool ProfileSaver::ProcessProfilingInfo(bool force_save, /*out*/uint16_t* number total_number_of_failed_writes_++; } } - // Trim the maps to madvise the pages used for profile info. - // It is unlikely we will need them again in the near feature. - Runtime::Current()->GetArenaPool()->TrimMaps(); } + // Trim the maps to madvise the pages used for profile info. + // It is unlikely we will need them again in the near feature. + Runtime::Current()->GetArenaPool()->TrimMaps(); + return profile_file_saved; } @@ -621,12 +706,13 @@ void ProfileSaver::Stop(bool dump_info) { profile_saver->period_condition_.Signal(Thread::Current()); } + // Force save everything before destroying the thread since we want profiler_pthread_ to remain + // valid. + instance_->ProcessProfilingInfo(/*force_save*/true, /*number_of_new_methods*/nullptr); + // Wait for the saver thread to stop. CHECK_PTHREAD_CALL(pthread_join, (profiler_pthread, nullptr), "profile saver thread shutdown"); - // Force save everything before destroying the instance. - instance_->ProcessProfilingInfo(/*force_save*/true, /*number_of_new_methods*/nullptr); - { MutexLock profiler_mutex(Thread::Current(), *Locks::profiler_lock_); if (dump_info) { @@ -713,16 +799,18 @@ void ProfileSaver::ForceProcessProfiles() { } } -bool ProfileSaver::HasSeenMethod(const std::string& profile, - const DexFile* dex_file, - uint16_t method_idx) { +bool ProfileSaver::HasSeenMethod(const std::string& profile, bool hot, MethodReference ref) { MutexLock mu(Thread::Current(), *Locks::profiler_lock_); if (instance_ != nullptr) { ProfileCompilationInfo info(Runtime::Current()->GetArenaPool()); if (!info.Load(profile, /*clear_if_invalid*/false)) { return false; } - return info.GetMethodHotness(MethodReference(dex_file, method_idx)).IsInProfile(); + ProfileCompilationInfo::MethodHotness hotness = info.GetMethodHotness(ref); + // Ignore hot parameter for now since it was causing test 595 to be flaky. TODO: Investigate. + // b/63635729 + UNUSED(hot); + return hotness.IsInProfile(); } return false; } diff --git a/runtime/jit/profile_saver.h b/runtime/jit/profile_saver.h index 01d72fec3d..ce8233bbea 100644 --- a/runtime/jit/profile_saver.h +++ b/runtime/jit/profile_saver.h @@ -19,6 +19,7 @@ #include "base/mutex.h" #include "jit_code_cache.h" +#include "method_reference.h" #include "profile_compilation_info.h" #include "profile_saver_options.h" #include "safe_map.h" @@ -55,10 +56,8 @@ class ProfileSaver { // For testing or manual purposes (SIGUSR1). static void ForceProcessProfiles(); - // Just for testing purpose. - static bool HasSeenMethod(const std::string& profile, - const DexFile* dex_file, - uint16_t method_idx); + // Just for testing purposes. + static bool HasSeenMethod(const std::string& profile, bool hot, MethodReference ref); private: ProfileSaver(const ProfileSaverOptions& options, @@ -97,7 +96,7 @@ class ProfileSaver { // Fetches the current resolved classes and methods from the ClassLinker and stores them in the // profile_cache_ for later save. - void FetchAndCacheResolvedClassesAndMethods(); + void FetchAndCacheResolvedClassesAndMethods(bool startup); void DumpInfo(std::ostream& os); diff --git a/runtime/mirror/class-inl.h b/runtime/mirror/class-inl.h index 419a4db0fc..003cd4e2bc 100644 --- a/runtime/mirror/class-inl.h +++ b/runtime/mirror/class-inl.h @@ -673,11 +673,7 @@ inline uint32_t Class::GetReferenceInstanceOffsets() { } inline void Class::SetClinitThreadId(pid_t new_clinit_thread_id) { - if (Runtime::Current()->IsActiveTransaction()) { - SetField32<true>(OFFSET_OF_OBJECT_MEMBER(Class, clinit_thread_id_), new_clinit_thread_id); - } else { - SetField32<false>(OFFSET_OF_OBJECT_MEMBER(Class, clinit_thread_id_), new_clinit_thread_id); - } + SetField32Transaction(OFFSET_OF_OBJECT_MEMBER(Class, clinit_thread_id_), new_clinit_thread_id); } inline String* Class::GetName() { @@ -685,11 +681,7 @@ inline String* Class::GetName() { } inline void Class::SetName(ObjPtr<String> name) { - if (Runtime::Current()->IsActiveTransaction()) { - SetFieldObject<true>(OFFSET_OF_OBJECT_MEMBER(Class, name_), name); - } else { - SetFieldObject<false>(OFFSET_OF_OBJECT_MEMBER(Class, name_), name); - } + SetFieldObjectTransaction(OFFSET_OF_OBJECT_MEMBER(Class, name_), name); } template<VerifyObjectFlags kVerifyFlags> diff --git a/runtime/mirror/class.cc b/runtime/mirror/class.cc index e4b53209e9..b0e5b6adbf 100644 --- a/runtime/mirror/class.cc +++ b/runtime/mirror/class.cc @@ -191,7 +191,7 @@ void Class::SetStatus(Handle<Class> h_this, Status new_status, Thread* self) { } void Class::SetDexCache(ObjPtr<DexCache> new_dex_cache) { - SetFieldObject<false>(OFFSET_OF_OBJECT_MEMBER(Class, dex_cache_), new_dex_cache); + SetFieldObjectTransaction(OFFSET_OF_OBJECT_MEMBER(Class, dex_cache_), new_dex_cache); } void Class::SetClassSize(uint32_t new_class_size) { @@ -200,8 +200,7 @@ void Class::SetClassSize(uint32_t new_class_size) { LOG(FATAL_WITHOUT_ABORT) << new_class_size << " vs " << GetClassSize(); LOG(FATAL) << "class=" << PrettyTypeOf(); } - // Not called within a transaction. - SetField32<false>(OFFSET_OF_OBJECT_MEMBER(Class, class_size_), new_class_size); + SetField32Transaction(OFFSET_OF_OBJECT_MEMBER(Class, class_size_), new_class_size); } // Return the class' name. The exact format is bizarre, but it's the specified behavior for diff --git a/runtime/mirror/class.h b/runtime/mirror/class.h index 00498bc30a..b60ddcf60c 100644 --- a/runtime/mirror/class.h +++ b/runtime/mirror/class.h @@ -408,7 +408,7 @@ class MANAGED Class FINAL : public Object { DCHECK_EQ(v32 & kPrimitiveTypeMask, v32) << "upper 16 bits aren't zero"; // Store the component size shift in the upper 16 bits. v32 |= Primitive::ComponentSizeShift(new_type) << kPrimitiveTypeSizeShiftShift; - SetField32<false>(OFFSET_OF_OBJECT_MEMBER(Class, primitive_type_), v32); + SetField32Transaction(OFFSET_OF_OBJECT_MEMBER(Class, primitive_type_), v32); } template<VerifyObjectFlags kVerifyFlags = kDefaultVerifyFlags> @@ -1169,8 +1169,7 @@ class MANAGED Class FINAL : public Object { } void SetDexClassDefIndex(uint16_t class_def_idx) REQUIRES_SHARED(Locks::mutator_lock_) { - // Not called within a transaction. - SetField32<false>(OFFSET_OF_OBJECT_MEMBER(Class, dex_class_def_idx_), class_def_idx); + SetField32Transaction(OFFSET_OF_OBJECT_MEMBER(Class, dex_class_def_idx_), class_def_idx); } dex::TypeIndex GetDexTypeIndex() REQUIRES_SHARED(Locks::mutator_lock_) { @@ -1179,8 +1178,7 @@ class MANAGED Class FINAL : public Object { } void SetDexTypeIndex(dex::TypeIndex type_idx) REQUIRES_SHARED(Locks::mutator_lock_) { - // Not called within a transaction. - SetField32<false>(OFFSET_OF_OBJECT_MEMBER(Class, dex_type_idx_), type_idx.index_); + SetField32Transaction(OFFSET_OF_OBJECT_MEMBER(Class, dex_type_idx_), type_idx.index_); } dex::TypeIndex FindTypeIndexInOtherDexFile(const DexFile& dex_file) diff --git a/runtime/mirror/method_handles_lookup.cc b/runtime/mirror/method_handles_lookup.cc index 0c25fa8ba0..9eada6dfdd 100644 --- a/runtime/mirror/method_handles_lookup.cc +++ b/runtime/mirror/method_handles_lookup.cc @@ -20,7 +20,10 @@ #include "gc_root-inl.h" #include "object-inl.h" #include "handle_scope.h" +#include "jni_internal.h" +#include "mirror/method_handle_impl.h" #include "modifiers.h" +#include "well_known_classes.h" namespace art { namespace mirror { @@ -54,5 +57,27 @@ MethodHandlesLookup* MethodHandlesLookup::Create(Thread* const self, Handle<Clas return mhl.Get(); } +MethodHandlesLookup* MethodHandlesLookup::GetDefault(Thread* const self) { + ArtMethod* lookup = jni::DecodeArtMethod(WellKnownClasses::java_lang_invoke_MethodHandles_lookup); + JValue result; + lookup->Invoke(self, nullptr, 0, &result, "L"); + return down_cast<MethodHandlesLookup*>(result.GetL()); +} + +MethodHandle* MethodHandlesLookup::FindConstructor(Thread* const self, + Handle<Class> klass, + Handle<MethodType> method_type) { + ArtMethod* findConstructor = + jni::DecodeArtMethod(WellKnownClasses::java_lang_invoke_MethodHandles_Lookup_findConstructor); + uint32_t args[] = { + static_cast<uint32_t>(reinterpret_cast<uintptr_t>(this)), + static_cast<uint32_t>(reinterpret_cast<uintptr_t>(klass.Get())), + static_cast<uint32_t>(reinterpret_cast<uintptr_t>(method_type.Get())) + }; + JValue result; + findConstructor->Invoke(self, args, sizeof(args), &result, "LLL"); + return down_cast<MethodHandle*>(result.GetL()); +} + } // namespace mirror } // namespace art diff --git a/runtime/mirror/method_handles_lookup.h b/runtime/mirror/method_handles_lookup.h index 63eb428f94..2109f601ee 100644 --- a/runtime/mirror/method_handles_lookup.h +++ b/runtime/mirror/method_handles_lookup.h @@ -30,6 +30,9 @@ class RootVisitor; namespace mirror { +class MethodHandle; +class MethodType; + // C++ mirror of java.lang.invoke.MethodHandles.Lookup class MANAGED MethodHandlesLookup : public Object { public: @@ -45,6 +48,16 @@ class MANAGED MethodHandlesLookup : public Object { static void ResetClass() REQUIRES_SHARED(Locks::mutator_lock_); static void VisitRoots(RootVisitor* visitor) REQUIRES_SHARED(Locks::mutator_lock_); + // Returns the result of java.lang.invoke.MethodHandles.lookup(). + static mirror::MethodHandlesLookup* GetDefault(Thread* const self) + REQUIRES_SHARED(Locks::mutator_lock_); + + // Find constructor using java.lang.invoke.MethodHandles$Lookup.findConstructor(). + mirror::MethodHandle* FindConstructor(Thread* const self, + Handle<Class> klass, + Handle<MethodType> method_type) + REQUIRES_SHARED(Locks::mutator_lock_); + private: static MemberOffset AllowedModesOffset() { return MemberOffset(OFFSETOF_MEMBER(MethodHandlesLookup, allowed_modes_)); diff --git a/runtime/mirror/object-inl.h b/runtime/mirror/object-inl.h index 95f829dc23..43d70b74ec 100644 --- a/runtime/mirror/object-inl.h +++ b/runtime/mirror/object-inl.h @@ -560,6 +560,15 @@ inline void Object::SetField32Volatile(MemberOffset field_offset, int32_t new_va SetField32<kTransactionActive, kCheckTransaction, kVerifyFlags, true>(field_offset, new_value); } +template<bool kCheckTransaction, VerifyObjectFlags kVerifyFlags, bool kIsVolatile> +inline void Object::SetField32Transaction(MemberOffset field_offset, int32_t new_value) { + if (Runtime::Current()->IsActiveTransaction()) { + SetField32<true, kCheckTransaction, kVerifyFlags, kIsVolatile>(field_offset, new_value); + } else { + SetField32<false, kCheckTransaction, kVerifyFlags, kIsVolatile>(field_offset, new_value); + } +} + // TODO: Pass memory_order_ and strong/weak as arguments to avoid code duplication? template<bool kTransactionActive, bool kCheckTransaction, VerifyObjectFlags kVerifyFlags> @@ -657,6 +666,15 @@ inline void Object::SetField64Volatile(MemberOffset field_offset, int64_t new_va new_value); } +template<bool kCheckTransaction, VerifyObjectFlags kVerifyFlags, bool kIsVolatile> +inline void Object::SetField64Transaction(MemberOffset field_offset, int32_t new_value) { + if (Runtime::Current()->IsActiveTransaction()) { + SetField64<true, kCheckTransaction, kVerifyFlags, kIsVolatile>(field_offset, new_value); + } else { + SetField64<false, kCheckTransaction, kVerifyFlags, kIsVolatile>(field_offset, new_value); + } +} + template<typename kSize> inline kSize Object::GetFieldAcquire(MemberOffset field_offset) { const uint8_t* raw_addr = reinterpret_cast<const uint8_t*>(this) + field_offset.Int32Value(); @@ -775,6 +793,15 @@ inline void Object::SetFieldObjectVolatile(MemberOffset field_offset, ObjPtr<Obj new_value); } +template<bool kCheckTransaction, VerifyObjectFlags kVerifyFlags, bool kIsVolatile> +inline void Object::SetFieldObjectTransaction(MemberOffset field_offset, ObjPtr<Object> new_value) { + if (Runtime::Current()->IsActiveTransaction()) { + SetFieldObject<true, kCheckTransaction, kVerifyFlags, kIsVolatile>(field_offset, new_value); + } else { + SetFieldObject<false, kCheckTransaction, kVerifyFlags, kIsVolatile>(field_offset, new_value); + } +} + template <VerifyObjectFlags kVerifyFlags> inline HeapReference<Object>* Object::GetFieldObjectReferenceAddr(MemberOffset field_offset) { if (kVerifyFlags & kVerifyThis) { diff --git a/runtime/mirror/object.h b/runtime/mirror/object.h index 9cf42522d1..886780f051 100644 --- a/runtime/mirror/object.h +++ b/runtime/mirror/object.h @@ -312,6 +312,11 @@ class MANAGED LOCKABLE Object { ObjPtr<Object> new_value) REQUIRES_SHARED(Locks::mutator_lock_); + template<bool kCheckTransaction = true, VerifyObjectFlags kVerifyFlags = kDefaultVerifyFlags, + bool kIsVolatile = false> + ALWAYS_INLINE void SetFieldObjectTransaction(MemberOffset field_offset, ObjPtr<Object> new_value) + REQUIRES_SHARED(Locks::mutator_lock_); + template<bool kTransactionActive, bool kCheckTransaction = true, VerifyObjectFlags kVerifyFlags = kDefaultVerifyFlags> @@ -470,6 +475,12 @@ class MANAGED LOCKABLE Object { ALWAYS_INLINE void SetField32Volatile(MemberOffset field_offset, int32_t new_value) REQUIRES_SHARED(Locks::mutator_lock_); + template<bool kCheckTransaction = true, + VerifyObjectFlags kVerifyFlags = kDefaultVerifyFlags, + bool kIsVolatile = false> + ALWAYS_INLINE void SetField32Transaction(MemberOffset field_offset, int32_t new_value) + REQUIRES_SHARED(Locks::mutator_lock_); + template<bool kTransactionActive, bool kCheckTransaction = true, VerifyObjectFlags kVerifyFlags = kDefaultVerifyFlags> ALWAYS_INLINE bool CasFieldWeakSequentiallyConsistent32(MemberOffset field_offset, @@ -525,6 +536,12 @@ class MANAGED LOCKABLE Object { ALWAYS_INLINE void SetField64Volatile(MemberOffset field_offset, int64_t new_value) REQUIRES_SHARED(Locks::mutator_lock_); + template<bool kCheckTransaction = true, + VerifyObjectFlags kVerifyFlags = kDefaultVerifyFlags, + bool kIsVolatile = false> + ALWAYS_INLINE void SetField64Transaction(MemberOffset field_offset, int32_t new_value) + REQUIRES_SHARED(Locks::mutator_lock_); + template<bool kTransactionActive, bool kCheckTransaction = true, VerifyObjectFlags kVerifyFlags = kDefaultVerifyFlags> bool CasFieldWeakSequentiallyConsistent64(MemberOffset field_offset, int64_t old_value, diff --git a/runtime/mirror/string-inl.h b/runtime/mirror/string-inl.h index 75606391ad..84587c871c 100644 --- a/runtime/mirror/string-inl.h +++ b/runtime/mirror/string-inl.h @@ -251,6 +251,7 @@ inline String* String::AllocFromByteArray(Thread* self, int32_t byte_length, Handle<ByteArray> array, int32_t offset, int32_t high_byte, gc::AllocatorType allocator_type) { const uint8_t* const src = reinterpret_cast<uint8_t*>(array->GetData()) + offset; + high_byte &= 0xff; // Extract the relevant bits before determining `compressible`. const bool compressible = kUseStringCompression && String::AllASCII<uint8_t>(src, byte_length) && (high_byte == 0); const int32_t length_with_flag = String::GetFlaggedCount(byte_length, compressible); diff --git a/runtime/monitor.cc b/runtime/monitor.cc index 3e3eaae13a..5c63dcad78 100644 --- a/runtime/monitor.cc +++ b/runtime/monitor.cc @@ -910,7 +910,8 @@ void Monitor::InflateThinLocked(Thread* self, Handle<mirror::Object> obj, LockWo // Go ahead and inflate the lock. Inflate(self, owner, obj.Get(), hash_code); } - thread_list->Resume(owner, SuspendReason::kInternal); + bool resumed = thread_list->Resume(owner, SuspendReason::kInternal); + DCHECK(resumed); } self->SetMonitorEnterObject(nullptr); } diff --git a/runtime/native/dalvik_system_VMStack.cc b/runtime/native/dalvik_system_VMStack.cc index 7d2d0e5bb9..2aeef60d00 100644 --- a/runtime/native/dalvik_system_VMStack.cc +++ b/runtime/native/dalvik_system_VMStack.cc @@ -62,7 +62,8 @@ static jobject GetThreadStack(const ScopedFastNativeObjectAccess& soa, jobject p trace = thread->CreateInternalStackTrace<false>(soa); } // Restart suspended thread. - thread_list->Resume(thread, SuspendReason::kInternal); + bool resumed = thread_list->Resume(thread, SuspendReason::kInternal); + DCHECK(resumed); } else if (timed_out) { LOG(ERROR) << "Trying to get thread's stack failed as the thread failed to suspend within a " "generous timeout."; diff --git a/runtime/native/java_lang_Thread.cc b/runtime/native/java_lang_Thread.cc index 8b76327fa8..4ce72edd7b 100644 --- a/runtime/native/java_lang_Thread.cc +++ b/runtime/native/java_lang_Thread.cc @@ -155,7 +155,8 @@ static void Thread_nativeSetName(JNIEnv* env, jobject peer, jstring java_name) { ScopedObjectAccess soa(env); thread->SetThreadName(name.c_str()); } - thread_list->Resume(thread, SuspendReason::kInternal); + bool resumed = thread_list->Resume(thread, SuspendReason::kInternal); + DCHECK(resumed); } else if (timed_out) { LOG(ERROR) << "Trying to set thread name to '" << name.c_str() << "' failed as the thread " "failed to suspend within a generous timeout."; diff --git a/runtime/native/org_apache_harmony_dalvik_ddmc_DdmVmInternal.cc b/runtime/native/org_apache_harmony_dalvik_ddmc_DdmVmInternal.cc index c516b66d93..125d737958 100644 --- a/runtime/native/org_apache_harmony_dalvik_ddmc_DdmVmInternal.cc +++ b/runtime/native/org_apache_harmony_dalvik_ddmc_DdmVmInternal.cc @@ -76,7 +76,8 @@ static jobjectArray DdmVmInternal_getStackTraceById(JNIEnv* env, jclass, jint th trace = Thread::InternalStackTraceToStackTraceElementArray(soa, internal_trace); } // Restart suspended thread. - thread_list->Resume(thread, SuspendReason::kInternal); + bool resumed = thread_list->Resume(thread, SuspendReason::kInternal); + DCHECK(resumed); } else { if (timed_out) { LOG(ERROR) << "Trying to get thread's stack by id failed as the thread failed to suspend " diff --git a/runtime/oat.h b/runtime/oat.h index 521cc40764..5e61907d82 100644 --- a/runtime/oat.h +++ b/runtime/oat.h @@ -32,7 +32,8 @@ class InstructionSetFeatures; class PACKED(4) OatHeader { public: static constexpr uint8_t kOatMagic[] = { 'o', 'a', 't', '\n' }; - static constexpr uint8_t kOatVersion[] = { '1', '2', '7', '\0' }; // .bss ArtMethod* section. + // Last oat version changed reason: update classpath key format. + static constexpr uint8_t kOatVersion[] = { '1', '2', '8', '\0' }; static constexpr const char* kImageLocationKey = "image-location"; static constexpr const char* kDex2OatCmdLineKey = "dex2oat-cmdline"; diff --git a/runtime/oat_file.cc b/runtime/oat_file.cc index 888de457dc..1c1189d7de 100644 --- a/runtime/oat_file.cc +++ b/runtime/oat_file.cc @@ -1574,28 +1574,6 @@ CompilerFilter::Filter OatFile::GetCompilerFilter() const { return GetOatHeader().GetCompilerFilter(); } -static constexpr char kDexClassPathEncodingSeparator = '*'; - -std::string OatFile::EncodeDexFileDependencies(const std::vector<const DexFile*>& dex_files, - std::string& base_dir) { - std::ostringstream out; - - for (const DexFile* dex_file : dex_files) { - const std::string& location = dex_file->GetLocation(); - // Find paths that were relative and convert them back from absolute. - if (!base_dir.empty() && location.substr(0, base_dir.length()) == base_dir) { - out << location.substr(base_dir.length() + 1).c_str(); - } else { - out << dex_file->GetLocation().c_str(); - } - out << kDexClassPathEncodingSeparator; - out << dex_file->GetLocationChecksum(); - out << kDexClassPathEncodingSeparator; - } - - return out.str(); -} - OatFile::OatClass OatFile::FindOatClass(const DexFile& dex_file, uint16_t class_def_idx, bool* found) { diff --git a/runtime/oat_file.h b/runtime/oat_file.h index 66ed44f1b9..b112b84564 100644 --- a/runtime/oat_file.h +++ b/runtime/oat_file.h @@ -296,11 +296,6 @@ class OatFile { static std::string ResolveRelativeEncodedDexLocation( const char* abs_dex_location, const std::string& rel_dex_location); - // Create a dependency list (dex locations and checksums) for the given dex files. - // Removes dex file paths prefixed with base_dir to convert them back to relative paths. - static std::string EncodeDexFileDependencies(const std::vector<const DexFile*>& dex_files, - std::string& base_dir); - // Finds the associated oat class for a dex_file and descriptor. Returns an invalid OatClass on // error and sets found to false. static OatClass FindOatClass(const DexFile& dex_file, uint16_t class_def_idx, bool* found); diff --git a/runtime/oat_file_assistant.cc b/runtime/oat_file_assistant.cc index 4820feb56c..c8766578c4 100644 --- a/runtime/oat_file_assistant.cc +++ b/runtime/oat_file_assistant.cc @@ -298,28 +298,38 @@ std::string OatFileAssistant::GetStatusDump() { } std::vector<std::unique_ptr<const DexFile>> OatFileAssistant::LoadDexFiles( - const OatFile& oat_file, const char* dex_location) { + const OatFile &oat_file, const char *dex_location) { std::vector<std::unique_ptr<const DexFile>> dex_files; + if (LoadDexFiles(oat_file, dex_location, &dex_files)) { + return dex_files; + } else { + return std::vector<std::unique_ptr<const DexFile>>(); + } +} +bool OatFileAssistant::LoadDexFiles( + const OatFile &oat_file, + const std::string& dex_location, + std::vector<std::unique_ptr<const DexFile>>* out_dex_files) { // Load the main dex file. std::string error_msg; const OatFile::OatDexFile* oat_dex_file = oat_file.GetOatDexFile( - dex_location, nullptr, &error_msg); + dex_location.c_str(), nullptr, &error_msg); if (oat_dex_file == nullptr) { LOG(WARNING) << error_msg; - return std::vector<std::unique_ptr<const DexFile>>(); + return false; } std::unique_ptr<const DexFile> dex_file = oat_dex_file->OpenDexFile(&error_msg); if (dex_file.get() == nullptr) { LOG(WARNING) << "Failed to open dex file from oat dex file: " << error_msg; - return std::vector<std::unique_ptr<const DexFile>>(); + return false; } - dex_files.push_back(std::move(dex_file)); + out_dex_files->push_back(std::move(dex_file)); // Load the rest of the multidex entries - for (size_t i = 1; ; i++) { - std::string multidex_dex_location = DexFile::GetMultiDexLocation(i, dex_location); + for (size_t i = 1;; i++) { + std::string multidex_dex_location = DexFile::GetMultiDexLocation(i, dex_location.c_str()); oat_dex_file = oat_file.GetOatDexFile(multidex_dex_location.c_str(), nullptr); if (oat_dex_file == nullptr) { // There are no more multidex entries to load. @@ -329,11 +339,11 @@ std::vector<std::unique_ptr<const DexFile>> OatFileAssistant::LoadDexFiles( dex_file = oat_dex_file->OpenDexFile(&error_msg); if (dex_file.get() == nullptr) { LOG(WARNING) << "Failed to open dex file from oat dex file: " << error_msg; - return std::vector<std::unique_ptr<const DexFile>>(); + return false; } - dex_files.push_back(std::move(dex_file)); + out_dex_files->push_back(std::move(dex_file)); } - return dex_files; + return true; } bool OatFileAssistant::HasOriginalDexFiles() { diff --git a/runtime/oat_file_assistant.h b/runtime/oat_file_assistant.h index 03d9ca38a8..92d87eaeae 100644 --- a/runtime/oat_file_assistant.h +++ b/runtime/oat_file_assistant.h @@ -207,6 +207,13 @@ class OatFileAssistant { static std::vector<std::unique_ptr<const DexFile>> LoadDexFiles( const OatFile& oat_file, const char* dex_location); + // Same as `std::vector<std::unique_ptr<const DexFile>> LoadDexFiles(...)` with the difference: + // - puts the dex files in the given vector + // - returns whether or not all dex files were successfully opened + static bool LoadDexFiles(const OatFile& oat_file, + const std::string& dex_location, + std::vector<std::unique_ptr<const DexFile>>* out_dex_files); + // Returns true if there are dex files in the original dex location that can // be compiled with dex2oat for this dex location. // Returns false if there is no original dex file, or if the original dex diff --git a/runtime/oat_file_manager.cc b/runtime/oat_file_manager.cc index 630945a829..b166961795 100644 --- a/runtime/oat_file_manager.cc +++ b/runtime/oat_file_manager.cc @@ -28,6 +28,7 @@ #include "base/stl_util.h" #include "base/systrace.h" #include "class_linker.h" +#include "class_loader_context.h" #include "dex_file-inl.h" #include "dex_file_tracking_registrar.h" #include "gc/scoped_gc_critical_section.h" @@ -421,38 +422,47 @@ static void GetDexFilesFromDexElementsArray( } } -static bool AreSharedLibrariesOk(const std::string& shared_libraries, - std::vector<const DexFile*>& dex_files) { - // If no shared libraries, we expect no dex files. - if (shared_libraries.empty()) { - return dex_files.empty(); - } - // If we find the special shared library, skip the shared libraries check. - if (shared_libraries.compare(OatFile::kSpecialSharedLibrary) == 0) { - return true; +static bool AreSharedLibrariesOk(const std::string& context_spec, + std::vector<const DexFile*>& dex_files, + std::string* error_msg) { + std::vector<std::string> classpath; + std::vector<uint32_t> checksums; + bool is_special_shared_library; + if (!ClassLoaderContext::DecodePathClassLoaderContextFromOatFileKey( + context_spec, &classpath, &checksums, &is_special_shared_library)) { + *error_msg = "Could not decode the class loader context from the oat file key."; + return false; } - // Shared libraries is a series of dex file paths and their checksums, each separated by '*'. - std::vector<std::string> shared_libraries_split; - Split(shared_libraries, '*', &shared_libraries_split); - // Sanity check size of dex files and split shared libraries. Should be 2x as many entries in - // the split shared libraries since it contains pairs of filename/checksum. - if (dex_files.size() * 2 != shared_libraries_split.size()) { + DCHECK_EQ(classpath.size(), checksums.size()); + + // The classpath size should match the number of dex files. + if (classpath.size() != dex_files.size()) { + *error_msg = "The number of loaded dex files does not match the number of files " + "specified in the context. Expected=" + std::to_string(classpath.size()) + + ", found=" + std::to_string(dex_files.size()); return false; } + // If we find the special shared library, skip the shared libraries check. + if (is_special_shared_library) { + return true; + } + // Check that the loaded dex files have the same order and checksums as the shared libraries. for (size_t i = 0; i < dex_files.size(); ++i) { + const std::string& dex_location = dex_files[i]->GetLocation(); + uint32_t dex_location_checksum = dex_files[i]->GetLocationChecksum(); std::string absolute_library_path = - OatFile::ResolveRelativeEncodedDexLocation(dex_files[i]->GetLocation().c_str(), - shared_libraries_split[i * 2]); - if (dex_files[i]->GetLocation() != absolute_library_path) { + OatFile::ResolveRelativeEncodedDexLocation(dex_location.c_str(), classpath[i]); + if (dex_location != absolute_library_path) { + *error_msg = "SharedLibraryCheck: expected=" + absolute_library_path + ", found=" + + dex_location; return false; } - char* end; - size_t shared_lib_checksum = strtoul(shared_libraries_split[i * 2 + 1].c_str(), &end, 10); - uint32_t dex_checksum = dex_files[i]->GetLocationChecksum(); - if (*end != '\0' || dex_checksum != shared_lib_checksum) { + if (dex_location_checksum != checksums[i]) { + *error_msg = "SharedLibraryCheck: checksum mismatch for " + dex_location + ". Expected=" + + std::to_string(checksums[i]) + ", found=" + std::to_string(dex_location_checksum); return false; } } @@ -586,7 +596,7 @@ bool OatFileManager::HasCollisions(const OatFile* oat_file, // Exit if shared libraries are ok. Do a full duplicate classes check otherwise. const std::string shared_libraries(oat_file->GetOatHeader().GetStoreValueByKey(OatHeader::kClassPathKey)); - if (AreSharedLibrariesOk(shared_libraries, dex_files_loaded)) { + if (AreSharedLibrariesOk(shared_libraries, dex_files_loaded, error_msg)) { return false; } diff --git a/runtime/openjdkjvm/OpenjdkJvm.cc b/runtime/openjdkjvm/OpenjdkJvm.cc index 4560dda4dd..6a8f2cedca 100644 --- a/runtime/openjdkjvm/OpenjdkJvm.cc +++ b/runtime/openjdkjvm/OpenjdkJvm.cc @@ -432,7 +432,8 @@ JNIEXPORT void JVM_SetNativeThreadName(JNIEnv* env, jobject jthread, jstring jav art::ScopedObjectAccess soa(env); thread->SetThreadName(name.c_str()); } - thread_list->Resume(thread, art::SuspendReason::kInternal); + bool resumed = thread_list->Resume(thread, art::SuspendReason::kInternal); + DCHECK(resumed); } else if (timed_out) { LOG(ERROR) << "Trying to set thread name to '" << name.c_str() << "' failed as the thread " "failed to suspend within a generous timeout."; diff --git a/runtime/openjdkjvmti/OpenjdkJvmTi.cc b/runtime/openjdkjvmti/OpenjdkJvmTi.cc index d3e8798bd6..3c1311b18a 100644 --- a/runtime/openjdkjvmti/OpenjdkJvmTi.cc +++ b/runtime/openjdkjvmti/OpenjdkJvmTi.cc @@ -133,34 +133,34 @@ class JvmtiFunctions { return ThreadUtil::GetAllThreads(env, threads_count_ptr, threads_ptr); } - static jvmtiError SuspendThread(jvmtiEnv* env, jthread thread ATTRIBUTE_UNUSED) { + static jvmtiError SuspendThread(jvmtiEnv* env, jthread thread) { ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_suspend); - return ERR(NOT_IMPLEMENTED); + return ThreadUtil::SuspendThread(env, thread); } static jvmtiError SuspendThreadList(jvmtiEnv* env, - jint request_count ATTRIBUTE_UNUSED, - const jthread* request_list ATTRIBUTE_UNUSED, - jvmtiError* results ATTRIBUTE_UNUSED) { + jint request_count, + const jthread* request_list, + jvmtiError* results) { ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_suspend); - return ERR(NOT_IMPLEMENTED); + return ThreadUtil::SuspendThreadList(env, request_count, request_list, results); } - static jvmtiError ResumeThread(jvmtiEnv* env, jthread thread ATTRIBUTE_UNUSED) { + static jvmtiError ResumeThread(jvmtiEnv* env, jthread thread) { ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_suspend); - return ERR(NOT_IMPLEMENTED); + return ThreadUtil::ResumeThread(env, thread); } static jvmtiError ResumeThreadList(jvmtiEnv* env, - jint request_count ATTRIBUTE_UNUSED, - const jthread* request_list ATTRIBUTE_UNUSED, - jvmtiError* results ATTRIBUTE_UNUSED) { + jint request_count, + const jthread* request_list, + jvmtiError* results) { ENSURE_VALID_ENV(env); ENSURE_HAS_CAP(env, can_suspend); - return ERR(NOT_IMPLEMENTED); + return ThreadUtil::ResumeThreadList(env, request_count, request_list, results); } static jvmtiError StopThread(jvmtiEnv* env, @@ -1498,10 +1498,11 @@ class JvmtiFunctions { static jvmtiError DisposeEnvironment(jvmtiEnv* env) { ENSURE_VALID_ENV(env); - gEventHandler.RemoveArtJvmTiEnv(ArtJvmTiEnv::AsArtJvmTiEnv(env)); - art::Runtime::Current()->RemoveSystemWeakHolder( - ArtJvmTiEnv::AsArtJvmTiEnv(env)->object_tag_table.get()); - delete env; + ArtJvmTiEnv* tienv = ArtJvmTiEnv::AsArtJvmTiEnv(env); + gEventHandler.RemoveArtJvmTiEnv(tienv); + art::Runtime::Current()->RemoveSystemWeakHolder(tienv->object_tag_table.get()); + ThreadUtil::RemoveEnvironment(tienv); + delete tienv; return OK; } @@ -1671,6 +1672,7 @@ static bool IsJvmtiVersion(jint version) { } extern const jvmtiInterface_1 gJvmtiInterface; + ArtJvmTiEnv::ArtJvmTiEnv(art::JavaVMExt* runtime, EventHandler* event_handler) : art_vm(runtime), local_data(nullptr), diff --git a/runtime/openjdkjvmti/art_jvmti.h b/runtime/openjdkjvmti/art_jvmti.h index c63e50252b..4d5bb95f2c 100644 --- a/runtime/openjdkjvmti/art_jvmti.h +++ b/runtime/openjdkjvmti/art_jvmti.h @@ -233,7 +233,7 @@ const jvmtiCapabilities kPotentialCapabilities = { .can_generate_exception_events = 0, .can_generate_frame_pop_events = 0, .can_generate_breakpoint_events = 1, - .can_suspend = 0, + .can_suspend = 1, .can_redefine_any_class = 0, .can_get_current_thread_cpu_time = 0, .can_get_thread_cpu_time = 0, diff --git a/runtime/openjdkjvmti/ti_thread.cc b/runtime/openjdkjvmti/ti_thread.cc index 2cc2a26c3b..d1cee9ab3f 100644 --- a/runtime/openjdkjvmti/ti_thread.cc +++ b/runtime/openjdkjvmti/ti_thread.cc @@ -159,6 +159,17 @@ jvmtiError ThreadUtil::GetCurrentThread(jvmtiEnv* env ATTRIBUTE_UNUSED, jthread* return ERR(NONE); } +static art::Thread* GetNativeThreadLocked(jthread thread, + const art::ScopedObjectAccessAlreadyRunnable& soa) + REQUIRES_SHARED(art::Locks::mutator_lock_) + REQUIRES(art::Locks::thread_list_lock_) { + if (thread == nullptr) { + return art::Thread::Current(); + } + + return art::Thread::FromManagedThread(soa, thread); +} + // Get the native thread. The spec says a null object denotes the current thread. static art::Thread* GetNativeThread(jthread thread, const art::ScopedObjectAccessAlreadyRunnable& soa) @@ -433,6 +444,9 @@ jvmtiError ThreadUtil::GetThreadState(jvmtiEnv* env ATTRIBUTE_UNUSED, if (native_thread->IsInterrupted()) { jvmti_state |= JVMTI_THREAD_STATE_INTERRUPTED; } + if (native_thread->IsSuspended()) { + jvmti_state |= JVMTI_THREAD_STATE_SUSPENDED; + } // Java state is derived from nativeGetState. // Note: Our implementation assigns "runnable" to suspended. As such, we will have slightly @@ -492,40 +506,82 @@ jvmtiError ThreadUtil::GetAllThreads(jvmtiEnv* env, return ERR(NONE); } -jvmtiError ThreadUtil::SetThreadLocalStorage(jvmtiEnv* env ATTRIBUTE_UNUSED, - jthread thread, - const void* data) { - art::ScopedObjectAccess soa(art::Thread::Current()); - art::Thread* self = GetNativeThread(thread, soa); - if (self == nullptr && thread == nullptr) { +// The struct that we store in the art::Thread::custom_tls_ that maps the jvmtiEnvs to the data +// stored with that thread. This is needed since different jvmtiEnvs are not supposed to share TLS +// data but we only have a single slot in Thread objects to store data. +struct JvmtiGlobalTLSData { + std::unordered_map<jvmtiEnv*, const void*> data GUARDED_BY(art::Locks::thread_list_lock_); +}; + +static void RemoveTLSData(art::Thread* target, void* ctx) REQUIRES(art::Locks::thread_list_lock_) { + jvmtiEnv* env = reinterpret_cast<jvmtiEnv*>(ctx); + art::Locks::thread_list_lock_->AssertHeld(art::Thread::Current()); + JvmtiGlobalTLSData* global_tls = reinterpret_cast<JvmtiGlobalTLSData*>(target->GetCustomTLS()); + if (global_tls != nullptr) { + global_tls->data.erase(env); + } +} + +void ThreadUtil::RemoveEnvironment(jvmtiEnv* env) { + art::Thread* self = art::Thread::Current(); + art::MutexLock mu(self, *art::Locks::thread_list_lock_); + art::ThreadList* list = art::Runtime::Current()->GetThreadList(); + list->ForEach(RemoveTLSData, env); +} + +jvmtiError ThreadUtil::SetThreadLocalStorage(jvmtiEnv* env, jthread thread, const void* data) { + art::Thread* self = art::Thread::Current(); + art::ScopedObjectAccess soa(self); + art::MutexLock mu(self, *art::Locks::thread_list_lock_); + art::Thread* target = GetNativeThreadLocked(thread, soa); + if (target == nullptr && thread == nullptr) { return ERR(INVALID_THREAD); } - if (self == nullptr) { + if (target == nullptr) { return ERR(THREAD_NOT_ALIVE); } - self->SetCustomTLS(data); + JvmtiGlobalTLSData* global_tls = reinterpret_cast<JvmtiGlobalTLSData*>(target->GetCustomTLS()); + if (global_tls == nullptr) { + target->SetCustomTLS(new JvmtiGlobalTLSData); + global_tls = reinterpret_cast<JvmtiGlobalTLSData*>(target->GetCustomTLS()); + } + + global_tls->data[env] = data; return ERR(NONE); } -jvmtiError ThreadUtil::GetThreadLocalStorage(jvmtiEnv* env ATTRIBUTE_UNUSED, +jvmtiError ThreadUtil::GetThreadLocalStorage(jvmtiEnv* env, jthread thread, void** data_ptr) { if (data_ptr == nullptr) { return ERR(NULL_POINTER); } - art::ScopedObjectAccess soa(art::Thread::Current()); - art::Thread* self = GetNativeThread(thread, soa); - if (self == nullptr && thread == nullptr) { + art::Thread* self = art::Thread::Current(); + art::ScopedObjectAccess soa(self); + art::MutexLock mu(self, *art::Locks::thread_list_lock_); + art::Thread* target = GetNativeThreadLocked(thread, soa); + if (target == nullptr && thread == nullptr) { return ERR(INVALID_THREAD); } - if (self == nullptr) { + if (target == nullptr) { return ERR(THREAD_NOT_ALIVE); } - *data_ptr = const_cast<void*>(self->GetCustomTLS()); + JvmtiGlobalTLSData* global_tls = reinterpret_cast<JvmtiGlobalTLSData*>(target->GetCustomTLS()); + if (global_tls == nullptr) { + *data_ptr = nullptr; + return OK; + } + auto it = global_tls->data.find(env); + if (it != global_tls->data.end()) { + *data_ptr = const_cast<void*>(it->second); + } else { + *data_ptr = nullptr; + } + return ERR(NONE); } @@ -605,4 +661,200 @@ jvmtiError ThreadUtil::RunAgentThread(jvmtiEnv* jvmti_env, return ERR(NONE); } +// Suspends the current thread if it has any suspend requests on it. +static void SuspendCheck(art::Thread* self) + REQUIRES(!art::Locks::mutator_lock_, !art::Locks::user_code_suspension_lock_) { + art::ScopedObjectAccess soa(self); + // Really this is only needed if we are in FastJNI and actually have the mutator_lock_ already. + self->FullSuspendCheck(); +} + +jvmtiError ThreadUtil::SuspendOther(art::Thread* self, + jthread target_jthread, + art::Thread* target) { + // Loop since we need to bail out and try again if we would end up getting suspended while holding + // the user_code_suspension_lock_ due to a SuspendReason::kForUserCode. In this situation we + // release the lock, wait to get resumed and try again. + do { + // Suspend ourself if we have any outstanding suspends. This is so we won't suspend due to + // another SuspendThread in the middle of suspending something else potentially causing a + // deadlock. We need to do this in the loop because if we ended up back here then we had + // outstanding SuspendReason::kForUserCode suspensions and we should wait for them to be cleared + // before continuing. + SuspendCheck(self); + art::MutexLock mu(self, *art::Locks::user_code_suspension_lock_); + { + art::MutexLock thread_list_mu(self, *art::Locks::thread_suspend_count_lock_); + // Make sure we won't be suspended in the middle of holding the thread_suspend_count_lock_ by + // a user-code suspension. We retry and do another SuspendCheck to clear this. + if (self->GetUserCodeSuspendCount() != 0) { + continue; + } else if (target->GetUserCodeSuspendCount() != 0) { + return ERR(THREAD_SUSPENDED); + } + } + bool timeout = true; + while (timeout) { + art::ThreadState state = target->GetState(); + if (state == art::ThreadState::kTerminated || state == art::ThreadState::kStarting) { + return ERR(THREAD_NOT_ALIVE); + } + target = art::Runtime::Current()->GetThreadList()->SuspendThreadByPeer( + target_jthread, + /* request_suspension */ true, + art::SuspendReason::kForUserCode, + &timeout); + if (target == nullptr && !timeout) { + // TODO It would be good to get more information about why exactly the thread failed to + // suspend. + return ERR(INTERNAL); + } + } + return OK; + } while (true); + UNREACHABLE(); +} + +jvmtiError ThreadUtil::SuspendSelf(art::Thread* self) { + CHECK(self == art::Thread::Current()); + { + art::MutexLock mu(self, *art::Locks::user_code_suspension_lock_); + art::MutexLock thread_list_mu(self, *art::Locks::thread_suspend_count_lock_); + if (self->GetUserCodeSuspendCount() != 0) { + // This can only happen if we race with another thread to suspend 'self' and we lose. + return ERR(THREAD_SUSPENDED); + } + // We shouldn't be able to fail this. + if (!self->ModifySuspendCount(self, +1, nullptr, art::SuspendReason::kForUserCode)) { + // TODO More specific error would be nice. + return ERR(INTERNAL); + } + } + // Once we have requested the suspend we actually go to sleep. We need to do this after releasing + // the suspend_lock to make sure we can be woken up. This call gains the mutator lock causing us + // to go to sleep until we are resumed. + SuspendCheck(self); + return OK; +} + +jvmtiError ThreadUtil::SuspendThread(jvmtiEnv* env ATTRIBUTE_UNUSED, jthread thread) { + art::Thread* self = art::Thread::Current(); + art::Thread* target; + { + art::ScopedObjectAccess soa(self); + target = GetNativeThread(thread, soa); + } + if (target == nullptr) { + return ERR(INVALID_THREAD); + } + if (target == self) { + return SuspendSelf(self); + } else { + return SuspendOther(self, thread, target); + } +} + +jvmtiError ThreadUtil::ResumeThread(jvmtiEnv* env ATTRIBUTE_UNUSED, + jthread thread) { + if (thread == nullptr) { + return ERR(NULL_POINTER); + } + art::Thread* self = art::Thread::Current(); + art::Thread* target; + { + // NB This does a SuspendCheck (during thread state change) so we need to make sure we don't + // have the 'suspend_lock' locked here. + art::ScopedObjectAccess soa(self); + target = GetNativeThread(thread, soa); + } + if (target == nullptr) { + return ERR(INVALID_THREAD); + } else if (target == self) { + // We would have paused until we aren't suspended anymore due to the ScopedObjectAccess so we + // can just return THREAD_NOT_SUSPENDED. Unfortunately we cannot do any real DCHECKs about + // current state since it's all concurrent. + return ERR(THREAD_NOT_SUSPENDED); + } + // Now that we know we aren't getting suspended ourself (since we have a mutator lock) we lock the + // suspend_lock to start suspending. + art::MutexLock mu(self, *art::Locks::user_code_suspension_lock_); + { + // The JVMTI spec requires us to return THREAD_NOT_SUSPENDED if it is alive but we really cannot + // tell why resume failed. + art::MutexLock thread_list_mu(self, *art::Locks::thread_suspend_count_lock_); + if (target->GetUserCodeSuspendCount() == 0) { + return ERR(THREAD_NOT_SUSPENDED); + } + } + if (target->GetState() == art::ThreadState::kTerminated) { + return ERR(THREAD_NOT_ALIVE); + } + DCHECK(target != self); + if (!art::Runtime::Current()->GetThreadList()->Resume(target, art::SuspendReason::kForUserCode)) { + // TODO Give a better error. + // This is most likely THREAD_NOT_SUSPENDED but we cannot really be sure. + return ERR(INTERNAL); + } + return OK; +} + +// Suspends all the threads in the list at the same time. Getting this behavior is a little tricky +// since we can have threads in the list multiple times. This generally doesn't matter unless the +// current thread is present multiple times. In that case we need to suspend only once and either +// return the same error code in all the other slots if it failed or return ERR(THREAD_SUSPENDED) if +// it didn't. We also want to handle the current thread last to make the behavior of the code +// simpler to understand. +jvmtiError ThreadUtil::SuspendThreadList(jvmtiEnv* env, + jint request_count, + const jthread* threads, + jvmtiError* results) { + if (request_count == 0) { + return ERR(ILLEGAL_ARGUMENT); + } else if (results == nullptr || threads == nullptr) { + return ERR(NULL_POINTER); + } + // This is the list of the indexes in 'threads' and 'results' that correspond to the currently + // running thread. These indexes we need to handle specially since we need to only actually + // suspend a single time. + std::vector<jint> current_thread_indexes; + art::Thread* self = art::Thread::Current(); + for (jint i = 0; i < request_count; i++) { + { + art::ScopedObjectAccess soa(self); + if (threads[i] == nullptr || GetNativeThread(threads[i], soa) == self) { + current_thread_indexes.push_back(i); + continue; + } + } + results[i] = env->SuspendThread(threads[i]); + } + if (!current_thread_indexes.empty()) { + jint first_current_thread_index = current_thread_indexes[0]; + // Suspend self. + jvmtiError res = env->SuspendThread(threads[first_current_thread_index]); + results[first_current_thread_index] = res; + // Fill in the rest of the error values as appropriate. + jvmtiError other_results = (res != OK) ? res : ERR(THREAD_SUSPENDED); + for (auto it = ++current_thread_indexes.begin(); it != current_thread_indexes.end(); ++it) { + results[*it] = other_results; + } + } + return OK; +} + +jvmtiError ThreadUtil::ResumeThreadList(jvmtiEnv* env, + jint request_count, + const jthread* threads, + jvmtiError* results) { + if (request_count == 0) { + return ERR(ILLEGAL_ARGUMENT); + } else if (results == nullptr || threads == nullptr) { + return ERR(NULL_POINTER); + } + for (jint i = 0; i < request_count; i++) { + results[i] = env->ResumeThread(threads[i]); + } + return OK; +} + } // namespace openjdkjvmti diff --git a/runtime/openjdkjvmti/ti_thread.h b/runtime/openjdkjvmti/ti_thread.h index 939aea7a29..d07dc0682b 100644 --- a/runtime/openjdkjvmti/ti_thread.h +++ b/runtime/openjdkjvmti/ti_thread.h @@ -35,8 +35,11 @@ #include "jni.h" #include "jvmti.h" +#include "base/mutex.h" + namespace art { class ArtField; +class Thread; } // namespace art namespace openjdkjvmti { @@ -51,6 +54,9 @@ class ThreadUtil { // To be called when it is safe to cache data. static void CacheData(); + // Handle a jvmtiEnv going away. + static void RemoveEnvironment(jvmtiEnv* env); + static jvmtiError GetAllThreads(jvmtiEnv* env, jint* threads_count_ptr, jthread** threads_ptr); static jvmtiError GetCurrentThread(jvmtiEnv* env, jthread* thread_ptr); @@ -68,7 +74,33 @@ class ThreadUtil { const void* arg, jint priority); + static jvmtiError SuspendThread(jvmtiEnv* env, jthread thread); + static jvmtiError ResumeThread(jvmtiEnv* env, jthread thread); + + static jvmtiError SuspendThreadList(jvmtiEnv* env, + jint request_count, + const jthread* threads, + jvmtiError* results); + static jvmtiError ResumeThreadList(jvmtiEnv* env, + jint request_count, + const jthread* threads, + jvmtiError* results); + private: + // We need to make sure only one thread tries to suspend threads at a time so we can get the + // 'suspend-only-once' behavior the spec requires. Internally, ART considers suspension to be a + // counted state, allowing a single thread to be suspended multiple times by different users. This + // makes mapping into the JVMTI idea of thread suspension difficult. We have decided to split the + // difference and ensure that JVMTI tries to treat suspension as the boolean flag as much as + // possible with the suspend/resume methods but only do best effort. On the other hand + // GetThreadState will be totally accurate as much as possible. This means that calling + // ResumeThread on a thread that has state JVMTI_THREAD_STATE_SUSPENDED will not necessarily + // cause the thread to wake up if the thread is suspended for the debugger or gc or something. + static jvmtiError SuspendSelf(art::Thread* self) + REQUIRES(!art::Locks::mutator_lock_, !art::Locks::user_code_suspension_lock_); + static jvmtiError SuspendOther(art::Thread* self, jthread target_jthread, art::Thread* target) + REQUIRES(!art::Locks::mutator_lock_, !art::Locks::user_code_suspension_lock_); + static art::ArtField* context_class_loader_; }; diff --git a/runtime/quicken_info.h b/runtime/quicken_info.h new file mode 100644 index 0000000000..5b72468fcd --- /dev/null +++ b/runtime/quicken_info.h @@ -0,0 +1,55 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_RUNTIME_QUICKEN_INFO_H_ +#define ART_RUNTIME_QUICKEN_INFO_H_ + +#include "dex_instruction.h" + +namespace art { + +// QuickenInfoTable is a table of 16 bit dex indices. There is one slot fo every instruction that is +// possibly dequickenable. +class QuickenInfoTable { + public: + explicit QuickenInfoTable(const uint8_t* data) : data_(data) {} + + bool IsNull() const { + return data_ == nullptr; + } + + uint16_t GetData(size_t index) const { + return data_[index * 2] | (static_cast<uint16_t>(data_[index * 2 + 1]) << 8); + } + + // Returns true if the dex instruction has an index in the table. (maybe dequickenable). + static bool NeedsIndexForInstruction(const Instruction* inst) { + return inst->IsQuickened() || inst->Opcode() == Instruction::NOP; + } + + static size_t NumberOfIndices(size_t bytes) { + return bytes / sizeof(uint16_t); + } + + private: + const uint8_t* const data_; + + DISALLOW_COPY_AND_ASSIGN(QuickenInfoTable); +}; + +} // namespace art + +#endif // ART_RUNTIME_QUICKEN_INFO_H_ diff --git a/runtime/safe_map.h b/runtime/safe_map.h index b54f587715..f29869172e 100644 --- a/runtime/safe_map.h +++ b/runtime/safe_map.h @@ -79,6 +79,9 @@ class SafeMap { iterator lower_bound(const K& k) { return map_.lower_bound(k); } const_iterator lower_bound(const K& k) const { return map_.lower_bound(k); } + iterator upper_bound(const K& k) { return map_.upper_bound(k); } + const_iterator upper_bound(const K& k) const { return map_.upper_bound(k); } + size_type count(const K& k) const { return map_.count(k); } // Note that unlike std::map's operator[], this doesn't return a reference to the value. diff --git a/runtime/signal_catcher.cc b/runtime/signal_catcher.cc index 8c934d52a2..f0b6ee4a58 100644 --- a/runtime/signal_catcher.cc +++ b/runtime/signal_catcher.cc @@ -168,7 +168,7 @@ void SignalCatcher::Output(const std::string& s) { } #if defined(ART_TARGET_ANDROID) - if (!tombstoned_notify_completion(tombstone_fd)) { + if (use_tombstoned_stack_trace_fd_ && !tombstoned_notify_completion(tombstone_fd)) { LOG(WARNING) << "Unable to notify tombstoned of dump completion."; } #endif diff --git a/runtime/suspend_reason.h b/runtime/suspend_reason.h index 27c4d3207b..289a1a4fb3 100644 --- a/runtime/suspend_reason.h +++ b/runtime/suspend_reason.h @@ -28,6 +28,8 @@ enum class SuspendReason { kInternal, // Suspending for debugger (code in Dbg::*, runtime/jdwp/, etc.). kForDebugger, + // Suspending due to non-runtime, user controlled, code. (For example Thread#Suspend()). + kForUserCode, }; std::ostream& operator<<(std::ostream& os, const SuspendReason& thread); diff --git a/runtime/thread-inl.h b/runtime/thread-inl.h index 95608b5f63..b5a962691b 100644 --- a/runtime/thread-inl.h +++ b/runtime/thread-inl.h @@ -121,10 +121,20 @@ inline bool Thread::IsThreadSuspensionAllowable() const { return false; } for (int i = kLockLevelCount - 1; i >= 0; --i) { - if (i != kMutatorLock && GetHeldMutex(static_cast<LockLevel>(i)) != nullptr) { + if (i != kMutatorLock && + i != kUserCodeSuspensionLock && + GetHeldMutex(static_cast<LockLevel>(i)) != nullptr) { return false; } } + // Thread autoanalysis isn't able to understand that the GetHeldMutex(...) or AssertHeld means we + // have the mutex meaning we need to do this hack. + auto is_suspending_for_user_code = [this]() NO_THREAD_SAFETY_ANALYSIS { + return tls32_.user_code_suspend_count != 0; + }; + if (GetHeldMutex(kUserCodeSuspensionLock) != nullptr && is_suspending_for_user_code()) { + return false; + } return true; } @@ -136,8 +146,9 @@ inline void Thread::AssertThreadSuspensionIsAllowable(bool check_locks) const { if (check_locks) { bool bad_mutexes_held = false; for (int i = kLockLevelCount - 1; i >= 0; --i) { - // We expect no locks except the mutator_lock_ or thread list suspend thread lock. - if (i != kMutatorLock) { + // We expect no locks except the mutator_lock_. User code suspension lock is OK as long as + // we aren't going to be held suspended due to SuspendReason::kForUserCode. + if (i != kMutatorLock && i != kUserCodeSuspensionLock) { BaseMutex* held_mutex = GetHeldMutex(static_cast<LockLevel>(i)); if (held_mutex != nullptr) { LOG(ERROR) << "holding \"" << held_mutex->GetName() @@ -146,6 +157,19 @@ inline void Thread::AssertThreadSuspensionIsAllowable(bool check_locks) const { } } } + // Make sure that if we hold the user_code_suspension_lock_ we aren't suspending due to + // user_code_suspend_count which would prevent the thread from ever waking up. Thread + // autoanalysis isn't able to understand that the GetHeldMutex(...) or AssertHeld means we + // have the mutex meaning we need to do this hack. + auto is_suspending_for_user_code = [this]() NO_THREAD_SAFETY_ANALYSIS { + return tls32_.user_code_suspend_count != 0; + }; + if (GetHeldMutex(kUserCodeSuspensionLock) != nullptr && is_suspending_for_user_code()) { + LOG(ERROR) << "suspending due to user-code while holding \"" + << Locks::user_code_suspension_lock_->GetName() << "\"! Thread would never " + << "wake up."; + bad_mutexes_held = true; + } if (gAborting == 0) { CHECK(!bad_mutexes_held); } diff --git a/runtime/thread.cc b/runtime/thread.cc index 36ecd3398c..004b68e204 100644 --- a/runtime/thread.cc +++ b/runtime/thread.cc @@ -1208,6 +1208,15 @@ bool Thread::ModifySuspendCountInternal(Thread* self, Locks::thread_list_lock_->AssertHeld(self); } } + // User code suspensions need to be checked more closely since they originate from code outside of + // the runtime's control. + if (UNLIKELY(reason == SuspendReason::kForUserCode)) { + Locks::user_code_suspension_lock_->AssertHeld(self); + if (UNLIKELY(delta + tls32_.user_code_suspend_count < 0)) { + LOG(ERROR) << "attempting to modify suspend count in an illegal way."; + return false; + } + } if (UNLIKELY(delta < 0 && tls32_.suspend_count <= 0)) { UnsafeLogFatalForSuspendCount(self, this); return false; @@ -1241,6 +1250,9 @@ bool Thread::ModifySuspendCountInternal(Thread* self, case SuspendReason::kForDebugger: tls32_.debug_suspend_count += delta; break; + case SuspendReason::kForUserCode: + tls32_.user_code_suspend_count += delta; + break; case SuspendReason::kInternal: break; } @@ -2134,6 +2146,10 @@ void Thread::Destroy() { ScopedObjectAccess soa(self); // We may need to call user-supplied managed code, do this before final clean-up. HandleUncaughtExceptions(soa); + Runtime* runtime = Runtime::Current(); + if (runtime != nullptr) { + runtime->GetRuntimeCallbacks()->ThreadDeath(self); + } RemoveFromThreadGroup(soa); // this.nativePeer = 0; @@ -2144,11 +2160,6 @@ void Thread::Destroy() { jni::DecodeArtField(WellKnownClasses::java_lang_Thread_nativePeer) ->SetLong<false>(tlsPtr_.opeer, 0); } - Runtime* runtime = Runtime::Current(); - if (runtime != nullptr) { - runtime->GetRuntimeCallbacks()->ThreadDeath(self); - } - // Thread.join() is implemented as an Object.wait() on the Thread.lock object. Signal anyone // who is waiting. @@ -2862,6 +2873,7 @@ void Thread::DumpThreadOffset(std::ostream& os, uint32_t offset) { DO_THREAD_OFFSET(SelfOffset<ptr_size>(), "self") DO_THREAD_OFFSET(StackEndOffset<ptr_size>(), "stack_end") DO_THREAD_OFFSET(ThinLockIdOffset<ptr_size>(), "thin_lock_thread_id") + DO_THREAD_OFFSET(IsGcMarkingOffset<ptr_size>(), "is_gc_marking") DO_THREAD_OFFSET(TopOfManagedStackOffset<ptr_size>(), "top_quick_frame_method") DO_THREAD_OFFSET(TopShadowFrameOffset<ptr_size>(), "top_shadow_frame") DO_THREAD_OFFSET(TopHandleScopeOffset<ptr_size>(), "top_handle_scope") diff --git a/runtime/thread.h b/runtime/thread.h index e785ddc803..e1102ed322 100644 --- a/runtime/thread.h +++ b/runtime/thread.h @@ -228,6 +228,11 @@ class Thread { return tls32_.suspend_count; } + int GetUserCodeSuspendCount() const REQUIRES(Locks::thread_suspend_count_lock_, + Locks::user_code_suspension_lock_) { + return tls32_.user_code_suspend_count; + } + int GetDebugSuspendCount() const REQUIRES(Locks::thread_suspend_count_lock_) { return tls32_.debug_suspend_count; } @@ -656,6 +661,17 @@ class Thread { OFFSETOF_MEMBER(tls_ptr_sized_values, jni_entrypoints) + jni_entrypoint_offset); } + // Return the entry point offset integer value for ReadBarrierMarkRegX, where X is `reg`. + template <PointerSize pointer_size> + static int32_t ReadBarrierMarkEntryPointsOffset(size_t reg) { + // The entry point list defines 30 ReadBarrierMarkRegX entry points. + DCHECK_LT(reg, 30u); + // The ReadBarrierMarkRegX entry points are ordered by increasing + // register number in Thread::tls_Ptr_.quick_entrypoints. + return QUICK_ENTRYPOINT_OFFSET(pointer_size, pReadBarrierMarkReg00).Int32Value() + + static_cast<size_t>(pointer_size) * reg; + } + template<PointerSize pointer_size> static ThreadOffset<pointer_size> SelfOffset() { return ThreadOffsetFromTlsPtr<pointer_size>(OFFSETOF_MEMBER(tls_ptr_sized_values, self)); @@ -1166,11 +1182,11 @@ class Thread { return debug_disallow_read_barrier_; } - const void* GetCustomTLS() const { + void* GetCustomTLS() const REQUIRES(Locks::thread_list_lock_) { return custom_tls_; } - void SetCustomTLS(const void* data) { + void SetCustomTLS(void* data) REQUIRES(Locks::thread_list_lock_) { custom_tls_ = data; } @@ -1381,7 +1397,7 @@ class Thread { thread_exit_check_count(0), handling_signal_(false), is_transitioning_to_runnable(false), ready_for_debug_invoke(false), debug_method_entry_(false), is_gc_marking(false), weak_ref_access_enabled(true), - disable_thread_flip_count(0) { + disable_thread_flip_count(0), user_code_suspend_count(0) { } union StateAndFlags state_and_flags; @@ -1456,6 +1472,12 @@ class Thread { // levels of (nested) JNI critical sections the thread is in and is used to detect a nested JNI // critical section enter. uint32_t disable_thread_flip_count; + + // How much of 'suspend_count_' is by request of user code, used to distinguish threads + // suspended by the runtime from those suspended by user code. + // This should have GUARDED_BY(Locks::user_code_suspension_lock_) but auto analysis cannot be + // told that AssertHeld should be good enough. + int user_code_suspend_count GUARDED_BY(Locks::thread_suspend_count_lock_); } tls32_; struct PACKED(8) tls_64bit_sized_values { @@ -1661,7 +1683,7 @@ class Thread { // Custom TLS field that can be used by plugins. // TODO: Generalize once we have more plugins. - const void* custom_tls_; + void* custom_tls_; // True if the thread is allowed to call back into java (for e.g. during class resolution). // By default this is true. diff --git a/runtime/thread_list.cc b/runtime/thread_list.cc index fc767ed899..9c938ffe18 100644 --- a/runtime/thread_list.cc +++ b/runtime/thread_list.cc @@ -828,7 +828,7 @@ void ThreadList::ResumeAll() { } } -void ThreadList::Resume(Thread* thread, SuspendReason reason) { +bool ThreadList::Resume(Thread* thread, SuspendReason reason) { // This assumes there was an ATRACE_BEGIN when we suspended the thread. ATRACE_END(); @@ -841,16 +841,23 @@ void ThreadList::Resume(Thread* thread, SuspendReason reason) { MutexLock mu(self, *Locks::thread_list_lock_); // To check IsSuspended. MutexLock mu2(self, *Locks::thread_suspend_count_lock_); - DCHECK(thread->IsSuspended()); + if (UNLIKELY(!thread->IsSuspended())) { + LOG(ERROR) << "Resume(" << reinterpret_cast<void*>(thread) + << ") thread not suspended"; + return false; + } if (!Contains(thread)) { // We only expect threads within the thread-list to have been suspended otherwise we can't // stop such threads from delete-ing themselves. LOG(ERROR) << "Resume(" << reinterpret_cast<void*>(thread) << ") thread not within thread list"; - return; + return false; + } + if (UNLIKELY(!thread->ModifySuspendCount(self, -1, nullptr, reason))) { + LOG(ERROR) << "Resume(" << reinterpret_cast<void*>(thread) + << ") could not modify suspend count."; + return false; } - bool updated = thread->ModifySuspendCount(self, -1, nullptr, reason); - DCHECK(updated); } { @@ -860,6 +867,7 @@ void ThreadList::Resume(Thread* thread, SuspendReason reason) { } VLOG(threads) << "Resume(" << reinterpret_cast<void*>(thread) << ") complete"; + return true; } static void ThreadSuspendByPeerWarning(Thread* self, diff --git a/runtime/thread_list.h b/runtime/thread_list.h index 41c5e328b0..11f272c48c 100644 --- a/runtime/thread_list.h +++ b/runtime/thread_list.h @@ -65,8 +65,8 @@ class ThreadList { void ResumeAll() REQUIRES(!Locks::thread_list_lock_, !Locks::thread_suspend_count_lock_) UNLOCK_FUNCTION(Locks::mutator_lock_); - void Resume(Thread* thread, SuspendReason reason = SuspendReason::kInternal) - REQUIRES(!Locks::thread_suspend_count_lock_); + bool Resume(Thread* thread, SuspendReason reason = SuspendReason::kInternal) + REQUIRES(!Locks::thread_suspend_count_lock_) WARN_UNUSED; // Suspends all threads and gets exclusive access to the mutator_lock_. // If long_suspend is true, then other threads who try to suspend will never timeout. diff --git a/runtime/utils/dex_cache_arrays_layout-inl.h b/runtime/utils/dex_cache_arrays_layout-inl.h index 95904af011..72f63c6c07 100644 --- a/runtime/utils/dex_cache_arrays_layout-inl.h +++ b/runtime/utils/dex_cache_arrays_layout-inl.h @@ -132,7 +132,7 @@ inline size_t DexCacheArraysLayout::FieldsSize(size_t num_elements) const { if (num_elements < cache_size) { cache_size = num_elements; } - return 2u * static_cast<size_t>(pointer_size_) * num_elements; + return 2u * static_cast<size_t>(pointer_size_) * cache_size; } inline size_t DexCacheArraysLayout::FieldsAlignment() const { diff --git a/runtime/vdex_file.h b/runtime/vdex_file.h index 93d282b9bb..4845c0239a 100644 --- a/runtime/vdex_file.h +++ b/runtime/vdex_file.h @@ -65,8 +65,8 @@ class VdexFile { private: static constexpr uint8_t kVdexMagic[] = { 'v', 'd', 'e', 'x' }; - // Last update: Disable in-place vdex update - static constexpr uint8_t kVdexVersion[] = { '0', '0', '6', '\0' }; + // Last update: Smaller quickening info + static constexpr uint8_t kVdexVersion[] = { '0', '0', '7', '\0' }; uint8_t magic_[4]; uint8_t version_[4]; diff --git a/runtime/well_known_classes.cc b/runtime/well_known_classes.cc index 8d505e2582..f72fdb4b2a 100644 --- a/runtime/well_known_classes.cc +++ b/runtime/well_known_classes.cc @@ -94,6 +94,8 @@ jmethodID WellKnownClasses::java_lang_Float_valueOf; jmethodID WellKnownClasses::java_lang_Integer_valueOf; jmethodID WellKnownClasses::java_lang_invoke_MethodHandle_invoke; jmethodID WellKnownClasses::java_lang_invoke_MethodHandle_invokeExact; +jmethodID WellKnownClasses::java_lang_invoke_MethodHandles_lookup; +jmethodID WellKnownClasses::java_lang_invoke_MethodHandles_Lookup_findConstructor; jmethodID WellKnownClasses::java_lang_Long_valueOf; jmethodID WellKnownClasses::java_lang_ref_FinalizerReference_add; jmethodID WellKnownClasses::java_lang_ref_ReferenceQueue_add; @@ -173,8 +175,8 @@ static jfieldID CacheField(JNIEnv* env, jclass c, bool is_static, return fid; } -jmethodID CacheMethod(JNIEnv* env, jclass c, bool is_static, - const char* name, const char* signature) { +static jmethodID CacheMethod(JNIEnv* env, jclass c, bool is_static, + const char* name, const char* signature) { jmethodID mid = is_static ? env->GetStaticMethodID(c, name, signature) : env->GetMethodID(c, name, signature); if (mid == nullptr) { @@ -190,6 +192,12 @@ jmethodID CacheMethod(JNIEnv* env, jclass c, bool is_static, return mid; } +static jmethodID CacheMethod(JNIEnv* env, const char* klass, bool is_static, + const char* name, const char* signature) { + ScopedLocalRef<jclass> java_class(env, env->FindClass(klass)); + return CacheMethod(env, java_class.get(), is_static, name, signature); +} + static jmethodID CachePrimitiveBoxingMethod(JNIEnv* env, char prim_name, const char* boxed_name) { ScopedLocalRef<jclass> boxed_class(env, env->FindClass(boxed_name)); return CacheMethod(env, boxed_class.get(), true, "valueOf", @@ -322,16 +330,12 @@ void WellKnownClasses::Init(JNIEnv* env) { java_lang_Daemons_requestHeapTrim = CacheMethod(env, java_lang_Daemons, true, "requestHeapTrim", "()V"); java_lang_Daemons_start = CacheMethod(env, java_lang_Daemons, true, "start", "()V"); java_lang_Daemons_stop = CacheMethod(env, java_lang_Daemons, true, "stop", "()V"); - java_lang_invoke_MethodHandle_invoke = - CacheMethod(env, java_lang_invoke_MethodHandle, false, - "invoke", "([Ljava/lang/Object;)Ljava/lang/Object;"); - java_lang_invoke_MethodHandle_invokeExact = - CacheMethod(env, java_lang_invoke_MethodHandle, false, - "invokeExact", "([Ljava/lang/Object;)Ljava/lang/Object;"); - ScopedLocalRef<jclass> java_lang_ref_FinalizerReference(env, env->FindClass("java/lang/ref/FinalizerReference")); - java_lang_ref_FinalizerReference_add = CacheMethod(env, java_lang_ref_FinalizerReference.get(), true, "add", "(Ljava/lang/Object;)V"); - ScopedLocalRef<jclass> java_lang_ref_ReferenceQueue(env, env->FindClass("java/lang/ref/ReferenceQueue")); - java_lang_ref_ReferenceQueue_add = CacheMethod(env, java_lang_ref_ReferenceQueue.get(), true, "add", "(Ljava/lang/ref/Reference;)V"); + java_lang_invoke_MethodHandle_invoke = CacheMethod(env, java_lang_invoke_MethodHandle, false, "invoke", "([Ljava/lang/Object;)Ljava/lang/Object;"); + java_lang_invoke_MethodHandle_invokeExact = CacheMethod(env, java_lang_invoke_MethodHandle, false, "invokeExact", "([Ljava/lang/Object;)Ljava/lang/Object;"); + java_lang_invoke_MethodHandles_lookup = CacheMethod(env, "java/lang/invoke/MethodHandles", true, "lookup", "()Ljava/lang/invoke/MethodHandles$Lookup;"); + java_lang_invoke_MethodHandles_Lookup_findConstructor = CacheMethod(env, "java/lang/invoke/MethodHandles$Lookup", false, "findConstructor", "(Ljava/lang/Class;Ljava/lang/invoke/MethodType;)Ljava/lang/invoke/MethodHandle;"); + java_lang_ref_FinalizerReference_add = CacheMethod(env, "java/lang/ref/FinalizerReference", true, "add", "(Ljava/lang/Object;)V"); + java_lang_ref_ReferenceQueue_add = CacheMethod(env, "java/lang/ref/ReferenceQueue", true, "add", "(Ljava/lang/ref/Reference;)V"); java_lang_reflect_Parameter_init = CacheMethod(env, java_lang_reflect_Parameter, false, "<init>", "(Ljava/lang/String;ILjava/lang/reflect/Executable;I)V"); java_lang_String_charAt = CacheMethod(env, java_lang_String, false, "charAt", "(I)C"); diff --git a/runtime/well_known_classes.h b/runtime/well_known_classes.h index c5a16c1c76..2f2f1ad3f4 100644 --- a/runtime/well_known_classes.h +++ b/runtime/well_known_classes.h @@ -33,8 +33,6 @@ class Class; // them up. Similar to libcore's JniConstants (except there's no overlap, so // we keep them separate). -jmethodID CacheMethod(JNIEnv* env, jclass c, bool is_static, const char* name, const char* signature); - struct WellKnownClasses { public: static void Init(JNIEnv* env); // Run before native methods are registered. @@ -104,6 +102,8 @@ struct WellKnownClasses { static jmethodID java_lang_Integer_valueOf; static jmethodID java_lang_invoke_MethodHandle_invoke; static jmethodID java_lang_invoke_MethodHandle_invokeExact; + static jmethodID java_lang_invoke_MethodHandles_lookup; + static jmethodID java_lang_invoke_MethodHandles_Lookup_findConstructor; static jmethodID java_lang_Long_valueOf; static jmethodID java_lang_ref_FinalizerReference_add; static jmethodID java_lang_ref_ReferenceQueue_add; diff --git a/test/021-string2/src/Main.java b/test/021-string2/src/Main.java index 194f4a1a7d..3b81d8e623 100644 --- a/test/021-string2/src/Main.java +++ b/test/021-string2/src/Main.java @@ -700,6 +700,11 @@ public class Main { $noinline$constNonAsciiString35Equals("\u0440123456789012345678901234567890123x")); Assert.assertFalse( $noinline$constNonAsciiString35Equals("01234567890123456789012345678901234")); + + // Regression test for incorrectly creating an uncompressed string when the + // string should be compressed. Only the low 8 bits are relevant but the whole + // `hibyte` was erroneously tested. Bug: 63661357 + Assert.assertTrue("A".equals(new String(new byte[] { (byte)'A' }, /* hibyte */ 0x100))); } public static boolean $noinline$equalsConstString0(String s) { diff --git a/test/064-field-access/jasmin/SubClassUsingInaccessibleField.j b/test/064-field-access/jasmin/SubClassUsingInaccessibleField.j new file mode 100644 index 0000000000..3422f85434 --- /dev/null +++ b/test/064-field-access/jasmin/SubClassUsingInaccessibleField.j @@ -0,0 +1,36 @@ +; Copyright (C) 2017 The Android Open Source Project +; +; Licensed under the Apache License, Version 2.0 (the "License"); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, software +; distributed under the License is distributed on an "AS IS" BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions and +; limitations under the License. + +.class public SubClassUsingInaccessibleField +.super other/PublicClass + +.method public <init>()V + .limit stack 1 + .limit locals 1 + aload_0 + invokespecial other/PublicClass/<init>()V + return +.end method + +; Regression test for compiler DCHECK() failure (bogus check) when referencing +; a package-private field from an indirectly inherited package-private class, +; using this very class as the declaring class in the FieldId, bug: 27684368 . +.method public test()I + .limit stack 1 + .limit locals 1 + aload_0 + getfield SubClassUsingInaccessibleField/otherProtectedClassPackageIntInstanceField I + ireturn +.end method + diff --git a/test/064-field-access/smali/SubClassUsingInaccessibleField.smali b/test/064-field-access/smali/SubClassUsingInaccessibleField.smali deleted file mode 100644 index 224b431e2e..0000000000 --- a/test/064-field-access/smali/SubClassUsingInaccessibleField.smali +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright (C) 2016 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -.class public LSubClassUsingInaccessibleField; - -.super Lother/PublicClass; - -.method public constructor <init>()V - .registers 1 - invoke-direct {p0}, Lother/PublicClass;-><init>()V - return-void -.end method - -# Regression test for compiler DCHECK() failure (bogus check) when referencing -# a package-private field from an indirectly inherited package-private class, -# using this very class as the declaring class in the FieldId, bug: 27684368 . -.method public test()I - .registers 2 - iget v0, p0, LSubClassUsingInaccessibleField;->otherProtectedClassPackageIntInstanceField:I - return v0 -.end method diff --git a/test/079-phantom/src/Bitmap.java b/test/079-phantom/src/Bitmap.java index ff43749e76..0d6e2d84e7 100644 --- a/test/079-phantom/src/Bitmap.java +++ b/test/079-phantom/src/Bitmap.java @@ -17,6 +17,7 @@ import java.lang.ref.ReferenceQueue; import java.lang.ref.PhantomReference; import java.util.ArrayList; +import java.util.concurrent.CountDownLatch; public class Bitmap { String mName; /* for debugging */ @@ -76,11 +77,14 @@ public class Bitmap { PhantomWrapper phan = new PhantomWrapper(wrapper, sPhantomQueue, nativeData); sPhantomList.add(phan); + wrapper.mPhantomWrapper = phan; return wrapper; } - static void freeNativeStorage(int nativeDataPtr) { + static void freeNativeStorage(int nativeDataPtr, CountDownLatch freeSignal) { System.out.println("freeNativeStorage: " + nativeDataPtr); + // Wake up the main thread that is [or will be] blocked until this native data is freed. + freeSignal.countDown(); } /* @@ -93,6 +97,9 @@ public class Bitmap { } public int mNativeData; + // The PhantomWrapper corresponding to this NativeWrapper. + public PhantomWrapper mPhantomWrapper; + /* @Override protected void finalize() throws Throwable { @@ -118,6 +125,8 @@ class PhantomWrapper extends PhantomReference { } public int mNativeData; + // This will be signaled once mNativeData has been freed. + public CountDownLatch mFreeSignal = new CountDownLatch(1); } /* @@ -137,8 +146,7 @@ class BitmapWatcher extends Thread { PhantomWrapper ref = (PhantomWrapper) mQueue.remove(); //System.out.println("dequeued ref " + ref.mNativeData + // " - " + ref); - Bitmap.freeNativeStorage(ref.mNativeData); - //ref.clear(); + Bitmap.freeNativeStorage(ref.mNativeData, ref.mFreeSignal); } catch (InterruptedException ie) { System.out.println("intr"); break; diff --git a/test/079-phantom/src/Main.java b/test/079-phantom/src/Main.java index daead2e1dd..ae2c688603 100644 --- a/test/079-phantom/src/Main.java +++ b/test/079-phantom/src/Main.java @@ -14,8 +14,11 @@ * limitations under the License. */ +import java.util.concurrent.CountDownLatch; + public class Main { Bitmap mBitmap1, mBitmap2, mBitmap3, mBitmap4; + CountDownLatch mFreeSignalA, mFreeSignalB; public static void sleep(int ms) { try { @@ -31,7 +34,6 @@ public class Main { Main main = new Main(); main.run(); - sleep(1000); System.out.println("done"); } @@ -46,22 +48,30 @@ public class Main { System.out.println("nulling 1"); mBitmap1 = null; Runtime.getRuntime().gc(); - sleep(500); + try { + mFreeSignalA.await(); // Block until dataA is definitely freed. + } catch (InterruptedException e) { + System.out.println("got unexpected InterruptedException e: " + e); + } System.out.println("nulling 2"); mBitmap2 = null; Runtime.getRuntime().gc(); - sleep(500); + sleep(200); System.out.println("nulling 3"); mBitmap3 = null; Runtime.getRuntime().gc(); - sleep(500); + sleep(200); System.out.println("nulling 4"); mBitmap4 = null; Runtime.getRuntime().gc(); - sleep(500); + try { + mFreeSignalB.await(); // Block until dataB is definitely freed. + } catch (InterruptedException e) { + System.out.println("got unexpected InterruptedException e: " + e); + } Bitmap.shutDown(); } @@ -77,7 +87,10 @@ public class Main { */ public void createBitmaps() { Bitmap.NativeWrapper dataA = Bitmap.allocNativeStorage(10, 10); + mFreeSignalA = dataA.mPhantomWrapper.mFreeSignal; Bitmap.NativeWrapper dataB = Bitmap.allocNativeStorage(20, 20); + mFreeSignalB = dataB.mPhantomWrapper.mFreeSignal; + mBitmap1 = new Bitmap("one", 10, 10, dataA); mBitmap2 = new Bitmap("two", 20, 20, dataB); mBitmap3 = mBitmap4 = new Bitmap("three/four", 20, 20, dataB); diff --git a/test/1901-get-bytecodes/run b/test/1901-get-bytecodes/run index c6e62ae6cd..e92b873956 100755 --- a/test/1901-get-bytecodes/run +++ b/test/1901-get-bytecodes/run @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright 2016 The Android Open Source Project +# Copyright 2017 The Android Open Source Project # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/test/1901-get-bytecodes/src/art/Test1901.java b/test/1901-get-bytecodes/src/art/Test1901.java index 6940491e3c..9827e3f968 100644 --- a/test/1901-get-bytecodes/src/art/Test1901.java +++ b/test/1901-get-bytecodes/src/art/Test1901.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2016 The Android Open Source Project + * Copyright (C) 2017 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/test/1902-suspend/expected.txt b/test/1902-suspend/expected.txt new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/test/1902-suspend/expected.txt diff --git a/test/1902-suspend/info.txt b/test/1902-suspend/info.txt new file mode 100644 index 0000000000..c49a20f971 --- /dev/null +++ b/test/1902-suspend/info.txt @@ -0,0 +1,2 @@ +Test basic jvmti Suspend/ResumeThread behavior + diff --git a/test/1902-suspend/run b/test/1902-suspend/run new file mode 100755 index 0000000000..e92b873956 --- /dev/null +++ b/test/1902-suspend/run @@ -0,0 +1,17 @@ +#!/bin/bash +# +# Copyright 2017 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +./default-run "$@" --jvmti diff --git a/test/1902-suspend/src/Main.java b/test/1902-suspend/src/Main.java new file mode 100644 index 0000000000..0bc7ba1a39 --- /dev/null +++ b/test/1902-suspend/src/Main.java @@ -0,0 +1,21 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +public class Main { + public static void main(String[] args) throws Exception { + art.Test1902.run(); + } +} diff --git a/test/1902-suspend/src/art/Suspension.java b/test/1902-suspend/src/art/Suspension.java new file mode 100644 index 0000000000..16e62ccac9 --- /dev/null +++ b/test/1902-suspend/src/art/Suspension.java @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package art; + +public class Suspension { + // Suspends a thread using jvmti. + public native static void suspend(Thread thr); + + // Resumes a thread using jvmti. + public native static void resume(Thread thr); + + public native static boolean isSuspended(Thread thr); + + public native static int[] suspendList(Thread... threads); + public native static int[] resumeList(Thread... threads); +} diff --git a/test/1902-suspend/src/art/Test1902.java b/test/1902-suspend/src/art/Test1902.java new file mode 100644 index 0000000000..2bbfacf8ef --- /dev/null +++ b/test/1902-suspend/src/art/Test1902.java @@ -0,0 +1,118 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package art; + +public class Test1902 { + public static final Object lock = new Object(); + + public static volatile boolean OTHER_THREAD_CONTINUE = true; + public static volatile boolean OTHER_THREAD_DID_SOMETHING = true; + public static volatile boolean OTHER_THREAD_STARTED = false; + + public static class OtherThread implements Runnable { + @Override + public void run() { + OTHER_THREAD_STARTED = true; + while (OTHER_THREAD_CONTINUE) { + OTHER_THREAD_DID_SOMETHING = true; + } + } + } + + public static void waitFor(long millis) { + try { + lock.wait(millis); + } catch (Exception e) { + System.out.println("Unexpected error: " + e); + e.printStackTrace(); + } + } + + public static void waitForSuspension(Thread target) { + while (!Suspension.isSuspended(target)) { + waitFor(100); + } + } + + public static void waitForStart() { + while (!OTHER_THREAD_STARTED) { + waitFor(100); + } + } + + + public static void run() { + synchronized (lock) { + Thread other = new Thread(new OtherThread(), "TARGET THREAD"); + try { + other.start(); + + waitForStart(); + + // Try to resume ourself. + try { + Suspension.resume(Thread.currentThread()); + } catch (Exception e) { + if (!e.getMessage().equals("JVMTI_ERROR_THREAD_NOT_SUSPENDED")) { + System.out.println("incorrect error for resuming a non-suspended thread"); + } + } + try { + Suspension.resume(other); + } catch (Exception e) { + if (!e.getMessage().equals("JVMTI_ERROR_THREAD_NOT_SUSPENDED")) { + System.out.println("incorrect error for resuming a non-suspended thread"); + } + } + + Suspension.suspend(other); + // Wait 1 second for the other thread to suspend. + waitForSuspension(other); + OTHER_THREAD_DID_SOMETHING = false; + // Wait a second to see if anything happens. + waitFor(1000); + + if (OTHER_THREAD_DID_SOMETHING) { + System.out.println("Looks like other thread did something while suspended!"); + } + // Resume always. + Suspension.resume(other); + + // Wait another second. + waitFor(1000); + + if (!OTHER_THREAD_DID_SOMETHING) { + System.out.println("Doesn't look like the thread unsuspended!"); + } + + // Stop the other thread. + OTHER_THREAD_CONTINUE = false; + // Wait for 1 second for it to die. + other.join(1000); + + if (other.isAlive()) { + System.out.println("other thread didn't terminate in a reasonable time!"); + Runtime.getRuntime().halt(1); + } + } catch (Throwable t) { + System.out.println("something was thrown. Runtime might be in unrecoverable state: " + t); + t.printStackTrace(); + Runtime.getRuntime().halt(2); + } + } + } +} diff --git a/test/1903-suspend-self/expected.txt b/test/1903-suspend-self/expected.txt new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/test/1903-suspend-self/expected.txt diff --git a/test/1903-suspend-self/info.txt b/test/1903-suspend-self/info.txt new file mode 100644 index 0000000000..779becc020 --- /dev/null +++ b/test/1903-suspend-self/info.txt @@ -0,0 +1 @@ +Test jvmti suspend/resume of the current thread. diff --git a/test/1903-suspend-self/run b/test/1903-suspend-self/run new file mode 100755 index 0000000000..e92b873956 --- /dev/null +++ b/test/1903-suspend-self/run @@ -0,0 +1,17 @@ +#!/bin/bash +# +# Copyright 2017 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +./default-run "$@" --jvmti diff --git a/test/1903-suspend-self/src/Main.java b/test/1903-suspend-self/src/Main.java new file mode 100644 index 0000000000..bd2028f323 --- /dev/null +++ b/test/1903-suspend-self/src/Main.java @@ -0,0 +1,21 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +public class Main { + public static void main(String[] args) throws Exception { + art.Test1903.run(); + } +} diff --git a/test/1903-suspend-self/src/art/Suspension.java b/test/1903-suspend-self/src/art/Suspension.java new file mode 100644 index 0000000000..16e62ccac9 --- /dev/null +++ b/test/1903-suspend-self/src/art/Suspension.java @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package art; + +public class Suspension { + // Suspends a thread using jvmti. + public native static void suspend(Thread thr); + + // Resumes a thread using jvmti. + public native static void resume(Thread thr); + + public native static boolean isSuspended(Thread thr); + + public native static int[] suspendList(Thread... threads); + public native static int[] resumeList(Thread... threads); +} diff --git a/test/1903-suspend-self/src/art/Test1903.java b/test/1903-suspend-self/src/art/Test1903.java new file mode 100644 index 0000000000..cf2a55c44a --- /dev/null +++ b/test/1903-suspend-self/src/art/Test1903.java @@ -0,0 +1,91 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package art; + +public class Test1903 { + public static final Object lock = new Object(); + + public static volatile boolean OTHER_THREAD_CONTINUE = true; + public static volatile boolean OTHER_THREAD_DID_SOMETHING = true; + public static volatile boolean OTHER_THREAD_STARTED = false; + public static volatile boolean OTHER_THREAD_RESUMED = false; + + public static class OtherThread implements Runnable { + @Override + public void run() { + // Wake up main thread. + OTHER_THREAD_STARTED = true; + try { + Suspension.suspend(Thread.currentThread()); + OTHER_THREAD_RESUMED = true; + } catch (Throwable t) { + System.out.println("Unexpected error occurred " + t); + t.printStackTrace(); + Runtime.getRuntime().halt(2); + } + } + } + + public static void waitFor(long millis) { + try { + lock.wait(millis); + } catch (Exception e) { + System.out.println("Unexpected error: " + e); + e.printStackTrace(); + } + } + + public static void waitForSuspension(Thread target) { + while (!Suspension.isSuspended(target)) { + waitFor(100); + } + } + + public static void waitForStart() { + while (!OTHER_THREAD_STARTED) { + waitFor(100); + } + } + + public static void run() { + synchronized (lock) { + Thread other = new Thread(new OtherThread(), "TARGET THREAD"); + try { + other.start(); + + // Wait for the other thread to actually start doing things. + + waitForStart(); + waitForSuspension(other); + + Suspension.resume(other); + for (int i = 0; i < 1000; i++) { + waitFor(100); + if (OTHER_THREAD_RESUMED) { + return; + } + } + System.out.println("Failed to resume thread!"); + Runtime.getRuntime().halt(4); + } catch (Throwable t) { + System.out.println("something was thrown. Runtime might be in unrecoverable state: " + t); + t.printStackTrace(); + Runtime.getRuntime().halt(2); + } + } + } +} diff --git a/test/1904-double-suspend/expected.txt b/test/1904-double-suspend/expected.txt new file mode 100644 index 0000000000..321b8a34f1 --- /dev/null +++ b/test/1904-double-suspend/expected.txt @@ -0,0 +1 @@ +Got exception JVMTI_ERROR_THREAD_SUSPENDED diff --git a/test/1904-double-suspend/info.txt b/test/1904-double-suspend/info.txt new file mode 100644 index 0000000000..5d2415b29d --- /dev/null +++ b/test/1904-double-suspend/info.txt @@ -0,0 +1 @@ +Test jvmti suspending a thread more than once. diff --git a/test/1904-double-suspend/run b/test/1904-double-suspend/run new file mode 100755 index 0000000000..e92b873956 --- /dev/null +++ b/test/1904-double-suspend/run @@ -0,0 +1,17 @@ +#!/bin/bash +# +# Copyright 2017 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +./default-run "$@" --jvmti diff --git a/test/1904-double-suspend/src/Main.java b/test/1904-double-suspend/src/Main.java new file mode 100644 index 0000000000..a0e71c6a45 --- /dev/null +++ b/test/1904-double-suspend/src/Main.java @@ -0,0 +1,21 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +public class Main { + public static void main(String[] args) throws Exception { + art.Test1904.run(); + } +} diff --git a/test/1904-double-suspend/src/art/Suspension.java b/test/1904-double-suspend/src/art/Suspension.java new file mode 100644 index 0000000000..16e62ccac9 --- /dev/null +++ b/test/1904-double-suspend/src/art/Suspension.java @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package art; + +public class Suspension { + // Suspends a thread using jvmti. + public native static void suspend(Thread thr); + + // Resumes a thread using jvmti. + public native static void resume(Thread thr); + + public native static boolean isSuspended(Thread thr); + + public native static int[] suspendList(Thread... threads); + public native static int[] resumeList(Thread... threads); +} diff --git a/test/1904-double-suspend/src/art/Test1904.java b/test/1904-double-suspend/src/art/Test1904.java new file mode 100644 index 0000000000..8a52aa0e73 --- /dev/null +++ b/test/1904-double-suspend/src/art/Test1904.java @@ -0,0 +1,109 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package art; + +public class Test1904 { + public static final Object lock = new Object(); + + public static volatile boolean OTHER_THREAD_CONTINUE = true; + public static volatile boolean OTHER_THREAD_DID_SOMETHING = true; + public static volatile boolean OTHER_THREAD_STARTED = false; + + public static class OtherThread implements Runnable { + @Override + public void run() { + OTHER_THREAD_STARTED = true; + while (OTHER_THREAD_CONTINUE) { + OTHER_THREAD_DID_SOMETHING = true; + } + } + } + + public static void waitFor(long millis) { + try { + lock.wait(millis); + } catch (Exception e) { + System.out.println("Unexpected error: " + e); + e.printStackTrace(); + } + } + + public static void waitForSuspension(Thread target) { + while (!Suspension.isSuspended(target)) { + waitFor(100); + } + } + + public static void waitForStart() { + while (!OTHER_THREAD_STARTED) { + waitFor(100); + } + } + + + public static void run() { + synchronized (lock) { + Thread other = new Thread(new OtherThread(), "TARGET THREAD"); + try { + other.start(); + + waitForStart(); + + Suspension.suspend(other); + + waitForSuspension(other); + OTHER_THREAD_DID_SOMETHING = false; + // Wait a second to see if anything happens. + waitFor(1000); + + if (OTHER_THREAD_DID_SOMETHING) { + System.out.println("Looks like other thread did something while suspended!"); + } + + try { + Suspension.suspend(other); + } catch (Exception e) { + System.out.println("Got exception " + e.getMessage()); + } + + // Resume always. + Suspension.resume(other); + + // Wait another second. + waitFor(1000); + + if (!OTHER_THREAD_DID_SOMETHING) { + System.out.println("Doesn't look like the thread unsuspended!"); + } + + // Stop the other thread. + OTHER_THREAD_CONTINUE = false; + // Wait for 1 second for it to die. + other.join(1000); + + if (other.isAlive()) { + System.out.println("other thread didn't terminate in a reasonable time!"); + Runtime.getRuntime().halt(1); + } + } catch (Throwable t) { + System.out.println("something was thrown. Runtime might be in unrecoverable state: " + t); + t.printStackTrace(); + Runtime.getRuntime().halt(2); + } + } + } +} diff --git a/test/1905-suspend-native/expected.txt b/test/1905-suspend-native/expected.txt new file mode 100644 index 0000000000..43b2669fab --- /dev/null +++ b/test/1905-suspend-native/expected.txt @@ -0,0 +1,8 @@ +Resumer: isNativeThreadSpinning() = true +Resumer: isSuspended(spinner) = false +Resumer: Suspended spinner while native spinning +Resumer: isNativeThreadSpinning() = true +Resumer: isSuspended(spinner) = true +Resumer: resumed spinner while native spinning +Resumer: isNativeThreadSpinning() = true +Resumer: isSuspended(spinner) = false diff --git a/test/1905-suspend-native/info.txt b/test/1905-suspend-native/info.txt new file mode 100644 index 0000000000..3545d5937c --- /dev/null +++ b/test/1905-suspend-native/info.txt @@ -0,0 +1 @@ +Tests jvmti suspending of a thread that is spinning in native code. diff --git a/test/1905-suspend-native/native_suspend.cc b/test/1905-suspend-native/native_suspend.cc new file mode 100644 index 0000000000..95b8da22ce --- /dev/null +++ b/test/1905-suspend-native/native_suspend.cc @@ -0,0 +1,51 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <atomic> + +#include "android-base/logging.h" +#include "jni.h" +#include "scoped_local_ref.h" +#include "scoped_primitive_array.h" + +#include "jvmti.h" + +// Test infrastructure +#include "jvmti_helper.h" +#include "test_env.h" + +namespace art { +namespace Test1905NativeSuspend { + +std::atomic<bool> done(false); +std::atomic<bool> started(false); + +extern "C" JNIEXPORT void JNICALL Java_art_Test1905_nativeSpin(JNIEnv*, jclass) { + while (!done.load()) { + started.store(true); + } +} + +extern "C" JNIEXPORT jboolean JNICALL Java_art_Test1905_isNativeThreadSpinning(JNIEnv*, jclass) { + return started.load(); +} + +extern "C" JNIEXPORT void JNICALL Java_art_Test1905_nativeResume(JNIEnv*, jclass) { + done.store(true); +} + +} // namespace Test1905NativeSuspend +} // namespace art diff --git a/test/1905-suspend-native/run b/test/1905-suspend-native/run new file mode 100755 index 0000000000..e92b873956 --- /dev/null +++ b/test/1905-suspend-native/run @@ -0,0 +1,17 @@ +#!/bin/bash +# +# Copyright 2017 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +./default-run "$@" --jvmti diff --git a/test/1905-suspend-native/src/Main.java b/test/1905-suspend-native/src/Main.java new file mode 100644 index 0000000000..42c02d0f42 --- /dev/null +++ b/test/1905-suspend-native/src/Main.java @@ -0,0 +1,21 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +public class Main { + public static void main(String[] args) throws Exception { + art.Test1905.run(); + } +} diff --git a/test/1905-suspend-native/src/art/Suspension.java b/test/1905-suspend-native/src/art/Suspension.java new file mode 100644 index 0000000000..16e62ccac9 --- /dev/null +++ b/test/1905-suspend-native/src/art/Suspension.java @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package art; + +public class Suspension { + // Suspends a thread using jvmti. + public native static void suspend(Thread thr); + + // Resumes a thread using jvmti. + public native static void resume(Thread thr); + + public native static boolean isSuspended(Thread thr); + + public native static int[] suspendList(Thread... threads); + public native static int[] resumeList(Thread... threads); +} diff --git a/test/1905-suspend-native/src/art/Test1905.java b/test/1905-suspend-native/src/art/Test1905.java new file mode 100644 index 0000000000..ec3901935f --- /dev/null +++ b/test/1905-suspend-native/src/art/Test1905.java @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package art; + +public class Test1905 { + public static void run() throws Exception { + final Thread spinner = new Thread(() -> { + nativeSpin(); + }, "Spinner"); + + final Thread resumer = new Thread(() -> { + String me = Thread.currentThread().getName(); + + // wait for the other thread to start spinning. + while (!isNativeThreadSpinning()) { } + + System.out.println(me + ": isNativeThreadSpinning() = " + isNativeThreadSpinning()); + System.out.println(me + ": isSuspended(spinner) = " + Suspension.isSuspended(spinner)); + + // Suspend it from java. + Suspension.suspend(spinner); + + System.out.println(me + ": Suspended spinner while native spinning"); + System.out.println(me + ": isNativeThreadSpinning() = " + isNativeThreadSpinning()); + System.out.println(me + ": isSuspended(spinner) = " + Suspension.isSuspended(spinner)); + + // Resume it from java. It is still native spinning. + Suspension.resume(spinner); + + System.out.println(me + ": resumed spinner while native spinning"); + System.out.println(me + ": isNativeThreadSpinning() = " + isNativeThreadSpinning()); + System.out.println(me + ": isSuspended(spinner) = " + Suspension.isSuspended(spinner)); + nativeResume(); + }, "Resumer"); + + spinner.start(); + resumer.start(); + + spinner.join(); + resumer.join(); + } + + public static native void nativeSpin(); + public static native void nativeResume(); + public static native boolean isNativeThreadSpinning(); +} diff --git a/test/1906-suspend-list-me-first/expected.txt b/test/1906-suspend-list-me-first/expected.txt new file mode 100644 index 0000000000..503d728cb0 --- /dev/null +++ b/test/1906-suspend-list-me-first/expected.txt @@ -0,0 +1 @@ +Second thread suspended before first thread suspended self! diff --git a/test/1906-suspend-list-me-first/info.txt b/test/1906-suspend-list-me-first/info.txt new file mode 100644 index 0000000000..2b2f4e15ae --- /dev/null +++ b/test/1906-suspend-list-me-first/info.txt @@ -0,0 +1 @@ +Test jvmti SuspendThreadList with the current thread as the first thread in the list. diff --git a/test/1906-suspend-list-me-first/run b/test/1906-suspend-list-me-first/run new file mode 100755 index 0000000000..e92b873956 --- /dev/null +++ b/test/1906-suspend-list-me-first/run @@ -0,0 +1,17 @@ +#!/bin/bash +# +# Copyright 2017 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +./default-run "$@" --jvmti diff --git a/test/1906-suspend-list-me-first/src/Main.java b/test/1906-suspend-list-me-first/src/Main.java new file mode 100644 index 0000000000..1c8432c27a --- /dev/null +++ b/test/1906-suspend-list-me-first/src/Main.java @@ -0,0 +1,21 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +public class Main { + public static void main(String[] args) throws Exception { + art.Test1906.run(); + } +} diff --git a/test/1906-suspend-list-me-first/src/art/Suspension.java b/test/1906-suspend-list-me-first/src/art/Suspension.java new file mode 100644 index 0000000000..16e62ccac9 --- /dev/null +++ b/test/1906-suspend-list-me-first/src/art/Suspension.java @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package art; + +public class Suspension { + // Suspends a thread using jvmti. + public native static void suspend(Thread thr); + + // Resumes a thread using jvmti. + public native static void resume(Thread thr); + + public native static boolean isSuspended(Thread thr); + + public native static int[] suspendList(Thread... threads); + public native static int[] resumeList(Thread... threads); +} diff --git a/test/1906-suspend-list-me-first/src/art/Test1906.java b/test/1906-suspend-list-me-first/src/art/Test1906.java new file mode 100644 index 0000000000..9bb272eded --- /dev/null +++ b/test/1906-suspend-list-me-first/src/art/Test1906.java @@ -0,0 +1,89 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package art; + +public class Test1906 { + public static final Object lock = new Object(); + + public static volatile boolean SECOND_THREAD_RUN = true; + public static volatile boolean SECOND_THREAD_RUNNING = false; + + public static void waitFor(long millis) { + try { + lock.wait(millis); + } catch (Exception e) { + System.out.println("Unexpected error: " + e); + e.printStackTrace(); + } + } + + public static void waitForSuspension(Thread target) { + while (!Suspension.isSuspended(target)) { + waitFor(100); + } + } + + public static void run() { + synchronized (lock) { + final Thread second_thread = new Thread( + () -> { + while (SECOND_THREAD_RUN) { SECOND_THREAD_RUNNING = true; } + }, + "SECONDARY THREAD"); + Thread self_thread = new Thread( + () -> { + try { + // Wait for second thread to start doing stuff. + while (!SECOND_THREAD_RUNNING) { } + Suspension.suspendList(Thread.currentThread(), second_thread); + } catch (Throwable t) { + System.out.println("Unexpected error occurred " + t); + t.printStackTrace(); + Runtime.getRuntime().halt(2); + } + }, + "TARGET THREAD"); + try { + second_thread.start(); + self_thread.start(); + + waitForSuspension(self_thread); + + // Wait to see if second thread is running. + SECOND_THREAD_RUNNING = false; + waitFor(1000); + + if (SECOND_THREAD_RUNNING) { + System.out.println("Second thread running after first thread suspended self!"); + } else { + System.out.println("Second thread suspended before first thread suspended self!"); + } + + Suspension.resume(self_thread); + waitForSuspension(second_thread); + Suspension.resume(second_thread); + self_thread.join(); + SECOND_THREAD_RUN = false; + second_thread.join(); + } catch (Throwable t) { + System.out.println("something was thrown. Runtime might be in unrecoverable state: " + t); + t.printStackTrace(); + Runtime.getRuntime().halt(2); + } + } + } +} diff --git a/test/1907-suspend-list-self-twice/expected.txt b/test/1907-suspend-list-self-twice/expected.txt new file mode 100644 index 0000000000..cd9b53f4a8 --- /dev/null +++ b/test/1907-suspend-list-self-twice/expected.txt @@ -0,0 +1,2 @@ +Suspend self twice returned: [0, 14] +Thread was no longer suspended after one resume. diff --git a/test/1907-suspend-list-self-twice/info.txt b/test/1907-suspend-list-self-twice/info.txt new file mode 100644 index 0000000000..923c545421 --- /dev/null +++ b/test/1907-suspend-list-self-twice/info.txt @@ -0,0 +1 @@ +Test jvmti SuspendThreadList with the current thread on it twice. diff --git a/test/1907-suspend-list-self-twice/run b/test/1907-suspend-list-self-twice/run new file mode 100755 index 0000000000..e92b873956 --- /dev/null +++ b/test/1907-suspend-list-self-twice/run @@ -0,0 +1,17 @@ +#!/bin/bash +# +# Copyright 2017 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +./default-run "$@" --jvmti diff --git a/test/1907-suspend-list-self-twice/src/Main.java b/test/1907-suspend-list-self-twice/src/Main.java new file mode 100644 index 0000000000..910848ad1c --- /dev/null +++ b/test/1907-suspend-list-self-twice/src/Main.java @@ -0,0 +1,21 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +public class Main { + public static void main(String[] args) throws Exception { + art.Test1907.run(); + } +} diff --git a/test/1907-suspend-list-self-twice/src/art/Suspension.java b/test/1907-suspend-list-self-twice/src/art/Suspension.java new file mode 100644 index 0000000000..16e62ccac9 --- /dev/null +++ b/test/1907-suspend-list-self-twice/src/art/Suspension.java @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package art; + +public class Suspension { + // Suspends a thread using jvmti. + public native static void suspend(Thread thr); + + // Resumes a thread using jvmti. + public native static void resume(Thread thr); + + public native static boolean isSuspended(Thread thr); + + public native static int[] suspendList(Thread... threads); + public native static int[] resumeList(Thread... threads); +} diff --git a/test/1907-suspend-list-self-twice/src/art/Test1907.java b/test/1907-suspend-list-self-twice/src/art/Test1907.java new file mode 100644 index 0000000000..504f7f3381 --- /dev/null +++ b/test/1907-suspend-list-self-twice/src/art/Test1907.java @@ -0,0 +1,82 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package art; + +import java.util.Arrays; + +public class Test1907 { + public static final Object lock = new Object(); + + public static void waitFor(long millis) { + try { + lock.wait(millis); + } catch (Exception e) { + System.out.println("Unexpected error: " + e); + e.printStackTrace(); + } + } + + public static void waitForSuspension(Thread target) { + while (!Suspension.isSuspended(target)) { + waitFor(100); + } + } + + public static void run() { + synchronized (lock) { + Thread thrd = new Thread( + () -> { + try { + // Put self twice in the suspend list + System.out.println("Suspend self twice returned: " + + Arrays.toString( + Suspension.suspendList(Thread.currentThread(), Thread.currentThread()))); + } catch (Throwable t) { + System.out.println("Unexpected error occurred " + t); + t.printStackTrace(); + Runtime.getRuntime().halt(2); + } + }, + "TARGET THREAD"); + try { + thrd.start(); + + // Wait for at least one suspend to happen. + waitForSuspension(thrd); + + // Wake it up. + Suspension.resume(thrd); + waitFor(1000); + + // Is it suspended. + if (Suspension.isSuspended(thrd)) { + Suspension.resume(thrd); + thrd.join(); + System.out.println("Thread was still suspended after one resume."); + } else { + thrd.join(); + System.out.println("Thread was no longer suspended after one resume."); + } + + } catch (Throwable t) { + System.out.println("something was thrown. Runtime might be in unrecoverable state: " + t); + t.printStackTrace(); + Runtime.getRuntime().halt(2); + } + } + } +} diff --git a/test/1908-suspend-native-resume-self/expected.txt b/test/1908-suspend-native-resume-self/expected.txt new file mode 100644 index 0000000000..13cc517d68 --- /dev/null +++ b/test/1908-suspend-native-resume-self/expected.txt @@ -0,0 +1,10 @@ +Resumer: isNativeThreadSpinning() = true +Resumer: isSuspended(spinner) = false +Resumer: Suspended spinner while native spinning +Resumer: isNativeThreadSpinning() = true +Resumer: isSuspended(spinner) = true +Resuming other thread +other thread attempting self resume +Resumer: isSuspended(spinner) = true +real resume +other thread resumed. diff --git a/test/1908-suspend-native-resume-self/info.txt b/test/1908-suspend-native-resume-self/info.txt new file mode 100644 index 0000000000..3545d5937c --- /dev/null +++ b/test/1908-suspend-native-resume-self/info.txt @@ -0,0 +1 @@ +Tests jvmti suspending of a thread that is spinning in native code. diff --git a/test/1908-suspend-native-resume-self/native_suspend_resume.cc b/test/1908-suspend-native-resume-self/native_suspend_resume.cc new file mode 100644 index 0000000000..158b22cce6 --- /dev/null +++ b/test/1908-suspend-native-resume-self/native_suspend_resume.cc @@ -0,0 +1,67 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <atomic> + +#include "android-base/logging.h" +#include "jni.h" +#include "scoped_local_ref.h" +#include "scoped_primitive_array.h" + +#include "jvmti.h" + +// Test infrastructure +#include "jvmti_helper.h" +#include "test_env.h" + +namespace art { +namespace Test1908NativeSuspendResume { + +std::atomic<bool> done(false); +std::atomic<bool> started(false); +std::atomic<bool> resumed(false); +std::atomic<bool> resuming(false); + +extern "C" JNIEXPORT jint JNICALL Java_art_Test1908_nativeSpinAndResume(JNIEnv*, + jclass, + jthread thr) { + while (!done.load()) { + started.store(true); + } + resuming.store(true); + jint ret = jvmti_env->ResumeThread(thr); + resumed.store(true); + return ret; +} + +extern "C" JNIEXPORT jboolean JNICALL Java_art_Test1908_isNativeThreadSpinning(JNIEnv*, jclass) { + return started.load(); +} + +extern "C" JNIEXPORT void JNICALL Java_art_Test1908_waitForNativeResumeStarted(JNIEnv*, jclass) { + while (!resuming.load()) {} +} + +extern "C" JNIEXPORT void JNICALL Java_art_Test1908_waitForNativeResumeFinished(JNIEnv*, jclass) { + while (!resumed.load()) {} +} + +extern "C" JNIEXPORT void JNICALL Java_art_Test1908_nativeResume(JNIEnv*, jclass) { + done.store(true); +} + +} // namespace Test1908NativeSuspendResume +} // namespace art diff --git a/test/1908-suspend-native-resume-self/run b/test/1908-suspend-native-resume-self/run new file mode 100755 index 0000000000..e92b873956 --- /dev/null +++ b/test/1908-suspend-native-resume-self/run @@ -0,0 +1,17 @@ +#!/bin/bash +# +# Copyright 2017 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +./default-run "$@" --jvmti diff --git a/test/1908-suspend-native-resume-self/src/Main.java b/test/1908-suspend-native-resume-self/src/Main.java new file mode 100644 index 0000000000..312adc44de --- /dev/null +++ b/test/1908-suspend-native-resume-self/src/Main.java @@ -0,0 +1,21 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +public class Main { + public static void main(String[] args) throws Exception { + art.Test1908.run(); + } +} diff --git a/test/1908-suspend-native-resume-self/src/art/Suspension.java b/test/1908-suspend-native-resume-self/src/art/Suspension.java new file mode 100644 index 0000000000..16e62ccac9 --- /dev/null +++ b/test/1908-suspend-native-resume-self/src/art/Suspension.java @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package art; + +public class Suspension { + // Suspends a thread using jvmti. + public native static void suspend(Thread thr); + + // Resumes a thread using jvmti. + public native static void resume(Thread thr); + + public native static boolean isSuspended(Thread thr); + + public native static int[] suspendList(Thread... threads); + public native static int[] resumeList(Thread... threads); +} diff --git a/test/1908-suspend-native-resume-self/src/art/Test1908.java b/test/1908-suspend-native-resume-self/src/art/Test1908.java new file mode 100644 index 0000000000..9b7020adb8 --- /dev/null +++ b/test/1908-suspend-native-resume-self/src/art/Test1908.java @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package art; + +public class Test1908 { + public static void run() throws Exception { + final Thread spinner = new Thread(() -> { + int ret = nativeSpinAndResume(Thread.currentThread()); + if (ret != 13) { + System.out.println("Got " + ret + " instead of JVMTI_ERROR_THREAD_NOT_SUSPENDED"); + } + }, "Spinner"); + + final Thread resumer = new Thread(() -> { + String me = Thread.currentThread().getName(); + + // wait for the other thread to start spinning. + while (!isNativeThreadSpinning()) { } + + System.out.println(me + ": isNativeThreadSpinning() = " + isNativeThreadSpinning()); + System.out.println(me + ": isSuspended(spinner) = " + Suspension.isSuspended(spinner)); + + // Suspend it from java. + Suspension.suspend(spinner); + + System.out.println(me + ": Suspended spinner while native spinning"); + System.out.println(me + ": isNativeThreadSpinning() = " + isNativeThreadSpinning()); + System.out.println(me + ": isSuspended(spinner) = " + Suspension.isSuspended(spinner)); + + System.out.println("Resuming other thread"); + nativeResume(); + waitForNativeResumeStarted(); + // Wait for the other thread to try to resume itself + try { Thread.currentThread().sleep(1000); } catch (Exception e) {} + + System.out.println("other thread attempting self resume"); + System.out.println(me + ": isSuspended(spinner) = " + Suspension.isSuspended(spinner)); + + System.out.println("real resume"); + Suspension.resume(spinner); + waitForNativeResumeFinished(); + System.out.println("other thread resumed."); + }, "Resumer"); + + spinner.start(); + resumer.start(); + + spinner.join(); + resumer.join(); + } + + public static native int nativeSpinAndResume(Thread cur); + public static native void nativeResume(); + public static native boolean isNativeThreadSpinning(); + public static native void waitForNativeResumeFinished(); + public static native void waitForNativeResumeStarted(); +} diff --git a/test/1909-per-agent-tls/agent_tls.cc b/test/1909-per-agent-tls/agent_tls.cc new file mode 100644 index 0000000000..14c82e38de --- /dev/null +++ b/test/1909-per-agent-tls/agent_tls.cc @@ -0,0 +1,75 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "jvmti.h" + +// Test infrastructure +#include "jvmti_helper.h" +#include "scoped_local_ref.h" +#include "test_env.h" + +namespace art { +namespace Test1909AgentTLS { + +extern "C" JNIEXPORT void JNICALL Java_art_Test1909_setTLS(JNIEnv* env, + jclass, + jlong jvmti_env_ptr, + jthread thr, + jlong data) { + JvmtiErrorToException(env, + reinterpret_cast<jvmtiEnv*>(jvmti_env_ptr), + reinterpret_cast<jvmtiEnv*>(jvmti_env_ptr)->SetThreadLocalStorage( + thr, reinterpret_cast<const void*>(static_cast<intptr_t>(data)))); +} + +extern "C" JNIEXPORT jlong JNICALL Java_art_Test1909_getTLS(JNIEnv* env, + jclass, + jlong jvmti_env_ptr, + jthread thr) { + void* res = nullptr; + JvmtiErrorToException( + env, + reinterpret_cast<jvmtiEnv*>(jvmti_env_ptr), + reinterpret_cast<jvmtiEnv*>(jvmti_env_ptr)->GetThreadLocalStorage(thr, &res)); + return static_cast<jlong>(reinterpret_cast<intptr_t>(res)); +} + +extern "C" JNIEXPORT void Java_art_Test1909_destroyJvmtiEnv(JNIEnv* env, + jclass, + jlong jvmti_env_ptr) { + JvmtiErrorToException(env, + jvmti_env, + reinterpret_cast<jvmtiEnv*>(jvmti_env_ptr)->DisposeEnvironment()); +} + +extern "C" JNIEXPORT jlong Java_art_Test1909_newJvmtiEnv(JNIEnv* env, jclass) { + JavaVM* vm = nullptr; + if (env->GetJavaVM(&vm) != 0) { + ScopedLocalRef<jclass> rt_exception(env, env->FindClass("java/lang/RuntimeException")); + env->ThrowNew(rt_exception.get(), "Unable to get JavaVM"); + return -1; + } + jvmtiEnv* new_env = nullptr; + if (vm->GetEnv(reinterpret_cast<void**>(&new_env), JVMTI_VERSION_1_0) != 0) { + ScopedLocalRef<jclass> rt_exception(env, env->FindClass("java/lang/RuntimeException")); + env->ThrowNew(rt_exception.get(), "Unable to create new jvmtiEnv"); + return -1; + } + return static_cast<jlong>(reinterpret_cast<intptr_t>(new_env)); +} + +} // namespace Test1909AgentTLS +} // namespace art diff --git a/test/1909-per-agent-tls/expected.txt b/test/1909-per-agent-tls/expected.txt new file mode 100644 index 0000000000..386f3d2fd8 --- /dev/null +++ b/test/1909-per-agent-tls/expected.txt @@ -0,0 +1 @@ +Test passed diff --git a/test/1909-per-agent-tls/info.txt b/test/1909-per-agent-tls/info.txt new file mode 100644 index 0000000000..00acefd065 --- /dev/null +++ b/test/1909-per-agent-tls/info.txt @@ -0,0 +1 @@ +Tests jvmti behavior of GetThreadLocalStorage with multiple threads. diff --git a/test/1909-per-agent-tls/run b/test/1909-per-agent-tls/run new file mode 100755 index 0000000000..c6e62ae6cd --- /dev/null +++ b/test/1909-per-agent-tls/run @@ -0,0 +1,17 @@ +#!/bin/bash +# +# Copyright 2016 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +./default-run "$@" --jvmti diff --git a/test/1909-per-agent-tls/src/Main.java b/test/1909-per-agent-tls/src/Main.java new file mode 100644 index 0000000000..befebea0d3 --- /dev/null +++ b/test/1909-per-agent-tls/src/Main.java @@ -0,0 +1,21 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +public class Main { + public static void main(String[] args) throws Exception { + art.Test1909.run(); + } +} diff --git a/test/1909-per-agent-tls/src/art/Main.java b/test/1909-per-agent-tls/src/art/Main.java new file mode 100644 index 0000000000..aa5498bd62 --- /dev/null +++ b/test/1909-per-agent-tls/src/art/Main.java @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package art; + +// Binder class so the agent's C code has something that can be bound and exposed to tests. +// In a package to separate cleanly and work around CTS reference issues (though this class +// should be replaced in the CTS version). +public class Main { + // Load the given class with the given classloader, and bind all native methods to corresponding + // C methods in the agent. Will abort if any of the steps fail. + public static native void bindAgentJNI(String className, ClassLoader classLoader); + // Same as above, giving the class directly. + public static native void bindAgentJNIForClass(Class<?> klass); + + // Common infrastructure. + public static native void setTag(Object o, long tag); + public static native long getTag(Object o); +} diff --git a/test/1909-per-agent-tls/src/art/Test1909.java b/test/1909-per-agent-tls/src/art/Test1909.java new file mode 100644 index 0000000000..245397dc46 --- /dev/null +++ b/test/1909-per-agent-tls/src/art/Test1909.java @@ -0,0 +1,176 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package art; + +import java.lang.ref.WeakReference; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Semaphore; + +public class Test1909 { + + public static class ThreadHolder { + public Thread thr; + public ThreadHolder(Thread thr) { + this.thr = thr; + } + + public long getTLS(long jvmtienv) { + return Test1909.getTLS(jvmtienv, this.thr); + } + public void setTLS(long jvmtienv, long ptr) { + Test1909.setTLS(jvmtienv, this.thr, ptr); + } + } + + public static class ThreadWaiter { + public boolean exit; + public Thread thr; + public final Object lock; + + public ThreadWaiter() { + this.exit = false; + this.lock = new Object(); + this.thr = new Thread(() -> { + try { + synchronized (lock) { + while (!this.exit) { + this.lock.wait(); + } + } + } catch (Exception e) { + e.printStackTrace(); + } + }); + // Kill threads if we exit. + thr.setDaemon(true); + thr.start(); + } + + public void cleanup() throws Exception { + synchronized (lock) { + exit = true; + lock.notifyAll(); + } + thr.join(); + } + public long getTLS(long jvmtienv) { + return Test1909.getTLS(jvmtienv, this.thr); + } + public void setTLS(long jvmtienv, long ptr) { + Test1909.setTLS(jvmtienv, this.thr, ptr); + } + } + + public static void checkEq(long a, long b) { + if (a != b) { + throw new Error("Expected: " + a + " got: " + b); + } + } + + public static void run() throws Exception { + ThreadHolder tc = new ThreadHolder(Thread.currentThread()); + ThreadWaiter t1 = new ThreadWaiter(); + long e1 = newJvmtiEnv(); + long e2 = newJvmtiEnv(); + + // Everything should be 0 + checkEq(0, tc.getTLS(e1)); + checkEq(0, t1.getTLS(e1)); + checkEq(0, tc.getTLS(e2)); + checkEq(0, t1.getTLS(e2)); + + // Set in one pair. + tc.setTLS(e1, 1); + checkEq(1, tc.getTLS(e1)); + checkEq(0, t1.getTLS(e1)); + checkEq(0, tc.getTLS(e2)); + checkEq(0, t1.getTLS(e2)); + + // Set in another pair. + t1.setTLS(e1, 2); + checkEq(1, tc.getTLS(e1)); + checkEq(2, t1.getTLS(e1)); + checkEq(0, tc.getTLS(e2)); + checkEq(0, t1.getTLS(e2)); + + // Set in third pair. + tc.setTLS(e2, 3); + checkEq(1, tc.getTLS(e1)); + checkEq(2, t1.getTLS(e1)); + checkEq(3, tc.getTLS(e2)); + checkEq(0, t1.getTLS(e2)); + + // Set in fourth pair. + t1.setTLS(e2, 4); + checkEq(1, tc.getTLS(e1)); + checkEq(2, t1.getTLS(e1)); + checkEq(3, tc.getTLS(e2)); + checkEq(4, t1.getTLS(e2)); + + // Create a new thread and make sure everything is 0. + ThreadWaiter t2 = new ThreadWaiter(); + checkEq(1, tc.getTLS(e1)); + checkEq(2, t1.getTLS(e1)); + checkEq(0, t2.getTLS(e1)); + checkEq(3, tc.getTLS(e2)); + checkEq(4, t1.getTLS(e2)); + checkEq(0, t2.getTLS(e2)); + + // Create a new jvmtienv and make sure everything is 0. + long e3 = newJvmtiEnv(); + checkEq(1, tc.getTLS(e1)); + checkEq(2, t1.getTLS(e1)); + checkEq(0, t2.getTLS(e1)); + checkEq(3, tc.getTLS(e2)); + checkEq(4, t1.getTLS(e2)); + checkEq(0, t2.getTLS(e2)); + checkEq(0, tc.getTLS(e3)); + checkEq(0, t1.getTLS(e3)); + checkEq(0, t2.getTLS(e3)); + + // Delete an env without data and make sure everything is still there. + destroyJvmtiEnv(e3); + checkEq(1, tc.getTLS(e1)); + checkEq(2, t1.getTLS(e1)); + checkEq(0, t2.getTLS(e1)); + checkEq(3, tc.getTLS(e2)); + checkEq(4, t1.getTLS(e2)); + checkEq(0, t2.getTLS(e2)); + + // Delete an env with data and make sure everything is still there. + destroyJvmtiEnv(e2); + checkEq(1, tc.getTLS(e1)); + checkEq(2, t1.getTLS(e1)); + checkEq(0, t2.getTLS(e1)); + + // Delete a thread. Make sure other thread still has data. + t1.cleanup(); + checkEq(1, tc.getTLS(e1)); + checkEq(0, t2.getTLS(e1)); + + t2.cleanup(); + + System.out.println("Test passed"); + } + + public static native long getTLS(long jvmtienv, Thread thr); + public static native void setTLS(long jvmtienv, Thread thr, long ptr); + public static native long newJvmtiEnv(); + public static native void destroyJvmtiEnv(long jvmtienv); +} diff --git a/test/536-checker-intrinsic-optimization/src/Main.java b/test/536-checker-intrinsic-optimization/src/Main.java index 3dce23fb31..6d3abb1026 100644 --- a/test/536-checker-intrinsic-optimization/src/Main.java +++ b/test/536-checker-intrinsic-optimization/src/Main.java @@ -340,7 +340,7 @@ public class Main { /// CHECK-START-MIPS: boolean Main.stringArgumentNotNull(java.lang.Object) disassembly (after) /// CHECK: InvokeVirtual {{.*\.equals.*}} intrinsic:StringEquals - /// CHECK-NOT: beq r0, + /// CHECK-NOT: beq zero, /// CHECK-NOT: beqz /// CHECK-NOT: beqzc // Terminate the scope for the CHECK-NOT search at the class field or length comparison, diff --git a/test/595-profile-saving/profile-saving.cc b/test/595-profile-saving/profile-saving.cc index 0bdbadef48..00aa944726 100644 --- a/test/595-profile-saving/profile-saving.cc +++ b/test/595-profile-saving/profile-saving.cc @@ -38,15 +38,19 @@ extern "C" JNIEXPORT void JNICALL Java_Main_ensureProfilingInfo(JNIEnv* env, CHECK(method != nullptr); ScopedObjectAccess soa(env); ObjPtr<mirror::Executable> exec = soa.Decode<mirror::Executable>(method); - ProfilingInfo::Create(soa.Self(), exec->GetArtMethod(), /* retry_allocation */ true); + ArtMethod* art_method = exec->GetArtMethod(); + if (!ProfilingInfo::Create(soa.Self(), art_method, /* retry_allocation */ true)) { + LOG(ERROR) << "Failed to create profiling info for method " << art_method->PrettyMethod(); + } } extern "C" JNIEXPORT void JNICALL Java_Main_ensureProfileProcessing(JNIEnv*, jclass) { ProfileSaver::ForceProcessProfiles(); } -extern "C" JNIEXPORT jboolean JNICALL Java_Main_presentInProfile(JNIEnv* env, +extern "C" JNIEXPORT jboolean JNICALL Java_Main_profileHasMethod(JNIEnv* env, jclass, + jboolean hot, jstring filename, jobject method) { ScopedUtfChars filename_chars(env, filename); @@ -55,8 +59,9 @@ extern "C" JNIEXPORT jboolean JNICALL Java_Main_presentInProfile(JNIEnv* env, ObjPtr<mirror::Executable> exec = soa.Decode<mirror::Executable>(method); ArtMethod* art_method = exec->GetArtMethod(); return ProfileSaver::HasSeenMethod(std::string(filename_chars.c_str()), - art_method->GetDexFile(), - art_method->GetDexMethodIndex()); + hot != JNI_FALSE, + MethodReference(art_method->GetDexFile(), + art_method->GetDexMethodIndex())); } } // namespace diff --git a/test/595-profile-saving/src/Main.java b/test/595-profile-saving/src/Main.java index 18c0598bef..197c4e74ff 100644 --- a/test/595-profile-saving/src/Main.java +++ b/test/595-profile-saving/src/Main.java @@ -42,6 +42,14 @@ public class Main { System.out.println("Class loader does not match boot class"); } testAddMethodToProfile(file, bootMethod); + + // Test a sampled method that is only warm and not hot. + Method reflectMethod = Main.class.getDeclaredMethod("testReflectionInvoke"); + reflectMethod.invoke(null); + testSampledMethodInProfile(file, reflectMethod); + if (staticObj == null) { + throw new AssertionError("Object was not set"); + } } finally { if (file != null) { file.delete(); @@ -49,23 +57,38 @@ public class Main { } } + static Object staticObj = null; + + static void testReflectionInvoke() { + staticObj = new Object(); + } + static void testAddMethodToProfile(File file, Method m) { // Make sure we have a profile info for this method without the need to loop. ensureProfilingInfo(m); - // Make sure the profile gets saved. + // Make sure the profile gets processed. ensureProfileProcessing(); // Verify that the profile was saved and contains the method. - if (!presentInProfile(file.getPath(), m)) { + if (!profileHasMethod(true, file.getPath(), m)) { throw new RuntimeException("Method with index " + m + " not in the profile"); } } + static void testSampledMethodInProfile(File file, Method m) { + // Make sure the profile gets processed. + ensureProfileProcessing(); + // Verify that the profile was saved and contains the method. + if (!profileHasMethod(false, file.getPath(), m)) { + throw new RuntimeException("Method with index " + m + " not sampled in the profile"); + } + } + // Ensure a method has a profiling info. public static native void ensureProfilingInfo(Method method); // Ensures the profile saver does its usual processing. public static native void ensureProfileProcessing(); - // Checks if the profiles saver knows about the method. - public static native boolean presentInProfile(String profile, Method method); + // Checks if the profile saver has the method as a warm/sampled method. + public static native boolean profileHasMethod(boolean hot, String profile, Method method); private static final String TEMP_FILE_NAME_PREFIX = "dummy"; private static final String TEMP_FILE_NAME_SUFFIX = "-file"; diff --git a/test/606-erroneous-class/jasmin-multidex/ClassA.j b/test/606-erroneous-class/jasmin-multidex/ClassA.j new file mode 100644 index 0000000000..50c6755b4b --- /dev/null +++ b/test/606-erroneous-class/jasmin-multidex/ClassA.j @@ -0,0 +1,30 @@ +; Copyright (C) 2017 The Android Open Source Project +; +; Licensed under the Apache License, Version 2.0 (the "License"); +; you may not use this file except in compliance with the License. +; You may obtain a copy of the License at +; +; http://www.apache.org/licenses/LICENSE-2.0 +; +; Unless required by applicable law or agreed to in writing, software +; distributed under the License is distributed on an "AS IS" BASIS, +; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; See the License for the specific language governing permissions and +; limitations under the License. + +.class public final ClassA +.super java/lang/Object + +.method public static foo()V + .limit stack 1 + .limit locals 0 + ; Obtain the ErrClass type from Dex cache of the first Dex file. Note that + ; because the first Dex file has already been verified, we know the class + ; is erroneous at this point. + getstatic ClassB/g LErrClass; + ; Use the object in a way that will try to store the ErrClass type in + ; the Dex cache of the second Dex file. + invokevirtual ErrClass/foo()V + return +.end method + diff --git a/test/606-erroneous-class/smali-multidex/ClassA.smali b/test/606-erroneous-class/smali-multidex/ClassA.smali deleted file mode 100644 index f87fcb2469..0000000000 --- a/test/606-erroneous-class/smali-multidex/ClassA.smali +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright (C) 2016 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -.class public final LClassA; -.super Ljava/lang/Object; - -.method public static foo()V - .registers 1 - # Obtain the ErrClass type from Dex cache of the first Dex file. Note that - # because the first Dex file has already been verified, we know the class - # is erroneous at this point. - sget-object v0, LClassB;->g:LErrClass; - # Use the object in a way that will try to store the ErrClass type in - # the Dex cache of the second Dex file. - invoke-virtual {v0}, LErrClass;->foo()V -.end method diff --git a/test/912-classes/src-art/art/Test912.java b/test/912-classes/src-art/art/Test912.java index 9896eacfeb..fbf8794c50 100644 --- a/test/912-classes/src-art/art/Test912.java +++ b/test/912-classes/src-art/art/Test912.java @@ -228,7 +228,8 @@ public class Test912 { // The JIT may deeply inline and load some classes. Preload these for test determinism. final String PRELOAD_FOR_JIT[] = { "java.nio.charset.CoderMalfunctionError", - "java.util.NoSuchElementException" + "java.util.NoSuchElementException", + "java.io.FileNotFoundException", // b/63581208 }; for (String s : PRELOAD_FOR_JIT) { Class.forName(s); diff --git a/test/952-invoke-custom-kinds/classes/invokecustom/InvokeCustom.class b/test/952-invoke-custom-kinds/classes/invokecustom/InvokeCustom.class Binary files differindex 4f0f497c72..b8dcd5559a 100644 --- a/test/952-invoke-custom-kinds/classes/invokecustom/InvokeCustom.class +++ b/test/952-invoke-custom-kinds/classes/invokecustom/InvokeCustom.class diff --git a/test/952-invoke-custom-kinds/classes/invokecustom/Super.class b/test/952-invoke-custom-kinds/classes/invokecustom/Super.class Binary files differindex 5990f28d47..7906f99d2d 100644 --- a/test/952-invoke-custom-kinds/classes/invokecustom/Super.class +++ b/test/952-invoke-custom-kinds/classes/invokecustom/Super.class diff --git a/test/952-invoke-custom-kinds/classes/invokecustom/TestGenerator.class b/test/952-invoke-custom-kinds/classes/invokecustom/TestGenerator.class Binary files differindex 711db2343b..03dc23396c 100644 --- a/test/952-invoke-custom-kinds/classes/invokecustom/TestGenerator.class +++ b/test/952-invoke-custom-kinds/classes/invokecustom/TestGenerator.class diff --git a/test/952-invoke-custom-kinds/expected.txt b/test/952-invoke-custom-kinds/expected.txt index c72da25c99..c41b5c6d88 100644 --- a/test/952-invoke-custom-kinds/expected.txt +++ b/test/952-invoke-custom-kinds/expected.txt @@ -12,7 +12,7 @@ c String bsmLookupStaticWithExtraArgs [1, 123456789, 123.456, 123456.789123] targetMethodTest3 from InvokeCustom -bsmCreateCallSite [MethodHandle(Super)void] +bsmCreateCallSite [MethodHandle(InvokeCustom)void] targetMethodTest4 from Super bsmLookupStatic [] targetMethodTest5 1000 + -923 = 77 @@ -34,5 +34,7 @@ targetMethodTest9 ()void checkStaticFieldTest9: old 0 new 1985229328 expected 1985229328 OK checkFieldTest9: old 0.0 new 1.99E-19 expected 1.99E-19 OK helperMethodTest9 in class invokecustom.InvokeCustom +InvokeCustom.<init>(3) run() for Test9 +InvokeCustom.privateMethodTest9() targetMethodTest9() diff --git a/test/952-invoke-custom-kinds/info.txt b/test/952-invoke-custom-kinds/info.txt index cddb96dcf0..33b4cffc1d 100644 --- a/test/952-invoke-custom-kinds/info.txt +++ b/test/952-invoke-custom-kinds/info.txt @@ -1,7 +1,4 @@ This test checks call sites and constant method handles in DEX files used by invoke-custom. -It is derived from a dx test (dalvik/dx/tests/135-invoke-custom) which -generates the invoke-custom using ASM to generate class files. The -test here differs as it not include an instance a constant method -handle with a constructor kind (see b/62774190). +The class files come from dalvik/dx/tests/135-invoke-custom. diff --git a/test/990-method-handle-and-mr/build b/test/990-method-handle-and-mr/build new file mode 100755 index 0000000000..12a8e18d0b --- /dev/null +++ b/test/990-method-handle-and-mr/build @@ -0,0 +1,20 @@ +#!/bin/bash +# +# Copyright 2017 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Exit on failure. +set -e + +./default-build "$@" --experimental method-handles diff --git a/test/990-method-handle-and-mr/expected.txt b/test/990-method-handle-and-mr/expected.txt new file mode 100644 index 0000000000..8483fb5045 --- /dev/null +++ b/test/990-method-handle-and-mr/expected.txt @@ -0,0 +1,4 @@ +Test +Test +Test +passed diff --git a/test/990-method-handle-and-mr/info.txt b/test/990-method-handle-and-mr/info.txt new file mode 100644 index 0000000000..85a957ceea --- /dev/null +++ b/test/990-method-handle-and-mr/info.txt @@ -0,0 +1,2 @@ +Test stressing code generated for invoke-polymorphic instructions with +respect to Marking Register (on architectures supporting MR). diff --git a/test/990-method-handle-and-mr/src/Main.java b/test/990-method-handle-and-mr/src/Main.java new file mode 100644 index 0000000000..739b8eb551 --- /dev/null +++ b/test/990-method-handle-and-mr/src/Main.java @@ -0,0 +1,91 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This test was inspired by benchmarks.MicroMethodHandles.java.MicroMethodHandles. + +import java.io.PrintStream; +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; + +class A { + public Long binaryFunction(int x, double y) { + return 1000l; + } +} + +class Test { + Test() throws Throwable { + this.handle = MethodHandles.lookup().findVirtual(A.class, "binaryFunction", + MethodType.methodType(Long.class, int.class, + double.class)); + this.a = new A(); + this.x = new Integer(72); + this.y = new Double(-1.39e-31); + } + + void execute() { + try { + executeFor(2000); + System.out.println(getName()); + } catch (Throwable t) { + System.err.println("Exception during the execution of " + getName()); + System.err.println(t); + t.printStackTrace(new PrintStream(System.err)); + System.exit(1); + } + } + + void executeFor(long timeMinimumMillis) throws Throwable { + long startTime = System.currentTimeMillis(); + long elapsed = 0; + while (elapsed < timeMinimumMillis) { + exercise(); + elapsed = System.currentTimeMillis() - startTime; + } + } + + void exercise() throws Throwable { + for (int i = 0; i < EXERCISE_ITERATIONS; ++i) { + run(); + } + } + + void run() throws Throwable { + long result = (long) handle.invoke(a, x, y); + } + + String getName() { + return getClass().getSimpleName(); + } + + private static final int EXERCISE_ITERATIONS = 500; + + private MethodHandle handle; + private A a; + private Integer x; + private Double y; +} + +public class Main { + public static void main(String[] args) throws Throwable { + Test[] tests = new Test[] { new Test(), new Test(), new Test() }; + for (Test test : tests) { + test.execute(); + } + System.out.println("passed"); + } +} diff --git a/test/Android.bp b/test/Android.bp index 591684b887..44cb4f6e8a 100644 --- a/test/Android.bp +++ b/test/Android.bp @@ -253,6 +253,7 @@ art_cc_defaults { "ti-agent/breakpoint_helper.cc", "ti-agent/common_helper.cc", "ti-agent/redefinition_helper.cc", + "ti-agent/suspension_helper.cc", "ti-agent/trace_helper.cc", // This is the list of non-special OnLoad things and excludes BCI and anything that depends // on ART internals. @@ -288,6 +289,9 @@ art_cc_defaults { "996-breakpoint-obsolete/obsolete_breakpoints.cc", "1900-track-alloc/alloc.cc", "1901-get-bytecodes/bytecodes.cc", + "1905-suspend-native/native_suspend.cc", + "1908-suspend-native-resume-self/native_suspend_resume.cc", + "1909-per-agent-tls/agent_tls.cc", ], shared_libs: [ "libbase", diff --git a/test/etc/default-build b/test/etc/default-build index 13f430185e..bafd415802 100755 --- a/test/etc/default-build +++ b/test/etc/default-build @@ -24,6 +24,13 @@ else HAS_SMALI=false fi +# .j files in jasmin get compiled into classes.jar +if [ -d jasmin ]; then + HAS_JASMIN=true +else + HAS_JASMIN=false +fi + if [ -d src ]; then HAS_SRC=true else @@ -55,6 +62,13 @@ else HAS_SMALI_MULTIDEX=false fi +# .j files in jasmin-multidex get compiled into classes2.jar +if [ -d jasmin-multidex ]; then + HAS_JASMIN_MULTIDEX=true +else + HAS_JASMIN_MULTIDEX=false +fi + if [ -d src-ex ]; then HAS_SRC_EX=true else @@ -80,7 +94,6 @@ ZIP_ALIGN_BYTES="-1" DX_FLAGS="--min-sdk-version=24" DX_VM_FLAGS="" -SKIP_DX_MERGER="false" EXPERIMENTAL="" BUILD_MODE="target" @@ -219,6 +232,21 @@ function zip() { fi } +function make_jasmin() { + local out_directory="$1" + shift + local jasmin_sources=("$@") + + mkdir -p "$out_directory" + + if [[ $DEV_MODE == yes ]]; then + echo ${JASMIN} -d "$out_directory" "${jasmin_sources[@]}" + ${JASMIN} -d "$out_directory" "${jasmin_sources[@]}" + else + ${JASMIN} -d "$out_directory" "${jasmin_sources[@]}" >/dev/null + fi +} + function desugar() { local desugar_args=--mode=host if [[ $BUILD_MODE == target ]]; then @@ -268,6 +296,26 @@ function make_dex() { ${DX} -JXmx256m ${DX_VM_FLAGS} --debug --dex --dump-to=${name}.lst --output=${name}.dex --dump-width=1000 ${DX_FLAGS} "${dx_input}" } +# Merge all the dex files in $1..$N into $1. Skip non-existing files, but at least 1 file must exist. +function make_dexmerge() { + # Dex file that acts as the destination. + local dst_file="$1" + + # Dex files that act as the source. + local dex_files_to_merge=() + + # Skip any non-existing files. + while [[ $# -gt 0 ]]; do + if [[ -e "$1" ]]; then + dex_files_to_merge+=("$1") + fi + shift + done + + # Should have at least 1 dex_files_to_merge here, otherwise dxmerger will print the help. + ${DXMERGER} "$dst_file" "${dex_files_to_merge[@]}" +} + # Print the directory name only if it exists. function maybe_dir() { local dirname="$1" @@ -281,11 +329,6 @@ if [ -e classes.dex ]; then exit 0 fi -if ! [ "${HAS_SRC}" = "true" ] && ! [ "${HAS_SRC2}" = "true" ] && ! [ "${HAS_SRC_ART}" = "true" ]; then - # No src directory? Then forget about trying to run dx. - SKIP_DX_MERGER="true" -fi - if [ ${HAS_SRC_DEX2OAT_UNRESOLVED} = "true" ]; then mkdir classes mkdir classes-ex @@ -332,7 +375,7 @@ else fi # Compile jack files into a DEX file. - if [ "${HAS_SRC}" = "true" ] || [ "${HAS_SRC2}" = "true" ] || [ "${HAS_SRC_ART}" ]; then + if [ "${HAS_SRC}" = "true" ] || [ "${HAS_SRC2}" = "true" ] || [ "${HAS_SRC_ART}" = "true" ]; then ${JACK} ${JACK_ARGS} ${jack_extra_args} --output-dex . fi else @@ -361,22 +404,49 @@ else fi if [[ "${HAS_SRC}" == "true" || "${HAS_SRC2}" == "true" || "${HAS_SRC_ART}" == "true" ]]; then - if [ ${NEED_DEX} = "true" -a ${SKIP_DX_MERGER} = "false" ]; then + if [ ${NEED_DEX} = "true" ]; then make_dex classes fi fi fi fi +if [[ "${HAS_JASMIN}" == true ]]; then + # Compile Jasmin classes as if they were part of the classes.dex file. + make_jasmin jasmin_classes $(find 'jasmin' -name '*.j') + if [[ "${NEED_DEX}" == "true" ]]; then + # Disable desugar because it won't handle intentional linkage errors. + USE_DESUGAR=false make_dex jasmin_classes + make_dexmerge classes.dex jasmin_classes.dex + else + # Move jasmin classes into classes directory so that they are picked up with -cp classes. + mkdir -p classes + mv jasmin_classes/* classes + fi +fi + if [ "${HAS_SMALI}" = "true" -a ${NEED_DEX} = "true" ]; then # Compile Smali classes ${SMALI} -JXmx512m assemble ${SMALI_ARGS} --output smali_classes.dex `find smali -name '*.smali'` - # Don't bother with dexmerger if we provide our own main function in a smali file. - if [ ${SKIP_DX_MERGER} = "false" ]; then - ${DXMERGER} classes.dex classes.dex smali_classes.dex + # Merge smali files into classes.dex, this takes priority over any jasmin files. + make_dexmerge classes.dex smali_classes.dex +fi + +# Compile Jasmin classes in jasmin-multidex as if they were part of the classes2.jar +if [[ "$HAS_JASMIN_MULTIDEX" == true ]]; then + make_jasmin jasmin_classes2 $(find 'jasmin-multidex' -name '*.j') + + if [[ "${NEED_DEX}" == "true" ]]; then + # Disable desugar because it won't handle intentional linkage errors. + USE_DESUGAR=false make_dex jasmin_classes2 + + # Merge jasmin_classes2.dex into classes2.dex + make_dexmerge classes2.dex jasmin_classes2.dex else - mv smali_classes.dex classes.dex + # Move jasmin classes into classes2 directory so that they are picked up with -cp classes2. + mkdir -p classes2 + mv jasmin_classes2/* classes2 fi fi @@ -384,12 +454,8 @@ if [ "${HAS_SMALI_MULTIDEX}" = "true" -a ${NEED_DEX} = "true" ]; then # Compile Smali classes ${SMALI} -JXmx512m assemble ${SMALI_ARGS} --output smali_classes2.dex `find smali-multidex -name '*.smali'` - # Don't bother with dexmerger if we provide our own main function in a smali file. - if [ ${HAS_SRC_MULTIDEX} = "true" ]; then - ${DXMERGER} classes2.dex classes2.dex smali_classes2.dex - else - mv smali_classes2.dex classes2.dex - fi + # Merge smali_classes2.dex into classes2.dex + make_dexmerge classes2.dex smali_classes2.dex fi @@ -430,9 +496,9 @@ if [ ${HAS_SRC_EX} = "true" ]; then fi fi -# Create a single jar with two dex files for multidex. +# Create a single dex jar with two dex files for multidex. if [ ${NEED_DEX} = "true" ]; then - if [ ${HAS_SRC_MULTIDEX} = "true" ] || [ ${HAS_SMALI_MULTIDEX} = "true" ]; then + if [ ${HAS_SRC_MULTIDEX} = "true" ] || [ ${HAS_JASMIN_MULTIDEX} = "true" ] || [ ${HAS_SMALI_MULTIDEX} = "true" ]; then zip $TEST_NAME.jar classes.dex classes2.dex else zip $TEST_NAME.jar classes.dex diff --git a/test/knownfailures.json b/test/knownfailures.json index 87d857eaa6..3edb0a8a45 100644 --- a/test/knownfailures.json +++ b/test/knownfailures.json @@ -100,6 +100,11 @@ "bug": "http://b/35800768" }, { + "tests": "163-app-image-methods", + "variant": "gcstress", + "description": ["This test sometimes runs out of memory initializing the boot classpath."] + }, + { "tests": ["908-gc-start-finish", "913-heaps"], "variant": "gcstress", @@ -640,6 +645,36 @@ "env_vars": {"SANITIZE_TARGET": "address"} }, { + "tests": [ + "059-finalizer-throw", + "074-gc-thrash", + "911-get-stack-trace", + "913-heaps", + "980-redefine-object" + ], + "description": [ + "Interpreter with access checks stack frames are too large and result in", + "StackOverFlow errors being thrown." + ], + "variant": "interp-ac & host", + "env_vars": {"SANITIZE_HOST": "address"} + }, + { + "tests": [ + "059-finalizer-throw", + "074-gc-thrash", + "911-get-stack-trace", + "913-heaps", + "980-redefine-object" + ], + "description": [ + "Interpreter with access checks stack frames are too large and result in", + "StackOverFlow errors being thrown." + ], + "variant": "interp-ac & target", + "env_vars": {"SANITIZE_TARGET": "address"} + }, + { "tests": "071-dexfile-map-clean", "description": [ "We use prebuilt zipalign on master-art-host to avoid pulling in a lot", "of the framework. But a non-sanitized zipalign binary does not work with", diff --git a/test/ti-agent/suspension_helper.cc b/test/ti-agent/suspension_helper.cc new file mode 100644 index 0000000000..b685cb2a55 --- /dev/null +++ b/test/ti-agent/suspension_helper.cc @@ -0,0 +1,98 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "jni.h" +#include "jvmti.h" + +#include <vector> + +#include "jvmti_helper.h" +#include "test_env.h" + +namespace art { +namespace common_suspension { + +extern "C" JNIEXPORT jboolean JNICALL Java_art_Suspension_isSuspended( + JNIEnv* env, jclass, jthread thr) { + jint state; + if (JvmtiErrorToException(env, jvmti_env, jvmti_env->GetThreadState(thr, &state))) { + return false; + } + return (state & JVMTI_THREAD_STATE_SUSPENDED) != 0; +} + +static std::vector<jthread> CopyToVector(JNIEnv* env, jobjectArray thrs) { + jsize len = env->GetArrayLength(thrs); + std::vector<jthread> ret; + for (jsize i = 0; i < len; i++) { + ret.push_back(reinterpret_cast<jthread>(env->GetObjectArrayElement(thrs, i))); + } + return ret; +} + +extern "C" JNIEXPORT jintArray JNICALL Java_art_Suspension_resumeList(JNIEnv* env, + jclass, + jobjectArray thr) { + static_assert(sizeof(jvmtiError) == sizeof(jint), "cannot use jintArray as jvmtiError array"); + std::vector<jthread> threads(CopyToVector(env, thr)); + if (env->ExceptionCheck()) { + return nullptr; + } + jintArray ret = env->NewIntArray(threads.size()); + if (env->ExceptionCheck()) { + return nullptr; + } + jint* elems = env->GetIntArrayElements(ret, nullptr); + JvmtiErrorToException(env, jvmti_env, + jvmti_env->ResumeThreadList(threads.size(), + threads.data(), + reinterpret_cast<jvmtiError*>(elems))); + env->ReleaseIntArrayElements(ret, elems, 0); + return ret; +} + +extern "C" JNIEXPORT jintArray JNICALL Java_art_Suspension_suspendList(JNIEnv* env, + jclass, + jobjectArray thrs) { + static_assert(sizeof(jvmtiError) == sizeof(jint), "cannot use jintArray as jvmtiError array"); + std::vector<jthread> threads(CopyToVector(env, thrs)); + if (env->ExceptionCheck()) { + return nullptr; + } + jintArray ret = env->NewIntArray(threads.size()); + if (env->ExceptionCheck()) { + return nullptr; + } + jint* elems = env->GetIntArrayElements(ret, nullptr); + JvmtiErrorToException(env, jvmti_env, + jvmti_env->SuspendThreadList(threads.size(), + threads.data(), + reinterpret_cast<jvmtiError*>(elems))); + env->ReleaseIntArrayElements(ret, elems, 0); + return ret; +} + +extern "C" JNIEXPORT void JNICALL Java_art_Suspension_resume(JNIEnv* env, jclass, jthread thr) { + JvmtiErrorToException(env, jvmti_env, jvmti_env->ResumeThread(thr)); +} + +extern "C" JNIEXPORT void JNICALL Java_art_Suspension_suspend(JNIEnv* env, jclass, jthread thr) { + JvmtiErrorToException(env, jvmti_env, jvmti_env->SuspendThread(thr)); +} + +} // namespace common_suspension +} // namespace art + diff --git a/tools/generate-boot-image-profile.sh b/tools/generate-boot-image-profile.sh new file mode 100755 index 0000000000..d87123ad71 --- /dev/null +++ b/tools/generate-boot-image-profile.sh @@ -0,0 +1,73 @@ +#!/bin/bash +# +# Copyright (C) 2017 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# This script creates a boot image profile based on input profiles. +# + +if [[ "$#" -lt 2 ]]; then + echo "Usage $0 <output> <profman args> <profiles>+" + echo "Also outputs <output>.txt and <output>.preloaded-classes" + echo 'Example: generate-boot-image-profile.sh boot.prof --profman-arg --boot-image-sampled-method-threshold=1 profiles/0/*/primary.prof' + exit 1 +fi + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +TOP="$DIR/../.." +source "${TOP}/build/envsetup.sh" >&/dev/null # import get_build_var + +OUT_PROFILE=$1 +shift + +# Read the profman args. +profman_args=() +while [[ "$#" -ge 2 ]] && [[ "$1" = '--profman-arg' ]]; do + profman_args+=("$2") + shift 2 +done + +# Remaining args are all the profiles. +for file in "$@"; do + if [[ -s $file ]]; then + profman_args+=("--profile-file=$file") + fi +done + +jar_args=() +boot_jars=$("$ANDROID_BUILD_TOP"/art/tools/bootjars.sh --target) +jar_dir=$ANDROID_BUILD_TOP/$(get_build_var TARGET_OUT_JAVA_LIBRARIES) +for file in $boot_jars; do + filename="$jar_dir/$file.jar" + jar_args+=("--apk=$filename") + jar_args+=("--dex-location=$filename") +done +profman_args+=("${jar_args[@]}") + +# Generate the profile. +"$ANDROID_HOST_OUT/bin/profman" --generate-boot-image-profile "--reference-profile-file=$OUT_PROFILE" "${profman_args[@]}" + +# Convert it to text. +echo Dumping profile to $OUT_PROFILE.txt +"$ANDROID_HOST_OUT/bin/profman" --dump-classes-and-methods "--profile-file=$OUT_PROFILE" "${jar_args[@]}" > "$OUT_PROFILE.txt" + +# Generate preloaded classes +# Filter only classes by using grep -v +# Remove first and last characters L and ; +# Replace / with . to make dot format +grep -v "\\->" "$OUT_PROFILE.txt" | sed 's/.\(.*\)./\1/g' | tr "/" "." > "$OUT_PROFILE.preloaded-classes" + +# You may need to filter some classes out since creating threads is not allowed in the zygote. +# i.e. using: grep -v -E '(android.net.ConnectivityThread\$Singleton)' diff --git a/tools/jfuzz/jfuzz.cc b/tools/jfuzz/jfuzz.cc index 82683f2186..4cd23356bf 100644 --- a/tools/jfuzz/jfuzz.cc +++ b/tools/jfuzz/jfuzz.cc @@ -14,6 +14,7 @@ * limitations under the License. */ +#include <cmath> #include <random> #include <inttypes.h> @@ -54,7 +55,7 @@ static constexpr const char* kRelOps[] = { "==", "!=", ">", ">=", "<", "<=" * to preserve the property that a given version of JFuzz yields the same * fuzzed program for a deterministic random seed. */ -const char* VERSION = "1.2"; +const char* VERSION = "1.3"; /* * Maximum number of array dimensions, together with corresponding maximum size @@ -698,6 +699,72 @@ class JFuzz { return mayFollow; } + // Emit one dimension of an array initializer, where parameter dim >= 1 + // denotes the number of remaining dimensions that should be emitted. + void emitArrayInitDim(int dim) { + if (dim == 1) { + // Last dimension: set of values. + fputs("{ ", out_); + for (uint32_t i = 0; i < array_size_; i++) { + emitExpression(array_type_); + fputs(", ", out_); + } + fputs("}", out_); + + } else { + // Outer dimensions: set of sets. + fputs("{\n", out_); + indentation_ += 2; + emitIndentation(); + + for (uint32_t i = 0; i < array_size_; i++) { + emitArrayInitDim(dim - 1); + if (i != array_size_ - 1) { + fputs(",\n", out_); + emitIndentation(); + } + } + + fputs(",\n", out_); + indentation_ -= 2; + emitIndentation(); + fputs("}", out_); + } + } + + // Emit an array initializer of the following form. + // { + // type[]..[] tmp = { .. }; + // mArray = tmp; + // } + bool emitArrayInit() { + // Avoid elaborate array initializers. + uint64_t p = pow(array_size_, array_dim_); + if (p > 20) { + return emitAssignment(); // fall back + } + + fputs("{\n", out_); + + indentation_ += 2; + emitIndentation(); + emitType(array_type_); + for (uint32_t i = 0; i < array_dim_; i++) { + fputs("[]", out_); + } + fputs(" tmp = ", out_); + emitArrayInitDim(array_dim_); + fputs(";\n", out_); + + emitIndentation(); + fputs("mArray = tmp;\n", out_); + + indentation_ -= 2; + emitIndentation(); + fputs("}\n", out_); + return true; + } + // Emit a for loop. bool emitForLoop() { // Continuing loop nest becomes less likely as the depth grows. @@ -874,10 +941,11 @@ class JFuzz { case 2: return emitContinue(); break; case 3: return emitBreak(); break; case 4: return emitScope(); break; - case 5: return emitForLoop(); break; - case 6: return emitDoLoop(); break; - case 7: return emitIfStmt(); break; - case 8: return emitSwitch(); break; + case 5: return emitArrayInit(); break; + case 6: return emitForLoop(); break; + case 7: return emitDoLoop(); break; + case 8: return emitIfStmt(); break; + case 9: return emitSwitch(); break; default: return emitAssignment(); break; } } diff --git a/tools/libcore_failures.txt b/tools/libcore_failures.txt index 8a4c2df2c1..c6553f89fa 100644 --- a/tools/libcore_failures.txt +++ b/tools/libcore_failures.txt @@ -223,5 +223,10 @@ result: EXEC_FAILED, bug: 62408076, names: ["libcore.java.lang.reflect.annotations.AnnotatedElementParameterTest#testImplicitConstructorParameters_singleAnnotation"] +}, +{ + description: "java.io.IOException: Error writing ASN.1 encoding", + result: EXEC_FAILED, + names: ["libcore.javax.crypto.spec.AlgorithmParametersTestGCM#testEncoding"] } ] diff --git a/tools/runtime_memusage/sanitizer_logcat_analysis.sh b/tools/runtime_memusage/sanitizer_logcat_analysis.sh index 75cb9a927e..66b48fafd0 100755 --- a/tools/runtime_memusage/sanitizer_logcat_analysis.sh +++ b/tools/runtime_memusage/sanitizer_logcat_analysis.sh @@ -18,13 +18,22 @@ # # This script takes in a logcat containing Sanitizer traces and outputs several # files, prints information regarding the traces, and plots information as well. +ALL_PIDS=false USE_TEMP=true DO_REDO=false +PACKAGE_NAME="" # EXACT_ARG and MIN_ARG are passed to prune_sanitizer_output.py EXACT_ARG="" MIN_ARG="" +OFFSET_ARGS="" +TIME_ARGS="" usage() { echo "Usage: $0 [options] [LOGCAT_FILE] [CATEGORIES...]" + echo " -a" + echo " Forces all pids associated with registered dex" + echo " files in the logcat to be processed." + echo " default: only the last pid is processed" + echo echo " -d OUT_DIRECTORY" echo " Puts all output in specified directory." echo " If not given, output will be put in a local" @@ -37,7 +46,7 @@ usage() { echo " the -m argument or by prune_sanitizer_output.py" echo echo " -f" - echo " forces redo of all commands even if output" + echo " Forces redo of all commands even if output" echo " files exist. Steps are skipped if their output" echo " exist already and this is not enabled." echo @@ -46,6 +55,22 @@ usage() { echo " at least MINIMUM_CALLS_PER_TRACE lines." echo " default: specified by prune_sanitizer_output.py" echo + echo " -o [OFFSET],[OFFSET]" + echo " Filters out all Dex File offsets outside the" + echo " range between provided offsets. 'inf' can be" + echo " provided for infinity." + echo " default: 0,inf" + echo + echo " -p [PACKAGE_NAME]" + echo " Using the package name, uses baksmali to get" + echo " a dump of the Dex File format for the package." + echo + echo " -t [TIME_OFFSET],[TIME_OFFSET]" + echo " Filters out all time offsets outside the" + echo " range between provided offsets. 'inf' can be" + echo " provided for infinity." + echo " default: 0,inf" + echo echo " CATEGORIES are words that are expected to show in" echo " a large subset of symbolized traces. Splits" echo " output based on each word." @@ -55,33 +80,61 @@ usage() { } -while [[ $# -gt 1 ]]; do -case $1 in - -d) - shift - USE_TEMP=false - OUT_DIR=$1 - shift - break - ;; - -e) - shift - EXACT_ARG='-e' - ;; - -f) - shift - DO_REDO=true - ;; - -m) - shift - MIN_ARG='-m '"$1"'' - shift - ;; - *) - usage - exit +while getopts ":ad:efm:o:p:t:" opt ; do +case ${opt} in + a) + ALL_PIDS=true + ;; + d) + USE_TEMP=false + OUT_DIR=$OPTARG + ;; + e) + EXACT_ARG='-e' + ;; + f) + DO_REDO=true + ;; + m) + if ! [ "$OPTARG" -eq "$OPTARG" ]; then + usage + exit + fi + MIN_ARG='-m '"$OPTARG" + ;; + o) + set -f + OLD_IFS=$IFS + IFS="," + OFFSET_ARGS=( $OPTARG ) + if [ "${#OFFSET_ARGS[@]}" -ne 2 ]; then + usage + exit + fi + OFFSET_ARGS=( "--offsets" "${OFFSET_ARGS[@]}" ) + IFS=$OLD_IFS + ;; + t) + set -f + OLD_IFS=$IFS + IFS="," + TIME_ARGS=( $OPTARG ) + if [ "${#TIME_ARGS[@]}" -ne 2 ]; then + usage + exit + fi + TIME_ARGS=( "--times" "${TIME_ARGS[@]}" ) + IFS=$OLD_IFS + ;; + p) + PACKAGE_NAME=$OPTARG + ;; + \?) + usage + exit esac done +shift $((OPTIND -1)) if [ $# -lt 1 ]; then usage @@ -103,78 +156,147 @@ if [ ! -d "$OUT_DIR" ]; then fi # Note: Steps are skipped if their output exists until -f flag is enabled -# Step 1 - Only output lines related to Sanitizer -# Folder that holds all file output echo "Output folder: $OUT_DIR" -ASAN_OUT=$OUT_DIR/asan_output -if [ ! -f $ASAN_OUT ] || [ $DO_REDO = true ]; then - DO_REDO=true - echo "Extracting ASAN output" - grep "app_process64" $LOGCAT_FILE > $ASAN_OUT -else - echo "Skipped: Extracting ASAN output" +unique_pids=( $(grep "RegisterDexFile" "$LOGCAT_FILE" | grep -v "zygote64" | tr -s ' ' | cut -f3 -d' ' | awk '!a[$0]++') ) +echo "List of pids: ${unique_pids[@]}" +if [ $ALL_PIDS = false ]; then + unique_pids=( ${unique_pids[-1]} ) fi -# Step 2 - Only output lines containing Dex File Start Addresses -DEX_START=$OUT_DIR/dex_start -if [ ! -f $DEX_START ] || [ $DO_REDO = true ]; then - DO_REDO=true - echo "Extracting Start of Dex File(s)" - grep "RegisterDexFile" $LOGCAT_FILE > $DEX_START -else - echo "Skipped: Extracting Start of Dex File(s)" -fi +for pid in "${unique_pids[@]}" +do + echo + echo "Current pid: $pid" + echo + PID_DIR=$OUT_DIR/$pid + if [ ! -d "$PID_DIR" ]; then + mkdir $PID_DIR + DO_REDO[$pid]=true + fi -# Step 3 - Clean Sanitizer output from Step 2 since logcat cannot -# handle large amounts of output. -ASAN_OUT_FILTERED=$OUT_DIR/asan_output_filtered -if [ ! -f $ASAN_OUT_FILTERED ] || [ $DO_REDO = true ]; then - DO_REDO=true - echo "Filtering/Cleaning ASAN output" - python $ANDROID_BUILD_TOP/art/tools/runtime_memusage/prune_sanitizer_output.py \ - $EXACT_ARG $MIN_ARG -d $OUT_DIR $ASAN_OUT -else - echo "Skipped: Filtering/Cleaning ASAN output" -fi + INTERMEDIATES_DIR=$PID_DIR/intermediates + RESULTS_DIR=$PID_DIR/results + LOGCAT_PID_FILE=$PID_DIR/logcat -# Step 4 - Retrieve symbolized stack traces from Step 3 output -SYM_FILTERED=$OUT_DIR/sym_filtered -if [ ! -f $SYM_FILTERED ] || [ $DO_REDO = true ]; then - DO_REDO=true - echo "Retrieving symbolized traces" - $ANDROID_BUILD_TOP/development/scripts/stack $ASAN_OUT_FILTERED > $SYM_FILTERED -else - echo "Skipped: Retrieving symbolized traces" -fi + if [ ! -f "$PID_DIR/logcat" ] || [ "${DO_REDO[$pid]}" = true ] || [ $DO_REDO = true ]; then + DO_REDO[$pid]=true + awk '{if($3 == '$pid') print $0}' $LOGCAT_FILE > $LOGCAT_PID_FILE + fi -# Step 5 - Using Steps 2, 3, 4 outputs in order to output graph data -# and trace data -# Only the category names are needed for the commands giving final output -shift -TIME_OUTPUT=($OUT_DIR/time_output_*.dat) -if [ ! -e ${TIME_OUTPUT[0]} ] || [ $DO_REDO = true ]; then - DO_REDO=true - echo "Creating Categorized Time Table" - python $ANDROID_BUILD_TOP/art/tools/runtime_memusage/symbol_trace_info.py \ - -d $OUT_DIR $ASAN_OUT_FILTERED $SYM_FILTERED $DEX_START $@ -else - echo "Skipped: Creating Categorized Time Table" -fi + if [ ! -d "$INTERMEDIATES_DIR" ]; then + mkdir $INTERMEDIATES_DIR + DO_REDO[$pid]=true + fi -# Step 6 - Use graph data from Step 5 to plot graph -# Contains the category names used for legend of gnuplot -PLOT_CATS=`echo \"Uncategorized $@\"` -echo "Plotting Categorized Time Table" -# Plots the information from logcat -gnuplot --persist -e \ - 'filename(n) = sprintf("'"$OUT_DIR"'/time_output_%d.dat", n); - catnames = '"$PLOT_CATS"'; - set title "Dex File Offset vs. Time accessed since App Start"; - set xlabel "Time (milliseconds)"; - set ylabel "Dex File Offset (bytes)"; - plot for [i=0:'"$NUM_CAT"'] filename(i) using 1:2 title word(catnames, i + 1);' + # Step 1 - Only output lines related to Sanitizer + # Folder that holds all file output + ASAN_OUT=$INTERMEDIATES_DIR/asan_output + if [ ! -f $ASAN_OUT ] || [ "${DO_REDO[$pid]}" = true ] || [ $DO_REDO = true ]; then + DO_REDO[$pid]=true + echo "Extracting ASAN output" + grep "app_process64" $LOGCAT_PID_FILE > $ASAN_OUT + else + echo "Skipped: Extracting ASAN output" + fi -if [ $USE_TEMP = true ]; then - echo "Removing temp directory and files" - rm -rf $OUT_DIR -fi + # Step 2 - Only output lines containing Dex File Start Addresses + DEX_START=$INTERMEDIATES_DIR/dex_start + if [ ! -f $DEX_START ] || [ "${DO_REDO[$pid]}" = true ] || [ $DO_REDO = true ]; then + DO_REDO[$pid]=true + echo "Extracting Start of Dex File(s)" + grep "RegisterDexFile" $LOGCAT_PID_FILE > $DEX_START + else + echo "Skipped: Extracting Start of Dex File(s)" + fi + + # Step 3 - Clean Sanitizer output from Step 2 since logcat cannot + # handle large amounts of output. + ASAN_OUT_FILTERED=$INTERMEDIATES_DIR/asan_output_filtered + if [ ! -f $ASAN_OUT_FILTERED ] || [ "${DO_REDO[$pid]}" = true ] || [ $DO_REDO = true ]; then + DO_REDO[$pid]=true + echo "Filtering/Cleaning ASAN output" + python $ANDROID_BUILD_TOP/art/tools/runtime_memusage/prune_sanitizer_output.py \ + $EXACT_ARG $MIN_ARG -d $INTERMEDIATES_DIR $ASAN_OUT + else + echo "Skipped: Filtering/Cleaning ASAN output" + fi + + # Step 4 - Retrieve symbolized stack traces from Step 3 output + SYM_FILTERED=$INTERMEDIATES_DIR/sym_filtered + if [ ! -f $SYM_FILTERED ] || [ "${DO_REDO[$pid]}" = true ] || [ $DO_REDO = true ]; then + DO_REDO[$pid]=true + echo "Retrieving symbolized traces" + $ANDROID_BUILD_TOP/development/scripts/stack $ASAN_OUT_FILTERED > $SYM_FILTERED + else + echo "Skipped: Retrieving symbolized traces" + fi + + # Step 4.5 - Obtain Dex File Format of dex file related to package + BAKSMALI_DMP_OUT="$INTERMEDIATES_DIR""/baksmali_dex_file" + BAKSMALI_DMP_ARG="--dex-file="$BAKSMALI_DMP_OUT + if [ ! -f $BAKSMALI_DMP_OUT ] || [ "${DO_REDO[$pid]}" = true ] || [ $DO_REDO = true ]; then + if [ $PACKAGE_NAME != "" ]; then + # Extracting Dex File path on device from Dex File related to package + apk_directory=$(dirname $(grep $PACKAGE_NAME $DEX_START | tail -n1 | awk '{print $8}')) + apk_dex_files=$(adb shell find $apk_directory -name "*.?dex" -type f 2> /dev/null) + for apk_file in $apk_dex_files; do + base_name=$(basename $apk_file) + adb pull $apk_file $INTERMEDIATES_DIR/base."${base_name#*.}" + done + oatdump --oat-file=$INTERMEDIATES_DIR/base.odex --export-dex-to=$INTERMEDIATES_DIR --output=/dev/null + export_dex=( $INTERMEDIATES_DIR/*apk_export* ) + baksmali -JXmx1024M dump $export_dex > $BAKSMALI_DMP_OUT 2> /dev/null + if ! [ -s $BAKSMALI_DMP_OUT ]; then + rm $BAKSMALI_DMP_OUT + BAKSMALI_DMP_ARG="" + echo "Failed to retrieve Dex File format" + fi + else + BAKSMALI_DMP_ARG="" + echo "Failed to retrieve Dex File format" + fi + else + echo "Skipped: Retrieving Dex File format from baksmali" + fi + + if [ ! -d "$RESULTS_DIR" ]; then + mkdir $RESULTS_DIR + DO_REDO[$pid]=true + fi + + # Step 5 - Using Steps 2, 3, 4 outputs in order to output graph data + # and trace data + # Only the category names are needed for the commands giving final output + shift + TIME_OUTPUT=($RESULTS_DIR/time_output_*.dat) + if [ ! -e ${TIME_OUTPUT[0]} ] || [ "${DO_REDO[$pid]}" = true ] || [ $DO_REDO = true ]; then + DO_REDO[$pid]=true + echo "Creating Categorized Time Table" + python $ANDROID_BUILD_TOP/art/tools/runtime_memusage/symbol_trace_info.py \ + -d $RESULTS_DIR ${OFFSET_ARGS[@]} ${TIME_ARGS[@]} $BAKSMALI_DMP_ARG $ASAN_OUT_FILTERED $SYM_FILTERED $DEX_START $@ + else + echo "Skipped: Creating Categorized Time Table" + fi + + # Step 6 - Use graph data from Step 5 to plot graph + # Contains the category names used for legend of gnuplot + PLOT_CATS=`echo \"Uncategorized $@\"` + PACKAGE_STRING="" + if [ $PACKAGE_NAME != "" ]; then + PACKAGE_STRING="Package name: "$PACKAGE_NAME" " + fi + echo "Plotting Categorized Time Table" + # Plots the information from logcat + gnuplot --persist -e \ + 'filename(n) = sprintf("'"$RESULTS_DIR"'/time_output_%d.dat", n); + catnames = '"$PLOT_CATS"'; + set title "'"$PACKAGE_STRING"'PID: '"$pid"'"; + set xlabel "Time (milliseconds)"; + set ylabel "Dex File Offset (bytes)"; + plot for [i=0:'"$NUM_CAT"'] filename(i) using 1:2 title word(catnames, i + 1);' + + if [ $USE_TEMP = true ]; then + echo "Removing temp directory and files" + rm -rf $OUT_DIR + fi +done diff --git a/tools/runtime_memusage/symbol_trace_info.py b/tools/runtime_memusage/symbol_trace_info.py index e539be2217..a5ced380f0 100755 --- a/tools/runtime_memusage/symbol_trace_info.py +++ b/tools/runtime_memusage/symbol_trace_info.py @@ -25,7 +25,7 @@ from datetime import datetime import argparse import bisect import os -import sys +import re def find_match(list_substrings, big_string): @@ -36,8 +36,13 @@ def find_match(list_substrings, big_string): return list_substrings.index("Uncategorized") -def absolute_to_relative(plot_list, dex_start_list, cat_list): +def absolute_to_relative(data_lists, symbol_traces): """Address changed to Dex File offset and shifting time to 0 min in ms.""" + plot_list = data_lists["plot_list"] + dex_start_list = data_lists["dex_start_list"] + cat_list = data_lists["cat_list"] + offsets = data_lists["offsets"] + time_offsets = data_lists["time_offsets"] time_format_str = "%H:%M:%S.%f" first_access_time = datetime.strptime(plot_list[0][0], time_format_str) @@ -52,9 +57,22 @@ def absolute_to_relative(plot_list, dex_start_list, cat_list): dex_file_start = dex_start_list[bisect.bisect(dex_start_list, address_access) - 1 ] - elem.insert(1, address_access - dex_file_start) - # Category that a data point belongs to - elem.insert(2, cat_list[ind]) + dex_offset = address_access - dex_file_start + # Meant to nullify data that does not meet offset criteria if specified + # Assumes that offsets is already sorted + if (dex_offset >= offsets[0] and dex_offset < offsets[1] and + elem[0] >= time_offsets[0] and elem[0] < time_offsets[1]): + + elem.insert(1, dex_offset) + # Category that a data point belongs to + elem.insert(2, cat_list[ind]) + else: + elem[0] = None + elem[1] = None + elem.append(None) + elem.append(None) + symbol_traces[ind] = None + cat_list[ind] = None def print_category_info(cat_split, outname, out_dir_name, title): @@ -67,7 +85,7 @@ def print_category_info(cat_split, outname, out_dir_name, title): str(len(trace_counts_list_ordered))) print("\tSum of trace counts: " + str(sum([trace[1] for trace in trace_counts_list_ordered]))) - print("\n\tCount: How many traces appeared with count\n\t") + print("\n\tCount: How many traces appeared with count\n\t", end="") print(Counter([trace[1] for trace in trace_counts_list_ordered])) with open(os.path.join(out_dir_name, outname), "w") as output_file: for trace in trace_counts_list_ordered: @@ -79,6 +97,8 @@ def print_category_info(cat_split, outname, out_dir_name, title): def print_categories(categories, symbol_file_split, out_dir_name): """Prints details of all categories.""" + symbol_file_split = [trace for trace in symbol_file_split + if trace is not None] # Info of traces containing a call to current category for cat_num, cat_name in enumerate(categories[1:]): print("\nCategory #%d" % (cat_num + 1)) @@ -123,6 +143,26 @@ def parse_args(argv): parser.add_argument("-d", action="store", default="", dest="out_dir_name", type=is_directory, help="Output Directory") + parser.add_argument("--dex-file", action="store", + default=None, dest="dex_file", + type=argparse.FileType("r"), + help="Baksmali Dex File Dump") + parser.add_argument("--offsets", action="store", nargs=2, + default=[float(0), float("inf")], + dest="offsets", + metavar="OFFSET", + type=float, + help="Filters out accesses not between provided" + " offsets if provided. Can provide 'inf'" + " for infinity") + parser.add_argument("--times", action="store", nargs=2, + default=[float(0), float("inf")], + dest="times", + metavar="TIME", + type=float, + help="Filters out accesses not between provided" + " time offsets if provided. Can provide 'inf'" + " for infinity") parser.add_argument("sanitizer_trace", action="store", type=argparse.FileType("r"), help="File containing sanitizer traces filtered by " @@ -141,6 +181,14 @@ def parse_args(argv): return parser.parse_args(argv) +def get_dex_offset_data(line, dex_file_item): + """ Returns a tuple of dex file offset, item name, and data of a line.""" + return (int(line[:line.find(":")], 16), + (dex_file_item, + line.split("|")[1].strip()) + ) + + def read_data(parsed_argv): """Reads data from filepath arguments and parses them into lists.""" # Using a dictionary to establish relation between lists added @@ -149,6 +197,12 @@ def read_data(parsed_argv): # Makes sure each trace maps to some category categories.insert(0, "Uncategorized") + data_lists["offsets"] = parsed_argv.offsets + data_lists["offsets"].sort() + + data_lists["times"] = parsed_argv.times + data_lists["times"].sort() + logcat_file_data = parsed_argv.sanitizer_trace.readlines() parsed_argv.sanitizer_trace.close() @@ -159,6 +213,25 @@ def read_data(parsed_argv): dex_start_file_data = parsed_argv.dex_starts.readlines() parsed_argv.dex_starts.close() + if parsed_argv.dex_file != None: + dex_file_data = parsed_argv.dex_file.read() + parsed_argv.dex_file.close() + # Splits baksmali dump by each item + item_split = [s.splitlines() for s in re.split(r"\|\[[0-9]+\] ", + dex_file_data)] + # Splits each item by line and creates a list of offsets and a + # corresponding list of the data associated with that line + offset_list, offset_data = zip(*[get_dex_offset_data(line, item[0]) + for item in item_split + for line in item[1:] + if re.search("[0-9a-f]{6}:", line) + is not None + and line.find("|") != -1]) + data_lists["offset_list"] = offset_list + data_lists["offset_data"] = offset_data + else: + dex_file_data = None + # Each element is a tuple of time and address accessed data_lists["plot_list"] = [[elem[1] for elem in enumerate(line.split()) if elem[0] in (1, 11) @@ -184,23 +257,26 @@ def read_data(parsed_argv): return data_lists, categories, symbol_file_split -def main(argv=None): +def main(): """Takes in trace information and outputs details about them.""" - if argv is None: - argv = sys.argv - parsed_argv = parse_args(argv[1:]) + parsed_argv = parse_args(None) data_lists, categories, symbol_file_split = read_data(parsed_argv) + # Formats plot_list such that each element is a data point - absolute_to_relative(data_lists["plot_list"], data_lists["dex_start_list"], - data_lists["cat_list"]) + #absolute_to_relative(data_lists["plot_list"], data_lists["dex_start_list"], + # data_lists["cat_list"], data_lists["offsets"], + # data_lists["times"], symbol_file_split) + absolute_to_relative(data_lists, symbol_file_split) for file_ext, cat_name in enumerate(categories): out_file_name = os.path.join(parsed_argv.out_dir_name, "time_output_" + str(file_ext) + ".dat") with open(out_file_name, "w") as output_file: output_file.write("# Category: " + cat_name + "\n") - output_file.write("# Time, Dex File Offset, Address \n") + output_file.write("# Time, Dex File Offset_10, Dex File Offset_16," + " Address, Item Accessed, Item Member Accessed" + " Unaligned\n") for time, dex_offset, category, address in data_lists["plot_list"]: if category == cat_name: output_file.write( @@ -208,9 +284,23 @@ def main(argv=None): " " + str(dex_offset) + " #" + - str(address) + - "\n") - + hex(dex_offset) + + " " + + str(address)) + if data_lists.has_key("offset_list"): + dex_offset_index = bisect.bisect( + data_lists["offset_list"], + dex_offset) - 1 + aligned_dex_offset = (data_lists["offset_list"] + [dex_offset_index]) + dex_offset_data = (data_lists["offset_data"] + [dex_offset_index]) + output_file.write( + " " + + "|".join(dex_offset_data) + + " " + + str(aligned_dex_offset != dex_offset)) + output_file.write("\n") print_categories(categories, symbol_file_split, parsed_argv.out_dir_name) diff --git a/tools/test_presubmit.py b/tools/test_presubmit.py new file mode 100755 index 0000000000..f6e6df9a68 --- /dev/null +++ b/tools/test_presubmit.py @@ -0,0 +1,159 @@ +#!/usr/bin/python3 +# +# Copyright 2017, The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# There are many run-tests which generate their sources automatically. +# It is desirable to keep the checked-in source code, as we re-run generators very rarely. +# +# This script will re-run the generators only if their dependent files have changed and then +# complain if the outputs no longer matched what's in the source tree. +# + +import os +import pathlib +import subprocess +import sys +import tempfile + +THIS_PATH = os.path.dirname(os.path.realpath(__file__)) + +TOOLS_GEN_SRCS = [ + # tool -> path to a script to generate a file + # reference_files -> list of files that the script can generate + # args -> lambda(path) that generates arguments the 'tool' in order to output to 'path' + # interesting_files -> which files much change in order to re-run the tool. + # interesting_to_reference_files: lambda(x,reference_files) + # given the interesting file 'x' and a list of reference_files, + # return exactly one reference file that corresponds to it. + { 'tool' : 'test/988-method-trace/gen_srcs.py', + 'reference_files' : ['test/988-method-trace/src/art/Test988Intrinsics.java'], + 'args' : lambda output_path: [output_path], + 'interesting_files' : ['compiler/intrinsics_list.h'], + 'interesting_to_reference_file' : lambda interesting, references: references[0], + }, +] + +DEBUG = False + +def debug_print(msg): + if DEBUG: + print("[DEBUG]: " + msg, file=sys.stderr) + +def is_interesting(f, tool_dict): + """ + Returns true if this is a file we want to run this tool before uploading. False otherwise. + """ + path = pathlib.Path(f) + return str(path) in tool_dict['interesting_files'] + +def get_changed_files(commit): + """ + Gets the files changed in the given commit. + """ + return subprocess.check_output( + ["git", 'diff-tree', '--no-commit-id', '--name-only', '-r', commit], + stderr=subprocess.STDOUT, + universal_newlines=True).split() + +def command_line_for_tool(tool_dict, output): + """ + Calculate the command line for this tool when ran against the output file 'output'. + """ + proc_args = [tool_dict['tool']] + tool_dict['args'](output) + return proc_args + +def run_tool(tool_dict, output): + """ + Execute this tool by passing the tool args to the tool. + """ + proc_args = command_line_for_tool(tool_dict, output) + debug_print("PROC_ARGS: %s" %(proc_args)) + succ = subprocess.call(proc_args) + return succ + +def get_reference_file(changed_file, tool_dict): + """ + Lookup the file that the tool is generating in response to changing an interesting file + """ + return tool_dict['interesting_to_reference_file'](changed_file, tool_dict['reference_files']) + +def run_diff(changed_file, tool_dict, original_file): + ref_file = get_reference_file(changed_file, tool_dict) + + return subprocess.call(["diff", ref_file, original_file]) != 0 + +def run_gen_srcs(files): + """ + Runs test tools only for interesting files that were changed in this commit. + """ + if len(files) == 0: + return + + success = 0 # exit code 0 = success, >0 error. + had_diffs = False + + for tool_dict in TOOLS_GEN_SRCS: + tool_ran_at_least_once = False + for f in files: + if is_interesting(f, tool_dict): + tmp_file = tempfile.mktemp() + reference_file = get_reference_file(f, tool_dict) + + # Generate the source code with a temporary file as the output. + success = run_tool(tool_dict, tmp_file) + if success != 0: + # Immediately abort if the tool fails with a non-0 exit code, do not go any further. + print("[FATAL] Error when running tool (return code %s)" %(success), file=sys.stderr) + print("$> %s" %(" ".join(command_line_for_tool(tool_dict, tmp_file))), file=sys.stderr) + sys.exit(success) + if run_diff(f, tool_dict, tmp_file): + # If the tool succeeded, but there was a diff, then the generated code has diverged. + # Output the diff information and continue to the next files/tools. + had_diffs = True + print("-----------------------------------------------------------", file=sys.stderr) + print("File '%s' diverged from generated file; please re-run tools:" %(reference_file), file=sys.stderr) + print("$> %s" %(" ".join(command_line_for_tool(tool_dict, reference_file))), file=sys.stderr) + else: + debug_print("File %s is consistent with tool %s" %(reference_file, tool_dict['tool'])) + + tool_ran_at_least_once = True + + if not tool_ran_at_least_once: + debug_print("Interesting files %s unchanged, skipping tool '%s'" %(tool_dict['interesting_files'], tool_dict['tool'])) + + if had_diffs: + success = 1 + # Always return non-0 exit code when there were diffs so that the presubmit hooks are FAILED. + + return success + + +def main(): + if 'PREUPLOAD_COMMIT' in os.environ: + commit = os.environ['PREUPLOAD_COMMIT'] + else: + print("WARNING: Not running as a pre-upload hook. Assuming commit to check = 'HEAD'", file=sys.stderr) + commit = "HEAD" + + os.chdir(os.path.join(THIS_PATH, '..')) # run tool relative to 'art' directory + debug_print("CWD: %s" %(os.getcwd())) + + changed_files = get_changed_files(commit) + debug_print("Changed files: %s" %(changed_files)) + return run_gen_srcs(changed_files) + +if __name__ == '__main__': + sys.exit(main()) |